#!/usr/bin/env python3 # Repro: pandas.read_parquet(filters=...) does not accept pandas Period values, # and there is no documented way to pass the correct physical scalar via pandas API. # # Expected: Either accept Period in filters (map to physical storage), or document # an official helper to build Arrow-coercible filters from pandas logical types. import os, sys, json, tempfile import numpy as np import pandas as pd import pyarrow as pa import pyarrow.parquet as pq import pyarrow.dataset as ds print("Versions:") print(" python :", sys.version.split()[0]) print(" pandas :", pd.__version__) print(" pyarrow :", pa.__version__) print() # --- build a tiny dataset: ints, Period[M], datetimes --- months = pd.period_range("2024-01", "2024-12", freq="M") df = pd.DataFrame({ "n": np.arange(1, 13, dtype="int64"), "per_m": months, # pandas Period[M] "ts_m": months.to_timestamp(how="start") # datetime64[ns], first of month }) # write to a temp dir/file tmpdir = tempfile.mkdtemp(prefix="pandas_period_filter_repro_") path = os.path.join(tmpdir, "mini.parquet") df.to_parquet(path, engine="pyarrow", index=False) print("Wrote:", path) print() # --- inspect physical schema + pandas metadata --- dset = ds.dataset(path, format="parquet") schema = dset.schema print("Arrow physical schema:", schema) pmeta = json.loads((schema.metadata or {}).get(b"pandas", b"{}").decode() or "{}") cur_meta = {c["name"]: c for c in pmeta.get("columns", [])} print("Pandas metadata for columns:") for k, v in cur_meta.items(): if "metadata" in v and isinstance(v["metadata"], dict): v = dict(v, metadata={"keys": list(v["metadata"].keys())}) print(" ", k, "→ pandas_type:", v.get("pandas_type"), "metadata:", v.get("metadata")) print() # Helper for pretty printing results def show(label, pdf): print(label) print(pdf.sort_values(["n"]).reset_index(drop=True)) print() # --- 1) numeric filter: works --- pdf_num = pd.read_parquet( path, engine="pyarrow", columns=["n", "per_m", "ts_m"], filters=[("n", ">=", 7)] ) show("Numeric filter n>=7 (EXPECTED TO WORK):", pdf_num) # --- 2) timestamp range filter: works --- start = pd.Timestamp("2024-07-01") pdf_ts = pd.read_parquet( path, engine="pyarrow", columns=["n", "per_m", "ts_m"], filters=[("ts_m", ">=", start)] ) show("Timestamp range filter July 2024 (EXPECTED TO WORK):", pdf_ts) # --- 3a) period equality filter: fails (cannot convert Period) --- m = pd.Period("2024-07", freq="M") try: pdf_per = pd.read_parquet( path, engine="pyarrow", columns=["n", "per_m", "ts_m"], filters=[("per_m", ">=", m)] ) show("Period equality filter per_m >= 2024-07 (UNEXPECTED, but if this prints it worked):", pdf_per) except Exception as e: print("Period equality filter per_m>=Period('2024-07','M') raised (EXPECTED BUG/UNSUPPORTED):") print(" ", type(e).__name__ + ":", e) print() # --- 3b) period ordinal equality filter: also does not match via pandas filters --- # Even though the physical column is integer (ordinals), passing m.ordinal here still # goes through pandas' filter adapter, which treats values as Python scalars and does # not apply the pandas metadata mapping used on read. try: pdf_ord = pd.read_parquet( path, engine="pyarrow", columns=["n", "per_m", "ts_m"], filters=[("per_m", ">=", m.ordinal)] # try integer ordinal directly ) show("Period ordinal filter per_m>=m.ordinal (CURRENT BEHAVIOR):", pdf_ord) if pdf_ord.empty: print(" Note: No rows returned. Ordinal inequality through pandas filters did not match.\n") except Exception as e: print("Period ordinal filter raised:") print(" ", type(e).__name__ + ":", e) print()