import sys import pandas as pd print(f"Python version: {sys.version}") print(f"pandas version: {pd.__version__}") print() num_indices = 100000 # OK with 10,000; fails with 100,000 metrics = [ "apple", "banana", ] data_rows = [] for idx in range(num_indices): data_rows.append({"idx": idx, "metric": "apple", "value": 2 * idx}) data_rows.append({"idx": idx, "metric": "banana", "value": 3 * idx}) data_rows.append({"idx": idx, "metric": "coconut", "value": 4 * idx}) df = pd.DataFrame(data_rows) print(f"Generated dataset: {len(df):,} rows") print(f"Expected rows after pivot: {num_indices:,}") print() print("Pivoting data...") pivoted = df.pivot_table( index=["idx"], columns="metric", values="value", aggfunc="first", ) print("After pivot:") print(f" Total rows: {len(pivoted):,}") print(f" Unique indices: {pivoted.index.nunique():,}") print(f" Has duplicate indices: {pivoted.index.duplicated().any()}") if pivoted.index.duplicated().any(): print(" BUG: DUPLICATE INDICES") print() print("Example duplicates:") dup_indices = pivoted.index[pivoted.index.duplicated(keep=False)] for idx in dup_indices.unique()[:3]: print(pivoted.loc[idx]) print() else: print() print("OK") status = 0 if not pivoted.index.duplicated().any() else 1 sys.exit(status)