BUG: groupby.describe on a frame with duplicate column names

doc/source/whatsnew/v2.0.0.rst

@@ -1130,6 +1130,7 @@ Groupby/resample/rolling
 - Bug in :meth:`.SeriesGroupBy.value_counts` did not respect ``sort=False`` (:issue:`50482`)
 - Bug in :meth:`.DataFrameGroupBy.resample` raises ``KeyError`` when getting the result from a key list when resampling on time index (:issue:`50840`)
 - Bug in :meth:`.DataFrameGroupBy.transform` and :meth:`.SeriesGroupBy.transform` would raise incorrectly when grouper had ``axis=1`` for ``"ngroup"`` argument (:issue:`45986`)
+- Bug in :meth:`.DataFrameGroupBy.describe` produced incorrect results when data had duplicate columns (:issue:`50806`)
 -
 
 Reshaping

pandas/core/groupby/groupby.py

@@ -610,7 +610,7 @@ def f(self):
 
 
 class BaseGroupBy(PandasObject, SelectionMixin[NDFrameT], GroupByIndexingMixin):
- _group_selection: IndexLabel | None = None
+ _group_selection: np.ndarray | None = None
  _hidden_attrs = PandasObject._hidden_attrs | {
  "as_index",
  "axis",
@@ -726,7 +726,9 @@ def _selected_obj(self):
 
  if self._selection is None or isinstance(self.obj, Series):
  if self._group_selection is not None:
- return self.obj[self._group_selection]
+ return self.obj._take(
+ self._group_selection, axis=1, convert_indices=False
+ )
  return self.obj
  else:
  return self.obj[self._selection]
@@ -1023,7 +1025,14 @@ def _set_group_selection(self) -> None:
  if len(groupers):
  # GH12839 clear selected obj cache when group selection changes
  ax = self.obj._info_axis
- self._group_selection = ax.difference(Index(groupers), sort=False).tolist()
+ if len(ax) < 1000:
+ # Determined experimentally, larger is slower than the NumPy version
+ self._group_selection = np.array(
+ [idx for idx, label in enumerate(ax) if label not in groupers]
+ )
+ else:
+ indexer = ax.get_indexer_for(list(groupers))
+ self._group_selection = np.delete(np.arange(len(ax)), indexer)
  self._reset_cache("_selected_obj")
 
  @final

pandas/tests/groupby/test_function.py

@@ -1254,6 +1254,27 @@ def test_describe_with_duplicate_output_column_names(as_index, keys):
  tm.assert_frame_equal(result, expected)
 
 
+def test_describe_duplicate_columns():
+ # GH#50806
+ df = DataFrame([[0, 1, 2, 3]])
+ df.columns = [0, 1, 2, 0]
+ gb = df.groupby(df[1])
+ result = gb.describe(percentiles=[])
+
+ columns = ["count", "mean", "std", "min", "50%", "max"]
+ frames = [
+ DataFrame([[1.0, val, np.nan, val, val, val]], index=[1], columns=columns)
+ for val in (0.0, 2.0, 3.0)
+ ]
+ expected = pd.concat(frames, axis=1)
+ expected.columns = MultiIndex(
+ levels=[[0, 2], columns],
+ codes=[6 * [0] + 6 * [1] + 6 * [0], 3 * list(range(6))],
+ )
+ expected.index.names = [1]
+ tm.assert_frame_equal(result, expected)
+
+
 def test_groupby_mean_no_overflow():
  # Regression test for (#22487)
  df = DataFrame(
@@ -1594,3 +1615,29 @@ def test_multiindex_group_all_columns_when_empty(groupby_func):
  result = method(*args).index
  expected = df.index
  tm.assert_index_equal(result, expected)
+
+
+def test_duplicate_columns(request, groupby_func, as_index):
+ # GH#50806
+ if groupby_func == "corrwith":
+ msg = "GH#50845 - corrwith fails when there are duplicate columns"
+ request.node.add_marker(pytest.mark.xfail(reason=msg))
+ df = DataFrame([[1, 3, 6], [1, 4, 7], [2, 5, 8]], columns=list("abb"))
+ args = get_groupby_method_args(groupby_func, df)
+ gb = df.groupby("a", as_index=as_index)
+ result = getattr(gb, groupby_func)(*args)
+
+ if groupby_func in ("size", "ngroup", "cumcount"):
+ expected = getattr(
+ df.take([0, 1], axis=1).groupby("a", as_index=as_index), groupby_func
+ )(*args)
+ tm.assert_equal(result, expected)
+ else:
+ expected_df = df.copy()
+ expected_df.columns = ["a", "b", "c"]
+ expected_args = get_groupby_method_args(groupby_func, expected_df)
+ expected = getattr(expected_df.groupby("a", as_index=as_index), groupby_func)(
+ *expected_args
+ )
+ expected = expected.rename(columns={"c": "b"})
+ tm.assert_frame_equal(result, expected)

pandas/tests/groupby/test_groupby.py

@@ -2867,3 +2867,14 @@ def test_groupby_method_drop_na(method):
  else:
  expected = DataFrame({"A": ["a", "b", "c"], "B": [0, 2, 4]}, index=[0, 2, 4])
  tm.assert_frame_equal(result, expected)
+
+
+def test_selected_obj_duplicate_columns():
+ # GH#50806
+ df = DataFrame([[0, 1, 2, 3]])
+ df.columns = [0, 1, 2, 0]
+ gb = df.groupby(df[1])
+ with gb._group_selection_context():
+ result = gb._selected_obj
+ expected = df.take([0, 2, 3], axis=1)
+ tm.assert_frame_equal(result, expected)

pandas/tests/groupby/test_groupby_dropna.py

@@ -544,9 +544,12 @@ def test_categorical_reducers(
 
  gb_filled = df_filled.groupby(keys, observed=observed, sort=sort, as_index=True)
  expected = getattr(gb_filled, reduction_func)(*args_filled).reset_index()
- expected["x"] = expected["x"].replace(4, None)
+ # Workaround since we can't use replace (GH#50872)
+ mask = expected["x"] == 4
+ expected["x"] = expected["x"].mask(mask, None).cat.remove_categories([4])
  if index_kind == "multi":
- expected["x2"] = expected["x2"].replace(4, None)
+ mask = expected["x2"] == 4
+ expected["x2"] = expected["x2"].mask(mask, None).cat.remove_categories([4])
  if as_index:
  if index_kind == "multi":
  expected = expected.set_index(["x", "x2"])
@@ -578,6 +581,8 @@ def test_categorical_reducers(
  result = getattr(gb_keepna, reduction_func)(*args)
 
  # size will return a Series, others are DataFrame
+ print(result.index.dtype)
+ print(expected.index.dtype)
  tm.assert_equal(result, expected)