Merge origin

Author: rhshadrach

doc/source/whatsnew/v3.0.0.rst

@@ -392,9 +392,9 @@ and users can skip the check by explicitly specifying ``sort=True`` or
 ``sort=False``.
 
 This deprecation can also impact pandas' internal usage of :func:`concat`.
-While we have investigated uses of :func:`concat` to determine if this could lead
-to a change in behavior of other functions and methods in the API, it is
-possible some have been missed. In order to be cautious here, pandas has *not*
+Here cases where :func:`concat` was sorting a :class:`DatetimeIndex` but not
+other indexes are considered bugs and have been fixed as noted below. However
+it is possible some have been missed. In order to be cautious here, pandas has *not*
 added ``sort=False`` to any internal calls where we believe behavior should not change.
 If we have missed something, users will not experience a behavior change but they
 will receive a warning about :func:`concat` even though they are not directly
@@ -431,6 +431,14 @@ we may address any potential behavior changes.
 
  pd.concat([df1, df2], axis=1, sort=False)
 
+Cases where pandas' internal usage of :func:`concat` resulted in inconsistent sorting
+that are now fixed in this release are as follows.
+
+- :meth:`Series.apply` and :meth:`DataFrame.apply` with a list-like or dict-like ``func`` argument.
+- :meth:`Series.shift`, :meth:`DataFrame.shift`, :meth:`.SeriesGroupBy.shift`, :meth:`.DataFrameGroupBy.shift` with the ``periods`` argument a list of length greater than 1.
+- :meth:`DataFrame.join` with ``other`` a list of one or more Series or DataFrames and ``how="inner"``, ``how="left"``, or ``how="right"``.
+- :meth:`Series.str.cat` with ``others`` a Series or DataFrame.
+
 .. _whatsnew_300.api_breaking.value_counts_sorting:
 
 Changed behavior in :meth:`DataFrame.value_counts` and :meth:`DataFrameGroupBy.value_counts` when ``sort=False``
@@ -1243,7 +1251,6 @@ Groupby/resample/rolling
 - Bug in :meth:`DataFrameGroupBy.apply` with ``as_index=False`` that was returning :class:`MultiIndex` instead of returning :class:`Index`. (:issue:`58291`)
 - Bug in :meth:`DataFrameGroupBy.cumsum` and :meth:`DataFrameGroupBy.cumprod` where ``numeric_only`` parameter was passed indirectly through kwargs instead of passing directly. (:issue:`58811`)
 - Bug in :meth:`DataFrameGroupBy.cumsum` where it did not return the correct dtype when the label contained ``None``. (:issue:`58811`)
-- Bug in :meth:`DataFrameGroupBy.shift` where the resulting index would be sorted if the input is a :class:`DatetimeIndex` and multiple periods are specified (:issue:`62843`)
 - Bug in :meth:`DataFrameGroupby.transform` and :meth:`SeriesGroupby.transform` with a reducer and ``observed=False`` that coerces dtype to float when there are unobserved categories. (:issue:`55326`)
 - Bug in :meth:`Rolling.apply` for ``method="table"`` where column order was not being respected due to the columns getting sorted by default. (:issue:`59666`)
 - Bug in :meth:`Rolling.apply` where the applied function could be called on fewer than ``min_period`` periods if ``method="table"``. (:issue:`58868`)

pandas/core/apply.py

@@ -382,7 +382,7 @@ def transform_dict_like(self, func) -> DataFrame:
  for name, how in func.items():
  colg = obj._gotitem(name, ndim=1)
  results[name] = colg.transform(how, 0, *args, **kwargs)
- return concat(results, axis=1) # nobug
+ return concat(results, axis=1)
 
  def transform_str_or_callable(self, func) -> DataFrame | Series:
  """
@@ -485,7 +485,7 @@ def wrap_results_list_like(
  obj = self.obj
 
  try:
- return concat(results, keys=keys, axis=1, sort=False) # maybebug
+ return concat(results, keys=keys, axis=1, sort=False)
  except TypeError as err:
  # we are concatting non-NDFrame objects,
  # e.g. a list of scalars
@@ -635,10 +635,11 @@ def wrap_results_dict_like(
  keys_to_use = ktu
 
  axis: AxisInt = 0 if isinstance(obj, ABCSeries) else 1
- result = concat( # maybebug
+ result = concat(
  results,
  axis=axis,
  keys=keys_to_use,
+ sort=False,
  )
  elif any(is_ndframe):
  # There is a mix of NDFrames and scalars

pandas/core/arrays/arrow/accessors.py

@@ -496,6 +496,6 @@ def explode(self) -> DataFrame:
  from pandas import concat
 
  pa_type = self._pa_array.type
- return concat( # nobug
+ return concat(
  [self.field(i) for i in range(pa_type.num_fields)], axis="columns"
  )

pandas/core/arrays/categorical.py

@@ -2686,7 +2686,7 @@ def describe(self) -> DataFrame:
  from pandas import Index
  from pandas.core.reshape.concat import concat
 
- result = concat([counts, freqs], ignore_index=True, axis=1) # nobug
+ result = concat([counts, freqs], ignore_index=True, axis=1)
  result.columns = Index(["counts", "freqs"])
  result.index.name = "categories"
 

pandas/core/frame.py

@@ -6137,7 +6137,7 @@ def shift(
  .shift(periods=period, freq=freq, axis=axis, fill_value=fill_value)
  .add_suffix(f"{suffix}_{period}" if suffix else f"_{period}")
  )
- return concat(shifted_dataframes, axis=1) # bug
+ return concat(shifted_dataframes, axis=1, sort=False)
  elif suffix:
  raise ValueError("Cannot specify `suffix` if `periods` is an int.")
  periods = cast(int, periods)
@@ -11166,7 +11166,7 @@ def _append_internal(
 
  from pandas.core.reshape.concat import concat
 
- result = concat( # possible bug
+ result = concat(
  [self, row_df],
  ignore_index=ignore_index,
  )
@@ -11394,7 +11394,7 @@ def join(
  # join indexes only using concat
  if can_concat:
  if how == "left" or how == "right":
- res = concat( # nobug
+ res = concat(
  frames, axis=1, join="outer", verify_integrity=True, sort=sort
  )
  index = self.index if how == "left" else frames[-1].index
@@ -11405,7 +11405,7 @@ def join(
  else:
  if how == "outer":
  sort = True
- return concat( # bug
+ return concat(
  frames, axis=1, join=how, verify_integrity=True, sort=sort
  )
 
@@ -11595,9 +11595,7 @@ def _series_round(ser: Series, decimals: int) -> Series:
 
  if new_cols is not None and len(new_cols) > 0:
  return self._constructor(
- concat(new_cols, axis=1),
- index=self.index,
- columns=self.columns, # nobug
+ concat(new_cols, axis=1), index=self.index, columns=self.columns
  ).__finalize__(self, method="round")
  else:
  return self.copy(deep=False)
@@ -14181,7 +14179,7 @@ def isin(self, values: Series | DataFrame | Sequence | Mapping) -> DataFrame:
  from pandas.core.reshape.concat import concat
 
  values = collections.defaultdict(list, values)
- result = concat( # nobug
+ result = concat(
  (
  self.iloc[:, [i]].isin(values[col])
  for i, col in enumerate(self.columns)

pandas/core/generic.py

@@ -6512,7 +6512,7 @@ def astype(
  return self.copy(deep=False)
 
  # GH 19920: retain column metadata after concat
- result = concat(results, axis=1) # nobug
+ result = concat(results, axis=1)
  # GH#40810 retain subclass
  # error: Incompatible types in assignment
  # (expression has type "Self", variable has type "DataFrame")
@@ -9494,7 +9494,7 @@ def compare(
 
  # error: List item 0 has incompatible type "NDFrame"; expected
  # "Union[Series, DataFrame]"
- diff = concat( # bug
+ diff = concat(
  [self, other], # type: ignore[list-item]
  axis=axis,
  keys=result_names,

pandas/core/groupby/generic.py

@@ -582,7 +582,7 @@ def _aggregate_multiple_funcs(self, arg, *args, **kwargs) -> DataFrame:
  if any(isinstance(x, DataFrame) for x in results.values()):
  from pandas import concat
 
- res_df = concat( # nobug
+ res_df = concat(
  results.values(), axis=1, keys=[key.label for key in results]
  )
  return res_df
@@ -753,7 +753,7 @@ def _transform_general(
  if results:
  from pandas.core.reshape.concat import concat
 
- concatenated = concat(results, ignore_index=True) # nobug
+ concatenated = concat(results, ignore_index=True)
  result = self._set_result_index_ordered(concatenated)
  else:
  result = self.obj._constructor(dtype=np.float64)
@@ -2263,7 +2263,7 @@ def _transform_general(self, func, engine, engine_kwargs, *args, **kwargs):
  applied.append(res)
 
  concat_index = obj.columns
- concatenated = concat( # nobug
+ concatenated = concat(
  applied, axis=0, verify_integrity=False, ignore_index=True
  )
  concatenated = concatenated.reindex(concat_index, axis=1)
@@ -2553,7 +2553,7 @@ def _apply_to_column_groupbys(self, func) -> DataFrame:
  # concat would raise
  res_df = DataFrame([], columns=columns, index=self._grouper.result_index)
  else:
- res_df = concat(results, keys=columns, axis=1) # nobug
+ res_df = concat(results, keys=columns, axis=1)
 
  if not self.as_index:
  res_df.index = default_index(len(res_df))
@@ -3413,9 +3413,7 @@ def _wrap_transform_general_frame(
  # other dimension; this will preserve dtypes
  # GH14457
  if res.index.is_(obj.index):
- res_frame = concat(
- [res] * len(group.columns), axis=1, ignore_index=True
- ) # nobug
+ res_frame = concat([res] * len(group.columns), axis=1, ignore_index=True)
  res_frame.columns = group.columns
  res_frame.index = group.index
  else:

pandas/core/groupby/groupby.py

@@ -1141,7 +1141,7 @@ def _concat_objects(
  group_levels = self._grouper.levels
  group_names = self._grouper.names
 
- result = concat( # maybebug
+ result = concat(
  values,
  axis=0,
  keys=group_keys,
@@ -1150,10 +1150,10 @@ def _concat_objects(
  sort=False,
  )
  else:
- result = concat(values, axis=0) # maybebug
+ result = concat(values, axis=0)
 
  elif not not_indexed_same:
- result = concat(values, axis=0) # maybebug
+ result = concat(values, axis=0)
 
  ax = self._selected_obj.index
  if self.dropna:
@@ -1176,7 +1176,7 @@ def _concat_objects(
  result = result.reindex(ax, axis=0)
 
  else:
- result = concat(values, axis=0) # maybebug
+ result = concat(values, axis=0)
 
  if self.obj.ndim == 1:
  name = self.obj.name
@@ -5231,7 +5231,7 @@ def shift(
  return (
  shifted_dataframes[0]
  if len(shifted_dataframes) == 1
- else concat(shifted_dataframes, axis=1, sort=False) # nobug
+ else concat(shifted_dataframes, axis=1, sort=False)
  )
 
  @final

pandas/core/indexes/base.py

@@ -5384,9 +5384,9 @@ def append(self, other: Index | Sequence[Index]) -> Index:
  names = {obj.name for obj in to_concat}
  name = None if len(names) > 1 else self.name
 
- return self._concat(to_concat, name) # nobug
+ return self._concat(to_concat, name)
 
- def _concat(self, to_concat: list[Index], name: Hashable) -> Index: # nobug
+ def _concat(self, to_concat: list[Index], name: Hashable) -> Index:
  """
  Concatenate multiple Index objects.
  """

pandas/core/indexes/range.py

@@ -1181,7 +1181,7 @@ def insert(self, loc: int, item) -> Index:
 
  return super().insert(loc, item)
 
- def _concat(self, indexes: list[Index], name: Hashable) -> Index: # nobug
+ def _concat(self, indexes: list[Index], name: Hashable) -> Index:
  """
  Overriding parent method for the case of all RangeIndex instances.
 
@@ -1191,7 +1191,7 @@ def _concat(self, indexes: list[Index], name: Hashable) -> Index: # nobug
  indexes = [RangeIndex(3), RangeIndex(4, 6)] -> Index([0,1,2,4,5], dtype='int64')
  """
  if not all(isinstance(x, RangeIndex) for x in indexes):
- result = super()._concat(indexes, name) # nobug
+ result = super()._concat(indexes, name)
  if result.dtype.kind == "i":
  return self._shallow_copy(result._values)
  return result