REF: Decouple Series.apply from Series.agg (#53400)

Author: topper-123

doc/source/whatsnew/v2.1.0.rst

@@ -101,6 +101,7 @@ Other enhancements
 - :meth:`DataFrame.unstack` gained the ``sort`` keyword to dictate whether the resulting :class:`MultiIndex` levels are sorted (:issue:`15105`)
 - :meth:`SeriesGroupby.agg` and :meth:`DataFrameGroupby.agg` now support passing in multiple functions for ``engine="numba"`` (:issue:`53486`)
 - Added ``engine_kwargs`` parameter to :meth:`DataFrame.to_excel` (:issue:`53220`)
+- Added a new parameter ``by_row`` to :meth:`Series.apply`. When set to ``False`` the supplied callables will always operate on the whole Series (:issue:`53400`).
 - Many read/to_* functions, such as :meth:`DataFrame.to_pickle` and :func:`read_csv`, support forwarding compression arguments to lzma.LZMAFile (:issue:`52979`)
 - Performance improvement in :func:`concat` with homogeneous ``np.float64`` or ``np.float32`` dtypes (:issue:`52685`)
 - Performance improvement in :meth:`DataFrame.filter` when ``items`` is given (:issue:`52941`)

pandas/core/apply.py

@@ -16,6 +16,7 @@
  Iterable,
  Iterator,
  List,
+ Literal,
  Sequence,
  cast,
 )
@@ -288,6 +289,11 @@ def agg_list_like(self) -> DataFrame | Series:
  -------
  Result of aggregation.
  """
+ return self.agg_or_apply_list_like(op_name="agg")
+
+ def agg_or_apply_list_like(
+ self, op_name: Literal["agg", "apply"]
+ ) -> DataFrame | Series:
  from pandas.core.groupby.generic import (
  DataFrameGroupBy,
  SeriesGroupBy,
@@ -296,6 +302,9 @@ def agg_list_like(self) -> DataFrame | Series:
 
  obj = self.obj
  func = cast(List[AggFuncTypeBase], self.func)
+ kwargs = self.kwargs
+ if op_name == "apply":
+ kwargs = {**kwargs, "by_row": False}
 
  if getattr(obj, "axis", 0) == 1:
  raise NotImplementedError("axis other than 0 is not supported")
@@ -313,8 +322,6 @@ def agg_list_like(self) -> DataFrame | Series:
  keys = []
 
  is_groupby = isinstance(obj, (DataFrameGroupBy, SeriesGroupBy))
- is_ser_or_df = isinstance(obj, (ABCDataFrame, ABCSeries))
- this_args = [self.axis, *self.args] if is_ser_or_df else self.args
 
  context_manager: ContextManager
  if is_groupby:
@@ -323,12 +330,19 @@ def agg_list_like(self) -> DataFrame | Series:
  context_manager = com.temp_setattr(obj, "as_index", True)
  else:
  context_manager = nullcontext()
+
+ def include_axis(colg) -> bool:
+ return isinstance(colg, ABCDataFrame) or (
+ isinstance(colg, ABCSeries) and op_name == "agg"
+ )
+
  with context_manager:
  # degenerate case
  if selected_obj.ndim == 1:
  for a in func:
  colg = obj._gotitem(selected_obj.name, ndim=1, subset=selected_obj)
- new_res = colg.aggregate(a, *this_args, **self.kwargs)
+ args = [self.axis, *self.args] if include_axis(colg) else self.args
+ new_res = getattr(colg, op_name)(a, *args, **kwargs)
  results.append(new_res)
 
  # make sure we find a good name
@@ -339,7 +353,8 @@ def agg_list_like(self) -> DataFrame | Series:
  indices = []
  for index, col in enumerate(selected_obj):
  colg = obj._gotitem(col, ndim=1, subset=selected_obj.iloc[:, index])
- new_res = colg.aggregate(func, *this_args, **self.kwargs)
+ args = [self.axis, *self.args] if include_axis(colg) else self.args
+ new_res = getattr(colg, op_name)(func, *args, **kwargs)
  results.append(new_res)
  indices.append(index)
  keys = selected_obj.columns.take(indices)
@@ -366,15 +381,23 @@ def agg_dict_like(self) -> DataFrame | Series:
  -------
  Result of aggregation.
  """
+ return self.agg_or_apply_dict_like(op_name="agg")
+
+ def agg_or_apply_dict_like(
+ self, op_name: Literal["agg", "apply"]
+ ) -> DataFrame | Series:
  from pandas import Index
  from pandas.core.groupby.generic import (
  DataFrameGroupBy,
  SeriesGroupBy,
  )
  from pandas.core.reshape.concat import concat
 
+ assert op_name in ["agg", "apply"]
+
  obj = self.obj
  func = cast(AggFuncTypeDict, self.func)
+ kwargs = {"by_row": False} if op_name == "apply" else {}
 
  if getattr(obj, "axis", 0) == 1:
  raise NotImplementedError("axis other than 0 is not supported")
@@ -387,7 +410,7 @@ def agg_dict_like(self) -> DataFrame | Series:
  selected_obj = obj._selected_obj
  selection = obj._selection
 
- func = self.normalize_dictlike_arg("agg", selected_obj, func)
+ func = self.normalize_dictlike_arg(op_name, selected_obj, func)
 
  is_groupby = isinstance(obj, (DataFrameGroupBy, SeriesGroupBy))
  context_manager: ContextManager
@@ -404,17 +427,18 @@ def agg_dict_like(self) -> DataFrame | Series:
  )
 
  # Numba Groupby engine/engine-kwargs passthrough
- kwargs = {}
  if is_groupby:
  engine = self.kwargs.get("engine", None)
  engine_kwargs = self.kwargs.get("engine_kwargs", None)
- kwargs = {"engine": engine, "engine_kwargs": engine_kwargs}
+ kwargs.update({"engine": engine, "engine_kwargs": engine_kwargs})
 
  with context_manager:
  if selected_obj.ndim == 1:
  # key only used for output
  colg = obj._gotitem(selection, ndim=1)
- result_data = [colg.agg(how, **kwargs) for _, how in func.items()]
+ result_data = [
+ getattr(colg, op_name)(how, **kwargs) for _, how in func.items()
+ ]
  result_index = list(func.keys())
  elif is_non_unique_col:
  # key used for column selection and output
@@ -429,7 +453,9 @@ def agg_dict_like(self) -> DataFrame | Series:
  label_to_indices[label].append(index)
 
  key_data = [
- selected_obj._ixs(indice, axis=1).agg(how, **kwargs)
+ getattr(selected_obj._ixs(indice, axis=1), op_name)(
+ how, **kwargs
+ )
  for label, indices in label_to_indices.items()
  for indice in indices
  ]
@@ -439,7 +465,7 @@ def agg_dict_like(self) -> DataFrame | Series:
  else:
  # key used for column selection and output
  result_data = [
- obj._gotitem(key, ndim=1).agg(how, **kwargs)
+ getattr(obj._gotitem(key, ndim=1), op_name)(how, **kwargs)
  for key, how in func.items()
  ]
  result_index = list(func.keys())
@@ -535,7 +561,7 @@ def apply_str(self) -> DataFrame | Series:
  self.kwargs["axis"] = self.axis
  return self._apply_str(obj, func, *self.args, **self.kwargs)
 
- def apply_multiple(self) -> DataFrame | Series:
+ def apply_list_or_dict_like(self) -> DataFrame | Series:
  """
  Compute apply in case of a list-like or dict-like.
 
@@ -551,9 +577,9 @@ def apply_multiple(self) -> DataFrame | Series:
  kwargs = self.kwargs
 
  if is_dict_like(func):
- result = self.agg_dict_like()
+ result = self.agg_or_apply_dict_like(op_name="apply")
  else:
- result = self.agg_list_like()
+ result = self.agg_or_apply_list_like(op_name="apply")
 
  result = reconstruct_and_relabel_result(result, func, **kwargs)
 
@@ -692,9 +718,9 @@ def values(self):
 
  def apply(self) -> DataFrame | Series:
  """compute the results"""
- # dispatch to agg
+ # dispatch to handle list-like or dict-like
  if is_list_like(self.func):
- return self.apply_multiple()
+ return self.apply_list_or_dict_like()
 
  # all empty
  if len(self.columns) == 0 and len(self.index) == 0:
@@ -1041,13 +1067,15 @@ def infer_to_same_shape(self, results: ResType, res_index: Index) -> DataFrame:
 class SeriesApply(NDFrameApply):
  obj: Series
  axis: AxisInt = 0
+ by_row: bool # only relevant for apply()
 
  def __init__(
  self,
  obj: Series,
  func: AggFuncType,
  *,
  convert_dtype: bool | lib.NoDefault = lib.no_default,
+ by_row: bool = True,
  args,
  kwargs,
  ) -> None:
@@ -1062,6 +1090,7 @@ def __init__(
  stacklevel=find_stack_level(),
  )
  self.convert_dtype = convert_dtype
+ self.by_row = by_row
 
  super().__init__(
  obj,
@@ -1078,9 +1107,9 @@ def apply(self) -> DataFrame | Series:
  if len(obj) == 0:
  return self.apply_empty_result()
 
- # dispatch to agg
+ # dispatch to handle list-like or dict-like
  if is_list_like(self.func):
- return self.apply_multiple()
+ return self.apply_list_or_dict_like()
 
  if isinstance(self.func, str):
  # if we are a string, try to dispatch
@@ -1126,6 +1155,8 @@ def apply_standard(self) -> DataFrame | Series:
  if isinstance(func, np.ufunc):
  with np.errstate(all="ignore"):
  return func(obj, *self.args, **self.kwargs)
+ elif not self.by_row:
+ return func(obj, *self.args, **self.kwargs)
 
  if self.args or self.kwargs:
  # _map_values does not support args/kwargs

pandas/core/series.py

@@ -4496,6 +4496,8 @@ def apply(
  func: AggFuncType,
  convert_dtype: bool | lib.NoDefault = lib.no_default,
  args: tuple[Any, ...] = (),
+ *,
+ by_row: bool = True,
  **kwargs,
  ) -> DataFrame | Series:
  """
@@ -4523,6 +4525,12 @@ def apply(
  instead if you want ``convert_dtype=False``.
  args : tuple
  Positional arguments passed to func after the series value.
+ by_row : bool, default True
+ If False, the func will be passed the whole Series at once.
+ If True, will func will be passed each element of the Series, like
+ Series.map (backward compatible).
+
+ .. versionadded:: 2.1.0
  **kwargs
  Additional keyword arguments passed to func.
 
@@ -4611,7 +4619,12 @@ def apply(
  dtype: float64
  """
  return SeriesApply(
- self, func, convert_dtype=convert_dtype, args=args, kwargs=kwargs
+ self,
+ func,
+ convert_dtype=convert_dtype,
+ by_row=by_row,
+ args=args,
+ kwargs=kwargs,
  ).apply()
 
  def _reindex_indexer(