DEPR: df.iloc[:, foo] = bar attempt to set inplace (#45333)

Author: jbrockmendel

doc/source/user_guide/10min.rst

@@ -328,6 +328,7 @@ Setting values by position:
 Setting by assigning with a NumPy array:
 
 .. ipython:: python
+ :okwarning:
 
  df.loc[:, "D"] = np.array([5] * len(df))
 

doc/source/whatsnew/v1.5.0.rst

@@ -533,6 +533,73 @@ As ``group_keys=True`` is the default value of :meth:`DataFrame.groupby` and
 raise a ``FutureWarning``. This can be silenced and the previous behavior
 retained by specifying ``group_keys=False``.
 
+.. _whatsnew_150.notable_bug_fixes.setitem_column_try_inplace:
+ _ see also _whatsnew_130.notable_bug_fixes.setitem_column_try_inplace
+
+Try operating inplace when setting values with ``loc`` and ``iloc``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Most of the time setting values with ``frame.iloc`` attempts to set values
+in-place, only falling back to inserting a new array if necessary. In the past,
+setting entire columns has been an exception to this rule:
+
+.. ipython:: python
+
+ values = np.arange(4).reshape(2, 2)
+ df = pd.DataFrame(values)
+ ser = df[0]
+
+*Old behavior*:
+
+.. code-block:: ipython
+
+ In [3]: df.iloc[:, 0] = np.array([10, 11])
+ In [4]: ser
+ Out[4]:
+ 0 0
+ 1 2
+ Name: 0, dtype: int64
+
+This behavior is deprecated. In a future version, setting an entire column with
+iloc will attempt to operate inplace.
+
+*Future behavior*:
+
+.. code-block:: ipython
+
+ In [3]: df.iloc[:, 0] = np.array([10, 11])
+ In [4]: ser
+ Out[4]:
+ 0 10
+ 1 11
+ Name: 0, dtype: int64
+
+To get the old behavior, use :meth:`DataFrame.__setitem__` directly:
+
+*Future behavior*:
+
+.. code-block:: ipython
+
+ In [5]: df[0] = np.array([21, 31])
+ In [4]: ser
+ Out[4]:
+ 0 10
+ 1 11
+ Name: 0, dtype: int64
+
+In the case where ``df.columns`` is not unique, use :meth:`DataFrame.isetitem`:
+
+*Future behavior*:
+
+.. code-block:: ipython
+
+ In [5]: df.columns = ["A", "A"]
+ In [5]: df.isetitem(0, np.array([21, 31]))
+ In [4]: ser
+ Out[4]:
+ 0 10
+ 1 11
+ Name: 0, dtype: int64
+
 .. _whatsnew_150.deprecations.numeric_only_default:
 
 ``numeric_only`` default value

pandas/core/frame.py

@@ -3687,6 +3687,29 @@ def _get_value(self, index, col, takeable: bool = False) -> Scalar:
  loc = engine.get_loc(index)
  return series._values[loc]
 
+ def isetitem(self, loc, value) -> None:
+ """
+ Set the given value in the column with position 'loc'.
+
+ This is a positional analogue to __setitem__.
+
+ Parameters
+ ----------
+ loc : int or sequence of ints
+ value : scalar or arraylike
+
+ Notes
+ -----
+ Unlike `frame.iloc[:, i] = value`, `frame.isetitem(loc, value)` will
+ _never_ try to set the values in place, but will always insert a new
+ array.
+
+ In cases where `frame.columns` is unique, this is equivalent to
+ `frame[frame.columns[i]] = value`.
+ """
+ arraylike = self._sanitize_column(value)
+ self._iset_item_mgr(loc, arraylike, inplace=False)
+
  def __setitem__(self, key, value):
  key = com.apply_if_callable(key, self)
 
@@ -5467,7 +5490,7 @@ def _replace_columnwise(
  target, value = mapping[ax[i]]
  newobj = ser.replace(target, value, regex=regex)
 
- res.iloc[:, i] = newobj
+ res._iset_item(i, newobj)
 
  if inplace:
  return

pandas/core/groupby/groupby.py

@@ -2257,7 +2257,13 @@ def sem(self, ddof: int = 1, numeric_only: bool | lib.NoDefault = lib.no_default
  counts = self.count()
  result_ilocs = result.columns.get_indexer_for(cols)
  count_ilocs = counts.columns.get_indexer_for(cols)
- result.iloc[:, result_ilocs] /= np.sqrt(counts.iloc[:, count_ilocs])
+ with warnings.catch_warnings():
+ # TODO(2.0): once iloc[:, foo] = bar depecation is enforced,
+ # this catching will be unnecessary
+ warnings.filterwarnings(
+ "ignore", ".*will attempt to set the values inplace.*"
+ )
+ result.iloc[:, result_ilocs] /= np.sqrt(counts.iloc[:, count_ilocs])
  return result
 
  @final

pandas/core/indexing.py

@@ -1937,24 +1937,25 @@ def _setitem_single_column(self, loc: int, value, plane_indexer):
  pi = plane_indexer
 
  ser = self.obj._ixs(loc, axis=1)
+ orig_values = ser._values
 
  # perform the equivalent of a setitem on the info axis
  # as we have a null slice or a slice with full bounds
  # which means essentially reassign to the columns of a
  # multi-dim object
  # GH#6149 (null slice), GH#10408 (full bounds)
  if com.is_null_slice(pi) or com.is_full_slice(pi, len(self.obj)):
- ser = value
+ pass
  elif (
  is_array_like(value)
  and is_exact_shape_match(ser, value)
  and not is_empty_indexer(pi)
  ):
  if is_list_like(pi):
- ser = value[np.argsort(pi)]
+ value = value[np.argsort(pi)]
  else:
  # in case of slice
- ser = value[pi]
+ value = value[pi]
  else:
  # set the item, first attempting to operate inplace, then
  # falling back to casting if necessary; see
@@ -1970,8 +1971,40 @@ def _setitem_single_column(self, loc: int, value, plane_indexer):
  self.obj._iset_item(loc, ser)
  return
 
- # reset the sliced object if unique
- self.obj._iset_item(loc, ser)
+ # We will not operate in-place, but will attempt to in the future.
+ # To determine whether we need to issue a FutureWarning, see if the
+ # setting in-place would work, i.e. behavior will change.
+ warn = can_hold_element(ser._values, value)
+ # Don't issue the warning yet, as we can still trim a few cases where
+ # behavior will not change.
+
+ self.obj._iset_item(loc, value)
+
+ if warn:
+ new_values = self.obj._ixs(loc, axis=1)._values
+
+ if (
+ isinstance(new_values, np.ndarray)
+ and isinstance(orig_values, np.ndarray)
+ and np.shares_memory(new_values, orig_values)
+ ):
+ # TODO: get something like tm.shares_memory working?
+ # The values were set inplace after all, no need to warn,
+ # e.g. test_rename_nocopy
+ pass
+ else:
+ warnings.warn(
+ "In a future version, `df.iloc[:, i] = newvals` will attempt "
+ "to set the values inplace instead of always setting a new "
+ "array. To retain the old behavior, use either "
+ "`df[df.columns[i]] = newvals` or, if columns are non-unique, "
+ "`df.isetitem(i, newvals)`",
+ FutureWarning,
+ stacklevel=find_stack_level(),
+ )
+ # TODO: how to get future behavior?
+ # TODO: what if we got here indirectly via loc?
+ return
 
  def _setitem_single_block(self, indexer, value, name: str):
  """
@@ -1981,7 +2014,6 @@ def _setitem_single_block(self, indexer, value, name: str):
 
  info_axis = self.obj._info_axis_number
  item_labels = self.obj._get_axis(info_axis)
-
  if isinstance(indexer, tuple):
 
  # if we are setting on the info axis ONLY
@@ -1996,7 +2028,9 @@ def _setitem_single_block(self, indexer, value, name: str):
  if len(item_labels.get_indexer_for([col])) == 1:
  # e.g. test_loc_setitem_empty_append_expands_rows
  loc = item_labels.get_loc(col)
- self.obj._iset_item(loc, value)
+ # Go through _setitem_single_column to get
+ # FutureWarning if relevant.
+ self._setitem_single_column(loc, value, indexer[0])
  return
 
  indexer = maybe_convert_ix(*indexer) # e.g. test_setitem_frame_align

pandas/tests/extension/base/setitem.py

@@ -1,6 +1,13 @@
 import numpy as np
 import pytest
 
+from pandas.core.dtypes.dtypes import (
+ DatetimeTZDtype,
+ IntervalDtype,
+ PandasDtype,
+ PeriodDtype,
+)
+
 import pandas as pd
 import pandas._testing as tm
 from pandas.tests.extension.base.base import BaseExtensionTests
@@ -361,6 +368,11 @@ def test_setitem_frame_2d_values(self, data):
  # GH#44514
  df = pd.DataFrame({"A": data})
 
+ # These dtypes have non-broken implementations of _can_hold_element
+ has_can_hold_element = isinstance(
+ data.dtype, (PandasDtype, PeriodDtype, IntervalDtype, DatetimeTZDtype)
+ )
+
  # Avoiding using_array_manager fixture
  # https://github.com/pandas-dev/pandas/pull/44514#discussion_r754002410
  using_array_manager = isinstance(df._mgr, pd.core.internals.ArrayManager)
@@ -369,13 +381,24 @@ def test_setitem_frame_2d_values(self, data):
 
  orig = df.copy()
 
- df.iloc[:] = df
+ msg = "will attempt to set the values inplace instead"
+ warn = None
+ if has_can_hold_element and not isinstance(data.dtype, PandasDtype):
+ # PandasDtype excluded because it isn't *really* supported.
+ warn = FutureWarning
+
+ with tm.assert_produces_warning(warn, match=msg):
+ df.iloc[:] = df
  self.assert_frame_equal(df, orig)
 
  df.iloc[:-1] = df.iloc[:-1]
  self.assert_frame_equal(df, orig)
 
- df.iloc[:] = df.values
+ if isinstance(data.dtype, DatetimeTZDtype):
+ # no warning bc df.values casts to object dtype
+ warn = None
+ with tm.assert_produces_warning(warn, match=msg):
+ df.iloc[:] = df.values
  self.assert_frame_equal(df, orig)
  if not using_array_manager:
  # GH#33457 Check that this setting occurred in-place

pandas/tests/frame/indexing/test_coercion.py

@@ -36,7 +36,10 @@ def test_loc_setitem_multiindex_columns(self, consolidate):
  A.loc[2:3, (1, slice(2, 3))] = np.ones((2, 2), dtype=np.float32)
  assert (A.dtypes == np.float32).all()
 
- A.loc[0:5, (1, slice(2, 3))] = np.ones((6, 2), dtype=np.float32)
+ msg = "will attempt to set the values inplace instead"
+ with tm.assert_produces_warning(FutureWarning, match=msg):
+ A.loc[0:5, (1, slice(2, 3))] = np.ones((6, 2), dtype=np.float32)
+
  assert (A.dtypes == np.float32).all()
 
  A.loc[:, (1, slice(2, 3))] = np.ones((6, 2), dtype=np.float32)
@@ -129,7 +132,10 @@ def test_iloc_setitem_unnecesssary_float_upcasting():
  orig = df.copy()
 
  values = df[0].values.reshape(2, 1)
- df.iloc[:, 0:1] = values
+
+ msg = "will attempt to set the values inplace instead"
+ with tm.assert_produces_warning(FutureWarning, match=msg):
+ df.iloc[:, 0:1] = values
 
  tm.assert_frame_equal(df, orig)
 

pandas/tests/frame/indexing/test_indexing.py

@@ -688,7 +688,7 @@ def test_getitem_setitem_boolean_multi(self):
  expected.loc[[0, 2], [1]] = 5
  tm.assert_frame_equal(df, expected)
 
- def test_getitem_setitem_float_labels(self):
+ def test_getitem_setitem_float_labels(self, using_array_manager):
  index = Index([1.5, 2, 3, 4, 5])
  df = DataFrame(np.random.randn(5, 5), index=index)
 
@@ -771,7 +771,10 @@ def test_getitem_setitem_float_labels(self):
  assert len(result) == 5
 
  cp = df.copy()
- cp.loc[1.0:5.0] = 0
+ warn = FutureWarning if using_array_manager else None
+ msg = "will attempt to set the values inplace"
+ with tm.assert_produces_warning(warn, match=msg):
+ cp.loc[1.0:5.0] = 0
  result = cp.loc[1.0:5.0]
  assert (result == 0).values.all()
 

pandas/tests/frame/indexing/test_setitem.py

@@ -399,10 +399,14 @@ def test_setitem_frame_length_0_str_key(self, indexer):
 
  def test_setitem_frame_duplicate_columns(self, using_array_manager):
  # GH#15695
+ warn = FutureWarning if using_array_manager else None
+ msg = "will attempt to set the values inplace"
+
  cols = ["A", "B", "C"] * 2
  df = DataFrame(index=range(3), columns=cols)
  df.loc[0, "A"] = (0, 3)
- df.loc[:, "B"] = (1, 4)
+ with tm.assert_produces_warning(warn, match=msg):
+ df.loc[:, "B"] = (1, 4)
  df["C"] = (2, 5)
  expected = DataFrame(
  [
@@ -769,7 +773,9 @@ def test_setitem_string_column_numpy_dtype_raising(self):
  def test_setitem_empty_df_duplicate_columns(self):
  # GH#38521
  df = DataFrame(columns=["a", "b", "b"], dtype="float64")
- df.loc[:, "a"] = list(range(2))
+ msg = "will attempt to set the values inplace instead"
+ with tm.assert_produces_warning(FutureWarning, match=msg):
+ df.loc[:, "a"] = list(range(2))
  expected = DataFrame(
  [[0, np.nan, np.nan], [1, np.nan, np.nan]], columns=["a", "b", "b"]
  )

pandas/tests/frame/indexing/test_where.py

@@ -365,7 +365,7 @@ def test_where_bug_transposition(self):
  result = a.where(do_not_replace, b)
  tm.assert_frame_equal(result, expected)
 
- def test_where_datetime(self):
+ def test_where_datetime(self, using_array_manager):
 
  # GH 3311
  df = DataFrame(
@@ -385,7 +385,11 @@ def test_where_datetime(self):
 
  expected = df.copy()
  expected.loc[[0, 1], "A"] = np.nan
- expected.loc[:, "C"] = np.nan
+
+ warn = FutureWarning if using_array_manager else None
+ msg = "will attempt to set the values inplace"
+ with tm.assert_produces_warning(warn, match=msg):
+ expected.loc[:, "C"] = np.nan
  tm.assert_frame_equal(result, expected)
 
  def test_where_none(self):
@@ -513,7 +517,7 @@ def test_where_axis_with_upcast(self):
  assert return_value is None
  tm.assert_frame_equal(result, expected)
 
- def test_where_axis_multiple_dtypes(self):
+ def test_where_axis_multiple_dtypes(self, using_array_manager):
  # Multiple dtypes (=> multiple Blocks)
  df = pd.concat(
  [
@@ -569,7 +573,10 @@ def test_where_axis_multiple_dtypes(self):
 
  d2 = df.copy().drop(1, axis=1)
  expected = df.copy()
- expected.loc[:, 1] = np.nan
+ warn = FutureWarning if using_array_manager else None
+ msg = "will attempt to set the values inplace"
+ with tm.assert_produces_warning(warn, match=msg):
+ expected.loc[:, 1] = np.nan
 
  result = df.where(mask, d2)
  tm.assert_frame_equal(result, expected)