REF: ArrowEA _data->_pa_array (#50987)

Author: jbrockmendel

pandas/_testing/__init__.py

@@ -1028,8 +1028,8 @@ def shares_memory(left, right) -> bool:
  left = cast("ArrowExtensionArray", left)
  if isinstance(right, ExtensionArray) and right.dtype == "string[pyarrow]":
  right = cast("ArrowExtensionArray", right)
- left_pa_data = left._data
- right_pa_data = right._data
+ left_pa_data = left._pa_array
+ right_pa_data = right._pa_array
  left_buf1 = left_pa_data.chunk(0).buffers()[1]
  right_buf1 = right_pa_data.chunk(0).buffers()[1]
  return left_buf1 == right_buf1

pandas/core/arrays/arrow/array.py

@@ -5,6 +5,7 @@
  Callable,
  Union,
 )
+import warnings
 
 import numpy as np
 
@@ -18,6 +19,7 @@
  npt,
 )
 from pandas.compat import pa_version_under7p0
+from pandas.util._exceptions import find_stack_level
 
 from pandas.core.dtypes.common import (
  is_bool_dtype,
@@ -112,7 +114,7 @@ def __init__(self, values) -> None:
  super().__init__(values)
  self._dtype = StringDtype(storage="pyarrow")
 
- if not pa.types.is_string(self._data.type):
+ if not pa.types.is_string(self._pa_array.type):
  raise ValueError(
  "ArrowStringArray requires a PyArrow (chunked) array of string type"
  )
@@ -125,7 +127,7 @@ def __len__(self) -> int:
  -------
  length : int
  """
- return len(self._data)
+ return len(self._pa_array)
 
  @classmethod
  def _from_sequence(cls, scalars, dtype: Dtype | None = None, copy: bool = False):
@@ -193,7 +195,7 @@ def isin(self, values) -> npt.NDArray[np.bool_]:
  if not len(value_set):
  return np.zeros(len(self), dtype=bool)
 
- result = pc.is_in(self._data, value_set=pa.array(value_set))
+ result = pc.is_in(self._pa_array, value_set=pa.array(value_set))
  # pyarrow 2.0.0 returned nulls, so we explicily specify dtype to convert nulls
  # to False
  return np.array(result, dtype=np.bool_)
@@ -206,13 +208,24 @@ def astype(self, dtype, copy: bool = True):
  return self.copy()
  return self
  elif isinstance(dtype, NumericDtype):
- data = self._data.cast(pa.from_numpy_dtype(dtype.numpy_dtype))
+ data = self._pa_array.cast(pa.from_numpy_dtype(dtype.numpy_dtype))
  return dtype.__from_arrow__(data)
  elif isinstance(dtype, np.dtype) and np.issubdtype(dtype, np.floating):
  return self.to_numpy(dtype=dtype, na_value=np.nan)
 
  return super().astype(dtype, copy=copy)
 
+ @property
+ def _data(self):
+ # dask accesses ._data directlys
+ warnings.warn(
+ f"{type(self).__name__}._data is a deprecated and will be removed "
+ "in a future version, use ._pa_array instead",
+ FutureWarning,
+ stacklevel=find_stack_level(),
+ )
+ return self._pa_array
+
  # ------------------------------------------------------------------------
  # String methods interface
 
@@ -292,12 +305,12 @@ def _str_contains(
  fallback_performancewarning()
  return super()._str_contains(pat, case, flags, na, regex)
  else:
- result = pc.match_substring_regex(self._data, pat)
+ result = pc.match_substring_regex(self._pa_array, pat)
  else:
  if case:
- result = pc.match_substring(self._data, pat)
+ result = pc.match_substring(self._pa_array, pat)
  else:
- result = pc.match_substring(pc.utf8_upper(self._data), pat.upper())
+ result = pc.match_substring(pc.utf8_upper(self._pa_array), pat.upper())
  result = BooleanDtype().__from_arrow__(result)
  if not isna(na):
  result[isna(result)] = bool(na)
@@ -325,7 +338,7 @@ def _str_replace(
  return super()._str_replace(pat, repl, n, case, flags, regex)
 
  func = pc.replace_substring_regex if regex else pc.replace_substring
- result = func(self._data, pattern=pat, replacement=repl, max_replacements=n)
+ result = func(self._pa_array, pattern=pat, replacement=repl, max_replacements=n)
  return type(self)(result)
 
  def _str_match(
@@ -343,68 +356,68 @@ def _str_fullmatch(
  return self._str_match(pat, case, flags, na)
 
  def _str_isalnum(self):
- result = pc.utf8_is_alnum(self._data)
+ result = pc.utf8_is_alnum(self._pa_array)
  return BooleanDtype().__from_arrow__(result)
 
  def _str_isalpha(self):
- result = pc.utf8_is_alpha(self._data)
+ result = pc.utf8_is_alpha(self._pa_array)
  return BooleanDtype().__from_arrow__(result)
 
  def _str_isdecimal(self):
- result = pc.utf8_is_decimal(self._data)
+ result = pc.utf8_is_decimal(self._pa_array)
  return BooleanDtype().__from_arrow__(result)
 
  def _str_isdigit(self):
- result = pc.utf8_is_digit(self._data)
+ result = pc.utf8_is_digit(self._pa_array)
  return BooleanDtype().__from_arrow__(result)
 
  def _str_islower(self):
- result = pc.utf8_is_lower(self._data)
+ result = pc.utf8_is_lower(self._pa_array)
  return BooleanDtype().__from_arrow__(result)
 
  def _str_isnumeric(self):
- result = pc.utf8_is_numeric(self._data)
+ result = pc.utf8_is_numeric(self._pa_array)
  return BooleanDtype().__from_arrow__(result)
 
  def _str_isspace(self):
- result = pc.utf8_is_space(self._data)
+ result = pc.utf8_is_space(self._pa_array)
  return BooleanDtype().__from_arrow__(result)
 
  def _str_istitle(self):
- result = pc.utf8_is_title(self._data)
+ result = pc.utf8_is_title(self._pa_array)
  return BooleanDtype().__from_arrow__(result)
 
  def _str_isupper(self):
- result = pc.utf8_is_upper(self._data)
+ result = pc.utf8_is_upper(self._pa_array)
  return BooleanDtype().__from_arrow__(result)
 
  def _str_len(self):
- result = pc.utf8_length(self._data)
+ result = pc.utf8_length(self._pa_array)
  return Int64Dtype().__from_arrow__(result)
 
  def _str_lower(self):
- return type(self)(pc.utf8_lower(self._data))
+ return type(self)(pc.utf8_lower(self._pa_array))
 
  def _str_upper(self):
- return type(self)(pc.utf8_upper(self._data))
+ return type(self)(pc.utf8_upper(self._pa_array))
 
  def _str_strip(self, to_strip=None):
  if to_strip is None:
- result = pc.utf8_trim_whitespace(self._data)
+ result = pc.utf8_trim_whitespace(self._pa_array)
  else:
- result = pc.utf8_trim(self._data, characters=to_strip)
+ result = pc.utf8_trim(self._pa_array, characters=to_strip)
  return type(self)(result)
 
  def _str_lstrip(self, to_strip=None):
  if to_strip is None:
- result = pc.utf8_ltrim_whitespace(self._data)
+ result = pc.utf8_ltrim_whitespace(self._pa_array)
  else:
- result = pc.utf8_ltrim(self._data, characters=to_strip)
+ result = pc.utf8_ltrim(self._pa_array, characters=to_strip)
  return type(self)(result)
 
  def _str_rstrip(self, to_strip=None):
  if to_strip is None:
- result = pc.utf8_rtrim_whitespace(self._data)
+ result = pc.utf8_rtrim_whitespace(self._pa_array)
  else:
- result = pc.utf8_rtrim(self._data, characters=to_strip)
+ result = pc.utf8_rtrim(self._pa_array, characters=to_strip)
  return type(self)(result)

pandas/core/arrays/string_arrow.py

@@ -218,7 +218,7 @@ def isocalendar(self):
  result = (
  cast(ArrowExtensionArray, self._parent.array)
  ._dt_isocalendar()
- ._data.combine_chunks()
+ ._pa_array.combine_chunks()
  )
  iso_calendar_df = DataFrame(
  {

pandas/core/indexes/accessors.py

@@ -192,7 +192,9 @@ def test_astype_arrow_timestamp(using_copy_on_write):
  result = df.astype("timestamp[ns][pyarrow]")
  if using_copy_on_write:
  assert not result._mgr._has_no_reference(0)
- assert np.shares_memory(get_array(df, "a").asi8, get_array(result, "a")._data)
+ assert np.shares_memory(
+ get_array(df, "a").asi8, get_array(result, "a")._pa_array
+ )
 
 
 def test_convert_dtypes_infer_objects(using_copy_on_write):

pandas/tests/copy_view/test_astype.py

@@ -274,14 +274,14 @@ def test_from_dtype(self, data, request):
 
  def test_from_sequence_pa_array(self, data):
  # https://github.com/pandas-dev/pandas/pull/47034#discussion_r955500784
- # data._data = pa.ChunkedArray
- result = type(data)._from_sequence(data._data)
+ # data._pa_array = pa.ChunkedArray
+ result = type(data)._from_sequence(data._pa_array)
  tm.assert_extension_array_equal(result, data)
- assert isinstance(result._data, pa.ChunkedArray)
+ assert isinstance(result._pa_array, pa.ChunkedArray)
 
- result = type(data)._from_sequence(data._data.combine_chunks())
+ result = type(data)._from_sequence(data._pa_array.combine_chunks())
  tm.assert_extension_array_equal(result, data)
- assert isinstance(result._data, pa.ChunkedArray)
+ assert isinstance(result._pa_array, pa.ChunkedArray)
 
  def test_from_sequence_pa_array_notimplemented(self, request):
  with pytest.raises(NotImplementedError, match="Converting strings to"):
@@ -317,7 +317,7 @@ def test_from_sequence_of_strings_pa_array(self, data, request):
  ),
  )
  )
- pa_array = data._data.cast(pa.string())
+ pa_array = data._pa_array.cast(pa.string())
  result = type(data)._from_sequence_of_strings(pa_array, dtype=data.dtype)
  tm.assert_extension_array_equal(result, data)
 
@@ -1456,7 +1456,7 @@ def test_quantile(data, interpolation, quantile, request):
  or (pa.types.is_decimal(pa_dtype) and not pa_version_under7p0)
  ):
  pass
- elif pa.types.is_temporal(data._data.type):
+ elif pa.types.is_temporal(data._pa_array.type):
  pass
  else:
  request.node.add_marker(
@@ -1619,7 +1619,7 @@ def test_pickle_roundtrip(data):
 
 def test_astype_from_non_pyarrow(data):
  # GH49795
- pd_array = data._data.to_pandas().array
+ pd_array = data._pa_array.to_pandas().array
  result = pd_array.astype(data.dtype)
  assert not isinstance(pd_array.dtype, ArrowDtype)
  assert isinstance(result.dtype, ArrowDtype)
@@ -1638,11 +1638,11 @@ def test_to_numpy_with_defaults(data):
  # GH49973
  result = data.to_numpy()
 
- pa_type = data._data.type
+ pa_type = data._pa_array.type
  if pa.types.is_duration(pa_type) or pa.types.is_timestamp(pa_type):
  expected = np.array(list(data))
  else:
- expected = np.array(data._data)
+ expected = np.array(data._pa_array)
 
  if data._hasna:
  expected = expected.astype(object)
@@ -1668,7 +1668,7 @@ def test_setitem_null_slice(data):
  result = orig.copy()
  result[:] = data[0]
  expected = ArrowExtensionArray(
- pa.array([data[0]] * len(data), type=data._data.type)
+ pa.array([data[0]] * len(data), type=data._pa_array.type)
  )
  tm.assert_extension_array_equal(result, expected)
 
@@ -1685,7 +1685,7 @@ def test_setitem_null_slice(data):
 
 def test_setitem_invalid_dtype(data):
  # GH50248
- pa_type = data._data.type
+ pa_type = data._pa_array.type
  if pa.types.is_string(pa_type) or pa.types.is_binary(pa_type):
  fill_value = 123
  err = TypeError

pandas/tests/extension/test_arrow.py

@@ -35,7 +35,7 @@ def split_array(arr):
  def _split_array(arr):
  import pyarrow as pa
 
- arrow_array = arr._data
+ arrow_array = arr._pa_array
  split = len(arrow_array) // 2
  arrow_array = pa.chunked_array(
  [*arrow_array[:split].chunks, *arrow_array[split:].chunks]

pandas/tests/extension/test_string.py

@@ -578,7 +578,7 @@ def test_use_nullable_dtypes(self, read_ext, dtype_backend, option):
  )
  # pyarrow by default infers timestamp resolution as us, not ns
  expected["i"] = ArrowExtensionArray(
- expected["i"].array._data.cast(pa.timestamp(unit="us"))
+ expected["i"].array._pa_array.cast(pa.timestamp(unit="us"))
  )
  # pyarrow supports a null type, so don't have to default to Int64
  expected["j"] = ArrowExtensionArray(pa.array([None, None]))