COMPAT: Iteration should always yield a python scalar (pandas-dev#17491)

xref pandas-dev#10904 closes pandas-dev#13236 closes pandas-dev#13256 xref pandas-dev#14216

Author: jreback

doc/source/whatsnew/v0.21.0.txt

@@ -188,6 +188,53 @@ the target. Now, a ``ValueError`` will be raised when such an input is passed in
  ...
  ValueError: Cannot operate inplace if there is no assignment
 
+.. _whatsnew_0210.api_breaking.iteration_scalars:
+
+Iteration of Series/Index will now return python scalars
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Previously, when using certain iteration methods for a ``Series`` with dtype ``int`` or ``float``, you would receive a ``numpy`` scalar, e.g. a ``np.int64``, rather than a python ``int``. Issue (:issue:`10904`) corrected this for ``Series.tolist()`` and ``list(Series)``. This change makes all iteration methods consistent, in particular, for ``__iter__()`` and ``.map()``; note that this only affect int/float dtypes. (:issue:`13236`, :issue:`13258`, :issue:`14216`).
+
+.. ipython:: python
+
+ s = Series([1, 2, 3])
+ s
+
+Previously:
+
+.. code-block:: python
+
+ In [2]: type(list(s)[0])
+ Out[2]: numpy.int64
+
+New Behaviour:
+
+.. ipython:: python
+
+ type(list(s)[0])
+
+Furthermore this will now correctly box the results of iteration for :func:`DataFrame.to_dict` as well.
+
+.. ipython:: python
+
+ d = {'a':[1], 'b':['b']}
+ df = DataFrame(d)
+
+Previously:
+
+.. code-block:: python
+
+ In [8]: type(df.to_dict()['a'][0])
+ Out[8]: numpy.int64
+
+New Behaviour:
+
+.. ipython:: python
+
+ type(df.to_dict()['a'][0])
+
+.. _whatsnew_0210.api_breaking.dtype_conversions:
+
 Dtype Conversions
 ^^^^^^^^^^^^^^^^^
 

pandas/core/base.py

@@ -8,7 +8,12 @@
 
 from pandas.core.dtypes.missing import isna
 from pandas.core.dtypes.generic import ABCDataFrame, ABCSeries, ABCIndexClass
-from pandas.core.dtypes.common import is_object_dtype, is_list_like, is_scalar
+from pandas.core.dtypes.common import (
+ is_object_dtype,
+ is_list_like,
+ is_scalar,
+ is_datetimelike)
+
 from pandas.util._validators import validate_bool_kwarg
 
 from pandas.core import common as com
@@ -18,7 +23,8 @@
 from pandas.compat import PYPY
 from pandas.util._decorators import (Appender, cache_readonly,
  deprecate_kwarg, Substitution)
-from pandas.core.common import AbstractMethodError
+from pandas.core.common import AbstractMethodError, _maybe_box_datetimelike
+
 from pandas.core.accessor import DirNamesMixin
 
 _shared_docs = dict()
@@ -884,6 +890,21 @@ def argmin(self, axis=None):
  """
  return nanops.nanargmin(self.values)
 
+ def tolist(self):
+ """
+ return a list of the values; box to scalars
+ """
+ return list(self.__iter__())
+
+ def __iter__(self):
+ """
+ provide iteration over the values; box to scalars
+ """
+ if is_datetimelike(self):
+ return (_maybe_box_datetimelike(x) for x in self._values)
+ else:
+ return iter(self._values.tolist())
+
  @cache_readonly
  def hasnans(self):
  """ return if I have any nans; enables various perf speedups """

pandas/core/categorical.py

@@ -399,6 +399,12 @@ def itemsize(self):
  """ return the size of a single category """
  return self.categories.itemsize
 
+ def tolist(self):
+ """
+ return a list of my values
+ """
+ return np.array(self).tolist()
+
  def reshape(self, new_shape, *args, **kwargs):
  """
  .. deprecated:: 0.19.0

pandas/core/indexes/base.py

@@ -585,12 +585,6 @@ def memory_usage(self, deep=False):
  return result
 
  # ops compat
- def tolist(self):
- """
- return a list of the Index values
- """
- return list(self.values)
-
  @deprecate_kwarg(old_arg_name='n', new_arg_name='repeats')
  def repeat(self, repeats, *args, **kwargs):
  """
@@ -1601,9 +1595,6 @@ def is_all_dates(self):
  return False
  return is_datetime_array(_ensure_object(self.values))
 
- def __iter__(self):
- return iter(self.values)
-
  def __reduce__(self):
  d = dict(data=self._data)
  d.update(self._get_attributes_dict())

pandas/core/indexes/category.py

@@ -253,6 +253,10 @@ def get_values(self):
  """ return the underlying data as an ndarray """
  return self._data.get_values()
 
+ def __iter__(self):
+ """ iterate like Categorical """
+ return self._data.__iter__()
+
  @property
  def codes(self):
  return self._data.codes

pandas/core/series.py

@@ -19,7 +19,6 @@
  is_integer, is_integer_dtype,
  is_float_dtype,
  is_extension_type, is_datetimetz,
- is_datetimelike,
  is_datetime64tz_dtype,
  is_timedelta64_dtype,
  is_list_like,
@@ -1095,14 +1094,6 @@ def to_string(self, buf=None, na_rep='NaN', float_format=None, header=True,
  with open(buf, 'w') as f:
  f.write(result)
 
- def __iter__(self):
- """ provide iteration over the values of the Series
- box values if necessary """
- if is_datetimelike(self):
- return (_maybe_box_datetimelike(x) for x in self._values)
- else:
- return iter(self._values)
-
  def iteritems(self):
  """
  Lazily iterate over (index, value) tuples
@@ -1118,10 +1109,6 @@ def keys(self):
  """Alias for index"""
  return self.index
 
- def tolist(self):
- """ Convert Series to a nested list """
- return list(self.asobject)
-
  def to_dict(self, into=dict):
  """
  Convert Series to {label -> value} dict or dict-like object.

pandas/core/sparse/array.py

@@ -407,8 +407,18 @@ def to_dense(self, fill=None):
  return self.values
 
  def __iter__(self):
+ if np.issubdtype(self.dtype, np.floating):
+ boxer = float
+ elif np.issubdtype(self.dtype, np.integer):
+ boxer = int
+ else:
+ boxer = lambda x: x
+
  for i in range(len(self)):
- yield self._get_val_at(i)
+ r = self._get_val_at(i)
+
+ # box em
+ yield boxer(r)
 
  def __getitem__(self, key):
  """

pandas/tests/frame/test_api.py

@@ -9,7 +9,7 @@
 import sys
 from distutils.version import LooseVersion
 
-from pandas.compat import range, lrange
+from pandas.compat import range, lrange, long
 from pandas import compat
 
 from numpy.random import randn
@@ -205,15 +205,18 @@ def test_itertuples(self):
  'ints': lrange(5)}, columns=['floats', 'ints'])
 
  for tup in df.itertuples(index=False):
- assert isinstance(tup[1], np.integer)
+ assert isinstance(tup[1], (int, long))
 
  df = self.klass(data={"a": [1, 2, 3], "b": [4, 5, 6]})
  dfaa = df[['a', 'a']]
 
  assert (list(dfaa.itertuples()) ==
  [(0, 1, 1), (1, 2, 2), (2, 3, 3)])
- assert (repr(list(df.itertuples(name=None))) ==
- '[(0, 1, 4), (1, 2, 5), (2, 3, 6)]')
+
+ # repr with be int/long on windows
+ if not compat.is_platform_windows():
+ assert (repr(list(df.itertuples(name=None))) ==
+ '[(0, 1, 4), (1, 2, 5), (2, 3, 6)]')
 
  tup = next(df.itertuples(name='TestName'))
 

pandas/tests/frame/test_convert_to.py

@@ -5,6 +5,7 @@
 import numpy as np
 
 from pandas import compat
+from pandas.compat import long
 from pandas import (DataFrame, Series, MultiIndex, Timestamp,
  date_range)
 
@@ -236,3 +237,15 @@ def test_to_records_datetimeindex_with_tz(self, tz):
 
  # both converted to UTC, so they are equal
  tm.assert_numpy_array_equal(result, expected)
+
+ def test_to_dict_box_scalars(self):
+ # 14216
+ # make sure that we are boxing properly
+ d = {'a': [1], 'b': ['b']}
+
+ result = DataFrame(d).to_dict()
+ assert isinstance(list(result['a'])[0], (int, long))
+ assert isinstance(list(result['b'])[0], (int, long))
+
+ result = DataFrame(d).to_dict(orient='records')
+ assert isinstance(result[0]['a'], (int, long))

pandas/tests/series/test_io.py

@@ -10,7 +10,7 @@
 
 from pandas import Series, DataFrame
 
-from pandas.compat import StringIO, u, long
+from pandas.compat import StringIO, u
 from pandas.util.testing import (assert_series_equal, assert_almost_equal,
  assert_frame_equal, ensure_clean)
 import pandas.util.testing as tm
@@ -178,37 +178,3 @@ def test_to_dict(self, mapping):
  from_method = Series(ts.to_dict(collections.Counter))
  from_constructor = Series(collections.Counter(ts.iteritems()))
  tm.assert_series_equal(from_method, from_constructor)
-
-
-class TestSeriesToList(TestData):
-
- def test_tolist(self):
- rs = self.ts.tolist()
- xp = self.ts.values.tolist()
- assert_almost_equal(rs, xp)
-
- # datetime64
- s = Series(self.ts.index)
- rs = s.tolist()
- assert self.ts.index[0] == rs[0]
-
- def test_tolist_np_int(self):
- # GH10904
- for t in ['int8', 'int16', 'int32', 'int64']:
- s = pd.Series([1], dtype=t)
- assert isinstance(s.tolist()[0], (int, long))
-
- def test_tolist_np_uint(self):
- # GH10904
- for t in ['uint8', 'uint16']:
- s = pd.Series([1], dtype=t)
- assert isinstance(s.tolist()[0], int)
- for t in ['uint32', 'uint64']:
- s = pd.Series([1], dtype=t)
- assert isinstance(s.tolist()[0], long)
-
- def test_tolist_np_float(self):
- # GH10904
- for t in ['float16', 'float32', 'float64']:
- s = pd.Series([1], dtype=t)
- assert isinstance(s.tolist()[0], float)