BUG: Better handle larger numbers in to_numeric (#24956)

* BUG: Better handle larger numbers in to_numeric * Warn about lossiness when passing really large numbers that exceed (u)int64 ranges. * Coerce negative numbers to float when requested instead of crashing and returning object. * Consistently parse numbers as integers / floats, even if we know that the resulting container has to be float. This is to ensure consistent error behavior when inputs numbers are too large. Closes gh-24910. * MAINT: Address comments

Author: gfyoung

doc/source/whatsnew/v0.25.0.rst

@@ -110,6 +110,8 @@ Timezones
 Numeric
 ^^^^^^^
 
+- Bug in :meth:`to_numeric` in which large negative numbers were being improperly handled (:issue:`24910`)
+- Bug in :meth:`to_numeric` in which numbers were being coerced to float, even though ``errors`` was not ``coerce`` (:issue:`24910`)
 -
 -
 -

pandas/_libs/lib.pyx

@@ -1828,7 +1828,7 @@ def maybe_convert_numeric(ndarray[object] values, set na_values,
  except (ValueError, OverflowError, TypeError):
  pass
 
- # otherwise, iterate and do full infererence
+ # Otherwise, iterate and do full inference.
  cdef:
  int status, maybe_int
  Py_ssize_t i, n = values.size
@@ -1865,10 +1865,10 @@ def maybe_convert_numeric(ndarray[object] values, set na_values,
  else:
  seen.float_ = True
 
- if val <= oINT64_MAX:
+ if oINT64_MIN <= val <= oINT64_MAX:
  ints[i] = val
 
- if seen.sint_ and seen.uint_:
+ if val < oINT64_MIN or (seen.sint_ and seen.uint_):
  seen.float_ = True
 
  elif util.is_bool_object(val):
@@ -1910,23 +1910,28 @@ def maybe_convert_numeric(ndarray[object] values, set na_values,
  else:
  seen.saw_int(as_int)
 
- if not (seen.float_ or as_int in na_values):
+ if as_int not in na_values:
  if as_int < oINT64_MIN or as_int > oUINT64_MAX:
- raise ValueError('Integer out of range.')
+ if seen.coerce_numeric:
+ seen.float_ = True
+ else:
+ raise ValueError("Integer out of range.")
+ else:
+ if as_int >= 0:
+ uints[i] = as_int
 
- if as_int >= 0:
- uints[i] = as_int
- if as_int <= oINT64_MAX:
- ints[i] = as_int
+ if as_int <= oINT64_MAX:
+ ints[i] = as_int
 
  seen.float_ = seen.float_ or (seen.uint_ and seen.sint_)
  else:
  seen.float_ = True
  except (TypeError, ValueError) as e:
  if not seen.coerce_numeric:
- raise type(e)(str(e) + ' at position {pos}'.format(pos=i))
+ raise type(e)(str(e) + " at position {pos}".format(pos=i))
  elif "uint64" in str(e): # Exception from check functions.
  raise
+
  seen.saw_null()
  floats[i] = NaN
 

pandas/core/tools/numeric.py

@@ -19,6 +19,14 @@ def to_numeric(arg, errors='raise', downcast=None):
  depending on the data supplied. Use the `downcast` parameter
  to obtain other dtypes.
 
+ Please note that precision loss may occur if really large numbers
+ are passed in. Due to the internal limitations of `ndarray`, if
+ numbers smaller than `-9223372036854775808` (np.iinfo(np.int64).min)
+ or larger than `18446744073709551615` (np.iinfo(np.uint64).max) are
+ passed in, it is very likely they will be converted to float so that
+ they can stored in an `ndarray`. These warnings apply similarly to
+ `Series` since it internally leverages `ndarray`.
+
  Parameters
  ----------
  arg : scalar, list, tuple, 1-d array, or Series

pandas/tests/tools/test_numeric.py

@@ -4,11 +4,50 @@
 from numpy import iinfo
 import pytest
 
+import pandas.compat as compat
+
 import pandas as pd
 from pandas import DataFrame, Index, Series, to_numeric
 from pandas.util import testing as tm
 
 
+@pytest.fixture(params=[None, "ignore", "raise", "coerce"])
+def errors(request):
+ return request.param
+
+
+@pytest.fixture(params=[True, False])
+def signed(request):
+ return request.param
+
+
+@pytest.fixture(params=[lambda x: x, str], ids=["identity", "str"])
+def transform(request):
+ return request.param
+
+
+@pytest.fixture(params=[
+ 47393996303418497800,
+ 100000000000000000000
+])
+def large_val(request):
+ return request.param
+
+
+@pytest.fixture(params=[True, False])
+def multiple_elts(request):
+ return request.param
+
+
+@pytest.fixture(params=[
+ (lambda x: Index(x, name="idx"), tm.assert_index_equal),
+ (lambda x: Series(x, name="ser"), tm.assert_series_equal),
+ (lambda x: np.array(Index(x).values), tm.assert_numpy_array_equal)
+])
+def transform_assert_equal(request):
+ return request.param
+
+
 @pytest.mark.parametrize("input_kwargs,result_kwargs", [
  (dict(), dict(dtype=np.int64)),
  (dict(errors="coerce", downcast="integer"), dict(dtype=np.int8))
@@ -172,7 +211,6 @@ def test_all_nan():
  tm.assert_series_equal(result, expected)
 
 
-@pytest.mark.parametrize("errors", [None, "ignore", "raise", "coerce"])
 def test_type_check(errors):
  # see gh-11776
  df = DataFrame({"a": [1, -3.14, 7], "b": ["4", "5", "6"]})
@@ -183,11 +221,100 @@ def test_type_check(errors):
  to_numeric(df, **kwargs)
 
 
-@pytest.mark.parametrize("val", [
- 1, 1.1, "1", "1.1", -1.5, "-1.5"
-])
-def test_scalar(val):
- assert to_numeric(val) == float(val)
+@pytest.mark.parametrize("val", [1, 1.1, 20001])
+def test_scalar(val, signed, transform):
+ val = -val if signed else val
+ assert to_numeric(transform(val)) == float(val)
+
+
+def test_really_large_scalar(large_val, signed, transform, errors):
+ # see gh-24910
+ kwargs = dict(errors=errors) if errors is not None else dict()
+ val = -large_val if signed else large_val
+
+ val = transform(val)
+ val_is_string = isinstance(val, str)
+
+ if val_is_string and errors in (None, "raise"):
+ msg = "Integer out of range. at position 0"
+ with pytest.raises(ValueError, match=msg):
+ to_numeric(val, **kwargs)
+ else:
+ expected = float(val) if (errors == "coerce" and
+ val_is_string) else val
+ assert tm.assert_almost_equal(to_numeric(val, **kwargs), expected)
+
+
+def test_really_large_in_arr(large_val, signed, transform,
+ multiple_elts, errors):
+ # see gh-24910
+ kwargs = dict(errors=errors) if errors is not None else dict()
+ val = -large_val if signed else large_val
+ val = transform(val)
+
+ extra_elt = "string"
+ arr = [val] + multiple_elts * [extra_elt]
+
+ val_is_string = isinstance(val, str)
+ coercing = errors == "coerce"
+
+ if errors in (None, "raise") and (val_is_string or multiple_elts):
+ if val_is_string:
+ msg = "Integer out of range. at position 0"
+ else:
+ msg = 'Unable to parse string "string" at position 1'
+
+ with pytest.raises(ValueError, match=msg):
+ to_numeric(arr, **kwargs)
+ else:
+ result = to_numeric(arr, **kwargs)
+
+ exp_val = float(val) if (coercing and val_is_string) else val
+ expected = [exp_val]
+
+ if multiple_elts:
+ if coercing:
+ expected.append(np.nan)
+ exp_dtype = float
+ else:
+ expected.append(extra_elt)
+ exp_dtype = object
+ else:
+ exp_dtype = float if isinstance(exp_val, (
+ int, compat.long, float)) else object
+
+ tm.assert_almost_equal(result, np.array(expected, dtype=exp_dtype))
+
+
+def test_really_large_in_arr_consistent(large_val, signed,
+ multiple_elts, errors):
+ # see gh-24910
+ #
+ # Even if we discover that we have to hold float, does not mean
+ # we should be lenient on subsequent elements that fail to be integer.
+ kwargs = dict(errors=errors) if errors is not None else dict()
+ arr = [str(-large_val if signed else large_val)]
+
+ if multiple_elts:
+ arr.insert(0, large_val)
+
+ if errors in (None, "raise"):
+ index = int(multiple_elts)
+ msg = "Integer out of range. at position {index}".format(index=index)
+
+ with pytest.raises(ValueError, match=msg):
+ to_numeric(arr, **kwargs)
+ else:
+ result = to_numeric(arr, **kwargs)
+
+ if errors == "coerce":
+ expected = [float(i) for i in arr]
+ exp_dtype = float
+ else:
+ expected = arr
+ exp_dtype = object
+
+ tm.assert_almost_equal(result, np.array(expected, dtype=exp_dtype))
 
 
 @pytest.mark.parametrize("errors,checker", [
@@ -205,15 +332,6 @@ def test_scalar_fail(errors, checker):
  assert checker(to_numeric(scalar, errors=errors))
 
 
-@pytest.fixture(params=[
- (lambda x: Index(x, name="idx"), tm.assert_index_equal),
- (lambda x: Series(x, name="ser"), tm.assert_series_equal),
- (lambda x: np.array(Index(x).values), tm.assert_numpy_array_equal)
-])
-def transform_assert_equal(request):
- return request.param
-
-
 @pytest.mark.parametrize("data", [
  [1, 2, 3],
  [1., np.nan, 3, np.nan]