BUG: Series creation with datetime64 with non-ns unit as object dtype

closes #11275 closes #11745

doc/source/whatsnew/v0.20.0.txt

@@ -27,7 +27,6 @@ Check the :ref:`API Changes <whatsnew_0200.api_breaking>` and :ref:`deprecations
 New features
 ~~~~~~~~~~~~
 
-
 .. _whatsnew_0200.enhancements.dataio_dtype:
 
 ``dtype`` keyword for data IO
@@ -55,6 +54,19 @@ fixed-width text files, and :func:`read_excel` for parsing Excel files.
  pd.read_fwf(StringIO(data)).dtypes
  pd.read_fwf(StringIO(data), dtype={'a':'float64', 'b':'object'}).dtypes
 
+.. _whatsnew_0120.enhancements.datetime_origin:
+
+to_datetime can be used with Offset
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+``pd.to_datetime`` has gained a new parameter, ``origin``, to define an offset
+from where to compute the resulting ``DatetimeIndex``. (:issue:`11276`, :issue:`11745`)
+
+.. ipython:: python
+
+ to_datetime([1,2,3], unit='D', origin=pd.Timestamp('1960-01-01'))
+
+The above code would return days with offset from origin as defined by timestamp set by origin.
+
 .. _whatsnew_0200.enhancements.groupby_access:
 
 Groupby Enhancements

pandas/tests/indexes/datetimes/test_tools.py

@@ -1515,3 +1515,47 @@ def test_normalize_date():
 
  result = normalize_date(value)
  assert (result == datetime(2012, 9, 7))
+
+
+def test_to_datetime_origin():
+ units = ['D', 's', 'ms', 'us', 'ns']
+ # gh-11276, gh-11745
+ # for origin as julian
+
+ julian_dates = pd.date_range(
+ '2014-1-1', periods=10).to_julian_date().values
+ result = Series(pd.to_datetime(
+ julian_dates, unit='D', origin='julian'))
+ expected = Series(pd.to_datetime(
+ julian_dates - pd.Timestamp(0).to_julian_date(), unit='D'))
+ assert_series_equal(result, expected)
+
+ # checking for invalid combination of origin='julian' and unit != D
+ for unit in units:
+ if unit == 'D':
+ continue
+ with pytest.raises(ValueError):
+ pd.to_datetime(julian_dates, unit=unit, origin='julian')
+
+ # for origin as 1960-01-01
+ epoch_1960 = pd.Timestamp('1960-01-01')
+ epoch_timestamp_convertible = [epoch_1960, epoch_1960.to_datetime(),
+ epoch_1960.to_datetime64(),
+ str(epoch_1960)]
+ invalid_origins = ['random_string', '13-24-1990', '0001-01-01']
+ units_from_epoch = [0, 1, 2, 3, 4]
+
+ for unit in units:
+ for epoch in epoch_timestamp_convertible:
+ expected = Series(
+ [pd.Timedelta(x, unit=unit) +
+ epoch_1960 for x in units_from_epoch])
+ result = Series(pd.to_datetime(
+ units_from_epoch, unit=unit, origin=epoch))
+ assert_series_equal(result, expected)
+
+ # check for invalid origins
+ for origin in invalid_origins:
+ with pytest.raises(ValueError):
+ pd.to_datetime(units_from_epoch, unit=unit,
+ origin=origin)

pandas/tseries/tools.py

@@ -176,8 +176,8 @@ def _guess_datetime_format_for_array(arr, **kwargs):
 
 
 def to_datetime(arg, errors='raise', dayfirst=False, yearfirst=False,
- utc=None, box=True, format=None, exact=True,
- unit=None, infer_datetime_format=False):
+ utc=None, box=True, format=None, exact=True, coerce=None,
+ unit=None, infer_datetime_format=False, origin='epoch'):
  """
  Convert argument to datetime.
 
@@ -236,6 +236,19 @@ def to_datetime(arg, errors='raise', dayfirst=False, yearfirst=False,
  datetime strings, and if it can be inferred, switch to a faster
  method of parsing them. In some cases this can increase the parsing
  speed by ~5-10x.
+ origin : scalar convertible to Timestamp / string ('julian', 'epoch'),
+ default 'epoch'.
+ Define reference date. The numeric values would be parsed as number
+ of units (defined by `unit`) since this reference date.
+
+ - If 'epoch', origin is set to 1970-01-01.
+ - If 'julian', unit must be 'D', and origin is set to beginning of
+ Julian Calendar. Julian day number 0 is assigned to the day starting
+ at noon on January 1, 4713 BC.
+ - If Timestamp convertible, origin is set to Timestamp identified by
+ origin.
+
+ .. versionadded: 0.19.0
 
  Returns
  -------
@@ -297,8 +310,14 @@ def to_datetime(arg, errors='raise', dayfirst=False, yearfirst=False,
  >>> %timeit pd.to_datetime(s,infer_datetime_format=False)
  1 loop, best of 3: 471 ms per loop
 
- """
+ Using non-epoch origins to parse date
+
+ >>> pd.to_datetime([1,2,3], unit='D', origin=pd.Timestamp('1960-01-01'))
+ 0 1960-01-02
+ 1 1960-01-03
+ 2 1960-01-04
 
+ """
  from pandas.tseries.index import DatetimeIndex
 
  tz = 'utc' if utc else None
@@ -409,22 +428,43 @@ def _convert_listlike(arg, box, format, name=None, tz=tz):
  except (ValueError, TypeError):
  raise e
 
- if arg is None:
- return arg
- elif isinstance(arg, tslib.Timestamp):
- return arg
- elif isinstance(arg, ABCSeries):
- from pandas import Series
- values = _convert_listlike(arg._values, False, format)
- return Series(values, index=arg.index, name=arg.name)
- elif isinstance(arg, (ABCDataFrame, MutableMapping)):
- return _assemble_from_unit_mappings(arg, errors=errors)
- elif isinstance(arg, ABCIndexClass):
- return _convert_listlike(arg, box, format, name=arg.name)
- elif is_list_like(arg):
- return _convert_listlike(arg, box, format)
+ def intermediate_result(arg):
+ if origin == 'julian':
+ if unit != 'D':
+ raise ValueError("unit must be 'D' for origin='julian'")
+ try:
+ arg = arg - tslib.Timestamp(0).to_julian_date()
+ except:
+ raise ValueError("incompatible 'arg' type for given "
+ "'origin'='julian'")
+ if arg is None:
+ return arg
+ elif isinstance(arg, tslib.Timestamp):
+ return arg
+ elif isinstance(arg, ABCSeries):
+ from pandas import Series
+ values = _convert_listlike(arg._values, False, format)
+ return Series(values, index=arg.index, name=arg.name)
+ elif isinstance(arg, (ABCDataFrame, MutableMapping)):
+ return _assemble_from_unit_mappings(arg, errors=errors)
+ elif isinstance(arg, ABCIndexClass):
+ return _convert_listlike(arg, box, format, name=arg.name)
+ elif is_list_like(arg):
+ return _convert_listlike(arg, box, format)
+ return _convert_listlike(np.array([arg]), box, format)[0]
+
+ result = intermediate_result(arg)
+
+ offset = None
+ if origin not in ['epoch', 'julian']:
+ try:
+ offset = tslib.Timestamp(origin) - tslib.Timestamp(0)
+ except ValueError:
+ raise ValueError("Invalid 'origin' or 'origin' Out of Bound")
 
- return _convert_listlike(np.array([arg]), box, format)[0]
+ if offset is not None:
+ result = result + offset
+ return result
 
 
 # mappings for assembling units