BUG: combine_first does not retain dtypes with Timestamp DataFrames (#38145)

Author: arw2019

doc/source/whatsnew/v1.2.0.rst

@@ -554,6 +554,7 @@ Categorical
 
 Datetimelike
 ^^^^^^^^^^^^
+- Bug in :meth:`DataFrame.combine_first` that would convert datetime-like column on other :class:`DataFrame` to integer when the column is not present in original :class:`DataFrame` (:issue:`28481`)
 - Bug in :attr:`.DatetimeArray.date` where a ``ValueError`` would be raised with a read-only backing array (:issue:`33530`)
 - Bug in ``NaT`` comparisons failing to raise ``TypeError`` on invalid inequality comparisons (:issue:`35046`)
 - Bug in :class:`.DateOffset` where attributes reconstructed from pickle files differ from original objects when input values exceed normal ranges (e.g months=12) (:issue:`34511`)

pandas/core/frame.py

@@ -6386,7 +6386,7 @@ def combine(
  otherSeries = otherSeries.astype(new_dtype)
 
  arr = func(series, otherSeries)
- arr = maybe_downcast_to_dtype(arr, this_dtype)
+ arr = maybe_downcast_to_dtype(arr, new_dtype)
 
  result[col] = arr
 

pandas/tests/frame/methods/test_combine_first.py

@@ -103,6 +103,7 @@ def test_combine_first_mixed_bug(self):
  combined = frame1.combine_first(frame2)
  assert len(combined.columns) == 5
 
+ def test_combine_first_same_as_in_update(self):
  # gh 3016 (same as in update)
  df = DataFrame(
  [[1.0, 2.0, False, True], [4.0, 5.0, True, False]],
@@ -118,6 +119,7 @@ def test_combine_first_mixed_bug(self):
  df.loc[0, "A"] = 45
  tm.assert_frame_equal(result, df)
 
+ def test_combine_first_doc_example(self):
  # doc example
  df1 = DataFrame(
  {"A": [1.0, np.nan, 3.0, 5.0, np.nan], "B": [np.nan, 2.0, 3.0, np.nan, 6.0]}
@@ -134,38 +136,56 @@ def test_combine_first_mixed_bug(self):
  expected = DataFrame({"A": [1, 2, 3, 5, 3, 7.0], "B": [np.nan, 2, 3, 4, 6, 8]})
  tm.assert_frame_equal(result, expected)
 
- # GH3552, return object dtype with bools
+ def test_combine_first_return_obj_type_with_bools(self):
+ # GH3552
+
  df1 = DataFrame(
  [[np.nan, 3.0, True], [-4.6, np.nan, True], [np.nan, 7.0, False]]
  )
  df2 = DataFrame([[-42.6, np.nan, True], [-5.0, 1.6, False]], index=[1, 2])
 
- result = df1.combine_first(df2)[2]
- expected = Series([True, True, False], name=2)
- tm.assert_series_equal(result, expected)
-
- # GH 3593, converting datetime64[ns] incorrectly
- df0 = DataFrame(
- {"a": [datetime(2000, 1, 1), datetime(2000, 1, 2), datetime(2000, 1, 3)]}
- )
- df1 = DataFrame({"a": [None, None, None]})
- df2 = df1.combine_first(df0)
- tm.assert_frame_equal(df2, df0)
-
- df2 = df0.combine_first(df1)
- tm.assert_frame_equal(df2, df0)
-
- df0 = DataFrame(
- {"a": [datetime(2000, 1, 1), datetime(2000, 1, 2), datetime(2000, 1, 3)]}
- )
- df1 = DataFrame({"a": [datetime(2000, 1, 2), None, None]})
- df2 = df1.combine_first(df0)
- result = df0.copy()
- result.iloc[0, :] = df1.iloc[0, :]
- tm.assert_frame_equal(df2, result)
+ expected = Series([True, True, False], name=2, dtype=object)
+
+ result_12 = df1.combine_first(df2)[2]
+ tm.assert_series_equal(result_12, expected)
+
+ result_21 = df2.combine_first(df1)[2]
+ tm.assert_series_equal(result_21, expected)
+
+ @pytest.mark.parametrize(
+ "data1, data2, data_expected",
+ (
+ (
+ [datetime(2000, 1, 1), datetime(2000, 1, 2), datetime(2000, 1, 3)],
+ [None, None, None],
+ [datetime(2000, 1, 1), datetime(2000, 1, 2), datetime(2000, 1, 3)],
+ ),
+ (
+ [None, None, None],
+ [datetime(2000, 1, 1), datetime(2000, 1, 2), datetime(2000, 1, 3)],
+ [datetime(2000, 1, 1), datetime(2000, 1, 2), datetime(2000, 1, 3)],
+ ),
+ (
+ [datetime(2000, 1, 2), None, None],
+ [datetime(2000, 1, 1), datetime(2000, 1, 2), datetime(2000, 1, 3)],
+ [datetime(2000, 1, 2), datetime(2000, 1, 2), datetime(2000, 1, 3)],
+ ),
+ (
+ [datetime(2000, 1, 1), datetime(2000, 1, 2), datetime(2000, 1, 3)],
+ [datetime(2000, 1, 2), None, None],
+ [datetime(2000, 1, 1), datetime(2000, 1, 2), datetime(2000, 1, 3)],
+ ),
+ ),
+ )
+ def test_combine_first_convert_datatime_correctly(
+ self, data1, data2, data_expected
+ ):
+ # GH 3593
 
- df2 = df0.combine_first(df1)
- tm.assert_frame_equal(df2, df0)
+ df1, df2 = DataFrame({"a": data1}), DataFrame({"a": data2})
+ result = df1.combine_first(df2)
+ expected = DataFrame({"a": data_expected})
+ tm.assert_frame_equal(result, expected)
 
  def test_combine_first_align_nan(self):
  # GH 7509 (not fixed)
@@ -339,9 +359,14 @@ def test_combine_first_int(self):
  df1 = DataFrame({"a": [0, 1, 3, 5]}, dtype="int64")
  df2 = DataFrame({"a": [1, 4]}, dtype="int64")
 
- res = df1.combine_first(df2)
- tm.assert_frame_equal(res, df1)
- assert res["a"].dtype == "int64"
+ result_12 = df1.combine_first(df2)
+ expected_12 = DataFrame({"a": [0, 1, 3, 5]}, dtype="float64")
+ tm.assert_frame_equal(result_12, expected_12)
+
+ result_21 = df2.combine_first(df1)
+ expected_21 = DataFrame({"a": [1, 4, 3, 5]}, dtype="float64")
+
+ tm.assert_frame_equal(result_21, expected_21)
 
  @pytest.mark.parametrize("val", [1, 1.0])
  def test_combine_first_with_asymmetric_other(self, val):
@@ -367,6 +392,26 @@ def test_combine_first_string_dtype_only_na(self):
  tm.assert_frame_equal(result, expected)
 
 
+@pytest.mark.parametrize(
+ "scalar1, scalar2",
+ [
+ (datetime(2020, 1, 1), datetime(2020, 1, 2)),
+ (pd.Period("2020-01-01", "D"), pd.Period("2020-01-02", "D")),
+ (pd.Timedelta("89 days"), pd.Timedelta("60 min")),
+ (pd.Interval(left=0, right=1), pd.Interval(left=2, right=3, closed="left")),
+ ],
+)
+def test_combine_first_timestamp_bug(scalar1, scalar2, nulls_fixture):
+ # GH28481
+ na_value = nulls_fixture
+ frame = DataFrame([[na_value, na_value]], columns=["a", "b"])
+ other = DataFrame([[scalar1, scalar2]], columns=["b", "c"])
+
+ result = frame.combine_first(other)
+ expected = DataFrame([[na_value, scalar1, scalar2]], columns=["a", "b", "c"])
+ tm.assert_frame_equal(result, expected)
+
+
 def test_combine_first_with_nan_multiindex():
  # gh-36562