Merge remote-tracking branch 'upstream/master' into api-doc-methods

Author: TomAugspurger

doc/source/whatsnew/v0.21.1.txt

@@ -73,6 +73,7 @@ Conversion
 Indexing
 ^^^^^^^^
 
+- Bug in a boolean comparison of a ``datetime.datetime`` and a ``datetime64[ns]`` dtype Series (:issue:`17965`)
 - Bug where a ``MultiIndex`` with more than a million records was not raising ``AttributeError`` when trying to access a missing attribute (:issue:`18165`)
 -
 -
@@ -85,6 +86,7 @@ I/O
 - Bug in :func:`read_csv` for handling null values in index columns when specifying ``na_filter=False`` (:issue:`5239`)
 - Bug in :meth:`DataFrame.to_csv` when the table had ``MultiIndex`` columns, and a list of strings was passed in for ``header`` (:issue:`5539`)
 - :func:`read_parquet` now allows to specify the columns to read from a parquet file (:issue:`18154`)
+- :func:`read_parquet` now allows to specify kwargs which are passed to the respective engine (:issue:`18216`)
 
 Plotting
 ^^^^^^^^

doc/source/whatsnew/v0.22.0.txt

@@ -62,7 +62,7 @@ Removal of prior version deprecations/changes
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 - Warnings against the obsolete usage ``Categorical(codes, categories)``, which were emitted for instance when the first two arguments to ``Categorical()`` had different dtypes, and recommended the use of ``Categorical.from_codes``, have now been removed (:issue:`8074`)
--
+- The ``levels`` and ``labels`` attributes of a ``MultiIndex`` can no longer be set directly (:issue:`4039`).
 -
 
 .. _whatsnew_0220.performance:

pandas/_libs/index.pyx

@@ -19,7 +19,7 @@ from hashtable cimport HashTable
 
 from pandas._libs import algos, period as periodlib, hashtable as _hash
 from pandas._libs.tslib import Timestamp, Timedelta
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, date
 
 from cpython cimport PyTuple_Check, PyList_Check
 
@@ -549,7 +549,7 @@ cpdef convert_scalar(ndarray arr, object value):
  if arr.descr.type_num == NPY_DATETIME:
  if isinstance(value, np.ndarray):
  pass
- elif isinstance(value, datetime):
+ elif isinstance(value, (datetime, np.datetime64, date)):
  return Timestamp(value).value
  elif value is None or value != value:
  return iNaT

pandas/core/frame.py

@@ -4029,15 +4029,30 @@ def combine(self, other, func, fill_value=None, overwrite=True):
  ----------
  other : DataFrame
  func : function
+ Function that takes two series as inputs and return a Series or a
+ scalar
  fill_value : scalar value
  overwrite : boolean, default True
  If True then overwrite values for common keys in the calling frame
 
  Returns
  -------
  result : DataFrame
- """
 
+ Examples
+ --------
+ >>> df1 = DataFrame({'A': [0, 0], 'B': [4, 4]})
+ >>> df2 = DataFrame({'A': [1, 1], 'B': [3, 3]})
+ >>> df1.combine(df2, lambda s1, s2: s1 if s1.sum() < s2.sum() else s2)
+ A B
+ 0 0 3
+ 1 0 3
+
+ See Also
+ --------
+ DataFrame.combine_first : Combine two DataFrame objects and default to
+ non-null values in frame calling the method
+ """
  other_idxlen = len(other.index) # save for compare
 
  this, other = self.align(other, copy=False)
@@ -4125,16 +4140,24 @@ def combine_first(self, other):
  ----------
  other : DataFrame
 
+ Returns
+ -------
+ combined : DataFrame
+
  Examples
  --------
- a's values prioritized, use values from b to fill holes:
-
- >>> a.combine_first(b)
+ df1's values prioritized, use values from df2 to fill holes:
 
+ >>> df1 = pd.DataFrame([[1, np.nan]])
+ >>> df2 = pd.DataFrame([[3, 4]])
+ >>> df1.combine_first(df2)
+ 0 1
+ 0 1 4.0
 
- Returns
- -------
- combined : DataFrame
+ See Also
+ --------
+ DataFrame.combine : Perform series-wise operation on two DataFrames
+ using a given function
  """
  import pandas.core.computation.expressions as expressions
 
@@ -5782,7 +5805,12 @@ def idxmin(self, axis=0, skipna=True):
  0 or 'index' for row-wise, 1 or 'columns' for column-wise
  skipna : boolean, default True
  Exclude NA/null values. If an entire row/column is NA, the result
- will be NA
+ will be NA.
+
+ Raises
+ ------
+ ValueError
+ * If the row/column is empty
 
  Returns
  -------
@@ -5813,7 +5841,12 @@ def idxmax(self, axis=0, skipna=True):
  0 or 'index' for row-wise, 1 or 'columns' for column-wise
  skipna : boolean, default True
  Exclude NA/null values. If an entire row/column is NA, the result
- will be first index.
+ will be NA.
+
+ Raises
+ ------
+ ValueError
+ * If the row/column is empty
 
  Returns
  -------

pandas/core/indexes/multi.py

@@ -2,7 +2,6 @@
 # pylint: disable=E1101,E1103,W0232
 import datetime
 import warnings
-from functools import partial
 from sys import getsizeof
 
 import numpy as np
@@ -28,8 +27,7 @@
  is_true_slices)
 
 import pandas.core.base as base
-from pandas.util._decorators import (Appender, cache_readonly,
- deprecate, deprecate_kwarg)
+from pandas.util._decorators import Appender, cache_readonly, deprecate_kwarg
 import pandas.core.common as com
 import pandas.core.missing as missing
 import pandas.core.algorithms as algos
@@ -201,7 +199,8 @@ def _verify_integrity(self, labels=None, levels=None):
  " inconsistent state" % (i, label.max(),
  len(level)))
 
- def _get_levels(self):
+ @property
+ def levels(self):
  return self._levels
 
  def _set_levels(self, levels, level=None, copy=False, validate=True,
@@ -303,14 +302,8 @@ def set_levels(self, levels, level=None, inplace=False,
  if not inplace:
  return idx
 
- # remove me in 0.14 and change to read only property
- __set_levels = deprecate("setting `levels` directly",
- partial(set_levels, inplace=True,
- verify_integrity=True),
- alt_name="set_levels")
- levels = property(fget=_get_levels, fset=__set_levels)
-
- def _get_labels(self):
+ @property
+ def labels(self):
  return self._labels
 
  def _set_labels(self, labels, level=None, copy=False, validate=True,
@@ -403,13 +396,6 @@ def set_labels(self, labels, level=None, inplace=False,
  if not inplace:
  return idx
 
- # remove me in 0.14 and change to readonly property
- __set_labels = deprecate("setting labels directly",
- partial(set_labels, inplace=True,
- verify_integrity=True),
- alt_name="set_labels")
- labels = property(fget=_get_labels, fset=__set_labels)
-
  def copy(self, names=None, dtype=None, levels=None, labels=None,
  deep=False, _set_identity=False, **kwargs):
  """
@@ -844,9 +830,10 @@ def duplicated(self, keep='first'):
 
  return duplicated_int64(ids, keep)
 
- @Appender(ibase._index_shared_docs['fillna'])
  def fillna(self, value=None, downcast=None):
- # isna is not implemented for MultiIndex
+ """
+ fillna is not implemented for MultiIndex
+ """
  raise NotImplementedError('isna is not defined for MultiIndex')
 
  @Appender(_index_shared_docs['dropna'])

pandas/core/series.py

@@ -1306,7 +1306,13 @@ def idxmin(self, axis=None, skipna=True, *args, **kwargs):
  Parameters
  ----------
  skipna : boolean, default True
- Exclude NA/null values
+ Exclude NA/null values. If the entire Series is NA, the result
+ will be NA.
+
+ Raises
+ ------
+ ValueError
+ * If the Series is empty
 
  Returns
  -------
@@ -1336,7 +1342,13 @@ def idxmax(self, axis=None, skipna=True, *args, **kwargs):
  Parameters
  ----------
  skipna : boolean, default True
- Exclude NA/null values
+ Exclude NA/null values. If the entire Series is NA, the result
+ will be NA.
+
+ Raises
+ ------
+ ValueError
+ * If the Series is empty
 
  Returns
  -------
@@ -1361,13 +1373,13 @@ def idxmax(self, axis=None, skipna=True, *args, **kwargs):
 
  # ndarray compat
  argmin = deprecate('argmin', idxmin,
- msg="'argmin' is deprecated. Use 'idxmin' instead. "
+ msg="'argmin' is deprecated, use 'idxmin' instead. "
  "The behavior of 'argmin' will be corrected to "
  "return the positional minimum in the future. "
  "Use 'series.values.argmin' to get the position of "
  "the minimum now.")
  argmax = deprecate('argmax', idxmax,
- msg="'argmax' is deprecated. Use 'idxmax' instead. "
+ msg="'argmax' is deprecated, use 'idxmax' instead. "
  "The behavior of 'argmax' will be corrected to "
  "return the positional maximum in the future. "
  "Use 'series.values.argmax' to get the position of "
@@ -1731,11 +1743,26 @@ def combine(self, other, func, fill_value=np.nan):
  ----------
  other : Series or scalar value
  func : function
+ Function that takes two scalars as inputs and return a scalar
  fill_value : scalar value
 
  Returns
  -------
  result : Series
+
+ Examples
+ --------
+ >>> s1 = Series([1, 2])
+ >>> s2 = Series([0, 3])
+ >>> s1.combine(s2, lambda x1, x2: x1 if x1 < x2 else x2)
+ 0 0
+ 1 2
+ dtype: int64
+
+ See Also
+ --------
+ Series.combine_first : Combine Series values, choosing the calling
+ Series's values first
  """
  if isinstance(other, Series):
  new_index = self.index.union(other.index)
@@ -1764,7 +1791,21 @@ def combine_first(self, other):
 
  Returns
  -------
- y : Series
+ combined : Series
+
+ Examples
+ --------
+ >>> s1 = pd.Series([1, np.nan])
+ >>> s2 = pd.Series([3, 4])
+ >>> s1.combine_first(s2)
+ 0 1.0
+ 1 4.0
+ dtype: float64
+
+ See Also
+ --------
+ Series.combine : Perform elementwise operation on two Series
+ using a given function
  """
  new_index = self.index.union(other.index)
  this = self.reindex(new_index, copy=False)

pandas/io/parquet.py

@@ -76,9 +76,10 @@ def write(self, df, path, compression='snappy',
  table, path, compression=compression,
  coerce_timestamps=coerce_timestamps, **kwargs)
 
- def read(self, path, columns=None):
+ def read(self, path, columns=None, **kwargs):
  path, _, _ = get_filepath_or_buffer(path)
- return self.api.parquet.read_table(path, columns=columns).to_pandas()
+ return self.api.parquet.read_table(path, columns=columns,
+ **kwargs).to_pandas()
 
 
 class FastParquetImpl(object):
@@ -115,9 +116,9 @@ def write(self, df, path, compression='snappy', **kwargs):
  self.api.write(path, df,
  compression=compression, **kwargs)
 
- def read(self, path, columns=None):
+ def read(self, path, columns=None, **kwargs):
  path, _, _ = get_filepath_or_buffer(path)
- return self.api.ParquetFile(path).to_pandas(columns=columns)
+ return self.api.ParquetFile(path).to_pandas(columns=columns, **kwargs)
 
 
 def to_parquet(df, path, engine='auto', compression='snappy', **kwargs):
@@ -175,7 +176,7 @@ def to_parquet(df, path, engine='auto', compression='snappy', **kwargs):
  if df.columns.inferred_type not in valid_types:
  raise ValueError("parquet must have string column names")
 
- return impl.write(df, path, compression=compression)
+ return impl.write(df, path, compression=compression, **kwargs)
 
 
 def read_parquet(path, engine='auto', columns=None, **kwargs):
@@ -205,4 +206,4 @@ def read_parquet(path, engine='auto', columns=None, **kwargs):
  """
 
  impl = get_engine(engine)
- return impl.read(path, columns=columns)
+ return impl.read(path, columns=columns, **kwargs)

pandas/tests/indexes/datetimes/test_partial_slicing.py

@@ -2,9 +2,10 @@
 
 import pytest
 
-from datetime import datetime
+from datetime import datetime, date
 import numpy as np
 import pandas as pd
+import operator as op
 
 from pandas import (DatetimeIndex, Series, DataFrame,
  date_range, Index, Timedelta, Timestamp)
@@ -330,3 +331,21 @@ def test_loc_datetime_length_one(self):
 
  result = df.loc['2016-10-01T00:00:00':]
  tm.assert_frame_equal(result, df)
+
+ @pytest.mark.parametrize('datetimelike', [
+ Timestamp('20130101'), datetime(2013, 1, 1),
+ date(2013, 1, 1), np.datetime64('2013-01-01T00:00', 'ns')])
+ @pytest.mark.parametrize('op,expected', [
+ (op.lt, [True, False, False, False]),
+ (op.le, [True, True, False, False]),
+ (op.eq, [False, True, False, False]),
+ (op.gt, [False, False, False, True])])
+ def test_selection_by_datetimelike(self, datetimelike, op, expected):
+ # GH issue #17965, test for ability to compare datetime64[ns] columns
+ # to datetimelike
+ df = DataFrame({'A': [pd.Timestamp('20120101'),
+ pd.Timestamp('20130101'),
+ np.nan, pd.Timestamp('20130103')]})
+ result = op(df.A, datetimelike)
+ expected = Series(expected, name='A')
+ tm.assert_series_equal(result, expected)