REF: Simplify quantile, remove reduction from BlockManager (#24597)

Author: jbrockmendel

pandas/core/internals/blocks.py

@@ -42,6 +42,7 @@
 from pandas.core.indexing import check_setitem_lengths
 from pandas.core.internals.arrays import extract_array
 import pandas.core.missing as missing
+from pandas.core.nanops import nanpercentile
 
 from pandas.io.formats.printing import pprint_thing
 
@@ -1438,7 +1439,7 @@ def _unstack(self, unstacker_func, new_columns, n_rows, fill_value):
  blocks = [make_block(new_values, placement=new_placement)]
  return blocks, mask
 
- def quantile(self, qs, interpolation='linear', axis=0, axes=None):
+ def quantile(self, qs, interpolation='linear', axis=0):
  """
  compute the quantiles of the
 
@@ -1447,94 +1448,53 @@ def quantile(self, qs, interpolation='linear', axis=0, axes=None):
  qs: a scalar or list of the quantiles to be computed
  interpolation: type of interpolation, default 'linear'
  axis: axis to compute, default 0
- axes : BlockManager.axes
 
  Returns
  -------
- tuple of (axis, block)
-
+ Block
  """
- kw = {'interpolation': interpolation}
  values = self.get_values()
  values, _ = self._try_coerce_args(values, values)
 
- def _nanpercentile1D(values, mask, q, **kw):
- # mask is Union[ExtensionArray, ndarray]
- values = values[~mask]
-
- if len(values) == 0:
- if lib.is_scalar(q):
- return self._na_value
- else:
- return np.array([self._na_value] * len(q),
- dtype=values.dtype)
-
- return np.percentile(values, q, **kw)
-
- def _nanpercentile(values, q, axis, **kw):
-
- mask = isna(self.values)
- if not lib.is_scalar(mask) and mask.any():
- if self.ndim == 1:
- return _nanpercentile1D(values, mask, q, **kw)
- else:
- # for nonconsolidatable blocks mask is 1D, but values 2D
- if mask.ndim < values.ndim:
- mask = mask.reshape(values.shape)
- if axis == 0:
- values = values.T
- mask = mask.T
- result = [_nanpercentile1D(val, m, q, **kw) for (val, m)
- in zip(list(values), list(mask))]
- result = np.array(result, dtype=values.dtype, copy=False).T
- return result
- else:
- return np.percentile(values, q, axis=axis, **kw)
-
- from pandas import Float64Index
  is_empty = values.shape[axis] == 0
- if is_list_like(qs):
- ax = Float64Index(qs)
+ orig_scalar = not is_list_like(qs)
+ if orig_scalar:
+ # make list-like, unpack later
+ qs = [qs]
 
- if is_empty:
- if self.ndim == 1:
- result = self._na_value
- else:
- # create the array of na_values
- # 2d len(values) * len(qs)
- result = np.repeat(np.array([self._na_value] * len(qs)),
- len(values)).reshape(len(values),
- len(qs))
+ if is_empty:
+ if self.ndim == 1:
+ result = self._na_value
  else:
- result = _nanpercentile(values, np.array(qs) * 100,
- axis=axis, **kw)
-
- result = np.array(result, copy=False)
- if self.ndim > 1:
- result = result.T
-
+ # create the array of na_values
+ # 2d len(values) * len(qs)
+ result = np.repeat(np.array([self._na_value] * len(qs)),
+ len(values)).reshape(len(values),
+ len(qs))
  else:
+ mask = isna(self.values)
+ result = nanpercentile(values, np.array(qs) * 100,
+ axis=axis, na_value=self._na_value,
+ mask=mask, ndim=self.ndim,
+ interpolation=interpolation)
 
- if self.ndim == 1:
- ax = Float64Index([qs])
- else:
- ax = axes[0]
+ result = np.array(result, copy=False)
+ if self.ndim > 1:
+ result = result.T
 
- if is_empty:
- if self.ndim == 1:
- result = self._na_value
- else:
- result = np.array([self._na_value] * len(self))
- else:
- result = _nanpercentile(values, qs * 100, axis=axis, **kw)
+ if orig_scalar and not lib.is_scalar(result):
+ # result could be scalar in case with is_empty and self.ndim == 1
+ assert result.shape[-1] == 1, result.shape
+ result = result[..., 0]
+ result = lib.item_from_zerodim(result)
 
  ndim = getattr(result, 'ndim', None) or 0
  result = self._try_coerce_result(result)
  if lib.is_scalar(result):
- return ax, self.make_block_scalar(result)
- return ax, make_block(result,
-  placement=np.arange(len(result)),
-  ndim=ndim)
+ return self.make_block_scalar(result)
+ return make_block(result,
+ placement=np.arange(len(result)),
+ ndim=ndim)
 
  def _replace_coerce(self, to_replace, value, inplace=True, regex=False,
  convert=False, mask=None):

pandas/core/internals/managers.py

@@ -16,7 +16,7 @@
  maybe_promote)
 from pandas.core.dtypes.common import (
  _NS_DTYPE, is_datetimelike_v_numeric, is_extension_array_dtype,
- is_extension_type, is_numeric_v_string_like, is_scalar)
+ is_extension_type, is_list_like, is_numeric_v_string_like, is_scalar)
 import pandas.core.dtypes.concat as _concat
 from pandas.core.dtypes.generic import ABCExtensionArray, ABCSeries
 from pandas.core.dtypes.missing import isna
@@ -402,34 +402,47 @@ def apply(self, f, axes=None, filter=None, do_integrity_check=False,
  bm._consolidate_inplace()
  return bm
 
- def reduction(self, f, axis=0, consolidate=True, transposed=False,
-  **kwargs):
+ def quantile(self, axis=0, consolidate=True, transposed=False,
+ interpolation='linear', qs=None, numeric_only=None):
  """
- iterate over the blocks, collect and create a new block manager.
+ Iterate over blocks applying quantile reduction.
  This routine is intended for reduction type operations and
  will do inference on the generated blocks.
 
  Parameters
  ----------
- f: the callable or function name to operate on at the block level
  axis: reduction axis, default 0
  consolidate: boolean, default True. Join together blocks having same
  dtype
  transposed: boolean, default False
  we are holding transposed data
+ interpolation : type of interpolation, default 'linear'
+ qs : a scalar or list of the quantiles to be computed
+ numeric_only : ignored
 
  Returns
  -------
  Block Manager (new object)
-
  """
 
  if consolidate:
  self._consolidate_inplace()
 
+ def get_axe(block, qs, axes):
+ from pandas import Float64Index
+ if is_list_like(qs):
+ ax = Float64Index(qs)
+ elif block.ndim == 1:
+ ax = Float64Index([qs])
+ else:
+ ax = axes[0]
+ return ax
+
  axes, blocks = [], []
  for b in self.blocks:
- axe, block = getattr(b, f)(axis=axis, axes=self.axes, **kwargs)
+ block = b.quantile(axis=axis, qs=qs, interpolation=interpolation)
+
+ axe = get_axe(b, qs, axes=self.axes)
 
  axes.append(axe)
  blocks.append(block)
@@ -496,9 +509,6 @@ def isna(self, func, **kwargs):
  def where(self, **kwargs):
  return self.apply('where', **kwargs)
 
- def quantile(self, **kwargs):
- return self.reduction('quantile', **kwargs)
-
  def setitem(self, **kwargs):
  return self.apply('setitem', **kwargs)
 

pandas/core/nanops.py

@@ -1194,3 +1194,75 @@ def f(x, y):
 nanle = make_nancomp(operator.le)
 naneq = make_nancomp(operator.eq)
 nanne = make_nancomp(operator.ne)
+
+
+def _nanpercentile_1d(values, mask, q, na_value, interpolation):
+ """
+ Wraper for np.percentile that skips missing values, specialized to
+ 1-dimensional case.
+
+ Parameters
+ ----------
+ values : array over which to find quantiles
+ mask : ndarray[bool]
+ locations in values that should be considered missing
+ q : scalar or array of quantile indices to find
+ na_value : scalar
+ value to return for empty or all-null values
+ interpolation : str
+
+ Returns
+ -------
+ quantiles : scalar or array
+ """
+ # mask is Union[ExtensionArray, ndarray]
+ values = values[~mask]
+
+ if len(values) == 0:
+ if lib.is_scalar(q):
+ return na_value
+ else:
+ return np.array([na_value] * len(q),
+ dtype=values.dtype)
+
+ return np.percentile(values, q, interpolation=interpolation)
+
+
+def nanpercentile(values, q, axis, na_value, mask, ndim, interpolation):
+ """
+ Wraper for np.percentile that skips missing values.
+
+ Parameters
+ ----------
+ values : array over which to find quantiles
+ q : scalar or array of quantile indices to find
+ axis : {0, 1}
+ na_value : scalar
+ value to return for empty or all-null values
+ mask : ndarray[bool]
+ locations in values that should be considered missing
+ ndim : {1, 2}
+ interpolation : str
+
+ Returns
+ -------
+ quantiles : scalar or array
+ """
+ if not lib.is_scalar(mask) and mask.any():
+ if ndim == 1:
+ return _nanpercentile_1d(values, mask, q, na_value,
+ interpolation=interpolation)
+ else:
+ # for nonconsolidatable blocks mask is 1D, but values 2D
+ if mask.ndim < values.ndim:
+ mask = mask.reshape(values.shape)
+ if axis == 0:
+ values = values.T
+ mask = mask.T
+ result = [_nanpercentile_1d(val, m, q, na_value,
+ interpolation=interpolation)
+ for (val, m) in zip(list(values), list(mask))]
+ result = np.array(result, dtype=values.dtype, copy=False).T
+ return result
+ else:
+ return np.percentile(values, q, axis=axis, interpolation=interpolation)