Merge pull request #3065 from jreback/replace

BUG/ENH: guarantee blocks will upcast as needed, and split as needed

Author: jreback

RELEASE.rst

@@ -79,7 +79,7 @@ pandas 0.11.0
  doesn't have nans, then an int will be returned)
  - backfill/pad/take/diff/ohlc will now support ``float32/int16/int8``
  operations
- - Integer block types will upcast as needed in where operations (GH2793_)
+ - Block types will upcast as needed in where/masking operations (GH2793_)
  - Series now automatically will try to set the correct dtype based on passed
  datetimelike objects (datetime/Timestamp)
 

pandas/core/common.py

@@ -221,6 +221,13 @@ def mask_missing(arr, values_to_mask):
  for x in nonna:
  if mask is None:
  mask = arr == x
+
+ # if x is a string and mask is not, then we get a scalar
+ # return value, which is not good
+ if not isinstance(mask,np.ndarray):
+ m = mask
+ mask = np.empty(arr.shape,dtype=np.bool)
+ mask.fill(m)
  else:
  mask = mask | (arr == x)
 
@@ -730,6 +737,11 @@ def _maybe_promote(dtype, fill_value=np.nan):
  dtype = np.complex128
  else:
  dtype = np.object_
+
+ # in case we have a string that looked like a number
+ if issubclass(np.dtype(dtype).type, basestring):
+ dtype = np.object_
+
  return dtype, fill_value
 
 

pandas/core/internals.py

@@ -4,7 +4,7 @@
 from numpy import nan
 import numpy as np
 
-from pandas.core.common import _possibly_downcast_to_dtype
+from pandas.core.common import _possibly_downcast_to_dtype, isnull
 from pandas.core.index import Index, _ensure_index, _handle_legacy_indexes
 from pandas.core.indexing import _check_slice_bounds, _maybe_convert_indices
 import pandas.core.common as com
@@ -260,32 +260,14 @@ def _try_cast_result(self, result):
  return result
 
  def replace(self, to_replace, value, inplace=False):
- new_values = self.values if inplace else self.values.copy()
- if self._can_hold_element(value):
- value = self._try_cast(value)
-
- if not isinstance(to_replace, (list, np.ndarray)):
- if self._can_hold_element(to_replace):
- to_replace = self._try_cast(to_replace)
- msk = com.mask_missing(new_values, to_replace)
- np.putmask(new_values, msk, value)
- else:
- try:
- to_replace = np.array(to_replace, dtype=self.dtype)
- msk = com.mask_missing(new_values, to_replace)
- np.putmask(new_values, msk, value)
- except Exception:
- to_replace = np.array(to_replace, dtype=object)
- for r in to_replace:
- if self._can_hold_element(r):
- r = self._try_cast(r)
- msk = com.mask_missing(new_values, to_replace)
- np.putmask(new_values, msk, value)
-
- if inplace:
- return self
- else:
- return make_block(new_values, self.items, self.ref_items)
+ """ replace the to_replace value with value, possible to create new blocks here
+ this is just a call to putmask """
+ mask = com.mask_missing(self.values, to_replace)
+ if not mask.any():
+ if inplace:
+ return [ self ]
+ return [ self.copy() ]
+ return self.putmask(mask, value, inplace=inplace)
 
  def putmask(self, mask, new, inplace=False):
  """ putmask the data to the block; it is possible that we may create a new dtype of block
@@ -309,19 +291,34 @@ def putmask(self, mask, new, inplace=False):
 
  # maybe upcast me
  elif mask.any():
- # type of the new block
- if ((isinstance(new, np.ndarray) and issubclass(new.dtype, np.number)) or
- isinstance(new, float)):
- typ = np.float64
- else:
- typ = np.object_
 
- # we need to exiplicty astype here to make a copy
- new_values = new_values.astype(typ)
+ # need to go column by column
+ new_blocks = []
+ for i, item in enumerate(self.items):
 
- # we create a new block type
- np.putmask(new_values, mask, new)
- return [ make_block(new_values, self.items, self.ref_items) ]
+ m = mask[i]
+
+ # need a new block
+ if m.any():
+
+ n = new[i] if isinstance(new, np.ndarray) else new
+
+ # type of the new block
+ dtype, _ = com._maybe_promote(np.array(n).dtype)
+
+ # we need to exiplicty astype here to make a copy
+ nv = new_values[i].astype(dtype)
+
+ # we create a new block type
+ np.putmask(nv, m, n)
+
+ else:
+ nv = new_values[i] if inplace else new_values[i].copy()
+
+ nv = _block_shape(nv)
+ new_blocks.append(make_block(nv, [ item ], self.ref_items))
+
+ return new_blocks
 
  if inplace:
  return [ self ]
@@ -350,7 +347,7 @@ def interpolate(self, method='pad', axis=0, inplace=False,
  if missing is None:
  mask = None
  else: # todo create faster fill func without masking
- mask = _mask_missing(transf(values), missing)
+ mask = com.mask_missing(transf(values), missing)
 
  if method == 'pad':
  com.pad_2d(transf(values), limit=limit, mask=mask)
@@ -532,31 +529,14 @@ def create_block(result, items, transpose = True):
  if len(result) == 1:
  result = np.repeat(result,self.shape[1:])
 
- result = result.reshape(((1,) + self.shape[1:]))
+ result = _block_shape(result,ndim=self.ndim,shape=self.shape[1:])
  result_blocks.append(create_block(result, item, transpose = False))
 
  return result_blocks
  else:
  result = func(cond,values,other)
  return create_block(result, self.items)
 
-def _mask_missing(array, missing_values):
- if not isinstance(missing_values, (list, np.ndarray)):
- missing_values = [missing_values]
-
- mask = None
- missing_values = np.array(missing_values, dtype=object)
- if com.isnull(missing_values).any():
- mask = com.isnull(array)
- missing_values = missing_values[com.notnull(missing_values)]
-
- for v in missing_values:
- if mask is None:
- mask = array == missing_values
- else:
- mask |= array == missing_values
- return mask
-
 class NumericBlock(Block):
  is_numeric = True
  _can_hold_na = True
@@ -659,7 +639,7 @@ def convert(self, convert_dates = True, convert_numeric = True, copy = True):
  values = self.get(c)
 
  values = com._possibly_convert_objects(values, convert_dates=convert_dates, convert_numeric=convert_numeric)
- values = values.reshape(((1,) + values.shape))
+ values = _block_shape(values)
  items = self.items.take([i])
  newb = make_block(values, items, self.ref_items)
  blocks.append(newb)
@@ -949,23 +929,37 @@ def replace(self, *args, **kwargs):
 
  def replace_list(self, src_lst, dest_lst, inplace=False):
  """ do a list replace """
- if not inplace:
- self = self.copy()
-
- sset = set(src_lst)
- if any([k in sset for k in dest_lst]):
- masks = {}
- for s in src_lst:
- masks[s] = [b.values == s for b in self.blocks]
-
- for s, d in zip(src_lst, dest_lst):
- [b.putmask(masks[s][i], d, inplace=True) for i, b in
- enumerate(self.blocks)]
- else:
- for s, d in zip(src_lst, dest_lst):
- self.replace(s, d, inplace=True)
 
- return self
+ # figure out our mask a-priori to avoid repeated replacements
+ values = self.as_matrix()
+ def comp(s):
+ if isnull(s):
+ return isnull(values)
+ return values == s
+ masks = [ comp(s) for i, s in enumerate(src_lst) ]
+
+ result_blocks = []
+ for blk in self.blocks:
+
+ # its possible to get multiple result blocks here
+ # replace ALWAYS will return a list
+ rb = [ blk if inplace else blk.copy() ]
+ for i, d in enumerate(dest_lst):
+ new_rb = []
+ for b in rb:
+ # get our mask for this element, sized to this 
+ # particular block
+ m = masks[i][b.ref_locs]
+ if m.any():
+ new_rb.extend(b.putmask(m, d, inplace=True))
+ else:
+ new_rb.append(b)
+ rb = new_rb
+ result_blocks.extend(rb)
+
+ bm = self.__class__(result_blocks, self.axes)
+ bm._consolidate_inplace()
+ return bm
 
  def is_consolidated(self):
  """
@@ -1302,8 +1296,7 @@ def set(self, item, value):
  Set new item in-place. Does not consolidate. Adds new Block if not
  contained in the current set of items
  """
- if value.ndim == self.ndim - 1:
- value = value.reshape((1,) + value.shape)
+ value = _block_shape(value,self.ndim-1)
  if value.shape[1:] != self.shape[1:]:
  raise AssertionError('Shape of new values must be compatible '
  'with manager shape')
@@ -1873,6 +1866,14 @@ def _merge_blocks(blocks, items):
  return new_block.reindex_items_from(items)
 
 
+def _block_shape(values, ndim=1, shape=None):
+ """ guarantee the shape of the values to be at least 1 d """
+ if values.ndim == ndim:
+ if shape is None:
+ shape = values.shape
+ values = values.reshape(tuple((1,) + shape))
+ return values
+
 def _vstack(to_stack):
  if all(x.dtype == _NS_DTYPE for x in to_stack):
  # work around NumPy 1.6 bug

pandas/tests/test_frame.py

@@ -5596,6 +5596,31 @@ def test_replace_mixed(self):
  assert_frame_equal(result, expected)
  assert_frame_equal(result.replace(-1e8, nan), self.mixed_frame)
 
+ # int block upcasting
+ df = DataFrame({ 'A' : Series([1.0,2.0],dtype='float64'), 'B' : Series([0,1],dtype='int64') })
+ expected = DataFrame({ 'A' : Series([1.0,2.0],dtype='float64'), 'B' : Series([0.5,1],dtype='float64') })
+ result = df.replace(0, 0.5)
+ assert_frame_equal(result,expected)
+
+ df.replace(0, 0.5, inplace=True) 
+ assert_frame_equal(df,expected)
+
+ # int block splitting
+ df = DataFrame({ 'A' : Series([1.0,2.0],dtype='float64'), 'B' : Series([0,1],dtype='int64'), 'C' : Series([1,2],dtype='int64') })
+ expected = DataFrame({ 'A' : Series([1.0,2.0],dtype='float64'), 'B' : Series([0.5,1],dtype='float64'), 'C' : Series([1,2],dtype='int64') })
+ result = df.replace(0, 0.5)
+ assert_frame_equal(result,expected)
+
+ # to object block upcasting
+ df = DataFrame({ 'A' : Series([1.0,2.0],dtype='float64'), 'B' : Series([0,1],dtype='int64') })
+ expected = DataFrame({ 'A' : Series([1,'foo'],dtype='object'), 'B' : Series([0,1],dtype='int64') })
+ result = df.replace(2, 'foo')
+ assert_frame_equal(result,expected)
+
+ expected = DataFrame({ 'A' : Series(['foo','bar'],dtype='object'), 'B' : Series([0,'foo'],dtype='object') })
+ result = df.replace([1,2], ['foo','bar'])
+ assert_frame_equal(result,expected)
+
  def test_replace_interpolate(self):
  padded = self.tsframe.replace(nan, method='pad')
  assert_frame_equal(padded, self.tsframe.fillna(method='pad'))