Merge pull request #4714 from jreback/hdf_nan

API: for HDFStore, add the keyword dropna=True to append to change whether to write ALL nan rows to the store (GH4625)

Author: jreback

doc/source/release.rst

@@ -109,6 +109,9 @@ pandas 0.13
  be raised if you try to use ``mode='w'`` with an OPEN file handle (:issue:`4367`)
  - allow a passed locations array or mask as a ``where`` condition (:issue:`4467`)
  - the ``fmt`` keyword now replaces the ``table`` keyword; allowed values are ``s|t``
+ - add the keyword ``dropna=True`` to ``append`` to change whether ALL nan rows are not written
+ to the store (default is ``True``, ALL nan rows are NOT written), also settable
+ via the option ``io.hdf.dropna_table`` (:issue:`4625`)
  - ``JSON``
 
  - added ``date_unit`` parameter to specify resolution of timestamps. Options

doc/source/v0.13.0.txt

@@ -98,6 +98,9 @@ API changes
 
  import os
  os.remove(path)
+ - add the keyword ``dropna=True`` to ``append`` to change whether ALL nan rows are not written
+ to the store (default is ``True``, ALL nan rows are NOT written), also settable
+ via the option ``io.hdf.dropna_table`` (:issue:`4625`)
 
  - Changes to how ``Index`` and ``MultiIndex`` handle metadata (``levels``,
  ``labels``, and ``names``) (:issue:`4039`):

pandas/io/pytables.py

@@ -32,6 +32,7 @@
 from pandas.tools.merge import concat
 from pandas import compat
 from pandas.io.common import PerformanceWarning
+from pandas.core.config import get_option
 
 import pandas.lib as lib
 import pandas.algos as algos
@@ -165,6 +166,17 @@ class DuplicateWarning(Warning):
  Panel4D: [1, 2, 3],
 }
 
+# register our configuration options
+from pandas.core import config
+dropna_doc = """
+: boolean
+ drop ALL nan rows when appending to a table
+"""
+
+with config.config_prefix('io.hdf'):
+ config.register_option('dropna_table', True, dropna_doc,
+ validator=config.is_bool)
+
 # oh the troubles to reduce import time
 _table_mod = None
 _table_supports_index = False
@@ -730,7 +742,7 @@ def remove(self, key, where=None, start=None, stop=None):
  'can only remove with where on objects written as tables')
  return s.delete(where=where, start=start, stop=stop)
 
- def append(self, key, value, fmt=None, append=True, columns=None, **kwargs):
+ def append(self, key, value, fmt=None, append=True, columns=None, dropna=None, **kwargs):
  """
  Append to Table in file. Node must already exist and be Table
  format.
@@ -751,7 +763,8 @@ def append(self, key, value, fmt=None, append=True, columns=None, **kwargs):
  chunksize : size to chunk the writing
  expectedrows : expected TOTAL row size of this table
  encoding : default None, provide an encoding for strings
-
+ dropna : boolean, default True, do not write an ALL nan row to the store
+ settable by the option 'io.hdf.dropna_table'
  Notes
  -----
  Does *not* check if data being appended overlaps with existing
@@ -761,8 +774,10 @@ def append(self, key, value, fmt=None, append=True, columns=None, **kwargs):
  raise Exception(
  "columns is not a supported keyword in append, try data_columns")
 
+ if dropna is None:
+ dropna = get_option("io.hdf.dropna_table")
  kwargs = self._validate_format(fmt or 't', kwargs)
- self._write_to_group(key, value, append=append, **kwargs)
+ self._write_to_group(key, value, append=append, dropna=dropna, **kwargs)
 
  def append_to_multiple(self, d, value, selector, data_columns=None, axes=None, **kwargs):
  """
@@ -3219,7 +3234,7 @@ class AppendableTable(LegacyTable):
 
  def write(self, obj, axes=None, append=False, complib=None,
  complevel=None, fletcher32=None, min_itemsize=None, chunksize=None,
- expectedrows=None, **kwargs):
+ expectedrows=None, dropna=True, **kwargs):
 
  if not append and self.is_exists:
  self._handle.removeNode(self.group, 'table')
@@ -3254,29 +3269,36 @@ def write(self, obj, axes=None, append=False, complib=None,
  a.validate_and_set(table, append)
 
  # add the rows
- self.write_data(chunksize)
+ self.write_data(chunksize, dropna=dropna)
 
- def write_data(self, chunksize):
+ def write_data(self, chunksize, dropna=True):
  """ we form the data into a 2-d including indexes,values,mask
  write chunk-by-chunk """
 
  names = self.dtype.names
  nrows = self.nrows_expected
 
- # create the masks & values
- masks = []
- for a in self.values_axes:
+ # if dropna==True, then drop ALL nan rows
+ if dropna:
+
+ masks = []
+ for a in self.values_axes:
+
+ # figure the mask: only do if we can successfully process this
+ # column, otherwise ignore the mask
+ mask = com.isnull(a.data).all(axis=0)
+ masks.append(mask.astype('u1'))
 
- # figure the mask: only do if we can successfully process this
- # column, otherwise ignore the mask
- mask = com.isnull(a.data).all(axis=0)
- masks.append(mask.astype('u1'))
+ # consolidate masks
+ mask = masks[0]
+ for m in masks[1:]:
+ mask = mask & m
+ mask = mask.ravel()
+
+ else:
 
- # consolidate masks
- mask = masks[0]
- for m in masks[1:]:
- mask = mask & m
- mask = mask.ravel()
+ mask = np.empty(nrows, dtype='u1')
+ mask.fill(False)
 
  # broadcast the indexes if needed
  indexes = [a.cvalues for a in self.index_axes]

pandas/io/tests/test_pytables.py

@@ -757,45 +757,76 @@ def test_append_some_nans(self):
  store.append('df3', df3[10:])
  tm.assert_frame_equal(store['df3'], df3)
 
- ##### THIS IS A BUG, should not drop these all-nan rows
- ##### BUT need to store the index which we don't want to do....
- # nan some entire rows
+ def test_append_all_nans(self):
+
+ with ensure_clean(self.path) as store:
+
  df = DataFrame({'A1' : np.random.randn(20),
  'A2' : np.random.randn(20)},
  index=np.arange(20))
+ df.ix[0:15,:] = np.nan
+
+
+ # nan some entire rows (dropna=True)
+ _maybe_remove(store, 'df')
+ store.append('df', df[:10], dropna=True)
+ store.append('df', df[10:], dropna=True)
+ tm.assert_frame_equal(store['df'], df[-4:])
+
+ # nan some entire rows (dropna=False)
+ _maybe_remove(store, 'df2')
+ store.append('df2', df[:10], dropna=False)
+ store.append('df2', df[10:], dropna=False)
+ tm.assert_frame_equal(store['df2'], df)
+
+ # tests the option io.hdf.dropna_table
+ pandas.set_option('io.hdf.dropna_table',False)
+ _maybe_remove(store, 'df3')
+ store.append('df3', df[:10])
+ store.append('df3', df[10:])
+ tm.assert_frame_equal(store['df3'], df)
 
+ pandas.set_option('io.hdf.dropna_table',True)
  _maybe_remove(store, 'df4')
- df.ix[0:15,:] = np.nan
  store.append('df4', df[:10])
  store.append('df4', df[10:])
  tm.assert_frame_equal(store['df4'], df[-4:])
- self.assert_(store.get_storer('df4').nrows == 4)
 
  # nan some entire rows (string are still written!)
  df = DataFrame({'A1' : np.random.randn(20),
  'A2' : np.random.randn(20),
  'B' : 'foo', 'C' : 'bar'},
  index=np.arange(20))
 
- _maybe_remove(store, 'df5')
  df.ix[0:15,:] = np.nan
- store.append('df5', df[:10])
- store.append('df5', df[10:])
- tm.assert_frame_equal(store['df5'], df)
- self.assert_(store.get_storer('df5').nrows == 20)
+
+ _maybe_remove(store, 'df')
+ store.append('df', df[:10], dropna=True)
+ store.append('df', df[10:], dropna=True)
+ tm.assert_frame_equal(store['df'], df)
+
+ _maybe_remove(store, 'df2')
+ store.append('df2', df[:10], dropna=False)
+ store.append('df2', df[10:], dropna=False)
+ tm.assert_frame_equal(store['df2'], df)
 
  # nan some entire rows (but since we have dates they are still written!)
  df = DataFrame({'A1' : np.random.randn(20),
  'A2' : np.random.randn(20),
  'B' : 'foo', 'C' : 'bar', 'D' : Timestamp("20010101"), 'E' : datetime.datetime(2001,1,2,0,0) },
  index=np.arange(20))
 
- _maybe_remove(store, 'df6')
  df.ix[0:15,:] = np.nan
- store.append('df6', df[:10])
- store.append('df6', df[10:])
- tm.assert_frame_equal(store['df6'], df)
- self.assert_(store.get_storer('df6').nrows == 20)
+
+ _maybe_remove(store, 'df')
+ store.append('df', df[:10], dropna=True)
+ store.append('df', df[10:], dropna=True)
+ tm.assert_frame_equal(store['df'], df)
+
+ _maybe_remove(store, 'df2')
+ store.append('df2', df[:10], dropna=False)
+ store.append('df2', df[10:], dropna=False)
+ tm.assert_frame_equal(store['df2'], df)
 
  def test_append_frame_column_oriented(self):