Categorical: fix describe with np.nan

Author: jankatins

pandas/core/categorical.py

@@ -931,12 +931,25 @@ def describe(self):
  'values' : self._codes }
  ).groupby('codes').count()
 
- counts.index = self.levels.take(counts.index)
- counts = counts.reindex(self.levels)
  freqs = counts / float(counts.sum())
 
  from pandas.tools.merge import concat
  result = concat([counts,freqs],axis=1)
- result.index.name = 'levels'
  result.columns = ['counts','freqs']
+
+ # Up to now we have codes -> fill in the levels
+ # object in case we need to handle NaNs
+ levels = np.asarray(self.levels, dtype=object)
+ # use arange to also include not used levels
+ index = np.arange(0, len(levels))
+ # handle nan
+ if -1 in result.index:
+ # take[...,-1] returns the last element. So put np.nan there...
+ levels = np.append(levels, np.nan)
+ # also sort the -1 to the last position in the index
+ index = np.append(index, -1)
+ result = result.reindex(index)
+ result.index = levels.take(result.index)
+ result.index.name = 'levels'
+
  return result

pandas/tests/test_categorical.py

@@ -217,6 +217,16 @@ def test_describe(self):
  ).set_index('levels')
  tm.assert_frame_equal(desc, expected)
 
+ # check unused levels
+ cat = self.factor.copy()
+ cat.levels = ["a","b","c","d"]
+ desc = cat.describe()
+ expected = DataFrame.from_dict(dict(counts=[3, 2, 3, np.nan],
+ freqs=[3/8., 2/8., 3/8., np.nan],
+ levels=['a', 'b', 'c', 'd'])
+ ).set_index('levels')
+ tm.assert_frame_equal(desc, expected)
+
  # check an integer one
  desc = Categorical([1,2,3,1,2,3,3,2,1,1,1]).describe()
  expected = DataFrame.from_dict(dict(counts=[5, 3, 3],
@@ -226,6 +236,29 @@ def test_describe(self):
  ).set_index('levels')
  tm.assert_frame_equal(desc, expected)
 
+ # https://github.com/pydata/pandas/issues/3678
+ # describe should work with NaN
+ cat = pd.Categorical([np.nan,1, 2, 2])
+ desc = cat.describe()
+ expected = DataFrame.from_dict(dict(counts=[1, 2, 1],
+ freqs=[1/4., 2/4., 1/4.],
+ levels=[1,2,np.nan]
+ )
+ ).set_index('levels')
+ tm.assert_frame_equal(desc, expected)
+
+ # having NaN as level and as "not available" should also print two NaNs in describe!
+ cat = pd.Categorical([np.nan,1, 2, 2])
+ cat.levels = [1,2,np.nan]
+ desc = cat.describe()
+ expected = DataFrame.from_dict(dict(counts=[1, 2, np.nan, 1],
+ freqs=[1/4., 2/4., np.nan, 1/4.],
+ levels=[1,2,np.nan,np.nan]
+ )
+ ).set_index('levels')
+ tm.assert_frame_equal(desc, expected)
+
+
  def test_print(self):
  expected = [" a", " b", " b", " a", " a", " c", " c", " c",
  "Levels (3, object): [a < b < c]"]