code cleanup

Author: jreback

asv_bench/benchmarks/reindex.py

@@ -16,8 +16,8 @@ def setup(self):
  data=np.random.rand(10000, 30), columns=range(30))
 
  # multi-index
- N = 1000
- K = 20
+ N = 5000
+ K = 200
  level1 = tm.makeStringIndex(N).values.repeat(K)
  level2 = np.tile(tm.makeStringIndex(K).values, N)
  index = MultiIndex.from_arrays([level1, level2])

pandas/indexes/multi.py

@@ -667,14 +667,9 @@ def _has_complex_internals(self):
  @cache_readonly
  def is_monotonic(self):
 
- def level_values(level):
- unique = self.levels[level]
- labels = self.labels[level]
- return algos.take_1d(unique.values, labels,
- fill_value=unique._na_value)
-
  # reversed() because lexsort() wants the most significant key last.
- values = [level_values(i) for i in reversed(range(len(self.levels)))]
+ values = [self._get_level_values(i)
+ for i in reversed(range(len(self.levels)))]
  try:
  sort_order = np.lexsort(values)
  return Index(sort_order).is_monotonic
@@ -827,26 +822,44 @@ def _try_mi(k):
 
  raise InvalidIndexError(key)
 
- def get_level_values(self, level):
+ def _get_level_values(self, level):
  """
- Return vector of label values for requested level, equal to the length
- of the index
+ Return vector of label values for requested level,
+ equal to the length of the index
+
+ **this is an internal method**
 
  Parameters
  ----------
- level : int or level name
+ level : int level
 
  Returns
  -------
  values : ndarray
  """
- num = self._get_level_number(level)
- unique = self.levels[num] # .values
- labels = self.labels[num]
- filled = algos.take_1d(unique.values, labels,
+
+ unique = self.levels[level]
+ labels = self.labels[level]
+ filled = algos.take_1d(unique._values, labels,
  fill_value=unique._na_value)
- values = unique._shallow_copy(filled)
- return values
+ return filled
+
+ def get_level_values(self, level):
+ """
+ Return vector of label values for requested level,
+ equal to the length of the index
+
+ Parameters
+ ----------
+ level : int or level name
+
+ Returns
+ -------
+ values : Index
+ """
+ level = self._get_level_number(level)
+ values = self._get_level_values(level)
+ return self.levels[level]._shallow_copy(values)
 
  def format(self, space=2, sparsify=None, adjoin=True, names=False,
  na_rep=None, formatter=None):

pandas/tools/hashing.py

@@ -5,7 +5,6 @@
 
 import numpy as np
 from pandas import _hash, Series, factorize, Categorical, Index, MultiIndex
-import pandas.core.algorithms as algos
 from pandas.lib import is_bool_array
 from pandas.types.generic import ABCIndexClass, ABCSeries, ABCDataFrame
 from pandas.types.common import (is_categorical_dtype, is_numeric_dtype,
@@ -115,7 +114,7 @@ def hash_pandas_object(obj, index=True, encoding='utf8', hash_key=None,
  return h
 
 
-def hash_tuples(vals, encoding='utf8', hash_key=None):
+def hash_tuples(vals, encoding='utf8', hash_key=None, categorize=True):
  """
  Hash an MultiIndex / list-of-tuples efficiently
 
@@ -126,6 +125,9 @@ def hash_tuples(vals, encoding='utf8', hash_key=None):
  vals : MultiIndex, list-of-tuples, or single tuple
  encoding : string, default 'utf8'
  hash_key : string key to encode, default to _default_hash_key
+ categorize : bool, default True
+ Whether to first categorize object arrays before hashing. This is more
+ efficient when the array contains duplicate values.
 
  Returns
  -------
@@ -143,18 +145,14 @@ def hash_tuples(vals, encoding='utf8', hash_key=None):
  vals = MultiIndex.from_tuples(vals)
 
  # create a list-of-ndarrays
- def get_level_values(num):
- unique = vals.levels[num] # .values
- labels = vals.labels[num]
- filled = algos.take_1d(unique._values, labels,
- fill_value=unique._na_value)
- return filled
-
- vals = [get_level_values(level)
+ vals = [vals._get_level_values(level)
  for level in range(vals.nlevels)]
 
  # hash the list-of-ndarrays
- hashes = (hash_array(l, encoding=encoding, hash_key=hash_key)
+ hashes = (hash_array(l,
+ encoding=encoding,
+ hash_key=hash_key,
+ categorize=categorize)
  for l in vals)
  h = _combine_hash_arrays(hashes, len(vals))
  if is_tuple: