check for hash collisions

Author: jreback

pandas/indexes/multi.py

@@ -2181,7 +2181,9 @@ def equals(self, other):
  return True
 
  if not isinstance(other, Index):
- return False
+ if not isinstance(other, tuple):
+ return False
+ other = Index([other])
 
  if not isinstance(other, MultiIndex):
  return array_equivalent(self._values,

pandas/src/hashtable_class_helper.pxi.in

@@ -859,24 +859,50 @@ cdef class MultiIndexHashTable(HashTable):
  sizeof(size_t) + # vals
  sizeof(uint32_t)) # flags
 
+ def _check_for_collisions(self, int64_t[:] locs, object mi):
+ # validate that the locs map to the actual values
+ # provided in the mi
+ # we can only check if we *don't* have any missing values
+ # :<
+ cdef:
+ ndarray[int64_t] alocs
+
+ alocs = np.asarray(locs)
+ if (alocs!=-1).all():
+
+ result = self.mi.take(locs)
+ if not result.equals(mi):
+ raise ValueError("hash collision alert")
+
  def __contains__(self, object key):
  cdef:
  khiter_t k
  uint64_t value
 
  value = self.mi._hashed_indexing_key(key)
  k = kh_get_uint64(self.table, value)
- return k != self.table.n_buckets
+ if k != self.table.n_buckets:
+ loc = self.table.vals[k]
+ locs = np.array([loc], dtype=np.int64)
+ self._check_for_collisions(locs, key)
+ return True
+
+ return False
 
  cpdef get_item(self, object key):
  cdef:
  khiter_t k
  uint64_t value
+ int64_t[:] locs
+ Py_ssize_t loc
 
  value = self.mi._hashed_indexing_key(key)
  k = kh_get_uint64(self.table, value)
  if k != self.table.n_buckets:
- return self.table.vals[k]
+ loc = self.table.vals[k]
+ locs = np.array([loc], dtype=np.int64)
+ self._check_for_collisions(locs, key)
+ return loc
  else:
  raise KeyError(key)
 
@@ -927,6 +953,7 @@ cdef class MultiIndexHashTable(HashTable):
  else:
  locs[i] = -1
 
+ self._check_for_collisions(locs, mi)
  return np.asarray(locs)
 
  def unique(self, object mi):

pandas/tests/indexes/test_multi.py

@@ -1317,6 +1317,18 @@ def test_get_indexer_nearest(self):
  with tm.assertRaises(NotImplementedError):
  midx.get_indexer(['a'], method='pad', tolerance=2)
 
+ def test_hash_collisions(self):
+ # non-smoke test that we don't get hash collisions
+
+ index = MultiIndex.from_product([np.arange(1000), np.arange(1000)],
+ names=['one','two'])
+ result = index.get_indexer(index.values)
+ self.assert_numpy_array_equal(result, np.arange(len(index), dtype='int64'))
+
+ for i in [0, 1, len(index)-2, len(index)-1]:
+ result = index.get_loc(index[i])
+ self.assertEqual(result, i)
+
  def test_format(self):
  self.index.format()
  self.index[:0].format()