BUG: Index._searchsorted_monotonic(..., side='right') returns the lef…

…t side position for monotonic decreasing indexes (#17272)

doc/source/whatsnew/v0.21.0.txt

@@ -400,6 +400,7 @@ Indexing
 - Bug in ``.isin()`` in which checking membership in empty ``Series`` objects raised an error (:issue:`16991`)
 - Bug in ``CategoricalIndex`` reindexing in which specified indices containing duplicates were not being respected (:issue:`17323`)
 - Bug in intersection of ``RangeIndex`` with negative step (:issue:`17296`)
+- Bug in ``IntervalIndex`` where performing a scalar lookup fails for included right endpoints of non-overlapping monotonic decreasing indexes (:issue:`16417`, :issue:`17271`)
 
 I/O
 ^^^

pandas/core/indexes/base.py

@@ -3465,7 +3465,7 @@ def _searchsorted_monotonic(self, label, side='left'):
  # everything for it to work (element ordering, search side and
  # resulting value).
  pos = self[::-1].searchsorted(label, side='right' if side == 'left'
- else 'right')
+ else 'left')
  return len(self) - pos
 
  raise ValueError('index must be monotonic increasing or decreasing')

pandas/tests/indexes/common.py

@@ -11,6 +11,7 @@
  RangeIndex, MultiIndex, CategoricalIndex, DatetimeIndex,
  TimedeltaIndex, PeriodIndex, IntervalIndex,
  notna, isna)
+from pandas.core.indexes.base import InvalidIndexError
 from pandas.core.indexes.datetimelike import DatetimeIndexOpsMixin
 from pandas.core.dtypes.common import needs_i8_conversion
 from pandas._libs.tslib import iNaT
@@ -138,9 +139,14 @@ def test_get_indexer_consistency(self):
  if isinstance(index, IntervalIndex):
  continue
 
- indexer = index.get_indexer(index[0:2])
- assert isinstance(indexer, np.ndarray)
- assert indexer.dtype == np.intp
+ if index.is_unique or isinstance(index, CategoricalIndex):
+ indexer = index.get_indexer(index[0:2])
+ assert isinstance(indexer, np.ndarray)
+ assert indexer.dtype == np.intp
+ else:
+ e = "Reindexing only valid with uniquely valued Index objects"
+ with tm.assert_raises_regex(InvalidIndexError, e):
+ indexer = index.get_indexer(index[0:2])
 
  indexer, _ = index.get_indexer_non_unique(index[0:2])
  assert isinstance(indexer, np.ndarray)
@@ -632,7 +638,8 @@ def test_difference_base(self):
  pass
  elif isinstance(idx, (DatetimeIndex, TimedeltaIndex)):
  assert result.__class__ == answer.__class__
- tm.assert_numpy_array_equal(result.asi8, answer.asi8)
+ tm.assert_numpy_array_equal(result.sort_values().asi8,
+ answer.sort_values().asi8)
  else:
  result = first.difference(case)
  assert tm.equalContents(result, answer)
@@ -954,3 +961,47 @@ def test_join_self_unique(self, how):
  if index.is_unique:
  joined = index.join(index, how=how)
  assert (index == joined).all()
+
+ def test_searchsorted_monotonic(self):
+ # GH17271
+ for index in self.indices.values():
+ # not implemented for tuple searches in MultiIndex
+ # or Intervals searches in IntervalIndex
+ if isinstance(index, (MultiIndex, IntervalIndex)):
+ continue
+
+ # nothing to test if the index is empty
+ if index.empty:
+ continue
+ value = index[0]
+
+ # determine the expected results (handle dupes for 'right')
+ expected_left, expected_right = 0, (index == value).argmin()
+ if expected_right == 0:
+ # all values are the same, expected_right should be length
+ expected_right = len(index)
+
+ # test _searchsorted_monotonic in all cases
+ # test searchsorted only for increasing
+ if index.is_monotonic_increasing:
+ ssm_left = index._searchsorted_monotonic(value, side='left')
+ assert expected_left == ssm_left
+
+ ssm_right = index._searchsorted_monotonic(value, side='right')
+ assert expected_right == ssm_right
+
+ ss_left = index.searchsorted(value, side='left')
+ assert expected_left == ss_left
+
+ ss_right = index.searchsorted(value, side='right')
+ assert expected_right == ss_right
+ elif index.is_monotonic_decreasing:
+ ssm_left = index._searchsorted_monotonic(value, side='left')
+ assert expected_left == ssm_left
+
+ ssm_right = index._searchsorted_monotonic(value, side='right')
+ assert expected_right == ssm_right
+ else:
+ # non-monotonic should raise.
+ with pytest.raises(ValueError):
+ index._searchsorted_monotonic(value, side='left')

pandas/tests/indexes/datetimes/test_datetimelike.py

@@ -12,7 +12,9 @@ class TestDatetimeIndex(DatetimeLike):
  _holder = DatetimeIndex
 
  def setup_method(self, method):
- self.indices = dict(index=tm.makeDateIndex(10))
+ self.indices = dict(index=tm.makeDateIndex(10),
+ index_dec=date_range('20130110', periods=10,
+ freq='-1D'))
  self.setup_indices()
 
  def create_index(self):

pandas/tests/indexes/period/test_period.py

@@ -18,7 +18,9 @@ class TestPeriodIndex(DatetimeLike):
  _multiprocess_can_split_ = True
 
  def setup_method(self, method):
- self.indices = dict(index=tm.makePeriodIndex(10))
+ self.indices = dict(index=tm.makePeriodIndex(10),
+ index_dec=period_range('20130101', periods=10,
+ freq='D')[::-1])
  self.setup_indices()
 
  def create_index(self):

pandas/tests/indexes/test_base.py

@@ -46,7 +46,8 @@ def setup_method(self, method):
  catIndex=tm.makeCategoricalIndex(100),
  empty=Index([]),
  tuples=MultiIndex.from_tuples(lzip(
- ['foo', 'bar', 'baz'], [1, 2, 3])))
+ ['foo', 'bar', 'baz'], [1, 2, 3])),
+ repeats=Index([0, 0, 1, 1, 2, 2]))
  self.setup_indices()
 
  def create_index(self):

pandas/tests/indexes/test_numeric.py

@@ -181,7 +181,9 @@ class TestFloat64Index(Numeric):
 
  def setup_method(self, method):
  self.indices = dict(mixed=Float64Index([1.5, 2, 3, 4, 5]),
- float=Float64Index(np.arange(5) * 2.5))
+ float=Float64Index(np.arange(5) * 2.5),
+ mixed_dec=Float64Index([5, 4, 3, 2, 1.5]),
+ float_dec=Float64Index(np.arange(4, -1, -1) * 2.5))
  self.setup_indices()
 
  def create_index(self):
@@ -654,7 +656,8 @@ class TestInt64Index(NumericInt):
  _holder = Int64Index
 
  def setup_method(self, method):
- self.indices = dict(index=Int64Index(np.arange(0, 20, 2)))
+ self.indices = dict(index=Int64Index(np.arange(0, 20, 2)),
+ index_dec=Int64Index(np.arange(19, -1, -1)))
  self.setup_indices()
 
  def create_index(self):
@@ -949,8 +952,9 @@ class TestUInt64Index(NumericInt):
  _holder = UInt64Index
 
  def setup_method(self, method):
- self.indices = dict(index=UInt64Index([2**63, 2**63 + 10, 2**63 + 15,
- 2**63 + 20, 2**63 + 25]))
+ vals = [2**63, 2**63 + 10, 2**63 + 15, 2**63 + 20, 2**63 + 25]
+ self.indices = dict(index=UInt64Index(vals),
+ index_dec=UInt64Index(reversed(vals)))
  self.setup_indices()
 
  def create_index(self):

pandas/tests/indexes/test_range.py

@@ -25,7 +25,8 @@ class TestRangeIndex(Numeric):
  _compat_props = ['shape', 'ndim', 'size', 'itemsize']
 
  def setup_method(self, method):
- self.indices = dict(index=RangeIndex(0, 20, 2, name='foo'))
+ self.indices = dict(index=RangeIndex(0, 20, 2, name='foo'),
+ index_dec=RangeIndex(18, -1, -2, name='bar'))
  self.setup_indices()
 
  def create_index(self):

pandas/tests/indexing/test_interval.py

@@ -3,6 +3,7 @@
 import pandas as pd
 
 from pandas import Series, DataFrame, IntervalIndex, Interval
+from pandas.compat import product
 import pandas.util.testing as tm
 
 
@@ -14,16 +15,6 @@ def setup_method(self, method):
  def test_loc_with_scalar(self):
 
  s = self.s
- expected = 0
-
- result = s.loc[0.5]
- assert result == expected
-
- result = s.loc[1]
- assert result == expected
-
- with pytest.raises(KeyError):
- s.loc[0]
 
  expected = s.iloc[:3]
  tm.assert_series_equal(expected, s.loc[:3])
@@ -42,16 +33,6 @@ def test_loc_with_scalar(self):
  def test_getitem_with_scalar(self):
 
  s = self.s
- expected = 0
-
- result = s[0.5]
- assert result == expected
-
- result = s[1]
- assert result == expected
-
- with pytest.raises(KeyError):
- s[0]
 
  expected = s.iloc[:3]
  tm.assert_series_equal(expected, s[:3])
@@ -67,6 +48,41 @@ def test_getitem_with_scalar(self):
  expected = s.iloc[2:5]
  tm.assert_series_equal(expected, s[s >= 2])
 
+ @pytest.mark.parametrize('direction, closed',
+ product(('increasing', 'decreasing'),
+ ('left', 'right', 'neither', 'both')))
+ def test_nonoverlapping_monotonic(self, direction, closed):
+ tpls = [(0, 1), (2, 3), (4, 5)]
+ if direction == 'decreasing':
+ tpls = reversed(tpls)
+
+ idx = IntervalIndex.from_tuples(tpls, closed=closed)
+ s = Series(list('abc'), idx)
+
+ for key, expected in zip(idx.left, s):
+ if idx.closed_left:
+ assert s[key] == expected
+ assert s.loc[key] == expected
+ else:
+ with pytest.raises(KeyError):
+ s[key]
+ with pytest.raises(KeyError):
+ s.loc[key]
+
+ for key, expected in zip(idx.right, s):
+ if idx.closed_right:
+ assert s[key] == expected
+ assert s.loc[key] == expected
+ else:
+ with pytest.raises(KeyError):
+ s[key]
+ with pytest.raises(KeyError):
+ s.loc[key]
+
+ for key, expected in zip(idx.mid, s):
+ assert s[key] == expected
+ assert s.loc[key] == expected
+
  def test_with_interval(self):
 
  s = self.s