MDEV-34413 Index Condition Pushdown for reverse ordered scans

Allows index condition pushdown for reverse ordered scans, a previously disabled feature due to poor performance. This patch adds a new API to the handler class called set_end_range which allows callers to tell the handler what the end of the index range will be when scanning. Combined with a pushed index condition, the handler can scan the index efficiently and not read beyond the end of the given range. When checking if the pushed index condition matches, the handler will also check if scanning has reached the end of the provided range and stop if so. If we instead only enabled ICP for reverse ordered scans without also calling this new API, then the handler would perform unnecessary index condition checks. In fact this would continue until the end of the index is reached. These changes are agnostic of storage engine. That is, any storage engine that supports index condition pushdown will inhereit this new behavior as it is implemented in the SQL and storage engine API layers. The partitioned tables storage meta-engine (ha_partition) adds an override of set_end_range which recursively calls set_end_range on its child storage engine (handler) implementations. This commit updates the test made in an earlier commit to show that ICP matches happen for the reverse ordered case. This patch is based on changes written by Olav Sandstaa in MySQL commit da1d92fd46071cd86de61058b6ea39fd9affcd87

Author: DaveGosselin-MariaDB

mysql-test/main/mdev-34413-icp-reverse-order.result

@@ -19,7 +19,7 @@ a	b	c
 10	10	10
 explain select * from t10 force index(a) where a between 10 and 20 and b+1 <3333 order by a desc, b desc;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
-1	SIMPLE	t10	range	a	a	5	NULL	11	Using where
+1	SIMPLE	t10	range	a	a	5	NULL	11	Using index condition
 flush status;
 select * from t10 force index(a) where a between 10 and 20 and b+1 <3333 order by a desc, b desc;
 a	b	c
@@ -36,8 +36,8 @@ a	b	c
 10	10	10
 SELECT * FROM information_schema.SESSION_STATUS WHERE VARIABLE_NAME LIKE '%icp%';
 VARIABLE_NAME	VARIABLE_VALUE
-HANDLER_ICP_ATTEMPTS0
-HANDLER_ICP_MATCH0
+HANDLER_ICP_ATTEMPTS11
+HANDLER_ICP_MATCH11
 select * from t10 force index(a) where a between 10 and 20 and b+1 <3333 order by a asc, b asc;
 a	b	c
 10	10	10
@@ -77,15 +77,15 @@ a	b	c
 10	10	10
 explain select * from t10 force index(a) where a=10 and b+1 <3333 order by a desc, b desc;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
-1	SIMPLE	t10	ref	a	a	5	const	1	Using where
+1	SIMPLE	t10	ref	a	a	5	const	1	Using index condition; Using where
 flush status;
 select * from t10 force index(a) where a=10 and b+1 <3333 order by a desc, b desc;
 a	b	c
 10	10	10
 SELECT * FROM information_schema.SESSION_STATUS WHERE VARIABLE_NAME LIKE '%icp%';
 VARIABLE_NAME	VARIABLE_VALUE
-HANDLER_ICP_ATTEMPTS0
-HANDLER_ICP_MATCH0
+HANDLER_ICP_ATTEMPTS1
+HANDLER_ICP_MATCH1
 select * from t10 force index(a) where a=10 and b+1 <3333 order by a asc, b asc;
 a	b	c
 10	10	10
@@ -105,15 +105,15 @@ a	b	c
 10	10	10
 explain select * from t10 force index(a) where a=10 and b+1 <3333 order by a asc, b desc;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
-1	SIMPLE	t10	ref	a	a	5	const	1	Using where
+1	SIMPLE	t10	ref	a	a	5	const	1	Using index condition; Using where
 flush status;
 select * from t10 force index(a) where a=10 and b+1 <3333 order by a asc, b desc;
 a	b	c
 10	10	10
 SELECT * FROM information_schema.SESSION_STATUS WHERE VARIABLE_NAME LIKE '%icp%';
 VARIABLE_NAME	VARIABLE_VALUE
-HANDLER_ICP_ATTEMPTS0
-HANDLER_ICP_MATCH0
+HANDLER_ICP_ATTEMPTS1
+HANDLER_ICP_MATCH1
 select * from t10 force index(a) where a=10 and b+1 <3333 order by a desc, b asc;
 a	b	c
 10	10	10
@@ -135,15 +135,15 @@ a	b	c
 3	30	300
 explain select * from t1 where a >= 3 and a <= 3 order by a desc, b desc;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
-1	SIMPLE	t1	range	a	a	5	NULL	1	Using where
+1	SIMPLE	t1	range	a	a	5	NULL	1	Using index condition
 flush status;
 select * from t1 where a >= 3 and a <= 3 order by a desc, b desc;
 a	b	c
 3	30	300
 SELECT * FROM information_schema.SESSION_STATUS WHERE VARIABLE_NAME LIKE '%icp%';
 VARIABLE_NAME	VARIABLE_VALUE
-HANDLER_ICP_ATTEMPTS0
-HANDLER_ICP_MATCH0
+HANDLER_ICP_ATTEMPTS1
+HANDLER_ICP_MATCH1
 select * from t1 where a >= 3 and a <= 3 order by a asc, b asc;
 a	b	c
 3	30	300
@@ -166,15 +166,15 @@ a	b	c
 2	20	200
 explain select * from t1 where a >= 2 and a <= 2 order by a desc, b desc;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
-1	SIMPLE	t1	range	a	a	5	NULL	1	Using where
+1	SIMPLE	t1	range	a	a	5	NULL	1	Using index condition
 flush status;
 select * from t1 where a >= 2 and a <= 2 order by a desc, b desc;
 a	b	c
 2	20	200
 SELECT * FROM information_schema.SESSION_STATUS WHERE VARIABLE_NAME LIKE '%icp%';
 VARIABLE_NAME	VARIABLE_VALUE
-HANDLER_ICP_ATTEMPTS0
-HANDLER_ICP_MATCH0
+HANDLER_ICP_ATTEMPTS1
+HANDLER_ICP_MATCH1
 select * from t1 where a >= 2 and a <= 2 order by a asc, b asc;
 a	b	c
 2	20	200
@@ -214,7 +214,7 @@ a	b	c
 3	3	3
 explain select * from t1 where a >= 3 and a <= 7 order by a desc;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
-1	SIMPLE	t1	range	a	a	4	NULL	5	Using where
+1	SIMPLE	t1	range	a	a	4	NULL	5	Using index condition
 flush status;
 select * from t1 where a >= 3 and a <= 7 order by a desc;
 a	b	c
@@ -225,8 +225,8 @@ a	b	c
 3	3	3
 SELECT * FROM information_schema.SESSION_STATUS WHERE VARIABLE_NAME LIKE '%icp%';
 VARIABLE_NAME	VARIABLE_VALUE
-HANDLER_ICP_ATTEMPTS0
-HANDLER_ICP_MATCH0
+HANDLER_ICP_ATTEMPTS6
+HANDLER_ICP_MATCH6
 select * from t1 where a >= 3 and a <= 7 order by a desc, b desc;
 a	b	c
 7	7	7
@@ -236,7 +236,7 @@ a	b	c
 3	3	3
 explain select * from t1 where a >= 3 and a <= 7 order by a desc, b desc;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
-1	SIMPLE	t1	range	a	a	4	NULL	5	Using where
+1	SIMPLE	t1	range	a	a	4	NULL	5	Using index condition
 flush status;
 select * from t1 where a >= 3 and a <= 7 order by a desc, b desc;
 a	b	c
@@ -247,8 +247,8 @@ a	b	c
 3	3	3
 SELECT * FROM information_schema.SESSION_STATUS WHERE VARIABLE_NAME LIKE '%icp%';
 VARIABLE_NAME	VARIABLE_VALUE
-HANDLER_ICP_ATTEMPTS0
-HANDLER_ICP_MATCH0
+HANDLER_ICP_ATTEMPTS6
+HANDLER_ICP_MATCH6
 drop table t1;
 create table t1 (
 pk int primary key,
@@ -273,6 +273,6 @@ select * from t1 where kp1 between 950 and 960 and kp2+1 >33333 order by kp1 des
 pk	kp1	kp2	col1
 SELECT * FROM information_schema.SESSION_STATUS WHERE VARIABLE_NAME LIKE '%icp%';
 VARIABLE_NAME	VARIABLE_VALUE
-HANDLER_ICP_ATTEMPTS0
+HANDLER_ICP_ATTEMPTS11
 HANDLER_ICP_MATCH	0
 drop table t1;

mysql-test/main/mdev-34413-icp-reverse-order.test

@@ -1,4 +1,3 @@
---source include/have_innodb.inc
 --source include/have_partition.inc
 --source include/have_sequence.inc
 

mysql-test/main/myisam_explain_non_select_all.result

@@ -1779,7 +1779,7 @@ FLUSH STATUS;
 FLUSH TABLES;
 EXPLAIN EXTENDED SELECT * FROM t2 WHERE i > 10 AND i <= 18 ORDER BY i DESC LIMIT 5;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
-1	SIMPLE	t2	range	PRIMARY	PRIMARY	4	NULL	8	100.00	Using where
+1	SIMPLE	t2	range	PRIMARY	PRIMARY	4	NULL	8	100.00	Using index condition
 Warnings:
 Note	1003	select `test`.`t2`.`a` AS `a`,`test`.`t2`.`i` AS `i` from `test`.`t2` where `test`.`t2`.`i` > 10 and `test`.`t2`.`i` <= 18 order by `test`.`t2`.`i` desc limit 5
 # Status of EXPLAIN EXTENDED "equivalent" SELECT query execution
@@ -2291,7 +2291,7 @@ FLUSH STATUS;
 FLUSH TABLES;
 EXPLAIN EXTENDED SELECT * FROM t2 WHERE i > 10 AND i <= 18 ORDER BY i DESC LIMIT 5;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
-1	SIMPLE	t2	range	PRIMARY	PRIMARY	4	NULL	8	100.00	Using where
+1	SIMPLE	t2	range	PRIMARY	PRIMARY	4	NULL	8	100.00	Using index condition
 Warnings:
 Note	1003	select `test`.`t2`.`a` AS `a`,`test`.`t2`.`i` AS `i` from `test`.`t2` where `test`.`t2`.`i` > 10 and `test`.`t2`.`i` <= 18 order by `test`.`t2`.`i` desc limit 5
 # Status of EXPLAIN EXTENDED "equivalent" SELECT query execution

mysql-test/main/myisam_icp.result

@@ -165,7 +165,7 @@ WHERE ts BETWEEN '0000-00-00' AND '2010-00-01 00:00:00'
 ORDER BY ts DESC
 LIMIT 2;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
-1	SIMPLE	t1	range	PRIMARY	PRIMARY	4	NULL	4	Using where
+1	SIMPLE	t1	range	PRIMARY	PRIMARY	4	NULL	4	Using index condition
 
 DROP TABLE t1;
 #
@@ -1016,12 +1016,12 @@ set optimizer_switch='mrr=off';
 # Must not use ICP:
 explain select * from t1 where a between 5 and 8 order by a desc, col desc;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
-1	SIMPLE	t1	range	a	a	5	NULL	40	Using where
+1	SIMPLE	t1	range	a	a	5	NULL	40	Using index condition
 set optimizer_switch= @tmp_10000051;
 # Must not use ICP:
 explain select * from t1 where a=3 and col > 500 order by a desc, col desc;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
-1	SIMPLE	t1	range	a	a	10	NULL	10	Using where
+1	SIMPLE	t1	range	a	a	10	NULL	10	Using index condition
 drop table t0, t1;
 #
 # MDEV-13628: ORed condition in pushed index condition is not removed from the WHERE

mysql-test/main/order_by_optimizer_innodb.result

@@ -42,11 +42,11 @@ pk1	count(*)
 # The following should use range(ux_pk1_fd5), two key parts (key_len=5+8=13)
 EXPLAIN SELECT * FROM t2 USE INDEX(ux_pk1_fd5) WHERE pk1=9 AND fd5 < 500 ORDER BY fd5 DESC LIMIT 10;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
-1	SIMPLE	t2	range	ux_pk1_fd5	ux_pk1_fd5	13	NULL	138	Using where
+1	SIMPLE	t2	range	ux_pk1_fd5	ux_pk1_fd5	13	NULL	138	Using index condition
 # This also must use range, not ref. key_len must be 13
 EXPLAIN SELECT * FROM t2 WHERE pk1=9 AND fd5 < 500 ORDER BY fd5 DESC LIMIT 10;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
-1	SIMPLE	t2	range	PRIMARY,ux_pk1_fd5	ux_pk1_fd5	13	NULL	138	Using where
+1	SIMPLE	t2	range	PRIMARY,ux_pk1_fd5	ux_pk1_fd5	13	NULL	138	Using index condition
 drop table t2;
 #
 # MDEV-6814: Server crashes in calculate_key_len on query with ORDER BY

mysql-test/main/rowid_filter_innodb.result

@@ -3826,15 +3826,18 @@ ANALYZE
  "loops": 1,
  "r_loops": 1,
  "rows": 250,
- "r_rows": 250,
+ "r_index_rows": 250,
+ "r_rows": 2,
  "cost": "REPLACED",
  "r_table_time_ms": "REPLACED",
  "r_other_time_ms": "REPLACED",
  "r_engine_stats": REPLACED,
  "filtered": 4.799086094,
  "r_total_filtered": 0.8,
- "attached_condition": "t1.a <=> 30100 and t1.b in (30100,30101,30102) and 30100 + t1.pk > 38539",
- "r_filtered": 0.8
+ "index_condition": "30100 + t1.pk > 38539",
+ "r_icp_filtered": 0.8,
+ "attached_condition": "t1.a <=> 30100 and t1.b in (30100,30101,30102)",
+ "r_filtered": 100
  }
  }
  ]
@@ -3870,15 +3873,18 @@ ANALYZE
  "loops": 1,
  "r_loops": 1,
  "rows": 250,
- "r_rows": 250,
+ "r_index_rows": 250,
+ "r_rows": 2,
  "cost": "REPLACED",
  "r_table_time_ms": "REPLACED",
  "r_other_time_ms": "REPLACED",
  "r_engine_stats": REPLACED,
  "filtered": 4.799086094,
  "r_total_filtered": 0.8,
- "attached_condition": "t1.a <=> 30100 and t1.b in (30100,30101,30102) and 30100 + t1.pk > 38539",
- "r_filtered": 0.8
+ "index_condition": "30100 + t1.pk > 38539",
+ "r_icp_filtered": 0.8,
+ "attached_condition": "t1.a <=> 30100 and t1.b in (30100,30101,30102)",
+ "r_filtered": 100
  }
  }
  ]

mysql-test/main/subselect.result

@@ -3197,7 +3197,7 @@ ON r.a = (SELECT t2.a FROM t2 WHERE t2.c = t1.a AND t2.b <= '359899'
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	PRIMARY	t1	system	PRIMARY	NULL	NULL	NULL	1
 1	PRIMARY	r	const	PRIMARY	PRIMARY	4	const	1
-2	SUBQUERY	t2	range	cb	cb	40	NULL	3	Using where
+2	SUBQUERY	t2	range	cb	cb	40	NULL	3	Using index condition
 SELECT sql_no_cache t1.a, r.a, r.b FROM t1 LEFT JOIN t2 r
 ON r.a = (SELECT t2.a FROM t2 WHERE t2.c = t1.a AND t2.b <= '359899'
  ORDER BY t2.c DESC, t2.b DESC LIMIT 1) WHERE t1.a = 10;

mysql-test/main/subselect_no_exists_to_in.result

@@ -3200,7 +3200,7 @@ ON r.a = (SELECT t2.a FROM t2 WHERE t2.c = t1.a AND t2.b <= '359899'
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	PRIMARY	t1	system	PRIMARY	NULL	NULL	NULL	1
 1	PRIMARY	r	const	PRIMARY	PRIMARY	4	const	1
-2	SUBQUERY	t2	range	cb	cb	40	NULL	3	Using where
+2	SUBQUERY	t2	range	cb	cb	40	NULL	3	Using index condition
 SELECT sql_no_cache t1.a, r.a, r.b FROM t1 LEFT JOIN t2 r
 ON r.a = (SELECT t2.a FROM t2 WHERE t2.c = t1.a AND t2.b <= '359899'
  ORDER BY t2.c DESC, t2.b DESC LIMIT 1) WHERE t1.a = 10;

mysql-test/main/subselect_no_mat.result

@@ -3202,7 +3202,7 @@ ON r.a = (SELECT t2.a FROM t2 WHERE t2.c = t1.a AND t2.b <= '359899'
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	PRIMARY	t1	system	PRIMARY	NULL	NULL	NULL	1
 1	PRIMARY	r	const	PRIMARY	PRIMARY	4	const	1
-2	SUBQUERY	t2	range	cb	cb	40	NULL	3	Using where
+2	SUBQUERY	t2	range	cb	cb	40	NULL	3	Using index condition
 SELECT sql_no_cache t1.a, r.a, r.b FROM t1 LEFT JOIN t2 r
 ON r.a = (SELECT t2.a FROM t2 WHERE t2.c = t1.a AND t2.b <= '359899'
  ORDER BY t2.c DESC, t2.b DESC LIMIT 1) WHERE t1.a = 10;

mysql-test/main/subselect_no_opts.result

@@ -3198,7 +3198,7 @@ ON r.a = (SELECT t2.a FROM t2 WHERE t2.c = t1.a AND t2.b <= '359899'
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 1	PRIMARY	t1	system	PRIMARY	NULL	NULL	NULL	1
 1	PRIMARY	r	const	PRIMARY	PRIMARY	4	const	1
-2	SUBQUERY	t2	range	cb	cb	40	NULL	3	Using where
+2	SUBQUERY	t2	range	cb	cb	40	NULL	3	Using index condition
 SELECT sql_no_cache t1.a, r.a, r.b FROM t1 LEFT JOIN t2 r
 ON r.a = (SELECT t2.a FROM t2 WHERE t2.c = t1.a AND t2.b <= '359899'
  ORDER BY t2.c DESC, t2.b DESC LIMIT 1) WHERE t1.a = 10;