Bug#36578613: Hypergraph result difference when querying I_S.TABLES

Some query plans combining hash joins and sorting caused wrong results with the hypergraph optimizer. The problem occurred when two SORT paths were stacked on top of each other without any materialization step in between, and the sorting required row IDs, and the input to the lower SORT path came from a hash join. The cause of the problem is that the sorting iterators are not updating the row ID in handler::ref when they have positioned the handler on the row to return. This makes the next sort see the same row ID for every row, and it therefore returns the same row multiple times instead of sorting them. Fixed by making FindTablesToGetRowidFor() take into account that row IDs are not preserved through SORT the same way as they are through HASH_JOIN, so that any SORT or HASH_JOIN above a SORT will fetch row IDs afresh from the handler. Change-Id: I15d83fbfc748d7cf26bcaecb2f826e485b193c39

Author: kahatlen

mysql-test/include/keyring_tests/mats/table_encrypt_2.inc

@@ -119,7 +119,6 @@ CREATE TABLE not_encrypted2 (col1 INT) ENCRYPTION='N';
 CREATE TABLE encrypted1 (col1 INT) ENCRYPTION='y';
 CREATE TABLE encrypted2 (col1 INT) ENCRYPTION='Y';
 
---skip_if_hypergraph # See Bug#36578613.
 SELECT TABLE_SCHEMA, TABLE_NAME, CREATE_OPTIONS
  FROM INFORMATION_SCHEMA.TABLES
  WHERE TABLE_NAME like '%encrypted%'

mysql-test/r/hypergraph_bugs.result

@@ -1739,3 +1739,32 @@ SELECT NULL FROM t AS t1, t AS t2 STRAIGHT_JOIN t AS t3 ON t2.x = t3.x
 ) OR x = 1;
 1
 DROP TABLE t;
+#
+# Bug#36578613: Hypergraph result difference when querying I_S.TABLES
+#
+CREATE TABLE t(x INT, y LONGTEXT);
+INSERT INTO t VALUES (1, 'a'), (2, 'b'), (3, 'c'), (4, 'd'), (5, 'e'), (6, 'f');
+ANALYZE TABLE t;
+Table	Op	Msg_type	Msg_text
+test.t	analyze	status	OK
+EXPLAIN FORMAT=JSON INTO @explain SELECT DISTINCT t1.x, t1.y
+FROM t AS t1, t AS t2 WHERE t1.x = t2.x
+ORDER BY -t1.x;
+SELECT
+JSON_EXTRACT(@explain, '$.access_type') AS sort_for_order_by,
+JSON_EXTRACT(@explain, '$.inputs[0].access_type') AS sort_for_distinct,
+JSON_EXTRACT(@explain, '$.inputs[0].inputs[0].join_algorithm') AS hash_join;
+sort_for_order_by	"sort"
+sort_for_distinct	"sort"
+hash_join	"hash"
+SELECT DISTINCT t1.x, t1.y
+FROM t AS t1, t AS t2 WHERE t1.x = t2.x
+ORDER BY -t1.x;
+x	y
+6	f
+5	e
+4	d
+3	c
+2	b
+1	a
+DROP TABLE t;

mysql-test/suite/innodb/t/row_format.test

@@ -3,7 +3,6 @@
 --echo # Bug#31479542 : MYSQL 8 ORPHANED TABLE DUE TO UNCHECKED NON-EXISTENT ROW FORMAT CHECK.
 --echo #
 
---source include/not_hypergraph.inc # Fails, see Bug#36578613
 --source include/have_innodb_16k.inc
 --source include/not_valgrind.inc
 

mysql-test/suite/innodb/t/table_encrypt_2.test

@@ -127,7 +127,6 @@ CREATE TABLE not_encrypted2 (col1 INT) ENCRYPTION='N';
 CREATE TABLE encrypted1 (col1 INT) ENCRYPTION='y';
 CREATE TABLE encrypted2 (col1 INT) ENCRYPTION='Y';
 
---skip_if_hypergraph # Bug#36578613
 SELECT TABLE_SCHEMA, TABLE_NAME, CREATE_OPTIONS
  FROM INFORMATION_SCHEMA.TABLES
  WHERE TABLE_NAME like '%encrypted%'

mysql-test/suite/perfschema/t/table_component_lifecycle.test

@@ -26,7 +26,6 @@ select * from performance_schema.pfs_example_continent;
 
 INSTALL COMPONENT "file://component_pfs_example_component_population";
 
---skip_if_hypergraph # Bug#36578613, see comment.
 select TABLE_SCHEMA, TABLE_NAME, ENGINE, VERSION, TABLE_COMMENT
  from INFORMATION_SCHEMA.tables
  where TABLE_NAME like "pfs_example_%"
@@ -80,7 +79,6 @@ select * from performance_schema.pfs_example_continent;
 
 INSTALL COMPONENT "file://component_pfs_example_component_population";
 
---skip_if_hypergraph # Bug#36578613, see comment.
 select TABLE_SCHEMA, TABLE_NAME, ENGINE, VERSION, TABLE_COMMENT
  from INFORMATION_SCHEMA.tables
  where TABLE_NAME like "pfs_example_%"
@@ -98,7 +96,6 @@ select * from performance_schema.pfs_example_continent;
 
 --source include/restart_mysqld.inc
 
---skip_if_hypergraph # Bug#36578613, see comment.
 select TABLE_SCHEMA, TABLE_NAME, ENGINE, VERSION, TABLE_COMMENT
  from INFORMATION_SCHEMA.tables
  where TABLE_NAME like "pfs_example_%"

mysql-test/suite/perfschema/t/table_plugin_in_use.test

@@ -6,8 +6,6 @@
 #
 ################################################################################
 
---source include/not_hypergraph.inc # Appears susceptible to Bug#36578613.
-
 connect(con1, localhost, root,,);
 
 connect(con2, localhost, root,,);

mysql-test/suite/perfschema/t/table_plugin_lifecycle.test

@@ -4,8 +4,6 @@
 #
 ################################################################################
 
---source include/not_hypergraph.inc # Appears susceptible to Bug#36578613.
-
 --echo #################
 --echo # INITIAL STATE #
 --echo #################

mysql-test/t/hypergraph_bugs.test

@@ -1444,4 +1444,33 @@ FROM t WHERE x IN (
 ) OR x = 1;
 DROP TABLE t;
 
+--echo #
+--echo # Bug#36578613: Hypergraph result difference when querying I_S.TABLES
+--echo #
+
+# Create a table with a LONGTEXT column, to ensure that filesort uses
+# row IDs, and with enough rows to make a hash join look more
+# attractive than a nested loop join.
+CREATE TABLE t(x INT, y LONGTEXT);
+INSERT INTO t VALUES (1, 'a'), (2, 'b'), (3, 'c'), (4, 'd'), (5, 'e'), (6, 'f');
+ANALYZE TABLE t;
+
+let $query =
+SELECT DISTINCT t1.x, t1.y
+FROM t AS t1, t AS t2 WHERE t1.x = t2.x
+ORDER BY -t1.x;
+
+# To reproduce the bug, the plan should have two SORT nodes (one for
+# ORDER BY and one for DISTINCT) on top of a HASH_JOIN node.
+--eval EXPLAIN FORMAT=JSON INTO @explain $query
+query_vertical
+SELECT
+ JSON_EXTRACT(@explain, '$.access_type') AS sort_for_order_by,
+ JSON_EXTRACT(@explain, '$.inputs[0].access_type') AS sort_for_distinct,
+ JSON_EXTRACT(@explain, '$.inputs[0].inputs[0].join_algorithm') AS hash_join;
+
+--eval $query
+
+DROP TABLE t;
+
 --source include/disable_hypergraph.inc

mysql-test/t/information_schema_cs.test

@@ -901,6 +901,7 @@ sql security definer view v2 as select 1;
 connect (con16681,localhost,mysqltest_1,,test);
 connection con16681;
 
+--sorted_result
 select * from information_schema.views
 where table_name='v1' or table_name='v2';
 connection default;
@@ -1046,7 +1047,6 @@ select 1 as f1 from information_schema.tables where "COLUMN_PRIVILEGES"=
 (select cast(table_name as char) from information_schema.tables
  where table_schema != 'performance_schema' order by table_name limit 1) limit 1;
 
---skip_if_hypergraph # Flaky, see Bug#36579239
 select t.table_name, group_concat(t.table_schema, '.', t.table_name),
  count(*) as num1
 from information_schema.tables t

sql/join_optimizer/access_path.cc

@@ -1339,6 +1339,12 @@ unique_ptr_destroy_only<RowIterator> CreateIteratorFromAccessPath(
 }
 
 void FindTablesToGetRowidFor(AccessPath *path) {
+ // A map of the tables for which other paths further down in the tree will
+ // take care of copying the correct row ID into table->file->ref. The hash
+ // join iterators and the BKA join iterators do that, so iterators higher up
+ // should not call handler::position(), as that would overwrite the copied row
+ // ID with the row ID of the last row that was read by the join. Sorting
+ // iterators, on the other hand, do not
  table_map handled_by_others = 0;
 
  auto add_tables_handled_by_others = [path, &handled_by_others](
@@ -1367,6 +1373,13 @@ void FindTablesToGetRowidFor(AccessPath *path) {
  // Doesn't really matter, we don't cross query blocks anyway.
  return true;
  }
+ case AccessPath::SORT:
+ // The sorting iterators do not populate handler::ref with the row ID
+ // while returning rows, so row IDs in any tables handled by paths below
+ // it have to be fetched again from the handler by the paths above the
+ // sort. Therefore, we don't add any of the tables in the subtree below
+ // SORT to handled_by_others.
+ return true; // Skip the rest of the subtree.
  default:
  return false;
  }