MDEV-33281 Make BNL() hint work for join_cache_levels from 0 to 3

BNL() hint effectively increases join_cache_level up to 4 if it is set to value less than 4. This commit also makes the BKA() hint override not only `join_cache_bka` optimizer switch but `join_cache_level` as well. I.e., BKA() hint enables BKA and BKAH join buffers both flat and incremental despite `join_cache_level` and `join_cache_bka` setting.

Author: Olernov

mysql-test/main/opt_hints.result

@@ -353,6 +353,32 @@ id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
 1	SIMPLE	t13	ref	a	a	5	test.t12.a	1	100.00	Using index condition; Using where; Using join buffer (flat, BKAH join); Rowid-ordered scan
 Warnings:
 Note	1003	select /*+ QB_NAME(`QB1`) BKA(`t13`@`QB1`) */ `test`.`t12`.`a` AS `a`,`test`.`t12`.`b` AS `b`,`test`.`t13`.`a` AS `a`,`test`.`t13`.`b` AS `b`,`test`.`t13`.`c` AS `c`,`test`.`t13`.`filler` AS `filler` from `test`.`t12` join `test`.`t13` where `test`.`t13`.`a` = `test`.`t12`.`a` and `test`.`t13`.`b` + 1 <= `test`.`t13`.`b` + 1
+# Hint overrides both join_cache_level and optimizer switch
+set join_cache_level = 0;
+EXPLAIN EXTENDED SELECT /*+ BKA(t12, t13) */ * FROM t12, t13
+WHERE t12.a=t13.a AND (t13.b+1 <= t13.b+1);
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
+1	SIMPLE	t12	ALL	NULL	NULL	NULL	NULL	10	100.00	Using where
+1	SIMPLE	t13	ref	a	a	5	test.t12.a	1	100.00	Using index condition; Using where; Using join buffer (flat, BKAH join); Rowid-ordered scan
+Warnings:
+Note	1003	select /*+ BKA(`t12`@`select#1`) BKA(`t13`@`select#1`) */ `test`.`t12`.`a` AS `a`,`test`.`t12`.`b` AS `b`,`test`.`t13`.`a` AS `a`,`test`.`t13`.`b` AS `b`,`test`.`t13`.`c` AS `c`,`test`.`t13`.`filler` AS `filler` from `test`.`t12` join `test`.`t13` where `test`.`t13`.`a` = `test`.`t12`.`a` and `test`.`t13`.`b` + 1 <= `test`.`t13`.`b` + 1
+set join_cache_level = 2;
+EXPLAIN EXTENDED SELECT /*+ BKA(t12, t13) */ * FROM t12, t13
+WHERE t12.a=t13.a AND (t13.b+1 <= t13.b+1);
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
+1	SIMPLE	t12	ALL	NULL	NULL	NULL	NULL	10	100.00	Using where
+1	SIMPLE	t13	ref	a	a	5	test.t12.a	1	100.00	Using index condition; Using where; Using join buffer (flat, BKAH join); Rowid-ordered scan
+Warnings:
+Note	1003	select /*+ BKA(`t12`@`select#1`) BKA(`t13`@`select#1`) */ `test`.`t12`.`a` AS `a`,`test`.`t12`.`b` AS `b`,`test`.`t13`.`a` AS `a`,`test`.`t13`.`b` AS `b`,`test`.`t13`.`c` AS `c`,`test`.`t13`.`filler` AS `filler` from `test`.`t12` join `test`.`t13` where `test`.`t13`.`a` = `test`.`t12`.`a` and `test`.`t13`.`b` + 1 <= `test`.`t13`.`b` + 1
+set join_cache_level = 4;
+EXPLAIN EXTENDED SELECT /*+ BKA(t12, t13) */ * FROM t12, t13
+WHERE t12.a=t13.a AND (t13.b+1 <= t13.b+1);
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
+1	SIMPLE	t12	ALL	NULL	NULL	NULL	NULL	10	100.00	Using where
+1	SIMPLE	t13	ref	a	a	5	test.t12.a	1	100.00	Using index condition; Using where; Using join buffer (flat, BKAH join); Rowid-ordered scan
+Warnings:
+Note	1003	select /*+ BKA(`t12`@`select#1`) BKA(`t13`@`select#1`) */ `test`.`t12`.`a` AS `a`,`test`.`t12`.`b` AS `b`,`test`.`t13`.`a` AS `a`,`test`.`t13`.`b` AS `b`,`test`.`t13`.`c` AS `c`,`test`.`t13`.`filler` AS `filler` from `test`.`t12` join `test`.`t13` where `test`.`t13`.`a` = `test`.`t12`.`a` and `test`.`t13`.`b` + 1 <= `test`.`t13`.`b` + 1
+set join_cache_level = 8;
 # Enable BKA
 set optimizer_switch='join_cache_bka=on';
 EXPLAIN EXTENDED SELECT /*+ NO_BKA(t13) */ * FROM t12, t13
@@ -1175,6 +1201,40 @@ LEFT JOIN t4 ON t4.a4 <> 0) LEFT JOIN t5 ON t5.a5 = t2.a2;
 a4	b5
 9	0
 9	NULL
+# BNL() hint overrides join_cache_levels from 0 to 3 increasing it to 4
+set join_cache_level = 1;
+EXPLAIN
+SELECT /*+ BNL(t4, t5) */ t4.a4, t5.b5
+FROM ((t1 LEFT JOIN (t2 JOIN t3 ON t2.a2 = t3.a3) ON t2.b2 = t1.a1)
+LEFT JOIN t4 ON t4.a4 <> 0) LEFT JOIN t5 ON t5.a5 = t2.a2;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t1	ALL	NULL	NULL	NULL	NULL	2
+1	SIMPLE	t3	index	PRIMARY	PRIMARY	4	NULL	1	Using where; Using index
+1	SIMPLE	t2	eq_ref	PRIMARY	PRIMARY	4	test.t3.a3	1	Using where
+1	SIMPLE	t4	ALL	NULL	NULL	NULL	NULL	1	Using where; Using join buffer (flat, BNL join)
+1	SIMPLE	t5	hash_ALL	PRIMARY	#hash#PRIMARY	4	test.t2.a2	1	Using where; Using join buffer (incremental, BNLH join)
+set join_cache_level = 2;
+EXPLAIN
+SELECT /*+ BNL(t4, t5) */ t4.a4, t5.b5
+FROM ((t1 LEFT JOIN (t2 JOIN t3 ON t2.a2 = t3.a3) ON t2.b2 = t1.a1)
+LEFT JOIN t4 ON t4.a4 <> 0) LEFT JOIN t5 ON t5.a5 = t2.a2;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t1	ALL	NULL	NULL	NULL	NULL	2
+1	SIMPLE	t3	index	PRIMARY	PRIMARY	4	NULL	1	Using where; Using index
+1	SIMPLE	t2	eq_ref	PRIMARY	PRIMARY	4	test.t3.a3	1	Using where
+1	SIMPLE	t4	ALL	NULL	NULL	NULL	NULL	1	Using where; Using join buffer (flat, BNL join)
+1	SIMPLE	t5	hash_ALL	PRIMARY	#hash#PRIMARY	4	test.t2.a2	1	Using where; Using join buffer (incremental, BNLH join)
+set join_cache_level = 3;
+EXPLAIN
+SELECT /*+ BNL(t4, t5) */ t4.a4, t5.b5
+FROM ((t1 LEFT JOIN (t2 JOIN t3 ON t2.a2 = t3.a3) ON t2.b2 = t1.a1)
+LEFT JOIN t4 ON t4.a4 <> 0) LEFT JOIN t5 ON t5.a5 = t2.a2;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t1	ALL	NULL	NULL	NULL	NULL	2
+1	SIMPLE	t2	ALL	PRIMARY	NULL	NULL	NULL	1	Using where
+1	SIMPLE	t3	eq_ref	PRIMARY	PRIMARY	4	test.t2.a2	1	Using index
+1	SIMPLE	t4	ALL	NULL	NULL	NULL	NULL	1	Using where; Using join buffer (flat, BNL join)
+1	SIMPLE	t5	hash_ALL	PRIMARY	#hash#PRIMARY	4	test.t2.a2	1	Using where; Using join buffer (incremental, BNLH join)
 # Enable all join buffering capabilities:
 SET SESSION join_cache_level = 8;
 EXPLAIN

mysql-test/main/opt_hints.test

@@ -197,6 +197,21 @@ EXPLAIN EXTENDED SELECT /*+ BKA(t12) */ * FROM t12, t13
 EXPLAIN EXTENDED SELECT /*+ QB_NAME(QB1) BKA(t13@QB1) */ * FROM t12, t13 
  WHERE t12.a=t13.a AND (t13.b+1 <= t13.b+1);
 
+--echo # Hint overrides both join_cache_level and optimizer switch
+set join_cache_level = 0;
+EXPLAIN EXTENDED SELECT /*+ BKA(t12, t13) */ * FROM t12, t13
+ WHERE t12.a=t13.a AND (t13.b+1 <= t13.b+1);
+
+set join_cache_level = 2;
+EXPLAIN EXTENDED SELECT /*+ BKA(t12, t13) */ * FROM t12, t13
+ WHERE t12.a=t13.a AND (t13.b+1 <= t13.b+1);
+
+set join_cache_level = 4;
+EXPLAIN EXTENDED SELECT /*+ BKA(t12, t13) */ * FROM t12, t13
+ WHERE t12.a=t13.a AND (t13.b+1 <= t13.b+1);
+
+set join_cache_level = 8;
+
 --echo # Enable BKA
 set optimizer_switch='join_cache_bka=on';
 
@@ -603,6 +618,25 @@ SELECT /*+ BNL(t4, t5) */ t4.a4, t5.b5
  FROM ((t1 LEFT JOIN (t2 JOIN t3 ON t2.a2 = t3.a3) ON t2.b2 = t1.a1)
  LEFT JOIN t4 ON t4.a4 <> 0) LEFT JOIN t5 ON t5.a5 = t2.a2;
 
+--echo # BNL() hint overrides join_cache_levels from 0 to 3 increasing it to 4
+set join_cache_level = 1;
+EXPLAIN
+SELECT /*+ BNL(t4, t5) */ t4.a4, t5.b5
+ FROM ((t1 LEFT JOIN (t2 JOIN t3 ON t2.a2 = t3.a3) ON t2.b2 = t1.a1)
+ LEFT JOIN t4 ON t4.a4 <> 0) LEFT JOIN t5 ON t5.a5 = t2.a2;
+
+set join_cache_level = 2;
+EXPLAIN
+SELECT /*+ BNL(t4, t5) */ t4.a4, t5.b5
+ FROM ((t1 LEFT JOIN (t2 JOIN t3 ON t2.a2 = t3.a3) ON t2.b2 = t1.a1)
+ LEFT JOIN t4 ON t4.a4 <> 0) LEFT JOIN t5 ON t5.a5 = t2.a2;
+
+set join_cache_level = 3;
+EXPLAIN
+SELECT /*+ BNL(t4, t5) */ t4.a4, t5.b5
+ FROM ((t1 LEFT JOIN (t2 JOIN t3 ON t2.a2 = t3.a3) ON t2.b2 = t1.a1)
+ LEFT JOIN t4 ON t4.a4 <> 0) LEFT JOIN t5 ON t5.a5 = t2.a2;
+
 --echo # Enable all join buffering capabilities:
 SET SESSION join_cache_level = 8;
 EXPLAIN

sql/sql_select.cc

@@ -15715,24 +15715,35 @@ uint check_join_cache_usage(JOIN_TAB *tab,
  bool no_bka_cache= !hint_table_state_or_fallback(join->thd,
  tab->tab_list->table, BKA_HINT_ENUM,
  join->allowed_join_cache_types & JOIN_CACHE_BKA_BIT);
+ bool hint_forces_bka= hint_table_state_or_fallback(join->thd,
+ tab->tab_list->table,
+ BKA_HINT_ENUM, false);
  join->return_tab= 0;
 
  if (tab->no_forced_join_cache || (no_bnl_cache && no_bka_cache))
  goto no_join_cache;
 
- if (cache_level == 0 && hint_table_state_or_fallback(join->thd,
+ if (cache_level < 4 && hint_table_state_or_fallback(join->thd,
  tab->tab_list->table,
  BNL_HINT_ENUM, false))
  {
  cache_level= 4; // BNL() hint present, raise join_cache_level to BNLH
  }
 
  /*
- Don't use join cache if @@join_cache_level==0 or this table is the first
+ Don't use join cache if @@join_cache_level==0 and hint BKA()
+ is not specified or this table is the first
  one join suborder (either at top level or inside a bush)
  */
- if (cache_level == 0 || !prev_tab)
+ if ((cache_level == 0 && !hint_forces_bka) || !prev_tab)
+ {
+ /*
+ We could have cache_level==0 but join cache was forced for some previous
+ table (PT) with a hint. Proceed to cancel join cache for PT if it is not
+ allowed to use join cache for PT without using it for this table.
+ */
  goto no_join_cache;
+ }
 
  if (force_unlinked_cache && (cache_level%2 == 0))
  cache_level--;
@@ -15871,6 +15882,8 @@ uint check_join_cache_usage(JOIN_TAB *tab,
  case JT_CONST:
  case JT_REF:
  case JT_EQ_REF:
+ if (hint_forces_bka)
+ cache_level= 8; // Increase to BKAH incremental
  if (cache_level <=2 || (no_hashed_cache && no_bka_cache))
  goto no_join_cache;
  if (tab->ref.is_access_triggered())