MDEV-7565: Server crash with Signal 6 (part 2)

Preparation of subselect moved earlier (before checks which needs it prepared).

Author: sanja-byelkin

mysql-test/r/subselect.result

@@ -7015,3 +7015,15 @@ select exists(select 1 from t1 group by `c` in (select `c` from t1));
 exists(select 1 from t1 group by `c` in (select `c` from t1))
 0
 drop table t1;
+#
+# MDEV-7565: Server crash with Signal 6 (part 2)
+#
+Select 
+(Select Sum(`TestCase`.Revenue) From mysql.slow_log E 
+Where TestCase.TemplateID not in (Select 1 from mysql.slow_log where 2=2)
+) As `ControlRev`
+From 
+(Select 3 as Revenue, 4 as TemplateID) As `TestCase` 
+Group By TestCase.Revenue, TestCase.TemplateID;
+ControlRev
+NULL

mysql-test/r/subselect_no_mat.result

@@ -7012,6 +7012,18 @@ select exists(select 1 from t1 group by `c` in (select `c` from t1));
 exists(select 1 from t1 group by `c` in (select `c` from t1))
 0
 drop table t1;
+#
+# MDEV-7565: Server crash with Signal 6 (part 2)
+#
+Select 
+(Select Sum(`TestCase`.Revenue) From mysql.slow_log E 
+Where TestCase.TemplateID not in (Select 1 from mysql.slow_log where 2=2)
+) As `ControlRev`
+From 
+(Select 3 as Revenue, 4 as TemplateID) As `TestCase` 
+Group By TestCase.Revenue, TestCase.TemplateID;
+ControlRev
+NULL
 set optimizer_switch=default;
 select @@optimizer_switch like '%materialization=on%';
 @@optimizer_switch like '%materialization=on%'

mysql-test/r/subselect_no_opts.result

@@ -7010,4 +7010,16 @@ select exists(select 1 from t1 group by `c` in (select `c` from t1));
 exists(select 1 from t1 group by `c` in (select `c` from t1))
 0
 drop table t1;
+#
+# MDEV-7565: Server crash with Signal 6 (part 2)
+#
+Select 
+(Select Sum(`TestCase`.Revenue) From mysql.slow_log E 
+Where TestCase.TemplateID not in (Select 1 from mysql.slow_log where 2=2)
+) As `ControlRev`
+From 
+(Select 3 as Revenue, 4 as TemplateID) As `TestCase` 
+Group By TestCase.Revenue, TestCase.TemplateID;
+ControlRev
+NULL
 set @optimizer_switch_for_subselect_test=null;

mysql-test/r/subselect_no_scache.result

@@ -7021,6 +7021,18 @@ select exists(select 1 from t1 group by `c` in (select `c` from t1));
 exists(select 1 from t1 group by `c` in (select `c` from t1))
 0
 drop table t1;
+#
+# MDEV-7565: Server crash with Signal 6 (part 2)
+#
+Select 
+(Select Sum(`TestCase`.Revenue) From mysql.slow_log E 
+Where TestCase.TemplateID not in (Select 1 from mysql.slow_log where 2=2)
+) As `ControlRev`
+From 
+(Select 3 as Revenue, 4 as TemplateID) As `TestCase` 
+Group By TestCase.Revenue, TestCase.TemplateID;
+ControlRev
+NULL
 set optimizer_switch=default;
 select @@optimizer_switch like '%subquery_cache=on%';
 @@optimizer_switch like '%subquery_cache=on%'

mysql-test/r/subselect_no_semijoin.result

@@ -7010,5 +7010,17 @@ select exists(select 1 from t1 group by `c` in (select `c` from t1));
 exists(select 1 from t1 group by `c` in (select `c` from t1))
 0
 drop table t1;
+#
+# MDEV-7565: Server crash with Signal 6 (part 2)
+#
+Select 
+(Select Sum(`TestCase`.Revenue) From mysql.slow_log E 
+Where TestCase.TemplateID not in (Select 1 from mysql.slow_log where 2=2)
+) As `ControlRev`
+From 
+(Select 3 as Revenue, 4 as TemplateID) As `TestCase` 
+Group By TestCase.Revenue, TestCase.TemplateID;
+ControlRev
+NULL
 set @optimizer_switch_for_subselect_test=null;
 set @join_cache_level_for_subselect_test=NULL;

mysql-test/t/subselect.test

@@ -5890,3 +5890,15 @@ DROP TABLE t1,t2;
 create table t1 (c int);
 select exists(select 1 from t1 group by `c` in (select `c` from t1));
 drop table t1;
+
+--echo #
+--echo # MDEV-7565: Server crash with Signal 6 (part 2)
+--echo #
+Select 
+ (Select Sum(`TestCase`.Revenue) From mysql.slow_log E 
+ Where TestCase.TemplateID not in (Select 1 from mysql.slow_log where 2=2)
+ ) As `ControlRev`
+From 
+(Select 3 as Revenue, 4 as TemplateID) As `TestCase` 
+Group By TestCase.Revenue, TestCase.TemplateID;
+

sql/opt_subselect.cc

@@ -703,6 +703,12 @@ int check_and_do_in_subquery_rewrites(JOIN *join)
  if (!optimizer_flag(thd, OPTIMIZER_SWITCH_IN_TO_EXISTS) &&
  !optimizer_flag(thd, OPTIMIZER_SWITCH_MATERIALIZATION))
  my_error(ER_ILLEGAL_SUBQUERY_OPTIMIZER_SWITCHES, MYF(0));
+ /*
+ Transform each subquery predicate according to its overloaded
+ transformer.
+ */
+ if (subselect->select_transformer(join))
+ DBUG_RETURN(-1);
 
  /*
  If the subquery predicate is IN/=ANY, analyse and set all possible
@@ -754,12 +760,6 @@ int check_and_do_in_subquery_rewrites(JOIN *join)
  allany_subs->add_strategy(strategy);
  }
 
- /*
- Transform each subquery predicate according to its overloaded
- transformer.
- */
- if (subselect->select_transformer(join))
- DBUG_RETURN(-1);
  }
  }
  DBUG_RETURN(0);