MDEV-13012 Assertion `share->error' failed in discover_handlerton upon executing statement with max_session_mem_used = 8192

and MDEV-13011 Server crashes in THD::handle_condition or Assertion `! is_set() || m_can_overwrite_status' fails upon attempt to connect with max_session_mem_used = 8192 errors when a connection is killed in the * TABLE_SHARE::init_from_sql_statement_string() * THD::init() also, safety-wise, don't check max_mem_used on free() and when some error was already issued.

Author: vuvova

mysql-test/r/errors.result

@@ -168,3 +168,7 @@ UPDATE t1 SET a = 'new'
 WHERE COLUMN_CREATE( 1, 'v', 1, 'w' ) IS NULL;
 ERROR 22007: Illegal value used as argument of dynamic column function
 drop table t1;
+set max_session_mem_used = 8192;
+select * from seq_1_to_1000;
+ERROR HY000: Engine SEQUENCE failed to discover table `test`.`seq_1_to_1000` with 'create table seq (seq bigint unsigned primary key)'
+set global max_session_mem_used = default;

mysql-test/t/errors.test

@@ -1,6 +1,7 @@
 #
 # Test some error conditions
 #
+--source include/have_sequence.inc
 
 --disable_warnings
 drop table if exists t1;
@@ -198,3 +199,11 @@ CREATE TABLE t1 (a CHAR(3), b BLOB);
 UPDATE t1 SET a = 'new'
 WHERE COLUMN_CREATE( 1, 'v', 1, 'w' ) IS NULL;
 drop table t1;
+
+#
+# errors caused by max_session_mem_used
+#
+set max_session_mem_used = 8192;
+--error ER_SQL_DISCOVER_ERROR
+select * from seq_1_to_1000;
+set global max_session_mem_used = default;

sql/handler.cc

@@ -4857,7 +4857,12 @@ static my_bool discover_handlerton(THD *thd, plugin_ref plugin,
  {
  if (error)
  {
- DBUG_ASSERT(share->error); // tdc_lock_share needs that
+ if (!share->error)
+ {
+ share->error= OPEN_FRM_ERROR_ALREADY_ISSUED;
+ plugin_unlock(0, share->db_plugin);
+ }
+
  /*
  report an error, unless it is "generic" and a more
  specific one was already reported

sql/mysqld.cc

@@ -3939,8 +3939,9 @@ static void my_malloc_size_cb_func(long long size, my_bool is_thread_specific)
  (longlong) thd->status_var.local_memory_used,
  size));
  thd->status_var.local_memory_used+= size;
- if (thd->status_var.local_memory_used > (int64)thd->variables.max_mem_used &&
- !thd->killed)
+ if (size > 0 &&
+ thd->status_var.local_memory_used > (int64)thd->variables.max_mem_used &&
+ !thd->killed && !thd->get_stmt_da()->is_set())
  {
  char buf[1024];
  thd->killed= KILL_QUERY;
@@ -6646,11 +6647,12 @@ void handle_connections_sockets()
  */
 
  DBUG_PRINT("info", ("Creating THD for new connection"));
- if (!(thd= new THD))
+ if (!(thd= new THD) || thd->is_error())
  {
  (void) mysql_socket_shutdown(new_sock, SHUT_RDWR);
  (void) mysql_socket_close(new_sock);
  statistic_increment(connection_errors_internal, &LOCK_status);
+ delete thd;
  continue;
  }
  /* Set to get io buffers to be part of THD */

sql/sql_class.cc

@@ -879,6 +879,7 @@ THD::THD(bool is_wsrep_applier)
  debug_sync_control(0),
 #endif /* defined(ENABLED_DEBUG_SYNC) */
  wait_for_commit_ptr(0),
+ m_internal_handler(0),
  main_da(0, false, false),
  m_stmt_da(&main_da),
  tdc_hash_pins(0),
@@ -1073,7 +1074,6 @@ THD::THD(bool is_wsrep_applier)
  MYF(MY_WME|MY_THREAD_SPECIFIC));
  }
 
- m_internal_handler= NULL;
  m_binlog_invoker= INVOKER_NONE;
  memset(&invoker_user, 0, sizeof(invoker_user));
  memset(&invoker_host, 0, sizeof(invoker_host));