MDEV-8208: Sporadic SEGFAULT on startup

Problem: When mysqld starts as a galera node, it creates 2 system threads (applier & rollbacker) using start_wsrep_THD(). These threads are created before plugin initialization (plugin_init()) for SST methods like rsync and xtrabackup. The threads' initialization itself can proceed in parallel to mysqld's main thread of execution. As a result, the thread initialization code (start_wsrep_THD()) can end up accessing some un/partially initialized structures (like maria_hton, in this particular case) resulting in segfault. Solution: Fixed by calling THD::init_for_queries() (which accesses maria_hton) only after the plugins have been initialized.

Author: Nirbhay Choubey

sql/mysqld.cc

@@ -5125,11 +5125,6 @@ static int init_server_components()
  {
  set_ports(); // this is also called in network_init() later but we need
  // to know mysqld_port now - lp:1071882
- /*
- Plugin initialization (plugin_init()) hasn't happened yet, set
- maria_hton to 0.
- */
- maria_hton= 0;
  wsrep_init_startup(true);
  }
  }
@@ -5204,6 +5199,29 @@ static int init_server_components()
  }
  plugins_are_initialized= TRUE; /* Don't separate from init function */
 
+#ifdef WITH_WSREP
+ /* Wait for wsrep threads to get created. */
+ if (wsrep_creating_startup_threads == 1) {
+ mysql_mutex_lock(&LOCK_thread_count);
+ while (wsrep_running_threads < 2)
+ {
+ mysql_cond_wait(&COND_thread_count, &LOCK_thread_count);
+ }
+
+ /* Now is the time to initialize threads for queries. */
+ THD *tmp;
+ I_List_iterator<THD> it(threads);
+ while ((tmp= it++))
+ {
+ if (tmp->wsrep_applier == true)
+ {
+ tmp->init_for_queries();
+ }
+ }
+ mysql_mutex_unlock(&LOCK_thread_count);
+ }
+#endif
+
  /* we do want to exit if there are any other unknown options */
  if (remaining_argc > 1)
  {
@@ -5412,7 +5430,6 @@ static void create_shutdown_thread()
 
 #endif /* EMBEDDED_LIBRARY */
 
-
 #if (defined(_WIN32) || defined(HAVE_SMEM)) && !defined(EMBEDDED_LIBRARY)
 static void handle_connections_methods()
 {

sql/wsrep_mysqld.cc

@@ -91,6 +91,12 @@ my_bool wsrep_slave_UK_checks = 0; // slave thread does UK checks
 my_bool wsrep_slave_FK_checks = 0; // slave thread does FK checks
 bool wsrep_new_cluster = false; // Bootstrap the cluster ?
 
+/*
+ Set during the creation of first wsrep applier and rollback threads.
+ Since these threads are critical, abort if the thread creation fails.
+*/
+my_bool wsrep_creating_startup_threads = 0;
+
 // Use wsrep_gtid_domain_id for galera transactions?
 bool wsrep_gtid_mode = 0;
 // gtid_domain_id for galera transactions.
@@ -792,6 +798,7 @@ void wsrep_init_startup (bool first)
 
  if (!wsrep_start_replication()) unireg_abort(1);
 
+ wsrep_creating_startup_threads= 1;
  wsrep_create_rollbacker();
  wsrep_create_appliers(1);
 
@@ -1719,16 +1726,11 @@ pthread_handler_t start_wsrep_THD(void *arg)
  THD *thd;
  wsrep_thd_processor_fun processor= (wsrep_thd_processor_fun)arg;
 
- if (my_thread_init())
+ if (my_thread_init() || (!(thd= new THD(true))))
  {
- WSREP_ERROR("Could not initialize thread");
- return(NULL);
+ goto error;
  }
 
- if (!(thd= new THD(true)))
- {
- return(NULL);
- }
  mysql_mutex_lock(&LOCK_thread_count);
  thd->thread_id=thread_id++;
 
@@ -1765,7 +1767,7 @@ pthread_handler_t start_wsrep_THD(void *arg)
  statistic_increment(aborted_connects,&LOCK_status);
  MYSQL_CALLBACK(thread_scheduler, end_thread, (thd, 0));
 
- return(NULL);
+ goto error;
  }
 
 // </5.1.17>
@@ -1788,8 +1790,7 @@ pthread_handler_t start_wsrep_THD(void *arg)
  statistic_increment(aborted_connects,&LOCK_status);
  MYSQL_CALLBACK(thread_scheduler, end_thread, (thd, 0));
  delete thd;
-
- return(NULL);
+ goto error;
  }
 
  thd->system_thread= SYSTEM_THREAD_SLAVE_SQL;
@@ -1799,12 +1800,21 @@ pthread_handler_t start_wsrep_THD(void *arg)
  //thd->version= refresh_version;
  thd->proc_info= 0;
  thd->set_command(COM_SLEEP);
- thd->set_time();
- thd->init_for_queries();
+
+ if (plugins_are_initialized)
+ {
+ thd->init_for_queries();
+ }
 
  mysql_mutex_lock(&LOCK_thread_count);
  wsrep_running_threads++;
  mysql_cond_broadcast(&COND_thread_count);
+
+ if (wsrep_running_threads > 2)
+ {
+ wsrep_creating_startup_threads= 0;
+ }
+
  mysql_mutex_unlock(&LOCK_thread_count);
 
  processor(thd);
@@ -1842,6 +1852,15 @@ pthread_handler_t start_wsrep_THD(void *arg)
  mysql_mutex_unlock(&LOCK_thread_count);
  }
  return(NULL);
+
+error:
+ WSREP_ERROR("Failed to create/initialize system thread");
+
+ /* Abort if its the first applier/rollbacker thread. */
+ if (wsrep_creating_startup_threads < 2)
+ unireg_abort(1);
+ else
+ return NULL;
 }
 
 

sql/wsrep_mysqld.h

@@ -86,6 +86,7 @@ extern my_bool wsrep_slave_FK_checks;
 extern my_bool wsrep_slave_UK_checks;
 extern ulong wsrep_running_threads;
 extern bool wsrep_new_cluster;
+extern my_bool wsrep_creating_startup_threads;
 extern bool wsrep_gtid_mode;
 extern uint32 wsrep_gtid_domain_id;
 
@@ -341,6 +342,7 @@ int wsrep_create_trigger_query(THD *thd, uchar** buf, size_t* buf_len);
 #define wsrep_thr_init() do {} while(0)
 #define wsrep_thr_deinit() do {} while(0)
 #define wsrep_running_threads (0)
+#define wsrep_creating_startup_threads (0)
 
 #endif /* WITH_WSREP */
 #endif /* WSREP_MYSQLD_H */