MDEV-8208: Sporadic SEGFAULT on startup

Problem:
When mysqld starts as a galera node, it creates 2 system threads
(applier & rollbacker) using start_wsrep_THD(). These threads are
created before plugin initialization (plugin_init()) for SST methods
like rsync and xtrabackup.

The threads' initialization itself can proceed in parallel to mysqld's
main thread of execution. As a result, the thread initialization code
(start_wsrep_THD()) can end up accessing some un/partially initialized
structures (like maria_hton, in this particular case) resulting in
segfault.

Solution:
Fixed by calling THD::init_for_queries() (which accesses maria_hton)
only after the plugins have been initialized.

sql/mysqld.cc

@@ -5125,11 +5125,6 @@ static int init_server_components()
       {
         set_ports(); // this is also called in network_init() later but we need
                      // to know mysqld_port now - lp:1071882
-        /*
-          Plugin initialization (plugin_init()) hasn't happened yet, set
-          maria_hton to 0.
-        */
-        maria_hton= 0;
         wsrep_init_startup(true);
       }
     }
@@ -5204,6 +5199,29 @@ static int init_server_components()
   }
   plugins_are_initialized= TRUE;  /* Don't separate from init function */
 
+#ifdef WITH_WSREP
+  /* Wait for wsrep threads to get created. */
+  if (wsrep_creating_startup_threads == 1) {
+    mysql_mutex_lock(&LOCK_thread_count);
+    while (wsrep_running_threads < 2)
+    {
+      mysql_cond_wait(&COND_thread_count, &LOCK_thread_count);
+    }
+
+    /* Now is the time to initialize threads for queries. */
+    THD *tmp;
+    I_List_iterator<THD> it(threads);
+    while ((tmp= it++))
+    {
+      if (tmp->wsrep_applier == true)
+      {
+        tmp->init_for_queries();
+      }
+    }
+    mysql_mutex_unlock(&LOCK_thread_count);
+  }
+#endif
+
   /* we do want to exit if there are any other unknown options */
   if (remaining_argc > 1)
   {
@@ -5412,7 +5430,6 @@ static void create_shutdown_thread()
 
 #endif /* EMBEDDED_LIBRARY */
 
-
 #if (defined(_WIN32) || defined(HAVE_SMEM)) && !defined(EMBEDDED_LIBRARY)
 static void handle_connections_methods()
 {

sql/wsrep_mysqld.cc

@@ -91,6 +91,12 @@ my_bool wsrep_slave_UK_checks          = 0; // slave thread does UK checks
 my_bool wsrep_slave_FK_checks          = 0; // slave thread does FK checks
 bool wsrep_new_cluster                 = false; // Bootstrap the cluster ?
 
+/*
+  Set during the creation of first wsrep applier and rollback threads.
+  Since these threads are critical, abort if the thread creation fails.
+*/
+my_bool wsrep_creating_startup_threads = 0;
+
 // Use wsrep_gtid_domain_id for galera transactions?
 bool wsrep_gtid_mode                   = 0;
 // gtid_domain_id for galera transactions.
@@ -792,6 +798,7 @@ void wsrep_init_startup (bool first)
 
   if (!wsrep_start_replication()) unireg_abort(1);
 
+  wsrep_creating_startup_threads= 1;
   wsrep_create_rollbacker();
   wsrep_create_appliers(1);
 
@@ -1719,16 +1726,11 @@ pthread_handler_t start_wsrep_THD(void *arg)
   THD *thd;
   wsrep_thd_processor_fun processor= (wsrep_thd_processor_fun)arg;
 
-  if (my_thread_init())
+  if (my_thread_init() || (!(thd= new THD(true))))
   {
-    WSREP_ERROR("Could not initialize thread");
-    return(NULL);
+    goto error;
   }
 
-  if (!(thd= new THD(true)))
-  {
-    return(NULL);
-  }
   mysql_mutex_lock(&LOCK_thread_count);
   thd->thread_id=thread_id++;
 
@@ -1765,7 +1767,7 @@ pthread_handler_t start_wsrep_THD(void *arg)
     statistic_increment(aborted_connects,&LOCK_status);
     MYSQL_CALLBACK(thread_scheduler, end_thread, (thd, 0));
 
-    return(NULL);
+    goto error;
   }
 
 // </5.1.17>
@@ -1788,8 +1790,7 @@ pthread_handler_t start_wsrep_THD(void *arg)
     statistic_increment(aborted_connects,&LOCK_status);
     MYSQL_CALLBACK(thread_scheduler, end_thread, (thd, 0));
     delete thd;
-
-    return(NULL);
+    goto error;
   }
 
   thd->system_thread= SYSTEM_THREAD_SLAVE_SQL;
@@ -1799,12 +1800,21 @@ pthread_handler_t start_wsrep_THD(void *arg)
   //thd->version= refresh_version;
   thd->proc_info= 0;
   thd->set_command(COM_SLEEP);
-  thd->set_time();
-  thd->init_for_queries();
+
+  if (plugins_are_initialized)
+  {
+    thd->init_for_queries();
+  }
 
   mysql_mutex_lock(&LOCK_thread_count);
   wsrep_running_threads++;
   mysql_cond_broadcast(&COND_thread_count);
+
+  if (wsrep_running_threads > 2)
+  {
+    wsrep_creating_startup_threads= 0;
+  }
+
   mysql_mutex_unlock(&LOCK_thread_count);
 
   processor(thd);
@@ -1842,6 +1852,15 @@ pthread_handler_t start_wsrep_THD(void *arg)
     mysql_mutex_unlock(&LOCK_thread_count);
   }
   return(NULL);
+
+error:
+  WSREP_ERROR("Failed to create/initialize system thread");
+
+  /* Abort if its the first applier/rollbacker thread. */
+  if (wsrep_creating_startup_threads < 2)
+    unireg_abort(1);
+  else
+    return NULL;
 }
 
 

sql/wsrep_mysqld.h

@@ -86,6 +86,7 @@ extern my_bool     wsrep_slave_FK_checks;
 extern my_bool     wsrep_slave_UK_checks;
 extern ulong       wsrep_running_threads;
 extern bool        wsrep_new_cluster;
+extern my_bool     wsrep_creating_startup_threads;
 extern bool        wsrep_gtid_mode;
 extern uint32      wsrep_gtid_domain_id;
 
@@ -341,6 +342,7 @@ int wsrep_create_trigger_query(THD *thd, uchar** buf, size_t* buf_len);
 #define wsrep_thr_init() do {} while(0)
 #define wsrep_thr_deinit() do {} while(0)
 #define wsrep_running_threads (0)
+#define wsrep_creating_startup_threads (0)
 
 #endif /* WITH_WSREP */
 #endif /* WSREP_MYSQLD_H */