MDEV-14485 Server hangs on startup in THD::init

Solve 3 way deadlock between plugin_initialiaze(), THD::init() and
mysql_sys_var_char().

The deadlock exists because of the lock order inversion between
LOCK_global_system_variables mutex and LOCK_system_variables_hash
read-write lock-

In this case, it is enough to change LOCK_system_variables_hash to prefer
reads to fix the deadlock, i.e change it to mysql_prlock_t

extra/mariabackup/xtrabackup.cc

@@ -5070,7 +5070,7 @@ handle_options(int argc, char **argv, char ***argv_client, char ***argv_server)
 	setup_error_messages();
 	sys_var_init();
 	plugin_mutex_init();
-	mysql_rwlock_init(key_rwlock_LOCK_system_variables_hash, &LOCK_system_variables_hash);
+	mysql_prlock_init(key_rwlock_LOCK_system_variables_hash, &LOCK_system_variables_hash);
 	opt_stack_trace = 1;
 	test_flags |=  TEST_SIGINT;
 	init_signals();

sql/mysqld.cc

@@ -778,7 +778,7 @@ mysql_mutex_t LOCK_prepared_stmt_count;
 mysql_mutex_t LOCK_des_key_file;
 #endif
 mysql_rwlock_t LOCK_grant, LOCK_sys_init_connect, LOCK_sys_init_slave;
-mysql_rwlock_t LOCK_system_variables_hash;
+mysql_prlock_t LOCK_system_variables_hash;
 mysql_cond_t COND_thread_count, COND_start_thread;
 pthread_t signal_thread;
 pthread_attr_t connection_attrib;
@@ -2354,7 +2354,7 @@ static void clean_up_mutexes()
   mysql_rwlock_destroy(&LOCK_sys_init_connect);
   mysql_rwlock_destroy(&LOCK_sys_init_slave);
   mysql_mutex_destroy(&LOCK_global_system_variables);
-  mysql_rwlock_destroy(&LOCK_system_variables_hash);
+  mysql_prlock_destroy(&LOCK_system_variables_hash);
   mysql_mutex_destroy(&LOCK_short_uuid_generator);
   mysql_mutex_destroy(&LOCK_prepared_stmt_count);
   mysql_mutex_destroy(&LOCK_error_messages);
@@ -4680,7 +4680,7 @@ static int init_thread_environment()
                    &LOCK_global_system_variables, MY_MUTEX_INIT_FAST);
   mysql_mutex_record_order(&LOCK_active_mi, &LOCK_global_system_variables);
   mysql_mutex_record_order(&LOCK_status, &LOCK_thread_count);
-  mysql_rwlock_init(key_rwlock_LOCK_system_variables_hash,
+  mysql_prlock_init(key_rwlock_LOCK_system_variables_hash,
                     &LOCK_system_variables_hash);
   mysql_mutex_init(key_LOCK_prepared_stmt_count,
                    &LOCK_prepared_stmt_count, MY_MUTEX_INIT_FAST);

sql/mysqld.h

@@ -577,7 +577,7 @@ extern mysql_mutex_t LOCK_des_key_file;
 extern mysql_mutex_t LOCK_server_started;
 extern mysql_cond_t COND_server_started;
 extern mysql_rwlock_t LOCK_grant, LOCK_sys_init_connect, LOCK_sys_init_slave;
-extern mysql_rwlock_t LOCK_system_variables_hash;
+extern mysql_prlock_t LOCK_system_variables_hash;
 extern mysql_cond_t COND_thread_count, COND_start_thread;
 extern mysql_cond_t COND_manager;
 extern mysql_cond_t COND_slave_background;

sql/set_var.cc

@@ -595,10 +595,10 @@ int mysql_del_sys_var_chain(sys_var *first)
 {
   int result= 0;
 
-  mysql_rwlock_wrlock(&LOCK_system_variables_hash);
+  mysql_prlock_wrlock(&LOCK_system_variables_hash);
   for (sys_var *var= first; var; var= var->next)
     result|= my_hash_delete(&system_variable_hash, (uchar*) var);
-  mysql_rwlock_unlock(&LOCK_system_variables_hash);
+  mysql_prlock_unlock(&LOCK_system_variables_hash);
 
   return result;
 }
@@ -1067,7 +1067,7 @@ int fill_sysvars(THD *thd, TABLE_LIST *tables, COND *cond)
 
   cond= make_cond_for_info_schema(thd, cond, tables);
   thd->count_cuted_fields= CHECK_FIELD_WARN;
-  mysql_rwlock_rdlock(&LOCK_system_variables_hash);
+  mysql_prlock_rdlock(&LOCK_system_variables_hash);
 
   for (uint i= 0; i < system_variable_hash.records; i++)
   {
@@ -1229,7 +1229,7 @@ int fill_sysvars(THD *thd, TABLE_LIST *tables, COND *cond)
   }
   res= 0;
 end:
-  mysql_rwlock_unlock(&LOCK_system_variables_hash);
+  mysql_prlock_unlock(&LOCK_system_variables_hash);
   thd->count_cuted_fields= save_count_cuted_fields;
   return res;
 }

sql/sql_plugin.cc

@@ -1394,10 +1394,10 @@ static int plugin_initialize(MEM_ROOT *tmp_root, struct st_plugin_int *plugin,
 
   mysql_mutex_unlock(&LOCK_plugin);
 
-  mysql_rwlock_wrlock(&LOCK_system_variables_hash);
+  mysql_prlock_wrlock(&LOCK_system_variables_hash);
   if (test_plugin_options(tmp_root, plugin, argc, argv))
     state= PLUGIN_IS_DISABLED;
-  mysql_rwlock_unlock(&LOCK_system_variables_hash);
+  mysql_prlock_unlock(&LOCK_system_variables_hash);
 
   if (options_only || state == PLUGIN_IS_DISABLED)
   {
@@ -2797,11 +2797,11 @@ sys_var *find_sys_var_ex(THD *thd, const char *str, size_t length,
 
   if (!locked)
     mysql_mutex_lock(&LOCK_plugin);
-  mysql_rwlock_rdlock(&LOCK_system_variables_hash);
+  mysql_prlock_rdlock(&LOCK_system_variables_hash);
   if ((var= intern_find_sys_var(str, length)) &&
       (pi= var->cast_pluginvar()))
   {
-    mysql_rwlock_unlock(&LOCK_system_variables_hash);
+    mysql_prlock_unlock(&LOCK_system_variables_hash);
     LEX *lex= thd ? thd->lex : 0;
     if (!(plugin= intern_plugin_lock(lex, plugin_int_to_ref(pi->plugin))))
       var= NULL; /* failed to lock it, it must be uninstalling */
@@ -2814,7 +2814,7 @@ sys_var *find_sys_var_ex(THD *thd, const char *str, size_t length,
     }
   }
   else
-    mysql_rwlock_unlock(&LOCK_system_variables_hash);
+    mysql_prlock_unlock(&LOCK_system_variables_hash);
   if (!locked)
     mysql_mutex_unlock(&LOCK_plugin);
 
@@ -3043,9 +3043,9 @@ static uchar *intern_sys_var_ptr(THD* thd, int offset, bool global_lock)
   if (!thd->variables.dynamic_variables_ptr ||
       (uint)offset > thd->variables.dynamic_variables_head)
   {
-    mysql_rwlock_rdlock(&LOCK_system_variables_hash);
+    mysql_prlock_rdlock(&LOCK_system_variables_hash);
     sync_dynamic_session_variables(thd, global_lock);
-    mysql_rwlock_unlock(&LOCK_system_variables_hash);
+    mysql_prlock_unlock(&LOCK_system_variables_hash);
   }
   DBUG_RETURN((uchar*)thd->variables.dynamic_variables_ptr + offset);
 }
@@ -3160,7 +3160,7 @@ static void cleanup_variables(struct system_variables *vars)
   st_bookmark *v;
   uint idx;
 
-  mysql_rwlock_rdlock(&LOCK_system_variables_hash);
+  mysql_prlock_rdlock(&LOCK_system_variables_hash);
   for (idx= 0; idx < bookmark_hash.records; idx++)
   {
     v= (st_bookmark*) my_hash_element(&bookmark_hash, idx);
@@ -3179,7 +3179,7 @@ static void cleanup_variables(struct system_variables *vars)
       *ptr= NULL;
     }
   }
-  mysql_rwlock_unlock(&LOCK_system_variables_hash);
+  mysql_prlock_unlock(&LOCK_system_variables_hash);
 
   DBUG_ASSERT(vars->table_plugin == NULL);
   DBUG_ASSERT(vars->tmp_table_plugin == NULL);
@@ -4234,10 +4234,10 @@ int thd_key_create(MYSQL_THD_KEY_T *key)
              PLUGIN_VAR_NOSYSVAR | PLUGIN_VAR_NOCMDOPT;
   char namebuf[256];
   snprintf(namebuf, sizeof(namebuf), "%u", thd_key_no++);
-  mysql_rwlock_wrlock(&LOCK_system_variables_hash);
+  mysql_prlock_wrlock(&LOCK_system_variables_hash);
   // non-letters in the name as an extra safety
   st_bookmark *bookmark= register_var("\a\v\a\t\a\r", namebuf, flags);
-  mysql_rwlock_unlock(&LOCK_system_variables_hash);
+  mysql_prlock_unlock(&LOCK_system_variables_hash);
   if (bookmark)
   {
     *key= bookmark->offset;

sql/sql_show.cc

@@ -7399,7 +7399,7 @@ int fill_variables(THD *thd, TABLE_LIST *tables, COND *cond)
 
   COND *partial_cond= make_cond_for_info_schema(thd, cond, tables);
 
-  mysql_rwlock_rdlock(&LOCK_system_variables_hash);
+  mysql_prlock_rdlock(&LOCK_system_variables_hash);
 
   /*
     Avoid recursive LOCK_system_variables_hash acquisition in
@@ -7414,7 +7414,7 @@ int fill_variables(THD *thd, TABLE_LIST *tables, COND *cond)
   res= show_status_array(thd, wild, enumerate_sys_vars(thd, sorted_vars, scope),
                          scope, NULL, "", tables->table,
                          upper_case_names, partial_cond);
-  mysql_rwlock_unlock(&LOCK_system_variables_hash);
+  mysql_prlock_unlock(&LOCK_system_variables_hash);
   DBUG_RETURN(res);
 }