Bug#17638477 UNINSTALL AND INSTALL SEMI-SYNC PLUGIN CAUSES SLAVES TO BREAK

Problem: Uninstallation of semi sync plugin causes replication to
break.

Analysis: A semisync enabled replication is mutual agreement between
Master and Slave when the connection (I/O thread) is established.
Once I/O thread is started and if semisync is enabled on both
master and slave, master appends special magic header to events
using semisync plugin functions and sends it to slave. And slave
expects that each event will have that special magic header format
and reads those bytes using semisync plugin functions.

When semi sync replication is in use if users execute
uninstallation of the plugin on master, slave gets confused while
interpreting that event's content because it expects special 
magic header at the beginning of the event. Slave SQL thread will
be stopped with "Missing magic number in the header" error.

Similar problem will happen if uninstallation of the plugin happens
on slave when semi sync replication is in in use. Master sends
the events with magic header and slave does not know about the
added magic header and thinks that it received a corrupted event.
Hence slave SQL thread stops with "Found  corrupted event" error.

Fix: Uninstallation of semisync plugin will be blocked when semisync
replication is in use and will throw 'ER_UNKNOWN_ERROR' error.
To detect that semisync replication is in use, this patch uses
semisync status variable values.
 > On Master, it checks for 'Rpl_semi_sync_master_status' to be OFF
    before allowing the uninstallation of rpl_semi_sync_master plugin.
    >> Rpl_semi_sync_master_status is OFF when
        >>> there is no dump thread running
        >>> there are no semisync slaves
 > On Slave, it checks for 'Rpl_semi_sync_slave_status' to be OFF
    before allowing the uninstallation of rpl_semi_sync_slave plugin.
    >> Rpl_semi_sync_slave_status is OFF when
       >>> there is no I/O thread running
       >>> replication is asynchronous replication.

mysql-test/include/stop_dump_threads.inc

+# ==== Purpose ====
+#
+# Stop all dump threads on the server of the current connection.
+#
+# ==== Usage ====
+#
+# --source include/stop_dump_threads.inc
+
+--let $include_filename= stop_dump_threads.inc
+--source include/begin_include_file.inc
+
+
+--let $_sdt_show_rpl_debug_info_old= $show_rpl_debug_info
+--let $show_rpl_debug_info= 1
+--disable_query_log
+--disable_result_log
+
+--let $_sdt_dump_thread_id= `SELECT ID FROM INFORMATION_SCHEMA.PROCESSLIST WHERE COMMAND = 'Binlog dump'`
+
+while ($_sdt_dump_thread_id != '')
+{
+  eval KILL $_sdt_dump_thread_id;
+  --let $wait_condition= SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE ID = $_sdt_dump_thread_id
+  --source include/wait_condition.inc
+
+  --let $_sdt_dump_thread_id= `SELECT ID FROM INFORMATION_SCHEMA.PROCESSLIST WHERE COMMAND = 'Binlog dump'`
+}
+
+--let $show_rpl_debug_info= $_sdt_show_rpl_debug_info_old
+
+--let $include_filename= stop_dump_threads.inc
+--source include/end_include_file.inc

mysql-test/include/uninstall_semisync.inc

@@ -13,6 +13,11 @@
 UNINSTALL PLUGIN rpl_semi_sync_slave;
 
 --connection master
+# After BUG#17638477 fix, uninstallation of rpl_semi_sync_master
+#  is not allowed when there are semi sync slaves. Hence kill
+#  all dump threads before uninstalling it.
+SET GLOBAL rpl_semi_sync_master_enabled = OFF;
+--source include/stop_dump_threads.inc
 UNINSTALL PLUGIN rpl_semi_sync_master;
 --enable_warnings
 

mysql-test/suite/rpl/r/rpl_semi_sync.result

@@ -439,9 +439,8 @@ Rpl_semi_sync_slave_status	OFF
 #
 # Clean up
 #
+include/uninstall_semisync.inc
 include/stop_slave.inc
-UNINSTALL PLUGIN rpl_semi_sync_slave;
-UNINSTALL PLUGIN rpl_semi_sync_master;
 change master to master_user='root',master_password='';
 include/start_slave.inc
 drop table t1;

mysql-test/suite/rpl/r/rpl_semi_sync_uninstall_plugin.result

+include/master-slave.inc
+[connection master]
+INSTALL PLUGIN rpl_semi_sync_master SONAME 'semisync_master.so';
+INSTALL PLUGIN rpl_semi_sync_slave SONAME 'semisync_slave.so';
+UNINSTALL PLUGIN rpl_semi_sync_slave;
+UNINSTALL PLUGIN rpl_semi_sync_master;
+CREATE TABLE t1(i int);
+INSERT INTO t1 values (1);
+DROP TABLE t1;
+include/install_semisync.inc
+call mtr.add_suppression("Plugin 'rpl_semi_sync_slave' cannot be uninstalled now");
+UNINSTALL PLUGIN rpl_semi_sync_slave;
+ERROR HY000: Unknown error
+call mtr.add_suppression("Plugin 'rpl_semi_sync_master' cannot be uninstalled now");
+UNINSTALL PLUGIN rpl_semi_sync_master;
+ERROR HY000: Unknown error
+CREATE TABLE t1(i int);
+INSERT INTO t1 values (2);
+DROP TABLE t1;
+include/assert.inc [semi sync slave status should be ON.]
+include/assert.inc [semi sync master status should be ON.]
+include/assert.inc [semi sync master clients should be 1.]
+SET GLOBAL rpl_semi_sync_master_enabled = OFF;
+include/assert.inc [semi sync master clients should be 1.]
+UNINSTALL PLUGIN rpl_semi_sync_master;
+ERROR HY000: Unknown error
+include/stop_slave.inc
+SET GLOBAL rpl_semi_sync_slave_enabled = OFF;
+include/start_slave.inc
+UNINSTALL PLUGIN rpl_semi_sync_slave;
+include/assert.inc [semi sync master clients should be 0.]
+UNINSTALL PLUGIN rpl_semi_sync_master;
+CREATE TABLE t1(i int);
+INSERT INTO t1 values (3);
+DROP TABLE t1;
+include/rpl_end.inc

mysql-test/suite/rpl/t/rpl_semi_sync.test

@@ -598,19 +598,10 @@ SHOW STATUS LIKE 'Rpl_semi_sync_slave_status';
 --echo #
 --echo # Clean up
 --echo #
+--source include/uninstall_semisync.inc
 
 connection slave;
 source include/stop_slave.inc;
-UNINSTALL PLUGIN rpl_semi_sync_slave;
-
-connection master;
-# The dump thread may still be running on the master, and so the following
-# UNINSTALL could generate a warning about the plugin is busy.
-disable_warnings;
-UNINSTALL PLUGIN rpl_semi_sync_master;
-enable_warnings;
-
-connection slave;
 change master to master_user='root',master_password='';
 source include/start_slave.inc;
 

mysql-test/suite/rpl/t/rpl_semi_sync_uninstall_plugin-master.opt

+$SEMISYNC_PLUGIN_OPT

mysql-test/suite/rpl/t/rpl_semi_sync_uninstall_plugin-slave.opt

+$SEMISYNC_PLUGIN_OPT

mysql-test/suite/rpl/t/rpl_semi_sync_uninstall_plugin.test

+###############################################################################
+# Bug#17638477 UNINSTALL AND INSTALL SEMI-SYNC PLUGIN CAUSES SLAVES TO BREAK
+#  Problem: Uninstallation of Semi sync plugin should be blocked when it is
+#  in use.
+#  Test case: Uninstallation of semi sync should be allowed
+#   On Master:
+#     1) When there is no dump thread
+#     2) When there are no semi sync slaves (i.e., async replication).
+#   On Slave:
+#     1) When there is no I/O thread
+#     2) When there are no semi sync enabled I/O thread (i.e.,async replication).
+###############################################################################
+
+--source include/have_semisync_plugin.inc
+--source include/not_embedded.inc
+--source include/have_binlog_format_statement.inc
+--source include/master-slave.inc
+
+###############################################################################
+# Case 1: Uninstallation of semi sync plugins should be allowed when it is
+#  not in use i.e., when asynchronous replication is active.
+###############################################################################
+# Step 1.1: Install semi sync master plugin on master
+eval INSTALL PLUGIN rpl_semi_sync_master SONAME '$SEMISYNC_MASTER_PLUGIN';
+
+# Step 1.2: Install semi sync slave plugin on slave
+--connection slave
+eval INSTALL PLUGIN rpl_semi_sync_slave SONAME '$SEMISYNC_SLAVE_PLUGIN';
+
+# Step 1.3: Uninstallation of semisync plugin on master and slave should be
+#  allowed at this state as there is no semi sync replication enabled between
+#  master and slave.
+UNINSTALL PLUGIN rpl_semi_sync_slave;
+--connection master
+UNINSTALL PLUGIN rpl_semi_sync_master;
+
+# Step 1.4: Check that replication is working fine at the end of the test case.
+CREATE TABLE t1(i int);
+INSERT INTO t1 values (1);
+DROP TABLE t1;
+--sync_slave_with_master
+
+###############################################################################
+# Case 2: Uninstallation of semi sync plugins should be disallowed
+#  when it is in use i.e., when semi sync replication is active
+###############################################################################
+# Step 2.1: Install and enable semi sync replication between master and slave
+--source include/install_semisync.inc
+
+# Step 2.2: Check that rpl_semi_sync_slave uninstallation on Slave is not
+#  possible at this state
+--connection slave
+call mtr.add_suppression("Plugin 'rpl_semi_sync_slave' cannot be uninstalled now");
+--error ER_UNKNOWN_ERROR
+UNINSTALL PLUGIN rpl_semi_sync_slave;
+
+# Step 2.3: Check that rpl_semi_sync_master uninstallation on Master is not
+#  possible at this state
+--connection master
+call mtr.add_suppression("Plugin 'rpl_semi_sync_master' cannot be uninstalled now");
+--error ER_UNKNOWN_ERROR
+UNINSTALL PLUGIN rpl_semi_sync_master;
+
+# Step 2.4: Check that replication is working fine at the end of the test case.
+CREATE TABLE t1(i int);
+INSERT INTO t1 values (2);
+DROP TABLE t1;
+--sync_slave_with_master
+
+# Step 2.5: Make sure rpl_semi_sync_master_status on Master and
+# rpl_semi_sync_slave_staus on Slave are ON
+--let $slave_status=[show status like "Rpl_semi_sync_slave_status", Value, 1]
+--let assert_cond= "$slave_status" = "ON"
+--let assert_text= semi sync slave status should be ON.
+--source include/assert.inc
+
+--connection master
+--let $master_status=[show status like "Rpl_semi_sync_master_status", Value, 1]
+--let assert_cond= "$master_status" = "ON"
+--let assert_text= semi sync master status should be ON.
+--source include/assert.inc
+
+--let $master_clients=[show status like "Rpl_semi_sync_master_clients", Value, 1]
+--let assert_cond= $master_clients = 1
+--let assert_text= semi sync master clients should be 1.
+--source include/assert.inc
+
+###############################################################################
+# Case 3: Uninstallation of semi sync plugin should be disallowed when there
+#  are semi sync slaves even though rpl_semi_sync_master_enabled= OFF;.
+###############################################################################
+# Step 3.1: Disable semi sync on master
+--connection master
+SET GLOBAL rpl_semi_sync_master_enabled = OFF;
+
+# Step 3.2: Check that still Rpl_semi_sync_master_clients is 1
+--let $master_clients=[show status like "Rpl_semi_sync_master_clients", Value, 1]
+--let assert_cond= $master_clients = 1
+--let assert_text= semi sync master clients should be 1.
+--source include/assert.inc
+
+# Step 3.3: Since Rpl_semi_sync_master_clients is 1, uninstallation of
+#  rpl_semi_sync_master should be disallowed.
+--error ER_UNKNOWN_ERROR
+UNINSTALL PLUGIN rpl_semi_sync_master;
+
+###############################################################################
+# Case 4: Uninstallation of semi sync plugin should be allowed when it is not
+#  in use. Same as Case 1 but this case is to check the case after enabling and
+#  disabling semi sync replication.
+###############################################################################
+
+# Step 4.1: Stop IO thread on slave.
+--connection slave
+--source include/stop_slave.inc
+
+# Step 4.2: Disable semi sync on slave.
+SET GLOBAL rpl_semi_sync_slave_enabled = OFF;
+
+# Step 4.3: Start IO thread on slave.
+--source include/start_slave.inc
+
+# Step 4.4: Uninstall semi sync plugin, it should be successful now.
+UNINSTALL PLUGIN rpl_semi_sync_slave;
+
+# Step 4.5: On Master, check that semi sync slaves are now '0'.
+--connection master
+--let $master_clients=[show status like "Rpl_semi_sync_master_clients", Value, 1]
+--let assert_cond= $master_clients = 0
+--let assert_text= semi sync master clients should be 0.
+--source include/assert.inc
+
+# Step 4.6: So uninstalling semi sync plugin should be allowed
+UNINSTALL PLUGIN rpl_semi_sync_master;
+
+# Step 4.7: Check that replication is working fine at the end of the test case
+CREATE TABLE t1(i int);
+INSERT INTO t1 values (3);
+DROP TABLE t1;
+--sync_slave_with_master
+
+# Cleanup
+source include/rpl_end.inc;

sql/sql_plugin.cc

@@ -1930,6 +1930,49 @@ bool mysql_uninstall_plugin(THD *thd, const LEX_STRING *name)
     goto err;
   }
 
+#ifdef HAVE_REPLICATION
+  /* Block Uninstallation of semi_sync plugins (Master/Slave)
+     when they are busy
+   */
+  char buff[20];
+  /*
+    Master: If there are active semi sync slaves for this Master,
+    then that means it is busy and rpl_semi_sync_master plugin
+    cannot be uninstalled. To check whether the master
+    has any semi sync slaves or not, check Rpl_semi_sync_master_cliens
+    status variable value, if it is not 0, that means it is busy.
+  */
+  if (!strcmp(name->str, "rpl_semi_sync_master") &&
+      get_status_var(thd,
+                     plugin->plugin->status_vars,
+                     "Rpl_semi_sync_master_clients",buff) &&
+      strcmp(buff,"0") )
+  {
+    sql_print_error("Plugin 'rpl_semi_sync_master' cannot be uninstalled now. "
+                    "Stop any active semisynchronous slaves of this master "
+                    "first.\n");
+    my_error(ER_UNKNOWN_ERROR, MYF(0), name->str);
+    goto err;
+  }
+  /* Slave: If there is semi sync enabled IO thread active on this Slave,
+    then that means plugin is busy and rpl_semi_sync_slave plugin
+    cannot be uninstalled. To check whether semi sync
+    IO thread is active or not, check Rpl_semi_sync_slave_status status
+    variable value, if it is ON, that means it is busy.
+  */
+  if (!strcmp(name->str, "rpl_semi_sync_slave") &&
+      get_status_var(thd, plugin->plugin->status_vars,
+                     "Rpl_semi_sync_slave_status", buff) &&
+      !strcmp(buff,"ON") )
+  {
+    sql_print_error("Plugin 'rpl_semi_sync_slave' cannot be uninstalled now. "
+                    "Stop any active semisynchronous I/O threads on this slave "
+                    "first.\n");
+    my_error(ER_UNKNOWN_ERROR, MYF(0), name->str);
+    goto err;
+  }
+#endif
+
   plugin->state= PLUGIN_IS_DELETED;
   if (plugin->ref_count)
     push_warning(thd, MYSQL_ERROR::WARN_LEVEL_WARN,

sql/sql_show.cc

@@ -2117,6 +2117,59 @@ void free_status_vars()
   delete_dynamic(&all_status_vars);
 }
 
+/**
+  @brief           Get the value of given status variable
+
+  @param[in]       thd        thread handler
+  @param[in]       list       list of SHOW_VAR objects in which function should
+                              search
+  @param[in]       name       name of the status variable
+  @param[in/out]   value      buffer in which value of the status variable
+                              needs to be filled in
+
+  @return          status
+    @retval        FALSE      if variable is not found in the list
+    @retval        TRUE       if variable is found in the list
+  NOTE: Currently this function is implemented just to support 'bool' status
+  variables and 'long' status variables *only*. It can be extended very easily
+  for further show_types in future if required.
+  TODO: Currently show_status_arary switch case is tightly coupled with
+  pos, end, buff, value variables and also it stores the values in a 'table'.
+  Decouple the switch case to fill the buffer value so that it can be used
+  in show_status_array() and get_status_var() to avoid duplicate code.
+ */
+
+bool get_status_var(THD* thd, SHOW_VAR *list, const char * name, char * const value)
+{
+  for (; list->name; list++)
+  {
+    int res= strcmp(list->name, name);
+    if (res == 0)
+    {
+      /*
+        if var->type is SHOW_FUNC, call the function.
+        Repeat as necessary, if new var is again SHOW_FUNC
+      */
+      SHOW_VAR tmp;
+      for (; list->type == SHOW_FUNC; list= &tmp)
+        ((mysql_show_var_func)(list->value))(thd, &tmp, value);
+      switch (list->type) {
+      case SHOW_BOOL:
+        strmov(value, *(bool*) list->value ? "ON" : "OFF");
+        break;
+      case SHOW_LONG:
+        int10_to_str(*(long*) list->value, value, 10);
+        break;
+      default:
+        /* not supported type */
+        DBUG_ASSERT(0);
+      }
+      return TRUE;
+    }
+  }
+  return FALSE;
+}
+
 /*
   Removes an array of SHOW_VAR entries from the output of SHOW STATUS
 

sql/sql_show.h

@@ -113,6 +113,7 @@ int add_status_vars(SHOW_VAR *list);
 void remove_status_vars(SHOW_VAR *list);
 void init_status_vars();
 void free_status_vars();
+bool get_status_var(THD* thd, SHOW_VAR *list, const char *name, char * const buff);
 void reset_status_vars();
 bool show_create_trigger(THD *thd, const sp_name *trg_name);
 void view_store_options(THD *thd, TABLE_LIST *table, String *buff);