Galera4

.gitmodules

@@ -4,3 +4,7 @@
 [submodule "storage/rocksdb/rocksdb"]
 	path = storage/rocksdb/rocksdb
 	url = https://github.com/facebook/rocksdb.git
+[submodule "wsrep-lib"]
+	path = wsrep-lib
+	url = https://github.com/codership/wsrep-lib.git
+	branch = master

CMakeLists.txt

@@ -143,6 +143,7 @@ INCLUDE(misc)
 INCLUDE(mysql_version)
 INCLUDE(cpack_source_ignore_files)
 INCLUDE(install_layout)
+INCLUDE(submodules)
 INCLUDE(wsrep)
 INCLUDE(cpack_rpm)
 INCLUDE(cpack_deb)
@@ -377,7 +378,6 @@ ENDIF()
 
 SET (MYSQLD_STATIC_PLUGIN_LIBS "" CACHE INTERNAL "")
 
-INCLUDE(submodules)
 INCLUDE(mariadb_connector_c) # this does ADD_SUBDIRECTORY(libmariadb)
 
 # Add storage engines and plugins.
@@ -405,7 +405,7 @@ IF(NOT WITHOUT_SERVER)
   ENDIF(WITH_EMBEDDED_SERVER)
 
   IF(WITH_WSREP)
-    ADD_SUBDIRECTORY(wsrep)
+    ADD_SUBDIRECTORY(wsrep-lib)
   ENDIF()
 
   ADD_SUBDIRECTORY(mysql-test)

cmake/cpack_rpm.cmake

@@ -171,7 +171,7 @@ SETA(CPACK_RPM_server_PACKAGE_REQUIRES
 
 IF(WITH_WSREP)
   SETA(CPACK_RPM_server_PACKAGE_REQUIRES
-    "galera" "rsync" "lsof" "grep" "gawk" "iproute"
+    "galera-4" "rsync" "lsof" "grep" "gawk" "iproute"
     "coreutils" "findutils" "tar")
 ENDIF()
 

cmake/make_dist.cmake.in

@@ -50,6 +50,14 @@ IF(GIT_EXECUTABLE)
   IF(NOT RESULT EQUAL 0)
     SET(GIT_EXECUTABLE)
   ENDIF()
+  EXECUTE_PROCESS(
+    COMMAND "${GIT_EXECUTABLE}" submodule foreach "${GIT_EXECUTABLE} checkout-index --all --prefix=${PACKAGE_DIR}/wsrep-lib/$path/"
+    WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/wsrep-lib
+    RESULT_VARIABLE RESULT
+  )
+  IF(NOT RESULT EQUAL 0)
+    SET(GIT_EXECUTABLE)
+  ENDIF()
 ENDIF()
 
 CONFIGURE_FILE(${CMAKE_BINARY_DIR}/include/source_revision.h

cmake/submodules.cmake

@@ -14,16 +14,16 @@ IF(GIT_EXECUTABLE AND EXISTS "${CMAKE_SOURCE_DIR}/.git")
     SET(update_result 0)
   ELSEIF (cmake_update_submodules MATCHES force)
     MESSAGE(STATUS "Updating submodules (forced)")
-    EXECUTE_PROCESS(COMMAND "${GIT_EXECUTABLE}" submodule update --init --force
+    EXECUTE_PROCESS(COMMAND "${GIT_EXECUTABLE}" submodule update --init --force --recursive
                     WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}"
                     RESULT_VARIABLE update_result)
   ELSEIF (cmake_update_submodules MATCHES yes)
-    EXECUTE_PROCESS(COMMAND "${GIT_EXECUTABLE}" submodule update --init
+    EXECUTE_PROCESS(COMMAND "${GIT_EXECUTABLE}" submodule update --init --recursive
                     WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}"
                     RESULT_VARIABLE update_result)
   ELSE()
     MESSAGE(STATUS "Updating submodules")
-    EXECUTE_PROCESS(COMMAND "${GIT_EXECUTABLE}" submodule update --init
+    EXECUTE_PROCESS(COMMAND "${GIT_EXECUTABLE}" submodule update --init --recursive
                     WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}"
                     RESULT_VARIABLE update_result)
   ENDIF()

cmake/wsrep.cmake

@@ -24,12 +24,15 @@ ELSE()
 ENDIF()
 
 OPTION(WITH_WSREP "WSREP replication API (to use, e.g. Galera Replication library)" ${with_wsrep_default})
+OPTION(WITH_WSREP_ALL
+  "Build all components of WSREP (unit tests, sample programs)"
+  OFF)
 
 # Set the patch version
-SET(WSREP_PATCH_VERSION "23")
+SET(WSREP_PATCH_VERSION "22")
 
 # Obtain wsrep API version
-FILE(STRINGS "${MySQL_SOURCE_DIR}/wsrep/wsrep_api.h" WSREP_API_VERSION
+FILE(STRINGS "${CMAKE_SOURCE_DIR}/wsrep-lib/wsrep-API/v26/wsrep_api.h" WSREP_API_VERSION
      LIMIT_COUNT 1 REGEX "WSREP_INTERFACE_VERSION")
 STRING(REGEX MATCH "([0-9]+)" WSREP_API_VERSION "${WSREP_API_VERSION}")
 
@@ -40,4 +43,12 @@ SET(WSREP_PROC_INFO ${WITH_WSREP})
 
 IF(WITH_WSREP)
   SET(WSREP_PATCH_VERSION "wsrep_${WSREP_VERSION}")
+  if (NOT WITH_WSREP_ALL)
+    SET(WSREP_LIB_WITH_UNIT_TESTS OFF CACHE BOOL
+      "Disable unit tests for wsrep-lib")
+    SET(WSREP_LIB_WITH_DBSIM OFF CACHE BOOL
+      "Disable building dbsim for wsrep-lib")
+  endif()
+  INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/wsrep-lib/include)
+  INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/wsrep-lib/wsrep-API/v26)
 ENDIF()

debian/control

@@ -430,7 +430,7 @@ Recommends: libhtml-template-perl
 Pre-Depends: adduser (>= 3.40),
              debconf,
              mariadb-common (>= ${source:Version})
-Depends: galera-3 (>=25.3),
+Depends: galera-4 (>=26.4),
          gawk,
          iproute | iproute2,
          libdbi-perl,

extra/mariabackup/CMakeLists.txt

@@ -40,9 +40,6 @@ IF(NOT HAVE_SYSTEM_REGEX)
   INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/pcre)
 ENDIF()
 
-IF(WITH_WSREP)
-  INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/wsrep)
-ENDIF()
 
 ADD_DEFINITIONS(-UMYSQL_SERVER)
 ########################################################################

include/thr_lock.h

@@ -168,9 +168,9 @@ void thr_set_lock_wait_callback(void (*before_wait)(void),
                                 void (*after_wait)(void));
 
 #ifdef WITH_WSREP
-  typedef my_bool (* wsrep_thd_is_brute_force_fun)(void *, my_bool);
-  typedef int (* wsrep_abort_thd_fun)(void *, void *, my_bool);
-  typedef int (* wsrep_on_fun)(void *);
+  typedef my_bool (* wsrep_thd_is_brute_force_fun)(const MYSQL_THD, my_bool);
+  typedef my_bool(* wsrep_abort_thd_fun)(const MYSQL_THD, MYSQL_THD, my_bool);
+  typedef my_bool (* wsrep_on_fun)(const MYSQL_THD);
   void wsrep_thr_lock_init(
     wsrep_thd_is_brute_force_fun bf_fun, wsrep_abort_thd_fun abort_fun,
     my_bool debug, my_bool convert_LOCK_to_trx, wsrep_on_fun on_fun);

include/wsrep.h

@@ -13,11 +13,11 @@
    along with this program; if not, write to the Free Software
    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111-1301 USA */
 
-#include <my_config.h>
-
 #ifndef WSREP_INCLUDED
 #define WSREP_INCLUDED
 
+#include <my_config.h>
+
 #ifdef WITH_WSREP
 #define IF_WSREP(A,B) A
 #define DBUG_ASSERT_IF_WSREP(A) DBUG_ASSERT(A)
@@ -28,12 +28,14 @@
     goto wsrep_error_label;
 
 #define WSREP_TO_ISOLATION_BEGIN_ALTER(db_, table_, table_list_, alter_info_) \
-  if (WSREP_ON && WSREP(thd) && wsrep_to_isolation_begin(thd, db_, table_,    \
+  if (WSREP(thd) && wsrep_thd_is_local(thd) &&                          \
+      wsrep_to_isolation_begin(thd, db_, table_,                        \
                                table_list_, alter_info_))               \
     goto wsrep_error_label;
 
 #define WSREP_TO_ISOLATION_END                                          \
-  if (WSREP_ON && (WSREP(thd) || (thd && thd->wsrep_exec_mode==TOTAL_ORDER))) \
+  if ((WSREP(thd) && wsrep_thd_is_local_toi(thd)) ||                    \
+      wsrep_thd_is_in_rsu(thd))                                         \
     wsrep_to_isolation_end(thd);
 
 /*
@@ -54,19 +56,22 @@
     { if (WSREP_CLIENT(thd_) &&                                         \
           wsrep_sync_wait(thd_, before_)) goto wsrep_error_label; }
 
-#else
+#else /* !WITH_WSREP */
+
+/* These macros are needed to compile MariaDB without WSREP support
+ * (e.g. embedded) */
+
 #define IF_WSREP(A,B) B
-#define DBUG_ASSERT_IF_WSREP(A)
+//#define DBUG_ASSERT_IF_WSREP(A)
 #define WSREP_DEBUG(...)
-#define WSREP_INFO(...)
-#define WSREP_WARN(...)
+//#define WSREP_INFO(...)
+//#define WSREP_WARN(...)
 #define WSREP_ERROR(...)
 #define WSREP_TO_ISOLATION_BEGIN(db_, table_, table_list_) do { } while(0)
 #define WSREP_TO_ISOLATION_BEGIN_ALTER(db_, table_, table_list_, alter_info_)
 #define WSREP_TO_ISOLATION_END
 #define WSREP_TO_ISOLATION_BEGIN_WRTCHK(db_, table_, table_list_)
 #define WSREP_SYNC_WAIT(thd_, before_)
-
 #endif /* WITH_WSREP */
 
 #endif /* WSREP_INCLUDED */

mysql-test/include/check-testcase.test

@@ -103,5 +103,56 @@ cat_file $datadir.tempfiles.txt;
 remove_file $datadir.tempfiles.txt;
 list_files $datadir/mysql #sql*;
 
+#
+# Check that SHOW ENGINE INNODB STATUS does not show any active transactions
+# We do this only if wsrep provider is loaded, to avoid disturbing any non-Galera MTR tests
+#
+if (`SELECT COUNT(*)=1 FROM INFORMATION_SCHEMA.PLUGINS WHERE PLUGIN_NAME = 'wsrep' AND PLUGIN_STATUS='ACTIVE'`) {
+  if (`SELECT @@wsrep_on`) {
+    if (`SELECT COUNT(*) FROM information_schema.innodb_trx WHERE trx_mysql_thread_id != 0`) {
+      if ($before) {
+        --echo Before test start.
+      }
+      if (!$before) {
+        --echo After test end.
+      }
+      --echo There is one or more active InnoDB transaction(s) when there should be none. Dumping some diagnostics.
+
+      --let $status_locks = `SELECT @@innodb_status_output_locks`
+      --let $status_output = `SELECT @@innodb_status_output`
+      --enable_query_log
+      SET GLOBAL innodb_status_output_locks=ON;
+      SHOW ENGINE INNODB STATUS;
+      --disable_query_log
+      --eval SET GLOBAL innodb_status_output_locks=$status_locks;
+      --eval SET GLOBAL innodb_status_output=$status_output;
+      --enable_query_log
+
+      --vertical_results
+      if ($before) {
+        --replace_regex /$/ /
+      }
+      SELECT * FROM information_schema.processlist;
+
+      if ($before) {
+        --replace_regex /$/ /
+      }
+      SELECT * FROM information_schema.innodb_trx;
+
+      if ($before) {
+        --replace_regex /$/ /
+      }
+      SELECT * FROM information_schema.innodb_locks;
+
+      if ($before) {
+        --replace_regex /$/ /
+      }
+      SELECT * FROM information_schema.innodb_lock_waits;
+      --horizontal_results
+      --disable_query_log
+    }
+  }
+}
+
 --enable_query_log
 

mysql-test/include/galera_cluster.inc

@@ -8,5 +8,11 @@
 
 --let $galera_cluster_size = 2
 --source include/galera_init.inc
+--source include/have_innodb.inc
+--source include/galera_wait_ready.inc
 
+--connection node_2
+--source include/galera_wait_ready.inc
 --source include/have_innodb.inc
+
+--connection node_1

mysql-test/include/galera_wait_sync_point.inc

 --let $wait_timeout = 10
 --let $wsrep_on_orig = `SELECT @@wsrep_on`
 SET SESSION wsrep_on = 0;
+
+#
+# following is only for debugging purposes
+# should be commented out when test wporks as planned
+#
+#--sleep 1
+# SHOW PROCESSLIST;
+#SHOW STATUS LIKE 'wsrep_%';
+#--echo $galera_sync_point
+
 --let $wait_condition = SELECT 1 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_debug_sync_waiters' AND VARIABLE_VALUE = '$galera_sync_point'
 --source include/wait_condition.inc
 --eval SET SESSION wsrep_on = $wsrep_on_orig
+

mysql-test/include/have_wsrep_enabled.inc

 # To be used in a test which requires wsrep plugin to be ACTIVE and enabled
 # (i.e. wsrep_on=ON). It includes have_wsrep.inc.
 
---source include/have_wsrep.inc
 --source include/have_innodb.inc
 
 if (`SELECT COUNT(*)=0 FROM INFORMATION_SCHEMA.GLOBAL_VARIABLES WHERE VARIABLE_NAME = 'wsrep_on' AND VARIABLE_VALUE='ON'`)

mysql-test/include/kill_galera.inc

+--echo Killing server ...
+
+# Write file to make mysql-test-run.pl expect the crash, but don't start it
+--let $_server_id= `SELECT @@server_id`
+--let $_expect_file_name= $MYSQLTEST_VARDIR/tmp/mysqld.$_server_id.expect
+--exec echo "wait" > $_expect_file_name
+
+# Kill the connected server
+--disable_reconnect
+--let KILL_NODE_PIDFILE = `SELECT @@pid_file`
+
+--perl
+        my $pid_filename = $ENV{'KILL_NODE_PIDFILE'};
+        my $mysqld_pid = `cat $pid_filename`;
+        chomp($mysqld_pid);
+        system("kill -9 $mysqld_pid");
+        exit(0);
+EOF
+
+--source include/wait_until_disconnected.inc

mysql-test/include/wsrep_wait_disconnect.inc

+let $wait_condition = SELECT 1 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_ready' AND VARIABLE_VALUE = 'OFF';
+# since this is called until AFTER provider disconnects,we need to allow
+# queries in non-prim
+#
+# We are also forced to use a hard-coded value for wsrep_sync_wait here because
+# we can not issue a SELECT query to obtain the original value and then restore
+# it
+disable_query_log;
+SET SESSION wsrep_sync_wait = 7;
+--let $restore_wsrep_on = `SHOW VARIABLES WHERE Variable_name = 'wsrep_on' AND Value = 'ON'`
+SET SESSION wsrep_on = OFF;
+
+--source include/wait_condition.inc
+
+if ($restore_wsrep_on != "")
+{
+  --eval SET SESSION wsrep_on = ON
+}
+SET SESSION wsrep_sync_wait = 15;
+enable_query_log;

mysql-test/mysql-test-run.pl

@@ -2896,15 +2896,44 @@ sub mysql_server_start($) {
     # Save this test case information, so next can examine it
     $mysqld->{'started_tinfo'}= $tinfo;
   }
+
+  # If wsrep is on, we need to wait until the first
+  # server starts and bootstraps the cluster before
+  # starting other servers. The bootsrap server in the
+  # configuration should always be the first which has
+  # wsrep_on=ON
+  if (wsrep_on($mysqld) && wsrep_is_bootstrap_server($mysqld))
+  {
+    mtr_verbose("Waiting for wsrep bootstrap server to start");
+    if ($mysqld->{WAIT}->($mysqld))
+    {
+      return 1;
+    }
+  }
 }
 
 sub mysql_server_wait {
-  my ($mysqld) = @_;
+  my ($mysqld, $tinfo) = @_;
 
-  return not sleep_until_file_created($mysqld->value('pid-file'),
+  if (!sleep_until_file_created($mysqld->value('pid-file'),
                                 $opt_start_timeout,
                                 $mysqld->{'proc'},
-                                      $warn_seconds);
+                                $warn_seconds))
+  {
+    $tinfo->{comment}= "Failed to start ".$mysqld->name() . "\n";
+    return 1;
+  }
+
+  if (wsrep_on($mysqld))
+  {
+    mtr_verbose("Waiting for wsrep server " . $mysqld->name() . " to be ready");
+    if (!wait_wsrep_ready($tinfo, $mysqld))
+    {
+      return 1;
+    }
+  }
+
+  return 0;
 }
 
 sub create_config_file_for_extern {
@@ -5408,6 +5437,118 @@ sub stop_servers($$) {
   }
 }
 
+#
+# run_query_output
+#
+# Run a query against a server using mysql client. The output of
+# the query will be written into outfile.
+#
+sub run_query_output {
+  my ($mysqld, $query, $outfile)= @_;
+  my $args;
+
+  mtr_init_args(\$args);
+  mtr_add_arg($args, "--defaults-file=%s", $path_config_file);
+  mtr_add_arg($args, "--defaults-group-suffix=%s", $mysqld->after('mysqld'));
+  mtr_add_arg($args, "--silent");
+  mtr_add_arg($args, "--execute=%s", $query);
+
+  my $res= My::SafeProcess->run
+  (
+    name          => "run_query_output -> ".$mysqld->name(),
+    path          => $exe_mysql,
+    args          => \$args,
+    output        => $outfile,
+    error         => $outfile
+  );
+
+  return $res
+}
+
+
+#
+# wsrep_wait_ready
+#
+# Wait until the server has been joined to the cluster and is
+# ready for operation.
+#
+# RETURN
+# 1 Server is ready
+# 0 Server didn't transition to ready state within start timeout
+#
+sub wait_wsrep_ready($$) {
+  my ($tinfo, $mysqld)= @_;
+
+  my $sleeptime= 100; # Milliseconds
+  my $loops= ($opt_start_timeout * 1000) / $sleeptime;
+
+  my $name= $mysqld->name();
+  my $outfile= "$opt_vardir/tmp/$name.wsrep_ready";
+  my $query= "SET SESSION wsrep_sync_wait = 0;
+              SELECT VARIABLE_NAME, VARIABLE_VALUE
+              FROM INFORMATION_SCHEMA.GLOBAL_STATUS
+              WHERE VARIABLE_NAME = 'wsrep_ready'";
+
+  for (my $loop= 1; $loop <= $loops; $loop++)
+  {
+    # Careful... if MTR runs with option 'verbose' then the
+    # file contains also SafeProcess verbose output
+    if (run_query_output($mysqld, $query, $outfile) == 0 &&
+        mtr_grab_file($outfile) =~ /WSREP_READY\s+ON/)
+    {
+      unlink($outfile);
+      return 1;
+    }
+    mtr_milli_sleep($sleeptime);
+  }
+
+  $tinfo->{logfile}= "WSREP did not transition to state READY";
+  return 0;
+}
+
+#
+# wsrep_is_bootstrap_server
+#
+# Check if the server is the first one to be started in the
+# cluster.
+#
+# RETURN
+# 1 The server is a bootstrap server
+# 0 The server is not a bootstrap server
+#
+sub wsrep_is_bootstrap_server($) {
+  my $mysqld= shift;
+
+  my $cluster_address= $mysqld->if_exist('wsrep-cluster-address') ||
+                       $mysqld->if_exist('wsrep_cluster_address');
+  if (defined $cluster_address)
+  {
+    return $cluster_address eq "gcomm://" || $cluster_address eq "'gcomm://'";
+  }
+  return 0;
+}
+
+#
+# wsrep_on
+#
+# Check if wsrep has been enabled for a server.
+#
+# RETURN
+# 1 Wsrep has been enabled
+# 0 Wsrep is not enabled
+#
+sub wsrep_on($) {
+  my $mysqld= shift;
+  #check if wsrep_on=  is set in configuration
+  if ($mysqld->if_exist('wsrep-on')) {
+    my $on= "".$mysqld->value('wsrep-on');
+    if ($on eq "1" || $on eq "ON") {
+      return 1;
+    }
+  }
+  return 0;
+}
+
 
 #
 # start_servers
@@ -5427,8 +5568,7 @@ sub start_servers($) {
 
   for (all_servers()) {
     next unless $_->{WAIT} and started($_);
-    if ($_->{WAIT}->($_)) {
-      $tinfo->{comment}= "Failed to start ".$_->name() . "\n";
+    if ($_->{WAIT}->($_, $tinfo)) {
       return 1;
     }
   }

mysql-test/suite/galera/disabled.def

@@ -9,13 +9,14 @@
 #  Do not use any TAB characters for whitespace.
 #
 ##############################################################################
-
+MW-360 : needs rewrite to be MariaDB gtid compatible
 galera_flush :  MariaDB does not have global.thread_statistics
 galera_account_management : MariaDB 10.0 does not support ALTER USER
 galera_binlog_rows_query_log_events: MariaDB does not support binlog_rows_query_log_events
 galera_migrate : MariaDB does not support START SLAVE USER
 galera_as_master_gtid : Requires MySQL GTID
 galera_as_master_gtid_change_master : Requires MySQL GTID
+galera_as_slave_gtid_replicate_do_db_cc : Requires MySQL GTID
 galera_as_slave_preordered : wsrep-preordered feature not merged to MariaDB
 GAL-419 : MDEV-13549 Galera test failures
 galera_var_notify_cmd : MDEV-13549 Galera test failures
@@ -52,3 +53,13 @@ galera.galera_var_reject_queries : assertion in inline_mysql_socket_send
 query_cache : MDEV-18137: Galera test failure on query_cache
 galera.galera_autoinc_sst_mariabackup : MDEV-18177 Galera test failure on galera_autoinc_sst_mariabackup
 galera_gcache_recover_manytrx : MDEV-15740
+galera.galera_ist_mariabackup : Leaves port open
+galera.galera_sst_rsync2 : MDEV-18178 Galera test failure on galera_sst_rsync2
+galera.galera_kill_largechanges : MDEV-18179 Galera test failure on galera.galera_kill_largechanges
+galera.galera_concurrent_ctas : MDEV-18180 Galera test failure on galera.galera_concurrent_ctas
+galera.galera_var_retry_autocommit: MDEV-18181 Galera test failure on galera.galera_var_retry_autocommit
+galera.galera_many_tables_nopk : MDEV-18182 Galera test failure on galera.galera_many_tables_nopk
+galera.galera_kill_nochanges : MDEV-18280 Galera test failure on galera_split_brain and galera_kill_nochanges
+galera.galera_split_brain : MDEV-18280 Galera test failure on galera_split_brain and galera_kill_nochanges
+galera.galera_bf_abort_group_commit : MDEV-18282 Galera test failure on galera.galera_bf_abort_group_commit
+galera.GCF-1081 : MDEV-18283 Galera test failure on galera.GCF-1081

mysql-test/suite/galera/galera_2nodes.cnf

@@ -2,7 +2,7 @@
 !include include/default_mysqld.cnf
 
 [mysqld]
-wsrep-on=1
+loose-innodb
 binlog-format=row
 innodb-autoinc-lock-mode=2
 default-storage-engine=innodb
@@ -10,20 +10,35 @@ wsrep-provider=@ENV.WSREP_PROVIDER
 wsrep_node_address=127.0.0.1
 # enforce read-committed characteristics across the cluster
 wsrep-sync-wait=15
+# lock schedule alg appears to be VATS by default, and it is not
+# yet compatible with galera
+innodb_lock_schedule_algorithm=FCFS
 
 [mysqld.1]
+loose-innodb
 #galera_port=@OPT.port
 #ist_port=@OPT.port
 #sst_port=@OPT.port
+wsrep-on=1
 wsrep-cluster-address=gcomm://
 wsrep_provider_options='repl.causal_read_timeout=PT90S;base_port=@mysqld.1.#galera_port;evs.suspect_timeout=PT10S;evs.inactive_timeout=PT30S;evs.install_timeout=PT15S;gcache.size=10M'
 wsrep_node_incoming_address=127.0.0.1:@mysqld.1.port
 wsrep_sst_receive_address='127.0.0.1:@mysqld.1.#sst_port'
 
+# enforce read-committed characteristics across the cluster
+wsrep_causal_reads=ON
+wsrep_sync_wait = 15
+# lock schedule alg appears to be VATS by default, and it is not
+# yet compatible with galera
+innodb_lock_schedule_algorithm=FCFS
+
 [mysqld.2]
+loose-innodb
+# debug=d:t:i:o,/tmp/mysqld.2.trace
 #galera_port=@OPT.port
 #ist_port=@OPT.port
 #sst_port=@OPT.port
+wsrep-on=1
 wsrep_cluster_address='gcomm://127.0.0.1:@mysqld.1.#galera_port'
 wsrep_provider_options='repl.causal_read_timeout=PT90S;base_port=@mysqld.2.#galera_port;evs.suspect_timeout=PT10S;evs.inactive_timeout=PT30S;evs.install_timeout=PT15S'
 
@@ -34,8 +49,14 @@ wsrep_sync_wait = 15
 wsrep_node_address=127.0.0.1
 wsrep_sst_receive_address=127.0.0.2:@mysqld.2.#sst_port
 wsrep_node_incoming_address=127.0.0.1:@mysqld.2.port
-wsrep_sst_receive_address='127.0.0.1:@mysqld.2.#sst_port'
+wsrep_sst_receive_address='127.0.0.2:@mysqld.2.#sst_port'
 
+# enforce read-committed characteristics across the cluster
+wsrep_causal_reads=ON
+wsrep_sync_wait = 15
+# lock schedule alg appears to be VATS by default, and it is not
+# yet compatible with galera
+innodb_lock_schedule_algorithm=FCFS
 
 [ENV]
 NODE_MYPORT_1= @mysqld.1.port
@@ -43,4 +64,3 @@ NODE_MYSOCK_1= @mysqld.1.socket
 
 NODE_MYPORT_2= @mysqld.2.port
 NODE_MYSOCK_2= @mysqld.2.socket
-

mysql-test/suite/galera/galera_2nodes_as_master.cnf

@@ -11,6 +11,9 @@ log-bin=mysqld-bin
 binlog-format=row
 innodb-autoinc-lock-mode=2
 default-storage-engine=innodb
+# lock schedule alg appears to be VATS by default, and it is not
+# yet compatible with galera
+innodb_lock_schedule_algorithm=FCFS
 
 [mysqld.1]
 #galera_port=@OPT.port
@@ -30,6 +33,9 @@ wsrep_node_incoming_address=127.0.0.1:@mysqld.1.port
 # enforce read-committed characteristics across the cluster
 wsrep-causal-reads=ON
 wsrep-sync-wait=15
+# lock schedule alg appears to be VATS by default, and it is not
+# yet compatible with galera
+innodb_lock_schedule_algorithm=FCFS
 
 [mysqld.2]
 #galera_port=@OPT.port
@@ -49,9 +55,15 @@ wsrep_node_incoming_address=127.0.0.1:@mysqld.2.port
 # enforce read-committed characteristics across the cluster
 wsrep-causal-reads=ON
 wsrep-sync-wait=15
+# lock schedule alg appears to be VATS by default, and it is not
+# yet compatible with galera
+innodb_lock_schedule_algorithm=FCFS
 
 [mysqld.3]
 server-id=3
+# lock schedule alg appears to be VATS by default, and it is not
+# yet compatible with galera
+innodb_lock_schedule_algorithm=FCFS
 
 [ENV]
 NODE_MYPORT_1= @mysqld.1.port
@@ -62,9 +74,3 @@ NODE_MYSOCK_2= @mysqld.2.socket
 
 NODE_MYPORT_3= @mysqld.3.port
 NODE_MYSOCK_3= @mysqld.3.socket
-
-NODE_GALERAPORT_1= @mysqld.1.#galera_port
-NODE_GALERAPORT_2= @mysqld.2.#galera_port
-
-NODE_SSTPORT_1= @mysqld.1.#sst_port
-NODE_SSTPORT_2= @mysqld.2.#sst_port

mysql-test/suite/galera/galera_2nodes_as_slave.cnf

@@ -9,17 +9,13 @@
 binlog-format=row
 
 [mysqld.1]
-log-bin
-server-id=1
-
-[mysqld.2]
 #galera_port=@OPT.port
 #ist_port=@OPT.port
 #sst_port=@OPT.port
-
 wsrep-on=1
 
-log-bin
+log-bin=master-bin
+log-bin-index=master-bin
 log-slave-updates
 
 innodb-autoinc-lock-mode=2
@@ -27,38 +23,53 @@ default-storage-engine=innodb
 wsrep-provider=@ENV.WSREP_PROVIDER
 wsrep_node_address=127.0.0.1
 wsrep-cluster-address=gcomm://
-wsrep_provider_options='base_port=@mysqld.2.#galera_port;gcache.size=10M'
-wsrep_node_incoming_address=127.0.0.1:@mysqld.2.port
-wsrep_sst_receive_address='127.0.0.1:@mysqld.2.#sst_port'
+wsrep_provider_options='base_port=@mysqld.1.#galera_port;gcache.size=10M'
+wsrep_node_incoming_address=127.0.0.1:@mysqld.1.port
+wsrep_sst_receive_address='127.0.0.1:@mysqld.1.#sst_port'
 
 # enforce read-committed characteristics across the cluster
 wsrep-causal-reads=ON
 wsrep-sync-wait=15
-server-id=2
+server-id=1
+# lock schedule alg appears to be VATS by default, and it is not
+# yet compatible with galera
+innodb_lock_schedule_algorithm=FCFS
 
-[mysqld.3]
+[mysqld.2]
 #galera_port=@OPT.port
 #ist_port=@OPT.port
 #sst_port=@OPT.port
-
 wsrep-on=1
 
-log-bin
+log-bin=master-bin
+log-bin-index=master-bin
 log-slave-updates
 
 innodb-autoinc-lock-mode=2
 default-storage-engine=innodb
 wsrep-provider=@ENV.WSREP_PROVIDER
 wsrep_node_address=127.0.0.1
-wsrep_cluster_address='gcomm://127.0.0.1:@mysqld.2.#galera_port'
-wsrep_provider_options='base_port=@mysqld.3.#galera_port;gcache.size=10M'
-wsrep_node_incoming_address=127.0.0.1:@mysqld.3.port
-wsrep_sst_receive_address='127.0.0.1:@mysqld.3.#sst_port'
+wsrep_cluster_address='gcomm://127.0.0.1:@mysqld.1.#galera_port'
+wsrep_provider_options='base_port=@mysqld.2.#galera_port;gcache.size=10M'
+wsrep_node_incoming_address=127.0.0.1:@mysqld.2.port
+wsrep_sst_receive_address='127.0.0.1:@mysqld.2.#sst_port'
 
 # enforce read-committed characteristics across the cluster
 wsrep-causal-reads=ON
 wsrep-sync-wait=15
+server-id=2
+# lock schedule alg appears to be VATS by default, and it is not
+# yet compatible with galera
+innodb_lock_schedule_algorithm=FCFS
+
+[mysqld.3]
+log-bin=master-bin
+log-bin-index=master-bin
 server-id=3
+# lock schedule alg appears to be VATS by default, and it is not
+# yet compatible with galera
+innodb_lock_schedule_algorithm=FCFS
+
 
 [ENV]
 NODE_MYPORT_1= @mysqld.1.port
@@ -69,9 +80,3 @@ NODE_MYSOCK_2= @mysqld.2.socket
 
 NODE_MYPORT_3= @mysqld.3.port
 NODE_MYSOCK_3= @mysqld.3.socket
-
-NODE_GALERAPORT_2= @mysqld.2.#galera_port
-NODE_GALERAPORT_3= @mysqld.3.#galera_port
-
-NODE_SSTPORT_2= @mysqld.2.#sst_port
-NODE_SSTPORT_3= @mysqld.3.#sst_port

mysql-test/suite/galera/galera_3nodes_as_slave.cnf

@@ -8,14 +8,15 @@
 [mysqld]
 log-bin
 binlog-format=row
+# lock schedule alg appears to be VATS by default, and it is not
+# yet compatible with galera
+innodb_lock_schedule_algorithm=FCFS
 
 [mysqld.1]
-server-id=1
-
-[mysqld.2]
 #galera_port=@OPT.port
 #ist_port=@OPT.port
 #sst_port=@OPT.port
+wsrep-on=1
 
 log-slave-updates
 
@@ -24,19 +25,23 @@ default-storage-engine=innodb
 wsrep-provider=@ENV.WSREP_PROVIDER
 wsrep_node_address=127.0.0.1
 wsrep-cluster-address=gcomm://
-wsrep_provider_options='base_port=@mysqld.2.#galera_port;evs.install_timeout = PT15S;evs.max_install_timeouts=1;gcache.size=10M'
-wsrep_node_incoming_address=127.0.0.1:@mysqld.2.port
-wsrep_sst_receive_address='127.0.0.1:@mysqld.2.#sst_port'
+wsrep_provider_options='base_port=@mysqld.1.#galera_port;evs.install_timeout = PT15S;evs.max_install_timeouts=1;gcache.size=10M'
+wsrep_node_incoming_address=127.0.0.1:@mysqld.1.port
+wsrep_sst_receive_address='127.0.0.1:@mysqld.1.#sst_port'
 
 # enforce read-committed characteristics across the cluster
 wsrep-causal-reads=ON
 wsrep-sync-wait=15
-server-id=2
+server-id=1
+# lock schedule alg appears to be VATS by default, and it is not
+# yet compatible with galera
+innodb_lock_schedule_algorithm=FCFS
 
-[mysqld.3]
+[mysqld.2]
 #galera_port=@OPT.port
 #ist_port=@OPT.port
 #sst_port=@OPT.port
+wsrep-on=1
 
 log-slave-updates
 
@@ -44,20 +49,24 @@ innodb-autoinc-lock-mode=2
 default-storage-engine=innodb
 wsrep-provider=@ENV.WSREP_PROVIDER
 wsrep_node_address=127.0.0.1
-wsrep_cluster_address='gcomm://127.0.0.1:@mysqld.2.#galera_port'
-wsrep_provider_options='base_port=@mysqld.3.#galera_port;evs.install_timeout=PT15S;evs.max_install_timeouts=1;gcache.size=10M'
-wsrep_node_incoming_address=127.0.0.1:@mysqld.3.port
-wsrep_sst_receive_address='127.0.0.1:@mysqld.3.#sst_port'
+wsrep_cluster_address='gcomm://127.0.0.1:@mysqld.1.#galera_port'
+wsrep_provider_options='base_port=@mysqld.2.#galera_port;evs.install_timeout=PT15S;evs.max_install_timeouts=1;gcache.size=10M'
+wsrep_node_incoming_address=127.0.0.1:@mysqld.2.port
+wsrep_sst_receive_address='127.0.0.1:@mysqld.2.#sst_port'
 
 # enforce read-committed characteristics across the cluster
 wsrep-causal-reads=ON
 wsrep-sync-wait=15
-server-id=3
+server-id=2
+# lock schedule alg appears to be VATS by default, and it is not
+# yet compatible with galera
+innodb_lock_schedule_algorithm=FCFS
 
-[mysqld.4]
+[mysqld.3]
 #galera_port=@OPT.port
 #ist_port=@OPT.port
 #sst_port=@OPT.port
+wsrep-on=1
 
 log-slave-updates
 
@@ -65,15 +74,24 @@ innodb-autoinc-lock-mode=2
 default-storage-engine=innodb
 wsrep-provider=@ENV.WSREP_PROVIDER
 wsrep_node_address=127.0.0.1
-wsrep_cluster_address='gcomm://127.0.0.1:@mysqld.2.#galera_port'
-wsrep_provider_options='base_port=@mysqld.4.#galera_port;evs.install_timeout=PT15S;evs.max_install_timeouts=1;gcache.size=10M'
-wsrep_node_incoming_address=127.0.0.1:@mysqld.4.port
-wsrep_sst_receive_address='127.0.0.1:@mysqld.4.#sst_port'
+wsrep_cluster_address='gcomm://127.0.0.1:@mysqld.1.#galera_port'
+wsrep_provider_options='base_port=@mysqld.3.#galera_port;evs.install_timeout=PT15S;evs.max_install_timeouts=1;gcache.size=10M'
+wsrep_node_incoming_address=127.0.0.1:@mysqld.3.port
+wsrep_sst_receive_address='127.0.0.1:@mysqld.3.#sst_port'
 
 # enforce read-committed characteristics across the cluster
 wsrep-causal-reads=ON
 wsrep-sync-wait=15
+server-id=3
+# lock schedule alg appears to be VATS by default, and it is not
+# yet compatible with galera
+innodb_lock_schedule_algorithm=FCFS
+
+[mysqld.4]
 server-id=4
+# lock schedule alg appears to be VATS by default, and it is not
+# yet compatible with galera
+innodb_lock_schedule_algorithm=FCFS
 
 [ENV]
 NODE_MYPORT_1= @mysqld.1.port
@@ -87,11 +105,3 @@ NODE_MYSOCK_3= @mysqld.3.socket
 
 NODE_MYPORT_4= @mysqld.4.port
 NODE_MYSOCK_4= @mysqld.4.socket
-
-NODE_GALERAPORT_2= @mysqld.2.#galera_port
-NODE_GALERAPORT_3= @mysqld.3.#galera_port
-NODE_GALERAPORT_4= @mysqld.4.#galera_port
-
-NODE_SSTPORT_2= @mysqld.2.#sst_port
-NODE_SSTPORT_3= @mysqld.3.#sst_port
-NODE_SSTPORT_4= @mysqld.4.#sst_port

mysql-test/suite/galera/galera_4nodes.cnf

@@ -5,50 +5,68 @@
 binlog-format=row
 innodb-autoinc-lock-mode=2
 default-storage-engine=innodb
-wsrep-on=1
 wsrep-provider=@ENV.WSREP_PROVIDER
 wsrep_node_address=127.0.0.1
 # enforce read-committed characteristics across the cluster
 wsrep-causal-reads=ON
 wsrep-sync-wait=15
+# lock schedule alg appears to be VATS by default, and it is not
+# yet compatible with galera
+innodb_lock_schedule_algorithm=FCFS
 
 [mysqld.1]
 #galera_port=@OPT.port
 #ist_port=@OPT.port
 #sst_port=@OPT.port
+wsrep-on=1
 wsrep-cluster-address=gcomm://
 wsrep_provider_options='base_port=@mysqld.1.#galera_port;gcache.size=10M'
 wsrep_node_incoming_address=127.0.0.1:@mysqld.1.port
 wsrep_sst_receive_address='127.0.0.1:@mysqld.1.#sst_port'
+# lock schedule alg appears to be VATS by default, and it is not
+# yet compatible with galera
+innodb_lock_schedule_algorithm=FCFS
 
 [mysqld.2]
 #galera_port=@OPT.port
 #ist_port=@OPT.port
 #sst_port=@OPT.port
+wsrep-on=1
 wsrep_cluster_address='gcomm://127.0.0.1:@mysqld.1.#galera_port'
 wsrep_provider_options='base_port=@mysqld.2.#galera_port;gcache.size=10M'
 wsrep_node_incoming_address=127.0.0.1:@mysqld.2.port
 wsrep_sst_receive_address='127.0.0.1:@mysqld.2.#sst_port'
+# lock schedule alg appears to be VATS by default, and it is not
+# yet compatible with galera
+innodb_lock_schedule_algorithm=FCFS
 
 
 [mysqld.3]
 #galera_port=@OPT.port
 #ist_port=@OPT.port
 #sst_port=@OPT.port
+wsrep-on=1
 wsrep_cluster_address='gcomm://127.0.0.1:@mysqld.1.#galera_port'
 wsrep_provider_options='base_port=@mysqld.3.#galera_port;gcache.size=10M'
 wsrep_node_incoming_address=127.0.0.1:@mysqld.3.port
 wsrep_sst_receive_address='127.0.0.1:@mysqld.3.#sst_port'
+# lock schedule alg appears to be VATS by default, and it is not
+# yet compatible with galera
+innodb_lock_schedule_algorithm=FCFS
 
 
 [mysqld.4]
 #galera_port=@OPT.port
 #ist_port=@OPT.port
 #sst_port=@OPT.port
+wsrep-on=1
 wsrep_cluster_address='gcomm://127.0.0.1:@mysqld.1.#galera_port'
 wsrep_provider_options='base_port=@mysqld.4.#galera_port;gcache.size=10M'
 wsrep_node_incoming_address=127.0.0.1:@mysqld.4.port
 wsrep_sst_receive_address='127.0.0.1:@mysqld.4.#sst_port'
+# lock schedule alg appears to be VATS by default, and it is not
+# yet compatible with galera
+innodb_lock_schedule_algorithm=FCFS
 
 [ENV]
 NODE_MYPORT_1= @mysqld.1.port
@@ -62,13 +80,3 @@ NODE_MYSOCK_3= @mysqld.3.socket
 
 NODE_MYPORT_4= @mysqld.4.port
 NODE_MYSOCK_4= @mysqld.4.socket
-
-NODE_GALERAPORT_1= @mysqld.1.#galera_port
-NODE_GALERAPORT_2= @mysqld.2.#galera_port
-NODE_GALERAPORT_3= @mysqld.3.#galera_port
-NODE_GALERAPORT_4= @mysqld.4.#galera_port
-
-NODE_SSTPORT_1= @mysqld.1.#sst_port
-NODE_SSTPORT_2= @mysqld.2.#sst_port
-NODE_SSTPORT_3= @mysqld.3.#sst_port
-NODE_SSTPORT_4= @mysqld.4.#sst_port

mysql-test/suite/galera/include/galera_base_port.inc

+#
+# Extract base_port from galera node.
+#
+
+# Convert "... base_port = N; ..." to "N; ..."
+--let $s1 = `SELECT SUBSTR(@@wsrep_provider_options, LOCATE('base_port =', @@wsrep_provider_options) + LENGTH('base_port = '))`
+# Convert "N; ..." to "N"
+--let $_NODE_GALERAPORT = `SELECT SUBSTR('$s1', 1, LOCATE(';', '$s1') - 1)`

mysql-test/suite/galera/include/galera_concurrent_test.inc

+#
+# Perform a quick concurrent test on two nodes using a set of predefined statements.
+#
+# Such tests are not deterministic, so we are hoping to catch assertions, slave apply errors
+# and cases where the two nodes diverge
+#
+# Parameters:
+# - $wsrep_trx_fragment_size
+# - $count
+# - $query_node_1
+# - $query_node_1a (optional)
+# - $query_node_2
+#
+
+if (!$count) {
+	--let $count = 50;
+}
+
+if (!$node_1a_connected) {
+	--connect node_1a, 127.0.0.1, root, , test, $NODE_MYPORT_1
+	--let $node_1a_connected = 1
+}
+
+--echo Running a concurrent test with the following queries:
+--echo $query_node_1
+--echo $query_node_1a
+--echo $query_node_2
+
+--connection node_1
+--disable_warnings
+DROP TABLE IF EXISTS t1;
+--enable_warnings
+
+CREATE TABLE t1 (f1 INTEGER PRIMARY KEY AUTO_INCREMENT, f2 CHAR(255)) ENGINE=InnoDB;
+
+SET SESSION wsrep_sync_wait = 0;
+--eval SET SESSION wsrep_trx_fragment_size = $wsrep_trx_fragment_size;
+
+--connection node_1a
+SET SESSION wsrep_sync_wait = 0;
+--eval SET SESSION wsrep_trx_fragment_size = $wsrep_trx_fragment_size;
+
+--connection node_2
+SET SESSION wsrep_sync_wait = 0;
+--eval SET SESSION wsrep_trx_fragment_size = $wsrep_trx_fragment_size;
+
+--disable_query_log
+--let $i = `SELECT $count`
+while ($i)
+{
+	--connection node_1
+	--send_eval $query_node_1
+
+        --connection node_1a
+        if ($query_node_1a) {
+		--send_eval $query_node_1a
+	}
+
+	--connection node_2
+	--send_eval $query_node_2
+
+        --connection node_1
+        --error 0,ER_QUERY_INTERRUPTED,ER_LOCK_DEADLOCK,ER_DUP_ENTRY
+	--reap
+
+	--connection node_1a
+        if ($query_node_1a) {
+	        --error 0,ER_QUERY_INTERRUPTED,ER_LOCK_DEADLOCK,ER_DUP_ENTRY
+	        --reap
+	}
+
+        --connection node_2
+        --error 0,ER_QUERY_INTERRUPTED,ER_LOCK_DEADLOCK,ER_DUP_ENTRY
+        --reap
+        --dec $i
+}
+
+SET SESSION wsrep_sync_wait = 15;
+--enable_query_log
+
+--let $diff_servers = 1 2
+--source include/diff_servers.inc
+
+DROP TABLE t1;
+
+--let $query_node_1 = ""
+--let $query_node_1a = ""
+--let $query_node_2 = ""
+
+--echo Concurrent test end

mysql-test/suite/galera/include/galera_dump_sr_table.inc

+#
+# Dump the contents of the SR table using mysqldump
+#
+
+--let $sr_min = `SELECT MIN(seqno) FROM mysql.wsrep_streaming_log`
+--let $sr_max = `SELECT MAX(seqno) FROM mysql.wsrep_streaming_log`
+
+--let $seqno = $sr_min
+while ($seqno <= $sr_max)
+{
+  --let $sr_fragment_file = $MYSQLTEST_VARDIR/tmp/sr_fragment.log
+  --exec rm -rf $sr_fragment_file
+  --disable_query_log
+  --eval SELECT frag FROM mysql.wsrep_streaming_log WHERE seqno = $seqno INTO DUMPFILE '$sr_fragment_file'
+  --enable_query_log
+
+  --let $sr_binlog_file = $MYSQLTEST_VARDIR/tmp/sr_binlog.log
+  --exec rm -rf $sr_binlog_file
+
+  --exec cp std_data/binlog-header.log $sr_binlog_file
+  --exec cat $sr_fragment_file >> $sr_binlog_file
+
+  --replace_regex /SET TIMESTAMP=[0-9]+/SET TIMESTAMP=<TIMESTAMP>/ /#[0-9]+ +[0-9]+:[0-9]+:[0-9]+/<ISO TIMESTAMP>/ /pseudo_thread_id=[0-9]+/pseudo_thread_id=<PSEUDO_THREAD_ID>/ /thread_id=[0-9]+/thread_id=<QUERY_THREAD_ID>/ /table id [0-9]+/table id <TABLE_ID>/ /mapped to number [0-9]+/mapped to number <TABLE_ID>/ /auto_increment_increment=[0-9]+/auto_increment_increment=<AUTO_INCREMENT_INCREMENT>/ /auto_increment_offset=[0-9]+/auto_increment_offset=<AUTO_INCREMENT_OFFSET>/ /exec_time=[0-9]+/exec_time=<EXEC_TIME>/
+  --exec $MYSQL_BINLOG --skip-gtids $sr_binlog_file --base64-output=decode-rows --start-position=120 | grep -v 'SET @' 2>&1
+
+  --inc $seqno
+}
+

mysql-test/suite/galera/include/galera_load_provider.inc

@@ -2,7 +2,75 @@
 
 --disable_query_log
 --eval SET GLOBAL wsrep_provider = '$wsrep_provider_orig';
+
+#
+# count occurences of successful node starts in error log
+#
+perl;
+  use strict;
+   my $test_log=$ENV{'LOG_FILE'} or die "LOG_FILE not set";
+   my $test_log_copy=$test_log . '.copy';
+   if (-e $test_log_copy) {
+      unlink $test_log_copy;
+   }
+
+EOF
+--copy_file $LOG_FILE $LOG_FILE.copy
+
+#
+#  now join to the cluster
+#
 --eval SET GLOBAL wsrep_cluster_address = '$wsrep_cluster_address_orig';
+
+--enable_query_log
+
+#
+# Cluster address change above, will launch SST/IST
+# if mysqldump ST has been configured, mysqld will close all
+# client connections, and it will be hard for us to poll for
+# cluster status during SST process, therefore wait_until_connected_again.inc
+# and wait_until_ready.inc may fail in this phase
+# To workaround this, we do first lazy polling here just to see when
+# client connections will be possible, and after that check for node readyness
+#
+--disable_result_log
+--disable_query_log
+
+--error 0,1
+perl;
+  use strict;
+  my $logfile = $ENV{'LOG_FILE'} or die ("no error log file set");
+
+  my $counter = 1000;
+  #my $found = false
+  
+  while ($counter > 0) {
+ 
+    open(FILE, "$logfile") or die("Unable to open $logfile : $!\n");
+    my $new_sync_count = () = grep(/Synchronized with group/g,<FILE>);
+    close(FILE);
+
+    open(FILEN, "$logfile.copy") or die("Unable to open $logfile.copy : $!\n");
+    my $old_sync_count = () = grep(/Synchronized with group/g,<FILEN>);
+    close(FILEN);
+
+    if ($new_sync_count > $old_sync_count ) {
+       exit(0);
+    }
+    $counter--;
+    sleep(5);
+  }
+  exit(1);
+EOF
+if ($errno)
+{
+--echo "SST failed $errno"
+}
+
+--remove_file $LOG_FILE.copy
+
 --enable_query_log
+--enable_result_log
+#--eval SET GLOBAL log_error = $log_error_;
 
 --source include/galera_wait_ready.inc

mysql-test/suite/galera/include/galera_sst_restore.inc

@@ -20,7 +20,7 @@ CALL mtr.add_suppression("Can't open and lock time zone table");
 CALL mtr.add_suppression("Can't open and lock privilege tables");
 CALL mtr.add_suppression("Info table is not ready to be used");
 CALL mtr.add_suppression("Native table .* has the wrong structure");
-
+CALL mtr.add_suppression("Table \'mysql.gtid_slave_pos\' doesn\'t exist");
 --disable_query_log
 --eval SET GLOBAL wsrep_sst_method = '$wsrep_sst_method_orig';
 --eval SET GLOBAL wsrep_sst_receive_address = '$wsrep_sst_receive_address_orig';

mysql-test/suite/galera/include/galera_st_disconnect_slave.inc

@@ -55,6 +55,14 @@ INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
 --connection node_2
 --source suite/galera/include/galera_load_provider.inc
 
+#
+# client connections were killed by provider load, so have to re-open here
+#
+--disconnect node_2
+--connect node_2, 127.0.0.1, root, , test, $NODE_MYPORT_2
+--connection node_2
+--enable_reconnect
+
 --let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'
 --source include/wait_condition.inc
 

mysql-test/suite/galera/include/galera_unload_provider.inc

@@ -3,5 +3,13 @@
 --let $wsrep_cluster_address_orig = `SELECT @@wsrep_cluster_address`
 --let $wsrep_provider_orig = `SELECT @@wsrep_provider`
 --let $wsrep_provider_options_orig = `SELECT @@wsrep_provider_options`
+--let $wsrep_error_log_orig = `SELECT @@log_error`
+if(!$wsrep_log_error_orig)
+{
+    # MySQL Server on windows is started with --console and thus
+    # does not know the location of its .err log, use default location
+    let $wsrep_log_error_orig = $MYSQLTEST_VARDIR/log/mysqld.2.err;
+}
+--let LOG_FILE= $wsrep_log_error_orig
 
 SET GLOBAL wsrep_provider = 'none';

mysql-test/suite/galera/r/GAL-382.result

+connection node_2;
+connection node_1;
 connection node_1;
 create table t1 (i int, j int, k int, primary key pk(i)) engine=innodb;
 insert into t1 values (1, 1, 1), (2, 2, 2), (3, 3, 3);

mysql-test/suite/galera/r/GAL-401.result

+connection node_2;
+connection node_1;
 SET GLOBAL wsrep_provider_options = 'pc.ignore_sb=true';
 connection node_2;
 SET @@global.wsrep_desync = 1;

mysql-test/suite/galera/r/GAL-480.result

+connection node_2;
+connection node_1;
 connection node_1;
 CREATE TABLE t1 (f1 CHAR(10), f0 integer) ENGINE=InnoDB;
 FLUSH TABLE t1 FOR EXPORT;

mysql-test/suite/galera/r/GCF-1081.result

+connection node_2;
+connection node_1;
+connection node_1;
+CREATE TABLE t1 (f1 INTEGER PRIMARY KEY, f2 INTEGER) ENGINE=InnoDB;
+INSERT INTO t1 VALUES (1, 0), (3, 0);
+CREATE PROCEDURE proc_update ()
+BEGIN
+UPDATE t1 SET f2 = 1 where f1 > 0;
+END|
+connect node_1a, 127.0.0.1, root, , test, $NODE_MYPORT_1;
+SET GLOBAL wsrep_provider_options = 'dbug=d,commit_monitor_master_enter_sync';
+connection node_1;
+CALL proc_update ();;
+connection node_1a;
+SET SESSION wsrep_sync_wait = 0;
+SET SESSION wsrep_on = 0;
+SET SESSION wsrep_on = 1;
+connection node_1a;
+SET GLOBAL DEBUG = 'd,sync.wsrep_before_BF_victim_unlock';
+Warnings:
+Warning	1287	'@@debug' is deprecated and will be removed in a future release. Please use '@@debug_dbug' instead
+connection node_2;
+INSERT INTO t1 VALUES (2, 2);;
+connection node_1a;
+SET SESSION DEBUG_SYNC = 'now WAIT_FOR sync.wsrep_before_BF_victim_unlock_reached';
+SET GLOBAL DEBUG = '';
+Warnings:
+Warning	1287	'@@debug' is deprecated and will be removed in a future release. Please use '@@debug_dbug' instead
+connection node_1a;
+SET GLOBAL wsrep_provider_options = 'dbug=';
+SET GLOBAL wsrep_provider_options = 'signal=commit_monitor_master_enter_sync';
+connection node_2;
+SELECT * FROM t1;
+f1	f2
+1	1
+2	2
+3	1
+connection node_1;
+SELECT * FROM t1;
+f1	f2
+1	1
+2	2
+3	1
+wsrep_local_replays
+1
+DROP PROCEDURE proc_update;
+DROP TABLE t1;

mysql-test/suite/galera/r/GCF-939.result

+connection node_2;
+connection node_1;
+connection node_1;
+DROP TABLE t1;
+ERROR 42S02: Unknown table 'test.t1'
+CREATE TABLE t1 (f1 INTEGER) ENGINE=InnoDB;
+INSERT INTO t1 VALUES (1);
+GRA_.log
+GRA_.log
+DROP TABLE t1;
+CALL mtr.add_suppression("Ignoring error 'Unknown table 'test.t1'' on query");
+connection node_2;
+CALL mtr.add_suppression("Error 'Unknown table 'test.t1'' on query");

mysql-test/suite/galera/r/MDEV-15443.result

+connection node_2;
+connection node_1;
 CREATE TABLE t1 (f1 INT PRIMARY KEY) ENGINE=InnoDB;
 INSERT INTO t1 VALUES (1);
 connection node_2;

mysql-test/suite/galera/r/MW-252.result

+connection node_2;
+connection node_1;
 connection node_1;
 CREATE TABLE t1 (f1 INTEGER) ENGINE=InnoDB;
 FLUSH TABLES WITH READ LOCK;

mysql-test/suite/galera/r/MW-258.result

+connection node_2;
+connection node_1;
 connection node_1;
 CREATE TABLE t1 (f1 INTEGER);
 LOCK TABLE t1 WRITE;

mysql-test/suite/galera/r/MW-259.result

+connection node_2;
+connection node_1;
 connect node_1a, 127.0.0.1, root, , test, $NODE_MYPORT_1;
 connect node_1b, 127.0.0.1, root, , test, $NODE_MYPORT_1;
 connection node_1;

mysql-test/suite/galera/r/MW-284.result

+connection node_2;
+connection node_1;
 connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3;
 connection node_1;
 CREATE TABLE t1 (f1 INTEGER) ENGINE=InnoDB;
@@ -24,7 +26,9 @@ RESET SLAVE ALL;
 CALL mtr.add_suppression('failed registering on master');
 CALL mtr.add_suppression('You need to use --log-bin to make --binlog-format work');
 connection node_1;
+set global wsrep_on=OFF;
 RESET MASTER;
+set global wsrep_on=ON;
 CALL mtr.add_suppression('WSREP: Last Applied Action message in non-primary configuration from member');
 connection node_2;
 CALL mtr.add_suppression('WSREP: Last Applied Action message in non-primary configuration from member');

mysql-test/suite/galera/r/MW-285.result

+connection node_2;
+connection node_1;
 CREATE TABLE parent1 ( id INT PRIMARY KEY, KEY (id) ) ENGINE=InnoDB;
 CREATE TABLE parent2 ( id INT PRIMARY KEY, KEY (id) ) ENGINE=InnoDB;
 CREATE TABLE child (

mysql-test/suite/galera/r/MW-286.result

+connection node_2;
+connection node_1;
 connection node_1;
-CREATE TABLE ten (f1 INTEGER) Engine=InnoDB;
-INSERT INTO ten VALUES (0),(1),(2),(3),(4),(5),(6),(7),(8),(9);
 CREATE TABLE t1 (f1 INTEGER) Engine=InnoDB;
-INSERT INTO t1 (f1) SELECT 000000 + (10000 * a1.f1) + (1000 * a2.f1) + (100 * a3.f1) + (10 * a4.f1) + a5.f1 FROM ten AS a1, ten AS a2, ten AS a3, ten AS a4, ten AS a5;
-INSERT INTO t1 (f1) SELECT 100000 + (10000 * a1.f1) + (1000 * a2.f1) + (100 * a3.f1) + (10 * a4.f1) + a5.f1 FROM ten AS a1, ten AS a2, ten AS a3, ten AS a4, ten AS a5;;
+INSERT INTO t1 (f1) VALUES (1), (2), (3);
 connection node_2;
 SET GLOBAL wsrep_desync = TRUE;
 SET wsrep_on = FALSE;
-ALTER TABLE t1 ADD PRIMARY KEY (f1);
+SET SESSION debug_sync = "alter_table_inplace_after_lock_upgrade SIGNAL mdl_locked WAIT_FOR mdl_continue";
+ALTER TABLE t1 ADD PRIMARY KEY (f1);;
+connect node_2a, 127.0.0.1, root, , test, $NODE_MYPORT_2;
+connection node_2a;
+SET SESSION debug_sync = "now WAIT_FOR mdl_locked";
+connection node_1;
+INSERT INTO t1(f1) VALUES (11);
+connection node_2a;
+SET debug_sync = "now SIGNAL mdl_continue";
+SET debug_sync='RESET';
+connection node_2;
+ERROR 70100: Query execution was interrupted
 SET wsrep_on = TRUE;
 SET GLOBAL wsrep_desync = FALSE;
 connection node_1;
 DROP TABLE t1;
-DROP TABLE ten;

mysql-test/suite/galera/r/MW-292.result

+connection node_2;
+connection node_1;
 CREATE TABLE rand_table (f1 FLOAT);
 CREATE TABLE t1 (f1 INTEGER PRIMARY KEY, f2 CHAR(1));
 INSERT INTO t1 VALUES (1, 'a');
@@ -10,19 +12,28 @@ SELECT * FROM t1 WHERE f1 = 2 FOR UPDATE;
 f1	f2
 2	a
 connect node_1a, 127.0.0.1, root, , test, $NODE_MYPORT_1;
-SET GLOBAL wsrep_provider_options = 'dbug=d,commit_monitor_enter_sync';
-connection node_1;
-COMMIT;;
-connection node_1a;
-SET SESSION wsrep_sync_wait = 0;
-SET SESSION wsrep_on = 0;
-SET SESSION wsrep_on = 1;
+SET SESSION wsrep_sync_wait=0;
+SET GLOBAL wsrep_provider_options = 'dbug=d,apply_monitor_slave_enter_sync';
 connection node_2;
 UPDATE t1 SET f2 = 'c' WHERE f1 = 2;
 connection node_1a;
+SET SESSION wsrep_on = 0;
+SET SESSION wsrep_on = 1;
+SET GLOBAL wsrep_provider_options = 'dbug=';
+SET GLOBAL wsrep_provider_options = 'dbug=d,commit_monitor_master_enter_sync';
+connection node_1;
+COMMIT;
 connection node_1a;
+SET SESSION wsrep_on = 0;
+SET SESSION wsrep_on = 1;
+SET GLOBAL wsrep_provider_options = 'dbug=';
+SET GLOBAL wsrep_provider_options = 'dbug=d,abort_trx_end';
+SET GLOBAL wsrep_provider_options = 'signal=apply_monitor_slave_enter_sync';
+SET SESSION wsrep_on = 0;
+SET SESSION wsrep_on = 1;
 SET GLOBAL wsrep_provider_options = 'dbug=';
-SET GLOBAL wsrep_provider_options = 'signal=commit_monitor_enter_sync';
+SET GLOBAL wsrep_provider_options = 'signal=abort_trx_end';
+SET GLOBAL wsrep_provider_options = 'signal=commit_monitor_master_enter_sync';
 connection node_1;
 SELECT TIMEDIFF(SYSDATE(), NOW()) < 2;
 TIMEDIFF(SYSDATE(), NOW()) < 2

mysql-test/suite/galera/r/MW-309.result

+connection node_2;
+connection node_1;
 CREATE TABLE t1 (f1 INTEGER) ENGINE=InnoDB;
 INSERT INTO t1 VALUES (1);
 INSERT INTO t1 SELECT * FROM t1;

mysql-test/suite/galera/r/MW-313.result

+connection node_2;
+connection node_1;
 CREATE TABLE t1 (f1 INTEGER) ENGINE=InnoDB;
 INSERT INTO t1 VALUES (1);
 INSERT INTO t1 SELECT * FROM t1;

mysql-test/suite/galera/r/MW-328A.result

+connection node_2;
+connection node_1;
 CREATE TABLE t1 (f1 INTEGER AUTO_INCREMENT PRIMARY KEY, f2 CHAR(20) DEFAULT 'abc') ENGINE=InnoDB;
 INSERT INTO t1 (f1) VALUES (1);
 CREATE TABLE t2 (f1 CHAR(20)) ENGINE=InnoDB;
@@ -14,10 +16,6 @@ connection node_1X;
 CALL proc_update();;
 connection node_2;
 SET SESSION wsrep_retry_autocommit = 0;
-have_successes
-1
-have_deadlocks
-1
 connection node_1;
 connection node_1X;
 Got one of the listed errors
@@ -25,3 +23,22 @@ connection node_1;
 DROP PROCEDURE proc_update;
 DROP TABLE t1, t2;
 CALL mtr.add_suppression("conflict state 3 after post commit");
+connection node_1;
+CREATE TABLE t1 (i int primary key, j int) engine=innodb;
+INSERT INTO t1 values (1,0);
+BEGIN;
+UPDATE t1 SET j=1 WHERE i=1;
+connection node_2;
+UPDATE t1 SET j=2 WHERE i=1;
+connection node_1;
+COMMIT;
+ERROR 40001: Deadlock found when trying to get lock; try restarting transaction
+SELECT * FROM t1;
+i	j
+1	2
+connection node_2;
+SELECT * FROM t1;
+i	j
+1	2
+connection node_1;
+DROP TABLE t1;

mysql-test/suite/galera/r/MW-328B.result

+connection node_2;
+connection node_1;
 CREATE TABLE t1 (f1 INTEGER AUTO_INCREMENT PRIMARY KEY, f2 CHAR(20) DEFAULT 'abc') ENGINE=InnoDB;
 INSERT INTO t1 (f1) VALUES (1);
 CREATE TABLE t2 (f1 CHAR(20)) ENGINE=InnoDB;

mysql-test/suite/galera/r/MW-328C.result

+connection node_2;
+connection node_1;
 CREATE TABLE t1 (f1 INTEGER AUTO_INCREMENT PRIMARY KEY, f2 CHAR(20) DEFAULT 'abc') ENGINE=InnoDB;
 INSERT INTO t1 (f1) VALUES (1);
 CREATE TABLE t2 (f1 CHAR(20)) ENGINE=InnoDB;

mysql-test/suite/galera/r/MW-328D.result

+connection node_2;
+connection node_1;
 CREATE TABLE t1 (i INT) ENGINE = InnoDB;
 INSERT INTO t1 (i) VALUES(1);
 CREATE TABLE t2 (i INT) ENGINE = InnoDB;

mysql-test/suite/galera/r/MW-328E.result

+connection node_2;
+connection node_1;
 create table t1 (i int primary key, j int) engine=innodb;
 create table t2 (i int primary key, j int) engine=innodb;
 insert into t1 values (1,0);

mysql-test/suite/galera/r/MW-329.result

 CALL mtr.add_suppression("WSREP: .*conflict state . after post commit .*");
 CREATE TABLE t1 (f1 INTEGER, f2 CHAR(20) DEFAULT 'abc') ENGINE=InnoDB;
 INSERT INTO t1 (f1) VALUES (1),(65535);
-FLUSH STATUS;
-SELECT VARIABLE_VALUE = 0 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_local_replays';
-VARIABLE_VALUE = 0
-1
 CREATE PROCEDURE proc_insert ()
 BEGIN
 DECLARE CONTINUE HANDLER FOR SQLEXCEPTION BEGIN END;
@@ -16,10 +12,7 @@ END|
 connect node_1b, 127.0.0.1, root, , test, $NODE_MYPORT_1;
 connection node_1b;
 CALL proc_insert();;
-connection node_2;
-CALL mtr.add_suppression("WSREP: Failed to report last committed .*");
-SELECT VARIABLE_VALUE > 0 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_local_replays';
-VARIABLE_VALUE > 0
+wsrep_local_replays
 1
 connection node_1;
 connection node_1b;

mysql-test/suite/galera/r/MW-336.result

+connection node_2;
+connection node_1;
 CREATE TABLE t1 (f1 INTEGER) Engine=InnoDB;
 INSERT INTO t1 values(0);
 connection node_1;

mysql-test/suite/galera/r/MW-357.result

 connection node_2;
+connection node_1;
+connection node_2;
 SET GLOBAL wsrep_slave_threads = 0;
 Warnings:
 Warning	1292	Truncated incorrect wsrep_slave_threads value: '0'

mysql-test/suite/galera/r/MW-360.result

+SET GLOBAL wsrep_on=OFF;
+RESET MASTER;
+SET GLOBAL wsrep_on=ON;
+SET GLOBAL wsrep_on=OFF;
+RESET MASTER;
+SET GLOBAL wsrep_on=ON;
+CREATE TEMPORARY TABLE t1 (f1 INTEGER) ENGINE=InnoDB;
+INSERT INTO t1 VALUES (1);
+DROP TABLE t1;
+CREATE TEMPORARY TABLE t1 (f1 INTEGER) ENGINE=InnoDB;
+INSERT INTO t1 VALUES (1);
+CREATE TABLE t2 (f1 INTEGER) ENGINE=InnoDB;
+INSERT INTO t2 VALUES (2);
+DROP TABLE t1, t2;
+CREATE TABLE t1 (f1 INTEGER) ENGINE=InnoDB;
+INSERT INTO t1 VALUES (1);
+CREATE TEMPORARY TABLE t2 (f1 INTEGER) ENGINE=InnoDB;
+INSERT INTO t2 VALUES (2);
+DROP TABLE t1, t2;
+CREATE TEMPORARY TABLE t1 (f1 INTEGER) ENGINE=InnoDB;
+INSERT INTO t1 VALUES (1);
+CREATE TABLE t2 (f1 INTEGER) ENGINE=InnoDB;
+INSERT INTO t2 VALUES (2);
+CREATE TEMPORARY TABLE t3 (f1 INTEGER) ENGINE=InnoDB;
+INSERT INTO t3 VALUES (3);
+DROP TABLE t1, t2, t3;
+CREATE TABLE t1 (f1 INTEGER) ENGINE=InnoDB;
+INSERT INTO t1 VALUES (1);
+CREATE TEMPORARY TABLE t2 (f1 INTEGER) ENGINE=InnoDB;
+INSERT INTO t2 VALUES (2);
+CREATE TABLE t3 (f1 INTEGER) ENGINE=InnoDB;
+INSERT INTO t3 VALUES (3);
+DROP TABLE t1, t2, t3;
+CREATE TABLE t1 (f1 INTEGER) ENGINE=InnoDB;
+INSERT INTO t1 VALUES (1);
+CREATE TEMPORARY TABLE t1 (f1 INTEGER) ENGINE=InnoDB;
+INSERT INTO t1 VALUES (2);
+DROP TABLE t1;
+DROP TABLE t1;
+gtid_executed_equal
+1

mysql-test/suite/galera/r/MW-369.result

+connection node_2;
+connection node_1;
 CREATE TABLE p (f1 INTEGER PRIMARY KEY, f2 INTEGER) ENGINE=INNODB;
 CREATE TABLE c (f1 INTEGER PRIMARY KEY, p_id INTEGER,
 CONSTRAINT fk_1 FOREIGN KEY (p_id) REFERENCES p (f1))  ;
 INSERT INTO p VALUES (1, 0);
 INSERT INTO p VALUES (2, 0);
+connect node_1a, 127.0.0.1, root, , test, $NODE_MYPORT_1;
+connection node_1;
 SET AUTOCOMMIT=ON;
 START TRANSACTION;
 DELETE FROM p WHERE f1 = 1;
+connection node_1a;
 SET SESSION wsrep_sync_wait = 0;
 SET GLOBAL wsrep_provider_options = 'dbug=d,apply_monitor_slave_enter_sync';
+connection node_2;
 INSERT INTO c VALUES (1, 1);
+connection node_1a;
 SET SESSION wsrep_on = 0;
 SET SESSION wsrep_on = 1;
 SET GLOBAL wsrep_provider_options = 'dbug=';
-SET GLOBAL wsrep_provider_options = 'dbug=d,local_monitor_enter_sync';
+SET GLOBAL wsrep_provider_options = 'dbug=d,local_monitor_master_enter_sync';
+connection node_1;
 COMMIT;
+connection node_1a;
 SET SESSION wsrep_on = 0;
 SET SESSION wsrep_on = 1;
 SET GLOBAL wsrep_provider_options = 'signal=apply_monitor_slave_enter_sync';
-SET GLOBAL wsrep_provider_options = 'signal=local_monitor_enter_sync';
+SET GLOBAL wsrep_provider_options = 'signal=local_monitor_master_enter_sync';
 SET GLOBAL wsrep_provider_options = 'dbug=';
+connection node_1;
 ERROR 40001: Deadlock found when trying to get lock; try restarting transaction
+connection node_2;
 SELECT * FROM p;
 f1	f2
 1	0
@@ -29,6 +40,7 @@ f1	p_id
 1	1
 DROP TABLE c;
 DROP TABLE p;
+connection node_1;
 CREATE TABLE p (f1 INTEGER PRIMARY KEY, f2 INTEGER) ENGINE=INNODB;
 CREATE TABLE c (f1 INTEGER PRIMARY KEY, p_id INTEGER,
 f2 INTEGER,
@@ -36,22 +48,30 @@ CONSTRAINT fk_1 FOREIGN KEY (p_id) REFERENCES p (f1))  ;
 INSERT INTO p VALUES (1, 0);
 INSERT INTO p VALUES (2, 0);
 INSERT INTO c VALUES (1, 1, 0);
+connection node_1;
 SET AUTOCOMMIT=ON;
 START TRANSACTION;
 UPDATE p SET f2 = 1 WHERE f1 = 1;
+connection node_1a;
 SET SESSION wsrep_sync_wait = 0;
 SET GLOBAL wsrep_provider_options = 'dbug=d,apply_monitor_slave_enter_sync';
+connection node_2;
 UPDATE c SET f2 = 1 WHERE f1 = 1;
+connection node_1a;
 SET SESSION wsrep_on = 0;
 SET SESSION wsrep_on = 1;
 SET GLOBAL wsrep_provider_options = 'dbug=';
-SET GLOBAL wsrep_provider_options = 'dbug=d,local_monitor_enter_sync';
+SET GLOBAL wsrep_provider_options = 'dbug=d,local_monitor_master_enter_sync';
+connection node_1;
 COMMIT;
+connection node_1a;
 SET SESSION wsrep_on = 0;
 SET SESSION wsrep_on = 1;
 SET GLOBAL wsrep_provider_options = 'signal=apply_monitor_slave_enter_sync';
-SET GLOBAL wsrep_provider_options = 'signal=local_monitor_enter_sync';
+SET GLOBAL wsrep_provider_options = 'signal=local_monitor_master_enter_sync';
 SET GLOBAL wsrep_provider_options = 'dbug=';
+connection node_1;
+connection node_2;
 SELECT * FROM p;
 f1	f2
 1	1
@@ -61,28 +81,37 @@ f1	p_id	f2
 1	1	1
 DROP TABLE c;
 DROP TABLE p;
+connection node_1;
 CREATE TABLE p (f1 INTEGER PRIMARY KEY, f2 INTEGER) ENGINE=INNODB;
 CREATE TABLE c (f1 INTEGER PRIMARY KEY, p_id INTEGER,
 CONSTRAINT fk_1 FOREIGN KEY (p_id) REFERENCES p (f1))  ;
 INSERT INTO p VALUES (1, 0);
 INSERT INTO p VALUES (2, 0);
 INSERT INTO c VALUES (1, 1);
+connection node_1;
 SET AUTOCOMMIT=ON;
 START TRANSACTION;
 UPDATE p SET f2 = 1 WHERE f1 = 1;
+connection node_1a;
 SET SESSION wsrep_sync_wait = 0;
 SET GLOBAL wsrep_provider_options = 'dbug=d,apply_monitor_slave_enter_sync';
+connection node_2;
 DELETE FROM c WHERE f1 = 1;
+connection node_1a;
 SET SESSION wsrep_on = 0;
 SET SESSION wsrep_on = 1;
 SET GLOBAL wsrep_provider_options = 'dbug=';
-SET GLOBAL wsrep_provider_options = 'dbug=d,local_monitor_enter_sync';
+SET GLOBAL wsrep_provider_options = 'dbug=d,local_monitor_master_enter_sync';
+connection node_1;
 COMMIT;
+connection node_1a;
 SET SESSION wsrep_on = 0;
 SET SESSION wsrep_on = 1;
 SET GLOBAL wsrep_provider_options = 'signal=apply_monitor_slave_enter_sync';
-SET GLOBAL wsrep_provider_options = 'signal=local_monitor_enter_sync';
+SET GLOBAL wsrep_provider_options = 'signal=local_monitor_master_enter_sync';
 SET GLOBAL wsrep_provider_options = 'dbug=';
+connection node_1;
+connection node_2;
 SELECT * FROM p;
 f1	f2
 1	1
@@ -95,23 +124,31 @@ CREATE TABLE p (f1 INTEGER PRIMARY KEY, f2 INTEGER UNIQUE KEY) ENGINE=INNODB;
 CREATE TABLE c (f1 INTEGER PRIMARY KEY, p_id INTEGER,
 CONSTRAINT fk_1 FOREIGN KEY (p_id) REFERENCES p (f2))  ;
 INSERT INTO p VALUES (1, 0);
+connection node_1;
 SET AUTOCOMMIT=ON;
 START TRANSACTION;
 UPDATE p SET f2 = 1 WHERE f1 = 1;
+connection node_1a;
 SET SESSION wsrep_sync_wait = 0;
 SET GLOBAL wsrep_provider_options = 'dbug=d,apply_monitor_slave_enter_sync';
+connection node_2;
 INSERT INTO c VALUES (1, 0);;
+connection node_1a;
 SET SESSION wsrep_on = 0;
 SET SESSION wsrep_on = 1;
 SET GLOBAL wsrep_provider_options = 'dbug=';
-SET GLOBAL wsrep_provider_options = 'dbug=d,local_monitor_enter_sync';
+SET GLOBAL wsrep_provider_options = 'dbug=d,local_monitor_master_enter_sync';
+connection node_1;
 COMMIT;
+connection node_1a;
 SET SESSION wsrep_on = 0;
 SET SESSION wsrep_on = 1;
 SET GLOBAL wsrep_provider_options = 'signal=apply_monitor_slave_enter_sync';
-SET GLOBAL wsrep_provider_options = 'signal=local_monitor_enter_sync';
+SET GLOBAL wsrep_provider_options = 'signal=local_monitor_master_enter_sync';
 SET GLOBAL wsrep_provider_options = 'dbug=';
+connection node_1;
 ERROR 40001: Deadlock found when trying to get lock; try restarting transaction
+connection node_2;
 SELECT * FROM p;
 f1	f2
 1	0
@@ -127,23 +164,31 @@ ON DELETE CASCADE)  ;
 INSERT INTO p VALUES (1, 0);
 INSERT INTO p VALUES (2, 0);
 INSERT INTO c VALUES (1, 1, 0);
+connection node_1;
 SET AUTOCOMMIT=ON;
 START TRANSACTION;
 DELETE FROM p WHERE f1 = 1;
+connection node_1a;
 SET SESSION wsrep_sync_wait = 0;
 SET GLOBAL wsrep_provider_options = 'dbug=d,apply_monitor_slave_enter_sync';
+connection node_2;
 UPDATE c SET f2 = 1 WHERE f1 = 1;
+connection node_1a;
 SET SESSION wsrep_on = 0;
 SET SESSION wsrep_on = 1;
 SET GLOBAL wsrep_provider_options = 'dbug=';
-SET GLOBAL wsrep_provider_options = 'dbug=d,local_monitor_enter_sync';
+SET GLOBAL wsrep_provider_options = 'dbug=d,local_monitor_master_enter_sync';
+connection node_1;
 COMMIT;
+connection node_1a;
 SET SESSION wsrep_on = 0;
 SET SESSION wsrep_on = 1;
 SET GLOBAL wsrep_provider_options = 'signal=apply_monitor_slave_enter_sync';
-SET GLOBAL wsrep_provider_options = 'signal=local_monitor_enter_sync';
+SET GLOBAL wsrep_provider_options = 'signal=local_monitor_master_enter_sync';
 SET GLOBAL wsrep_provider_options = 'dbug=';
+connection node_1;
 ERROR 40001: Deadlock found when trying to get lock; try restarting transaction
+connection node_2;
 SELECT * FROM p;
 f1	f2
 1	0
@@ -153,3 +198,87 @@ f1	p_id	f2
 1	1	1
 DROP TABLE c;
 DROP TABLE p;
+#
+# Start of 10.4 tests
+#
+connection node_1;
+CREATE TABLE pf (f1 INTEGER PRIMARY KEY) ENGINE=INNODB;
+CREATE TABLE cf (
+f1 INTEGER PRIMARY KEY,
+p_id INTEGER,
+CONSTRAINT fk_1 FOREIGN KEY (p_id) REFERENCES pf (f1)
+);
+INSERT INTO pf VALUES (1);
+connection node_1;
+SET AUTOCOMMIT=ON;
+START TRANSACTION;
+INSERT INTO cf (f1, p_id) VALUES (10, 1);
+connection node_1a;
+SET SESSION wsrep_sync_wait = 0;
+SET GLOBAL wsrep_provider_options = 'dbug=d,apply_monitor_slave_enter_sync';
+connection node_2;
+INSERT INTO cf (f1, p_id) VALUES (20, 1);
+connection node_1a;
+SET SESSION wsrep_on = 0;
+SET SESSION wsrep_on = 1;
+SET GLOBAL wsrep_provider_options = 'dbug=';
+SET GLOBAL wsrep_provider_options = 'dbug=d,local_monitor_master_enter_sync';
+connection node_1;
+COMMIT;
+connection node_1a;
+SET SESSION wsrep_on = 0;
+SET SESSION wsrep_on = 1;
+SET GLOBAL wsrep_provider_options = 'signal=apply_monitor_slave_enter_sync';
+SET GLOBAL wsrep_provider_options = 'signal=local_monitor_master_enter_sync';
+SET GLOBAL wsrep_provider_options = 'dbug=';
+connection node_1;
+connection node_2;
+SELECT * FROM pf;
+f1
+1
+SELECT * FROM cf;
+f1	p_id
+10	1
+20	1
+DROP TABLE cf;
+DROP TABLE pf;
+connection node_1;
+CREATE TABLE pg (f1 INTEGER PRIMARY KEY, f2 INTEGER) ENGINE=INNODB;
+CREATE TABLE cg (f1 INTEGER PRIMARY KEY, p_id INTEGER,
+f2 INTEGER,
+CONSTRAINT fk_1 FOREIGN KEY (p_id) REFERENCES pg (f1))  ;
+INSERT INTO pg VALUES (1, 0);
+INSERT INTO pg VALUES (2, 0);
+connection node_1;
+SET AUTOCOMMIT=ON;
+START TRANSACTION;
+UPDATE pg SET f2 = 1 WHERE f1 = 1;
+connection node_1a;
+SET SESSION wsrep_sync_wait = 0;
+SET GLOBAL wsrep_provider_options = 'dbug=d,apply_monitor_slave_enter_sync';
+connection node_2;
+INSERT INTO cg VALUES (1, 1, 0);
+connection node_1a;
+SET SESSION wsrep_on = 0;
+SET SESSION wsrep_on = 1;
+SET GLOBAL wsrep_provider_options = 'dbug=';
+SET GLOBAL wsrep_provider_options = 'dbug=d,local_monitor_master_enter_sync';
+connection node_1;
+COMMIT;
+connection node_1a;
+SET SESSION wsrep_on = 0;
+SET SESSION wsrep_on = 1;
+SET GLOBAL wsrep_provider_options = 'signal=apply_monitor_slave_enter_sync';
+SET GLOBAL wsrep_provider_options = 'signal=local_monitor_master_enter_sync';
+SET GLOBAL wsrep_provider_options = 'dbug=';
+connection node_1;
+connection node_2;
+SELECT * FROM pg;
+f1	f2
+1	1
+2	0
+SELECT * FROM cg;
+f1	p_id	f2
+1	1	0
+DROP TABLE cg;
+DROP TABLE pg;

mysql-test/suite/galera/r/MW-388.result

+connection node_2;
+connection node_1;
 connection node_1;
 CREATE TABLE t1 (f1 INTEGER PRIMARY KEY, f2 CHAR(255)) Engine=InnoDB;
 CREATE PROCEDURE insert_proc ()
@@ -18,7 +20,7 @@ connection node_1a;
 SET SESSION DEBUG_SYNC = "now WAIT_FOR sync.wsrep_apply_cb_reached";
 connection node_1;
 SET SESSION wsrep_sync_wait = 0;
-SET SESSION DEBUG_SYNC = 'wsrep_after_replication SIGNAL wsrep_after_replication_reached WAIT_FOR wsrep_after_replication_continue';
+SET SESSION DEBUG_SYNC = 'wsrep_after_certification SIGNAL wsrep_after_certification_reached WAIT_FOR wsrep_after_certification_continue';
 CALL insert_proc ();;
 connection node_1a;
 SET SESSION DEBUG_SYNC = "now WAIT_FOR wsrep_after_replication_reached";
@@ -29,7 +31,7 @@ connection node_2;
 connection node_1;
 SELECT @errno = 1213;
 @errno = 1213
-0
+1
 SELECT * FROM t1;
 f1	f2
 1	node 2

mysql-test/suite/galera/r/MW-402.result

+connection node_2;
+connection node_1;
 connect node_1a, 127.0.0.1, root, , test, $NODE_MYPORT_1;
 connection node_1;
 CREATE TABLE p (f1 INTEGER PRIMARY KEY, f2 INTEGER) ENGINE=INNODB;
@@ -20,14 +22,14 @@ connection node_1a;
 SET SESSION wsrep_on = 0;
 SET SESSION wsrep_on = 1;
 SET GLOBAL wsrep_provider_options = 'dbug=';
-SET GLOBAL wsrep_provider_options = 'dbug=d,local_monitor_enter_sync';
+SET GLOBAL wsrep_provider_options = 'dbug=d,local_monitor_master_enter_sync';
 connection node_1;
 COMMIT;
 connection node_1a;
 SET SESSION wsrep_on = 0;
 SET SESSION wsrep_on = 1;
 SET GLOBAL wsrep_provider_options = 'signal=apply_monitor_slave_enter_sync';
-SET GLOBAL wsrep_provider_options = 'signal=local_monitor_enter_sync';
+SET GLOBAL wsrep_provider_options = 'signal=local_monitor_master_enter_sync';
 SET GLOBAL wsrep_provider_options = 'dbug=';
 connection node_1;
 ERROR 40001: Deadlock found when trying to get lock; try restarting transaction
@@ -60,14 +62,14 @@ connection node_1a;
 SET SESSION wsrep_on = 0;
 SET SESSION wsrep_on = 1;
 SET GLOBAL wsrep_provider_options = 'dbug=';
-SET GLOBAL wsrep_provider_options = 'dbug=d,local_monitor_enter_sync';
+SET GLOBAL wsrep_provider_options = 'dbug=d,local_monitor_master_enter_sync';
 connection node_1;
 COMMIT;
 connection node_1a;
 SET SESSION wsrep_on = 0;
 SET SESSION wsrep_on = 1;
 SET GLOBAL wsrep_provider_options = 'signal=apply_monitor_slave_enter_sync';
-SET GLOBAL wsrep_provider_options = 'signal=local_monitor_enter_sync';
+SET GLOBAL wsrep_provider_options = 'signal=local_monitor_master_enter_sync';
 SET GLOBAL wsrep_provider_options = 'dbug=';
 connection node_1;
 ERROR 40001: Deadlock found when trying to get lock; try restarting transaction
@@ -102,14 +104,14 @@ connection node_1a;
 SET SESSION wsrep_on = 0;
 SET SESSION wsrep_on = 1;
 SET GLOBAL wsrep_provider_options = 'dbug=';
-SET GLOBAL wsrep_provider_options = 'dbug=d,local_monitor_enter_sync';
+SET GLOBAL wsrep_provider_options = 'dbug=d,local_monitor_master_enter_sync';
 connection node_1;
 COMMIT;
 connection node_1a;
 SET SESSION wsrep_on = 0;
 SET SESSION wsrep_on = 1;
 SET GLOBAL wsrep_provider_options = 'signal=apply_monitor_slave_enter_sync';
-SET GLOBAL wsrep_provider_options = 'signal=local_monitor_enter_sync';
+SET GLOBAL wsrep_provider_options = 'signal=local_monitor_master_enter_sync';
 SET GLOBAL wsrep_provider_options = 'dbug=';
 connection node_1;
 ERROR 40001: Deadlock found when trying to get lock; try restarting transaction
@@ -135,14 +137,14 @@ connection node_1a;
 SET SESSION wsrep_on = 0;
 SET SESSION wsrep_on = 1;
 SET GLOBAL wsrep_provider_options = 'dbug=';
-SET GLOBAL wsrep_provider_options = 'dbug=d,local_monitor_enter_sync';
+SET GLOBAL wsrep_provider_options = 'dbug=d,local_monitor_master_enter_sync';
 connection node_1;
 COMMIT;
 connection node_1a;
 SET SESSION wsrep_on = 0;
 SET SESSION wsrep_on = 1;
 SET GLOBAL wsrep_provider_options = 'signal=apply_monitor_slave_enter_sync';
-SET GLOBAL wsrep_provider_options = 'signal=local_monitor_enter_sync';
+SET GLOBAL wsrep_provider_options = 'signal=local_monitor_master_enter_sync';
 SET GLOBAL wsrep_provider_options = 'dbug=';
 connection node_1;
 ERROR 40001: Deadlock found when trying to get lock; try restarting transaction
@@ -159,8 +161,10 @@ DROP TABLE p;
 connection node_1;
 CREATE TABLE p1 (f1 INTEGER PRIMARY KEY, f2 INTEGER) ENGINE=INNODB;
 CREATE TABLE p2 (f1 INTEGER PRIMARY KEY, f2 INTEGER) ENGINE=INNODB;
-CREATE TABLE c (f1 INTEGER PRIMARY KEY, p1_id INTEGER, p2_id INTEGER, f2 INTEGER,
-CONSTRAINT fk_1 FOREIGN KEY (p1_id) REFERENCES p1 (f1) ON DELETE CASCADE,
+CREATE TABLE c (f1 INTEGER PRIMARY KEY, p1_id INTEGER, p2_id INTEGER,
+f2 INTEGER,
+CONSTRAINT fk_1 FOREIGN KEY (p1_id) REFERENCES p1 (f1)
+ON DELETE CASCADE,
 CONSTRAINT fk_2 FOREIGN KEY (p2_id) REFERENCES p2 (f1));
 INSERT INTO p1 VALUES (1, 0);
 INSERT INTO p2 VALUES (1, 0);
@@ -179,14 +183,14 @@ connection node_1a;
 SET SESSION wsrep_on = 0;
 SET SESSION wsrep_on = 1;
 SET GLOBAL wsrep_provider_options = 'dbug=';
-SET GLOBAL wsrep_provider_options = 'dbug=d,local_monitor_enter_sync';
+SET GLOBAL wsrep_provider_options = 'dbug=d,local_monitor_master_enter_sync';
 connection node_1;
 COMMIT;
 connection node_1a;
 SET SESSION wsrep_on = 0;
 SET SESSION wsrep_on = 1;
 SET GLOBAL wsrep_provider_options = 'signal=apply_monitor_slave_enter_sync';
-SET GLOBAL wsrep_provider_options = 'signal=local_monitor_enter_sync';
+SET GLOBAL wsrep_provider_options = 'signal=local_monitor_master_enter_sync';
 SET GLOBAL wsrep_provider_options = 'dbug=';
 connection node_1;
 connection node_2;
@@ -197,4 +201,52 @@ f1	f2
 1	2
 SELECT * FROM c;
 f1	p1_id	p2_id	f2
+DROP TABLE c;
+DROP TABLE p1;
+DROP TABLE p2;
+connection node_1;
+CREATE TABLE p1 (f1 INTEGER PRIMARY KEY, f2 INTEGER) ENGINE=INNODB;
+CREATE TABLE p2 (f1 INTEGER PRIMARY KEY, f2 INTEGER) ENGINE=INNODB;
+CREATE TABLE c (f1 INTEGER PRIMARY KEY, p1_id INTEGER, p2_id INTEGER,
+f2 INTEGER,
+CONSTRAINT fk_1 FOREIGN KEY (p1_id) REFERENCES p1 (f1)
+ON DELETE CASCADE,
+CONSTRAINT fk_2 FOREIGN KEY (p2_id) REFERENCES p2 (f1)
+ON DELETE CASCADE);
+INSERT INTO p1 VALUES (1, 0);
+INSERT INTO p2 VALUES (1, 0);
+INSERT INTO c VALUES (1, 1, 1, 0);
+connection node_1a;
+connection node_1;
+SET AUTOCOMMIT=ON;
+START TRANSACTION;
+DELETE FROM p2 WHERE f1=1;
+connection node_1a;
+SET SESSION wsrep_sync_wait = 0;
+SET GLOBAL wsrep_provider_options = 'dbug=d,apply_monitor_slave_enter_sync';
+connection node_2;
+DELETE FROM p1 WHERE f1=1;
+connection node_1a;
+SET SESSION wsrep_on = 0;
+SET SESSION wsrep_on = 1;
+SET GLOBAL wsrep_provider_options = 'dbug=';
+SET GLOBAL wsrep_provider_options = 'dbug=d,local_monitor_master_enter_sync';
+connection node_1;
+COMMIT;
+connection node_1a;
+SET SESSION wsrep_on = 0;
+SET SESSION wsrep_on = 1;
+SET GLOBAL wsrep_provider_options = 'signal=apply_monitor_slave_enter_sync';
+SET GLOBAL wsrep_provider_options = 'signal=local_monitor_master_enter_sync';
+SET GLOBAL wsrep_provider_options = 'dbug=';
+connection node_1;
+ERROR 40001: Deadlock found when trying to get lock; try restarting transaction
+connection node_2;
+SELECT * FROM p1;
+f1	f2
+SELECT * FROM p2;
+f1	f2
+1	0
+SELECT * FROM c;
+f1	p1_id	p2_id	f2
 DROP TABLE c,p1,p2;

mysql-test/suite/galera/r/MW-416.result

@@ -109,6 +109,5 @@ mtr
 mysql
 performance_schema
 test
-SHOW GLOBAL STATUS LIKE 'wsrep_replicated';
-Variable_name	Value
-wsrep_replicated	3
+wsrep_replicated_after_diff
+1

mysql-test/suite/galera/r/MW-86-wait1.result

 connection node_2;
+connection node_1;
+SET @orig_debug=@@debug;
+connection node_2;
 SELECT @@debug_sync;
 @@debug_sync
 ON - current signal: ''
+set debug_sync='RESET';
 SET SESSION wsrep_sync_wait = 1;
-SET GLOBAL debug = "+d,sync.wsrep_apply_cb";
-Warnings:
-Warning	1287	'@@debug' is deprecated and will be removed in a future release. Please use '@@debug_dbug' instead
+SET GLOBAL debug_dbug = "+d,sync.wsrep_apply_cb";
 connection node_1;
 CREATE TABLE t_wait1 (f1 INTEGER) ENGINE=InnoDB;
 INSERT INTO t_wait1 VALUES (1);
@@ -36,16 +38,11 @@ SHOW TABLES;
 SHOW TRIGGERS;
 SHOW GLOBAL VARIABLES LIKE 'foo_bar';
 SHOW WARNINGS;
-SET GLOBAL debug = "-d,sync.wsrep_apply_cb";
-Warnings:
-Warning	1287	'@@debug' is deprecated and will be removed in a future release. Please use '@@debug_dbug' instead
+SET GLOBAL debug_dbug = @orig_debug;
 SET SESSION debug_sync = "now SIGNAL signal.wsrep_apply_cb";
+SET debug_sync='RESET';
 SET SESSION wsrep_sync_wait = default;
 DROP TABLE t_wait1;
-SET GLOBAL debug = NULL;
-Warnings:
-Warning	1287	'@@debug' is deprecated and will be removed in a future release. Please use '@@debug_dbug' instead
-SET debug_sync='RESET';
 SELECT @@debug_sync;
 @@debug_sync
 ON - current signal: ''

mysql-test/suite/galera/r/MW-86-wait8.result

 connection node_2;
+connection node_1;
+SET @orig_debug=@@debug;
+connection node_2;
 SELECT @@debug_sync;
 @@debug_sync
 ON - current signal: ''
 SET SESSION wsrep_sync_wait = 8;
-SET GLOBAL debug = "+d,sync.wsrep_apply_cb";
-Warnings:
-Warning	1287	'@@debug' is deprecated and will be removed in a future release. Please use '@@debug_dbug' instead
+SET GLOBAL debug_dbug = "+d,sync.wsrep_apply_cb";
 connection node_1;
 CREATE TABLE t_wait8 (f1 INTEGER) ENGINE=InnoDB;
 INSERT INTO t_wait8 VALUES (1);
@@ -38,16 +39,11 @@ SHOW TABLES;
 SHOW TRIGGERS;
 SHOW GLOBAL VARIABLES LIKE 'foo_bar';
 SHOW WARNINGS;
-SET GLOBAL debug = "-d,sync.wsrep_apply_cb";
-Warnings:
-Warning	1287	'@@debug' is deprecated and will be removed in a future release. Please use '@@debug_dbug' instead
+SET GLOBAL debug_dbug = @orig_debug;
 SET DEBUG_SYNC = "now SIGNAL signal.wsrep_apply_cb";
+SET debug_sync='RESET';
 SET SESSION wsrep_sync_wait = default;
 DROP TABLE t_wait8;
-SET GLOBAL debug = NULL;
-Warnings:
-Warning	1287	'@@debug' is deprecated and will be removed in a future release. Please use '@@debug_dbug' instead
-SET debug_sync='RESET';
 SELECT @@debug_sync;
 @@debug_sync
 ON - current signal: ''

mysql-test/suite/galera/r/MW-86.result

+connection node_2;
+connection node_1;
+connection node_2;
+SET SESSION wsrep_sync_wait = 1;
+SET GLOBAL DEBUG = "d,sync.wsrep_apply_cb";
+Warnings:
+Warning	1287	'@@debug' is deprecated and will be removed in a future release. Please use '@@debug_dbug' instead
+connection node_1;
+CREATE DATABASE db1;
+CREATE TABLE bar (f1 INTEGER) ENGINE=InnoDB;
+INSERT INTO bar VALUES (1);
+connection node_2;
+SHOW BINARY LOGS;
+SHOW BINLOG EVENTS;
+SHOW COLUMNS FROM t1;
+SHOW CREATE DATABASE db1;
+SHOW CREATE EVENT e1;
+SHOW CREATE FUNCTION f1;
+SHOW CREATE PROCEDURE p1;
+SHOW CREATE TABLE t1;
+SHOW CREATE TRIGGER tr1;
+SHOW CREATE VIEW v1;
+SHOW DATABASES;
+SHOW ENGINE InnoDB STATUS;
+SHOW FUNCTION CODE f1;
+SHOW FUNCTION STATUS;
+SHOW GRANTS FOR 'root'@'localhost';
+SHOW INDEX FROM t1;
+SHOW OPEN TABLES;
+SHOW PROCEDURE CODE p1;
+SHOW PROCEDURE STATUS;
+SHOW PRIVILEGES;
+SHOW STATUS LIKE 'wsrep_cluster_size';
+SHOW TABLE STATUS;
+SHOW TABLES;
+SHOW TRIGGERS;
+SHOW GLOBAL VARIABLES LIKE 'foo_bar';
+SHOW WARNINGS;
+SET GLOBAL DEBUG = "";
+SET DEBUG_SYNC = "now SIGNAL signal.wsrep_apply_cb";
+SET SESSION wsrep_sync_wait = 8;
+DROP DATABASE db1;
+connection node_2;
+SET GLOBAL wsrep_provider_options = "repl.causal_read_timeout=PT0.1S";
+SET SESSION wsrep_sync_wait = 8;
+SET GLOBAL DEBUG = "d,sync.wsrep_apply_cb";
+connection node_1;
+CREATE TABLE q (f1 INTEGER) ENGINE=InnoDB;
+INSERT INTO q VALUES (1);
+connection node_2;
+SHOW BINARY LOGS;
+SHOW BINLOG EVENTS;
+SHOW COLUMNS FROM t1;
+SHOW CREATE DATABASE db1;
+SHOW CREATE EVENT e1;
+SHOW CREATE FUNCTION f1;
+SHOW CREATE PROCEDURE p1;
+SHOW CREATE TABLE t1;
+SHOW CREATE TRIGGER tr1;
+SHOW CREATE VIEW v1;
+SHOW DATABASES;
+SHOW ENGINE InnoDB STATUS;
+SHOW FUNCTION CODE f1;
+SHOW FUNCTION STATUS;
+SHOW GRANTS FOR 'root'@'localhost';
+SHOW INDEX FROM t1;
+SHOW OPEN TABLES;
+SHOW PROCEDURE CODE p1;
+SHOW PROCEDURE STATUS;
+SHOW PRIVILEGES;
+SHOW STATUS LIKE 'wsrep_cluster_size';
+SHOW TABLE STATUS;
+SHOW TABLES;
+SHOW TRIGGERS;
+SHOW GLOBAL VARIABLES LIKE 'foo_bar';
+SHOW WARNINGS;
+SET GLOBAL DEBUG = "";
+SET DEBUG_SYNC = "now SIGNAL signal.wsrep_apply_cb";

mysql-test/suite/galera/r/basic.result

+connection node_2;
+connection node_1;
 USE test;
 CREATE TABLE t1(c1 INT PRIMARY KEY) ENGINE=INNODB;
 INSERT INTO t1 VALUES (1), (2), (3), (4), (5);

mysql-test/suite/galera/r/binlog_checksum.result

+connection node_2;
+connection node_1;
 # On node_1
 connection node_1;
 SET @binlog_checksum_saved= @@GLOBAL.BINLOG_CHECKSUM;

mysql-test/suite/galera/r/create.result

+connection node_2;
+connection node_1;
 #
 # MDEV-6924 : Server crashed on CREATE TABLE ... SELECT
 #

mysql-test/suite/galera/r/enforce_storage_engine.result

+connection node_2;
+connection node_1;
 #
 # MDEV-8831 : enforce_storage_engine doesn't block table creation on
 # other nodes (galera cluster)

mysql-test/suite/galera/r/enforce_storage_engine2.result

+connection node_2;
+connection node_1;
 #
 # MDEV-9312: storage engine not enforced during galera cluster
 # replication

mysql-test/suite/galera/r/ev51914.result

+connection node_2;
+connection node_1;
 connection node_1;
 SAVEPOINT in a stored function should be forbidden
 CREATE FUNCTION f1 () RETURNS INT BEGIN

mysql-test/suite/galera/r/fk.result

+connection node_2;
+connection node_1;
 USE test;
 
 # On node_1

mysql-test/suite/galera/r/galera#414.result

+connection node_2;
+connection node_1;
 connection node_1;
 connection node_2;
 connection node_2;

mysql-test/suite/galera/r/galera#500.result

+connection node_2;
+connection node_1;
+connection node_1;
+connection node_2;
+connection node_2;
 SET SESSION wsrep_sync_wait = 0;
 SET GLOBAL wsrep_provider_options="gmcast.isolate=2";
+connection node_1;
 SET SESSION wsrep_sync_wait = 0;
 SHOW STATUS LIKE 'wsrep_cluster_status';
 Variable_name	Value
 wsrep_cluster_status	non-Primary
 SET SESSION wsrep_sync_wait = default;
 SET GLOBAL wsrep_provider_options="pc.bootstrap=1";
+connection node_2;
 SET SESSION wsrep_on=0;
+connection node_2;
 CALL mtr.add_suppression("WSREP: exception from gcomm, backend must be restarted: Gcomm backend termination was requested by setting gmcast.isolate=2.");

mysql-test/suite/galera/r/galera#505.result

+connection node_2;
+connection node_1;
 connection node_1;
 SET SESSION wsrep_sync_wait=0;
 SET SESSION wsrep_sync_wait=DEFAULT;

mysql-test/suite/galera/r/galera_FK_duplicate_client_insert.result

+connection node_2;
+connection node_1;
 CREATE TABLE user(id int primary key, j int) ENGINE=InnoDB;
 CREATE TABLE user_session(id int primary key, fk1 int, fk2 int) ENGINE=InnoDB;
 alter table user_session add foreign key (fk1) references user(id);

mysql-test/suite/galera/r/galera_admin.result

+connection node_2;
+connection node_1;
 DROP TABLE IF EXISTS t1, t2;
 DROP TABLE IF EXISTS x1, x2;
 connection node_1;

mysql-test/suite/galera/r/galera_alter_engine_innodb.result

+connection node_2;
+connection node_1;
 CREATE TABLE t1 (f1 INTEGER) ENGINE=InnoDB;
 INSERT INTO t1 VALUES (1);
 ALTER TABLE t1 ENGINE=InnoDB;

mysql-test/suite/galera/r/galera_alter_engine_myisam.result

+connection node_2;
+connection node_1;
 SET GLOBAL wsrep_replicate_myisam = TRUE;
 CREATE TABLE t1 (f1 INTEGER) ENGINE=MyISAM;
 INSERT INTO t1 VALUES (1);

mysql-test/suite/galera/r/galera_alter_table_force.result

+connection node_2;
+connection node_1;
 CREATE TABLE t1 (f1 INTEGER) ENGINE=InnoDB;
 INSERT INTO t1 VALUES (1);
 ALTER TABLE t1 FORCE;

mysql-test/suite/galera/r/galera_applier_ftwrl_table.result

+connection node_2;
+connection node_1;
 connection node_1;
 SET SESSION wsrep_sync_wait = 0;
 CREATE TABLE t1 (f1 INTEGER PRIMARY KEY) ENGINE=InnoDB;

mysql-test/suite/galera/r/galera_applier_ftwrl_table_alter.result

+connection node_2;
+connection node_1;
 connection node_1;
 SET SESSION wsrep_sync_wait = 0;
 SET SESSION lock_wait_timeout = 60;

mysql-test/suite/galera/r/galera_as_master.result

+connection node_2;
+connection node_1;
 connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3;
 START SLAVE;
 connection node_1;
@@ -54,4 +56,6 @@ STOP SLAVE;
 RESET SLAVE ALL;
 CALL mtr.add_suppression('You need to use --log-bin to make --binlog-format work');
 connection node_1;
+set global wsrep_on=OFF;
 RESET MASTER;
+set global wsrep_on=ON;

mysql-test/suite/galera/r/galera_as_master_gtid.result

@@ -5,55 +5,19 @@ CREATE TABLE t1 (f1 INTEGER PRIMARY KEY) ENGINE=InnoDB;
 INSERT INTO t1 VALUES(1);
 uuids_do_not_match
 1
-SHOW BINLOG EVENTS IN 'mysqld-bin.000002' FROM 120;
-Log_name	Pos	Event_type	Server_id	End_log_pos	Info
-mysqld-bin.000002	120	Previous_gtids	1	151	
-mysqld-bin.000002	151	Gtid	1	199	SET @@SESSION.GTID_NEXT= '<effective_uuid>:1'
-mysqld-bin.000002	199	Query	1	327	use `test`; CREATE TABLE t1 (f1 INTEGER PRIMARY KEY) ENGINE=InnoDB
-mysqld-bin.000002	327	Gtid	1	375	SET @@SESSION.GTID_NEXT= '<effective_uuid>:2'
-mysqld-bin.000002	375	Query	1	452	BEGIN
-mysqld-bin.000002	452	Table_map	1	497	table_id: # (test.t1)
-mysqld-bin.000002	497	Write_rows	1	537	table_id: # flags: STMT_END_F
-mysqld-bin.000002	537	Xid	1	568	COMMIT /* xid=# */
 INSERT INTO t1 VALUES(2);
 uuids_do_not_match
 1
 uuids_match
 1
-SHOW BINLOG EVENTS IN 'mysqld-bin.000003' FROM 120;
-Log_name	Pos	Event_type	Server_id	End_log_pos	Info
-mysqld-bin.000003	120	Previous_gtids	2	151	
-mysqld-bin.000003	151	Gtid	1	199	SET @@SESSION.GTID_NEXT= '<effective_uuid>:1'
-mysqld-bin.000003	199	Query	1	327	use `test`; CREATE TABLE t1 (f1 INTEGER PRIMARY KEY) ENGINE=InnoDB
-mysqld-bin.000003	327	Gtid	1	375	SET @@SESSION.GTID_NEXT= '<effective_uuid>:2'
-mysqld-bin.000003	375	Query	1	443	BEGIN
-mysqld-bin.000003	443	Table_map	1	488	table_id: # (test.t1)
-mysqld-bin.000003	488	Write_rows	1	528	table_id: # flags: STMT_END_F
-mysqld-bin.000003	528	Xid	1	559	COMMIT /* xid=# */
-mysqld-bin.000003	559	Gtid	2	607	SET @@SESSION.GTID_NEXT= '<effective_uuid>:3'
-mysqld-bin.000003	607	Query	2	684	BEGIN
-mysqld-bin.000003	684	Table_map	2	729	table_id: # (test.t1)
-mysqld-bin.000003	729	Write_rows	2	769	table_id: # flags: STMT_END_F
-mysqld-bin.000003	769	Xid	2	800	COMMIT /* xid=# */
 uuids_do_not_match
 1
 uuids_match
 1
-SHOW BINLOG EVENTS IN 'mysqld-bin.000001' FROM 120;
-Log_name	Pos	Event_type	Server_id	End_log_pos	Info
-mysqld-bin.000001	120	Previous_gtids	3	151	
-mysqld-bin.000001	151	Gtid	1	199	SET @@SESSION.GTID_NEXT= '<effective_uuid>:1'
-mysqld-bin.000001	199	Query	1	327	use `test`; CREATE TABLE t1 (f1 INTEGER PRIMARY KEY) ENGINE=InnoDB
-mysqld-bin.000001	327	Gtid	1	375	SET @@SESSION.GTID_NEXT= '<effective_uuid>:2'
-mysqld-bin.000001	375	Query	1	443	BEGIN
-mysqld-bin.000001	443	Table_map	1	488	table_id: # (test.t1)
-mysqld-bin.000001	488	Write_rows	1	528	table_id: # flags: STMT_END_F
-mysqld-bin.000001	528	Xid	1	559	COMMIT /* xid=# */
-mysqld-bin.000001	559	Gtid	2	607	SET @@SESSION.GTID_NEXT= '<effective_uuid>:3'
-mysqld-bin.000001	607	Query	2	675	BEGIN
-mysqld-bin.000001	675	Table_map	2	720	table_id: # (test.t1)
-mysqld-bin.000001	720	Write_rows	2	760	table_id: # flags: STMT_END_F
-mysqld-bin.000001	760	Xid	2	791	COMMIT /* xid=# */
 DROP TABLE t1;
+gtid_executed_equal
+1
+gtid_executed_equal
+1
 STOP SLAVE;
 RESET SLAVE ALL;

mysql-test/suite/galera/r/galera_as_master_large.result

@@ -2,6 +2,8 @@
 # MDEV-9044 : Getting binlog corruption on my Galera cluster (10.1.8)
 # making it impossible to async slave.
 #
+connection node_2;
+connection node_1;
 connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3;
 START SLAVE;
 connection node_1;

mysql-test/suite/galera/r/galera_as_slave.result

-connect node_2a, 127.0.0.1, root, , test, $NODE_MYPORT_2;
 connection node_2;
-START SLAVE;
 connection node_1;
+connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3;
+connection node_2;
+START SLAVE;
+connection node_3;
 CREATE TABLE t1 (f1 INTEGER PRIMARY KEY) ENGINE=InnoDB;
 INSERT INTO t1 VALUES(1);
 connection node_2;
 INSERT INTO t1 VALUES (2);
-connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3;
+connection node_1;
 SELECT COUNT(*) = 2 FROM t1;
 COUNT(*) = 2
 1
@@ -15,10 +17,10 @@ connection node_2;
 SELECT COUNT(*) = 3 FROM t1;
 COUNT(*) = 3
 1
-connection node_1;
+connection node_3;
 DROP TABLE t1;
 connection node_2;
 STOP SLAVE;
 RESET SLAVE ALL;
-connection node_1;
+connection node_3;
 RESET MASTER;

mysql-test/suite/galera/r/galera_as_slave_autoinc.result

-connect node_2a, 127.0.0.1, root, , test, $NODE_MYPORT_2;
 connection node_2;
-START SLAVE;
 connection node_1;
+connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3;
+connection node_2;
+START SLAVE;
+connection node_3;
 SET SESSION binlog_format='STATEMENT';
 CREATE TABLE t1 (
 i int(11) NOT NULL AUTO_INCREMENT,
@@ -60,7 +62,7 @@ binlog_format	ROW
 show variables like 'auto_increment_increment';
 Variable_name	Value
 auto_increment_increment	2
-connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3;
+connection node_1;
 select * from t1;
 i	c
 1	dummy_text
@@ -79,10 +81,10 @@ binlog_format	ROW
 show variables like 'auto_increment_increment';
 Variable_name	Value
 auto_increment_increment	2
-connection node_1;
+connection node_3;
 DROP TABLE t1;
 connection node_2;
 STOP SLAVE;
 RESET SLAVE ALL;
-connection node_1;
+connection node_3;
 RESET MASTER;

mysql-test/suite/galera/r/galera_as_slave_gtid.result

-connect node_2a, 127.0.0.1, root, , test, $NODE_MYPORT_2;
 connection node_2;
-START SLAVE;
 connection node_1;
+connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3;
+connection node_2;
+START SLAVE;
+connection node_3;
 CREATE TABLE t1 (f1 INTEGER PRIMARY KEY) ENGINE=InnoDB;
 INSERT INTO t1 VALUES(1);
 SELECT LENGTH(@@global.gtid_binlog_state) > 1;
@@ -10,15 +12,15 @@ LENGTH(@@global.gtid_binlog_state) > 1
 connection node_2;
 gtid_binlog_state_equal
 1
-connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3;
+connection node_1;
 SELECT COUNT(*) = 1 FROM t1;
 COUNT(*) = 1
 1
 gtid_binlog_state_equal
 1
-connection node_1;
-DROP TABLE t1;
 connection node_3;
+DROP TABLE t1;
+connection node_1;
 connection node_2;
 STOP SLAVE;
 RESET SLAVE ALL;

mysql-test/suite/galera/r/galera_as_slave_gtid_replicate_do_db.result

+connection node_2;
+connection node_1;
+connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3;
+connection node_1;
+SET global wsrep_on=OFF;
+RESET MASTER;
+SET global wsrep_on=ON;
+connection node_3;
+RESET MASTER;
+connection node_2;
+SET global wsrep_on=OFF;
+RESET MASTER;
+SET global wsrep_on=ON;
+START SLAVE;
+connection node_3;
+CREATE SCHEMA test1;
+CREATE SCHEMA test2;
+USE test1;
+CREATE TABLE t1 (f1 INTEGER PRIMARY KEY,f2 CHAR(5) DEFAULT 'abc') ENGINE=InnoDB;
+USE test2;
+CREATE TABLE t1 (f1 INTEGER PRIMARY KEY,f2 CHAR(5) DEFAULT 'abc') ENGINE=InnoDB;
+INSERT INTO test1.t1 (f1) VALUES (1);
+INSERT INTO test2.t1 (f1) VALUES (1);
+INSERT INTO test1.t1 (f1) VALUES (2);
+INSERT INTO test2.t1 (f1) VALUES (2);
+INSERT INTO test1.t1 (f1) VALUES (3);
+INSERT INTO test2.t1 (f1) VALUES (3);
+UPDATE test2.t1 SET test2.t1.f2 = 'cde';
+UPDATE test1.t1, test2.t1 SET test1.t1.f2 = 'klm', test2.t1.f2 = 'xyz';
+DELETE test1.t1, test2.t1 FROM test1.t1 INNER JOIN test2.t1 WHERE test1.t1.f1 = test2.t1.f1 AND test1.t1.f1 = 3;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO test2.t1 (f1) VALUES (999);
+INSERT INTO test2.t1 (f1) VALUES (9999);
+COMMIT;
+START TRANSACTION;
+INSERT INTO test1.t1 (f1) VALUES (111);
+INSERT INTO test1.t1 (f1) VALUES (222);
+COMMIT;
+START TRANSACTION;
+INSERT INTO test1.t1 (f1) VALUES (333);
+INSERT INTO test2.t1 (f1) VALUES (99999);
+COMMIT;
+connection node_2;
+SHOW BINLOG EVENTS IN 'master-bin.000001' FROM 256;
+Log_name	Pos	Event_type	Server_id	End_log_pos	Info
+master-bin.000001	256	Gtid_list	2	285	[]
+master-bin.000001	285	Binlog_checkpoint	2	329	master-bin.000001
+master-bin.000001	329	Gtid	3	371	GTID 0-3-1
+master-bin.000001	371	Query	3	458	CREATE SCHEMA test1
+master-bin.000001	458	Gtid	3	500	GTID 0-3-3
+master-bin.000001	500	Query	3	647	use `test1`; CREATE TABLE t1 (f1 INTEGER PRIMARY KEY,f2 CHAR(5) DEFAULT 'abc') ENGINE=InnoDB
+master-bin.000001	647	Gtid	3	689	BEGIN GTID 0-3-5
+master-bin.000001	689	Annotate_rows	3	748	INSERT INTO test1.t1 (f1) VALUES (1)
+master-bin.000001	748	Table_map	3	797	table_id: ### (test1.t1)
+master-bin.000001	797	Write_rows_v1	3	839	table_id: ### flags: STMT_END_F
+master-bin.000001	839	Xid	3	870	COMMIT /* xid=### */
+master-bin.000001	870	Gtid	3	912	BEGIN GTID 0-3-7
+master-bin.000001	912	Annotate_rows	3	971	INSERT INTO test1.t1 (f1) VALUES (2)
+master-bin.000001	971	Table_map	3	1020	table_id: ### (test1.t1)
+master-bin.000001	1020	Write_rows_v1	3	1062	table_id: ### flags: STMT_END_F
+master-bin.000001	1062	Xid	3	1093	COMMIT /* xid=### */
+master-bin.000001	1093	Gtid	3	1135	BEGIN GTID 0-3-9
+master-bin.000001	1135	Annotate_rows	3	1194	INSERT INTO test1.t1 (f1) VALUES (3)
+master-bin.000001	1194	Table_map	3	1243	table_id: ### (test1.t1)
+master-bin.000001	1243	Write_rows_v1	3	1285	table_id: ### flags: STMT_END_F
+master-bin.000001	1285	Xid	3	1316	COMMIT /* xid=### */
+master-bin.000001	1316	Gtid	3	1358	BEGIN GTID 0-3-12
+master-bin.000001	1358	Annotate_rows	3	1451	UPDATE test1.t1, test2.t1 SET test1.t1.f2 = 'klm', test2.t1.f2 = 'xyz'
+master-bin.000001	1451	Table_map	3	1500	table_id: ### (test1.t1)
+master-bin.000001	1500	Update_rows_v1	3	1588	table_id: ### flags: STMT_END_F
+master-bin.000001	1588	Xid	3	1619	COMMIT /* xid=### */
+master-bin.000001	1619	Gtid	3	1661	BEGIN GTID 0-3-13
+master-bin.000001	1661	Annotate_rows	3	1795	DELETE test1.t1, test2.t1 FROM test1.t1 INNER JOIN test2.t1 WHERE test1.t1.f1 = test2.t1.f1 AND test1.t1.f1 = 3
+master-bin.000001	1795	Table_map	3	1844	table_id: ### (test1.t1)
+master-bin.000001	1844	Delete_rows_v1	3	1886	table_id: ### flags: STMT_END_F
+master-bin.000001	1886	Xid	3	1917	COMMIT /* xid=### */
+master-bin.000001	1917	Gtid	3	1959	BEGIN GTID 0-3-15
+master-bin.000001	1959	Annotate_rows	3	2020	INSERT INTO test1.t1 (f1) VALUES (111)
+master-bin.000001	2020	Table_map	3	2069	table_id: ### (test1.t1)
+master-bin.000001	2069	Write_rows_v1	3	2111	table_id: ### flags: STMT_END_F
+master-bin.000001	2111	Annotate_rows	3	2172	INSERT INTO test1.t1 (f1) VALUES (222)
+master-bin.000001	2172	Table_map	3	2221	table_id: ### (test1.t1)
+master-bin.000001	2221	Write_rows_v1	3	2263	table_id: ### flags: STMT_END_F
+master-bin.000001	2263	Xid	3	2294	COMMIT /* xid=### */
+master-bin.000001	2294	Gtid	3	2336	BEGIN GTID <effective_uuid>
+master-bin.000001	2336	Annotate_rows	3	2397	INSERT INTO test1.t1 (f1) VALUES (333)
+master-bin.000001	2397	Table_map	3	2446	table_id: ### (test1.t1)
+master-bin.000001	2446	Write_rows_v1	3	2488	table_id: ### flags: STMT_END_F
+master-bin.000001	2488	Xid	3	2519	COMMIT /* xid=### */
+connection node_1;
+gtid_executed_equal
+0
+SHOW BINLOG EVENTS IN 'master-bin.000001' FROM 256;
+Log_name	Pos	Event_type	Server_id	End_log_pos	Info
+master-bin.000001	256	Gtid_list	1	285	[]
+master-bin.000001	285	Binlog_checkpoint	1	329	master-bin.000001
+master-bin.000001	329	Gtid	3	371	GTID 0-3-1
+master-bin.000001	371	Query	3	458	CREATE SCHEMA test1
+master-bin.000001	458	Gtid	3	500	GTID 0-3-2
+master-bin.000001	500	Query	3	647	use `test1`; CREATE TABLE t1 (f1 INTEGER PRIMARY KEY,f2 CHAR(5) DEFAULT 'abc') ENGINE=InnoDB
+master-bin.000001	647	Gtid	3	689	BEGIN GTID 0-3-3
+master-bin.000001	689	Annotate_rows	3	748	INSERT INTO test1.t1 (f1) VALUES (1)
+master-bin.000001	748	Table_map	3	797	table_id: ### (test1.t1)
+master-bin.000001	797	Write_rows_v1	3	839	table_id: ### flags: STMT_END_F
+master-bin.000001	839	Xid	3	870	COMMIT /* xid=### */
+master-bin.000001	870	Gtid	3	912	BEGIN GTID 0-3-4
+master-bin.000001	912	Annotate_rows	3	971	INSERT INTO test1.t1 (f1) VALUES (2)
+master-bin.000001	971	Table_map	3	1020	table_id: ### (test1.t1)
+master-bin.000001	1020	Write_rows_v1	3	1062	table_id: ### flags: STMT_END_F
+master-bin.000001	1062	Xid	3	1093	COMMIT /* xid=### */
+master-bin.000001	1093	Gtid	3	1135	BEGIN GTID 0-3-5
+master-bin.000001	1135	Annotate_rows	3	1194	INSERT INTO test1.t1 (f1) VALUES (3)
+master-bin.000001	1194	Table_map	3	1243	table_id: ### (test1.t1)
+master-bin.000001	1243	Write_rows_v1	3	1285	table_id: ### flags: STMT_END_F
+master-bin.000001	1285	Xid	3	1316	COMMIT /* xid=### */
+master-bin.000001	1316	Gtid	3	1358	BEGIN GTID 0-3-6
+master-bin.000001	1358	Annotate_rows	3	1451	UPDATE test1.t1, test2.t1 SET test1.t1.f2 = 'klm', test2.t1.f2 = 'xyz'
+master-bin.000001	1451	Table_map	3	1500	table_id: ### (test1.t1)
+master-bin.000001	1500	Update_rows_v1	3	1588	table_id: ### flags: STMT_END_F
+master-bin.000001	1588	Xid	3	1619	COMMIT /* xid=### */
+master-bin.000001	1619	Gtid	3	1661	BEGIN GTID 0-3-7
+master-bin.000001	1661	Annotate_rows	3	1795	DELETE test1.t1, test2.t1 FROM test1.t1 INNER JOIN test2.t1 WHERE test1.t1.f1 = test2.t1.f1 AND test1.t1.f1 = 3
+master-bin.000001	1795	Table_map	3	1844	table_id: ### (test1.t1)
+master-bin.000001	1844	Delete_rows_v1	3	1886	table_id: ### flags: STMT_END_F
+master-bin.000001	1886	Xid	3	1917	COMMIT /* xid=### */
+master-bin.000001	1917	Gtid	3	1959	BEGIN GTID 0-3-8
+master-bin.000001	1959	Annotate_rows	3	2020	INSERT INTO test1.t1 (f1) VALUES (111)
+master-bin.000001	2020	Table_map	3	2069	table_id: ### (test1.t1)
+master-bin.000001	2069	Write_rows_v1	3	2111	table_id: ### flags: STMT_END_F
+master-bin.000001	2111	Annotate_rows	3	2172	INSERT INTO test1.t1 (f1) VALUES (222)
+master-bin.000001	2172	Table_map	3	2221	table_id: ### (test1.t1)
+master-bin.000001	2221	Write_rows_v1	3	2263	table_id: ### flags: STMT_END_F
+master-bin.000001	2263	Xid	3	2294	COMMIT /* xid=### */
+master-bin.000001	2294	Gtid	3	2336	BEGIN GTID 0-3-9
+master-bin.000001	2336	Annotate_rows	3	2397	INSERT INTO test1.t1 (f1) VALUES (333)
+master-bin.000001	2397	Table_map	3	2446	table_id: ### (test1.t1)
+master-bin.000001	2446	Write_rows_v1	3	2488	table_id: ### flags: STMT_END_F
+master-bin.000001	2488	Xid	3	2519	COMMIT /* xid=### */
+include/diff_servers.inc [servers=1 2]
+connection node_1;
+SELECT COUNT(*) = 2 FROM test1.t1 WHERE f1 IN (1,2);
+COUNT(*) = 2
+1
+SELECT COUNT(*) = 3 FROM test1.t1 WHERE f1 IN (111,222,333);
+COUNT(*) = 3
+1
+SELECT COUNT(*) = 2 FROM test1.t1 WHERE f2 = 'klm';
+COUNT(*) = 2
+1
+USE test2;
+ERROR 42000: Unknown database 'test2'
+connection node_3;
+DROP SCHEMA test1;
+DROP SCHEMA test2;
+connection node_1;
+connection node_2;
+STOP SLAVE;
+RESET SLAVE ALL;

mysql-test/suite/galera/r/galera_as_slave_nonprim.result

+connection node_2;
+connection node_1;
+connect node_4, 127.0.0.1, root, , test, $NODE_MYPORT_4;
+connection node_2;
 START SLAVE;
 SET SESSION wsrep_sync_wait = 0;
+connection node_4;
 CREATE TABLE t1 (f1 INTEGER PRIMARY KEY) ENGINE=InnoDB;
+connection node_2;
 SET GLOBAL wsrep_provider_options = 'gmcast.isolate=1';
+connection node_1;
+connection node_4;
 INSERT INTO t1 VALUES (1),(2),(3),(4),(5);
+connection node_2;
+connection node_1;
 expected_error
 1
+connection node_2;
 SET GLOBAL wsrep_provider_options = 'gmcast.isolate=0';
+connection node_1;
+connection node_2;
 START SLAVE;
+connection node_4;
 DROP TABLE t1;
+connection node_2;
 STOP SLAVE;
 RESET SLAVE ALL;
 CALL mtr.add_suppression("Slave SQL: Error 'Unknown command' on query");
@@ -15,4 +30,5 @@ CALL mtr.add_suppression("Slave: Unknown command Error_code: 1047");
 CALL mtr.add_suppression("Transport endpoint is not connected");
 CALL mtr.add_suppression("Slave SQL: Error in Xid_log_event: Commit could not be completed, 'Deadlock found when trying to get lock; try restarting transaction', Error_code: 1213");
 CALL mtr.add_suppression("Slave SQL: Node has dropped from cluster, Error_code: 1047");
+connection node_4;
 RESET MASTER;

mysql-test/suite/galera/r/galera_autoinc_sst_mariabackup.result

+connection node_2;
+connection node_1;
 connection node_1;
 connection node_2;
 connection node_1;

mysql-test/suite/galera/r/galera_bf_abort.result

+connection node_2;
+connection node_1;
 CREATE TABLE t1 (f1 INTEGER PRIMARY KEY, f2 CHAR(6)) ENGINE=InnoDB;
 connect node_2a, 127.0.0.1, root, , test, $NODE_MYPORT_2;
 connection node_2;
@@ -9,7 +11,7 @@ INSERT INTO t1 VALUES (1,'node_1');
 connection node_2a;
 connection node_2;
 INSERT INTO t1 VALUES (2, 'node_2');
-ERROR 40001: Deadlock: wsrep aborted transaction
+ERROR 40001: Deadlock found when trying to get lock; try restarting transaction
 wsrep_local_aborts_increment
 1
 DROP TABLE t1;

mysql-test/suite/galera/r/galera_bf_abort_flush_for_export.result

+connection node_2;
+connection node_1;
 CREATE TABLE t1 (f1 INTEGER PRIMARY KEY) ENGINE=InnoDB;
 connection node_2;
 SET AUTOCOMMIT=OFF;

mysql-test/suite/galera/r/galera_bf_abort_for_update.result

+connection node_2;
+connection node_1;
 CREATE TABLE t1 (f1 INTEGER PRIMARY KEY, f2 INT) ENGINE=InnoDB;
 INSERT INTO t1 VALUES (1, 10);
 connection node_1;
@@ -10,7 +12,7 @@ UPDATE t1 SET f1 = 2;
 connect node_1a, 127.0.0.1, root, , test, $NODE_MYPORT_1;
 connection node_1;
 COMMIT;
-ERROR 40001: Deadlock: wsrep aborted transaction
+ERROR 40001: Deadlock found when trying to get lock; try restarting transaction
 wsrep_local_bf_aborts_diff
 1
 connection node_1;
@@ -23,7 +25,7 @@ UPDATE t1 SET f2 = 20;
 connection node_1a;
 connection node_1;
 COMMIT;
-ERROR 40001: Deadlock: wsrep aborted transaction
+ERROR 40001: Deadlock found when trying to get lock; try restarting transaction
 wsrep_local_bf_aborts_diff
 1
 DROP TABLE t1;

mysql-test/suite/galera/r/galera_bf_abort_ftwrl.result

+connection node_2;
+connection node_1;
 CREATE TABLE t1 (f1 INTEGER PRIMARY KEY) ENGINE=InnoDB;
 connection node_2;
 SET AUTOCOMMIT=OFF;

mysql-test/suite/galera/r/galera_bf_abort_get_lock.result

+connection node_2;
+connection node_1;
 CREATE TABLE t1 (f1 INTEGER PRIMARY KEY) ENGINE=InnoDB;
 connection node_2a;
 SELECT GET_LOCK("foo", 1000);
@@ -10,7 +12,7 @@ SELECT GET_LOCK("foo", 1000);;
 connection node_1;
 INSERT INTO t1 VALUES (1);
 connection node_2;
-ERROR 40001: Deadlock: wsrep aborted transaction
+ERROR 40001: Deadlock found when trying to get lock; try restarting transaction
 wsrep_local_aborts_increment
 1
 DROP TABLE t1;

mysql-test/suite/galera/r/galera_bf_abort_lock_table.result

+connection node_2;
+connection node_1;
 CREATE TABLE t1 (f1 INTEGER PRIMARY KEY) ENGINE=InnoDB;
 connection node_2;
 SET AUTOCOMMIT=OFF;

mysql-test/suite/galera/r/galera_bf_abort_shutdown.result

+connection node_2;
+connection node_1;
+connection node_1;
+connection node_2;
+connection node_1;
+CREATE TABLE t1 (f1 INT PRIMARY KEY);
+connection node_2;
+SET DEBUG_SYNC = 'wsrep_before_certification WAIT_FOR continue';
+INSERT INTO t1 VALUES (1);
+connect node_2a, 127.0.0.1, root, , test, $NODE_MYPORT_2;
+connection node_1;
+DROP TABLE t1;

mysql-test/suite/galera/r/galera_bf_abort_sleep.result

+connection node_2;
+connection node_1;
 CREATE TABLE t1 (f1 INTEGER PRIMARY KEY) ENGINE=InnoDB;
 connection node_2;
 SET AUTOCOMMIT=OFF;
@@ -6,7 +8,7 @@ SELECT SLEEP(1000);;
 connection node_1;
 INSERT INTO t1 VALUES (1);
 connection node_2;
-ERROR 40001: Deadlock: wsrep aborted transaction
+ERROR 40001: Deadlock found when trying to get lock; try restarting transaction
 wsrep_local_aborts_increment
 1
 DROP TABLE t1;

mysql-test/suite/galera/r/galera_bf_background_statistics.result

+connection node_2;
+connection node_1;
 SELECT @@innodb_stats_persistent;
 @@innodb_stats_persistent
 1
@@ -27,7 +29,7 @@ SELECT SLEEP(1000);;
 connection node_1;
 ALTER TABLE t1 CHANGE f2 f2 INTEGER NOT NULL DEFAULT 1;
 connection node_2;
-ERROR 40001: Deadlock: wsrep aborted transaction
+ERROR 40001: Deadlock found when trying to get lock; try restarting transaction
 wsrep_local_aborts_increment
 1
 DROP TABLE t1;