DEV-17835: Remove wsrep-sst-method=xtrabackup

The use of the xtrabackup and xtrabackup-v2 methods for SST
has been declared obsolete since version 10.2, now it cannot
be used because of the different redo log format. Accordingly,
we need to remove the xtrabackup-related scripts and dynamically
replace the call to xtrabackup[-v2] to the mariabackup (with a
corresponding warning in the log) when the server performs SST.

https://jira.mariadb.org/browse/MDEV-17835

.gitignore

@@ -128,8 +128,6 @@ scripts/wsrep_sst_mysqldump
 scripts/wsrep_sst_rsync
 scripts/wsrep_sst_rsync_wan
 scripts/wsrep_sst_mariabackup
-scripts/wsrep_sst_xtrabackup
-scripts/wsrep_sst_xtrabackup-v2
 scripts/maria_add_gis_sp.sql
 scripts/maria_add_gis_sp_bootstrap.sql
 scripts/galera_new_cluster

Docs/README-wsrep

@@ -137,7 +137,7 @@ Additional packages to consider (if not yet installed):
    * galera (multi-master replication provider, https://launchpad.net/galera)
    * MySQL-client-community (for connecting to server and mysqldump-based SST)
    * rsync (for rsync-based SST)
-   * xtrabackup and nc (for xtrabackup-based SST)
+   * mariabackup and nc (for mariabackup-based SST)
 
 2.2 Upgrade system tables.
 
@@ -380,14 +380,14 @@ to join or start a cluster.
 wsrep_sst_method=rsync
    What method to use to copy database state to a newly joined node. Supported
    methods:
-   - mysqldump:  slow (except for small datasets) but allows for upgrade
-                 between major MySQL versions or InnoDB features.
-   - rsync:      much faster on large datasets (default).
-   - rsync_wan:  same as rsync but with deltaxfer to minimize network traffic.
-   - xtrabackup: very fast and practically non-blocking SST method based on
-                 Percona's xtrabackup tool.
-
-   (for xtrabackup to work the following settings must be present in my.cnf
+   - mysqldump:   slow (except for small datasets) but allows for upgrade
+                  between major MySQL versions or InnoDB features.
+   - rsync:       much faster on large datasets (default).
+   - rsync_wan:   same as rsync but with deltaxfer to minimize network traffic.
+   - mariabackup: very fast and practically non-blocking SST method based on
+                  mariabackup tool (enhanced version of Percona's xtrabackup).
+
+   (for mariabackup to work the following settings must be present in my.cnf
     on all nodes:
       [mysqld]
       wsrep_sst_auth=root:<root password>

mysql-test/suite/galera/r/galera_autoinc_sst_mariabackup.result

 connection node_1;
+connection node_2;
+connection node_1;
 CREATE TABLE t1 (f1 INTEGER PRIMARY KEY AUTO_INCREMENT) ENGINE=InnoDB;
 CREATE PROCEDURE p1 ()
 BEGIN

mysql-test/suite/galera/r/galera_ist_innodb_flush_logs.result

+Performing State Transfer on a server that has been killed and restarted
+connection node_1;
+CREATE TABLE t1 (f1 CHAR(255)) ENGINE=InnoDB;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES ('node1_committed_before');
+INSERT INTO t1 VALUES ('node1_committed_before');
+INSERT INTO t1 VALUES ('node1_committed_before');
+INSERT INTO t1 VALUES ('node1_committed_before');
+INSERT INTO t1 VALUES ('node1_committed_before');
+COMMIT;
+connection node_2;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES ('node2_committed_before');
+INSERT INTO t1 VALUES ('node2_committed_before');
+INSERT INTO t1 VALUES ('node2_committed_before');
+INSERT INTO t1 VALUES ('node2_committed_before');
+INSERT INTO t1 VALUES ('node2_committed_before');
+COMMIT;
+Killing server ...
+connection node_1;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES ('node1_committed_during');
+INSERT INTO t1 VALUES ('node1_committed_during');
+INSERT INTO t1 VALUES ('node1_committed_during');
+INSERT INTO t1 VALUES ('node1_committed_during');
+INSERT INTO t1 VALUES ('node1_committed_during');
+COMMIT;
+START TRANSACTION;
+INSERT INTO t1 VALUES ('node1_to_be_committed_after');
+INSERT INTO t1 VALUES ('node1_to_be_committed_after');
+INSERT INTO t1 VALUES ('node1_to_be_committed_after');
+INSERT INTO t1 VALUES ('node1_to_be_committed_after');
+INSERT INTO t1 VALUES ('node1_to_be_committed_after');
+connect node_1a_galera_st_kill_slave, 127.0.0.1, root, , test, $NODE_MYPORT_1;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
+connection node_2;
+Performing --wsrep-recover ...
+Starting server ...
+Using --wsrep-start-position when starting mysqld ...
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES ('node2_committed_after');
+INSERT INTO t1 VALUES ('node2_committed_after');
+INSERT INTO t1 VALUES ('node2_committed_after');
+INSERT INTO t1 VALUES ('node2_committed_after');
+INSERT INTO t1 VALUES ('node2_committed_after');
+COMMIT;
+connection node_1;
+INSERT INTO t1 VALUES ('node1_to_be_committed_after');
+INSERT INTO t1 VALUES ('node1_to_be_committed_after');
+INSERT INTO t1 VALUES ('node1_to_be_committed_after');
+INSERT INTO t1 VALUES ('node1_to_be_committed_after');
+INSERT INTO t1 VALUES ('node1_to_be_committed_after');
+COMMIT;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES ('node1_committed_after');
+INSERT INTO t1 VALUES ('node1_committed_after');
+INSERT INTO t1 VALUES ('node1_committed_after');
+INSERT INTO t1 VALUES ('node1_committed_after');
+INSERT INTO t1 VALUES ('node1_committed_after');
+COMMIT;
+connection node_1a_galera_st_kill_slave;
+INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
+ROLLBACK;
+SELECT COUNT(*) = 35 FROM t1;
+COUNT(*) = 35
+1
+SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1;
+COUNT(*) = 0
+1
+COMMIT;
+SET AUTOCOMMIT=ON;
+connection node_1;
+SELECT COUNT(*) = 35 FROM t1;
+COUNT(*) = 35
+1
+SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1;
+COUNT(*) = 0
+1
+DROP TABLE t1;
+COMMIT;
+SET AUTOCOMMIT=ON;
+Performing State Transfer on a server that has been killed and restarted
+while a DDL was in progress on it
+connection node_1;
+CREATE TABLE t1 (f1 CHAR(255)) ENGINE=InnoDB;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES ('node1_committed_before');
+INSERT INTO t1 VALUES ('node1_committed_before');
+INSERT INTO t1 VALUES ('node1_committed_before');
+INSERT INTO t1 VALUES ('node1_committed_before');
+INSERT INTO t1 VALUES ('node1_committed_before');
+connection node_2;
+START TRANSACTION;
+INSERT INTO t1 VALUES ('node2_committed_before');
+INSERT INTO t1 VALUES ('node2_committed_before');
+INSERT INTO t1 VALUES ('node2_committed_before');
+INSERT INTO t1 VALUES ('node2_committed_before');
+INSERT INTO t1 VALUES ('node2_committed_before');
+COMMIT;
+SET GLOBAL debug_dbug = 'd,sync.alter_opened_table';
+connection node_1;
+ALTER TABLE t1 ADD COLUMN f2 INTEGER;
+connection node_2;
+SET wsrep_sync_wait = 0;
+Killing server ...
+connection node_1;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 (f1) VALUES ('node1_committed_during');
+INSERT INTO t1 (f1) VALUES ('node1_committed_during');
+INSERT INTO t1 (f1) VALUES ('node1_committed_during');
+INSERT INTO t1 (f1) VALUES ('node1_committed_during');
+INSERT INTO t1 (f1) VALUES ('node1_committed_during');
+COMMIT;
+START TRANSACTION;
+INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after');
+INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after');
+INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after');
+INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after');
+INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after');
+connect node_1a_galera_st_kill_slave_ddl, 127.0.0.1, root, , test, $NODE_MYPORT_1;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after');
+INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after');
+INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after');
+INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after');
+INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after');
+connection node_2;
+Performing --wsrep-recover ...
+connection node_2;
+Starting server ...
+Using --wsrep-start-position when starting mysqld ...
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 (f1) VALUES ('node2_committed_after');
+INSERT INTO t1 (f1) VALUES ('node2_committed_after');
+INSERT INTO t1 (f1) VALUES ('node2_committed_after');
+INSERT INTO t1 (f1) VALUES ('node2_committed_after');
+INSERT INTO t1 (f1) VALUES ('node2_committed_after');
+COMMIT;
+connection node_1;
+INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after');
+INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after');
+INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after');
+INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after');
+INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after');
+COMMIT;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 (f1) VALUES ('node1_committed_after');
+INSERT INTO t1 (f1) VALUES ('node1_committed_after');
+INSERT INTO t1 (f1) VALUES ('node1_committed_after');
+INSERT INTO t1 (f1) VALUES ('node1_committed_after');
+INSERT INTO t1 (f1) VALUES ('node1_committed_after');
+COMMIT;
+connection node_1a_galera_st_kill_slave_ddl;
+INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after');
+INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after');
+INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after');
+INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after');
+INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after');
+ROLLBACK;
+SELECT COUNT(*) = 2 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = 't1';
+COUNT(*) = 2
+1
+SELECT COUNT(*) = 35 FROM t1;
+COUNT(*) = 35
+1
+SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1;
+COUNT(*) = 0
+1
+COMMIT;
+SET AUTOCOMMIT=ON;
+connection node_1;
+SELECT COUNT(*) = 2 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = 't1';
+COUNT(*) = 2
+1
+SELECT COUNT(*) = 35 FROM t1;
+COUNT(*) = 35
+1
+SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1;
+COUNT(*) = 0
+1
+DROP TABLE t1;
+COMMIT;
+SET AUTOCOMMIT=ON;
+SET GLOBAL debug_dbug = $debug_orig;

mysql-test/suite/galera/t/galera_autoinc_sst_mariabackup.cnf

@@ -9,4 +9,3 @@ wsrep_provider_options='base_port=@mysqld.1.#galera_port;gcache.size=1;pc.ignore
 
 [mysqld.2]
 wsrep_provider_options='base_port=@mysqld.2.#galera_port;gcache.size=1;pc.ignore_sb=true'
-

mysql-test/suite/galera/t/galera_autoinc_sst_mariabackup.test

@@ -8,6 +8,10 @@
 --source include/have_innodb.inc
 --source include/have_mariabackup.inc
 
+--let $node_1=node_1
+--let $node_2=node_2
+--source include/auto_increment_offset_save.inc
+
 --connection node_1
 --let $connection_id = `SELECT CONNECTION_ID()`
 
@@ -94,3 +98,7 @@ DROP TABLE t1;
 
 CALL mtr.add_suppression("gcs_caused\\(\\) returned -1 \\(Operation not permitted\\)");
 CALL mtr.add_suppression("WSREP: Action message in non-primary configuration from member 0");
+
+--let $node_1=node_1a
+--let $node_2=node_2a
+--source include/auto_increment_offset_restore.inc

mysql-test/suite/galera/t/galera_ist_innodb_flush_logs.cnf

+!include ../galera_2nodes.cnf
+
+[mysqld]
+wsrep_sst_method=mariabackup
+wsrep_sst_auth=root:
+
+innodb_flush_log_at_trx_commit=0
+
+[mysqld.1]
+wsrep_provider_options='base_port=@mysqld.1.#galera_port;pc.ignore_sb=true'
+
+[mysqld.2]
+wsrep_provider_options='base_port=@mysqld.2.#galera_port;pc.ignore_sb=true'

mysql-test/suite/galera/t/galera_ist_innodb_flush_logs.test

+#
+# This test performs server kill and IST while innodb_flush_logs_on_trx_commit = 0
+# This confirms that IST can properly catch up even in the face of relaxed single-node durability
+#
+#
+
+--source include/big_test.inc
+--source include/galera_cluster.inc
+--source include/have_innodb.inc
+--source include/have_mariabackup.inc
+
+--source suite/galera/include/galera_st_kill_slave.inc
+--source suite/galera/include/galera_st_kill_slave_ddl.inc

mysql-test/suite/galera/t/galera_ist_mariabackup_innodb_flush_logs.test

@@ -7,6 +7,7 @@
 --source include/big_test.inc
 --source include/galera_cluster.inc
 --source include/have_innodb.inc
+--source include/have_mariabackup.inc
 
 --source suite/galera/include/galera_st_kill_slave.inc
 --source suite/galera/include/galera_st_kill_slave_ddl.inc

mysql-test/suite/sys_vars/r/wsrep_sst_method_basic.result

@@ -33,6 +33,10 @@ SET @@global.wsrep_sst_method="xtrabackup-v2";
 SELECT @@global.wsrep_sst_method;
 @@global.wsrep_sst_method
 xtrabackup-v2
+SET @@global.wsrep_sst_method="mariabackup";
+SELECT @@global.wsrep_sst_method;
+@@global.wsrep_sst_method
+mariabackup
 SET @@global.wsrep_sst_method=default;
 SELECT @@global.wsrep_sst_method;
 @@global.wsrep_sst_method

mysql-test/suite/sys_vars/t/wsrep_sst_method_basic.test

@@ -23,10 +23,15 @@ SET @@global.wsrep_sst_method=rsync;
 SELECT @@global.wsrep_sst_method;
 SET @@global.wsrep_sst_method=mysqldump;
 SELECT @@global.wsrep_sst_method;
+# The xtrabackup and xtrabackup-v2 methods are obsolete,
+# but we can still select them (they will be automatically
+# replaced to mariabackup):
 SET @@global.wsrep_sst_method=xtrabackup;
 SELECT @@global.wsrep_sst_method;
 SET @@global.wsrep_sst_method="xtrabackup-v2";
 SELECT @@global.wsrep_sst_method;
+SET @@global.wsrep_sst_method="mariabackup";
+SELECT @@global.wsrep_sst_method;
 SET @@global.wsrep_sst_method=default;
 SELECT @@global.wsrep_sst_method;
 

scripts/CMakeLists.txt

@@ -274,8 +274,6 @@ ELSE()
     SET(WSREP_SCRIPTS
       wsrep_sst_mysqldump
       wsrep_sst_rsync
-      wsrep_sst_xtrabackup
-      wsrep_sst_xtrabackup-v2
       wsrep_sst_mariabackup
     )
     # The following script is sourced from other SST scripts, so it should

sql/wsrep_sst.cc

@@ -803,6 +803,7 @@ ssize_t wsrep_sst_prepare (void** msg)
 {
   const char* addr_in=  NULL;
   const char* addr_out= NULL;
+  const char* method;
 
   if (!strcmp(wsrep_sst_method, WSREP_SST_SKIP))
   {
@@ -861,7 +862,8 @@ ssize_t wsrep_sst_prepare (void** msg)
   }
 
   ssize_t addr_len= -ENOSYS;
-  if (!strcmp(wsrep_sst_method, WSREP_SST_MYSQLDUMP))
+  method = wsrep_sst_method;
+  if (!strcmp(method, WSREP_SST_MYSQLDUMP))
   {
     addr_len= sst_prepare_mysqldump (addr_in, &addr_out);
     if (addr_len < 0) unireg_abort(1);
@@ -871,6 +873,13 @@ ssize_t wsrep_sst_prepare (void** msg)
     /*! A heuristic workaround until we learn how to stop and start engines */
     if (SE_initialized)
     {
+      if (!strcmp(method, WSREP_SST_XTRABACKUP) ||
+          !strcmp(method, WSREP_SST_XTRABACKUPV2))
+      {
+         WSREP_WARN("The %s SST method is deprecated, so it is automatically "
+                    "replaced by %s", method, WSREP_SST_MARIABACKUP);
+         method = WSREP_SST_MARIABACKUP;
+      }
       // we already did SST at initializaiton, now engines are running
       // sql_print_information() is here because the message is too long
       // for WSREP_INFO.
@@ -880,28 +889,28 @@ ssize_t wsrep_sst_prepare (void** msg)
                  "Wsrep provider won't be able to fall back to it "
                  "if other means of state transfer are unavailable. "
                  "In that case you will need to restart the server.",
-                 wsrep_sst_method);
+                 method);
       *msg = 0;
       return 0;
     }
 
-    addr_len = sst_prepare_other (wsrep_sst_method, sst_auth_real,
+    addr_len = sst_prepare_other (method, sst_auth_real,
                                   addr_in, &addr_out);
     if (addr_len < 0)
     {
       WSREP_ERROR("Failed to prepare for '%s' SST. Unrecoverable.",
-                   wsrep_sst_method);
+                   method);
       unireg_abort(1);
     }
   }
 
-  size_t const method_len(strlen(wsrep_sst_method));
+  size_t const method_len(strlen(method));
   size_t const msg_len   (method_len + addr_len + 2 /* + auth_len + 1*/);
 
   *msg = malloc (msg_len);
   if (NULL != *msg) {
     char* const method_ptr(reinterpret_cast<char*>(*msg));
-    strcpy (method_ptr, wsrep_sst_method);
+    strcpy (method_ptr, method);
     char* const addr_ptr(method_ptr + method_len + 1);
     strcpy (addr_ptr, addr_out);
 

sql/wsrep_sst.h

@@ -49,6 +49,9 @@
 #define WSREP_SST_MYSQLDUMP    "mysqldump"
 #define WSREP_SST_RSYNC        "rsync"
 #define WSREP_SST_SKIP         "skip"
+#define WSREP_SST_MARIABACKUP  "mariabackup"
+#define WSREP_SST_XTRABACKUP   "xtrabackup"
+#define WSREP_SST_XTRABACKUPV2 "xtrabackupv2"
 #define WSREP_SST_DEFAULT      WSREP_SST_RSYNC
 #define WSREP_SST_ADDRESS_AUTO "AUTO"
 #define WSREP_SST_AUTH_MASK    "********"