Merge pull request #828 from tempesta-tech/sysprg/10.1-MDEV-10754

MDEV-10754 wsrep_sst_rsync does not support innodb_data_home_dir

mysql-test/suite/galera/r/galera_sst_rsync_data_dir.result

+Performing State Transfer on a server that has been shut down cleanly and restarted
+CREATE TABLE t1 (f1 CHAR(255)) ENGINE=InnoDB;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES ('node1_committed_before');
+INSERT INTO t1 VALUES ('node1_committed_before');
+INSERT INTO t1 VALUES ('node1_committed_before');
+INSERT INTO t1 VALUES ('node1_committed_before');
+INSERT INTO t1 VALUES ('node1_committed_before');
+COMMIT;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES ('node2_committed_before');
+INSERT INTO t1 VALUES ('node2_committed_before');
+INSERT INTO t1 VALUES ('node2_committed_before');
+INSERT INTO t1 VALUES ('node2_committed_before');
+INSERT INTO t1 VALUES ('node2_committed_before');
+COMMIT;
+Shutting down server ...
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES ('node1_committed_during');
+INSERT INTO t1 VALUES ('node1_committed_during');
+INSERT INTO t1 VALUES ('node1_committed_during');
+INSERT INTO t1 VALUES ('node1_committed_during');
+INSERT INTO t1 VALUES ('node1_committed_during');
+COMMIT;
+START TRANSACTION;
+INSERT INTO t1 VALUES ('node1_to_be_committed_after');
+INSERT INTO t1 VALUES ('node1_to_be_committed_after');
+INSERT INTO t1 VALUES ('node1_to_be_committed_after');
+INSERT INTO t1 VALUES ('node1_to_be_committed_after');
+INSERT INTO t1 VALUES ('node1_to_be_committed_after');
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
+Starting server ...
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES ('node2_committed_after');
+INSERT INTO t1 VALUES ('node2_committed_after');
+INSERT INTO t1 VALUES ('node2_committed_after');
+INSERT INTO t1 VALUES ('node2_committed_after');
+INSERT INTO t1 VALUES ('node2_committed_after');
+COMMIT;
+INSERT INTO t1 VALUES ('node1_to_be_committed_after');
+INSERT INTO t1 VALUES ('node1_to_be_committed_after');
+INSERT INTO t1 VALUES ('node1_to_be_committed_after');
+INSERT INTO t1 VALUES ('node1_to_be_committed_after');
+INSERT INTO t1 VALUES ('node1_to_be_committed_after');
+COMMIT;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES ('node1_committed_after');
+INSERT INTO t1 VALUES ('node1_committed_after');
+INSERT INTO t1 VALUES ('node1_committed_after');
+INSERT INTO t1 VALUES ('node1_committed_after');
+INSERT INTO t1 VALUES ('node1_committed_after');
+COMMIT;
+INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
+ROLLBACK;
+SELECT COUNT(*) = 35 FROM t1;
+COUNT(*) = 35
+1
+SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1;
+COUNT(*) = 0
+1
+COMMIT;
+SET AUTOCOMMIT=ON;
+SELECT COUNT(*) = 35 FROM t1;
+COUNT(*) = 35
+1
+SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1;
+COUNT(*) = 0
+1
+DROP TABLE t1;
+COMMIT;
+SET AUTOCOMMIT=ON;
+Performing State Transfer on a server that starts from a clean var directory
+This is accomplished by shutting down node #2 and removing its var directory before restarting it
+CREATE TABLE t1 (f1 CHAR(255)) ENGINE=InnoDB;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES ('node1_committed_before');
+INSERT INTO t1 VALUES ('node1_committed_before');
+INSERT INTO t1 VALUES ('node1_committed_before');
+INSERT INTO t1 VALUES ('node1_committed_before');
+INSERT INTO t1 VALUES ('node1_committed_before');
+COMMIT;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES ('node2_committed_before');
+INSERT INTO t1 VALUES ('node2_committed_before');
+INSERT INTO t1 VALUES ('node2_committed_before');
+INSERT INTO t1 VALUES ('node2_committed_before');
+INSERT INTO t1 VALUES ('node2_committed_before');
+COMMIT;
+Shutting down server ...
+Cleaning var directory ...
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES ('node1_committed_during');
+INSERT INTO t1 VALUES ('node1_committed_during');
+INSERT INTO t1 VALUES ('node1_committed_during');
+INSERT INTO t1 VALUES ('node1_committed_during');
+INSERT INTO t1 VALUES ('node1_committed_during');
+COMMIT;
+START TRANSACTION;
+INSERT INTO t1 VALUES ('node1_to_be_committed_after');
+INSERT INTO t1 VALUES ('node1_to_be_committed_after');
+INSERT INTO t1 VALUES ('node1_to_be_committed_after');
+INSERT INTO t1 VALUES ('node1_to_be_committed_after');
+INSERT INTO t1 VALUES ('node1_to_be_committed_after');
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
+Starting server ...
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES ('node2_committed_after');
+INSERT INTO t1 VALUES ('node2_committed_after');
+INSERT INTO t1 VALUES ('node2_committed_after');
+INSERT INTO t1 VALUES ('node2_committed_after');
+INSERT INTO t1 VALUES ('node2_committed_after');
+COMMIT;
+INSERT INTO t1 VALUES ('node1_to_be_committed_after');
+INSERT INTO t1 VALUES ('node1_to_be_committed_after');
+INSERT INTO t1 VALUES ('node1_to_be_committed_after');
+INSERT INTO t1 VALUES ('node1_to_be_committed_after');
+INSERT INTO t1 VALUES ('node1_to_be_committed_after');
+COMMIT;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES ('node1_committed_after');
+INSERT INTO t1 VALUES ('node1_committed_after');
+INSERT INTO t1 VALUES ('node1_committed_after');
+INSERT INTO t1 VALUES ('node1_committed_after');
+INSERT INTO t1 VALUES ('node1_committed_after');
+COMMIT;
+INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
+ROLLBACK;
+SELECT COUNT(*) = 35 FROM t1;
+COUNT(*) = 35
+1
+SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1;
+COUNT(*) = 0
+1
+COMMIT;
+SET AUTOCOMMIT=ON;
+SELECT COUNT(*) = 35 FROM t1;
+COUNT(*) = 35
+1
+SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1;
+COUNT(*) = 0
+1
+DROP TABLE t1;
+COMMIT;
+SET AUTOCOMMIT=ON;
+Performing State Transfer on a server that has been killed and restarted
+CREATE TABLE t1 (f1 CHAR(255)) ENGINE=InnoDB;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES ('node1_committed_before');
+INSERT INTO t1 VALUES ('node1_committed_before');
+INSERT INTO t1 VALUES ('node1_committed_before');
+INSERT INTO t1 VALUES ('node1_committed_before');
+INSERT INTO t1 VALUES ('node1_committed_before');
+COMMIT;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES ('node2_committed_before');
+INSERT INTO t1 VALUES ('node2_committed_before');
+INSERT INTO t1 VALUES ('node2_committed_before');
+INSERT INTO t1 VALUES ('node2_committed_before');
+INSERT INTO t1 VALUES ('node2_committed_before');
+COMMIT;
+Killing server ...
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES ('node1_committed_during');
+INSERT INTO t1 VALUES ('node1_committed_during');
+INSERT INTO t1 VALUES ('node1_committed_during');
+INSERT INTO t1 VALUES ('node1_committed_during');
+INSERT INTO t1 VALUES ('node1_committed_during');
+COMMIT;
+START TRANSACTION;
+INSERT INTO t1 VALUES ('node1_to_be_committed_after');
+INSERT INTO t1 VALUES ('node1_to_be_committed_after');
+INSERT INTO t1 VALUES ('node1_to_be_committed_after');
+INSERT INTO t1 VALUES ('node1_to_be_committed_after');
+INSERT INTO t1 VALUES ('node1_to_be_committed_after');
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
+Performing --wsrep-recover ...
+Starting server ...
+Using --wsrep-start-position when starting mysqld ...
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES ('node2_committed_after');
+INSERT INTO t1 VALUES ('node2_committed_after');
+INSERT INTO t1 VALUES ('node2_committed_after');
+INSERT INTO t1 VALUES ('node2_committed_after');
+INSERT INTO t1 VALUES ('node2_committed_after');
+COMMIT;
+INSERT INTO t1 VALUES ('node1_to_be_committed_after');
+INSERT INTO t1 VALUES ('node1_to_be_committed_after');
+INSERT INTO t1 VALUES ('node1_to_be_committed_after');
+INSERT INTO t1 VALUES ('node1_to_be_committed_after');
+INSERT INTO t1 VALUES ('node1_to_be_committed_after');
+COMMIT;
+SET AUTOCOMMIT=OFF;
+START TRANSACTION;
+INSERT INTO t1 VALUES ('node1_committed_after');
+INSERT INTO t1 VALUES ('node1_committed_after');
+INSERT INTO t1 VALUES ('node1_committed_after');
+INSERT INTO t1 VALUES ('node1_committed_after');
+INSERT INTO t1 VALUES ('node1_committed_after');
+COMMIT;
+INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
+INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
+ROLLBACK;
+SELECT COUNT(*) = 35 FROM t1;
+COUNT(*) = 35
+1
+SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1;
+COUNT(*) = 0
+1
+COMMIT;
+SET AUTOCOMMIT=ON;
+SELECT COUNT(*) = 35 FROM t1;
+COUNT(*) = 35
+1
+SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1;
+COUNT(*) = 0
+1
+DROP TABLE t1;
+COMMIT;
+SET AUTOCOMMIT=ON;

mysql-test/suite/galera/t/galera_sst_rsync_data_dir.cnf

+!include ../galera_2nodes.cnf
+
+[mysqld]
+wsrep_sst_method=rsync
+
+[mysqld.1]
+wsrep_provider_options='base_port=@mysqld.1.#galera_port;gcache.size=1;pc.ignore_sb=true'
+
+[mysqld.2]
+innodb_data_home_dir=@ENV.MYSQL_TMP_DIR/rsync_test_2
+wsrep_provider_options='base_port=@mysqld.2.#galera_port;gcache.size=1;pc.ignore_sb=true'

mysql-test/suite/galera/t/galera_sst_rsync_data_dir.test

+--source include/big_test.inc
+--source include/galera_cluster.inc
+
+--let $node_1=node_1
+--let $node_2=node_2
+--source include/auto_increment_offset_save.inc
+
+--source suite/galera/include/galera_st_shutdown_slave.inc
+--source suite/galera/include/galera_st_clean_slave.inc
+
+--source suite/galera/include/galera_st_kill_slave.inc
+--source suite/galera/include/galera_st_kill_slave_ddl.inc
+--source include/auto_increment_offset_restore.inc
+
+# cleanup temporary database files:
+--remove_files_wildcard $MYSQL_TMP_DIR/rsync_test_2 *

scripts/wsrep_sst_common.sh

@@ -27,6 +27,7 @@ WSREP_SST_OPT_PSWD=${WSREP_SST_OPT_PSWD:-}
 WSREP_SST_OPT_DEFAULT=""
 WSREP_SST_OPT_EXTRA_DEFAULT=""
 WSREP_SST_OPT_SUFFIX_DEFAULT=""
+WSREP_SST_OPT_SUFFIX_VALUE=""
 
 while [ $# -gt 0 ]; do
 case "$1" in
@@ -76,6 +77,7 @@ case "$1" in
         ;;
     '--defaults-group-suffix')
         readonly WSREP_SST_OPT_SUFFIX_DEFAULT="$1=$2"
+        readonly WSREP_SST_OPT_SUFFIX_VALUE="$2"
         shift
         ;;
     '--host')

scripts/wsrep_sst_rsync.sh

@@ -121,6 +121,20 @@ else
     WSREP_LOG_DIR=$(cd $WSREP_SST_OPT_DATA; pwd -P)
 fi
 
+INNODB_DATA_HOME_DIR=${INNODB_DATA_HOME_DIR:-""}
+# if INNODB_DATA_HOME_DIR env. variable is not set, try to get it from my.cnf
+if [ -z "$INNODB_DATA_HOME_DIR" ]; then
+    INNODB_DATA_HOME_DIR=$(parse_cnf mysqld$WSREP_SST_OPT_SUFFIX_VALUE innodb-data-home-dir '')
+fi
+
+if [ -n "$INNODB_DATA_HOME_DIR" ]; then
+    # handle both relative and absolute paths
+    INNODB_DATA_HOME_DIR=$(cd $WSREP_SST_OPT_DATA; mkdir -p "$INNODB_DATA_HOME_DIR"; cd $INNODB_DATA_HOME_DIR; pwd -P)
+else
+    # default to datadir
+    INNODB_DATA_HOME_DIR=$(cd $WSREP_SST_OPT_DATA; pwd -P)
+fi
+
 # Old filter - include everything except selected
 # FILTER=(--exclude '*.err' --exclude '*.pid' --exclude '*.sock' \
 #         --exclude '*.conf' --exclude core --exclude 'galera.*' \
@@ -129,7 +143,8 @@ fi
 
 # New filter - exclude everything except dirs (schemas) and innodb files
 FILTER=(-f '- /lost+found' -f '- /.fseventsd' -f '- /.Trashes'
-        -f '+ /wsrep_sst_binlog.tar' -f '+ /ib_lru_dump' -f '+ /ibdata*' -f '+ /*/' -f '- /*')
+        -f '+ /wsrep_sst_binlog.tar' -f '- $INNODB_DATA_HOME_DIR/ib_lru_dump'
+        -f '- $INNODB_DATA_HOME_DIR/ibdata*' -f '+ /*/' -f '- /*')
 
 SSTKEY=$(parse_cnf sst tkey "")
 SSTCERT=$(parse_cnf sst tcert "")
@@ -235,6 +250,19 @@ EOF
             exit $RC
         fi
 
+        # Transfer InnoDB data files
+        rsync ${STUNNEL:+--rsh="$STUNNEL"} \
+              --owner --group --perms --links --specials \
+              --ignore-times --inplace --dirs --delete --quiet \
+              $WHOLE_FILE_OPT -f '+ /ibdata*' -f '+ /ib_lru_dump' \
+              -f '- **' "$INNODB_DATA_HOME_DIR/" \
+              rsync://$WSREP_SST_OPT_ADDR-data_dir >&2 || RC=$?
+
+        if [ $RC -ne 0 ]; then
+            wsrep_log_error "rsync innodb_data_home_dir returned code $RC:"
+            exit 255 # unknown error
+        fi
+
         # second, we transfer InnoDB log files
         rsync ${STUNNEL:+--rsh="$STUNNEL"} \
               --owner --group --perms --links --specials \
@@ -333,6 +361,8 @@ $SILENT
     path = $WSREP_SST_OPT_DATA
 [$MODULE-log_dir]
     path = $WSREP_LOG_DIR
+[$MODULE-data_dir]
+    path = $INNODB_DATA_HOME_DIR
 EOF
 
 #    rm -rf "$DATA"/ib_logfile* # we don't want old logs around

sql/wsrep_mysqld.h

@@ -160,6 +160,7 @@ extern "C" time_t wsrep_thd_query_start(THD *thd);
 extern "C" query_id_t wsrep_thd_query_id(THD *thd);
 extern "C" query_id_t wsrep_thd_wsrep_last_query_id(THD *thd);
 extern "C" void wsrep_thd_set_wsrep_last_query_id(THD *thd, query_id_t id);
+extern "C" void wsrep_set_data_home_dir(const char *data_dir);
 
 extern "C" void wsrep_thd_auto_increment_variables(THD*,
                                                    unsigned long long *offset,

sql/wsrep_sst.cc

@@ -67,6 +67,13 @@ bool wsrep_sst_method_update (sys_var *self, THD* thd, enum_var_type type)
     return 0;
 }
 
+static const char* data_home_dir = NULL;
+
+extern "C"
+void wsrep_set_data_home_dir(const char *data_dir)
+{
+  data_home_dir= (data_dir && *data_dir) ? data_dir : NULL;
+}
 
 static void make_wsrep_defaults_file()
 {
@@ -549,6 +556,29 @@ static int sst_append_auth_env(wsp::env& env, const char* sst_auth)
   return -env.error();
 }
 
+#define DATA_HOME_DIR_ENV "INNODB_DATA_HOME_DIR"
+
+static int sst_append_data_dir(wsp::env& env, const char* data_dir)
+{
+  int const data_dir_size= strlen(DATA_HOME_DIR_ENV) + 1 /* = */
+    + (data_dir ? strlen(data_dir) : 0) + 1 /* \0 */;
+
+  wsp::string data_dir_str(data_dir_size); // for automatic cleanup on return
+  if (!data_dir_str()) return -ENOMEM;
+
+  int ret= snprintf(data_dir_str(), data_dir_size, "%s=%s",
+                    DATA_HOME_DIR_ENV, data_dir ? data_dir : "");
+
+  if (ret < 0 || ret >= data_dir_size)
+  {
+    WSREP_ERROR("sst_append_data_dir(): snprintf() failed: %d", ret);
+    return (ret < 0 ? ret : -EMSGSIZE);
+  }
+
+  env.append(data_dir_str());
+  return -env.error();
+}
+
 static ssize_t sst_prepare_other (const char*  method,
                                   const char*  sst_auth,
                                   const char*  addr_in,
@@ -610,6 +640,16 @@ static ssize_t sst_prepare_other (const char*  method,
     return ret;
   }
 
+  if (data_home_dir)
+  {
+    if ((ret= sst_append_data_dir(env, data_home_dir)))
+    {
+      WSREP_ERROR("sst_prepare_other(): appending data "
+                  "directory failed: %d", ret);
+      return ret;
+    }
+  }
+
   pthread_t tmp;
   sst_thread_arg arg(cmd_str(), env());
   mysql_mutex_lock (&arg.lock);
@@ -1301,6 +1341,16 @@ wsrep_cb_status_t wsrep_sst_donate_cb (void* app_ctx, void* recv_ctx,
     return WSREP_CB_FAILURE;
   }
 
+  if (data_home_dir)
+  {
+    if ((ret= sst_append_data_dir(env, data_home_dir)))
+    {
+      WSREP_ERROR("wsrep_sst_donate_cb(): appending data "
+                  "directory failed: %d", ret);
+      return WSREP_CB_FAILURE;
+    }
+  }
+
   if (!strcmp (WSREP_SST_MYSQLDUMP, method))
   {
     ret = sst_donate_mysqldump(data, &current_gtid->uuid, uuid_str,

storage/innobase/handler/ha_innodb.cc

@@ -3623,6 +3623,12 @@ innobase_init(
 	srv_data_home = (innobase_data_home_dir ? innobase_data_home_dir :
 			 default_path);
 
+#ifdef WITH_WSREP
+	/* If we use the wsrep API, then we need to tell the server
+	the path to the data files (for passing it to the SST scripts): */
+	wsrep_set_data_home_dir(innobase_data_home_dir);
+#endif /* WITH_WSREP */
+
 	/* Set default InnoDB data file size to 12 MB and let it be
 	auto-extending. Thus users can use InnoDB in >= 4.0 without having
 	to specify any startup options. */