MDEV-28423: Galera IST is failing on Joiner node

This commit fixes an issue with IST handling in
version 10.9 which is a regression after MDEV-26971
and related to trying to get a non-existent "total"
tag on the IST branch (this tag is only defined in
SST mode).

mysql-test/suite/galera/r/galera_ist_MDEV-28423,debug.rdiff

+--- r/galera_ist_MDEV-28423.result
++++ r/galera_ist_MDEV-28423,debug.reject
+@@ -517,3 +517,187 @@
+ 1
+ DROP TABLE t1;
+ COMMIT;
++Performing State Transfer on a server that has been killed and restarted
++while a DDL was in progress on it
++connection node_1;
++CREATE TABLE t1 (id int not null primary key,f1 CHAR(255)) ENGINE=InnoDB;
++SET AUTOCOMMIT=OFF;
++START TRANSACTION;
++INSERT INTO t1 VALUES (1,'node1_committed_before');
++INSERT INTO t1 VALUES (2,'node1_committed_before');
++INSERT INTO t1 VALUES (3,'node1_committed_before');
++INSERT INTO t1 VALUES (4,'node1_committed_before');
++INSERT INTO t1 VALUES (5,'node1_committed_before');
++connection node_2;
++START TRANSACTION;
++INSERT INTO t1 VALUES (6,'node2_committed_before');
++INSERT INTO t1 VALUES (7,'node2_committed_before');
++INSERT INTO t1 VALUES (8,'node2_committed_before');
++INSERT INTO t1 VALUES (9,'node2_committed_before');
++INSERT INTO t1 VALUES (10,'node2_committed_before');
++COMMIT;
++SET GLOBAL debug_dbug = 'd,sync.alter_opened_table';
++connection node_1;
++ALTER TABLE t1 ADD COLUMN f2 INTEGER;
++connection node_2;
++SET wsrep_sync_wait = 0;
++Killing server ...
++connection node_1;
++SET AUTOCOMMIT=OFF;
++START TRANSACTION;
++INSERT INTO t1 (id,f1) VALUES (11,'node1_committed_during');
++INSERT INTO t1 (id,f1) VALUES (12,'node1_committed_during');
++INSERT INTO t1 (id,f1) VALUES (13,'node1_committed_during');
++INSERT INTO t1 (id,f1) VALUES (14,'node1_committed_during');
++INSERT INTO t1 (id,f1) VALUES (15,'node1_committed_during');
++COMMIT;
++START TRANSACTION;
++INSERT INTO t1 (id,f1) VALUES (16,'node1_to_be_committed_after');
++INSERT INTO t1 (id,f1) VALUES (17,'node1_to_be_committed_after');
++INSERT INTO t1 (id,f1) VALUES (18,'node1_to_be_committed_after');
++INSERT INTO t1 (id,f1) VALUES (19,'node1_to_be_committed_after');
++INSERT INTO t1 (id,f1) VALUES (20,'node1_to_be_committed_after');
++connect node_1a_galera_st_kill_slave_ddl, 127.0.0.1, root, , test, $NODE_MYPORT_1;
++SET AUTOCOMMIT=OFF;
++START TRANSACTION;
++INSERT INTO t1 (id,f1) VALUES (21,'node1_to_be_rollbacked_after');
++INSERT INTO t1 (id,f1) VALUES (22,'node1_to_be_rollbacked_after');
++INSERT INTO t1 (id,f1) VALUES (23,'node1_to_be_rollbacked_after');
++INSERT INTO t1 (id,f1) VALUES (24,'node1_to_be_rollbacked_after');
++INSERT INTO t1 (id,f1) VALUES (25,'node1_to_be_rollbacked_after');
++connection node_2;
++Performing --wsrep-recover ...
++connection node_2;
++Starting server ...
++Using --wsrep-start-position when starting mysqld ...
++SET AUTOCOMMIT=OFF;
++START TRANSACTION;
++INSERT INTO t1 (id,f1) VALUES (26,'node2_committed_after');
++INSERT INTO t1 (id,f1) VALUES (27,'node2_committed_after');
++INSERT INTO t1 (id,f1) VALUES (28,'node2_committed_after');
++INSERT INTO t1 (id,f1) VALUES (29,'node2_committed_after');
++INSERT INTO t1 (id,f1) VALUES (30,'node2_committed_after');
++COMMIT;
++connection node_1;
++INSERT INTO t1 (id,f1) VALUES (31,'node1_to_be_committed_after');
++INSERT INTO t1 (id,f1) VALUES (32,'node1_to_be_committed_after');
++INSERT INTO t1 (id,f1) VALUES (33,'node1_to_be_committed_after');
++INSERT INTO t1 (id,f1) VALUES (34,'node1_to_be_committed_after');
++INSERT INTO t1 (id,f1) VALUES (35,'node1_to_be_committed_after');
++COMMIT;
++SET AUTOCOMMIT=OFF;
++START TRANSACTION;
++INSERT INTO t1 (id,f1) VALUES (36,'node1_committed_after');
++INSERT INTO t1 (id,f1) VALUES (37,'node1_committed_after');
++INSERT INTO t1 (id,f1) VALUES (38,'node1_committed_after');
++INSERT INTO t1 (id,f1) VALUES (39,'node1_committed_after');
++INSERT INTO t1 (id,f1) VALUES (40,'node1_committed_after');
++COMMIT;
++connection node_1a_galera_st_kill_slave_ddl;
++INSERT INTO t1 (id,f1) VALUES (41,'node1_to_be_rollbacked_after');
++INSERT INTO t1 (id,f1) VALUES (42,'node1_to_be_rollbacked_after');
++INSERT INTO t1 (id,f1) VALUES (43,'node1_to_be_rollbacked_after');
++INSERT INTO t1 (id,f1) VALUES (44,'node1_to_be_rollbacked_after');
++INSERT INTO t1 (id,f1) VALUES (45,'node1_to_be_rollbacked_after');
++ROLLBACK;
++SET AUTOCOMMIT=ON;
++SET SESSION wsrep_sync_wait=15;
++SELECT COUNT(*) AS EXPECT_3 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = 't1';
++EXPECT_3
++3
++SELECT COUNT(*) AS EXPECT_35 FROM t1;
++EXPECT_35
++35
++SELECT * FROM t1;
++id	f1	f2
++1	node1_committed_before	NULL
++2	node1_committed_before	NULL
++3	node1_committed_before	NULL
++4	node1_committed_before	NULL
++5	node1_committed_before	NULL
++6	node2_committed_before	NULL
++7	node2_committed_before	NULL
++8	node2_committed_before	NULL
++9	node2_committed_before	NULL
++10	node2_committed_before	NULL
++11	node1_committed_during	NULL
++12	node1_committed_during	NULL
++13	node1_committed_during	NULL
++14	node1_committed_during	NULL
++15	node1_committed_during	NULL
++16	node1_to_be_committed_after	NULL
++17	node1_to_be_committed_after	NULL
++18	node1_to_be_committed_after	NULL
++19	node1_to_be_committed_after	NULL
++20	node1_to_be_committed_after	NULL
++26	node2_committed_after	NULL
++27	node2_committed_after	NULL
++28	node2_committed_after	NULL
++29	node2_committed_after	NULL
++30	node2_committed_after	NULL
++31	node1_to_be_committed_after	NULL
++32	node1_to_be_committed_after	NULL
++33	node1_to_be_committed_after	NULL
++34	node1_to_be_committed_after	NULL
++35	node1_to_be_committed_after	NULL
++36	node1_committed_after	NULL
++37	node1_committed_after	NULL
++38	node1_committed_after	NULL
++39	node1_committed_after	NULL
++40	node1_committed_after	NULL
++SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1;
++COUNT(*) = 0
++1
++COMMIT;
++connection node_1;
++SET AUTOCOMMIT=ON;
++SET SESSION wsrep_sync_wait=15;
++SELECT COUNT(*) AS EXPECT_3 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = 't1';
++EXPECT_3
++3
++SELECT COUNT(*) AS EXPECT_35 FROM t1;
++EXPECT_35
++35
++SELECT * FROM t1;
++id	f1	f2
++1	node1_committed_before	NULL
++2	node1_committed_before	NULL
++3	node1_committed_before	NULL
++4	node1_committed_before	NULL
++5	node1_committed_before	NULL
++6	node2_committed_before	NULL
++7	node2_committed_before	NULL
++8	node2_committed_before	NULL
++9	node2_committed_before	NULL
++10	node2_committed_before	NULL
++11	node1_committed_during	NULL
++12	node1_committed_during	NULL
++13	node1_committed_during	NULL
++14	node1_committed_during	NULL
++15	node1_committed_during	NULL
++16	node1_to_be_committed_after	NULL
++17	node1_to_be_committed_after	NULL
++18	node1_to_be_committed_after	NULL
++19	node1_to_be_committed_after	NULL
++20	node1_to_be_committed_after	NULL
++26	node2_committed_after	NULL
++27	node2_committed_after	NULL
++28	node2_committed_after	NULL
++29	node2_committed_after	NULL
++30	node2_committed_after	NULL
++31	node1_to_be_committed_after	NULL
++32	node1_to_be_committed_after	NULL
++33	node1_to_be_committed_after	NULL
++34	node1_to_be_committed_after	NULL
++35	node1_to_be_committed_after	NULL
++36	node1_committed_after	NULL
++37	node1_committed_after	NULL
++38	node1_committed_after	NULL
++39	node1_committed_after	NULL
++40	node1_committed_after	NULL
++SELECT COUNT(*) = 0 FROM (SELECT COUNT(*) AS c, f1 FROM t1 GROUP BY f1 HAVING c NOT IN (5, 10)) AS a1;
++COUNT(*) = 0
++1
++DROP TABLE t1;
++COMMIT;
++SET GLOBAL debug_dbug = $debug_orig;

mysql-test/suite/galera/t/galera_ist_MDEV-28423.cnf

+!include ../galera_2nodes.cnf
+
+[mysqld.1]
+# server-id=101
+#wsrep-debug=1
+innodb_file_per_table
+innodb_autoinc_lock_mode=2
+#wsrep_sst_method=rsync
+wsrep_sst_method=mariabackup
+wsrep_sst_auth=root:
+binlog_format=ROW
+core-file
+log-output=none
+wsrep_slave_threads=2
+wsrep_on=1
+gtid_strict_mode=1
+log_slave_updates=ON
+log_bin=binlog
+
+[mysqld.2]
+# server-id=102
+#wsrep-debug=1
+innodb_file_per_table
+innodb_autoinc_lock_mode=2
+#wsrep_sst_method=rsync
+wsrep_sst_method=mariabackup
+wsrep_sst_auth=root:
+binlog_format=ROW
+core-file
+log-output=none
+wsrep_slave_threads=2
+wsrep_on=1
+gtid_strict_mode=1
+log_slave_updates=ON
+log_bin=binlog
+
+[mysqld.1]
+wsrep_provider_options='base_port=@mysqld.1.#galera_port;pc.ignore_sb=true'
+
+[mysqld.2]
+wsrep_provider_options='base_port=@mysqld.2.#galera_port;pc.ignore_sb=true'
+
+[sst]
+transferfmt=@ENV.MTR_GALERA_TFMT

mysql-test/suite/galera/t/galera_ist_MDEV-28423.test

+# MDEV-28423: Galera IST is failing on Joiner node
+
+--source include/big_test.inc
+--source include/galera_cluster.inc
+--source include/have_innodb.inc
+--source include/have_mariabackup.inc
+
+--let $node_1=node_1
+--let $node_2=node_2
+--source include/auto_increment_offset_save.inc
+
+--source suite/galera/include/galera_st_disconnect_slave.inc
+--source suite/galera/include/galera_st_shutdown_slave.inc
+
+--source suite/galera/include/galera_st_kill_slave.inc
+--source suite/galera/include/galera_st_kill_slave_ddl.inc
+
+--source include/auto_increment_offset_restore.inc

scripts/wsrep_sst_mariabackup.sh

@@ -86,6 +86,7 @@ encrypt_threads=""
 encrypt_chunk=""
 
 readonly SECRET_TAG='secret'
+readonly TOTAL_TAG='total'
 
 # Required for backup locks
 # For backup locks it is 1 sent by joiner
@@ -324,7 +325,8 @@ get_transfer()
             if [ -z "$ssl_dhparams" ]; then
                 # Determine the socat version
                 SOCAT_VERSION=$(socat -V 2>&1 | \
-                                grep -m1 -owE '[0-9]+(\.[0-9]+)+' | head -n1)
+                                grep -m1 -owE '[0-9]+(\.[0-9]+)+' | \
+                                head -n1 || :)
                 if [ -z "$SOCAT_VERSION" ]; then
                     wsrep_log_error "******** FATAL ERROR ******************"
                     wsrep_log_error "* Cannot determine the socat version. *"
@@ -770,25 +772,26 @@ recv_joiner()
 
     if [ $checkf -eq 1 ]; then
         if [ ! -r "$MAGIC_FILE" ]; then
-            # this message should cause joiner to abort
+            # this message should cause joiner to abort:
             wsrep_log_error "receiving process ended without creating" \
-                            "'$MAGIC_FILE'"
-            wsrep_log_info "Contents of datadir"
+                            "magic file ($MAGIC_FILE)"
+            wsrep_log_info "Contents of datadir:"
             wsrep_log_info $(ls -l "$dir/"*)
             exit 32
         fi
 
         # check donor supplied secret
-        SECRET=$(grep -F -- "$SECRET_TAG " "$MAGIC_FILE" 2>/dev/null | \
-                 cut -d ' ' -f2)
+        SECRET=$(grep -m1 -E "^$SECRET_TAG[[:space:]]" -- "$MAGIC_FILE" || :)
+        SECRET=$(trim_string "${SECRET#$SECRET_TAG}")
         if [ "$SECRET" != "$MY_SECRET" ]; then
             wsrep_log_error "Donor does not know my secret!"
             wsrep_log_info "Donor: '$SECRET', my: '$MY_SECRET'"
             exit 32
         fi
 
-        # remove secret from the magic file
-        grep -v -F -- "$SECRET_TAG " "$MAGIC_FILE" > "$MAGIC_FILE.new"
+        # remove secret and total from the magic file
+        grep -v -E "^($SECRET_TAG|$TOTAL_TAG)[[:space:]]" -- \
+             "$MAGIC_FILE" > "$MAGIC_FILE.new"
         mv "$MAGIC_FILE.new" "$MAGIC_FILE"
     fi
 }
@@ -1256,8 +1259,7 @@ then
 
     recv_joiner "$STATDIR" "$stagemsg-gtid" $stimeout 1 1
 
-    if ! ps -p "$WSREP_SST_OPT_PARENT" >/dev/null 2>&1
-    then
+    if ! ps -p "$WSREP_SST_OPT_PARENT" >/dev/null 2>&1; then
         wsrep_log_error "Parent mysqld process (PID: $WSREP_SST_OPT_PARENT)" \
                         "terminated unexpectedly."
         exit 32

scripts/wsrep_sst_rsync.sh

@@ -804,8 +804,7 @@ EOF
         sleep 1
     done
 
-    if ! ps -p $MYSQLD_PID >/dev/null 2>&1
-    then
+    if ! ps -p $MYSQLD_PID >/dev/null 2>&1; then
         wsrep_log_error \
             "Parent mysqld process (PID: $MYSQLD_PID) terminated unexpectedly."
         kill -- -$MYSQLD_PID
@@ -813,23 +812,23 @@ EOF
         exit 32
     fi
 
-    if [ -r "$MAGIC_FILE" ]; then
+    if [ ! -r "$MAGIC_FILE" ]; then
+        # This message should cause joiner to abort:
+        wsrep_log_info "rsync process ended without creating" \
+                       "magic file ($MAGIC_FILE)"
+        exit 32
+    fi
+
     if [ -n "$MY_SECRET" ]; then
         # Check donor supplied secret:
-            SECRET=$(grep -F -- "$SECRET_TAG " "$MAGIC_FILE" 2>/dev/null | \
-                     cut -d ' ' -f2)
+        SECRET=$(grep -m1 -E "^$SECRET_TAG[[:space:]]" -- "$MAGIC_FILE" || :)
+        SECRET=$(trim_string "${SECRET#$SECRET_TAG}")
         if [ "$SECRET" != "$MY_SECRET" ]; then
             wsrep_log_error "Donor does not know my secret!"
             wsrep_log_info "Donor: '$SECRET', my: '$MY_SECRET'"
             exit 32
         fi
     fi
-    else
-        # This message should cause joiner to abort:
-        wsrep_log_info "rsync process ended without creating magic file"
-        echo "rsync process ended without creating '$MAGIC_FILE'"
-        exit 32
-    fi
 
     if [ -n "$WSREP_SST_OPT_BINLOG" ]; then
         binlog_tar_present=0
@@ -907,7 +906,7 @@ EOF
     if [ -n "$MY_SECRET" ]; then
         # remove secret from the magic file, and output
         # the UUID:seqno & wsrep_gtid_domain_id:
-        grep -v -F -- "$SECRET_TAG " "$MAGIC_FILE"
+        grep -v -E "^$SECRET_TAG[[:space:]]" -- "$MAGIC_FILE"
     else
         # Output the UUID:seqno and wsrep_gtid_domain_id:
         cat "$MAGIC_FILE"