MDEV-25719: stunnel uses "verifyChain" without subject checks

Another batch of changes that should make the SST process
more reliable in all scenarios:

 1) Added hostname or CN verification when stunnel is used
    with certificate chain verification (verifyChain = yes);
 2) Added check for the absence of the stunnel utility for
    mtr tests;
 3) Deletion of working files before and after SST is done
    more accurately;
 4) rsync on joiner can be run even if the path to its
    configuration file contains spaces;
 5) More accurate directory creation (for data files and
    for logs);
 6) IST with mysqldump no longer turns off statement logging;
 7) Reset password for mysqldump when password is empty but
    username is specified;
 8) More reliable quoting when generating statements in
    wsrep_sst_mysqldump;
 9) Added explicit generation of 2048-bit Diffie-Hellman
    parameters for sockat < 1.7.3, by analogy with xtrabackup;
10) Compression parameters for qpress are read from all
    suitable server groups in configuration file, as well as
    from the [sst] and [xtrabackup] groups;
11) Added a test that checks compression using qpress;
12) Checking for optional utilities is modified to work even
    if they implemented as built-in shell commands (unlikely
    on real systems, but more reliable).

mysql-test/suite/galera/disabled.def

@@ -43,11 +43,11 @@ galera_var_notify_cmd : MDEV-21905 Galera test galera_var_notify_cmd causes hang
 galera_var_reject_queries : assertion in inline_mysql_socket_send
 galera_var_replicate_myisam_on : MDEV-24062 Galera test failure on galera_var_replicate_myisam_on
 galera_var_retry_autocommit: MDEV-18181 Galera test failure on galera.galera_var_retry_autocommit
-galera_wan : MDEV-17259 Test failure on galera.galera_wan
+#galera_wan : MDEV-17259 Test failure on galera.galera_wan
 galera_wsrep_provider_unset_set: wsrep_provider is read-only for security reasons
 mysql-wsrep#198 : MDEV-24446: galera.mysql-wsrep#198 MTR failed: query 'reap' failed: 2000: Unknown MySQL error
 partition : MDEV-19958 Galera test failure on galera.partition
 pxc-421: wsrep_provider is read-only for security reasons
 query_cache: MDEV-15805 Test failure on galera.query_cache
-sql_log_bin : MDEV-21491 galera.sql_log_bin
+#sql_log_bin : MDEV-21491 galera.sql_log_bin
 versioning_trx_id: MDEV-18590: galera.versioning_trx_id: Test failure: mysqltest: Result content mismatch

mysql-test/suite/galera/include/have_qpress.inc

+#
+# suite.pm will make sure that all tests including this file
+# will be skipped as needed
+#

mysql-test/suite/galera/include/have_stunnel.inc

+#
+# suite.pm will make sure that all tests including this file
+# will be skipped as needed
+#

mysql-test/suite/galera/r/galera_sst_mariabackup_qpress.result

+SELECT 1;
+1
+1
+include/assert_grep.inc [Compressed qpress files found]

mysql-test/suite/galera/suite.pm

@@ -67,9 +67,16 @@ push @::global_suppressions,
      qr|WSREP: Wait for gtid returned error 3 while waiting for prior transactions to commit before setting position|,
    );
 
+sub which($) { return `sh -c "command -v $_[0]"` }
+
 sub skip_combinations {
   my %skip = ();
-  $skip{'include/have_mariabackup.inc'} = 'Need ss' unless `ss -V`;
+  $skip{'include/have_mariabackup.inc'} = 'Need socket statistics utility'
+            unless which("lsof") || which("sockstat") || which("ss");
+  $skip{'include/have_stunnel.inc'} = "Need 'stunnel' utility"
+            unless which("stunnel");
+  $skip{'include/have_qpress.inc'} = "Need 'qpress' utility"
+            unless which("qpress");
   %skip;
 }
 

mysql-test/suite/galera/t/galera_ist_rsync.cnf

@@ -10,4 +10,3 @@ wsrep_sync_wait=1
 [mysqld.2]
 wsrep_provider_options='base_port=@mysqld.2.#galera_port;pc.ignore_sb=true'
 wsrep_sync_wait=1
-

mysql-test/suite/galera/t/galera_log_bin.cnf

@@ -7,4 +7,3 @@ log-slave-updates
 [mysqld.2]
 log-bin
 log-slave-updates
-

mysql-test/suite/galera/t/galera_log_bin_opt.cnf

@@ -12,4 +12,4 @@ wsrep_provider_options='base_port=@mysqld.2.#galera_port;gcache.size=1;pc.ignore
 
 [sst]
 transferfmt=@ENV.MTR_GALERA_TFMT
-streamfmt=xbstream
+streamfmt=mbstream

mysql-test/suite/galera/t/galera_sst_mariabackup_qpress.cnf

+!include ../galera_2nodes.cnf
+
+[mysqld]
+wsrep_sst_method=mariabackup
+wsrep_sst_auth="root:"
+wsrep_debug=ON
+
+[sst]
+transferfmt=@ENV.MTR_GALERA_TFMT
+compress=quicklz
+compress-threads=2
+compress-chunk-size=32768

mysql-test/suite/galera/t/galera_sst_mariabackup_qpress.test

+#
+# This test checks that qpress compression works with mariabackup
+# Initial SST happens via mariabackup, so there is not much to do in the body
+# of the test
+#
+
+--source include/big_test.inc
+--source include/galera_cluster.inc
+--source include/have_innodb.inc
+--source include/have_mariabackup.inc
+--source include/have_qpress.inc
+
+SELECT 1;
+
+--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size';
+--source include/wait_condition.inc
+
+# Confirm that transfer was compressed using qpress algorithm
+--let $assert_text = Compressed qpress files found
+--let $assert_select = Compressed qpress files found
+--let $assert_count = 1
+--let $assert_file = $MYSQLTEST_VARDIR/log/mysqld.2.err
+--let $assert_only_after = CURRENT_TEST
+--source include/assert_grep.inc

mysql-test/suite/galera/t/galera_sst_rsync_encrypt_with_key.test

 --source include/big_test.inc
 --source include/galera_cluster.inc
 --source include/have_debug.inc
+--source include/have_stunnel.inc
 
 # Save original auto_increment_offset values.
 --let $node_1=node_1

mysql-test/suite/galera/t/galera_sst_rsync_encrypt_with_server.test

 --source include/big_test.inc
 --source include/galera_cluster.inc
 --source include/have_debug.inc
+--source include/have_stunnel.inc
 
 # Save original auto_increment_offset values.
 --let $node_1=node_1

mysql-test/suite/galera/t/galera_wan_restart_sst.test

@@ -4,7 +4,7 @@
 # We can not easily restart the first node, so instead we restart all the other nodes. MTR does not allow multiple nodes
 # to be down at the same time, so restarts are sequential.
 #
-# We can not test any of the actual WAN optimizations from inside MTR and no 
+# We can not test any of the actual WAN optimizations from inside MTR and no
 # status variables are provided. So we only check that simple replication works.
 #
 
@@ -98,7 +98,6 @@ INSERT INTO t1 VALUES (33);
 
 INSERT INTO t1 VALUES (341);
 
-
 #
 # Check all nodes
 #

mysql-test/suite/galera_3nodes/suite.pm

@@ -67,4 +67,13 @@ push @::global_suppressions,
      qr|WSREP: Wait for gtid returned error 3 while waiting for prior transactions to commit before setting position|,
    );
 
+sub which($) { return `sh -c "command -v $_[0]"` }
+
+sub skip_combinations {
+  my %skip = ();
+  $skip{'include/have_mariabackup.inc'} = 'Need ss'
+            unless which("lsof") || which("sockstat") || which("ss");
+  %skip;
+}
+
 bless { };

scripts/wsrep_sst_common.sh

-# Copyright (C) 2012-2015 Codership Oy
 # Copyright (C) 2017-2021 MariaDB
+# Copyright (C) 2012-2015 Codership Oy
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -822,14 +822,15 @@ wsrep_log_info()
 
 wsrep_cleanup_progress_file()
 {
-    [ -n "$SST_PROGRESS_FILE" ] && rm -f "$SST_PROGRESS_FILE" 2>/dev/null || true
+    [ -n "$SST_PROGRESS_FILE" -a \
+      -f "$SST_PROGRESS_FILE" ] && rm -f "$SST_PROGRESS_FILE" 2>/dev/null || true
 }
 
 wsrep_check_program()
 {
     local prog="$1"
     local cmd=$(command -v "$prog")
-    if [ ! -x "$cmd" ]; then
+    if [ -z "$cmd" ]; then
         echo "'$prog' not found in PATH"
         return 2 # no such file or directory
     fi
@@ -865,9 +866,9 @@ get_openssl()
     fi
     # Let's look for openssl:
     OPENSSL_BINARY="$(command -v openssl)"
-    if [ ! -x "$OPENSSL_BINARY" ]; then
+    if [ -z "$OPENSSL_BINARY" ]; then
         OPENSSL_BINARY='/usr/bin/openssl'
-        if [ ! -x "$OPENSSL_BINARY" ]; then
+        if [ -z "$OPENSSL_BINARY" ]; then
             OPENSSL_BINARY=""
         fi
     fi
@@ -899,14 +900,14 @@ is_local_ip()
     [ "$1" = "$(hostname -d)" ] && return 0
 
     local ip_util="$(command -v ip)"
-    if [ -x "$ip_util" ]; then
+    if [ -n "$ip_util" ]; then
         # ip address show ouput format is " inet[6] <address>/<mask>":
         "$ip_util" address show \
              | grep -E "^[[:space:]]*inet.? [^[:space:]]+/" -o \
              | grep -F " $1/" >/dev/null && return 0
     else
         local ifconfig_util="$(command -v ifconfig)"
-        if [ -x "$ifconfig_util" ]; then
+        if [ -n "$ifconfig_util" ]; then
             # ifconfig output format is " inet[6] <address> ...":
             "$ifconfig_util" \
                  | grep -E "^[[:space:]]*inet.? [^[:space:]]+ " -o \
@@ -923,16 +924,79 @@ check_sockets_utils()
     sockstat_available=0
     ss_available=0
 
-    [ -x "$(command -v lsof)" ] && lsof_available=1
-    [ -x "$(command -v sockstat)" ] && sockstat_available=1
-    [ -x "$(command -v ss)" ] && ss_available=1
+    [ -n "$(command -v lsof)" ] && lsof_available=1
+    [ -n "$(command -v sockstat)" ] && sockstat_available=1
+    [ -n "$(command -v ss)" ] && ss_available=1
 
     if [ $lsof_available -eq 0 -a \
          $sockstat_available -eq 0 -a \
          $ss_available -eq 0 ]
     then
-        wsrep_log_error "Neither lsof tool, nor ss or sockstat was found in " \
-                        "the PATH! Make sure you have it installed."
+        wsrep_log_error "Neither lsof, nor sockstat or ss tool was found in " \
+                        "the PATH. Make sure you have it installed."
         exit 2 # ENOENT
     fi
 }
+
+#
+# If the ssl_dhparams variable is already set, uses that as a source
+# of dh parameters for OpenSSL. Otherwise, looks for dhparams.pem in
+# the datadir, and creates it there if it can't find the file.
+#
+check_for_dhparams()
+{
+    if [ -z "$ssl_dhparams" ]; then
+        ssl_dhparams="$DATA/dhparams.pem"
+        if [ ! -r "$ssl_dhparams" ]; then
+            get_openssl
+            if [ -n "$OPENSSL_BINARY" ]; then
+                wsrep_log_info "Could not find dhparams file, creating $ssl_dhparams"
+                if ! "$OPENSSL_BINARY" dhparam -out "$ssl_dhparams" 2048 >/dev/null 2>&1
+                then
+                    wsrep_log_error "******** ERROR *****************************************"
+                    wsrep_log_error "* Could not create the dhparams.pem file with OpenSSL. *"
+                    wsrep_log_error "********************************************************"
+                    ssl_dhparams=""
+                fi
+            else
+                # Rollback: if openssl is not installed, then use
+                # the default parameters:
+                ssl_dhparams=""
+            fi
+        fi
+    fi
+}
+
+#
+# Compares two version strings.
+# The first parameter is the version to be checked;
+# The second parameter is the minimum version required;
+# Returns 1 (failure) if $1 >= $2, 0 (success) otherwise.
+#
+check_for_version()
+{
+    y1=${1#*.}
+    [ "$y1" = "$1" ] && y1=""
+    z1=${y1#*.}
+    [ "$z1" = "$y1" ] && z1=""
+    x1=${1%%.*}
+    y1=${y1%%.*}
+    z1=${z1%%.*}
+    [ -z "$y1" ] && y1=0
+    [ -z "$z1" ] && z1=0
+    y2=${2#*.}
+    [ "$y2" = "$2" ] && y2=""
+    z2=${y2#*.}
+    [ "$z2" = "$y2" ] && z2=""
+    x2=${2%%.*}
+    y2=${y2%%.*}
+    z2=${z2%%.*}
+    [ -z "$y2" ] && y2=0
+    [ -z "$z2" ] && z2=0
+    [ $x1 -lt $x2 ] && return 1
+    [ $x1 -gt $x2 ] && return 0
+    [ $y1 -lt $y2 ] && return 1
+    [ $y1 -gt $y2 ] && return 0
+    [ $z1 -lt $z2 ] && return 1
+    return 0
+}

scripts/wsrep_sst_mariabackup.sh

@@ -73,6 +73,12 @@ xtmpdir=""
 scomp=""
 sdecomp=""
 
+ssl_dhparams=""
+
+compress='none'
+compress_chunk=""
+compress_threads=""
+
 readonly SECRET_TAG="secret"
 
 # Required for backup locks
@@ -80,25 +86,24 @@ readonly SECRET_TAG="secret"
 # 5.6.21 PXC and later can't donate to an older joiner
 sst_ver=1
 
-if [ -x "$(command -v pv)" ] && pv --help | grep -qw -- '-F'; then
+if [ -n "$(command -v pv)" ] && pv --help | grep -qw -- '-F'; then
     pvopts="$pvopts $pvformat"
 fi
 pcmd="pv $pvopts"
 declare -a RC
 
-set +e
 MARIABACKUP_BIN="$(command -v mariabackup)"
 if [ ! -x "$MARIABACKUP_BIN" ]; then
     wsrep_log_error 'mariabackup binary not found in $PATH'
     exit 42
 fi
-set -e
 MBSTREAM_BIN=mbstream
 
 DATA="$WSREP_SST_OPT_DATA"
 INFO_FILE="xtrabackup_galera_info"
 IST_FILE="xtrabackup_ist"
 MAGIC_FILE="$DATA/$INFO_FILE"
+
 INNOAPPLYLOG="$DATA/mariabackup.prepare.log"
 INNOMOVELOG="$DATA/mariabackup.move.log"
 INNOBACKUPLOG="$DATA/mariabackup.backup.log"
@@ -184,7 +189,7 @@ get_keys()
             ecmd="$ecmd -k '$ekey'"
         fi
     elif [ "$eformat" = 'xbcrypt' ]; then
-        if [ ! -x "$(command -v xbcrypt)" ]; then
+        if [ -z "$(command -v xbcrypt)" ]; then
             wsrep_log_error "If encryption using the xbcrypt is enabled, " \
                             "then you need to install xbcrypt"
             exit 2
@@ -268,6 +273,22 @@ get_transfer()
             exit 2
         fi
 
+        # Determine the socat version
+        SOCAT_VERSION=$(socat -V 2>&1 | grep -m1 -oe '[0-9]\.[0-9][\.0-9]*')
+        if [ -z "$SOCAT_VERSION" ]; then
+            wsrep_log_error "******** FATAL ERROR ******************"
+            wsrep_log_error "* Cannot determine the socat version. *"
+            wsrep_log_error "***************************************"
+            exit 2
+        fi
+
+        if ! check_for_version "$SOCAT_VERSION" "1.7.3"; then
+            # socat versions < 1.7.3 will have 512-bit dhparams (too small)
+            # so create 2048-bit dhparams and send that as a parameter:
+            check_for_dhparams
+            sockopt=",dhparam='$ssl_dhparams'$sockopt"
+        fi
+
         if [ $encrypt -eq 2 ]; then
             wsrep_log_info "Using openssl based encryption with socat: with crt and pem"
             if [ -z "$tpem" -o -z "$tcert" ]; then
@@ -328,7 +349,7 @@ get_footprint()
 {
     pushd "$WSREP_SST_OPT_DATA" 1>/dev/null
     payload=$(find . -regex '.*\.ibd$\|.*\.MYI$\|.*\.MYD$\|.*ibdata1$' -type f -print0 | du --files0-from=- --block-size=1 -c | awk 'END { print $1 }')
-    if $MY_PRINT_DEFAULTS xtrabackup | grep -q -- "--compress"; then
+    if [ "$compress" != 'none' ]; then
         # QuickLZ has around 50% compression ratio
         # When compression/compaction used, the progress is only an approximate.
         payload=$(( payload*1/2 ))
@@ -340,7 +361,7 @@ get_footprint()
 
 adjust_progress()
 {
-    if [ ! -x "$(command -v pv)" ]; then
+    if [ -z "$(command -v pv)" ]; then
         wsrep_log_error "pv not found in path: $PATH"
         wsrep_log_error "Disabling all progress/rate-limiting"
         pcmd=""
@@ -447,6 +468,14 @@ read_cnf()
     if [ $ssyslog -ne -1 ]; then
         ssyslog=$(in_config 'mysqld_safe' 'syslog')
     fi
+
+    if [ "$WSREP_SST_OPT_ROLE" = 'donor' ]; then
+        compress=$(parse_cnf "$encgroups" 'compress' 'none')
+        if [ "$compress" != 'none' ]; then
+            compress_chunk=$(parse_cnf "$encgroups" 'compress-chunk-size')
+            compress_threads=$(parse_cnf "$encgroups" 'compress-threads')
+        fi
+    fi
 }
 
 get_stream()
@@ -480,7 +509,7 @@ get_proc()
 sig_joiner_cleanup()
 {
     wsrep_log_error "Removing $MAGIC_FILE file due to signal"
-    rm -f "$MAGIC_FILE"
+    [ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE"
 }
 
 cleanup_joiner()
@@ -540,7 +569,7 @@ cleanup_donor()
         fi
     fi
 
-    rm -f "$DATA/$IST_FILE" || true
+    [ -f "$DATA/$IST_FILE" ] && rm -f "$DATA/$IST_FILE"
 
     if [ -n "$progress" -a -p "$progress" ]; then
         wsrep_log_info "Cleaning up fifo file $progress"
@@ -549,13 +578,8 @@ cleanup_donor()
 
     wsrep_log_info "Cleaning up temporary directories"
 
-    if [ -n "$xtmpdir" ]; then
-       [ -d "$xtmpdir" ] && rm -rf "$xtmpdir" || true
-    fi
-
-    if [ -n "$itmpdir" ]; then
-       [ -d "$itmpdir" ] && rm -rf "$itmpdir" || true
-    fi
+    [ -n "$xtmpdir" -a -d "$xtmpdir" ] && rm -rf "$xtmpdir" || true
+    [ -n "$itmpdir" -a -d "$itmpdir" ] && rm -rf "$itmpdir" || true
 
     # Final cleanup
     pgid=$(ps -o pgid= $$ | grep -o '[0-9]*')
@@ -682,12 +706,9 @@ recv_joiner()
         return
     fi
 
-    pushd "$dir" 1>/dev/null
-    set +e
-
     local ltcmd="$tcmd"
     if [ $tmt -gt 0 ]; then
-        if [ -x "$(command -v timeout)" ]; then
+        if [ -n "$(command -v timeout)" ]; then
             if timeout --help | grep -qw -- '-k'; then
                 ltcmd="timeout -k $(( tmt+10 )) $tmt $tcmd"
             else
@@ -696,6 +717,9 @@ recv_joiner()
         fi
     fi
 
+    pushd "$dir" 1>/dev/null
+    set +e
+
     if [ $wait -ne 0 ]; then
         wait_for_listen "$SST_PORT" "$ADDR" "$MODULE" &
     fi
@@ -781,7 +805,7 @@ monitor_process()
 
 wsrep_check_programs "$MARIABACKUP_BIN"
 
-rm -f "$MAGIC_FILE"
+[ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE"
 
 if [ "$WSREP_SST_OPT_ROLE" != 'joiner' -a "$WSREP_SST_OPT_ROLE" != 'donor' ]; then
     wsrep_log_error "Invalid role ${WSREP_SST_OPT_ROLE}"
@@ -795,13 +819,6 @@ if "$MARIABACKUP_BIN" --help 2>/dev/null | grep -qw -- '--version-check'; then
     disver='--no-version-check'
 fi
 
-iopts="$iopts --databases-exclude='lost+found'"
-
-if [ ${FORCE_FTWRL:-0} -eq 1 ]; then
-    wsrep_log_info "Forcing FTWRL due to environment variable FORCE_FTWRL equal to $FORCE_FTWRL"
-    iopts="$iopts --no-backup-locks"
-fi
-
 # if no command line argument and INNODB_DATA_HOME_DIR environment variable
 # is not set, try to get it from my.cnf:
 if [ -z "$INNODB_DATA_HOME_DIR" ]; then
@@ -810,19 +827,19 @@ fi
 
 OLD_PWD="$(pwd)"
 
+cd "$WSREP_SST_OPT_DATA"
 if [ -n "$INNODB_DATA_HOME_DIR" ]; then
     # handle both relative and absolute paths
-    INNODB_DATA_HOME_DIR=$(cd "$DATA"; mkdir -p "$INNODB_DATA_HOME_DIR"; cd "$INNODB_DATA_HOME_DIR"; pwd -P)
-else
-    # default to datadir
-    INNODB_DATA_HOME_DIR=$(cd "$DATA"; pwd -P)
+    [ ! -d "$INNODB_DATA_HOME_DIR" ] && mkdir -p "$INNODB_DATA_HOME_DIR"
+    cd "$INNODB_DATA_HOME_DIR"
 fi
+INNODB_DATA_HOME_DIR=$(pwd -P)
 
 cd "$OLD_PWD"
 
 if [ $ssyslog -eq 1 ]; then
 
-    if [ -x "$(command -v logger)" ]; then
+    if [ -n "$(command -v logger)" ]; then
         wsrep_log_info "Logging all stderr of SST/mariabackup to syslog"
 
         exec 2> >(logger -p daemon.err -t ${ssystag}wsrep-sst-$WSREP_SST_OPT_ROLE)
@@ -850,10 +867,8 @@ if [ $sstlogarchive -eq 1 ]
 then
     ARCHIVETIMESTAMP=$(date "+%Y.%m.%d-%H.%M.%S.%N")
 
-    if [ -n "$sstlogarchivedir" ]
-    then
-        if [ ! -d "$sstlogarchivedir" ]
-        then
+    if [ -n "$sstlogarchivedir" ]; then
+        if [ ! -d "$sstlogarchivedir" ]; then
             mkdir -p "$sstlogarchivedir"
         fi
     fi
@@ -927,7 +942,6 @@ then
 
     if [ $WSREP_SST_OPT_BYPASS -eq 0 ]
     then
-        usrst=0
         if [ -z "$sst_ver" ]; then
             wsrep_log_error "Upgrade joiner to 5.6.21 or higher for backup locks support"
             wsrep_log_error "The joiner is not supported for this version of donor"
@@ -944,6 +958,7 @@ then
         itmpdir="$(mktemp -d)"
         wsrep_log_info "Using $itmpdir as mariabackup temporary directory"
 
+        usrst=0
         if [ -n "$WSREP_SST_OPT_USER" ]; then
            INNOEXTRA="$INNOEXTRA --user='$WSREP_SST_OPT_USER'"
            usrst=1
@@ -1008,6 +1023,25 @@ then
             tcmd="$ecmd | $tcmd"
         fi
 
+        iopts="$iopts --databases-exclude='lost+found'"
+
+        if [ ${FORCE_FTWRL:-0} -eq 1 ]; then
+           wsrep_log_info "Forcing FTWRL due to environment variable FORCE_FTWRL equal to $FORCE_FTWRL"
+           iopts="$iopts --no-backup-locks"
+        fi
+
+        # if compression is enabled for backup files, then add the
+        # appropriate options to the mariabackup command line:
+        if [ "$compress" != 'none' ]; then
+            iopts="$iopts --compress${compress:+=$compress}"
+            if [ -n "$compress_threads" ]; then
+                iopts="$iopts --compress-threads=$compress_threads"
+            fi
+            if [ -n "$compress_chunk" ]; then
+                iopts="$iopts --compress-chunk-size=$compress_chunk"
+            fi
+        fi
+
         setup_commands
         set +e
         timeit "$stagemsg-SST" "$INNOBACKUP | $tcmd; RC=( "\${PIPESTATUS[@]}" )"
@@ -1082,10 +1116,12 @@ then
 
     MODULE="xtrabackup_sst"
 
-    rm -f "$DATA/$IST_FILE"
+    [ -f "$DATA/$IST_FILE" ] && rm -f "$DATA/$IST_FILE"
 
     # May need xtrabackup_checkpoints later on
-    rm -f "$DATA/xtrabackup_binary" "$DATA/xtrabackup_galera_info" "$DATA/ib_logfile0"
+    [ -f "$DATA/xtrabackup_binary"      ] && rm -f "$DATA/xtrabackup_binary"
+    [ -f "$DATA/xtrabackup_galera_info" ] && rm -f "$DATA/xtrabackup_galera_info"
+    [ -f "$DATA/ib_logfile0"            ] && rm -f "$DATA/ib_logfile0"
 
     ADDR="$WSREP_SST_OPT_ADDR"
 
@@ -1185,8 +1221,6 @@ then
         wsrep_log_info "Waiting for SST streaming to complete!"
         monitor_process $jpid
 
-        get_proc
-
         if [ ! -s "$DATA/xtrabackup_checkpoints" ]; then
             wsrep_log_error "xtrabackup_checkpoints missing, failed mariabackup/SST on donor"
             exit 2
@@ -1203,11 +1237,15 @@ then
         if [ -n "$qpfiles" ]; then
             wsrep_log_info "Compressed qpress files found"
 
-            if [ ! -x "$(command -v qpress)" ]; then
-                wsrep_log_error "qpress not found in path: $PATH"
+            if [ -z "$(command -v qpress)" ]; then
+                wsrep_log_error "qpress utility not found in the path"
                 exit 22
             fi
 
+            get_proc
+
+            dcmd="xargs -n 2 qpress -dT$nproc"
+
             if [ -n "$progress" ] && pv --help | grep -qw -- '--line-mode'; then
                 count=$(find "$DATA" -type f -name '*.qp' | wc -l)
                 count=$(( count*2 ))
@@ -1217,9 +1255,7 @@ then
                 fi
                 pcmd="pv $pvopts"
                 adjust_progress
-                dcmd="$pcmd | xargs -n 2 qpress -T${nproc}d"
-            else
-                dcmd="xargs -n 2 qpress -T${nproc}d"
+                dcmd="$pcmd | $dcmd"
             fi
 
             # Decompress the qpress files
@@ -1267,6 +1303,7 @@ then
         MAGIC_FILE="$TDATA/$INFO_FILE"
         wsrep_log_info "Moving the backup to ${TDATA}"
         timeit "mariabackup move stage" "$INNOMOVE"
+
         if [ $? -eq 0 ]; then
             wsrep_log_info "Move successful, removing ${DATA}"
             rm -rf "$DATA"

scripts/wsrep_sst_mysqldump.sh

@@ -45,7 +45,12 @@ then
     exit $EINVAL
 fi
 
-[ -n "$WSREP_SST_OPT_USER" ] && AUTH="-u$WSREP_SST_OPT_USER" || AUTH=
+AUTH=""
+usrst=0
+if [ -n "$WSREP_SST_OPT_USER" ]; then
+    AUTH="-u$WSREP_SST_OPT_USER"
+    usrst=1
+fi
 
 # Refs https://github.com/codership/mysql-wsrep/issues/141
 # Passing password in MYSQL_PWD environment variable is considered
@@ -56,9 +61,14 @@ fi
 # whereas (at least on Linux) unprivileged user can't see process environment
 # that he does not own. So while it may be not secure in the NSA sense of the
 # word, it is arguably more secure than passing password on the command line.
-[ -n "$WSREP_SST_OPT_PSWD" ] && export MYSQL_PWD="$WSREP_SST_OPT_PSWD"
+if [ -n "$WSREP_SST_OPT_PSWD" ]; then
+    export MYSQL_PWD="$WSREP_SST_OPT_PSWD"
+elif [ $usrst -eq 1 ]; then
+    # Empty password, used for testing, debugging etc.
+    unset MYSQL_PWD
+fi
 
-STOP_WSREP="SET wsrep_on=OFF;"
+STOP_WSREP='SET wsrep_on=OFF;'
 
 # mysqldump cannot restore CSV tables, fix this issue
 CSV_TABLES_FIX="
@@ -99,15 +109,15 @@ MYSQL="$MYSQL_CLIENT $WSREP_SST_OPT_CONF "\
 
 # Check if binary logging is enabled on the joiner node.
 # Note: SELECT cannot be used at this point.
-LOG_BIN=$(echo "set statement wsrep_sync_wait=0 for SHOW VARIABLES LIKE 'log_bin'" | $MYSQL |\
+LOG_BIN=$(echo "set statement wsrep_sync_wait=0 for SHOW VARIABLES LIKE 'log_bin'" | $MYSQL | \
 tail -1 | awk -F ' ' '{ print $2 }')
 
 # Check the joiner node's server version.
-SERVER_VERSION=$(echo "set statement wsrep_sync_wait=0 for SHOW VARIABLES LIKE 'version'" | $MYSQL |\
+SERVER_VERSION=$(echo "set statement wsrep_sync_wait=0 for SHOW VARIABLES LIKE 'version'" | $MYSQL | \
 tail -1 | awk -F ' ' '{ print $2 }')
 
 # Retrieve the donor's @@global.gtid_binlog_state.
-GTID_BINLOG_STATE=$(echo "SHOW GLOBAL VARIABLES LIKE 'gtid_binlog_state'" | $MYSQL |\
+GTID_BINLOG_STATE=$(echo "SHOW GLOBAL VARIABLES LIKE 'gtid_binlog_state'" | $MYSQL | \
 tail -1 | awk -F ' ' '{ print $2 }')
 
 RESET_MASTER=""
@@ -115,7 +125,7 @@ SET_GTID_BINLOG_STATE=""
 SQL_LOG_BIN_OFF=""
 
 # Safety check
-if [ "${SERVER_VERSION%%.*}" != '5' ]
+if [ ${SERVER_VERSION%%.*} -gt 5 ]
 then
     # If binary logging is enabled on the joiner node, we need to copy donor's
     # gtid_binlog_state to joiner. In order to do that, a RESET MASTER must be
@@ -135,29 +145,29 @@ MYSQLDUMP="$MYSQLDUMP $WSREP_SST_OPT_CONF $AUTH -S$WSREP_SST_OPT_SOCKET \
 --disable-keys --extended-insert --skip-lock-tables --quick --set-charset \
 --skip-comments --flush-privileges --all-databases --events"
 
-# need to disable logging when loading the dump
-# reason is that dump contains ALTER TABLE for log tables, and
-# this causes an error if logging is enabled
-GENERAL_LOG_OPT=`$MYSQL --skip-column-names -e "$STOP_WSREP SELECT @@GENERAL_LOG"`
-SLOW_LOG_OPT=`$MYSQL --skip-column-names -e "$STOP_WSREP SELECT @@SLOW_QUERY_LOG"`
-$MYSQL -e "$STOP_WSREP SET GLOBAL GENERAL_LOG=OFF"
-$MYSQL -e "$STOP_WSREP SET GLOBAL SLOW_QUERY_LOG=OFF"
-
-# commands to restore log settings
-RESTORE_GENERAL_LOG="SET GLOBAL GENERAL_LOG=$GENERAL_LOG_OPT;"
-RESTORE_SLOW_QUERY_LOG="SET GLOBAL SLOW_QUERY_LOG=$SLOW_LOG_OPT;"
-
 if [ $WSREP_SST_OPT_BYPASS -eq 0 ]
 then
-    (echo $STOP_WSREP && echo $RESET_MASTER && \
-     echo $SET_GTID_BINLOG_STATE && echo $SQL_LOG_BIN_OFF && \
-     echo $STOP_WSREP && $MYSQLDUMP && echo $CSV_TABLES_FIX && \
-     echo $RESTORE_GENERAL_LOG && echo $RESTORE_SLOW_QUERY_LOG && \
-     echo $SET_START_POSITION && echo $SET_WSREP_GTID_DOMAIN_ID \
+    # need to disable logging when loading the dump
+    # reason is that dump contains ALTER TABLE for log tables, and
+    # this causes an error if logging is enabled
+    GENERAL_LOG_OPT=$($MYSQL --skip-column-names -e "$STOP_WSREP SELECT @@GENERAL_LOG")
+    SLOW_LOG_OPT=$($MYSQL --skip-column-names -e "$STOP_WSREP SELECT @@SLOW_QUERY_LOG")
+
+    LOG_OFF="SET GLOBAL GENERAL_LOG=OFF; SET GLOBAL SLOW_QUERY_LOG=OFF;"
+
+    # commands to restore log settings
+    RESTORE_GENERAL_LOG="SET GLOBAL GENERAL_LOG=$GENERAL_LOG_OPT;"
+    RESTORE_SLOW_QUERY_LOG="SET GLOBAL SLOW_QUERY_LOG=$SLOW_LOG_OPT;"
+
+    (echo "$STOP_WSREP" && echo "$LOG_OFF" && echo "$RESET_MASTER" && \
+     echo "$SET_GTID_BINLOG_STATE" && echo "$SQL_LOG_BIN_OFF" && \
+     echo "$STOP_WSREP" && $MYSQLDUMP && echo "$CSV_TABLES_FIX" && \
+     echo "$RESTORE_GENERAL_LOG" && echo "$RESTORE_SLOW_QUERY_LOG" && \
+     echo "$SET_START_POSITION" && echo "$SET_WSREP_GTID_DOMAIN_ID" \
      || echo "SST failed to complete;") | $MYSQL
 else
     wsrep_log_info "Bypassing state dump."
-    echo $SET_START_POSITION | $MYSQL
+    echo "$SET_START_POSITION" | $MYSQL
 fi
 
 #

scripts/wsrep_sst_rsync.sh

@@ -40,13 +40,12 @@ cleanup_joiner()
     [ "0" != "$RSYNC_REAL_PID" ]            && \
     kill $RSYNC_REAL_PID                    && \
     sleep 0.5                               && \
-    kill -9 $RSYNC_REAL_PID >/dev/null 2>&1 || \
-    :
-    rm -rf "$RSYNC_CONF"
-    rm -f "$STUNNEL_CONF"
-    rm -f "$STUNNEL_PID"
-    rm -rf "$MAGIC_FILE"
-    rm -rf "$RSYNC_PID"
+    kill -9 $RSYNC_REAL_PID >/dev/null 2>&1 || :
+    [ -f "$RSYNC_CONF"   ] && rm -f "$RSYNC_CONF"
+    [ -f "$STUNNEL_CONF" ] && rm -f "$STUNNEL_CONF"
+    [ -f "$STUNNEL_PID"  ] && rm -f "$STUNNEL_PID"
+    [ -f "$MAGIC_FILE"   ] && rm -f "$MAGIC_FILE"
+    [ -f "$RSYNC_PID"    ] && rm -f "$RSYNC_PID"
     wsrep_log_info "Joiner cleanup done."
     if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then
         wsrep_cleanup_progress_file
@@ -125,17 +124,13 @@ check_pid_and_port()
 }
 
 STUNNEL_CONF="$WSREP_SST_OPT_DATA/stunnel.conf"
-rm -f "$STUNNEL_CONF"
 
 STUNNEL_PID="$WSREP_SST_OPT_DATA/stunnel.pid"
-rm -f "$STUNNEL_PID"
 
 MAGIC_FILE="$WSREP_SST_OPT_DATA/rsync_sst_complete"
-rm -rf "$MAGIC_FILE"
 
 BINLOG_TAR_FILE="$WSREP_SST_OPT_DATA/wsrep_sst_binlog.tar"
 BINLOG_N_FILES=1
-rm -f "$BINLOG_TAR_FILE" || :
 
 get_binlog
 
@@ -154,13 +149,13 @@ OLD_PWD="$(pwd)"
 
 WSREP_LOG_DIR="$INNODB_LOG_GROUP_HOME"
 
+cd "$WSREP_SST_OPT_DATA"
 if [ -n "$WSREP_LOG_DIR" ]; then
     # handle both relative and absolute paths
-    WSREP_LOG_DIR=$(cd "$WSREP_SST_OPT_DATA"; mkdir -p "$WSREP_LOG_DIR"; cd "$WSREP_LOG_DIR"; pwd -P)
-else
-    # default to datadir
-    WSREP_LOG_DIR=$(cd "$WSREP_SST_OPT_DATA"; pwd -P)
+    [ ! -d "$WSREP_LOG_DIR" ] && mkdir -p "$WSREP_LOG_DIR"
+    cd "$WSREP_LOG_DIR"
 fi
+WSREP_LOG_DIR=$(pwd -P)
 
 cd "$OLD_PWD"
 
@@ -170,13 +165,13 @@ if [ -z "$INNODB_DATA_HOME_DIR" ]; then
     INNODB_DATA_HOME_DIR=$(parse_cnf '--mysqld' 'innodb-data-home-dir')
 fi
 
+cd "$WSREP_SST_OPT_DATA"
 if [ -n "$INNODB_DATA_HOME_DIR" ]; then
     # handle both relative and absolute paths
-    INNODB_DATA_HOME_DIR=$(cd "$WSREP_SST_OPT_DATA"; mkdir -p "$INNODB_DATA_HOME_DIR"; cd "$INNODB_DATA_HOME_DIR"; pwd -P)
-else
-    # default to datadir
-    INNODB_DATA_HOME_DIR=$(cd "$WSREP_SST_OPT_DATA"; pwd -P)
+    [ ! -d "$INNODB_DATA_HOME_DIR" ] && mkdir -p "$INNODB_DATA_HOME_DIR"
+    cd "$INNODB_DATA_HOME_DIR"
 fi
+INNODB_DATA_HOME_DIR=$(pwd -P)
 
 cd "$OLD_PWD"
 
@@ -185,13 +180,13 @@ if [ -z "$INNODB_UNDO_DIR" ]; then
     INNODB_UNDO_DIR=$(parse_cnf '--mysqld' 'innodb-undo-directory')
 fi
 
+cd "$WSREP_SST_OPT_DATA"
 if [ -n "$INNODB_UNDO_DIR" ]; then
     # handle both relative and absolute paths
-    INNODB_UNDO_DIR=$(cd "$WSREP_SST_OPT_DATA"; mkdir -p "$INNODB_UNDO_DIR"; cd "$INNODB_UNDO_DIR"; pwd -P)
-else
-    # default to datadir
-    INNODB_UNDO_DIR=$(cd "$WSREP_SST_OPT_DATA"; pwd -P)
+    [ ! -d "$INNODB_UNDO_DIR" ] && mkdir -p "$INNODB_UNDO_DIR"
+    cd "$INNODB_UNDO_DIR"
 fi
+INNODB_UNDO_DIR=$(pwd -P)
 
 cd "$OLD_PWD"
 
@@ -239,7 +234,7 @@ if [ -z "$SSLMODE" ]; then
     # Implicit verification if CA is set and the SSL mode
     # is not specified by user:
     if [ -n "$SSTCA" ]; then
-        if [ -x "$(command -v stunnel)" ]; then
+        if [ -n "$(command -v stunnel)" ]; then
             SSLMODE='VERIFY_CA'
         fi
     # Require SSL by default if SSL key and cert are present:
@@ -260,28 +255,36 @@ then
     case "$SSLMODE" in
     'VERIFY_IDENTITY')
         VERIFY_OPT='verifyPeer = yes'
+        CHECK_OPT=""
         ;;
     'VERIFY_CA')
         VERIFY_OPT='verifyChain = yes'
+        if is_local_ip "$WSREP_SST_OPT_HOST_UNESCAPED"; then
+            CHECK_OPT='checkHost = localhost'
+        else
+            CHECK_OPT='checkHost = $WSREP_SST_OPT_HOST_UNESCAPED'
+        fi
         ;;
     *)
         wsrep_log_error "Unrecognized ssl-mode option: '$SSLMODE'"
         exit 22 # EINVAL
     esac
-    if [ -z "$CAFILE_OPT" ]
-    then
-        wsrep_log_error "Can't have ssl-mode=$SSLMODE without CA file"
+    if [ -z "$CAFILE_OPT" ]; then
+        wsrep_log_error "Can't have ssl-mode='$SSLMODE' without CA file"
         exit 22 # EINVAL
     fi
 else
     VERIFY_OPT=""
+    CHECK_OPT=""
 fi
 
 STUNNEL=""
-if [ -n "$SSLMODE" -a "$SSLMODE" != 'DISABLED' ] && wsrep_check_programs stunnel
-then
-    wsrep_log_info "Using stunnel for SSL encryption: CAfile: '$SSTCA', SSLMODE: '$SSLMODE'"
-    STUNNEL="stunnel $STUNNEL_CONF"
+if [ -n "$SSLMODE" -a "$SSLMODE" != 'DISABLED' ]; then
+    STUNNEL_BIN="$(command -v stunnel)"
+    if [ -n "$STUNNEL_BIN" ]; then
+        wsrep_log_info "Using stunnel for SSL encryption: CAfile: '$SSTCA', ssl-mode='$SSLMODE'"
+        STUNNEL="$STUNNEL_BIN $STUNNEL_CONF"
+    fi
 fi
 
 readonly SECRET_TAG="secret"
@@ -289,7 +292,13 @@ readonly SECRET_TAG="secret"
 if [ "$WSREP_SST_OPT_ROLE" = 'donor' ]
 then
 
-cat << EOF > "$STUNNEL_CONF"
+    [ -f "$MAGIC_FILE"      ] && rm -f "$MAGIC_FILE"
+    [ -f "$BINLOG_TAR_FILE" ] && rm -f "$BINLOG_TAR_FILE"
+
+    if [ -n "$STUNNEL" ]
+    then
+        [ -f "$STUNNEL_PID" ] && rm -f "$STUNNEL_PID"
+        cat << EOF > "$STUNNEL_CONF"
 key = $SSTKEY
 cert = $SSTCERT
 ${CAFILE_OPT}
@@ -300,7 +309,9 @@ client = yes
 connect = $WSREP_SST_OPT_HOST_UNESCAPED:$WSREP_SST_OPT_PORT
 TIMEOUTclose = 0
 ${VERIFY_OPT}
+${CHECK_OPT}
 EOF
+    fi
 
     if [ $WSREP_SST_OPT_BYPASS -eq 0 ]
     then
@@ -366,7 +377,7 @@ EOF
 
         # first, the normal directories, so that we can detect incompatible protocol
         RC=0
-        eval rsync ${STUNNEL:+--rsh=\"$STUNNEL\"} \
+        eval rsync ${STUNNEL:+"'--rsh=$STUNNEL'"} \
               --owner --group --perms --links --specials \
               --ignore-times --inplace --dirs --delete --quiet \
               $WHOLE_FILE_OPT $FILTER "'$WSREP_SST_OPT_DATA/'" \
@@ -449,7 +460,7 @@ EOF
 
     fi
 
-    echo "continue" # now server can resume updating data
+    echo 'continue' # now server can resume updating data
 
     echo "$STATE" > "$MAGIC_FILE"
 
@@ -487,7 +498,10 @@ then
         wsrep_log_error "rsync daemon already running."
         exit 114 # EALREADY
     fi
-    rm -rf "$RSYNC_PID"
+
+    [ -f "$RSYNC_PID"       ] && rm -f "$RSYNC_PID"
+    [ -f "$MAGIC_FILE"      ] && rm -f "$MAGIC_FILE"
+    [ -f "$BINLOG_TAR_FILE" ] && rm -f "$BINLOG_TAR_FILE"
 
     ADDR="$WSREP_SST_OPT_ADDR"
     RSYNC_PORT="$WSREP_SST_OPT_PORT"
@@ -541,20 +555,40 @@ EOF
         rsync --daemon --no-detach --port "$RSYNC_PORT" --config "$RSYNC_CONF" $RSYNC_EXTRA_ARGS &
         RSYNC_REAL_PID=$!
     else
-        cat << EOF > "$STUNNEL_CONF"
+        [ -f "$STUNNEL_PID" ] && rm -f "$STUNNEL_PID"
+        # Let's check if the path to the config file contains a space?
+        if [ "${RSYNC_CONF#* }" = "$RSYNC_CONF" ]; then
+            cat << EOF > "$STUNNEL_CONF"
 key = $SSTKEY
 cert = $SSTCERT
 ${CAFILE_OPT}
 foreground = yes
 pid = $STUNNEL_PID
 debug = warning
-debug = 6
 client = no
 [rsync]
 accept = $STUNNEL_ACCEPT
 exec = $(command -v rsync)
 execargs = rsync --server --daemon --config=$RSYNC_CONF .
 EOF
+        else
+            # The path contains a space, so we will run it via
+            # shell with "eval" command:
+            export RSYNC_CMD="eval $(command -v rsync) --server --daemon --config='$RSYNC_CONF' ."
+            cat << EOF > "$STUNNEL_CONF"
+key = $SSTKEY
+cert = $SSTCERT
+${CAFILE_OPT}
+foreground = yes
+pid = $STUNNEL_PID
+debug = warning
+client = no
+[rsync]
+accept = $STUNNEL_ACCEPT
+exec = $SHELL
+execargs = $SHELL -c \$RSYNC_CMD
+EOF
+        fi
         stunnel "$STUNNEL_CONF" &
         RSYNC_REAL_PID=$!
         RSYNC_PID="$STUNNEL_PID"
@@ -655,6 +689,6 @@ else
     exit 22 # EINVAL
 fi
 
-rm -f "$BINLOG_TAR_FILE" || :
+[ -f "$BINLOG_TAR_FILE" ] && rm -f "$BINLOG_TAR_FILE"
 
 exit 0