MDEV-25818: RSYNC SST failed due to busy port

This commit reduces the likelihood of getting a busy port on
quick restarts with rsync SST (problem MDEV-25818) and fixes
a number of other flaws in SST scripts, adds new functionality,
and also synchronizes the xtrabackup-v2 script with the
mariabackup script (the latter applies only to the 10.2 branch):

 1) SST via rsync: rsync and stunnel does not always get the right
    time to complete by correctly handling SIGTERM. These utilities
    are now given more time to complete normally (via normal SIGTERM
    processing) before we move on to using "kill -9";
 2) SST via rsync: attempts to terminate an rsync or stunnel process
    (via "kill" utility) are only made if it did not terminated on
    its own;
 3) SST via rsync: if a combination of stunnel and rsync is used,
    then we need to wait for both utilities to finish or stop, not
    just one of them;
 4) The config file and pid file for stunnel are now deleted after
    successful completion of SST on the donor node;
 5) The configs and pid files from rsync and stunnel should not be
    deleted unless these utilities succeed (or are sucessfully
    terminated) on the joiner node;
 6) The configs and pid files now excluded from transfer via rsync;
 7) Spaces in paths are now valid for config files as well (when
    used with SST via rsync or mariabackup / xtrabackup[-v2]);
 8) SST via mariabackup: added preliminary verification of keys and
    certificates that are used when establishing a connection using
    SSL (to avoid long timeouts and improve diagnostics) - by analogy
    with how it is done for the xtrabackup-v2 (plus check for CA file),
    while that check is skipped if the user does not have openssl
    installed (or does not have diff utility);
 9) Added backup-threads=<n> configuration option which adds
    "--parallel=<n>" for mariabackup / xtrabackup at backup and
    move-back stages;
10) Added encrypt-threads and encrypt-chunk-size configuration
    options for xbcrypt management (when xbcrypt is used);
11) Small optimization: checking the socat version and adding
    a file with parameters for 2048-bit Diffie-Hellman (if necessary)
    is done only if the user has not specified "dhparam=" in the
    "sockopt" option value;
12) SST via rsync now supports "backup-threads" configuration option
    (in server-related sections or in the "[sst]");
13) Determining the number of available processors is now supported
    for FreeBSD + mariabackup/xtrabackup: before that we might have
    problems with "--compact" (rebuild indexes) or qpress on FreeBSD;
14) The check_pid() function should not raise an error state in
    the rare cases when the pid file was created, but it is empty,
    or if it is deleted right during the check, or when zero is read
    from the pid file;
15) Iproved templates that are used to check if a requested socket
    is "listening" when using the ss utility;
16) Shortened some other templates for socket state utilities;
17) Temporary files created by mariabackup / xtrabackup are moved
    to a separate subdirectory inside tmpdir (so they don't get
    mixed with other temporary files, which can make debugging
    more difficult);
18) 10.2 only: the script for SST via xtrabackup-v2 has been brought
    in full compliance with all the bugfixes made for mariabackup (as
    it previously contained many flaws compared to the updated script
    for mariabackup).

mysql-test/suite/galera/t/galera_autoinc_sst_xtrabackup.cnf

@@ -10,4 +10,3 @@ wsrep_provider_options='base_port=@mysqld.1.#galera_port;gcache.size=1;pc.ignore
 
 [mysqld.2]
 wsrep_provider_options='base_port=@mysqld.2.#galera_port;gcache.size=1;pc.ignore_sb=true'
-

mysql-test/suite/galera/t/galera_ist_mariabackup.cnf

@@ -9,4 +9,3 @@ wsrep_provider_options='base_port=@mysqld.1.#galera_port;pc.ignore_sb=true'
 
 [mysqld.2]
 wsrep_provider_options='base_port=@mysqld.2.#galera_port;pc.ignore_sb=true'
-

mysql-test/suite/galera/t/galera_ist_mariabackup_innodb_flush_logs.cnf

@@ -11,4 +11,3 @@ wsrep_provider_options='base_port=@mysqld.1.#galera_port;pc.ignore_sb=true'
 
 [mysqld.2]
 wsrep_provider_options='base_port=@mysqld.2.#galera_port;pc.ignore_sb=true'
-

mysql-test/suite/galera/t/galera_ist_mysqldump.cnf

@@ -8,5 +8,3 @@ wsrep_provider_options='base_port=@mysqld.1.#galera_port;pc.ignore_sb=true'
 
 [mysqld.2]
 wsrep_provider_options='base_port=@mysqld.2.#galera_port;pc.ignore_sb=true'
-
-

mysql-test/suite/galera/t/galera_ist_xtrabackup-v2.cnf

@@ -5,10 +5,8 @@ wsrep_sst_method=xtrabackup-v2
 wsrep_sst_auth=root:
 innodb_safe_truncate=OFF
 
-
 [mysqld.1]
 wsrep_provider_options='base_port=@mysqld.1.#galera_port;pc.ignore_sb=true'
 
 [mysqld.2]
 wsrep_provider_options='base_port=@mysqld.2.#galera_port;pc.ignore_sb=true'
-

mysql-test/suite/galera/t/galera_sst_mariabackup.cnf

@@ -13,4 +13,4 @@ wsrep_provider_options='base_port=@mysqld.2.#galera_port;gcache.size=1;pc.ignore
 
 [sst]
 transferfmt=@ENV.MTR_GALERA_TFMT
-streamfmt=xbstream
+streamfmt=mbstream

mysql-test/suite/galera/t/galera_sst_mariabackup_qpress.cnf

@@ -10,3 +10,4 @@ transferfmt=@ENV.MTR_GALERA_TFMT
 compress=quicklz
 compress-threads=2
 compress-chunk-size=32768
+backup-threads=2

mysql-test/suite/galera/t/galera_sst_mariabackup_table_options.cnf

@@ -15,4 +15,4 @@ wsrep_provider_options='base_port=@mysqld.2.#galera_port;gcache.size=1;pc.ignore
 
 [sst]
 transferfmt=@ENV.MTR_GALERA_TFMT
-streamfmt=xbstream
+streamfmt=mbstream

mysql-test/suite/galera/t/galera_sst_rsync.cnf

@@ -8,4 +8,3 @@ wsrep_provider_options='base_port=@mysqld.1.#galera_port;gcache.size=1;pc.ignore
 
 [mysqld.2]
 wsrep_provider_options='base_port=@mysqld.2.#galera_port;gcache.size=1;pc.ignore_sb=true'
-

mysql-test/suite/galera/t/galera_sst_rsync2.cnf

@@ -12,4 +12,3 @@ log_bin_index=@ENV.MYSQLTEST_VARDIR/tmp/server1_binlog_index.index
 wsrep_provider_options='base_port=@mysqld.2.#galera_port;gcache.size=1;pc.ignore_sb=true'
 log_bin=@ENV.MYSQLTEST_VARDIR/server2_binlog
 log_bin_index=@ENV.MYSQLTEST_VARDIR/tmp/server2_binlog_index.index
-

mysql-test/suite/galera/t/galera_sst_rsync_data_dir.cnf

@@ -9,3 +9,6 @@ wsrep_provider_options='base_port=@mysqld.1.#galera_port;gcache.size=1;pc.ignore
 [mysqld.2]
 innodb_data_home_dir=@ENV.MYSQL_TMP_DIR/rsync_test_2
 wsrep_provider_options='base_port=@mysqld.2.#galera_port;gcache.size=1;pc.ignore_sb=true'
+
+[sst]
+backup_threads=2

scripts/wsrep_sst_common.sh

@@ -29,7 +29,9 @@ WSREP_SST_OPT_USER="${WSREP_SST_OPT_USER:-}"
 WSREP_SST_OPT_PSWD="${WSREP_SST_OPT_PSWD:-}"
 WSREP_SST_OPT_REMOTE_AUTH="${WSREP_SST_OPT_REMOTE_AUTH:-}"
 WSREP_SST_OPT_DEFAULT=""
+WSREP_SST_OPT_DEFAULTS=""
 WSREP_SST_OPT_EXTRA_DEFAULT=""
+WSREP_SST_OPT_EXTRA_DEFAULTS=""
 WSREP_SST_OPT_SUFFIX_DEFAULT=""
 WSREP_SST_OPT_SUFFIX_VALUE=""
 WSREP_SST_OPT_MYSQLD=""
@@ -152,10 +154,12 @@ case "$1" in
         ;;
     '--defaults-file')
         readonly WSREP_SST_OPT_DEFAULT="$1=$2"
+        readonly WSREP_SST_OPT_DEFAULTS="$1='$2'"
         shift
         ;;
     '--defaults-extra-file')
         readonly WSREP_SST_OPT_EXTRA_DEFAULT="$1=$2"
+        readonly WSREP_SST_OPT_EXTRA_DEFAULTS="$1='$2'"
         shift
         ;;
     '--defaults-group-suffix')
@@ -611,24 +615,54 @@ else
     MYSQLDUMP="$(command -v mysqldump)"
 fi
 
+wsrep_log()
+{
+    # echo everything to stderr so that it gets into common error log
+    # deliberately made to look different from the rest of the log
+    local readonly tst="$(date +%Y%m%d\ %H:%M:%S.%N | cut -b -21)"
+    echo "WSREP_SST: $* ($tst)" >&2
+}
+
+wsrep_log_error()
+{
+    wsrep_log "[ERROR] $*"
+}
+
+wsrep_log_warning()
+{
+    wsrep_log "[WARNING] $*"
+}
+
+wsrep_log_info()
+{
+    wsrep_log "[INFO] $*"
+}
+
 if [ -x "$SCRIPTS_DIR/my_print_defaults" ]; then
     MY_PRINT_DEFAULTS="$SCRIPTS_DIR/my_print_defaults"
 elif [ -x "$EXTRA_DIR/my_print_defaults" ]; then
     MY_PRINT_DEFAULTS="$EXTRA_DIR/my_print_defaults"
 else
     MY_PRINT_DEFAULTS="$(command -v my_print_defaults)"
+    if [ -z "$MY_PRINT_DEFAULTS" ]; then
+        wsrep_log_error "my_print_defaults not found in path"
+        exit 2
+    fi
 fi
 
+readonly MY_PRINT_DEFAULTS
+
+wsrep_defaults="$WSREP_SST_OPT_DEFAULTS"
+wsrep_defaults="$wsrep_defaults${wsrep_defaults:+ }$WSREP_SST_OPT_EXTRA_DEFAULTS"
+wsrep_defaults="$wsrep_defaults${wsrep_defaults:+ }$WSREP_SST_OPT_SUFFIX_DEFAULT"
+
+readonly WSREP_SST_OPT_CONF="$wsrep_defaults"
+
 wsrep_defaults="$WSREP_SST_OPT_DEFAULT"
-if [ -n "$wsrep_defaults" ]; then
-    wsrep_defaults="$wsrep_defaults "
-fi
-wsrep_defaults="$wsrep_defaults$WSREP_SST_OPT_EXTRA_DEFAULT"
-if [ -n "$wsrep_defaults" ]; then
-    wsrep_defaults="$wsrep_defaults "
-fi
-readonly WSREP_SST_OPT_CONF="$wsrep_defaults$WSREP_SST_OPT_SUFFIX_DEFAULT"
-readonly MY_PRINT_DEFAULTS="$MY_PRINT_DEFAULTS $WSREP_SST_OPT_CONF"
+wsrep_defaults="$wsrep_defaults${wsrep_defaults:+ }$WSREP_SST_OPT_EXTRA_DEFAULT"
+wsrep_defaults="$wsrep_defaults${wsrep_defaults:+ }$WSREP_SST_OPT_SUFFIX_DEFAULT"
+
+readonly WSREP_SST_OPT_CONF_UNQUOTED="$wsrep_defaults"
 
 #
 # User can specify mariabackup specific settings that will be used during sst
@@ -663,13 +697,21 @@ parse_cnf()
         # If the group name is the same as the "mysqld" without "--" prefix,
         # then try to use it together with the group suffix:
         if [ "$group" = 'mysqld' -a -n "$WSREP_SST_OPT_SUFFIX_VALUE" ]; then
-            reval=$($MY_PRINT_DEFAULTS "mysqld$WSREP_SST_OPT_SUFFIX_VALUE" | awk "$pattern")
+            reval=$("$MY_PRINT_DEFAULTS" \
+                    ${WSREP_SST_OPT_DEFAULT:+"$WSREP_SST_OPT_DEFAULT"} \
+                    ${WSREP_SST_OPT_EXTRA_DEFAULT:+"$WSREP_SST_OPT_EXTRA_DEFAULT"} \
+                    ${WSREP_SST_OPT_SUFFIX_DEFAULT:+"$WSREP_SST_OPT_SUFFIX_DEFAULT"} \
+                    "mysqld$WSREP_SST_OPT_SUFFIX_VALUE" | awk "$pattern")
             if [ -n "$reval" ]; then
                 break
             fi
         fi
         # Let's try to use the group name as it is:
-        reval=$($MY_PRINT_DEFAULTS "$group" | awk "$pattern")
+        reval=$("$MY_PRINT_DEFAULTS" \
+                ${WSREP_SST_OPT_DEFAULT:+"$WSREP_SST_OPT_DEFAULT"} \
+                ${WSREP_SST_OPT_EXTRA_DEFAULT:+"$WSREP_SST_OPT_EXTRA_DEFAULT"} \
+                ${WSREP_SST_OPT_SUFFIX_DEFAULT:+"$WSREP_SST_OPT_SUFFIX_DEFAULT"} \
+                "$group" | awk "$pattern")
         if [ -n "$reval" ]; then
             break
         fi
@@ -710,13 +752,21 @@ in_config()
         # If the group name is the same as the "mysqld" without "--" prefix,
         # then try to use it together with the group suffix:
         if [ "$group" = 'mysqld' -a -n "$WSREP_SST_OPT_SUFFIX_VALUE" ]; then
-            found=$($MY_PRINT_DEFAULTS "mysqld$WSREP_SST_OPT_SUFFIX_VALUE" | awk "$pattern")
+            found=$("$MY_PRINT_DEFAULTS" \
+                    ${WSREP_SST_OPT_DEFAULT:+"$WSREP_SST_OPT_DEFAULT"} \
+                    ${WSREP_SST_OPT_EXTRA_DEFAULT:+"$WSREP_SST_OPT_EXTRA_DEFAULT"} \
+                    ${WSREP_SST_OPT_SUFFIX_DEFAULT:+"$WSREP_SST_OPT_SUFFIX_DEFAULT"} \
+                    "mysqld$WSREP_SST_OPT_SUFFIX_VALUE" | awk "$pattern")
             if [ $found -ne 0 ]; then
                 break
             fi
         fi
         # Let's try to use the group name as it is:
-        found=$($MY_PRINT_DEFAULTS "$group" | awk "$pattern")
+        found=$($MY_PRINT_DEFAULTS \
+                ${WSREP_SST_OPT_DEFAULT:+"$WSREP_SST_OPT_DEFAULT"} \
+                ${WSREP_SST_OPT_EXTRA_DEFAULT:+"$WSREP_SST_OPT_EXTRA_DEFAULT"} \
+                ${WSREP_SST_OPT_SUFFIX_DEFAULT:+"$WSREP_SST_OPT_SUFFIX_DEFAULT"} \
+                "$group" | awk "$pattern")
         if [ $found -ne 0 ]; then
             break
         fi
@@ -797,29 +847,6 @@ else
     SST_PROGRESS_FILE=""
 fi
 
-wsrep_log()
-{
-    # echo everything to stderr so that it gets into common error log
-    # deliberately made to look different from the rest of the log
-    local readonly tst="$(date +%Y%m%d\ %H:%M:%S.%N | cut -b -21)"
-    echo "WSREP_SST: $* ($tst)" >&2
-}
-
-wsrep_log_error()
-{
-    wsrep_log "[ERROR] $*"
-}
-
-wsrep_log_warning()
-{
-    wsrep_log "[WARNING] $*"
-}
-
-wsrep_log_info()
-{
-    wsrep_log "[INFO] $*"
-}
-
 wsrep_cleanup_progress_file()
 {
     [ -n "$SST_PROGRESS_FILE" -a \
@@ -959,6 +986,43 @@ check_sockets_utils()
     fi
 }
 
+#
+# Check if the port is in the "listen" state.
+# The first parameter is the PID of the process that should
+# listen on the port - if it is not known, you can specify
+# an empty string or zero.
+# The second parameter is the port number.
+# The third parameter is a list of the names of utilities
+# (via "|") that can listen on this port during the state
+# transfer.
+#
+check_port()
+{
+    local pid="$1"
+    local port="$2"
+    local utils="$3"
+
+    [ -z "$pid" ] || [ $pid -eq 0 ] && pid='[0-9]+'
+
+    local rc=1
+
+    if [ $lsof_available -ne 0 ]; then
+        lsof -Pnl -i ":$port" 2>/dev/null | \
+        grep -q -E "^($utils)[^[:space:]]*[[:space:]]+$pid[[:space:]].*\\(LISTEN\\)" && rc=0
+    elif [ $sockstat_available -ne 0 ]; then
+        sockstat -p "$port" 2>/dev/null | \
+        grep -q -E "[[:space:]]+($utils)[^[:space:]]*[[:space:]]+$pid[[:space:]].*[[:space:]]LISTEN" && rc=0
+    elif [ $ss_available -ne 0 ]; then
+        ss -nlpH "( sport = :$port )" 2>/dev/null | \
+        grep -q -E "users:\\(.*\\(\"($utils)[^[:space:]]*\"[^)]*,pid=$pid(,[^)]*)?\\)" && rc=0
+    else
+        wsrep_log_error "unknown sockets utility"
+        exit 2 # ENOENT
+    fi
+
+    return $rc
+}
+
 #
 # If the ssl_dhparams variable is already set, uses that as a source
 # of dh parameters for OpenSSL. Otherwise, looks for dhparams.pem in
@@ -966,28 +1030,95 @@ check_sockets_utils()
 #
 check_for_dhparams()
 {
-    if [ -z "$ssl_dhparams" ]; then
-        ssl_dhparams="$DATA/dhparams.pem"
-        if [ ! -r "$ssl_dhparams" ]; then
-            get_openssl
-            if [ -n "$OPENSSL_BINARY" ]; then
-                wsrep_log_info "Could not find dhparams file, creating $ssl_dhparams"
-                if ! "$OPENSSL_BINARY" dhparam -out "$ssl_dhparams" 2048 >/dev/null 2>&1
-                then
-                    wsrep_log_error "******** ERROR *****************************************"
-                    wsrep_log_error "* Could not create the dhparams.pem file with OpenSSL. *"
-                    wsrep_log_error "********************************************************"
-                    ssl_dhparams=""
-                fi
-            else
-                # Rollback: if openssl is not installed, then use
-                # the default parameters:
+    ssl_dhparams="$DATA/dhparams.pem"
+    if [ ! -r "$ssl_dhparams" ]; then
+        get_openssl
+        if [ -n "$OPENSSL_BINARY" ]; then
+            wsrep_log_info "Could not find dhparams file, creating $ssl_dhparams"
+            if ! "$OPENSSL_BINARY" dhparam -out "$ssl_dhparams" 2048 >/dev/null 2>&1
+            then
+                wsrep_log_error "******** ERROR *****************************************"
+                wsrep_log_error "* Could not create the dhparams.pem file with OpenSSL. *"
+                wsrep_log_error "********************************************************"
                 ssl_dhparams=""
-            fi
+             fi
+        else
+            # Rollback: if openssl is not installed, then use
+            # the default parameters:
+            ssl_dhparams=""
         fi
     fi
 }
 
+#
+# Verifies that the CA file verifies the certificate.
+# Doing this here lets us generate better error messages.
+#
+# 1st param: path to the CA file.
+# 2nd param: path to the certificate.
+#
+verify_ca_matches_cert()
+{
+    local ca_path="$1"
+    local cert_path="$2"
+
+    # If the openssl utility is not installed, then
+    # we will not do this certificate check:
+    get_openssl
+    if [ -z "$OPENSSL_BINARY" ]; then
+        return
+    fi
+
+    if ! "$OPENSSL_BINARY" verify -verbose -CAfile "$ca_path" "$cert_path" >/dev/null 2>&1
+    then
+        wsrep_log_error "******** FATAL ERROR ********************************************"
+        wsrep_log_error "* The certifcate and CA (certificate authority) do not match.   *"
+        wsrep_log_error "* It does not appear that the certificate was issued by the CA. *"
+        wsrep_log_error "* Please check your certificate and CA files.                   *"
+        wsrep_log_error "*****************************************************************"
+        exit 22
+    fi
+}
+
+#
+# Verifies that the certificate matches the private key.
+# Doing this will save us having to wait for a timeout that would
+# otherwise occur.
+#
+# 1st param: path to the certificate.
+# 2nd param: path to the private key.
+#
+verify_cert_matches_key()
+{
+    local cert_path="$1"
+    local key_path="$2"
+
+    # If the diff utility is not installed, then
+    # we will not do this certificate check:
+    if [ -z "$(command -v diff)" ]; then
+        return
+    fi
+
+    # If the openssl utility is not installed, then
+    # we will not do this certificate check:
+    get_openssl
+    if [ -z "$OPENSSL_BINARY" ]; then
+        return
+    fi
+
+    # Generate the public key from the cert and the key.
+    # They should match (otherwise we can't create an SSL connection).
+    if ! diff <("$OPENSSL_BINARY" x509 -in "$cert_path" -pubkey -noout 2>/dev/null) \
+              <("$OPENSSL_BINARY" pkey -in "$key_path" -pubout 2>/dev/null) >/dev/null 2>&1
+    then
+        wsrep_log_error "******************* FATAL ERROR ****************"
+        wsrep_log_error "* The certifcate and private key do not match. *"
+        wsrep_log_error "* Please check your certificate and key files. *"
+        wsrep_log_error "************************************************"
+        exit 22
+    fi
+}
+
 #
 # Compares two version strings.
 # The first parameter is the version to be checked;
@@ -996,22 +1127,22 @@ check_for_dhparams()
 #
 check_for_version()
 {
-    y1=${1#*.}
+    y1="${1#*.}"
     [ "$y1" = "$1" ] && y1=""
     z1=${y1#*.}
     [ "$z1" = "$y1" ] && z1=""
-    x1=${1%%.*}
-    y1=${y1%%.*}
-    z1=${z1%%.*}
+    x1="${1%%.*}"
+    y1="${y1%%.*}"
+    z1="${z1%%.*}"
     [ -z "$y1" ] && y1=0
     [ -z "$z1" ] && z1=0
-    y2=${2#*.}
+    y2="${2#*.}"
     [ "$y2" = "$2" ] && y2=""
-    z2=${y2#*.}
+    z2="${y2#*.}"
     [ "$z2" = "$y2" ] && z2=""
-    x2=${2%%.*}
-    y2=${y2%%.*}
-    z2=${z2%%.*}
+    x2="${2%%.*}"
+    y2="${y2%%.*}"
+    z2="${z2%%.*}"
     [ -z "$y2" ] && y2=0
     [ -z "$z2" ] && z2=0
     [ $x1 -lt $x2 ] && return 1
@@ -1032,8 +1163,8 @@ trim_string()
         if [ $x -ne $z ]; then
             local y="${1%$pattern*}"
             y=${#y}
-            x=$(( $z-$x-1 ))
-            y=$(( $y-$x+1 ))
+            x=$(( z-x-1 ))
+            y=$(( y-x+1 ))
             printf '%s' "${1:$x:$y}"
         else
             printf ''
@@ -1043,3 +1174,105 @@ trim_string()
         echo "$1" | sed -E "s/^$pattern+|$pattern+\$//g"
     fi
 }
+
+#
+# Check whether process is still running.
+# The first parameter contains the name of the PID file.
+# The second parameter is the flag of the need to delete
+# the PID file.
+# If the second parameter is not zero and not empty,
+# then if the process terminates, the corresponding
+# PID file will be deleted.
+# This function also sets the CHECK_PID variable to zero
+# if the process has already exited, or writes the PID
+# of the process there if it is still running.
+#
+check_pid()
+{
+    local pid_file="$1"
+    local remove=${2:-0}
+    if [ -r "$pid_file" ]; then
+        local pid=$(cat "$pid_file" 2>/dev/null)
+        if [ -n "$pid" ]; then
+            if [ $pid -ne 0 ]; then
+                if ps -p "$pid" >/dev/null 2>&1; then
+                    CHECK_PID=$pid
+                    return 0
+                fi
+            fi
+        fi
+        if [ $remove -eq 1 ]; then
+            rm -f "$pid_file"
+        fi
+    fi
+    CHECK_PID=0
+    return 1
+}
+
+#
+# Checking that the process with the specified PID is still
+# running and killing it in this case by sending SIGTERM
+# (using the "kill" operation).
+# The first parameter contains PID of the process.
+# The second and third parameters (both optional) are the names
+# of the PID and the configuration files, which should be removed
+# after the process ends.
+# If the first parameter (PID of the process) is zero, then
+# the function immediately deletes the PID and the configuration
+# files (if specified), without any additional checks.
+#
+cleanup_pid()
+{
+    local pid="$1"
+    local pid_file="${2:-}"
+    local config="${3:-}"
+
+    if [ $pid -ne 0 ]; then
+        if ps -p $pid >/dev/null 2>&1; then
+            if kill $pid >/dev/null 2>&1; then
+                sleep 0.5
+                local round=0
+                local force=0
+                while ps -p $pid >/dev/null 2>&1; do
+                   sleep 1
+                   round=$(( round+1 ))
+                   if [ $round -eq 16 ]; then
+                       if [ $force -eq 0 ]; then
+                           round=8
+                           force=1
+                           kill -9 $pid >/dev/null 2>&1
+                       else
+                           return 1;
+                       fi
+                   fi
+                done
+            elif ps -p $pid >/dev/null 2>&1; then
+                wsrep_log_warning "Unable to kill PID=$pid ($pid_file)"
+                return 1
+            fi
+        fi
+    fi
+
+    [ -n "$pid_file" ] && [ -f "$pid_file" ] && rm -f "$pid_file"
+    [ -n "$config" ]   && [ -f "$config"   ] && rm -f "$config"
+
+    return 0
+}
+
+nproc=""
+
+get_proc()
+{
+    if [ -z "$nproc" ]; then
+        set +e
+        if [ "$OS" = 'Linux' ]; then
+            nproc=$(grep -c processor /proc/cpuinfo 2>/dev/null)
+        elif [ "$OS" = 'Darwin' -o "$OS" = 'FreeBSD' ]; then
+            nproc=$(sysctl -n hw.ncpu)
+        fi
+        if [ -z "$nproc" ] || [ $nproc -eq 0 ]; then
+            nproc=1
+        fi
+        set -e
+    fi
+}

scripts/wsrep_sst_mariabackup.sh

@@ -29,11 +29,10 @@ eformat=""
 ekey=""
 ekeyfile=""
 encrypt=0
-nproc=1
 ecode=0
 ssyslog=""
 ssystag=""
-MARIABACKUP_PID=""
+BACKUP_PID=""
 tcert=""
 tpem=""
 tkey=""
@@ -77,6 +76,11 @@ compress='none'
 compress_chunk=""
 compress_threads=""
 
+backup_threads=""
+
+encrypt_threads=""
+encrypt_chunk=""
+
 readonly SECRET_TAG="secret"
 
 # Required for backup locks
@@ -90,8 +94,8 @@ fi
 pcmd="pv $pvopts"
 declare -a RC
 
-MARIABACKUP_BIN="$(command -v mariabackup)"
-if [ ! -x "$MARIABACKUP_BIN" ]; then
+BACKUP_BIN="$(command -v mariabackup)"
+if [ ! -x "$BACKUP_BIN" ]; then
     wsrep_log_error 'mariabackup binary not found in path'
     exit 42
 fi
@@ -108,7 +112,8 @@ INNOBACKUPLOG="$DATA/mariabackup.backup.log"
 # Setting the path for ss and ip
 export PATH="/usr/sbin:/sbin:$PATH"
 
-timeit(){
+timeit()
+{
     local stage="$1"
     shift
     local cmd="$@"
@@ -198,6 +203,12 @@ get_keys()
         else
             ecmd="xbcrypt --encrypt-algo='$ealgo' --encrypt-key='$ekey'"
         fi
+        if [ -n "$encrypt_threads" ]; then
+            ecmd="$ecmd --encrypt-threads=$encrypt_threads"
+        fi
+        if [ -n "$encrypt_chunk" ]; then
+            ecmd="$ecmd --encrypt-chunk-size=$encrypt_chunk"
+        fi
     else
         wsrep_log_error "Unknown encryption format='$eformat'"
         exit 2
@@ -298,15 +309,6 @@ get_transfer()
             exit 2
         fi
 
-        # Determine the socat version
-        SOCAT_VERSION=$(socat -V 2>&1 | grep -m1 -oe '[0-9]\.[0-9][\.0-9]*')
-        if [ -z "$SOCAT_VERSION" ]; then
-            wsrep_log_error "******** FATAL ERROR ******************"
-            wsrep_log_error "* Cannot determine the socat version. *"
-            wsrep_log_error "***************************************"
-            exit 2
-        fi
-
         local action='Decrypting'
         if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then
             tcmd="socat -u openssl-listen:$SST_PORT,reuseaddr"
@@ -315,10 +317,22 @@ get_transfer()
             action='Encrypting'
         fi
 
-        if ! check_for_version "$SOCAT_VERSION" '1.7.3'; then
-            # socat versions < 1.7.3 will have 512-bit dhparams (too small)
-            # so create 2048-bit dhparams and send that as a parameter:
-            check_for_dhparams
+        if [ "${sockopt#*,dhparam=}" != "$sockopt" ]; then
+            if [ -z "$ssl_dhparams" ]; then
+                # Determine the socat version
+                SOCAT_VERSION=$(socat -V 2>&1 | grep -m1 -oe '[0-9]\.[0-9][\.0-9]*')
+                if [ -z "$SOCAT_VERSION" ]; then
+                    wsrep_log_error "******** FATAL ERROR ******************"
+                    wsrep_log_error "* Cannot determine the socat version. *"
+                    wsrep_log_error "***************************************"
+                    exit 2
+                fi
+                if ! check_for_version "$SOCAT_VERSION" '1.7.3'; then
+                    # socat versions < 1.7.3 will have 512-bit dhparams (too small)
+                    # so create 2048-bit dhparams and send that as a parameter:
+                    check_for_dhparams
+                fi
+            fi
             if [ -n "$ssl_dhparams" ]; then
                 tcmd="$tcmd,dhparam='$ssl_dhparams'"
             fi
@@ -330,6 +344,11 @@ get_transfer()
                 wsrep_log_error "Both PEM and CRT files required"
                 exit 22
             fi
+            if [ ! -r "$tpem" -o ! -r "$tcert" ]; then
+                wsrep_log_error "Both PEM and CRT files must be readable"
+                exit 22
+            fi
+            verify_ca_matches_cert "$tcert" "$tpem"
             tcmd="$tcmd,cert='$tpem',cafile='$tcert'$sockopt"
             stagemsg="$stagemsg-OpenSSL-Encrypted-2"
             wsrep_log_info "$action with cert=$tpem, cafile=$tcert"
@@ -339,6 +358,11 @@ get_transfer()
                 wsrep_log_error "Both certificate and key files required"
                 exit 22
             fi
+            if [ ! -r "$tpem" -o ! -r "$tkey" ]; then
+                wsrep_log_error "Both certificate and key files must be readable"
+                exit 22
+            fi
+            verify_cert_matches_key "$tpem" "$tkey"
             stagemsg="$stagemsg-OpenSSL-Encrypted-3"
             if [ -z "$tcert" ]; then
                 if [ $encrypt -eq 4 ]; then
@@ -350,6 +374,11 @@ get_transfer()
                 wsrep_log_info "$action with cert=$tpem, key=$tkey, verify=0"
             else
                 # CA verification
+                if [ ! -r "$tcert" ]; then
+                    wsrep_log_error "Certificate file must be readable"
+                    exit 22
+                fi
+                verify_ca_matches_cert "$tcert" "$tpem"
                 if [ -n "$WSREP_SST_OPT_REMOTE_USER" ]; then
                     CN_option=",commonname='$WSREP_SST_OPT_REMOTE_USER'"
                 elif [ $encrypt -eq 4 ]; then
@@ -440,10 +469,10 @@ read_cnf()
             tpem=$(parse_cnf 'sst' 'tcert')
             tkey=$(parse_cnf 'sst' 'tkey')
         fi
-        if [ "$tmode" != 'DISABLED' ]
-        then # backward-incompatible behavior
-            if [ -z "$tpem" -a -z "$tkey" -a -z "$tcert" ]
-            then # no old-style SSL config in [sst]
+        if [ "$tmode" != 'DISABLED' ]; then
+            # backward-incompatible behavior
+            if [ -z "$tpem" -a -z "$tkey" -a -z "$tcert" ]; then
+                # no old-style SSL config in [sst]
                 check_server_ssl_config
             fi
             if [ 0 -eq $encrypt -a -n "$tpem" -a -n "$tkey" ]
@@ -504,21 +533,28 @@ read_cnf()
             compress_threads=$(parse_cnf "$encgroups" 'compress-threads')
         fi
     fi
+
+    backup_threads=$(parse_cnf "$encgroups" 'backup-threads')
+
+    if [ "$eformat" = 'xbcrypt' ]; then
+        encrypt_threads=$(parse_cnf "$encgroups" 'encrypt-threads')
+        encrypt_chunk=$(parse_cnf "$encgroups" 'encrypt-chunk-size')
+    fi
 }
 
 get_stream()
 {
     if [ "$sfmt" = 'mbstream' -o "$sfmt" = 'xbstream' ]; then
         sfmt='mbstream'
-        MBSTREAM_BIN="$(command -v mbstream)"
-        if [ -z "$MBSTREAM_BIN" ]; then
+        STREAM_BIN="$(command -v mbstream)"
+        if [ -z "$STREAM_BIN" ]; then
             wsrep_log_error "Streaming with $sfmt, but $sfmt not found in path"
             exit 42
         fi
         if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then
-            strmcmd="'$MBSTREAM_BIN' -x"
+            strmcmd="'$STREAM_BIN' -x"
         else
-            strmcmd="'$MBSTREAM_BIN' -c '$INFO_FILE'"
+            strmcmd="'$STREAM_BIN' -c '$INFO_FILE'"
         fi
     else
         sfmt='tar'
@@ -531,79 +567,33 @@ get_stream()
     wsrep_log_info "Streaming with $sfmt"
 }
 
-get_proc()
-{
-    set +e
-    nproc=$(grep -c processor /proc/cpuinfo)
-    [ -z $nproc -o $nproc -eq 0 ] && nproc=1
-    set -e
-}
-
 sig_joiner_cleanup()
 {
     wsrep_log_error "Removing $MAGIC_FILE file due to signal"
     [ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE"
 }
 
-cleanup_joiner()
+cleanup_at_exit()
 {
     # Since this is invoked just after exit NNN
     local estatus=$?
     if [ $estatus -ne 0 ]; then
         wsrep_log_error "Cleanup after exit with status:$estatus"
-    elif [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then
-        wsrep_log_info "Removing the sst_in_progress file"
-        wsrep_cleanup_progress_file
-    fi
-    if [ -n "$progress" -a -p "$progress" ]; then
-        wsrep_log_info "Cleaning up fifo file $progress"
-        rm "$progress"
-    fi
-
-    if [ -n "$STATDIR" ]; then
-       [ -d "$STATDIR" ] && rm -rf "$STATDIR"
     fi
 
-    # Final cleanup
-    pgid=$(ps -o pgid= $$ | grep -o '[0-9]*')
-
-    # This means no setsid done in mysqld.
-    # We don't want to kill mysqld here otherwise.
-    if [ $$ -eq $pgid ]; then
-        # This means a signal was delivered to the process.
-        # So, more cleanup.
-        if [ $estatus -ge 128 ]; then
-            kill -KILL -$$ || true
-        fi
-    fi
-
-    exit $estatus
-}
-
-check_pid()
-{
-    local pid_file="$1"
-    [ -r "$pid_file" ] && ps -p $(cat "$pid_file") 2>&1 >/dev/null
-}
-
-cleanup_donor()
-{
-    # Since this is invoked just after exit NNN
-    local estatus=$?
-    if [ $estatus -ne 0 ]; then
-        wsrep_log_error "Cleanup after exit with status:$estatus"
-    fi
-
-    if [ -n "$MARIABACKUP_PID" ]; then
-        if check_pid $MARIABACKUP_PID
-        then
-            wsrep_log_error "mariabackup process is still running. Killing..."
-            kill_mariabackup
+    if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then
+        wsrep_log_info "Removing the sst_in_progress file"
+        wsrep_cleanup_progress_file
+    else
+        if [ -n "$BACKUP_PID" ]; then
+            if check_pid "$BACKUP_PID" 1; then
+                wsrep_log_error "mariabackup process is still running. Killing..."
+                cleanup_pid $CHECK_PID "$BACKUP_PID"
+            fi
         fi
+        [ -f "$DATA/$IST_FILE" ] && rm -f "$DATA/$IST_FILE"
     fi
 
-    [ -f "$DATA/$IST_FILE" ] && rm -f "$DATA/$IST_FILE"
-
     if [ -n "$progress" -a -p "$progress" ]; then
         wsrep_log_info "Cleaning up fifo file $progress"
         rm -f "$progress" || true
@@ -611,8 +601,14 @@ cleanup_donor()
 
     wsrep_log_info "Cleaning up temporary directories"
 
-    [ -n "$xtmpdir" -a -d "$xtmpdir" ] && rm -rf "$xtmpdir" || true
-    [ -n "$itmpdir" -a -d "$itmpdir" ] && rm -rf "$itmpdir" || true
+    if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then
+        if [ -n "$STATDIR" ]; then
+           [ -d "$STATDIR" ] && rm -rf "$STATDIR"
+        fi
+    else
+        [ -n "$xtmpdir" -a -d "$xtmpdir" ] && rm -rf "$xtmpdir" || true
+        [ -n "$itmpdir" -a -d "$itmpdir" ] && rm -rf "$itmpdir" || true
+    fi
 
     # Final cleanup
     pgid=$(ps -o pgid= $$ | grep -o '[0-9]*')
@@ -623,21 +619,13 @@ cleanup_donor()
         # This means a signal was delivered to the process.
         # So, more cleanup.
         if [ $estatus -ge 128 ]; then
-            kill -KILL -$$ || true
+            kill -KILL -- -$$ || true
         fi
     fi
 
     exit $estatus
 }
 
-kill_mariabackup()
-{
-    local PID=$(cat "$MARIABACKUP_PID")
-    [ -n "$PID" -a "0" != "$PID" ] && kill $PID && (kill $PID && kill -9 $PID) || :
-    wsrep_log_info "Removing mariabackup pid file ($MARIABACKUP_PID)"
-    rm -f "$MARIABACKUP_PID" || true
-}
-
 setup_ports()
 {
     SST_PORT="$WSREP_SST_OPT_PORT"
@@ -648,51 +636,17 @@ setup_ports()
     fi
 }
 
-check_port()
-{
-    local PORT="$1"
-    local UTILS="$2"
-
-    local port_info is_util
-
-    if [ $lsof_available -ne 0 ]; then
-        port_info=$(lsof -i ":$PORT" -Pn 2>/dev/null | \
-            grep -F '(LISTEN)')
-        is_util=$(echo "$port_info" | \
-            grep -E "^($UTILS)[^[:space:]]*[[:space:]]+[0-9]+[[:space:]]+")
-    elif [ $sockstat_available -ne 0 ]; then
-        port_info=$(sockstat -p "$PORT" 2>/dev/null | \
-            grep -F 'LISTEN')
-        is_util=$(echo "$port_info" | \
-            grep -E "[[:space:]]+($UTILS)[^[:space:]]*[[:space:]]+[0-9]+[[:space:]]+")
-    elif [ $ss_available -ne 0 ]; then
-        port_info=$(ss -H -p -n -l "( sport = :$PORT )" 2>/dev/null)
-        is_util=$(echo "$port_info" | \
-            grep -E "users:\\(.*\\(\"($UTILS)[^[:space:]]*\".*\<pid=[0-9]+\>.*\\)")
-    else
-        wsrep_log_error "unknown sockets utility"
-        exit 2 # ENOENT
-    fi
-
-    if [ -z "$is_util" ]; then
-        return 1
-    fi
-
-    return 0
-}
-
-# waits ~10 seconds for nc to open the port and then reports ready
-# (regardless of timeout)
+#
+# Waits ~30 seconds for socat or nc to open the port and
+# then reports ready, regardless of timeout.
+#
 wait_for_listen()
 {
     local PORT="$1"
     local ADDR="$2"
     local MODULE="$3"
-
-    for i in {1..50}
-    do
-        if check_port "$PORT" 'socat|nc'
-        then
+    for i in {1..150}; do
+        if check_port "" "$PORT" 'socat|nc'; then
             break
         fi
         sleep 0.2
@@ -708,8 +662,8 @@ check_extra()
         if [ "$thread_handling" = 'pool-of-threads' ]; then
             local eport=$(parse_cnf '--mysqld' 'extra-port')
             if [ -n "$eport" ]; then
-                # mariabackup works only locally, hence,
-                # setting host to 127.0.0.1 unconditionally:
+                # mariabackup works only locally.
+                # Hence, setting host to 127.0.0.1 unconditionally:
                 wsrep_log_info "SST through extra_port $eport"
                 INNOEXTRA="$INNOEXTRA --host=127.0.0.1 --port=$eport"
                 use_socket=0
@@ -825,11 +779,12 @@ monitor_process()
     local sst_stream_pid=$1
 
     while true ; do
-        if ! ps -p "$WSREP_SST_OPT_PARENT" &>/dev/null; then
+        if ! ps -p "$WSREP_SST_OPT_PARENT" >/dev/null 2>&1; then
             wsrep_log_error "Parent mysqld process (PID: $WSREP_SST_OPT_PARENT) terminated unexpectedly."
+            kill -- -"$WSREP_SST_OPT_PARENT"
             exit 32
         fi
-        if ! ps -p "$sst_stream_pid" &>/dev/null; then
+        if ! ps -p "$sst_stream_pid" >/dev/null 2>&1; then
             break
         fi
         sleep 0.1
@@ -839,14 +794,14 @@ monitor_process()
 [ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE"
 
 if [ "$WSREP_SST_OPT_ROLE" != 'joiner' -a "$WSREP_SST_OPT_ROLE" != 'donor' ]; then
-    wsrep_log_error "Invalid role ${WSREP_SST_OPT_ROLE}"
+    wsrep_log_error "Invalid role '$WSREP_SST_OPT_ROLE'"
     exit 22
 fi
 
 read_cnf
 setup_ports
 
-if "$MARIABACKUP_BIN" --help 2>/dev/null | grep -qw -- '--version-check'; then
+if "$BACKUP_BIN" --help 2>/dev/null | grep -qw -- '--version-check'; then
     disver='--no-version-check'
 fi
 
@@ -942,8 +897,8 @@ else
             gzip "$newfile"
         fi
     fi
-    INNOAPPLY="&> '$INNOAPPLYLOG'"
-    INNOMOVE="&> '$INNOMOVELOG'"
+    INNOAPPLY="> '$INNOAPPLYLOG' 2>&1"
+    INNOMOVE="> '$INNOMOVELOG' 2>&1"
     INNOBACKUP="2> '$INNOBACKUPLOG'"
 fi
 
@@ -953,9 +908,9 @@ setup_commands()
     if [ -n "$WSREP_SST_OPT_MYSQLD" ]; then
         mysqld_args="--mysqld-args $WSREP_SST_OPT_MYSQLD"
     fi
-    INNOAPPLY="$MARIABACKUP_BIN --prepare $disver $iapts $INNOEXTRA --target-dir='$DATA' --datadir='$DATA' $mysqld_args $INNOAPPLY"
-    INNOMOVE="$MARIABACKUP_BIN $WSREP_SST_OPT_CONF --move-back $disver $impts --force-non-empty-directories --target-dir='$DATA' --datadir='${TDATA:-$DATA}' $INNOMOVE"
-    INNOBACKUP="$MARIABACKUP_BIN $WSREP_SST_OPT_CONF --backup $disver $iopts $tmpopts $INNOEXTRA --galera-info --stream=$sfmt --target-dir='$itmpdir' --datadir='$DATA' $mysqld_args $INNOBACKUP"
+    INNOAPPLY="$BACKUP_BIN --prepare $disver $iapts $INNOEXTRA --target-dir='$DATA' --datadir='$DATA' $mysqld_args $INNOAPPLY"
+    INNOMOVE="$BACKUP_BIN $WSREP_SST_OPT_CONF --move-back $disver $impts --force-non-empty-directories --target-dir='$DATA' --datadir='${TDATA:-$DATA}' $INNOMOVE"
+    INNOBACKUP="$BACKUP_BIN $WSREP_SST_OPT_CONF --backup $disver $iopts $tmpopts $INNOEXTRA --galera-info --stream=$sfmt --target-dir='$itmpdir' --datadir='$DATA' $mysqld_args $INNOBACKUP"
 }
 
 get_stream
@@ -963,7 +918,7 @@ get_transfer
 
 if [ "$WSREP_SST_OPT_ROLE" = 'donor' ]
 then
-    trap cleanup_donor EXIT
+    trap cleanup_at_exit EXIT
 
     if [ $WSREP_SST_OPT_BYPASS -eq 0 ]
     then
@@ -976,12 +931,15 @@ then
         tmpdir=$(parse_cnf "$encgroups" 'tmpdir')
         if [ -z "$tmpdir" ]; then
             xtmpdir="$(mktemp -d)"
-            tmpopts="--tmpdir='$xtmpdir'"
-            wsrep_log_info "Using $xtmpdir as mariabackup temporary directory"
+        else
+            xtmpdir=$(mktemp '-d' "--tmpdir=$tmpdir")
         fi
 
+        wsrep_log_info "Using '$xtmpdir' as mariabackup temporary directory"
+        tmpopts="--tmpdir='$xtmpdir'"
+
         itmpdir="$(mktemp -d)"
-        wsrep_log_info "Using $itmpdir as mariabackup temporary directory"
+        wsrep_log_info "Using '$itmpdir' as mariabackup working directory"
 
         usrst=0
         if [ -n "$WSREP_SST_OPT_USER" ]; then
@@ -1048,25 +1006,29 @@ then
             tcmd="$ecmd | $tcmd"
         fi
 
-        iopts="$iopts --databases-exclude='lost+found'"
+        iopts="--databases-exclude='lost+found' $iopts"
 
         if [ ${FORCE_FTWRL:-0} -eq 1 ]; then
             wsrep_log_info "Forcing FTWRL due to environment variable FORCE_FTWRL equal to $FORCE_FTWRL"
-            iopts="$iopts --no-backup-locks"
+            iopts="--no-backup-locks $iopts"
         fi
 
         # if compression is enabled for backup files, then add the
         # appropriate options to the mariabackup command line:
         if [ "$compress" != 'none' ]; then
-            iopts="$iopts --compress${compress:+=$compress}"
+            iopts="--compress${compress:+=$compress} $iopts"
             if [ -n "$compress_threads" ]; then
-                iopts="$iopts --compress-threads=$compress_threads"
+                iopts="--compress-threads=$compress_threads $iopts"
             fi
             if [ -n "$compress_chunk" ]; then
-                iopts="$iopts --compress-chunk-size=$compress_chunk"
+                iopts="--compress-chunk-size=$compress_chunk $iopts"
             fi
         fi
 
+        if [ -n "$backup_threads" ]; then
+            iopts="--parallel=$backup_threads $iopts"
+        fi
+
         setup_commands
         set +e
         timeit "$stagemsg-SST" "$INNOBACKUP | $tcmd; RC=( "\${PIPESTATUS[@]}" )"
@@ -1082,7 +1044,7 @@ then
         fi
 
         # mariabackup implicitly writes PID to fixed location in $xtmpdir
-        MARIABACKUP_PID="$xtmpdir/xtrabackup_pid"
+        BACKUP_PID="$xtmpdir/xtrabackup_pid"
 
     else # BYPASS FOR IST
 
@@ -1134,6 +1096,10 @@ then
 
     ib_undo_dir="$INNODB_UNDO_DIR"
 
+    if [ -n "$backup_threads" ]; then
+        impts="--parallel=$backup_threads $impts"
+    fi
+
     stagemsg='Joiner-Recv'
 
     sencrypted=1
@@ -1173,7 +1139,7 @@ then
     fi
 
     trap sig_joiner_cleanup HUP PIPE INT TERM
-    trap cleanup_joiner EXIT
+    trap cleanup_at_exit EXIT
 
     if [ -n "$progress" ]; then
         adjust_progress
@@ -1196,7 +1162,7 @@ then
 
     recv_joiner "$STATDIR" "$stagemsg-gtid" $stimeout 1 1
 
-    if ! ps -p "$WSREP_SST_OPT_PARENT" &>/dev/null
+    if ! ps -p "$WSREP_SST_OPT_PARENT" >/dev/null 2>&1
     then
         wsrep_log_error "Parent mysqld process (PID: $WSREP_SST_OPT_PARENT) terminated unexpectedly."
         exit 32
@@ -1316,7 +1282,7 @@ then
 
         fi
 
-        wsrep_log_info "Preparing the backup at ${DATA}"
+        wsrep_log_info "Preparing the backup at $DATA"
         setup_commands
         timeit "mariabackup prepare stage" "$INNOAPPLY"
 
@@ -1327,26 +1293,26 @@ then
 
         MAGIC_FILE="$TDATA/$INFO_FILE"
 
-        wsrep_log_info "Moving the backup to ${TDATA}"
+        wsrep_log_info "Moving the backup to $TDATA"
         timeit "mariabackup move stage" "$INNOMOVE"
         if [ $? -eq 0 ]; then
-            wsrep_log_info "Move successful, removing ${DATA}"
+            wsrep_log_info "Move successful, removing $DATA"
             rm -rf "$DATA"
             DATA="$TDATA"
         else
-            wsrep_log_error "Move failed, keeping ${DATA} for further diagnosis"
+            wsrep_log_error "Move failed, keeping '$DATA' for further diagnosis"
             wsrep_log_error "Check syslog or '$INNOMOVELOG' for details"
             exit 22
         fi
 
     else
 
-        wsrep_log_info "${IST_FILE} received from donor: Running IST"
+        wsrep_log_info "'$IST_FILE' received from donor: Running IST"
 
     fi
 
     if [ ! -r "$MAGIC_FILE" ]; then
-        wsrep_log_error "SST magic file ${MAGIC_FILE} not found/readable"
+        wsrep_log_error "SST magic file '$MAGIC_FILE' not found/readable"
         exit 2
     fi
 

scripts/wsrep_sst_mysqldump.sh

@@ -103,7 +103,7 @@ then
     DROP PREPARE stmt;"
 fi
 
-MYSQL="$MYSQL_CLIENT $WSREP_SST_OPT_CONF "\
+MYSQL="$MYSQL_CLIENT $WSREP_SST_OPT_CONF_UNQUOTED "\
 "$AUTH -h$WSREP_SST_OPT_HOST_UNESCAPED "\
 "-P$WSREP_SST_OPT_PORT --disable-reconnect --connect_timeout=10"
 
@@ -140,7 +140,7 @@ then
 fi
 
 # NOTE: we don't use --routines here because we're dumping mysql.proc table
-MYSQLDUMP="$MYSQLDUMP $WSREP_SST_OPT_CONF $AUTH -S$WSREP_SST_OPT_SOCKET \
+MYSQLDUMP="$MYSQLDUMP $WSREP_SST_OPT_CONF_UNQUOTED $AUTH -S$WSREP_SST_OPT_SOCKET \
 --add-drop-database --add-drop-table --skip-add-locks --create-options \
 --disable-keys --extended-insert --skip-lock-tables --quick --set-charset \
 --skip-comments --flush-privileges --all-databases --events"

scripts/wsrep_sst_rsync.sh

 #!/bin/bash -ue
 
-# Copyright (C) 2010-2014 Codership Oy
 # Copyright (C) 2017-2021 MariaDB
+# Copyright (C) 2010-2014 Codership Oy
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -19,9 +19,8 @@
 
 # This is a reference script for rsync-based state snapshot tansfer
 
-RSYNC_PID=                                      # rsync pid file
-RSYNC_CONF=                                     # rsync configuration file
-RSYNC_REAL_PID=                                 # rsync process id
+RSYNC_REAL_PID=0   # rsync process id
+STUNNEL_REAL_PID=0 # stunnel process id
 
 OS="$(uname)"
 [ "$OS" = 'Darwin' ] && export -n LD_LIBRARY_PATH
@@ -36,95 +35,95 @@ wsrep_check_programs rsync
 
 cleanup_joiner()
 {
-    wsrep_log_info "Joiner cleanup. rsync PID: $RSYNC_REAL_PID"
-    [ "0" != "$RSYNC_REAL_PID" ]            && \
-    kill $RSYNC_REAL_PID                    && \
-    sleep 0.5                               && \
-    kill -9 $RSYNC_REAL_PID >/dev/null 2>&1 || :
-    [ -f "$RSYNC_CONF"   ] && rm -f "$RSYNC_CONF"
-    [ -f "$STUNNEL_CONF" ] && rm -f "$STUNNEL_CONF"
-    [ -f "$STUNNEL_PID"  ] && rm -f "$STUNNEL_PID"
-    [ -f "$MAGIC_FILE"   ] && rm -f "$MAGIC_FILE"
-    [ -f "$RSYNC_PID"    ] && rm -f "$RSYNC_PID"
+    local failure=0
+
+    wsrep_log_info "Joiner cleanup: rsync PID=$RSYNC_REAL_PID, stunnel PID=$STUNNEL_REAL_PID"
+
+    if [ -n "$STUNNEL" ]; then
+        if cleanup_pid $STUNNEL_REAL_PID "$STUNNEL_PID" "$STUNNEL_CONF"; then
+            if [ $RSYNC_REAL_PID -eq 0 ]; then
+                if [ -r "$RSYNC_PID" ]; then
+                    RSYNC_REAL_PID=$(cat "$RSYNC_PID" 2>/dev/null)
+                    if [ -z "$RSYNC_REAL_PID" ]; then
+                        RSYNC_REAL_PID=0
+                    fi
+                fi
+            fi
+        else
+            wsrep_log_warning "stunnel cleanup failed."
+            failure=1
+        fi
+    fi
+
+    if [ $failure -eq 0 ]; then
+        if cleanup_pid $RSYNC_REAL_PID "$RSYNC_PID" "$RSYNC_CONF"; then
+            [ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE"
+        else
+            wsrep_log_warning "rsync cleanup failed."
+        fi
+    fi
+
     wsrep_log_info "Joiner cleanup done."
+
     if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then
         wsrep_cleanup_progress_file
     fi
 }
 
-# Check whether rsync process is still running.
-check_pid()
-{
-    local pid_file="$1"
-    [ -r "$pid_file" ] && ps -p $(cat "$pid_file") 2>&1 >/dev/null
-}
-
 check_pid_and_port()
 {
     local pid_file="$1"
-    local rsync_pid=$2
-    local rsync_addr="$3"
-    local rsync_port="$4"
+    local pid=$2
+    local addr="$3"
+    local port="$4"
 
-    if [ -z "$rsync_port" -o -z "$rsync_addr" -o -z "$rsync_pid" ]; then
-        wsrep_log_error "check_pid_and_port(): bad arguments"
-        exit 2 # ENOENT
-    fi
+    local utils='rsync|stunnel'
 
-    local port_info is_rsync
-
-    if [ $lsof_available -ne 0 ]; then
-        port_info=$(lsof -i ":$rsync_port" -Pn 2>/dev/null | \
-            grep -F '(LISTEN)')
-        is_rsync=$(echo "$port_info" | \
-            grep -E "^(rsync|stunnel)[^[:space:]]*[[:space:]]+$rsync_pid[[:space:]]+")
-    elif [ $sockstat_available -ne 0 ]; then
-        port_info=$(sockstat -p "$rsync_port" 2>/dev/null | \
-            grep -F 'LISTEN')
-        is_rsync=$(echo "$port_info" | \
-            grep -E "[[:space:]]+(rsync|stunnel)[^[:space:]]*[[:space:]]+$rsync_pid[[:space:]]+")
-    elif [ $ss_available -ne 0 ]; then
-        port_info=$(ss -H -p -n -l "( sport = :$rsync_port )" 2>/dev/null)
-        is_rsync=$(echo "$port_info" | \
-            grep -E "users:\\(.*\\(\"(rsync|stunnel)[^[:space:]]*\".*\<pid=$rsync_pid\>.*\\)")
-    else
-        wsrep_log_error "unknown sockets utility"
-        exit 2 # ENOENT
-    fi
+    if ! check_port "$pid" "$port" "$utils"; then
+        local port_info
+        local busy=0
 
-    if [ -z "$is_rsync" ]; then
-        local is_listening_all
         if [ $lsof_available -ne 0 ]; then
-            is_listening_all=$(echo "$port_info" | \
-                grep -E "[[:space:]](\\*|\\[?::\\]?):$rsync_port[[:space:]]")
+            port_info=$(lsof -Pnl -i ":$port" 2>/dev/null | \
+                grep -F '(LISTEN)')
+            echo "$port_info" | \
+            grep -q -E "[[:space:]](\\*|\\[?::\\]?):$port[[:space:]]" && busy=1
         else
-            if [ $sockstat_available -eq 0 ]; then
-                 port_info=$(echo "$port_info" | grep -q -F 'users:(')
+            local filter='([^[:space:]]+[[:space:]]+){4}[^[:space:]]+'
+            if [ $sockstat_available -eq 1 ]; then
+                port_info=$(sockstat -p "$port" 2>/dev/null | \
+                    grep -E '[[:space:]]LISTEN' | grep -o -E "$filter")
+            else
+                port_info=$(ss -nlpH "( sport = :$port )" 2>/dev/null | \
+                    grep -F 'users:(' | grep -o -E "$filter")
             fi
-            port_info=$(echo "$port_info" | \
-                grep -E "[^[:space:]]+[[:space:]]+[^[:space:]]+[[:space:]]+[^[:space:]]+[[:space:]]+[^[:space:]]+[[:space:]]+[^[:space:]]+" -o)
-            is_listening_all=$(echo "$port_info" | \
-                grep -E "[[:space:]](\\*|\\[?::\\]?):$rsync_port\$")
+            echo "$port_info" | \
+            grep -q -E "[[:space:]](\\*|\\[?::\\]?):$port\$" && busy=1
         fi
-        local is_listening_addr=$(echo "$port_info" | \
-            grep -w -F -- "$rsync_addr:$rsync_port")
-        if [ -z "$is_listening_addr" ]; then
-            is_listening_addr=$(echo "$port_info" | \
-                grep -w -F "[$rsync_addr]:$rsync_port")
+
+        if [ $busy -eq 0 ]; then
+            if echo "$port_info" | grep -qw -F "[$addr]:$port" || \
+               echo "$port_info" | grep -qw -F -- "$addr:$port"
+            then
+                busy=1
+            fi
         fi
-        if [ -n "$is_listening_all" -o -n "$is_listening_addr" ]; then
-            wsrep_log_error "rsync or stunnel daemon port '$rsync_port' " \
+
+        if [ $busy -eq 0 ]; then
+            return 1
+        fi
+
+        if ! check_port "$pid" "$port" "$utils"; then
+            wsrep_log_error "rsync or stunnel daemon port '$port' " \
                             "has been taken by another program"
             exit 16 # EBUSY
         fi
-        return 1
     fi
 
-    check_pid "$pid_file" && [ $(cat "$pid_file") -eq $rsync_pid ]
+    check_pid "$pid_file" && [ $CHECK_PID -eq $pid ]
 }
 
 STUNNEL_CONF="$WSREP_SST_OPT_DATA/stunnel.conf"
-
 STUNNEL_PID="$WSREP_SST_OPT_DATA/stunnel.pid"
 
 MAGIC_FILE="$WSREP_SST_OPT_DATA/rsync_sst_complete"
@@ -201,6 +200,8 @@ FILTER="-f '- /lost+found'
         -f '- /.zfs'
         -f '- /.fseventsd'
         -f '- /.Trashes'
+        -f '- /.pid'
+        -f '- /.conf'
         -f '+ /wsrep_sst_binlog.tar'
         -f '- $INNODB_DATA_HOME_DIR/ib_lru_dump'
         -f '- $INNODB_DATA_HOME_DIR/ibdata*'
@@ -266,7 +267,7 @@ then
         else
             # check if the address is an ip-address (v4 or v6):
             if echo "$WSREP_SST_OPT_HOST_UNESCAPED" | \
-               grep -q -E '^([0-9]+(\.[0-9]+){3,3}|[0-9a-fA-F]*(\:[0-9a-fA-F]*)+)$'
+               grep -q -E '^([0-9]+(\.[0-9]+){3}|[0-9a-fA-F]*(\:[0-9a-fA-F]*)+)$'
             then
                 CHECK_OPT="checkIP = $WSREP_SST_OPT_HOST_UNESCAPED"
             else
@@ -303,10 +304,10 @@ then
 
     [ -f "$MAGIC_FILE"      ] && rm -f "$MAGIC_FILE"
     [ -f "$BINLOG_TAR_FILE" ] && rm -f "$BINLOG_TAR_FILE"
+    [ -f "$STUNNEL_PID"     ] && rm -f "$STUNNEL_PID"
 
     if [ -n "$STUNNEL" ]
     then
-        [ -f "$STUNNEL_PID" ] && rm -f "$STUNNEL_PID"
         cat << EOF > "$STUNNEL_CONF"
 key = $SSTKEY
 cert = $SSTCERT
@@ -321,6 +322,8 @@ ${VERIFY_OPT}
 ${CHECK_OPT}
 ${CHECK_OPT_LOCAL}
 EOF
+    else
+        [ -f "$STUNNEL_CONF" ] && rm -f "$STUNNEL_CONF"
     fi
 
     if [ $WSREP_SST_OPT_BYPASS -eq 0 ]
@@ -329,13 +332,8 @@ EOF
         FLUSHED="$WSREP_SST_OPT_DATA/tables_flushed"
         ERROR="$WSREP_SST_OPT_DATA/sst_error"
 
-        rm -rf "$FLUSHED"
-        rm -rf "$ERROR"
-
-        # Use deltaxfer only for WAN
-        inv=$(basename "$0")
-        [ "$inv" = "wsrep_sst_rsync_wan" ] && WHOLE_FILE_OPT="" \
-                                           || WHOLE_FILE_OPT="--whole-file"
+        [ -f "$FLUSHED" ] && rm -f "$FLUSHED"
+        [ -f "$ERROR"   ] && rm -f "$ERROR"
 
         echo "flush tables"
 
@@ -350,15 +348,14 @@ EOF
             if [ -f "$ERROR" ]
             then
                 # Flush tables operation failed.
-                rm -rf "$ERROR"
+                rm -f "$ERROR"
                 exit 255
             fi
-
             sleep 0.2
         done
 
         STATE=$(cat "$FLUSHED")
-        rm -rf "$FLUSHED"
+        rm -f "$FLUSHED"
 
         sync
 
@@ -385,6 +382,13 @@ EOF
             cd "$OLD_PWD"
         fi
 
+        # Use deltaxfer only for WAN
+        inv=$(basename "$0")
+        WHOLE_FILE_OPT=""
+        if [ "${inv%wsrep_sst_rsync_wan*}" != "$inv" ]; then
+            WHOLE_FILE_OPT="--whole-file"
+        fi
+
         # first, the normal directories, so that we can detect incompatible protocol
         RC=0
         eval rsync ${STUNNEL:+"'--rsh=$STUNNEL'"} \
@@ -436,16 +440,18 @@ EOF
         fi
 
         # then, we parallelize the transfer of database directories,
-        # use . so that path concatenation works:
+        # use '.' so that path concatenation works:
 
         cd "$WSREP_SST_OPT_DATA"
 
-        count=1
-        [ "$OS" = 'Linux' ] && count=$(grep -c processor /proc/cpuinfo)
-        [ "$OS" = 'Darwin' -o "$OS" = 'FreeBSD' ] && count=$(sysctl -n hw.ncpu)
+        backup_threads=$(parse_cnf "--mysqld|sst" 'backup-threads')
+        if [ -z "$backup_threads" ]; then
+            get_proc
+            backup_threads=$nproc
+        fi
 
         find . -maxdepth 1 -mindepth 1 -type d -not -name 'lost+found' \
-             -not -name '.zfs' -print0 | xargs -I{} -0 -P $count \
+             -not -name '.zfs' -print0 | xargs -I{} -0 -P $backup_threads \
              rsync ${STUNNEL:+--rsh="$STUNNEL"} \
              --owner --group --perms --links --specials \
              --ignore-times --inplace --recursive --delete --quiet \
@@ -484,35 +490,52 @@ EOF
 
     echo "done $STATE"
 
+    if [ -n "$STUNNEL" ]; then
+        [ -f "$STUNNEL_CONF" ] && rm -f "$STUNNEL_CONF"
+        [ -f "$STUNNEL_PID"  ] && rm -f "$STUNNEL_PID"
+    fi
+
 elif [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]
 then
     check_sockets_utils
 
-    touch "$SST_PROGRESS_FILE"
-    MYSQLD_PID="$WSREP_SST_OPT_PARENT"
+    # give some time for lingering stunnel from previous SST to complete
+    check_round=0
+    while check_pid "$STUNNEL_PID" 1
+    do
+        wsrep_log_info "lingering stunnel daemon found at startup, waiting for it to exit"
+        check_round=$(( check_round + 1 ))
+        if [ $check_round -eq 10 ]; then
+            wsrep_log_error "stunnel daemon already running."
+            exit 114 # EALREADY
+        fi
+        sleep 1
+    done
 
     MODULE="rsync_sst"
-
     RSYNC_PID="$WSREP_SST_OPT_DATA/$MODULE.pid"
+    RSYNC_CONF="$WSREP_SST_OPT_DATA/$MODULE.conf"
+
     # give some time for lingering rsync from previous SST to complete
     check_round=0
-    while check_pid "$RSYNC_PID" && [ $check_round -lt 10 ]
+    while check_pid "$RSYNC_PID" 1
     do
         wsrep_log_info "lingering rsync daemon found at startup, waiting for it to exit"
         check_round=$(( check_round + 1 ))
+        if [ $check_round -eq 10 ]; then
+            wsrep_log_error "rsync daemon already running."
+            exit 114 # EALREADY
+        fi
         sleep 1
     done
 
-    if check_pid "$RSYNC_PID"
-    then
-        wsrep_log_error "rsync daemon already running."
-        exit 114 # EALREADY
-    fi
-
-    [ -f "$RSYNC_PID"       ] && rm -f "$RSYNC_PID"
     [ -f "$MAGIC_FILE"      ] && rm -f "$MAGIC_FILE"
     [ -f "$BINLOG_TAR_FILE" ] && rm -f "$BINLOG_TAR_FILE"
 
+    if [ -z "$STUNNEL" ]; then
+        [ -f "$STUNNEL_CONF" ] && rm -f "$STUNNEL_CONF"
+    fi
+
     ADDR="$WSREP_SST_OPT_ADDR"
     RSYNC_PORT="$WSREP_SST_OPT_PORT"
     RSYNC_ADDR="$WSREP_SST_OPT_HOST"
@@ -522,7 +545,7 @@ then
     trap "exit 3"  INT TERM ABRT
     trap cleanup_joiner EXIT
 
-    RSYNC_CONF="$WSREP_SST_OPT_DATA/$MODULE.conf"
+    touch "$SST_PROGRESS_FILE"
 
     if [ -n "${MYSQL_TMP_DIR:-}" ]; then
         SILENT="log file = $MYSQL_TMP_DIR/rsyncd.log"
@@ -545,18 +568,18 @@ $SILENT
     path = $INNODB_DATA_HOME_DIR
 EOF
 
-#    rm -rf "$DATA"/ib_logfile* # we don't want old logs around
+#   rm -rf "$DATA/ib_logfile"* # we don't want old logs around
 
-    # If the IP is local listen only in it
+    # If the IP is local, listen only on it:
     if is_local_ip "$RSYNC_ADDR_UNESCAPED"
     then
         RSYNC_EXTRA_ARGS="--address $RSYNC_ADDR_UNESCAPED"
         STUNNEL_ACCEPT="$RSYNC_ADDR_UNESCAPED:$RSYNC_PORT"
     else
-        # Not local, possibly a NAT, listen on all interfaces
+        # Not local, possibly a NAT, listen on all interfaces:
         RSYNC_EXTRA_ARGS=""
         STUNNEL_ACCEPT="$RSYNC_PORT"
-        # Overwrite address with all
+        # Overwrite address with all:
         RSYNC_ADDR="*"
     fi
 
@@ -564,8 +587,9 @@ EOF
     then
         rsync --daemon --no-detach --port "$RSYNC_PORT" --config "$RSYNC_CONF" $RSYNC_EXTRA_ARGS &
         RSYNC_REAL_PID=$!
+        TRANSFER_REAL_PID="$RSYNC_REAL_PID"
+        TRANSFER_PID=$RSYNC_PID
     else
-        [ -f "$STUNNEL_PID" ] && rm -f "$STUNNEL_PID"
         # Let's check if the path to the config file contains a space?
         if [ "${RSYNC_CONF#* }" = "$RSYNC_CONF" ]; then
             cat << EOF > "$STUNNEL_CONF"
@@ -606,15 +630,11 @@ execargs = $SHELL -c \$RSYNC_CMD
 EOF
         fi
         stunnel "$STUNNEL_CONF" &
-        RSYNC_REAL_PID=$!
-        RSYNC_PID="$STUNNEL_PID"
+        STUNNEL_REAL_PID=$!
+        TRANSFER_REAL_PID="$STUNNEL_REAL_PID"
+        TRANSFER_PID=$STUNNEL_PID
     fi
 
-    until check_pid_and_port "$RSYNC_PID" "$RSYNC_REAL_PID" "$RSYNC_ADDR_UNESCAPED" "$RSYNC_PORT"
-    do
-        sleep 0.2
-    done
-
     if [ "${SSLMODE#VERIFY}" != "$SSLMODE" ]
     then # backward-incompatible behavior
         CN=""
@@ -635,19 +655,26 @@ EOF
         ADDR="$CN:$MY_SECRET@$WSREP_SST_OPT_HOST"
     else
         MY_SECRET="" # for check down in recv_joiner()
-        ADDR=$WSREP_SST_OPT_HOST
+        ADDR="$WSREP_SST_OPT_HOST"
     fi
 
+    until check_pid_and_port "$TRANSFER_PID" $TRANSFER_REAL_PID "$RSYNC_ADDR_UNESCAPED" "$RSYNC_PORT"
+    do
+        sleep 0.2
+    done
+
     echo "ready $ADDR:$RSYNC_PORT/$MODULE"
 
+    MYSQLD_PID="$WSREP_SST_OPT_PARENT"
+
     # wait for SST to complete by monitoring magic file
-    while [ ! -r "$MAGIC_FILE" ] && check_pid "$RSYNC_PID" && \
-          ps -p $MYSQLD_PID >/dev/null
+    while [ ! -r "$MAGIC_FILE" ] && check_pid "$TRANSFER_PID" && \
+          ps -p $MYSQLD_PID >/dev/null 2>&1
     do
         sleep 1
     done
 
-    if ! ps -p $MYSQLD_PID >/dev/null
+    if ! ps -p $MYSQLD_PID >/dev/null 2>&1
     then
         wsrep_log_error \
         "Parent mysqld process (PID: $MYSQLD_PID) terminated unexpectedly."
@@ -698,7 +725,7 @@ EOF
         echo "rsync process ended without creating '$MAGIC_FILE'"
     fi
 
-    wsrep_cleanup_progress_file
+#   wsrep_cleanup_progress_file
 #   cleanup_joiner
 else
     wsrep_log_error "Unrecognized role: '$WSREP_SST_OPT_ROLE'"

scripts/wsrep_sst_xtrabackup-v2.sh

 #!/bin/bash -ue
-# Copyright (C) 2013 Percona Inc
 # Copyright (C) 2017-2021 MariaDB
+# Copyright (C) 2013 Percona Inc
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -23,27 +23,31 @@
 . $(dirname "$0")/wsrep_sst_common
 wsrep_check_datadir
 
+OS="$(uname)"
 ealgo=""
+eformat=""
 ekey=""
 ekeyfile=""
 encrypt=0
-nproc=1
 ecode=0
 ssyslog=""
 ssystag=""
-XTRABACKUP_PID=""
+BACKUP_PID=""
 tcert=""
 tpem=""
 tkey=""
+tmode="DISABLED"
 sockopt=""
 progress=""
 ttime=0
 totime=0
+lsn=""
 ecmd=""
 rlimit=""
 # Initially
-stagemsg="${WSREP_SST_OPT_ROLE}"
+stagemsg="$WSREP_SST_OPT_ROLE"
 cpat=""
+speciald=1
 ib_home_dir=""
 ib_log_dir=""
 ib_undo_dir=""
@@ -67,41 +71,54 @@ xtmpdir=""
 
 scomp=""
 sdecomp=""
+
 ssl_dhparams=""
 
-ssl_cert=""
-ssl_ca=""
-ssl_key=""
+compress='none'
+compress_chunk=""
+compress_threads=""
+
+backup_threads=""
+
+encrypt_threads=""
+encrypt_chunk=""
+
+readonly SECRET_TAG="secret"
+
+sst_ver=-1
 
 if [ -n "$(command -v pv)" ] && pv --help | grep -qw -- '-F'; then
-    pvopts+=$pvformat
+    pvopts="$pvopts $pvformat"
 fi
 pcmd="pv $pvopts"
 declare -a RC
 
-INNOBACKUPEX_BIN='innobackupex'
-DATA="${WSREP_SST_OPT_DATA}"
+BACKUP_BIN="$(command -v innobackupex)"
+if [ ! -x "$BACKUP_BIN" ]; then
+    wsrep_log_error 'innobackupex binary not found in path'
+    exit 42
+fi
+
+DATA="$WSREP_SST_OPT_DATA"
 INFO_FILE="xtrabackup_galera_info"
 IST_FILE="xtrabackup_ist"
-MAGIC_FILE="${DATA}/${INFO_FILE}"
+MAGIC_FILE="$DATA/$INFO_FILE"
+
+INNOAPPLYLOG="$DATA/innobackupex.prepare.log"
+INNOMOVELOG="$DATA/innobackupex.move.log"
+INNOBACKUPLOG="$DATA/innobackupex.backup.log"
 
 # Setting the path for ss and ip
 export PATH="/usr/sbin:/sbin:$PATH"
 
-OS="$(uname)"
-
-if [ -z "$(command -v lsof)" ]; then
-    wsrep_log_error "lsof tool not found in PATH! Make sure you have it installed."
-    exit 2 # ENOENT
-fi
-
-timeit(){
+timeit()
+{
     local stage="$1"
     shift
     local cmd="$@"
     local x1 x2 took extcode
 
-    if [[ $ttime -eq 1 ]];then
+    if [ $ttime -eq 1 ]; then
         x1=$(date +%s)
         wsrep_log_info "Evaluating $cmd"
         eval "$cmd"
@@ -121,105 +138,89 @@ timeit(){
 get_keys()
 {
     # $encrypt -eq 1 is for internal purposes only
-    if [[ $encrypt -ge 2 || $encrypt -eq -1 ]];then
+    if [ $encrypt -ge 2 -o $encrypt -eq -1 ]; then
         return
     fi
 
-    if [[ $encrypt -eq 0 ]];then
-        if $MY_PRINT_DEFAULTS xtrabackup | grep -q encrypt;then
-            wsrep_log_error "Unexpected option combination. SST may fail. Refer to http://www.percona.com/doc/percona-xtradb-cluster/manual/xtrabackup_sst.html "
+    if [ $encrypt -eq 0 ]; then
+        if [ -n "$ealgo" -o -n "$ekey" -o -n "$ekeyfile" ]; then
+            wsrep_log_error "Options for encryption are specified, " \
+                            "but encryption itself is disabled. SST may fail."
         fi
         return
     fi
 
-    if [[ "$sfmt" == 'tar' ]];then
-        wsrep_log_info "NOTE: Xtrabackup-based encryption - encrypt=1 - cannot be enabled with tar format"
+    if [ $sfmt = 'tar' ]; then
+        wsrep_log_info "NOTE: key-based encryption (encrypt=1) " \
+                       "cannot be enabled with tar format"
         encrypt=-1
         return
     fi
 
-    wsrep_log_info "Xtrabackup based encryption enabled in my.cnf - Supported only from Xtrabackup 2.1.4"
+    wsrep_log_info "Key based encryption enabled in my.cnf - supported only from Xtrabackup 2.1.4"
 
-    if [[ -z "$ealgo" ]];then
+    if [ -z "$ealgo" ]; then
         wsrep_log_error "FATAL: Encryption algorithm empty from my.cnf, bailing out"
         exit 3
     fi
 
-    if [[ -z "$ekey" && ! -r "$ekeyfile" ]];then
+    if [ -z "$ekey" -a ! -r "$ekeyfile" ]; then
         wsrep_log_error "FATAL: Either key or keyfile must be readable"
         exit 3
     fi
 
-    if [[ -z "$ekey" ]];then
-        ecmd="xbcrypt --encrypt-algo=$ealgo --encrypt-key-file=$ekeyfile"
+    if [ "$eformat" = 'openssl' ]; then
+        get_openssl
+        if [ -z "$OPENSSL_BINARY" ]; then
+            wsrep_log_error "If encryption using the openssl is enabled, " \
+                            "then you need to install openssl"
+            exit 2
+        fi
+        ecmd="'$OPENSSL_BINARY' enc -$ealgo"
+        if "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -- '-pbkdf2'; then
+            ecmd="$ecmd -pbkdf2"
+        elif "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -- '-iter'; then
+            ecmd="$ecmd -iter 1"
+        elif "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -- '-md'; then
+            ecmd="$ecmd -md sha256"
+        fi
+        if [ -z "$ekey" ]; then
+            ecmd="$ecmd -kfile '$ekeyfile'"
+        else
+            ecmd="$ecmd -k '$ekey'"
+        fi
+    elif [ "$eformat" = 'xbcrypt' ]; then
+        if [ -z "$(command -v xbcrypt)" ]; then
+            wsrep_log_error "If encryption using the xbcrypt is enabled, " \
+                            "then you need to install xbcrypt"
+            exit 2
+        fi
+        wsrep_log_info "NOTE: xbcrypt-based encryption, " \
+                       "supported only from Xtrabackup 2.1.4"
+        if [ -z "$ekey" ]; then
+            ecmd="xbcrypt --encrypt-algo='$ealgo' --encrypt-key-file='$ekeyfile'"
+        else
+            wsrep_log_warning "Using the 'encrypt-key' option causes the encryption key"
+            wsrep_log_warning "to be set via the command-line and is considered insecure."
+            wsrep_log_warning "It is recommended to use the 'encrypt-key-file' option instead."
+            ecmd="xbcrypt --encrypt-algo='$ealgo' --encrypt-key='$ekey'"
+        fi
+        if [ -n "$encrypt_threads" ]; then
+            ecmd="$ecmd --encrypt-threads=$encrypt_threads"
+        fi
+        if [ -n "$encrypt_chunk" ]; then
+            ecmd="$ecmd --encrypt-chunk-size=$encrypt_chunk"
+        fi
     else
-        wsrep_log_warning "Using the 'encrypt-key' option causes the encryption key"
-        wsrep_log_warning "to be set via the command-line and is considered insecure."
-        wsrep_log_warning "It is recommended to use the 'encrypt-key-file' option instead."
-
-        ecmd="xbcrypt --encrypt-algo=$ealgo --encrypt-key=$ekey"
+        wsrep_log_error "Unknown encryption format='$eformat'"
+        exit 2
     fi
 
-    if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then
-        ecmd+=" -d"
+    if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then
+        ecmd="$ecmd -d"
     fi
 
-    stagemsg+="-XB-Encrypted"
-}
-
-#
-# verifies that the certificate matches the private key
-# doing this will save us having to wait for a timeout that would
-# otherwise occur.
-#
-# 1st param: path to the cert
-# 2nd param: path to the private key
-#
-verify_cert_matches_key()
-{
-    local cert_path="$1"
-    local key_path="$2"
-
-    wsrep_check_programs openssl diff
-
-    # generate the public key from the cert and the key
-    # they should match (otherwise we can't create an SSL connection)
-    if ! diff <(openssl x509 -in "$cert_path" -pubkey -noout) <(openssl rsa -in "$key_path" -pubout 2>/dev/null) >/dev/null 2>&1
-    then
-        wsrep_log_error "******** FATAL ERROR ************************* "
-        wsrep_log_error "* The certifcate and private key do not match. "
-        wsrep_log_error "* Please check your certificate and key files. "
-        wsrep_log_error "********************************************** "
-        exit 22
-    fi
-}
-
-# Checks to see if the file exists
-# If the file does not exist (or cannot be read), issues an error
-# and exits
-#
-# 1st param: file name to be checked (for read access)
-# 2nd param: 1st error message (header)
-# 3rd param: 2nd error message (footer, optional)
-#
-verify_file_exists()
-{
-    local file_path="$1"
-    local error_message1="$2"
-    local error_message2="$3"
-
-    if ! [[ -r "$file_path" ]]; then
-        wsrep_log_error "******** FATAL ERROR ************************* "
-        wsrep_log_error "* $error_message1 "
-        wsrep_log_error "* Could not find/access : $file_path "
-
-        if ! [[ -z "$error_message2" ]]; then
-            wsrep_log_error "* $error_message2 "
-        fi
-
-        wsrep_log_error "********************************************** "
-        exit 22
-    fi
+    stagemsg="$stagemsg-XB-Encrypted"
 }
 
 get_transfer()
@@ -310,15 +311,6 @@ get_transfer()
             exit 2
         fi
 
-        # Determine the socat version
-        SOCAT_VERSION=$(socat -V 2>&1 | grep -m1 -oe '[0-9]\.[0-9][\.0-9]*')
-        if [ -z "$SOCAT_VERSION" ]; then
-            wsrep_log_error "******** FATAL ERROR ******************"
-            wsrep_log_error "* Cannot determine the socat version. *"
-            wsrep_log_error "***************************************"
-            exit 2
-        fi
-
         local action='Decrypting'
         if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then
             tcmd="socat -u openssl-listen:$SST_PORT,reuseaddr"
@@ -327,11 +319,25 @@ get_transfer()
             action='Encrypting'
         fi
 
-        if ! check_for_version "$SOCAT_VERSION" '1.7.3'; then
-            # socat versions < 1.7.3 will have 512-bit dhparams (too small)
-            # so create 2048-bit dhparams and send that as a parameter:
-            check_for_dhparams
-            tcmd="$tcmd,dhparam='$ssl_dhparams'"
+        if [ "${sockopt#*,dhparam=}" != "$sockopt" ]; then
+            if [ -z "$ssl_dhparams" ]; then
+                # Determine the socat version
+                SOCAT_VERSION=$(socat -V 2>&1 | grep -m1 -oe '[0-9]\.[0-9][\.0-9]*')
+                if [ -z "$SOCAT_VERSION" ]; then
+                    wsrep_log_error "******** FATAL ERROR ******************"
+                    wsrep_log_error "* Cannot determine the socat version. *"
+                    wsrep_log_error "***************************************"
+                    exit 2
+                fi
+                if ! check_for_version "$SOCAT_VERSION" '1.7.3'; then
+                    # socat versions < 1.7.3 will have 512-bit dhparams (too small)
+                    # so create 2048-bit dhparams and send that as a parameter:
+                    check_for_dhparams
+                fi
+            fi
+            if [ -n "$ssl_dhparams" ]; then
+                tcmd="$tcmd,dhparam='$ssl_dhparams'"
+            fi
         fi
 
         if [ $encrypt -eq 2 ]; then
@@ -340,6 +346,11 @@ get_transfer()
                 wsrep_log_error "Both PEM and CRT files required"
                 exit 22
             fi
+            if [ ! -r "$tpem" -o ! -r "$tcert" ]; then
+                wsrep_log_error "Both PEM and CRT files must be readable"
+                exit 22
+            fi
+            verify_ca_matches_cert "$tcert" "$tpem"
             tcmd="$tcmd,cert='$tpem',cafile='$tcert'$sockopt"
             stagemsg="$stagemsg-OpenSSL-Encrypted-2"
             wsrep_log_info "$action with cert=$tpem, cafile=$tcert"
@@ -349,6 +360,11 @@ get_transfer()
                 wsrep_log_error "Both certificate and key files required"
                 exit 22
             fi
+            if [ ! -r "$tpem" -o ! -r "$tkey" ]; then
+                wsrep_log_error "Both certificate and key files must be readable"
+                exit 22
+            fi
+            verify_cert_matches_key "$tpem" "$tkey"
             stagemsg="$stagemsg-OpenSSL-Encrypted-3"
             if [ -z "$tcert" ]; then
                 if [ $encrypt -eq 4 ]; then
@@ -360,6 +376,11 @@ get_transfer()
                 wsrep_log_info "$action with cert=$tpem, key=$tkey, verify=0"
             else
                 # CA verification
+                if [ ! -r "$tcert" ]; then
+                    wsrep_log_error "Certificate file must be readable"
+                    exit 22
+                fi
+                verify_ca_matches_cert "$tcert" "$tpem"
                 if [ -n "$WSREP_SST_OPT_REMOTE_USER" ]; then
                     CN_option=",commonname='$WSREP_SST_OPT_REMOTE_USER'"
                 elif [ $encrypt -eq 4 ]; then
@@ -387,13 +408,13 @@ get_footprint()
 {
     pushd "$WSREP_SST_OPT_DATA" 1>/dev/null
     payload=$(find . -regex '.*\.ibd$\|.*\.MYI$\|.*\.MYD$\|.*ibdata1$' -type f -print0 | du --files0-from=- --block-size=1 -c -s | awk 'END { print $1 }')
-    if [ $(in_config 'xtrabackup' 'compress') -eq 1 ]; then
+    if [ "$compress" != 'none' ]; then
         # QuickLZ has around 50% compression ratio
         # When compression/compaction used, the progress is only an approximate.
         payload=$(( payload*1/2 ))
     fi
     popd 1>/dev/null
-    pcmd+=" -s $payload"
+    pcmd="$pcmd -s $payload"
     adjust_progress
 }
 
@@ -408,93 +429,141 @@ adjust_progress()
         return
     fi
 
-    if [[ -n "$progress" && "$progress" != '1' ]];then
-        if [[ -e "$progress" ]];then
-            pcmd+=" 2>>$progress"
+    if [ -n "$progress" -a "$progress" != '1' ]; then
+        if [ -e "$progress" ]; then
+            pcmd="$pcmd 2>>'$progress'"
         else
-            pcmd+=" 2>$progress"
+            pcmd="$pcmd 2>'$progress'"
         fi
-    elif [[ -z "$progress" && -n "$rlimit" ]];then
+    elif [ -z "$progress" -a -n "$rlimit" ]; then
             # When rlimit is non-zero
             pcmd="pv -q"
     fi
 
-    if [[ -n "$rlimit" && "$WSREP_SST_OPT_ROLE" == "donor" ]];then
+    if [ -n "$rlimit" -a "$WSREP_SST_OPT_ROLE" = 'donor' ]; then
         wsrep_log_info "Rate-limiting SST to $rlimit"
-        pcmd+=" -L \$rlimit"
+        pcmd="$pcmd -L \$rlimit"
     fi
 }
 
+encgroups='--mysqld|sst|xtrabackup'
+
+check_server_ssl_config()
+{
+    tcert=$(parse_cnf "$encgroups" 'ssl-ca')
+    tpem=$(parse_cnf "$encgroups" 'ssl-cert')
+    tkey=$(parse_cnf "$encgroups" 'ssl-key')
+}
+
 read_cnf()
 {
-    sfmt=$(parse_cnf sst streamfmt "xbstream")
-    tfmt=$(parse_cnf sst transferfmt "socat")
-    tcert=$(parse_cnf sst tca "")
-    tpem=$(parse_cnf sst tcert "")
-    tkey=$(parse_cnf sst tkey "")
-    encrypt=$(parse_cnf sst encrypt 0)
+    sfmt=$(parse_cnf sst streamfmt 'xbstream')
+    tfmt=$(parse_cnf sst transferfmt 'socat')
+
+    encrypt=$(parse_cnf "$encgroups" 'encrypt' 0)
+    tmode=$(parse_cnf "$encgroups" 'ssl-mode' 'DISABLED' | tr [:lower:] [:upper:])
+
+    if [ $encrypt -eq 0 -o $encrypt -ge 2 ]
+    then
+        if [ "$tmode" != 'DISABLED' -o $encrypt -ge 2 ]
+        then
+            tcert=$(parse_cnf 'sst' 'tca')
+            tpem=$(parse_cnf 'sst' 'tcert')
+            tkey=$(parse_cnf 'sst' 'tkey')
+        fi
+        if [ "$tmode" != 'DISABLED' ]; then
+            # backward-incompatible behavior
+            if [ -z "$tpem" -a -z "$tkey" -a -z "$tcert" ]; then
+                # no old-style SSL config in [sst]
+                check_server_ssl_config
+            fi
+            if [ 0 -eq $encrypt -a -n "$tpem" -a -n "$tkey" ]
+            then
+                encrypt=3 # enable cert/key SSL encyption
+
+                # avoid CA verification if not set explicitly:
+                # nodes may happen to have different CA if self-generated
+                # zeroing up tcert does the trick
+                [ "${tmode#VERIFY}" != "$tmode" ] || tcert=""
+            fi
+        fi
+    elif [ $encrypt -eq 1 ]; then
+        ealgo=$(parse_cnf "$encgroups" 'encrypt-algo')
+        eformat=$(parse_cnf "$encgroups" 'encrypt-format' 'xbcrypt')
+        ekey=$(parse_cnf "$encgroups" 'encrypt-key')
+        ekeyfile=$(parse_cnf "$encgroups" 'encrypt-key-file')
+    fi
+
+    wsrep_log_info "SSL configuration: CA='$tcert', CERT='$tpem'," \
+                   "KEY='$tkey', MODE='$tmode', encrypt='$encrypt'"
+
     sockopt=$(parse_cnf sst sockopt "")
     progress=$(parse_cnf sst progress "")
     rebuild=$(parse_cnf sst rebuild 0)
     ttime=$(parse_cnf sst time 0)
     cpat='.*\.pem$\|.*init\.ok$\|.*galera\.cache$\|.*sst_in_progress$\|.*\.sst$\|.*gvwstate\.dat$\|.*grastate\.dat$\|.*\.err$\|.*\.log$\|.*RPM_UPGRADE_MARKER$\|.*RPM_UPGRADE_HISTORY$'
     [ "$OS" = 'FreeBSD' ] && cpat=$(echo "$cpat" | sed 's/\\|/|/g')
-    ealgo=$(parse_cnf xtrabackup encrypt "")
-    ekey=$(parse_cnf xtrabackup encrypt-key "")
-    ekeyfile=$(parse_cnf xtrabackup encrypt-key-file "")
+    cpat=$(parse_cnf sst cpat "$cpat")
     scomp=$(parse_cnf sst compressor "")
     sdecomp=$(parse_cnf sst decompressor "")
 
-    # Refer to http://www.percona.com/doc/percona-xtradb-cluster/manual/xtrabackup_sst.html
-    if [[ -z "$ealgo" ]];then
-        ealgo=$(parse_cnf sst encrypt-algo "")
-        ekey=$(parse_cnf sst encrypt-key "")
-        ekeyfile=$(parse_cnf sst encrypt-key-file "")
-    fi
-
-    # Pull the parameters needed for encrypt=4
-    ssl_ca=$(parse_cnf sst ssl-ca "")
-    if [[ -z "$ssl_ca" ]]; then
-        ssl_ca=$(parse_cnf --mysqld ssl-ca "")
-    fi
-    ssl_cert=$(parse_cnf sst ssl-cert "")
-    if [[ -z "$ssl_cert" ]]; then
-        ssl_cert=$(parse_cnf --mysqld ssl-cert "")
-    fi
-    ssl_key=$(parse_cnf sst ssl-key "")
-    if [[ -z "$ssl_key" ]]; then
-        ssl_key=$(parse_cnf --mysqld ssl-key "")
-    fi
-
     rlimit=$(parse_cnf sst rlimit "")
     uextra=$(parse_cnf sst use-extra 0)
+    speciald=$(parse_cnf sst sst-special-dirs 1)
     iopts=$(parse_cnf sst inno-backup-opts "")
     iapts=$(parse_cnf sst inno-apply-opts "")
     impts=$(parse_cnf sst inno-move-opts "")
-    stimeout=$(parse_cnf sst sst-initial-timeout 100)
+    stimeout=$(parse_cnf sst sst-initial-timeout 300)
     ssyslog=$(parse_cnf sst sst-syslog 0)
     ssystag=$(parse_cnf mysqld_safe syslog-tag "${SST_SYSLOG_TAG:-}")
-    ssystag+="-"
+    ssystag="$ssystag-"
+    sstlogarchive=$(parse_cnf sst sst-log-archive 1)
+    sstlogarchivedir=$(parse_cnf sst sst-log-archive-dir '/tmp/sst_log_archive')
+
+    if [ $speciald -eq 0 ]; then
+        wsrep_log_error "sst-special-dirs equal to 0 is not supported, falling back to 1"
+        speciald=1
+    fi
 
     if [ $ssyslog -ne -1 ]; then
         ssyslog=$(in_config 'mysqld_safe' 'syslog')
     fi
+
+    if [ "$WSREP_SST_OPT_ROLE" = 'donor' ]; then
+        compress=$(parse_cnf "$encgroups" 'compress' 'none')
+        if [ "$compress" != 'none' ]; then
+            compress_chunk=$(parse_cnf "$encgroups" 'compress-chunk-size')
+            compress_threads=$(parse_cnf "$encgroups" 'compress-threads')
+        fi
+    fi
+
+    backup_threads=$(parse_cnf "$encgroups" 'backup-threads')
+
+    if [ "$eformat" = 'xbcrypt' ]; then
+        encrypt_threads=$(parse_cnf "$encgroups" 'encrypt-threads')
+        encrypt_chunk=$(parse_cnf "$encgroups" 'encrypt-chunk-size')
+    fi
 }
 
 get_stream()
 {
     if [ "$sfmt" = 'mbstream' -o "$sfmt" = 'xbstream' ]; then
-        XBSTREAM_BIN=$(command -v "$sfmt")
-        if [ -z "$XBSTREAM_BIN" ]; then
-            if [ -z "$XBSTREAM_BIN" ]; then
-                wsrep_log_error "Streaming with $sfmt, but $sfmt not found in path"
-                exit 42
+        STREAM_BIN=$(command -v "$sfmt")
+        if [ -z "$STREAM_BIN" ]; then
+            if [ "$sfmt" = 'xbstream' ]; then
+                STREAM_BIN="$(command -v mbstream)"
+            else
+                STREAM_BIN="$(command -v xbstream)"
             fi
         fi
+        if [ -z "$STREAM_BIN" ]; then
+            wsrep_log_error "Streaming with $sfmt, but $sfmt not found in path"
+            exit 42
+        fi
         if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then
-            strmcmd="'$XBSTREAM_BIN' -x"
+            strmcmd="'$STREAM_BIN' -x"
         else
-            strmcmd="'$XBSTREAM_BIN' -c '$INFO_FILE'"
+            strmcmd="'$STREAM_BIN' -c '$INFO_FILE'"
         fi
     else
         sfmt='tar'
@@ -507,91 +576,47 @@ get_stream()
     wsrep_log_info "Streaming with $sfmt"
 }
 
-get_proc()
-{
-    set +e
-    nproc=$(grep -c processor /proc/cpuinfo)
-    [[ -z $nproc || $nproc -eq 0 ]] && nproc=1
-    set -e
-}
-
 sig_joiner_cleanup()
 {
     wsrep_log_error "Removing $MAGIC_FILE file due to signal"
-    rm -f "$MAGIC_FILE"
-}
-
-cleanup_joiner()
-{
-    # Since this is invoked just after exit NNN
-    local estatus=$?
-    if [[ $estatus -ne 0 ]];then
-        wsrep_log_error "Cleanup after exit with status:$estatus"
-    elif [ "${WSREP_SST_OPT_ROLE}" = "joiner" ];then
-        wsrep_log_info "Removing the sst_in_progress file"
-        wsrep_cleanup_progress_file
-    fi
-    if [[ -n "$progress" && -p "$progress" ]];then
-        wsrep_log_info "Cleaning up fifo file $progress"
-        rm $progress
-    fi
-    if [[ -n "${STATDIR:-}" ]];then
-       [[ -d "$STATDIR" ]] && rm -rf "$STATDIR"
-    fi
-
-    # Final cleanup
-    pgid=$(ps -o pgid= $$ | grep -o '[0-9]*')
-
-    # This means no setsid done in mysqld.
-    # We don't want to kill mysqld here otherwise.
-    if [[ $$ -eq $pgid ]];then
-        # This means a signal was delivered to the process.
-        # So, more cleanup.
-        if [[ $estatus -ge 128 ]];then
-            kill -KILL -$$ || true
-        fi
-    fi
-
-    exit $estatus
-}
-
-check_pid()
-{
-    local pid_file="$1"
-    [ -r "$pid_file" ] && ps -p $(cat "$pid_file") >/dev/null 2>&1
+    [ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE"
 }
 
-cleanup_donor()
+cleanup_at_exit()
 {
     # Since this is invoked just after exit NNN
     local estatus=$?
-    if [[ $estatus -ne 0 ]];then
+    if [ $estatus -ne 0 ]; then
         wsrep_log_error "Cleanup after exit with status:$estatus"
     fi
 
-    if [[ -n "${XTRABACKUP_PID:-}" ]];then
-        if check_pid "$XTRABACKUP_PID"
-        then
-            wsrep_log_error "xtrabackup process is still running. Killing... "
-            kill_xtrabackup
+    if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then
+        wsrep_log_info "Removing the sst_in_progress file"
+        wsrep_cleanup_progress_file
+    else
+        if [ -n "$BACKUP_PID" ]; then
+            if check_pid "$BACKUP_PID" 1; then
+                wsrep_log_error "xtrabackup process is still running. Killing..."
+                cleanup_pid $CHECK_PID "$BACKUP_PID"
+            fi
         fi
+        [ -f "$DATA/$IST_FILE" ] && rm -f "$DATA/$IST_FILE"
     fi
 
-    rm -f "${DATA}/${IST_FILE}" || true
-
-    if [[ -n "$progress" && -p "$progress" ]];then
+    if [ -n "$progress" -a -p "$progress" ]; then
         wsrep_log_info "Cleaning up fifo file $progress"
         rm -f "$progress" || true
     fi
 
     wsrep_log_info "Cleaning up temporary directories"
 
-    if [[ -n "$xtmpdir" ]];then
-       [[ -d "$xtmpdir" ]] &&  rm -rf "$xtmpdir" || true
-    fi
-
-    if [[ -n "$itmpdir" ]];then
-       [[ -d "$itmpdir" ]] &&  rm -rf "$itmpdir" || true
+    if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then
+        if [ -n "$STATDIR" ]; then
+           [ -d "$STATDIR" ] && rm -rf "$STATDIR"
+        fi
+    else
+        [ -n "$xtmpdir" -a -d "$xtmpdir" ] && rm -rf "$xtmpdir" || true
+        [ -n "$itmpdir" -a -d "$itmpdir" ] && rm -rf "$itmpdir" || true
     fi
 
     # Final cleanup
@@ -599,44 +624,43 @@ cleanup_donor()
 
     # This means no setsid done in mysqld.
     # We don't want to kill mysqld here otherwise.
-    if [[ $$ -eq $pgid ]];then
-
+    if [ $$ -eq $pgid ]; then
         # This means a signal was delivered to the process.
         # So, more cleanup.
-        if [[ $estatus -ge 128 ]];then
-            kill -KILL -$$ || true
+        if [ $estatus -ge 128 ]; then
+            kill -KILL -- -$$ || true
         fi
-
     fi
 
     exit $estatus
 }
 
-kill_xtrabackup()
+setup_ports()
 {
-    local PID=$(cat "$XTRABACKUP_PID")
-    [ -n "$PID" -a "0" != "$PID" ] && kill $PID && (kill $PID && kill -9 $PID) || :
-    wsrep_log_info "Removing xtrabackup pid file $XTRABACKUP_PID"
-    rm -f "$XTRABACKUP_PID" || true
+    SST_PORT="$WSREP_SST_OPT_PORT"
+    if [ "$WSREP_SST_OPT_ROLE" = 'donor' ]; then
+        REMOTEIP="$WSREP_SST_OPT_HOST"
+        lsn="$WSREP_SST_OPT_LSN"
+        sst_ver="$WSREP_SST_OPT_SST_VER"
+    fi
 }
 
-# waits ~1 minute for nc/socat to open the port and then reports ready
-# (regardless of timeout)
+#
+# Waits ~30 seconds for socat or nc to open the port and
+# then reports ready, regardless of timeout.
+#
 wait_for_listen()
 {
-    local HOST="$1"
-    local PORT="$2"
+    local PORT="$1"
+    local ADDR="$2"
     local MODULE="$3"
-    local LSOF_OUT
-
-    for i in {1..300}
-    do
-        LSOF_OUT=$(lsof -sTCP:LISTEN -i TCP:${PORT} -a -c nc -c socat -F c 2> /dev/null || :)
-        [ -n "${LSOF_OUT}" ] && break
+    for i in {1..150}; do
+        if check_port "" "$PORT" 'socat|nc'; then
+            break
+        fi
         sleep 0.2
     done
-
-    echo "ready ${HOST}:${PORT}/${MODULE}//${WSREP_SST_OPT_SST_VER:-1}"
+    echo "ready $ADDR/$MODULE//$sst_ver"
 }
 
 check_extra()
@@ -648,9 +672,9 @@ check_extra()
             local eport=$(parse_cnf '--mysqld' 'extra-port')
             if [ -n "$eport" ]; then
                 # Xtrabackup works only locally.
-                # Hence, setting host to 127.0.0.1 unconditionally.
+                # Hence, setting host to 127.0.0.1 unconditionally:
                 wsrep_log_info "SST through extra_port $eport"
-                INNOEXTRA+=" --host=127.0.0.1 --port=$eport"
+                INNOEXTRA="$INNOEXTRA --host=127.0.0.1 --port=$eport"
                 use_socket=0
             else
                 wsrep_log_error "Extra port $eport null, failing"
@@ -661,7 +685,7 @@ check_extra()
         fi
     fi
     if [ $use_socket -eq 1 -a -n "$WSREP_SST_OPT_SOCKET" ]; then
-        INNOEXTRA+=" --socket='$WSREP_SST_OPT_SOCKET'"
+        INNOEXTRA="$INNOEXTRA --socket='$WSREP_SST_OPT_SOCKET'"
     fi
 }
 
@@ -671,49 +695,71 @@ recv_joiner()
     local msg="$2"
     local tmt=$3
     local checkf=$4
-    local ltcmd
+    local wait=$5
 
-    if [[ ! -d "${dir}" ]];then
+    if [ ! -d "$dir" ]; then
         # This indicates that IST is in progress
         return
     fi
 
-    pushd "${dir}" 1>/dev/null
+    local ltcmd="$tcmd"
+    if [ $tmt -gt 0 ]; then
+        if [ -n "$(command -v timeout)" ]; then
+            if timeout --help | grep -qw -- '-k'; then
+                ltcmd="timeout -k $(( tmt+10 )) $tmt $tcmd"
+            else
+                ltcmd="timeout -s9 $tmt $tcmd"
+            fi
+        fi
+    fi
+
+    pushd "$dir" 1>/dev/null
     set +e
 
-    if [ $tmt -gt 0 -a -n "$(command -v timeout)" ]; then
-        if timeout --help | grep -qw -- '-k';then
-            ltcmd="timeout -k $(( tmt+10 )) $tmt $tcmd"
-        else
-            ltcmd="timeout -s9 $tmt $tcmd"
-        fi
-        timeit "$msg" "$ltcmd | $strmcmd; RC=( "\${PIPESTATUS[@]}" )"
-    else
-        timeit "$msg" "$tcmd | $strmcmd; RC=( "\${PIPESTATUS[@]}" )"
+    if [ $wait -ne 0 ]; then
+        wait_for_listen "$SST_PORT" "$ADDR" "$MODULE" &
     fi
 
+    timeit "$msg" "$ltcmd | $strmcmd; RC=( "\${PIPESTATUS[@]}" )"
+
     set -e
     popd 1>/dev/null
 
-    if [[ ${RC[0]} -eq 124 ]];then
-        wsrep_log_error "Possible timeout in receving first data from donor in gtid stage"
+    if [ ${RC[0]} -eq 124 ]; then
+        wsrep_log_error "Possible timeout in receiving first data from " \
+                        "donor in gtid stage: exit codes: ${RC[@]}"
         exit 32
     fi
 
-    for ecode in "${RC[@]}";do
-        if [[ $ecode -ne 0 ]];then
+    for ecode in "${RC[@]}"; do
+        if [ $ecode -ne 0 ]; then
             wsrep_log_error "Error while getting data from donor node: " \
                             "exit codes: ${RC[@]}"
             exit 32
         fi
     done
 
-    if [[ $checkf -eq 1 && ! -r "${MAGIC_FILE}" ]];then
-        # this message should cause joiner to abort
-        wsrep_log_error "xtrabackup process ended without creating '${MAGIC_FILE}'"
-        wsrep_log_info "Contents of datadir"
-        wsrep_log_info $(ls -l "$dir/"*)
-        exit 32
+    if [ $checkf -eq 1 ]; then
+        if [ ! -r "$MAGIC_FILE" ]; then
+            # this message should cause joiner to abort
+            wsrep_log_error "receiving process ended without creating " \
+                            "'$MAGIC_FILE'"
+            wsrep_log_info "Contents of datadir"
+            wsrep_log_info $(ls -l "$dir/"*)
+            exit 32
+        fi
+
+        # check donor supplied secret
+        SECRET=$(grep -- "$SECRET_TAG " "$MAGIC_FILE" 2>/dev/null | cut -d ' ' -f 2)
+        if [ "$SECRET" != "$MY_SECRET" ]; then
+            wsrep_log_error "Donor does not know my secret!"
+            wsrep_log_info "Donor:'$SECRET', my:'$MY_SECRET'"
+            exit 32
+        fi
+
+        # remove secret from magic file
+        grep -v -- "$SECRET_TAG " "$MAGIC_FILE" > "$MAGIC_FILE.new"
+        mv "$MAGIC_FILE.new" "$MAGIC_FILE"
     fi
 }
 
@@ -722,14 +768,14 @@ send_donor()
     local dir="$1"
     local msg="$2"
 
-    pushd "${dir}" 1>/dev/null
+    pushd "$dir" 1>/dev/null
     set +e
     timeit "$msg" "$strmcmd | $tcmd; RC=( "\${PIPESTATUS[@]}" )"
     set -e
     popd 1>/dev/null
 
-    for ecode in "${RC[@]}";do
-        if [[ $ecode -ne 0 ]];then
+    for ecode in "${RC[@]}"; do
+        if [ $ecode -ne 0 ]; then
             wsrep_log_error "Error while sending data to joiner node: " \
                             "exit codes: ${RC[@]}"
             exit 32
@@ -742,68 +788,68 @@ monitor_process()
     local sst_stream_pid=$1
 
     while true ; do
-
-        if ! ps --pid "${WSREP_SST_OPT_PARENT}" &>/dev/null; then
+        if ! ps -p "$WSREP_SST_OPT_PARENT" >/dev/null 2>&1; then
             wsrep_log_error "Parent mysqld process (PID: $WSREP_SST_OPT_PARENT) terminated unexpectedly."
-            kill -- -"${WSREP_SST_OPT_PARENT}"
+            kill -- -"$WSREP_SST_OPT_PARENT"
             exit 32
         fi
-
-        if ! ps --pid "${sst_stream_pid}" &>/dev/null; then
+        if ! ps -p "$sst_stream_pid" >/dev/null 2>&1; then
             break
         fi
-
         sleep 0.1
-
     done
 }
 
-innobackup=$(command -v "$INNOBACKUPEX_BIN")
-if [ ! -x "$innobackup" ]; then
-    wsrep_log_error "innobackupex not in path: $PATH"
-    exit 2
-fi
-
 # check the version, we require XB-2.4 to ensure that we can pass the
 # datadir via the command-line option
 XB_REQUIRED_VERSION="2.3.5"
 
-XB_VERSION=`$INNOBACKUPEX_BIN --version 2>&1 | grep -oe '[0-9]\.[0-9][\.0-9]*' | head -n1`
+XB_VERSION=`$BACKUP_BIN --version 2>&1 | grep -oe '[0-9]\.[0-9][\.0-9]*' | head -n1`
 if [[ -z "$XB_VERSION" ]]; then
-    wsrep_log_error "FATAL: Cannot determine the $INNOBACKUPEX_BIN version. Needs xtrabackup-$XB_REQUIRED_VERSION or higher to perform SST"
+    wsrep_log_error "FATAL: Cannot determine the $BACKUP_BIN version. Needs xtrabackup-$XB_REQUIRED_VERSION or higher to perform SST"
     exit 2
 fi
 
 if ! check_for_version "$XB_VERSION" "$XB_REQUIRED_VERSION"; then
-    wsrep_log_error "FATAL: The $INNOBACKUPEX_BIN version is $XB_VERSION. Needs xtrabackup-$XB_REQUIRED_VERSION or higher to perform SST"
+    wsrep_log_error "FATAL: The $BACKUP_BIN version is $XB_VERSION. Needs xtrabackup-$XB_REQUIRED_VERSION or higher to perform SST"
     exit 2
 fi
 
-rm -f "${MAGIC_FILE}"
+[ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE"
 
-if [[ ! "${WSREP_SST_OPT_ROLE}" == 'joiner' && ! "${WSREP_SST_OPT_ROLE}" == 'donor' ]];then
-    wsrep_log_error "Invalid role ${WSREP_SST_OPT_ROLE}"
+if [ "$WSREP_SST_OPT_ROLE" != 'joiner' -a "$WSREP_SST_OPT_ROLE" != 'donor' ]; then
+    wsrep_log_error "Invalid role '$WSREP_SST_OPT_ROLE'"
     exit 22
 fi
 
 read_cnf
+setup_ports
 
-if ${INNOBACKUPEX_BIN} /tmp --help 2>/dev/null | grep -qw -- '--version-check'; then
+if "$BACKUP_BIN" --help 2>/dev/null | grep -qw -- '--version-check'; then
     disver='--no-version-check'
 fi
 
-iopts+=" --databases-exclude='lost+found'"
+# if no command line argument and INNODB_DATA_HOME_DIR environment variable
+# is not set, try to get it from my.cnf:
+if [ -z "$INNODB_DATA_HOME_DIR" ]; then
+    INNODB_DATA_HOME_DIR=$(parse_cnf '--mysqld' 'innodb-data-home-dir')
+fi
+
+OLD_PWD="$(pwd)"
 
-if [ ${FORCE_FTWRL:-0} -eq 1 ]; then
-    wsrep_log_info "Forcing FTWRL due to environment variable FORCE_FTWRL equal to $FORCE_FTWRL"
-    iopts+=' --no-backup-locks'
+cd "$WSREP_SST_OPT_DATA"
+if [ -n "$INNODB_DATA_HOME_DIR" ]; then
+    # handle both relative and absolute paths
+    [ ! -d "$INNODB_DATA_HOME_DIR" ] && mkdir -p "$INNODB_DATA_HOME_DIR"
+    cd "$INNODB_DATA_HOME_DIR"
 fi
+INNODB_DATA_HOME_DIR=$(pwd -P)
 
-if [[ $ssyslog -eq 1 ]];then
-    if [ -z "$(command -v logger)" ]; then
-        wsrep_log_error "logger not in path: $PATH. Ignoring"
-    else
-        wsrep_log_info "Logging all stderr of SST/Innobackupex to syslog"
+cd "$OLD_PWD"
+
+if [ $ssyslog -eq 1 ]; then
+    if [ -n "$(command -v logger)" ]; then
+        wsrep_log_info "Logging all stderr of SST/xtrabackup to syslog"
 
         exec 2> >(logger -p daemon.err -t ${ssystag}wsrep-sst-$WSREP_SST_OPT_ROLE)
 
@@ -816,69 +862,108 @@ if [[ $ssyslog -eq 1 ]];then
         {
             logger  -p daemon.info -t ${ssystag}wsrep-sst-$WSREP_SST_OPT_ROLE "$@"
         }
-
-        INNOAPPLY="2>&1 | logger -p daemon.err -t ${ssystag}innobackupex-apply"
-        INNOMOVE="2>&1 | logger -p daemon.err -t ${ssystag}innobackupex-move"
-        INNOBACKUP="2> >(logger -p daemon.err -t ${ssystag}innobackupex-backup)"
+    else
+        wsrep_log_error "logger not in path: $PATH. Ignoring"
     fi
+    INNOAPPLY="2>&1 | logger -p daemon.err -t ${ssystag}innobackupex-apply"
+    INNOMOVE="2>&1 | logger -p daemon.err -t ${ssystag}innobackupex-move"
+    INNOBACKUP="2> >(logger -p daemon.err -t ${ssystag}innobackupex-backup)"
 else
-    INNOAPPLY="&>'$DATA/innobackup.prepare.log'"
-    INNOMOVE="&>'$DATA/innobackup.move.log'"
-    INNOBACKUP="2>'$DATA/innobackup.backup.log'"
-fi
+    if [ $sstlogarchive -eq 1 ]
+    then
+        ARCHIVETIMESTAMP=$(date "+%Y.%m.%d-%H.%M.%S.%N")
 
-get_stream
-get_transfer
+        if [ -n "$sstlogarchivedir" ]; then
+            if [ ! -d "$sstlogarchivedir" ]; then
+                mkdir -p "$sstlogarchivedir"
+            fi
+        fi
 
-# if no command line argument and INNODB_DATA_HOME_DIR environment variable
-# is not set, try to get it from my.cnf:
-if [ -z "$INNODB_DATA_HOME_DIR" ]; then
-    INNODB_DATA_HOME_DIR=$(parse_cnf '--mysqld' 'innodb-data-home-dir')
-fi
+        if [ -e "$INNOAPPLYLOG" ]
+        then
+            if [ -n "$sstlogarchivedir" ]
+            then
+                newfile=$(basename "$INNOAPPLYLOG")
+                newfile="$sstlogarchivedir/$newfile.$ARCHIVETIMESTAMP"
+            else
+                newfile="$INNOAPPLYLOG.$ARCHIVETIMESTAMP"
+            fi
+            wsrep_log_info "Moving '$INNOAPPLYLOG' to '$newfile'"
+            mv "$INNOAPPLYLOG" "$newfile"
+            gzip "$newfile"
+        fi
 
-OLD_PWD="$(pwd)"
+        if [ -e "$INNOMOVELOG" ]
+        then
+            if [ -n "$sstlogarchivedir" ]
+            then
+                newfile=$(basename "$INNOMOVELOG")
+                newfile="$sstlogarchivedir/$newfile.$ARCHIVETIMESTAMP"
+            else
+                newfile="$INNOMOVELOG.$ARCHIVETIMESTAMP"
+            fi
+            wsrep_log_info "Moving '$INNOMOVELOG' to '$newfile'"
+            mv "$INNOMOVELOG" "$newfile"
+            gzip "$newfile"
+        fi
 
-cd "$WSREP_SST_OPT_DATA"
-if [ -n "$INNODB_DATA_HOME_DIR" ]; then
-    # handle both relative and absolute paths
-    [ ! -d "$INNODB_DATA_HOME_DIR" ] && mkdir -p "$INNODB_DATA_HOME_DIR"
-    cd "$INNODB_DATA_HOME_DIR"
+        if [ -e "$INNOBACKUPLOG" ]
+        then
+            if [ -n "$sstlogarchivedir" ]
+            then
+                newfile=$(basename "$INNOBACKUPLOG")
+                newfile="$sstlogarchivedir/$newfile.$ARCHIVETIMESTAMP"
+            else
+                newfile="$INNOBACKUPLOG.$ARCHIVETIMESTAMP"
+            fi
+            wsrep_log_info "Moving '$INNOBACKUPLOG' to '$newfile'"
+            mv "$INNOBACKUPLOG" "$newfile"
+            gzip "$newfile"
+        fi
+    fi
+    INNOAPPLY="> '$INNOAPPLYLOG' 2>&1"
+    INNOMOVE="> '$INNOMOVELOG' 2>&1"
+    INNOBACKUP="2> '$INNOBACKUPLOG'"
 fi
-INNODB_DATA_HOME_DIR=$(pwd -P)
-
-cd "$OLD_PWD"
 
-setup_commands () {
-    INNOAPPLY="$INNOBACKUPEX_BIN $disver $iapts $INNOEXTRA --apply-log $rebuildcmd '$DATA' $INNOAPPLY"
-    INNOMOVE="$INNOBACKUPEX_BIN $WSREP_SST_OPT_CONF $disver $impts --move-back --force-non-empty-directories '$DATA' $INNOMOVE"
+setup_commands()
+{
+    INNOAPPLY="$BACKUP_BIN $disver $iapts $INNOEXTRA --apply-log $rebuildcmd '$DATA' $INNOAPPLY"
+    INNOMOVE="$BACKUP_BIN $WSREP_SST_OPT_CONF --move-back $disver $impts --force-non-empty-directories '$DATA' $INNOMOVE"
     sfmt_work="$sfmt"
     if [ "$sfmt" = 'mbstream' ]; then
         sfmt_work='xbstream'
     fi
-    INNOBACKUP="$INNOBACKUPEX_BIN $WSREP_SST_OPT_CONF $disver $iopts $tmpopts $INNOEXTRA --galera-info --stream=$sfmt_work '$itmpdir' $INNOBACKUP"
+    INNOBACKUP="$BACKUP_BIN $WSREP_SST_OPT_CONF $disver $iopts $tmpopts $INNOEXTRA --galera-info --stream=$sfmt_work '$itmpdir' $INNOBACKUP"
 }
 
-if [ "$WSREP_SST_OPT_ROLE" = "donor" ]
+get_stream
+get_transfer
+
+if [ "$WSREP_SST_OPT_ROLE" = 'donor' ]
 then
-    trap cleanup_donor EXIT
+    trap cleanup_at_exit EXIT
 
     if [ $WSREP_SST_OPT_BYPASS -eq 0 ]
     then
-        if [ -z "$WSREP_SST_OPT_SST_VER" ]; then
+        if [ -z "$sst_ver" ]; then
             wsrep_log_error "Upgrade joiner to 5.6.21 or higher for backup locks support"
             wsrep_log_error "The joiner is not supported for this version of donor"
             exit 93
         fi
 
-        tmpdir=$(parse_cnf "--mysqld|sst|xtrabackup" 'tmpdir')
+        tmpdir=$(parse_cnf "$encgroups" 'tmpdir')
         if [ -z "$tmpdir" ]; then
-            xtmpdir=$(mktemp -d)
-            tmpopts=" --tmpdir='$xtmpdir'"
-            wsrep_log_info "Using $xtmpdir as xtrabackup temporary directory"
+            xtmpdir="$(mktemp -d)"
+        else
+            xtmpdir=$(mktemp '-d' "--tmpdir=$tmpdir")
         fi
 
-        itmpdir=$(mktemp -d)
-        wsrep_log_info "Using $itmpdir as innobackupex temporary directory"
+        wsrep_log_info "Using '$xtmpdir' as xtrabackup temporary directory"
+        tmpopts="--tmpdir='$xtmpdir'"
+
+        itmpdir="$(mktemp -d)"
+        wsrep_log_info "Using '$itmpdir' as xtrabackup working directory"
 
         usrst=0
         if [ -n "$WSREP_SST_OPT_USER" ]; then
@@ -887,37 +972,41 @@ then
         fi
 
         if [ -n "$WSREP_SST_OPT_PSWD" ]; then
-            export MYSQL_PWD="$WSREP_SST_OPT_PSWD"
+            INNOEXTRA="$INNOEXTRA --password='$WSREP_SST_OPT_PSWD'"
         elif [ $usrst -eq 1 ]; then
             # Empty password, used for testing, debugging etc.
-            unset MYSQL_PWD
+            INNOEXTRA="$INNOEXTRA --password="
         fi
 
-        get_keys
         check_extra
 
         wsrep_log_info "Streaming GTID file before SST"
 
         # Store donor's wsrep GTID (state ID) and wsrep_gtid_domain_id
         # (separated by a space).
-        echo "${WSREP_SST_OPT_GTID} ${WSREP_SST_OPT_GTID_DOMAIN_ID}" > "${MAGIC_FILE}"
+        echo "$WSREP_SST_OPT_GTID $WSREP_SST_OPT_GTID_DOMAIN_ID" > "$MAGIC_FILE"
+
+        if [ -n "$WSREP_SST_OPT_REMOTE_PSWD" ]; then
+            # Let joiner know that we know its secret
+            echo "$SECRET_TAG $WSREP_SST_OPT_REMOTE_PSWD" >> "$MAGIC_FILE"
+        fi
 
         ttcmd="$tcmd"
 
-        if [ $encrypt -eq 1 ]; then
-            if [ -n "$scomp" ]; then
-                tcmd="\$ecmd | $scomp | $tcmd"
-            else
-                tcmd="\$ecmd | $tcmd"
-            fi
-        elif [ -n "$scomp" ]; then
+        if [ -n "$scomp" ]; then
             tcmd="$scomp | $tcmd"
         fi
 
-        send_donor $DATA "${stagemsg}-gtid"
+        get_keys
+        if [ $encrypt -eq 1 ]; then
+            tcmd="$ecmd | $tcmd"
+        fi
+
+        send_donor "$DATA" "$stagemsg-gtid"
 
         # Restore the transport commmand to its original state
         tcmd="$ttcmd"
+
         if [ -n "$progress" ]; then
             get_footprint
             tcmd="$pcmd | $tcmd"
@@ -929,34 +1018,57 @@ then
         wsrep_log_info "Sleeping before data transfer for SST"
         sleep 10
 
-        wsrep_log_info "Streaming the backup to joiner at ${WSREP_SST_OPT_HOST}:${WSREP_SST_OPT_PORT}"
+        wsrep_log_info "Streaming the backup to joiner at $REMOTEIP:$SST_PORT"
 
         # Add compression to the head of the stream (if specified)
-        if [[ -n "$scomp" ]]; then
+        if [ -n "$scomp" ]; then
             tcmd="$scomp | $tcmd"
         fi
 
         # Add encryption to the head of the stream (if specified)
-        if [[ $encrypt -eq 1 ]]; then
-            tcmd="\$ecmd | $tcmd"
+        if [ $encrypt -eq 1 ]; then
+            tcmd="$ecmd | $tcmd"
+        fi
+
+        iopts="--databases-exclude='lost+found' $iopts"
+
+        if [ ${FORCE_FTWRL:-0} -eq 1 ]; then
+            wsrep_log_info "Forcing FTWRL due to environment variable FORCE_FTWRL equal to $FORCE_FTWRL"
+            iopts="--no-backup-locks $iopts"
+        fi
+
+        # if compression is enabled for backup files, then add the
+        # appropriate options to the innobackupex command line:
+        if [ "$compress" != 'none' ]; then
+            iopts="--compress${compress:+=$compress} $iopts"
+            if [ -n "$compress_threads" ]; then
+                iopts="--compress-threads=$compress_threads $iopts"
+            fi
+            if [ -n "$compress_chunk" ]; then
+                iopts="--compress-chunk-size=$compress_chunk $iopts"
+            fi
+        fi
+
+        if [ -n "$backup_threads" ]; then
+            iopts="--parallel=$backup_threads $iopts"
         fi
 
         setup_commands
         set +e
-        timeit "${stagemsg}-SST" "$INNOBACKUP | $tcmd; RC=( "\${PIPESTATUS[@]}" )"
+        timeit "$stagemsg-SST" "$INNOBACKUP | $tcmd; RC=( "\${PIPESTATUS[@]}" )"
         set -e
 
         if [ ${RC[0]} -ne 0 ]; then
-          wsrep_log_error "${INNOBACKUPEX_BIN} finished with error: ${RC[0]}. " \
-                          "Check ${DATA}/innobackup.backup.log"
-          exit 22
-        elif [[ ${RC[$(( ${#RC[@]}-1 ))]} -eq 1 ]];then
-          wsrep_log_error "$tcmd finished with error: ${RC[1]}"
-          exit 22
+            wsrep_log_error "innobackupex finished with error: ${RC[0]}. " \
+                            "Check syslog or '$INNOBACKUPLOG' for details"
+            exit 22
+        elif [ ${RC[$(( ${#RC[@]}-1 ))]} -eq 1 ]; then
+            wsrep_log_error "$tcmd finished with error: ${RC[1]}"
+            exit 22
         fi
 
         # innobackupex implicitly writes PID to fixed location in $xtmpdir
-        XTRABACKUP_PID="$xtmpdir/xtrabackup_pid"
+        BACKUP_PID="$xtmpdir/xtrabackup_pid"
 
     else # BYPASS FOR IST
 
@@ -965,31 +1077,31 @@ then
 
         # Store donor's wsrep GTID (state ID) and wsrep_gtid_domain_id
         # (separated by a space).
-        echo "${WSREP_SST_OPT_GTID} ${WSREP_SST_OPT_GTID_DOMAIN_ID}" > "${MAGIC_FILE}"
-        echo "1" > "${DATA}/${IST_FILE}"
+        echo "$WSREP_SST_OPT_GTID $WSREP_SST_OPT_GTID_DOMAIN_ID" > "$MAGIC_FILE"
+        echo "1" > "$DATA/$IST_FILE"
+
+        if [ -n "$scomp" ]; then
+            tcmd="$scomp | $tcmd"
+        fi
+
         get_keys
         if [ $encrypt -eq 1 ]; then
-            if [ -n "$scomp" ]; then
-                tcmd="\$ecmd | $scomp | $tcmd"
-            else
-                tcmd="\$ecmd | $tcmd"
-            fi
-        elif [ -n "$scomp" ]; then
-            tcmd="$scomp | $tcmd"
+            tcmd="$ecmd | $tcmd"
         fi
-        strmcmd+=" \${IST_FILE}"
 
-        send_donor "$DATA" "${stagemsg}-IST"
+        strmcmd="$strmcmd '$IST_FILE'"
+
+        send_donor "$DATA" "$stagemsg-IST"
 
     fi
 
-    echo "done ${WSREP_SST_OPT_GTID}"
+    echo "done $WSREP_SST_OPT_GTID"
     wsrep_log_info "Total time on donor: $totime seconds"
 
-elif [ "${WSREP_SST_OPT_ROLE}" = "joiner" ]
+elif [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]
 then
-    [[ -e "$SST_PROGRESS_FILE" ]] && wsrep_log_info "Stale sst_in_progress file: $SST_PROGRESS_FILE"
-    [[ -n "$SST_PROGRESS_FILE" ]] && touch "$SST_PROGRESS_FILE"
+    [ -e "$SST_PROGRESS_FILE" ] && wsrep_log_info "Stale sst_in_progress file: $SST_PROGRESS_FILE"
+    [ -n "$SST_PROGRESS_FILE" ] && touch "$SST_PROGRESS_FILE"
 
     ib_home_dir="$INNODB_DATA_HOME_DIR"
 
@@ -1008,66 +1120,101 @@ then
 
     ib_undo_dir="$INNODB_UNDO_DIR"
 
-    stagemsg="Joiner-Recv"
+    if [ -n "$backup_threads" ]; then
+        impts="--parallel=$backup_threads $impts"
+    fi
+
+    stagemsg='Joiner-Recv'
 
     sencrypted=1
     nthreads=1
 
     MODULE="xtrabackup_sst"
 
-    rm -f "${DATA}/${IST_FILE}"
+    [ -f "$DATA/$IST_FILE" ] && rm -f "$DATA/$IST_FILE"
 
     # May need xtrabackup_checkpoints later on
-    rm -f "${DATA}/xtrabackup_binary" "${DATA}/xtrabackup_galera_info" "${DATA}/xtrabackup_logfile"
+    [ -f "$DATA/xtrabackup_binary"      ] && rm -f "$DATA/xtrabackup_binary"
+    [ -f "$DATA/xtrabackup_galera_info" ] && rm -f "$DATA/xtrabackup_galera_info"
+    [ -f "$DATA/ib_logfile0"            ] && rm -f "$DATA/ib_logfile0"
+
+    ADDR="$WSREP_SST_OPT_ADDR"
 
-    wait_for_listen ${WSREP_SST_OPT_HOST} ${WSREP_SST_OPT_PORT} ${MODULE} &
+    if [ "${tmode#VERIFY}" != "$tmode" ]
+    then # backward-incompatible behavior
+        CN=""
+        if [ -n "$tpem" ]
+        then
+            # find out my Common Name
+            get_openssl
+            if [ -z "$OPENSSL_BINARY" ]; then
+                wsrep_log_error 'openssl not found but it is required for authentication'
+                exit 42
+            fi
+            CN=$("$OPENSSL_BINARY" x509 -noout -subject -in "$tpem" | \
+                 tr "," "\n" | grep -F 'CN =' | cut -d= -f2 | sed s/^\ // | \
+                 sed s/\ %//)
+        fi
+        MY_SECRET="$(wsrep_gen_secret)"
+        # Add authentication data to address
+        ADDR="$CN:$MY_SECRET@$ADDR"
+    else
+        MY_SECRET="" # for check down in recv_joiner()
+    fi
 
     trap sig_joiner_cleanup HUP PIPE INT TERM
-    trap cleanup_joiner EXIT
+    trap cleanup_at_exit EXIT
 
-    if [[ -n "$progress" ]];then
+    if [ -n "$progress" ]; then
         adjust_progress
-        tcmd+=" | $pcmd"
+        tcmd="$tcmd | $pcmd"
     fi
 
     get_keys
     if [ $encrypt -eq 1 -a $sencrypted -eq 1 ]; then
-        if [ -n "$sdecomp" ]; then
-            strmcmd="$sdecomp | \$ecmd | $strmcmd"
-        else
-            strmcmd="\$ecmd | $strmcmd"
-        fi
-    elif [ -n "$sdecomp" ]; then
-            strmcmd="$sdecomp | $strmcmd"
+        strmcmd="$ecmd | $strmcmd"
     fi
 
-    STATDIR=$(mktemp -d)
-    MAGIC_FILE="${STATDIR}/${INFO_FILE}"
-    recv_joiner "$STATDIR" "${stagemsg}-gtid" $stimeout 1
+    if [ -n "$sdecomp" ]; then
+        strmcmd="$sdecomp | $strmcmd"
+    fi
+
+    check_sockets_utils
+
+    STATDIR="$(mktemp -d)"
+    MAGIC_FILE="$STATDIR/$INFO_FILE"
 
-    if ! ps -p ${WSREP_SST_OPT_PARENT} &>/dev/null
+    recv_joiner "$STATDIR" "$stagemsg-gtid" $stimeout 1 1
+
+    if ! ps -p "$WSREP_SST_OPT_PARENT" >/dev/null 2>&1
     then
         wsrep_log_error "Parent mysqld process (PID: $WSREP_SST_OPT_PARENT) terminated unexpectedly."
         exit 32
     fi
 
-    if [ ! -r "${STATDIR}/${IST_FILE}" ]
+    if [ ! -r "$STATDIR/$IST_FILE" ]
     then
 
-        if [[ -d "${DATA}/.sst" ]];then
-            wsrep_log_info "WARNING: Stale temporary SST directory: ${DATA}/.sst from previous state transfer. Removing"
-            rm -rf "${DATA}/.sst"
+        if [ -d "$DATA/.sst" ]; then
+            wsrep_log_info "WARNING: Stale temporary SST directory: '$DATA/.sst' from previous state transfer. Removing"
+            rm -rf "$DATA/.sst"
         fi
-        mkdir -p "${DATA}/.sst"
-        (recv_joiner "$DATA/.sst" "${stagemsg}-SST" 0 0) &
+        mkdir -p "$DATA/.sst"
+        (recv_joiner "$DATA/.sst" "$stagemsg-SST" 0 0 0) &
         jpid=$!
         wsrep_log_info "Proceeding with SST"
 
         wsrep_log_info "Cleaning the existing datadir and innodb-data/log directories"
-        if [ "${OS}" = "FreeBSD" ]; then
-            find -E $ib_home_dir $ib_log_dir $ib_undo_dir $DATA -mindepth 1 -prune -regex $cpat -o -exec rm -rfv {} 1>&2 \+
+        if [ "$OS" = 'FreeBSD' ]; then
+            find -E ${ib_home_dir:+"$ib_home_dir"} \
+                    ${ib_undo_dir:+"$ib_undo_dir"} \
+                    ${ib_log_dir:+"$ib_log_dir"} \
+                    "$DATA" -mindepth 1 -prune -regex "$cpat" -o -exec rm -rfv {} 1>&2 \+
         else
-            find $ib_home_dir $ib_log_dir $ib_undo_dir $DATA -mindepth 1 -prune -regex $cpat -o -exec rm -rfv {} 1>&2 \+
+            find ${ib_home_dir:+"$ib_home_dir"} \
+                 ${ib_undo_dir:+"$ib_undo_dir"} \
+                 ${ib_log_dir:+"$ib_log_dir"} \
+                 "$DATA" -mindepth 1 -prune -regex "$cpat" -o -exec rm -rfv {} 1>&2 \+
         fi
 
         get_binlog
@@ -1082,50 +1229,46 @@ then
             cd "$OLD_PWD"
         fi
 
-        TDATA="${DATA}"
-        DATA="${DATA}/.sst"
+        TDATA="$DATA"
+        DATA="$DATA/.sst"
 
-        MAGIC_FILE="${DATA}/${INFO_FILE}"
+        MAGIC_FILE="$DATA/$INFO_FILE"
         wsrep_log_info "Waiting for SST streaming to complete!"
         monitor_process $jpid
 
-        get_proc
-
-        if [[ ! -s "${DATA}/xtrabackup_checkpoints" ]];then
-            wsrep_log_error "xtrabackup_checkpoints missing, failed innobackupex/SST on donor"
+        if [ ! -s "$DATA/xtrabackup_checkpoints" ]; then
+            wsrep_log_error "xtrabackup_checkpoints missing, failed xtrabackup/SST on donor"
             exit 2
         fi
 
         # Rebuild indexes for compact backups
-        if grep -q 'compact = 1' "${DATA}/xtrabackup_checkpoints";then
+        if grep -q -F 'compact = 1' "$DATA/xtrabackup_checkpoints"; then
             wsrep_log_info "Index compaction detected"
-            rebuild=1
-        fi
-
-        if [[ $rebuild -eq 1 ]];then
-            nthreads=$(parse_cnf xtrabackup rebuild-threads $nproc)
+            get_proc
+            nthreads=$(parse_cnf "$encgroups" 'rebuild-threads' $nproc)
             wsrep_log_info "Rebuilding during prepare with $nthreads threads"
-            rebuildcmd="--rebuild-indexes --rebuild-threads='$nthreads'"
+            rebuildcmd="--rebuild-indexes --rebuild-threads=$nthreads"
         fi
 
-        qpfiles=$(find "${DATA}" -maxdepth 1 -type f -name '*.qp' -print -quit)
+        qpfiles=$(find "$DATA" -maxdepth 1 -type f -name '*.qp' -print -quit)
         if [ -n "$qpfiles" ]; then
             wsrep_log_info "Compressed qpress files found"
 
             if [ -z "$(command -v qpress)" ]; then
-                wsrep_log_error "qpress not found in path: $PATH"
+                wsrep_log_error "qpress utility not found in the path"
                 exit 22
             fi
 
+            get_proc
+
             dcmd="xargs -n 2 qpress -dT$nproc"
 
-            if [[ -n "$progress" ]] && pv --help | grep -qw '--line-mode';then
-                count=$(find "${DATA}" -type f -name '*.qp' | wc -l)
+            if [ -n "$progress" ] && pv --help | grep -qw -- '--line-mode'; then
+                count=$(find "$DATA" -type f -name '*.qp' | wc -l)
                 count=$(( count*2 ))
-                if pv --help | grep -qw -F '-F';then
-                    pvopts="-f -s $count -l -N Decompression -F '%N => Rate:%r Elapsed:%t %e Progress: [%b/$count]'"
-                else
-                    pvopts="-f -s $count -l -N Decompression"
+                pvopts="-f -s $count -l -N Decompression"
+                if pv --help | grep -qw -- '-F'; then
+                    pvopts="$pvopts -F '%N => Rate:%r Elapsed:%t %e Progress: [%b/$count]'"
                 fi
                 pcmd="pv $pvopts"
                 adjust_progress
@@ -1134,13 +1277,13 @@ then
 
             # Decompress the qpress files
             wsrep_log_info "Decompression with $nproc threads"
-            timeit "Joiner-Decompression" "find ${DATA} -type f -name '*.qp' -printf '%p\n%h\n' | $dcmd"
+            timeit "Joiner-Decompression" "find '$DATA' -type f -name '*.qp' -printf '%p\n%h\n' | $dcmd"
             extcode=$?
 
-            if [[ $extcode -eq 0 ]];then
+            if [ $extcode -eq 0 ]; then
                 wsrep_log_info "Removing qpress files after decompression"
                 find "$DATA" -type f -name '*.qp' -delete
-                if [[ $? -ne 0 ]];then
+                if [ $? -ne 0 ]; then
                     wsrep_log_error "Something went wrong with deletion of qpress files. Investigate"
                 fi
             else
@@ -1165,38 +1308,39 @@ then
 
         fi
 
-        wsrep_log_info "Preparing the backup at ${DATA}"
+        wsrep_log_info "Preparing the backup at $DATA"
         setup_commands
         timeit "Xtrabackup prepare stage" "$INNOAPPLY"
 
-        if [ $? -ne 0 ];
-        then
-            wsrep_log_error "${INNOBACKUPEX_BIN} apply finished with errors. Check ${DATA}/innobackup.prepare.log"
+        if [ $? -ne 0 ]; then
+            wsrep_log_error "xtrabackup apply finished with errors. Check '$INNOAPPLYLOG' for details"
             exit 22
         fi
 
-        MAGIC_FILE="${TDATA}/${INFO_FILE}"
-        set +e
-        rm "$TDATA/innobackup.prepare.log" "$TDATA/innobackup.move.log"
-        set -e
-        wsrep_log_info "Moving the backup to ${TDATA}"
+        # [ -f "$INNOAPPLYLOG" ] && rm "$INNOAPPLYLOG"
+
+        MAGIC_FILE="$TDATA/$INFO_FILE"
+
+        wsrep_log_info "Moving the backup to $TDATA"
         timeit "Xtrabackup move stage" "$INNOMOVE"
-        if [[ $? -eq 0 ]];then
-            wsrep_log_info "Move successful, removing ${DATA}"
+        if [ $? -eq 0 ]; then
+            wsrep_log_info "Move successful, removing $DATA"
             rm -rf "$DATA"
-            DATA="${TDATA}"
+            DATA="$TDATA"
         else
-            wsrep_log_error "Move failed, keeping ${DATA} for further diagnosis"
-            wsrep_log_error "Check ${DATA}/innobackup.move.log for details"
+            wsrep_log_error "Move failed, keeping '$DATA' for further diagnosis"
+            wsrep_log_error "Check syslog or '$INNOMOVELOG' for details"
             exit 22
         fi
 
     else
-        wsrep_log_info "${IST_FILE} received from donor: Running IST"
+
+        wsrep_log_info "'$IST_FILE' received from donor: Running IST"
+
     fi
 
-    if [[ ! -r "${MAGIC_FILE}" ]];then
-        wsrep_log_error "SST magic file ${MAGIC_FILE} not found/readable"
+    if [ ! -r "$MAGIC_FILE" ]; then
+        wsrep_log_error "SST magic file '$MAGIC_FILE' not found/readable"
         exit 2
     fi
 

scripts/wsrep_sst_xtrabackup.sh

@@ -97,8 +97,9 @@ get_keys()
     fi
 
     if [[ $encrypt -eq 0 ]];then
-        if $MY_PRINT_DEFAULTS xtrabackup | grep -q encrypt;then
-            wsrep_log_error "Unexpected option combination. SST may fail. Refer to http://www.percona.com/doc/percona-xtradb-cluster/manual/xtrabackup_sst.html "
+        if [ -n "$ealgo" -o -n "$ekey" -o -n "$ekeyfile" ]; then
+            wsrep_log_error "Options for encryption are specified, " \
+                            "but encryption itself is disabled. SST may fail."
         fi
         return
     fi
@@ -324,12 +325,6 @@ cleanup_joiner()
     fi
 }
 
-check_pid()
-{
-    local pid_file="$1"
-    [ -r "$pid_file" ] && ps -p $(cat "$pid_file") >/dev/null 2>&1
-}
-
 cleanup_donor()
 {
     # Since this is invoked just after exit NNN
@@ -339,12 +334,11 @@ cleanup_donor()
     fi
 
     if [[ -n "$XTRABACKUP_PID" ]];then
-        if check_pid "$XTRABACKUP_PID"
+        if check_pid "$XTRABACKUP_PID" 1
         then
             wsrep_log_error "xtrabackup process is still running. Killing... "
-            kill_xtrabackup
+            cleanup_pid $CHECK_PID "$XTRABACKUP_PID"
         fi
-        rm -f "$XTRABACKUP_PID"
     fi
 
     rm -f "${DATA}/${IST_FILE}"
@@ -355,13 +349,6 @@ cleanup_donor()
     fi
 }
 
-kill_xtrabackup()
-{
-    local PID=$(cat "$XTRABACKUP_PID")
-    [ -n "$PID" -a $PID -ne 0 ] && kill $PID && (kill $PID && kill -9 $PID) || :
-    rm -f "$XTRABACKUP_PID"
-}
-
 # waits ~10 seconds for nc to open the port and then reports ready
 # (regardless of timeout)
 wait_for_listen()