MDEV-18215: mariabackup does not report unknown command line options

MDEV-21298: mariabackup doesn't read from the [mariadbd] and [mariadbd-X.Y]
server option groups from configuration files
MDEV-21301: mariabackup doesn't read [mariadb-backup] option group in
configuration file

All three issues require to change the same code, that is why their
fixes are joined in one commit.

The fix is in invoking load_defaults_or_exit() and handle_options() for
backup-specific groups separately from client-server groups to let the last
handle_options() call fail on unknown backup-specific options.

The order of options procesing is the following:
1) Load server groups and process server options, ignore unknown
options
2) Load client groups and process client options, ignore unknown
options
3) Load backup groups and process client-server options, exit on
unknown option
4) Process --mysqld-args command line options, ignore unknown options

New global flag my_handle_options_init_variables was added to have
ability to invoke handle_options() for the same allowed options set
several times without re-initialising previously set option values.

--password value destroying is moved from option processing callback to
mariabackup's handle_options() function to have ability to invoke server's
handle_options() several times for the same possible allowed options
set.

Galera invokes wsrep_sst_mariabackup.sh with mysqld command line
options to configure mariabackup as close to the server as possible.
It is not known what server options are supported by mariabackup when the
script is invoked. That is why new mariabackup option "--mysqld-args" is added,
all unknown options that follow this option will be silently ignored.

wsrep_sst_mariabackup.sh was also changed to:
- use "--mysqld-args" mariabackup option to pass mysqld options,
- remove deprecated innobackupex mode,
- remove unsupported mariabackup options:
    --encrypt
    --encrypt-key
    --rebuild-indexes
    --rebuild-threads

extra/mariabackup/innobackupex.cc

@@ -112,6 +112,8 @@ bool ibx_partial_backup = false;
 char *ibx_position_arg = NULL;
 char *ibx_backup_directory = NULL;
 
+extern bool xb_opt_destroy_password;
+
 /* copy of proxied xtrabackup options */
 my_bool ibx_xb_close_files;
 const char *ibx_xtrabackup_compress_alg;
@@ -771,17 +773,7 @@ ibx_get_one_option(int optid,
 		xtrabackup_compress = TRUE;
 		break;
 	case 'p':
-		if (argument)
-		{
-		        char *start = argument;
-			my_free(opt_ibx_password);
-			opt_ibx_password= my_strdup(argument, MYF(MY_FAE));
-			/*  Destroy argument */
-			while (*argument)
-				*argument++= 'x';
-			if (*start)
-				start[1]=0 ;
-		}
+    opt_ibx_password= argument;
 		break;
         }
 	return(0);

include/my_getopt.h

@@ -106,6 +106,7 @@ extern char *autoset_my_option;
 extern my_bool my_getopt_print_errors;
 extern my_bool my_getopt_skip_unknown;
 extern my_bool my_getopt_prefix_matching;
+extern my_bool my_handle_options_init_variables;
 extern my_error_reporter my_getopt_error_reporter;
 
 extern int handle_options (int *argc, char ***argv, 

mysql-test/suite/mariabackup/apply-log-only-incr.test

@@ -36,7 +36,7 @@ connection default;
 
 --disable_result_log
 exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf  --backup --ftwrl-wait-timeout=5 --ftwrl-wait-threshold=300 --ftwrl-wait-query-type=all --target-dir=$incremental_dir --incremental-basedir=$basedir ;
-exec $XTRABACKUP --prepare --verbose --apply-log-only --target-dir=$basedir ;
+exec $XTRABACKUP --prepare --verbose --target-dir=$basedir ;
 --enable_result_log
 
 let SEARCH_FILE=$MYSQLTEST_VARDIR/log/current_test;
@@ -44,7 +44,7 @@ let SEARCH_FILE=$MYSQLTEST_VARDIR/log/current_test;
 --source include/search_pattern_in_file.inc
 --echo # expect NOT FOUND
 
-exec $XTRABACKUP --prepare --verbose --apply-log-only --target-dir=$basedir --incremental-dir=$incremental_dir ;
+exec $XTRABACKUP --prepare --verbose --target-dir=$basedir --incremental-dir=$incremental_dir ;
 
 --source include/search_pattern_in_file.inc
 --echo # expect NOT FOUND

mysql-test/suite/mariabackup/apply-log-only.test

@@ -11,7 +11,7 @@ INSERT INTO t VALUES(1);
 exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf  --backup --target-dir=$basedir;
 --enable_result_log
 
-exec $XTRABACKUP --prepare --apply-log-only --target-dir=$basedir ;
+exec $XTRABACKUP --prepare --target-dir=$basedir ;
 
 let SEARCH_FILE=$MYSQLTEST_VARDIR/log/current_test;
 --let SEARCH_PATTERN= Rolled back recovered transaction

mysql-test/suite/mariabackup/binlog.test

@@ -12,7 +12,7 @@ SHOW VARIABLES like 'log_bin';
 exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf  --backup --target-dir=$basedir;
 --enable_result_log
 
-exec $XTRABACKUP --prepare --binlog-info=1 --apply-log-only --target-dir=$basedir ;
+exec $XTRABACKUP --prepare --binlog-info=1 --target-dir=$basedir ;
 
 let SEARCH_FILE=$MYSQLTEST_VARDIR/log/current_test;
 --let SEARCH_PATTERN= Last binlog file .*, position .*

mysql-test/suite/mariabackup/incremental_ddl_before_backup.test

@@ -33,7 +33,7 @@ select count(*) from t7;
 exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf  --backup --target-dir=$incremental_dir --incremental-basedir=$basedir;
 
 --echo # XTRABACKUP PREPARE
-exec $XTRABACKUP --apply-log-only --prepare --target-dir=$basedir;
+exec $XTRABACKUP --prepare --target-dir=$basedir;
 
 --echo # XTRABACKUP INCREMENTAL PREPARE
 exec $XTRABACKUP --prepare --target-dir=$basedir --incremental-dir=$incremental_dir;

mysql-test/suite/mariabackup/incremental_ddl_during_backup.test

@@ -25,7 +25,7 @@ exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf  --backup --target-dir
 --let after_load_tablespaces=
 --disable_result_log
 echo # Prepare full backup, apply incremental one;
-exec $XTRABACKUP --apply-log-only --prepare --target-dir=$basedir;
+exec $XTRABACKUP --prepare --target-dir=$basedir;
 exec $XTRABACKUP --prepare --target-dir=$basedir --incremental-dir=$incremental_dir ;
 
 echo # Restore and check results;

mysql-test/suite/mariabackup/options_check.result

+# Check for unknown options in command-line
+# Check for unknown options in "mariabackup" group
+# Check for unknown options in "xtrabackup" group
+# Check for unknown options in "mariadb-backup" group
+# Check for options overwriting
+# Check if uknown options that follow --mysqld-args are ingored

mysql-test/suite/mariabackup/options_check.test

+--let $targetdir=$MYSQLTEST_VARDIR/tmp/backup
+--let $custom_cnf=$MYSQLTEST_VARDIR/tmp/custom_my.cnf
+
+--echo # Check for unknown options in command-line
+--disable_result_log
+--error 7
+exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --unknown-option=xxx --target-dir=$targetdir;
+--error 2
+exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --unknown-option --target-dir=$targetdir;
+--enable_result_log
+
+--echo # Check for unknown options in "mariabackup" group
+--write_file $custom_cnf
+[mariabackup]
+unknown-option=XXX
+EOF
+--error 7
+exec $XTRABACKUP --defaults-file=$custom_cnf --backup --target-dir=$targetdir;
+--remove_file $custom_cnf
+
+--echo # Check for unknown options in "xtrabackup" group
+--write_file $custom_cnf
+[xtrabackup]
+unknown-option=XXX
+EOF
+--error 7
+exec $XTRABACKUP --defaults-file=$custom_cnf --backup --target-dir=$targetdir;
+--remove_file $custom_cnf
+
+--echo # Check for unknown options in "mariadb-backup" group
+--write_file $custom_cnf
+[mariadb-backup]
+unknown-option=XXX
+EOF
+--error 7
+exec $XTRABACKUP --defaults-file=$custom_cnf --backup --target-dir=$targetdir;
+--remove_file $custom_cnf
+
+--echo # Check for options overwriting
+--write_file $custom_cnf
+[mariadbd]
+innodb-flush-method=O_DIRECT
+
+[mariabackup]
+innodb-flush-method=blablabla
+EOF
+--error 13
+exec $XTRABACKUP --defaults-file=$custom_cnf --backup --target-dir=$targetdir;
+--remove_file $custom_cnf
+
+--echo # Check if uknown options that follow --mysqld-args are ingored
+exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir --mysqld-args --unknown-option;
+--rmdir $targetdir

mysql-test/suite/mariabackup/undo_space_id.test

@@ -17,7 +17,7 @@ exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf  --backup --target-dir
 list_files $basedir undo*;
 
 --echo # xtrabackup prepare
-exec $XTRABACKUP --prepare --apply-log-only --target-dir=$basedir;
+exec $XTRABACKUP --prepare --target-dir=$basedir;
 --echo # Display undo log files from targer directory
 list_files $basedir undo*;
 

mysql-test/suite/mariabackup/xb_fulltext_encrypted.test

@@ -16,7 +16,7 @@ let $targetdir=$MYSQLTEST_VARDIR/tmp/backup;
 --disable_result_log
 
 exec $INNOBACKUPEX --defaults-file=$MYSQLTEST_VARDIR/my.cnf --no-timestamp $targetdir;
-exec $INNOBACKUPEX --apply-log --rebuild-indexes --rebuild-threads=2 $targetdir;
+exec $INNOBACKUPEX --apply-log $targetdir;
 --source include/restart_and_restore.inc
 
 --enable_result_log

mysql-test/suite/mariabackup/xb_partition.test

@@ -38,7 +38,7 @@ INSERT INTO isam_p VALUES (1), (101), (201), (301);
 let $targetdir=$MYSQLTEST_VARDIR/tmp;
 
 --disable_result_log
-exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --no-timestamp --backup --target-dir=$targetdir/full;
+exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir/full;
 --enable_result_log
 
 DROP TABLE t1;

mysys/my_getopt.c

@@ -79,6 +79,12 @@ my_bool my_getopt_skip_unknown= 0;
 */
 my_bool my_getopt_prefix_matching= 1;
 
+/*
+   This is a flag that can be set in client programs. 1 means that
+   handle_options()  will not initialize options to default values.
+*/
+my_bool my_handle_options_init_variables = 1;
+
 static void default_reporter(enum loglevel level,
                              const char *format, ...)
 {
@@ -212,7 +218,8 @@ int handle_options(int *argc, char ***argv,
   DBUG_ASSERT(argv && *argv);
   (*argc)--; /* Skip the program name */
   (*argv)++; /*      --- || ----      */
-  init_variables(longopts, init_one_value);
+  if (my_handle_options_init_variables)
+    init_variables(longopts, init_one_value);
 
   /*
     Search for args_separator, if found, then the first part of the

scripts/wsrep_sst_mariabackup.sh

@@ -56,8 +56,6 @@ sfmt="tar"
 strmcmd=""
 tfmt=""
 tcmd=""
-rebuild=0
-rebuildcmd=""
 payload=0
 pvformat="-F '%N => Rate:%r Avg:%a Elapsed:%t %e Bytes: %b %p' "
 pvopts="-f -i 10 -N $WSREP_SST_OPT_ROLE "
@@ -339,7 +337,6 @@ read_cnf()
     encrypt=$(parse_cnf sst encrypt 0)
     sockopt=$(parse_cnf sst sockopt "")
     progress=$(parse_cnf sst progress "")
-    rebuild=$(parse_cnf sst rebuild 0)
     ttime=$(parse_cnf sst time 0)
     cpat=$(parse_cnf sst cpat '.*galera\.cache$\|.*sst_in_progress$\|.*\.sst$\|.*gvwstate\.dat$\|.*grastate\.dat$\|.*\.err$\|.*\.log$\|.*RPM_UPGRADE_MARKER$\|.*RPM_UPGRADE_HISTORY$')
     [[ $OS == "FreeBSD" ]] && cpat=$(parse_cnf sst cpat '.*galera\.cache$|.*sst_in_progress$|.*\.sst$|.*gvwstate\.dat$|.*grastate\.dat$|.*\.err$|.*\.log$|.*RPM_UPGRADE_MARKER$|.*RPM_UPGRADE_HISTORY$')
@@ -708,7 +705,7 @@ if [[ ${FORCE_FTWRL:-0} -eq 1 ]];then
     iopts+=" --no-backup-locks "
 fi
 
-INNOEXTRA=$WSREP_SST_OPT_MYSQLD
+INNOEXTRA=
 
 INNODB_DATA_HOME_DIR=${INNODB_DATA_HOME_DIR:-""}
 # Try to set INNODB_DATA_HOME_DIR from the command line:
@@ -754,9 +751,9 @@ if [[ $ssyslog -eq 1 ]];then
             logger  -p daemon.info -t ${ssystag}wsrep-sst-$WSREP_SST_OPT_ROLE "$@" 
         }
 
-        INNOAPPLY="${INNOBACKUPEX_BIN} --innobackupex $disver $iapts \$INNOEXTRA --apply-log \$rebuildcmd \${DATA} 2>&1 | logger -p daemon.err -t ${ssystag}innobackupex-apply"
-        INNOMOVE="${INNOBACKUPEX_BIN} --innobackupex ${WSREP_SST_OPT_CONF} $disver $impts  --move-back --force-non-empty-directories \${DATA} 2>&1 | logger -p daemon.err -t ${ssystag}innobackupex-move"
-        INNOBACKUP="${INNOBACKUPEX_BIN} --innobackupex ${WSREP_SST_OPT_CONF} $disver $iopts \$tmpopts \$INNOEXTRA --galera-info --stream=\$sfmt \$itmpdir 2> >(logger -p daemon.err -t ${ssystag}innobackupex-backup)"
+        INNOAPPLY="${INNOBACKUPEX_BIN} --prepare $disver $iapts \$INNOEXTRA --target-dir=\${DATA} --mysqld-args \$WSREP_SST_OPT_MYSQLD  2>&1 | logger -p daemon.err -t ${ssystag}innobackupex-apply"
+        INNOMOVE="${INNOBACKUPEX_BIN} ${WSREP_SST_OPT_CONF} --move-back $disver $impts --force-non-empty-directories --target-dir=\${DATA} 2>&1 | logger -p daemon.err -t ${ssystag}innobackupex-move"
+        INNOBACKUP="${INNOBACKUPEX_BIN} ${WSREP_SST_OPT_CONF} --backup $disver $iopts \$tmpopts \$INNOEXTRA --galera-info --stream=\$sfmt --target-dir=\$itmpdir --mysqld-args \$WSREP_SST_OPT_MYSQLD 2> >(logger -p daemon.err -t ${ssystag}innobackupex-backup)"
     fi
 
 else
@@ -818,9 +815,9 @@ then
 
 fi
  
-    INNOAPPLY="${INNOBACKUPEX_BIN} --innobackupex $disver $iapts \$INNOEXTRA --apply-log \$rebuildcmd \${DATA} &> ${INNOAPPLYLOG}"
-    INNOMOVE="${INNOBACKUPEX_BIN} --innobackupex ${WSREP_SST_OPT_CONF} $disver $impts  --move-back --force-non-empty-directories \${DATA} &> ${INNOMOVELOG}"
-    INNOBACKUP="${INNOBACKUPEX_BIN} --innobackupex ${WSREP_SST_OPT_CONF} $disver $iopts \$tmpopts \$INNOEXTRA --galera-info --stream=\$sfmt \$itmpdir 2> ${INNOBACKUPLOG}"
+    INNOAPPLY="${INNOBACKUPEX_BIN} --prepare $disver $iapts \$INNOEXTRA --target-dir=\${DATA} --mysqld-args \$WSREP_SST_OPT_MYSQLD &> ${INNOAPPLYLOG}"
+    INNOMOVE="${INNOBACKUPEX_BIN} ${WSREP_SST_OPT_CONF} --move-back $disver $impts  --move-back --force-non-empty-directories --target-dir=\${DATA} &> ${INNOMOVELOG}"
+    INNOBACKUP="${INNOBACKUPEX_BIN} ${WSREP_SST_OPT_CONF} --backup $disver $iopts \$tmpopts \$INNOEXTRA --galera-info --stream=\$sfmt --target-dir=\$itmpdir 2> ${INNOBACKUPLOG}"
 fi
 
 get_stream
@@ -862,15 +859,6 @@ then
            INNOEXTRA+=" --password="
         fi
 
-        get_keys
-        if [[ $encrypt -eq 1 ]];then
-            if [[ -n $ekey ]];then
-                INNOEXTRA+=" --encrypt=$ealgo --encrypt-key=$ekey"
-            else 
-                INNOEXTRA+=" --encrypt=$ealgo --encrypt-key-file=$ekeyfile"
-            fi
-        fi
-
         check_extra
 
         wsrep_log_info "Streaming GTID file before SST"
@@ -1091,18 +1079,6 @@ then
             exit 2
         fi
 
-        # Rebuild indexes for compact backups
-        if grep -q 'compact = 1' ${DATA}/xtrabackup_checkpoints;then 
-            wsrep_log_info "Index compaction detected"
-            rebuild=1
-        fi
-
-        if [[ $rebuild -eq 1 ]];then 
-            nthreads=$(parse_cnf xtrabackup rebuild-threads $nproc)
-            wsrep_log_info "Rebuilding during prepare with $nthreads threads"
-            rebuildcmd="--rebuild-indexes --rebuild-threads=$nthreads"
-        fi
-
         if test -n "$(find ${DATA} -maxdepth 1 -type f -name '*.qp' -print -quit)";then
 
             wsrep_log_info "Compressed qpress files found"