Merge 10.3 into 10.4

We omit commit a3bdce8f
and commit a0e2a293
because they would make the test galera_3nodes.galera_gtid_2_cluster
fail and disable it.

extra/mariabackup/xtrabackup.cc

@@ -2768,6 +2768,7 @@ static bool xtrabackup_copy_logfile(bool last = false)
 	ut_a(dst_log_file != NULL);
 	ut_ad(recv_sys.is_initialised());
 
+	bool overwritten_block = false;
 	lsn_t	start_lsn;
 	lsn_t	end_lsn;
 
@@ -2793,6 +2794,12 @@ static bool xtrabackup_copy_logfile(bool last = false)
 		}
 
 		if (lsn == start_lsn) {
+			overwritten_block= !recv_sys.found_corrupt_log
+				&& (innodb_log_checksums || log_sys.log.is_encrypted())
+				&& log_block_calc_checksum_crc32(log_sys.buf) ==
+					log_block_get_checksum(log_sys.buf)
+				&& log_block_get_hdr_no(log_sys.buf) >
+					log_block_convert_lsn_to_no(start_lsn);
 			start_lsn = 0;
 		} else {
 			mutex_enter(&recv_sys.mutex);
@@ -2803,9 +2810,15 @@ static bool xtrabackup_copy_logfile(bool last = false)
 		log_mutex_exit();
 
 		if (!start_lsn) {
-			die(recv_sys.found_corrupt_log
-			    ? "xtrabackup_copy_logfile() failed: corrupt log."
-			    : "xtrabackup_copy_logfile() failed.");
+			const char *reason = recv_sys.found_corrupt_log
+				? "corrupt log."
+				: (overwritten_block
+					? "redo log block is overwritten, please increase redo log size with innodb_log_file_size parameter."
+					: ((innodb_log_checksums || log_sys.log.is_encrypted())
+						? "redo log block checksum does not match."
+						: "unknown reason as innodb_log_checksums is switched off and redo"
+							" log is not encrypted."));
+			die("xtrabackup_copy_logfile() failed: %s", reason);
 			return true;
 		}
 	} while (start_lsn == end_lsn);

mysql-test/suite/encryption/r/create_or_replace.result

+SET @save_threads = @@GLOBAL.innodb_encryption_threads;
+SET @save_tables = @@GLOBAL.innodb_encrypt_tables;
 SET default_storage_engine = InnoDB;
-CREATE TABLE t1 (pk INT PRIMARY KEY, c VARCHAR(256));
-CREATE TABLE t2 AS SELECT * FROM t1;
-drop table t1,t2;
-SET GLOBAL innodb_encryption_threads = 0;
 SET GLOBAL innodb_encryption_threads = 4;
 CREATE TABLE `table10_int_autoinc` (`col_int_key` int, pk int auto_increment, `col_int` int, key (`col_int_key` ),primary key (pk)) engine=innodb;
 INSERT /*! IGNORE */ INTO table10_int_autoinc VALUES  (NULL, NULL, -474021888) ,  (1, NULL, NULL) ,  (1141047296, NULL, NULL) ,  (NULL, NULL, NULL) ,  (NULL, NULL, 1) ,  (NULL, NULL, 9) ,  (0, NULL, 1225785344) ,  (NULL, NULL, 1574174720) ,  (2, NULL, NULL) ,  (6, NULL, 3);
@@ -14,14 +12,10 @@ INSERT IGNORE INTO `table1_int_autoinc` ( `col_int` ) VALUES ( 1 ), ( 0 ), ( 7 )
 INSERT IGNORE INTO `table10_int_autoinc` ( `col_int` ) VALUES ( 6 ), ( 2 ), ( 3 ), ( 6 );
 connect  con1,localhost,root,,test;
 connect  con2,localhost,root,,test;
-connection default;
-drop table if exists create_or_replace_t, table1_int_autoinc, table0_int_autoinc, table10_int_autoinc;
 disconnect con1;
 disconnect con2;
-SET GLOBAL innodb_encrypt_tables = OFF;
-SET GLOBAL innodb_encryption_threads = 4;
-# Wait max 10 min for key encryption threads to decrypt all spaces
-# Success!
-SET GLOBAL innodb_encryption_threads = 0;
-SET GLOBAL innodb_encrypt_tables = OFF;
-# restart
+connection default;
+drop table create_or_replace_t, table1_int_autoinc, table0_int_autoinc,
+table10_int_autoinc;
+SET GLOBAL innodb_encryption_threads = @save_threads;
+SET GLOBAL innodb_encrypt_tables = @save_tables;

mysql-test/suite/encryption/r/create_or_replace_big.result

+SET default_storage_engine = InnoDB;
+CREATE TABLE t1 (pk INT PRIMARY KEY, c VARCHAR(256));
+CREATE TABLE t2 AS SELECT * FROM t1;
+drop table t1,t2;
+SET GLOBAL innodb_encryption_threads = 0;
+SET GLOBAL innodb_encryption_threads = 4;
+CREATE TABLE `table10_int_autoinc` (`col_int_key` int, pk int auto_increment, `col_int` int, key (`col_int_key` ),primary key (pk)) engine=innodb;
+INSERT /*! IGNORE */ INTO table10_int_autoinc VALUES  (NULL, NULL, -474021888) ,  (1, NULL, NULL) ,  (1141047296, NULL, NULL) ,  (NULL, NULL, NULL) ,  (NULL, NULL, 1) ,  (NULL, NULL, 9) ,  (0, NULL, 1225785344) ,  (NULL, NULL, 1574174720) ,  (2, NULL, NULL) ,  (6, NULL, 3);
+CREATE TABLE `table1_int_autoinc` (`col_int_key` int, pk int auto_increment, `col_int` int,key (`col_int_key` ), primary key (pk)) engine=innodb;
+CREATE TABLE `table0_int_autoinc` (`col_int_key` int, pk int auto_increment, `col_int` int, key (`col_int_key` ),primary key (pk)) engine=innodb;
+INSERT /*! IGNORE */ INTO table1_int_autoinc VALUES  (4, NULL, NULL);
+INSERT IGNORE INTO `table0_int_autoinc` ( `col_int_key` ) VALUES ( 1 ), ( 3 ), ( 4 ), ( 1 );
+INSERT IGNORE INTO `table1_int_autoinc` ( `col_int` ) VALUES ( 1 ), ( 0 ), ( 7 ), ( 9 );
+INSERT IGNORE INTO `table10_int_autoinc` ( `col_int` ) VALUES ( 6 ), ( 2 ), ( 3 ), ( 6 );
+# Wait max 10 min for key encryption threads to decrypt all spaces
+# Success!
+SET GLOBAL innodb_encryption_threads = 0;
+SET GLOBAL innodb_encrypt_tables = OFF;
+DROP TABLE table0_int_autoinc, table1_int_autoinc, table10_int_autoinc;
+# restart

mysql-test/suite/encryption/t/create_or_replace.test

 --source include/have_innodb.inc
 --source include/have_file_key_management_plugin.inc
---source include/not_embedded.inc
-# This is needed for longer testcase timeout at least P7/P8
---source include/big_test.inc
+--source include/count_sessions.inc
 
-#
-# MDEV-8164: Server crashes in pfs_mutex_enter_func after fil_crypt_is_closing or alike
-#
-SET default_storage_engine = InnoDB;
-
-CREATE TABLE t1 (pk INT PRIMARY KEY, c VARCHAR(256));
-CREATE TABLE t2 AS SELECT * FROM t1;
-
---disable_abort_on_error
---disable_warnings
---disable_query_log
+SET @save_threads = @@GLOBAL.innodb_encryption_threads;
+SET @save_tables = @@GLOBAL.innodb_encrypt_tables;
 
-let $i = 40;
-while ($i)
-{
-SET GLOBAL innodb_encrypt_tables = ON;
-SET GLOBAL innodb_encryption_threads = 1;
-CREATE OR REPLACE TABLE t1 AS SELECT * FROM t2;
-CREATE OR REPLACE TABLE t2 AS SELECT * FROM t1;
-SET GLOBAL innodb_encryption_rotation_iops = 100;
-SET GLOBAL innodb_encrypt_tables = OFF;
-CREATE OR REPLACE TABLE t2 AS SELECT * FROM t1;
-CREATE OR REPLACE TABLE t1 AS SELECT * FROM t2;
-dec $i;
-}
-
---enable_abort_on_error
---enable_warnings
---enable_query_log
-
-drop table t1,t2;
-SET GLOBAL innodb_encryption_threads = 0;
+SET default_storage_engine = InnoDB;
 
 #
 # MDEV-8173: InnoDB; Failing assertion: crypt_data->type == 1
@@ -58,11 +28,9 @@ INSERT IGNORE INTO `table10_int_autoinc` ( `col_int` ) VALUES ( 6 ), ( 2 ), ( 3
 --connect (con1,localhost,root,,test)
 --connect (con2,localhost,root,,test)
 
---disable_abort_on_error
---disable_warnings
 --disable_query_log
 
-let $i = 500;
+let $i = 100;
 while ($i)
 {
 connection con1;
@@ -101,42 +69,12 @@ dec $i;
 }
 
 --enable_query_log
+disconnect con1;
+disconnect con2;
 connection default;
-drop table if exists create_or_replace_t, table1_int_autoinc, table0_int_autoinc, table10_int_autoinc;
---disconnect con1
---disconnect con2
---enable_abort_on_error
---enable_warnings
-
-SET GLOBAL innodb_encrypt_tables = OFF;
-SET GLOBAL innodb_encryption_threads = 4;
-
---echo # Wait max 10 min for key encryption threads to decrypt all spaces
-let $cnt=600;
-while ($cnt)
-{
-    let $success=`SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0`;
-    if ($success)
-    {
-        let $cnt=0;
-    }
-    if (!$success)
-    {
-        real_sleep 1;
-        dec $cnt;
-    }
-}
-if (!$success)
-{
-    SELECT * FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION;
-    SHOW STATUS LIKE 'innodb_encryption%';
-    -- die Timeout waiting for encryption threads
-}
---echo # Success!
-
-SET GLOBAL innodb_encryption_threads = 0;
-SET GLOBAL innodb_encrypt_tables = OFF;
-
-# Make sure that all dirty pages are flushed
+drop table create_or_replace_t, table1_int_autoinc, table0_int_autoinc,
+table10_int_autoinc;
 
--- source include/restart_mysqld.inc
+SET GLOBAL innodb_encryption_threads = @save_threads;
+SET GLOBAL innodb_encrypt_tables = @save_tables;
+--source include/wait_until_count_sessions.inc

mysql-test/suite/encryption/t/create_or_replace_big.test

+--source include/have_innodb.inc
+--source include/have_file_key_management_plugin.inc
+--source include/not_embedded.inc
+# This is needed for longer testcase timeout at least P7/P8
+--source include/big_test.inc
+
+#
+# MDEV-8164: Server crashes in pfs_mutex_enter_func after fil_crypt_is_closing or alike
+#
+SET default_storage_engine = InnoDB;
+
+CREATE TABLE t1 (pk INT PRIMARY KEY, c VARCHAR(256));
+CREATE TABLE t2 AS SELECT * FROM t1;
+
+--disable_abort_on_error
+--disable_warnings
+--disable_query_log
+
+let $i = 40;
+while ($i)
+{
+SET GLOBAL innodb_encrypt_tables = ON;
+SET GLOBAL innodb_encryption_threads = 1;
+CREATE OR REPLACE TABLE t1 AS SELECT * FROM t2;
+CREATE OR REPLACE TABLE t2 AS SELECT * FROM t1;
+SET GLOBAL innodb_encryption_rotation_iops = 100;
+SET GLOBAL innodb_encrypt_tables = OFF;
+CREATE OR REPLACE TABLE t2 AS SELECT * FROM t1;
+CREATE OR REPLACE TABLE t1 AS SELECT * FROM t2;
+dec $i;
+}
+
+--enable_abort_on_error
+--enable_warnings
+--enable_query_log
+
+drop table t1,t2;
+SET GLOBAL innodb_encryption_threads = 0;
+
+#
+# MDEV-8173: InnoDB; Failing assertion: crypt_data->type == 1
+#
+
+SET GLOBAL innodb_encryption_threads = 4;
+
+CREATE TABLE `table10_int_autoinc` (`col_int_key` int, pk int auto_increment, `col_int` int, key (`col_int_key` ),primary key (pk)) engine=innodb;
+INSERT /*! IGNORE */ INTO table10_int_autoinc VALUES  (NULL, NULL, -474021888) ,  (1, NULL, NULL) ,  (1141047296, NULL, NULL) ,  (NULL, NULL, NULL) ,  (NULL, NULL, 1) ,  (NULL, NULL, 9) ,  (0, NULL, 1225785344) ,  (NULL, NULL, 1574174720) ,  (2, NULL, NULL) ,  (6, NULL, 3);
+
+CREATE TABLE `table1_int_autoinc` (`col_int_key` int, pk int auto_increment, `col_int` int,key (`col_int_key` ), primary key (pk)) engine=innodb;
+
+CREATE TABLE `table0_int_autoinc` (`col_int_key` int, pk int auto_increment, `col_int` int, key (`col_int_key` ),primary key (pk)) engine=innodb;
+
+INSERT /*! IGNORE */ INTO table1_int_autoinc VALUES  (4, NULL, NULL);
+INSERT IGNORE INTO `table0_int_autoinc` ( `col_int_key` ) VALUES ( 1 ), ( 3 ), ( 4 ), ( 1 );
+INSERT IGNORE INTO `table1_int_autoinc` ( `col_int` ) VALUES ( 1 ), ( 0 ), ( 7 ), ( 9 );
+INSERT IGNORE INTO `table10_int_autoinc` ( `col_int` ) VALUES ( 6 ), ( 2 ), ( 3 ), ( 6 );
+
+--echo # Wait max 10 min for key encryption threads to decrypt all spaces
+let $cnt=600;
+while ($cnt)
+{
+    let $success=`SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION <> 0`;
+    if ($success)
+    {
+        let $cnt=0;
+    }
+    if (!$success)
+    {
+        real_sleep 1;
+        dec $cnt;
+    }
+}
+if (!$success)
+{
+    SELECT * FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION;
+    SHOW STATUS LIKE 'innodb_encryption%';
+    -- die Timeout waiting for encryption threads
+}
+--echo # Success!
+
+SET GLOBAL innodb_encryption_threads = 0;
+SET GLOBAL innodb_encrypt_tables = OFF;
+
+DROP TABLE table0_int_autoinc, table1_int_autoinc, table10_int_autoinc;
+
+-- source include/restart_mysqld.inc

mysql-test/suite/mariabackup/innodb_redo_overwrite.opt

+--loose-innodb-log-file-size=1048576 --loose-innodb-log-files-in-group=2

mysql-test/suite/mariabackup/innodb_redo_overwrite.result

+CREATE TABLE t(i INT) ENGINE=INNODB;
+INSERT INTO t VALUES
+(0), (1), (2), (3), (4), (5), (6), (7), (8), (9),
+(0), (1), (2), (3), (4), (5), (6), (7), (8), (9),
+(0), (1), (2), (3), (4), (5), (6), (7), (8), (9),
+(0), (1), (2), (3), (4), (5), (6), (7), (8), (9),
+(0), (1), (2), (3), (4), (5), (6), (7), (8), (9),
+(0), (1), (2), (3), (4), (5), (6), (7), (8), (9),
+(0), (1), (2), (3), (4), (5), (6), (7), (8), (9),
+(0), (1), (2), (3), (4), (5), (6), (7), (8), (9),
+(0), (1), (2), (3), (4), (5), (6), (7), (8), (9),
+(0), (1), (2), (3), (4), (5), (6), (7), (8), (9);
+# Generate enough data to overwrite innodb redo log
+# on the next "INSERT INTO t SELECT * FROM t" execution.
+INSERT INTO t SELECT * FROM t;
+INSERT INTO t SELECT * FROM t;
+INSERT INTO t SELECT * FROM t;
+INSERT INTO t SELECT * FROM t;
+INSERT INTO t SELECT * FROM t;
+INSERT INTO t SELECT * FROM t;
+INSERT INTO t SELECT * FROM t;
+INSERT INTO t SELECT * FROM t;
+INSERT INTO t SELECT * FROM t;
+# xtrabackup backup
+FOUND 1 /failed: redo log block is overwritten/ in backup.log
+FOUND 1 /failed: redo log block checksum does not match/ in backup.log
+FOUND 1 /failed: unknown reason/ in backup.log
+DROP TABLE t;

mysql-test/suite/mariabackup/innodb_redo_overwrite.test

+--source include/have_innodb.inc
+--source include/have_debug_sync.inc
+
+CREATE TABLE t(i INT) ENGINE=INNODB;
+
+INSERT INTO t VALUES
+  (0), (1), (2), (3), (4), (5), (6), (7), (8), (9),
+  (0), (1), (2), (3), (4), (5), (6), (7), (8), (9),
+  (0), (1), (2), (3), (4), (5), (6), (7), (8), (9),
+  (0), (1), (2), (3), (4), (5), (6), (7), (8), (9),
+  (0), (1), (2), (3), (4), (5), (6), (7), (8), (9),
+  (0), (1), (2), (3), (4), (5), (6), (7), (8), (9),
+  (0), (1), (2), (3), (4), (5), (6), (7), (8), (9),
+  (0), (1), (2), (3), (4), (5), (6), (7), (8), (9),
+  (0), (1), (2), (3), (4), (5), (6), (7), (8), (9),
+  (0), (1), (2), (3), (4), (5), (6), (7), (8), (9);
+--echo # Generate enough data to overwrite innodb redo log
+--echo # on the next "INSERT INTO t SELECT * FROM t" execution.
+--let $i = 0
+while ($i < 9) {
+INSERT INTO t SELECT * FROM t;
+--inc $i
+}
+
+--echo # xtrabackup backup
+--let $targetdir=$MYSQLTEST_VARDIR/tmp/backup
+--let $backuplog=$MYSQLTEST_VARDIR/tmp/backup.log
+
+--let before_innodb_log_copy_thread_started=INSERT INTO test.t SELECT * FROM test.t
+
+--disable_result_log
+--error 1
+--exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir --dbug=+d,mariabackup_events > $backuplog
+--enable_result_log
+
+--let SEARCH_PATTERN=failed: redo log block is overwritten
+--let SEARCH_FILE=$backuplog
+--source include/search_pattern_in_file.inc
+--remove_file $backuplog
+--rmdir $targetdir
+
+--let before_innodb_log_copy_thread_started=INSERT INTO test.t VALUES (0), (1), (2), (3), (4), (5), (6), (7), (8), (9)
+
+--disable_result_log
+--error 1
+--exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir --dbug=+d,mariabackup_events,log_checksum_mismatch > $backuplog
+--enable_result_log
+
+--let SEARCH_PATTERN=failed: redo log block checksum does not match
+--let SEARCH_FILE=$backuplog
+--source include/search_pattern_in_file.inc
+--remove_file $backuplog
+--rmdir $targetdir
+
+--let before_innodb_log_copy_thread_started=INSERT INTO test.t SELECT * FROM test.t LIMIT 50000
+
+--disable_result_log
+--error 1
+--exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir --innodb-log-checksums=0 --dbug=+d,mariabackup_events  > $backuplog
+--enable_result_log
+
+--let SEARCH_PATTERN=failed: unknown reason
+--let SEARCH_FILE=$backuplog
+--source include/search_pattern_in_file.inc
+--remove_file $backuplog
+--rmdir $targetdir
+
+--let before_innodb_log_copy_thread_started=
+
+DROP TABLE t;

storage/innobase/fil/fil0crypt.cc

@@ -1684,6 +1684,11 @@ static bool fil_crypt_find_space_to_rotate(
 {
 	/* we need iops to start rotating */
 	while (!state->should_shutdown() && !fil_crypt_alloc_iops(state)) {
+		if (state->space && state->space->is_stopping()) {
+			state->space->release();
+			state->space = NULL;
+		}
+
 		os_event_reset(fil_crypt_threads_event);
 		os_event_wait_time(fil_crypt_threads_event, 100000);
 	}
@@ -2699,6 +2704,7 @@ fil_space_crypt_close_tablespace(
 
 		/* wakeup throttle (all) sleepers */
 		os_event_set(fil_crypt_throttle_sleep_event);
+		os_event_set(fil_crypt_threads_event);
 
 		os_thread_sleep(20000);
 		dict_mutex_enter_for_mysql();

storage/innobase/fts/fts0fts.cc

@@ -5671,22 +5671,6 @@ fts_savepoint_rollback(
 	}
 }
 
-/*********************************************************************//**
-Compare two fts_aux_table_t parent_ids.
-@return < 0 if n1 < n2, 0 if n1 == n2, > 0 if n1 > n2 */
-UNIV_INLINE
-int
-fts_check_aux_table_parent_id_cmp(
-/*==============================*/
-	const void*	p1,		/*!< in: id1 */
-	const void*	p2)		/*!< in: id2 */
-{
-	const fts_aux_table_t*	fa1 = static_cast<const fts_aux_table_t*>(p1);
-	const fts_aux_table_t*	fa2 = static_cast<const fts_aux_table_t*>(p2);
-
-	return static_cast<int>(fa1->parent_id - fa2->parent_id);
-}
-
 bool fts_check_aux_table(const char *name,
                          table_id_t *table_id,
                          index_id_t *index_id)

storage/innobase/log/log0recv.cc

@@ -969,6 +969,8 @@ bool log_t::files::read_log_seg(lsn_t* start_lsn, lsn_t end_lsn)
 					 }
 			 });
 
+			DBUG_EXECUTE_IF("log_checksum_mismatch", { cksum = crc + 1; });
+
 			if (crc != cksum) {
 				ib::error() << "Invalid log block checksum."
 					    << " block: " << block_number