Merge 10.1 into 10.2

extra/mariabackup/fil_cur.cc

@@ -31,6 +31,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
 
 #include "fil_cur.h"
 #include "fil0crypt.h"
+#include "fil0pagecompress.h"
 #include "common.h"
 #include "read_filt.h"
 #include "xtrabackup.h"
@@ -357,6 +358,7 @@ xb_fil_cur_read(
 	for (page = cursor->buf, i = 0; i < npages;
 	     page += page_size, i++) {
 		ulint page_no = cursor->buf_page_no + i;
+		ulint page_type = mach_read_from_2(page + FIL_PAGE_TYPE);
 
                 if (cursor->space_id == TRX_SYS_SPACE &&
                     page_no >= FSP_EXTENT_SIZE &&
@@ -378,12 +380,33 @@ xb_fil_cur_read(
 			memcpy(tmp_page, page, page_size);
 
 			if (!fil_space_decrypt(space, tmp_frame,
-					       tmp_page, &decrypted)
+					       tmp_page, &decrypted)) {
+				goto corrupted;
+			}
+
+			if (page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) {
+				goto page_decomp;
+			}
+
+			if (buf_page_is_corrupted(true, tmp_page,
+						  cursor->page_size, space)) {
+				goto corrupted;
+			}
+
+		} else if (page_type == FIL_PAGE_PAGE_COMPRESSED) {
+			memcpy(tmp_page, page, cursor->page_size.physical());
+page_decomp:
+			ulint decomp = fil_page_decompress(tmp_frame, tmp_page);
+
+			if (!decomp
+			    || (decomp != srv_page_size
+				&& cursor->page_size.is_compressed())
 			    || buf_page_is_corrupted(true, tmp_page,
 						     cursor->page_size,
 						     space)) {
 				goto corrupted;
 			}
+
 		} else if (buf_page_is_corrupted(true, page, cursor->page_size,
 						 space)) {
 corrupted:

mysql-test/suite/encryption/r/innodb-bad-key-change.result

@@ -2,6 +2,7 @@ call mtr.add_suppression("Plugin 'file_key_management' init function returned er
 call mtr.add_suppression("Plugin 'file_key_management' registration.*failed");
 call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file '.*test.t[12]\\.ibd' cannot be decrypted\\.");
 call mtr.add_suppression("failed to read or decrypt \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\]");
+call mtr.add_suppression("InnoDB: Encrypted page \\[page id: space=[1-9][0-9]*, page number=3\\] in file .*test.t1.ibd looks corrupted; key_version=1");
 call mtr.add_suppression("File '.*mysql-test.std_data.keysbad3\\.txt' not found");
 # Start server with keys2.txt
 SET GLOBAL innodb_file_per_table = ON;

mysql-test/suite/encryption/r/innodb-encryption-disable.result

 call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file '.*test.t[15]\\.ibd' cannot be decrypted\\.");
 call mtr.add_suppression("failed to read or decrypt \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\]");
+call mtr.add_suppression("InnoDB: Encrypted page \\[page id: space=[1-9][0-9]*, page number=3\\] in file .*test.t[15].ibd looks corrupted; key_version=1");
 call mtr.add_suppression("Couldn't load plugins from 'file_key_management");
 create table t5 (
 `intcol1` int(32) DEFAULT NULL,

mysql-test/suite/encryption/r/innodb-missing-key.result

 call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file '.*test.t[123]\\.ibd' cannot be decrypted\\.");
 call mtr.add_suppression("failed to read or decrypt \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\]");
+call mtr.add_suppression("InnoDB: Encrypted page \\[page id: space=[1-9][0-9]*, page number=3\\] in file .*test.t[12].ibd looks corrupted; key_version=1");
 # Start server with keys2.txt
 CREATE TABLE t1(a int not null primary key auto_increment, b varchar(128)) engine=innodb ENCRYPTED=YES ENCRYPTION_KEY_ID=19;
 CREATE TABLE t2(a int not null primary key auto_increment, b varchar(128)) engine=innodb ENCRYPTED=YES ENCRYPTION_KEY_ID=1;

mysql-test/suite/encryption/t/innodb-bad-key-change.test

@@ -12,6 +12,7 @@ call mtr.add_suppression("Plugin 'file_key_management' init function returned er
 call mtr.add_suppression("Plugin 'file_key_management' registration.*failed");
 call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file '.*test.t[12]\\.ibd' cannot be decrypted\\.");
 call mtr.add_suppression("failed to read or decrypt \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\]");
+call mtr.add_suppression("InnoDB: Encrypted page \\[page id: space=[1-9][0-9]*, page number=3\\] in file .*test.t1.ibd looks corrupted; key_version=1");
 call mtr.add_suppression("File '.*mysql-test.std_data.keysbad3\\.txt' not found");
 
 --echo # Start server with keys2.txt

mysql-test/suite/encryption/t/innodb-encryption-disable.test

@@ -9,6 +9,7 @@
 
 call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file '.*test.t[15]\\.ibd' cannot be decrypted\\.");
 call mtr.add_suppression("failed to read or decrypt \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\]");
+call mtr.add_suppression("InnoDB: Encrypted page \\[page id: space=[1-9][0-9]*, page number=3\\] in file .*test.t[15].ibd looks corrupted; key_version=1");
 
 # Suppression for builds where file_key_management plugin is linked statically
 call mtr.add_suppression("Couldn't load plugins from 'file_key_management");

mysql-test/suite/encryption/t/innodb-missing-key.test

@@ -9,6 +9,7 @@
 
 call mtr.add_suppression("InnoDB: The page \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\] in file '.*test.t[123]\\.ibd' cannot be decrypted\\.");
 call mtr.add_suppression("failed to read or decrypt \\[page id: space=[1-9][0-9]*, page number=[1-9][0-9]*\\]");
+call mtr.add_suppression("InnoDB: Encrypted page \\[page id: space=[1-9][0-9]*, page number=3\\] in file .*test.t[12].ibd looks corrupted; key_version=1");
 
 --echo # Start server with keys2.txt
 -- let $restart_parameters=--file-key-management-filename=$MYSQL_TEST_DIR/std_data/keys2.txt

mysql-test/suite/mariabackup/encrypted_page_compressed.opt

+--innodb-encryption-rotate-key-age=2
+--innodb-encryption-threads=4
+--innodb-tablespaces-encryption
+--plugin-load-add=$FILE_KEY_MANAGEMENT_SO
+--loose-file-key-management
+--loose-file-key-management-filename=$MYSQL_TEST_DIR/std_data/logkey.txt

mysql-test/suite/mariabackup/encrypted_page_compressed.result

+CREATE TABLE t1 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT, c char(200)) ENGINE=InnoDB page_compressed=yes encrypted=yes;
+insert into t1(b, c) values("mariadb", "mariabackup");
+# Corrupt the table
+# xtrabackup backup
+FOUND 1 /Database page corruption detected/ in backup.log
+drop table t1;

mysql-test/suite/mariabackup/encrypted_page_compressed.test

+source include/have_file_key_management.inc;
+CREATE TABLE t1 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT, c char(200)) ENGINE=InnoDB page_compressed=yes encrypted=yes;
+insert into t1(b, c) values("mariadb", "mariabackup");
+
+let $MYSQLD_DATADIR=`select @@datadir`;
+let t1_IBD = $MYSQLD_DATADIR/test/t1.ibd;
+let INNODB_PAGE_SIZE=`select @@innodb_page_size`;
+
+--source include/shutdown_mysqld.inc
+
+--echo # Corrupt the table
+
+perl;
+use strict;
+use warnings;
+use Fcntl qw(:DEFAULT :seek);
+
+my $ibd_file = $ENV{'t1_IBD'};
+
+my $chunk;
+my $page_size = $ENV{'INNODB_PAGE_SIZE'};
+
+sysopen IBD_FILE, $ibd_file, O_RDWR || die "Unable to open $ibd_file";
+sysseek IBD_FILE,  $page_size * 3 + 75, SEEK_CUR;
+$chunk = '\xAA\xAA\xAA\xAA';
+syswrite IBD_FILE, $chunk, 4;
+
+close IBD_FILE;
+EOF
+
+--source include/start_mysqld.inc
+
+echo # xtrabackup backup;
+--disable_result_log
+let $targetdir=$MYSQLTEST_VARDIR/tmp/backup;
+let $backuplog=$MYSQLTEST_VARDIR/tmp/backup.log;
+--error 1
+exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir > $backuplog;
+--enable_result_log
+
+--let SEARCH_PATTERN=Database page corruption detected
+--let SEARCH_FILE=$backuplog
+--source include/search_pattern_in_file.inc
+remove_file $backuplog;
+
+drop table t1;
+rmdir $targetdir;

mysql-test/suite/mariabackup/unencrypted_page_compressed.result

+SET GLOBAL innodb_purge_rseg_truncate_frequency = 1;
+CREATE TABLE t1 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT, c char(200)) ENGINE=InnoDB page_compressed=yes;
+insert into t1(b, c) values("mariadb", "mariabackup");
+InnoDB		0 transactions not purged
+# Corrupt the table
+# xtrabackup backup
+FOUND 1 /Database page corruption detected/ in backup.log
+drop table t1;

mysql-test/suite/mariabackup/unencrypted_page_compressed.test

+SET GLOBAL innodb_purge_rseg_truncate_frequency = 1;
+CREATE TABLE t1 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT, c char(200)) ENGINE=InnoDB page_compressed=yes;
+insert into t1(b, c) values("mariadb", "mariabackup");
+--source ../innodb/include/wait_all_purged.inc
+
+let $MYSQLD_DATADIR=`select @@datadir`;
+let t1_IBD = $MYSQLD_DATADIR/test/t1.ibd;
+let INNODB_PAGE_SIZE=`select @@innodb_page_size`;
+
+--source include/shutdown_mysqld.inc
+
+--echo # Corrupt the table
+
+perl;
+use strict;
+use warnings;
+use Fcntl qw(:DEFAULT :seek);
+
+my $ibd_file = $ENV{'t1_IBD'};
+
+my $chunk;
+my $page_size = $ENV{'INNODB_PAGE_SIZE'};
+
+sysopen IBD_FILE, $ibd_file, O_RDWR || die "Unable to open $ibd_file";
+sysseek IBD_FILE, 16384 * 3 + 75, SEEK_CUR;
+$chunk = '\xAA\xAA\xAA\xAA';
+syswrite IBD_FILE, $chunk, 4;
+
+close IBD_FILE;
+EOF
+
+--let $restart_parameters= --skip-innodb-buffer-pool-load-at-startup
+--source include/start_mysqld.inc
+
+echo # xtrabackup backup;
+--disable_result_log
+let $targetdir=$MYSQLTEST_VARDIR/tmp/backup;
+let $backuplog=$MYSQLTEST_VARDIR/tmp/backup.log;
+--error 1
+exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir > $backuplog;
+--enable_result_log
+
+--let SEARCH_PATTERN=Database page corruption detected
+--let SEARCH_FILE=$backuplog
+--source include/search_pattern_in_file.inc
+remove_file $backuplog;
+
+drop table t1;
+rmdir $targetdir;

storage/innobase/buf/buf0buf.cc

@@ -526,13 +526,13 @@ static bool buf_page_decrypt_after_read(buf_page_t* bpage, fil_space_t* space)
 		/* Verify encryption checksum before we even try to
 		decrypt. */
 		if (!fil_space_verify_crypt_checksum(dst_frame, bpage->size)) {
+decrypt_failed:
 			ib::error() << "Encrypted page " << bpage->id
 				    << " in file " << space->chain.start->name
 				    << " looks corrupted; key_version="
 				    << mach_read_from_4(
 					    FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION
 					    + dst_frame);
-decrypt_failed:
 			/* Mark page encrypted in case it should be. */
 			if (space->crypt_data->type
 			    != CRYPT_SCHEME_UNENCRYPTED) {

storage/xtradb/buf/buf0buf.cc

@@ -482,6 +482,8 @@ static bool buf_page_decrypt_after_read(buf_page_t* bpage, fil_space_t* space)
 		decrypt. */
 		if (!fil_space_verify_crypt_checksum(
 			    dst_frame, buf_page_get_zip_size(bpage))) {
+
+decrypt_failed:
 			ib_logf(IB_LOG_LEVEL_ERROR,
 				"Encrypted page %u:%u in file %s"
 				" looks corrupted; key_version=" ULINTPF,
@@ -490,7 +492,7 @@ static bool buf_page_decrypt_after_read(buf_page_t* bpage, fil_space_t* space)
 				mach_read_from_4(
 					FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION
 					+ dst_frame));
-decrypt_failed:
+
 			/* Mark page encrypted in case it should be. */
 			if (space->crypt_data->type
 			    != CRYPT_SCHEME_UNENCRYPTED) {
@@ -4797,7 +4799,6 @@ static dberr_t buf_page_check_corrupt(buf_page_t* bpage, fil_space_t* space)
 		not anymore encrypted. */
 		corrupted = buf_page_is_corrupted(true, dst_frame, zip_size,
 						  space);
-
 		if (!corrupted) {
 			bpage->encrypted = false;
 		} else {