MDEV-8773: InnoDB innochecksum does not work with encrypted or page compressed tables

extra/innochecksum.cc

@@ -62,6 +62,7 @@ The parts not included are excluded by #ifndef UNIV_INNOCHECKSUM. */
 #include "fsp0fsp.h"             /* fsp_flags_get_page_size() &
                                     fsp_flags_get_zip_size() */
 #include "ut0crc32.h"            /* ut_crc32_init() */
+#include "fsp0pagecompress.h"    /* fil_get_compression_alg_name */
 
 #ifdef UNIV_NONINL
 # include "fsp0fsp.ic"
@@ -109,6 +110,8 @@ int n_fil_page_type_xdes;
 int n_fil_page_type_blob;
 int n_fil_page_type_zblob;
 int n_fil_page_type_other;
+int n_fil_page_type_page_compressed;
+int n_fil_page_type_page_compressed_encrypted;
 
 int n_fil_page_max_index_id;
 
@@ -152,6 +155,8 @@ struct per_index_stats {
 
 std::map<unsigned long long, per_index_stats> index_ids;
 
+bool encrypted = false;
+
 /* Get the page size of the filespace from the filespace header. */
 static
 my_bool
@@ -197,6 +202,8 @@ get_page_size(
   {
     compressed= true;
   }
+
+
   return TRUE;
 }
 
@@ -515,6 +522,18 @@ parse_page(
                }
                n_fil_page_type_zblob++;
                break;
+       case FIL_PAGE_PAGE_COMPRESSED:
+	       if (per_page_details) {
+		       printf("FIL_PAGE_PAGE_COMPRESSED\n");
+	       }
+	       n_fil_page_type_page_compressed++;
+	       break;
+       case FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED:
+	       if (per_page_details) {
+		       printf("FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED\n");
+	       }
+	       n_fil_page_type_page_compressed_encrypted++;
+	       break;
        default:
                if (per_page_details) {
                        printf("FIL_PAGE_TYPE_OTHER\n");
@@ -604,6 +623,8 @@ print_stats()
        printf("%d\tFIL_PAGE_TYPE_XDES\n", n_fil_page_type_xdes);
        printf("%d\tFIL_PAGE_TYPE_BLOB\n", n_fil_page_type_blob);
        printf("%d\tFIL_PAGE_TYPE_ZBLOB\n", n_fil_page_type_zblob);
+       printf("%d\tFIL_PAGE_PAGE_COMPRESSED\n", n_fil_page_type_page_compressed);
+       printf("%d\tFIL_PAGE_PAGE_COMPRESSED_ENCRYPTED\n", n_fil_page_type_page_compressed_encrypted);
        printf("%d\tother\n", n_fil_page_type_other);
        printf("%d\tmax index_id\n", n_fil_page_max_index_id);
        printf("undo type: %d insert, %d update, %d other\n",
@@ -791,7 +812,9 @@ int main(int argc, char **argv)
   while (!feof(f))
   {
     int page_ok = 1;
+
     bytes= fread(buf, 1, physical_page_size, f);
+
     if (!bytes && feof(f))
     {
       print_stats();
@@ -809,8 +832,54 @@ int main(int argc, char **argv)
       return 1;
     }
 
-    if (compressed) {
+    ulint page_type = mach_read_from_2(buf+FIL_PAGE_TYPE);
+    ulint key_version = mach_read_from_4(buf + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION);
+
+    if (key_version && page_type != FIL_PAGE_PAGE_COMPRESSED) {
+	    encrypted = true;
+    } else {
+	    encrypted = false;
+    }
+
+    ulint comp_method = 0;
+
+    if (encrypted) {
+	    comp_method = mach_read_from_2(buf+FIL_PAGE_DATA+FIL_PAGE_COMPRESSED_SIZE);
+    } else {
+	    comp_method = mach_read_from_8(buf+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION);
+    }
+
+    ulint comp_size = mach_read_from_2(buf+FIL_PAGE_DATA);
+    ib_uint32_t encryption_checksum = mach_read_from_4(buf+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4);
+
+
+    if (page_type == FIL_PAGE_PAGE_COMPRESSED) {
+      /* Page compressed tables do not have any checksum */
+      if (debug)
+	fprintf(stderr, "Page %lu page compressed with method %s real_size %lu\n", ct,
+	        fil_get_compression_alg_name(comp_method), comp_size);
+      page_ok = 1;
+    } else if (compressed) {
         /* compressed pages */
+	ulint crccsum = page_zip_calc_checksum(buf, physical_page_size, SRV_CHECKSUM_ALGORITHM_CRC32);
+	ulint icsum = page_zip_calc_checksum(buf, physical_page_size,  SRV_CHECKSUM_ALGORITHM_INNODB);
+
+        if (debug) {
+	  if (key_version != 0) {
+	    fprintf(stderr,
+		    "Page %lu encrypted key_version %lu calculated = %lu; crc32 = %lu; recorded = %u\n",
+		    ct, key_version, icsum, crccsum, encryption_checksum);
+	  }
+        }
+
+	if (encrypted) {
+	  if (encryption_checksum != 0 && crccsum != encryption_checksum && icsum != encryption_checksum) {
+	    if (debug)
+	      fprintf(stderr, "page %lu: compressed: calculated = %lu; crc32 = %lu; recorded = %u\n",
+		      ct, icsum, crccsum, encryption_checksum);
+	    fprintf(stderr, "Fail; page %lu invalid (fails compressed page checksum).\n", ct);
+	  }
+	} else {
           if (!page_zip_verify_checksum(buf, physical_page_size)) {
             fprintf(stderr, "Fail; page %lu invalid (fails compressed page checksum).\n", ct);
             if (!skip_corrupt)
@@ -821,8 +890,25 @@ int main(int argc, char **argv)
             }
             page_ok = 0;
           }
+	}
+    } else {
+      if (key_version != 0) {
+      /* Encrypted page */
+        if (debug) {
+	  if (page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) {
+	    fprintf(stderr,
+		    "Page %lu page compressed with method %s real_size %lu and encrypted key_version %lu checksum %u\n",
+		    ct, fil_get_compression_alg_name(comp_method), comp_size, key_version, encryption_checksum);
 	  } else {
+	    fprintf(stderr,
+		    "Page %lu encrypted key_version %lu checksum %u\n",
+		    ct, key_version, encryption_checksum);
+	  }
+        }
+      }
 
+      /* Page compressed tables do not contain FIL tailer */
+      if (page_type != FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED && page_type != FIL_PAGE_PAGE_COMPRESSED) {
         /* check the "stored log sequence numbers" */
          logseq= mach_read_from_4(buf + FIL_PAGE_LSN + 4);
         logseqfield= mach_read_from_4(buf + logical_page_size - FIL_PAGE_END_LSN_OLD_CHKSUM + 4);
@@ -856,11 +942,16 @@ int main(int argc, char **argv)
           }
           page_ok = 0;
         }
+      }
 
       /* now check the new method */
       csum= buf_calc_page_new_checksum(buf);
       crc32= buf_calc_page_crc32(buf);
       csumfield= mach_read_from_4(buf + FIL_PAGE_SPACE_OR_CHKSUM);
+
+      if (key_version)
+	      csumfield = encryption_checksum;
+
       if (debug)
         printf("page %lu: new style: calculated = %lu; crc32 = %lu; recorded = %lu\n",
                ct, csum, crc32, csumfield);
@@ -903,7 +994,10 @@ int main(int argc, char **argv)
       continue;
     }
 
+    /* Can't parse compressed or/and encrypted pages */
+    if (page_type != FIL_PAGE_PAGE_COMPRESSED && !encrypted) {
       parse_page(buf, xdes);
+    }
 
     if (verbose)
     {

mysql-test/suite/encryption/r/innochecksum.result

+SET GLOBAL innodb_file_format = `Barracuda`;
+SET GLOBAL innodb_file_per_table = ON;
+set global innodb_compression_algorithm = 1;
+# Create and populate a tables
+CREATE TABLE t1 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB ENCRYPTED=YES ENCRYPTION_KEY_ID=4;
+CREATE TABLE t2 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB ROW_FORMAT=COMPRESSED ENCRYPTED=YES ENCRYPTION_KEY_ID=4;
+CREATE TABLE t3 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB ROW_FORMAT=COMPRESSED ENCRYPTED=NO;
+CREATE TABLE t4 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB PAGE_COMPRESSED=1;
+CREATE TABLE t5 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB PAGE_COMPRESSED=1 ENCRYPTED=YES ENCRYPTION_KEY_ID=4;
+# Write file to make mysql-test-run.pl expect the "crash", but don't
+# start it until it's told to
+# We give 30 seconds to do a clean shutdown because we do not want
+# to redo apply the pages of t1.ibd at the time of recovery.
+# We want SQL to initiate the first access to t1.ibd.
+# Wait until disconnected.
+# Run innochecksum on t1
+# Run innochecksum on t2
+# Run innochecksum on t3
+# Run innochecksum on t4
+# Run innochecksum on t4
+# Write file to make mysql-test-run.pl start up the server again
+# Cleanup
+DROP TABLE t1, t2, t3, t4, t5;

mysql-test/suite/encryption/r/innodb-bad-key-change.result

@@ -49,8 +49,8 @@ SELECT * FROM t2;
 ERROR HY000: Got error 192 'Table encrypted but decryption failed. This could be because correct encryption management plugin is not loaded, used encryption key is not available or encryption method does not match.' from InnoDB
 SHOW WARNINGS;
 Level	Code	Message
-Warning	192	Table test/t2 in tablespace 7 is encrypted but encryption service or used key_id is not available.  Can't continue reading table.
-Warning	192	Table test/t2 in tablespace 7 is encrypted but encryption service or used key_id is not available.  Can't continue reading table.
+Warning	192	 is encrypted but encryption service or used key_id is not available.  Can't continue reading table.
+Warning	192	 is encrypted but encryption service or used key_id is not available.  Can't continue reading table.
 Warning	192	Table test/t2 is encrypted but encryption service or used key_id is not available.  Can't continue reading table.
 Error	1296	Got error 192 'Table encrypted but decryption failed. This could be because correct encryption management plugin is not loaded, used encryption key is not available or encryption method does not match.' from InnoDB
 SELECT * FROM t2 where id = 1;

mysql-test/suite/encryption/t/innochecksum.test

+#
+# MDEV-8773: InnoDB innochecksum does not work with encrypted or page compressed tables
+#
+
+# Don't test under embedded
+-- source include/not_embedded.inc
+# Require InnoDB
+-- source include/have_innodb.inc
+-- source include/have_file_key_management_plugin.inc
+
+if (!$INNOCHECKSUM) {
+  --echo Need innochecksum binary
+  --die Need innochecksum binary
+}
+
+--disable_query_log
+let $innodb_compression_algorithm_orig=`SELECT @@innodb_compression_algorithm`;
+let $innodb_file_format_orig = `SELECT @@innodb_file_format`;
+let $innodb_file_per_table_orig = `SELECT @@innodb_file_per_table`;
+--enable_query_log
+
+SET GLOBAL innodb_file_format = `Barracuda`;
+SET GLOBAL innodb_file_per_table = ON;
+# zlib
+set global innodb_compression_algorithm = 1;
+
+--echo # Create and populate a tables
+CREATE TABLE t1 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB ENCRYPTED=YES ENCRYPTION_KEY_ID=4;
+CREATE TABLE t2 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB ROW_FORMAT=COMPRESSED ENCRYPTED=YES ENCRYPTION_KEY_ID=4;
+CREATE TABLE t3 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB ROW_FORMAT=COMPRESSED ENCRYPTED=NO;
+CREATE TABLE t4 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB PAGE_COMPRESSED=1;
+CREATE TABLE t5 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB PAGE_COMPRESSED=1 ENCRYPTED=YES ENCRYPTION_KEY_ID=4;
+
+--disable_query_log
+--let $i = 1000
+while ($i)
+{
+  INSERT INTO t1 (b) VALUES (REPEAT('abcdefghijklmnopqrstuvwxyz', 100));
+  dec $i;
+}
+INSERT INTO t2 SELECT * FROM t1;
+INSERT INTO t3 SELECT * FROM t1;
+INSERT INTO t4 SELECT * FROM t1;
+INSERT INTO t5 SELECT * FROM t1;
+--enable_query_log
+
+let $MYSQLD_DATADIR=`select @@datadir`;
+let t1_IBD = $MYSQLD_DATADIR/test/t1.ibd;
+let t2_IBD = $MYSQLD_DATADIR/test/t2.ibd;
+let t3_IBD = $MYSQLD_DATADIR/test/t3.ibd;
+let t4_IBD = $MYSQLD_DATADIR/test/t4.ibd;
+let t5_IBD = $MYSQLD_DATADIR/test/t5.ibd;
+
+--echo # Write file to make mysql-test-run.pl expect the "crash", but don't
+--echo # start it until it's told to
+--exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
+
+--echo # We give 30 seconds to do a clean shutdown because we do not want
+--echo # to redo apply the pages of t1.ibd at the time of recovery.
+--echo # We want SQL to initiate the first access to t1.ibd.
+shutdown_server 30;
+
+--echo # Wait until disconnected.
+--source include/wait_until_disconnected.inc
+
+--echo # Run innochecksum on t1
+-- disable_result_log
+--exec $INNOCHECKSUM $t1_IBD
+
+--echo # Run innochecksum on t2
+
+--exec $INNOCHECKSUM $t2_IBD
+
+--echo # Run innochecksum on t3
+
+--exec $INNOCHECKSUM $t3_IBD
+
+--echo # Run innochecksum on t4
+
+--exec $INNOCHECKSUM $t4_IBD
+
+--echo # Run innochecksum on t4
+
+--exec $INNOCHECKSUM $t4_IBD
+
+--enable_result_log
+
+--echo # Write file to make mysql-test-run.pl start up the server again
+--exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
+--enable_reconnect
+--source include/wait_until_connected_again.inc
+
+--echo # Cleanup
+DROP TABLE t1, t2, t3, t4, t5;
+
+# reset system
+--disable_query_log
+EVAL SET GLOBAL innodb_compression_algorithm = $innodb_compression_algorithm_orig;
+EVAL SET GLOBAL innodb_file_per_table = $innodb_file_per_table_orig;
+EVAL SET GLOBAL innodb_file_format = $innodb_file_format_orig;
+--enable_query_log

mysql-test/suite/encryption/t/innodb-bad-key-change.test

@@ -79,6 +79,7 @@ INSERT INTO t2 VALUES ('foobar',1,2);
 
 --error ER_GET_ERRMSG
 SELECT * FROM t2;
+--replace_regex /.*tablespace [0-9]*//
 SHOW WARNINGS;
 --error ER_GET_ERRMSG
 SELECT * FROM t2 where id = 1;

storage/innobase/fil/fil0crypt.cc

@@ -115,17 +115,6 @@ fil_crypt_needs_rotation(
 	uint			latest_key_version,	/*!< in: Latest key version */
 	uint			rotate_key_age);	/*!< in: When to rotate */
 
-/**
-* Magic pattern in start of crypt data on page 0
-*/
-#define MAGIC_SZ 6
-
-static const unsigned char CRYPT_MAGIC[MAGIC_SZ] = {
-	's', 0xE, 0xC, 'R', 'E', 't' };
-
-static const unsigned char EMPTY_PATTERN[MAGIC_SZ] = {
-	0x0, 0x0, 0x0, 0x0, 0x0, 0x0 };
-
 /*********************************************************************
 Init space crypt */
 UNIV_INTERN

storage/innobase/include/fil0crypt.h

@@ -26,6 +26,17 @@ Created 04/01/2015 Jan Lindström
 #ifndef fil0crypt_h
 #define fil0crypt_h
 
+/**
+* Magic pattern in start of crypt data on page 0
+*/
+#define MAGIC_SZ 6
+
+static const unsigned char CRYPT_MAGIC[MAGIC_SZ] = {
+	's', 0xE, 0xC, 'R', 'E', 't' };
+
+static const unsigned char EMPTY_PATTERN[MAGIC_SZ] = {
+	0x0, 0x0, 0x0, 0x0, 0x0, 0x0 };
+
 /* This key will be used if nothing else is given */
 #define FIL_DEFAULT_ENCRYPTION_KEY ENCRYPTION_KEY_SYSTEM_DATA
 

storage/innobase/include/fsp0pagecompress.ic

@@ -96,6 +96,7 @@ fil_page_is_compressed_encrypted(
 	return(mach_read_from_2(buf+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED);
 }
 
+#ifndef UNIV_INNOCHECKSUM
 /*******************************************************************//**
 Returns the page compression level of the space, or 0 if the space
 is not compressed. The tablespace must be cached in the memory cache.
@@ -140,6 +141,8 @@ fil_space_is_page_compressed(
 	return(flags);
 }
 
+#endif /* UNIV_INNOCHECKSUM */
+
 /****************************************************************//**
 Get the name of the compression algorithm used for page
 compression.
@@ -166,13 +169,19 @@ fil_get_compression_alg_name(
 	case PAGE_LZMA_ALGORITHM:
 		return ("LZMA");
 		break;
-	default:
-		return("UNKNOWN");
-		ut_error;
+	case PAGE_BZIP2_ALGORITHM:
+		return ("BZIP2");
+		break;
+	case PAGE_SNAPPY_ALGORITHM:
+		return ("SNAPPY");
 		break;
+	/* No default to get compiler warning */
 	}
+
+	return ("NULL");
 }
 
+#ifndef UNIV_INNOCHECKSUM
 /*******************************************************************//**
 Returns the atomic writes flag of the space, or false if the space
 is not using atomic writes. The tablespace must be cached in the memory cache.
@@ -209,3 +218,5 @@ fil_page_is_lzo_compressed(
 	       (mach_read_from_2(buf+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED &&
 			mach_read_from_2(buf+FIL_PAGE_DATA+FIL_PAGE_COMPRESSED_SIZE) == PAGE_LZO_ALGORITHM));
 }
+
+#endif /* UNIV_INNOCHECKSUM */

storage/xtradb/fil/fil0crypt.cc

@@ -115,17 +115,6 @@ fil_crypt_needs_rotation(
 	uint			latest_key_version,	/*!< in: Latest key version */
 	uint			rotate_key_age);	/*!< in: When to rotate */
 
-/**
-* Magic pattern in start of crypt data on page 0
-*/
-#define MAGIC_SZ 6
-
-static const unsigned char CRYPT_MAGIC[MAGIC_SZ] = {
-	's', 0xE, 0xC, 'R', 'E', 't' };
-
-static const unsigned char EMPTY_PATTERN[MAGIC_SZ] = {
-	0x0, 0x0, 0x0, 0x0, 0x0, 0x0 };
-
 /*********************************************************************
 Init space crypt */
 UNIV_INTERN

storage/xtradb/include/fil0crypt.h

@@ -26,6 +26,17 @@ Created 04/01/2015 Jan Lindström
 #ifndef fil0crypt_h
 #define fil0crypt_h
 
+/**
+* Magic pattern in start of crypt data on page 0
+*/
+#define MAGIC_SZ 6
+
+static const unsigned char CRYPT_MAGIC[MAGIC_SZ] = {
+	's', 0xE, 0xC, 'R', 'E', 't' };
+
+static const unsigned char EMPTY_PATTERN[MAGIC_SZ] = {
+	0x0, 0x0, 0x0, 0x0, 0x0, 0x0 };
+
 /* This key will be used if nothing else is given */
 #define FIL_DEFAULT_ENCRYPTION_KEY ENCRYPTION_KEY_SYSTEM_DATA
 

storage/xtradb/include/fsp0pagecompress.ic

@@ -96,6 +96,7 @@ fil_page_is_compressed_encrypted(
 	return(mach_read_from_2(buf+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED);
 }
 
+#ifndef UNIV_INNOCHECKSUM
 /*******************************************************************//**
 Returns the page compression level of the space, or 0 if the space
 is not compressed. The tablespace must be cached in the memory cache.
@@ -140,6 +141,8 @@ fil_space_is_page_compressed(
 	return(flags);
 }
 
+#endif /* UNIV_INNOCHECKSUM */
+
 /****************************************************************//**
 Get the name of the compression algorithm used for page
 compression.
@@ -166,13 +169,19 @@ fil_get_compression_alg_name(
 	case PAGE_LZMA_ALGORITHM:
 		return ("LZMA");
 		break;
-	default:
-		return("UNKNOWN");
-		ut_error;
+	case PAGE_BZIP2_ALGORITHM:
+		return ("BZIP2");
+		break;
+	case PAGE_SNAPPY_ALGORITHM:
+		return ("SNAPPY");
 		break;
+	/* No default to get compiler warning */
 	}
+
+	return ("NULL");
 }
 
+#ifndef UNIV_INNOCHECKSUM
 /*******************************************************************//**
 Returns the atomic writes flag of the space, or false if the space
 is not using atomic writes. The tablespace must be cached in the memory cache.
@@ -209,3 +218,5 @@ fil_page_is_lzo_compressed(
 	       (mach_read_from_2(buf+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED &&
 			mach_read_from_2(buf+FIL_PAGE_DATA+FIL_PAGE_COMPRESSED_SIZE) == PAGE_LZO_ALGORITHM));
 }
+
+#endif /* UNIV_INNOCHECKSUM */