MDEV-11782: Redefine the innodb_encrypt_log format

Write only one encryption key to the checkpoint page.
Use 4 bytes of nonce. Encrypt more of each redo log block,
only skipping the 4-byte field LOG_BLOCK_HDR_NO which the
initialization vector is derived from.

Issue notes, not warning messages for rewriting the redo log files.

recv_recovery_from_checkpoint_finish(): Do not generate any redo log,
because we must avoid that before rewriting the redo log files, or
otherwise a crash during a redo log rewrite (removing or adding
encryption) may end up making the database unrecoverable.
Instead, do these tasks in innobase_start_or_create_for_mysql().

Issue a firm "Missing MLOG_CHECKPOINT" error message. Remove some
unreachable code and duplicated error messages for log corruption.

LOG_HEADER_FORMAT_ENCRYPTED: A flag for identifying an encrypted redo
log format.

log_group_t::is_encrypted(), log_t::is_encrypted(): Determine
if the redo log is in encrypted format.

recv_find_max_checkpoint(): Interpret LOG_HEADER_FORMAT_ENCRYPTED.

srv_prepare_to_delete_redo_log_files(): Display NOTE messages about
adding or removing encryption. Do not issue warnings for redo log
resizing any more.

innobase_start_or_create_for_mysql(): Rebuild the redo logs also when
the encryption changes.

innodb_log_checksums_func_update(): Always use the CRC-32C checksum
if innodb_encrypt_log. If needed, issue a warning
that innodb_encrypt_log implies innodb_log_checksums.

log_group_write_buf(): Compute the checksum on the encrypted
block contents, so that transmission errors or incomplete blocks can be
detected without decrypting.

Rewrite most of the redo log encryption code. Only remember one
encryption key at a time (but remember up to 5 when upgrading from the
MariaDB 10.1 format.)

mysql-test/mysql-test-run.pl

@@ -4345,10 +4345,7 @@ sub extract_warning_lines ($$) {
      qr/error .*connecting to master/,
      qr/InnoDB: Error: in ALTER TABLE `test`.`t[12]`/,
      qr/InnoDB: Error: table `test`.`t[12]` .*does not exist in the InnoDB internal/,
-     qr/InnoDB: Warning: Setting innodb_use_sys_malloc/,
      qr/InnoDB: Warning: a long semaphore wait:/,
-     qr/InnoDB: Disabling redo log encryption/,
-     qr/InnoDB: Redo log crypto: Can't initialize to key version -1u/,
      qr/InnoDB: Dumping buffer pool.*/,
      qr/InnoDB: Buffer pool.*/,
      qr/InnoDB: Warning: Writer thread is waiting this semaphore/,
@@ -4422,9 +4419,6 @@ sub extract_warning_lines ($$) {
      qr|InnoDB: TABLE to scan your table for corruption|,
      qr/InnoDB: See also */,
      qr/InnoDB: Cannot open .*ib_buffer_pool.* for reading: No such file or directory*/,
-     qr/InnoDB: Upgrading redo log:*/,
-     qr|InnoDB: Starting to delete and rewrite log files.|,
-     qr/InnoDB: New log files created, LSN=*/,
      qr|InnoDB: Creating foreign key constraint system tables.|,
      qr/InnoDB: Table .*mysql.*innodb_table_stats.* not found./,
      qr/InnoDB: User stopword table .* does not exist./

mysql-test/suite/encryption/r/innodb-log-encrypt-crash.result

-call mtr.add_suppression("InnoDB: New log files created, LSN=.*");
-call mtr.add_suppression("InnoDB: Creating foreign key constraint system tables.");
-call mtr.add_suppression("InnoDB: Error: Table .*");
-CREATE TABLE t1 (
-pk bigint auto_increment,
-col_int int,
-col_int_key int,
-col_char char(12),
-col_char_key char(12),
-primary key (pk),
-key (`col_int_key` ),
-key (`col_char_key` )
-) ENGINE=InnoDB;
-CREATE TABLE t2 LIKE t1;
-INSERT INTO t1 VALUES (NULL,1,1,'foo','foo'),(NULL,2,2,'bar','bar'),(NULL,3,3,'baz','baz'),(NULL,4,4,'qux','qux');
-INSERT INTO t2 
-SELECT NULL, a1.col_int, a1.col_int_key, a1.col_char, a1.col_char_key 
-FROM t1 a1, t1 a2, t1 a3, t1 a4, t1 a5, t1 a6, t1 a7, t1 a8, t1 a9, t1 a10;
-DROP TABLE t1, t2;

mysql-test/suite/encryption/r/innodb-log-encrypt.result

-create table t1(c1 bigint not null, b char(200), c varchar(200)) engine=innodb encrypted=yes encryption_key_id=1;
-show warnings;
-Level	Code	Message
-create procedure innodb_insert_proc (repeat_count int)
-begin
-declare current_num int;
-set current_num = 0;
-while current_num < repeat_count do
-insert into t1 values(current_num, substring(MD5(RAND()), -64), REPEAT('privatejanprivate',10));
-set current_num = current_num + 1;
-end while;
-end//
-commit;
-begin;
-call innodb_insert_proc(2000);
-commit;
-update t1 set c1 = c1 +1;
-select count(*) from t1;
-count(*)
-2000
-# Kill the server
-# ibdata1 yes on expecting NOT FOUND
-NOT FOUND /privatejanprivate/ in ibdata1
-# t1 yes on expecting NOT FOUND
-NOT FOUND /privatejanprivate/ in t1.ibd
-# log0 yes on expecting NOT FOUND
-NOT FOUND /privatejanprivate/ in ib_logfile0
-# log1 yes on expecting NOT FOUND
-NOT FOUND /privatejanprivate/ in ib_logfile1
-# Restart mysqld --innodb_encrypt_log=0
-insert into t1 values(5000, substring(MD5(RAND()), -64), REPEAT('publicmessage',10));
-insert into t1 values(5001, substring(MD5(RAND()), -64), REPEAT('publicmessage',10));
-insert into t1 values(5002, substring(MD5(RAND()), -64), REPEAT('publicmessage',10));
-insert into t1 values(5003, substring(MD5(RAND()), -64), REPEAT('publicmessage',10));
-insert into t1 values(5004, substring(MD5(RAND()), -64), REPEAT('publicmessage',10));
-# ibdata1 yes on expecting NOT FOUND
-NOT FOUND /privatejanprivate/ in ibdata1
-# t1 yes on expecting NOT FOUND
-NOT FOUND /privatejanprivate/ in t1.ibd
-# log0 yes on expecting NOT FOUND
-NOT FOUND /privatejanprivate/ in ib_logfile0
-# log1 yes on expecting NOT FOUND
-NOT FOUND /privatejanprivate/ in ib_logfile1
-# ibdata1 yes on expecting NOT FOUND
-NOT FOUND /publicmessage/ in ibdata1
-# t1 yes on expecting NOT FOUND
-NOT FOUND /publicmessage/ in t1.ibd
-# log0 no on expecting FOUND/NOTFOUND depending where insert goes
-FOUND /publicmessage/ in ib_logfile0
-# log1 no on expecting FOUND/NOTFOUND depending where insert goes
-NOT FOUND /publicmessage/ in ib_logfile1
-drop procedure innodb_insert_proc;
-drop table t1;

mysql-test/suite/encryption/r/innodb_encrypt_log.result

+#
+# MDEV-9011: Redo log encryption does not work
+#
+#
+# MDEV-9422 Encrypted redo log checksum errors
+# on restart after killing busy server instance
+#
+SET GLOBAL innodb_log_checksums=0;
+Warnings:
+Warning	138	innodb_encrypt_log implies innodb_log_checksums
+SELECT @@global.innodb_log_checksums;
+@@global.innodb_log_checksums
+1
+CREATE TABLE t0 (
+pk bigint auto_increment,
+col_int int,
+col_int_key int,
+col_char char(12),
+col_char_key char(12),
+primary key (pk),
+key (col_int_key),
+key (col_char_key)
+) ENGINE=InnoDB ENCRYPTED=YES ENCRYPTION_KEY_ID=1;
+CREATE TEMPORARY TABLE t LIKE t0;
+INSERT INTO t VALUES
+(NULL,1,1,'private','secret'),(NULL,2,2,'sacred','success'),
+(NULL,3,3,'story','secure'),(NULL,4,4,'security','sacrament');
+SET GLOBAL innodb_flush_log_at_trx_commit=1;
+INSERT INTO t0
+SELECT NULL, t1.col_int, t1.col_int_key, t1.col_char, t1.col_char_key
+FROM t t1, t t2, t t3, t t4, t t5;
+# Kill the server
+# ibdata1 expecting NOT FOUND
+NOT FOUND /private|secret|sacr(ed|ament)|success|story|secur(e|ity)/ in ibdata1
+# t0.ibd expecting NOT FOUND
+NOT FOUND /private|secret|sacr(ed|ament)|success|story|secur(e|ity)/ in t0.ibd
+# ib_logfile0 expecting NOT FOUND
+NOT FOUND /private|secret|sacr(ed|ament)|success|story|secur(e|ity)/ in ib_logfile0
+# ib_logfile1 expecting NOT FOUND
+NOT FOUND /private|secret|sacr(ed|ament)|success|story|secur(e|ity)/ in ib_logfile1
+# Restart without redo log encryption
+SELECT COUNT(*) FROM t0;
+COUNT(*)
+1024
+CHECK TABLE t0;
+Table	Op	Msg_type	Msg_text
+test.t0	check	status	OK
+SET GLOBAL innodb_flush_log_at_trx_commit=1;
+INSERT INTO t0 VALUES(NULL, 5, 5, 'public', 'gossip');
+# Kill the server
+# ib_logfile0 expecting NOT FOUND
+NOT FOUND /private|secret|sacr(ed|ament)|success|story|secur(e|ity)/ in ib_logfile0
+# ib_logfile0 expecting FOUND
+FOUND /public|gossip/ in ib_logfile0
+# ibdata1 expecting NOT FOUND
+NOT FOUND /private|secret|sacr(ed|ament)|success|story|secur(e|ity)|public|gossip/ in ibdata1
+# t0.ibd expecting NOT FOUND
+NOT FOUND /private|secret|sacr(ed|ament)|success|story|secur(e|ity)|public|gossip/ in t0.ibd
+SELECT COUNT(*) FROM t0;
+COUNT(*)
+1025
+CHECK TABLE t0;
+Table	Op	Msg_type	Msg_text
+test.t0	check	status	OK
+DROP TABLE t0;

mysql-test/suite/encryption/r/innodb_encrypt_log_corruption.result

@@ -41,7 +41,7 @@ WHERE engine = 'innodb'
 AND support IN ('YES', 'DEFAULT', 'ENABLED');
 ENGINE	SUPPORT	COMMENT	TRANSACTIONS	XA	SAVEPOINTS
 FOUND /InnoDB: Invalid log block checksum. block: 2372 checkpoint no: 1 expected: 3362026715 found: 144444122/ in mysqld.1.err
-FOUND /InnoDB: Ignoring the redo log due to missing MLOG_CHECKPOINT between the checkpoint 1213964 and the end 1213952\./ in mysqld.1.err
+FOUND /InnoDB: Missing MLOG_CHECKPOINT between the checkpoint 1213964 and the end 1213952\./ in mysqld.1.err
 # --innodb-force-recovery=6 (skip the entire redo log)
 SELECT * FROM INFORMATION_SCHEMA.ENGINES
 WHERE engine = 'innodb'
@@ -54,7 +54,6 @@ SELECT * FROM INFORMATION_SCHEMA.ENGINES
 WHERE engine = 'innodb'
 AND support IN ('YES', 'DEFAULT', 'ENABLED');
 ENGINE	SUPPORT	COMMENT	TRANSACTIONS	XA	SAVEPOINTS
-FOUND /InnoDB: Ignoring the redo log due to missing MLOG_CHECKPOINT between the checkpoint 1213964 and the end 1213952\./ in mysqld.1.err
 # --innodb-force-recovery=6 (skip the entire redo log)
 SELECT * FROM INFORMATION_SCHEMA.ENGINES
 WHERE engine = 'innodb'
@@ -90,6 +89,7 @@ SELECT COUNT(*) `1` FROM INFORMATION_SCHEMA.ENGINES WHERE engine='innodb'
 AND support IN ('YES', 'DEFAULT', 'ENABLED');
 1
 1
+FOUND /InnoDB: Encrypting redo log/ in mysqld.1.err
 ib_buffer_pool
 ib_logfile0
 ib_logfile1

mysql-test/suite/encryption/t/innodb-log-encrypt-crash.opt

---innodb-encrypt-log=ON
---plugin-load-add=$FILE_KEY_MANAGEMENT_SO
---loose-file-key-management
---loose-file-key-management-filename=$MYSQL_TEST_DIR/std_data/logkey.txt
---file-key-management-encryption-algorithm=aes_cbc
---innodb-buffer-pool-size=128M

mysql-test/suite/encryption/t/innodb-log-encrypt-crash.test

--- source include/have_innodb.inc
--- source include/not_embedded.inc
-# test takes very long time on debug build
--- source include/not_debug.inc
--- source include/big_test.inc
--- source filekeys_plugin.inc
-
-call mtr.add_suppression("InnoDB: New log files created, LSN=.*");
-call mtr.add_suppression("InnoDB: Creating foreign key constraint system tables.");
-call mtr.add_suppression("InnoDB: Error: Table .*");
-
-#
-# MDEV-9422: Checksum errors on restart when killing busy instance that uses encrypted XtraDB tables
-#
-
-CREATE TABLE t1 (
-  pk bigint auto_increment,
-  col_int int,
-  col_int_key int,
-  col_char char(12),
-  col_char_key char(12),
-  primary key (pk),
-  key (`col_int_key` ),
-  key (`col_char_key` )
-) ENGINE=InnoDB;
-CREATE TABLE t2 LIKE t1;
-
-INSERT INTO t1 VALUES (NULL,1,1,'foo','foo'),(NULL,2,2,'bar','bar'),(NULL,3,3,'baz','baz'),(NULL,4,4,'qux','qux');
-INSERT INTO t2 
-  SELECT NULL, a1.col_int, a1.col_int_key, a1.col_char, a1.col_char_key 
-    FROM t1 a1, t1 a2, t1 a3, t1 a4, t1 a5, t1 a6, t1 a7, t1 a8, t1 a9, t1 a10;
-
---exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
---shutdown_server 0
---source include/wait_until_disconnected.inc
-
---exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
---enable_reconnect
---source include/wait_until_connected_again.inc
-
-DROP TABLE t1, t2;

mysql-test/suite/encryption/t/innodb-log-encrypt.test

--- source include/have_innodb.inc
--- source include/not_embedded.inc
--- source filekeys_plugin.inc
-
-#
-# MDEV-9011: Redo log encryption does not work
-#
-
-create table t1(c1 bigint not null, b char(200), c varchar(200)) engine=innodb encrypted=yes encryption_key_id=1;
-show warnings;
-
-delimiter //;
-create procedure innodb_insert_proc (repeat_count int)
-begin
-  declare current_num int;
-  set current_num = 0;
-  while current_num < repeat_count do
-    insert into t1 values(current_num, substring(MD5(RAND()), -64), REPEAT('privatejanprivate',10));
-    set current_num = current_num + 1;
-  end while;
-end//
-delimiter ;//
-commit;
-
-begin;
-call innodb_insert_proc(2000);
-commit;
-
-update t1 set c1 = c1 +1;
-select count(*) from t1;
-
---let $MYSQLD_DATADIR=`select @@datadir`
---let ib1_IBD = $MYSQLD_DATADIR/ibdata1
---let t1_IBD = $MYSQLD_DATADIR/test/t1.ibd
---let log0 = $MYSQLD_DATADIR/ib_logfile0
---let log1 = $MYSQLD_DATADIR/ib_logfile1
---let SEARCH_RANGE = 10000000
---let SEARCH_PATTERN=privatejanprivate
-
--- source include/kill_mysqld.inc
-
---echo # ibdata1 yes on expecting NOT FOUND
--- let SEARCH_FILE=$ib1_IBD
--- source include/search_pattern_in_file.inc
---echo # t1 yes on expecting NOT FOUND
--- let SEARCH_FILE=$t1_IBD
--- source include/search_pattern_in_file.inc
---echo # log0 yes on expecting NOT FOUND
--- let SEARCH_FILE=$log0
--- source include/search_pattern_in_file.inc
---echo # log1 yes on expecting NOT FOUND
--- let SEARCH_FILE=$log1
--- source include/search_pattern_in_file.inc
-
---echo # Restart mysqld --innodb_encrypt_log=0
--- let $restart_parameters=--innodb_encrypt_log=0
--- source include/start_mysqld.inc
-
-insert into t1 values(5000, substring(MD5(RAND()), -64), REPEAT('publicmessage',10));
-insert into t1 values(5001, substring(MD5(RAND()), -64), REPEAT('publicmessage',10));
-insert into t1 values(5002, substring(MD5(RAND()), -64), REPEAT('publicmessage',10));
-insert into t1 values(5003, substring(MD5(RAND()), -64), REPEAT('publicmessage',10));
-insert into t1 values(5004, substring(MD5(RAND()), -64), REPEAT('publicmessage',10));
-
---let SEARCH_PATTERN=privatejanprivate
---echo # ibdata1 yes on expecting NOT FOUND
--- let SEARCH_FILE=$ib1_IBD
--- source include/search_pattern_in_file.inc
---echo # t1 yes on expecting NOT FOUND
--- let SEARCH_FILE=$t1_IBD
--- source include/search_pattern_in_file.inc
---echo # log0 yes on expecting NOT FOUND
--- let SEARCH_FILE=$log0
--- source include/search_pattern_in_file.inc
---echo # log1 yes on expecting NOT FOUND
--- let SEARCH_FILE=$log1
--- source include/search_pattern_in_file.inc
-
---let SEARCH_PATTERN=publicmessage
---echo # ibdata1 yes on expecting NOT FOUND
--- let SEARCH_FILE=$ib1_IBD
--- source include/search_pattern_in_file.inc
---echo # t1 yes on expecting NOT FOUND
--- let SEARCH_FILE=$t1_IBD
--- source include/search_pattern_in_file.inc
---echo # log0 no on expecting FOUND/NOTFOUND depending where insert goes
--- let SEARCH_FILE=$log0
--- source include/search_pattern_in_file.inc
---echo # log1 no on expecting FOUND/NOTFOUND depending where insert goes
--- let SEARCH_FILE=$log1
--- source include/search_pattern_in_file.inc
-
-drop procedure innodb_insert_proc;
-drop table t1;

mysql-test/suite/encryption/t/innodb_encrypt_log.test

+-- source include/have_innodb.inc
+-- source include/not_embedded.inc
+-- source filekeys_plugin.inc
+
+--echo #
+--echo # MDEV-9011: Redo log encryption does not work
+--echo #
+
+--echo #
+--echo # MDEV-9422 Encrypted redo log checksum errors
+--echo # on restart after killing busy server instance
+--echo #
+
+--let $MYSQLD_DATADIR=`select @@datadir`
+
+SET GLOBAL innodb_log_checksums=0;
+SELECT @@global.innodb_log_checksums;
+
+CREATE TABLE t0 (
+  pk bigint auto_increment,
+  col_int int,
+  col_int_key int,
+  col_char char(12),
+  col_char_key char(12),
+  primary key (pk),
+  key (col_int_key),
+  key (col_char_key)
+) ENGINE=InnoDB ENCRYPTED=YES ENCRYPTION_KEY_ID=1;
+CREATE TEMPORARY TABLE t LIKE t0;
+
+INSERT INTO t VALUES
+(NULL,1,1,'private','secret'),(NULL,2,2,'sacred','success'),
+(NULL,3,3,'story','secure'),(NULL,4,4,'security','sacrament');
+
+# Force a redo log flush at the next commit.
+SET GLOBAL innodb_flush_log_at_trx_commit=1;
+INSERT INTO t0
+  SELECT NULL, t1.col_int, t1.col_int_key, t1.col_char, t1.col_char_key
+    FROM t t1, t t2, t t3, t t4, t t5;
+
+--source include/kill_mysqld.inc
+
+--let SEARCH_RANGE = 10000000
+--let SEARCH_PATTERN=private|secret|sacr(ed|ament)|success|story|secur(e|ity)
+
+--echo # ibdata1 expecting NOT FOUND
+-- let SEARCH_FILE=$MYSQLD_DATADIR/ibdata1
+-- source include/search_pattern_in_file.inc
+--echo # t0.ibd expecting NOT FOUND
+-- let SEARCH_FILE=$MYSQLD_DATADIR/test/t0.ibd
+-- source include/search_pattern_in_file.inc
+--echo # ib_logfile0 expecting NOT FOUND
+-- let SEARCH_FILE=$MYSQLD_DATADIR/ib_logfile0
+-- source include/search_pattern_in_file.inc
+--echo # ib_logfile1 expecting NOT FOUND
+-- let SEARCH_FILE=$MYSQLD_DATADIR/ib_logfile1
+-- source include/search_pattern_in_file.inc
+
+--echo # Restart without redo log encryption
+-- let $restart_parameters=--skip-innodb-encrypt-log --innodb-log-files-in-group=1
+-- source include/start_mysqld.inc
+
+SELECT COUNT(*) FROM t0;
+CHECK TABLE t0;
+# Force a redo log flush at the next commit.
+SET GLOBAL innodb_flush_log_at_trx_commit=1;
+# If we tested with UPDATE, we would get clear-text redo log for
+# encrypted undo log written with the old secret values.
+INSERT INTO t0 VALUES(NULL, 5, 5, 'public', 'gossip');
+
+--source include/kill_mysqld.inc
+
+--echo # ib_logfile0 expecting NOT FOUND
+-- let SEARCH_FILE=$MYSQLD_DATADIR/ib_logfile0
+-- source include/search_pattern_in_file.inc
+--let SEARCH_PATTERN=public|gossip
+--echo # ib_logfile0 expecting FOUND
+-- let SEARCH_FILE=$MYSQLD_DATADIR/ib_logfile0
+-- source include/search_pattern_in_file.inc
+
+--let SEARCH_PATTERN=private|secret|sacr(ed|ament)|success|story|secur(e|ity)|public|gossip
+--echo # ibdata1 expecting NOT FOUND
+-- let SEARCH_FILE=$MYSQLD_DATADIR/ibdata1
+-- source include/search_pattern_in_file.inc
+--echo # t0.ibd expecting NOT FOUND
+-- let SEARCH_FILE=$MYSQLD_DATADIR/test/t0.ibd
+-- source include/search_pattern_in_file.inc
+
+--let $restart_parameters=
+--source include/start_mysqld.inc
+
+SELECT COUNT(*) FROM t0;
+CHECK TABLE t0;
+DROP TABLE t0;

mysql-test/suite/encryption/t/innodb_encrypt_log_corruption.test

@@ -3,9 +3,12 @@
 
 SELECT COUNT(*) `1` FROM INFORMATION_SCHEMA.ENGINES WHERE engine='innodb'
 AND support IN ('YES', 'DEFAULT', 'ENABLED');
+--source include/shutdown_mysqld.inc
+--let SEARCH_PATTERN= InnoDB: Encrypting redo log
+--source include/search_pattern_in_file.inc
 
 --let $restart_parameters=
---source include/restart_mysqld.inc
+--source include/start_mysqld.inc
 
 --list_files $bugdir
 --remove_files_wildcard $bugdir

mysql-test/suite/innodb/r/innodb-32k.result

-call mtr.add_suppression("InnoDB: Warning: innodb_page_size has been changed from default value ");
-call mtr.add_suppression("InnoDB: Resizing redo log from ");
-call mtr.add_suppression("InnoDB: Starting to delete and rewrite log files");
-call mtr.add_suppression("InnoDB: New log files created, LSN=");
 call mtr.add_suppression("Innodb: Cannot add field.*row size is");
 # Test 1) Show the page size from Information Schema
 SELECT variable_value FROM information_schema.global_status

mysql-test/suite/innodb/r/innodb-64k.result

-call mtr.add_suppression("InnoDB: Warning: innodb_page_size has been changed from default value *");
-call mtr.add_suppression("InnoDB: Resizing redo log from *");
-call mtr.add_suppression("InnoDB: Starting to delete and rewrite log files.");
-call mtr.add_suppression("InnoDB: New log files created, LSN=*");
 # Test 1) Show the page size from Information Schema
 SELECT variable_value FROM information_schema.global_status
 WHERE LOWER(variable_name) = 'innodb_page_size';

mysql-test/suite/innodb/r/innodb-bigblob.result

-call mtr.add_suppression("Resizing redo log from *");
-call mtr.add_suppression("Starting to delete and rewrite log files.");
-call mtr.add_suppression("New log files created, LSN=*");
-call mtr.add_suppression("Writer thread is waiting this semaphore");
 create table foo (id varchar(37) not null, content longblob) engine=INNODB;
 insert into foo (id, content) values('xyz', '');
 update foo set content=repeat('a', 43941888) where id='xyz';

mysql-test/suite/innodb/r/log_corruption.result

@@ -41,7 +41,7 @@ WHERE engine = 'innodb'
 AND support IN ('YES', 'DEFAULT', 'ENABLED');
 ENGINE	SUPPORT	COMMENT	TRANSACTIONS	XA	SAVEPOINTS
 FOUND /InnoDB: Invalid log block checksum. block: 2372 checkpoint no: 1 expected: 3362026715 found: 144444122/ in mysqld.1.err
-FOUND /InnoDB: Ignoring the redo log due to missing MLOG_CHECKPOINT between the checkpoint 1213964 and the end 1213952\./ in mysqld.1.err
+FOUND /InnoDB: Missing MLOG_CHECKPOINT between the checkpoint 1213964 and the end 1213952\./ in mysqld.1.err
 # --innodb-force-recovery=6 (skip the entire redo log)
 SELECT * FROM INFORMATION_SCHEMA.ENGINES
 WHERE engine = 'innodb'
@@ -54,7 +54,6 @@ SELECT * FROM INFORMATION_SCHEMA.ENGINES
 WHERE engine = 'innodb'
 AND support IN ('YES', 'DEFAULT', 'ENABLED');
 ENGINE	SUPPORT	COMMENT	TRANSACTIONS	XA	SAVEPOINTS
-FOUND /InnoDB: Ignoring the redo log due to missing MLOG_CHECKPOINT between the checkpoint 1213964 and the end 1213952\./ in mysqld.1.err
 # --innodb-force-recovery=6 (skip the entire redo log)
 SELECT * FROM INFORMATION_SCHEMA.ENGINES
 WHERE engine = 'innodb'
@@ -90,7 +89,7 @@ SELECT * FROM INFORMATION_SCHEMA.ENGINES
 WHERE engine = 'innodb'
 AND support IN ('YES', 'DEFAULT', 'ENABLED');
 ENGINE	SUPPORT	COMMENT	TRANSACTIONS	XA	SAVEPOINTS
-FOUND /InnoDB: Redo log crypto: getting mysqld crypto key from key version failed err = 4294967295/ in mysqld.1.err
+FOUND /InnoDB: Obtaining redo log encryption key version 1 failed/ in mysqld.1.err
 FOUND /InnoDB: Decrypting checkpoint failed/ in mysqld.1.err
 ib_buffer_pool
 ib_logfile0

mysql-test/suite/innodb/r/log_file.result

@@ -340,7 +340,7 @@ WHERE engine='innodb'
 AND support IN ('YES', 'DEFAULT', 'ENABLED');
 1
 1
-FOUND /Resizing redo log from 1\*\d+ to 3\*\d+ pages, LSN=\d+/ in mysqld.1.err
+FOUND /Resizing redo log from 1\*\d+ to 3\*\d+ pages; LSN=\d+/ in mysqld.1.err
 # Cleanup
 bak_ib_logfile0
 bak_ib_logfile1

mysql-test/suite/innodb/t/innodb-32k.test

@@ -3,10 +3,6 @@
 --source include/have_innodb.inc
 --source include/have_innodb_32k.inc
 
-call mtr.add_suppression("InnoDB: Warning: innodb_page_size has been changed from default value ");
-call mtr.add_suppression("InnoDB: Resizing redo log from ");
-call mtr.add_suppression("InnoDB: Starting to delete and rewrite log files");
-call mtr.add_suppression("InnoDB: New log files created, LSN=");
 call mtr.add_suppression("Innodb: Cannot add field.*row size is");
 
 let $MYSQLD_DATADIR= `select @@datadir`;

mysql-test/suite/innodb/t/innodb-64k.test

@@ -3,11 +3,6 @@
 --source include/have_innodb.inc
 --source include/have_innodb_64k.inc
 
-call mtr.add_suppression("InnoDB: Warning: innodb_page_size has been changed from default value *");
-call mtr.add_suppression("InnoDB: Resizing redo log from *");
-call mtr.add_suppression("InnoDB: Starting to delete and rewrite log files.");
-call mtr.add_suppression("InnoDB: New log files created, LSN=*");
-
 let $MYSQLD_DATADIR= `select @@datadir`;
 
 --echo # Test 1) Show the page size from Information Schema

mysql-test/suite/innodb/t/innodb-bigblob.test

@@ -6,11 +6,6 @@
 let $status_orig=`SELECT @@innodb_status_output`;
 --enable_query_log
 
-call mtr.add_suppression("Resizing redo log from *");
-call mtr.add_suppression("Starting to delete and rewrite log files.");
-call mtr.add_suppression("New log files created, LSN=*");
-call mtr.add_suppression("Writer thread is waiting this semaphore");
-
 create table foo (id varchar(37) not null, content longblob) engine=INNODB;
 insert into foo (id, content) values('xyz', '');
 update foo set content=repeat('a', 43941888) where id='xyz';

mysql-test/suite/innodb/t/log_corruption.test

@@ -9,11 +9,11 @@ call mtr.add_suppression("Plugin 'InnoDB' registration as a STORAGE ENGINE faile
 call mtr.add_suppression("InnoDB: Unsupported redo log format");
 call mtr.add_suppression("InnoDB: No valid checkpoint found");
 call mtr.add_suppression("InnoDB: Invalid (log block|redo log header) checksum");
-call mtr.add_suppression("InnoDB: Ignoring the redo log due to missing MLOG_CHECKPOINT");
+call mtr.add_suppression("InnoDB: Missing MLOG_CHECKPOINT");
 call mtr.add_suppression("InnoDB: MLOG_FILE_NAME incorrect");
 call mtr.add_suppression("InnoDB: ############### CORRUPT LOG RECORD FOUND");
-call mtr.add_suppression("InnoDB: Found corrupted log");
-call mtr.add_suppression("InnoDB: Redo log crypto: getting mysqld crypto key from key version failed");
+call mtr.add_suppression("InnoDB: Log scan aborted at LSN");
+call mtr.add_suppression("InnoDB: Obtaining redo log encryption key version 1 failed");
 call mtr.add_suppression("InnoDB: Decrypting checkpoint failed");
 --enable_query_log
 
@@ -206,7 +206,7 @@ eval $check_no_innodb;
 --source include/shutdown_mysqld.inc
 let SEARCH_PATTERN=InnoDB: Invalid log block checksum. block: 2372 checkpoint no: 1 expected: 3362026715 found: 144444122;
 --source include/search_pattern_in_file.inc
-let SEARCH_PATTERN=InnoDB: Ignoring the redo log due to missing MLOG_CHECKPOINT between the checkpoint 1213964 and the end 1213952\.;
+let SEARCH_PATTERN=InnoDB: Missing MLOG_CHECKPOINT between the checkpoint 1213964 and the end 1213952\.;
 --source include/search_pattern_in_file.inc
 --echo # --innodb-force-recovery=6 (skip the entire redo log)
 --let $restart_parameters= $dirs --innodb-force-recovery=6
@@ -232,14 +232,12 @@ print OUT pack("H*x[5]", "C0DEBA5E0022000c0000000138");
 print OUT pack("H*x[475]H*", "12860cb7809781e80006626f677573", "089C0ADA");
 EOF
 --copy_file $bugdir/ib_logfile0 $bugdir/ib_logfile
-# Anything below innodb_force_recovery=6 must find a valid redo log.
+# Anything below innodb_force_recovery=6 must find an invalid redo log.
 # Missing tablespace files are tolerated already with innodb_force_recovery=1.
 --let $restart_parameters= $dirs --innodb-force-recovery=5
 --source include/start_mysqld.inc
 eval $check_no_innodb;
 --source include/shutdown_mysqld.inc
-let SEARCH_PATTERN=InnoDB: Ignoring the redo log due to missing MLOG_CHECKPOINT between the checkpoint 1213964 and the end 1213952\.;
---source include/search_pattern_in_file.inc
 --echo # --innodb-force-recovery=6 (skip the entire redo log)
 --let $restart_parameters= $dirs --innodb-force-recovery=6
 --source include/start_mysqld.inc
@@ -340,7 +338,7 @@ EOF
 if (!$no_cleanup) {
 eval $check_no_innodb;
 --source include/shutdown_mysqld.inc
---let SEARCH_PATTERN= InnoDB: Redo log crypto: getting mysqld crypto key from key version failed err = 4294967295
+--let SEARCH_PATTERN= InnoDB: Obtaining redo log encryption key version 1 failed
 --source include/search_pattern_in_file.inc
 --let SEARCH_PATTERN= InnoDB: Decrypting checkpoint failed
 --source include/search_pattern_in_file.inc

mysql-test/suite/innodb/t/log_file.test

@@ -219,7 +219,7 @@ eval $check_no_innodb;
 --source include/start_mysqld.inc
 eval $check_yes_innodb;
 --source include/shutdown_mysqld.inc
---let SEARCH_PATTERN=Resizing redo log from 1\*\d+ to 3\*\d+ pages, LSN=\d+
+--let SEARCH_PATTERN=Resizing redo log from 1\*\d+ to 3\*\d+ pages; LSN=\d+
 --source include/search_pattern_in_file.inc
 
 --let $restart_parameters=

mysql-test/suite/innodb/t/log_file_name.test

@@ -160,7 +160,7 @@ call mtr.add_suppression("InnoDB: Cannot read first page in datafile: .*test.*ib
 call mtr.add_suppression("InnoDB: Datafile '.*test.*ibd' is corrupted");
 call mtr.add_suppression("InnoDB:  Cannot replay file rename. Remove either file and try again");
 call mtr.add_suppression("InnoDB: Cannot rename.*because the target file exists");
-call mtr.add_suppression("InnoDB: Found corrupted log");
+call mtr.add_suppression("InnoDB: Log scan aborted at LSN");
 # The following are for the --innodb-force-recovery=1 with broken u* tables:
 call mtr.add_suppression("InnoDB: Header page consists of zero bytes in datafile: .*u1.ibd");
 call mtr.add_suppression("InnoDB: The error means the system cannot find the path specified");

mysql-test/suite/innodb/t/log_file_size.test

@@ -10,9 +10,6 @@ if (`SELECT @@innodb_log_file_size = 1048576`) {
 }
 
 --disable_query_log
-call mtr.add_suppression("InnoDB: Resizing redo log");
-call mtr.add_suppression("InnoDB: Starting to delete and rewrite log files");
-call mtr.add_suppression("InnoDB: New log files created");
 call mtr.add_suppression("InnoDB: The log sequence numbers [0-9]+ and [0-9]+ in ibdata files do not match the log sequence number [0-9]+ in the ib_logfiles");
 call mtr.add_suppression("syntax error in innodb_log_group_home_dir");
 call mtr.add_suppression("Plugin 'InnoDB' init function returned error");
@@ -100,7 +97,7 @@ let SEARCH_PATTERN= InnoDB: innodb_read_only prevents crash recovery;
 SELECT * FROM t1;
 let SEARCH_PATTERN= InnoDB: Starting an apply batch of log records;
 --source include/search_pattern_in_file.inc
-let SEARCH_PATTERN= InnoDB: Resizing redo log from 3\*[0-9]+ to 2\*[0-9]+ pages;
+let SEARCH_PATTERN= redo log from 3\*[0-9]+ to 2\*[0-9]+ pages;
 --source include/search_pattern_in_file.inc
 
 --let $restart_parameters= --debug=d,innodb_log_abort_5
@@ -109,7 +106,7 @@ let SEARCH_PATTERN= InnoDB: Resizing redo log from 3\*[0-9]+ to 2\*[0-9]+ pages;
 SELECT * FROM t1;
 let SEARCH_PATTERN= InnoDB: Starting an apply batch of log records;
 --source include/search_pattern_in_file.inc
-let SEARCH_PATTERN= InnoDB: Resizing redo log from 3\*[0-9]+ to 2\*[0-9]+ pages;
+let SEARCH_PATTERN= redo log from 3\*[0-9]+ to 2\*[0-9]+ pages;
 --source include/search_pattern_in_file.inc
 
 --let $restart_parameters= --innodb-read-only
@@ -126,7 +123,7 @@ SELECT * FROM t1;
 
 let SEARCH_PATTERN= InnoDB: Starting an apply batch of log records;
 --source include/search_pattern_in_file.inc
-let SEARCH_PATTERN= InnoDB: Resizing redo log from 3\*[0-9]+ to 2\*[0-9]+ pages;
+let SEARCH_PATTERN= redo log from 3\*[0-9]+ to 2\*[0-9]+ pages;
 --source include/search_pattern_in_file.inc
 
 --let $restart_parameters= --debug=d,innodb_log_abort_7

mysql-test/suite/innodb/t/log_file_size_checkpoint.test

@@ -8,11 +8,6 @@ let $n=250;
 let $t=veryLongTableNameToCreateMLOG_FILE_NAMErecords;
 
 --disable_query_log
-call mtr.add_suppression("InnoDB: Resizing redo log");
-call mtr.add_suppression("InnoDB: Starting to delete and rewrite log files");
-call mtr.add_suppression("InnoDB: New log files created");
-FLUSH TABLES;
-
 let $i=$n;
 while ($i)
 {

storage/innobase/handler/ha_innodb.cc

@@ -89,7 +89,7 @@ this program; if not, write to the Free Software Foundation, Inc.,
 #include "fts0types.h"
 #include "ibuf0ibuf.h"
 #include "lock0lock.h"
-#include "log0log.h"
+#include "log0crypt.h"
 #include "mem0mem.h"
 #include "mtr0mtr.h"
 #include "os0file.h"
@@ -3748,14 +3748,42 @@ static const char*	deprecated_use_trim
 
 /** Update log_checksum_algorithm_ptr with a pointer to the function
 corresponding to whether checksums are enabled.
-@param[in]	check	whether redo log block checksums are enabled */
-static
-void
-innodb_log_checksums_func_update(bool	check)
+@param[in,out]	thd	client session, or NULL if at startup
+@param[in]	check	whether redo log block checksums are enabled
+@return whether redo log block checksums are enabled */
+static inline
+bool
+innodb_log_checksums_func_update(THD* thd, bool check)
 {
-	log_checksum_algorithm_ptr = check
-		? log_block_calc_checksum_crc32
-		: log_block_calc_checksum_none;
+	static const char msg[] = "innodb_encrypt_log implies"
+		" innodb_log_checksums";
+
+	ut_ad(!thd == !srv_was_started);
+
+	if (!check) {
+		check = srv_encrypt_log;
+		if (!check) {
+		} else if (thd) {
+			push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN,
+					    HA_ERR_UNSUPPORTED, msg);
+		} else {
+			sql_print_warning(msg);
+		}
+	}
+
+	if (thd) {
+		log_mutex_enter();
+		log_checksum_algorithm_ptr = check
+			? log_block_calc_checksum_crc32
+			: log_block_calc_checksum_none;
+		log_mutex_exit();
+	} else {
+		log_checksum_algorithm_ptr = check
+			? log_block_calc_checksum_crc32
+			: log_block_calc_checksum_none;
+	}
+
+	return(check);
 }
 
 /****************************************************************//**
@@ -4244,7 +4272,8 @@ innobase_init(
 		srv_checksum_algorithm = SRV_CHECKSUM_ALGORITHM_NONE;
 	}
 
-	innodb_log_checksums_func_update(innodb_log_checksums);
+	innodb_log_checksums = innodb_log_checksums_func_update(
+		NULL, innodb_log_checksums);
 
 #ifdef HAVE_LINUX_LARGE_PAGES
 	if ((os_use_large_pages = my_use_large_pages)) {
@@ -20229,13 +20258,8 @@ innodb_log_checksums_update(
 	void*				var_ptr,
 	const void*			save)
 {
-	my_bool	check = *static_cast<my_bool*>(var_ptr)
-		= *static_cast<const my_bool*>(save);
-
-	/* Make sure we are the only log user */
-	mutex_enter(&log_sys->mutex);
-	innodb_log_checksums_func_update(check);
-	mutex_exit(&log_sys->mutex);
+	*static_cast<my_bool*>(var_ptr) = innodb_log_checksums_func_update(
+		thd, *static_cast<const my_bool*>(save));
 }
 
 static SHOW_VAR innodb_status_variables_export[]= {

storage/innobase/include/fil0crypt.h

@@ -27,6 +27,7 @@ Created 04/01/2015 Jan Lindström
 #define fil0crypt_h
 
 #include "os0event.h"
+#include "my_crypt.h"
 
 /**
 * Magic pattern in start of crypt data on page 0

storage/innobase/include/log0crypt.h

@@ -22,27 +22,21 @@ Innodb log encrypt/decrypt
 
 Created 11/25/2013 Minli Zhu
 Modified           Jan Lindström jan.lindstrom@mariadb.com
+MDEV-11782: Rewritten for MariaDB 10.2 by Marko Mäkelä, MariaDB Corporation.
 *******************************************************/
 #ifndef log0crypt_h
 #define log0crypt_h
 
-#include "univ.i"
-#include "ut0byte.h"
-#include "my_crypt.h"
+#include "log0log.h"
 
-typedef int Crypt_result;
-
-/* If true, enable redo log encryption. */
+/** innodb_encrypt_log: whether to encrypt the redo log */
 extern my_bool srv_encrypt_log;
 
-/***********************************************************************
-Set next checkpoint's key version to latest one, and generate new key */
+/** Initialize the redo log encryption key.
+@return whether the operation succeeded */
 UNIV_INTERN
-void
-log_crypt_set_ver_and_key(
-/*======================*/
-	ib_uint64_t next_checkpoint_no);/*!< in: next checkpoint no */
-
+bool
+log_crypt_init();
 
 /*********************************************************************//**
 Writes the crypto (version, msg and iv) info, which has been used for
@@ -64,78 +58,23 @@ log_crypt_101_read_checkpoint(const byte* buf);
 /** Decrypt a MariaDB 10.1 redo log block.
 @param[in,out]	buf	log block
 @return	whether the decryption was successful */
+UNIV_INTERN
 bool
 log_crypt_101_read_block(byte* buf);
 
-/*********************************************************************//**
-Read the crypto (version, msg and iv) info, which has been used for
-log blocks with lsn <= this checkpoint's lsn, from a log header's
-checkpoint buf. */
+/** Read the checkpoint crypto (version, msg and iv) info.
+@param[in]	buf	checkpoint buffer
+@return	whether the operation was successful */
 UNIV_INTERN
 bool
-log_crypt_read_checkpoint_buf(
-/*===========================*/
-	const byte*	buf);		/*!< in: checkpoint buffer */
-
-/********************************************************
-Encrypt one or more log block before it is flushed to disk */
-UNIV_INTERN
-void
-log_encrypt_before_write(
-/*=====================*/
-	ib_uint64_t	next_checkpoint_no,	/*!< in: log group to be flushed */
-	byte* 		block,			/*!< in/out: pointer to a log block */
-	const ulint 	size);			/*!< in: size of log blocks */
-
-/********************************************************
-Decrypt a specified log segment after they are read from a log file to a buffer.
-*/
-UNIV_INTERN
-void
-log_decrypt_after_read(
-/*===================*/
-	byte*		frame,	/*!< in/out: log segment */
-	const ulint	size);	/*!< in: log segment size */
-
-/* Error codes for crypt info */
-typedef enum {
-	LOG_UNENCRYPTED = 0,
-	LOG_CRYPT_KEY_NOT_FOUND = 1,
-	LOG_DECRYPT_MAYBE_FAILED = 2
-} log_crypt_err_t;
+log_crypt_read_checkpoint_buf(const byte* buf);
 
-/********************************************************
-Check is the checkpoint information encrypted. This check
-is based on fact has log group crypt info and based
-on this crypt info was the key version different from
-unencrypted key version. There is no realible way to
-distinguish encrypted log block from corrupted log block,
-but if log block corruption is found this function is
-used to find out if log block is maybe encrypted but
-encryption key, key management plugin or encryption
-algorithm does not match.
-@return TRUE, if log block may be encrypted */
+/** Encrypt or decrypt log blocks.
+@param[in,out]	buf	log blocks to encrypt or decrypt
+@param[in]	size	size of the buffer, in bytes
+@param[in]	decrypt	whether to decrypt instead of encrypting */
 UNIV_INTERN
-ibool
-log_crypt_block_maybe_encrypted(
-/*============================*/
-	const byte* 		log_block, 	/*!< in: log block */
-	log_crypt_err_t* 	err_info); 	/*!< out: error info */
-
-/********************************************************
-Print crypt error message to error log */
-UNIV_INTERN
-void
-log_crypt_print_error(
-/*==================*/
-	log_crypt_err_t 	err_info); 	/*!< out: error info */
-
-/*********************************************************************//**
-Print checkpoint no from log block and all encryption keys from
-checkpoints if they are present. Used for problem analysis. */
 void
-log_crypt_print_checkpoint_keys(
-/*============================*/
-	const byte* log_block);
+log_crypt(byte* buf, ulint size, bool decrypt = false);
 
 #endif  // log0crypt.h

storage/innobase/include/log0log.h

@@ -37,13 +37,9 @@ Created 12/9/1995 Heikki Tuuri
 #include "univ.i"
 #include "dyn0buf.h"
 #include "sync0rw.h"
-#include "log0crypt.h"
 #include "log0types.h"
 #include "os0event.h"
 
-/** Redo log buffer */
-struct log_t;
-
 /** Redo log group */
 struct log_group_t;
 
@@ -275,15 +271,6 @@ objects! */
 void
 log_check_margins(void);
 
-/******************************************************//**
-Reads a specified log segment to a buffer. */
-void
-log_group_read_log_seg(
-/*===================*/
-	byte*		buf,		/*!< in: buffer where to read */
-	log_group_t*	group,		/*!< in: log group */
-	lsn_t		start_lsn,	/*!< in: read area start */
-	lsn_t		end_lsn);	/*!< in: read area end */
 /********************************************************//**
 Sets the field values in group to correspond to a given lsn. For this function
 to work, the values must already be correctly initialized to correspond to
@@ -449,9 +436,6 @@ void
 log_mem_free(void);
 /*==============*/
 
-/** Redo log system */
-extern log_t*	log_sys;
-
 /** Whether to generate and require checksums on the redo log pages */
 extern my_bool	innodb_log_checksums;
 
@@ -508,6 +492,12 @@ extern my_bool	innodb_log_checksums;
 #define LOG_CHECKPOINT_LSN		8
 #define LOG_CHECKPOINT_OFFSET		16
 #define LOG_CHECKPOINT_LOG_BUF_SIZE	24
+/** MariaDB 10.2.5 encrypted redo log encryption key version (32 bits)*/
+#define LOG_CHECKPOINT_CRYPT_KEY	32
+/** MariaDB 10.2.5 encrypted redo log random nonce (32 bits) */
+#define LOG_CHECKPOINT_CRYPT_NONCE	36
+/** MariaDB 10.2.5 encrypted redo log random message (MY_AES_BLOCK_SIZE) */
+#define LOG_CHECKPOINT_CRYPT_MESSAGE	40
 
 /** Offsets of a log file header */
 /* @{ */
@@ -538,19 +528,11 @@ or the MySQL version that created the redo log file. */
 /** The redo log format identifier corresponding to the current format version.
 Stored in LOG_HEADER_FORMAT. */
 #define LOG_HEADER_FORMAT_CURRENT	1
+/** Encrypted MariaDB redo log */
+#define LOG_HEADER_FORMAT_ENCRYPTED	(1U<<31)
 
 /* @} */
 
-/** MariaDB Server 10.1 encrypted redo log offsets */
-/* @{ */
-#define LOG_CRYPT_VER			(20 + 32 * 9)
-#define LOG_CRYPT_MAX_ENTRIES           (5)
-#define LOG_CRYPT_ENTRY_SIZE            (4 + 4 + 2 * MY_AES_BLOCK_SIZE)
-#define LOG_CRYPT_SIZE                  (1 + 1 +			\
-					 (LOG_CRYPT_MAX_ENTRIES *	\
-					  LOG_CRYPT_ENTRY_SIZE))
-/* @} */
-
 #define LOG_CHECKPOINT_1	OS_FILE_LOG_BLOCK_SIZE
 					/* first checkpoint field in the log
 					header; we write alternately to the
@@ -609,6 +591,12 @@ struct log_group_t{
 	byte*				checkpoint_buf;
 	/** list of log groups */
 	UT_LIST_NODE_T(log_group_t)	log_groups;
+
+	/** @return whether the redo log is encrypted */
+	bool is_encrypted() const
+	{
+		return((format & LOG_HEADER_FORMAT_ENCRYPTED) != 0);
+	}
 };
 
 /** Redo log buffer */
@@ -750,8 +738,17 @@ struct log_t{
 	byte*		checkpoint_buf;	/*!< checkpoint header is read to this
 					buffer */
 	/* @} */
+
+	/** @return whether the redo log is encrypted */
+	bool is_encrypted() const
+	{
+		return(UT_LIST_GET_FIRST(log_groups)->is_encrypted());
+	}
 };
 
+/** Redo log system */
+extern log_t*	log_sys;
+
 /** Test if flush order mutex is owned. */
 #define log_flush_order_mutex_own()			\
 	mutex_own(&log_sys->log_flush_order_mutex)

storage/innobase/include/log0recv.h

@@ -297,13 +297,4 @@ use these free frames to read in pages when we start applying the
 log records to the database. */
 extern ulint	recv_n_pool_free_frames;
 
-/******************************************************//**
-Checks the 4-byte checksum to the trailer checksum field of a log
-block.  */
-bool
-log_block_checksum_is_ok(
-/*===================================*/
-	const byte*	block,	/*!< in: pointer to a log block */
-	bool            print_err); /*!< in print error ? */
-
 #endif

storage/innobase/log/log0log.cc

@@ -40,6 +40,7 @@ Created 12/9/1995 Heikki Tuuri
 #include "log0log.ic"
 #endif
 
+#include "log0crypt.h"
 #include "mem0mem.h"
 #include "buf0buf.h"
 #include "buf0flu.h"
@@ -56,9 +57,6 @@ Created 12/9/1995 Heikki Tuuri
 #include "srv0mon.h"
 #include "sync0sync.h"
 
-/* Used for debugging */
-// #define DEBUG_CRYPT 1
-
 /*
 General philosophy of InnoDB redo-logs:
 
@@ -898,7 +896,9 @@ log_group_init(
 
 	group->id = id;
 	group->n_files = n_files;
-	group->format = LOG_HEADER_FORMAT_CURRENT;
+	group->format = srv_encrypt_log
+		? LOG_HEADER_FORMAT_CURRENT | LOG_HEADER_FORMAT_ENCRYPTED
+		: LOG_HEADER_FORMAT_CURRENT;
 	group->file_size = file_size;
 	group->space_id = space_id;
 	group->state = LOG_GROUP_OK;
@@ -987,11 +987,13 @@ log_group_file_header_flush(
 	ut_ad(!recv_no_log_write);
 	ut_ad(group->id == 0);
 	ut_a(nth_file < group->n_files);
+	ut_ad((group->format & ~LOG_HEADER_FORMAT_ENCRYPTED)
+	      == LOG_HEADER_FORMAT_CURRENT);
 
 	buf = *(group->file_header_bufs + nth_file);
 
 	memset(buf, 0, OS_FILE_LOG_BLOCK_SIZE);
-	mach_write_to_4(buf + LOG_HEADER_FORMAT, LOG_HEADER_FORMAT_CURRENT);
+	mach_write_to_4(buf + LOG_HEADER_FORMAT, group->format);
 	mach_write_to_8(buf + LOG_HEADER_START_LSN, start_lsn);
 	strcpy(reinterpret_cast<char*>(buf) + LOG_HEADER_CREATOR,
 	       LOG_HEADER_CREATOR_CURRENT);
@@ -1115,6 +1117,10 @@ log_group_write_buf(
 	      || log_block_get_hdr_no(buf)
 		 == log_block_convert_lsn_to_no(start_lsn));
 
+	if (log_sys->is_encrypted()) {
+		log_crypt(buf, write_len);
+	}
+
 	/* Calculate the checksums for each log block and write them to
 	the trailer fields of the log blocks */
 
@@ -1135,8 +1141,6 @@ log_group_write_buf(
 
 	ut_a(next_offset / UNIV_PAGE_SIZE <= ULINT_MAX);
 
-	log_encrypt_before_write(log_sys->next_checkpoint_no,
-				buf, write_len);
 	const ulint	page_no
 		= (ulint) (next_offset / univ_page_size.physical());
 
@@ -1625,7 +1629,9 @@ log_group_checkpoint(
 	mach_write_to_8(buf + LOG_CHECKPOINT_NO, log_sys->next_checkpoint_no);
 	mach_write_to_8(buf + LOG_CHECKPOINT_LSN, log_sys->next_checkpoint_lsn);
 
-	log_crypt_write_checkpoint_buf(buf);
+	if (log_sys->is_encrypted()) {
+		log_crypt_write_checkpoint_buf(buf);
+	}
 
 	lsn_offset = log_group_calc_lsn_offset(log_sys->next_checkpoint_lsn,
 					       group);
@@ -1689,30 +1695,16 @@ log_group_header_read(
 /** Write checkpoint info to the log header and invoke log_mutex_exit().
 @param[in]	sync	whether to wait for the write to complete */
 void
-log_write_checkpoint_info(
-	bool	sync)
+log_write_checkpoint_info(bool sync)
 {
-	log_group_t*	group;
-
 	ut_ad(log_mutex_own());
+	ut_ad(!srv_read_only_mode);
 
-	if (!srv_read_only_mode) {
-		for (group = UT_LIST_GET_FIRST(log_sys->log_groups);
-		     group;
-		     group = UT_LIST_GET_NEXT(log_groups, group)) {
-
-			log_group_checkpoint(group);
-		}
-	}
+	for (log_group_t* group = UT_LIST_GET_FIRST(log_sys->log_groups);
+	     group;
+	     group = UT_LIST_GET_NEXT(log_groups, group)) {
 
-	/* generate key version and key used to encrypt future blocks,
-	*
-	* NOTE: the +1 is as the next_checkpoint_no will be updated once
-	* the checkpoint info has been written and THEN blocks will be encrypted
-	* with new key
-	*/
-	if (srv_encrypt_log) {
-		log_crypt_set_ver_and_key(log_sys->next_checkpoint_no + 1);
+		log_group_checkpoint(group);
 	}
 
 	log_mutex_exit();
@@ -1991,78 +1983,6 @@ log_checkpoint_margin(void)
 	}
 }
 
-/******************************************************//**
-Reads a specified log segment to a buffer. */
-void
-log_group_read_log_seg(
-/*===================*/
-	byte*		buf,		/*!< in: buffer where to read */
-	log_group_t*	group,		/*!< in: log group */
-	lsn_t		start_lsn,	/*!< in: read area start */
-	lsn_t		end_lsn)	/*!< in: read area end */
-{
-	ulint	len;
-	lsn_t	source_offset;
-
-	ut_ad(log_mutex_own());
-
-loop:
-	source_offset = log_group_calc_lsn_offset(start_lsn, group);
-
-	ut_a(end_lsn - start_lsn <= ULINT_MAX);
-	len = (ulint) (end_lsn - start_lsn);
-
-	ut_ad(len != 0);
-
-	if ((source_offset % group->file_size) + len > group->file_size) {
-
-		/* If the above condition is true then len (which is ulint)
-		is > the expression below, so the typecast is ok */
-		len = (ulint) (group->file_size -
-			(source_offset % group->file_size));
-	}
-
-	log_sys->n_log_ios++;
-
-	MONITOR_INC(MONITOR_LOG_IO);
-
-	ut_a(source_offset / UNIV_PAGE_SIZE <= ULINT_MAX);
-
-	const ulint	page_no
-		= (ulint) (source_offset / univ_page_size.physical());
-
-	fil_io(IORequestLogRead, true,
-	       page_id_t(group->space_id, page_no),
-	       univ_page_size,
-	       (ulint) (source_offset % univ_page_size.physical()),
-	       len, buf, NULL);
-
-#ifdef DEBUG_CRYPT
-	fprintf(stderr, "BEFORE DECRYPT: block: %lu checkpoint: %lu %.8lx %.8lx offset %lu\n",
-		log_block_get_hdr_no(buf),
-			log_block_get_checkpoint_no(buf),
-			log_block_calc_checksum(buf),
-		log_block_get_checksum(buf), source_offset);
-#endif
-
-	log_decrypt_after_read(buf, len);
-
-#ifdef DEBUG_CRYPT
-	fprintf(stderr, "AFTER DECRYPT: block: %lu checkpoint: %lu %.8lx %.8lx\n",
-			log_block_get_hdr_no(buf),
-			log_block_get_checkpoint_no(buf),
-			log_block_calc_checksum(buf),
-			log_block_get_checksum(buf));
-#endif
-	start_lsn += len;
-	buf += len;
-
-	if (start_lsn != end_lsn) {
-
-		goto loop;
-	}
-}
-
 /**
 Checks that there is enough free space in the log to start a new query step.
 Flushes the log buffer or makes a new checkpoint if necessary. NOTE: this

storage/innobase/mysql-test/storage_engine/define_engine.inc

@@ -41,9 +41,5 @@ let $default_char_type = CHAR(8);
 # e.g. creation of an additional schema or table, etc.
 # The cleanup part should be defined in cleanup_engine.inc
 
-CALL mtr.add_suppression("InnoDB: Resizing redo log from .* to .* pages, LSN=.*");
-CALL mtr.add_suppression("InnoDB: Starting to delete and rewrite log files.");
-CALL mtr.add_suppression("InnoDB: New log files created, LSN=.*");
-
 --enable_query_log
 --enable_result_log

storage/innobase/srv/srv0start.cc

@@ -63,7 +63,7 @@ Created 2/16/1996 Heikki Tuuri
 #include "fsp0fsp.h"
 #include "rem0rec.h"
 #include "mtr0mtr.h"
-#include "log0log.h"
+#include "log0crypt.h"
 #include "log0recv.h"
 #include "page0page.h"
 #include "page0cur.h"
@@ -481,6 +481,9 @@ create_log_files(
 
 	/* Create a log checkpoint. */
 	log_mutex_enter();
+	if (log_sys->is_encrypted() && !log_crypt_init()) {
+		return(DB_ERROR);
+	}
 	ut_d(recv_no_log_write = false);
 	recv_reset_logs(lsn);
 	log_mutex_exit();
@@ -536,7 +539,7 @@ create_log_files_rename(
 
 	fil_open_log_and_system_tablespace_files();
 
-	ib::warn() << "New log files created, LSN=" << lsn;
+	ib::info() << "New log files created, LSN=" << lsn;
 
 	return(err);
 }
@@ -1362,17 +1365,35 @@ srv_prepare_to_delete_redo_log_files(
 		flushed_lsn = log_sys->lsn;
 
 		{
-			ib::warn	warning;
+			ib::info	info;
 			if (srv_log_file_size == 0) {
-				warning << "Upgrading redo log: ";
+				info << "Upgrading redo log: ";
+			} else if (n_files != srv_n_log_files
+				   || srv_log_file_size
+				   != srv_log_file_size_requested) {
+				if (srv_encrypt_log
+				    == log_sys->is_encrypted()) {
+					info << (srv_encrypt_log
+						 ? "Resizing encrypted"
+						 : "Resizing");
+				} else if (srv_encrypt_log) {
+					info << "Encrypting and resizing";
+				} else {
+					info << "Removing encryption"
+						" and resizing";
+				}
+
+				info << " redo log from " << n_files
+				     << "*" << srv_log_file_size << " to ";
+			} else if (srv_encrypt_log) {
+				info << "Encrypting redo log: ";
 			} else {
-				warning << "Resizing redo log from "
-					<< n_files << "*"
-					<< srv_log_file_size << " to ";
+				info << "Removing redo log encryption: ";
 			}
-			warning << srv_n_log_files << "*"
-				<< srv_log_file_size_requested
-				<< " pages, LSN=" << flushed_lsn;
+
+			info << srv_n_log_files << "*"
+			     << srv_log_file_size_requested
+			     << " pages; LSN=" << flushed_lsn;
 		}
 
 		/* Flush the old log files. */
@@ -2180,6 +2201,14 @@ innobase_start_or_create_for_mysql(void)
 
 		recv_sys->dblwr.pages.clear();
 
+		if (err == DB_SUCCESS && !srv_read_only_mode) {
+			log_mutex_enter();
+			if (log_sys->is_encrypted() && !log_crypt_init()) {
+				err = DB_ERROR;
+			}
+			log_mutex_exit();
+		}
+
 		if (err == DB_SUCCESS) {
 			/* Initialize the change buffer. */
 			err = dict_boot();
@@ -2291,6 +2320,115 @@ innobase_start_or_create_for_mysql(void)
 
 		recv_recovery_from_checkpoint_finish();
 
+		/* Upgrade or resize or rebuild the redo logs before
+		generating any dirty pages, so that the old redo log
+		files will not be written to. */
+
+		if (srv_force_recovery == SRV_FORCE_NO_LOG_REDO) {
+			/* Completely ignore the redo log. */
+		} else if (srv_read_only_mode) {
+			/* Leave the redo log alone. */
+		} else if (srv_log_file_size_requested == srv_log_file_size
+			   && srv_n_log_files_found == srv_n_log_files
+			   && log_sys->is_encrypted() == srv_encrypt_log) {
+			/* No need to upgrade or resize the redo log. */
+		} else {
+			/* Prepare to delete the old redo log files */
+			flushed_lsn = srv_prepare_to_delete_redo_log_files(i);
+
+			DBUG_EXECUTE_IF("innodb_log_abort_1",
+					return(srv_init_abort(DB_ERROR)););
+			/* Prohibit redo log writes from any other
+			threads until creating a log checkpoint at the
+			end of create_log_files(). */
+			ut_d(recv_no_log_write = true);
+			ut_ad(!buf_pool_check_no_pending_io());
+
+			DBUG_EXECUTE_IF("innodb_log_abort_3",
+					return(srv_init_abort(DB_ERROR)););
+
+			/* Stamp the LSN to the data files. */
+			err = fil_write_flushed_lsn(flushed_lsn);
+
+			DBUG_EXECUTE_IF("innodb_log_abort_4", err = DB_ERROR;);
+
+			if (err != DB_SUCCESS) {
+				return(srv_init_abort(err));
+			}
+
+			/* Close and free the redo log files, so that
+			we can replace them. */
+			fil_close_log_files(true);
+
+			DBUG_EXECUTE_IF("innodb_log_abort_5",
+					return(srv_init_abort(DB_ERROR)););
+
+			/* Free the old log file space. */
+			log_group_close_all();
+
+			ib::info() << "Starting to delete and rewrite log"
+				" files.";
+
+			srv_log_file_size = srv_log_file_size_requested;
+
+			err = create_log_files(
+				logfilename, dirnamelen, flushed_lsn,
+				logfile0);
+
+			if (err == DB_SUCCESS) {
+				err = create_log_files_rename(
+					logfilename, dirnamelen, flushed_lsn,
+					logfile0);
+			}
+
+			if (err != DB_SUCCESS) {
+				return(srv_init_abort(err));
+			}
+		}
+
+
+		/* Validate a few system page types that were left
+		uninitialized by older versions of MySQL. */
+		if (!high_level_read_only) {
+			mtr_t		mtr;
+			buf_block_t*	block;
+			mtr.start();
+			mtr.set_sys_modified();
+			/* Bitmap page types will be reset in
+			buf_dblwr_check_block() without redo logging. */
+			block = buf_page_get(
+				page_id_t(IBUF_SPACE_ID,
+					  FSP_IBUF_HEADER_PAGE_NO),
+				univ_page_size, RW_X_LATCH, &mtr);
+			fil_block_check_type(block, FIL_PAGE_TYPE_SYS, &mtr);
+			/* Already MySQL 3.23.53 initialized
+			FSP_IBUF_TREE_ROOT_PAGE_NO to
+			FIL_PAGE_INDEX. No need to reset that one. */
+			block = buf_page_get(
+				page_id_t(TRX_SYS_SPACE, TRX_SYS_PAGE_NO),
+				univ_page_size, RW_X_LATCH, &mtr);
+			fil_block_check_type(block, FIL_PAGE_TYPE_TRX_SYS,
+					     &mtr);
+			block = buf_page_get(
+				page_id_t(TRX_SYS_SPACE,
+					  FSP_FIRST_RSEG_PAGE_NO),
+				univ_page_size, RW_X_LATCH, &mtr);
+			fil_block_check_type(block, FIL_PAGE_TYPE_SYS, &mtr);
+			block = buf_page_get(
+				page_id_t(TRX_SYS_SPACE, FSP_DICT_HDR_PAGE_NO),
+				univ_page_size, RW_X_LATCH, &mtr);
+			fil_block_check_type(block, FIL_PAGE_TYPE_SYS, &mtr);
+			mtr.commit();
+		}
+
+		/* Roll back any recovered data dictionary transactions, so
+		that the data dictionary tables will be free of any locks.
+		The data dictionary latch should guarantee that there is at
+		most one data dictionary transaction active at a time. */
+		if (srv_force_recovery < SRV_FORCE_NO_TRX_UNDO) {
+			trx_rollback_or_clean_recovered(FALSE);
+		}
+
 		/* Fix-up truncate of tables in the system tablespace
 		if server crashed while truncate was active. The non-
 		system tables are done after tablespace discovery. Do
@@ -2344,71 +2482,6 @@ innobase_start_or_create_for_mysql(void)
 			return(srv_init_abort(err));
 		}
 
-		if (!srv_force_recovery
-		    && !recv_sys->found_corrupt_log
-		    && (srv_log_file_size_requested != srv_log_file_size
-			|| srv_n_log_files_found != srv_n_log_files)) {
-			/* Prepare to replace the redo log files. */
-
-			if (srv_read_only_mode) {
-				ib::error() << "Cannot resize log files"
-					" in read-only mode.";
-				return(srv_init_abort(DB_READ_ONLY));
-			}
-
-			/* Prepare to delete the old redo log files */
-			flushed_lsn = srv_prepare_to_delete_redo_log_files(i);
-
-			DBUG_EXECUTE_IF("innodb_log_abort_1",
-					return(srv_init_abort(DB_ERROR)););
-			/* Prohibit redo log writes from any other
-			threads until creating a log checkpoint at the
-			end of create_log_files(). */
-			ut_d(recv_no_log_write = true);
-			ut_ad(!buf_pool_check_no_pending_io());
-
-			DBUG_EXECUTE_IF("innodb_log_abort_3",
-					return(srv_init_abort(DB_ERROR)););
-
-			/* Stamp the LSN to the data files. */
-			err = fil_write_flushed_lsn(flushed_lsn);
-
-			DBUG_EXECUTE_IF("innodb_log_abort_4", err = DB_ERROR;);
-
-			if (err != DB_SUCCESS) {
-				return(srv_init_abort(err));
-			}
-
-			/* Close and free the redo log files, so that
-			we can replace them. */
-			fil_close_log_files(true);
-
-			DBUG_EXECUTE_IF("innodb_log_abort_5",
-					return(srv_init_abort(DB_ERROR)););
-
-			/* Free the old log file space. */
-			log_group_close_all();
-
-			ib::warn() << "Starting to delete and rewrite log"
-				" files.";
-
-			srv_log_file_size = srv_log_file_size_requested;
-
-			err = create_log_files(
-				logfilename, dirnamelen, flushed_lsn,
-				logfile0);
-
-			if (err == DB_SUCCESS) {
-				err = create_log_files_rename(
-					logfilename, dirnamelen, flushed_lsn,
-					logfile0);
-			}
-
-			if (err != DB_SUCCESS) {
-				return(srv_init_abort(err));
-			}
-		}
-
 		recv_recovery_rollback_active();
 
 		/* It is possible that file_format tag has never