MDEV-11814 Refuse innodb_read_only startup if crash recovery is needed

recv_scan_log_recs(): Remember if redo log apply is needed,
even if starting up in innodb_read_only mode.

recv_recovery_from_checkpoint_start_func(): Refuse
innodb_read_only startup if redo log apply is needed.

mysql-test/include/kill_and_restart_mysqld.inc

+--let $_server_id= `SELECT @@server_id`
+--let $_expect_file_name= $MYSQLTEST_VARDIR/tmp/mysqld.$_server_id.expect
+
+if ($restart_parameters)
+{
+  --echo # Kill and restart: $restart_parameters
+  --exec echo "restart: $restart_parameters" > $_expect_file_name
+}
+if (!$restart_parameters)
+{
+  --echo # Kill and restart
+  --exec echo "restart" > $_expect_file_name
+}
+
+--shutdown_server 0
+--source include/wait_until_disconnected.inc
+--enable_reconnect
+--source include/wait_until_connected_again.inc
+--disable_reconnect

mysql-test/include/kill_mysqld.inc

+--let $_server_id= `SELECT @@server_id`
+--let $_expect_file_name= $MYSQLTEST_VARDIR/tmp/mysqld.$_server_id.expect
+
+--echo # Kill the server
+--exec echo "wait" > $_expect_file_name
+--shutdown_server 0
+--source include/wait_until_disconnected.inc

mysql-test/include/search_pattern_in_file.inc

@@ -60,25 +60,36 @@
 
 perl;
     use strict;
-    my $search_file=    $ENV{'SEARCH_FILE'}    or die "SEARCH_FILE not set";
+    die "SEARCH_FILE not set" unless $ENV{'SEARCH_FILE'};
+    my @search_files= glob($ENV{'SEARCH_FILE'});
     my $search_pattern= $ENV{'SEARCH_PATTERN'} or die "SEARCH_PATTERN not set";
     my $search_range=   $ENV{'SEARCH_RANGE'};
-    my $file_content;
+    my $content;
     $search_range= 50000 unless $search_range =~ /-?[0-9]+/;
-    open(FILE, '<', $search_file) or die("Unable to open '$search_file': $!\n");
-    if ($search_range >= 0) {
-       read(FILE, $file_content, $search_range, 0);
-    } else {
-       my $size= -s $search_file;
-       $search_range = -$size if $size > -$search_range;
-       seek(FILE, $search_range, 2);
-       read(FILE, $file_content, -$search_range, 0);
+    foreach my $search_file (@search_files) {
+        open(FILE, '<', $search_file) or die("Unable to open '$search_file': $!\n");
+        my $file_content;
+        if ($search_range >= 0) {
+           read(FILE, $file_content, $search_range, 0);
+        } else {
+           my $size= -s $search_file;
+           $search_range = -$size if $size > -$search_range;
+           seek(FILE, $search_range, 2);
+           read(FILE, $file_content, -$search_range, 0);
+        }
+        close(FILE);
+        $content.= $file_content;
     }
-    close(FILE);
-    $search_file =~ s{^.*?([^/\\]+)$}{$1};
-    if ($file_content =~ m{$search_pattern}) {
-      print "FOUND /$search_pattern/ in $search_file\n"
+    $ENV{'SEARCH_FILE'} =~ s{^.*?([^/\\]+)$}{$1};
+    if ($content =~ m{$search_pattern}) {
+      die "FOUND /$search_pattern/ in $ENV{'SEARCH_FILE'}\n"
+      if $ENV{SEARCH_ABORT} eq 'FOUND';
+      print "FOUND /$search_pattern/ in $ENV{'SEARCH_FILE'}\n"
+      unless defined $ENV{SEARCH_ABORT};
     } else {
-      print "NOT FOUND /$search_pattern/ in $search_file\n"
+      die "NOT FOUND /$search_pattern/ in $ENV{'SEARCH_FILE'}\n"
+      if $ENV{SEARCH_ABORT} eq 'NOT FOUND';
+      print "NOT FOUND /$search_pattern/ in $ENV{'SEARCH_FILE'}\n"
+      unless defined $ENV{SEARCH_ABORT};
     }
 EOF

mysql-test/include/start_mysqld.inc

 # Include this script only after using shutdown_mysqld.inc
 # where $_expect_file_name was initialized.
 # Write file to make mysql-test-run.pl start up the server again
---exec echo "restart" > $_expect_file_name
+if ($restart_parameters)
+{
+  --exec echo "restart: $restart_parameters" > $_expect_file_name
+}
+if (!$restart_parameters)
+{
+  --exec echo "restart" > $_expect_file_name
+}
 
 # Turn on reconnect
 --enable_reconnect

mysql-test/suite/innodb/r/log_file_size.result

+call mtr.add_suppression("InnoDB: Resizing redo log");
+call mtr.add_suppression("InnoDB: Starting to delete and rewrite log files");
+call mtr.add_suppression("InnoDB: New log files created");
+call mtr.add_suppression("InnoDB: The log sequence numbers [0-9]+ and [0-9]+ in ibdata files do not match the log sequence number [0-9]+ in the ib_logfiles");
+CREATE TABLE t1(a INT PRIMARY KEY) ENGINE=InnoDB;
+BEGIN;
+INSERT INTO t1 VALUES (42);
+# Kill and restart: --innodb-log-file-size=6M
+SELECT * FROM t1;
+a
+INSERT INTO t1 VALUES (42);
+BEGIN;
+DELETE FROM t1;
+# Kill and restart: --innodb-log-files-in-group=3 --innodb-log-file-size=5M
+SELECT * FROM t1;
+a
+42
+INSERT INTO t1 VALUES (123);
+BEGIN;
+DELETE FROM t1;
+# Kill the server
+--innodb-force-recovery-crash=1
+--innodb-force-recovery-crash=3
+--innodb-force-recovery-crash=4
+--innodb-force-recovery-crash=5
+--innodb-force-recovery-crash=6
+--innodb-force-recovery-crash=7
+--innodb-force-recovery-crash=8
+--innodb-force-recovery-crash=9
+--innodb-force-recovery-crash=10
+SELECT * FROM t1;
+a
+42
+123
+DROP TABLE t1;

mysql-test/suite/innodb/t/log_file_size.test

+# Test resizing the InnoDB redo log.
+
+--source include/have_innodb.inc
+
+# Embedded server does not support crashing
+--source include/not_embedded.inc
+# Avoid CrashReporter popup on Mac
+--source include/not_crashrep.inc
+# innodb-force-recovery-crash needs debug
+--source include/have_debug.inc
+
+if (`SELECT @@innodb_log_file_size = 1048576`) {
+  --skip Test requires innodb_log_file_size>1M.
+}
+
+call mtr.add_suppression("InnoDB: Resizing redo log");
+call mtr.add_suppression("InnoDB: Starting to delete and rewrite log files");
+call mtr.add_suppression("InnoDB: New log files created");
+call mtr.add_suppression("InnoDB: The log sequence numbers [0-9]+ and [0-9]+ in ibdata files do not match the log sequence number [0-9]+ in the ib_logfiles");
+
+CREATE TABLE t1(a INT PRIMARY KEY) ENGINE=InnoDB;
+BEGIN;
+INSERT INTO t1 VALUES (42);
+
+let $restart_parameters = --innodb-log-file-size=6M;
+--source include/kill_and_restart_mysqld.inc
+
+SELECT * FROM t1;
+
+INSERT INTO t1 VALUES (42);
+BEGIN;
+DELETE FROM t1;
+
+let $restart_parameters = --innodb-log-files-in-group=3 --innodb-log-file-size=5M;
+--source include/kill_and_restart_mysqld.inc
+
+SELECT * FROM t1;
+
+INSERT INTO t1 VALUES (123);
+
+let MYSQLD_DATADIR= `select @@datadir`;
+let SEARCH_ABORT = NOT FOUND;
+let SEARCH_FILE= $MYSQLTEST_VARDIR/log/my_restart.err;
+let $args=--innodb --unknown-option --loose-console --core-file > $SEARCH_FILE 2>&1;
+let $crash=--innodb --unknown-option --loose-console > $SEARCH_FILE 2>&1 --innodb-force-recovery-crash;
+
+BEGIN;
+DELETE FROM t1;
+
+--source include/kill_mysqld.inc
+
+--error 2
+--exec $MYSQLD_CMD $args --innodb-log-group-home-dir=foo\;bar
+let SEARCH_PATTERN= syntax error in innodb_log_group_home_dir;
+--source include/search_pattern_in_file.inc
+--remove_file $SEARCH_FILE
+
+--echo --innodb-force-recovery-crash=1
+--error 3
+--exec $MYSQLD_CMD $crash=1
+let SEARCH_PATTERN= InnoDB: Starting an apply batch of log records;
+--source include/search_pattern_in_file.inc
+--remove_file $SEARCH_FILE
+--echo --innodb-force-recovery-crash=3
+--error 3
+--exec $MYSQLD_CMD $crash=3
+let SEARCH_PATTERN= InnoDB: Starting an apply batch of log records;
+--source include/search_pattern_in_file.inc
+--remove_file $SEARCH_FILE
+
+--error 2
+--exec $MYSQLD_CMD $args --innodb-read-only
+let SEARCH_PATTERN= InnoDB: innodb_read_only prevents crash recovery;
+--source include/search_pattern_in_file.inc
+--remove_file $SEARCH_FILE
+
+--echo --innodb-force-recovery-crash=4
+--error 3
+--exec $MYSQLD_CMD $crash=4
+let SEARCH_PATTERN= InnoDB: Starting an apply batch of log records;
+--source include/search_pattern_in_file.inc
+let SEARCH_PATTERN= InnoDB: Resizing redo log from 3\*[0-9]+ to 2\*[0-9]+ pages;
+--source include/search_pattern_in_file.inc
+--remove_file $SEARCH_FILE
+--echo --innodb-force-recovery-crash=5
+--error 3
+--exec $MYSQLD_CMD $crash=5
+let SEARCH_PATTERN= InnoDB: Starting an apply batch of log records;
+--source include/search_pattern_in_file.inc
+let SEARCH_PATTERN= InnoDB: Resizing redo log from 3\*[0-9]+ to 2\*[0-9]+ pages;
+--source include/search_pattern_in_file.inc
+--remove_file $SEARCH_FILE
+
+--error 2
+--exec $MYSQLD_CMD $args --innodb-read-only
+let SEARCH_PATTERN= InnoDB: innodb_read_only prevents crash recovery;
+--source include/search_pattern_in_file.inc
+--remove_file $SEARCH_FILE
+
+--echo --innodb-force-recovery-crash=6
+--error 3
+--exec $MYSQLD_CMD $crash=6
+let SEARCH_PATTERN= InnoDB: Starting an apply batch of log records;
+--source include/search_pattern_in_file.inc
+let SEARCH_PATTERN= InnoDB: Resizing redo log from 3\*[0-9]+ to 2\*[0-9]+ pages;
+--source include/search_pattern_in_file.inc
+--remove_file $SEARCH_FILE
+--echo --innodb-force-recovery-crash=7
+--error 3
+--exec $MYSQLD_CMD $crash=7
+# this crashes right after deleting all log files
+--remove_file $SEARCH_FILE
+
+--error 2
+--exec $MYSQLD_CMD $args --innodb-read-only
+let SEARCH_PATTERN= InnoDB: Cannot create log files in read-only mode;
+--source include/search_pattern_in_file.inc
+--remove_file $SEARCH_FILE
+
+--echo --innodb-force-recovery-crash=8
+--error 3
+--exec $MYSQLD_CMD $crash=8
+let SEARCH_PATTERN= InnoDB: Setting log file .*ib_logfile[0-9]+ size to;
+--source include/search_pattern_in_file.inc
+--remove_file $SEARCH_FILE
+--echo --innodb-force-recovery-crash=9
+--error 3
+--exec $MYSQLD_CMD $crash=9
+let SEARCH_PATTERN= InnoDB: Setting log file .*ib_logfile[0-9]+ size to;
+--source include/search_pattern_in_file.inc
+--remove_file $SEARCH_FILE
+
+# We should have perfectly synced files here.
+# Rename the log files, and trigger an error in recovery.
+--move_file $MYSQLD_DATADIR/ib_logfile101 $MYSQLD_DATADIR/ib_logfile0
+--move_file $MYSQLD_DATADIR/ib_logfile1 $MYSQLD_DATADIR/ib_logfile1_hidden
+--error 2
+--exec $MYSQLD_CMD $args
+let SEARCH_PATTERN= InnoDB: Only one log file found;
+--source include/search_pattern_in_file.inc
+--remove_file $SEARCH_FILE
+--move_file $MYSQLD_DATADIR/ib_logfile0 $MYSQLD_DATADIR/ib_logfile101
+
+perl;
+die unless open(FILE, ">$ENV{MYSQLD_DATADIR}/ib_logfile0");
+print FILE "garbage";
+close(FILE);
+EOF
+--error 2
+--exec $MYSQLD_CMD $args
+let SEARCH_PATTERN= InnoDB: Log file .*ib_logfile0 size 7 is not a multiple of innodb_page_size;
+--source include/search_pattern_in_file.inc
+--remove_file $SEARCH_FILE
+--remove_file $MYSQLD_DATADIR/ib_logfile0
+--move_file $MYSQLD_DATADIR/ib_logfile101 $MYSQLD_DATADIR/ib_logfile0
+
+perl;
+die unless open(FILE, ">$ENV{MYSQLD_DATADIR}/ib_logfile1");
+print FILE "junkfill" x 131072;
+close(FILE);
+EOF
+
+--error 2
+--exec $MYSQLD_CMD $args
+let SEARCH_PATTERN= InnoDB: Log file .*ib_logfile1 is of different size 1048576 bytes than other log files;
+--source include/search_pattern_in_file.inc
+--remove_file $SEARCH_FILE
+--remove_file $MYSQLD_DATADIR/ib_logfile1
+--move_file $MYSQLD_DATADIR/ib_logfile0 $MYSQLD_DATADIR/ib_logfile101
+--move_file $MYSQLD_DATADIR/ib_logfile1_hidden $MYSQLD_DATADIR/ib_logfile1
+
+--echo --innodb-force-recovery-crash=10
+--error 3
+--exec $MYSQLD_CMD $crash=10
+let SEARCH_PATTERN= InnoDB: Setting log file .*ib_logfile[0-9]+ size to;
+--source include/search_pattern_in_file.inc
+let SEARCH_PATTERN= InnoDB: Renaming log file .*ib_logfile101 to .*ib_logfile0;
+--source include/search_pattern_in_file.inc
+--remove_file $SEARCH_FILE
+
+--let $restart_parameters=
+--source include/start_mysqld.inc
+
+SELECT * FROM t1;
+DROP TABLE t1;

storage/innobase/log/log0recv.cc

@@ -2806,11 +2806,10 @@ recv_scan_log_recs(
 
 					recv_init_crash_recovery();
 				} else {
-
-					ib_logf(IB_LOG_LEVEL_WARN,
-						"Recovery skipped, "
-						"--innodb-read-only set!");
-
+					ib_logf(IB_LOG_LEVEL_ERROR,
+						"innodb_read_only prevents"
+						" crash recovery");
+					recv_needed_recovery = TRUE;
 					return(TRUE);
 				}
 			}
@@ -3227,6 +3226,11 @@ recv_recovery_from_checkpoint_start_func(
 
 	/* Done with startup scan. Clear the flag. */
 	recv_log_scan_is_startup_type = FALSE;
+
+	if (srv_read_only_mode && recv_needed_recovery) {
+		return(DB_READ_ONLY);
+	}
+
 	if (TYPE_CHECKPOINT) {
 		/* NOTE: we always do a 'recovery' at startup, but only if
 		there is something wrong we will print a message to the

storage/xtradb/log/log0recv.cc

@@ -2,6 +2,7 @@
 
 Copyright (c) 1997, 2016, Oracle and/or its affiliates. All Rights Reserved.
 Copyright (c) 2012, Facebook Inc.
+Copyright (c) 2017, MariaDB Corporation. All Rights Reserved.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -2895,11 +2896,10 @@ recv_scan_log_recs(
 
 					recv_init_crash_recovery();
 				} else {
-
-					ib_logf(IB_LOG_LEVEL_WARN,
-						"Recovery skipped, "
-						"--innodb-read-only set!");
-
+					ib_logf(IB_LOG_LEVEL_ERROR,
+						"innodb_read_only prevents"
+						" crash recovery");
+					recv_needed_recovery = TRUE;
 					return(TRUE);
 				}
 			}
@@ -3323,6 +3323,11 @@ recv_recovery_from_checkpoint_start_func(
 
 	/* Done with startup scan. Clear the flag. */
 	recv_log_scan_is_startup_type = FALSE;
+
+	if (srv_read_only_mode && recv_needed_recovery) {
+		return(DB_READ_ONLY);
+	}
+
 	if (TYPE_CHECKPOINT) {
 		/* NOTE: we always do a 'recovery' at startup, but only if
 		there is something wrong we will print a message to the