MDEV-20060: Failing assertion: srv_log_file_size <= 512ULL << 30 while preparing backup

The general reason why innodb redo log file is limited by 512G is that
log_block_convert_lsn_to_no() returns value limited by 1G. But there is no
need to have unique log block numbers in log group. The fix removes 512G
limit and limits log group size by
(uint32_t maximum value) * (minimum page size), which, in turns, can be
removed if fil_io() is no longer used for innodb redo log io.

extra/mariabackup/xtrabackup.cc

@@ -1298,7 +1298,7 @@ struct my_option xb_server_options[] =
   {"innodb_log_file_size", OPT_INNODB_LOG_FILE_SIZE,
    "Ignored for mysqld option compatibility",
    (G_PTR*) &srv_log_file_size, (G_PTR*) &srv_log_file_size, 0,
-   GET_ULL, REQUIRED_ARG, 48 << 20, 1 << 20, 512ULL << 30, 0,
+   GET_ULL, REQUIRED_ARG, 48 << 20, 1 << 20, log_group_max_size, 0,
    UNIV_PAGE_SIZE_MAX, 0},
   {"innodb_log_files_in_group", OPT_INNODB_LOG_FILES_IN_GROUP,
    "Ignored for mysqld option compatibility",

mysql-test/mysql-test-run.pl

@@ -3333,6 +3333,7 @@ sub mysql_install_db {
   # Create the bootstrap.sql file
   # ----------------------------------------------------------------------
   my $bootstrap_sql_file= "$opt_vardir/log/bootstrap.sql";
+  $ENV{'MYSQL_BOOTSTRAP_SQL_FILE'}= $bootstrap_sql_file;
 
   if (! -e $bootstrap_sql_file)
   {

mysql-test/suite/mariabackup/big_innodb_log.result

+# Kill the server
+CREATE TABLE t(i INT) ENGINE InnoDB;
+INSERT INTO t VALUES
+(0), (1), (2), (3), (4), (5), (6), (7), (8), (9),
+(0), (1), (2), (3), (4), (5), (6), (7), (8), (9),
+(0), (1), (2), (3), (4), (5), (6), (7), (8), (9),
+(0), (1), (2), (3), (4), (5), (6), (7), (8), (9),
+(0), (1), (2), (3), (4), (5), (6), (7), (8), (9),
+(0), (1), (2), (3), (4), (5), (6), (7), (8), (9),
+(0), (1), (2), (3), (4), (5), (6), (7), (8), (9),
+(0), (1), (2), (3), (4), (5), (6), (7), (8), (9),
+(0), (1), (2), (3), (4), (5), (6), (7), (8), (9),
+(0), (1), (2), (3), (4), (5), (6), (7), (8), (9);
+# xtrabackup backup, execute the following query after test.t is copied:
+# BEGIN NOT ATOMIC INSERT INTO test.t SELECT * FROM test.t; UPDATE test.t SET i = 10 WHERE i = 0; DELETE FROM test.t WHERE i = 1; END
+SELECT count(*) FROM t WHERE i = 0;
+count(*)
+0
+# xtrabackup prepare
+# shutdown server
+# remove datadir
+# xtrabackup move back
+# restart server
+SELECT count(*) FROM t WHERE i = 0;
+count(*)
+0
+Ok
+Ok
+DROP TABLE t;
+# Kill the server

mysql-test/suite/mariabackup/big_innodb_log.test

+# The general reason why innodb redo log file is limited by 512G is that
+# log_block_convert_lsn_to_no() returns value limited by 1G. But there is no
+# need to have unique log block numbers in log group. This test forces innodb
+# to generate redo log files with non-unique log block numbers and tests
+# recovery process with such numbers.
+--source include/have_innodb.inc
+--source include/have_debug.inc
+
+--let MYSQLD_DATADIR= `select @@datadir`
+let $MYSQLD_BOOTSTRAP_CMD= $MYSQLD_BOOTSTRAP_CMD --datadir=$MYSQLD_DATADIR --debug-dbug=+d,innodb_small_log_block_no_limit;
+
+--source include/kill_mysqld.inc
+--rmdir $MYSQLD_DATADIR
+--mkdir $MYSQLD_DATADIR
+--mkdir $MYSQLD_DATADIR/mysql
+--mkdir $MYSQLD_DATADIR/test
+--exec $MYSQLD_BOOTSTRAP_CMD < $MYSQL_BOOTSTRAP_SQL_FILE >> $MYSQLTEST_VARDIR/tmp/bootstrap.log 2>&1
+let $old_restart_parameters=$restart_parameters;
+let $restart_parameters= $old_restart_parameters --debug-dbug=+d,innodb_small_log_block_no_limit;
+--source include/start_mysqld.inc
+
+CREATE TABLE t(i INT) ENGINE InnoDB;
+INSERT INTO t VALUES
+  (0), (1), (2), (3), (4), (5), (6), (7), (8), (9),
+  (0), (1), (2), (3), (4), (5), (6), (7), (8), (9),
+  (0), (1), (2), (3), (4), (5), (6), (7), (8), (9),
+  (0), (1), (2), (3), (4), (5), (6), (7), (8), (9),
+  (0), (1), (2), (3), (4), (5), (6), (7), (8), (9),
+  (0), (1), (2), (3), (4), (5), (6), (7), (8), (9),
+  (0), (1), (2), (3), (4), (5), (6), (7), (8), (9),
+  (0), (1), (2), (3), (4), (5), (6), (7), (8), (9),
+  (0), (1), (2), (3), (4), (5), (6), (7), (8), (9),
+  (0), (1), (2), (3), (4), (5), (6), (7), (8), (9);
+
+--let after_copy_test_t=BEGIN NOT ATOMIC INSERT INTO test.t SELECT * FROM test.t; UPDATE test.t SET i = 10 WHERE i = 0; DELETE FROM test.t WHERE i = 1; END
+
+--echo # xtrabackup backup, execute the following query after test.t is copied:
+--echo # $after_copy_test_t
+let $targetdir=$MYSQLTEST_VARDIR/tmp/backup;
+
+--disable_result_log
+exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --parallel=10 --target-dir=$targetdir --dbug=+d,mariabackup_events,innodb_small_log_block_no_limit;
+--enable_result_log
+
+--let $total_before=`SELECT count(*) FROM t`
+SELECT count(*) FROM t WHERE i = 0;
+--let $updated_before=`SELECT count(*) FROM t WHERE i = 10`
+
+echo # xtrabackup prepare;
+--disable_result_log
+exec $XTRABACKUP  --prepare --target-dir=$targetdir --dbug=+d,innodb_small_log_block_no_limit;
+--source include/restart_and_restore.inc
+--enable_result_log
+
+--let $total_after=`SELECT count(*) FROM t`
+SELECT count(*) FROM t WHERE i = 0;
+--let $updated_after=`SELECT count(*) FROM t WHERE i = 10`
+
+if ($total_before == $total_after) {
+--echo Ok
+}
+if ($total_before != $total_after) {
+--echo Failed
+}
+if ($updated_before == $updated_after) {
+--echo Ok
+}
+if ($updated_before != $updated_after) {
+--echo Failed
+}
+
+DROP TABLE t;
+rmdir $targetdir;
+--source include/kill_mysqld.inc
+--rmdir $MYSQLD_DATADIR
+
+perl;
+use lib "lib";
+use My::File::Path;
+my $install_db_dir = ($ENV{MTR_PARALLEL} == 1) ?
+  "$ENV{'MYSQLTEST_VARDIR'}/install.db" :
+  "$ENV{'MYSQLTEST_VARDIR'}/../install.db";
+copytree($install_db_dir, $ENV{'MYSQLD_DATADIR'});
+EOF
+
+--let $restart_parameters= $old_restart_parameters
+--source include/start_mysqld.inc

mysql-test/suite/sys_vars/r/sysvars_innodb.result

@@ -1793,7 +1793,7 @@ VARIABLE_SCOPE	GLOBAL
 VARIABLE_TYPE	BIGINT UNSIGNED
 VARIABLE_COMMENT	Size of each log file in a log group.
 NUMERIC_MIN_VALUE	1048576
-NUMERIC_MAX_VALUE	549755813888
+NUMERIC_MAX_VALUE	17592186044415
 NUMERIC_BLOCK_SIZE	65536
 ENUM_VALUE_LIST	NULL
 READ_ONLY	YES

storage/innobase/handler/ha_innodb.cc

@@ -20453,7 +20453,7 @@ static MYSQL_SYSVAR_LONG(log_buffer_size, innobase_log_buffer_size,
 static MYSQL_SYSVAR_ULONGLONG(log_file_size, srv_log_file_size,
   PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY,
   "Size of each log file in a log group.",
-  NULL, NULL, 48 << 20, 1 << 20, 512ULL << 30, UNIV_PAGE_SIZE_MAX);
+  NULL, NULL, 48 << 20, 1 << 20, log_group_max_size, UNIV_PAGE_SIZE_MAX);
 /* OS_FILE_LOG_BLOCK_SIZE would be more appropriate than UNIV_PAGE_SIZE_MAX,
 but fil_space_t is being used for the redo log, and it uses data pages. */
 

storage/innobase/include/log0log.h

@@ -40,6 +40,10 @@ Created 12/9/1995 Heikki Tuuri
 #include "os0event.h"
 #include "os0file.h"
 
+#ifndef UINT32_MAX
+#define UINT32_MAX             (4294967295U)
+#endif
+
 /** Redo log group */
 struct log_group_t;
 
@@ -527,6 +531,12 @@ MariaDB 10.2.18 and later will use the 10.3 format, but LOG_HEADER_SUBFORMAT
 					header */
 #define LOG_FILE_HDR_SIZE	(4 * OS_FILE_LOG_BLOCK_SIZE)
 
+/* As long as fil_io() is used to handle log io, log group max size is limited
+by (maximum page number) * (minimum page size). Page number type is uint32_t.
+Remove this limitation if page number is no longer used for log file io. */
+static const ulonglong log_group_max_size =
+	((ulonglong(UINT32_MAX) + 1) * UNIV_PAGE_SIZE_MIN - 1);
+
 /** The state of a log group */
 enum log_group_state_t {
 	/** No corruption detected */

storage/innobase/include/log0log.ic

@@ -185,7 +185,9 @@ log_block_convert_lsn_to_no(
 /*========================*/
 	lsn_t	lsn)	/*!< in: lsn of a byte within the block */
 {
-	return(((ulint) (lsn / OS_FILE_LOG_BLOCK_SIZE) & 0x3FFFFFFFUL) + 1);
+	return(((ulint) (lsn / OS_FILE_LOG_BLOCK_SIZE) &
+		DBUG_EVALUATE_IF("innodb_small_log_block_no_limit",
+			0xFUL, 0x3FFFFFFFUL)) + 1);
 }
 
 /************************************************************//**

storage/innobase/log/log0log.cc

@@ -942,11 +942,14 @@ log_group_write_buf(
 	the trailer fields of the log blocks */
 
 	for (i = 0; i < write_len / OS_FILE_LOG_BLOCK_SIZE; i++) {
+#ifdef UNIV_DEBUG
+		ulint hdr_no_2 = log_block_get_hdr_no(buf) + i;
+		DBUG_EXECUTE_IF("innodb_small_log_block_no_limit",
+				hdr_no_2 = ((hdr_no_2 - 1) & 0xFUL) + 1;);
+#endif
 		ut_ad(pad_len >= len
 			|| i * OS_FILE_LOG_BLOCK_SIZE >= len - pad_len
-		      || log_block_get_hdr_no(
-			      buf + i * OS_FILE_LOG_BLOCK_SIZE)
-			 == log_block_get_hdr_no(buf) + i);
+			|| log_block_get_hdr_no(buf + i * OS_FILE_LOG_BLOCK_SIZE) == hdr_no_2);
 		log_block_store_checksum(buf + i * OS_FILE_LOG_BLOCK_SIZE);
 	}
 

storage/innobase/srv/srv0start.cc

@@ -1852,13 +1852,11 @@ innobase_start_or_create_for_mysql()
 		srv_start_state_set(SRV_START_STATE_IO);
 	}
 
-	if (srv_n_log_files * srv_log_file_size >= 512ULL << 30) {
-		/* log_block_convert_lsn_to_no() limits the returned block
-		number to 1G and given that OS_FILE_LOG_BLOCK_SIZE is 512
-		bytes, then we have a limit of 512 GB. If that limit is to
-		be raised, then log_block_convert_lsn_to_no() must be
-		modified. */
-		ib::error() << "Combined size of log files must be < 512 GB";
+	if (srv_n_log_files * srv_log_file_size >= log_group_max_size) {
+		/* Log group size is limited by the size of page number. Remove this
+		limitation when fil_io() is not used for recovery log io. */
+		ib::error() << "Combined size of log files must be < "
+			<< log_group_max_size << " GB";
 
 		return(srv_init_abort(DB_ERROR));
 	}
@@ -2070,7 +2068,7 @@ innobase_start_or_create_for_mysql()
 		ut_a(fil_validate());
 		ut_a(log_space);
 
-		ut_a(srv_log_file_size <= 512ULL << 30);
+		ut_a(srv_log_file_size <= log_group_max_size);
 
 		const ulint size = 1 + ulint((srv_log_file_size - 1)
 					     >> srv_page_size_shift);