MDEV-14585 Automatically remove #sql- tables in InnoDB dictionary during recovery

This is a backport of the following commits:
commit b4165985
commit 69e88de0
commit 40f4525f
commit 656f66de

Now that MDEV-14717 made RENAME TABLE crash-safe within InnoDB,
it should be safe to drop the #sql- tables within InnoDB during
crash recovery. These tables can be one of two things:

(1) #sql-ib related to deferred DROP TABLE (follow-up to MDEV-13407)
or to table-rebuilding ALTER TABLE...ALGORITHM=INPLACE
(since MDEV-14378, only related to the intermediate copy of a table),

(2) #sql- related to the intermediate copy of a table during
ALTER TABLE...ALGORITHM=COPY

We will not drop tables whose name starts with #sql2, because
the server can be killed during an ALGORITHM=COPY operation at
a point where the original table was renamed to #sql2 but the
finished intermediate copy was not yet renamed from #sql-
to the original table name.

If an old version of MariaDB Server before 10.2.13 (MDEV-11415)
was killed while ALTER TABLE...ALGORITHM=COPY was in progress,
after recovery there could be undo log records for some records that were
inserted into an intermediate copy of the table. Due to these undo log
records, InnoDB would resurrect locks at recovery, and the intermediate
table would be locked while we are trying to drop it. This would cause
a call to row_rename_table_for_mysql(), either from
row_mysql_drop_garbage_tables() or from the rollback of a RENAME
operation that was part of the ALTER TABLE.

row_rename_table_for_mysql(): Do not attempt to parse FOREIGN KEY
constraints when renaming from #sql-something to #sql-something-else,
because it does not make any sense.

row_drop_table_for_mysql(): When deferring DROP TABLE due to locks,
do not rename the table if its name already starts with the #sql-
prefix, which is what row_mysql_drop_garbage_tables() uses.
Previously, the too strict prefix #sql-ib was used, and some
tables were renamed unnecessarily.

mysql-test/suite/innodb/r/alter_copy.result

@@ -183,7 +183,6 @@ CHECK TABLE t1;
 Table	Op	Msg_type	Msg_text
 test.t1	check	status	OK
 #sql-temporary.frm
-#sql-temporary.ibd
 FTS_INDEX_1.ibd
 FTS_INDEX_2.ibd
 FTS_INDEX_3.ibd
@@ -212,4 +211,3 @@ t.ibd
 t1.frm
 t1.ibd
 DROP TABLE t1,t;
-DROP TABLE `#mysql50##sql-temporary`;

mysql-test/suite/innodb/r/alter_crash.result

@@ -83,9 +83,8 @@ ALTER TABLE t2 ADD PRIMARY KEY (f2, f1);
 ERROR HY000: Lost connection to MySQL server during query
 # Startup the server after the crash
 SELECT * FROM information_schema.innodb_sys_tables
-WHERE name LIKE 'test/#sql-ib%';
+WHERE name LIKE 'test/#sql-%';
 TABLE_ID	NAME	FLAG	N_COLS	SPACE	FILE_FORMAT	ROW_FORMAT	ZIP_PAGE_SIZE	SPACE_TYPE
-# Drop the orphaned rebuilt table.
 SHOW TABLES;
 Tables_in_test
 t2
@@ -123,7 +122,6 @@ ERROR HY000: Lost connection to MySQL server during query
 SELECT * FROM information_schema.innodb_sys_tables
 WHERE table_id = ID;
 TABLE_ID	NAME	FLAG	N_COLS	SPACE	FILE_FORMAT	ROW_FORMAT	ZIP_PAGE_SIZE	SPACE_TYPE
-FLUSH TABLES;
 # Files in datadir after manual recovery.
 t1.frm
 t1.ibd

mysql-test/suite/innodb/r/innodb-alter-tempfile.result

@@ -8,6 +8,7 @@ CREATE TABLE t1 (f1 INT NOT NULL, f2 INT NOT NULL) ENGINE=innodb;
 SET debug_dbug='+d,innodb_alter_commit_crash_before_commit';
 ALTER TABLE t1 ADD PRIMARY KEY (f2, f1);
 ERROR HY000: Lost connection to MySQL server during query
+# Startup the server after the crash
 show create table t1;
 Table	Create Table
 t1	CREATE TABLE `t1` (

mysql-test/suite/innodb/r/rename_table_debug.result

-CREATE TABLE t1 (a INT UNSIGNED PRIMARY KEY) ENGINE=InnoDB;
-INSERT INTO t1 VALUES(42);
+FLUSH TABLES;
+CREATE TABLE t1 (a SERIAL, b INT, c INT, d INT) ENGINE=InnoDB;
+INSERT INTO t1 () VALUES ();
 connect  con1,localhost,root,,test;
 SET DEBUG_SYNC='before_rename_table_commit SIGNAL renamed WAIT_FOR ever';
 RENAME TABLE t1 TO t2;
@@ -7,6 +8,6 @@ connection default;
 SET DEBUG_SYNC='now WAIT_FOR renamed';
 disconnect con1;
 SELECT * FROM t1;
-a
-42
+a	b	c	d
+1	NULL	NULL	NULL
 DROP TABLE t1;

mysql-test/suite/innodb/r/truncate_crash.result

+FLUSH TABLES;
+CREATE TABLE t1 (a INT PRIMARY KEY) ENGINE=InnoDB;
+INSERT INTO t1 SET a=1;
+connect  wait,localhost,root,,test;
+SET DEBUG_SYNC='after_trx_committed_in_memory SIGNAL c WAIT_FOR ever';
+TRUNCATE TABLE t1;
+connection default;
+SET DEBUG_SYNC='now WAIT_FOR c';
+disconnect wait;
+SELECT * FROM t1;
+a
+1
+TRUNCATE TABLE t1;
+DROP TABLE t1;

mysql-test/suite/innodb/t/alter_copy.test

@@ -88,8 +88,4 @@ CHECK TABLE t1;
 DROP TABLE t1,t;
 
 # Work around missing crash recovery at the SQL layer.
-let $temp_table_name = `SELECT SUBSTRING(name,6)
-	FROM information_schema.innodb_sys_tables
-	WHERE name LIKE "test/#sql-%"`;
---replace_regex /#sql-[0-9a-f_]*/#sql-temporary/
-eval DROP TABLE `#mysql50#$temp_table_name`;
+--remove_files_wildcard $datadir/test #sql-*.frm

mysql-test/suite/innodb/t/alter_crash.test

@@ -72,9 +72,6 @@ let $orig_table_id = `SELECT table_id
 --error 2013
 ALTER TABLE t1 ADD PRIMARY KEY (f2, f1);
 
---echo # Restart mysqld after the crash and reconnect.
---source include/start_mysqld.inc
-
 let TABLENAME_INC= $MYSQLTEST_VARDIR/tmp/tablename.inc;
 perl;
 die unless open OUT, ">$ENV{TABLENAME_INC}";
@@ -86,12 +83,15 @@ EOF
 source $TABLENAME_INC;
 remove_file $TABLENAME_INC;
 
+move_file $datadir/test/$tablename.frm $datadir/test/t1.frm;
+
+--echo # Restart mysqld after the crash and reconnect.
+--source include/start_mysqld.inc
+
 --replace_result $orig_table_id ID
 eval SELECT * FROM information_schema.innodb_sys_tables
 WHERE table_id = $orig_table_id;
 
-move_file $datadir/test/$tablename.frm $datadir/test/t1.frm;
-
 --echo # Files in datadir after manual recovery.
 --list_files $MYSQLD_DATADIR/test
 
@@ -125,27 +125,13 @@ let $orig_table_id = `SELECT table_id
 --error 2013
 ALTER TABLE t2 ADD PRIMARY KEY (f2, f1);
 
+remove_files_wildcard $datadir/test #sql-*.frm;
+
 --echo # Startup the server after the crash
 --source include/start_mysqld.inc
 
 SELECT * FROM information_schema.innodb_sys_tables
-WHERE name LIKE 'test/#sql-ib%';
-
-let TABLENAME_INC= $MYSQLTEST_VARDIR/tmp/tablename.inc;
-perl;
-die unless open OUT, ">$ENV{TABLENAME_INC}";
-chdir "$ENV{'datadir'}/test";
-my @frm_file = map { substr($_, 0, -4) } glob "#sql-*.frm";
-print OUT 'let $tablename=', $frm_file[0], ';';
-close OUT or die;
-EOF
-source $TABLENAME_INC;
-remove_file $TABLENAME_INC;
-
---echo # Drop the orphaned rebuilt table.
---disable_query_log
-eval DROP TABLE `#mysql50#$tablename`;
---enable_query_log
+WHERE name LIKE 'test/#sql-%';
 
 SHOW TABLES;
 INSERT INTO t2 VALUES (5,6),(7,8);
@@ -181,13 +167,6 @@ let $orig_table_id = `select table_id from
 --error 2013
 ALTER TABLE t1 ADD INDEX (b), CHANGE c d int, ALGORITHM=INPLACE;
 
---echo # Restart mysqld after the crash and reconnect.
---source include/start_mysqld.inc
-
---replace_result $orig_table_id ID
-eval SELECT * FROM information_schema.innodb_sys_tables
-WHERE table_id = $orig_table_id;
-
 perl;
 die unless open OUT, ">$ENV{TABLENAME_INC}";
 chdir "$ENV{'datadir'}/test";
@@ -195,12 +174,17 @@ my @frm_file = map { substr($_, 0, -4) } glob "#sql-*.frm";
 print OUT 'let $tablename=', $frm_file[0], ';';
 close OUT or die;
 EOF
-
 source $TABLENAME_INC;
 remove_file $TABLENAME_INC;
+
 move_file $datadir/test/$tablename.frm $datadir/test/t1.frm;
 
-FLUSH TABLES;
+--echo # Restart mysqld after the crash and reconnect.
+--source include/start_mysqld.inc
+
+--replace_result $orig_table_id ID
+eval SELECT * FROM information_schema.innodb_sys_tables
+WHERE table_id = $orig_table_id;
 
 --echo # Files in datadir after manual recovery.
 --list_files $MYSQLD_DATADIR/test

mysql-test/suite/innodb/t/innodb-alter-tempfile.test

@@ -31,25 +31,14 @@ SET debug_dbug='+d,innodb_alter_commit_crash_before_commit';
 
 --error 2013
 ALTER TABLE t1 ADD PRIMARY KEY (f2, f1);
+--echo # Startup the server after the crash
 
-let TABLENAME_INC= $MYSQLTEST_VARDIR/tmp/tablename.inc;
-perl;
-die unless open OUT, ">$ENV{TABLENAME_INC}";
-chdir "$ENV{'datadir'}/test";
-my @frm_file = map { substr($_, 0, -4) } glob "#sql-*.frm";
-print OUT 'let $temp_table_name=', $frm_file[0], ';';
-close OUT or die;
-EOF
-source $TABLENAME_INC;
-remove_file $TABLENAME_INC;
+remove_files_wildcard $datadir/test #sql-*.frm;
 
 --source include/start_mysqld.inc
 
 show create table t1;
 --echo # Consecutive Alter table does not create same temporary file name
 ALTER TABLE t1 ADD PRIMARY KEY (f2, f1);
---disable_query_log
-eval DROP TABLE `#mysql50#$temp_table_name`;
---enable_query_log
 show create table t1;
 drop table t1;

mysql-test/suite/innodb/t/rename_table_debug.test

@@ -3,8 +3,11 @@
 --source include/have_debug_sync.inc
 --source include/not_embedded.inc
 
-CREATE TABLE t1 (a INT UNSIGNED PRIMARY KEY) ENGINE=InnoDB;
-INSERT INTO t1 VALUES(42);
+FLUSH TABLES;
+LET $datadir= `SELECT @@datadir`;
+
+CREATE TABLE t1 (a SERIAL, b INT, c INT, d INT) ENGINE=InnoDB;
+INSERT INTO t1 () VALUES ();
 
 --connect (con1,localhost,root,,test)
 SET DEBUG_SYNC='before_rename_table_commit SIGNAL renamed WAIT_FOR ever';

mysql-test/suite/innodb/t/truncate_crash.test

+--source include/have_innodb.inc
+--source include/have_debug.inc
+--source include/have_debug_sync.inc
+--source include/not_embedded.inc
+
+FLUSH TABLES;
+CREATE TABLE t1 (a INT PRIMARY KEY) ENGINE=InnoDB;
+INSERT INTO t1 SET a=1;
+
+connect (wait,localhost,root,,test);
+SET DEBUG_SYNC='after_trx_committed_in_memory SIGNAL c WAIT_FOR ever';
+send TRUNCATE TABLE t1;
+
+connection default;
+SET DEBUG_SYNC='now WAIT_FOR c';
+--let $shutdown_timeout=0
+--source include/restart_mysqld.inc
+disconnect wait;
+
+SELECT * FROM t1;
+TRUNCATE TABLE t1;
+DROP TABLE t1;

storage/innobase/handler/ha_innodb.cc

@@ -13509,11 +13509,7 @@ int ha_innobase::truncate()
 
 	if (!err) {
 		/* Reopen the newly created table, and drop the
-		original table that was renamed to temp_name.
-
-		Note: In MariaDB 10.2 (before MDEV-14585), if the
-		server is killed and restarted before the original
-		table is dropped, the table will remain orphaned. */
+		original table that was renamed to temp_name. */
 		close();
 		err = open(name, 0, 0);
 		if (!err) {

storage/innobase/row/row0mysql.cc

@@ -2773,7 +2773,7 @@ row_mysql_drop_garbage_tables()
 		table_name = mem_heap_strdupl(
 			heap,
 			reinterpret_cast<const char*>(field), len);
-		if (strstr(table_name, "/" TEMP_FILE_PREFIX_INNODB)) {
+		if (strstr(table_name, "/" TEMP_FILE_PREFIX "-")) {
 			btr_pcur_store_position(&pcur, &mtr);
 			btr_pcur_commit_specify_mtr(&pcur, &mtr);
 
@@ -3615,7 +3615,7 @@ row_drop_table_for_mysql(
 
 	if (table->n_foreign_key_checks_running > 0) {
 defer:
-		if (!strstr(table->name.m_name, "/" TEMP_FILE_PREFIX_INNODB)) {
+		if (!strstr(table->name.m_name, "/" TEMP_FILE_PREFIX)) {
 			heap = mem_heap_create(FN_REFLEN);
 			const char* tmp_name
 				= dict_mem_create_temporary_tablename(
@@ -4399,7 +4399,7 @@ row_rename_table_for_mysql(
 
 		goto funct_exit;
 
-	} else if (new_is_tmp) {
+	} else if (!old_is_tmp && new_is_tmp) {
 		/* MySQL is doing an ALTER TABLE command and it renames the
 		original table to a temporary table name. We want to preserve
 		the original foreign key constraint definitions despite the