Bug #23070734 CONCURRENT TRUNCATE TABLES CAUSE STALLS

PROBLEM

When truncating single tablespace tables, we need to scan the entire
buffer pool to remove the pages of the table from the buffer pool.
During this scan and removal dict_sys->mutex is being held ,causing
stalls in other DDL operations.

FIX

Release the dict_sys->mutex during the scan and reacquire it after the
scan. Make sure that purge thread doesn't purge the records of the table
being truncated and background stats collection thread skips the updation
of stats for the table being truncated.

[#rb 14564 Approved by Jimmy and satya ]

mysql-test/suite/innodb/r/innodb-truncate-debug.result

+#
+# Bug #23070734	CONCURRENT TRUNCATE TABLES CAUSE STALLS
+#
+Test_1 :- Check if DDL operations are possible on
+table being truncated. Also check if
+DDL operations on other tables succeed.
+create table t1 (f1 int,f2 int,key(f2),f3 int) engine=innodb;
+create index idx1 on t1(f3);
+create table t2 (f1 int,f2 int,key(f2),f3 int) engine=innodb;
+create table t3 (f1 int,f2 int,key(f2)) engine=innodb;
+insert into t1 values (10,20,30),(30,40,50),(50,60,70);
+insert into t1 select * from t1;
+insert into t1 select * from t1;
+insert into t1 select * from t1;
+insert into t2 values (10,20,30),(30,40,50),(50,60,70);
+insert into t2 select * from t2;
+insert into t2 select * from t2;
+insert into t2 select * from t2;
+insert into t3 values (10,20),(30,40),(50,50);
+insert into t3 select * from t3;
+insert into t3 select * from t3;
+SET session lock_wait_timeout = 1;
+show variables like 'lock_wait_timeout%';
+Variable_name	Value
+lock_wait_timeout	1
+connection 1
+SET DEBUG_SYNC= 'simulate_buffer_pool_scan SIGNAL opened WAIT_FOR finish_scan';
+truncate table t1;;
+connection default
+SET DEBUG_SYNC= 'now WAIT_FOR opened';
+Check Analyze table. Gives lock time out error.
+analyze table t1;
+Table	Op	Msg_type	Msg_text
+test.t1	analyze	Error	Lock wait timeout exceeded; try restarting transaction
+test.t1	analyze	status	Operation failed
+Check if we can turn off auto recalculation.
+alter table t1 STATS_AUTO_RECALC=0;
+ERROR HY000: Lock wait timeout exceeded; try restarting transaction
+Check if we can turn off persistent stats on the table
+alter table t1 STATS_PERSISTENT=0;
+ERROR HY000: Lock wait timeout exceeded; try restarting transaction
+Check if DML is possible on table being truncated
+insert into t1 values (10,89,99);
+ERROR HY000: Lock wait timeout exceeded; try restarting transaction
+Check if DDL is possible on table being truncated
+alter table t1 add column f4 int;
+ERROR HY000: Lock wait timeout exceeded; try restarting transaction
+check if table can be created with the same name
+create table t1 (bd int) engine=innodb;
+ERROR HY000: Lock wait timeout exceeded; try restarting transaction
+check if index can be created on table being truncated
+create index idx1 on t1(f1);
+ERROR HY000: Lock wait timeout exceeded; try restarting transaction
+Check if DROP of table is possible during truncate
+drop table t1;
+ERROR HY000: Lock wait timeout exceeded; try restarting transaction
+Check if select is possible during truncate
+select * from t1;
+ERROR HY000: Lock wait timeout exceeded; try restarting transaction
+select * from t2 where t2.f1=t1.f1;
+ERROR 42S22: Unknown column 't1.f1' in 'where clause'
+Check concurrent truncate operation on table;
+truncate table t1;
+ERROR HY000: Lock wait timeout exceeded; try restarting transaction
+Check concurrent selects  and inserts on the other table
+insert into t3 values (10,20),(30,40),(50,50);
+select * from t3 where f1=40;
+f1	f2
+Concurrent truncate on other tables
+truncate table t2;
+Concurrent alters on the other tables
+alter table t2 add column f4 int;
+check if index can be created on the other table
+create index idx1 on t2(f3);
+Check if we can turn off persistent stats off entire instance
+SET GLOBAL innodb_stats_persistent=off;
+connection con2
+set global innodb_adaptive_hash_index=off;;
+connection default
+SET DEBUG_SYNC= 'now SIGNAL finish_scan';
+connection con1
+connection con2
+connection default
+SET session lock_wait_timeout=default;
+set global innodb_adaptive_hash_index=default;
+drop table t1,t2,t3;
+SET @@global.innodb_stats_persistent=default;
+Test_2 :- Check if purge thread ignores the undo  
+log records of the table being truncated.
+create table t1 (f1 int ,f2 int,key(f2)) engine=innodb;
+insert into t1 values (10,45),(20,78),(30,88),(40,23),(50,78),(60,11),(70,56),(80,90);
+insert into t1 select * from t1;
+insert into t1 select * from t1;
+insert into t1 select * from t1;
+insert into t1 select * from t1;
+insert into t1 select * from t1;
+insert into t1 select * from t1;
+insert into t1 select * from t1;
+insert into t1 select * from t1;
+insert into t1 select * from t1;
+insert into t1 select * from t1;
+insert into t1 select * from t1;
+insert into t1 select * from t1;
+insert into t1 select * from t1;
+insert into t1 select * from t1;
+select count(*) from t1;
+count(*)
+131072
+Stop the purge thread
+set global innodb_purge_stop_now = 1;
+delete from t1;
+connection con1
+SET DEBUG_SYNC= 'simulate_buffer_pool_scan SIGNAL opened WAIT_FOR finish_scan';
+truncate table t1;;
+Connection default
+SET DEBUG_SYNC= 'now WAIT_FOR opened';
+set global innodb_purge_run_now=1;
+SET DEBUG_SYNC= 'now SIGNAL finish_scan';
+connection con1
+connection default
+drop table t1;
+Pattern "InnoDB: Record with space id \d+ belongs to table which is being truncated therefore skipping this undo record." found
+# restart server
+# restart:

mysql-test/suite/innodb/t/innodb-truncate-debug.test

+--source include/have_innodb.inc
+--source include/have_debug.inc
+--source include/have_debug_sync.inc
+
+--echo #
+--echo # Bug #23070734	CONCURRENT TRUNCATE TABLES CAUSE STALLS
+--echo #
+
+--echo Test_1 :- Check if DDL operations are possible on
+--echo		 table being truncated. Also check if
+--echo		 DDL operations on other tables succeed.
+
+create table t1 (f1 int,f2 int,key(f2),f3 int) engine=innodb;
+create index idx1 on t1(f3);
+create table t2 (f1 int,f2 int,key(f2),f3 int) engine=innodb;
+create table t3 (f1 int,f2 int,key(f2)) engine=innodb;
+
+insert into t1 values (10,20,30),(30,40,50),(50,60,70);
+insert into t1 select * from t1;
+insert into t1 select * from t1;
+insert into t1 select * from t1;
+insert into t2 values (10,20,30),(30,40,50),(50,60,70);
+
+insert into t2 select * from t2;
+insert into t2 select * from t2;
+insert into t2 select * from t2;
+
+insert into t3 values (10,20),(30,40),(50,50);
+insert into t3 select * from t3;
+insert into t3 select * from t3;
+
+SET session lock_wait_timeout = 1;
+show variables like 'lock_wait_timeout%';
+
+connect (con1,localhost,root,,);
+connect (con2,localhost,root,,);
+
+--echo connection 1
+connection con1;
+SET DEBUG_SYNC= 'simulate_buffer_pool_scan SIGNAL opened WAIT_FOR finish_scan';
+--send truncate table t1;
+
+--echo connection default
+connection default;
+SET DEBUG_SYNC= 'now WAIT_FOR opened';
+
+--echo Check Analyze table. Gives lock time out error.
+analyze table t1;
+
+--echo Check if we can turn off auto recalculation.
+--error ER_LOCK_WAIT_TIMEOUT
+alter table t1 STATS_AUTO_RECALC=0;
+
+--echo Check if we can turn off persistent stats on the table
+--error ER_LOCK_WAIT_TIMEOUT
+alter table t1 STATS_PERSISTENT=0;
+
+--echo Check if DML is possible on table being truncated
+--error ER_LOCK_WAIT_TIMEOUT
+insert into t1 values (10,89,99);
+
+--echo Check if DDL is possible on table being truncated
+--error ER_LOCK_WAIT_TIMEOUT
+alter table t1 add column f4 int;
+
+--echo check if table can be created with the same name
+--error ER_LOCK_WAIT_TIMEOUT
+create table t1 (bd int) engine=innodb;
+
+--echo check if index can be created on table being truncated
+--error ER_LOCK_WAIT_TIMEOUT
+create index idx1 on t1(f1);
+
+--echo Check if DROP of table is possible during truncate
+--error ER_LOCK_WAIT_TIMEOUT
+drop table t1;
+
+--echo Check if select is possible during truncate
+--error ER_LOCK_WAIT_TIMEOUT
+select * from t1;
+
+--error ER_BAD_FIELD_ERROR
+select * from t2 where t2.f1=t1.f1;
+
+--echo Check concurrent truncate operation on table;
+--error ER_LOCK_WAIT_TIMEOUT
+truncate table t1;
+
+--echo Check concurrent selects  and inserts on the other table
+insert into t3 values (10,20),(30,40),(50,50);
+select * from t3 where f1=40;
+
+--echo Concurrent truncate on other tables
+truncate table t2;
+
+--echo Concurrent alters on the other tables
+alter table t2 add column f4 int;
+
+--echo check if index can be created on the other table
+create index idx1 on t2(f3);
+
+
+--echo Check if we can turn off persistent stats off entire instance
+SET GLOBAL innodb_stats_persistent=off;
+
+--echo connection con2
+connection con2;
+--send set global innodb_adaptive_hash_index=off;
+
+--echo connection default
+connection default;
+SET DEBUG_SYNC= 'now SIGNAL finish_scan';
+
+--echo connection con1
+connection con1;
+reap;
+
+--echo connection con2
+connection con2;
+reap;
+
+--echo connection default
+connection default;
+disconnect con1;
+disconnect con2;
+
+SET session lock_wait_timeout=default;
+set global innodb_adaptive_hash_index=default;
+
+drop table t1,t2,t3;
+SET @@global.innodb_stats_persistent=default;
+
+
+--echo Test_2 :- Check if purge thread ignores the undo  
+--echo		 log records of the table being truncated.
+
+let $restart_parameters = restart: --innodb_purge_threads=1 --innodb_purge_batch_size=1 --innodb_stats_persistent=OFF
+--source include/restart_mysqld.inc;
+
+create table t1 (f1 int ,f2 int,key(f2)) engine=innodb;
+insert into t1 values (10,45),(20,78),(30,88),(40,23),(50,78),(60,11),(70,56),(80,90);
+insert into t1 select * from t1;
+insert into t1 select * from t1;
+insert into t1 select * from t1;
+insert into t1 select * from t1;
+insert into t1 select * from t1;
+insert into t1 select * from t1;
+insert into t1 select * from t1;
+insert into t1 select * from t1;
+insert into t1 select * from t1;
+insert into t1 select * from t1;
+insert into t1 select * from t1;
+insert into t1 select * from t1;
+insert into t1 select * from t1;
+insert into t1 select * from t1;
+select count(*) from t1;
+
+--echo Stop the purge thread
+set global innodb_purge_stop_now = 1;
+delete from t1;
+
+connect (con1,localhost,root,,);
+
+--echo connection con1
+connection con1;
+SET DEBUG_SYNC= 'simulate_buffer_pool_scan SIGNAL opened WAIT_FOR finish_scan';
+--send truncate table t1;
+
+--echo Connection default
+connection default;
+
+SET DEBUG_SYNC= 'now WAIT_FOR opened';
+set global innodb_purge_run_now=1;
+--sleep 60
+SET DEBUG_SYNC= 'now SIGNAL finish_scan';
+
+--echo connection con1
+connection con1;
+reap;
+
+--echo connection default
+connection default;
+disconnect con1;
+
+drop table t1;
+
+let SEARCH_FILE = $MYSQLTEST_VARDIR/log/mysqld.1.err;
+let SEARCH_PATTERN = InnoDB: Record with space id \d+ belongs to table which is being truncated therefore skipping this undo record.;
+--source include/search_pattern.inc
+
+#cleanup
+--echo # restart server
+let $restart_parameters = restart:;
+--source include/restart_mysqld.inc

storage/innobase/dict/dict0stats_bg.cc

 /*****************************************************************************
 
-Copyright (c) 2012, 2016, Oracle and/or its affiliates. All Rights Reserved.
-Copyright (c) 2017, MariaDB Corporation. All Rights Reserved.
+Copyright (c) 2012, 2017, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2017, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -31,6 +31,7 @@ Created Apr 25, 2012 Vasil Dimov
 #include "row0mysql.h"
 #include "srv0start.h"
 #include "ut0new.h"
+#include "fil0fil.h"
 
 #include <vector>
 
@@ -318,6 +319,12 @@ dict_stats_process_entry_from_recalc_pool()
 		return;
 	}
 
+	if (fil_space_is_being_truncated(table->space)) {
+		dict_table_close(table, TRUE, FALSE);
+		mutex_exit(&dict_sys->mutex);
+		return;
+	}
+
 	/* Check whether table is corrupted */
 	if (table->corrupted) {
 		dict_table_close(table, TRUE, FALSE);

storage/innobase/fil/fil0fil.cc

 /*****************************************************************************
 
-Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 1995, 2017, Oracle and/or its affiliates. All Rights Reserved.
 Copyright (c) 2014, 2017, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
@@ -30,6 +30,7 @@ Created 10/25/1995 Heikki Tuuri
 #include "fil0crypt.h"
 
 #include "btr0btr.h"
+#include "btr0sea.h"
 #include "buf0buf.h"
 #include "dict0boot.h"
 #include "dict0dict.h"
@@ -55,9 +56,6 @@ Created 10/25/1995 Heikki Tuuri
 #include "os0event.h"
 #include "sync0sync.h"
 #include "buf0flu.h"
-#include "srv0start.h"
-#include "trx0purge.h"
-#include "ut0new.h"
 #include "os0api.h"
 
 /** Tries to close a file in the LRU list. The caller must hold the fil_sys
@@ -3266,20 +3264,34 @@ fil_prepare_for_truncate(
 	return(err);
 }
 
-/**********************************************************************//**
-Reinitialize the original tablespace header with the same space id
-for single tablespace */
+/** Reinitialize the original tablespace header with the same space id
+for single tablespace
+@param[in]      id              space id of the tablespace
+@param[in]      size            size in blocks
+@param[in]      trx             Transaction covering truncate */
 void
 fil_reinit_space_header(
-/*====================*/
-	ulint		id,	/*!< in: space id */
-	ulint		size)	/*!< in: size in blocks */
+	ulint		id,
+	ulint		size,
+	trx_t*		trx)
 {
 	ut_a(!is_system_tablespace(id));
 
 	/* Invalidate in the buffer pool all pages belonging
-	to the tablespace */
+	to the tablespace. The buffer pool scan may take long
+	time to complete, therefore we release dict_sys->mutex
+	and the dict operation lock during the scan and aquire
+	it again after the buffer pool scan.*/
+
+	row_mysql_unlock_data_dictionary(trx);
+
+	/* Lock the search latch in shared mode to prevent user
+	from disabling AHI during the scan */
+	btr_search_s_lock_all();
+	DEBUG_SYNC_C("simulate_buffer_pool_scan");
 	buf_LRU_flush_or_remove_pages(id, BUF_REMOVE_ALL_NO_WRITE, 0);
+	btr_search_s_unlock_all();
+	row_mysql_lock_data_dictionary(trx);
 
 	/* Remove all insert buffer entries for the tablespace */
 	ibuf_delete_for_discarded_space(id);

storage/innobase/include/fil0fil.h

 /*****************************************************************************
 
-Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 1995, 2017, Oracle and/or its affiliates. All Rights Reserved.
 Copyright (c) 2013, 2017, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
@@ -914,14 +914,18 @@ dberr_t
 fil_prepare_for_truncate(
 /*=====================*/
 	ulint	id);			/*!< in: space id */
-/**********************************************************************//**
-Reinitialize the original tablespace header with the same space id
-for single tablespace */
+
+/** Reinitialize the original tablespace header with the same space id
+for single tablespace
+@param[in]	id		space id of the tablespace
+@param[in]	size            size in blocks
+@param[in]	trx		Transaction covering truncate */
 void
 fil_reinit_space_header(
-/*====================*/
-	ulint		id,	/*!< in: space id */
-	ulint		size);	/*!< in: size in blocks */
+	ulint		id,
+	ulint		size,
+	trx_t*		trx);
+
 /*******************************************************************//**
 Closes a single-table tablespace. The tablespace must be cached in the
 memory cache. Free all pages used by the tablespace.

storage/innobase/row/row0purge.cc

 /*****************************************************************************
 
-Copyright (c) 1997, 2016, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 1997, 2017, Oracle and/or its affiliates. All Rights Reserved.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -43,6 +43,7 @@ Created 3/14/1997 Heikki Tuuri
 #include "srv0start.h"
 #include "handler.h"
 #include "ha_innodb.h"
+#include "fil0fil.h"
 
 /*************************************************************************
 IMPORTANT NOTE: Any operation that generates redo MUST check that there
@@ -854,6 +855,20 @@ row_purge_parse_undo_rec(
 	}
 	ut_ad(!dict_table_is_temporary(node->table));
 
+	if (fil_space_is_being_truncated(node->table->space)) {
+
+#if UNIV_DEBUG
+		ib::info() << "Record with space id "
+			   << node->table->space
+			   << " belongs to table which is being truncated"
+			   << " therefore skipping this undo record.";
+#endif
+		ut_ad(dict_table_is_file_per_table(node->table));
+		dict_table_close(node->table, FALSE, FALSE);
+		node->table = NULL;
+		goto err_exit;
+	}
+
 	if (node->table->n_v_cols && !node->table->vc_templ
 	    && dict_table_has_indexed_v_cols(node->table)) {
 		/* Need server fully up for virtual column computation */

storage/innobase/row/row0trunc.cc

 /*****************************************************************************
 
-Copyright (c) 2013, 2016, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2013, 2017, Oracle and/or its affiliates. All Rights Reserved.
 Copyright (c) 2017, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
@@ -2015,7 +2015,7 @@ row_truncate_table_for_mysql(
 			space_size -= ib_vector_size(table->fts->indexes);
 		}
 
-		fil_reinit_space_header(table->space, space_size);
+		fil_reinit_space_header(table->space, space_size, trx);
 	}
 
 	DBUG_EXECUTE_IF("ib_trunc_crash_with_intermediate_log_checkpoint",

storage/innobase/row/row0uins.cc

 /*****************************************************************************
 
-Copyright (c) 1997, 2016, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 1997, 2017, Oracle and/or its affiliates. All Rights Reserved.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -41,6 +41,7 @@ Created 2/25/1997 Heikki Tuuri
 #include "que0que.h"
 #include "ibuf0ibuf.h"
 #include "log0log.h"
+#include "fil0fil.h"
 
 /*************************************************************************
 IMPORTANT NOTE: Any operation that generates redo MUST check that there
@@ -347,6 +348,10 @@ row_undo_ins_parse_undo_rec(
 	if (UNIV_UNLIKELY(node->table == NULL)) {
 	} else if (UNIV_UNLIKELY(node->table->ibd_file_missing)) {
 close_table:
+		dict_table_close(node->table, dict_locked, FALSE);
+		node->table = NULL;
+	} else if (fil_space_is_being_truncated(node->table->space)) {
+
 		dict_table_close(node->table, dict_locked, FALSE);
 		node->table = NULL;
 	} else {

storage/innobase/row/row0umod.cc

 /*****************************************************************************
 
-Copyright (c) 1997, 2016, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 1997, 2017, Oracle and/or its affiliates. All Rights Reserved.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -1139,12 +1139,10 @@ row_undo_mod_parse_undo_rec(
 		return;
 	}
 
-	if (node->table->ibd_file_missing) {
+	if (node->table->ibd_file_missing ||
+	    fil_space_is_being_truncated(node->table->space) ) {
 		dict_table_close(node->table, dict_locked, FALSE);
-
-		/* We skip undo operations to missing .ibd files */
 		node->table = NULL;
-
 		return;
 	}