10.1 additions for MDEV-12052 Shutdown crash presumably due to master thread activity

btr_defragment_thread(): Create the thread in the same place as other
threads. Do not invoke btr_defragment_shutdown(), because
row_drop_tables_for_mysql_in_background() in the master thread can still
keep invoking btr_defragment_remove_table().

logs_empty_and_mark_files_at_shutdown(): Wait for btr_defragment_thread()
to exit.

innobase_start_or_create_for_mysql(), innobase_shutdown_for_mysql():
Skip encryption and scrubbing in innodb_read_only_mode.

srv_export_innodb_status(): Do not export encryption or scrubbing
statistics in innodb_read_only mode, because the threads will not
be running.

storage/innobase/btr/btr0defragment.cc

@@ -154,7 +154,6 @@ btr_defragment_init()
 		1000000.0 / srv_defragment_frequency);
 	mutex_create(btr_defragment_mutex_key, &btr_defragment_mutex,
 		     SYNC_ANY_LATCH);
-	os_thread_create(btr_defragment_thread, NULL, NULL);
 }
 
 /******************************************************************//**
@@ -735,14 +734,13 @@ btr_defragment_n_pages(
 	return current_block;
 }
 
-/******************************************************************//**
-Thread that merges consecutive b-tree pages into fewer pages to defragment
-the index. */
+/** Whether btr_defragment_thread is active */
+bool btr_defragment_thread_active;
+
+/** Merge consecutive b-tree pages into fewer pages to defragment indexes */
 extern "C" UNIV_INTERN
 os_thread_ret_t
-DECLARE_THREAD(btr_defragment_thread)(
-/*==========================================*/
-	void*	arg)	/*!< in: work queue */
+DECLARE_THREAD(btr_defragment_thread)(void*)
 {
 	btr_pcur_t*	pcur;
 	btr_cur_t*	cursor;
@@ -752,6 +750,8 @@ DECLARE_THREAD(btr_defragment_thread)(
 	buf_block_t*	last_block;
 
 	while (srv_shutdown_state == SRV_SHUTDOWN_NONE) {
+		ut_ad(btr_defragment_thread_active);
+
 		/* If defragmentation is disabled, sleep before
 		checking whether it's enabled. */
 		if (!srv_defragment) {
@@ -825,9 +825,9 @@ DECLARE_THREAD(btr_defragment_thread)(
 			btr_defragment_remove_item(item);
 		}
 	}
-	btr_defragment_shutdown();
+
+	btr_defragment_thread_active = false;
 	os_thread_exit(NULL);
 	OS_THREAD_DUMMY_RETURN;
 }
-
 #endif /* !UNIV_HOTBACKUP */

storage/innobase/include/btr0defragment.h

 /*****************************************************************************
 
 Copyright (C) 2013, 2014 Facebook, Inc. All Rights Reserved.
-Copyright (C) 2014, 2015, MariaDB Corporation.
+Copyright (C) 2014, 2017, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -89,16 +89,14 @@ UNIV_INTERN
 void
 btr_defragment_save_defrag_stats_if_needed(
 	dict_index_t*	index);	/*!< in: index */
-/******************************************************************//**
-Thread that merges consecutive b-tree pages into fewer pages to defragment
-the index. */
+
+/** Merge consecutive b-tree pages into fewer pages to defragment indexes */
 extern "C" UNIV_INTERN
 os_thread_ret_t
-DECLARE_THREAD(btr_defragment_thread)(
-/*==========================================*/
-	void*	arg);		/*!< in: a dummy parameter required by
-				os_thread_create */
+DECLARE_THREAD(btr_defragment_thread)(void*);
 
+/** Whether btr_defragment_thread is active */
+extern bool btr_defragment_thread_active;
 
 #endif /* !UNIV_HOTBACKUP */
 #endif

storage/innobase/log/log0log.cc

@@ -45,12 +45,13 @@ Created 12/9/1995 Heikki Tuuri
 #include "mem0mem.h"
 #include "buf0buf.h"
 #include "buf0flu.h"
-#include "srv0srv.h"
 #include "lock0lock.h"
 #include "log0recv.h"
 #include "fil0fil.h"
 #include "dict0boot.h"
-#include "dict0stats_bg.h" /* dict_stats_event */
+#include "dict0stats_bg.h"
+#include "btr0defragment.h"
+#include "srv0srv.h"
 #include "srv0start.h"
 #include "trx0sys.h"
 #include "trx0trx.h"
@@ -3300,6 +3301,8 @@ logs_empty_and_mark_files_at_shutdown(void)
 		thread_name = "lock_wait_timeout_thread";
 	} else if (srv_buf_dump_thread_active) {
 		thread_name = "buf_dump_thread";
+	} else if (btr_defragment_thread_active) {
+		thread_name = "btr_defragment_thread";
 	} else if (srv_fast_shutdown != 2 && trx_rollback_or_clean_is_active) {
 		thread_name = "rollback of recovered transactions";
 	} else {

storage/innobase/srv/srv0srv.cc

@@ -1457,8 +1457,10 @@ srv_export_innodb_status(void)
 	buf_get_total_stat(&stat);
 	buf_get_total_list_len(&LRU_len, &free_len, &flush_list_len);
 	buf_get_total_list_size_in_bytes(&buf_pools_list_size);
+	if (!srv_read_only_mode) {
 		fil_crypt_total_stat(&crypt_stat);
 		btr_scrub_total_stat(&scrub_stat);
+	}
 
 	mutex_enter(&srv_innodb_monitor_mutex);
 
@@ -1660,6 +1662,7 @@ srv_export_innodb_status(void)
 	export_vars.innodb_sec_rec_cluster_reads_avoided =
 		srv_stats.n_sec_rec_cluster_reads_avoided;
 
+	if (!srv_read_only_mode) {
 	export_vars.innodb_encryption_rotation_pages_read_from_cache =
 		crypt_stat.pages_read_from_cache;
 	export_vars.innodb_encryption_rotation_pages_read_from_disk =
@@ -1687,6 +1690,7 @@ srv_export_innodb_status(void)
 		scrub_stat.page_split_failures_missing_index;
 	export_vars.innodb_scrub_page_split_failures_unknown =
 		scrub_stat.page_split_failures_unknown;
+	}
 
 	mutex_exit(&srv_innodb_monitor_mutex);
 }

storage/innobase/srv/srv0start.cc

@@ -2981,13 +2981,15 @@ innobase_start_or_create_for_mysql(void)
 		fil_system_enter();
 		fil_crypt_threads_init();
 		fil_system_exit();
-	}
 
 		/* Init data for datafile scrub threads */
 		btr_scrub_init();
 
 		/* Initialize online defragmentation. */
 		btr_defragment_init();
+		btr_defragment_thread_active = true;
+		os_thread_create(btr_defragment_thread, NULL, NULL);
+	}
 
 	srv_was_started = TRUE;
 
@@ -3158,10 +3160,9 @@ innobase_shutdown_for_mysql(void)
 	if (!srv_read_only_mode) {
 		dict_stats_thread_deinit();
 		fil_crypt_threads_cleanup();
-	}
-
-	/* Cleanup data for datafile scrubbing */
 		btr_scrub_cleanup();
+		btr_defragment_shutdown();
+	}
 
 #ifdef __WIN__
 	/* MDEV-361: ha_innodb.dll leaks handles on Windows

storage/xtradb/btr/btr0defragment.cc

@@ -154,7 +154,6 @@ btr_defragment_init()
 		1000000.0 / srv_defragment_frequency);
 	mutex_create(btr_defragment_mutex_key, &btr_defragment_mutex,
 		     SYNC_ANY_LATCH);
-	os_thread_create(btr_defragment_thread, NULL, NULL);
 }
 
 /******************************************************************//**
@@ -735,14 +734,13 @@ btr_defragment_n_pages(
 	return current_block;
 }
 
-/******************************************************************//**
-Thread that merges consecutive b-tree pages into fewer pages to defragment
-the index. */
+/** Whether btr_defragment_thread is active */
+bool btr_defragment_thread_active;
+
+/** Merge consecutive b-tree pages into fewer pages to defragment indexes */
 extern "C" UNIV_INTERN
 os_thread_ret_t
-DECLARE_THREAD(btr_defragment_thread)(
-/*==========================================*/
-	void*	arg)	/*!< in: work queue */
+DECLARE_THREAD(btr_defragment_thread)(void*)
 {
 	btr_pcur_t*	pcur;
 	btr_cur_t*	cursor;
@@ -752,6 +750,8 @@ DECLARE_THREAD(btr_defragment_thread)(
 	buf_block_t*	last_block;
 
 	while (srv_shutdown_state == SRV_SHUTDOWN_NONE) {
+		ut_ad(btr_defragment_thread_active);
+
 		/* If defragmentation is disabled, sleep before
 		checking whether it's enabled. */
 		if (!srv_defragment) {
@@ -825,9 +825,9 @@ DECLARE_THREAD(btr_defragment_thread)(
 			btr_defragment_remove_item(item);
 		}
 	}
-	btr_defragment_shutdown();
+
+	btr_defragment_thread_active = false;
 	os_thread_exit(NULL);
 	OS_THREAD_DUMMY_RETURN;
 }
-
 #endif /* !UNIV_HOTBACKUP */

storage/xtradb/include/btr0defragment.h

 /*****************************************************************************
 
 Copyright (C) 2013, 2014 Facebook, Inc. All Rights Reserved.
-Copyright (C) 2014, 2015, MariaDB Corporation.
+Copyright (C) 2014, 2017, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -89,15 +89,14 @@ UNIV_INTERN
 void
 btr_defragment_save_defrag_stats_if_needed(
 	dict_index_t*	index);	/*!< in: index */
-/******************************************************************//**
-Thread that merges consecutive b-tree pages into fewer pages to defragment
-the index. */
+
+/** Merge consecutive b-tree pages into fewer pages to defragment indexes */
 extern "C" UNIV_INTERN
 os_thread_ret_t
-DECLARE_THREAD(btr_defragment_thread)(
-/*==========================================*/
-	void*	arg);		/*!< in: a dummy parameter required by
-				os_thread_create */
+DECLARE_THREAD(btr_defragment_thread)(void*);
+
+/** Whether btr_defragment_thread is active */
+extern bool btr_defragment_thread_active;
 
 #endif /* !UNIV_HOTBACKUP */
 #endif

storage/xtradb/log/log0log.cc

@@ -55,12 +55,13 @@ Created 12/9/1995 Heikki Tuuri
 #include "mem0mem.h"
 #include "buf0buf.h"
 #include "buf0flu.h"
-#include "srv0srv.h"
 #include "lock0lock.h"
 #include "log0recv.h"
 #include "fil0fil.h"
 #include "dict0boot.h"
-#include "dict0stats_bg.h" /* dict_stats_event */
+#include "dict0stats_bg.h"
+#include "dict0stats_bg.h"
+#include "btr0defragment.h"
 #include "srv0srv.h"
 #include "srv0start.h"
 #include "trx0sys.h"
@@ -3618,6 +3619,8 @@ logs_empty_and_mark_files_at_shutdown(void)
 		thread_name = "lock_wait_timeout_thread";
 	} else if (srv_buf_dump_thread_active) {
 		thread_name = "buf_dump_thread";
+	} else if (btr_defragment_thread_active) {
+		thread_name = "btr_defragment_thread";
 	} else if (srv_fast_shutdown != 2 && trx_rollback_or_clean_is_active) {
 		thread_name = "rollback of recovered transactions";
 	} else {

storage/xtradb/srv/srv0srv.cc

@@ -1784,8 +1784,10 @@ srv_export_innodb_status(void)
 	buf_get_total_stat(&stat);
 	buf_get_total_list_len(&LRU_len, &free_len, &flush_list_len);
 	buf_get_total_list_size_in_bytes(&buf_pools_list_size);
+	if (!srv_read_only_mode) {
 		fil_crypt_total_stat(&crypt_stat);
 		btr_scrub_total_stat(&scrub_stat);
+	}
 
 	mem_adaptive_hash = 0;
 
@@ -2099,6 +2101,7 @@ srv_export_innodb_status(void)
 	export_vars.innodb_sec_rec_cluster_reads_avoided =
 		srv_stats.n_sec_rec_cluster_reads_avoided;
 
+	if (!srv_read_only_mode) {
 	export_vars.innodb_encryption_rotation_pages_read_from_cache =
 		crypt_stat.pages_read_from_cache;
 	export_vars.innodb_encryption_rotation_pages_read_from_disk =
@@ -2126,6 +2129,7 @@ srv_export_innodb_status(void)
 		scrub_stat.page_split_failures_missing_index;
 	export_vars.innodb_scrub_page_split_failures_unknown =
 		scrub_stat.page_split_failures_unknown;
+	}
 
 	mutex_exit(&srv_innodb_monitor_mutex);
 }

storage/xtradb/srv/srv0start.cc

@@ -3089,13 +3089,15 @@ innobase_start_or_create_for_mysql(void)
 		fil_system_enter();
 		fil_crypt_threads_init();
 		fil_system_exit();
-	}
 
 		/* Init data for datafile scrub threads */
 		btr_scrub_init();
 
 		/* Initialize online defragmentation. */
 		btr_defragment_init();
+		btr_defragment_thread_active = true;
+		os_thread_create(btr_defragment_thread, NULL, NULL);
+	}
 
 	srv_was_started = TRUE;
 
@@ -3260,10 +3262,9 @@ innobase_shutdown_for_mysql(void)
 	if (!srv_read_only_mode) {
 		dict_stats_thread_deinit();
 		fil_crypt_threads_cleanup();
-	}
-
-	/* Cleanup data for datafile scrubbing */
 		btr_scrub_cleanup();
+		btr_defragment_shutdown();
+	}
 
 #ifdef __WIN__
 	/* MDEV-361: ha_innodb.dll leaks handles on Windows