Merge mysql-5.1-innodb -> mysql-5.1-bugteam

mysql-test/Makefile.am

@@ -100,7 +100,8 @@ TEST_DIRS = t r include std_data std_data/parts collections \
 	suite/rpl_ndb suite/rpl_ndb/t suite/rpl_ndb/r \
 	suite/parts suite/parts/t suite/parts/r suite/parts/inc \
 	suite/innodb suite/innodb/t suite/innodb/r suite/innodb/include \
-	suite/innodb_plugin suite/innodb_plugin/t suite/innodb_plugin/r suite/innodb_plugin/include \
+	suite/innodb_plugin suite/innodb_plugin/t suite/innodb_plugin/r \
+	suite/innodb_plugin/include \
 	suite/engines suite/engines/funcs suite/engines/iuds suite/engines/rr_trx \
 	suite/engines/funcs/r suite/engines/funcs/t suite/engines/iuds/r \
 	suite/engines/iuds/t suite/engines/rr_trx/include suite/engines/rr_trx/r \

mysql-test/suite/innodb/r/innodb_bug53756.result

+DROP TABLE IF EXISTS bug_53756 ;
+CREATE TABLE bug_53756 (pk INT, c1 INT) ENGINE=InnoDB;
+ALTER TABLE bug_53756 ADD PRIMARY KEY (pk);
+INSERT INTO bug_53756 VALUES(1, 11), (2, 22), (3, 33), (4, 44);
+
+# Select a less restrictive isolation level.
+SET GLOBAL TRANSACTION ISOLATION LEVEL READ COMMITTED;
+SET SESSION TRANSACTION ISOLATION LEVEL READ COMMITTED;
+COMMIT;
+
+# Start a transaction in the default connection for isolation.
+START TRANSACTION;
+SELECT @@tx_isolation;
+@@tx_isolation
+READ-COMMITTED
+SELECT * FROM bug_53756;
+pk	c1
+1	11
+2	22
+3	33
+4	44
+
+# connection con1 deletes row 1
+START TRANSACTION;
+SELECT @@tx_isolation;
+@@tx_isolation
+READ-COMMITTED
+DELETE FROM bug_53756 WHERE pk=1;
+
+# connection con2 deletes row 2
+START TRANSACTION;
+SELECT @@tx_isolation;
+@@tx_isolation
+READ-COMMITTED
+DELETE FROM bug_53756 WHERE pk=2;
+
+# connection con3 updates row 3
+START TRANSACTION;
+SELECT @@tx_isolation;
+@@tx_isolation
+READ-COMMITTED
+UPDATE bug_53756 SET c1=77 WHERE pk=3;
+
+# connection con4 updates row 4
+START TRANSACTION;
+SELECT @@tx_isolation;
+@@tx_isolation
+READ-COMMITTED
+UPDATE bug_53756 SET c1=88 WHERE pk=4;
+
+# connection con5 inserts row 5
+START TRANSACTION;
+SELECT @@tx_isolation;
+@@tx_isolation
+READ-COMMITTED
+INSERT INTO bug_53756 VALUES(5, 55);
+
+# connection con6 inserts row 6
+START TRANSACTION;
+SELECT @@tx_isolation;
+@@tx_isolation
+READ-COMMITTED
+INSERT INTO bug_53756 VALUES(6, 66);
+
+# connection con1 commits.
+COMMIT;
+
+# connection con3 commits.
+COMMIT;
+
+# connection con4 rolls back.
+ROLLBACK;
+
+# connection con6 rolls back.
+ROLLBACK;
+
+# The connections 2 and 5 stay open.
+
+# connection default selects resulting data.
+# Delete of row 1 was committed.
+# Update of row 3 was committed.
+# Due to isolation level read committed, these should be included.
+# All other changes should not be included.
+SELECT * FROM bug_53756;
+pk	c1
+2	22
+3	77
+4	44
+
+# connection default
+#
+# Crash server.
+START TRANSACTION;
+INSERT INTO bug_53756 VALUES (666,666);
+SET SESSION debug="+d,crash_commit_before";
+COMMIT;
+ERROR HY000: Lost connection to MySQL server during query
+
+#
+# disconnect con1, con2, con3, con4, con5, con6.
+#
+# Restart server.
+
+#
+# Select recovered data.
+# Delete of row 1 was committed.
+# Update of row 3 was committed.
+# These should be included.
+# All other changes should not be included.
+# Delete of row 2 and insert of row 5 should be rolled back
+SELECT * FROM bug_53756;
+pk	c1
+2	22
+3	77
+4	44
+
+# Clean up.
+DROP TABLE bug_53756;

mysql-test/suite/innodb/t/innodb_bug53756-master.opt

+--skip-stack-trace --skip-core-file

mysql-test/suite/innodb/t/innodb_bug53756.test

+# This is the test case for bug #53756. Alter table operation could
+# leave a deleted record for the temp table (later renamed to the altered
+# table) in the SYS_TABLES secondary index, we should ignore this row and
+# find the first non-deleted row for the specified table_id when load table
+# metadata in the function dict_load_table_on_id() during crash recovery.
+
+#
+# innobackup needs to connect to the server. Not supported in embedded.
+--source include/not_embedded.inc
+#
+# This test case needs to crash the server. Needs a debug server.
+--source include/have_debug.inc
+#
+# Don't test this under valgrind, memory leaks will occur.
+--source include/not_valgrind.inc
+#
+# This test case needs InnoDB.
+--source include/have_innodb.inc
+
+#
+# Precautionary clean up.
+#
+--disable_warnings
+DROP TABLE IF EXISTS bug_53756 ;
+--enable_warnings
+
+#
+# Create test data.
+#
+CREATE TABLE bug_53756 (pk INT, c1 INT) ENGINE=InnoDB;
+ALTER TABLE bug_53756 ADD PRIMARY KEY (pk);
+INSERT INTO bug_53756 VALUES(1, 11), (2, 22), (3, 33), (4, 44);
+
+--echo
+--echo # Select a less restrictive isolation level.
+# Don't use user variables. They won't survive server crash.
+--let $global_isolation= `SELECT @@global.tx_isolation`;
+--let $session_isolation= `SELECT @@session.tx_isolation`;
+SET GLOBAL TRANSACTION ISOLATION LEVEL READ COMMITTED;
+SET SESSION TRANSACTION ISOLATION LEVEL READ COMMITTED;
+COMMIT;
+
+--echo
+--echo # Start a transaction in the default connection for isolation.
+START TRANSACTION;
+SELECT @@tx_isolation;
+SELECT * FROM bug_53756;
+
+--echo
+--echo # connection con1 deletes row 1
+--connect (con1,localhost,root,,)
+START TRANSACTION;
+SELECT @@tx_isolation;
+DELETE FROM bug_53756 WHERE pk=1;
+
+--echo
+--echo # connection con2 deletes row 2
+--connect (con2,localhost,root,,)
+START TRANSACTION;
+SELECT @@tx_isolation;
+DELETE FROM bug_53756 WHERE pk=2;
+
+--echo
+--echo # connection con3 updates row 3
+--connect (con3,localhost,root,,)
+START TRANSACTION;
+SELECT @@tx_isolation;
+UPDATE bug_53756 SET c1=77 WHERE pk=3;
+
+--echo
+--echo # connection con4 updates row 4
+--connect (con4,localhost,root,,)
+START TRANSACTION;
+SELECT @@tx_isolation;
+UPDATE bug_53756 SET c1=88 WHERE pk=4;
+
+--echo
+--echo # connection con5 inserts row 5
+--connect (con5,localhost,root,,)
+START TRANSACTION;
+SELECT @@tx_isolation;
+INSERT INTO bug_53756 VALUES(5, 55);
+
+--echo
+--echo # connection con6 inserts row 6
+--connect (con6,localhost,root,,)
+START TRANSACTION;
+SELECT @@tx_isolation;
+INSERT INTO bug_53756 VALUES(6, 66);
+
+--echo
+--echo # connection con1 commits.
+--connection con1
+COMMIT;
+
+--echo
+--echo # connection con3 commits.
+--connection con3
+COMMIT;
+
+--echo
+--echo # connection con4 rolls back.
+--connection con4
+ROLLBACK;
+
+--echo
+--echo # connection con6 rolls back.
+--connection con6
+ROLLBACK;
+
+--echo
+--echo # The connections 2 and 5 stay open.
+
+--echo
+--echo # connection default selects resulting data.
+--echo # Delete of row 1 was committed.
+--echo # Update of row 3 was committed.
+--echo # Due to isolation level read committed, these should be included.
+--echo # All other changes should not be included.
+--connection default
+SELECT * FROM bug_53756;
+
+--echo
+--echo # connection default
+--connection default
+--echo #
+--echo # Crash server.
+#
+# Write file to make mysql-test-run.pl expect the "crash", but don't start
+# it until it's told to
+--exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
+#
+START TRANSACTION;
+INSERT INTO bug_53756 VALUES (666,666);
+#
+# Request a crash on next execution of commit.
+SET SESSION debug="+d,crash_commit_before";
+#
+# Execute the statement that causes the crash.
+--error 2013
+COMMIT;
+--echo
+--echo #
+--echo # disconnect con1, con2, con3, con4, con5, con6.
+--disconnect con1
+--disconnect con2
+--disconnect con3
+--disconnect con4
+--disconnect con5
+--disconnect con6
+--echo #
+--echo # Restart server.
+#
+# Write file to make mysql-test-run.pl start up the server again
+--exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
+#
+# Turn on reconnect
+--enable_reconnect
+#
+# Call script that will poll the server waiting for it to be back online again
+--source include/wait_until_connected_again.inc
+#
+# Turn off reconnect again
+--disable_reconnect
+--echo
+
+--echo #
+--echo # Select recovered data.
+--echo # Delete of row 1 was committed.
+--echo # Update of row 3 was committed.
+--echo # These should be included.
+--echo # All other changes should not be included.
+--echo # Delete of row 2 and insert of row 5 should be rolled back
+SELECT * FROM bug_53756;
+
+--echo
+--echo # Clean up.
+DROP TABLE bug_53756;
+
+--disable_query_log
+eval SET GLOBAL tx_isolation= '$global_isolation';
+eval SET SESSION tx_isolation= '$session_isolation';
+--enable_query_log
+

storage/innobase/dict/dict0load.c

@@ -927,6 +927,8 @@ dict_load_table_on_id(
 
 	ut_ad(mutex_own(&(dict_sys->mutex)));
 
+	table = NULL;
+
 	/* NOTE that the operation of this function is protected by
 	the dictionary mutex, and therefore no deadlocks can occur
 	with other dictionary operations. */
@@ -953,15 +955,17 @@ dict_load_table_on_id(
 				  BTR_SEARCH_LEAF, &pcur, &mtr);
 	rec = btr_pcur_get_rec(&pcur);
 
-	if (!btr_pcur_is_on_user_rec(&pcur, &mtr)
-	    || rec_get_deleted_flag(rec, 0)) {
+	if (!btr_pcur_is_on_user_rec(&pcur, &mtr)) {
 		/* Not found */
+		goto func_exit;
+	}
 
-		btr_pcur_close(&pcur);
-		mtr_commit(&mtr);
-		mem_heap_free(heap);
-
-		return(NULL);
+	/* Find the first record that is not delete marked */
+	while (rec_get_deleted_flag(rec, 0)) {
+		if (!btr_pcur_move_to_next_user_rec(&pcur, &mtr)) {
+			goto func_exit;
+		}
+		rec = btr_pcur_get_rec(&pcur);
 	}
 
 	/*---------------------------------------------------*/
@@ -974,19 +978,14 @@ dict_load_table_on_id(
 
 	/* Check if the table id in record is the one searched for */
 	if (ut_dulint_cmp(table_id, mach_read_from_8(field)) != 0) {
-
-		btr_pcur_close(&pcur);
-		mtr_commit(&mtr);
-		mem_heap_free(heap);
-
-		return(NULL);
+		goto func_exit;
 	}
 
 	/* Now we get the table name from the record */
 	field = rec_get_nth_field_old(rec, 1, &len);
 	/* Load the table definition to memory */
 	table = dict_load_table(mem_heap_strdupl(heap, (char*) field, len));
-
+func_exit:
 	btr_pcur_close(&pcur);
 	mtr_commit(&mtr);
 	mem_heap_free(heap);

storage/innodb_plugin/ChangeLog

+2010-07-27	The InnoDB Team
+
+	* include/mem0pool.h, mem/mem0mem.c, mem/mem0pool.c, srv/srv0start.c:
+	Fix Bug#55581 shutdown with innodb-use-sys-malloc=0: assert
+	mutex->magic_n == MUTEX_MAGIC_N.
+
+2010-06-30	The InnoDB Team
+
+	* btr/btr0sea.c, ha/ha0ha.c, handler/ha_innodb.cc, include/btr0sea.h:
+	Fix Bug#54311 Crash on CHECK PARTITION after concurrent LOAD DATA
+	and adaptive_hash_index=OFF
+
+2010-06-29	The InnoDB Team
+	* row/row0row.c, row/row0undo.c, row/row0upd.c:
+	Fix Bug#54408 txn rollback after recovery: row0umod.c:673
+	dict_table_get_format(index->table)
+
+2010-06-29	The InnoDB Team
+
+	* btr/btr0cur.c, include/btr0cur.h,
+	include/row0mysql.h, row/row0merge.c, row/row0sel.c:
+	Fix Bug#54358 READ UNCOMMITTED access failure of off-page DYNAMIC
+	or COMPRESSED columns
+
 2010-06-24	The InnoDB Team
 
 	* handler/ha_innodb.cc:

storage/innodb_plugin/btr/btr0cur.c

@@ -4814,7 +4814,7 @@ btr_copy_externally_stored_field(
 
 /*******************************************************************//**
 Copies an externally stored field of a record to mem heap.
-@return	the field copied to heap */
+@return	the field copied to heap, or NULL if the field is incomplete */
 UNIV_INTERN
 byte*
 btr_rec_copy_externally_stored_field(
@@ -4844,6 +4844,18 @@ btr_rec_copy_externally_stored_field(
 
 	data = rec_get_nth_field(rec, offsets, no, &local_len);
 
+	ut_a(local_len >= BTR_EXTERN_FIELD_REF_SIZE);
+
+	if (UNIV_UNLIKELY
+	    (!memcmp(data + local_len - BTR_EXTERN_FIELD_REF_SIZE,
+		     field_ref_zero, BTR_EXTERN_FIELD_REF_SIZE))) {
+		/* The externally stored field was not written yet.
+		This record should only be seen by
+		recv_recovery_rollback_active() or any
+		TRX_ISO_READ_UNCOMMITTED transactions. */
+		return(NULL);
+	}
+
 	return(btr_copy_externally_stored_field(len, data,
 						zip_size, local_len, heap));
 }

storage/innodb_plugin/btr/btr0sea.c

@@ -46,6 +46,7 @@ Created 2/17/1996 Heikki Tuuri
 /** Flag: has the search system been enabled?
 Protected by btr_search_latch and btr_search_enabled_mutex. */
 UNIV_INTERN char		btr_search_enabled	= TRUE;
+UNIV_INTERN ibool		btr_search_fully_disabled = FALSE;
 
 /** Mutex protecting btr_search_enabled */
 static mutex_t			btr_search_enabled_mutex;
@@ -201,12 +202,19 @@ btr_search_disable(void)
 	mutex_enter(&btr_search_enabled_mutex);
 	rw_lock_x_lock(&btr_search_latch);
 
+	/* Disable access to hash index, also tell ha_insert_for_fold()
+	stop adding new nodes to hash index, but still allow updating
+	existing nodes */
 	btr_search_enabled = FALSE;
 
 	/* Clear all block->is_hashed flags and remove all entries
 	from btr_search_sys->hash_index. */
 	buf_pool_drop_hash_index();
 
+	/* hash index has been cleaned up, disallow any operation to
+	the hash index */
+	btr_search_fully_disabled = TRUE;
+
 	/* btr_search_enabled_mutex should guarantee this. */
 	ut_ad(!btr_search_enabled);
 
@@ -225,6 +233,7 @@ btr_search_enable(void)
 	rw_lock_x_lock(&btr_search_latch);
 
 	btr_search_enabled = TRUE;
+	btr_search_fully_disabled = FALSE;
 
 	rw_lock_x_unlock(&btr_search_latch);
 	mutex_exit(&btr_search_enabled_mutex);
@@ -1363,7 +1372,7 @@ btr_search_build_page_hash_index(
 
 	rw_lock_x_lock(&btr_search_latch);
 
-	if (UNIV_UNLIKELY(!btr_search_enabled)) {
+	if (UNIV_UNLIKELY(btr_search_fully_disabled)) {
 		goto exit_func;
 	}
 

storage/innodb_plugin/ha/ha0ha.c

@@ -31,9 +31,7 @@ Created 8/22/1994 Heikki Tuuri
 #ifdef UNIV_DEBUG
 # include "buf0buf.h"
 #endif /* UNIV_DEBUG */
-#ifdef UNIV_SYNC_DEBUG
-# include "btr0sea.h"
-#endif /* UNIV_SYNC_DEBUG */
+#include "btr0sea.h"
 #include "page0page.h"
 
 /*************************************************************//**
@@ -127,7 +125,8 @@ ha_clear(
 /*************************************************************//**
 Inserts an entry into a hash table. If an entry with the same fold number
 is found, its node is updated to point to the new data, and no new node
-is inserted.
+is inserted. If btr_search_enabled is set to FALSE, we will only allow
+updating existing nodes, but no new node is allowed to be added.
 @return	TRUE if succeed, FALSE if no more memory could be allocated */
 UNIV_INTERN
 ibool
@@ -174,6 +173,7 @@ ha_insert_for_fold_func(
 				prev_block->n_pointers--;
 				block->n_pointers++;
 			}
+			ut_ad(!btr_search_fully_disabled);
 # endif /* !UNIV_HOTBACKUP */
 
 			prev_node->block = block;
@@ -186,6 +186,13 @@ ha_insert_for_fold_func(
 		prev_node = prev_node->next;
 	}
 
+	/* We are in the process of disabling hash index, do not add
+	new chain node */
+	if (!btr_search_enabled) {
+		ut_ad(!btr_search_fully_disabled);
+		return(TRUE);
+	}
+
 	/* We have to allocate a new chain node */
 
 	node = mem_heap_alloc(hash_get_heap(table, fold), sizeof(ha_node_t));

storage/innodb_plugin/handler/ha_innodb.cc

@@ -2270,6 +2270,7 @@ innobase_change_buffering_inited_ok:
 	/* Get the current high water mark format. */
 	innobase_file_format_check = (char*) trx_sys_file_format_max_get();
 
+	btr_search_fully_disabled = (!btr_search_enabled);
 	DBUG_RETURN(FALSE);
 error:
 	DBUG_RETURN(TRUE);

storage/innodb_plugin/include/btr0cur.h

@@ -570,7 +570,7 @@ btr_copy_externally_stored_field_prefix(
 	ulint		local_len);/*!< in: length of data, in bytes */
 /*******************************************************************//**
 Copies an externally stored field of a record to mem heap.
-@return	the field copied to heap */
+@return	the field copied to heap, or NULL if the field is incomplete */
 UNIV_INTERN
 byte*
 btr_rec_copy_externally_stored_field(

storage/innodb_plugin/include/btr0sea.h

@@ -190,7 +190,13 @@ btr_search_validate(void);
 
 /** Flag: has the search system been enabled?
 Protected by btr_search_latch and btr_search_enabled_mutex. */
-extern char btr_search_enabled;
+extern char	btr_search_enabled;
+
+/** Flag: whether the search system has completed its disabling process,
+It is set to TRUE right after buf_pool_drop_hash_index() in
+btr_search_disable(), indicating hash index entries are cleaned up.
+Protected by btr_search_latch and btr_search_enabled_mutex. */
+extern ibool	btr_search_fully_disabled;
 
 /** The search info struct in an index */
 struct btr_search_struct{

storage/innodb_plugin/include/mem0pool.h

@@ -100,18 +100,6 @@ mem_pool_get_reserved(
 /*==================*/
 	mem_pool_t*	pool);	/*!< in: memory pool */
 /********************************************************************//**
-Reserves the mem pool mutex. */
-UNIV_INTERN
-void
-mem_pool_mutex_enter(void);
-/*======================*/
-/********************************************************************//**
-Releases the mem pool mutex. */
-UNIV_INTERN
-void
-mem_pool_mutex_exit(void);
-/*=====================*/
-/********************************************************************//**
 Validates a memory pool.
 @return	TRUE if ok */
 UNIV_INTERN

storage/innodb_plugin/include/row0mysql.h

@@ -622,7 +622,11 @@ struct row_prebuilt_struct {
 					the secondary index, then this is
 					set to TRUE */
 	unsigned	templ_contains_blob:1;/*!< TRUE if the template contains
-					BLOB column(s) */
+					a column with DATA_BLOB ==
+					get_innobase_type_from_mysql_type();
+					not to be confused with InnoDB
+					externally stored columns
+					(VARCHAR can be off-page too) */
 	mysql_row_templ_t* mysql_template;/*!< template used to transform
 					rows fast between MySQL and Innobase
 					formats; memory for this template

storage/innodb_plugin/include/univ.i

@@ -46,7 +46,7 @@ Created 1/20/1994 Heikki Tuuri
 
 #define INNODB_VERSION_MAJOR	1
 #define INNODB_VERSION_MINOR	0
-#define INNODB_VERSION_BUGFIX	10
+#define INNODB_VERSION_BUGFIX	11
 
 /* The following is the InnoDB version as shown in
 SELECT plugin_version FROM information_schema.plugins;

storage/innodb_plugin/mem/mem0mem.c

@@ -367,7 +367,7 @@ mem_heap_create_block(
 	block->line = line;
 
 #ifdef MEM_PERIODIC_CHECK
-	mem_pool_mutex_enter();
+	mutex_enter(&(mem_comm_pool->mutex));
 
 	if (!mem_block_list_inited) {
 		mem_block_list_inited = TRUE;
@@ -376,7 +376,7 @@ mem_heap_create_block(
 
 	UT_LIST_ADD_LAST(mem_block_list, mem_block_list, block);
 
-	mem_pool_mutex_exit();
+	mutex_exit(&(mem_comm_pool->mutex));
 #endif
 	mem_block_set_len(block, len);
 	mem_block_set_type(block, type);
@@ -479,11 +479,11 @@ mem_heap_block_free(
 	UT_LIST_REMOVE(list, heap->base, block);
 
 #ifdef MEM_PERIODIC_CHECK
-	mem_pool_mutex_enter();
+	mutex_enter(&(mem_comm_pool->mutex));
 
 	UT_LIST_REMOVE(mem_block_list, mem_block_list, block);
 
-	mem_pool_mutex_exit();
+	mutex_exit(&(mem_comm_pool->mutex));
 #endif
 
 	ut_ad(heap->total_size >= block->len);
@@ -556,7 +556,7 @@ mem_validate_all_blocks(void)
 {
 	mem_block_t*	block;
 
-	mem_pool_mutex_enter();
+	mutex_enter(&(mem_comm_pool->mutex));
 
 	block = UT_LIST_GET_FIRST(mem_block_list);
 
@@ -568,6 +568,6 @@ mem_validate_all_blocks(void)
 		block = UT_LIST_GET_NEXT(mem_block_list, block);
 	}
 
-	mem_pool_mutex_exit();
+	mutex_exit(&(mem_comm_pool->mutex));
 }
 #endif

storage/innodb_plugin/mem/mem0pool.c

@@ -34,6 +34,7 @@ Created 5/12/1997 Heikki Tuuri
 #include "ut0lst.h"
 #include "ut0byte.h"
 #include "mem0mem.h"
+#include "srv0start.h"
 
 /* We would like to use also the buffer frames to allocate memory. This
 would be desirable, because then the memory consumption of the database
@@ -121,23 +122,33 @@ mysql@lists.mysql.com */
 UNIV_INTERN ulint	mem_n_threads_inside		= 0;
 
 /********************************************************************//**
-Reserves the mem pool mutex. */
-UNIV_INTERN
+Reserves the mem pool mutex if we are not in server shutdown. Use
+this function only in memory free functions, since only memory
+free functions are used during server shutdown. */
+UNIV_INLINE
 void
-mem_pool_mutex_enter(void)
-/*======================*/
+mem_pool_mutex_enter(
+/*=================*/
+	mem_pool_t*	pool)		/*!< in: memory pool */
 {
-	mutex_enter(&(mem_comm_pool->mutex));
+	if (srv_shutdown_state < SRV_SHUTDOWN_EXIT_THREADS) {
+		mutex_enter(&(pool->mutex));
+	}
 }
 
 /********************************************************************//**
-Releases the mem pool mutex. */
-UNIV_INTERN
+Releases the mem pool mutex if we are not in server shutdown. As
+its corresponding mem_pool_mutex_enter() function, use it only
+in memory free functions */
+UNIV_INLINE
 void
-mem_pool_mutex_exit(void)
-/*=====================*/
+mem_pool_mutex_exit(
+/*================*/
+	mem_pool_t*	pool)		/*!< in: memory pool */
 {
-	mutex_exit(&(mem_comm_pool->mutex));
+	if (srv_shutdown_state < SRV_SHUTDOWN_EXIT_THREADS) {
+		mutex_exit(&(pool->mutex));
+	}
 }
 
 /********************************************************************//**
@@ -567,7 +578,7 @@ mem_area_free(
 
 	n = ut_2_log(size);
 
-	mutex_enter(&(pool->mutex));
+	mem_pool_mutex_enter(pool);
 	mem_n_threads_inside++;
 
 	ut_a(mem_n_threads_inside == 1);
@@ -595,7 +606,7 @@ mem_area_free(
 		pool->reserved += ut_2_exp(n);
 
 		mem_n_threads_inside--;
-		mutex_exit(&(pool->mutex));
+		mem_pool_mutex_exit(pool);
 
 		mem_area_free(new_ptr, pool);
 
@@ -611,7 +622,7 @@ mem_area_free(
 	}
 
 	mem_n_threads_inside--;
-	mutex_exit(&(pool->mutex));
+	mem_pool_mutex_exit(pool);
 
 	ut_ad(mem_pool_validate(pool));
 }
@@ -630,7 +641,7 @@ mem_pool_validate(
 	ulint		free;
 	ulint		i;
 
-	mutex_enter(&(pool->mutex));
+	mem_pool_mutex_enter(pool);
 
 	free = 0;
 
@@ -658,7 +669,7 @@ mem_pool_validate(
 
 	ut_a(free + pool->reserved == pool->size);
 
-	mutex_exit(&(pool->mutex));
+	mem_pool_mutex_exit(pool);
 
 	return(TRUE);
 }

storage/innodb_plugin/row/row0merge.c

@@ -1780,6 +1780,11 @@ row_merge_copy_blobs(
 		(below). */
 		data = btr_rec_copy_externally_stored_field(
 			mrec, offsets, zip_size, i, &len, heap);
+		/* Because we have locked the table, any records
+		written by incomplete transactions must have been
+		rolled back already. There must not be any incomplete
+		BLOB columns. */
+		ut_a(data);
 
 		dfield_set_data(field, data, len);
 	}

storage/innodb_plugin/row/row0row.c

@@ -294,7 +294,13 @@ row_build(
 
 	ut_ad(dtuple_check_typed(row));
 
-	if (j) {
+	if (!ext) {
+		/* REDUNDANT and COMPACT formats store a local
+		768-byte prefix of each externally stored
+		column. No cache is needed. */
+		ut_ad(dict_table_get_format(index->table)
+		      < DICT_TF_FORMAT_ZIP);
+	} else if (j) {
 		*ext = row_ext_create(j, ext_cols, row,
 				      dict_table_zip_size(index->table),
 				      heap);

storage/innodb_plugin/row/row0sel.c

@@ -416,7 +416,7 @@ row_sel_fetch_columns(
 							      field_no))) {
 
 				/* Copy an externally stored field to the
-				temporary heap */
+				temporary heap, if possible. */
 
 				heap = mem_heap_create(1);
 
@@ -425,6 +425,17 @@ row_sel_fetch_columns(
 					dict_table_zip_size(index->table),
 					field_no, &len, heap);
 
+				/* data == NULL means that the
+				externally stored field was not
+				written yet. This record
+				should only be seen by
+				recv_recovery_rollback_active() or any
+				TRX_ISO_READ_UNCOMMITTED
+				transactions. The InnoDB SQL parser
+				(the sole caller of this function)
+				does not implement READ UNCOMMITTED,
+				and it is not involved during rollback. */
+				ut_a(data);
 				ut_a(len != UNIV_SQL_NULL);
 
 				needs_copy = TRUE;
@@ -926,6 +937,7 @@ row_sel_get_clust_rec(
 	when plan->clust_pcur was positioned.  The latch will not be
 	released until mtr_commit(mtr). */
 
+	ut_ad(!rec_get_deleted_flag(clust_rec, rec_offs_comp(offsets)));
 	row_sel_fetch_columns(index, clust_rec, offsets,
 			      UT_LIST_GET_FIRST(plan->columns));
 	*out_rec = clust_rec;
@@ -1628,6 +1640,13 @@ skip_lock:
 				}
 
 				if (old_vers == NULL) {
+					/* The record does not exist
+					in our read view. Skip it, but
+					first attempt to determine
+					whether the index segment we
+					are searching through has been
+					exhausted. */
+
 					offsets = rec_get_offsets(
 						rec, index, offsets,
 						ULINT_UNDEFINED, &heap);
@@ -2647,9 +2666,8 @@ Convert a row in the Innobase format to a row in the MySQL format.
 Note that the template in prebuilt may advise us to copy only a few
 columns to mysql_rec, other columns are left blank. All columns may not
 be needed in the query.
-@return TRUE if success, FALSE if could not allocate memory for a BLOB
-(though we may also assert in that case) */
-static
+@return TRUE on success, FALSE if not all columns could be retrieved */
+static __attribute__((warn_unused_result))
 ibool
 row_sel_store_mysql_rec(
 /*====================*/
@@ -2672,6 +2690,7 @@ row_sel_store_mysql_rec(
 	ut_ad(prebuilt->mysql_template);
 	ut_ad(prebuilt->default_rec);
 	ut_ad(rec_offs_validate(rec, NULL, offsets));
+	ut_ad(!rec_get_deleted_flag(rec, rec_offs_comp(offsets)));
 
 	if (UNIV_LIKELY_NULL(prebuilt->blob_heap)) {
 		mem_heap_free(prebuilt->blob_heap);
@@ -2719,6 +2738,21 @@ row_sel_store_mysql_rec(
 				dict_table_zip_size(prebuilt->table),
 				templ->rec_field_no, &len, heap);
 
+			if (UNIV_UNLIKELY(!data)) {
+				/* The externally stored field
+				was not written yet. This
+				record should only be seen by
+				recv_recovery_rollback_active()
+				or any TRX_ISO_READ_UNCOMMITTED
+				transactions. */
+
+				if (extern_field_heap) {
+					mem_heap_free(extern_field_heap);
+				}
+
+				return(FALSE);
+			}
+
 			ut_a(len != UNIV_SQL_NULL);
 		} else {
 			/* Field is stored in the row. */
@@ -3136,9 +3170,10 @@ row_sel_pop_cached_row_for_mysql(
 }
 
 /********************************************************************//**
-Pushes a row for MySQL to the fetch cache. */
-UNIV_INLINE
-void
+Pushes a row for MySQL to the fetch cache.
+@return TRUE on success, FALSE if the record contains incomplete BLOBs */
+UNIV_INLINE __attribute__((warn_unused_result))
+ibool
 row_sel_push_cache_row_for_mysql(
 /*=============================*/
 	row_prebuilt_t*	prebuilt,	/*!< in: prebuilt struct */
@@ -3180,10 +3215,11 @@ row_sel_push_cache_row_for_mysql(
 				  prebuilt->fetch_cache[
 					  prebuilt->n_fetch_cached],
 				  prebuilt, rec, offsets))) {
-		ut_error;
+		return(FALSE);
 	}
 
 	prebuilt->n_fetch_cached++;
+	return(TRUE);
 }
 
 /*********************************************************************//**
@@ -3578,11 +3614,21 @@ row_search_for_mysql(
 
 				if (!row_sel_store_mysql_rec(buf, prebuilt,
 							     rec, offsets)) {
-					err = DB_TOO_BIG_RECORD;
-
-					/* We let the main loop to do the
-					error handling */
-					goto shortcut_fails_too_big_rec;
+					/* Only fresh inserts may contain
+					incomplete externally stored
+					columns. Pretend that such
+					records do not exist. Such
+					records may only be accessed
+					at the READ UNCOMMITTED
+					isolation level or when
+					rolling back a recovered
+					transaction. Rollback happens
+					at a lower level, not here. */
+					ut_a(trx->isolation_level
+					     == TRX_ISO_READ_UNCOMMITTED);
+
+					/* Proceed as in case SEL_RETRY. */
+					break;
 				}
 
 				mtr_commit(&mtr);
@@ -3622,7 +3668,7 @@ release_search_latch_if_needed:
 			default:
 				ut_ad(0);
 			}
-shortcut_fails_too_big_rec:
+
 			mtr_commit(&mtr);
 			mtr_start(&mtr);
 		}
@@ -4357,9 +4403,18 @@ requires_clust_rec:
 		not cache rows because there the cursor is a scrollable
 		cursor. */
 
-		row_sel_push_cache_row_for_mysql(prebuilt, result_rec,
-						 offsets);
-		if (prebuilt->n_fetch_cached == MYSQL_FETCH_CACHE_SIZE) {
+		if (!row_sel_push_cache_row_for_mysql(prebuilt, result_rec,
+						      offsets)) {
+			/* Only fresh inserts may contain incomplete
+			externally stored columns. Pretend that such
+			records do not exist. Such records may only be
+			accessed at the READ UNCOMMITTED isolation
+			level or when rolling back a recovered
+			transaction. Rollback happens at a lower
+			level, not here. */
+			ut_a(trx->isolation_level == TRX_ISO_READ_UNCOMMITTED);
+		} else if (prebuilt->n_fetch_cached
+			   == MYSQL_FETCH_CACHE_SIZE) {
 
 			goto got_row;
 		}
@@ -4375,9 +4430,17 @@ requires_clust_rec:
 		} else {
 			if (!row_sel_store_mysql_rec(buf, prebuilt,
 						     result_rec, offsets)) {
-				err = DB_TOO_BIG_RECORD;
-
-				goto lock_wait_or_error;
+				/* Only fresh inserts may contain
+				incomplete externally stored
+				columns. Pretend that such records do
+				not exist. Such records may only be
+				accessed at the READ UNCOMMITTED
+				isolation level or when rolling back a
+				recovered transaction. Rollback
+				happens at a lower level, not here. */
+				ut_a(trx->isolation_level
+				     == TRX_ISO_READ_UNCOMMITTED);
+				goto next_rec;
 			}
 		}
 

storage/innodb_plugin/row/row0undo.c

@@ -199,8 +199,24 @@ row_undo_search_clust_to_pcur(
 
 		ret = FALSE;
 	} else {
+		row_ext_t**	ext;
+
+		if (dict_table_get_format(node->table) >= DICT_TF_FORMAT_ZIP) {
+			/* In DYNAMIC or COMPRESSED format, there is
+			no prefix of externally stored columns in the
+			clustered index record. Build a cache of
+			column prefixes. */
+			ext = &node->ext;
+		} else {
+			/* REDUNDANT and COMPACT formats store a local
+			768-byte prefix of each externally stored
+			column. No cache is needed. */
+			ext = NULL;
+			node->ext = NULL;
+		}
+
 		node->row = row_build(ROW_COPY_DATA, clust_index, rec,
-				      offsets, NULL, &node->ext, node->heap);
+				      offsets, NULL, ext, node->heap);
 		if (node->update) {
 			node->undo_row = dtuple_copy(node->row, node->heap);
 			row_upd_replace(node->undo_row, &node->undo_ext,

storage/innodb_plugin/row/row0upd.c

@@ -1398,6 +1398,7 @@ row_upd_store_row(
 	dict_index_t*	clust_index;
 	rec_t*		rec;
 	mem_heap_t*	heap		= NULL;
+	row_ext_t**	ext;
 	ulint		offsets_[REC_OFFS_NORMAL_SIZE];
 	const ulint*	offsets;
 	rec_offs_init(offsets_);
@@ -1414,8 +1415,22 @@ row_upd_store_row(
 
 	offsets = rec_get_offsets(rec, clust_index, offsets_,
 				  ULINT_UNDEFINED, &heap);
+
+	if (dict_table_get_format(node->table) >= DICT_TF_FORMAT_ZIP) {
+		/* In DYNAMIC or COMPRESSED format, there is no prefix
+		of externally stored columns in the clustered index
+		record. Build a cache of column prefixes. */
+		ext = &node->ext;
+	} else {
+		/* REDUNDANT and COMPACT formats store a local
+		768-byte prefix of each externally stored column.
+		No cache is needed. */
+		ext = NULL;
+		node->ext = NULL;
+	}
+
 	node->row = row_build(ROW_COPY_DATA, clust_index, rec, offsets,
-			      NULL, &node->ext, node->heap);
+			      NULL, ext, node->heap);
 	if (node->is_delete) {
 		node->upd_row = NULL;
 		node->upd_ext = NULL;

storage/innodb_plugin/srv/srv0start.c

@@ -2018,9 +2018,13 @@ innobase_shutdown_for_mysql(void)
 	pars_lexer_close();
 	log_mem_free();
 	buf_pool_free();
-	ut_free_all_mem();
 	mem_close();
 
+	/* ut_free_all_mem() frees all allocated memory not freed yet
+	in shutdown, and it will also free the ut_list_mutex, so it
+	should be the last one for all operation */
+	ut_free_all_mem();
+
 	if (os_thread_count != 0
 	    || os_event_count != 0
 	    || os_mutex_count != 0