Merge Bug #56680 from mysql-5.1.

Additional fixes in 5.5:

ibuf_set_del_mark(): Add diagnostics when setting a buffered delete-mark fails.

ibuf_delete(): Correct a misleading comment about non-found records.

rec_print(): Add a const qualifier to the index parameter.

Bug #56680 wrong InnoDB results from a case-insensitive covering index

row_search_for_mysql(): When a secondary index record might not be
visible in the current transaction's read view and we consult the
clustered index and optionally some undo log records, return the
relevant columns of the clustered index record to MySQL instead of the
secondary index record.

ibuf_insert_to_index_page_low(): New function, refactored from
ibuf_insert_to_index_page().

ibuf_insert_to_index_page(): When we are inserting a record in place
of a delete-marked record and some fields of the record differ, update
that record just like row_ins_sec_index_entry_by_modify() would do.

btr_cur_update_alloc_zip(): Make the function public.

mysql_row_templ_t: Add clust_rec_field_no.

row_sel_store_mysql_rec(), row_sel_push_cache_row_for_mysql(): Add the
flag rec_clust, for returning data at clust_rec_field_no instead of
rec_field_no. Resurrect the debug assertion that the record not be
marked for deletion. (Bug #55626)

[UNIV_DEBUG || UNIV_IBUF_DEBUG] ibuf_debug, buf_page_get_gen(),
buf_flush_page_try():
Implement innodb_change_buffering_debug=1 for evicting pages from the
buffer pool, so that change buffering will be attempted more
frequently.

mysql-test/suite/innodb/r/innodb_bug56680.result

+SET GLOBAL tx_isolation='REPEATABLE-READ';
+SET GLOBAL innodb_file_format=Barracuda;
+SET GLOBAL innodb_file_per_table=on;
+CREATE TABLE bug56680(
+a INT AUTO_INCREMENT PRIMARY KEY,
+b CHAR(1),
+c INT,
+INDEX(b))
+ENGINE=InnoDB;
+INSERT INTO bug56680 VALUES(0,'x',1);
+BEGIN;
+SELECT b FROM bug56680;
+b
+x
+BEGIN;
+UPDATE bug56680 SET b='X';
+SELECT b FROM bug56680;
+b
+x
+SELECT * FROM bug56680;
+a	b	c
+1	x	1
+ROLLBACK;
+SELECT b FROM bug56680;
+b
+x
+SET GLOBAL tx_isolation='READ-UNCOMMITTED';
+INSERT INTO bug56680 SELECT 0,b,c FROM bug56680;
+INSERT INTO bug56680 SELECT 0,b,c FROM bug56680;
+INSERT INTO bug56680 SELECT 0,b,c FROM bug56680;
+INSERT INTO bug56680 SELECT 0,b,c FROM bug56680;
+INSERT INTO bug56680 SELECT 0,b,c FROM bug56680;
+INSERT INTO bug56680 SELECT 0,b,c FROM bug56680;
+INSERT INTO bug56680 SELECT 0,b,c FROM bug56680;
+INSERT INTO bug56680 SELECT 0,b,c FROM bug56680;
+INSERT INTO bug56680 SELECT 0,b,c FROM bug56680;
+INSERT INTO bug56680 SELECT 0,b,c FROM bug56680;
+INSERT INTO bug56680 SELECT 0,b,c FROM bug56680;
+BEGIN;
+SELECT b FROM bug56680 LIMIT 2;
+b
+x
+x
+BEGIN;
+DELETE FROM bug56680 WHERE a=1;
+INSERT INTO bug56680 VALUES(1,'X',1);
+SELECT b FROM bug56680 LIMIT 3;
+b
+X
+x
+x
+SELECT b FROM bug56680 LIMIT 2;
+b
+x
+x
+CHECK TABLE bug56680;
+Table	Op	Msg_type	Msg_text
+test.bug56680	check	status	OK
+ROLLBACK;
+SELECT b FROM bug56680 LIMIT 2;
+b
+x
+x
+CHECK TABLE bug56680;
+Table	Op	Msg_type	Msg_text
+test.bug56680	check	status	OK
+SELECT b FROM bug56680 LIMIT 2;
+b
+x
+x
+CREATE TABLE bug56680_2(
+a INT AUTO_INCREMENT PRIMARY KEY,
+b VARCHAR(2) CHARSET latin1 COLLATE latin1_german2_ci,
+c INT,
+INDEX(b))
+ENGINE=InnoDB;
+INSERT INTO bug56680_2 SELECT 0,_latin1 0xdf,c FROM bug56680;
+BEGIN;
+SELECT HEX(b) FROM bug56680_2 LIMIT 2;
+HEX(b)
+DF
+DF
+DELETE FROM bug56680_2 WHERE a=1;
+INSERT INTO bug56680_2 VALUES(1,'SS',1);
+SELECT HEX(b) FROM bug56680_2 LIMIT 3;
+HEX(b)
+5353
+DF
+DF
+CHECK TABLE bug56680_2;
+Table	Op	Msg_type	Msg_text
+test.bug56680_2	check	status	OK
+ALTER TABLE bug56680_2 ROW_FORMAT=COMPRESSED KEY_BLOCK_SIZE=1;
+SELECT HEX(b) FROM bug56680_2 LIMIT 2;
+HEX(b)
+5353
+DF
+DELETE FROM bug56680_2 WHERE a=1;
+INSERT INTO bug56680_2 VALUES(1,_latin1 0xdf,1);
+SELECT HEX(b) FROM bug56680_2 LIMIT 3;
+HEX(b)
+DF
+DF
+DF
+CHECK TABLE bug56680_2;
+Table	Op	Msg_type	Msg_text
+test.bug56680_2	check	status	OK
+DROP TABLE bug56680_2;
+DROP TABLE bug56680;

mysql-test/suite/innodb/t/innodb_bug56680.test

+#
+# Bug #56680 InnoDB may return wrong results from a case-insensitive index
+#
+-- source include/have_innodb.inc
+
+-- disable_query_log
+SET @tx_isolation_orig = @@tx_isolation;
+SET @innodb_file_per_table_orig = @@innodb_file_per_table;
+SET @innodb_file_format_orig = @@innodb_file_format;
+SET @innodb_file_format_max_orig = @@innodb_file_format_max;
+# The flag innodb_change_buffering_debug is only available in debug builds.
+# It instructs InnoDB to try to evict pages from the buffer pool when
+# change buffering is possible, so that the change buffer will be used
+# whenever possible.
+-- error 0,ER_UNKNOWN_SYSTEM_VARIABLE
+SET @innodb_change_buffering_debug_orig = @@innodb_change_buffering_debug;
+-- error 0,ER_UNKNOWN_SYSTEM_VARIABLE
+SET GLOBAL innodb_change_buffering_debug = 1;
+-- enable_query_log
+SET GLOBAL tx_isolation='REPEATABLE-READ';
+SET GLOBAL innodb_file_format=Barracuda;
+SET GLOBAL innodb_file_per_table=on;
+
+CREATE TABLE bug56680(
+       a INT AUTO_INCREMENT PRIMARY KEY,
+       b CHAR(1),
+       c INT,
+       INDEX(b))
+ENGINE=InnoDB;
+
+INSERT INTO bug56680 VALUES(0,'x',1);
+BEGIN;
+SELECT b FROM bug56680;
+
+connect (con1,localhost,root,,);
+connection con1;
+BEGIN;
+UPDATE bug56680 SET b='X';
+
+connection default;
+# This should return the last committed value 'x', but would return 'X'
+# due to a bug in row_search_for_mysql().
+SELECT b FROM bug56680;
+# This would always return the last committed value 'x'.
+SELECT * FROM bug56680;
+
+connection con1;
+ROLLBACK;
+disconnect con1;
+
+connection default;
+
+SELECT b FROM bug56680;
+
+# For the rest of this test, use the READ UNCOMMITTED isolation level
+# to see what exists in the secondary index.
+SET GLOBAL tx_isolation='READ-UNCOMMITTED';
+
+# Create enough rows for the table, so that the insert buffer will be
+# used for modifying the secondary index page. There must be multiple
+# index pages, because changes to the root page are never buffered.
+
+INSERT INTO bug56680 SELECT 0,b,c FROM bug56680;
+INSERT INTO bug56680 SELECT 0,b,c FROM bug56680;
+INSERT INTO bug56680 SELECT 0,b,c FROM bug56680;
+INSERT INTO bug56680 SELECT 0,b,c FROM bug56680;
+INSERT INTO bug56680 SELECT 0,b,c FROM bug56680;
+INSERT INTO bug56680 SELECT 0,b,c FROM bug56680;
+INSERT INTO bug56680 SELECT 0,b,c FROM bug56680;
+INSERT INTO bug56680 SELECT 0,b,c FROM bug56680;
+INSERT INTO bug56680 SELECT 0,b,c FROM bug56680;
+INSERT INTO bug56680 SELECT 0,b,c FROM bug56680;
+INSERT INTO bug56680 SELECT 0,b,c FROM bug56680;
+
+BEGIN;
+SELECT b FROM bug56680 LIMIT 2;
+
+connect (con1,localhost,root,,);
+connection con1;
+BEGIN;
+DELETE FROM bug56680 WHERE a=1;
+# This should be buffered, if innodb_change_buffering_debug = 1 is in effect.
+INSERT INTO bug56680 VALUES(1,'X',1);
+
+# This should force an insert buffer merge, and return 'X' in the first row.
+SELECT b FROM bug56680 LIMIT 3;
+
+connection default;
+SELECT b FROM bug56680 LIMIT 2;
+CHECK TABLE bug56680;
+
+connection con1;
+ROLLBACK;
+SELECT b FROM bug56680 LIMIT 2;
+CHECK TABLE bug56680;
+
+connection default;
+disconnect con1;
+
+SELECT b FROM bug56680 LIMIT 2;
+
+CREATE TABLE bug56680_2(
+       a INT AUTO_INCREMENT PRIMARY KEY,
+       b VARCHAR(2) CHARSET latin1 COLLATE latin1_german2_ci,
+       c INT,
+       INDEX(b))
+ENGINE=InnoDB;
+
+INSERT INTO bug56680_2 SELECT 0,_latin1 0xdf,c FROM bug56680;
+
+BEGIN;
+SELECT HEX(b) FROM bug56680_2 LIMIT 2;
+DELETE FROM bug56680_2 WHERE a=1;
+# This should be buffered, if innodb_change_buffering_debug = 1 is in effect.
+INSERT INTO bug56680_2 VALUES(1,'SS',1);
+
+# This should force an insert buffer merge, and return 'SS' in the first row.
+SELECT HEX(b) FROM bug56680_2 LIMIT 3;
+CHECK TABLE bug56680_2;
+
+# Test this with compressed tables.
+ALTER TABLE bug56680_2 ROW_FORMAT=COMPRESSED KEY_BLOCK_SIZE=1;
+
+SELECT HEX(b) FROM bug56680_2 LIMIT 2;
+DELETE FROM bug56680_2 WHERE a=1;
+# This should be buffered, if innodb_change_buffering_debug = 1 is in effect.
+INSERT INTO bug56680_2 VALUES(1,_latin1 0xdf,1);
+
+# This should force an insert buffer merge, and return 0xdf in the first row.
+SELECT HEX(b) FROM bug56680_2 LIMIT 3;
+CHECK TABLE bug56680_2;
+
+DROP TABLE bug56680_2;
+DROP TABLE bug56680;
+
+-- disable_query_log
+SET GLOBAL tx_isolation = @tx_isolation_orig;
+SET GLOBAL innodb_file_per_table = @innodb_file_per_table_orig;
+SET GLOBAL innodb_file_format = @innodb_file_format_orig;
+SET GLOBAL innodb_file_format_max = @innodb_file_format_max_orig;
+-- error 0, ER_UNKNOWN_SYSTEM_VARIABLE
+SET GLOBAL innodb_change_buffering_debug = @innodb_change_buffering_debug_orig;

storage/innobase/btr/btr0cur.c

@@ -1744,7 +1744,7 @@ func_exit:
 See if there is enough place in the page modification log to log
 an update-in-place.
 @return	TRUE if enough place */
-static
+UNIV_INTERN
 ibool
 btr_cur_update_alloc_zip(
 /*=====================*/

storage/innobase/buf/buf0buf.c

@@ -3008,6 +3008,46 @@ wait_until_unfixed:
 	bytes. */
 	UNIV_MEM_ASSERT_RW(&block->page, sizeof block->page);
 #endif
+#if defined UNIV_DEBUG || defined UNIV_IBUF_DEBUG
+	if ((mode == BUF_GET_IF_IN_POOL || mode == BUF_GET_IF_IN_POOL_OR_WATCH)
+	    && ibuf_debug) {
+		/* Try to evict the block from the buffer pool, to use the
+		insert buffer (change buffer) as much as possible. */
+
+		if (buf_LRU_free_block(&block->page, TRUE, NULL)
+		    == BUF_LRU_FREED) {
+			mutex_exit(&block->mutex);
+			if (mode == BUF_GET_IF_IN_POOL_OR_WATCH) {
+				/* Set the watch, as it would have
+				been set if the page were not in the
+				buffer pool in the first place. */
+				block = (buf_block_t*) buf_pool_watch_set(
+					space, offset, fold);
+
+				if (UNIV_LIKELY_NULL(block)) {
+
+					/* The page entered the buffer
+					pool for some reason. Try to
+					evict it again. */
+					goto got_block;
+				}
+			}
+			buf_pool_mutex_exit(buf_pool);
+			fprintf(stderr,
+				"innodb_change_buffering_debug evict %u %u\n",
+				(unsigned) space, (unsigned) offset);
+			return(NULL);
+		} else if (buf_flush_page_try(buf_pool, block)) {
+			fprintf(stderr,
+				"innodb_change_buffering_debug flush %u %u\n",
+				(unsigned) space, (unsigned) offset);
+			guess = block;
+			goto loop;
+		}
+
+		/* Failed to evict the page; change it directly */
+	}
+#endif /* UNIV_DEBUG || UNIV_IBUF_DEBUG */
 
 	buf_block_buf_fix_inc(block, file, line);
 

storage/innobase/buf/buf0flu.c

@@ -1132,6 +1132,83 @@ buf_flush_write_block_low(
 	}
 }
 
+# if defined UNIV_DEBUG || defined UNIV_IBUF_DEBUG
+/********************************************************************//**
+Writes a flushable page asynchronously from the buffer pool to a file.
+NOTE: buf_pool_mutex and block->mutex must be held upon entering this
+function, and they will be released by this function after flushing.
+This is loosely based on buf_flush_batch() and buf_flush_page().
+@return TRUE if the page was flushed and the mutexes released */
+UNIV_INTERN
+ibool
+buf_flush_page_try(
+/*===============*/
+	buf_pool_t*	buf_pool,	/*!< in/out: buffer pool instance */
+	buf_block_t*	block)		/*!< in/out: buffer control block */
+{
+	ut_ad(buf_pool_mutex_own(buf_pool));
+	ut_ad(buf_block_get_state(block) == BUF_BLOCK_FILE_PAGE);
+	ut_ad(mutex_own(&block->mutex));
+
+	if (!buf_flush_ready_for_flush(&block->page, BUF_FLUSH_LRU)) {
+		return(FALSE);
+	}
+
+	if (buf_pool->n_flush[BUF_FLUSH_LRU] > 0
+	    || buf_pool->init_flush[BUF_FLUSH_LRU]) {
+		/* There is already a flush batch of the same type running */
+		return(FALSE);
+	}
+
+	buf_pool->init_flush[BUF_FLUSH_LRU] = TRUE;
+
+	buf_page_set_io_fix(&block->page, BUF_IO_WRITE);
+
+	buf_page_set_flush_type(&block->page, BUF_FLUSH_LRU);
+
+	if (buf_pool->n_flush[BUF_FLUSH_LRU]++ == 0) {
+
+		os_event_reset(buf_pool->no_flush[BUF_FLUSH_LRU]);
+	}
+
+	/* VERY IMPORTANT:
+	Because any thread may call the LRU flush, even when owning
+	locks on pages, to avoid deadlocks, we must make sure that the
+	s-lock is acquired on the page without waiting: this is
+	accomplished because buf_flush_ready_for_flush() must hold,
+	and that requires the page not to be bufferfixed. */
+
+	rw_lock_s_lock_gen(&block->lock, BUF_IO_WRITE);
+
+	/* Note that the s-latch is acquired before releasing the
+	buf_pool mutex: this ensures that the latch is acquired
+	immediately. */
+
+	mutex_exit(&block->mutex);
+	buf_pool_mutex_exit(buf_pool);
+
+	/* Even though block is not protected by any mutex at this
+	point, it is safe to access block, because it is io_fixed and
+	oldest_modification != 0.  Thus, it cannot be relocated in the
+	buffer pool or removed from flush_list or LRU_list. */
+
+	buf_flush_write_block_low(&block->page);
+
+	buf_pool_mutex_enter(buf_pool);
+	buf_pool->init_flush[BUF_FLUSH_LRU] = FALSE;
+
+	if (buf_pool->n_flush[BUF_FLUSH_LRU] == 0) {
+		/* The running flush batch has ended */
+		os_event_set(buf_pool->no_flush[BUF_FLUSH_LRU]);
+	}
+
+	buf_pool_mutex_exit(buf_pool);
+	buf_flush_buffered_writes();
+
+	return(TRUE);
+}
+# endif /* UNIV_DEBUG || UNIV_IBUF_DEBUG */
+
 /********************************************************************//**
 Writes a flushable page asynchronously from the buffer pool to a file.
 NOTE: in simulated aio we must call

storage/innobase/handler/ha_innodb.cc

@@ -4615,18 +4615,19 @@ include_field:
 		n_requested_fields++;
 
 		templ->col_no = i;
+		templ->clust_rec_field_no = dict_col_get_clust_pos(
+			col, clust_index);
+		ut_ad(templ->clust_rec_field_no != ULINT_UNDEFINED);
 
 		if (index == clust_index) {
-			templ->rec_field_no = dict_col_get_clust_pos(
-				col, index);
+			templ->rec_field_no = templ->clust_rec_field_no;
 		} else {
 			templ->rec_field_no = dict_index_get_nth_col_pos(
 								index, i);
-		}
-
 			if (templ->rec_field_no == ULINT_UNDEFINED) {
 				prebuilt->need_to_access_clustered = TRUE;
 			}
+		}
 
 		if (field->null_ptr) {
 			templ->mysql_null_byte_offset =
@@ -4675,9 +4676,7 @@ skip_field:
 		for (i = 0; i < n_requested_fields; i++) {
 			templ = prebuilt->mysql_template + i;
 
-			templ->rec_field_no = dict_col_get_clust_pos(
-				&index->table->cols[templ->col_no],
-				clust_index);
+			templ->rec_field_no = templ->clust_rec_field_no;
 		}
 	}
 }
@@ -11162,6 +11161,13 @@ static MYSQL_SYSVAR_STR(change_buffering, innobase_change_buffering,
   innodb_change_buffering_validate,
   innodb_change_buffering_update, "all");
 
+#if defined UNIV_DEBUG || defined UNIV_IBUF_DEBUG
+static MYSQL_SYSVAR_UINT(change_buffering_debug, ibuf_debug,
+  PLUGIN_VAR_RQCMDARG,
+  "Debug flags for InnoDB change buffering (0=none)",
+  NULL, NULL, 0, 0, 1, 0);
+#endif /* UNIV_DEBUG || UNIV_IBUF_DEBUG */
+
 static MYSQL_SYSVAR_ULONG(read_ahead_threshold, srv_read_ahead_threshold,
   PLUGIN_VAR_RQCMDARG,
   "Number of pages that must be accessed sequentially for InnoDB to "
@@ -11225,6 +11231,9 @@ static struct st_mysql_sys_var* innobase_system_variables[]= {
   MYSQL_SYSVAR(use_sys_malloc),
   MYSQL_SYSVAR(use_native_aio),
   MYSQL_SYSVAR(change_buffering),
+#if defined UNIV_DEBUG || defined UNIV_IBUF_DEBUG
+  MYSQL_SYSVAR(change_buffering_debug),
+#endif /* UNIV_DEBUG || UNIV_IBUF_DEBUG */
   MYSQL_SYSVAR(read_ahead_threshold),
   MYSQL_SYSVAR(io_capacity),
   MYSQL_SYSVAR(purge_threads),

storage/innobase/ibuf/ibuf0ibuf.c

@@ -49,6 +49,7 @@ Created 7/19/1997 Heikki Tuuri
 #include "btr0cur.h"
 #include "btr0pcur.h"
 #include "btr0btr.h"
+#include "row0upd.h"
 #include "sync0sync.h"
 #include "dict0boot.h"
 #include "fut0lst.h"
@@ -192,6 +193,11 @@ access order rules. */
 /** Operations that can currently be buffered. */
 UNIV_INTERN ibuf_use_t	ibuf_use		= IBUF_USE_ALL;
 
+#if defined UNIV_DEBUG || defined UNIV_IBUF_DEBUG
+/** Flag to control insert buffer debugging. */
+UNIV_INTERN uint	ibuf_debug;
+#endif /* UNIV_DEBUG || UNIV_IBUF_DEBUG */
+
 /** The insert buffer control structure */
 UNIV_INTERN ibuf_t*	ibuf			= NULL;
 
@@ -2761,9 +2767,8 @@ ibuf_get_volume_buffered_count(
 
 	switch (ibuf_op) {
 	case IBUF_OP_INSERT:
-		/* Inserts can be done by
-		btr_cur_set_deleted_flag_for_ibuf().  Because
-		delete-mark and insert operations can be pointing to
+		/* Inserts can be done by updating a delete-marked record.
+		Because delete-mark and insert operations can be pointing to
 		the same records, we must not count duplicates. */
 	case IBUF_OP_DELETE_MARK:
 		/* There must be a record to delete-mark.
@@ -3748,9 +3753,80 @@ During merge, inserts to an index page a secondary index entry extracted
 from the insert buffer. */
 static
 void
+ibuf_insert_to_index_page_low(
+/*==========================*/
+	const dtuple_t*	entry,	/*!< in: buffered entry to insert */
+	buf_block_t*	block,	/*!< in/out: index page where the buffered
+				entry should be placed */
+	dict_index_t*	index,	/*!< in: record descriptor */
+	mtr_t*		mtr,	/*!< in/out: mtr */
+	page_cur_t*	page_cur)/*!< in/out: cursor positioned on the record
+				after which to insert the buffered entry */
+{
+	const page_t*	page;
+	ulint		space;
+	ulint		page_no;
+	ulint		zip_size;
+	const page_t*	bitmap_page;
+	ulint		old_bits;
+
+	if (UNIV_LIKELY
+	    (page_cur_tuple_insert(page_cur, entry, index, 0, mtr) != NULL)) {
+		return;
+	}
+
+	/* If the record did not fit, reorganize */
+
+	btr_page_reorganize(block, index, mtr);
+	page_cur_search(block, index, entry, PAGE_CUR_LE, page_cur);
+
+	/* This time the record must fit */
+
+	if (UNIV_LIKELY
+	    (page_cur_tuple_insert(page_cur, entry, index, 0, mtr) != NULL)) {
+		return;
+	}
+
+	page = buf_block_get_frame(block);
+
+	ut_print_timestamp(stderr);
+
+	fprintf(stderr,
+		"  InnoDB: Error: Insert buffer insert fails;"
+		" page free %lu, dtuple size %lu\n",
+		(ulong) page_get_max_insert_size(page, 1),
+		(ulong) rec_get_converted_size(index, entry, 0));
+	fputs("InnoDB: Cannot insert index record ", stderr);
+	dtuple_print(stderr, entry);
+	fputs("\nInnoDB: The table where this index record belongs\n"
+	      "InnoDB: is now probably corrupt. Please run CHECK TABLE on\n"
+	      "InnoDB: that table.\n", stderr);
+
+	space = page_get_space_id(page);
+	zip_size = buf_block_get_zip_size(block);
+	page_no = page_get_page_no(page);
+
+	bitmap_page = ibuf_bitmap_get_map_page(space, page_no, zip_size, mtr);
+	old_bits = ibuf_bitmap_page_get_bits(bitmap_page, page_no, zip_size,
+					     IBUF_BITMAP_FREE, mtr);
+
+	fprintf(stderr,
+		"InnoDB: space %lu, page %lu, zip_size %lu, bitmap bits %lu\n",
+		(ulong) space, (ulong) page_no,
+		(ulong) zip_size, (ulong) old_bits);
+
+	fputs("InnoDB: Submit a detailed bug report"
+	      " to http://bugs.mysql.com\n", stderr);
+}
+
+/************************************************************************
+During merge, inserts to an index page a secondary index entry extracted
+from the insert buffer. */
+static
+void
 ibuf_insert_to_index_page(
 /*======================*/
-	dtuple_t*	entry,	/*!< in: buffered entry to insert */
+	const dtuple_t*	entry,	/*!< in: buffered entry to insert */
 	buf_block_t*	block,	/*!< in/out: index page where the buffered entry
 				should be placed */
 	dict_index_t*	index,	/*!< in: record descriptor */
@@ -3760,11 +3836,10 @@ ibuf_insert_to_index_page(
 	ulint		low_match;
 	page_t*		page		= buf_block_get_frame(block);
 	rec_t*		rec;
-	page_t*		bitmap_page;
-	ulint		old_bits;
 
 	ut_ad(ibuf_inside());
 	ut_ad(dtuple_check_typed(entry));
+	ut_ad(!buf_block_align(page)->is_hashed);
 
 	if (UNIV_UNLIKELY(dict_table_is_comp(index->table)
 			  != (ibool)!!page_is_comp(page))) {
@@ -3810,71 +3885,87 @@ dump:
 	low_match = page_cur_search(block, index, entry,
 				    PAGE_CUR_LE, &page_cur);
 
-	if (low_match == dtuple_get_n_fields(entry)) {
+	if (UNIV_UNLIKELY(low_match == dtuple_get_n_fields(entry))) {
+		mem_heap_t*	heap;
+		upd_t*		update;
+		ulint*		offsets;
 		page_zip_des_t*	page_zip;
 
 		rec = page_cur_get_rec(&page_cur);
-		page_zip = buf_block_get_page_zip(block);
 
-		btr_cur_set_deleted_flag_for_ibuf(rec, page_zip, FALSE, mtr);
-	} else {
-		rec = page_cur_tuple_insert(&page_cur, entry, index, 0, mtr);
+		/* This is based on
+		row_ins_sec_index_entry_by_modify(BTR_MODIFY_LEAF). */
+		ut_ad(rec_get_deleted_flag(rec, page_is_comp(page)));
+
+		heap = mem_heap_create(1024);
+
+		offsets = rec_get_offsets(rec, index, NULL, ULINT_UNDEFINED,
+					  &heap);
+		update = row_upd_build_sec_rec_difference_binary(
+			index, entry, rec, NULL, heap);
 
-		if (UNIV_LIKELY(rec != NULL)) {
+		page_zip = buf_block_get_page_zip(block);
+
+		if (update->n_fields == 0) {
+			/* The records only differ in the delete-mark.
+			Clear the delete-mark, like we did before
+			Bug #56680 was fixed. */
+			btr_cur_set_deleted_flag_for_ibuf(
+				rec, page_zip, FALSE, mtr);
+updated_in_place:
+			mem_heap_free(heap);
 			return;
 		}
 
-		/* If the record did not fit, reorganize */
+		/* Copy the info bits. Clear the delete-mark. */
+		update->info_bits = rec_get_info_bits(rec, page_is_comp(page));
+		update->info_bits &= ~REC_INFO_DELETED_FLAG;
 
-		btr_page_reorganize(block, index, mtr);
-		page_cur_search(block, index, entry, PAGE_CUR_LE, &page_cur);
+		/* We cannot invoke btr_cur_optimistic_update() here,
+		because we do not have a btr_cur_t or que_thr_t,
+		as the insert buffer merge occurs at a very low level. */
+		if (!row_upd_changes_field_size_or_external(index, offsets,
+							    update)
+		    && (!page_zip || btr_cur_update_alloc_zip(
+				page_zip, block, index,
+				rec_offs_size(offsets), FALSE, mtr))) {
+			/* This is the easy case. Do something similar
+			to btr_cur_update_in_place(). */
+			row_upd_rec_in_place(rec, index, offsets,
+					     update, page_zip);
+			goto updated_in_place;
+		}
 
-		/* This time the record must fit */
-		if (UNIV_UNLIKELY
-		    (!page_cur_tuple_insert(&page_cur, entry, index,
-					    0, mtr))) {
-			ulint	space;
-			ulint	page_no;
-			ulint	zip_size;
+		/* A collation may identify values that differ in
+		storage length.
+		Some examples (1 or 2 bytes):
+		utf8_turkish_ci: I = U+0131 LATIN SMALL LETTER DOTLESS I
+		utf8_general_ci: S = U+00DF LATIN SMALL LETTER SHARP S
+		utf8_general_ci: A = U+00E4 LATIN SMALL LETTER A WITH DIAERESIS
 
-			ut_print_timestamp(stderr);
+		latin1_german2_ci: SS = U+00DF LATIN SMALL LETTER SHARP S
 
-			fprintf(stderr,
-				"  InnoDB: Error: Insert buffer insert"
-				" fails; page free %lu,"
-				" dtuple size %lu\n",
-				(ulong) page_get_max_insert_size(
-					page, 1),
-				(ulong) rec_get_converted_size(
-					index, entry, 0));
-			fputs("InnoDB: Cannot insert index record ",
-			      stderr);
-			dtuple_print(stderr, entry);
-			fputs("\nInnoDB: The table where"
-			      " this index record belongs\n"
-			      "InnoDB: is now probably corrupt."
-			      " Please run CHECK TABLE on\n"
-			      "InnoDB: that table.\n", stderr);
+		Examples of a character (3-byte UTF-8 sequence)
+		identified with 2 or 4 characters (1-byte UTF-8 sequences):
 
-			space = page_get_space_id(page);
-			zip_size = buf_block_get_zip_size(block);
-			page_no = page_get_page_no(page);
+		utf8_unicode_ci: 'II' = U+2171 SMALL ROMAN NUMERAL TWO
+		utf8_unicode_ci: '(10)' = U+247D PARENTHESIZED NUMBER TEN
+		*/
 
-			bitmap_page = ibuf_bitmap_get_map_page(
-				space, page_no, zip_size, mtr);
-			old_bits = ibuf_bitmap_page_get_bits(
-				bitmap_page, page_no, zip_size,
-				IBUF_BITMAP_FREE, mtr);
+		/* Delete the different-length record, and insert the
+		buffered one. */
 
-			fprintf(stderr,
-				"InnoDB: space %lu, page %lu,"
-				" zip_size %lu, bitmap bits %lu\n",
-				(ulong) space, (ulong) page_no,
-				(ulong) zip_size, (ulong) old_bits);
+		lock_rec_store_on_page_infimum(block, rec);
+		page_cur_delete_rec(&page_cur, index, offsets, mtr);
+		page_cur_move_to_prev(&page_cur);
+		mem_heap_free(heap);
 
-			fputs("InnoDB: Submit a detailed bug report"
-			      " to http://bugs.mysql.com\n", stderr);
-		}
+		ibuf_insert_to_index_page_low(entry, block, index, mtr,
+					      &page_cur);
+		lock_rec_restore_from_page_infimum(block, rec, block);
+	} else {
+		ibuf_insert_to_index_page_low(entry, block, index, mtr,
+					      &page_cur);
 	}
 }
 
@@ -3906,9 +3997,31 @@ ibuf_set_del_mark(
 		rec = page_cur_get_rec(&page_cur);
 		page_zip = page_cur_get_page_zip(&page_cur);
 
+		if (UNIV_UNLIKELY
+		    (rec_get_deleted_flag(
+			    rec, dict_table_is_comp(index->table)))) {
+			ut_print_timestamp(stderr);
+			fputs("  InnoDB: record is already delete-marked\n",
+			      stderr);
+			goto failure;
+		}
+
 		btr_cur_set_deleted_flag_for_ibuf(rec, page_zip, TRUE, mtr);
 	} else {
-		/* This can happen benignly in some situations. */
+		ut_print_timestamp(stderr);
+		fputs("  InnoDB: unable to find a record to delete-mark\n",
+		      stderr);
+failure:
+		fputs("InnoDB: tuple ", stderr);
+		dtuple_print(stderr, entry);
+		fputs("\n"
+		      "InnoDB: record ", stderr);
+		rec_print(stderr, page_cur_get_rec(&page_cur), index);
+		putc('\n', stderr);
+		fputs("\n"
+		      "InnoDB: Submit a detailed bug report"
+		      " to http://bugs.mysql.com\n", stderr);
+		ut_ad(0);
 	}
 }
 
@@ -3983,10 +4096,7 @@ ibuf_delete(
 			mem_heap_free(heap);
 		}
 	} else {
-		/* This can happen benignly in some situations: either when
-		we crashed at just the right time, or on database startup
-		when we redo some old log entries (due to worse stored
-		position granularity on disk than in memory). */
+		/* The record must have been purged already. */
 	}
 }
 

storage/innobase/include/btr0cur.h

@@ -243,6 +243,22 @@ btr_cur_pessimistic_insert(
 	que_thr_t*	thr,	/*!< in: query thread or NULL */
 	mtr_t*		mtr);	/*!< in: mtr */
 /*************************************************************//**
+See if there is enough place in the page modification log to log
+an update-in-place.
+@return	TRUE if enough place */
+UNIV_INTERN
+ibool
+btr_cur_update_alloc_zip(
+/*=====================*/
+	page_zip_des_t*	page_zip,/*!< in/out: compressed page */
+	buf_block_t*	block,	/*!< in/out: buffer page */
+	dict_index_t*	index,	/*!< in: the index corresponding to the block */
+	ulint		length,	/*!< in: size needed */
+	ibool		create,	/*!< in: TRUE=delete-and-insert,
+				FALSE=update-in-place */
+	mtr_t*		mtr)	/*!< in: mini-transaction */
+	__attribute__((nonnull, warn_unused_result));
+/*************************************************************//**
 Updates a record when the update causes no size changes in its fields.
 @return	DB_SUCCESS or error number */
 UNIV_INTERN

storage/innobase/include/buf0flu.h

@@ -84,6 +84,21 @@ buf_flush_init_for_writing(
 	ib_uint64_t	newest_lsn);	/*!< in: newest modification lsn
 					to the page */
 #ifndef UNIV_HOTBACKUP
+# if defined UNIV_DEBUG || defined UNIV_IBUF_DEBUG
+/********************************************************************//**
+Writes a flushable page asynchronously from the buffer pool to a file.
+NOTE: buf_pool_mutex and block->mutex must be held upon entering this
+function, and they will be released by this function after flushing.
+This is loosely based on buf_flush_batch() and buf_flush_page().
+@return TRUE if the page was flushed and the mutexes released */
+UNIV_INTERN
+ibool
+buf_flush_page_try(
+/*===============*/
+	buf_pool_t*	buf_pool,	/*!< in/out: buffer pool instance */
+	buf_block_t*	block)		/*!< in/out: buffer control block */
+	__attribute__((nonnull, warn_unused_result));
+# endif /* UNIV_DEBUG || UNIV_IBUF_DEBUG */
 /*******************************************************************//**
 This utility flushes dirty blocks from the end of the LRU list.
 NOTE: The calling thread may own latches to pages: to avoid deadlocks,

storage/innobase/include/ibuf0ibuf.h

@@ -63,6 +63,11 @@ typedef enum {
 /** Operations that can currently be buffered. */
 extern ibuf_use_t	ibuf_use;
 
+#if defined UNIV_DEBUG || defined UNIV_IBUF_DEBUG
+/** Flag to control insert buffer debugging. */
+extern uint		ibuf_debug;
+#endif /* UNIV_DEBUG || UNIV_IBUF_DEBUG */
+
 /** The insert buffer control structure */
 extern ibuf_t*		ibuf;
 

storage/innobase/include/rem0rec.h

@@ -803,7 +803,7 @@ rec_print(
 /*======*/
 	FILE*			file,	/*!< in: file where to print */
 	const rec_t*		rec,	/*!< in: physical record */
-	dict_index_t*	index);	/*!< in: record descriptor */
+	const dict_index_t*	index);	/*!< in: record descriptor */
 #endif /* UNIV_HOTBACKUP */
 
 /* Maximum lengths for the data in a physical record if the offsets

storage/innobase/include/row0mysql.h

@@ -538,6 +538,10 @@ struct mysql_row_templ_struct {
 					Innobase record in the current index;
 					not defined if template_type is
 					ROW_MYSQL_WHOLE_ROW */
+	ulint	clust_rec_field_no;	/*!< field number of the column in an
+					Innobase record in the clustered index;
+					not defined if template_type is
+					ROW_MYSQL_WHOLE_ROW */
 	ulint	mysql_col_offset;	/*!< offset of the column in the MySQL
 					row format */
 	ulint	mysql_col_len;		/*!< length of the column in the MySQL

storage/innobase/include/row0upd.h

@@ -167,8 +167,11 @@ row_upd_changes_field_size_or_external(
 	const upd_t*	update);/*!< in: update vector */
 #endif /* !UNIV_HOTBACKUP */
 /***********************************************************//**
-Replaces the new column values stored in the update vector to the record
-given. No field size changes are allowed. */
+Replaces the new column values stored in the update vector to the
+record given. No field size changes are allowed. This function is
+usually invoked on a clustered index. The only use case for a
+secondary index is row_ins_sec_index_entry_by_modify() or its
+counterpart in ibuf_insert_to_index_page(). */
 UNIV_INTERN
 void
 row_upd_rec_in_place(

storage/innobase/rem/rem0rec.c

@@ -1751,7 +1751,7 @@ rec_print(
 /*======*/
 	FILE*			file,	/*!< in: file where to print */
 	const rec_t*		rec,	/*!< in: physical record */
-	dict_index_t*	index)	/*!< in: record descriptor */
+	const dict_index_t*	index)	/*!< in: record descriptor */
 {
 	ut_ad(index);
 

storage/innobase/row/row0mysql.c

@@ -503,7 +503,7 @@ row_mysql_convert_row_to_innobase(
 					row is used, as row may contain
 					pointers to this record! */
 {
-	mysql_row_templ_t*	templ;
+	const mysql_row_templ_t*templ;
 	dfield_t*		dfield;
 	ulint			i;
 

storage/innobase/row/row0sel.c

@@ -2667,21 +2667,22 @@ row_sel_store_mysql_rec(
 	row_prebuilt_t*	prebuilt,	/*!< in: prebuilt struct */
 	const rec_t*	rec,		/*!< in: Innobase record in the index
 					which was described in prebuilt's
-					template; must be protected by
-					a page latch */
+					template, or in the clustered index;
+					must be protected by a page latch */
+	ibool		rec_clust,	/*!< in: TRUE if rec is in the
+					clustered index instead of
+					prebuilt->index */
 	const ulint*	offsets)	/*!< in: array returned by
-					rec_get_offsets() */
+					rec_get_offsets(rec) */
 {
-	mysql_row_templ_t*	templ;
 	mem_heap_t*	extern_field_heap	= NULL;
 	mem_heap_t*	heap;
-	const byte*		data;
-	ulint			len;
 	ulint		i;
 
 	ut_ad(prebuilt->mysql_template);
 	ut_ad(prebuilt->default_rec);
 	ut_ad(rec_offs_validate(rec, NULL, offsets));
+	ut_ad(!rec_get_deleted_flag(rec, rec_offs_comp(offsets)));
 
 	if (UNIV_LIKELY_NULL(prebuilt->blob_heap)) {
 		mem_heap_free(prebuilt->blob_heap);
@@ -2690,10 +2691,15 @@ row_sel_store_mysql_rec(
 
 	for (i = 0; i < prebuilt->n_template; i++) {
 
-		templ = prebuilt->mysql_template + i;
+		const mysql_row_templ_t*templ = prebuilt->mysql_template + i;
+		const byte*		data;
+		ulint			len;
+		ulint			field_no;
 
-		if (UNIV_UNLIKELY(rec_offs_nth_extern(offsets,
-						      templ->rec_field_no))) {
+		field_no = rec_clust
+			? templ->clust_rec_field_no : templ->rec_field_no;
+
+		if (UNIV_UNLIKELY(rec_offs_nth_extern(offsets, field_no))) {
 
 			/* Copy an externally stored field to the temporary
 			heap */
@@ -2721,7 +2727,7 @@ row_sel_store_mysql_rec(
 			data = btr_rec_copy_externally_stored_field(
 				rec, offsets,
 				dict_table_zip_size(prebuilt->table),
-				templ->rec_field_no, &len, heap);
+				field_no, &len, heap);
 
 			if (UNIV_UNLIKELY(!data)) {
 				/* The externally stored field
@@ -2742,8 +2748,7 @@ row_sel_store_mysql_rec(
 		} else {
 			/* Field is stored in the row. */
 
-			data = rec_get_nth_field(rec, offsets,
-						 templ->rec_field_no, &len);
+			data = rec_get_nth_field(rec, offsets, field_no, &len);
 
 			if (UNIV_UNLIKELY(templ->type == DATA_BLOB)
 			    && len != UNIV_SQL_NULL) {
@@ -3105,7 +3110,7 @@ row_sel_pop_cached_row_for_mysql(
 	row_prebuilt_t*	prebuilt)	/*!< in: prebuilt struct */
 {
 	ulint			i;
-	mysql_row_templ_t*	templ;
+	const mysql_row_templ_t*templ;
 	byte*			cached_rec;
 	ut_ad(prebuilt->n_fetch_cached > 0);
 	ut_ad(prebuilt->mysql_prefix_len <= prebuilt->mysql_row_len);
@@ -3162,15 +3167,21 @@ ibool
 row_sel_push_cache_row_for_mysql(
 /*=============================*/
 	row_prebuilt_t*	prebuilt,	/*!< in: prebuilt struct */
-	const rec_t*	rec,		/*!< in: record to push; must
-					be protected by a page latch */
-	const ulint*	offsets)	/*!< in: rec_get_offsets() */
+	const rec_t*	rec,		/*!< in: record to push, in the index
+					which was described in prebuilt's
+					template, or in the clustered index;
+					must be protected by a page latch */
+	ibool		rec_clust,	/*!< in: TRUE if rec is in the
+					clustered index instead of
+					prebuilt->index */
+	const ulint*	offsets)	/*!< in: rec_get_offsets(rec) */
 {
 	byte*	buf;
 	ulint	i;
 
 	ut_ad(prebuilt->n_fetch_cached < MYSQL_FETCH_CACHE_SIZE);
 	ut_ad(rec_offs_validate(rec, NULL, offsets));
+	ut_ad(!rec_get_deleted_flag(rec, rec_offs_comp(offsets)));
 	ut_a(!prebuilt->templ_contains_blob);
 
 	if (prebuilt->fetch_cache[0] == NULL) {
@@ -3199,7 +3210,7 @@ row_sel_push_cache_row_for_mysql(
 	if (UNIV_UNLIKELY(!row_sel_store_mysql_rec(
 				  prebuilt->fetch_cache[
 					  prebuilt->n_fetch_cached],
-				  prebuilt, rec, offsets))) {
+				  prebuilt, rec, rec_clust, offsets))) {
 		return(FALSE);
 	}
 
@@ -3600,7 +3611,8 @@ row_search_for_mysql(
 				ut_ad(!rec_get_deleted_flag(rec, comp));
 
 				if (!row_sel_store_mysql_rec(buf, prebuilt,
-							     rec, offsets)) {
+							     rec, FALSE,
+							     offsets)) {
 					/* Only fresh inserts may contain
 					incomplete externally stored
 					columns. Pretend that such
@@ -4234,7 +4246,6 @@ no_gap_lock:
 			is necessary, because we can only get the undo
 			information via the clustered index record. */
 
-			ut_ad(index != clust_index);
 			ut_ad(!dict_index_is_clust(index));
 
 			if (!lock_sec_rec_cons_read_sees(
@@ -4350,34 +4361,19 @@ requires_clust_rec:
 			goto next_rec;
 		}
 
-		if (prebuilt->need_to_access_clustered) {
-
 		result_rec = clust_rec;
-
-			ut_ad(rec_offs_validate(result_rec, clust_index,
-						offsets));
+		ut_ad(rec_offs_validate(result_rec, clust_index, offsets));
 	} else {
-			/* We used 'offsets' for the clust rec, recalculate
-			them for 'rec' */
-			offsets = rec_get_offsets(rec, index, offsets,
-						  ULINT_UNDEFINED, &heap);
 		result_rec = rec;
 	}
 
-		/* result_rec can legitimately be delete-marked
-		now that it has been established that it points to a
-		clustered index record that exists in the read view. */
-	} else {
-		result_rec = rec;
-		ut_ad(!rec_get_deleted_flag(rec, comp));
-	}
-
 	/* We found a qualifying record 'result_rec'. At this point,
 	'offsets' are associated with 'result_rec'. */
 
 	ut_ad(rec_offs_validate(result_rec,
 				result_rec != rec ? clust_index : index,
 				offsets));
+	ut_ad(!rec_get_deleted_flag(result_rec, comp));
 
 	/* At this point, the clustered index record is protected
 	by a page latch that was acquired when pcur was positioned.
@@ -4402,6 +4398,7 @@ requires_clust_rec:
 		cursor. */
 
 		if (!row_sel_push_cache_row_for_mysql(prebuilt, result_rec,
+						      result_rec != rec,
 						      offsets)) {
 			/* Only fresh inserts may contain incomplete
 			externally stored columns. Pretend that such
@@ -4419,15 +4416,31 @@ requires_clust_rec:
 
 		goto next_rec;
 	} else {
-		if (prebuilt->template_type == ROW_MYSQL_DUMMY_TEMPLATE) {
+		if (UNIV_UNLIKELY
+		    (prebuilt->template_type == ROW_MYSQL_DUMMY_TEMPLATE)) {
+			/* CHECK TABLE: fetch the row */
+
+			if (result_rec != rec
+			    && !prebuilt->need_to_access_clustered) {
+				/* We used 'offsets' for the clust
+				rec, recalculate them for 'rec' */
+				offsets = rec_get_offsets(rec, index, offsets,
+							  ULINT_UNDEFINED,
+							  &heap);
+				result_rec = rec;
+			}
+
 			memcpy(buf + 4, result_rec
 			       - rec_offs_extra_size(offsets),
 			       rec_offs_size(offsets));
 			mach_write_to_4(buf,
 					rec_offs_extra_size(offsets) + 4);
 		} else {
-			if (!row_sel_store_mysql_rec(buf, prebuilt,
-						     result_rec, offsets)) {
+			/* Returning a row to MySQL */
+
+			if (!row_sel_store_mysql_rec(buf, prebuilt, result_rec,
+						     result_rec != rec,
+						     offsets)) {
 				/* Only fresh inserts may contain
 				incomplete externally stored
 				columns. Pretend that such records do

storage/innobase/row/row0upd.c

@@ -466,8 +466,11 @@ row_upd_changes_field_size_or_external(
 #endif /* !UNIV_HOTBACKUP */
 
 /***********************************************************//**
-Replaces the new column values stored in the update vector to the record
-given. No field size changes are allowed. */
+Replaces the new column values stored in the update vector to the
+record given. No field size changes are allowed. This function is
+usually invoked on a clustered index. The only use case for a
+secondary index is row_ins_sec_index_entry_by_modify() or its
+counterpart in ibuf_insert_to_index_page(). */
 UNIV_INTERN
 void
 row_upd_rec_in_place(