MDEV-11831 Make InnoDB mini-transaction memo checks stricter

InnoDB keeps track of buffer-fixed buf_block_t or acquired rw_lock_t
within a mini-transaction. There are some memo_contains assertions
in the code that document when certain blocks or rw_locks must be held.
But, these assertions only check the mini-transaction memo, not the fact
whether the rw_lock_t are actually being held by the caller.

btr_pcur_store_position(): Remove #ifdef, and assert that the block
is always buffer-fixed.

rtr_pcur_getnext_from_path(), rtr_pcur_open_low(),
ibuf_rec_get_page_no_func(), ibuf_rec_get_space_func(),
ibuf_rec_get_info_func(), ibuf_rec_get_op_type_func(),
ibuf_build_entry_from_ibuf_rec_func(), ibuf_rec_get_volume_func(),
ibuf_get_merge_page_nos_func(), ibuf_get_volume_buffered_count_func()
ibuf_get_entry_counter_low_func(), page_set_ssn_id(),
row_vers_old_has_index_entry(), row_vers_build_for_consistent_read(),
row_vers_build_for_semi_consistent_read(),
trx_undo_prev_version_build():
Make use of mtr_memo_contains_page_flagged().

mtr_t::memo_contains(): Take a const memo. Assert rw_lock_own().

FindPage, FlaggedCheck: Assert rw_lock_own_flagged().

storage/innobase/btr/btr0pcur.cc

 /*****************************************************************************
 
 Copyright (c) 1996, 2015, Oracle and/or its affiliates. All Rights Reserved.
-Copyright (c) 2016, MariaDB Corporation.
+Copyright (c) 2016, 2017, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -120,24 +120,17 @@ btr_pcur_store_position(
 	page = page_align(rec);
 	offs = page_offset(rec);
 
-#ifdef UNIV_DEBUG
-	if (dict_index_is_spatial(index)) {
-		/* For spatial index, when we do positioning on parent
-		buffer if necessary, it might not hold latches, but the
-		tree must be locked to prevent change on the page */
-		ut_ad((mtr_memo_contains_flagged(
-				mtr, dict_index_get_lock(index),
-				MTR_MEMO_X_LOCK | MTR_MEMO_SX_LOCK)
-		       || mtr_memo_contains_flagged(mtr, block,
-						    MTR_MEMO_PAGE_S_FIX
-						    | MTR_MEMO_PAGE_X_FIX))
-		      && (block->page.buf_fix_count > 0));
-	} else {
-		ut_ad(mtr_memo_contains_flagged(mtr, block,
-						MTR_MEMO_PAGE_S_FIX
-						| MTR_MEMO_PAGE_X_FIX));
-	}
-#endif /* UNIV_DEBUG */
+	ut_ad(block->page.buf_fix_count);
+	/* For spatial index, when we do positioning on parent
+	buffer if necessary, it might not hold latches, but the
+	tree must be locked to prevent change on the page */
+	ut_ad(mtr_memo_contains_flagged(mtr, block,
+					MTR_MEMO_PAGE_S_FIX
+					| MTR_MEMO_PAGE_X_FIX)
+	      || (dict_index_is_spatial(index)
+		  && mtr_memo_contains_flagged(
+			  mtr, dict_index_get_lock(index),
+			  MTR_MEMO_X_LOCK | MTR_MEMO_SX_LOCK)));
 
 	if (page_is_empty(page)) {
 		/* It must be an empty index tree; NOTE that in this case

storage/innobase/gis/gis0sea.cc

 /*****************************************************************************
 
 Copyright (c) 2016, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2017, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -129,12 +130,10 @@ rtr_pcur_getnext_from_path(
 		      || latch_mode & BTR_MODIFY_LEAF);
 		mtr_s_lock(dict_index_get_lock(index), mtr);
 	} else {
-		ut_ad(mtr_memo_contains(mtr, dict_index_get_lock(index),
-					MTR_MEMO_SX_LOCK)
-		      || mtr_memo_contains(mtr, dict_index_get_lock(index),
-					MTR_MEMO_S_LOCK)
-		      || mtr_memo_contains(mtr, dict_index_get_lock(index),
-					   MTR_MEMO_X_LOCK));
+		ut_ad(mtr_memo_contains_flagged(mtr, &index->lock,
+						MTR_MEMO_SX_LOCK
+						| MTR_MEMO_S_LOCK
+						| MTR_MEMO_X_LOCK));
 	}
 
 	const page_size_t&	page_size = dict_table_page_size(index->table);
@@ -601,10 +600,9 @@ rtr_pcur_open_low(
 	}
 
 	if (latch_mode & BTR_MODIFY_TREE) {
-		ut_ad(mtr_memo_contains(mtr, dict_index_get_lock(index),
-					MTR_MEMO_X_LOCK)
-		      || mtr_memo_contains(mtr, dict_index_get_lock(index),
-					   MTR_MEMO_SX_LOCK));
+		ut_ad(mtr_memo_contains_flagged(mtr, &index->lock,
+						MTR_MEMO_X_LOCK
+						| MTR_MEMO_SX_LOCK));
 		tree_latched = true;
 	}
 

storage/innobase/ibuf/ibuf0ibuf.cc

 /*****************************************************************************
 
 Copyright (c) 1997, 2016, Oracle and/or its affiliates. All Rights Reserved.
-Copyright (c) 2016, MariaDB Corporation.
+Copyright (c) 2016, 2017, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -1227,8 +1227,9 @@ ibuf_rec_get_page_no_func(
 	const byte*	field;
 	ulint		len;
 
-	ut_ad(mtr_memo_contains_page(mtr, rec, MTR_MEMO_PAGE_X_FIX)
-	      || mtr_memo_contains_page(mtr, rec, MTR_MEMO_PAGE_S_FIX));
+	ut_ad(mtr_memo_contains_page_flagged(mtr, rec,
+					     MTR_MEMO_PAGE_X_FIX
+					     | MTR_MEMO_PAGE_S_FIX));
 	ut_ad(ibuf_inside(mtr));
 	ut_ad(rec_get_n_fields_old(rec) > 2);
 
@@ -1265,8 +1266,8 @@ ibuf_rec_get_space_func(
 	const byte*	field;
 	ulint		len;
 
-	ut_ad(mtr_memo_contains_page(mtr, rec, MTR_MEMO_PAGE_X_FIX)
-	      || mtr_memo_contains_page(mtr, rec, MTR_MEMO_PAGE_S_FIX));
+	ut_ad(mtr_memo_contains_page_flagged(mtr, rec, MTR_MEMO_PAGE_X_FIX
+					     | MTR_MEMO_PAGE_S_FIX));
 	ut_ad(ibuf_inside(mtr));
 	ut_ad(rec_get_n_fields_old(rec) > 2);
 
@@ -1315,8 +1316,8 @@ ibuf_rec_get_info_func(
 	ulint		info_len_local;
 	ulint		counter_local;
 
-	ut_ad(mtr_memo_contains_page(mtr, rec, MTR_MEMO_PAGE_X_FIX)
-	      || mtr_memo_contains_page(mtr, rec, MTR_MEMO_PAGE_S_FIX));
+	ut_ad(mtr_memo_contains_page_flagged(mtr, rec, MTR_MEMO_PAGE_X_FIX
+					     | MTR_MEMO_PAGE_S_FIX));
 	ut_ad(ibuf_inside(mtr));
 	fields = rec_get_n_fields_old(rec);
 	ut_a(fields > IBUF_REC_FIELD_USER);
@@ -1387,8 +1388,8 @@ ibuf_rec_get_op_type_func(
 {
 	ulint		len;
 
-	ut_ad(mtr_memo_contains_page(mtr, rec, MTR_MEMO_PAGE_X_FIX)
-	      || mtr_memo_contains_page(mtr, rec, MTR_MEMO_PAGE_S_FIX));
+	ut_ad(mtr_memo_contains_page_flagged(mtr, rec, MTR_MEMO_PAGE_X_FIX
+					     | MTR_MEMO_PAGE_S_FIX));
 	ut_ad(ibuf_inside(mtr));
 	ut_ad(rec_get_n_fields_old(rec) > 2);
 
@@ -1586,8 +1587,8 @@ ibuf_build_entry_from_ibuf_rec_func(
 	ulint		comp;
 	dict_index_t*	index;
 
-	ut_ad(mtr_memo_contains_page(mtr, ibuf_rec, MTR_MEMO_PAGE_X_FIX)
-	      || mtr_memo_contains_page(mtr, ibuf_rec, MTR_MEMO_PAGE_S_FIX));
+	ut_ad(mtr_memo_contains_page_flagged(mtr, ibuf_rec, MTR_MEMO_PAGE_X_FIX
+					     | MTR_MEMO_PAGE_S_FIX));
 	ut_ad(ibuf_inside(mtr));
 
 	data = rec_get_nth_field_old(ibuf_rec, IBUF_REC_FIELD_MARKER, &len);
@@ -1708,8 +1709,8 @@ ibuf_rec_get_volume_func(
 	ibuf_op_t	op;
 	ulint		info_len;
 
-	ut_ad(mtr_memo_contains_page(mtr, ibuf_rec, MTR_MEMO_PAGE_X_FIX)
-	      || mtr_memo_contains_page(mtr, ibuf_rec, MTR_MEMO_PAGE_S_FIX));
+	ut_ad(mtr_memo_contains_page_flagged(mtr, ibuf_rec, MTR_MEMO_PAGE_X_FIX
+					     | MTR_MEMO_PAGE_S_FIX));
 	ut_ad(ibuf_inside(mtr));
 	ut_ad(rec_get_n_fields_old(ibuf_rec) > 2);
 
@@ -2291,8 +2292,8 @@ ibuf_get_merge_page_nos_func(
 	ulint	limit;
 	ulint	n_pages;
 
-	ut_ad(mtr_memo_contains_page(mtr, rec, MTR_MEMO_PAGE_X_FIX)
-	      || mtr_memo_contains_page(mtr, rec, MTR_MEMO_PAGE_S_FIX));
+	ut_ad(mtr_memo_contains_page_flagged(mtr, rec, MTR_MEMO_PAGE_X_FIX
+					     | MTR_MEMO_PAGE_S_FIX));
 	ut_ad(ibuf_inside(mtr));
 
 	*n_stored = 0;
@@ -2875,8 +2876,8 @@ ibuf_get_volume_buffered_count_func(
 	const byte*	types;
 	ulint		n_fields;
 
-	ut_ad(mtr_memo_contains_page(mtr, rec, MTR_MEMO_PAGE_X_FIX)
-	      || mtr_memo_contains_page(mtr, rec, MTR_MEMO_PAGE_S_FIX));
+	ut_ad(mtr_memo_contains_page_flagged(mtr, rec, MTR_MEMO_PAGE_X_FIX
+					     | MTR_MEMO_PAGE_S_FIX));
 	ut_ad(ibuf_inside(mtr));
 
 	n_fields = rec_get_n_fields_old(rec);
@@ -3248,8 +3249,8 @@ ibuf_get_entry_counter_low_func(
 	ulint		len;
 
 	ut_ad(ibuf_inside(mtr));
-	ut_ad(mtr_memo_contains_page(mtr, rec, MTR_MEMO_PAGE_X_FIX)
-	      || mtr_memo_contains_page(mtr, rec, MTR_MEMO_PAGE_S_FIX));
+	ut_ad(mtr_memo_contains_page_flagged(mtr, rec, MTR_MEMO_PAGE_X_FIX
+					     | MTR_MEMO_PAGE_S_FIX));
 	ut_ad(rec_get_n_fields_old(rec) > 2);
 
 	field = rec_get_nth_field_old(rec, IBUF_REC_FIELD_MARKER, &len);

storage/innobase/include/mtr0mtr.h

@@ -2,7 +2,7 @@
 
 Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved.
 Copyright (c) 2012, Facebook Inc.
-Copyright (c) 2013, 2016, MariaDB Corporation
+Copyright (c) 2013, 2017, MariaDB Corporation
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -487,9 +487,9 @@ struct mtr_t {
 	@param type	type of object
 	@return	true if contains */
 	static bool memo_contains(
-		mtr_buf_t*	memo,
-		const void*	object,
-		ulint		type)
+		const mtr_buf_t*	memo,
+		const void*		object,
+		ulint			type)
 		MY_ATTRIBUTE((warn_unused_result));
 
 	/** Check if memo contains the given item.

storage/innobase/include/mtr0types.h

 /*****************************************************************************
 
 Copyright (c) 1995, 2015, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2017, MariaDB Corporation
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -257,10 +258,10 @@ The record consists of a MLOG_CHECKPOINT byte followed by
 mach_write_to_8(checkpoint_lsn). */
 #define SIZE_OF_MLOG_CHECKPOINT	9
 
+#ifndef UNIV_INNOCHECKSUM
 /** Types for the mlock objects to store in the mtr memo; NOTE that the
 first 3 values must be RW_S_LATCH, RW_X_LATCH, RW_NO_LATCH */
 enum mtr_memo_type_t {
-#ifndef UNIV_INNOCHECKSUM
 	MTR_MEMO_PAGE_S_FIX = RW_S_LATCH,
 
 	MTR_MEMO_PAGE_X_FIX = RW_X_LATCH,
@@ -268,18 +269,18 @@ enum mtr_memo_type_t {
 	MTR_MEMO_PAGE_SX_FIX = RW_SX_LATCH,
 
 	MTR_MEMO_BUF_FIX = RW_NO_LATCH,
-#endif /* !UNIV_CHECKSUM */
 
 #ifdef UNIV_DEBUG
-	MTR_MEMO_MODIFY = 32,
+	MTR_MEMO_MODIFY = 16,
 #endif /* UNIV_DEBUG */
 
-	MTR_MEMO_S_LOCK = 64,
+	MTR_MEMO_S_LOCK = RW_S_LATCH << 5,
 
-	MTR_MEMO_X_LOCK = 128,
+	MTR_MEMO_X_LOCK = RW_X_LATCH << 5,
 
-	MTR_MEMO_SX_LOCK = 256
+	MTR_MEMO_SX_LOCK = RW_SX_LATCH << 5
 };
+#endif /* !UNIV_CHECKSUM */
 
 #ifdef UNIV_DEBUG
 # define MTR_MAGIC_N		54551

storage/innobase/include/page0page.ic

 /*****************************************************************************
 
 Copyright (c) 1994, 2015, Oracle and/or its affiliates. All Rights Reserved.
-Copyright (c) 2016, MariaDB Corporation.
+Copyright (c) 2016, 2017, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -147,8 +147,9 @@ page_set_ssn_id(
 {
 	page_t*	page = buf_block_get_frame(block);
 
-	ut_ad(!mtr || mtr_memo_contains(mtr, block, MTR_MEMO_PAGE_SX_FIX)
-	      || mtr_memo_contains(mtr, block, MTR_MEMO_PAGE_X_FIX));
+	ut_ad(!mtr || mtr_memo_contains_flagged(mtr, block,
+						MTR_MEMO_PAGE_SX_FIX
+						| MTR_MEMO_PAGE_X_FIX));
 
 	if (page_zip) {
 		mach_write_to_8(page + FIL_RTREE_SPLIT_SEQ_NUM, ssn_id);

storage/innobase/mtr/mtr0mtr.cc

 /*****************************************************************************
 
 Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2017, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -120,6 +121,8 @@ struct FindPage
 	FindPage(const void* ptr, ulint flags)
 		: m_ptr(ptr), m_flags(flags), m_slot(NULL)
 	{
+		/* There must be some flags to look for. */
+		ut_ad(flags);
 		/* We can only look for page-related flags. */
 		ut_ad(!(flags & ~(MTR_MEMO_PAGE_S_FIX
 				  | MTR_MEMO_PAGE_X_FIX
@@ -148,6 +151,11 @@ struct FindPage
 			return(true);
 		}
 
+		ut_ad(!(m_flags & (MTR_MEMO_PAGE_S_FIX
+				   | MTR_MEMO_PAGE_SX_FIX
+				   | MTR_MEMO_PAGE_X_FIX))
+		      || rw_lock_own_flagged(&block->lock, m_flags));
+
 		m_slot = slot;
 		return(false);
 	}
@@ -1009,14 +1017,30 @@ mtr_t::release_free_extents(ulint n_reserved)
 @return	true if contains */
 bool
 mtr_t::memo_contains(
-	mtr_buf_t*	memo,
-	const void*	object,
-	ulint		type)
+	const mtr_buf_t*	memo,
+	const void*		object,
+	ulint			type)
 {
 	Find		find(object, type);
 	Iterate<Find>	iterator(find);
 
-	return(!memo->for_each_block_in_reverse(iterator));
+	if (memo->for_each_block_in_reverse(iterator)) {
+		return(false);
+	}
+
+	switch (type) {
+	case MTR_MEMO_X_LOCK:
+		ut_ad(rw_lock_own((rw_lock_t*) object, RW_LOCK_X));
+		break;
+	case MTR_MEMO_SX_LOCK:
+		ut_ad(rw_lock_own((rw_lock_t*) object, RW_LOCK_SX));
+		break;
+	case MTR_MEMO_S_LOCK:
+		ut_ad(rw_lock_own((rw_lock_t*) object, RW_LOCK_S));
+		break;
+	}
+
+	return(true);
 }
 
 /** Debug check for flags */
@@ -1026,20 +1050,56 @@ struct FlaggedCheck {
 		m_ptr(ptr),
 		m_flags(flags)
 	{
-		// Do nothing
+		/* There must be some flags to look for. */
+		ut_ad(flags);
+		/* Look for rw-lock-related and page-related flags. */
+		ut_ad(!(flags & ~(MTR_MEMO_PAGE_S_FIX
+				  | MTR_MEMO_PAGE_X_FIX
+				  | MTR_MEMO_PAGE_SX_FIX
+				  | MTR_MEMO_BUF_FIX
+				  | MTR_MEMO_MODIFY
+				  | MTR_MEMO_X_LOCK
+				  | MTR_MEMO_SX_LOCK
+				  | MTR_MEMO_S_LOCK)));
+		/* Either some rw-lock-related or page-related flags
+		must be specified, but not both at the same time. */
+		ut_ad(!(flags & (MTR_MEMO_PAGE_S_FIX
+				 | MTR_MEMO_PAGE_X_FIX
+				 | MTR_MEMO_PAGE_SX_FIX
+				 | MTR_MEMO_BUF_FIX
+				 | MTR_MEMO_MODIFY))
+		      == !!(flags & (MTR_MEMO_X_LOCK
+				     | MTR_MEMO_SX_LOCK
+				     | MTR_MEMO_S_LOCK)));
 	}
 
+	/** Visit a memo entry.
+	@param[in]	slot	memo entry to visit
+	@retval	false	if m_ptr was found
+	@retval	true	if the iteration should continue */
 	bool operator()(const mtr_memo_slot_t* slot) const
 	{
-		if (m_ptr == slot->object && (m_flags & slot->type)) {
-			return(false);
+		if (m_ptr != slot->object || !(m_flags & slot->type)) {
+			return(true);
 		}
 
-		return(true);
+		if (ulint flags = m_flags & (MTR_MEMO_PAGE_S_FIX
+					     | MTR_MEMO_PAGE_SX_FIX
+					     | MTR_MEMO_PAGE_X_FIX)) {
+			rw_lock_t* lock = &static_cast<buf_block_t*>(
+				const_cast<void*>(m_ptr))->lock;
+			ut_ad(rw_lock_own_flagged(lock, flags));
+		} else {
+			rw_lock_t* lock = static_cast<rw_lock_t*>(
+				const_cast<void*>(m_ptr));
+			ut_ad(rw_lock_own_flagged(lock, m_flags >> 5));
+		}
+
+		return(false);
 	}
 
-	const void*	m_ptr;
-	ulint		m_flags;
+	const void*const	m_ptr;
+	const ulint		m_flags;
 };
 
 /** Check if memo contains the given item.

storage/innobase/row/row0vers.cc

 /*****************************************************************************
 
 Copyright (c) 1997, 2016, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2017, MariaDB Corporation
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -876,8 +877,8 @@ row_vers_old_has_index_entry(
 	mem_heap_t*	v_heap = NULL;
 	const dtuple_t*	cur_vrow = NULL;
 
-	ut_ad(mtr_memo_contains_page(mtr, rec, MTR_MEMO_PAGE_X_FIX)
-	      || mtr_memo_contains_page(mtr, rec, MTR_MEMO_PAGE_S_FIX));
+	ut_ad(mtr_memo_contains_page_flagged(mtr, rec, MTR_MEMO_PAGE_X_FIX
+					     | MTR_MEMO_PAGE_S_FIX));
 	ut_ad(!rw_lock_own(&(purge_sys->latch), RW_LOCK_S));
 
 	clust_index = dict_table_get_first_index(index->table);
@@ -1134,8 +1135,8 @@ row_vers_build_for_consistent_read(
 	dberr_t		err;
 
 	ut_ad(dict_index_is_clust(index));
-	ut_ad(mtr_memo_contains_page(mtr, rec, MTR_MEMO_PAGE_X_FIX)
-	      || mtr_memo_contains_page(mtr, rec, MTR_MEMO_PAGE_S_FIX));
+	ut_ad(mtr_memo_contains_page_flagged(mtr, rec, MTR_MEMO_PAGE_X_FIX
+					     | MTR_MEMO_PAGE_S_FIX));
 	ut_ad(!rw_lock_own(&(purge_sys->latch), RW_LOCK_S));
 
 	ut_ad(rec_offs_validate(rec, index, *offsets));
@@ -1246,8 +1247,8 @@ row_vers_build_for_semi_consistent_read(
 	trx_id_t	rec_trx_id	= 0;
 
 	ut_ad(dict_index_is_clust(index));
-	ut_ad(mtr_memo_contains_page(mtr, rec, MTR_MEMO_PAGE_X_FIX)
-	      || mtr_memo_contains_page(mtr, rec, MTR_MEMO_PAGE_S_FIX));
+	ut_ad(mtr_memo_contains_page_flagged(mtr, rec, MTR_MEMO_PAGE_X_FIX
+					     | MTR_MEMO_PAGE_S_FIX));
 	ut_ad(!rw_lock_own(&(purge_sys->latch), RW_LOCK_S));
 
 	ut_ad(rec_offs_validate(rec, index, *offsets));

storage/innobase/trx/trx0rec.cc

 /*****************************************************************************
 
 Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2017, MariaDB Corporation
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -2234,9 +2235,9 @@ trx_undo_prev_version_build(
 	byte*		buf;
 
 	ut_ad(!rw_lock_own(&purge_sys->latch, RW_LOCK_S));
-	ut_ad(mtr_memo_contains_page(index_mtr, index_rec, MTR_MEMO_PAGE_S_FIX)
-	      || mtr_memo_contains_page(index_mtr, index_rec,
-					MTR_MEMO_PAGE_X_FIX));
+	ut_ad(mtr_memo_contains_page_flagged(index_mtr, index_rec,
+					     MTR_MEMO_PAGE_S_FIX
+					     | MTR_MEMO_PAGE_X_FIX));
 	ut_ad(rec_offs_validate(rec, index, offsets));
 	ut_a(dict_index_is_clust(index));