Add page_has_prev(), page_has_next(), page_has_siblings()

Until now, InnoDB inefficiently compared the aligned fields
FIL_PAGE_PREV, FIL_PAGE_NEXT to the byte-order-agnostic value FIL_NULL.

storage/innobase/btr/btr0btr.cc

@@ -2,7 +2,7 @@
 
 Copyright (c) 1994, 2016, Oracle and/or its affiliates. All Rights Reserved.
 Copyright (c) 2012, Facebook Inc.
-Copyright (c) 2014, 2017, MariaDB Corporation.
+Copyright (c) 2014, 2018, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -2149,7 +2149,7 @@ btr_root_raise_and_insert(
 	they should already have been set.  The previous node field
 	must be FIL_NULL if root_page_zip != NULL, because the
 	REC_INFO_MIN_REC_FLAG (of the first user record) will be
-	set if and only if btr_page_get_prev() == FIL_NULL. */
+	set if and only if !page_has_prev(). */
 	btr_page_set_next(root, root_page_zip, FIL_NULL, mtr);
 	btr_page_set_prev(root, root_page_zip, FIL_NULL, mtr);
 
@@ -3542,8 +3542,7 @@ btr_lift_page_up(
 	bool		lift_father_up;
 	buf_block_t*	block_orig	= block;
 
-	ut_ad(btr_page_get_prev(page, mtr) == FIL_NULL);
-	ut_ad(btr_page_get_next(page, mtr) == FIL_NULL);
+	ut_ad(!page_has_siblings(page));
 	ut_ad(mtr_is_block_fix(mtr, block, MTR_MEMO_PAGE_X_FIX, index->table));
 
 	page_level = btr_page_get_level(page, mtr);
@@ -3610,8 +3609,7 @@ btr_lift_page_up(
 			page = buf_block_get_frame(block);
 			page_level = btr_page_get_level(page, mtr);
 
-			ut_ad(btr_page_get_prev(page, mtr) == FIL_NULL);
-			ut_ad(btr_page_get_next(page, mtr) == FIL_NULL);
+			ut_ad(!page_has_siblings(page));
 			ut_ad(mtr_is_block_fix(
 				mtr, block, MTR_MEMO_PAGE_X_FIX, index->table));
 
@@ -4276,8 +4274,7 @@ btr_discard_only_page_on_level(
 
 		ut_a(page_get_n_recs(page) == 1);
 		ut_a(page_level == btr_page_get_level(page, mtr));
-		ut_a(btr_page_get_prev(page, mtr) == FIL_NULL);
-		ut_a(btr_page_get_next(page, mtr) == FIL_NULL);
+		ut_a(!page_has_siblings(page));
 
 		ut_ad(mtr_is_block_fix(
 			mtr, block, MTR_MEMO_PAGE_X_FIX, index->table));
@@ -5281,13 +5278,13 @@ btr_validate_level(
 		if (left_page_no == FIL_NULL) {
 			ut_a(node_ptr == page_rec_get_next(
 				     page_get_infimum_rec(father_page)));
-			ut_a(btr_page_get_prev(father_page, &mtr) == FIL_NULL);
+			ut_a(!page_has_prev(father_page));
 		}
 
 		if (right_page_no == FIL_NULL) {
 			ut_a(node_ptr == page_rec_get_prev(
 				     page_get_supremum_rec(father_page)));
-			ut_a(btr_page_get_next(father_page, &mtr) == FIL_NULL);
+			ut_a(!page_has_next(father_page));
 		} else {
 			const rec_t*	right_node_ptr;
 

storage/innobase/btr/btr0cur.cc

@@ -804,10 +804,10 @@ btr_cur_will_modify_tree(
 
 		/* is first, 2nd or last record */
 		if (page_rec_is_first(rec, page)
-		    || (mach_read_from_4(page + FIL_PAGE_NEXT) != FIL_NULL
+		    || (page_has_next(page)
 			&& (page_rec_is_last(rec, page)
 			    || page_rec_is_second_last(rec, page)))
-		    || (mach_read_from_4(page + FIL_PAGE_PREV) != FIL_NULL
+		    || (page_has_prev(page)
 			&& page_rec_is_second(rec, page))) {
 			return(true);
 		}
@@ -892,13 +892,10 @@ btr_cur_need_opposite_intention(
 {
 	switch (lock_intention) {
 	case BTR_INTENTION_DELETE:
-		return((mach_read_from_4(page + FIL_PAGE_PREV) != FIL_NULL
-			&& page_rec_is_first(rec, page))
-		       || (mach_read_from_4(page + FIL_PAGE_NEXT) != FIL_NULL
-			   && page_rec_is_last(rec, page)));
+		return (page_has_prev(page) && page_rec_is_first(rec, page)) ||
+			(page_has_next(page) && page_rec_is_last(rec, page));
 	case BTR_INTENTION_INSERT:
-		return(mach_read_from_4(page + FIL_PAGE_NEXT) != FIL_NULL
-		       && page_rec_is_last(rec, page));
+		return page_has_next(page) && page_rec_is_last(rec, page);
 	case BTR_INTENTION_BOTH:
 		return(false);
 	}
@@ -1970,7 +1967,7 @@ btr_cur_search_to_nth_level_func(
 				MTR_MEMO_PAGE_S_FIX
 				| MTR_MEMO_PAGE_X_FIX));
 
-			if (btr_page_get_prev(page, mtr) != FIL_NULL
+			if (page_has_prev(page)
 			    && page_rec_is_first(node_ptr, page)) {
 
 				if (leftmost_from_level == 0) {
@@ -2087,7 +2084,7 @@ btr_cur_search_to_nth_level_func(
 	} else if (!dict_index_is_spatial(index)
 		   && latch_mode == BTR_MODIFY_TREE
 		   && lock_intention == BTR_INTENTION_INSERT
-		   && mach_read_from_4(page + FIL_PAGE_NEXT) != FIL_NULL
+		   && page_has_next(page)
 		   && page_rec_is_last(page_cur_get_rec(page_cursor), page)) {
 
 		/* btr_insert_into_right_sibling() might cause
@@ -5632,7 +5629,7 @@ btr_cur_pessimistic_delete(
 
 		rec_t*	next_rec = page_rec_get_next(rec);
 
-		if (btr_page_get_prev(page, mtr) == FIL_NULL) {
+		if (!page_has_prev(page)) {
 
 			/* If we delete the leftmost node pointer on a
 			non-leaf level, we must mark the new leftmost node
@@ -6614,7 +6611,8 @@ btr_estimate_number_of_different_key_vals(
 			}
 		}
 
-		if (n_cols == dict_index_get_n_unique_in_tree(index)) {
+		if (n_cols == dict_index_get_n_unique_in_tree(index)
+		    && page_has_siblings(page)) {
 
 			/* If there is more than one leaf page in the tree,
 			we add one because we know that the first record
@@ -6625,12 +6623,8 @@ btr_estimate_number_of_different_key_vals(
 			algorithm grossly underestimated the number of rows
 			in the table. */
 
-			if (btr_page_get_prev(page, &mtr) != FIL_NULL
-			    || btr_page_get_next(page, &mtr) != FIL_NULL) {
-
 			n_diff[n_cols - 1]++;
 		}
-		}
 
 		mtr_commit(&mtr);
 	}

storage/innobase/btr/btr0defragment.cc

 /*****************************************************************************
 
 Copyright (C) 2013, 2014 Facebook, Inc. All Rights Reserved.
-Copyright (C) 2014, 2017, MariaDB Corporation.
+Copyright (C) 2014, 2018, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -650,7 +650,7 @@ btr_defragment_n_pages(
 	}
 
 	if (n_pages == 1) {
-		if (btr_page_get_prev(first_page, mtr) == FIL_NULL) {
+		if (!page_has_prev(first_page)) {
 			/* last page in the index */
 			if (dict_index_get_page(index)
 			    == page_get_page_no(first_page))

storage/innobase/btr/btr0pcur.cc

@@ -132,8 +132,7 @@ btr_pcur_store_position(
 		we do not store the modify_clock, but always do a search
 		if we restore the cursor position */
 
-		ut_a(btr_page_get_next(page, mtr) == FIL_NULL);
-		ut_a(btr_page_get_prev(page, mtr) == FIL_NULL);
+		ut_a(!page_has_siblings(page));
 		ut_ad(page_is_leaf(page));
 		ut_ad(page_get_page_no(page) == index->page);
 

storage/innobase/dict/dict0stats.cc

@@ -1083,7 +1083,7 @@ dict_stats_analyze_index_level(
 	ut_a(btr_page_get_level(page, mtr) == level);
 
 	/* there should not be any pages on the left */
-	ut_a(btr_page_get_prev(page, mtr) == FIL_NULL);
+	ut_a(!page_has_prev(page));
 
 	if (REC_INFO_MIN_REC_FLAG & rec_get_info_bits(
 		    btr_pcur_get_rec(&pcur), page_is_comp(page))) {
@@ -1704,7 +1704,7 @@ dict_stats_analyze_index_for_n_prefix(
 	ut_a(btr_page_get_level(page, mtr) == n_diff_data->level);
 
 	/* there should not be any pages on the left */
-	ut_a(btr_page_get_prev(page, mtr) == FIL_NULL);
+	ut_a(!page_has_prev(page));
 
 	/* check whether the first record on the leftmost page is marked
 	as such; we are on a non-leaf level */

storage/innobase/gis/gis0rtree.cc

 /*****************************************************************************
 
 Copyright (c) 2016, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2018, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -1055,7 +1056,7 @@ rtr_page_split_and_insert(
 
 	page_no = block->page.id.page_no();
 
-	if (btr_page_get_prev(page, mtr) == FIL_NULL && !page_is_leaf(page)) {
+	if (!page_has_prev(page) && !page_is_leaf(page)) {
 		first_rec = page_rec_get_next(
 			page_get_infimum_rec(buf_block_get_frame(block)));
 	}

storage/innobase/gis/gis0sea.cc

@@ -1716,7 +1716,7 @@ rtr_cur_search_with_match(
 		first page as much as possible, as there will be problem
 		when update MIN_REC rec in compress table */
 		if (buf_block_get_page_zip(block)
-		    && mach_read_from_4(page + FIL_PAGE_PREV) == FIL_NULL
+		    && !page_has_prev(page)
 		    && page_get_n_recs(page) >= 2) {
 
 			rec = page_rec_get_next_const(rec);

storage/innobase/ibuf/ibuf0ibuf.cc

@@ -3323,8 +3323,7 @@ ibuf_get_entry_counter_func(
 		return(ULINT_UNDEFINED);
 	} else if (!page_rec_is_infimum(rec)) {
 		return(ibuf_get_entry_counter_low(mtr, rec, space, page_no));
-	} else if (only_leaf
-		   || fil_page_get_prev(page_align(rec)) == FIL_NULL) {
+	} else if (only_leaf || !page_has_prev(page_align(rec))) {
 		/* The parent node pointer did not contain the
 		searched for (space, page_no), which means that the
 		search ended on the correct page regardless of the

storage/innobase/include/btr0cur.ic

 /*****************************************************************************
 
 Copyright (c) 1994, 2015, Oracle and/or its affiliates. All Rights Reserved.
+Copyright (c) 2018, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -137,10 +138,9 @@ btr_cur_compress_recommendation(
 	LIMIT_OPTIMISTIC_INSERT_DEBUG(page_get_n_recs(page) * 2U,
 				      return(FALSE));
 
-	if ((page_get_data_size(page)
-	     < BTR_CUR_PAGE_COMPRESS_LIMIT(cursor->index))
-	    || ((btr_page_get_next(page, mtr) == FIL_NULL)
-		&& (btr_page_get_prev(page, mtr) == FIL_NULL))) {
+	if (page_get_data_size(page)
+	    < BTR_CUR_PAGE_COMPRESS_LIMIT(cursor->index)
+	    || !page_has_siblings(page)) {
 
 		/* The page fillfactor has dropped below a predefined
 		minimum value OR the level in the B-tree contains just
@@ -173,11 +173,9 @@ btr_cur_can_delete_without_compress(
 
 	page = btr_cur_get_page(cursor);
 
-	if ((page_get_data_size(page) - rec_size
-	     < BTR_CUR_PAGE_COMPRESS_LIMIT(cursor->index))
-	    || ((btr_page_get_next(page, mtr) == FIL_NULL)
-		&& (btr_page_get_prev(page, mtr) == FIL_NULL))
-	    || (page_get_n_recs(page) < 2)) {
+	if (page_get_data_size(page) - rec_size
+	    < BTR_CUR_PAGE_COMPRESS_LIMIT(cursor->index)
+	    || !page_has_siblings(page) || page_get_n_recs(page) < 2) {
 
 		/* The page fillfactor will drop below a predefined
 		minimum value, OR the level in the B-tree contains just

storage/innobase/include/btr0pcur.ic

@@ -219,12 +219,8 @@ btr_pcur_is_before_first_in_tree(
 	ut_ad(cursor->pos_state == BTR_PCUR_IS_POSITIONED);
 	ut_ad(cursor->latch_mode != BTR_NO_LATCHES);
 
-	if (btr_page_get_prev(btr_pcur_get_page(cursor), mtr) != FIL_NULL) {
-
-		return(FALSE);
-	}
-
-	return(page_cur_is_before_first(btr_pcur_get_page_cur(cursor)));
+	return !page_has_prev(btr_pcur_get_page(cursor))
+		&& page_cur_is_before_first(btr_pcur_get_page_cur(cursor));
 }
 
 /*********************************************************//**
@@ -240,12 +236,8 @@ btr_pcur_is_after_last_in_tree(
 	ut_ad(cursor->pos_state == BTR_PCUR_IS_POSITIONED);
 	ut_ad(cursor->latch_mode != BTR_NO_LATCHES);
 
-	if (btr_page_get_next(btr_pcur_get_page(cursor), mtr) != FIL_NULL) {
-
-		return(FALSE);
-	}
-
-	return(page_cur_is_after_last(btr_pcur_get_page_cur(cursor)));
+	return !page_has_next(btr_pcur_get_page(cursor))
+		&& page_cur_is_after_last(btr_pcur_get_page_cur(cursor));
 }
 
 /*********************************************************//**

storage/innobase/include/page0page.h

 /*****************************************************************************
 Copyright (c) 1994, 2016, Oracle and/or its affiliates. All Rights Reserved.
-Copyright (c) 2013, 2017, MariaDB Corporation.
+Copyright (c) 2013, 2018, MariaDB Corporation.
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of the GNU General Public License as published by the Free Software
@@ -733,14 +733,52 @@ ulint
 page_rec_get_heap_no(
 /*=================*/
 	const rec_t*	rec);	/*!< in: the physical record */
+/** Determine whether a page has any siblings.
+@param[in]	page	page frame
+@return true if the page has any siblings */
+inline
+bool
+page_has_siblings(const page_t* page)
+{
+	compile_time_assert(!(FIL_PAGE_PREV % 8));
+	compile_time_assert(FIL_PAGE_NEXT == FIL_PAGE_PREV + 4);
+	compile_time_assert(FIL_NULL == 0xffffffff);
+	return *reinterpret_cast<const uint64_t*>(page + FIL_PAGE_PREV)
+		!= ~uint64_t(0);
+}
+
 /** Determine whether a page is an index root page.
 @param[in]	page	page frame
 @return true if the page is a root page of an index */
-UNIV_INLINE
+inline
 bool
-page_is_root(
-	const page_t*	page)
-	MY_ATTRIBUTE((warn_unused_result));
+page_is_root(const page_t* page)
+{
+	return fil_page_index_page_check(page) && !page_has_siblings(page);
+}
+
+/** Determine whether a page has a predecessor.
+@param[in]	page	page frame
+@return true if the page has a predecessor */
+inline
+bool
+page_has_prev(const page_t* page)
+{
+	return *reinterpret_cast<const uint32_t*>(page + FIL_PAGE_PREV)
+		!= FIL_NULL;
+}
+
+/** Determine whether a page has a successor.
+@param[in]	page	page frame
+@return true if the page has a successor */
+inline
+bool
+page_has_next(const page_t* page)
+{
+	return *reinterpret_cast<const uint32_t*>(page + FIL_PAGE_NEXT)
+		!= FIL_NULL;
+}
+
 /************************************************************//**
 Gets the pointer to the next record on the page.
 @return pointer to next record */

storage/innobase/include/page0page.ic

@@ -275,31 +275,6 @@ page_rec_get_heap_no(
 	}
 }
 
-/** Determine whether a page is an index root page.
-@param[in]	page	page frame
-@return true if the page is a root page of an index */
-UNIV_INLINE
-bool
-page_is_root(
-	const page_t*	page)
-{
-#if FIL_PAGE_PREV % 8
-# error FIL_PAGE_PREV must be 64-bit aligned
-#endif
-#if FIL_PAGE_NEXT != FIL_PAGE_PREV + 4
-# error FIL_PAGE_NEXT must be adjacent to FIL_PAGE_PREV
-#endif
-#if FIL_NULL != 0xffffffff
-# error FIL_NULL != 0xffffffff
-#endif
-	/* Check that this is an index page and both the PREV and NEXT
-	pointers are FIL_NULL, because the root page does not have any
-	siblings. */
-	return(fil_page_index_page_check(page)
-	       && *reinterpret_cast<const ib_uint64_t*>(page + FIL_PAGE_PREV)
-	       == IB_UINT64_MAX);
-}
-
 /** Determine whether an index page record is a user record.
 @param[in]	rec	record in an index page
 @return true if a user record */

storage/innobase/page/page0cur.cc

@@ -521,7 +521,7 @@ page_cur_search_with_match(
 				ulint   rec_info = rec_get_info_bits(mid_rec,
                                                      rec_offs_comp(offsets));
 				ut_ad(rec_info & REC_INFO_MIN_REC_FLAG);
-				ut_ad(btr_page_get_prev(page, &mtr) == FIL_NULL);
+				ut_ad(!page_has_prev(page));
 				mtr_commit(&mtr);
 #endif
 
@@ -735,9 +735,7 @@ page_cur_search_with_match_bytes(
 					  mid_rec,
 					  dict_table_is_comp(index->table))
 				  & REC_INFO_MIN_REC_FLAG)) {
-			ut_ad(mach_read_from_4(FIL_PAGE_PREV
-					       + page_align(mid_rec))
-			      == FIL_NULL);
+			ut_ad(!page_has_prev(page_align(mid_rec)));
 			ut_ad(!page_rec_is_leaf(mid_rec)
 			      || rec_is_default_row(mid_rec, index));
 			cmp = 1;

storage/innobase/page/page0page.cc

@@ -2768,8 +2768,7 @@ page_delete_rec(
 	if (!rec_offs_any_extern(offsets)
 	    && ((page_get_data_size(page) - rec_offs_size(offsets)
 		< BTR_CUR_PAGE_COMPRESS_LIMIT(index))
-		|| (mach_read_from_4(page + FIL_PAGE_NEXT) == FIL_NULL
-		    && mach_read_from_4(page + FIL_PAGE_PREV) == FIL_NULL)
+		|| !page_has_siblings(page)
 		|| (page_get_n_recs(page) < 2))) {
 
 		ulint	root_page_no = dict_index_get_page(index);

storage/innobase/page/page0zip.cc

@@ -669,8 +669,7 @@ page_zip_dir_encode(
 		status = REC_STATUS_ORDINARY;
 	} else {
 		status = REC_STATUS_NODE_PTR;
-		if (UNIV_UNLIKELY
-		    (mach_read_from_4(page + FIL_PAGE_PREV) == FIL_NULL)) {
+		if (UNIV_UNLIKELY(!page_has_prev(page))) {
 			min_mark = REC_INFO_MIN_REC_FLAG;
 		}
 	}
@@ -3188,8 +3187,7 @@ page_zip_decompress_low(
 			goto err_exit;
 		}
 
-		info_bits = mach_read_from_4(page + FIL_PAGE_PREV) == FIL_NULL
-			? REC_INFO_MIN_REC_FLAG : 0;
+		info_bits = page_has_prev(page) ? 0 : REC_INFO_MIN_REC_FLAG;
 
 		if (UNIV_UNLIKELY(!page_zip_set_extra_bytes(page_zip, page,
 							    info_bits))) {
@@ -4907,9 +4905,8 @@ page_zip_copy_recs(
 	      + page_zip->m_end < page_zip_get_size(page_zip));
 
 	if (!page_is_leaf(src)
-	    && UNIV_UNLIKELY(mach_read_from_4(src + FIL_PAGE_PREV) == FIL_NULL)
-	    && UNIV_LIKELY(mach_read_from_4(page
-					    + FIL_PAGE_PREV) != FIL_NULL)) {
+	    && UNIV_UNLIKELY(!page_has_prev(src))
+	    && UNIV_LIKELY(page_has_prev(page))) {
 		/* Clear the REC_INFO_MIN_REC_FLAG of the first user record. */
 		ulint	offs = rec_get_next_offs(page + PAGE_NEW_INFIMUM,
 						 TRUE);