MDEV-6225: Idle replication slave keeps crashing.

  
  Analysis: Based on crashed the buffer pool instance identifier is
  not correct on block to be freed. Add LRU list mutex holding
  on functions calling free and add additional safety checks.

storage/xtradb/btr/btr0cur.c

@@ -4428,7 +4428,7 @@ btr_blob_free(
 	    && buf_block_get_space(block) == space
 	    && buf_block_get_page_no(block) == page_no) {
 
-		if (!buf_LRU_free_block(&block->page, all, &have_LRU_mutex)
+		if (!buf_LRU_free_block(&block->page, (void *)&block->mutex, all, &have_LRU_mutex)
 		    && all && block->page.zip.data
 		    /* Now, buf_LRU_free_block() may release mutex temporarily */
 		    && buf_block_get_state(block) == BUF_BLOCK_FILE_PAGE
@@ -4437,7 +4437,7 @@ btr_blob_free(
 			/* Attempt to deallocate the uncompressed page
 			if the whole block cannot be deallocted. */
 
-			buf_LRU_free_block(&block->page, FALSE, &have_LRU_mutex);
+			buf_LRU_free_block(&block->page, (void *)&block->mutex, FALSE, &have_LRU_mutex);
 		}
 	}
 

storage/xtradb/buf/buf0buf.c

@@ -2069,7 +2069,7 @@ buf_page_get_zip(
 		}
 
 		/* Discard the uncompressed page frame if possible. */
-		if (buf_LRU_free_block(bpage, FALSE, &have_LRU_mutex)) {
+		if (buf_LRU_free_block(bpage, (void *)block_mutex, FALSE, &have_LRU_mutex)) {
 			if (have_LRU_mutex) {
 				mutex_exit(&buf_pool->LRU_list_mutex);
 				have_LRU_mutex = FALSE;
@@ -2843,7 +2843,7 @@ buf_page_get_gen(
 		insert buffer (change buffer) as much as possible. */
 		ulint	page_no	= buf_block_get_page_no(block);
 
-		if (buf_LRU_free_block(&block->page, TRUE, &have_LRU_mutex)) {
+		if (buf_LRU_free_block(&block->page, (void *)block_mutex, TRUE, &have_LRU_mutex)) {
 			mutex_exit(block_mutex);
 			if (mode == BUF_GET_IF_IN_POOL_OR_WATCH) {
 				/* Set the watch, as it would have

storage/xtradb/buf/buf0lru.c

@@ -957,6 +957,7 @@ buf_LRU_free_from_unzip_LRU_list(
 {
 	buf_block_t*	block;
 	ulint		distance;
+	ibool		taken_LRU_mutex = FALSE;
 
 	//ut_ad(buf_pool_mutex_own(buf_pool));
 
@@ -976,6 +977,12 @@ buf_LRU_free_from_unzip_LRU_list(
 	distance = 100 + (n_iterations
 			  * UT_LIST_GET_LEN(buf_pool->unzip_LRU)) / 5;
 
+	if (!*have_LRU_mutex) {
+		mutex_enter(&buf_pool->LRU_list_mutex);
+		taken_LRU_mutex = TRUE;
+		*have_LRU_mutex = TRUE;
+	}
+
 restart:
 	for (block = UT_LIST_GET_LAST(buf_pool->unzip_LRU);
 	     UNIV_LIKELY(block != NULL) && UNIV_LIKELY(distance > 0);
@@ -990,18 +997,30 @@ buf_LRU_free_from_unzip_LRU_list(
 			goto restart;
 		}
 
+		ut_ad(mutex_own(&buf_pool->LRU_list_mutex));
+		ut_a(block->page.buf_pool_index < srv_buf_pool_instances);
 		ut_ad(buf_block_get_state(block) == BUF_BLOCK_FILE_PAGE);
 		ut_ad(block->in_unzip_LRU_list);
 		ut_ad(block->page.in_LRU_list);
 
-		freed = buf_LRU_free_block(&block->page, FALSE, have_LRU_mutex);
+		freed = buf_LRU_free_block(&block->page, (void *)&block->mutex, FALSE, have_LRU_mutex);
 		mutex_exit(&block->mutex);
 
 		if (freed) {
+
+			if (taken_LRU_mutex && *have_LRU_mutex) {
+				mutex_exit(&buf_pool->LRU_list_mutex);
+				*have_LRU_mutex = FALSE;
+			}
 			return(TRUE);
 		}
 	}
 
+	if (taken_LRU_mutex && *have_LRU_mutex) {
+		mutex_exit(&buf_pool->LRU_list_mutex);
+		*have_LRU_mutex = FALSE;
+	}
+
 	return(FALSE);
 }
 
@@ -1024,11 +1043,18 @@ buf_LRU_free_from_common_LRU_list(
 {
 	buf_page_t*	bpage;
 	ulint		distance;
+	ibool		taken_LRU_mutex = FALSE;
 
 	//ut_ad(buf_pool_mutex_own(buf_pool));
 
 	distance = 100 + (n_iterations * buf_pool->curr_size) / 10;
 
+	if (!*have_LRU_mutex) {
+		mutex_enter(&buf_pool->LRU_list_mutex);
+		taken_LRU_mutex = TRUE;
+		*have_LRU_mutex = TRUE;
+	}
+
 restart:
 	for (bpage = UT_LIST_GET_LAST(buf_pool->LRU);
 	     UNIV_LIKELY(bpage != NULL) && UNIV_LIKELY(distance > 0);
@@ -1048,11 +1074,13 @@ buf_LRU_free_from_common_LRU_list(
 			goto restart;
 		}
 
+		ut_ad(mutex_own(&buf_pool->LRU_list_mutex));
 		ut_ad(buf_page_in_file(bpage));
 		ut_ad(bpage->in_LRU_list);
+		ut_a(bpage->buf_pool_index < srv_buf_pool_instances);
 
 		accessed = buf_page_is_accessed(bpage);
-		freed = buf_LRU_free_block(bpage, TRUE, have_LRU_mutex);
+		freed = buf_LRU_free_block(bpage, (void *)block_mutex, TRUE, have_LRU_mutex);
 		mutex_exit(block_mutex);
 
 		if (freed) {
@@ -1062,10 +1090,21 @@ buf_LRU_free_from_common_LRU_list(
 			if (!accessed) {
 				++buf_pool->stat.n_ra_pages_evicted;
 			}
+
+			if (taken_LRU_mutex && *have_LRU_mutex) {
+				mutex_exit(&buf_pool->LRU_list_mutex);
+				*have_LRU_mutex = FALSE;
+			}
+
 			return(TRUE);
 		}
 	}
 
+	if (taken_LRU_mutex && *have_LRU_mutex) {
+		mutex_exit(&buf_pool->LRU_list_mutex);
+		*have_LRU_mutex = FALSE;
+	}
+
 	return(FALSE);
 }
 
@@ -1846,16 +1885,17 @@ ibool
 buf_LRU_free_block(
 /*===============*/
 	buf_page_t*	bpage,	/*!< in: block to be freed */
+	void*		block_mutex, /*!< in: block mutex or NULL */
 	ibool		zip,	/*!< in: TRUE if should remove also the
 				compressed page of an uncompressed page */
 	ibool*		have_LRU_mutex)
 {
 	buf_page_t*	b = NULL;
 	buf_pool_t*	buf_pool = buf_pool_from_bpage(bpage);
-	mutex_t*	block_mutex = buf_page_get_mutex(bpage);
+	//mutex_t*	block_mutex = buf_page_get_mutex(bpage);
 
 	//ut_ad(buf_pool_mutex_own(buf_pool));
-	ut_ad(mutex_own(block_mutex));
+	ut_ad(mutex_own((mutex_t*)block_mutex));
 	ut_ad(buf_page_in_file(bpage));
 	//ut_ad(bpage->in_LRU_list);
 	ut_ad(!bpage->in_flush_list == !bpage->oldest_modification);
@@ -1912,14 +1952,14 @@ buf_LRU_free_block(
 #endif /* UNIV_DEBUG */
 
 	/* not to break latch order, must re-enter block_mutex */
-	mutex_exit(block_mutex);
+	mutex_exit((mutex_t*)block_mutex);
 
 	if (!*have_LRU_mutex) {
 		mutex_enter(&buf_pool->LRU_list_mutex); /* optimistic */
 		*have_LRU_mutex = TRUE;
 	}
 	rw_lock_x_lock(&buf_pool->page_hash_latch);
-	mutex_enter(block_mutex);
+	mutex_enter((mutex_t*)block_mutex);
 
 	/* recheck states of block */
 	if (!bpage->in_LRU_list || block_mutex != buf_page_get_mutex(bpage)
@@ -2071,7 +2111,7 @@ buf_LRU_free_block(
 			*have_LRU_mutex = FALSE;
 		}
 		rw_lock_x_unlock(&buf_pool->page_hash_latch);
-		mutex_exit(block_mutex);
+		mutex_exit((mutex_t*)block_mutex);
 
 		/* Remove possible adaptive hash index on the page.
 		The page was declared uninitialized by
@@ -2107,7 +2147,7 @@ buf_LRU_free_block(
 			mutex_enter(&buf_pool->LRU_list_mutex);
 			*have_LRU_mutex = TRUE;
 		}
-		mutex_enter(block_mutex);
+		mutex_enter((mutex_t*)block_mutex);
 
 		if (b) {
 			mutex_enter(&buf_pool->zip_mutex);
@@ -2126,8 +2166,8 @@ buf_LRU_free_block(
 		/* The block_mutex should have been released by
 		buf_LRU_block_remove_hashed_page() when it returns
 		BUF_BLOCK_ZIP_FREE. */
-		ut_ad(block_mutex == &buf_pool->zip_mutex);
-		mutex_enter(block_mutex);
+		ut_ad((mutex_t*)block_mutex == &buf_pool->zip_mutex);
+		mutex_enter((mutex_t*)block_mutex);
 
 		if (*have_LRU_mutex) {
 			mutex_exit(&buf_pool->LRU_list_mutex);

storage/xtradb/handler/ha_innodb.cc

@@ -12704,7 +12704,7 @@ innodb_buffer_pool_evict_update(
 
 					mutex_enter(&block->mutex);
 					buf_LRU_free_block(&block->page,
-							   FALSE, &have_LRU_mutex);
+						(void *)&block->mutex,FALSE, &have_LRU_mutex);
 					mutex_exit(&block->mutex);
 					block = prev_block;
 				}

storage/xtradb/include/buf0buf.ic

@@ -60,7 +60,7 @@ buf_pool_index(
 {
 	ulint	i = buf_pool - buf_pool_ptr;
 	ut_ad(i < MAX_BUFFER_POOLS);
-	ut_ad(i < srv_buf_pool_instances);
+	ut_a(i < srv_buf_pool_instances);
 	return(i);
 }
 
@@ -75,7 +75,7 @@ buf_pool_from_bpage(
 {
 	ulint	i;
 	i = bpage->buf_pool_index;
-	ut_ad(i < srv_buf_pool_instances);
+	ut_a(i < srv_buf_pool_instances);
 	return(&buf_pool_ptr[i]);
 }
 
@@ -1097,7 +1097,7 @@ buf_pool_from_array(
 				buffer pool instance from */
 {
 	ut_ad(index < MAX_BUFFER_POOLS);
-	ut_ad(index < srv_buf_pool_instances);
+	ut_a(index < srv_buf_pool_instances);
 	return(&buf_pool_ptr[index]);
 }
 

storage/xtradb/include/buf0lru.h

@@ -99,6 +99,7 @@ ibool
 buf_LRU_free_block(
 /*===============*/
 	buf_page_t*	bpage,	/*!< in: block to be freed */
+	void*		block_mutex, /*!< in: block mutex or NULL */
 	ibool		zip,	/*!< in: TRUE if should remove also the
 				compressed page of an uncompressed page */
 	ibool*		have_LRU_mutex)