MDEV-9236: Dramatically overallocation of InnoDB buffer pool leads to crash

Part I: Add diagnostics to page allocation if state is not correct
but do not assert if it is incorrect.

storage/innobase/buf/buf0buf.cc

@@ -1452,6 +1452,9 @@ buf_pool_init_instance(
 
 	buf_pool_mutex_exit(buf_pool);
 
+	DBUG_EXECUTE_IF("buf_pool_init_instance_force_oom",
+		return(DB_ERROR); );
+
 	return(DB_SUCCESS);
 }
 

storage/innobase/buf/buf0lru.cc

@@ -2090,7 +2090,9 @@ buf_LRU_block_free_non_file_page(
 	case BUF_BLOCK_READY_FOR_USE:
 		break;
 	default:
-		ut_error;
+		fprintf(stderr, "InnoDB: Error: Block %p incorrect state %s in buf_LRU_block_free_non_file_page()\n",
+			block, buf_get_state_name(block));
+		return; /* Continue */
 	}
 
 #if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG

storage/innobase/include/buf0buf.h

@@ -863,7 +863,17 @@ UNIV_INLINE
 enum buf_page_state
 buf_page_get_state(
 /*===============*/
-	const buf_page_t*	bpage);	/*!< in: pointer to the control block */
+	const buf_page_t*	bpage);	/*!< in: pointer to the control
+					block */
+/*********************************************************************//**
+Gets the state name for state of a block
+@return	name or "CORRUPTED" */
+UNIV_INLINE
+const char*
+buf_get_state_name(
+/*===============*/
+	const buf_block_t*	block);	/*!< in: pointer to the control
+					block */
 /*********************************************************************//**
 Gets the state of a block.
 @return	state */

storage/innobase/include/buf0buf.ic

@@ -233,6 +233,41 @@ buf_block_get_state(
 {
 	return(buf_page_get_state(&block->page));
 }
+
+/*********************************************************************//**
+Gets the state name for state of a block
+@return	name or "CORRUPTED" */
+UNIV_INLINE
+const char*
+buf_get_state_name(
+/*===============*/
+	const buf_block_t*	block)	/*!< in: pointer to the control
+					block */
+{
+	enum buf_page_state	state = buf_page_get_state(&block->page);
+
+	switch (state) {
+	case BUF_BLOCK_POOL_WATCH:
+		return (const char *) "BUF_BLOCK_POOL_WATCH";
+	case BUF_BLOCK_ZIP_PAGE:
+		return (const char *) "BUF_BLOCK_ZIP_PAGE";
+	case BUF_BLOCK_ZIP_DIRTY:
+		return (const char *) "BUF_BLOCK_ZIP_DIRTY";
+	case BUF_BLOCK_NOT_USED:
+		return (const char *) "BUF_BLOCK_NOT_USED";
+	case BUF_BLOCK_READY_FOR_USE:
+		return (const char *) "BUF_BLOCK_NOT_USED";
+	case BUF_BLOCK_FILE_PAGE:
+		return (const char *) "BUF_BLOCK_FILE_PAGE";
+	case BUF_BLOCK_MEMORY:
+		return (const char *) "BUF_BLOCK_MEMORY";
+	case BUF_BLOCK_REMOVE_HASH:
+		return (const char *) "BUF_BLOCK_REMOVE_HASH";
+	default:
+		return (const char *) "CORRUPTED";
+	}
+}
+
 /*********************************************************************//**
 Sets the state of a block. */
 UNIV_INLINE

storage/innobase/include/fil0fil.ic

@@ -146,9 +146,9 @@ fil_page_type_validate(
 
 		/* Dump out the page info */
 		fprintf(stderr, "InnoDB: Space %lu offset %lu name %s page_type %lu page_type_name %s\n"
-			"InnoDB: key_version %u page_compressed %d lsn %lu compressed_len %lu\n",
+			"InnoDB: key_version %u page_compressed %d page_compressed_encrypted %d lsn %lu compressed_len %lu\n",
 			space, offset, rspace->name, page_type, fil_get_page_type_name(page_type),
-			key_version, page_compressed, lsn, compressed_len);
+			key_version, page_compressed, page_compressed_encrypted, lsn, compressed_len);
 		fflush(stderr);
 
 		ut_ad(page_type == FIL_PAGE_PAGE_COMPRESSED ||

storage/xtradb/buf/buf0buf.cc

@@ -1508,6 +1508,9 @@ buf_pool_init_instance(
 
 	buf_pool->try_LRU_scan = TRUE;
 
+	DBUG_EXECUTE_IF("buf_pool_init_instance_force_oom",
+		return(DB_ERROR); );
+
 	return(DB_SUCCESS);
 }
 

storage/xtradb/buf/buf0lru.cc

@@ -2237,7 +2237,9 @@ buf_LRU_block_free_non_file_page(
 	case BUF_BLOCK_READY_FOR_USE:
 		break;
 	default:
-		ut_error;
+		fprintf(stderr, "InnoDB: Error: Block %p incorrect state %s in buf_LRU_block_free_non_file_page()\n",
+			block, buf_get_state_name(block));
+		return; /* Continue */
 	}
 
 #if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG

storage/xtradb/include/buf0buf.h

@@ -861,7 +861,17 @@ UNIV_INLINE
 enum buf_page_state
 buf_page_get_state(
 /*===============*/
-	const buf_page_t*	bpage);	/*!< in: pointer to the control block */
+	const buf_page_t*	bpage);	/*!< in: pointer to the control
+					block */
+/*********************************************************************//**
+Gets the state name for state of a block
+@return	name or "CORRUPTED" */
+UNIV_INLINE
+const char*
+buf_get_state_name(
+/*===============*/
+	const buf_block_t*	block);	/*!< in: pointer to the control
+					block */
 /*********************************************************************//**
 Gets the state of a block.
 @return	state */

storage/xtradb/include/buf0buf.ic

@@ -236,6 +236,41 @@ buf_block_get_state(
 {
 	return(buf_page_get_state(&block->page));
 }
+
+/*********************************************************************//**
+Gets the state name for state of a block
+@return	name or "CORRUPTED" */
+UNIV_INLINE
+const char*
+buf_get_state_name(
+/*===============*/
+	const buf_block_t*	block)	/*!< in: pointer to the control
+					block */
+{
+	enum buf_page_state	state = buf_page_get_state(&block->page);
+
+	switch (state) {
+	case BUF_BLOCK_POOL_WATCH:
+		return (const char *) "BUF_BLOCK_POOL_WATCH";
+	case BUF_BLOCK_ZIP_PAGE:
+		return (const char *) "BUF_BLOCK_ZIP_PAGE";
+	case BUF_BLOCK_ZIP_DIRTY:
+		return (const char *) "BUF_BLOCK_ZIP_DIRTY";
+	case BUF_BLOCK_NOT_USED:
+		return (const char *) "BUF_BLOCK_NOT_USED";
+	case BUF_BLOCK_READY_FOR_USE:
+		return (const char *) "BUF_BLOCK_NOT_USED";
+	case BUF_BLOCK_FILE_PAGE:
+		return (const char *) "BUF_BLOCK_FILE_PAGE";
+	case BUF_BLOCK_MEMORY:
+		return (const char *) "BUF_BLOCK_MEMORY";
+	case BUF_BLOCK_REMOVE_HASH:
+		return (const char *) "BUF_BLOCK_REMOVE_HASH";
+	default:
+		return (const char *) "CORRUPTED";
+	}
+}
+
 /*********************************************************************//**
 Sets the state of a block. */
 UNIV_INLINE

storage/xtradb/include/fil0fil.ic

@@ -147,9 +147,9 @@ fil_page_type_validate(
 
 		/* Dump out the page info */
 		fprintf(stderr, "InnoDB: Space %lu offset %lu name %s page_type %lu page_type_name %s\n"
-			"InnoDB: key_version %u page_compressed %d lsn %lu compressed_len %lu\n",
+			"InnoDB: key_version %u page_compressed %d page_compressed_encrypted %d lsn %lu compressed_len %lu\n",
 			space, offset, rspace->name, page_type, fil_get_page_type_name(page_type),
-			key_version, page_compressed, lsn, compressed_len);
+			key_version, page_compressed, page_compressed_encrypted, lsn, compressed_len);
 		fflush(stderr);
 
 		ut_ad(page_type == FIL_PAGE_PAGE_COMPRESSED ||