Commit ae6ebafd authored by Marko Mäkelä's avatar Marko Mäkelä

Merge 10.5 into 10.6

parents c82f3f1b dc2741be
@@ -30,6 +30,9 @@
# Space ID mismatch
# Restore the original tables
# Corrupt FIL_DATA+10 (data)
+# FOUND 1 is expected for both.
+FOUND 1 /InnoDB: Crash recovery is broken due to insufficient innodb_log_file_size; last checkpoint LSN=\d+, current LSN=\d+\. Shutdown is in progress\..*InnoDB: Crash recovery was broken.*/ in mysqld.1.err
+FOUND 1 /InnoDB: Crash recovery was broken/ in mysqld.1.err
# Run innochecksum on t2
# Run innochecksum on t3
# Run innochecksum on t6
......@@ -9,6 +9,7 @@
-- source include/have_file_key_management_plugin.inc
-- source include/innodb_page_size_small.inc
-- source include/innodb_checksum_algorithm.inc
-- source include/maybe_debug.inc
if (!$INNOCHECKSUM) {
--echo Need innochecksum binary
......@@ -18,6 +19,10 @@ if (!$INNOCHECKSUM) {
--disable_query_log
# This may be triggered on a slow system or one that lacks native AIO.
call mtr.add_suppression("InnoDB: Trying to delete tablespace.*pending operations");
if ($have_debug) {
SET GLOBAL DEBUG_DBUG='+d,ib_log_checkpoint_avoid_hard';
call mtr.add_suppression("InnoDB: Crash recovery is broken due to insufficient innodb_log_file_size");
}
--enable_query_log
let $checksum_algorithm = `SELECT @@innodb_checksum_algorithm`;
......@@ -259,6 +264,15 @@ print FILE pack("H*", "c00lcafedeadb017");
close FILE or die "close";
EOF
if ($have_debug) {
--let SEARCH_FILE= $MYSQLTEST_VARDIR/log/mysqld.1.err
--let SEARCH_PATTERN= InnoDB: Crash recovery is broken due to insufficient innodb_log_file_size; last checkpoint LSN=\\d+, current LSN=\\d+\\. Shutdown is in progress\\..*InnoDB: Crash recovery was broken.*
--echo # FOUND 1 is expected for both.
--source include/search_pattern_in_file.inc
--let SEARCH_PATTERN= InnoDB: Crash recovery was broken
--source include/search_pattern_in_file.inc
}
-- disable_result_log
--error 1
--exec $INNOCHECKSUM $t1_IBD
......
......@@ -116,7 +116,7 @@ my_bool my_gethwaddr(uchar *to)
uint i;
for (i= 0; res && i < ifc.ifc_len / sizeof(ifr[0]); i++)
{
#if !defined(_AIX) || !defined(__linux__)
#if defined(_AIX) || defined(__linux__)
#if defined(__linux__)
#define HWADDR_DATA ifr[i].ifr_hwaddr.sa_data
#else
......
......@@ -280,8 +280,7 @@ SET(INNOBASE_SOURCES
include/handler0alter.h
include/hash0hash.h
include/ibuf0ibuf.h
include/ibuf0ibuf.inl/
include/ibuf0types.h
include/ibuf0ibuf.inl
include/lock0iter.h
include/lock0lock.h
include/lock0lock.inl
......
......@@ -1886,6 +1886,7 @@ ATTRIBUTE_COLD void buf_flush_wait_flushed(lsn_t sync_lsn)
write out before we can advance the checkpoint. */
if (sync_lsn > log_sys.get_flushed_lsn())
log_write_up_to(sync_lsn, true);
DBUG_EXECUTE_IF("ib_log_checkpoint_avoid_hard", return;);
log_checkpoint();
}
}
......@@ -1901,6 +1902,8 @@ ATTRIBUTE_COLD void buf_flush_ahead(lsn_t lsn, bool furious)
if (recv_recovery_is_on())
recv_sys.apply(true);
DBUG_EXECUTE_IF("ib_log_checkpoint_avoid_hard", return;);
Atomic_relaxed<lsn_t> &limit= furious
? buf_flush_sync_lsn : buf_flush_async_lsn;
......@@ -2253,6 +2256,7 @@ static void buf_flush_page_cleaner()
buf_pool.page_cleaner_set_idle(true);
DBUG_EXECUTE_IF("ib_log_checkpoint_avoid", continue;);
DBUG_EXECUTE_IF("ib_log_checkpoint_avoid_hard", continue;);
mysql_mutex_unlock(&buf_pool.flush_list_mutex);
......@@ -2336,6 +2340,7 @@ static void buf_flush_page_cleaner()
here should not affect correctness, because log_free_check()
should still be invoking checkpoints when needed. */
DBUG_EXECUTE_IF("ib_log_checkpoint_avoid", goto next;);
DBUG_EXECUTE_IF("ib_log_checkpoint_avoid_hard", goto next;);
if (!recv_recovery_is_on() && srv_operation == SRV_OPERATION_NORMAL)
log_checkpoint();
......
......@@ -2385,16 +2385,11 @@ static void ibuf_read_merge_pages(const uint32_t* space_ids,
#endif
}
/*********************************************************************//**
Contracts insert buffer trees by reading pages to the buffer pool.
/** Contract the change buffer by reading pages to the buffer pool.
@return a lower limit for the combined size in bytes of entries which
will be merged from ibuf trees to the pages read, 0 if ibuf is
empty */
static
ulint
ibuf_merge_pages(
/*=============*/
ulint* n_pages) /*!< out: number of pages to which merged */
will be merged from ibuf trees to the pages read
@retval 0 if ibuf.empty */
ulint ibuf_contract()
{
mtr_t mtr;
btr_pcur_t pcur;
......@@ -2402,8 +2397,6 @@ ibuf_merge_pages(
uint32_t page_nos[IBUF_MAX_N_PAGES_MERGED];
uint32_t space_ids[IBUF_MAX_N_PAGES_MERGED];
*n_pages = 0;
ibuf_mtr_start(&mtr);
/* Open a cursor to a randomly chosen leaf of the tree, at a random
......@@ -2436,13 +2429,14 @@ ibuf_merge_pages(
return(0);
}
ulint n_pages = 0;
sum_sizes = ibuf_get_merge_page_nos(TRUE,
btr_pcur_get_rec(&pcur), &mtr,
space_ids,
page_nos, n_pages);
page_nos, &n_pages);
ibuf_mtr_commit(&mtr);
ibuf_read_merge_pages(space_ids, page_nos, *n_pages);
ibuf_read_merge_pages(space_ids, page_nos, n_pages);
return(sum_sizes + 1);
}
......@@ -2514,73 +2508,6 @@ ibuf_merge_space(
return(n_pages);
}
/** Contract the change buffer by reading pages to the buffer pool.
@param[out] n_pages number of pages merged
@param[in] sync whether the caller waits for
the issued reads to complete
@return a lower limit for the combined size in bytes of entries which
will be merged from ibuf trees to the pages read, 0 if ibuf is
empty */
MY_ATTRIBUTE((warn_unused_result))
static ulint ibuf_merge(ulint* n_pages)
{
*n_pages = 0;
/* We perform a dirty read of ibuf.empty, without latching
the insert buffer root page. We trust this dirty read except
when a slow shutdown is being executed. During a slow
shutdown, the insert buffer merge must be completed. */
if (ibuf.empty && srv_shutdown_state <= SRV_SHUTDOWN_INITIATED) {
return(0);
#if defined UNIV_DEBUG || defined UNIV_IBUF_DEBUG
} else if (ibuf_debug) {
return(0);
#endif /* UNIV_DEBUG || UNIV_IBUF_DEBUG */
} else {
return ibuf_merge_pages(n_pages);
}
}
/** Contract the change buffer by reading pages to the buffer pool.
@return a lower limit for the combined size in bytes of entries which
will be merged from ibuf trees to the pages read, 0 if ibuf is empty */
static ulint ibuf_contract()
{
ulint n_pages;
return ibuf_merge_pages(&n_pages);
}
/** Contract the change buffer by reading pages to the buffer pool.
@return a lower limit for the combined size in bytes of entries which
will be merged from ibuf trees to the pages read, 0 if ibuf is
empty */
ulint ibuf_merge_all()
{
#if defined UNIV_DEBUG || defined UNIV_IBUF_DEBUG
if (ibuf_debug) {
return(0);
}
#endif /* UNIV_DEBUG || UNIV_IBUF_DEBUG */
ulint sum_bytes = 0;
ulint n_pages = srv_io_capacity;
for (ulint sum_pages = 0; sum_pages < n_pages; ) {
log_free_check();
ulint n_pag2;
ulint n_bytes = ibuf_merge(&n_pag2);
if (n_bytes == 0) {
break;
}
sum_bytes += n_bytes;
}
return sum_bytes;
}
/*********************************************************************//**
Contract insert buffer trees after insert if they are too big. */
UNIV_INLINE
......@@ -2590,13 +2517,7 @@ ibuf_contract_after_insert(
ulint entry_size) /*!< in: size of a record which was inserted
into an ibuf tree */
{
/* Perform dirty reads of ibuf.size and ibuf.max_size, to
reduce ibuf_mutex contention. ibuf.max_size remains constant
after ibuf_init_at_db_start(), but ibuf.size should be
protected by ibuf_mutex. Given that ibuf.size fits in a
machine word, this should be OK; at worst we are doing some
excessive ibuf_contract() or occasionally skipping a
ibuf_contract(). */
/* dirty comparison, to avoid contention on ibuf_mutex */
if (ibuf.size < ibuf.max_size) {
return;
}
......@@ -3224,16 +3145,17 @@ ibuf_insert_low(
do_merge = FALSE;
/* Perform dirty reads of ibuf.size and ibuf.max_size, to
reduce ibuf_mutex contention. Given that ibuf.max_size and
ibuf.size fit in a machine word, this should be OK; at worst
we are doing some excessive ibuf_contract() or occasionally
/* Perform dirty comparison of ibuf.max_size and ibuf.size to
reduce ibuf_mutex contention. This should be OK; at worst we
are doing some excessive ibuf_contract() or occasionally
skipping an ibuf_contract(). */
if (ibuf.max_size == 0) {
const ulint max_size = ibuf.max_size;
if (max_size == 0) {
return(DB_STRONG_FAIL);
}
if (ibuf.size >= ibuf.max_size + IBUF_CONTRACT_DO_NOT_INSERT) {
if (ibuf.size >= max_size + IBUF_CONTRACT_DO_NOT_INSERT) {
/* Insert buffer is now too big, contract it but do not try
to insert */
......@@ -4576,7 +4498,7 @@ ibuf_print(
fprintf(file,
"Ibuf: size " ULINTPF ", free list len " ULINTPF ","
" seg size " ULINTPF ", " ULINTPF " merges\n",
ibuf.size,
ulint{ibuf.size},
ibuf.free_list_len,
ibuf.seg_size,
ulint{ibuf.n_merges});
......
......@@ -30,7 +30,6 @@ Created 7/19/1997 Heikki Tuuri
#include "mtr0mtr.h"
#include "dict0mem.h"
#include "fsp0fsp.h"
#include "ibuf0types.h"
/** Default value for maximum on-disk size of change buffer in terms
of percentage of the buffer pool. */
......@@ -61,6 +60,37 @@ enum ibuf_use_t {
/** Operations that can currently be buffered. */
extern ulong innodb_change_buffering;
/** Insert buffer struct */
struct ibuf_t{
Atomic_relaxed<ulint> size; /*!< current size of the ibuf index
tree, in pages */
Atomic_relaxed<ulint> max_size; /*!< recommended maximum size of the
ibuf index tree, in pages */
ulint seg_size; /*!< allocated pages of the file
segment containing ibuf header and
tree */
bool empty; /*!< Protected by the page
latch of the root page of the
insert buffer tree
(FSP_IBUF_TREE_ROOT_PAGE_NO). true
if and only if the insert
buffer tree is empty. */
ulint free_list_len; /*!< length of the free list */
ulint height; /*!< tree height */
dict_index_t* index; /*!< insert buffer index */
/** number of pages merged */
Atomic_counter<ulint> n_merges;
Atomic_counter<ulint> n_merged_ops[IBUF_OP_COUNT];
/*!< number of operations of each type
merged to index pages */
Atomic_counter<ulint> n_discarded_ops[IBUF_OP_COUNT];
/*!< number of operations of each type
discarded without merging due to the
tablespace being deleted or the
index being dropped */
};
/** The insert buffer control structure */
extern ibuf_t ibuf;
......@@ -339,9 +369,9 @@ void ibuf_delete_for_discarded_space(ulint space);
/** Contract the change buffer by reading pages to the buffer pool.
@return a lower limit for the combined size in bytes of entries which
will be merged from ibuf trees to the pages read, 0 if ibuf is
empty */
ulint ibuf_merge_all();
will be merged from ibuf trees to the pages read
@retval 0 if ibuf.empty */
ulint ibuf_contract();
/** Contracts insert buffer trees by reading pages referring to space_id
to the buffer pool.
......
......@@ -64,37 +64,6 @@ ibuf_mtr_commit(
mtr_commit(mtr);
}
/** Insert buffer struct */
struct ibuf_t{
ulint size; /*!< current size of the ibuf index
tree, in pages */
ulint max_size; /*!< recommended maximum size of the
ibuf index tree, in pages */
ulint seg_size; /*!< allocated pages of the file
segment containing ibuf header and
tree */
bool empty; /*!< Protected by the page
latch of the root page of the
insert buffer tree
(FSP_IBUF_TREE_ROOT_PAGE_NO). true
if and only if the insert
buffer tree is empty. */
ulint free_list_len; /*!< length of the free list */
ulint height; /*!< tree height */
dict_index_t* index; /*!< insert buffer index */
/** number of pages merged */
Atomic_counter<ulint> n_merges;
Atomic_counter<ulint> n_merged_ops[IBUF_OP_COUNT];
/*!< number of operations of each type
merged to index pages */
Atomic_counter<ulint> n_discarded_ops[IBUF_OP_COUNT];
/*!< number of operations of each type
discarded without merging due to the
tablespace being deleted or the
index being dropped */
};
/************************************************************************//**
Sets the free bit of the page in the ibuf bitmap. This is done in a separate
mini-transaction, hence this operation does not restrict further work to only
......
/*****************************************************************************
Copyright (c) 1997, 2009, Oracle and/or its affiliates. All Rights Reserved.
This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software
Foundation; version 2 of the License.
This program is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
You should have received a copy of the GNU General Public License along with
this program; if not, write to the Free Software Foundation, Inc.,
51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA
*****************************************************************************/
/**************************************************//**
@file include/ibuf0types.h
Insert buffer global types
Created 7/29/1997 Heikki Tuuri
*******************************************************/
#ifndef ibuf0types_h
#define ibuf0types_h
struct ibuf_t;
#endif
......@@ -2,7 +2,7 @@
Copyright (c) 1995, 2017, Oracle and/or its affiliates. All rights reserved.
Copyright (c) 2009, Google Inc.
Copyright (c) 2017, 2021, MariaDB Corporation.
Copyright (c) 2017, 2022, MariaDB Corporation.
Portions of this file contain modifications contributed and copyrighted by
Google, Inc. Those modifications are gratefully acknowledged and are described
......@@ -466,6 +466,11 @@ struct log_t{
size_t buf_free;
/** recommended maximum size of buf, after which the buffer is flushed */
size_t max_buf_free;
/** Log sequence number when a log file overwrite (broken crash recovery)
was noticed. Protected by mutex. */
lsn_t overwrite_warned;
/** mutex to serialize access to the flush list when we are putting
dirty blocks in the list. The idea behind this mutex is to be able
to release log_sys.mutex during mtr_commit and still ensure that
......
......@@ -52,6 +52,7 @@ Created 12/9/1995 Heikki Tuuri
#include "srv0mon.h"
#include "buf0dump.h"
#include "log0sync.h"
#include "log.h"
/*
General philosophy of InnoDB redo-logs:
......@@ -988,11 +989,20 @@ ATTRIBUTE_COLD void log_write_checkpoint_info(lsn_t end_lsn)
DBUG_PRINT("ib_log", ("checkpoint ended at " LSN_PF
", flushed to " LSN_PF,
lsn_t{log_sys.last_checkpoint_lsn},
log_sys.next_checkpoint_lsn,
log_sys.get_flushed_lsn()));
MONITOR_INC(MONITOR_NUM_CHECKPOINT);
if (log_sys.overwrite_warned) {
sql_print_information("InnoDB: Crash recovery was broken "
"between LSN=" LSN_PF
" and checkpoint LSN=" LSN_PF ".",
log_sys.overwrite_warned,
log_sys.next_checkpoint_lsn);
log_sys.overwrite_warned = 0;
}
mysql_mutex_unlock(&log_sys.mutex);
}
......@@ -1020,10 +1030,15 @@ ATTRIBUTE_COLD static void log_checkpoint_margin()
const lsn_t sync_lsn= checkpoint + log_sys.max_checkpoint_age;
if (lsn <= sync_lsn)
{
#ifndef DBUG_OFF
skip_checkpoint:
#endif
log_sys.set_check_flush_or_checkpoint(false);
goto func_exit;
}
DBUG_EXECUTE_IF("ib_log_checkpoint_avoid_hard", goto skip_checkpoint;);
mysql_mutex_unlock(&log_sys.mutex);
/* We must wait to prevent the tail of the log overwriting the head. */
......@@ -1120,13 +1135,9 @@ ATTRIBUTE_COLD void logs_empty_and_mark_files_at_shutdown()
}
/* We need these threads to stop early in shutdown. */
const char* thread_name;
if (srv_fast_shutdown != 2 && trx_rollback_is_active) {
thread_name = "rollback of recovered transactions";
} else {
thread_name = NULL;
}
const char* thread_name = srv_fast_shutdown != 2
&& trx_rollback_is_active
? "rollback of recovered transactions" : nullptr;
if (thread_name) {
ut_ad(!srv_read_only_mode);
......
......@@ -24,18 +24,19 @@ Mini-transaction buffer
Created 11/26/1995 Heikki Tuuri
*******************************************************/
#include "mtr0mtr.h"
#include "mtr0log.h"
#include "buf0buf.h"
#include "buf0flu.h"
#include "fsp0sysspace.h"
#include "page0types.h"
#include "mtr0log.h"
#include "log0recv.h"
#include "my_cpu.h"
#ifdef BTR_CUR_HASH_ADAPT
# include "btr0sea.h"
#endif
#include "srv0start.h"
#include "log.h"
/** Iterate over a memo block in reverse. */
template <typename Functor>
......@@ -840,7 +841,6 @@ mtr_t::memo_release(const void* object, ulint type)
static bool log_margin_warned;
static time_t log_margin_warn_time;
static bool log_close_warned;
static time_t log_close_warn_time;
/** Check margin not to overwrite transaction log from the last checkpoint.
......@@ -878,8 +878,8 @@ static void log_margin_checkpoint_age(ulint len)
log_margin_warned= true;
log_margin_warn_time= t;
ib::error() << "innodb_log_file_size is too small "
"for mini-transaction size " << len;
sql_print_error("InnoDB: innodb_log_file_size is too small "
"for mini-transaction size " ULINTPF, len);
}
}
else if (UNIV_LIKELY(lsn + margin <= log_sys.last_checkpoint_lsn +
......@@ -1005,14 +1005,19 @@ static mtr_t::page_flush_ahead log_close(lsn_t lsn)
checkpoint_age != lsn)
{
time_t t= time(nullptr);
if (!log_close_warned || difftime(t, log_close_warn_time) > 15)
if (!log_sys.overwrite_warned || difftime(t, log_close_warn_time) > 15)
{
log_close_warned= true;
if (!log_sys.overwrite_warned)
log_sys.overwrite_warned= lsn;
log_close_warn_time= t;
ib::error() << "The age of the last checkpoint is " << checkpoint_age
<< ", which exceeds the log capacity "
<< log_sys.log_capacity << ".";
sql_print_error("InnoDB: Crash recovery is broken due to"
" insufficient innodb_log_file_size;"
" last checkpoint LSN=" LSN_PF ", current LSN=" LSN_PF
"%s.",
lsn_t{log_sys.last_checkpoint_lsn}, lsn,
srv_shutdown_state != SRV_SHUTDOWN_INITIATED
? ". Shutdown is in progress" : "");
}
}
else if (UNIV_LIKELY(checkpoint_age <= log_sys.max_modified_age_async))
......
......@@ -398,9 +398,6 @@ mysql_mutex_t srv_misc_tmpfile_mutex;
/** Temporary file for miscellanous diagnostic output */
FILE* srv_misc_tmpfile;
static ulint srv_main_thread_process_no;
static ulint srv_main_thread_id;
/* The following counts are used by the srv_master_callback. */
/** Iterations of the loop bounded by 'srv_active' label. */
......@@ -895,12 +892,7 @@ srv_printf_innodb_monitor(
n_reserved);
}
fprintf(file,
"Process ID=" ULINTPF
", Main thread ID=" ULINTPF
", state: %s\n",
srv_main_thread_process_no,
srv_main_thread_id,
fprintf(file, "Process ID=0, Main thread ID=0, state: %s\n",
srv_main_thread_op_info);
fprintf(file,
"Number of rows inserted " ULINTPF
......@@ -1487,31 +1479,29 @@ static void srv_sync_log_buffer_in_background()
}
}
/*********************************************************************//**
This function prints progress message every 60 seconds during server
shutdown, for any activities that master thread is pending on. */
static
void
srv_shutdown_print_master_pending(
/*==============================*/
time_t* last_print_time, /*!< last time the function
print the message */
ulint n_bytes_merged) /*!< number of change buffer
just merged */
/** Report progress during shutdown.
@param last time of last output
@param n_read number of page reads initiated for change buffer merge */
static void srv_shutdown_print(time_t &last, ulint n_read)
{
time_t current_time = time(NULL);
if (difftime(current_time, *last_print_time) > 60) {
*last_print_time = current_time;
time_t now= time(nullptr);
if (now - last >= 15)
{
last= now;
/* Check change buffer merge, we only wait for change buffer
merge if it is a slow shutdown */
if (!srv_fast_shutdown && n_bytes_merged) {
ib::info() << "Waiting for change buffer merge to"
" complete number of bytes of change buffer"
" just merged: " << n_bytes_merged;
}
}
const ulint ibuf_size= ibuf.size;
sql_print_information("Completing change buffer merge;"
" %zu page reads initiated;"
" %zu change buffer pages remain",
n_read, ibuf_size);
#if defined HAVE_SYSTEMD && !defined EMBEDDED_LIBRARY
service_manager_extend_timeout(INNODB_EXTEND_TIMEOUT_INTERVAL,
"Completing change buffer merge;"
" %zu page reads initiated;"
" %zu change buffer pages remain",
n_read, ibuf_size);
#endif
}
}
/** Perform periodic tasks whenever the server is active.
......@@ -1556,7 +1546,7 @@ Complete the shutdown tasks such as background DROP TABLE,
and optionally change buffer merge (on innodb_fast_shutdown=0). */
void srv_shutdown(bool ibuf_merge)
{
ulint n_bytes_merged = 0;
ulint n_read = 0;
time_t now = time(NULL);
do {
......@@ -1565,21 +1555,12 @@ void srv_shutdown(bool ibuf_merge)
++srv_main_shutdown_loops;
if (ibuf_merge) {
srv_main_thread_op_info = "checking free log space";
log_free_check();
srv_main_thread_op_info = "doing insert buffer merge";
n_bytes_merged = ibuf_merge_all();
/* Flush logs if needed */
srv_sync_log_buffer_in_background();
}
/* Print progress message every 60 seconds during shutdown */
if (srv_print_verbose_log) {
srv_shutdown_print_master_pending(&now,
n_bytes_merged);
log_free_check();
n_read = ibuf_contract();
srv_shutdown_print(now, n_read);
}
} while (n_bytes_merged);
} while (n_read);
}
/** The periodic master task controlling the server. */
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment