Skip to content

M
MariaDB
Commit 8b019f87 authored by Jan Lindström's avatar Jan Lindström
Browse files
Options

MDEV-11939: innochecksum mistakes a file for an encrypted one (page 0 invalid) 

Always read full page 0 to determine does tablespace contain
encryption metadata. Tablespaces that are page compressed or
page compressed and encrypted do not compare checksum as
it does not exists. For encrypted tables use checksum
verification written for encrypted tables and normal tables
use normal method.

buf_page_is_checksum_valid_crc32
buf_page_is_checksum_valid_innodb
buf_page_is_checksum_valid_none
	Add Innochecksum logging to file

buf_page_is_corrupted
        Remove ib_logf and page_warn_strict_checksum
        calls in innochecksum compilation. Add innochecksum
        logging to file.

fil0crypt.cc fil0crypt.h
        Modify to be able to use in innochecksum compilation and
	move fil_space_verify_crypt_checksum to end of the file.
	Add innochecksum logging to file.

univ.i
        Add innochecksum strict_verify, log_file and cur_page_num
        variables as extern.

page_zip_verify_checksum
        Add innochecksum logging to file.

innochecksum.cc
        Lot of changes most notable able to read encryption
        metadata from page 0 of the tablespace.

Added test case where we corrupt intentionally
FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION (encryption key version)
FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION+4 (post encryption checksum)
FIL_DATA+10 (data)
parent 56959e7b
Show whitespace changes
Inline Side-by-side
Showing with 2421 additions and 1203 deletions
extra/CMakeLists.txt
View file @ 8b019f87
......@@ -77,7 +77,9 @@ IF(WITH_INNOBASE_STORAGE_ENGINE OR WITH_XTRADB_STORAGE_ENGINE)
../storage/innobase/buf/buf0checksum.cc
../storage/innobase/ut/ut0crc32.cc
../storage/innobase/ut/ut0ut.cc
../storage/innobase/buf/buf0buf.cc
../storage/innobase/page/page0zip.cc
../storage/innobase/fil/fil0crypt.cc
)
IF(CMAKE_SYSTEM_PROCESSOR MATCHES "ppc64le")
......
extra/innochecksum.cc
View file @ 8b019f87
/*
Copyright (c) 2005, 2012, Oracle and/or its affiliates.
Copyright (c) 2014, 2015, MariaDB Corporation.
Copyright (c) 2005, 2016, Oracle and/or its affiliates. All rights reserved.
Copyright (c) 2014, 2017, MariaDB Corporation.
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
......@@ -42,6 +42,8 @@
/* Only parts of these files are included from the InnoDB codebase.
The parts not included are excluded by #ifndef UNIV_INNOCHECKSUM. */
typedef void fil_space_t;
#include "univ.i" /* include all of this */
#define FLST_BASE_NODE_SIZE (4 + 2 * FIL_ADDR_SIZE)
......@@ -55,6 +57,7 @@ The parts not included are excluded by #ifndef UNIV_INNOCHECKSUM. */
#include "fsp0types.h"
#include "rem0rec.h"
#include "buf0checksum.h" /* buf_calc_page_*() */
#include "buf0buf.h" /* buf_page_is_corrupted */
#include "fil0fil.h" /* FIL_* */
#include "page0page.h" /* PAGE_* */
#include "page0zip.h" /* page_zip_*() */
......@@ -63,6 +66,9 @@ The parts not included are excluded by #ifndef UNIV_INNOCHECKSUM. */
fsp_flags_get_zip_size() */
#include "ut0crc32.h" /* ut_crc32_init() */
#include "fsp0pagecompress.h" /* fil_get_compression_alg_name */
#include "fil0crypt.h" /* fil_space_verify_crypt_checksum */
#include <string.h>
#ifdef UNIV_NONINL
# include "fsp0fsp.ic"
......@@ -70,50 +76,104 @@ The parts not included are excluded by #ifndef UNIV_INNOCHECKSUM. */
# include "ut0rnd.ic"
#endif
#ifndef PRIuMAX
#define PRIuMAX "llu"
#endif
/* Global variables */
static my_bool verbose;
static my_bool debug;
static my_bool skip_corrupt;
static my_bool just_count;
static ulong start_page;
static ulong end_page;
static ulong do_page;
static my_bool use_end_page;
static my_bool do_one_page;
static my_bool per_page_details;
static bool verbose;
static bool just_count;
static ulint start_page;
static ulint end_page;
static ulint do_page;
static bool use_end_page;
static bool do_one_page;
static my_bool do_leaf;
static my_bool per_page_details;
static ulong n_merge;
ulong srv_page_size; /* replaces declaration in srv0srv.c */
extern ulong srv_checksum_algorithm;
static ulong physical_page_size; /* Page size in bytes on disk. */
static ulong logical_page_size; /* Page size when uncompressed. */
static bool compressed= false; /* Is tablespace compressed */
int n_undo_state_active;
int n_undo_state_cached;
int n_undo_state_to_free;
int n_undo_state_to_purge;
int n_undo_state_prepared;
int n_undo_state_other;
int n_undo_insert, n_undo_update, n_undo_other;
int n_bad_checksum;
int n_fil_page_index;
int n_fil_page_undo_log;
int n_fil_page_inode;
int n_fil_page_ibuf_free_list;
int n_fil_page_allocated;
int n_fil_page_ibuf_bitmap;
int n_fil_page_type_sys;
int n_fil_page_type_trx_sys;
int n_fil_page_type_fsp_hdr;
int n_fil_page_type_allocated;
int n_fil_page_type_xdes;
int n_fil_page_type_blob;
int n_fil_page_type_zblob;
int n_fil_page_type_other;
int n_fil_page_type_page_compressed;
int n_fil_page_type_page_compressed_encrypted;
int n_fil_page_max_index_id;
ulong srv_page_size;
/* Current page number (0 based). */
ulint cur_page_num;
/* Skip the checksum verification. */
static bool no_check;
/* Enabled for strict checksum verification. */
bool strict_verify = 0;
/* Enabled for rewrite checksum. */
static bool do_write;
/* Mismatches count allowed (0 by default). */
static ulint allow_mismatches;
static bool page_type_summary;
static bool page_type_dump;
/* Store filename for page-type-dump option. */
char* page_dump_filename = 0;
/* skip the checksum verification & rewrite if page is doublewrite buffer. */
static bool skip_page = 0;
const char *dbug_setting = "FALSE";
char* log_filename = NULL;
/* User defined filename for logging. */
FILE* log_file = NULL;
/* Enabled for log write option. */
static bool is_log_enabled = false;
#ifndef _WIN32
/* advisory lock for non-window system. */
struct flock lk;
#endif /* _WIN32 */
/* Strict check algorithm name. */
static ulong strict_check;
/* Rewrite checksum algorithm name. */
static ulong write_check;
/* Innodb page type. */
struct innodb_page_type {
int n_undo_state_active;
int n_undo_state_cached;
int n_undo_state_to_free;
int n_undo_state_to_purge;
int n_undo_state_prepared;
int n_undo_state_other;
int n_undo_insert;
int n_undo_update;
int n_undo_other;
int n_fil_page_index;
int n_fil_page_undo_log;
int n_fil_page_inode;
int n_fil_page_ibuf_free_list;
int n_fil_page_ibuf_bitmap;
int n_fil_page_type_sys;
int n_fil_page_type_trx_sys;
int n_fil_page_type_fsp_hdr;
int n_fil_page_type_allocated;
int n_fil_page_type_xdes;
int n_fil_page_type_blob;
int n_fil_page_type_zblob;
int n_fil_page_type_other;
int n_fil_page_type_zblob2;
int n_fil_page_type_page_compressed;
int n_fil_page_type_page_compressed_encrypted;
} page_type;
/* Possible values for "--strict-check" for strictly verify checksum
and "--write" for rewrite checksum. */
static const char *innochecksum_algorithms[] = {
"crc32",
"crc32",
"innodb",
"innodb",
"none",
"none",
NullS
};
/* Used to define an enumerate type of the "innochecksum algorithm". */
static TYPELIB innochecksum_algorithms_typelib = {
array_elements(innochecksum_algorithms)-1,"",
innochecksum_algorithms, NULL
};
#define SIZE_RANGES_FOR_PAGE 10
#define NUM_RETRIES 3
......@@ -143,7 +203,7 @@ struct per_index_stats {
last element for pages with more than logical_page_size */
unsigned long long pages_in_size_range[SIZE_RANGES_FOR_PAGE+2];
std::map<ulint, per_page_stats> leaves;
std::map<unsigned long long, per_page_stats> leaves;
per_index_stats():pages(0), leaf_pages(0), first_leaf_page(0),
count(0), free_pages(0), max_data_size(0), total_n_recs(0),
......@@ -155,219 +215,535 @@ struct per_index_stats {
std::map<unsigned long long, per_index_stats> index_ids;
bool encrypted = false;
void print_index_leaf_stats(
unsigned long long id,
const per_index_stats& index,
FILE* fil_out)
/* Get the page size of the filespace from the filespace header. */
static
my_bool
get_page_size(
/*==========*/
FILE* f, /*!< in: file pointer, must be open
and set to start of file */
byte* buf, /*!< in: buffer used to read the page */
ulong* logical_page_size, /*!< out: Logical/Uncompressed page size */
ulong* physical_page_size) /*!< out: Physical/Commpressed page size */
{
ulong flags;
ulint page_no = index.first_leaf_page;
std::map<unsigned long long, per_page_stats>::const_iterator it_page = index.leaves.find(page_no);
fprintf(fil_out, "\nindex: %llu leaf page stats: n_pages = %llu\n",
id, index.leaf_pages);
fprintf(fil_out, "page_no\tdata_size\tn_recs\n");
while (it_page != index.leaves.end()) {
const per_page_stats& stat = it_page->second;
fprintf(fil_out, "%llu\t%lu\t%lu\n", it_page->first, stat.data_size, stat.n_recs);
page_no = stat.right_page_no;
it_page = index.leaves.find(page_no);
}
}
void defrag_analysis(
unsigned long long id,
const per_index_stats& index,
FILE* fil_out)
{
// TODO: make it work for compressed pages too
std::map<unsigned long long, per_page_stats>::const_iterator it = index.leaves.find(index.first_leaf_page);
ulint n_pages = 0;
ulint n_leaf_pages = 0;
while (it != index.leaves.end()) {
ulint data_size_total = 0;
for (ulong i = 0; i < n_merge; i++) {
const per_page_stats& stat = it->second;
n_leaf_pages ++;
data_size_total += stat.data_size;
it = index.leaves.find(stat.right_page_no);
if (it == index.leaves.end()) {
break;
}
}
int bytes= fread(buf, 1, UNIV_PAGE_SIZE_MIN, f);
if (index.max_data_size) {
n_pages += data_size_total / index.max_data_size;
if (data_size_total % index.max_data_size != 0) {
n_pages += 1;
}
}
}
if (ferror(f))
{
perror("Error reading file header");
return FALSE;
if (index.leaf_pages) {
fprintf(fil_out, "count = %lu free = %lu\n", index.count, index.free_pages);
}
if (bytes != UNIV_PAGE_SIZE_MIN)
{
fprintf(stderr, "Error; Was not able to read the minimum page size ");
fprintf(stderr, "of %d bytes. Bytes read was %d\n", UNIV_PAGE_SIZE_MIN, bytes);
return FALSE;
if (!n_leaf_pages) {
n_leaf_pages = 1;
}
rewind(f);
fprintf(fil_out, "%llu\t\t%llu\t\t%lu\t\t%lu\t\t%lu\t\t%.2f\t%lu\n",
id, index.leaf_pages, n_leaf_pages, n_merge, n_pages,
1.0 - (double)n_pages / (double)n_leaf_pages, index.max_data_size);
}
flags = mach_read_from_4(buf + FIL_PAGE_DATA + FSP_SPACE_FLAGS);
void print_leaf_stats(
FILE* fil_out)
{
fprintf(fil_out, "\n**************************************************\n");
fprintf(fil_out, "index_id\t#leaf_pages\t#actual_leaf_pages\tn_merge\t"
"#leaf_after_merge\tdefrag\n");
for (std::map<unsigned long long, per_index_stats>::const_iterator it = index_ids.begin();
it != index_ids.end(); it++) {
const per_index_stats& index = it->second;
/* srv_page_size is used by InnoDB code as UNIV_PAGE_SIZE */
srv_page_size = *logical_page_size = fsp_flags_get_page_size(flags);
if (verbose) {
print_index_leaf_stats(it->first, index, fil_out);
}
/* fsp_flags_get_zip_size() will return zero if not compressed. */
*physical_page_size = fsp_flags_get_zip_size(flags);
if (*physical_page_size == 0)
{
*physical_page_size= *logical_page_size;
if (n_merge) {
defrag_analysis(it->first, index, fil_out);
}
else
{
compressed= true;
}
return TRUE;
}
#ifdef _WIN32
/***********************************************//*
@param [in] error error no. from the getLastError().
/* command line argument to do page checks (that's it) */
/* another argument to specify page ranges... seek to right spot and go from there */
static struct my_option innochecksum_options[] =
@retval error message corresponding to error no.
*/
static
char*
error_message(
int error)
{
{"help", '?', "Displays this help and exits.",
0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
{"info", 'I', "Synonym for --help.",
0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
{"version", 'V', "Displays version information and exits.",
0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
{"verbose", 'v', "Verbose (prints progress every 5 seconds).",
&verbose, &verbose, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
{"debug", 'd', "Debug mode (prints checksums for each page, implies verbose).",
&debug, &debug, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
{"skip_corrupt", 'u', "Skip corrupt pages.",
&skip_corrupt, &skip_corrupt, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
{"count", 'c', "Print the count of pages in the file.",
&just_count, &just_count, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
{"start_page", 's', "Start on this page number (0 based).",
&start_page, &start_page, 0, GET_ULONG, REQUIRED_ARG,
0, 0, (longlong) 2L*1024L*1024L*1024L, 0, 1, 0},
{"end_page", 'e', "End at this page number (0 based).",
&end_page, &end_page, 0, GET_ULONG, REQUIRED_ARG,
0, 0, (longlong) 2L*1024L*1024L*1024L, 0, 1, 0},
{"page", 'p', "Check only this page (0 based).",
&do_page, &do_page, 0, GET_ULONG, REQUIRED_ARG,
0, 0, (longlong) 2L*1024L*1024L*1024L, 0, 1, 0},
{"per_page_details", 'i', "Print out per-page detail information.",
&per_page_details, &per_page_details, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0}
,
{"leaf", 'l', "Examine leaf index pages",
&do_leaf, &do_leaf, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
{"merge", 'm', "leaf page count if merge given number of consecutive pages",
&n_merge, &n_merge, 0, GET_ULONG, REQUIRED_ARG,
0, 0, (longlong)10L, 0, 1, 0},
{0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}
};
static char err_msg[1024] = {'\0'};
FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM,
NULL, error, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
(LPTSTR)err_msg, sizeof(err_msg), NULL );
static void print_version(void)
{
printf("%s Ver %s, for %s (%s)\n",
my_progname, INNODB_VERSION_STR,
SYSTEM_TYPE, MACHINE_TYPE);
return (err_msg);
}
#endif /* _WIN32 */
static void usage(void)
/***********************************************//*
@param>>_______[in] name>_____name of file.
@retval file pointer; file pointer is NULL when error occured.
*/
FILE*
open_file(
const char* name)
{
print_version();
puts(ORACLE_WELCOME_COPYRIGHT_NOTICE("2000"));
printf("InnoDB offline file checksum utility.\n");
printf("Usage: %s [-c] [-s <start page>] [-e <end page>] [-p <page>] [-v] [-d] <filename>\n", my_progname);
my_print_help(innochecksum_options);
my_print_variables(innochecksum_options);
int fd; /* file descriptor. */
FILE* fil_in;
#ifdef _WIN32
HANDLE hFile; /* handle to open file. */
DWORD access; /* define access control */
int flags = 0; /* define the mode for file
descriptor */
if (do_write) {
access = GENERIC_READ | GENERIC_WRITE;
flags = _O_RDWR | _O_BINARY;
} else {
access = GENERIC_READ;
flags = _O_RDONLY | _O_BINARY;
}
/* CreateFile() also provide advisory lock with the usage of
access and share mode of the file.*/
hFile = CreateFile(
(LPCTSTR) name, access, 0L, NULL,
OPEN_EXISTING, NULL, NULL);
if (hFile == INVALID_HANDLE_VALUE) {
/* print the error message. */
fprintf(stderr, "Filename::%s %s\n", name,
error_message(GetLastError()));
return (NULL);
}
/* get the file descriptor. */
fd= _open_osfhandle((intptr_t)hFile, flags);
#else /* _WIN32 */
int create_flag;
/* define the advisory lock and open file mode. */
if (do_write) {
create_flag = O_RDWR;
lk.l_type = F_WRLCK;
} else {
create_flag = O_RDONLY;
lk.l_type = F_RDLCK;
}
fd = open(name, create_flag);
lk.l_whence = SEEK_SET;
lk.l_start = lk.l_len = 0;
if (fcntl(fd, F_SETLK, &lk) == -1) {
fprintf(stderr, "Error: Unable to lock file::"
" %s\n", name);
perror("fcntl");
return (NULL);
}
#endif /* _WIN32 */
if (do_write) {
fil_in = fdopen(fd, "rb+");
} else {
fil_in = fdopen(fd, "rb");
}
return (fil_in);
}
extern "C" my_bool
innochecksum_get_one_option(
/*========================*/
int optid,
const struct my_option *opt __attribute__((unused)),
char *argument __attribute__((unused)))
/************************************************************//*
Read the content of file
@param [in,out] buf read the file in buffer
@param [in] partial_page_read enable when to read the
remaining buffer for first page.
@param [in] physical_page_size Physical/Commpressed page size.
@param [in,out] fil_in file pointer created for the
tablespace.
@retval no. of bytes read.
*/
ulong read_file(
byte* buf,
bool partial_page_read,
ulong physical_page_size,
FILE* fil_in)
{
switch (optid) {
case 'd':
verbose=1; /* debug implies verbose... */
break;
case 'e':
use_end_page= 1;
break;
case 'p':
end_page= start_page= do_page;
use_end_page= 1;
do_one_page= 1;
break;
case 'V':
print_version();
exit(0);
break;
case 'I':
case '?':
usage();
exit(0);
break;
ulong bytes = 0;
DBUG_ASSERT(physical_page_size >= UNIV_ZIP_SIZE_MIN);
if (partial_page_read) {
buf += UNIV_ZIP_SIZE_MIN;
physical_page_size -= UNIV_ZIP_SIZE_MIN;
bytes = UNIV_ZIP_SIZE_MIN;
}
return 0;
bytes += ulong(fread(buf, 1, physical_page_size, fil_in));
return bytes;
}
static int get_options(
/*===================*/
int *argc,
char ***argv)
/** Check if page is corrupted or not.
@param[in] buf page frame
@param[in] page_size page size
@param[in] zip_size != if page row compressed
@param[in] is_encrypted true if page0 contained cryp_data
with crypt_scheme encrypted
@param[in] is_compressed true if page0 fsp_flags contained
page compression flag
@retval true if page is corrupted otherwise false. */
static
bool
is_page_corrupted(
byte* buf,
ulint page_size,
ulint zip_size,
bool is_encrypted,
bool is_compressed)
{
int ho_error;
if ((ho_error=handle_options(argc, argv, innochecksum_options, innochecksum_get_one_option)))
exit(ho_error);
/* enable if page is corrupted. */
bool is_corrupted;
/* use to store LSN values. */
ulint logseq;
ulint logseqfield;
ulint page_type = mach_read_from_2(buf+FIL_PAGE_TYPE);
ulint key_version = mach_read_from_4(buf+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION);
ulint space_id = mach_read_from_4(
buf + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID);
/* We can't trust only a page type, thus we take account
also fsp_flags or crypt_data on page 0 */
if ((page_type == FIL_PAGE_PAGE_COMPRESSED && is_compressed) ||
(page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED &&
is_compressed && is_encrypted)) {
/* Page compressed tables do not contain post compression
checksum. */
return (false);
}
if (!zip_size) {
/* check the stored log sequence numbers
for uncompressed tablespace. */
logseq = mach_read_from_4(buf + FIL_PAGE_LSN + 4);
logseqfield = mach_read_from_4(
buf + page_size -
FIL_PAGE_END_LSN_OLD_CHKSUM + 4);
if (is_log_enabled) {
fprintf(log_file,
"space::%" PRIuMAX " page::%" PRIuMAX
"; log sequence number:first = " ULINTPF
"; second = " ULINTPF "\n",
space_id, cur_page_num, logseq, logseqfield);
if (logseq != logseqfield) {
fprintf(log_file,
"Fail; space %" PRIuMAX " page %" PRIuMAX
" invalid (fails log "
"sequence number check)\n",
space_id, cur_page_num);
}
}
}
/* The next arg must be the filename */
if (!*argc)
{
usage();
return 1;
}
return 0;
} /* get_options */
/*********************************************************************//**
Gets the file page type.
@return type; NOTE that if the type has not been written to page, the
return value not defined */
ulint
fil_page_get_type(
/*==============*/
uchar* page) /*!< in: file page */
/* Again we can't trust only FIL_PAGE_FILE_FLUSH_LSN field
now repurposed as FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION,
we need to check also crypt_data contents. */
if (is_encrypted && key_version != 0) {
is_corrupted = !fil_space_verify_crypt_checksum(buf,
zip_size, NULL, cur_page_num);
} else {
is_corrupted = buf_page_is_corrupted(
true, buf, zip_size, NULL);
}
return(is_corrupted);
}
/********************************************//*
Check if page is doublewrite buffer or not.
@param [in] page buffer page
@retval true if page is doublewrite buffer otherwise false.
*/
static
bool
is_page_doublewritebuffer(
const byte* page)
{
return(mach_read_from_2(page + FIL_PAGE_TYPE));
if ((cur_page_num >= FSP_EXTENT_SIZE)
&& (cur_page_num < FSP_EXTENT_SIZE * 3)) {
/* page is doublewrite buffer. */
return (true);
}
return (false);
}
/**************************************************************//**
Gets the index id field of a page.
@return index id */
ib_uint64_t
btr_page_get_index_id(
/*==================*/
uchar* page) /*!< in: index page */
/*******************************************************//*
Check if page is empty or not.
@param [in] page page to checked for empty.
@param [in] len size of page.
@retval true if page is empty.
@retval false if page is not empty.
*/
static
bool
is_page_empty(
const byte* page,
size_t len)
{
return(mach_read_from_8(page + PAGE_HEADER + PAGE_INDEX_ID));
while (len--) {
if (*page++) {
return (false);
}
}
return (true);
}
/********************************************************//**
Gets the next index page number.
@return next page number */
ulint
btr_page_get_next(
/*==============*/
const page_t* page) /*!< in: index page */
/********************************************************************//**
Rewrite the checksum for the page.
@param [in/out] page page buffer
@param [in] physical_page_size page size in bytes on disk.
@param [in] iscompressed Is compressed/Uncompressed Page.
@retval true : do rewrite
@retval false : skip the rewrite as checksum stored match with
calculated or page is doublwrite buffer.
*/
bool
update_checksum(
byte* page,
ulong physical_page_size,
bool iscompressed)
{
return(mach_read_from_4(page + FIL_PAGE_NEXT));
ib_uint32_t checksum = 0;
byte stored1[4]; /* get FIL_PAGE_SPACE_OR_CHKSUM field checksum */
byte stored2[4]; /* get FIL_PAGE_END_LSN_OLD_CHKSUM field checksum */
ut_ad(page);
/* If page is doublewrite buffer, skip the rewrite of checksum. */
if (skip_page) {
return (false);
}
memcpy(stored1, page + FIL_PAGE_SPACE_OR_CHKSUM, 4);
memcpy(stored2, page + physical_page_size -
FIL_PAGE_END_LSN_OLD_CHKSUM, 4);
/* Check if page is empty, exclude the checksum field */
if (is_page_empty(page + 4, physical_page_size - 12)
&& is_page_empty(page + physical_page_size - 4, 4)) {
memset(page + FIL_PAGE_SPACE_OR_CHKSUM, 0, 4);
memset(page + physical_page_size -
FIL_PAGE_END_LSN_OLD_CHKSUM, 0, 4);
goto func_exit;
}
if (iscompressed) {
/* page is compressed */
checksum = page_zip_calc_checksum(
page, physical_page_size,
static_cast<srv_checksum_algorithm_t>(write_check));
mach_write_to_4(page + FIL_PAGE_SPACE_OR_CHKSUM, checksum);
if (is_log_enabled) {
fprintf(log_file, "page::%" PRIuMAX "; Updated checksum ="
" %u\n", cur_page_num, checksum);
}
} else {
/* page is uncompressed. */
/* Store the new formula checksum */
switch ((srv_checksum_algorithm_t) write_check) {
case SRV_CHECKSUM_ALGORITHM_CRC32:
case SRV_CHECKSUM_ALGORITHM_STRICT_CRC32:
checksum = buf_calc_page_crc32(page);
break;
case SRV_CHECKSUM_ALGORITHM_INNODB:
case SRV_CHECKSUM_ALGORITHM_STRICT_INNODB:
checksum = (ib_uint32_t)
buf_calc_page_new_checksum(page);
break;
case SRV_CHECKSUM_ALGORITHM_NONE:
case SRV_CHECKSUM_ALGORITHM_STRICT_NONE:
checksum = BUF_NO_CHECKSUM_MAGIC;
break;
/* no default so the compiler will emit a warning if new
enum is added and not handled here */
}
mach_write_to_4(page + FIL_PAGE_SPACE_OR_CHKSUM, checksum);
if (is_log_enabled) {
fprintf(log_file, "page::%" PRIuMAX "; Updated checksum field1"
" = %u\n", cur_page_num, checksum);
}
if (write_check == SRV_CHECKSUM_ALGORITHM_STRICT_INNODB
|| write_check == SRV_CHECKSUM_ALGORITHM_INNODB) {
checksum = (ib_uint32_t)
buf_calc_page_old_checksum(page);
}
mach_write_to_4(page + physical_page_size -
FIL_PAGE_END_LSN_OLD_CHKSUM,checksum);
if (is_log_enabled) {
fprintf(log_file, "page::%" PRIuMAX "; Updated checksum "
"field2 = %u\n", cur_page_num, checksum);
}
}
func_exit:
/* The following code is to check the stored checksum with the
calculated checksum. If it matches, then return FALSE to skip
the rewrite of checksum, otherwise return TRUE. */
if (iscompressed) {
if (!memcmp(stored1, page + FIL_PAGE_SPACE_OR_CHKSUM, 4)) {
return (false);
}
return (true);
}
if (!memcmp(stored1, page + FIL_PAGE_SPACE_OR_CHKSUM, 4)
&& !memcmp(stored2, page + physical_page_size -
FIL_PAGE_END_LSN_OLD_CHKSUM, 4)) {
return (false);
}
return (true);
}
/********************************************************//**
Gets the previous index page number.
@return prev page number */
ulint
btr_page_get_prev(
/*==============*/
const page_t* page) /*!< in: index page */
/**
Write the content to the file
@param[in] filename name of the file.
@param[in,out] file file pointer where content
have to be written
@param[in] buf file buffer read
@param[in] compressed Enabled if tablespace is
compressed.
@param[in,out] pos current file position.
@param[in] page_size page size in bytes on disk.
@retval true if successfully written
@retval false if a non-recoverable error occurred
*/
static
bool
write_file(
const char* filename,
FILE* file,
byte* buf,
bool compressed,
fpos_t* pos,
ulong page_size)
{
return(mach_read_from_4(page + FIL_PAGE_PREV));
bool do_update;
do_update = update_checksum(buf, page_size, compressed);
if (file != stdin) {
if (do_update) {
/* Set the previous file pointer position
saved in pos to current file position. */
if (0 != fsetpos(file, pos)) {
perror("fsetpos");
return(false);
}
} else {
/* Store the current file position in pos */
if (0 != fgetpos(file, pos)) {
perror("fgetpos");
return(false);
}
return(true);
}
}
if (page_size
!= fwrite(buf, 1, page_size, file == stdin ? stdout : file)) {
fprintf(stderr, "Failed to write page %" PRIuMAX " to %s: %s\n",
cur_page_num, filename, strerror(errno));
return(false);
}
if (file != stdin) {
fflush(file);
/* Store the current file position in pos */
if (0 != fgetpos(file, pos)) {
perror("fgetpos");
return(false);
}
}
return(true);
}
/*
Parse the page and collect/dump the information about page type
@param [in] page buffer page
@param [out] xdes extend descriptor page
@param [in] file file for diagnosis.
*/
void
parse_page(
/*=======*/
uchar* page, /* in: buffer page */
uchar* xdes) /* in: extend descriptor page */
const byte* page,
byte* xdes,
FILE* file)
{
ib_uint64_t id;
ulint x;
unsigned long long id;
ulint undo_page_type;
char str[20]={'\0'};
ulint n_recs;
ulint page_no;
ulint left_page_no;
......@@ -376,14 +752,22 @@ parse_page(
int is_leaf;
int size_range_id;
switch (fil_page_get_type(page)) {
/* Check whether page is doublewrite buffer. */
if(skip_page) {
strcpy(str, "Double_write_buffer");
} else {
strcpy(str, "-");
}
switch (mach_read_from_2(page + FIL_PAGE_TYPE)) {
case FIL_PAGE_INDEX:
n_fil_page_index++;
id = btr_page_get_index_id(page);
page_type.n_fil_page_index++;
id = mach_read_from_8(page + PAGE_HEADER + PAGE_INDEX_ID);
n_recs = page_get_n_recs(page);
page_no = page_get_page_no(page);
left_page_no = btr_page_get_prev(page);
right_page_no = btr_page_get_next(page);
left_page_no = mach_read_from_4(page + FIL_PAGE_PREV);
right_page_no = mach_read_from_4(page + FIL_PAGE_NEXT);
data_bytes = page_get_data_size(page);
is_leaf = page_is_leaf(page);
size_range_id = (data_bytes * SIZE_RANGES_FOR_PAGE
......@@ -394,8 +778,7 @@ parse_page(
size_range_id = SIZE_RANGES_FOR_PAGE + 1;
}
if (per_page_details) {
printf("index " IB_ID_FMT " page " ULINTPF
" leaf %d n_recs " ULINTPF " data_bytes " ULINTPF
printf("index %llu page %lu leaf %u n_recs %lu data_bytes %lu"
"\n", id, page_no, is_leaf, n_recs, data_bytes);
}
/* update per-index statistics */
......@@ -406,7 +789,7 @@ parse_page(
std::map<unsigned long long, per_index_stats>::iterator it;
it = index_ids.find(id);
per_index_stats &index = (it->second);
uchar* des = xdes + XDES_ARR_OFFSET
const byte* des = xdes + XDES_ARR_OFFSET
+ XDES_SIZE * ((page_no & (physical_page_size - 1))
/ FSP_EXTENT_SIZE);
if (xdes_get_bit(des, XDES_FREE_BIT,
......@@ -435,650 +818,1003 @@ parse_page(
index.pages_in_size_range[size_range_id] ++;
}
if (page_type_dump) {
fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tIndex page\t\t\t|"
"\tindex id=%llu,", cur_page_num, id);
fprintf(file,
" page level=" ULINTPF
", No. of records=" ULINTPF
", garbage=" ULINTPF ", %s\n",
page_header_get_field(page, PAGE_LEVEL),
page_header_get_field(page, PAGE_N_RECS),
page_header_get_field(page, PAGE_GARBAGE), str);
}
break;
case FIL_PAGE_UNDO_LOG:
if (per_page_details) {
printf("FIL_PAGE_UNDO_LOG\n");
}
n_fil_page_undo_log++;
x = mach_read_from_2(page + TRX_UNDO_PAGE_HDR +
TRX_UNDO_PAGE_TYPE);
if (x == TRX_UNDO_INSERT)
n_undo_insert++;
else if (x == TRX_UNDO_UPDATE)
n_undo_update++;
else
n_undo_other++;
x = mach_read_from_2(page + TRX_UNDO_SEG_HDR + TRX_UNDO_STATE);
switch (x) {
case TRX_UNDO_ACTIVE: n_undo_state_active++; break;
case TRX_UNDO_CACHED: n_undo_state_cached++; break;
case TRX_UNDO_TO_FREE: n_undo_state_to_free++; break;
case TRX_UNDO_TO_PURGE: n_undo_state_to_purge++; break;
case TRX_UNDO_PREPARED: n_undo_state_prepared++; break;
default: n_undo_state_other++; break;
page_type.n_fil_page_undo_log++;
undo_page_type = mach_read_from_2(page +
TRX_UNDO_PAGE_HDR + TRX_UNDO_PAGE_TYPE);
if (page_type_dump) {
fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tUndo log page\t\t\t|",
cur_page_num);
}
if (undo_page_type == TRX_UNDO_INSERT) {
page_type.n_undo_insert++;
if (page_type_dump) {
fprintf(file, "\t%s",
"Insert Undo log page");
}
} else if (undo_page_type == TRX_UNDO_UPDATE) {
page_type.n_undo_update++;
if (page_type_dump) {
fprintf(file, "\t%s",
"Update undo log page");
}
}
undo_page_type = mach_read_from_2(page + TRX_UNDO_SEG_HDR +
TRX_UNDO_STATE);
switch (undo_page_type) {
case TRX_UNDO_ACTIVE:
page_type.n_undo_state_active++;
if (page_type_dump) {
fprintf(file, ", %s", "Undo log of "
"an active transaction");
}
break;
case TRX_UNDO_CACHED:
page_type.n_undo_state_cached++;
if (page_type_dump) {
fprintf(file, ", %s", "Page is "
"cached for quick reuse");
}
break;
case TRX_UNDO_TO_FREE:
page_type.n_undo_state_to_free++;
if (page_type_dump) {
fprintf(file, ", %s", "Insert undo "
"segment that can be freed");
}
break;
case TRX_UNDO_TO_PURGE:
page_type.n_undo_state_to_purge++;
if (page_type_dump) {
fprintf(file, ", %s", "Will be "
"freed in purge when all undo"
"data in it is removed");
}
break;
case TRX_UNDO_PREPARED:
page_type.n_undo_state_prepared++;
if (page_type_dump) {
fprintf(file, ", %s", "Undo log of "
"an prepared transaction");
}
break;
default:
page_type.n_undo_state_other++;
break;
}
if(page_type_dump) {
fprintf(file, ", %s\n", str);
}
break;
case FIL_PAGE_INODE:
if (per_page_details) {
printf("FIL_PAGE_INODE\n");
page_type.n_fil_page_inode++;
if (page_type_dump) {
fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tInode page\t\t\t|"
"\t%s\n",cur_page_num, str);
}
n_fil_page_inode++;
break;
case FIL_PAGE_IBUF_FREE_LIST:
if (per_page_details) {
printf("FIL_PAGE_IBUF_FREE_LIST\n");
page_type.n_fil_page_ibuf_free_list++;
if (page_type_dump) {
fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tInsert buffer free list"
" page\t|\t%s\n", cur_page_num, str);
}
n_fil_page_ibuf_free_list++;
break;
case FIL_PAGE_TYPE_ALLOCATED:
if (per_page_details) {
printf("FIL_PAGE_TYPE_ALLOCATED\n");
page_type.n_fil_page_type_allocated++;
if (page_type_dump) {
fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tFreshly allocated "
"page\t\t|\t%s\n", cur_page_num, str);
}
n_fil_page_type_allocated++;
break;
case FIL_PAGE_IBUF_BITMAP:
if (per_page_details) {
printf("FIL_PAGE_IBUF_BITMAP\n");
page_type.n_fil_page_ibuf_bitmap++;
if (page_type_dump) {
fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tInsert Buffer "
"Bitmap\t\t|\t%s\n", cur_page_num, str);
}
n_fil_page_ibuf_bitmap++;
break;
case FIL_PAGE_TYPE_SYS:
if (per_page_details) {
printf("FIL_PAGE_TYPE_SYS\n");
page_type.n_fil_page_type_sys++;
if (page_type_dump) {
fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tSystem page\t\t\t|"
"\t%s\n",cur_page_num, str);
}
n_fil_page_type_sys++;
break;
case FIL_PAGE_TYPE_TRX_SYS:
if (per_page_details) {
printf("FIL_PAGE_TYPE_TRX_SYS\n");
page_type.n_fil_page_type_trx_sys++;
if (page_type_dump) {
fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tTransaction system "
"page\t\t|\t%s\n", cur_page_num, str);
}
n_fil_page_type_trx_sys++;
break;
case FIL_PAGE_TYPE_FSP_HDR:
if (per_page_details) {
printf("FIL_PAGE_TYPE_FSP_HDR\n");
}
page_type.n_fil_page_type_fsp_hdr++;
memcpy(xdes, page, physical_page_size);
n_fil_page_type_fsp_hdr++;
if (page_type_dump) {
fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tFile Space "
"Header\t\t|\t%s\n", cur_page_num, str);
}
break;
case FIL_PAGE_TYPE_XDES:
if (per_page_details) {
printf("FIL_PAGE_TYPE_XDES\n");
}
page_type.n_fil_page_type_xdes++;
memcpy(xdes, page, physical_page_size);
n_fil_page_type_xdes++;
if (page_type_dump) {
fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tExtent descriptor "
"page\t\t|\t%s\n", cur_page_num, str);
}
break;
case FIL_PAGE_TYPE_BLOB:
if (per_page_details) {
printf("FIL_PAGE_TYPE_BLOB\n");
page_type.n_fil_page_type_blob++;
if (page_type_dump) {
fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tBLOB page\t\t\t|\t%s\n",
cur_page_num, str);
}
n_fil_page_type_blob++;
break;
case FIL_PAGE_TYPE_ZBLOB:
page_type.n_fil_page_type_zblob++;
if (page_type_dump) {
fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tCompressed BLOB "
"page\t\t|\t%s\n", cur_page_num, str);
}
break;
case FIL_PAGE_TYPE_ZBLOB2:
if (per_page_details) {
printf("FIL_PAGE_TYPE_ZBLOB/2\n");
page_type.n_fil_page_type_zblob2++;
if (page_type_dump) {
fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tSubsequent Compressed "
"BLOB page\t|\t%s\n", cur_page_num, str);
}
n_fil_page_type_zblob++;
break;
case FIL_PAGE_PAGE_COMPRESSED:
if (per_page_details) {
printf("FIL_PAGE_PAGE_COMPRESSED\n");
page_type.n_fil_page_type_page_compressed++;
if (page_type_dump) {
fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tPage compressed "
"page\t|\t%s\n", cur_page_num, str);
}
n_fil_page_type_page_compressed++;
break;
case FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED:
if (per_page_details) {
printf("FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED\n");
page_type.n_fil_page_type_page_compressed_encrypted++;
if (page_type_dump) {
fprintf(file, "#::%8" PRIuMAX "\t\t|\t\tPage compressed encrypted "
"page\t|\t%s\n", cur_page_num, str);
}
n_fil_page_type_page_compressed_encrypted++;
break;
default:
if (per_page_details) {
printf("FIL_PAGE_TYPE_OTHER\n");
}
n_fil_page_type_other++;
page_type.n_fil_page_type_other++;
break;
}
}
/**
@param [in/out] file_name name of the filename
void print_index_leaf_stats(unsigned long long id, const per_index_stats& index)
@retval FILE pointer if successfully created else NULL when error occured.
*/
FILE*
create_file(
char* file_name)
{
ulint page_no = index.first_leaf_page;
std::map<ulint, per_page_stats>::const_iterator it_page = index.leaves.find(page_no);
printf("\nindex: %llu leaf page stats: n_pages = %llu\n",
id, index.leaf_pages);
printf("page_no\tdata_size\tn_recs\n");
while (it_page != index.leaves.end()) {
const per_page_stats& stat = it_page->second;
printf(ULINTPF "\t" ULINTPF "\t" ULINTPF "\n",
it_page->first, stat.data_size, stat.n_recs);
page_no = stat.right_page_no;
it_page = index.leaves.find(page_no);
}
}
FILE* file = NULL;
void defrag_analysis(unsigned long long id, const per_index_stats& index)
{
// TODO: make it work for compressed pages too
std::map<ulint, per_page_stats>::const_iterator it = index.leaves.find(index.first_leaf_page);
ulint n_pages = 0;
ulint n_leaf_pages = 0;
while (it != index.leaves.end()) {
ulint data_size_total = 0;
for (ulong i = 0; i < n_merge; i++) {
const per_page_stats& stat = it->second;
n_leaf_pages ++;
data_size_total += stat.data_size;
it = index.leaves.find(stat.right_page_no);
if (it == index.leaves.end()) {
break;
}
}
if (index.max_data_size) {
n_pages += data_size_total / index.max_data_size;
if (data_size_total % index.max_data_size != 0) {
n_pages += 1;
}
}
}
if (!n_leaf_pages) {
n_leaf_pages=1;
#ifndef _WIN32
file = fopen(file_name, "wb");
if (file == NULL) {
fprintf(stderr, "Failed to create file: %s: %s\n",
file_name, strerror(errno));
return(NULL);
}
printf("count = " ULINTPF " free = " ULINTPF "\n",
index.count, index.free_pages);
if (n_leaf_pages) {
printf("%llu\t\t%llu\t\t"
ULINTPF "\t\t%lu\t\t" ULINTPF "\t\t%.2f\t" ULINTPF "\n",
id, index.leaf_pages, n_leaf_pages, n_merge, n_pages,
1.0 - (double)n_pages / (double)n_leaf_pages, index.max_data_size);
}
}
#else
HANDLE hFile; /* handle to open file. */
int fd = 0;
hFile = CreateFile((LPCTSTR) file_name,
GENERIC_READ | GENERIC_WRITE,
FILE_SHARE_READ | FILE_SHARE_DELETE,
NULL, CREATE_NEW, NULL, NULL);
void print_leaf_stats()
{
printf("\n**************************************************\n");
printf("index_id\t#leaf_pages\t#actual_leaf_pages\tn_merge\t"
"#leaf_after_merge\tdefrag\n");
for (std::map<unsigned long long, per_index_stats>::const_iterator it = index_ids.begin(); it != index_ids.end(); it++) {
const per_index_stats& index = it->second;
if (verbose) {
print_index_leaf_stats(it->first, index);
}
if (n_merge) {
defrag_analysis(it->first, index);
}
if (hFile == INVALID_HANDLE_VALUE) {
/* print the error message. */
fprintf(stderr, "Filename::%s %s\n",
file_name,
error_message(GetLastError()));
return(NULL);
}
/* get the file descriptor. */
fd= _open_osfhandle((intptr_t)hFile, _O_RDWR | _O_BINARY);
file = fdopen(fd, "wb");
#endif /* _WIN32 */
return(file);
}
/*
Print the page type count of a tablespace.
@param [in] fil_out stream where the output goes.
*/
void
print_stats()
/*========*/
print_summary(
FILE* fil_out)
{
unsigned long long i;
printf("%d\tbad checksum\n", n_bad_checksum);
printf("%d\tFIL_PAGE_INDEX\n", n_fil_page_index);
printf("%d\tFIL_PAGE_UNDO_LOG\n", n_fil_page_undo_log);
printf("%d\tFIL_PAGE_INODE\n", n_fil_page_inode);
printf("%d\tFIL_PAGE_IBUF_FREE_LIST\n", n_fil_page_ibuf_free_list);
printf("%d\tFIL_PAGE_TYPE_ALLOCATED\n", n_fil_page_type_allocated);
printf("%d\tFIL_PAGE_IBUF_BITMAP\n", n_fil_page_ibuf_bitmap);
printf("%d\tFIL_PAGE_TYPE_SYS\n", n_fil_page_type_sys);
printf("%d\tFIL_PAGE_TYPE_TRX_SYS\n", n_fil_page_type_trx_sys);
printf("%d\tFIL_PAGE_TYPE_FSP_HDR\n", n_fil_page_type_fsp_hdr);
printf("%d\tFIL_PAGE_TYPE_XDES\n", n_fil_page_type_xdes);
printf("%d\tFIL_PAGE_TYPE_BLOB\n", n_fil_page_type_blob);
printf("%d\tFIL_PAGE_TYPE_ZBLOB\n", n_fil_page_type_zblob);
printf("%d\tFIL_PAGE_PAGE_COMPRESSED\n", n_fil_page_type_page_compressed);
printf("%d\tFIL_PAGE_PAGE_COMPRESSED_ENCRYPTED\n", n_fil_page_type_page_compressed_encrypted);
printf("%d\tother\n", n_fil_page_type_other);
printf("%d\tmax index_id\n", n_fil_page_max_index_id);
printf("undo type: %d insert, %d update, %d other\n",
n_undo_insert, n_undo_update, n_undo_other);
printf("undo state: %d active, %d cached, %d to_free, %d to_purge,"
" %d prepared, %d other\n", n_undo_state_active,
n_undo_state_cached, n_undo_state_to_free,
n_undo_state_to_purge, n_undo_state_prepared,
n_undo_state_other);
printf("index_id\t#pages\t\t#leaf_pages\t#recs_per_page"
fprintf(fil_out, "\n================PAGE TYPE SUMMARY==============\n");
fprintf(fil_out, "#PAGE_COUNT\tPAGE_TYPE");
fprintf(fil_out, "\n===============================================\n");
fprintf(fil_out, "%8d\tIndex page\n",
page_type.n_fil_page_index);
fprintf(fil_out, "%8d\tUndo log page\n",
page_type.n_fil_page_undo_log);
fprintf(fil_out, "%8d\tInode page\n",
page_type.n_fil_page_inode);
fprintf(fil_out, "%8d\tInsert buffer free list page\n",
page_type.n_fil_page_ibuf_free_list);
fprintf(fil_out, "%8d\tFreshly allocated page\n",
page_type.n_fil_page_type_allocated);
fprintf(fil_out, "%8d\tInsert buffer bitmap\n",
page_type.n_fil_page_ibuf_bitmap);
fprintf(fil_out, "%8d\tSystem page\n",
page_type.n_fil_page_type_sys);
fprintf(fil_out, "%8d\tTransaction system page\n",
page_type.n_fil_page_type_trx_sys);
fprintf(fil_out, "%8d\tFile Space Header\n",
page_type.n_fil_page_type_fsp_hdr);
fprintf(fil_out, "%8d\tExtent descriptor page\n",
page_type.n_fil_page_type_xdes);
fprintf(fil_out, "%8d\tBLOB page\n",
page_type.n_fil_page_type_blob);
fprintf(fil_out, "%8d\tCompressed BLOB page\n",
page_type.n_fil_page_type_zblob);
fprintf(fil_out, "%8d\tOther type of page\n",
page_type.n_fil_page_type_other);
fprintf(fil_out, "%8d\tPage compressed page\n",
page_type.n_fil_page_type_page_compressed);
fprintf(fil_out, "%8d\tPage compressed encrypted page\n",
page_type.n_fil_page_type_page_compressed_encrypted);
fprintf(fil_out, "\n===============================================\n");
fprintf(fil_out, "Additional information:\n");
fprintf(fil_out, "Undo page type: %d insert, %d update, %d other\n",
page_type.n_undo_insert,
page_type.n_undo_update,
page_type.n_undo_other);
fprintf(fil_out, "Undo page state: %d active, %d cached, %d to_free, %d"
" to_purge, %d prepared, %d other\n",
page_type.n_undo_state_active,
page_type.n_undo_state_cached,
page_type.n_undo_state_to_free,
page_type.n_undo_state_to_purge,
page_type.n_undo_state_prepared,
page_type.n_undo_state_other);
fprintf(fil_out, "index_id\t#pages\t\t#leaf_pages\t#recs_per_page"
"\t#bytes_per_page\n");
for (std::map<unsigned long long, per_index_stats>::const_iterator it = index_ids.begin(); it != index_ids.end(); it++) {
for (std::map<unsigned long long, per_index_stats>::const_iterator it = index_ids.begin();
it != index_ids.end(); it++) {
const per_index_stats& index = it->second;
ulonglong recs_per_page = index.total_n_recs;
ulonglong bytes_per_page = index.total_data_bytes;
if (index.total_n_recs && index.pages) {
recs_per_page = index.total_n_recs / index.pages;
}
if (index.total_data_bytes && index.pages) {
bytes_per_page = index.total_data_bytes / index.pages;
}
printf("%llu\t\t%llu\t\t%llu\t\t%llu\t\t%llu\n",
fprintf(fil_out, "%lld\t\t%lld\t\t%lld\t\t%lld\t\t%lld\n",
it->first, index.pages, index.leaf_pages,
recs_per_page,
bytes_per_page);
index.total_n_recs / index.pages,
index.total_data_bytes / index.pages);
}
printf("\n");
printf("index_id\tpage_data_bytes_histgram(empty,...,oversized)\n");
for (std::map<unsigned long long, per_index_stats>::const_iterator it = index_ids.begin(); it != index_ids.end(); it++) {
printf("%llu\t", it->first);
fprintf(fil_out, "\n");
fprintf(fil_out, "index_id\tpage_data_bytes_histgram(empty,...,oversized)\n");
for (std::map<unsigned long long, per_index_stats>::const_iterator it = index_ids.begin();
it != index_ids.end(); it++) {
fprintf(fil_out, "%lld\t", it->first);
const per_index_stats& index = it->second;
for (i = 0; i < SIZE_RANGES_FOR_PAGE+2; i++) {
printf("\t%llu", index.pages_in_size_range[i]);
for (ulint i = 0; i < SIZE_RANGES_FOR_PAGE+2; i++) {
fprintf(fil_out, "\t%lld", index.pages_in_size_range[i]);
}
printf("\n");
fprintf(fil_out, "\n");
}
if (do_leaf) {
print_leaf_stats();
print_leaf_stats(fil_out);
}
}
int main(int argc, char **argv)
/* command line argument for innochecksum tool. */
static struct my_option innochecksum_options[] = {
{"help", '?', "Displays this help and exits.",
0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
{"info", 'I', "Synonym for --help.",
0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
{"version", 'V', "Displays version information and exits.",
0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0},
{"verbose", 'v', "Verbose (prints progress every 5 seconds).",
&verbose, &verbose, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
#ifndef DBUG_OFF
{"debug", '#', "Output debug log. See " REFMAN "dbug-package.html",
&dbug_setting, &dbug_setting, 0, GET_STR, OPT_ARG, 0, 0, 0, 0, 0, 0},
#endif /* !DBUG_OFF */
{"count", 'c', "Print the count of pages in the file and exits.",
&just_count, &just_count, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
{"start_page", 's', "Start on this page number (0 based).",
&start_page, &start_page, 0, GET_ULL, REQUIRED_ARG,
0, 0, ULLONG_MAX, 0, 1, 0},
{"end_page", 'e', "End at this page number (0 based).",
&end_page, &end_page, 0, GET_ULL, REQUIRED_ARG,
0, 0, ULLONG_MAX, 0, 1, 0},
{"page", 'p', "Check only this page (0 based).",
&do_page, &do_page, 0, GET_ULL, REQUIRED_ARG,
0, 0, ULLONG_MAX, 0, 1, 0},
{"strict-check", 'C', "Specify the strict checksum algorithm by the user.",
&strict_check, &strict_check, &innochecksum_algorithms_typelib,
GET_ENUM, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
{"no-check", 'n', "Ignore the checksum verification.",
&no_check, &no_check, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
{"allow-mismatches", 'a', "Maximum checksum mismatch allowed.",
&allow_mismatches, &allow_mismatches, 0,
GET_ULL, REQUIRED_ARG, 0, 0, ULLONG_MAX, 0, 1, 0},
{"write", 'w', "Rewrite the checksum algorithm by the user.",
&write_check, &write_check, &innochecksum_algorithms_typelib,
GET_ENUM, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
{"page-type-summary", 'S', "Display a count of each page type "
"in a tablespace.", &page_type_summary, &page_type_summary, 0,
GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
{"page-type-dump", 'D', "Dump the page type info for each page in a "
"tablespace.", &page_dump_filename, &page_dump_filename, 0,
GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
{"per-page-details", 'i', "Print out per-page detail information.",
&per_page_details, &per_page_details, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
{"log", 'l', "log output.",
&log_filename, &log_filename, 0,
GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
{"leaf", 'f', "Examine leaf index pages",
&do_leaf, &do_leaf, 0, GET_BOOL, NO_ARG, 0, 0, 0, 0, 0, 0},
{"merge", 'm', "leaf page count if merge given number of consecutive pages",
&n_merge, &n_merge, 0, GET_ULONG, REQUIRED_ARG, 0, 0, (longlong)10L, 0, 1, 0},
{0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}
};
/* Print out the Innodb version and machine information. */
static void print_version(void)
{
FILE* f; /* our input file */
char* filename; /* our input filename. */
unsigned char *big_buf= 0, *buf;
unsigned char *big_xdes= 0, *xdes;
ulong bytes; /* bytes read count */
ulint ct; /* current page number (0 based) */
time_t now; /* current time */
time_t lastt; /* last time */
ulint oldcsum, oldcsumfield, csum, csumfield, crc32, logseq, logseqfield;
/* ulints for checksum storage */
unsigned long long int size; /* size of file (has to be 64 bits) */
ulint pages; /* number of pages in file */
long long offset= 0;
int fd;
#ifdef DBUG_OFF
printf("%s Ver %s, for %s (%s)\n",
my_progname, INNODB_VERSION_STR,
SYSTEM_TYPE, MACHINE_TYPE);
#else
printf("%s-debug Ver %s, for %s (%s)\n",
my_progname, INNODB_VERSION_STR,
SYSTEM_TYPE, MACHINE_TYPE);
#endif /* DBUG_OFF */
}
static void usage(void)
{
print_version();
puts(ORACLE_WELCOME_COPYRIGHT_NOTICE("2000"));
printf("InnoDB offline file checksum utility.\n");
printf("Usage: %s [-c] [-s <start page>] [-e <end page>] "
"[-p <page>] [-i] [-v] [-a <allow mismatches>] [-n] "
"[-C <strict-check>] [-w <write>] [-S] [-D <page type dump>] "
"[-l <log>] [-l] [-m <merge pages>] <filename or [-]>\n", my_progname);
printf("See " REFMAN "innochecksum.html for usage hints.\n");
my_print_help(innochecksum_options);
my_print_variables(innochecksum_options);
}
ut_crc32_init();
extern "C" my_bool
innochecksum_get_one_option(
int optid,
const struct my_option *opt MY_ATTRIBUTE((unused)),
char *argument MY_ATTRIBUTE((unused)))
{
switch (optid) {
#ifndef DBUG_OFF
case '#':
dbug_setting = argument
? argument
: IF_WIN("d:O,innochecksum.trace",
"d:o,/tmp/innochecksum.trace");
DBUG_PUSH(dbug_setting);
break;
#endif /* !DBUG_OFF */
case 'e':
use_end_page = true;
break;
case 'p':
end_page = start_page = do_page;
use_end_page = true;
do_one_page = true;
break;
case 'V':
print_version();
exit(EXIT_SUCCESS);
break;
case 'C':
strict_verify = true;
switch ((srv_checksum_algorithm_t) strict_check) {
case SRV_CHECKSUM_ALGORITHM_STRICT_CRC32:
case SRV_CHECKSUM_ALGORITHM_CRC32:
srv_checksum_algorithm =
SRV_CHECKSUM_ALGORITHM_STRICT_CRC32;
break;
MY_INIT(argv[0]);
case SRV_CHECKSUM_ALGORITHM_STRICT_INNODB:
case SRV_CHECKSUM_ALGORITHM_INNODB:
srv_checksum_algorithm =
SRV_CHECKSUM_ALGORITHM_STRICT_INNODB;
break;
if (get_options(&argc,&argv))
exit(1);
case SRV_CHECKSUM_ALGORITHM_STRICT_NONE:
case SRV_CHECKSUM_ALGORITHM_NONE:
srv_checksum_algorithm =
SRV_CHECKSUM_ALGORITHM_STRICT_NONE;
break;
default:
return(true);
}
break;
case 'n':
no_check = true;
break;
case 'a':
case 'S':
break;
case 'w':
do_write = true;
break;
case 'D':
page_type_dump = true;
break;
case 'l':
is_log_enabled = true;
break;
case 'I':
case '?':
usage();
my_end(0);
exit(EXIT_SUCCESS);
break;
}
if (verbose)
my_print_variables(innochecksum_options);
return(false);
}
/* The file name is not optional */
filename = *argv;
if (*filename == '\0')
{
fprintf(stderr, "Error; File name missing\n");
goto error_out;
static
bool
get_options(
int *argc,
char ***argv)
{
if (handle_options(argc, argv, innochecksum_options,
innochecksum_get_one_option))
exit(true);
/* The next arg must be the filename */
if (!*argc) {
usage();
return (true);
}
#ifdef _WIN32
/* Switch off OS file buffering for the file. */
return (false);
}
HANDLE h = CreateFile(filename, GENERIC_READ,
FILE_SHARE_READ|FILE_SHARE_WRITE, 0,
OPEN_EXISTING, FILE_FLAG_NO_BUFFERING, 0);
/** Check from page 0 if table is encrypted. */
static
bool check_encryption(
const char* filename,
ulint zip_size,
byte * page)
{
ulint offset = (FSP_HEADER_OFFSET + (XDES_ARR_OFFSET + XDES_SIZE *
(zip_size ? zip_size : UNIV_PAGE_SIZE) / FSP_EXTENT_SIZE));
if (!h)
{
fprintf(stderr, "Error; cant open file\n");
goto error;
if (memcmp(page + offset, CRYPT_MAGIC, MAGIC_SZ) != 0) {
return false;
}
if (!GetFileSizeEx(h, (LARGE_INTEGER *)&size))
{
fprintf(stderr, "Error; GetFileSize() failed\n");
goto error;
ulint type = mach_read_from_1(page + offset + MAGIC_SZ + 0);
if (! (type == CRYPT_SCHEME_UNENCRYPTED ||
type == CRYPT_SCHEME_1)) {
return false;
}
fd = _open_osfhandle ((intptr_t) h, _O_RDONLY);
if (fd < 0)
{
fprintf(stderr, "Error; _open_osfhandle() failed\n");
goto error;
ulint iv_length = mach_read_from_1(page + offset + MAGIC_SZ + 1);
if (iv_length != CRYPT_SCHEME_1_IV_LEN) {
return false;
}
f = _fdopen(fd, "rb");
if (!f)
{
fprintf(stderr, "Error; fdopen() failed\n");
goto error;
uint min_key_version = mach_read_from_4
(page + offset + MAGIC_SZ + 2 + iv_length);
uint key_id = mach_read_from_4
(page + offset + MAGIC_SZ + 2 + iv_length + 4);
if (type == CRYPT_SCHEME_1) {
if (is_log_enabled) {
fprintf(log_file,"Tablespace %s encrypted key_version %u key_id %u\n",
filename, min_key_version, key_id);
}
}
/*
Disable stdio buffering (FILE_FLAG_NO_BUFFERING requires properly IO buffers
which stdio does not guarantee.
*/
setvbuf(f, NULL, _IONBF, 0);
return (type == CRYPT_SCHEME_1);
}
int main(
int argc,
char **argv)
{
/* our input file. */
FILE* fil_in = NULL;
/* our input filename. */
char* filename;
/* Buffer to store pages read. */
byte* buf = NULL;
byte* xdes = NULL;
/* bytes read count */
ulong bytes;
/* current time */
time_t now;
/* last time */
time_t lastt;
/* stat, to get file size. */
#ifdef _WIN32
struct _stat64 st;
#else
struct stat st;
/* stat the file to get size and page count */
if (stat(filename, &st))
{
fprintf(stderr, "Error; %s cannot be found\n", filename);
goto error_out;
}
size= st.st_size;
/* Open the file for reading */
f= fopen(filename, "rb");
#endif
#endif /* _WIN32 */
if (f == NULL)
{
fprintf(stderr, "Error; %s cannot be opened", filename);
perror(" ");
goto error_out;
}
int exit_status = 0;
big_buf = (unsigned char *)malloc(2 * UNIV_PAGE_SIZE_MAX);
if (big_buf == NULL)
{
fprintf(stderr, "Error; failed to allocate memory\n");
perror("");
goto error_f;
}
/* size of file (has to be 64 bits) */
unsigned long long int size = 0;
/* number of pages in file */
ulint pages;
/* Make sure the page is aligned */
buf = (unsigned char*)ut_align_down(big_buf
+ UNIV_PAGE_SIZE_MAX, UNIV_PAGE_SIZE_MAX);
off_t offset = 0;
/* count the no. of page corrupted. */
ulint mismatch_count = 0;
/* Variable to ack the page is corrupted or not. */
bool is_corrupted = false;
big_xdes = (unsigned char *)malloc(2 * UNIV_PAGE_SIZE_MAX);
if (big_xdes == NULL)
{
fprintf(stderr, "Error; failed to allocate memory\n");
perror("");
goto error_big_buf;
}
bool partial_page_read = false;
/* Enabled when read from stdin is done. */
bool read_from_stdin = false;
FILE* fil_page_type = NULL;
fpos_t pos;
/* Make sure the page is aligned */
xdes = (unsigned char*)ut_align_down(big_xdes
+ UNIV_PAGE_SIZE_MAX, UNIV_PAGE_SIZE_MAX);
/* Use to check the space id of given file. If space_id is zero,
then check whether page is doublewrite buffer.*/
ulint space_id = 0UL;
/* enable when space_id of given file is zero. */
bool is_system_tablespace = false;
ut_crc32_init();
MY_INIT(argv[0]);
DBUG_ENTER("main");
DBUG_PROCESS(argv[0]);
if (!get_page_size(f, buf, &logical_page_size, &physical_page_size))
goto error;
if (get_options(&argc,&argv)) {
exit_status = 1;
goto my_exit;
}
if (compressed)
{
printf("Table is compressed\n");
printf("Key block size is %lu\n", physical_page_size);
if (strict_verify && no_check) {
fprintf(stderr, "Error: --strict-check option cannot be used "
"together with --no-check option.\n");
exit_status = 1;
goto my_exit;
}
else
{
printf("Table is uncompressed\n");
printf("Page size is %lu\n", physical_page_size);
if (no_check && !do_write) {
fprintf(stderr, "Error: --no-check must be associated with "
"--write option.\n");
exit_status = 1;
goto my_exit;
}
pages= (ulint) (size / physical_page_size);
if (page_type_dump) {
fil_page_type = create_file(page_dump_filename);
if (!fil_page_type) {
exit_status = 1;
goto my_exit;
}
}
if (just_count)
{
if (verbose)
printf("Number of pages: ");
printf(ULINTPF "\n", pages);
goto ok;
if (is_log_enabled) {
log_file = create_file(log_filename);
if (!log_file) {
exit_status = 1;
goto my_exit;
}
else if (verbose)
{
printf("file %s = %llu bytes (" ULINTPF " pages)...\n",
filename, size, pages);
if (do_one_page)
printf("InnoChecksum; checking page %lu\n", do_page);
else
printf("InnoChecksum; checking pages in range %lu to %lu\n", start_page, use_end_page ? end_page : (pages - 1));
fprintf(log_file, "InnoDB File Checksum Utility.\n");
}
#ifdef UNIV_LINUX
if (posix_fadvise(fileno(f), 0, 0, POSIX_FADV_SEQUENTIAL) ||
posix_fadvise(fileno(f), 0, 0, POSIX_FADV_NOREUSE))
{
perror("posix_fadvise failed");
if (verbose) {
my_print_variables(innochecksum_options);
}
#endif
/* seek to the necessary position */
if (start_page)
{
fd= fileno(f);
if (!fd)
{
perror("Error; Unable to obtain file descriptor number");
goto error;
buf = (byte*) malloc(UNIV_PAGE_SIZE_MAX * 2);
xdes = (byte*)malloc(UNIV_PAGE_SIZE_MAX * 2);
/* The file name is not optional. */
for (int i = 0; i < argc; ++i) {
/* Reset parameters for each file. */
filename = argv[i];
memset(&page_type, 0, sizeof(innodb_page_type));
is_corrupted = false;
partial_page_read = false;
skip_page = false;
if (is_log_enabled) {
fprintf(log_file, "Filename = %s\n", filename);
}
offset= (longlong)start_page * (longlong)physical_page_size;
if (*filename == '-') {
/* read from stdin. */
fil_in = stdin;
read_from_stdin = true;
}
/* stat the file to get size and page count. */
if (!read_from_stdin &&
#ifdef _WIN32
if (_lseeki64(fd, offset, SEEK_SET) != offset)
_stat64(filename, &st)) {
#else
if (lseek(fd, offset, SEEK_SET) != offset)
#endif
{
perror("Error; Unable to seek to necessary offset");
goto error;
stat(filename, &st)) {
#endif /* _WIN32 */
fprintf(stderr, "Error: %s cannot be found\n",
filename);
exit_status = 1;
goto my_exit;
}
if (!read_from_stdin) {
size = st.st_size;
fil_in = open_file(filename);
/*If fil_in is NULL, terminate as some error encountered */
if(fil_in == NULL) {
exit_status = 1;
goto my_exit;
}
/* Save the current file pointer in pos variable.*/
if (0 != fgetpos(fil_in, &pos)) {
perror("fgetpos");
exit_status = 1;
goto my_exit;
}
}
/* main checksumming loop */
ct= start_page;
lastt= 0;
while (!feof(f))
{
int page_ok = 1;
/* Read the minimum page size. */
bytes = ulong(fread(buf, 1, UNIV_ZIP_SIZE_MIN, fil_in));
partial_page_read = true;
bytes= fread(buf, 1, physical_page_size, f);
if (bytes != UNIV_ZIP_SIZE_MIN) {
fprintf(stderr, "Error: Was not able to read the "
"minimum page size ");
fprintf(stderr, "of %d bytes. Bytes read was %lu\n",
UNIV_ZIP_SIZE_MIN, bytes);
if (!bytes && feof(f))
goto ok;
exit_status = 1;
goto my_exit;
}
if (ferror(f))
{
fprintf(stderr, "Error reading %lu bytes", physical_page_size);
perror(" ");
goto error;
/* enable variable is_system_tablespace when space_id of given
file is zero. Use to skip the checksum verification and rewrite
for doublewrite pages. */
is_system_tablespace = (!memcmp(&space_id, buf +
FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID, 4))
? true : false;
/* Determine page size, zip_size and page compression
from fsp_flags and encryption metadata from page 0 */
ulint flags = mach_read_from_4(FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + buf);
ulint page_size = fsp_flags_get_page_size(flags);
ulint zip_size = fsp_flags_get_zip_size(flags);
logical_page_size = zip_size;
physical_page_size = page_size;
srv_page_size = page_size;
bool is_compressed = FSP_FLAGS_HAS_PAGE_COMPRESSION(flags);
if (page_size > UNIV_ZIP_SIZE_MIN) {
/* Read rest of the page 0 to determine crypt_data */
bytes = ulong(read_file(buf, partial_page_read, page_size, fil_in));
if (bytes != page_size) {
fprintf(stderr, "Error: Was not able to read the "
"rest of the page ");
fprintf(stderr, "of %lu bytes. Bytes read was %lu\n",
page_size - UNIV_ZIP_SIZE_MIN, bytes);
exit_status = 1;
goto my_exit;
}
partial_page_read = false;
}
ulint page_type = mach_read_from_2(buf+FIL_PAGE_TYPE);
ulint key_version = mach_read_from_4(buf + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION);
/* Now that we have full page 0 in buffer, check encryption */
bool is_encrypted = check_encryption(filename, zip_size, buf);
pages = (ulint) (size / page_size);
if (key_version && page_type != FIL_PAGE_PAGE_COMPRESSED) {
encrypted = true;
if (just_count) {
if (read_from_stdin) {
fprintf(stderr, "Number of pages:" ULINTPF "\n", pages);
} else {
encrypted = false;
printf("Number of pages:" ULINTPF "\n", pages);
}
continue;
} else if (verbose && !read_from_stdin) {
if (is_log_enabled) {
fprintf(log_file, "file %s = %llu bytes "
"(" ULINTPF " pages)\n", filename, size, pages);
if (do_one_page) {
fprintf(log_file, "Innochecksum: "
"checking page %" PRIuMAX "\n",
do_page);
}
}
} else {
if (is_log_enabled) {
fprintf(log_file, "Innochecksum: checking "
"pages in range %" PRIuMAX " to %" PRIuMAX "\n",
start_page, use_end_page ?
end_page : (pages - 1));
}
}
ulint comp_method = 0;
/* seek to the necessary position */
if (start_page) {
if (!read_from_stdin) {
/* If read is not from stdin, we can use
fseeko() to position the file pointer to
the desired page. */
partial_page_read = false;
offset = (off_t) start_page
* (off_t) page_size;
#ifdef _WIN32
if (_fseeki64(fil_in, offset, SEEK_SET)) {
#else
if (fseeko(fil_in, offset, SEEK_SET)) {
#endif /* _WIN32 */
perror("Error: Unable to seek to "
"necessary offset");
if (encrypted) {
comp_method = mach_read_from_2(buf+FIL_PAGE_DATA+FIL_PAGE_COMPRESSED_SIZE);
} else {
comp_method = mach_read_from_8(buf+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION);
exit_status = 1;
goto my_exit;
}
/* Save the current file pointer in
pos variable. */
if (0 != fgetpos(fil_in, &pos)) {
perror("fgetpos");
ulint comp_size = mach_read_from_2(buf+FIL_PAGE_DATA);
ib_uint32_t encryption_checksum = mach_read_from_4(buf+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4);
exit_status = 1;
goto my_exit;
}
} else {
ulong count = 0;
if (page_type == FIL_PAGE_PAGE_COMPRESSED) {
/* Page compressed tables do not have any checksum */
if (debug)
fprintf(stderr, "Page " ULINTPF
" page compressed with method %s real_size " ULINTPF "\n", ct,
fil_get_compression_alg_name(comp_method), comp_size);
page_ok = 1;
} else if (compressed) {
/* compressed pages */
ulint crccsum = page_zip_calc_checksum(buf, physical_page_size, SRV_CHECKSUM_ALGORITHM_CRC32);
ulint icsum = page_zip_calc_checksum(buf, physical_page_size, SRV_CHECKSUM_ALGORITHM_INNODB);
while (!feof(fil_in)) {
if (start_page == count) {
break;
}
/* We read a part of page to find the
minimum page size. We cannot reset
the file pointer to the beginning of
the page if we are reading from stdin
(fseeko() on stdin doesn't work). So
read only the remaining part of page,
if partial_page_read is enable. */
bytes = read_file(buf,
partial_page_read,
static_cast<ulong>(
page_size),
fil_in);
if (debug) {
if (key_version != 0) {
fprintf(stderr,
"Page " ULINTPF
" encrypted key_version " ULINTPF
" calculated = " ULINTPF "; crc32 = " ULINTPF
"; recorded = %u\n",
ct, key_version, icsum, crccsum, encryption_checksum);
partial_page_read = false;
count++;
if (!bytes || feof(fil_in)) {
fprintf(stderr, "Error: Unable "
"to seek to necessary "
"offset");
exit_status = 1;
goto my_exit;
}
}
}
}
if (encrypted) {
if (encryption_checksum != 0 && crccsum != encryption_checksum && icsum != encryption_checksum) {
if (debug)
fprintf(stderr, "page " ULINTPF
": compressed: calculated = " ULINTPF
"; crc32 = " ULINTPF "; recorded = %u\n",
ct, icsum, crccsum, encryption_checksum);
fprintf(stderr, "Fail; page " ULINTPF
" invalid (fails compressed page checksum).\n", ct);
if (page_type_dump) {
fprintf(fil_page_type,
"\n\nFilename::%s\n", filename);
fprintf(fil_page_type,
"========================================"
"======================================\n");
fprintf(fil_page_type,
"\tPAGE_NO\t\t|\t\tPAGE_TYPE\t\t"
"\t|\tEXTRA INFO\n");
fprintf(fil_page_type,
"========================================"
"======================================\n");
}
} else {
if (!page_zip_verify_checksum(buf, physical_page_size)) {
fprintf(stderr, "Fail; page " ULINTPF
" invalid (fails compressed page checksum).\n", ct);
if (!skip_corrupt)
goto error;
page_ok = 0;
/* main checksumming loop */
cur_page_num = start_page;
lastt = 0;
while (!feof(fil_in)) {
bytes = read_file(buf, partial_page_read,
static_cast<ulong>(
page_size), fil_in);
partial_page_read = false;
if (!bytes && feof(fil_in)) {
break;
}
if (ferror(fil_in)) {
fprintf(stderr, "Error reading " ULINTPF " bytes",
page_size);
perror(" ");
exit_status = 1;
goto my_exit;
}
if (bytes != page_size) {
fprintf(stderr, "Error: bytes read (%lu) "
"doesn't match page size (" ULINTPF ")\n",
bytes, page_size);
exit_status = 1;
goto my_exit;
}
if (is_system_tablespace) {
/* enable when page is double write buffer.*/
skip_page = is_page_doublewritebuffer(buf);
} else {
if (key_version != 0) {
/* Encrypted page */
if (debug) {
if (page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) {
fprintf(stderr,
"Page " ULINTPF
" page compressed with method %s real_size " ULINTPF
" and encrypted key_version " ULINTPF " checksum %u\n",
ct, fil_get_compression_alg_name(comp_method), comp_size, key_version, encryption_checksum);
} else {
fprintf(stderr,
"Page " ULINTPF
" encrypted key_version " ULINTPF " checksum %u\n",
ct, key_version, encryption_checksum);
skip_page = false;
}
/* If no-check is enabled, skip the
checksum verification.*/
if (!no_check) {
/* Checksum verification */
if (!skip_page) {
is_corrupted = is_page_corrupted(
buf, page_size, zip_size, is_encrypted, is_compressed);
if (is_corrupted) {
fprintf(stderr, "Fail: page "
"%" PRIuMAX " invalid\n",
cur_page_num);
mismatch_count++;
if(mismatch_count > allow_mismatches) {
fprintf(stderr,
"Exceeded the "
"maximum allowed "
"checksum mismatch "
"count::%" PRIuMAX "\n",
allow_mismatches);
exit_status = 1;
goto my_exit;
}
}
/* Page compressed tables do not contain FIL tailer */
if (page_type != FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED && page_type != FIL_PAGE_PAGE_COMPRESSED) {
/* check the "stored log sequence numbers" */
logseq= mach_read_from_4(buf + FIL_PAGE_LSN + 4);
logseqfield= mach_read_from_4(buf + logical_page_size - FIL_PAGE_END_LSN_OLD_CHKSUM + 4);
if (debug)
printf("page " ULINTPF
": log sequence number: first = " ULINTPF
"; second = " ULINTPF "\n",
ct, logseq, logseqfield);
if (logseq != logseqfield)
{
fprintf(stderr, "Fail; page " ULINTPF
" invalid (fails log sequence number check)\n", ct);
if (!skip_corrupt)
goto error;
page_ok = 0;
}
/* check old method of checksumming */
oldcsum= buf_calc_page_old_checksum(buf);
oldcsumfield= mach_read_from_4(buf + logical_page_size - FIL_PAGE_END_LSN_OLD_CHKSUM);
if (debug)
printf("page " ULINTPF
": old style: calculated = " ULINTPF
"; recorded = " ULINTPF "\n",
ct, oldcsum, oldcsumfield);
if (oldcsumfield != mach_read_from_4(buf + FIL_PAGE_LSN) && oldcsumfield != oldcsum)
{
fprintf(stderr, "Fail; page " ULINTPF
" invalid (fails old style checksum)\n", ct);
if (!skip_corrupt)
goto error;
page_ok = 0;
}
}
/* now check the new method */
csum= buf_calc_page_new_checksum(buf);
crc32= buf_calc_page_crc32(buf);
csumfield= mach_read_from_4(buf + FIL_PAGE_SPACE_OR_CHKSUM);
/* Rewrite checksum. Note that for encrypted and
page compressed tables this is not currently supported. */
if (do_write &&
!is_encrypted &&
!is_compressed
&& !write_file(filename, fil_in, buf,
zip_size != 0, &pos,
static_cast<ulong>(page_size))) {
if (key_version)
csumfield = encryption_checksum;
exit_status = 1;
goto my_exit;
}
if (debug)
printf("page " ULINTPF
": new style: calculated = " ULINTPF
"; crc32 = " ULINTPF "; recorded = " ULINTPF "\n",
ct, csum, crc32, csumfield);
if (csumfield != 0 && crc32 != csumfield && csum != csumfield)
{
fprintf(stderr, "Fail; page " ULINTPF
" invalid (fails innodb and crc32 checksum)\n", ct);
if (!skip_corrupt)
goto error;
page_ok = 0;
/* end if this was the last page we were supposed to check */
if (use_end_page && (cur_page_num >= end_page)) {
break;
}
if (per_page_details) {
printf("page %ld ", cur_page_num);
}
/* end if this was the last page we were supposed to check */
if (use_end_page && (ct >= end_page))
goto ok;
if (per_page_details)
{
printf("page " ULINTPF " ", ct);
if (page_type_summary || page_type_dump) {
parse_page(buf, xdes, fil_page_type);
}
/* do counter increase and progress printing */
ct++;
if (!page_ok)
{
if (per_page_details)
{
printf("BAD_CHECKSUM\n");
cur_page_num++;
if (verbose && !read_from_stdin) {
if ((cur_page_num % 64) == 0) {
now = time(0);
if (!lastt) {
lastt= now;
}
if (now - lastt >= 1
&& is_log_enabled) {
fprintf(log_file, "page %" PRIuMAX " "
"okay: %.3f%% done\n",
(cur_page_num - 1),
(float) cur_page_num / pages * 100);
lastt = now;
}
}
}
n_bad_checksum++;
continue;
}
/* Can't parse compressed or/and encrypted pages */
if (page_type != FIL_PAGE_PAGE_COMPRESSED && !encrypted) {
parse_page(buf, xdes);
if (!read_from_stdin) {
/* flcose() will flush the data and release the lock if
any acquired. */
fclose(fil_in);
}
if (verbose)
{
if (ct % 64 == 0)
{
now= time(0);
if (!lastt) lastt= now;
if (now - lastt >= 1)
{
printf("page " ULINTPF " okay: %.3f%% done\n",
(ct - 1), (float) ct / pages * 100);
lastt= now;
/* Enabled for page type summary. */
if (page_type_summary) {
if (!read_from_stdin) {
fprintf(stdout, "\nFile::%s",filename);
print_summary(stdout);
} else {
print_summary(stderr);
}
}
}
if (is_log_enabled) {
fclose(log_file);
}
ok:
if (!just_count)
print_stats();
free(big_xdes);
free(big_buf);
fclose(f);
my_end(0);
exit(0);
error:
free(big_xdes);
error_big_buf:
free(big_buf);
error_f:
fclose(f);
error_out:
my_end(0);
exit(1);
my_exit:
if (buf) {
free(buf);
}
my_end(exit_status);
exit(exit_status);
}
mysql-test/suite/encryption/r/innochecksum.result
View file @ 8b019f87
......@@ -7,6 +7,7 @@ CREATE TABLE t2 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB ROW_FOR
CREATE TABLE t3 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB ROW_FORMAT=COMPRESSED ENCRYPTED=NO;
CREATE TABLE t4 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB PAGE_COMPRESSED=1;
CREATE TABLE t5 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB PAGE_COMPRESSED=1 ENCRYPTED=YES ENCRYPTION_KEY_ID=4;
CREATE TABLE t6 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB;
# Write file to make mysql-test-run.pl expect the "crash", but don't
# start it until it's told to
# We give 30 seconds to do a clean shutdown because we do not want
......@@ -18,6 +19,27 @@ CREATE TABLE t5 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB PAGE_CO
# Run innochecksum on t3
# Run innochecksum on t4
# Run innochecksum on t4
# Run innochecksum on t5
# Run innochecksum on t6
# Backup tables before corrupting
# Corrupt FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION
# Run innochecksum on t2
# Run innochecksum on t3
# no encryption corrupting the field should not have effect
# Run innochecksum on t6
# no encryption corrupting the field should not have effect
# Restore the original tables
# Corrupt FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION+4 (post encryption checksum)
# Run innochecksum on t2
# Run innochecksum on t3
# Run innochecksum on t6
# no encryption corrupting the field should not have effect
# Restore the original tables
# Corrupt FIL_DATA+10 (data)
# Run innochecksum on t2
# Run innochecksum on t3
# Run innochecksum on t6
# Restore the original tables
# Write file to make mysql-test-run.pl start up the server again
# Cleanup
DROP TABLE t1, t2, t3, t4, t5;
DROP TABLE t1, t2, t3, t4, t5, t6;
mysql-test/suite/encryption/t/innochecksum.test
View file @ 8b019f87
......@@ -2,7 +2,7 @@
# MDEV-8773: InnoDB innochecksum does not work with encrypted or page compressed tables
#
# Don't test under embedded
# Don't test under embedded as we restart server
-- source include/not_embedded.inc
# Require InnoDB
-- source include/have_innodb.inc
......@@ -13,16 +13,12 @@ if (!$INNOCHECKSUM) {
--die Need innochecksum binary
}
--disable_query_log
let $innodb_compression_algorithm_orig=`SELECT @@innodb_compression_algorithm`;
let $innodb_file_format_orig = `SELECT @@innodb_file_format`;
let $innodb_file_per_table_orig = `SELECT @@innodb_file_per_table`;
--enable_query_log
--disable_warnings
SET GLOBAL innodb_file_format = `Barracuda`;
SET GLOBAL innodb_file_per_table = ON;
# zlib
set global innodb_compression_algorithm = 1;
--enable_warnings
--echo # Create and populate a tables
CREATE TABLE t1 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB ENCRYPTED=YES ENCRYPTION_KEY_ID=4;
......@@ -30,9 +26,11 @@ CREATE TABLE t2 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB ROW_FOR
CREATE TABLE t3 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB ROW_FORMAT=COMPRESSED ENCRYPTED=NO;
CREATE TABLE t4 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB PAGE_COMPRESSED=1;
CREATE TABLE t5 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB PAGE_COMPRESSED=1 ENCRYPTED=YES ENCRYPTION_KEY_ID=4;
CREATE TABLE t6 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT) ENGINE=InnoDB;
--disable_query_log
--let $i = 1000
begin;
while ($i)
{
INSERT INTO t1 (b) VALUES (REPEAT('abcdefghijklmnopqrstuvwxyz', 100));
......@@ -42,6 +40,8 @@ INSERT INTO t2 SELECT * FROM t1;
INSERT INTO t3 SELECT * FROM t1;
INSERT INTO t4 SELECT * FROM t1;
INSERT INTO t5 SELECT * FROM t1;
INSERT INTO t6 SELECT * FROM t1;
commit;
--enable_query_log
let $MYSQLD_DATADIR=`select @@datadir`;
......@@ -50,6 +50,10 @@ let t2_IBD = $MYSQLD_DATADIR/test/t2.ibd;
let t3_IBD = $MYSQLD_DATADIR/test/t3.ibd;
let t4_IBD = $MYSQLD_DATADIR/test/t4.ibd;
let t5_IBD = $MYSQLD_DATADIR/test/t5.ibd;
let t6_IBD = $MYSQLD_DATADIR/test/t6.ibd;
let INNODB_PAGE_SIZE=`select @@innodb_page_size`;
let MYSQLD_DATADIR=`select @@datadir`;
--echo # Write file to make mysql-test-run.pl expect the "crash", but don't
--echo # start it until it's told to
......@@ -83,19 +87,198 @@ shutdown_server 30;
--exec $INNOCHECKSUM $t4_IBD
--echo # Run innochecksum on t5
--exec $INNOCHECKSUM $t5_IBD
--echo # Run innochecksum on t6
--exec $INNOCHECKSUM $t6_IBD
--enable_result_log
--echo # Backup tables before corrupting
--copy_file $MYSQLD_DATADIR/test/t1.ibd $MYSQLD_DATADIR/test/t1.ibd.backup
--copy_file $MYSQLD_DATADIR/test/t2.ibd $MYSQLD_DATADIR/test/t2.ibd.backup
--copy_file $MYSQLD_DATADIR/test/t3.ibd $MYSQLD_DATADIR/test/t3.ibd.backup
--copy_file $MYSQLD_DATADIR/test/t4.ibd $MYSQLD_DATADIR/test/t4.ibd.backup
--copy_file $MYSQLD_DATADIR/test/t5.ibd $MYSQLD_DATADIR/test/t5.ibd.backup
--copy_file $MYSQLD_DATADIR/test/t6.ibd $MYSQLD_DATADIR/test/t6.ibd.backup
#
# MDEV-11939: innochecksum mistakes a file for an encrypted one
#
--echo # Corrupt FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION
perl;
open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t1.ibd") or die "open";
binmode FILE;
seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 26, SEEK_SET) or die "seek";
print FILE pack("H*", "c00lcafedeadb017");
close FILE or die "close";
open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t2.ibd") or die "open";
binmode FILE;
seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 26, SEEK_SET) or die "seek";
print FILE pack("H*", "c00lcafedeadb017");
close FILE or die "close";
open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t3.ibd") or die "open";
binmode FILE;
seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 26, SEEK_SET) or die "seek";
print FILE pack("H*", "c00lcafedeadb017");
close FILE or die "close";
open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t6.ibd") or die "open";
binmode FILE;
seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 26, SEEK_SET) or die "seek";
print FILE pack("H*", "c00lcafedeadb017");
close FILE or die "close";
EOF
-- disable_result_log
--error 1
--exec $INNOCHECKSUM $t1_IBD
--echo # Run innochecksum on t2
--error 1
--exec $INNOCHECKSUM $t2_IBD
--echo # Run innochecksum on t3
--echo # no encryption corrupting the field should not have effect
--exec $INNOCHECKSUM $t3_IBD
--echo # Run innochecksum on t6
--echo # no encryption corrupting the field should not have effect
--exec $INNOCHECKSUM $t6_IBD
--enable_result_log
--echo # Restore the original tables
--remove_file $MYSQLD_DATADIR/test/t1.ibd
--remove_file $MYSQLD_DATADIR/test/t2.ibd
--remove_file $MYSQLD_DATADIR/test/t3.ibd
--remove_file $MYSQLD_DATADIR/test/t4.ibd
--remove_file $MYSQLD_DATADIR/test/t5.ibd
--remove_file $MYSQLD_DATADIR/test/t6.ibd
--copy_file $MYSQLD_DATADIR/test/t1.ibd.backup $MYSQLD_DATADIR/test/t1.ibd
--copy_file $MYSQLD_DATADIR/test/t2.ibd.backup $MYSQLD_DATADIR/test/t2.ibd
--copy_file $MYSQLD_DATADIR/test/t3.ibd.backup $MYSQLD_DATADIR/test/t3.ibd
--copy_file $MYSQLD_DATADIR/test/t4.ibd.backup $MYSQLD_DATADIR/test/t4.ibd
--copy_file $MYSQLD_DATADIR/test/t5.ibd.backup $MYSQLD_DATADIR/test/t5.ibd
--copy_file $MYSQLD_DATADIR/test/t6.ibd.backup $MYSQLD_DATADIR/test/t6.ibd
--echo # Corrupt FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION+4 (post encryption checksum)
perl;
open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t1.ibd") or die "open";
binmode FILE;
seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 30, SEEK_SET) or die "seek";
print FILE pack("H*", "c00lcafedeadb017");
close FILE or die "close";
open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t2.ibd") or die "open";
binmode FILE;
seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 30, SEEK_SET) or die "seek";
print FILE pack("H*", "c00lcafedeadb017");
close FILE or die "close";
open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t3.ibd") or die "open";
binmode FILE;
seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 30, SEEK_SET) or die "seek";
print FILE pack("H*", "c00lcafedeadb017");
close FILE or die "close";
open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t6.ibd") or die "open";
binmode FILE;
seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 30, SEEK_SET) or die "seek";
print FILE pack("H*", "c00lcafedeadb017");
close FILE or die "close";
EOF
-- disable_result_log
--error 1
--exec $INNOCHECKSUM $t1_IBD
--echo # Run innochecksum on t2
--error 1
--exec $INNOCHECKSUM $t2_IBD
--echo # Run innochecksum on t3
--error 1
--exec $INNOCHECKSUM $t3_IBD
--echo # Run innochecksum on t6
--echo # no encryption corrupting the field should not have effect
--exec $INNOCHECKSUM $t6_IBD
--enable_result_log
--echo # Restore the original tables
--remove_file $MYSQLD_DATADIR/test/t1.ibd
--remove_file $MYSQLD_DATADIR/test/t2.ibd
--remove_file $MYSQLD_DATADIR/test/t3.ibd
--remove_file $MYSQLD_DATADIR/test/t4.ibd
--remove_file $MYSQLD_DATADIR/test/t5.ibd
--remove_file $MYSQLD_DATADIR/test/t6.ibd
--copy_file $MYSQLD_DATADIR/test/t1.ibd.backup $MYSQLD_DATADIR/test/t1.ibd
--copy_file $MYSQLD_DATADIR/test/t2.ibd.backup $MYSQLD_DATADIR/test/t2.ibd
--copy_file $MYSQLD_DATADIR/test/t3.ibd.backup $MYSQLD_DATADIR/test/t3.ibd
--copy_file $MYSQLD_DATADIR/test/t4.ibd.backup $MYSQLD_DATADIR/test/t4.ibd
--copy_file $MYSQLD_DATADIR/test/t5.ibd.backup $MYSQLD_DATADIR/test/t5.ibd
--copy_file $MYSQLD_DATADIR/test/t6.ibd.backup $MYSQLD_DATADIR/test/t6.ibd
--echo # Corrupt FIL_DATA+10 (data)
perl;
open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t1.ibd") or die "open";
binmode FILE;
seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 48, SEEK_SET) or die "seek";
print FILE pack("H*", "c00lcafedeadb017");
close FILE or die "close";
open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t2.ibd") or die "open";
binmode FILE;
seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 48, SEEK_SET) or die "seek";
print FILE pack("H*", "c00lcafedeadb017");
close FILE or die "close";
open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t3.ibd") or die "open";
binmode FILE;
seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 48, SEEK_SET) or die "seek";
print FILE pack("H*", "c00lcafedeadb017");
close FILE or die "close";
open(FILE, "+<", "$ENV{MYSQLD_DATADIR}/test/t6.ibd") or die "open";
binmode FILE;
seek(FILE, $ENV{'INNODB_PAGE_SIZE'} * 3 + 48, SEEK_SET) or die "seek";
print FILE pack("H*", "c00lcafedeadb017");
close FILE or die "close";
EOF
-- disable_result_log
--error 1
--exec $INNOCHECKSUM $t1_IBD
--echo # Run innochecksum on t2
--error 1
--exec $INNOCHECKSUM $t2_IBD
--echo # Run innochecksum on t3
--error 1
--exec $INNOCHECKSUM $t3_IBD
--echo # Run innochecksum on t6
--error 1
--exec $INNOCHECKSUM $t6_IBD
--enable_result_log
--echo # Restore the original tables
--move_file $MYSQLD_DATADIR/test/t1.ibd.backup $MYSQLD_DATADIR/test/t1.ibd
--move_file $MYSQLD_DATADIR/test/t2.ibd.backup $MYSQLD_DATADIR/test/t2.ibd
--move_file $MYSQLD_DATADIR/test/t3.ibd.backup $MYSQLD_DATADIR/test/t3.ibd
--move_file $MYSQLD_DATADIR/test/t4.ibd.backup $MYSQLD_DATADIR/test/t4.ibd
--move_file $MYSQLD_DATADIR/test/t5.ibd.backup $MYSQLD_DATADIR/test/t5.ibd
--move_file $MYSQLD_DATADIR/test/t6.ibd.backup $MYSQLD_DATADIR/test/t6.ibd
--echo # Write file to make mysql-test-run.pl start up the server again
--exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
--enable_reconnect
--source include/wait_until_connected_again.inc
--echo # Cleanup
DROP TABLE t1, t2, t3, t4, t5;
# reset system
--disable_query_log
EVAL SET GLOBAL innodb_compression_algorithm = $innodb_compression_algorithm_orig;
EVAL SET GLOBAL innodb_file_per_table = $innodb_file_per_table_orig;
EVAL SET GLOBAL innodb_file_format = $innodb_file_format_orig;
--enable_query_log
DROP TABLE t1, t2, t3, t4, t5, t6;
storage/innobase/buf/buf0buf.cc
View file @ 8b019f87
......@@ -31,12 +31,16 @@ The database buffer buf_pool
Created 11/5/1995 Heikki Tuuri
*******************************************************/
#include "univ.i"
#include "mach0data.h"
#include "buf0buf.h"
#include <string.h>
#ifdef UNIV_NONINL
#include "buf0buf.ic"
#endif
#ifndef UNIV_INNOCHECKSUM
#include "mem0mem.h"
#include "btr0btr.h"
#include "fil0fil.h"
......@@ -52,13 +56,15 @@ Created 11/5/1995 Heikki Tuuri
#include "srv0srv.h"
#include "dict0dict.h"
#include "log0recv.h"
#include "page0zip.h"
#include "srv0mon.h"
#include "buf0checksum.h"
#ifdef HAVE_LIBNUMA
#include <numa.h>
#include <numaif.h>
#endif // HAVE_LIBNUMA
#endif /* !UNIV_INNOCHECKSUM */
#include "page0zip.h"
#include "buf0checksum.h"
#ifndef UNIV_INNOCHECKSUM
#include "fil0pagecompress.h"
#include "ha_prototypes.h"
#include "ut0byte.h"
......@@ -528,6 +534,7 @@ buf_block_alloc(
return(block);
}
#endif /* !UNIV_HOTBACKUP */
#endif /* !UNIV_INNOCHECKSUM */
/** Check if a page is all zeroes.
@param[in] read_buf database page
......@@ -561,6 +568,17 @@ buf_page_is_checksum_valid_crc32(
{
ib_uint32_t crc32 = buf_calc_page_crc32(read_buf);
#ifdef UNIV_INNOCHECKSUM
if (log_file
&& srv_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_STRICT_CRC32) {
fprintf(log_file, "page::%lu;"
" crc32 calculated = %u;"
" recorded checksum field1 = %lu recorded"
" checksum field2 =%lu\n", cur_page_num,
crc32, checksum_field1, checksum_field2);
}
#endif /* UNIV_INNOCHECKSUM */
if (!(checksum_field1 == crc32 && checksum_field2 == crc32)) {
DBUG_PRINT("buf_checksum",
("Page checksum crc32 not valid field1 " ULINTPF
......@@ -595,12 +613,45 @@ buf_page_is_checksum_valid_innodb(
2. Newer InnoDB versions store the old formula checksum
(buf_calc_page_old_checksum()). */
ulint old_checksum = buf_calc_page_old_checksum(read_buf);
ulint new_checksum = buf_calc_page_new_checksum(read_buf);
#ifdef UNIV_INNOCHECKSUM
if (log_file
&& srv_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_INNODB) {
fprintf(log_file, "page::%lu;"
" old style: calculated ="
" %lu; recorded = %lu\n",
cur_page_num, old_checksum,
checksum_field2);
fprintf(log_file, "page::%lu;"
" new style: calculated ="
" %lu; crc32 = %u; recorded = %lu\n",
cur_page_num, new_checksum,
buf_calc_page_crc32(read_buf), checksum_field1);
}
if (log_file
&& srv_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_STRICT_INNODB) {
fprintf(log_file, "page::%lu;"
" old style: calculated ="
" %lu; recorded checksum = %lu\n",
cur_page_num, old_checksum,
checksum_field2);
fprintf(log_file, "page::%lu;"
" new style: calculated ="
" %lu; recorded checksum = %lu\n",
cur_page_num, new_checksum,
checksum_field1);
}
#endif /* UNIV_INNOCHECKSUM */
if (checksum_field2 != mach_read_from_4(read_buf + FIL_PAGE_LSN)
&& checksum_field2 != buf_calc_page_old_checksum(read_buf)) {
&& checksum_field2 != old_checksum) {
DBUG_PRINT("buf_checksum",
("Page checksum innodb not valid field1 " ULINTPF
" field2 " ULINTPF "crc32 " ULINTPF " lsn " ULINTPF ".",
checksum_field1, checksum_field2, buf_calc_page_old_checksum(read_buf),
checksum_field1, checksum_field2, old_checksum,
mach_read_from_4(read_buf + FIL_PAGE_LSN)));
return(false);
......@@ -612,11 +663,11 @@ buf_page_is_checksum_valid_innodb(
(always equal to 0), to FIL_PAGE_SPACE_OR_CHKSUM */
if (checksum_field1 != 0
&& checksum_field1 != buf_calc_page_new_checksum(read_buf)) {
&& checksum_field1 != new_checksum) {
DBUG_PRINT("buf_checksum",
("Page checksum innodb not valid field1 " ULINTPF
" field2 " ULINTPF "crc32 " ULINTPF " lsn " ULINTPF ".",
checksum_field1, checksum_field2, buf_calc_page_new_checksum(read_buf),
checksum_field1, checksum_field2, new_checksum,
mach_read_from_4(read_buf + FIL_PAGE_LSN)));
return(false);
......@@ -646,6 +697,18 @@ buf_page_is_checksum_valid_none(
mach_read_from_4(read_buf + FIL_PAGE_LSN)));
}
#ifdef UNIV_INNOCHECKSUM
if (log_file
&& srv_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_STRICT_NONE) {
fprintf(log_file,
"page::%lu; none checksum: calculated"
" = %lu; recorded checksum_field1 = %lu"
" recorded checksum_field2 = %lu\n",
cur_page_num, BUF_NO_CHECKSUM_MAGIC,
checksum_field1, checksum_field2);
}
#endif /* UNIV_INNOCHECKSUM */
return(checksum_field1 == checksum_field2
&& checksum_field1 == BUF_NO_CHECKSUM_MAGIC);
}
......@@ -662,14 +725,18 @@ buf_page_is_corrupted(
bool check_lsn,
const byte* read_buf,
ulint zip_size,
#ifndef UNIV_INNOCHECKSUM
const fil_space_t* space)
#else
const void* space)
#endif
{
ulint checksum_field1;
ulint checksum_field2;
ulint space_id = mach_read_from_4(
read_buf + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID);
ulint page_type = mach_read_from_2(
read_buf + FIL_PAGE_TYPE);
ulint checksum_field1 = 0;
ulint checksum_field2 = 0;
#ifndef UNIV_INNOCHECKSUM
ulint space_id = mach_read_from_4(read_buf + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID);
#endif
ulint page_type = mach_read_from_2(read_buf + FIL_PAGE_TYPE);
/* We can trust page type if page compression is set on tablespace
flags because page compression flag means file must have been
......@@ -682,7 +749,10 @@ buf_page_is_corrupted(
decompressed at this stage). */
if ((page_type == FIL_PAGE_PAGE_COMPRESSED ||
page_type == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED)
&& space && FSP_FLAGS_HAS_PAGE_COMPRESSION(space->flags)) {
#ifndef UNIV_INNOCHECKSUM
&& space && FSP_FLAGS_HAS_PAGE_COMPRESSION(space->flags)
#endif
) {
return (false);
}
......@@ -693,16 +763,17 @@ buf_page_is_corrupted(
/* Stored log sequence numbers at the start and the end
of page do not match */
#ifndef UNIV_INNOCHECKSUM
ib_logf(IB_LOG_LEVEL_INFO,
"Log sequence number at the start %lu and the end %lu do not match.",
mach_read_from_4(read_buf + FIL_PAGE_LSN + 4),
mach_read_from_4(read_buf + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM + 4));
#endif /* UNIV_INNOCHECKSUM */
return(true);
}
#ifndef UNIV_HOTBACKUP
#if !defined(UNIV_HOTBACKUP) && !defined(UNIV_INNOCHECKSUM)
if (check_lsn && recv_lsn_checks_on) {
lsn_t current_lsn;
......@@ -742,7 +813,7 @@ buf_page_is_corrupted(
}
if (zip_size) {
return(!page_zip_verify_checksum(read_buf, zip_size));
return(!page_zip_verify_checksum((const void *)read_buf, zip_size));
}
checksum_field1 = mach_read_from_4(
......@@ -762,9 +833,10 @@ buf_page_is_corrupted(
/* make sure that the page is really empty */
for (ulint i = 0; i < UNIV_PAGE_SIZE; i++) {
if (read_buf[i] != 0) {
#ifndef UNIV_INNOCHECKSUM
ib_logf(IB_LOG_LEVEL_INFO,
"Checksum fields zero but page is not empty.");
#endif
return(true);
}
}
......@@ -774,7 +846,9 @@ buf_page_is_corrupted(
DBUG_EXECUTE_IF("buf_page_is_corrupt_failure", return(true); );
#ifndef UNIV_INNOCHECKSUM
ulint page_no = mach_read_from_4(read_buf + FIL_PAGE_OFFSET);
#endif
const srv_checksum_algorithm_t curr_algo =
static_cast<srv_checksum_algorithm_t>(srv_checksum_algorithm);
......@@ -792,12 +866,32 @@ buf_page_is_corrupted(
checksum_field1, checksum_field2)) {
if (curr_algo
== SRV_CHECKSUM_ALGORITHM_STRICT_CRC32) {
#ifndef UNIV_INNOCHECKSUM
page_warn_strict_checksum(
curr_algo,
SRV_CHECKSUM_ALGORITHM_NONE,
space_id, page_no);
#endif /* !UNIV_INNOCHECKSUM */
}
#ifdef UNIV_INNOCHECKSUM
if (log_file) {
fprintf(log_file, "page::" ULINTPF ";"
" old style: calculated = " ULINTPF ";"
" recorded = " ULINTPF "\n",
cur_page_num,
buf_calc_page_old_checksum(read_buf),
checksum_field2);
fprintf(log_file, "page::" ULINTPF ";"
" new style: calculated = " ULINTPF ";"
" crc32 = %u; recorded = " ULINTPF "\n",
cur_page_num,
buf_calc_page_new_checksum(read_buf),
buf_calc_page_crc32(read_buf),
checksum_field1);
}
#endif /* UNIV_INNOCHECKSUM */
return(false);
}
......@@ -805,15 +899,24 @@ buf_page_is_corrupted(
checksum_field1, checksum_field2)) {
if (curr_algo
== SRV_CHECKSUM_ALGORITHM_STRICT_CRC32) {
#ifndef UNIV_INNOCHECKSUM
page_warn_strict_checksum(
curr_algo,
SRV_CHECKSUM_ALGORITHM_INNODB,
space_id, page_no);
#endif
}
return(false);
}
#ifdef UNIV_INNOCHECKSUM
if (log_file) {
fprintf(log_file, "Fail; page " ULINTPF
" invalid (fails crc32 checksum)\n",
cur_page_num);
}
#endif /* UNIV_INNOCHECKSUM */
return(true);
case SRV_CHECKSUM_ALGORITHM_INNODB:
......@@ -828,11 +931,29 @@ buf_page_is_corrupted(
checksum_field1, checksum_field2)) {
if (curr_algo
== SRV_CHECKSUM_ALGORITHM_STRICT_INNODB) {
#ifndef UNIV_INNOCHECKSUM
page_warn_strict_checksum(
curr_algo,
SRV_CHECKSUM_ALGORITHM_NONE,
space_id, page_no);
#endif
}
#ifdef UNIV_INNOCHECKSUM
if (log_file) {
fprintf(log_file, "page::" ULINTPF ";"
" old style: calculated = " ULINTPF ";"
" recorded = " ULINTPF "\n", cur_page_num,
buf_calc_page_old_checksum(read_buf),
checksum_field2);
fprintf(log_file, "page::" ULINTPF ";"
" new style: calculated = " ULINTPF ";"
" crc32 = %u; recorded = " ULINTPF "\n",
cur_page_num,
buf_calc_page_new_checksum(read_buf),
buf_calc_page_crc32(read_buf),
checksum_field1);
}
#endif /* UNIV_INNOCHECKSUM */
return(false);
}
......@@ -841,15 +962,25 @@ buf_page_is_corrupted(
checksum_field1, checksum_field2)) {
if (curr_algo
== SRV_CHECKSUM_ALGORITHM_STRICT_INNODB) {
#ifndef UNIV_INNOCHECKSUM
page_warn_strict_checksum(
curr_algo,
SRV_CHECKSUM_ALGORITHM_CRC32,
space_id, page_no);
#endif
}
return(false);
}
#ifdef UNIV_INNOCHECKSUM
if (log_file) {
fprintf(log_file, "Fail; page " ULINTPF
" invalid (fails innodb checksum)\n",
cur_page_num);
}
#endif /* UNIV_INNOCHECKSUM */
return(true);
case SRV_CHECKSUM_ALGORITHM_STRICT_NONE:
......@@ -861,22 +992,34 @@ buf_page_is_corrupted(
if (buf_page_is_checksum_valid_crc32(read_buf,
checksum_field1, checksum_field2)) {
#ifndef UNIV_INNOCHECKSUM
page_warn_strict_checksum(
curr_algo,
SRV_CHECKSUM_ALGORITHM_CRC32,
space_id, page_no);
#endif
return(false);
}
if (buf_page_is_checksum_valid_innodb(read_buf,
checksum_field1, checksum_field2)) {
#ifndef UNIV_INNOCHECKSUM
page_warn_strict_checksum(
curr_algo,
SRV_CHECKSUM_ALGORITHM_INNODB,
space_id, page_no);
#endif
return(false);
}
#ifdef UNIV_INNOCHECKSUM
if (log_file) {
fprintf(log_file, "Fail; page " ULINTPF
" invalid (fails none checksum)\n",
cur_page_num);
}
#endif /* UNIV_INNOCHECKSUM */
return(true);
case SRV_CHECKSUM_ALGORITHM_NONE:
......@@ -890,6 +1033,7 @@ buf_page_is_corrupted(
return(false);
}
#ifndef UNIV_INNOCHECKSUM
/********************************************************************//**
Prints a page to stderr. */
UNIV_INTERN
......@@ -6352,3 +6496,4 @@ buf_page_decrypt_after_read(buf_page_t* bpage, fil_space_t* space)
ut_ad(space->n_pending_ios > 0);
return (success);
}
#endif /* !UNIV_INNOCHECKSUM */
storage/innobase/buf/buf0checksum.cc
View file @ 8b019f87
......@@ -128,8 +128,6 @@ buf_calc_page_old_checksum(
return(checksum);
}
#ifndef UNIV_INNOCHECKSUM
/********************************************************************//**
Return a printable string describing the checksum algorithm.
@return algorithm name */
......@@ -158,4 +156,3 @@ buf_checksum_algorithm_name(
return(NULL);
}
#endif /* !UNIV_INNOCHECKSUM */
storage/innobase/fil/fil0crypt.cc
View file @ 8b019f87
......@@ -25,13 +25,18 @@ Modified Jan Lindström jan.lindstrom@mariadb.com
#include "fil0fil.h"
#include "fil0crypt.h"
#include "mach0data.h"
#include "page0zip.h"
#include "buf0buf.h"
#include "buf0checksum.h"
#ifndef UNIV_INNOCHECKSUM
#include "srv0srv.h"
#include "srv0start.h"
#include "mach0data.h"
#include "log0recv.h"
#include "mtr0mtr.h"
#include "mtr0log.h"
#include "page0zip.h"
#include "ut0ut.h"
#include "btr0scrub.h"
#include "fsp0fsp.h"
......@@ -107,13 +112,20 @@ UNIV_INTERN mysql_pfs_key_t fil_crypt_data_mutex_key;
extern my_bool srv_background_scrub_data_uncompressed;
extern my_bool srv_background_scrub_data_compressed;
/***********************************************************************
Check if a key needs rotation given a key_state
@param[in] encrypt_mode Encryption mode
@param[in] key_version Current key version
@param[in] latest_key_version Latest key version
@param[in] rotate_key_age when to rotate
@return true if key needs rotation, false if not */
static bool
fil_crypt_needs_rotation(
fil_encryption_t encrypt_mode, /*!< in: Encryption
mode */
uint key_version, /*!< in: Key version */
uint latest_key_version, /*!< in: Latest key version */
uint rotate_key_age); /*!< in: When to rotate */
fil_encryption_t encrypt_mode,
uint key_version,
uint latest_key_version,
uint rotate_key_age)
MY_ATTRIBUTE((warn_unused_result));
/*********************************************************************
Init space crypt */
......@@ -908,137 +920,6 @@ fil_crypt_calculate_checksum(
return checksum;
}
/*********************************************************************
Verify that post encryption checksum match calculated checksum.
This function should be called only if tablespace contains crypt_data
metadata (this is strong indication that tablespace is encrypted).
Function also verifies that traditional checksum does not match
calculated checksum as if it does page could be valid unencrypted,
encrypted, or corrupted.
@param[in] page Page to verify
@param[in] zip_size zip size
@param[in] space Tablespace
@param[in] pageno Page no
@return true if page is encrypted AND OK, false otherwise */
UNIV_INTERN
bool
fil_space_verify_crypt_checksum(
byte* page,
ulint zip_size,
const fil_space_t* space,
ulint pageno)
{
uint key_version = mach_read_from_4(page+ FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION);
/* If page is not encrypted, return false */
if (key_version == 0) {
return(false);
}
/* Read stored post encryption checksum. */
ib_uint32_t checksum = mach_read_from_4(
page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4);
/* Declare empty pages non-corrupted */
if (checksum == 0
&& *reinterpret_cast<const ib_uint64_t*>(page + FIL_PAGE_LSN) == 0
&& buf_page_is_zeroes(page, zip_size)) {
return(true);
}
/* Compressed and encrypted pages do not have checksum. Assume not
corrupted. Page verification happens after decompression in
buf_page_io_complete() using buf_page_is_corrupted(). */
if (mach_read_from_2(page+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) {
return (true);
}
ib_uint32_t cchecksum1 = 0;
ib_uint32_t cchecksum2 = 0;
/* Calculate checksums */
if (zip_size) {
cchecksum1 = page_zip_calc_checksum(
page, zip_size, SRV_CHECKSUM_ALGORITHM_CRC32);
if(cchecksum1 != checksum) {
cchecksum2 = page_zip_calc_checksum(
page, zip_size,
SRV_CHECKSUM_ALGORITHM_INNODB);
}
} else {
cchecksum1 = buf_calc_page_crc32(page);
if (cchecksum1 != checksum) {
cchecksum2 = (ib_uint32_t) buf_calc_page_new_checksum(
page);
}
}
/* If stored checksum matches one of the calculated checksums
page is not corrupted. */
bool encrypted = (checksum == cchecksum1 || checksum == cchecksum2
|| checksum == BUF_NO_CHECKSUM_MAGIC);
/* MySQL 5.6 and MariaDB 10.0 and 10.1 will write an LSN to the
first page of each system tablespace file at
FIL_PAGE_FILE_FLUSH_LSN offset. On other pages and in other files,
the field might have been uninitialized until MySQL 5.5. In MySQL 5.7
(and MariaDB Server 10.2.2) WL#7990 stopped writing the field for other
than page 0 of the system tablespace.
Starting from MariaDB 10.1 the field has been repurposed for
encryption key_version.
Starting with MySQL 5.7 (and MariaDB Server 10.2), the
field has been repurposed for SPATIAL INDEX pages for
FIL_RTREE_SPLIT_SEQ_NUM.
Note that FIL_PAGE_FILE_FLUSH_LSN is not included in the InnoDB page
checksum.
Thus, FIL_PAGE_FILE_FLUSH_LSN could contain any value. While the
field would usually be 0 for pages that are not encrypted, we cannot
assume that a nonzero value means that the page is encrypted.
Therefore we must validate the page both as encrypted and unencrypted
when FIL_PAGE_FILE_FLUSH_LSN does not contain 0.
*/
ulint checksum1 = mach_read_from_4(
page + FIL_PAGE_SPACE_OR_CHKSUM);
ulint checksum2 = checksum1;
bool valid;
if (zip_size) {
valid = (checksum1 == cchecksum1);
} else {
checksum2 = mach_read_from_4(
page + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM);
valid = (buf_page_is_checksum_valid_crc32(page,checksum1,checksum2)
|| buf_page_is_checksum_valid_innodb(page,checksum1, checksum2));
}
if (encrypted && valid) {
/* If page is encrypted and traditional checksums match,
page could be still encrypted, or not encrypted and valid or
corrupted. */
ib_logf(IB_LOG_LEVEL_ERROR,
" Page %lu in space %s (%lu) maybe corrupted."
" Post encryption checksum %u stored [%lu:%lu] key_version %u",
pageno,
space ? space->name : "N/A",
mach_read_from_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID),
checksum, checksum1, checksum2, key_version);
encrypted = false;
}
return(encrypted);
}
/***********************************************************************/
/** A copy of global key state */
......@@ -2555,8 +2436,9 @@ fil_space_crypt_close_tablespace(
if (now >= last + 30) {
ib_logf(IB_LOG_LEVEL_WARN,
"Waited %ld seconds to drop space: %s(" ULINTPF ").",
now - start, space->name, space->id);
"Waited %ld seconds to drop space: %s (" ULINTPF
") active threads %u flushing=%d.",
now - start, space->name, space->id, cnt, flushing);
last = now;
}
}
......@@ -2659,3 +2541,159 @@ fil_space_get_scrub_status(
mutex_exit(&crypt_data->mutex);
}
}
#endif /* !UNIV_INNOCHECKSUM */
/*********************************************************************
Verify that post encryption checksum match calculated checksum.
This function should be called only if tablespace contains crypt_data
metadata (this is strong indication that tablespace is encrypted).
Function also verifies that traditional checksum does not match
calculated checksum as if it does page could be valid unencrypted,
encrypted, or corrupted.
@param[in] page Page to verify
@param[in] zip_size zip size
@param[in] space Tablespace
@param[in] pageno Page no
@return true if page is encrypted AND OK, false otherwise */
UNIV_INTERN
bool
fil_space_verify_crypt_checksum(
byte* page,
ulint zip_size,
#ifndef UNIV_INNOCHECKSUM
const fil_space_t* space,
#else
const void* space,
#endif
ulint pageno)
{
uint key_version = mach_read_from_4(page+ FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION);
/* If page is not encrypted, return false */
if (key_version == 0) {
return(false);
}
srv_checksum_algorithm_t algorithm =
static_cast<srv_checksum_algorithm_t>(srv_checksum_algorithm);
/* If no checksum is used, can't continue checking. */
if (algorithm == SRV_CHECKSUM_ALGORITHM_NONE) {
return(true);
}
/* Read stored post encryption checksum. */
ib_uint32_t checksum = mach_read_from_4(
page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4);
/* Declare empty pages non-corrupted */
if (checksum == 0
&& *reinterpret_cast<const ib_uint64_t*>(page + FIL_PAGE_LSN) == 0
&& buf_page_is_zeroes(page, zip_size)) {
return(true);
}
/* Compressed and encrypted pages do not have checksum. Assume not
corrupted. Page verification happens after decompression in
buf_page_io_complete() using buf_page_is_corrupted(). */
if (mach_read_from_2(page+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) {
return (true);
}
ib_uint32_t cchecksum1 = 0;
ib_uint32_t cchecksum2 = 0;
/* Calculate checksums */
if (zip_size) {
cchecksum1 = page_zip_calc_checksum(
page, zip_size, SRV_CHECKSUM_ALGORITHM_CRC32);
cchecksum2 = (cchecksum1 == checksum)
? 0
: page_zip_calc_checksum(
page, zip_size,
SRV_CHECKSUM_ALGORITHM_INNODB);
} else {
cchecksum1 = buf_calc_page_crc32(page);
cchecksum2 = (cchecksum1 == checksum)
? 0
: buf_calc_page_new_checksum(page);
}
/* If stored checksum matches one of the calculated checksums
page is not corrupted. */
bool encrypted = (checksum == cchecksum1 || checksum == cchecksum2
|| checksum == BUF_NO_CHECKSUM_MAGIC);
/* MySQL 5.6 and MariaDB 10.0 and 10.1 will write an LSN to the
first page of each system tablespace file at
FIL_PAGE_FILE_FLUSH_LSN offset. On other pages and in other files,
the field might have been uninitialized until MySQL 5.5. In MySQL 5.7
(and MariaDB Server 10.2.2) WL#7990 stopped writing the field for other
than page 0 of the system tablespace.
Starting from MariaDB 10.1 the field has been repurposed for
encryption key_version.
Starting with MySQL 5.7 (and MariaDB Server 10.2), the
field has been repurposed for SPATIAL INDEX pages for
FIL_RTREE_SPLIT_SEQ_NUM.
Note that FIL_PAGE_FILE_FLUSH_LSN is not included in the InnoDB page
checksum.
Thus, FIL_PAGE_FILE_FLUSH_LSN could contain any value. While the
field would usually be 0 for pages that are not encrypted, we cannot
assume that a nonzero value means that the page is encrypted.
Therefore we must validate the page both as encrypted and unencrypted
when FIL_PAGE_FILE_FLUSH_LSN does not contain 0.
*/
uint32_t checksum1 = mach_read_from_4(page + FIL_PAGE_SPACE_OR_CHKSUM);
uint32_t checksum2;
bool valid;
if (zip_size) {
valid = (checksum1 == cchecksum1);
checksum2 = checksum1;
} else {
checksum2 = mach_read_from_4(
page + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM);
valid = (buf_page_is_checksum_valid_crc32(page,checksum1,checksum2)
|| buf_page_is_checksum_valid_innodb(page,checksum1, checksum2));
}
if (encrypted && valid) {
/* If page is encrypted and traditional checksums match,
page could be still encrypted, or not encrypted and valid or
corrupted. */
#ifndef UNIV_INNOCHECKSUM
ib_logf(IB_LOG_LEVEL_ERROR,
" Page " ULINTPF " in space %s (" ULINTPF ") maybe corrupted."
" Post encryption checksum %u stored [%u:%u] key_version %u",
pageno,
space ? space->name : "N/A",
mach_read_from_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID),
checksum, checksum1, checksum2, key_version);
#else
if (log_file) {
fprintf(log_file,
"Page " ULINTPF ":" ULINTPF " may be corrupted."
" Post encryption checksum %u"
" stored [%u:%u] key_version %u\n",
pageno,
mach_read_from_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID),
checksum, checksum1, checksum2,
key_version);
}
#endif /* UNIV_INNOCHECKSUM */
encrypted = false;
}
return(encrypted);
}
storage/innobase/include/buf0buf.h
View file @ 8b019f87
......@@ -31,6 +31,7 @@ Created 11/5/1995 Heikki Tuuri
#include "fil0fil.h"
#include "mtr0types.h"
#include "buf0types.h"
#ifndef UNIV_INNOCHECKSUM
#include "hash0hash.h"
#include "ut0byte.h"
#include "page0types.h"
......@@ -643,6 +644,8 @@ buf_block_unfix(
# define buf_block_modify_clock_inc(block) ((void) 0)
#endif /* !UNIV_HOTBACKUP */
#endif /* !UNIV_INNOCHECKSUM */
/** Checks if the page is in crc32 checksum format.
@param[in] read_buf database page
@param[in] checksum_field1 new checksum field
......@@ -691,8 +694,13 @@ buf_page_is_corrupted(
bool check_lsn,
const byte* read_buf,
ulint zip_size,
#ifndef UNIV_INNOCHECKSUM
const fil_space_t* space)
#else
const void* space = NULL)
#endif
MY_ATTRIBUTE((warn_unused_result));
/** Check if a page is all zeroes.
@param[in] read_buf database page
@param[in] zip_size ROW_FORMAT=COMPRESSED page size, or 0
......@@ -700,6 +708,9 @@ buf_page_is_corrupted(
UNIV_INTERN
bool
buf_page_is_zeroes(const byte* read_buf, ulint zip_size);
#ifndef UNIV_INNOCHECKSUM
#ifndef UNIV_HOTBACKUP
/**********************************************************************//**
Gets the space id, page offset, and byte offset within page of a
......@@ -2470,4 +2481,5 @@ struct CheckUnzipLRUAndLRUList {
#include "buf0buf.ic"
#endif
#endif /*! UNIV_INNOCHECKSUM */
#endif
storage/innobase/include/fil0crypt.h
View file @ 8b019f87
......@@ -71,9 +71,11 @@ struct key_struct
/** is encryption enabled */
extern ulong srv_encrypt_tables;
#ifndef UNIV_INNOCHECKSUM
#ifdef UNIV_PFS_MUTEX
extern mysql_pfs_key_t fil_crypt_data_mutex_key;
#endif
#endif /* !UNIV_INNOCHECKSUM */
/** Mutex helper for crypt_data->scheme
@param[in, out] schme encryption scheme
......@@ -102,6 +104,8 @@ struct fil_space_rotate_state_t
} scrubbing;
};
#ifndef UNIV_INNOCHECKSUM
struct fil_space_crypt_t : st_encryption_scheme
{
public:
......@@ -399,6 +403,8 @@ fil_crypt_calculate_checksum(
const byte* dst_frame)
MY_ATTRIBUTE((warn_unused_result));
#endif /* UNIV_INNOCHECKSUM */
/*********************************************************************
Verify that post encryption checksum match calculated checksum.
This function should be called only if tablespace contains crypt_data
......@@ -417,10 +423,16 @@ bool
fil_space_verify_crypt_checksum(
byte* page,
ulint zip_size,
#ifndef UNIV_INNOCHECKSUM
const fil_space_t* space,
#else
const void* space,
#endif
ulint pageno)
MY_ATTRIBUTE((warn_unused_result));
#ifndef UNIV_INNOCHECKSUM
/*********************************************************************
Adjust thread count for key rotation
@param[in] enw_cnt Number of threads to be used */
......@@ -508,4 +520,5 @@ fil_space_get_scrub_status(
#include "fil0crypt.ic"
#endif
#endif /* !UNIV_INNOCHECKSUM */
#endif /* fil0crypt_h */
storage/innobase/include/fil0fil.h
View file @ 8b019f87
......@@ -183,8 +183,6 @@ extern fil_addr_t fil_addr_null;
#define FIL_LOG 502 /*!< redo log */
/* @} */
#ifndef UNIV_INNOCHECKSUM
/** Structure containing encryption specification */
struct fil_space_crypt_t;
......@@ -209,6 +207,10 @@ extern ulint fil_n_pending_tablespace_flushes;
/** Number of files currently open */
extern ulint fil_n_file_opened;
#ifndef UNIV_INNOCHECKSUM
struct fil_space_t;
struct fsp_open_info {
ibool success; /*!< Has the tablespace been opened? */
const char* check_msg; /*!< fil_check_first_page() message */
......@@ -225,8 +227,6 @@ struct fsp_open_info {
dict_table_t* table; /*!< table */
};
struct fil_space_t;
/** File node of a tablespace or the log data space */
struct fil_node_t {
fil_space_t* space; /*!< backpointer to the space where this node
......
storage/innobase/include/mach0data.ic
View file @ 8b019f87
......@@ -154,6 +154,8 @@ mach_read_from_3(
);
}
#endif /* !UNIV_INNOCHECKSUM */
/*******************************************************//**
The following function is used to store data in four consecutive
bytes. We store the most significant byte to the lowest address. */
......@@ -172,8 +174,6 @@ mach_write_to_4(
b[3] = (byte) n;
}
#endif /* !UNIV_INNOCHECKSUM */
/********************************************************//**
The following function is used to fetch data from 4 consecutive
bytes. The most significant byte is at the lowest address.
......
storage/innobase/include/univ.i
View file @ 8b019f87
......@@ -482,6 +482,12 @@ typedef long int lint;
typedef unsigned long long int ullint;
#endif /* UNIV_HOTBACKUP */
#ifdef UNIV_INNOCHECKSUM
extern bool strict_verify;
extern FILE* log_file;
extern ulint cur_page_num;
#endif /* UNIV_INNOCHECKSUM */
#ifndef __WIN__
#if SIZEOF_LONG != SIZEOF_VOIDP
#error "Error: InnoDB's ulint must be of the same size as void*"
......
storage/innobase/page/page0zip.cc
View file @ 8b019f87
......@@ -4937,26 +4937,26 @@ page_zip_verify_checksum(
#error "FIL_PAGE_LSN must be 64 bit aligned"
#endif
#ifndef UNIV_INNOCHECKSUM
/* innochecksum doesn't compile with ut_d. Since we don't
need to check for empty pages when running innochecksum,
just don't include this code. */
/* Check if page is empty */
if (stored == 0
&& *reinterpret_cast<const ib_uint64_t*>(static_cast<const char*>(
data)
+ FIL_PAGE_LSN) == 0) {
/* make sure that the page is really empty */
ulint i;
for (i = 0; i < size; i++) {
for (ulint i = 0; i < size; i++) {
if (*((const char*) data + i) != 0) {
return(FALSE);
}
}
#ifdef UNIV_INNOCHECKSUM
if (log_file) {
fprintf(log_file, "Page::%lu is empty and"
" uncorrupted\n", cur_page_num);
}
#endif /* UNIV_INNOCHECKSUM */
/* Empty page */
return(TRUE);
}
#endif
const srv_checksum_algorithm_t curr_algo =
static_cast<srv_checksum_algorithm_t>(srv_checksum_algorithm);
......@@ -4968,6 +4968,33 @@ page_zip_verify_checksum(
calc = static_cast<ib_uint32_t>(page_zip_calc_checksum(
data, size, curr_algo));
#ifdef UNIV_INNOCHECKSUM
if (log_file) {
fprintf(log_file, "page::%lu;"
" %s checksum: calculated = %u;"
" recorded = %u\n", cur_page_num,
buf_checksum_algorithm_name(
static_cast<srv_checksum_algorithm_t>(
srv_checksum_algorithm)),
calc, stored);
}
if (!strict_verify) {
const uint32_t crc32 = page_zip_calc_checksum(
data, size, SRV_CHECKSUM_ALGORITHM_CRC32);
if (log_file) {
fprintf(log_file, "page::%lu: crc32 checksum:"
" calculated = %u; recorded = %u\n",
cur_page_num, crc32, stored);
fprintf(log_file, "page::%lu: none checksum:"
" calculated = %lu; recorded = %u\n",
cur_page_num, BUF_NO_CHECKSUM_MAGIC, stored);
}
}
#endif /* UNIV_INNOCHECKSUM */
if (stored == calc) {
return(TRUE);
}
......
storage/xtradb/fil/fil0crypt.cc
View file @ 8b019f87
......@@ -25,13 +25,18 @@ Modified Jan Lindström jan.lindstrom@mariadb.com
#include "fil0fil.h"
#include "fil0crypt.h"
#include "mach0data.h"
#include "page0zip.h"
#include "buf0buf.h"
#include "buf0checksum.h"
#ifndef UNIV_INNOCHECKSUM
#include "srv0srv.h"
#include "srv0start.h"
#include "mach0data.h"
#include "log0recv.h"
#include "mtr0mtr.h"
#include "mtr0log.h"
#include "page0zip.h"
#include "ut0ut.h"
#include "btr0scrub.h"
#include "fsp0fsp.h"
......@@ -107,13 +112,20 @@ UNIV_INTERN mysql_pfs_key_t fil_crypt_data_mutex_key;
extern my_bool srv_background_scrub_data_uncompressed;
extern my_bool srv_background_scrub_data_compressed;
/***********************************************************************
Check if a key needs rotation given a key_state
@param[in] encrypt_mode Encryption mode
@param[in] key_version Current key version
@param[in] latest_key_version Latest key version
@param[in] rotate_key_age when to rotate
@return true if key needs rotation, false if not */
static bool
fil_crypt_needs_rotation(
fil_encryption_t encrypt_mode, /*!< in: Encryption
mode */
uint key_version, /*!< in: Key version */
uint latest_key_version, /*!< in: Latest key version */
uint rotate_key_age); /*!< in: When to rotate */
fil_encryption_t encrypt_mode,
uint key_version,
uint latest_key_version,
uint rotate_key_age)
MY_ATTRIBUTE((warn_unused_result));
/*********************************************************************
Init space crypt */
......@@ -908,137 +920,6 @@ fil_crypt_calculate_checksum(
return checksum;
}
/*********************************************************************
Verify that post encryption checksum match calculated checksum.
This function should be called only if tablespace contains crypt_data
metadata (this is strong indication that tablespace is encrypted).
Function also verifies that traditional checksum does not match
calculated checksum as if it does page could be valid unencrypted,
encrypted, or corrupted.
@param[in] page Page to verify
@param[in] zip_size zip size
@param[in] space Tablespace
@param[in] pageno Page no
@return true if page is encrypted AND OK, false otherwise */
UNIV_INTERN
bool
fil_space_verify_crypt_checksum(
byte* page,
ulint zip_size,
const fil_space_t* space,
ulint pageno)
{
uint key_version = mach_read_from_4(page+ FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION);
/* If page is not encrypted, return false */
if (key_version == 0) {
return(false);
}
/* Read stored post encryption checksum. */
ib_uint32_t checksum = mach_read_from_4(
page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4);
/* Declare empty pages non-corrupted */
if (checksum == 0
&& *reinterpret_cast<const ib_uint64_t*>(page + FIL_PAGE_LSN) == 0
&& buf_page_is_zeroes(page, zip_size)) {
return(true);
}
/* Compressed and encrypted pages do not have checksum. Assume not
corrupted. Page verification happens after decompression in
buf_page_io_complete() using buf_page_is_corrupted(). */
if (mach_read_from_2(page+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) {
return (true);
}
ib_uint32_t cchecksum1 = 0;
ib_uint32_t cchecksum2 = 0;
/* Calculate checksums */
if (zip_size) {
cchecksum1 = page_zip_calc_checksum(
page, zip_size, SRV_CHECKSUM_ALGORITHM_CRC32);
if(cchecksum1 != checksum) {
cchecksum2 = page_zip_calc_checksum(
page, zip_size,
SRV_CHECKSUM_ALGORITHM_INNODB);
}
} else {
cchecksum1 = buf_calc_page_crc32(page);
if (cchecksum1 != checksum) {
cchecksum2 = (ib_uint32_t) buf_calc_page_new_checksum(
page);
}
}
/* If stored checksum matches one of the calculated checksums
page is not corrupted. */
bool encrypted = (checksum == cchecksum1 || checksum == cchecksum2
|| checksum == BUF_NO_CHECKSUM_MAGIC);
/* MySQL 5.6 and MariaDB 10.0 and 10.1 will write an LSN to the
first page of each system tablespace file at
FIL_PAGE_FILE_FLUSH_LSN offset. On other pages and in other files,
the field might have been uninitialized until MySQL 5.5. In MySQL 5.7
(and MariaDB Server 10.2.2) WL#7990 stopped writing the field for other
than page 0 of the system tablespace.
Starting from MariaDB 10.1 the field has been repurposed for
encryption key_version.
Starting with MySQL 5.7 (and MariaDB Server 10.2), the
field has been repurposed for SPATIAL INDEX pages for
FIL_RTREE_SPLIT_SEQ_NUM.
Note that FIL_PAGE_FILE_FLUSH_LSN is not included in the InnoDB page
checksum.
Thus, FIL_PAGE_FILE_FLUSH_LSN could contain any value. While the
field would usually be 0 for pages that are not encrypted, we cannot
assume that a nonzero value means that the page is encrypted.
Therefore we must validate the page both as encrypted and unencrypted
when FIL_PAGE_FILE_FLUSH_LSN does not contain 0.
*/
ulint checksum1 = mach_read_from_4(
page + FIL_PAGE_SPACE_OR_CHKSUM);
ulint checksum2 = checksum1;
bool valid;
if (zip_size) {
valid = (checksum1 == cchecksum1);
} else {
checksum1 = mach_read_from_4(
page + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM);
valid = (buf_page_is_checksum_valid_crc32(page,checksum1,checksum2)
|| buf_page_is_checksum_valid_innodb(page,checksum1, checksum2));
}
if (encrypted && valid) {
/* If page is encrypted and traditional checksums match,
page could be still encrypted, or not encrypted and valid or
corrupted. */
ib_logf(IB_LOG_LEVEL_ERROR,
" Page %lu in space %s (%lu) maybe corrupted."
" Post encryption checksum %u stored [%lu:%lu] key_version %u",
pageno,
space ? space->name : "N/A",
mach_read_from_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID),
checksum, checksum1, checksum2, key_version);
encrypted = false;
}
return(encrypted);
}
/***********************************************************************/
/** A copy of global key state */
......@@ -2660,3 +2541,159 @@ fil_space_get_scrub_status(
mutex_exit(&crypt_data->mutex);
}
}
#endif /* !UNIV_INNOCHECKSUM */
/*********************************************************************
Verify that post encryption checksum match calculated checksum.
This function should be called only if tablespace contains crypt_data
metadata (this is strong indication that tablespace is encrypted).
Function also verifies that traditional checksum does not match
calculated checksum as if it does page could be valid unencrypted,
encrypted, or corrupted.
@param[in] page Page to verify
@param[in] zip_size zip size
@param[in] space Tablespace
@param[in] pageno Page no
@return true if page is encrypted AND OK, false otherwise */
UNIV_INTERN
bool
fil_space_verify_crypt_checksum(
byte* page,
ulint zip_size,
#ifndef UNIV_INNOCHECKSUM
const fil_space_t* space,
#else
const void* space,
#endif
ulint pageno)
{
uint key_version = mach_read_from_4(page+ FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION);
/* If page is not encrypted, return false */
if (key_version == 0) {
return(false);
}
srv_checksum_algorithm_t algorithm =
static_cast<srv_checksum_algorithm_t>(srv_checksum_algorithm);
/* If no checksum is used, can't continue checking. */
if (algorithm == SRV_CHECKSUM_ALGORITHM_NONE) {
return(true);
}
/* Read stored post encryption checksum. */
ib_uint32_t checksum = mach_read_from_4(
page + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4);
/* Declare empty pages non-corrupted */
if (checksum == 0
&& *reinterpret_cast<const ib_uint64_t*>(page + FIL_PAGE_LSN) == 0
&& buf_page_is_zeroes(page, zip_size)) {
return(true);
}
/* Compressed and encrypted pages do not have checksum. Assume not
corrupted. Page verification happens after decompression in
buf_page_io_complete() using buf_page_is_corrupted(). */
if (mach_read_from_2(page+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) {
return (true);
}
ib_uint32_t cchecksum1 = 0;
ib_uint32_t cchecksum2 = 0;
/* Calculate checksums */
if (zip_size) {
cchecksum1 = page_zip_calc_checksum(
page, zip_size, SRV_CHECKSUM_ALGORITHM_CRC32);
cchecksum2 = (cchecksum1 == checksum)
? 0
: page_zip_calc_checksum(
page, zip_size,
SRV_CHECKSUM_ALGORITHM_INNODB);
} else {
cchecksum1 = buf_calc_page_crc32(page);
cchecksum2 = (cchecksum1 == checksum)
? 0
: buf_calc_page_new_checksum(page);
}
/* If stored checksum matches one of the calculated checksums
page is not corrupted. */
bool encrypted = (checksum == cchecksum1 || checksum == cchecksum2
|| checksum == BUF_NO_CHECKSUM_MAGIC);
/* MySQL 5.6 and MariaDB 10.0 and 10.1 will write an LSN to the
first page of each system tablespace file at
FIL_PAGE_FILE_FLUSH_LSN offset. On other pages and in other files,
the field might have been uninitialized until MySQL 5.5. In MySQL 5.7
(and MariaDB Server 10.2.2) WL#7990 stopped writing the field for other
than page 0 of the system tablespace.
Starting from MariaDB 10.1 the field has been repurposed for
encryption key_version.
Starting with MySQL 5.7 (and MariaDB Server 10.2), the
field has been repurposed for SPATIAL INDEX pages for
FIL_RTREE_SPLIT_SEQ_NUM.
Note that FIL_PAGE_FILE_FLUSH_LSN is not included in the InnoDB page
checksum.
Thus, FIL_PAGE_FILE_FLUSH_LSN could contain any value. While the
field would usually be 0 for pages that are not encrypted, we cannot
assume that a nonzero value means that the page is encrypted.
Therefore we must validate the page both as encrypted and unencrypted
when FIL_PAGE_FILE_FLUSH_LSN does not contain 0.
*/
uint32_t checksum1 = mach_read_from_4(page + FIL_PAGE_SPACE_OR_CHKSUM);
uint32_t checksum2;
bool valid;
if (zip_size) {
valid = (checksum1 == cchecksum1);
checksum2 = checksum1;
} else {
checksum2 = mach_read_from_4(
page + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM);
valid = (buf_page_is_checksum_valid_crc32(page,checksum1,checksum2)
|| buf_page_is_checksum_valid_innodb(page,checksum1, checksum2));
}
if (encrypted && valid) {
/* If page is encrypted and traditional checksums match,
page could be still encrypted, or not encrypted and valid or
corrupted. */
#ifndef UNIV_INNOCHECKSUM
ib_logf(IB_LOG_LEVEL_ERROR,
" Page " ULINTPF " in space %s (" ULINTPF ") maybe corrupted."
" Post encryption checksum %u stored [%u:%u] key_version %u",
pageno,
space ? space->name : "N/A",
mach_read_from_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID),
checksum, checksum1, checksum2, key_version);
#else
if (log_file) {
fprintf(log_file,
"Page " ULINTPF ":" ULINTPF " may be corrupted."
" Post encryption checksum %u"
" stored [%u:%u] key_version %u\n",
pageno,
mach_read_from_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID),
checksum, checksum1, checksum2,
key_version);
}
#endif /* UNIV_INNOCHECKSUM */
encrypted = false;
}
return(encrypted);
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7