Commit f5eb3712 authored by Marko Mäkelä's avatar Marko Mäkelä

MDEV-13103 Deal with page_compressed page corruption

fil_page_decompress(): Replaces fil_decompress_page().
Allow the caller detect errors. Remove
duplicated code. Use the "safe" instead of "fast" variants of
decompression routines.

fil_page_compress(): Replaces fil_compress_page().
The length of the input buffer always was srv_page_size (innodb_page_size).
Remove printouts, and remove the fil_space_t* parameter.

buf_tmp_buffer_t::reserved: Make private; the accessors acquire()
and release() will use atomic memory access.

buf_pool_reserve_tmp_slot(): Make static. Remove the second parameter.
Do not acquire any mutex. Remove the allocation of the buffers.

buf_tmp_reserve_crypt_buf(), buf_tmp_reserve_compression_buf():
Refactored away from buf_pool_reserve_tmp_slot().

buf_page_decrypt_after_read(): Make static, and simplify the logic.
Use the encryption buffer also for decompressing.

buf_page_io_complete(), buf_dblwr_process(): Check more failures.

fil_space_encrypt(): Simplify the debug checks.

fil_space_t::printed_compression_failure: Remove.

fil_get_compression_alg_name(): Remove.

fil_iterate(): Allocate a buffer for compression and decompression
only once, instead of allocating and freeing it for every page
that uses compression, during IMPORT TABLESPACE.

fil_node_get_space_id(), fil_page_is_index_page(),
fil_page_is_lzo_compressed(): Remove (unused code).
parent 72005b7a
call mtr.add_suppression("InnoDB: Compression failed for space [0-9]+ name test/innodb_page_compressed[0-9] len [0-9]+ err 2 write_size [0-9]+.");
set global innodb_file_format = `Barracuda`;
set global innodb_file_per_table = on;
create table innodb_normal (c1 int not null auto_increment primary key, b char(200)) engine=innodb;
......
call mtr.add_suppression("InnoDB: Compression failed for space [0-9]+ name test/innodb_page_compressed[0-9] len [0-9]+ err 2 write_size [0-9]+.");
set global innodb_compression_algorithm = snappy;
set global innodb_file_format = `Barracuda`;
set global innodb_file_per_table = on;
......
--source include/have_innodb.inc
--source include/not_embedded.inc
call mtr.add_suppression("InnoDB: Compression failed for space [0-9]+ name test/innodb_page_compressed[0-9] len [0-9]+ err 2 write_size [0-9]+.");
# All page compression test use the same
--source include/innodb-page-compression.inc
......
......@@ -2,8 +2,6 @@
-- source include/have_innodb_snappy.inc
--source include/not_embedded.inc
call mtr.add_suppression("InnoDB: Compression failed for space [0-9]+ name test/innodb_page_compressed[0-9] len [0-9]+ err 2 write_size [0-9]+.");
# snappy
set global innodb_compression_algorithm = snappy;
......
This diff is collapsed.
......@@ -510,10 +510,11 @@ buf_dblwr_process()
"Restoring possible half-written data pages "
"from the doublewrite buffer...");
unaligned_read_buf = static_cast<byte*>(ut_malloc(2 * UNIV_PAGE_SIZE));
unaligned_read_buf = static_cast<byte*>(ut_malloc(3 * UNIV_PAGE_SIZE));
read_buf = static_cast<byte*>(
ut_align(unaligned_read_buf, UNIV_PAGE_SIZE));
byte* const buf = read_buf + UNIV_PAGE_SIZE;
for (std::list<byte*>::iterator i = recv_dblwr.pages.begin();
i != recv_dblwr.pages.end(); ++i, ++page_no_dblwr ) {
......@@ -562,24 +563,23 @@ buf_dblwr_process()
ignore this page (there should be redo log
records to initialize it). */
} else {
if (fil_page_is_compressed_encrypted(read_buf) ||
fil_page_is_compressed(read_buf)) {
/* Decompress the page before
validating the checksum. */
fil_decompress_page(
NULL, read_buf, srv_page_size,
NULL, true);
/* Decompress the page before
validating the checksum. */
ulint decomp = fil_page_decompress(buf, read_buf);
if (!decomp || (decomp != srv_page_size && zip_size)) {
goto bad;
}
if (fil_space_verify_crypt_checksum(
read_buf, zip_size, NULL, page_no)
|| !buf_page_is_corrupted(
true, read_buf, zip_size, space())) {
read_buf, zip_size, NULL, page_no)
|| !buf_page_is_corrupted(
true, read_buf, zip_size, space())) {
/* The page is good; there is no need
to consult the doublewrite buffer. */
continue;
}
bad:
/* We intentionally skip this message for
is_all_zero pages. */
ib_logf(IB_LOG_LEVEL_INFO,
......@@ -588,18 +588,15 @@ buf_dblwr_process()
space_id, page_no);
}
/* Next, validate the doublewrite page. */
if (fil_page_is_compressed_encrypted(page) ||
fil_page_is_compressed(page)) {
/* Decompress the page before
validating the checksum. */
fil_decompress_page(
NULL, page, srv_page_size, NULL, true);
ulint decomp = fil_page_decompress(buf, page);
if (!decomp || (decomp != srv_page_size && zip_size)) {
goto bad_doublewrite;
}
if (!fil_space_verify_crypt_checksum(page, zip_size, NULL, page_no)
if (!fil_space_verify_crypt_checksum(page, zip_size, NULL,
page_no)
&& buf_page_is_corrupted(true, page, zip_size, space)) {
if (!is_all_zero) {
bad_doublewrite:
ib_logf(IB_LOG_LEVEL_WARN,
"A doublewrite copy of page "
ULINTPF ":" ULINTPF " is corrupted.",
......
......@@ -708,60 +708,39 @@ fil_space_encrypt(
#ifdef UNIV_DEBUG
if (tmp) {
/* Verify that encrypted buffer is not corrupted */
byte* tmp_mem = (byte *)malloc(UNIV_PAGE_SIZE);
dberr_t err = DB_SUCCESS;
byte* src = src_frame;
bool page_compressed_encrypted = (mach_read_from_2(tmp+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED);
byte* comp_mem = NULL;
byte* uncomp_mem = NULL;
byte uncomp_mem[UNIV_PAGE_SIZE_MAX];
byte tmp_mem[UNIV_PAGE_SIZE_MAX];
ulint size = (zip_size) ? zip_size : UNIV_PAGE_SIZE;
if (page_compressed_encrypted) {
comp_mem = (byte *)malloc(UNIV_PAGE_SIZE);
uncomp_mem = (byte *)malloc(UNIV_PAGE_SIZE);
memcpy(comp_mem, src_frame, UNIV_PAGE_SIZE);
fil_decompress_page(uncomp_mem, comp_mem,
srv_page_size, NULL);
src = uncomp_mem;
memcpy(uncomp_mem, src, srv_page_size);
ulint unzipped1 = fil_page_decompress(
tmp_mem, uncomp_mem);
ut_ad(unzipped1);
if (unzipped1 != srv_page_size) {
src = uncomp_mem;
}
}
bool corrupted1 = buf_page_is_corrupted(true, src, zip_size, space);
bool ok = fil_space_decrypt(crypt_data, tmp_mem, size, tmp, &err);
ut_ad(!buf_page_is_corrupted(true, src, zip_size, space));
ut_ad(fil_space_decrypt(crypt_data, tmp_mem, size, tmp, &err));
ut_ad(err == DB_SUCCESS);
/* Need to decompress the page if it was also compressed */
if (page_compressed_encrypted) {
memcpy(comp_mem, tmp_mem, UNIV_PAGE_SIZE);
fil_decompress_page(tmp_mem, comp_mem,
srv_page_size, NULL);
byte buf[UNIV_PAGE_SIZE_MAX];
memcpy(buf, tmp_mem, srv_page_size);
ulint unzipped2 = fil_page_decompress(tmp_mem, buf);
ut_ad(unzipped2);
}
bool corrupted = buf_page_is_corrupted(true, tmp_mem, zip_size, space);
memcpy(tmp_mem+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, src+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, 8);
bool different = memcmp(src, tmp_mem, size);
if (!ok || corrupted || corrupted1 || err != DB_SUCCESS || different) {
fprintf(stderr, "ok %d corrupted %d corrupted1 %d err %d different %d\n",
ok , corrupted, corrupted1, err, different);
fprintf(stderr, "src_frame\n");
buf_page_print(src_frame, zip_size);
fprintf(stderr, "encrypted_frame\n");
buf_page_print(tmp, zip_size);
fprintf(stderr, "decrypted_frame\n");
buf_page_print(tmp_mem, zip_size);
ut_ad(0);
}
free(tmp_mem);
if (comp_mem) {
free(comp_mem);
}
if (uncomp_mem) {
free(uncomp_mem);
}
memcpy(tmp_mem + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION,
src + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, 8);
ut_ad(!memcmp(src, tmp_mem, size));
}
#endif /* UNIV_DEBUG */
return tmp;
......
......@@ -342,19 +342,6 @@ fil_space_get_by_id(
return(space);
}
/****************************************************************//**
Get space id from fil node */
ulint
fil_node_get_space_id(
/*==================*/
fil_node_t* node) /*!< in: Compressed node*/
{
ut_ad(node);
ut_ad(node->space);
return (node->space->id);
}
/*******************************************************************//**
Returns the table space by a given name, NULL if not found. */
UNIV_INLINE
......
This diff is collapsed.
......@@ -5177,6 +5177,11 @@ ibuf_check_bitmap_on_import(
bitmap_page = ibuf_bitmap_get_map_page(
space_id, page_no, zip_size, &mtr);
if (!bitmap_page) {
mutex_exit(&ibuf_mutex);
return DB_CORRUPTION;
}
for (i = FSP_IBUF_BITMAP_OFFSET + 1; i < page_size; i++) {
const ulint offset = page_no + i;
......
/*****************************************************************************
Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved.
Copyright (c) 2013, 2017, MariaDB Corporation.
Copyright (c) 2013, 2018, MariaDB Corporation.
This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software
......@@ -39,6 +39,7 @@ Created 11/5/1995 Heikki Tuuri
#include "ut0rbt.h"
#include "os0proc.h"
#include "log0log.h"
#include "my_atomic.h"
/** @name Modes for buf_page_get_gen */
/* @{ */
......@@ -1506,45 +1507,16 @@ buf_page_encrypt_before_write(
buf_page_t* bpage,
byte* src_frame);
/**********************************************************************
The hook that is called after page is written to disk.
The function releases any resources needed for encryption that was allocated
in buf_page_encrypt_before_write */
UNIV_INTERN
ibool
buf_page_encrypt_after_write(
/*=========================*/
buf_page_t* page); /*!< in/out: buffer page that was flushed */
/********************************************************************//**
The hook that is called just before a page is read from disk.
The function allocates memory that is used to temporarily store disk content
before getting decrypted */
UNIV_INTERN
byte*
buf_page_decrypt_before_read(
/*=========================*/
buf_page_t* page, /*!< in/out: buffer page read from disk */
ulint zip_size); /*!< in: compressed page size, or 0 */
/********************************************************************//**
The hook that is called just after a page is read from disk.
The function decrypt disk content into buf_page_t and releases the
temporary buffer that was allocated in buf_page_decrypt_before_read */
UNIV_INTERN
bool
buf_page_decrypt_after_read(
/*========================*/
buf_page_t* page); /*!< in/out: buffer page read from disk */
/** @brief The temporary memory structure.
NOTE! The definition appears here only for other modules of this
directory (buf) to see it. Do not use from outside! */
typedef struct {
bool reserved; /*!< true if this slot is reserved
private:
int32 reserved; /*!< true if this slot is reserved
*/
public:
byte* crypt_buf; /*!< for encryption the data needs to be
copied to a separate buffer before it's
encrypted&written. this as a page can be
......@@ -1555,6 +1527,21 @@ typedef struct {
byte* out_buf; /*!< resulting buffer after
encryption/compression. This is a
pointer and not allocated. */
/** Release the slot */
void release()
{
my_atomic_store32_explicit(&reserved, false,
MY_MEMORY_ORDER_RELAXED);
}
/** Acquire the slot
@return whether the slot was acquired */
bool acquire()
{
return !my_atomic_fas32_explicit(&reserved, true,
MY_MEMORY_ORDER_RELAXED);
}
} buf_tmp_buffer_t;
/** The common buffer control block structure
......
......@@ -341,9 +341,6 @@ struct fil_space_t {
bool is_in_unflushed_spaces;
/*!< true if this space is currently in
unflushed_spaces */
bool printed_compression_failure;
/*!< true if we have already printed
compression failure */
fil_space_crypt_t* crypt_data;
/*!< tablespace crypt data or NULL */
ulint file_block_size;
......
......@@ -68,7 +68,6 @@ fil_get_page_type_name(
}
return "PAGE TYPE CORRUPTED";
}
/****************************************************************//**
......
/*****************************************************************************
Copyright (C) 2013, 2017 MariaDB Corporation. All Rights Reserved.
Copyright (C) 2013, 2018 MariaDB Corporation.
This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software
......@@ -30,70 +30,26 @@ atomic writes information to table space.
Created 11/12/2013 Jan Lindström jan.lindstrom@skysql.com
***********************************************************************/
/*******************************************************************//**
Find out wheather the page is index page or not
@return true if page type index page, false if not */
UNIV_INLINE
ibool
fil_page_is_index_page(
/*===================*/
byte *buf); /*!< in: page */
/****************************************************************//**
Get the name of the compression algorithm used for page
compression.
@return compression algorithm name or "UNKNOWN" if not known*/
UNIV_INLINE
const char*
fil_get_compression_alg_name(
/*=========================*/
ulint comp_alg); /*!<in: compression algorithm number */
/****************************************************************//**
For page compressed pages compress the page before actual write
operation.
@return compressed page to be written*/
UNIV_INTERN
byte*
fil_compress_page(
/*==============*/
fil_space_t* space, /*!< in,out: tablespace (NULL during IMPORT) */
byte* buf, /*!< in: buffer from which to write; in aio
this must be appropriately aligned */
byte* out_buf, /*!< out: compressed buffer */
ulint len, /*!< in: length of input buffer.*/
ulint level, /* in: compression level */
ulint block_size, /*!< in: block size */
bool encrypted, /*!< in: is page also encrypted */
ulint* out_len); /*!< out: actual length of compressed
page */
/****************************************************************//**
For page compressed pages decompress the page after actual read
operation. */
UNIV_INTERN
void
fil_decompress_page(
/*================*/
byte* page_buf, /*!< in: preallocated buffer or NULL */
byte* buf, /*!< out: buffer from which to read; in aio
this must be appropriately aligned */
ulong len, /*!< in: length of output buffer.*/
ulint* write_size, /*!< in/out: Actual payload size of
the compressed data. */
bool return_error=false);
/*!< in: true if only an error should
be produced when decompression fails.
By default this parameter is false. */
/****************************************************************//**
Get space id from fil node
@return space id*/
UNIV_INTERN
ulint
fil_node_get_space_id(
/*==================*/
fil_node_t* node); /*!< in: Node where to get space id*/
/** Compress a page_compressed page before writing to a data file.
@param[in] buf page to be compressed
@param[out] out_buf compressed page
@param[in] level compression level
@param[in] block_size file system block size
@param[in] encrypted whether the page will be subsequently encrypted
@return actual length of compressed page
@retval 0 if the page was not compressed */
UNIV_INTERN ulint fil_page_compress(const byte* buf, byte* out_buf, ulint level,
ulint block_size, bool encrypted)
MY_ATTRIBUTE((nonnull, warn_unused_result));
/** Decompress a page that may be subject to page_compressed compression.
@param[in,out] tmp_buf temporary buffer (of innodb_page_size)
@param[in,out] buf compressed page buffer
@return size of the compressed data
@retval 0 if decompression failed
@retval srv_page_size if the page was not compressed */
UNIV_INTERN ulint fil_page_decompress(byte* tmp_buf, byte* buf)
MY_ATTRIBUTE((nonnull, warn_unused_result));
/****************************************************************//**
Get block size from fil node
......@@ -120,13 +76,4 @@ ibool
fil_page_is_compressed_encrypted(
/*=============================*/
byte* buf); /*!< in: page */
/*******************************************************************//**
Find out wheather the page is page compressed with lzo method
@return true if page is page compressed with lzo method*/
UNIV_INLINE
ibool
fil_page_is_lzo_compressed(
/*=======================*/
byte* buf); /*!< in: page */
#endif
/*****************************************************************************
Copyright (C) 2013, 2017, MariaDB Corporation. All Rights Reserved.
Copyright (C) 2013, 2018, MariaDB Corporation.
This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software
......@@ -49,18 +49,6 @@ fsp_flags_get_atomic_writes(
return((atomic_writes_t)FSP_FLAGS_GET_ATOMIC_WRITES(flags));
}
/*******************************************************************//**
Find out wheather the page is index page or not
@return true if page type index page, false if not */
UNIV_INLINE
ibool
fil_page_is_index_page(
/*===================*/
byte* buf) /*!< in: page */
{
return(mach_read_from_2(buf+FIL_PAGE_TYPE) == FIL_PAGE_INDEX);
}
/*******************************************************************//**
Find out wheather the page is page compressed
@return true if page is page compressed, false if not */
......@@ -84,59 +72,3 @@ fil_page_is_compressed_encrypted(
{
return(mach_read_from_2(buf+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED);
}
/****************************************************************//**
Get the name of the compression algorithm used for page
compression.
@return compression algorithm name or "UNKNOWN" if not known*/
UNIV_INLINE
const char*
fil_get_compression_alg_name(
/*=========================*/
ulint comp_alg) /*!<in: compression algorithm number */
{
switch(comp_alg) {
case PAGE_UNCOMPRESSED:
return ("uncompressed");
break;
case PAGE_ZLIB_ALGORITHM:
return ("ZLIB");
break;
case PAGE_LZ4_ALGORITHM:
return ("LZ4");
break;
case PAGE_LZO_ALGORITHM:
return ("LZO");
break;
case PAGE_LZMA_ALGORITHM:
return ("LZMA");
break;
case PAGE_BZIP2_ALGORITHM:
return ("BZIP2");
break;
case PAGE_SNAPPY_ALGORITHM:
return ("SNAPPY");
break;
/* No default to get compiler warning */
}
return ("NULL");
}
#ifndef UNIV_INNOCHECKSUM
/*******************************************************************//**
Find out wheather the page is page compressed with lzo method
@return true if page is page compressed with lzo method, false if not */
UNIV_INLINE
ibool
fil_page_is_lzo_compressed(
/*=======================*/
byte* buf) /*!< in: page */
{
return((mach_read_from_2(buf+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED &&
mach_read_from_8(buf+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION) == PAGE_LZO_ALGORITHM) ||
(mach_read_from_2(buf+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED &&
mach_read_from_2(buf+FIL_PAGE_DATA+FIL_PAGE_COMPRESSED_SIZE) == PAGE_LZO_ALGORITHM));
}
#endif /* UNIV_INNOCHECKSUM */
......@@ -42,6 +42,12 @@ Created 2012-02-08 by Sunny Bains.
#include "srv0start.h"
#include "row0quiesce.h"
#include "fil0pagecompress.h"
#ifdef HAVE_LZO
#include "lzo/lzo1x.h"
#endif
#ifdef HAVE_SNAPPY
#include "snappy-c.h"
#endif
#include <vector>
......@@ -3365,15 +3371,30 @@ fil_iterate(
os_offset_t offset;
ulint n_bytes = iter.n_io_buffers * iter.page_size;
const ulint buf_size = srv_page_size
#ifdef HAVE_LZO
+ LZO1X_1_15_MEM_COMPRESS
#elif defined HAVE_SNAPPY
+ snappy_max_compressed_length(srv_page_size)
#endif
;
byte* page_compress_buf = static_cast<byte*>(
ut_malloc_low(buf_size, false));
ut_ad(!srv_read_only_mode);
if (!page_compress_buf) {
return DB_OUT_OF_MEMORY;
}
/* TODO: For ROW_FORMAT=COMPRESSED tables we do a lot of useless
copying for non-index pages. Unfortunately, it is
required by buf_zip_decompress() */
dberr_t err = DB_SUCCESS;
for (offset = iter.start; offset < iter.end; offset += n_bytes) {
if (callback.is_interrupted()) {
return DB_INTERRUPTED;
err = DB_INTERRUPTED;
goto func_exit;
}
byte* io_buffer = iter.io_buffer;
......@@ -3404,12 +3425,13 @@ fil_iterate(
if (!os_file_read_no_error_handling(iter.file, readptr,
offset, n_bytes)) {
ib_logf(IB_LOG_LEVEL_ERROR, "os_file_read() failed");
return DB_IO_ERROR;
err = DB_IO_ERROR;
goto func_exit;
}
bool updated = false;
ulint n_pages_read = (ulint) n_bytes / iter.page_size;
const ulint size = iter.page_size;
ulint n_pages_read = ulint(n_bytes) / size;
block->page.offset = offset / size;
for (ulint i = 0; i < n_pages_read;
......@@ -3438,11 +3460,11 @@ fil_iterate(
UINT64PF " looks corrupted.",
callback.filename(),
ulong(offset / size), offset);
return DB_CORRUPTION;
err = DB_CORRUPTION;
goto func_exit;
}
bool decrypted = false;
dberr_t err = DB_SUCCESS;
byte* dst = io_buffer + (i * size);
bool frame_changed = false;
ulint page_type = mach_read_from_2(src+FIL_PAGE_TYPE);
......@@ -3451,6 +3473,10 @@ fil_iterate(
== FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED
|| page_type == FIL_PAGE_PAGE_COMPRESSED;
if (page_compressed && block->page.zip.data) {
goto page_corrupted;
}
if (!encrypted) {
} else if (!mach_read_from_4(
FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION
......@@ -3476,7 +3502,7 @@ fil_iterate(
iter.page_size, src, &err);
if (err != DB_SUCCESS) {
return err;
goto func_exit;
}
if (!decrypted) {
......@@ -3489,8 +3515,12 @@ fil_iterate(
/* If the original page is page_compressed, we need
to decompress it before adjusting further. */
if (page_compressed) {
fil_decompress_page(NULL, dst, ulong(size),
NULL);
ulint compress_length = fil_page_decompress(
page_compress_buf, dst);
ut_ad(compress_length != srv_page_size);
if (compress_length == 0) {
goto page_corrupted;
}
updated = true;
} else if (buf_page_is_corrupted(
false,
......@@ -3501,7 +3531,7 @@ fil_iterate(
}
if ((err = callback(block)) != DB_SUCCESS) {
return err;
goto func_exit;
} else if (!updated) {
updated = buf_block_get_state(block)
== BUF_BLOCK_FILE_PAGE;
......@@ -3551,19 +3581,17 @@ fil_iterate(
src = io_buffer + (i * size);
if (page_compressed) {
ulint len = 0;
fil_compress_page(
NULL,
src,
NULL,
size,
0,/* FIXME: compression level */
512,/* FIXME: use proper block size */
encrypted,
&len);
updated = true;
if (fil_page_compress(
src,
page_compress_buf,
0,/* FIXME: compression level */
512,/* FIXME: proper block size */
encrypted)) {
/* FIXME: remove memcpy() */
memcpy(src, page_compress_buf,
srv_page_size);
}
}
/* If tablespace is encrypted, encrypt page before we
......@@ -3600,11 +3628,14 @@ fil_iterate(
offset, (ulint) n_bytes)) {
ib_logf(IB_LOG_LEVEL_ERROR, "os_file_write() failed");
return DB_IO_ERROR;
err = DB_IO_ERROR;
goto func_exit;
}
}
return DB_SUCCESS;
func_exit:
ut_free(page_compress_buf);
return err;
}
/********************************************************************//**
......
This diff is collapsed.
......@@ -510,10 +510,11 @@ buf_dblwr_process()
"Restoring possible half-written data pages "
"from the doublewrite buffer...");
unaligned_read_buf = static_cast<byte*>(ut_malloc(2 * UNIV_PAGE_SIZE));
unaligned_read_buf = static_cast<byte*>(ut_malloc(3 * UNIV_PAGE_SIZE));
read_buf = static_cast<byte*>(
ut_align(unaligned_read_buf, UNIV_PAGE_SIZE));
byte* const buf = read_buf + UNIV_PAGE_SIZE;
for (std::list<byte*>::iterator i = recv_dblwr.pages.begin();
i != recv_dblwr.pages.end(); ++i, ++page_no_dblwr ) {
......@@ -562,24 +563,26 @@ buf_dblwr_process()
ignore this page (there should be redo log
records to initialize it). */
} else {
if (fil_page_is_compressed_encrypted(read_buf) ||
fil_page_is_compressed(read_buf)) {
/* Decompress the page before
validating the checksum. */
fil_decompress_page(
NULL, read_buf, srv_page_size,
NULL, true);
/* Decompress the page before
validating the checksum. */
ulint decomp = fil_page_decompress(buf, read_buf);
if (!decomp) {
goto bad;
}
if (!decomp || (decomp != srv_page_size && zip_size)) {
goto bad;
}
if (fil_space_verify_crypt_checksum(
read_buf, zip_size, NULL, page_no)
|| !buf_page_is_corrupted(
true, read_buf, zip_size, space())) {
read_buf, zip_size, NULL, page_no)
|| !buf_page_is_corrupted(
true, read_buf, zip_size, space())) {
/* The page is good; there is no need
to consult the doublewrite buffer. */
continue;
}
bad:
/* We intentionally skip this message for
is_all_zero pages. */
ib_logf(IB_LOG_LEVEL_INFO,
......@@ -588,18 +591,15 @@ buf_dblwr_process()
space_id, page_no);
}
/* Next, validate the doublewrite page. */
if (fil_page_is_compressed_encrypted(page) ||
fil_page_is_compressed(page)) {
/* Decompress the page before
validating the checksum. */
fil_decompress_page(
NULL, page, srv_page_size, NULL, true);
ulint decomp = fil_page_decompress(buf, page);
if (!decomp || (decomp != srv_page_size && zip_size)) {
goto bad_doublewrite;
}
if (!fil_space_verify_crypt_checksum(page, zip_size, NULL, page_no)
if (!fil_space_verify_crypt_checksum(page, zip_size, NULL,
page_no)
&& buf_page_is_corrupted(true, page, zip_size, space)) {
if (!is_all_zero) {
bad_doublewrite:
ib_logf(IB_LOG_LEVEL_WARN,
"A doublewrite copy of page "
ULINTPF ":" ULINTPF " is corrupted.",
......
......@@ -708,60 +708,39 @@ fil_space_encrypt(
#ifdef UNIV_DEBUG
if (tmp) {
/* Verify that encrypted buffer is not corrupted */
byte* tmp_mem = (byte *)malloc(UNIV_PAGE_SIZE);
dberr_t err = DB_SUCCESS;
byte* src = src_frame;
bool page_compressed_encrypted = (mach_read_from_2(tmp+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED);
byte* comp_mem = NULL;
byte* uncomp_mem = NULL;
byte uncomp_mem[UNIV_PAGE_SIZE_MAX];
byte tmp_mem[UNIV_PAGE_SIZE_MAX];
ulint size = (zip_size) ? zip_size : UNIV_PAGE_SIZE;
if (page_compressed_encrypted) {
comp_mem = (byte *)malloc(UNIV_PAGE_SIZE);
uncomp_mem = (byte *)malloc(UNIV_PAGE_SIZE);
memcpy(comp_mem, src_frame, UNIV_PAGE_SIZE);
fil_decompress_page(uncomp_mem, comp_mem,
srv_page_size, NULL);
src = uncomp_mem;
memcpy(uncomp_mem, src, srv_page_size);
ulint unzipped1 = fil_page_decompress(
tmp_mem, uncomp_mem);
ut_ad(unzipped1);
if (unzipped1 != srv_page_size) {
src = uncomp_mem;
}
}
bool corrupted1 = buf_page_is_corrupted(true, src, zip_size, space);
bool ok = fil_space_decrypt(crypt_data, tmp_mem, size, tmp, &err);
ut_ad(!buf_page_is_corrupted(true, src, zip_size, space));
ut_ad(fil_space_decrypt(crypt_data, tmp_mem, size, tmp, &err));
ut_ad(err == DB_SUCCESS);
/* Need to decompress the page if it was also compressed */
if (page_compressed_encrypted) {
memcpy(comp_mem, tmp_mem, UNIV_PAGE_SIZE);
fil_decompress_page(tmp_mem, comp_mem,
srv_page_size, NULL);
byte buf[UNIV_PAGE_SIZE_MAX];
memcpy(buf, tmp_mem, srv_page_size);
ulint unzipped2 = fil_page_decompress(tmp_mem, buf);
ut_ad(unzipped2);
}
bool corrupted = buf_page_is_corrupted(true, tmp_mem, zip_size, space);
memcpy(tmp_mem+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, src+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, 8);
bool different = memcmp(src, tmp_mem, size);
if (!ok || corrupted || corrupted1 || err != DB_SUCCESS || different) {
fprintf(stderr, "ok %d corrupted %d corrupted1 %d err %d different %d\n",
ok , corrupted, corrupted1, err, different);
fprintf(stderr, "src_frame\n");
buf_page_print(src_frame, zip_size);
fprintf(stderr, "encrypted_frame\n");
buf_page_print(tmp, zip_size);
fprintf(stderr, "decrypted_frame\n");
buf_page_print(tmp_mem, zip_size);
ut_ad(0);
}
free(tmp_mem);
if (comp_mem) {
free(comp_mem);
}
if (uncomp_mem) {
free(uncomp_mem);
}
memcpy(tmp_mem + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION,
src + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION, 8);
ut_ad(!memcmp(src, tmp_mem, size));
}
#endif /* UNIV_DEBUG */
return tmp;
......
......@@ -351,19 +351,6 @@ fil_space_get_by_id(
return(space);
}
/****************************************************************//**
Get space id from fil node */
ulint
fil_node_get_space_id(
/*==================*/
fil_node_t* node) /*!< in: Compressed node*/
{
ut_ad(node);
ut_ad(node->space);
return (node->space->id);
}
/*******************************************************************//**
Returns the table space by a given name, NULL if not found. */
fil_space_t*
......
This diff is collapsed.
......@@ -5218,6 +5218,10 @@ ibuf_check_bitmap_on_import(
bitmap_page = ibuf_bitmap_get_map_page(
space_id, page_no, zip_size, &mtr);
if (!bitmap_page) {
mutex_exit(&ibuf_mutex);
return DB_CORRUPTION;
}
for (i = FSP_IBUF_BITMAP_OFFSET + 1; i < page_size; i++) {
const ulint offset = page_no + i;
......
/*****************************************************************************
Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved.
Copyright (c) 2013, 2017, MariaDB Corporation.
Copyright (c) 2013, 2018, MariaDB Corporation.
This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software
......@@ -38,6 +38,7 @@ Created 11/5/1995 Heikki Tuuri
#include "ut0rbt.h"
#include "os0proc.h"
#include "log0log.h"
#include "my_atomic.h"
/** @name Modes for buf_page_get_gen */
/* @{ */
......@@ -1528,45 +1529,16 @@ buf_page_encrypt_before_write(
buf_page_t* bpage,
byte* src_frame);
/**********************************************************************
The hook that is called after page is written to disk.
The function releases any resources needed for encryption that was allocated
in buf_page_encrypt_before_write */
UNIV_INTERN
ibool
buf_page_encrypt_after_write(
/*=========================*/
buf_page_t* page); /*!< in/out: buffer page that was flushed */
/********************************************************************//**
The hook that is called just before a page is read from disk.
The function allocates memory that is used to temporarily store disk content
before getting decrypted */
UNIV_INTERN
byte*
buf_page_decrypt_before_read(
/*=========================*/
buf_page_t* page, /*!< in/out: buffer page read from disk */
ulint zip_size); /*!< in: compressed page size, or 0 */
/********************************************************************//**
The hook that is called just after a page is read from disk.
The function decrypt disk content into buf_page_t and releases the
temporary buffer that was allocated in buf_page_decrypt_before_read */
UNIV_INTERN
bool
buf_page_decrypt_after_read(
/*========================*/
buf_page_t* page); /*!< in/out: buffer page read from disk */
/** @brief The temporary memory structure.
NOTE! The definition appears here only for other modules of this
directory (buf) to see it. Do not use from outside! */
typedef struct {
bool reserved; /*!< true if this slot is reserved
private:
int32 reserved; /*!< true if this slot is reserved
*/
public:
byte* crypt_buf; /*!< for encryption the data needs to be
copied to a separate buffer before it's
encrypted&written. this as a page can be
......@@ -1577,6 +1549,21 @@ typedef struct {
byte* out_buf; /*!< resulting buffer after
encryption/compression. This is a
pointer and not allocated. */
/** Release the slot */
void release()
{
my_atomic_store32_explicit(&reserved, false,
MY_MEMORY_ORDER_RELAXED);
}
/** Acquire the slot
@return whether the slot was acquired */
bool acquire()
{
return !my_atomic_fas32_explicit(&reserved, true,
MY_MEMORY_ORDER_RELAXED);
}
} buf_tmp_buffer_t;
/** The common buffer control block structure
......
......@@ -340,9 +340,6 @@ struct fil_space_t {
corrupted page. */
bool is_corrupt;
/*!< true if tablespace corrupted */
bool printed_compression_failure;
/*!< true if we have already printed
compression failure */
fil_space_crypt_t* crypt_data;
/*!< tablespace crypt data or NULL */
ulint file_block_size;
......
......@@ -68,7 +68,6 @@ fil_get_page_type_name(
}
return "PAGE TYPE CORRUPTED";
}
/****************************************************************//**
......
/*****************************************************************************
Copyright (C) 2013, 2017 MariaDB Corporation. All Rights Reserved.
Copyright (C) 2013, 2018 MariaDB Corporation.
This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software
......@@ -30,70 +30,26 @@ atomic writes information to table space.
Created 11/12/2013 Jan Lindström jan.lindstrom@skysql.com
***********************************************************************/
/*******************************************************************//**
Find out wheather the page is index page or not
@return true if page type index page, false if not */
UNIV_INLINE
ibool
fil_page_is_index_page(
/*===================*/
byte *buf); /*!< in: page */
/****************************************************************//**
Get the name of the compression algorithm used for page
compression.
@return compression algorithm name or "UNKNOWN" if not known*/
UNIV_INLINE
const char*
fil_get_compression_alg_name(
/*=========================*/
ulint comp_alg); /*!<in: compression algorithm number */
/****************************************************************//**
For page compressed pages compress the page before actual write
operation.
@return compressed page to be written*/
UNIV_INTERN
byte*
fil_compress_page(
/*==============*/
fil_space_t* space, /*!< in,out: tablespace (NULL during IMPORT) */
byte* buf, /*!< in: buffer from which to write; in aio
this must be appropriately aligned */
byte* out_buf, /*!< out: compressed buffer */
ulint len, /*!< in: length of input buffer.*/
ulint level, /* in: compression level */
ulint block_size, /*!< in: block size */
bool encrypted, /*!< in: is page also encrypted */
ulint* out_len); /*!< out: actual length of compressed
page */
/****************************************************************//**
For page compressed pages decompress the page after actual read
operation. */
UNIV_INTERN
void
fil_decompress_page(
/*================*/
byte* page_buf, /*!< in: preallocated buffer or NULL */
byte* buf, /*!< out: buffer from which to read; in aio
this must be appropriately aligned */
ulong len, /*!< in: length of output buffer.*/
ulint* write_size, /*!< in/out: Actual payload size of
the compressed data. */
bool return_error=false);
/*!< in: true if only an error should
be produced when decompression fails.
By default this parameter is false. */
/****************************************************************//**
Get space id from fil node
@return space id*/
UNIV_INTERN
ulint
fil_node_get_space_id(
/*==================*/
fil_node_t* node); /*!< in: Node where to get space id*/
/** Compress a page_compressed page before writing to a data file.
@param[in] buf page to be compressed
@param[out] out_buf compressed page
@param[in] level compression level
@param[in] block_size file system block size
@param[in] encrypted whether the page will be subsequently encrypted
@return actual length of compressed page
@retval 0 if the page was not compressed */
UNIV_INTERN ulint fil_page_compress(const byte* buf, byte* out_buf, ulint level,
ulint block_size, bool encrypted)
MY_ATTRIBUTE((nonnull, warn_unused_result));
/** Decompress a page that may be subject to page_compressed compression.
@param[in,out] tmp_buf temporary buffer (of innodb_page_size)
@param[in,out] buf compressed page buffer
@return size of the compressed data
@retval 0 if decompression failed
@retval srv_page_size if the page was not compressed */
UNIV_INTERN ulint fil_page_decompress(byte* tmp_buf, byte* buf)
MY_ATTRIBUTE((nonnull, warn_unused_result));
/****************************************************************//**
Get block size from fil node
......@@ -120,13 +76,4 @@ ibool
fil_page_is_compressed_encrypted(
/*=============================*/
byte* buf); /*!< in: page */
/*******************************************************************//**
Find out wheather the page is page compressed with lzo method
@return true if page is page compressed with lzo method*/
UNIV_INLINE
ibool
fil_page_is_lzo_compressed(
/*=======================*/
byte* buf); /*!< in: page */
#endif
/*****************************************************************************
Copyright (C) 2013, 2017, MariaDB Corporation. All Rights Reserved.
Copyright (C) 2013, 2018, MariaDB Corporation.
This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software
......@@ -49,18 +49,6 @@ fsp_flags_get_atomic_writes(
return((atomic_writes_t)FSP_FLAGS_GET_ATOMIC_WRITES(flags));
}
/*******************************************************************//**
Find out wheather the page is index page or not
@return true if page type index page, false if not */
UNIV_INLINE
ibool
fil_page_is_index_page(
/*===================*/
byte* buf) /*!< in: page */
{
return(mach_read_from_2(buf+FIL_PAGE_TYPE) == FIL_PAGE_INDEX);
}
/*******************************************************************//**
Find out wheather the page is page compressed
@return true if page is page compressed, false if not */
......@@ -84,59 +72,3 @@ fil_page_is_compressed_encrypted(
{
return(mach_read_from_2(buf+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED);
}
/****************************************************************//**
Get the name of the compression algorithm used for page
compression.
@return compression algorithm name or "UNKNOWN" if not known*/
UNIV_INLINE
const char*
fil_get_compression_alg_name(
/*=========================*/
ulint comp_alg) /*!<in: compression algorithm number */
{
switch(comp_alg) {
case PAGE_UNCOMPRESSED:
return ("uncompressed");
break;
case PAGE_ZLIB_ALGORITHM:
return ("ZLIB");
break;
case PAGE_LZ4_ALGORITHM:
return ("LZ4");
break;
case PAGE_LZO_ALGORITHM:
return ("LZO");
break;
case PAGE_LZMA_ALGORITHM:
return ("LZMA");
break;
case PAGE_BZIP2_ALGORITHM:
return ("BZIP2");
break;
case PAGE_SNAPPY_ALGORITHM:
return ("SNAPPY");
break;
/* No default to get compiler warning */
}
return ("NULL");
}
#ifndef UNIV_INNOCHECKSUM
/*******************************************************************//**
Find out wheather the page is page compressed with lzo method
@return true if page is page compressed with lzo method, false if not */
UNIV_INLINE
ibool
fil_page_is_lzo_compressed(
/*=======================*/
byte* buf) /*!< in: page */
{
return((mach_read_from_2(buf+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED &&
mach_read_from_8(buf+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION) == PAGE_LZO_ALGORITHM) ||
(mach_read_from_2(buf+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED &&
mach_read_from_2(buf+FIL_PAGE_DATA+FIL_PAGE_COMPRESSED_SIZE) == PAGE_LZO_ALGORITHM));
}
#endif /* UNIV_INNOCHECKSUM */
......@@ -42,6 +42,12 @@ Created 2012-02-08 by Sunny Bains.
#include "srv0start.h"
#include "row0quiesce.h"
#include "fil0pagecompress.h"
#ifdef HAVE_LZO
#include "lzo/lzo1x.h"
#endif
#ifdef HAVE_SNAPPY
#include "snappy-c.h"
#endif
#include <vector>
......@@ -3364,15 +3370,30 @@ fil_iterate(
os_offset_t offset;
ulint n_bytes = iter.n_io_buffers * iter.page_size;
const ulint buf_size = srv_page_size
#ifdef HAVE_LZO
+ LZO1X_1_15_MEM_COMPRESS
#elif defined HAVE_SNAPPY
+ snappy_max_compressed_length(srv_page_size)
#endif
;
byte* page_compress_buf = static_cast<byte*>(
ut_malloc_low(buf_size, false));
ut_ad(!srv_read_only_mode);
if (!page_compress_buf) {
return DB_OUT_OF_MEMORY;
}
/* TODO: For ROW_FORMAT=COMPRESSED tables we do a lot of useless
copying for non-index pages. Unfortunately, it is
required by buf_zip_decompress() */
dberr_t err = DB_SUCCESS;
for (offset = iter.start; offset < iter.end; offset += n_bytes) {
if (callback.is_interrupted()) {
return DB_INTERRUPTED;
err = DB_INTERRUPTED;
goto func_exit;
}
byte* io_buffer = iter.io_buffer;
......@@ -3403,12 +3424,13 @@ fil_iterate(
if (!os_file_read_no_error_handling(iter.file, readptr,
offset, n_bytes)) {
ib_logf(IB_LOG_LEVEL_ERROR, "os_file_read() failed");
return DB_IO_ERROR;
err = DB_IO_ERROR;
goto func_exit;
}
bool updated = false;
ulint n_pages_read = (ulint) n_bytes / iter.page_size;
const ulint size = iter.page_size;
ulint n_pages_read = ulint(n_bytes) / size;
block->page.offset = offset / size;
for (ulint i = 0; i < n_pages_read;
......@@ -3437,11 +3459,11 @@ fil_iterate(
UINT64PF " looks corrupted.",
callback.filename(),
ulong(offset / size), offset);
return DB_CORRUPTION;
err = DB_CORRUPTION;
goto func_exit;
}
bool decrypted = false;
dberr_t err = DB_SUCCESS;
byte* dst = io_buffer + (i * size);
bool frame_changed = false;
ulint page_type = mach_read_from_2(src+FIL_PAGE_TYPE);
......@@ -3450,6 +3472,10 @@ fil_iterate(
== FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED
|| page_type == FIL_PAGE_PAGE_COMPRESSED;
if (page_compressed && block->page.zip.data) {
goto page_corrupted;
}
if (!encrypted) {
} else if (!mach_read_from_4(
FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION
......@@ -3475,7 +3501,7 @@ fil_iterate(
iter.page_size, src, &err);
if (err != DB_SUCCESS) {
return err;
goto func_exit;
}
if (!decrypted) {
......@@ -3488,8 +3514,12 @@ fil_iterate(
/* If the original page is page_compressed, we need
to decompress it before adjusting further. */
if (page_compressed) {
fil_decompress_page(NULL, dst, ulong(size),
NULL);
ulint compress_length = fil_page_decompress(
page_compress_buf, dst);
ut_ad(compress_length != srv_page_size);
if (compress_length == 0) {
goto page_corrupted;
}
updated = true;
} else if (buf_page_is_corrupted(
false,
......@@ -3500,7 +3530,7 @@ fil_iterate(
}
if ((err = callback(block)) != DB_SUCCESS) {
return err;
goto func_exit;
} else if (!updated) {
updated = buf_block_get_state(block)
== BUF_BLOCK_FILE_PAGE;
......@@ -3550,19 +3580,17 @@ fil_iterate(
src = io_buffer + (i * size);
if (page_compressed) {
ulint len = 0;
fil_compress_page(
NULL,
src,
NULL,
size,
0,/* FIXME: compression level */
512,/* FIXME: use proper block size */
encrypted,
&len);
updated = true;
if (fil_page_compress(
src,
page_compress_buf,
0,/* FIXME: compression level */
512,/* FIXME: proper block size */
encrypted)) {
/* FIXME: remove memcpy() */
memcpy(src, page_compress_buf,
srv_page_size);
}
}
/* If tablespace is encrypted, encrypt page before we
......@@ -3599,11 +3627,14 @@ fil_iterate(
offset, (ulint) n_bytes)) {
ib_logf(IB_LOG_LEVEL_ERROR, "os_file_write() failed");
return DB_IO_ERROR;
err = DB_IO_ERROR;
goto func_exit;
}
}
return DB_SUCCESS;
func_exit:
ut_free(page_compress_buf);
return err;
}
/********************************************************************//**
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment