Skip to content

M
MariaDB
Commit 1a780eef authored by Marko Mäkelä's avatar Marko Mäkelä
Browse files
Options

MDEV-17958 Make bug-endian innodb_checksum_algorithm=crc32 optional 

In MySQL 5.7, it was noticed that files are not portable between
big-endian and little-endian processor architectures
(such as SPARC and x86), because the original implementation of
innodb_checksum_algorithm=crc32 was not byte order agnostic.

A byte order agnostic implementation of innodb_checksum_algorithm=crc32
was only added to MySQL 5.7, not backported to 5.6. Consequently,
MariaDB Server versions 10.0 and 10.1 only contain the CRC-32C
implementation that works incorrectly on big-endian architectures,
and MariaDB Server 10.2.2 got the byte-order agnostic CRC-32C
implementation from MySQL 5.7.

MySQL 5.7 introduced a "legacy crc32" variant that is functionally
equivalent to the big-endian version of the original crc32 implementation.
Thanks to this variant, old data files can be transferred from big-endian
systems to newer versions.

Introducing new variants of checksum algorithms (without introducing
new names for them, or something on the pages themselves to identify
the algorithm) generally is a bad idea, because each checksum algorithm
is like a lottery ticket. The more algorithms you try, the more likely
it will be for the checksum to match on a corrupted page.

So, essentially MySQL 5.7 weakened innodb_checksum_algorithm=crc32,
and MariaDB 10.2.2 inherited this weakening.

We introduce a build option that together with MDEV-17957
makes innodb_checksum_algorithm=strict_crc32 strict again
by only allowing one variant of the checksum to match.

WITH_INNODB_BUG_ENDIAN_CRC32: A new cmake option for enabling the
bug-compatible "legacy crc32" checksum. This is only enabled on
big-endian systems by default, to facilitate an upgrade from
MariaDB 10.0 or 10.1. Checked by #ifdef INNODB_BUG_ENDIAN_CRC32.

ut_crc32_byte_by_byte: Remove (unused function).

legacy_big_endian_checksum: Remove. This variable seems to have
unnecessarily complicated the logic. When the weakening is enabled,
we must always fall back to the buggy checksum.

buf_page_check_crc32(): A helper function to compute one or
two CRC-32C variants.
parent 2e5aea4b
Show whitespace changes
Inline Side-by-side
Showing with 211 additions and 253 deletions
storage/innobase/buf/buf0buf.cc
View file @ 1a780eef
......@@ -756,17 +756,14 @@ buf_page_is_zeroes(
@param[in] read_buf database page
@param[in] checksum_field1 new checksum field
@param[in] checksum_field2 old checksum field
@param[in] use_legacy_big_endian use legacy big endian algorithm
@return true if the page is in crc32 checksum format. */
bool
buf_page_is_checksum_valid_crc32(
const byte* read_buf,
ulint checksum_field1,
ulint checksum_field2,
bool use_legacy_big_endian)
ulint checksum_field2)
{
const uint32_t crc32 = buf_calc_page_crc32(read_buf,
use_legacy_big_endian);
const uint32_t crc32 = buf_calc_page_crc32(read_buf);
#ifdef UNIV_INNOCHECKSUM
if (log_file
......@@ -783,17 +780,11 @@ buf_page_is_checksum_valid_crc32(
return false;
}
if (checksum_field1 == crc32) {
return(true);
} else {
const uint32_t crc32_legacy = buf_calc_page_crc32(read_buf, true);
if (checksum_field1 == crc32_legacy) {
return(true);
}
}
return(false);
return checksum_field1 == crc32
#ifdef INNODB_BUG_ENDIAN_CRC32
|| checksum_field1 == buf_calc_page_crc32(read_buf, true)
#endif
;
}
/** Checks if the page is in innodb checksum format.
......@@ -922,6 +913,29 @@ buf_page_is_checksum_valid_none(
&& checksum_field1 == BUF_NO_CHECKSUM_MAGIC);
}
#ifdef INNODB_BUG_ENDIAN_CRC32
/** Validate the CRC-32C checksum of a page.
@param[in] page buffer page (srv_page_size bytes)
@param[in] checksum CRC-32C checksum stored on page
@return computed checksum */
static uint32_t buf_page_check_crc32(const byte* page, uint32_t checksum)
{
uint32_t crc32 = buf_calc_page_crc32(page);
if (checksum != crc32) {
crc32 = buf_calc_page_crc32(page, true);
}
return crc32;
}
#else /* INNODB_BUG_ENDIAN_CRC32 */
/** Validate the CRC-32C checksum of a page.
@param[in] page buffer page (srv_page_size bytes)
@param[in] checksum CRC-32C checksum stored on page
@return computed checksum */
# define buf_page_check_crc32(page, checksum) buf_calc_page_crc32(page)
#endif /* INNODB_BUG_ENDIAN_CRC32 */
/** Check if a page is corrupt.
@param[in] check_lsn whether the LSN should be checked
@param[in] read_buf database page
......@@ -1037,26 +1051,20 @@ buf_page_is_corrupted(
&& *reinterpret_cast<const ib_uint64_t*>(
read_buf + FIL_PAGE_LSN) == 0) {
ulint i;
/* make sure that the page is really empty */
for (ulint i = 0; i < UNIV_PAGE_SIZE; i++) {
for (ulint i = 0; i < page_size.logical(); i++) {
if (read_buf[i] != 0) {
return(true);
}
}
#ifdef UNIV_INNOCHECKSUM
if (i >= page_size.logical()) {
if (log_file) {
fprintf(log_file, "Page::%llu"
" is empty and uncorrupted\n",
cur_page_num);
}
return(false);
}
#else
return(i < page_size.logical());
#endif /* UNIV_INNOCHECKSUM */
return(false);
}
const srv_checksum_algorithm_t curr_algo =
......@@ -1065,10 +1073,7 @@ buf_page_is_corrupted(
switch (curr_algo) {
case SRV_CHECKSUM_ALGORITHM_STRICT_CRC32:
return !buf_page_is_checksum_valid_crc32(
read_buf, checksum_field1, checksum_field2, false)
&& !buf_page_is_checksum_valid_crc32(
read_buf, checksum_field1, checksum_field2,
true);
read_buf, checksum_field1, checksum_field2);
case SRV_CHECKSUM_ALGORITHM_STRICT_INNODB:
return !buf_page_is_checksum_valid_innodb(
read_buf, checksum_field1, checksum_field2);
......@@ -1111,19 +1116,10 @@ buf_page_is_corrupted(
if (srv_checksum_algorithm
== SRV_CHECKSUM_ALGORITHM_CRC32) {
crc32 = buf_calc_page_crc32(
read_buf, legacy_big_endian_checksum);
crc32 = buf_page_check_crc32(read_buf,
checksum_field2);
crc32_inited = true;
if (!legacy_big_endian_checksum
&& checksum_field2 != crc32) {
crc32 = buf_calc_page_crc32(read_buf,
true);
legacy_big_endian_checksum =
checksum_field2 == crc32;
}
if (checksum_field2 != crc32
&& checksum_field2
!= buf_calc_page_old_checksum(read_buf)) {
......@@ -1135,19 +1131,10 @@ buf_page_is_corrupted(
if (checksum_field2
!= buf_calc_page_old_checksum(read_buf)) {
crc32 = buf_calc_page_crc32(
read_buf,
legacy_big_endian_checksum);
crc32 = buf_page_check_crc32(
read_buf, checksum_field2);
crc32_inited = true;
if (!legacy_big_endian_checksum
&& checksum_field2 != crc32) {
crc32 = buf_calc_page_crc32(
read_buf, true);
legacy_big_endian_checksum =
checksum_field2 == crc32;
}
if (checksum_field2 != crc32) {
return true;
}
......@@ -1161,18 +1148,9 @@ buf_page_is_corrupted(
== SRV_CHECKSUM_ALGORITHM_CRC32) {
if (!crc32_inited) {
crc32 = buf_calc_page_crc32(
read_buf,
legacy_big_endian_checksum);
crc32 = buf_page_check_crc32(
read_buf, checksum_field2);
crc32_inited = true;
if (!legacy_big_endian_checksum
&& checksum_field2 != crc32) {
crc32 = buf_calc_page_crc32(
read_buf, true);
legacy_big_endian_checksum =
checksum_field2 == crc32;
}
}
if (checksum_field1 != crc32
......@@ -1188,18 +1166,9 @@ buf_page_is_corrupted(
!= buf_calc_page_new_checksum(read_buf)) {
if (!crc32_inited) {
crc32 = buf_calc_page_crc32(
read_buf,
legacy_big_endian_checksum);
crc32 = buf_page_check_crc32(
read_buf, checksum_field2);
crc32_inited = true;
if (!legacy_big_endian_checksum
&& checksum_field2 != crc32) {
crc32 = buf_calc_page_crc32(
read_buf, true);
legacy_big_endian_checksum =
checksum_field2 == crc32;
}
}
if (checksum_field1 != crc32) {
......@@ -1255,10 +1224,12 @@ buf_page_print(const byte* read_buf, const page_size_t& page_size)
<< page_zip_calc_checksum(
read_buf, page_size.physical(),
SRV_CHECKSUM_ALGORITHM_CRC32)
#ifdef INNODB_BUG_ENDIAN_CRC32
<< "/"
<< page_zip_calc_checksum(
read_buf, page_size.physical(),
SRV_CHECKSUM_ALGORITHM_CRC32, true)
#endif
<< ", "
<< buf_checksum_algorithm_name(
SRV_CHECKSUM_ALGORITHM_INNODB)
......@@ -1284,9 +1255,10 @@ buf_page_print(const byte* read_buf, const page_size_t& page_size)
} else {
const uint32_t crc32 = buf_calc_page_crc32(read_buf);
#ifdef INNODB_BUG_ENDIAN_CRC32
const uint32_t crc32_legacy = buf_calc_page_crc32(read_buf,
true);
#endif /* INNODB_BUG_ENDIAN_CRC32 */
ulint page_type = fil_page_get_type(read_buf);
ib::info() << "Uncompressed page, stored checksum in field1 "
......@@ -1295,7 +1267,10 @@ buf_page_print(const byte* read_buf, const page_size_t& page_size)
<< ", calculated checksums for field1: "
<< buf_checksum_algorithm_name(
SRV_CHECKSUM_ALGORITHM_CRC32) << " "
<< crc32 << "/" << crc32_legacy
<< crc32
#ifdef INNODB_BUG_ENDIAN_CRC32
<< "/" << crc32_legacy
#endif
<< ", "
<< buf_checksum_algorithm_name(
SRV_CHECKSUM_ALGORITHM_INNODB) << " "
......@@ -1312,7 +1287,10 @@ buf_page_print(const byte* read_buf, const page_size_t& page_size)
<< ", calculated checksums for field2: "
<< buf_checksum_algorithm_name(
SRV_CHECKSUM_ALGORITHM_CRC32) << " "
<< crc32 << "/" << crc32_legacy
<< crc32
#ifdef INNODB_BUG_ENDIAN_CRC32
<< "/" << crc32_legacy
#endif
<< ", "
<< buf_checksum_algorithm_name(
SRV_CHECKSUM_ALGORITHM_INNODB) << " "
......@@ -3950,10 +3928,12 @@ buf_zip_decompress(
<< ", crc32: "
<< page_zip_calc_checksum(
frame, size, SRV_CHECKSUM_ALGORITHM_CRC32)
#ifdef INNODB_BUG_ENDIAN_CRC32
<< "/"
<< page_zip_calc_checksum(
frame, size, SRV_CHECKSUM_ALGORITHM_CRC32,
true)
#endif
<< " innodb: "
<< page_zip_calc_checksum(
frame, size, SRV_CHECKSUM_ALGORITHM_INNODB)
......
storage/innobase/buf/buf0checksum.cc
View file @ 1a780eef
......@@ -39,44 +39,56 @@ ha_innodb.cc:12251: error: cannot convert 'srv_checksum_algorithm_t*' to
'long unsigned int*' in initialization */
ulong srv_checksum_algorithm = SRV_CHECKSUM_ALGORITHM_INNODB;
/** set if we have found pages matching legacy big endian checksum */
bool legacy_big_endian_checksum = false;
/** Calculates the CRC32 checksum of a page. The value is stored to the page
#ifdef INNODB_BUG_ENDIAN_CRC32
/** Calculate the CRC32 checksum of a page. The value is stored to the page
when it is written to a file and also checked for a match when reading from
the file. When reading we allow both normal CRC32 and CRC-legacy-big-endian
variants. Note that we must be careful to calculate the same value on 32-bit
and 64-bit architectures.
@param[in] page buffer page (UNIV_PAGE_SIZE bytes)
@param[in] use_legacy_big_endian if true then use big endian
byteorder when converting byte strings to integers
@return checksum */
uint32_t
buf_calc_page_crc32(
const byte* page,
bool use_legacy_big_endian /* = false */)
the file. Note that we must be careful to calculate the same value on all
architectures.
@param[in] page buffer page (srv_page_size bytes)
@param[in] bug_endian whether to use big endian byteorder
when converting byte strings to integers, for bug-compatibility with
big-endian architecture running MySQL 5.6, MariaDB 10.0 or MariaDB 10.1
@return CRC-32C */
uint32_t buf_calc_page_crc32(const byte* page, bool bug_endian)
{
/* Since the field FIL_PAGE_FILE_FLUSH_LSN, and in versions <= 4.1.x
FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID, are written outside the buffer pool
to the first pages of data files, we have to skip them in the page
checksum calculation.
We must also skip the field FIL_PAGE_SPACE_OR_CHKSUM where the
checksum is stored, and also the last 8 bytes of page because
there we store the old formula checksum. */
ut_crc32_func_t crc32_func = use_legacy_big_endian
? ut_crc32_legacy_big_endian
: ut_crc32;
const uint32_t c1 = crc32_func(
return bug_endian
? ut_crc32_legacy_big_endian(
page + FIL_PAGE_OFFSET,
FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION - FIL_PAGE_OFFSET);
const uint32_t c2 = crc32_func(
page + FIL_PAGE_DATA,
UNIV_PAGE_SIZE - FIL_PAGE_DATA - FIL_PAGE_END_LSN_OLD_CHKSUM);
return(c1 ^ c2);
FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION
- FIL_PAGE_OFFSET)
^ ut_crc32_legacy_big_endian(page + FIL_PAGE_DATA,
srv_page_size
- (FIL_PAGE_DATA
+ FIL_PAGE_END_LSN_OLD_CHKSUM))
: ut_crc32(page + FIL_PAGE_OFFSET,
FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION
- FIL_PAGE_OFFSET)
^ ut_crc32(page + FIL_PAGE_DATA,
srv_page_size
- (FIL_PAGE_DATA + FIL_PAGE_END_LSN_OLD_CHKSUM));
}
#else
/** Calculate the CRC32 checksum of a page. The value is stored to the page
when it is written to a file and also checked for a match when reading from
the file. Note that we must be careful to calculate the same value on all
architectures.
@param[in] page buffer page (srv_page_size bytes)
@return CRC-32C */
uint32_t buf_calc_page_crc32(const byte* page)
{
/* Note: innodb_checksum_algorithm=crc32 could and should have
included the entire page in the checksum, and CRC-32 values
should be combined with the CRC-32 function, not with
exclusive OR. We stick to the current algorithm in order to
remain compatible with old data files. */
return ut_crc32(page + FIL_PAGE_OFFSET,
FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION
- FIL_PAGE_OFFSET)
^ ut_crc32(page + FIL_PAGE_DATA,
srv_page_size
- (FIL_PAGE_DATA + FIL_PAGE_END_LSN_OLD_CHKSUM));
}
#endif
/** Calculate a checksum which is stored to the page when it is written
to a file. Note that we must be careful to calculate the same value on
......
storage/innobase/fil/fil0crypt.cc
View file @ 1a780eef
......@@ -2636,10 +2636,8 @@ fil_space_verify_crypt_checksum(
srv_checksum_algorithm);
switch (algorithm) {
case SRV_CHECKSUM_ALGORITHM_STRICT_CRC32:
/* We never supported upgrade from the "legacy crc32"
on big endian systems from MariaDB 10.1 to later. */
valid = buf_page_is_checksum_valid_crc32(
page, checksum1, checksum2, false);
page, checksum1, checksum2);
break;
case SRV_CHECKSUM_ALGORITHM_STRICT_INNODB:
valid = buf_page_is_checksum_valid_innodb(
......@@ -2649,13 +2647,11 @@ fil_space_verify_crypt_checksum(
case SRV_CHECKSUM_ALGORITHM_CRC32:
case SRV_CHECKSUM_ALGORITHM_INNODB:
case SRV_CHECKSUM_ALGORITHM_NONE:
/* We never supported upgrade from the "legacy crc32"
on big endian systems from MariaDB 10.1 to later.
We also never supported
/* never supported
innodb_checksum_algorithm=none or strict_none
for encrypted pages. */
valid = buf_page_is_checksum_valid_crc32(
page, checksum1, checksum2, false)
page, checksum1, checksum2)
|| buf_page_is_checksum_valid_innodb(
page, checksum1, checksum2);
break;
......@@ -2684,8 +2680,11 @@ fil_space_verify_crypt_checksum(
ib::info()
<< "If unencrypted: stored checksum [" << checksum1
<< ":" << checksum2 << "] calculated crc32 ["
<< buf_calc_page_crc32(page, false) << ":"
<< buf_calc_page_crc32(page, true) << "] innodb ["
<< buf_calc_page_crc32(page)
# ifdef INNODB_BUG_ENDIAN_CRC32
<< ":" << buf_calc_page_crc32(page, true)
# endif /* INNODB_BUG_ENDIAN_CRC32 */
<< "] innodb ["
<< buf_calc_page_old_checksum(page) << ":"
<< buf_calc_page_new_checksum(page) << "] LSN "
<< mach_read_from_4(page + FIL_PAGE_LSN);
......
storage/innobase/include/buf0buf.h
View file @ 1a780eef
......@@ -716,14 +716,12 @@ buf_block_unfix(
@param[in] read_buf database page
@param[in] checksum_field1 new checksum field
@param[in] checksum_field2 old checksum field
@param[in] use_legacy_big_endian use legacy big endian algorithm
@return true if the page is in crc32 checksum format. */
bool
buf_page_is_checksum_valid_crc32(
const byte* read_buf,
ulint checksum_field1,
ulint checksum_field2,
bool use_legacy_big_endian)
ulint checksum_field2)
MY_ATTRIBUTE((nonnull(1), warn_unused_result));
/** Checks if the page is in innodb checksum format.
......
storage/innobase/include/buf0checksum.h
View file @ 1a780eef
/*****************************************************************************
Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved.
Copyright (c) 2017, MariaDB Corporation.
Copyright (c) 2017, 2018, MariaDB Corporation.
This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software
......@@ -29,19 +29,26 @@ Created Aug 11, 2011 Vasil Dimov
#include "buf0types.h"
#ifdef INNODB_BUG_ENDIAN_CRC32
/** Calculate the CRC32 checksum of a page. The value is stored to the page
when it is written to a file and also checked for a match when reading from
the file. When reading we allow both normal CRC32 and CRC-legacy-big-endian
variants. Note that we must be careful to calculate the same value on 32-bit
and 64-bit architectures.
@param[in] page buffer page (UNIV_PAGE_SIZE bytes)
@param[in] use_legacy_big_endian if true then use big endian
byteorder when converting byte strings to integers
@return checksum */
uint32_t
buf_calc_page_crc32(
const byte* page,
bool use_legacy_big_endian = false);
the file. Note that we must be careful to calculate the same value on all
architectures.
@param[in] page buffer page (srv_page_size bytes)
@param[in] bug_endian whether to use big endian byteorder
when converting byte strings to integers, for bug-compatibility with
big-endian architecture running MySQL 5.6, MariaDB 10.0 or MariaDB 10.1
@return CRC-32C */
uint32_t buf_calc_page_crc32(const byte* page, bool bug_endian = false);
#else
/** Calculate the CRC32 checksum of a page. The value is stored to the page
when it is written to a file and also checked for a match when reading from
the file. Note that we must be careful to calculate the same value on all
architectures.
@param[in] page buffer page (srv_page_size bytes)
@return CRC-32C */
uint32_t buf_calc_page_crc32(const byte* page);
#endif
/** Calculate a checksum which is stored to the page when it is written
to a file. Note that we must be careful to calculate the same value on
......@@ -69,6 +76,5 @@ const char*
buf_checksum_algorithm_name(srv_checksum_algorithm_t algo);
extern ulong srv_checksum_algorithm;
extern bool legacy_big_endian_checksum;
#endif /* buf0checksum_h */
storage/innobase/include/page0zip.h
View file @ 1a780eef
......@@ -492,16 +492,17 @@ page_zip_parse_compress(
@param[in] data compressed page
@param[in] size size of compressed page
@param[in] algo algorithm to use
@param[in] use_legacy_big_endian only used if algo is
SRV_CHECKSUM_ALGORITHM_CRC32 or SRV_CHECKSUM_ALGORITHM_STRICT_CRC32 - if true
then use big endian byteorder when converting byte strings to integers.
@return page checksum */
uint32_t
page_zip_calc_checksum(
const void* data,
ulint size,
srv_checksum_algorithm_t algo,
bool use_legacy_big_endian = false);
srv_checksum_algorithm_t algo
#ifdef INNODB_BUG_ENDIAN_CRC32
/** for crc32, use the big-endian bug-compatible crc32 variant */
, bool use_legacy_big_endian = false
#endif
);
/**********************************************************************//**
Verify a compressed page's checksum.
......
storage/innobase/include/ut0crc32.h
View file @ 1a780eef
/*****************************************************************************
Copyright (c) 2011, 2015, Oracle and/or its affiliates. All Rights Reserved.
Copyright (c) 2016, MariaDB Corporation.
Copyright (c) 2016, 2018, MariaDB Corporation.
This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software
......@@ -47,13 +47,11 @@ typedef uint32_t (*ut_crc32_func_t)(const byte* ptr, ulint len);
/** Pointer to CRC32 calculation function. */
extern ut_crc32_func_t ut_crc32;
#ifdef INNODB_BUG_ENDIAN_CRC32
/** Pointer to CRC32 calculation function, which uses big-endian byte order
when converting byte strings to integers internally. */
extern ut_crc32_func_t ut_crc32_legacy_big_endian;
/** Pointer to CRC32-byte-by-byte calculation function (byte order agnostic,
but very slow). */
extern ut_crc32_func_t ut_crc32_byte_by_byte;
#endif /* INNODB_BUG_ENDIAN_CRC32 */
extern const char* ut_crc32_implementation;
......
storage/innobase/innodb.cmake
View file @ 1a780eef
......@@ -25,6 +25,7 @@ INCLUDE(lzma.cmake)
INCLUDE(bzip2.cmake)
INCLUDE(snappy.cmake)
INCLUDE(numa)
INCLUDE(TestBigEndian)
MYSQL_CHECK_LZ4()
MYSQL_CHECK_LZO()
......@@ -32,6 +33,7 @@ MYSQL_CHECK_LZMA()
MYSQL_CHECK_BZIP2()
MYSQL_CHECK_SNAPPY()
MYSQL_CHECK_NUMA()
TEST_BIG_ENDIAN(IS_BIG_ENDIAN)
IF(CMAKE_CROSSCOMPILING)
# Use CHECK_C_SOURCE_COMPILES instead of CHECK_C_SOURCE_RUNS when
......@@ -123,6 +125,11 @@ ELSEIF(WITH_INNODB_ROOT_GUESS)
ADD_DEFINITIONS(-DBTR_CUR_ADAPT)
ENDIF()
OPTION(WITH_INNODB_BUG_ENDIAN_CRC32 "Weaken innodb_checksum_algorithm=crc32 by supporting upgrade from big-endian systems running 5.6/10.0/10.1" ${IS_BIG_ENDIAN})
IF(WITH_INNODB_BUG_ENDIAN_CRC32)
ADD_DEFINITIONS(-DINNODB_BUG_ENDIAN_CRC32)
ENDIF()
OPTION(WITH_INNODB_EXTRA_DEBUG "Enable extra InnoDB debug checks" OFF)
IF(WITH_INNODB_EXTRA_DEBUG)
IF(NOT CMAKE_BUILD_TYPE STREQUAL "Debug")
......
storage/innobase/page/page0zip.cc
View file @ 1a780eef
......@@ -4909,18 +4909,17 @@ page_zip_parse_compress(
@param[in] data compressed page
@param[in] size size of compressed page
@param[in] algo algorithm to use
@param[in] use_legacy_big_endian only used if algo is
SRV_CHECKSUM_ALGORITHM_CRC32 or SRV_CHECKSUM_ALGORITHM_STRICT_CRC32 - if true
then use big endian byteorder when converting byte strings to integers.
SRV_CHECKSUM_ALGORITHM_CRC32 or SRV_CHECKSUM_ALGORITHM_STRICT_CRC32 - if true
then use big endian byteorder when converting byte strings to integers.
@return page checksum */
uint32_t
page_zip_calc_checksum(
const void* data,
ulint size,
srv_checksum_algorithm_t algo,
bool use_legacy_big_endian /* = false */)
srv_checksum_algorithm_t algo
#ifdef INNODB_BUG_ENDIAN_CRC32
/** for crc32, use the big-endian bug-compatible crc32 variant */
, bool use_legacy_big_endian
#endif
)
{
uLong adler;
const Bytef* s = static_cast<const byte*>(data);
......@@ -4931,25 +4930,25 @@ page_zip_calc_checksum(
switch (algo) {
case SRV_CHECKSUM_ALGORITHM_CRC32:
case SRV_CHECKSUM_ALGORITHM_STRICT_CRC32:
{
ut_ad(size > FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID);
ut_crc32_func_t crc32_func = use_legacy_big_endian
? ut_crc32_legacy_big_endian
: ut_crc32;
const uint32_t crc32
= crc32_func(
s + FIL_PAGE_OFFSET,
FIL_PAGE_LSN - FIL_PAGE_OFFSET)
^ crc32_func(
#ifdef INNODB_BUG_ENDIAN_CRC32
if (use_legacy_big_endian) {
return ut_crc32_legacy_big_endian(s + FIL_PAGE_OFFSET,
FIL_PAGE_LSN
- FIL_PAGE_OFFSET)
^ ut_crc32_legacy_big_endian(
s + FIL_PAGE_TYPE, 2)
^ crc32_func(
^ ut_crc32_legacy_big_endian(
s + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID,
size - FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID);
return(crc32);
size
- FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID);
}
#endif
return ut_crc32(s + FIL_PAGE_OFFSET,
FIL_PAGE_LSN - FIL_PAGE_OFFSET)
^ ut_crc32(s + FIL_PAGE_TYPE, 2)
^ ut_crc32(s + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID,
size - FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID);
case SRV_CHECKSUM_ALGORITHM_INNODB:
case SRV_CHECKSUM_ALGORITHM_STRICT_INNODB:
ut_ad(size > FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID);
......@@ -4984,13 +4983,8 @@ page_zip_verify_checksum(
const void* data, /*!< in: compressed page */
ulint size) /*!< in: size of compressed page */
{
ib_uint32_t stored;
ib_uint32_t calc;
ib_uint32_t crc32 = 0;
ib_uint32_t innodb = 0;
stored = static_cast<ib_uint32_t>(mach_read_from_4(
static_cast<const unsigned char*>(data) + FIL_PAGE_SPACE_OR_CHKSUM));
const uint32_t stored = mach_read_from_4(
static_cast<const byte*>(data) + FIL_PAGE_SPACE_OR_CHKSUM);
#if FIL_PAGE_LSN % 8
#error "FIL_PAGE_LSN must be 64 bit aligned"
......@@ -5034,8 +5028,7 @@ page_zip_verify_checksum(
return(TRUE);
}
calc = static_cast<ib_uint32_t>(page_zip_calc_checksum(
data, size, curr_algo));
uint32_t calc = page_zip_calc_checksum(data, size, curr_algo);
#ifdef UNIV_INNOCHECKSUM
if (log_file) {
......@@ -5070,13 +5063,11 @@ page_zip_verify_checksum(
switch (curr_algo) {
case SRV_CHECKSUM_ALGORITHM_STRICT_CRC32:
calc = page_zip_calc_checksum(data, size, curr_algo, true);
if (calc == stored) {
legacy_big_endian_checksum = true;
return TRUE;
}
return FALSE;
#ifdef INNODB_BUG_ENDIAN_CRC32
return stored == page_zip_calc_checksum(data, size, curr_algo,
true);
#endif
/* fall through */
case SRV_CHECKSUM_ALGORITHM_STRICT_INNODB:
case SRV_CHECKSUM_ALGORITHM_STRICT_NONE:
return FALSE;
......@@ -5085,29 +5076,29 @@ page_zip_verify_checksum(
return(TRUE);
}
calc = page_zip_calc_checksum(data, size, curr_algo, true);
crc32 = calc;
if (crc32 == stored) {
legacy_big_endian_checksum = true;
return TRUE;
}
innodb = static_cast<ib_uint32_t>(page_zip_calc_checksum(
data, size, SRV_CHECKSUM_ALGORITHM_INNODB));
break;
return
#ifdef INNODB_BUG_ENDIAN_CRC32
stored == page_zip_calc_checksum(data, size, curr_algo,
true) ||
#endif
stored == page_zip_calc_checksum(
data, size, SRV_CHECKSUM_ALGORITHM_INNODB);
case SRV_CHECKSUM_ALGORITHM_INNODB:
if (stored == BUF_NO_CHECKSUM_MAGIC) {
return TRUE;
}
crc32 = static_cast<ib_uint32_t>(page_zip_calc_checksum(
data, size, SRV_CHECKSUM_ALGORITHM_CRC32));
innodb = calc;
break;
return stored == page_zip_calc_checksum(
data, size, SRV_CHECKSUM_ALGORITHM_CRC32)
#ifdef INNODB_BUG_ENDIAN_CRC32
|| stored == page_zip_calc_checksum(
data, size,
SRV_CHECKSUM_ALGORITHM_CRC32, true)
#endif
;
case SRV_CHECKSUM_ALGORITHM_NONE:
return TRUE;
}
return (stored == crc32 || stored == innodb);
return FALSE;
}
storage/innobase/ut/ut0crc32.cc
View file @ 1a780eef
......@@ -2,7 +2,7 @@
Copyright (c) 2009, 2010 Facebook, Inc. All Rights Reserved.
Copyright (c) 2011, 2015, Oracle and/or its affiliates. All Rights Reserved.
Copyright (c) 2016, MariaDB Corporation.
Copyright (c) 2016, 2018, MariaDB Corporation.
This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software
......@@ -92,13 +92,11 @@ mysys/my_perf.c, contributed by Facebook under the following license.
/** Pointer to CRC32 calculation function. */
ut_crc32_func_t ut_crc32;
#ifdef INNODB_BUG_ENDIAN_CRC32
/** Pointer to CRC32 calculation function, which uses big-endian byte order
when converting byte strings to integers internally. */
ut_crc32_func_t ut_crc32_legacy_big_endian;
/** Pointer to CRC32-byte-by-byte calculation function (byte order agnostic,
but very slow). */
ut_crc32_func_t ut_crc32_byte_by_byte;
#endif /* INNODB_BUG_ENDIAN_CRC32 */
/** Text description of CRC32 implementation */
const char* ut_crc32_implementation;
......@@ -278,6 +276,7 @@ ut_crc32_64_hw(
*len -= 8;
}
#ifdef INNODB_BUG_ENDIAN_CRC32
/** Calculate CRC32 over 64-bit byte string using a hardware/CPU instruction.
The byte string is converted to a 64-bit integer using big endian byte order.
@param[in,out] crc crc32 checksum so far when this function is called,
......@@ -308,6 +307,7 @@ ut_crc32_64_legacy_big_endian_hw(
*data += 8;
*len -= 8;
}
#endif /* INNODB_BUG_ENDIAN_CRC32 */
/** Calculates CRC32 using hardware/CPU instructions.
@param[in] buf data over which to calculate CRC32
......@@ -396,6 +396,7 @@ ut_crc32_hw(
return(~crc);
}
# ifdef INNODB_BUG_ENDIAN_CRC32
/** Calculates CRC32 using hardware/CPU instructions.
This function uses big endian byte ordering when converting byte sequence to
integers.
......@@ -445,26 +446,7 @@ ut_crc32_legacy_big_endian_hw(
return(~crc);
}
/** Calculates CRC32 using hardware/CPU instructions.
This function processes one byte at a time (very slow) and thus it does
not depend on the byte order of the machine.
@param[in] buf data over which to calculate CRC32
@param[in] len data length
@return CRC-32C (polynomial 0x11EDC6F41) */
uint32_t
ut_crc32_byte_by_byte_hw(
const byte* buf,
ulint len)
{
uint32_t crc = 0xFFFFFFFFU;
while (len > 0) {
ut_crc32_8_hw(&crc, &buf, &len);
}
return(~crc);
}
# endif /* INNODB_BUG_ENDIAN_CRC32 */
#endif /* defined(__GNUC__) && defined(__x86_64__) || (_WIN64) */
/* CRC32 software implementation. */
......@@ -577,6 +559,7 @@ ut_crc32_64_sw(
*len -= 8;
}
#ifdef INNODB_BUG_ENDIAN_CRC32
/** Calculate CRC32 over 64-bit byte string using a software implementation.
The byte string is converted to a 64-bit integer using big endian byte order.
@param[in,out] crc crc32 checksum so far when this function is called,
......@@ -602,6 +585,7 @@ ut_crc32_64_legacy_big_endian_sw(
*data += 8;
*len -= 8;
}
#endif /* INNODB_BUG_ENDIAN_CRC32 */
/** Calculates CRC32 in software, without using CPU instructions.
@param[in] buf data over which to calculate CRC32
......@@ -653,6 +637,7 @@ ut_crc32_sw(
return(~crc);
}
#ifdef INNODB_BUG_ENDIAN_CRC32
/** Calculates CRC32 in software, without using CPU instructions.
This function uses big endian byte ordering when converting byte sequence to
integers.
......@@ -704,28 +689,7 @@ ut_crc32_legacy_big_endian_sw(
return(~crc);
}
/** Calculates CRC32 in software, without using CPU instructions.
This function processes one byte at a time (very slow) and thus it does
not depend on the byte order of the machine.
@param[in] buf data over which to calculate CRC32
@param[in] len data length
@return CRC-32C (polynomial 0x11EDC6F41) */
uint32_t
ut_crc32_byte_by_byte_sw(
const byte* buf,
ulint len)
{
uint32_t crc = 0xFFFFFFFFU;
ut_a(ut_crc32_slice8_table_initialized);
while (len > 0) {
ut_crc32_8_sw(&crc, &buf, &len);
}
return(~crc);
}
#endif /* INNODB_BUG_ENDIAN_CRC32 */
/********************************************************************//**
Initializes the data structures used by ut_crc32*(). Does not do any
......@@ -736,8 +700,9 @@ ut_crc32_init()
{
ut_crc32_slice8_table_init();
ut_crc32 = ut_crc32_sw;
#ifdef INNODB_BUG_ENDIAN_CRC32
ut_crc32_legacy_big_endian = ut_crc32_legacy_big_endian_sw;
ut_crc32_byte_by_byte = ut_crc32_byte_by_byte_sw;
#endif /* INNODB_BUG_ENDIAN_CRC32 */
ut_crc32_implementation = "Using generic crc32 instructions";
#if (defined(__GNUC__) && defined(__x86_64__)) || defined(_MSC_VER)
......@@ -770,8 +735,9 @@ ut_crc32_init()
if (features_ecx & 1 << 20) {
ut_crc32 = ut_crc32_hw;
#ifdef INNODB_BUG_ENDIAN_CRC32
ut_crc32_legacy_big_endian = ut_crc32_legacy_big_endian_hw;
ut_crc32_byte_by_byte = ut_crc32_byte_by_byte_hw;
#endif /* INNODB_BUG_ENDIAN_CRC32 */
ut_crc32_implementation = "Using SSE2 crc32 instructions";
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7