Skip to content

M
MariaDB
Commit 62d28f83 authored by Marko Mäkelä's avatar Marko Mäkelä
Browse files
Options

MDEV-17958: Remove WITH_INNODB_BUG_ENDIAN_CRC32 

Remove the bug-compatible crc32 algorithm variant that was added
to allow an upgrade from data files from big-endian systems where
innodb_checksum_algorithm=crc32 was used on MySQL 5.6
or MariaDB 10.0 or 10.1.

Affected users should be able to recompute page checksums using
innochecksum.
parent 6dbc50a3
Hide whitespace changes
Inline Side-by-side
Showing with 10 additions and 241 deletions
storage/innobase/buf/buf0buf.cc
View file @ 62d28f83
......@@ -770,11 +770,7 @@ buf_page_is_checksum_valid_crc32(
return false;
}
return checksum_field1 == crc32
#ifdef INNODB_BUG_ENDIAN_CRC32
|| checksum_field1 == buf_calc_page_crc32(read_buf, true)
#endif
;
return checksum_field1 == crc32;
}
/** Checks if the page is in innodb checksum format.
......@@ -903,29 +899,6 @@ buf_page_is_checksum_valid_none(
&& checksum_field1 == BUF_NO_CHECKSUM_MAGIC);
}
#ifdef INNODB_BUG_ENDIAN_CRC32
/** Validate the CRC-32C checksum of a page.
@param[in] page buffer page (srv_page_size bytes)
@param[in] checksum CRC-32C checksum stored on page
@return computed checksum */
static uint32_t buf_page_check_crc32(const byte* page, uint32_t checksum)
{
uint32_t crc32 = buf_calc_page_crc32(page);
if (checksum != crc32) {
crc32 = buf_calc_page_crc32(page, true);
}
return crc32;
}
#else /* INNODB_BUG_ENDIAN_CRC32 */
/** Validate the CRC-32C checksum of a page.
@param[in] page buffer page (srv_page_size bytes)
@param[in] checksum CRC-32C checksum stored on page
@return computed checksum */
# define buf_page_check_crc32(page, checksum) buf_calc_page_crc32(page)
#endif /* INNODB_BUG_ENDIAN_CRC32 */
/** Check if a page is corrupt.
@param[in] check_lsn whether the LSN should be checked
@param[in] read_buf database page
......@@ -1104,8 +1077,7 @@ buf_page_is_corrupted(
if (srv_checksum_algorithm
== SRV_CHECKSUM_ALGORITHM_CRC32) {
crc32 = buf_page_check_crc32(read_buf,
checksum_field2);
crc32 = buf_calc_page_crc32(read_buf);
crc32_inited = true;
if (checksum_field2 != crc32
......@@ -1119,8 +1091,7 @@ buf_page_is_corrupted(
if (checksum_field2
!= buf_calc_page_old_checksum(read_buf)) {
crc32 = buf_page_check_crc32(
read_buf, checksum_field2);
crc32 = buf_calc_page_crc32(read_buf);
crc32_inited = true;
if (checksum_field2 != crc32) {
......@@ -1136,8 +1107,7 @@ buf_page_is_corrupted(
== SRV_CHECKSUM_ALGORITHM_CRC32) {
if (!crc32_inited) {
crc32 = buf_page_check_crc32(
read_buf, checksum_field2);
crc32 = buf_calc_page_crc32(read_buf);
crc32_inited = true;
}
......@@ -1154,8 +1124,7 @@ buf_page_is_corrupted(
!= buf_calc_page_new_checksum(read_buf)) {
if (!crc32_inited) {
crc32 = buf_page_check_crc32(
read_buf, checksum_field2);
crc32 = buf_calc_page_crc32(read_buf);
crc32_inited = true;
}
......@@ -1262,12 +1231,6 @@ buf_page_print(const byte* read_buf, const page_size_t& page_size)
<< page_zip_calc_checksum(
read_buf, page_size.physical(),
SRV_CHECKSUM_ALGORITHM_CRC32)
#ifdef INNODB_BUG_ENDIAN_CRC32
<< "/"
<< page_zip_calc_checksum(
read_buf, page_size.physical(),
SRV_CHECKSUM_ALGORITHM_CRC32, true)
#endif
<< ", "
<< buf_checksum_algorithm_name(
SRV_CHECKSUM_ALGORITHM_INNODB)
......@@ -1293,10 +1256,6 @@ buf_page_print(const byte* read_buf, const page_size_t& page_size)
} else {
const uint32_t crc32 = buf_calc_page_crc32(read_buf);
#ifdef INNODB_BUG_ENDIAN_CRC32
const uint32_t crc32_legacy = buf_calc_page_crc32(read_buf,
true);
#endif /* INNODB_BUG_ENDIAN_CRC32 */
ulint page_type = fil_page_get_type(read_buf);
ib::info() << "Uncompressed page, stored checksum in field1 "
......@@ -1306,9 +1265,6 @@ buf_page_print(const byte* read_buf, const page_size_t& page_size)
<< buf_checksum_algorithm_name(
SRV_CHECKSUM_ALGORITHM_CRC32) << " "
<< crc32
#ifdef INNODB_BUG_ENDIAN_CRC32
<< "/" << crc32_legacy
#endif
<< ", "
<< buf_checksum_algorithm_name(
SRV_CHECKSUM_ALGORITHM_INNODB) << " "
......@@ -1326,9 +1282,6 @@ buf_page_print(const byte* read_buf, const page_size_t& page_size)
<< buf_checksum_algorithm_name(
SRV_CHECKSUM_ALGORITHM_CRC32) << " "
<< crc32
#ifdef INNODB_BUG_ENDIAN_CRC32
<< "/" << crc32_legacy
#endif
<< ", "
<< buf_checksum_algorithm_name(
SRV_CHECKSUM_ALGORITHM_INNODB) << " "
......@@ -3965,12 +3918,6 @@ buf_zip_decompress(
<< ", crc32: "
<< page_zip_calc_checksum(
frame, size, SRV_CHECKSUM_ALGORITHM_CRC32)
#ifdef INNODB_BUG_ENDIAN_CRC32
<< "/"
<< page_zip_calc_checksum(
frame, size, SRV_CHECKSUM_ALGORITHM_CRC32,
true)
#endif
<< " innodb: "
<< page_zip_calc_checksum(
frame, size, SRV_CHECKSUM_ALGORITHM_INNODB)
......
storage/innobase/buf/buf0checksum.cc
View file @ 62d28f83
......@@ -39,35 +39,6 @@ ha_innodb.cc:12251: error: cannot convert 'srv_checksum_algorithm_t*' to
'long unsigned int*' in initialization */
ulong srv_checksum_algorithm = SRV_CHECKSUM_ALGORITHM_INNODB;
#ifdef INNODB_BUG_ENDIAN_CRC32
/** Calculate the CRC32 checksum of a page. The value is stored to the page
when it is written to a file and also checked for a match when reading from
the file. Note that we must be careful to calculate the same value on all
architectures.
@param[in] page buffer page (srv_page_size bytes)
@param[in] bug_endian whether to use big endian byteorder
when converting byte strings to integers, for bug-compatibility with
big-endian architecture running MySQL 5.6, MariaDB 10.0 or MariaDB 10.1
@return CRC-32C */
uint32_t buf_calc_page_crc32(const byte* page, bool bug_endian)
{
return bug_endian
? ut_crc32_legacy_big_endian(
page + FIL_PAGE_OFFSET,
FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION
- FIL_PAGE_OFFSET)
^ ut_crc32_legacy_big_endian(page + FIL_PAGE_DATA,
srv_page_size
- (FIL_PAGE_DATA
+ FIL_PAGE_END_LSN_OLD_CHKSUM))
: ut_crc32(page + FIL_PAGE_OFFSET,
FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION
- FIL_PAGE_OFFSET)
^ ut_crc32(page + FIL_PAGE_DATA,
srv_page_size
- (FIL_PAGE_DATA + FIL_PAGE_END_LSN_OLD_CHKSUM));
}
#else
/** Calculate the CRC32 checksum of a page. The value is stored to the page
when it is written to a file and also checked for a match when reading from
the file. Note that we must be careful to calculate the same value on all
......@@ -88,7 +59,6 @@ uint32_t buf_calc_page_crc32(const byte* page)
srv_page_size
- (FIL_PAGE_DATA + FIL_PAGE_END_LSN_OLD_CHKSUM));
}
#endif
/** Calculate a checksum which is stored to the page when it is written
to a file. Note that we must be careful to calculate the same value on
......
storage/innobase/fil/fil0crypt.cc
View file @ 62d28f83
......@@ -2674,9 +2674,6 @@ fil_space_verify_crypt_checksum(
<< "If unencrypted: stored checksum [" << checksum1
<< ":" << checksum2 << "] calculated crc32 ["
<< buf_calc_page_crc32(page)
# ifdef INNODB_BUG_ENDIAN_CRC32
<< ":" << buf_calc_page_crc32(page, true)
# endif /* INNODB_BUG_ENDIAN_CRC32 */
<< "] innodb ["
<< buf_calc_page_old_checksum(page) << ":"
<< buf_calc_page_new_checksum(page) << "] LSN "
......
storage/innobase/include/buf0checksum.h
View file @ 62d28f83
......@@ -29,18 +29,6 @@ Created Aug 11, 2011 Vasil Dimov
#include "buf0types.h"
#ifdef INNODB_BUG_ENDIAN_CRC32
/** Calculate the CRC32 checksum of a page. The value is stored to the page
when it is written to a file and also checked for a match when reading from
the file. Note that we must be careful to calculate the same value on all
architectures.
@param[in] page buffer page (srv_page_size bytes)
@param[in] bug_endian whether to use big endian byteorder
when converting byte strings to integers, for bug-compatibility with
big-endian architecture running MySQL 5.6, MariaDB 10.0 or MariaDB 10.1
@return CRC-32C */
uint32_t buf_calc_page_crc32(const byte* page, bool bug_endian = false);
#else
/** Calculate the CRC32 checksum of a page. The value is stored to the page
when it is written to a file and also checked for a match when reading from
the file. Note that we must be careful to calculate the same value on all
......@@ -48,7 +36,6 @@ architectures.
@param[in] page buffer page (srv_page_size bytes)
@return CRC-32C */
uint32_t buf_calc_page_crc32(const byte* page);
#endif
/** Calculate a checksum which is stored to the page when it is written
to a file. Note that we must be careful to calculate the same value on
......
storage/innobase/include/page0zip.h
View file @ 62d28f83
......@@ -498,12 +498,7 @@ uint32_t
page_zip_calc_checksum(
const void* data,
ulint size,
srv_checksum_algorithm_t algo
#ifdef INNODB_BUG_ENDIAN_CRC32
/** for crc32, use the big-endian bug-compatible crc32 variant */
, bool use_legacy_big_endian = false
#endif
);
srv_checksum_algorithm_t algo);
/**********************************************************************//**
Verify a compressed page's checksum.
......
storage/innobase/include/ut0crc32.h
View file @ 62d28f83
......@@ -47,12 +47,6 @@ typedef uint32_t (*ut_crc32_func_t)(const byte* ptr, ulint len);
/** Pointer to CRC32 calculation function. */
extern ut_crc32_func_t ut_crc32;
#ifdef INNODB_BUG_ENDIAN_CRC32
/** Pointer to CRC32 calculation function, which uses big-endian byte order
when converting byte strings to integers internally. */
extern uint32_t ut_crc32_legacy_big_endian(const byte* buf, ulint len);
#endif /* INNODB_BUG_ENDIAN_CRC32 */
/** Text description of CRC32 implementation */
extern const char* ut_crc32_implementation;
......
storage/innobase/innodb.cmake
View file @ 62d28f83
......@@ -121,11 +121,6 @@ ELSEIF(WITH_INNODB_ROOT_GUESS)
ADD_DEFINITIONS(-DBTR_CUR_ADAPT)
ENDIF()
OPTION(WITH_INNODB_BUG_ENDIAN_CRC32 "Weaken innodb_checksum_algorithm=crc32 by supporting upgrade from big-endian systems running 5.6/10.0/10.1" ${IS_BIG_ENDIAN})
IF(WITH_INNODB_BUG_ENDIAN_CRC32)
ADD_DEFINITIONS(-DINNODB_BUG_ENDIAN_CRC32)
ENDIF()
OPTION(WITH_INNODB_EXTRA_DEBUG "Enable extra InnoDB debug checks" OFF)
IF(WITH_INNODB_EXTRA_DEBUG)
ADD_DEFINITIONS(-DUNIV_ZIP_DEBUG)
......
storage/innobase/page/page0zip.cc
View file @ 62d28f83
......@@ -4930,12 +4930,7 @@ uint32_t
page_zip_calc_checksum(
const void* data,
ulint size,
srv_checksum_algorithm_t algo
#ifdef INNODB_BUG_ENDIAN_CRC32
/** for crc32, use the big-endian bug-compatible crc32 variant */
, bool use_legacy_big_endian
#endif
)
srv_checksum_algorithm_t algo)
{
uLong adler;
const Bytef* s = static_cast<const byte*>(data);
......@@ -4947,19 +4942,6 @@ page_zip_calc_checksum(
case SRV_CHECKSUM_ALGORITHM_CRC32:
case SRV_CHECKSUM_ALGORITHM_STRICT_CRC32:
ut_ad(size > FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID);
#ifdef INNODB_BUG_ENDIAN_CRC32
if (use_legacy_big_endian) {
return ut_crc32_legacy_big_endian(s + FIL_PAGE_OFFSET,
FIL_PAGE_LSN
- FIL_PAGE_OFFSET)
^ ut_crc32_legacy_big_endian(
s + FIL_PAGE_TYPE, 2)
^ ut_crc32_legacy_big_endian(
s + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID,
size
- FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID);
}
#endif
return ut_crc32(s + FIL_PAGE_OFFSET,
FIL_PAGE_LSN - FIL_PAGE_OFFSET)
^ ut_crc32(s + FIL_PAGE_TYPE, 2)
......@@ -5077,11 +5059,6 @@ page_zip_verify_checksum(
switch (curr_algo) {
case SRV_CHECKSUM_ALGORITHM_STRICT_CRC32:
#ifdef INNODB_BUG_ENDIAN_CRC32
return stored == page_zip_calc_checksum(data, size, curr_algo,
true);
#endif
/* fall through */
case SRV_CHECKSUM_ALGORITHM_STRICT_INNODB:
case SRV_CHECKSUM_ALGORITHM_STRICT_NONE:
return FALSE;
......@@ -5090,26 +5067,15 @@ page_zip_verify_checksum(
return(TRUE);
}
return
#ifdef INNODB_BUG_ENDIAN_CRC32
stored == page_zip_calc_checksum(data, size, curr_algo,
true) ||
#endif
stored == page_zip_calc_checksum(
data, size, SRV_CHECKSUM_ALGORITHM_INNODB);
return stored == page_zip_calc_checksum(
data, size, SRV_CHECKSUM_ALGORITHM_INNODB);
case SRV_CHECKSUM_ALGORITHM_INNODB:
if (stored == BUF_NO_CHECKSUM_MAGIC) {
return TRUE;
}
return stored == page_zip_calc_checksum(
data, size, SRV_CHECKSUM_ALGORITHM_CRC32)
#ifdef INNODB_BUG_ENDIAN_CRC32
|| stored == page_zip_calc_checksum(
data, size,
SRV_CHECKSUM_ALGORITHM_CRC32, true)
#endif
;
data, size, SRV_CHECKSUM_ALGORITHM_CRC32);
case SRV_CHECKSUM_ALGORITHM_NONE:
return TRUE;
}
......
storage/innobase/ut/ut0crc32.cc
View file @ 62d28f83
......@@ -469,34 +469,6 @@ ut_crc32_64_sw(
*len -= 8;
}
#ifdef INNODB_BUG_ENDIAN_CRC32
/** Calculate CRC32 over 64-bit byte string using a software implementation.
The byte string is converted to a 64-bit integer using big endian byte order.
@param[in,out] crc crc32 checksum so far when this function is called,
when the function ends it will contain the new checksum
@param[in,out] data data to be checksummed, the pointer will be advanced
with 8 bytes
@param[in,out] len remaining bytes, it will be decremented with 8 */
inline
void
ut_crc32_64_legacy_big_endian_sw(
uint32_t* crc,
const byte** data,
ulint* len)
{
uint64_t data_int = *reinterpret_cast<const uint64_t*>(*data);
#ifndef WORDS_BIGENDIAN
data_int = ut_crc32_swap_byteorder(data_int);
#endif /* WORDS_BIGENDIAN */
*crc = ut_crc32_64_low_sw(*crc, data_int);
*data += 8;
*len -= 8;
}
#endif /* INNODB_BUG_ENDIAN_CRC32 */
/** Calculates CRC32 in software, without using CPU instructions.
@param[in] buf data over which to calculate CRC32
@param[in] len data length
......@@ -547,57 +519,6 @@ ut_crc32_sw(
return(~crc);
}
#ifdef INNODB_BUG_ENDIAN_CRC32
/** Calculates CRC32 in software, without using CPU instructions.
This function uses big endian byte ordering when converting byte sequence to
integers.
@param[in] buf data over which to calculate CRC32
@param[in] len data length
@return CRC-32C (polynomial 0x11EDC6F41) */
uint32_t ut_crc32_legacy_big_endian(const byte* buf, ulint len)
{
uint32_t crc = 0xFFFFFFFFU;
ut_a(ut_crc32_slice8_table_initialized);
/* Calculate byte-by-byte up to an 8-byte aligned address. After
this consume the input 8-bytes at a time. */
while (len > 0 && (reinterpret_cast<uintptr_t>(buf) & 7) != 0) {
ut_crc32_8_sw(&crc, &buf, &len);
}
while (len >= 128) {
/* This call is repeated 16 times. 16 * 8 = 128. */
ut_crc32_64_legacy_big_endian_sw(&crc, &buf, &len);
ut_crc32_64_legacy_big_endian_sw(&crc, &buf, &len);
ut_crc32_64_legacy_big_endian_sw(&crc, &buf, &len);
ut_crc32_64_legacy_big_endian_sw(&crc, &buf, &len);
ut_crc32_64_legacy_big_endian_sw(&crc, &buf, &len);
ut_crc32_64_legacy_big_endian_sw(&crc, &buf, &len);
ut_crc32_64_legacy_big_endian_sw(&crc, &buf, &len);
ut_crc32_64_legacy_big_endian_sw(&crc, &buf, &len);
ut_crc32_64_legacy_big_endian_sw(&crc, &buf, &len);
ut_crc32_64_legacy_big_endian_sw(&crc, &buf, &len);
ut_crc32_64_legacy_big_endian_sw(&crc, &buf, &len);
ut_crc32_64_legacy_big_endian_sw(&crc, &buf, &len);
ut_crc32_64_legacy_big_endian_sw(&crc, &buf, &len);
ut_crc32_64_legacy_big_endian_sw(&crc, &buf, &len);
ut_crc32_64_legacy_big_endian_sw(&crc, &buf, &len);
ut_crc32_64_legacy_big_endian_sw(&crc, &buf, &len);
}
while (len >= 8) {
ut_crc32_64_legacy_big_endian_sw(&crc, &buf, &len);
}
while (len > 0) {
ut_crc32_8_sw(&crc, &buf, &len);
}
return(~crc);
}
#endif /* INNODB_BUG_ENDIAN_CRC32 */
/********************************************************************//**
Initializes the data structures used by ut_crc32*(). Does not do any
allocations, would not hurt if called twice, but would be pointless. */
......@@ -637,9 +558,6 @@ ut_crc32_init()
if (features_ecx & 1 << 20) {
ut_crc32 = ut_crc32_hw;
#ifdef INNODB_BUG_ENDIAN_CRC32
ut_crc32_legacy_big_endian = ut_crc32_legacy_big_endian_hw;
#endif /* INNODB_BUG_ENDIAN_CRC32 */
ut_crc32_implementation = "Using SSE2 crc32 instructions";
}
#endif
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7