MDEV-17958: Remove WITH_INNODB_BUG_ENDIAN_CRC32

Remove the bug-compatible crc32 algorithm variant that was added
to allow an upgrade from data files from big-endian systems where
innodb_checksum_algorithm=crc32 was used on MySQL 5.6
or MariaDB 10.0 or 10.1.

Affected users should be able to recompute page checksums using
innochecksum.

storage/innobase/buf/buf0buf.cc

@@ -770,11 +770,7 @@ buf_page_is_checksum_valid_crc32(
 		return false;
 	}
 
-	return checksum_field1 == crc32
-#ifdef INNODB_BUG_ENDIAN_CRC32
-		|| checksum_field1 == buf_calc_page_crc32(read_buf, true)
-#endif
-		;
+	return checksum_field1 == crc32;
 }
 
 /** Checks if the page is in innodb checksum format.
@@ -903,29 +899,6 @@ buf_page_is_checksum_valid_none(
 	       && checksum_field1 == BUF_NO_CHECKSUM_MAGIC);
 }
 
-#ifdef INNODB_BUG_ENDIAN_CRC32
-/** Validate the CRC-32C checksum of a page.
-@param[in]	page		buffer page (srv_page_size bytes)
-@param[in]	checksum	CRC-32C checksum stored on page
-@return	computed checksum */
-static uint32_t buf_page_check_crc32(const byte* page, uint32_t checksum)
-{
-	uint32_t crc32 = buf_calc_page_crc32(page);
-
-	if (checksum != crc32) {
-		crc32 = buf_calc_page_crc32(page, true);
-	}
-
-	return crc32;
-}
-#else /* INNODB_BUG_ENDIAN_CRC32 */
-/** Validate the CRC-32C checksum of a page.
-@param[in]	page		buffer page (srv_page_size bytes)
-@param[in]	checksum	CRC-32C checksum stored on page
-@return	computed checksum */
-# define buf_page_check_crc32(page, checksum) buf_calc_page_crc32(page)
-#endif /* INNODB_BUG_ENDIAN_CRC32 */
-
 /** Check if a page is corrupt.
 @param[in]	check_lsn	whether the LSN should be checked
 @param[in]	read_buf	database page
@@ -1104,8 +1077,7 @@ buf_page_is_corrupted(
 
 			if (srv_checksum_algorithm
 			    == SRV_CHECKSUM_ALGORITHM_CRC32) {
-				crc32 = buf_page_check_crc32(read_buf,
-							     checksum_field2);
+				crc32 = buf_calc_page_crc32(read_buf);
 				crc32_inited = true;
 
 				if (checksum_field2 != crc32
@@ -1119,8 +1091,7 @@ buf_page_is_corrupted(
 
 				if (checksum_field2
 				    != buf_calc_page_old_checksum(read_buf)) {
-					crc32 = buf_page_check_crc32(
-						read_buf, checksum_field2);
+					crc32 = buf_calc_page_crc32(read_buf);
 					crc32_inited = true;
 
 					if (checksum_field2 != crc32) {
@@ -1136,8 +1107,7 @@ buf_page_is_corrupted(
 			   == SRV_CHECKSUM_ALGORITHM_CRC32) {
 
 			if (!crc32_inited) {
-				crc32 = buf_page_check_crc32(
-					read_buf, checksum_field2);
+				crc32 = buf_calc_page_crc32(read_buf);
 				crc32_inited = true;
 			}
 
@@ -1154,8 +1124,7 @@ buf_page_is_corrupted(
 			    != buf_calc_page_new_checksum(read_buf)) {
 
 				if (!crc32_inited) {
-					crc32 = buf_page_check_crc32(
-						read_buf, checksum_field2);
+					crc32 = buf_calc_page_crc32(read_buf);
 					crc32_inited = true;
 				}
 
@@ -1262,12 +1231,6 @@ buf_page_print(const byte* read_buf, const page_size_t& page_size)
 			<< page_zip_calc_checksum(
 				read_buf, page_size.physical(),
 				SRV_CHECKSUM_ALGORITHM_CRC32)
-#ifdef INNODB_BUG_ENDIAN_CRC32
-			<< "/"
-			<< page_zip_calc_checksum(
-				read_buf, page_size.physical(),
-				SRV_CHECKSUM_ALGORITHM_CRC32, true)
-#endif
 			<< ", "
 			<< buf_checksum_algorithm_name(
 				SRV_CHECKSUM_ALGORITHM_INNODB)
@@ -1293,10 +1256,6 @@ buf_page_print(const byte* read_buf, const page_size_t& page_size)
 
 	} else {
 		const uint32_t	crc32 = buf_calc_page_crc32(read_buf);
-#ifdef INNODB_BUG_ENDIAN_CRC32
-		const uint32_t	crc32_legacy = buf_calc_page_crc32(read_buf,
-								   true);
-#endif /* INNODB_BUG_ENDIAN_CRC32 */
 		ulint page_type = fil_page_get_type(read_buf);
 
 		ib::info() << "Uncompressed page, stored checksum in field1 "
@@ -1306,9 +1265,6 @@ buf_page_print(const byte* read_buf, const page_size_t& page_size)
 			<< buf_checksum_algorithm_name(
 				SRV_CHECKSUM_ALGORITHM_CRC32) << " "
 			<< crc32
-#ifdef INNODB_BUG_ENDIAN_CRC32
-			<< "/" << crc32_legacy
-#endif
 			<< ", "
 			<< buf_checksum_algorithm_name(
 				SRV_CHECKSUM_ALGORITHM_INNODB) << " "
@@ -1326,9 +1282,6 @@ buf_page_print(const byte* read_buf, const page_size_t& page_size)
 			<< buf_checksum_algorithm_name(
 				SRV_CHECKSUM_ALGORITHM_CRC32) << " "
 			<< crc32
-#ifdef INNODB_BUG_ENDIAN_CRC32
-			<< "/" << crc32_legacy
-#endif
 			<< ", "
 			<< buf_checksum_algorithm_name(
 				SRV_CHECKSUM_ALGORITHM_INNODB) << " "
@@ -3965,12 +3918,6 @@ buf_zip_decompress(
 			<< ", crc32: "
 			<< page_zip_calc_checksum(
 				frame, size, SRV_CHECKSUM_ALGORITHM_CRC32)
-#ifdef INNODB_BUG_ENDIAN_CRC32
-			<< "/"
-			<< page_zip_calc_checksum(
-				frame, size, SRV_CHECKSUM_ALGORITHM_CRC32,
-				true)
-#endif
 			<< " innodb: "
 			<< page_zip_calc_checksum(
 				frame, size, SRV_CHECKSUM_ALGORITHM_INNODB)

storage/innobase/buf/buf0checksum.cc

@@ -39,35 +39,6 @@ ha_innodb.cc:12251: error: cannot convert 'srv_checksum_algorithm_t*' to
   'long unsigned int*' in initialization */
 ulong	srv_checksum_algorithm = SRV_CHECKSUM_ALGORITHM_INNODB;
 
-#ifdef INNODB_BUG_ENDIAN_CRC32
-/** Calculate the CRC32 checksum of a page. The value is stored to the page
-when it is written to a file and also checked for a match when reading from
-the file. Note that we must be careful to calculate the same value on all
-architectures.
-@param[in]	page		buffer page (srv_page_size bytes)
-@param[in]	bug_endian	whether to use big endian byteorder
-when converting byte strings to integers, for bug-compatibility with
-big-endian architecture running MySQL 5.6, MariaDB 10.0 or MariaDB 10.1
-@return	CRC-32C */
-uint32_t buf_calc_page_crc32(const byte* page, bool bug_endian)
-{
-	return bug_endian
-		? ut_crc32_legacy_big_endian(
-			page + FIL_PAGE_OFFSET,
-			FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION
-			- FIL_PAGE_OFFSET)
-		^ ut_crc32_legacy_big_endian(page + FIL_PAGE_DATA,
-					     srv_page_size
-					     - (FIL_PAGE_DATA
-						+ FIL_PAGE_END_LSN_OLD_CHKSUM))
-		: ut_crc32(page + FIL_PAGE_OFFSET,
-			   FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION
-			   - FIL_PAGE_OFFSET)
-		^ ut_crc32(page + FIL_PAGE_DATA,
-			   srv_page_size
-			   - (FIL_PAGE_DATA + FIL_PAGE_END_LSN_OLD_CHKSUM));
-}
-#else
 /** Calculate the CRC32 checksum of a page. The value is stored to the page
 when it is written to a file and also checked for a match when reading from
 the file. Note that we must be careful to calculate the same value on all
@@ -88,7 +59,6 @@ uint32_t buf_calc_page_crc32(const byte* page)
 			   srv_page_size
 			   - (FIL_PAGE_DATA + FIL_PAGE_END_LSN_OLD_CHKSUM));
 }
-#endif
 
 /** Calculate a checksum which is stored to the page when it is written
 to a file. Note that we must be careful to calculate the same value on

storage/innobase/fil/fil0crypt.cc

@@ -2674,9 +2674,6 @@ fil_space_verify_crypt_checksum(
 			<< "If unencrypted: stored checksum [" << checksum1
 			<< ":" << checksum2 << "] calculated crc32 ["
 			<< buf_calc_page_crc32(page)
-# ifdef INNODB_BUG_ENDIAN_CRC32
-			<< ":" << buf_calc_page_crc32(page, true)
-# endif /* INNODB_BUG_ENDIAN_CRC32 */
 			<< "] innodb ["
 			<< buf_calc_page_old_checksum(page) << ":"
 			<< buf_calc_page_new_checksum(page) << "] LSN "

storage/innobase/include/buf0checksum.h

@@ -29,18 +29,6 @@ Created Aug 11, 2011 Vasil Dimov
 
 #include "buf0types.h"
 
-#ifdef INNODB_BUG_ENDIAN_CRC32
-/** Calculate the CRC32 checksum of a page. The value is stored to the page
-when it is written to a file and also checked for a match when reading from
-the file. Note that we must be careful to calculate the same value on all
-architectures.
-@param[in]	page		buffer page (srv_page_size bytes)
-@param[in]	bug_endian	whether to use big endian byteorder
-when converting byte strings to integers, for bug-compatibility with
-big-endian architecture running MySQL 5.6, MariaDB 10.0 or MariaDB 10.1
-@return	CRC-32C */
-uint32_t buf_calc_page_crc32(const byte* page, bool bug_endian = false);
-#else
 /** Calculate the CRC32 checksum of a page. The value is stored to the page
 when it is written to a file and also checked for a match when reading from
 the file. Note that we must be careful to calculate the same value on all
@@ -48,7 +36,6 @@ architectures.
 @param[in]	page			buffer page (srv_page_size bytes)
 @return	CRC-32C */
 uint32_t buf_calc_page_crc32(const byte* page);
-#endif
 
 /** Calculate a checksum which is stored to the page when it is written
 to a file. Note that we must be careful to calculate the same value on

storage/innobase/include/page0zip.h

@@ -498,12 +498,7 @@ uint32_t
 page_zip_calc_checksum(
 	const void*			data,
 	ulint				size,
-	srv_checksum_algorithm_t	algo
-#ifdef INNODB_BUG_ENDIAN_CRC32
-	/** for crc32, use the big-endian bug-compatible crc32 variant */
-	, bool				use_legacy_big_endian = false
-#endif
-);
+	srv_checksum_algorithm_t	algo);
 
 /**********************************************************************//**
 Verify a compressed page's checksum.

storage/innobase/include/ut0crc32.h

@@ -47,12 +47,6 @@ typedef uint32_t	(*ut_crc32_func_t)(const byte* ptr, ulint len);
 /** Pointer to CRC32 calculation function. */
 extern ut_crc32_func_t	ut_crc32;
 
-#ifdef INNODB_BUG_ENDIAN_CRC32
-/** Pointer to CRC32 calculation function, which uses big-endian byte order
-when converting byte strings to integers internally. */
-extern uint32_t ut_crc32_legacy_big_endian(const byte* buf, ulint len);
-#endif /* INNODB_BUG_ENDIAN_CRC32 */
-
 /** Text description of CRC32 implementation */
 extern const char*	ut_crc32_implementation;
 

storage/innobase/innodb.cmake

@@ -121,11 +121,6 @@ ELSEIF(WITH_INNODB_ROOT_GUESS)
   ADD_DEFINITIONS(-DBTR_CUR_ADAPT)
 ENDIF()
 
-OPTION(WITH_INNODB_BUG_ENDIAN_CRC32 "Weaken innodb_checksum_algorithm=crc32 by supporting upgrade from big-endian systems running 5.6/10.0/10.1" ${IS_BIG_ENDIAN})
-IF(WITH_INNODB_BUG_ENDIAN_CRC32)
-  ADD_DEFINITIONS(-DINNODB_BUG_ENDIAN_CRC32)
-ENDIF()
-
 OPTION(WITH_INNODB_EXTRA_DEBUG "Enable extra InnoDB debug checks" OFF)
 IF(WITH_INNODB_EXTRA_DEBUG)
   ADD_DEFINITIONS(-DUNIV_ZIP_DEBUG)

storage/innobase/page/page0zip.cc

@@ -4930,12 +4930,7 @@ uint32_t
 page_zip_calc_checksum(
 	const void*			data,
 	ulint				size,
-	srv_checksum_algorithm_t	algo
-#ifdef INNODB_BUG_ENDIAN_CRC32
-	/** for crc32, use the big-endian bug-compatible crc32 variant */
-	, bool				use_legacy_big_endian
-#endif
-)
+	srv_checksum_algorithm_t	algo)
 {
 	uLong		adler;
 	const Bytef*	s = static_cast<const byte*>(data);
@@ -4947,19 +4942,6 @@ page_zip_calc_checksum(
 	case SRV_CHECKSUM_ALGORITHM_CRC32:
 	case SRV_CHECKSUM_ALGORITHM_STRICT_CRC32:
 		ut_ad(size > FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID);
-#ifdef INNODB_BUG_ENDIAN_CRC32
-		if (use_legacy_big_endian) {
-			return ut_crc32_legacy_big_endian(s + FIL_PAGE_OFFSET,
-							  FIL_PAGE_LSN
-							  - FIL_PAGE_OFFSET)
-				^ ut_crc32_legacy_big_endian(
-					s + FIL_PAGE_TYPE, 2)
-				^ ut_crc32_legacy_big_endian(
-					s + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID,
-					size
-					- FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID);
-		}
-#endif
 		return ut_crc32(s + FIL_PAGE_OFFSET,
 				FIL_PAGE_LSN - FIL_PAGE_OFFSET)
 			^ ut_crc32(s + FIL_PAGE_TYPE, 2)
@@ -5077,11 +5059,6 @@ page_zip_verify_checksum(
 
 	switch (curr_algo) {
 	case SRV_CHECKSUM_ALGORITHM_STRICT_CRC32:
-#ifdef INNODB_BUG_ENDIAN_CRC32
-		return stored == page_zip_calc_checksum(data, size, curr_algo,
-							true);
-#endif
-		/* fall through */
 	case SRV_CHECKSUM_ALGORITHM_STRICT_INNODB:
 	case SRV_CHECKSUM_ALGORITHM_STRICT_NONE:
 		return FALSE;
@@ -5090,26 +5067,15 @@ page_zip_verify_checksum(
 			return(TRUE);
 		}
 
-		return
-#ifdef INNODB_BUG_ENDIAN_CRC32
-			stored == page_zip_calc_checksum(data, size, curr_algo,
-							 true) ||
-#endif
-			stored == page_zip_calc_checksum(
-				data, size, SRV_CHECKSUM_ALGORITHM_INNODB);
+		return stored == page_zip_calc_checksum(
+			data, size, SRV_CHECKSUM_ALGORITHM_INNODB);
 	case SRV_CHECKSUM_ALGORITHM_INNODB:
 		if (stored == BUF_NO_CHECKSUM_MAGIC) {
 			return TRUE;
 		}
 
 		return stored == page_zip_calc_checksum(
-			data, size, SRV_CHECKSUM_ALGORITHM_CRC32)
-#ifdef INNODB_BUG_ENDIAN_CRC32
-			|| stored == page_zip_calc_checksum(
-				data, size,
-				SRV_CHECKSUM_ALGORITHM_CRC32, true)
-#endif
-			;
+			data, size, SRV_CHECKSUM_ALGORITHM_CRC32);
 	case SRV_CHECKSUM_ALGORITHM_NONE:
 		return TRUE;
 	}

storage/innobase/ut/ut0crc32.cc

@@ -469,34 +469,6 @@ ut_crc32_64_sw(
 	*len -= 8;
 }
 
-#ifdef INNODB_BUG_ENDIAN_CRC32
-/** Calculate CRC32 over 64-bit byte string using a software implementation.
-The byte string is converted to a 64-bit integer using big endian byte order.
-@param[in,out]	crc	crc32 checksum so far when this function is called,
-when the function ends it will contain the new checksum
-@param[in,out]	data	data to be checksummed, the pointer will be advanced
-with 8 bytes
-@param[in,out]	len	remaining bytes, it will be decremented with 8 */
-inline
-void
-ut_crc32_64_legacy_big_endian_sw(
-	uint32_t*	crc,
-	const byte**	data,
-	ulint*		len)
-{
-	uint64_t	data_int = *reinterpret_cast<const uint64_t*>(*data);
-
-#ifndef WORDS_BIGENDIAN
-	data_int = ut_crc32_swap_byteorder(data_int);
-#endif /* WORDS_BIGENDIAN */
-
-	*crc = ut_crc32_64_low_sw(*crc, data_int);
-
-	*data += 8;
-	*len -= 8;
-}
-#endif /* INNODB_BUG_ENDIAN_CRC32 */
-
 /** Calculates CRC32 in software, without using CPU instructions.
 @param[in]	buf	data over which to calculate CRC32
 @param[in]	len	data length
@@ -547,57 +519,6 @@ ut_crc32_sw(
 	return(~crc);
 }
 
-#ifdef INNODB_BUG_ENDIAN_CRC32
-/** Calculates CRC32 in software, without using CPU instructions.
-This function uses big endian byte ordering when converting byte sequence to
-integers.
-@param[in]	buf	data over which to calculate CRC32
-@param[in]	len	data length
-@return CRC-32C (polynomial 0x11EDC6F41) */
-uint32_t ut_crc32_legacy_big_endian(const byte* buf, ulint len)
-{
-	uint32_t	crc = 0xFFFFFFFFU;
-
-	ut_a(ut_crc32_slice8_table_initialized);
-
-	/* Calculate byte-by-byte up to an 8-byte aligned address. After
-	this consume the input 8-bytes at a time. */
-	while (len > 0 && (reinterpret_cast<uintptr_t>(buf) & 7) != 0) {
-		ut_crc32_8_sw(&crc, &buf, &len);
-	}
-
-	while (len >= 128) {
-		/* This call is repeated 16 times. 16 * 8 = 128. */
-		ut_crc32_64_legacy_big_endian_sw(&crc, &buf, &len);
-		ut_crc32_64_legacy_big_endian_sw(&crc, &buf, &len);
-		ut_crc32_64_legacy_big_endian_sw(&crc, &buf, &len);
-		ut_crc32_64_legacy_big_endian_sw(&crc, &buf, &len);
-		ut_crc32_64_legacy_big_endian_sw(&crc, &buf, &len);
-		ut_crc32_64_legacy_big_endian_sw(&crc, &buf, &len);
-		ut_crc32_64_legacy_big_endian_sw(&crc, &buf, &len);
-		ut_crc32_64_legacy_big_endian_sw(&crc, &buf, &len);
-		ut_crc32_64_legacy_big_endian_sw(&crc, &buf, &len);
-		ut_crc32_64_legacy_big_endian_sw(&crc, &buf, &len);
-		ut_crc32_64_legacy_big_endian_sw(&crc, &buf, &len);
-		ut_crc32_64_legacy_big_endian_sw(&crc, &buf, &len);
-		ut_crc32_64_legacy_big_endian_sw(&crc, &buf, &len);
-		ut_crc32_64_legacy_big_endian_sw(&crc, &buf, &len);
-		ut_crc32_64_legacy_big_endian_sw(&crc, &buf, &len);
-		ut_crc32_64_legacy_big_endian_sw(&crc, &buf, &len);
-	}
-
-	while (len >= 8) {
-		ut_crc32_64_legacy_big_endian_sw(&crc, &buf, &len);
-	}
-
-	while (len > 0) {
-		ut_crc32_8_sw(&crc, &buf, &len);
-	}
-
-	return(~crc);
-}
-#endif /* INNODB_BUG_ENDIAN_CRC32 */
-
 /********************************************************************//**
 Initializes the data structures used by ut_crc32*(). Does not do any
 allocations, would not hurt if called twice, but would be pointless. */
@@ -637,9 +558,6 @@ ut_crc32_init()
 
 	if (features_ecx & 1 << 20) {
 		ut_crc32 = ut_crc32_hw;
-#ifdef INNODB_BUG_ENDIAN_CRC32
-		ut_crc32_legacy_big_endian = ut_crc32_legacy_big_endian_hw;
-#endif /* INNODB_BUG_ENDIAN_CRC32 */
 		ut_crc32_implementation = "Using SSE2 crc32 instructions";
 	}
 #endif