Merge 10.5 into 10.6

cmake/os/Windows.cmake

@@ -68,17 +68,12 @@ MACRO(ENABLE_SANITIZERS)
     MESSAGE(FATAL_ERROR "clang-cl or MSVC necessary to enable asan/ubsan")
   ENDIF()
   # currently, asan is broken with static CRT.
-  IF(NOT(MSVC_CRT_TYPE STREQUAL "/MD"))
-    IF(NOT DYNAMIC_UCRT_LINK)
-      MESSAGE(FATAL_ERROR "-DWITH_ASAN cmake parameter also requires -DMSVC_CRT_TYPE=/MD OR DYNAMIC_UCRT_LINK=ON")
-    ENDIF()
+  IF(CLANG_CL AND NOT(MSVC_CRT_TYPE STREQUAL "/MD"))
+    SET(MSVC_CRT_TYPE "/MD" CACHE  INTERNAL "" FORCE)
   ENDIF()
   IF(CMAKE_SIZEOF_VOID_P EQUAL 4)
     SET(ASAN_ARCH i386)
   ELSE()
-    IF(NOT CLANG_CL)
-      MESSAGE(FATAL_ERROR "sanitizers do not yet work on MSVC x64, try 32  bit or clang-cl")
-    ENDIF()
     SET(ASAN_ARCH x86_64)
   ENDIF()
 
@@ -88,8 +83,10 @@ MACRO(ENABLE_SANITIZERS)
   SET(SANITIZER_LINK_LIBRARIES)
   SET(SANITIZER_COMPILE_FLAGS)
   IF(WITH_ASAN)
+    IF(CLANG_CL)
       LIST(APPEND SANITIZER_LIBS
         clang_rt.asan_dynamic-${ASAN_ARCH}.lib clang_rt.asan_dynamic_runtime_thunk-${ASAN_ARCH}.lib)
+    ENDIF()
     STRING(APPEND SANITIZER_COMPILE_FLAGS " -fsanitize=address")
   ENDIF()
   IF(WITH_UBSAN)
@@ -134,7 +131,13 @@ IF(MSVC)
    INCLUDE(InstallRequiredSystemLibraries)
   ENDIF()
 
-  OPTION(DYNAMIC_UCRT_LINK "Link Universal CRT dynamically, if MSVC_CRT_TYPE=/MT" ON)
+  IF(WITH_ASAN AND (NOT CLANG_CL))
+    SET(DYNAMIC_UCRT_LINK_DEFAULT OFF)
+  ELSE()
+    SET(DYNAMIC_UCRT_LINK_DEFAULT ON)
+  ENDIF()
+
+  OPTION(DYNAMIC_UCRT_LINK "Link Universal CRT dynamically, if MSVC_CRT_TYPE=/MT" ${DYNAMIC_UCRT_LINK_DEFAULT})
   SET(DYNAMIC_UCRT_LINKER_OPTION " /NODEFAULTLIB:libucrt.lib /DEFAULTLIB:ucrt.lib")
 
   # Enable debug info also in Release build,
@@ -191,8 +194,15 @@ IF(MSVC)
 
   FOREACH(type EXE SHARED MODULE)
    STRING(REGEX REPLACE "/STACK:([^ ]+)" "" CMAKE_${type}_LINKER_FLAGS "${CMAKE_${type}_LINKER_FLAGS}")
-   STRING(REGEX REPLACE "/INCREMENTAL:([^ ]+)" "/INCREMENTAL:NO" CMAKE_${type}_LINKER_FLAGS_RELWITHDEBINFO "${CMAKE_${type}_LINKER_FLAGS_RELWITHDEBINFO}")
-   STRING(REGEX REPLACE "/INCREMENTAL$" "/INCREMENTAL:NO" CMAKE_${type}_LINKER_FLAGS_RELWITHDEBINFO "${CMAKE_${type}_LINKER_FLAGS_RELWITHDEBINFO}")
+   IF(WITH_ASAN)
+     SET(build_types RELWITHDEBINFO DEBUG)
+   ELSE()
+     SET(build_types RELWITHDEBINFO)
+   ENDIF()
+   FOREACH(btype ${build_types})
+     STRING(REGEX REPLACE "/INCREMENTAL:([^ ]+)" "/INCREMENTAL:NO" CMAKE_${type}_LINKER_FLAGS_${btype} "${CMAKE_${type}_LINKER_FLAGS_${btype}}")
+     STRING(REGEX REPLACE "/INCREMENTAL$" "/INCREMENTAL:NO" CMAKE_${type}_LINKER_FLAGS_${btype} "${CMAKE_${type}_LINKER_FLAGS_${btype}}")
+   ENDFOREACH()
    IF(NOT CLANG_CL)
      STRING(APPEND CMAKE_${type}_LINKER_FLAGS_RELWITHDEBINFO " /release /OPT:REF,ICF")
    ENDIF()

cmake/pcre.cmake

@@ -36,8 +36,8 @@ MACRO(BUNDLE_PCRE2)
     IF(MSVC)
       # Suppress a warning
       STRING(APPEND pcre2_flags${v} " /wd4244 " )
-      # Need this only for ASAN support
-      SET(stdlibs "-DCMAKE_C_STANDARD_LIBRARIES=${CMAKE_C_STANDARD_LIBRARIES}")
+      # Disable asan support
+      STRING(REPLACE "-fsanitize=address" "" pcre2_flags${v} "${CMAKE_C_FLAGS${v}}")
     ENDIF()
   ENDFOREACH()
   ExternalProject_Add(

include/my_pthread.h

@@ -666,7 +666,11 @@ extern void my_mutex_end(void);
   with the current number of keys and key parts.
 */
 #if defined(__SANITIZE_ADDRESS__) || defined(WITH_UBSAN)
+#ifndef DBUG_OFF
+#define DEFAULT_THREAD_STACK	(1024*1024L)
+#else
 #define DEFAULT_THREAD_STACK	(383*1024L) /* 392192 */
+#endif
 #else
 #define DEFAULT_THREAD_STACK	(292*1024L) /* 299008 */
 #endif

include/my_valgrind.h

@@ -47,7 +47,7 @@
 # define MEM_GET_VBITS(a,b,len) VALGRIND_GET_VBITS(a,b,len)
 # define MEM_SET_VBITS(a,b,len) VALGRIND_SET_VBITS(a,b,len)
 # define REDZONE_SIZE 8
-#elif defined(__SANITIZE_ADDRESS__)
+#elif defined(__SANITIZE_ADDRESS__) && (!defined(_MSC_VER) || defined (__clang__))
 # include <sanitizer/asan_interface.h>
 /* How to do manual poisoning:
 https://github.com/google/sanitizers/wiki/AddressSanitizerManualPoisoning */

mysql-test/suite/innodb/r/log_file_name.result

@@ -101,7 +101,7 @@ AND support IN ('YES', 'DEFAULT', 'ENABLED');
 ENGINE	SUPPORT	COMMENT	TRANSACTIONS	XA	SAVEPOINTS
 FOUND 1 /\[Note\] InnoDB: Header page consists of zero bytes in datafile: .*u1.ibd/ in mysqld.1.err
 FOUND 1 /InnoDB: At LSN: \d+: unable to open file .*u[1-5].ibd for tablespace/ in mysqld.1.err
-FOUND 1 /\[ERROR\] InnoDB: Cannot rename '.*u5.ibd' to '.*u6.ibd' because the target file exists/ in mysqld.1.err
+FOUND 1 /\[ERROR\] InnoDB: Cannot replay rename of tablespace \d+ from '.*u4.ibd' to '.*u6.ibd' because the target file exists/ in mysqld.1.err
 # restart: --innodb-force-recovery=1
 FOUND 1 /\[Note\] InnoDB: Header page consists of zero bytes in datafile: .*u1.ibd/ in mysqld.1.err
 FOUND 1 /InnoDB: At LSN: \d+: unable to open file .*u[1-5].ibd for tablespace/ in mysqld.1.err

mysql-test/suite/innodb/t/log_file_name.test

@@ -172,6 +172,7 @@ call mtr.add_suppression("InnoDB: Failed to find tablespace for table .* in the
 call mtr.add_suppression("InnoDB: Plugin initialization aborted");
 call mtr.add_suppression("Plugin 'InnoDB' \(init function returned error\|registration as a STORAGE ENGINE failed\)");
 call mtr.add_suppression("InnoDB: Table test/u[123] in the InnoDB data dictionary has tablespace id [1-9][0-9]*, but tablespace with that id or name does not exist\\. Have you deleted or moved \\.ibd files\\?");
+call mtr.add_suppression("InnoDB: Cannot replay rename of tablespace.*");
 FLUSH TABLES;
 --enable_query_log
 
@@ -246,7 +247,7 @@ let SEARCH_PATTERN= \[Note\] InnoDB: Header page consists of zero bytes in dataf
 let SEARCH_PATTERN= InnoDB: At LSN: \d+: unable to open file .*u[1-5].ibd for tablespace;
 --source include/search_pattern_in_file.inc
 
-let SEARCH_PATTERN= \[ERROR\] InnoDB: Cannot rename '.*u5.ibd' to '.*u6.ibd' because the target file exists;
+let SEARCH_PATTERN= \[ERROR\] InnoDB: Cannot replay rename of tablespace \d+ from '.*u4.ibd' to '.*u6.ibd' because the target file exists;
 --source include/search_pattern_in_file.inc
 
 --remove_file $MYSQLD_DATADIR/test/u6.ibd

storage/innobase/fil/fil0fil.cc

@@ -111,19 +111,6 @@ bool fil_space_t::try_to_close(bool print_info)
   return false;
 }
 
-/** Test if a tablespace file can be renamed to a new filepath by checking
-if that the old filepath exists and the new filepath does not exist.
-@param[in]	old_path	old filepath
-@param[in]	new_path	new filepath
-@param[in]	is_discarded	whether the tablespace is discarded
-@param[in]	replace_new	whether to ignore the existence of new_path
-@return innodb error code */
-static dberr_t
-fil_rename_tablespace_check(
-	const char*	old_path,
-	const char*	new_path,
-	bool		is_discarded,
-	bool		replace_new = false);
 /** Rename a single-table tablespace.
 The tablespace must exist in the memory cache.
 @param[in]	id		tablespace identifier
@@ -1583,89 +1570,6 @@ fil_name_write(
   mtr->log_file_op(FILE_MODIFY, space_id, name);
 }
 
-/** Replay a file rename operation if possible.
-@param[in]	space_id	tablespace identifier
-@param[in]	name		old file name
-@param[in]	new_name	new file name
-@return	whether the operation was successfully applied
-(the name did not exist, or new_name did not exist and
-name was successfully renamed to new_name)  */
-bool
-fil_op_replay_rename(
-	ulint		space_id,
-	const char*	name,
-	const char*	new_name)
-{
-	/* In order to replay the rename, the following must hold:
-	* The new name is not already used.
-	* A tablespace exists with the old name.
-	* The space ID for that tablepace matches this log entry.
-	This will prevent unintended renames during recovery. */
-	fil_space_t*	space = fil_space_get(space_id);
-
-	if (space == NULL) {
-		return(true);
-	}
-
-	const bool name_match
-		= strcmp(name, UT_LIST_GET_FIRST(space->chain)->name) == 0;
-
-	if (!name_match) {
-		return(true);
-	}
-
-	/* Create the database directory for the new name, if
-	it does not exist yet */
-
-	const char*	namend = strrchr(new_name, OS_PATH_SEPARATOR);
-	ut_a(namend != NULL);
-
-	char*		dir = static_cast<char*>(
-		ut_malloc_nokey(ulint(namend - new_name) + 1));
-
-	memcpy(dir, new_name, ulint(namend - new_name));
-	dir[namend - new_name] = '\0';
-
-	bool		success = os_file_create_directory(dir, false);
-	ut_a(success);
-
-	ulint		dirlen = 0;
-
-	if (const char* dirend = strrchr(dir, OS_PATH_SEPARATOR)) {
-		dirlen = ulint(dirend - dir) + 1;
-	}
-
-	ut_free(dir);
-
-	/* New path must not exist. */
-	dberr_t		err = fil_rename_tablespace_check(
-		name, new_name, false);
-	if (err != DB_SUCCESS) {
-		ib::error() << " Cannot replay file rename."
-			" Remove either file and try again.";
-		return(false);
-	}
-
-	char*		new_table = mem_strdupl(
-		new_name + dirlen,
-		strlen(new_name + dirlen)
-		- 4 /* remove ".ibd" */);
-
-	ut_ad(new_table[ulint(namend - new_name) - dirlen]
-	      == OS_PATH_SEPARATOR);
-#if OS_PATH_SEPARATOR != '/'
-	new_table[namend - new_name - dirlen] = '/';
-#endif
-
-	if (!fil_rename_tablespace(
-		    space_id, name, new_table, new_name)) {
-		ut_error;
-	}
-
-	ut_free(new_table);
-	return(true);
-}
-
 /** Check for pending operations.
 @param[in]	space	tablespace
 @param[in]	count	number of attempts so far
@@ -2092,22 +1996,18 @@ fil_make_filepath(
 if that the old filepath exists and the new filepath does not exist.
 @param[in]	old_path	old filepath
 @param[in]	new_path	new filepath
-@param[in]	is_discarded	whether the tablespace is discarded
 @param[in]	replace_new	whether to ignore the existence of new_path
 @return innodb error code */
 static dberr_t
 fil_rename_tablespace_check(
 	const char*	old_path,
 	const char*	new_path,
-	bool		is_discarded,
 	bool		replace_new)
 {
 	bool	exists = false;
 	os_file_type_t	ftype;
 
-	if (!is_discarded
-	    && os_file_status(old_path, &exists, &ftype)
-	    && !exists) {
+	if (os_file_status(old_path, &exists, &ftype) && !exists) {
 		ib::error() << "Cannot rename '" << old_path
 			<< "' to '" << new_path
 			<< "' because the source file"
@@ -2167,7 +2067,7 @@ dberr_t fil_space_t::rename(const char* name, const char* path, bool log,
 
 	if (log) {
 		dberr_t err = fil_rename_tablespace_check(
-			chain.start->name, path, false, replace);
+			chain.start->name, path, replace);
 		if (err != DB_SUCCESS) {
 			return(err);
 		}

storage/innobase/include/fil0fil.h

@@ -1565,20 +1565,6 @@ fil_write_flushed_lsn(
 	lsn_t	lsn)
 MY_ATTRIBUTE((warn_unused_result));
 
-/** Replay a file rename operation if possible.
-@param[in]	space_id	tablespace identifier
-@param[in]	name		old file name
-@param[in]	new_name	new file name
-@return	whether the operation was successfully applied
-(the name did not exist, or new_name did not exist and
-name was successfully renamed to new_name)  */
-bool
-fil_op_replay_rename(
-	ulint		space_id,
-	const char*	name,
-	const char*	new_name)
-	MY_ATTRIBUTE((warn_unused_result));
-
 /** Delete a tablespace and associated .ibd file.
 @param[in]	id		tablespace identifier
 @param[in]	if_exists	whether to ignore missing tablespace

storage/innobase/log/log0recv.cc

@@ -584,6 +584,9 @@ typedef std::map<
 
 static recv_spaces_t	recv_spaces;
 
+/** The last parsed FILE_RENAME records */
+static std::map<uint32_t,std::string> renamed_spaces;
+
 /** Report an operation to create, delete, or rename a file during backup.
 @param[in]	space_id	tablespace identifier
 @param[in]	create		whether the file is being created
@@ -943,6 +946,7 @@ void recv_sys_t::close()
   }
 
   recv_spaces.clear();
+  renamed_spaces.clear();
   mlog_init.clear();
 
   close_files();
@@ -2205,11 +2209,15 @@ bool recv_sys_t::parse(lsn_t checkpoint_lsn, store_t *store, bool apply)
                       l, static_cast<ulint>(fnend - fn),
                       reinterpret_cast<const byte*>(fn2),
                       fn2 ? static_cast<ulint>(fn2end - fn2) : 0);
-
-        if (!fn2 || !apply);
-        else if (UNIV_UNLIKELY(!fil_op_replay_rename(space_id, fn, fn2)))
-          found_corrupt_fs= true;
         const_cast<char&>(fn[rlen])= saved_end;
+
+        if (fn2 && apply)
+        {
+          const size_t len= fn2end - fn2;
+          auto r= renamed_spaces.emplace(space_id, std::string{fn2, len});
+          if (!r.second)
+            r.first->second= std::string{fn2, len};
+        }
         if (is_corrupt_fs())
           return true;
       }
@@ -2817,6 +2825,62 @@ void recv_sys_t::apply(bool last_batch)
     buf_pool_invalidate();
     mysql_mutex_lock(&log_sys.mutex);
   }
+#if 1 /* Mariabackup FIXME: Remove or adjust rename_table_in_prepare() */
+  else if (srv_operation != SRV_OPERATION_NORMAL);
+#endif
+  else
+  {
+    /* In the last batch, we will apply any rename operations. */
+    for (auto r : renamed_spaces)
+    {
+      const uint32_t id= r.first;
+      fil_space_t *space= fil_space_t::get(id);
+      if (!space)
+        continue;
+      ut_ad(UT_LIST_GET_LEN(space->chain) == 1);
+      const char *old= space->chain.start->name;
+      if (r.second != old)
+      {
+        bool exists;
+        os_file_type_t ftype;
+        const char *new_name= r.second.c_str();
+        if (!os_file_status(new_name, &exists, &ftype) || exists)
+        {
+          ib::error() << "Cannot replay rename of tablespace " << id
+                      << " from '" << old << "' to '" << r.second <<
+                      (exists ? "' because the target file exists" : "'");
+          found_corrupt_fs= true;
+        }
+        else
+        {
+          size_t base= r.second.rfind(OS_PATH_SEPARATOR);
+          ut_ad(base != std::string::npos);
+          size_t start= r.second.rfind(OS_PATH_SEPARATOR, base - 1);
+          if (start == std::string::npos)
+            start= 0;
+          else
+            ++start;
+          /* Keep only databasename/tablename without .ibd suffix */
+          std::string space_name(r.second, start, r.second.size() - start - 4);
+          ut_ad(space_name[base - start] == OS_PATH_SEPARATOR);
+#if OS_PATH_SEPARATOR != '/'
+          space_name[base - start]= '/';
+#endif
+          mysql_mutex_lock(&log_sys.mutex);
+          if (dberr_t err= space->rename(space_name.c_str(), r.second.c_str(),
+                                         false))
+          {
+            ib::error() << "Cannot replay rename of tablespace " << id
+                        << " to '" << r.second << "': " << err;
+            found_corrupt_fs= true;
+          }
+          mysql_mutex_unlock(&log_sys.mutex);
+        }
+      }
+      space->release();
+    }
+    renamed_spaces.clear();
+  }
 
   mysql_mutex_lock(&mutex);