MDEV-25123 support MSVC ASAN

cmake/os/Windows.cmake

@@ -68,17 +68,12 @@ MACRO(ENABLE_SANITIZERS)
     MESSAGE(FATAL_ERROR "clang-cl or MSVC necessary to enable asan/ubsan")
   ENDIF()
   # currently, asan is broken with static CRT.
-  IF(NOT(MSVC_CRT_TYPE STREQUAL "/MD"))
-    IF(NOT DYNAMIC_UCRT_LINK)
-      MESSAGE(FATAL_ERROR "-DWITH_ASAN cmake parameter also requires -DMSVC_CRT_TYPE=/MD OR DYNAMIC_UCRT_LINK=ON")
-    ENDIF()
+  IF(CLANG_CL AND NOT(MSVC_CRT_TYPE STREQUAL "/MD"))
+    SET(MSVC_CRT_TYPE "/MD" CACHE  INTERNAL "" FORCE)
   ENDIF()
   IF(CMAKE_SIZEOF_VOID_P EQUAL 4)
     SET(ASAN_ARCH i386)
   ELSE()
-    IF(NOT CLANG_CL)
-      MESSAGE(FATAL_ERROR "sanitizers do not yet work on MSVC x64, try 32  bit or clang-cl")
-    ENDIF()
     SET(ASAN_ARCH x86_64)
   ENDIF()
 
@@ -88,8 +83,10 @@ MACRO(ENABLE_SANITIZERS)
   SET(SANITIZER_LINK_LIBRARIES)
   SET(SANITIZER_COMPILE_FLAGS)
   IF(WITH_ASAN)
-    LIST(APPEND SANITIZER_LIBS
-      clang_rt.asan_dynamic-${ASAN_ARCH}.lib clang_rt.asan_dynamic_runtime_thunk-${ASAN_ARCH}.lib)
+    IF(CLANG_CL)
+      LIST(APPEND SANITIZER_LIBS
+        clang_rt.asan_dynamic-${ASAN_ARCH}.lib clang_rt.asan_dynamic_runtime_thunk-${ASAN_ARCH}.lib)
+    ENDIF()
     STRING(APPEND SANITIZER_COMPILE_FLAGS " -fsanitize=address")
   ENDIF()
   IF(WITH_UBSAN)
@@ -134,7 +131,13 @@ IF(MSVC)
    INCLUDE(InstallRequiredSystemLibraries)
   ENDIF()
 
-  OPTION(DYNAMIC_UCRT_LINK "Link Universal CRT dynamically, if MSVC_CRT_TYPE=/MT" ON)
+  IF(WITH_ASAN AND (NOT CLANG_CL))
+    SET(DYNAMIC_UCRT_LINK_DEFAULT OFF)
+  ELSE()
+    SET(DYNAMIC_UCRT_LINK_DEFAULT ON)
+  ENDIF()
+
+  OPTION(DYNAMIC_UCRT_LINK "Link Universal CRT dynamically, if MSVC_CRT_TYPE=/MT" ${DYNAMIC_UCRT_LINK_DEFAULT})
   SET(DYNAMIC_UCRT_LINKER_OPTION " /NODEFAULTLIB:libucrt.lib /DEFAULTLIB:ucrt.lib")
 
   # Enable debug info also in Release build,
@@ -191,8 +194,15 @@ IF(MSVC)
 
   FOREACH(type EXE SHARED MODULE)
    STRING(REGEX REPLACE "/STACK:([^ ]+)" "" CMAKE_${type}_LINKER_FLAGS "${CMAKE_${type}_LINKER_FLAGS}")
-   STRING(REGEX REPLACE "/INCREMENTAL:([^ ]+)" "/INCREMENTAL:NO" CMAKE_${type}_LINKER_FLAGS_RELWITHDEBINFO "${CMAKE_${type}_LINKER_FLAGS_RELWITHDEBINFO}")
-   STRING(REGEX REPLACE "/INCREMENTAL$" "/INCREMENTAL:NO" CMAKE_${type}_LINKER_FLAGS_RELWITHDEBINFO "${CMAKE_${type}_LINKER_FLAGS_RELWITHDEBINFO}")
+   IF(WITH_ASAN)
+     SET(build_types RELWITHDEBINFO DEBUG)
+   ELSE()
+     SET(build_types RELWITHDEBINFO)
+   ENDIF()
+   FOREACH(btype ${build_types})
+     STRING(REGEX REPLACE "/INCREMENTAL:([^ ]+)" "/INCREMENTAL:NO" CMAKE_${type}_LINKER_FLAGS_${btype} "${CMAKE_${type}_LINKER_FLAGS_${btype}}")
+     STRING(REGEX REPLACE "/INCREMENTAL$" "/INCREMENTAL:NO" CMAKE_${type}_LINKER_FLAGS_${btype} "${CMAKE_${type}_LINKER_FLAGS_${btype}}")
+   ENDFOREACH()
    IF(NOT CLANG_CL)
      STRING(APPEND CMAKE_${type}_LINKER_FLAGS_RELWITHDEBINFO " /release /OPT:REF,ICF")
    ENDIF()

cmake/pcre.cmake

@@ -36,8 +36,8 @@ MACRO(BUNDLE_PCRE2)
     IF(MSVC)
       # Suppress a warning
       STRING(APPEND pcre2_flags${v} " /wd4244 " )
-      # Need this only for ASAN support
-      SET(stdlibs "-DCMAKE_C_STANDARD_LIBRARIES=${CMAKE_C_STANDARD_LIBRARIES}")
+      # Disable asan support
+      STRING(REPLACE "-fsanitize=address" "" pcre2_flags${v} "${CMAKE_C_FLAGS${v}}")
     ENDIF()
   ENDFOREACH()
   ExternalProject_Add(

include/my_pthread.h

@@ -667,7 +667,11 @@ extern void my_mutex_end(void);
   with the current number of keys and key parts.
 */
 #if defined(__SANITIZE_ADDRESS__) || defined(WITH_UBSAN)
+#ifndef DBUG_OFF
+#define DEFAULT_THREAD_STACK	(1024*1024L)
+#else
 #define DEFAULT_THREAD_STACK	(383*1024L) /* 392192 */
+#endif
 #else
 #define DEFAULT_THREAD_STACK	(292*1024L) /* 299008 */
 #endif

include/my_valgrind.h

@@ -47,7 +47,7 @@
 # define MEM_GET_VBITS(a,b,len) VALGRIND_GET_VBITS(a,b,len)
 # define MEM_SET_VBITS(a,b,len) VALGRIND_SET_VBITS(a,b,len)
 # define REDZONE_SIZE 8
-#elif defined(__SANITIZE_ADDRESS__)
+#elif defined(__SANITIZE_ADDRESS__) && (!defined(_MSC_VER) || defined (__clang__))
 # include <sanitizer/asan_interface.h>
 /* How to do manual poisoning:
 https://github.com/google/sanitizers/wiki/AddressSanitizerManualPoisoning */