MSVC: ASAN support for VS 16.9

This enables ASAN support when used with VS 16.9
enable as usual in cmake with the WITH_COMPILER_ASAN
option, or when using make.bat just tag on `asan'
to the invocation, ie: `make lite 2019 asan`

MSVC: Asan support for 16.9

This enables ASAN support when used with VS 16.9
enable as usual in cmake with the WITH_COMPILER_ASAN
option, or when using make.bat just tag on `asan'
to the invocation, ie: `make lite 2019 asan`

Differential Revision: https://developer.blender.org/D7794
Reviewed By: brecht, sergey

3 files changed, 51 insertions, 22 deletions

diff --git a/build_files/cmake/platform/platform_win32.cmake b/build_files/cmake/platform/platform_win32.cmake
index acd6028e2ae..b6ec016aa95 100644
--- a/build_files/cmake/platform/platform_win32.cmake
+++ b/build_files/cmake/platform/platform_win32.cmake
@@ -149,11 +149,6 @@ add_definitions(-D_WIN32_WINNT=0x601)
 include(build_files/cmake/platform/platform_win32_bundle_crt.cmake)
 remove_cc_flag("/MDd" "/MD" "/Zi")
 
-if(WITH_WINDOWS_PDB)
-  set(PDB_INFO_OVERRIDE_FLAGS "/Z7")
-  set(PDB_INFO_OVERRIDE_LINKER_FLAGS "/DEBUG /OPT:REF /OPT:ICF /INCREMENTAL:NO")
-endif()
-
 if(MSVC_CLANG) # Clangs version of cl doesn't support all flags
   string(APPEND CMAKE_CXX_FLAGS " ${CXX_WARN_FLAGS} /nologo /J /Gd /EHsc -Wno-unused-command-line-argument -Wno-microsoft-enum-forward-reference ")
   set(CMAKE_C_FLAGS     "${CMAKE_C_FLAGS} /nologo /J /Gd -Wno-unused-command-line-argument -Wno-microsoft-enum-forward-reference")
@@ -162,6 +157,21 @@ else()
   set(CMAKE_C_FLAGS     "${CMAKE_C_FLAGS} /nologo /J /Gd /MP /bigobj")
 endif()
 
+# X64 ASAN is available and usable on MSVC 16.9 preview 4 and up)
+if(WITH_COMPILER_ASAN AND MSVC AND NOT MSVC_CLANG)
+  if(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 19.28.29828)
+    #set a flag so we don't have to do this comparison all the time
+    SET(MSVC_ASAN On)
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /fsanitize=address")
+    set(CMAKE_C_FLAGS     "${CMAKE_C_FLAGS} /fsanitize=address")
+    string(APPEND CMAKE_EXE_LINKER_FLAGS_DEBUG " /INCREMENTAL:NO")
+    string(APPEND CMAKE_SHARED_LINKER_FLAGS_DEBUG " /INCREMENTAL:NO")
+  else()
+    message("-- ASAN not supported on MSVC ${CMAKE_CXX_COMPILER_VERSION}")
+  endif()
+endif()
+
+
 # C++ standards conformace (/permissive-) is available on msvc 15.5 (1912) and up
 if(MSVC_VERSION GREATER 1911 AND NOT MSVC_CLANG)
   string(APPEND CMAKE_CXX_FLAGS " /permissive-")
@@ -174,14 +184,41 @@ if(WITH_WINDOWS_SCCACHE AND CMAKE_VS_MSBUILD_COMMAND)
     set(WITH_WINDOWS_SCCACHE Off)
 endif()
 
+# Debug Symbol format
+# sccache # MSVC_ASAN # format # why
+# On      # On        # Z7     # sccache will only play nice with Z7
+# On      # Off       # Z7     # sccache will only play nice with Z7
+# Off     # On        # Zi     # Asan will not play nice with Edit and Continue
+# Off     # Off       # ZI     # Neither asan nor sscache is enabled Edit and Continue is available
+
+# Release Symbol format
+# sccache # MSVC_ASAN # format # why
+# On      # On        # Z7     # sccache will only play nice with Z7
+# On      # Off       # Z7     # sccache will only play nice with Z7
+# Off     # On        # Zi     # Asan will not play nice with Edit and Continue
+# Off     # Off       # Zi     # Edit and Continue disables some optimizations
+
+
 if(WITH_WINDOWS_SCCACHE)
     set(CMAKE_C_COMPILER_LAUNCHER sccache)
     set(CMAKE_CXX_COMPILER_LAUNCHER sccache)
     set(SYMBOL_FORMAT /Z7)
+    set(SYMBOL_FORMAT_RELEASE /Z7)
 else()
     unset(CMAKE_C_COMPILER_LAUNCHER)
     unset(CMAKE_CXX_COMPILER_LAUNCHER)
-    set(SYMBOL_FORMAT /ZI)
+    if(MSVC_ASAN)
+      set(SYMBOL_FORMAT /Z7)
+      set(SYMBOL_FORMAT_RELEASE /Z7)
+    else()
+      set(SYMBOL_FORMAT /ZI)
+      set(SYMBOL_FORMAT_RELEASE /Zi)
+    endif()
+endif()
+
+if(WITH_WINDOWS_PDB)
+	set(PDB_INFO_OVERRIDE_FLAGS "${SYMBOL_FORMAT_RELEASE}")
+	set(PDB_INFO_OVERRIDE_LINKER_FLAGS "/DEBUG /OPT:REF /OPT:ICF /INCREMENTAL:NO")
 endif()
 
 string(APPEND CMAKE_CXX_FLAGS_DEBUG " /MDd ${SYMBOL_FORMAT}")
@@ -190,9 +227,11 @@ string(APPEND CMAKE_CXX_FLAGS_RELEASE " /MD ${PDB_INFO_OVERRIDE_FLAGS}")
 string(APPEND CMAKE_C_FLAGS_RELEASE " /MD ${PDB_INFO_OVERRIDE_FLAGS}")
 string(APPEND CMAKE_CXX_FLAGS_MINSIZEREL " /MD ${PDB_INFO_OVERRIDE_FLAGS}")
 string(APPEND CMAKE_C_FLAGS_MINSIZEREL " /MD ${PDB_INFO_OVERRIDE_FLAGS}")
-string(APPEND CMAKE_CXX_FLAGS_RELWITHDEBINFO " /MD ${SYMBOL_FORMAT}")
-string(APPEND CMAKE_C_FLAGS_RELWITHDEBINFO " /MD ${SYMBOL_FORMAT}")
+string(APPEND CMAKE_CXX_FLAGS_RELWITHDEBINFO " /MD ${SYMBOL_FORMAT_RELEASE}")
+string(APPEND CMAKE_C_FLAGS_RELWITHDEBINFO " /MD ${SYMBOL_FORMAT_RELEASE}")
 unset(SYMBOL_FORMAT)
+unset(SYMBOL_FORMAT_RELEASE)
+
 # JMC is available on msvc 15.8 (1915) and up
 if(MSVC_VERSION GREATER 1914 AND NOT MSVC_CLANG)
   string(APPEND CMAKE_CXX_FLAGS_DEBUG " /JMC")
diff --git a/build_files/windows/configure_msbuild.cmd b/build_files/windows/configure_msbuild.cmd
index 4956f1e3ea1..be69f99a8e8 100644
--- a/build_files/windows/configure_msbuild.cmd
+++ b/build_files/windows/configure_msbuild.cmd
@@ -9,14 +9,10 @@ if "%BUILD_WITH_SCCACHE%"=="1" (
 
 if "%WITH_CLANG%"=="1" (
 	set CLANG_CMAKE_ARGS=-T"llvm"
-	if "%WITH_ASAN%"=="1" (
+)
+
+if "%WITH_ASAN%"=="1" (
 		set ASAN_CMAKE_ARGS=-DWITH_COMPILER_ASAN=On
-	)
-) else (
-	if "%WITH_ASAN%"=="1" (
-		echo ASAN is only supported with clang.
-		exit /b 1 
-	)
 )
 
 if "%WITH_PYDEBUG%"=="1" (
diff --git a/build_files/windows/configure_ninja.cmd b/build_files/windows/configure_ninja.cmd
index d68d94a3bcf..90085feb2bd 100644
--- a/build_files/windows/configure_ninja.cmd
+++ b/build_files/windows/configure_ninja.cmd
@@ -46,16 +46,10 @@ set LLVM_DIR=
 		set CFLAGS=-m64 -fmsc-version=1914
 		set CXXFLAGS=-m64 -fmsc-version=1914
 	)
-	if "%WITH_ASAN%"=="1" (
-		set BUILD_CMAKE_ARGS=%BUILD_CMAKE_ARGS% -DWITH_COMPILER_ASAN=On
-	)	
 )
 
 if "%WITH_ASAN%"=="1" (
-	if "%WITH_CLANG%" == "" (
-		echo ASAN is only supported with clang.
-		exit /b 1 
-	)
+	set BUILD_CMAKE_ARGS=%BUILD_CMAKE_ARGS% -DWITH_COMPILER_ASAN=On
 )
 
 if NOT "%verbose%" == "" (