Add Asan support for clang on windows.

This will currently only work for the RelWithDebInfo configuration since asan
does not support the debug crt. for source line information in the reports,
you need a copy of llvm-symbolizer in the blender folder or set the
ASAN_SYMBOLIZER_PATH environment variable to point to it. Currently (as of
6.0.0) llvm-symbolizer does not ship with the binary clang/llvm distribution.

Reviewers: campbellbarton

Differential Revision: https://developer.blender.org/D3446

10 files changed, 49 insertions, 12 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 6ffe87da673..4543ca124bd 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -539,9 +539,7 @@ if(CMAKE_COMPILER_IS_GNUCC OR CMAKE_C_COMPILER_ID MATCHES "Clang")
 -fsanitize=enum \
 -fsanitize=float-cast-overflow \
 -fsanitize=float-divide-by-zero \
--fsanitize=leak \
 -fsanitize=nonnull-attribute \
--fsanitize=object-size \
 -fsanitize=returns-nonnull-attribute \
 -fsanitize=signed-integer-overflow \
 -fsanitize=undefined \
@@ -549,6 +547,9 @@ if(CMAKE_COMPILER_IS_GNUCC OR CMAKE_C_COMPILER_ID MATCHES "Clang")
 -fno-sanitize=alignment \
 ")
 
+		if(NOT MSVC) # not all sanitizers are supported with clang-cl, these two however are very vocal about it
+			set(_asan_defaults "${_asan_defaults} -fsanitize=leak -fsanitize=object-size" )
+		endif()
 		set(COMPILER_ASAN_CFLAGS "${_asan_defaults}" CACHE STRING "C flags for address sanitizer")
 		mark_as_advanced(COMPILER_ASAN_CFLAGS)
 		set(COMPILER_ASAN_CXXFLAGS "${_asan_defaults}" CACHE STRING "C++ flags for address sanitizer")
@@ -556,9 +557,15 @@ if(CMAKE_COMPILER_IS_GNUCC OR CMAKE_C_COMPILER_ID MATCHES "Clang")
 
 		unset(_asan_defaults)
 
-		find_library(COMPILER_ASAN_LIBRARY asan ${CMAKE_C_IMPLICIT_LINK_DIRECTORIES})
+		if(NOT MSVC)
+			find_library(COMPILER_ASAN_LIBRARY asan ${CMAKE_C_IMPLICIT_LINK_DIRECTORIES})
+		else()
+			find_library( COMPILER_ASAN_LIBRARY NAMES clang_rt.asan-x86_64
+				PATHS 	[HKEY_LOCAL_MACHINE\\SOFTWARE\\Wow6432Node\\LLVM\\LLVM;]/lib/clang/7.0.0/lib/windows
+						[HKEY_LOCAL_MACHINE\\SOFTWARE\\Wow6432Node\\LLVM\\LLVM;]/lib/clang/6.0.0/lib/windows
+						)
+		endif() 
 		mark_as_advanced(COMPILER_ASAN_LIBRARY)
-
 	endif()
 endif()
 
@@ -863,8 +870,12 @@ if(WITH_COMPILER_ASAN)
 
 	set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} ${COMPILER_ASAN_CXXFLAGS}")
 	set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} ${COMPILER_ASAN_CXXFLAGS}")
-
-	set(PLATFORM_LINKFLAGS_DEBUG "${COMPILER_ASAN_LIBRARY}")
+	if(MSVC)
+		set(COMPILER_ASAN_LINKER_FLAGS "/FUNCTIONPADMIN:6") 
+	endif()
+	set(PLATFORM_LINKLIBS "${PLATFORM_LINKLIBS};${COMPILER_ASAN_LIBRARY}")
+	set(PLATFORM_LINKFLAGS "${COMPILER_ASAN_LIBRARY} ${COMPILER_ASAN_LINKER_FLAGS}")
+	set(PLATFORM_LINKFLAGS_DEBUG "${COMPILER_ASAN_LIBRARY} ${COMPILER_ASAN_LINKER_FLAGS}") 
 endif()
 
 #-----------------------------------------------------------------------------
diff --git a/build_files/cmake/macros.cmake b/build_files/cmake/macros.cmake
index abf8da11c8e..11fc3e110c1 100644
--- a/build_files/cmake/macros.cmake
+++ b/build_files/cmake/macros.cmake
@@ -352,6 +352,11 @@ function(SETUP_LIBDIRS)
 	endif()
 endfunction()
 
+macro(setup_platform_linker_flags)
+	set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${PLATFORM_LINKFLAGS}")
+	set(CMAKE_EXE_LINKER_FLAGS_DEBUG "${CMAKE_EXE_LINKER_FLAGS_DEBUG} ${PLATFORM_LINKFLAGS_DEBUG}")
+endmacro()
+
 function(setup_liblinks
 	target
 	)
diff --git a/build_files/cmake/platform/platform_win32.cmake b/build_files/cmake/platform/platform_win32.cmake
index 3c35851bc5f..6e049c0048f 100644
--- a/build_files/cmake/platform/platform_win32.cmake
+++ b/build_files/cmake/platform/platform_win32.cmake
@@ -149,7 +149,7 @@ set(CMAKE_C_FLAGS_MINSIZEREL "${CMAKE_C_FLAGS_MINSIZEREL} /MT")
 set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} /MT")
 set(CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO} /MT")
 
-set(PLATFORM_LINKFLAGS "/SUBSYSTEM:CONSOLE /STACK:2097152 /INCREMENTAL:NO ")
+set(PLATFORM_LINKFLAGS "${PLATFORM_LINKFLAGS} /SUBSYSTEM:CONSOLE /STACK:2097152 /INCREMENTAL:NO ")
 set(PLATFORM_LINKFLAGS "${PLATFORM_LINKFLAGS} /NODEFAULTLIB:msvcrt.lib /NODEFAULTLIB:msvcmrt.lib /NODEFAULTLIB:msvcurt.lib /NODEFAULTLIB:msvcrtd.lib ")
 
 # Ignore meaningless for us linker warnings.
@@ -162,7 +162,7 @@ else()
 	set(PLATFORM_LINKFLAGS "/MACHINE:IX86 /LARGEADDRESSAWARE ${PLATFORM_LINKFLAGS}")
 endif()
 
-set(PLATFORM_LINKFLAGS_DEBUG "/IGNORE:4099 /NODEFAULTLIB:libcmt.lib /NODEFAULTLIB:libc.lib")
+set(PLATFORM_LINKFLAGS_DEBUG "${PLATFORM_LINKFLAGS_DEBUG} /IGNORE:4099 /NODEFAULTLIB:libcmt.lib /NODEFAULTLIB:libc.lib")
 
 if(NOT DEFINED LIBDIR)
 
diff --git a/build_files/windows/configure_msbuild.cmd b/build_files/windows/configure_msbuild.cmd
index f06f3e317ee..eee21f568be 100644
--- a/build_files/windows/configure_msbuild.cmd
+++ b/build_files/windows/configure_msbuild.cmd
@@ -12,6 +12,14 @@ if "%BUILD_ARCH%"=="x64" (
 
 if "%WITH_CLANG%"=="1" (
 	set BUILD_CMAKE_ARGS=%BUILD_CMAKE_ARGS% -T"LLVM-vs2017"
+	if "%WITH_ASAN%"=="1" (
+		set BUILD_CMAKE_ARGS=%BUILD_CMAKE_ARGS% -DWITH_COMPILER_ASAN=On
+	)
+) else (
+	if "%WITH_ASAN%"=="1" (
+		echo ASAN is only supported with clang.
+		exit /b 1 
+	)
 )
 
 if NOT EXIST %BUILD_DIR%\nul (
diff --git a/build_files/windows/configure_ninja.cmd b/build_files/windows/configure_ninja.cmd
index f882ca3dbf0..d3b002e9a24 100644
--- a/build_files/windows/configure_ninja.cmd
+++ b/build_files/windows/configure_ninja.cmd
@@ -27,6 +27,16 @@ set LLVM_DIR=
 	rem build and tested against 2017 15.7
 	set CFLAGS=-m64 -fmsc-version=1914
 	set CXXFLAGS=-m64 -fmsc-version=1914
+	if "%WITH_ASAN%"=="1" (
+		set BUILD_CMAKE_ARGS=%BUILD_CMAKE_ARGS% -DWITH_COMPILER_ASAN=On
+	)	
+)
+
+if "%WITH_ASAN%"=="1" (
+	if "%WITH_CLANG%" == "" (
+		echo ASAN is only supported with clang.
+		exit /b 1 
+	)
 )
 
 if NOT "%verbose%" == "" (
diff --git a/build_files/windows/parse_arguments.cmd b/build_files/windows/parse_arguments.cmd
index a8956d350ae..2cc0acfd243 100644
--- a/build_files/windows/parse_arguments.cmd
+++ b/build_files/windows/parse_arguments.cmd
@@ -42,6 +42,8 @@ if NOT "%1" == "" (
 	) else if "%1" == "release" (
 		set BUILD_CMAKE_ARGS=%BUILD_CMAKE_ARGS% -C"%BLENDER_DIR%\build_files\cmake\config\blender_release.cmake"
 		set TARGET=Release
+	) else if "%1" == "asan" (
+		set WITH_ASAN=1
 	) else if "%1" == "x86" (
 		set BUILD_ARCH=x86
 	) else if "%1" == "x64" (
diff --git a/build_files/windows/reset_variables.cmd b/build_files/windows/reset_variables.cmd
index d9a50a7f809..f933729b91c 100644
--- a/build_files/windows/reset_variables.cmd
+++ b/build_files/windows/reset_variables.cmd
@@ -21,4 +21,5 @@ set BUILD_UPDATE=
 set BUILD_SHOW_HASHES=
 set SHOW_HELP=
 set BUILD_WITH_NINJA=
-set WITH_CLANG=
\ No newline at end of file
+set WITH_CLANG=
+set WITH_ASAN=
\ No newline at end of file
diff --git a/source/blender/datatoc/CMakeLists.txt b/source/blender/datatoc/CMakeLists.txt
index af7f954cad1..0c68cdeb8e9 100644
--- a/source/blender/datatoc/CMakeLists.txt
+++ b/source/blender/datatoc/CMakeLists.txt
@@ -37,7 +37,7 @@ if(NOT WITH_HEADLESS)
 	set(SRC
 		datatoc_icon.c
 	)
-
+	setup_platform_linker_flags()
 	if(WIN32)
 		include_directories(
 			../blenlib
@@ -52,8 +52,6 @@ if(NOT WITH_HEADLESS)
 			../../../intern/utfconv/utfconv.c
 		)
 
-		set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${PLATFORM_LINKFLAGS}")
-		set(CMAKE_EXE_LINKER_FLAGS_DEBUG "${CMAKE_EXE_LINKER_FLAGS_DEBUG} ${PLATFORM_LINKFLAGS_DEBUG}")
 	endif()
 
 	include_directories(${PNG_INCLUDE_DIRS})
diff --git a/source/blender/makesdna/intern/CMakeLists.txt b/source/blender/makesdna/intern/CMakeLists.txt
index 5b8dcd97108..06fd23cdff2 100644
--- a/source/blender/makesdna/intern/CMakeLists.txt
+++ b/source/blender/makesdna/intern/CMakeLists.txt
@@ -53,6 +53,7 @@ endif()
 # SRC_DNA_INC is defined in the parent dir
 
 add_cc_flags_custom_test(makesdna)
+setup_platform_linker_flags()
 
 add_executable(makesdna ${SRC} ${SRC_DNA_INC})
 
diff --git a/source/blender/makesrna/intern/CMakeLists.txt b/source/blender/makesrna/intern/CMakeLists.txt
index 743d192f0c5..5e7438ce4ee 100644
--- a/source/blender/makesrna/intern/CMakeLists.txt
+++ b/source/blender/makesrna/intern/CMakeLists.txt
@@ -356,6 +356,7 @@ blender_include_dirs_sys(
 )
 
 add_cc_flags_custom_test(makesrna)
+setup_platform_linker_flags()
 
 add_executable(makesrna ${SRC} ${SRC_RNA_INC} ${SRC_DNA_INC})