Build: disable address sanitizer for Cycles optimized kernels with GCC

It's extremely slow to compile and run, so just disable it unless
WITH_CYCLES_KERNEL_ASAN is manually enabled. For Clang it's always
enabled since that appears to work ok.

This also limits the -fno-sanitize=vptr flag to the Cycles kernel, as it
was added specifically to work around an issue there.

Differential Revision: https://developer.blender.org/D5404

3 files changed, 19 insertions, 6 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 88295979fa7..6ced6e1d76d 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -413,6 +413,8 @@ unset(PLATFORM_DEFAULT)
 option(WITH_CYCLES_LOGGING  "Build Cycles with logging support" ON)
 option(WITH_CYCLES_DEBUG    "Build Cycles with extra debug capabilities" OFF)
 option(WITH_CYCLES_NATIVE_ONLY  "Build Cycles with native kernel only (which fits current CPU, use for development only)" OFF)
+option(WITH_CYCLES_KERNEL_ASAN  "Build Cycles kernels with address sanitizer when WITH_COMPILER_ASAN is on, even if it's very slow" OFF)
+mark_as_advanced(WITH_CYCLES_KERNEL_ASAN)
 mark_as_advanced(WITH_CYCLES_CUBIN_COMPILER)
 mark_as_advanced(WITH_CYCLES_LOGGING)
 mark_as_advanced(WITH_CYCLES_DEBUG)
@@ -822,12 +824,6 @@ if(NOT CMAKE_BUILD_TYPE MATCHES "Release")
 
     set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} ${COMPILER_ASAN_CXXFLAGS}")
     set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} ${COMPILER_ASAN_CXXFLAGS}")
-    if(WITH_CYCLES_OSL)
-      # With OSL, Cycles disables rtti in some modules, which then breaks at linking
-      # when trying to use vptr sanitizer (included into 'undefined' general option).
-      set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -fno-sanitize=vptr")
-      set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -fno-sanitize=vptr")
-    endif()
     if(MSVC)
       set(COMPILER_ASAN_LINKER_FLAGS "/FUNCTIONPADMIN:6")
     endif()
diff --git a/intern/cycles/kernel/CMakeLists.txt b/intern/cycles/kernel/CMakeLists.txt
index 8a8fee108ae..cd284c06259 100644
--- a/intern/cycles/kernel/CMakeLists.txt
+++ b/intern/cycles/kernel/CMakeLists.txt
@@ -486,6 +486,17 @@ endif()
 include_directories(${INC})
 include_directories(SYSTEM ${INC_SYS})
 
+if(CMAKE_COMPILER_IS_GNUCC AND (NOT WITH_CYCLES_KERNEL_ASAN))
+  # GCC hangs compiling the big kernel files with asan and release, so disable by default.
+  set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -fno-sanitize=all")
+  set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -fno-sanitize=vptr")
+elseif(CMAKE_C_COMPILER_ID MATCHES "Clang")
+  # With OSL, Cycles disables rtti in some modules, wich then breaks at linking
+  # when trying to use vptr sanitizer (included into 'undefined' general option).
+  set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -fno-sanitize=vptr")
+  set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -fno-sanitize=vptr")
+endif()
+
 set_source_files_properties(kernels/cpu/kernel.cpp PROPERTIES COMPILE_FLAGS "${CYCLES_KERNEL_FLAGS}")
 set_source_files_properties(kernels/cpu/kernel_split.cpp PROPERTIES COMPILE_FLAGS "${CYCLES_KERNEL_FLAGS}")
 set_source_files_properties(kernels/cpu/filter.cpp PROPERTIES COMPILE_FLAGS "${CYCLES_KERNEL_FLAGS}")
diff --git a/intern/elbeem/CMakeLists.txt b/intern/elbeem/CMakeLists.txt
index 926329be61b..383cfa66c15 100644
--- a/intern/elbeem/CMakeLists.txt
+++ b/intern/elbeem/CMakeLists.txt
@@ -119,4 +119,10 @@ else()
   add_definitions(-DPARALLEL=0)
 endif()
 
+# Work around hang with GCC and ASAN.
+if(CMAKE_COMPILER_IS_GNUCC OR CMAKE_C_COMPILER_ID MATCHES "Clang")
+  set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -fno-sanitize=vptr")
+  set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -fno-sanitize=vptr")
+endif()
+
 blender_add_lib_nolist(bf_intern_elbeem "${SRC}" "${INC}" "${INC_SYS}" "${LIB}")