Tests: fail automated tests on memory leaks and other internal errors

This adds a new `--debug-exit-on-error` flag. When it is set, Blender
will abort with a non-zero exit code when there are internal errors.
Currently, "internal errors" includes memory leaks detected by
guardedalloc and error/fatal log entries in clog.

The new flag is passed to Blender in various places where automated
tests are run. Furthermore, the `--debug-memory` flag is used in tests,
because that makes the verbose output more useful, when dealing
with memory leaks.

Reviewers: brecht, sergey

Differential Revision: https://developer.blender.org/D8665

10 files changed, 22 insertions, 10 deletions

diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 0ee3b500fdf..8f72db07d92 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -44,7 +44,7 @@ unset(_default_test_python_exe)
 
 # Standard Blender arguments for running tests.
 # Specify exit code so that if a Python script error happens, the test fails.
-set(TEST_BLENDER_EXE_PARAMS --background -noaudio --factory-startup --python-exit-code 1)
+set(TEST_BLENDER_EXE_PARAMS --background -noaudio --factory-startup --debug-memory --debug-exit-on-error --python-exit-code 1)
 
 # Python CTests
 if(WITH_BLENDER AND WITH_PYTHON)
diff --git a/tests/gtests/testing/testing_main.cc b/tests/gtests/testing/testing_main.cc
index e9313b31909..6238101b319 100644
--- a/tests/gtests/testing/testing_main.cc
+++ b/tests/gtests/testing/testing_main.cc
@@ -50,6 +50,7 @@ int main(int argc, char **argv)
 {
   MEM_use_guarded_allocator();
   MEM_init_memleak_detection();
+  MEM_enable_fail_on_memleak();
   testing::InitGoogleTest(&argc, argv);
   BLENDER_GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true);
   google::InitGoogleLogging(argv[0]);
diff --git a/tests/python/collada/CMakeLists.txt b/tests/python/collada/CMakeLists.txt
index bf22de6b762..1b24875578e 100644
--- a/tests/python/collada/CMakeLists.txt
+++ b/tests/python/collada/CMakeLists.txt
@@ -36,12 +36,12 @@ execute_process(COMMAND ${CMAKE_COMMAND} -E make_directory ${TEST_OUT_DIR})
 # all calls to blender use this
 if(APPLE)
   if(${CMAKE_GENERATOR} MATCHES "Xcode")
-    set(TEST_BLENDER_EXE_PARAMS --background -noaudio --factory-startup)
+    set(TEST_BLENDER_EXE_PARAMS --background -noaudio --factory-startup --debug-memory --debug-exit-on-error)
   else()
-    set(TEST_BLENDER_EXE_PARAMS --background -noaudio --factory-startup --env-system-scripts ${CMAKE_SOURCE_DIR}/release/scripts)
+    set(TEST_BLENDER_EXE_PARAMS --background -noaudio --factory-startup --debug-memory --debug-exit-on-error --env-system-scripts ${CMAKE_SOURCE_DIR}/release/scripts)
   endif()
 else()
-  set(TEST_BLENDER_EXE_PARAMS --background -noaudio --factory-startup --env-system-scripts ${CMAKE_SOURCE_DIR}/release/scripts)
+  set(TEST_BLENDER_EXE_PARAMS --background -noaudio --factory-startup --debug-memory --debug-exit-on-error --env-system-scripts ${CMAKE_SOURCE_DIR}/release/scripts)
 endif()
 
 # for testing with valgrind prefix: valgrind --track-origins=yes --error-limit=no
diff --git a/tests/python/cycles_render_tests.py b/tests/python/cycles_render_tests.py
index bdf4283eb3e..cc949248ce6 100644
--- a/tests/python/cycles_render_tests.py
+++ b/tests/python/cycles_render_tests.py
@@ -20,6 +20,8 @@ def get_arguments(filepath, output_filepath):
         "-noaudio",
         "--factory-startup",
         "--enable-autoexec",
+        "--debug-memory",
+        "--debug-exit-on-error",
         filepath,
         "-E", "CYCLES",
         "-o", output_filepath,
diff --git a/tests/python/eevee_render_tests.py b/tests/python/eevee_render_tests.py
index a7130136d0a..a90d7730ace 100644
--- a/tests/python/eevee_render_tests.py
+++ b/tests/python/eevee_render_tests.py
@@ -103,6 +103,8 @@ def get_arguments(filepath, output_filepath):
         "-noaudio",
         "--factory-startup",
         "--enable-autoexec",
+        "--debug-memory",
+        "--debug-exit-on-error",
         filepath,
         "-E", "BLENDER_EEVEE",
         "-P",
diff --git a/tests/python/modules/render_report.py b/tests/python/modules/render_report.py
index 506c1a1518a..b6cafc2ee24 100755
--- a/tests/python/modules/render_report.py
+++ b/tests/python/modules/render_report.py
@@ -448,16 +448,17 @@ class Report:
             crash = False
             output = None
             try:
-                output = subprocess.check_output(command)
-            except subprocess.CalledProcessError as e:
-                crash = True
+                completed_process = subprocess.run(command, stdout=subprocess.PIPE)
+                if completed_process.returncode != 0:
+                    crash = True
+                output = completed_process.stdout
             except BaseException as e:
                 crash = True
 
             if verbose:
                 print(" ".join(command))
-                if output:
-                    print(output.decode("utf-8"))
+            if (verbose or crash) and output:
+                print(output.decode("utf-8"))
 
             # Detect missing filepaths and consider those errors
             for filepath, output_filepath in zip(remaining_filepaths[:], output_filepaths):
diff --git a/tests/python/modules/test_utils.py b/tests/python/modules/test_utils.py
index e31db05ba61..9c23d511813 100755
--- a/tests/python/modules/test_utils.py
+++ b/tests/python/modules/test_utils.py
@@ -79,6 +79,8 @@ class AbstractBlenderRunnerTest(unittest.TestCase):
             '-noaudio',
             '--factory-startup',
             '--enable-autoexec',
+            '--debug-memory',
+            '--debug-exit-on-error',
         ]
 
         if blendfile:
diff --git a/tests/python/opengl_draw_tests.py b/tests/python/opengl_draw_tests.py
index ab4df63afd9..a2d9d510382 100644
--- a/tests/python/opengl_draw_tests.py
+++ b/tests/python/opengl_draw_tests.py
@@ -39,6 +39,8 @@ def get_arguments(filepath, output_filepath):
         "-noaudio",
         "--factory-startup",
         "--enable-autoexec",
+        "--debug-memory",
+        "--debug-exit-on-error",
         filepath,
         "-P",
         os.path.realpath(__file__),
diff --git a/tests/python/view_layer/CMakeLists.txt b/tests/python/view_layer/CMakeLists.txt
index 3f38ee4675f..3cac2b6d85f 100644
--- a/tests/python/view_layer/CMakeLists.txt
+++ b/tests/python/view_layer/CMakeLists.txt
@@ -30,7 +30,7 @@ execute_process(COMMAND ${CMAKE_COMMAND} -E make_directory ${TEST_OUT_DIR})
 # endif()
 
 # for testing with valgrind prefix: valgrind --track-origins=yes --error-limit=no
-set(TEST_BLENDER_EXE $<TARGET_FILE:blender> --background -noaudio --factory-startup --env-system-scripts ${CMAKE_SOURCE_DIR}/release/scripts)
+set(TEST_BLENDER_EXE $<TARGET_FILE:blender> --background -noaudio --factory-startup --debug-memory --debug-exit-on-error --env-system-scripts ${CMAKE_SOURCE_DIR}/release/scripts)
 
 
 # ------------------------------------------------------------------------------
diff --git a/tests/python/workbench_render_tests.py b/tests/python/workbench_render_tests.py
index 155b54098a8..7c9842d5733 100644
--- a/tests/python/workbench_render_tests.py
+++ b/tests/python/workbench_render_tests.py
@@ -39,6 +39,8 @@ def get_arguments(filepath, output_filepath):
         "-noaudio",
         "--factory-startup",
         "--enable-autoexec",
+        "--debug-memory",
+        "--debug-exit-on-error",
         filepath,
         "-E", "BLENDER_WORKBENCH",
         "-P",