From 8a9912eaf81bab73a12621a4c0987c37a865fe50 Mon Sep 17 00:00:00 2001 From: Jacques Lucke Date: Wed, 26 Aug 2020 22:02:02 +0200 Subject: Tests: fail automated tests on memory leaks and other internal errors This adds a new `--debug-exit-on-error` flag. When it is set, Blender will abort with a non-zero exit code when there are internal errors. Currently, "internal errors" includes memory leaks detected by guardedalloc and error/fatal log entries in clog. The new flag is passed to Blender in various places where automated tests are run. Furthermore, the `--debug-memory` flag is used in tests, because that makes the verbose output more useful, when dealing with memory leaks. Reviewers: brecht, sergey Differential Revision: https://developer.blender.org/D8665 --- tests/python/collada/CMakeLists.txt | 6 +++--- tests/python/cycles_render_tests.py | 2 ++ tests/python/eevee_render_tests.py | 2 ++ tests/python/modules/render_report.py | 11 ++++++----- tests/python/modules/test_utils.py | 2 ++ tests/python/opengl_draw_tests.py | 2 ++ tests/python/view_layer/CMakeLists.txt | 2 +- tests/python/workbench_render_tests.py | 2 ++ 8 files changed, 20 insertions(+), 9 deletions(-) (limited to 'tests/python') diff --git a/tests/python/collada/CMakeLists.txt b/tests/python/collada/CMakeLists.txt index bf22de6b762..1b24875578e 100644 --- a/tests/python/collada/CMakeLists.txt +++ b/tests/python/collada/CMakeLists.txt @@ -36,12 +36,12 @@ execute_process(COMMAND ${CMAKE_COMMAND} -E make_directory ${TEST_OUT_DIR}) # all calls to blender use this if(APPLE) if(${CMAKE_GENERATOR} MATCHES "Xcode") - set(TEST_BLENDER_EXE_PARAMS --background -noaudio --factory-startup) + set(TEST_BLENDER_EXE_PARAMS --background -noaudio --factory-startup --debug-memory --debug-exit-on-error) else() - set(TEST_BLENDER_EXE_PARAMS --background -noaudio --factory-startup --env-system-scripts ${CMAKE_SOURCE_DIR}/release/scripts) + set(TEST_BLENDER_EXE_PARAMS --background -noaudio --factory-startup --debug-memory --debug-exit-on-error --env-system-scripts ${CMAKE_SOURCE_DIR}/release/scripts) endif() else() - set(TEST_BLENDER_EXE_PARAMS --background -noaudio --factory-startup --env-system-scripts ${CMAKE_SOURCE_DIR}/release/scripts) + set(TEST_BLENDER_EXE_PARAMS --background -noaudio --factory-startup --debug-memory --debug-exit-on-error --env-system-scripts ${CMAKE_SOURCE_DIR}/release/scripts) endif() # for testing with valgrind prefix: valgrind --track-origins=yes --error-limit=no diff --git a/tests/python/cycles_render_tests.py b/tests/python/cycles_render_tests.py index bdf4283eb3e..cc949248ce6 100644 --- a/tests/python/cycles_render_tests.py +++ b/tests/python/cycles_render_tests.py @@ -20,6 +20,8 @@ def get_arguments(filepath, output_filepath): "-noaudio", "--factory-startup", "--enable-autoexec", + "--debug-memory", + "--debug-exit-on-error", filepath, "-E", "CYCLES", "-o", output_filepath, diff --git a/tests/python/eevee_render_tests.py b/tests/python/eevee_render_tests.py index a7130136d0a..a90d7730ace 100644 --- a/tests/python/eevee_render_tests.py +++ b/tests/python/eevee_render_tests.py @@ -103,6 +103,8 @@ def get_arguments(filepath, output_filepath): "-noaudio", "--factory-startup", "--enable-autoexec", + "--debug-memory", + "--debug-exit-on-error", filepath, "-E", "BLENDER_EEVEE", "-P", diff --git a/tests/python/modules/render_report.py b/tests/python/modules/render_report.py index 506c1a1518a..b6cafc2ee24 100755 --- a/tests/python/modules/render_report.py +++ b/tests/python/modules/render_report.py @@ -448,16 +448,17 @@ class Report: crash = False output = None try: - output = subprocess.check_output(command) - except subprocess.CalledProcessError as e: - crash = True + completed_process = subprocess.run(command, stdout=subprocess.PIPE) + if completed_process.returncode != 0: + crash = True + output = completed_process.stdout except BaseException as e: crash = True if verbose: print(" ".join(command)) - if output: - print(output.decode("utf-8")) + if (verbose or crash) and output: + print(output.decode("utf-8")) # Detect missing filepaths and consider those errors for filepath, output_filepath in zip(remaining_filepaths[:], output_filepaths): diff --git a/tests/python/modules/test_utils.py b/tests/python/modules/test_utils.py index e31db05ba61..9c23d511813 100755 --- a/tests/python/modules/test_utils.py +++ b/tests/python/modules/test_utils.py @@ -79,6 +79,8 @@ class AbstractBlenderRunnerTest(unittest.TestCase): '-noaudio', '--factory-startup', '--enable-autoexec', + '--debug-memory', + '--debug-exit-on-error', ] if blendfile: diff --git a/tests/python/opengl_draw_tests.py b/tests/python/opengl_draw_tests.py index ab4df63afd9..a2d9d510382 100644 --- a/tests/python/opengl_draw_tests.py +++ b/tests/python/opengl_draw_tests.py @@ -39,6 +39,8 @@ def get_arguments(filepath, output_filepath): "-noaudio", "--factory-startup", "--enable-autoexec", + "--debug-memory", + "--debug-exit-on-error", filepath, "-P", os.path.realpath(__file__), diff --git a/tests/python/view_layer/CMakeLists.txt b/tests/python/view_layer/CMakeLists.txt index 3f38ee4675f..3cac2b6d85f 100644 --- a/tests/python/view_layer/CMakeLists.txt +++ b/tests/python/view_layer/CMakeLists.txt @@ -30,7 +30,7 @@ execute_process(COMMAND ${CMAKE_COMMAND} -E make_directory ${TEST_OUT_DIR}) # endif() # for testing with valgrind prefix: valgrind --track-origins=yes --error-limit=no -set(TEST_BLENDER_EXE $ --background -noaudio --factory-startup --env-system-scripts ${CMAKE_SOURCE_DIR}/release/scripts) +set(TEST_BLENDER_EXE $ --background -noaudio --factory-startup --debug-memory --debug-exit-on-error --env-system-scripts ${CMAKE_SOURCE_DIR}/release/scripts) # ------------------------------------------------------------------------------ diff --git a/tests/python/workbench_render_tests.py b/tests/python/workbench_render_tests.py index 155b54098a8..7c9842d5733 100644 --- a/tests/python/workbench_render_tests.py +++ b/tests/python/workbench_render_tests.py @@ -39,6 +39,8 @@ def get_arguments(filepath, output_filepath): "-noaudio", "--factory-startup", "--enable-autoexec", + "--debug-memory", + "--debug-exit-on-error", filepath, "-E", "BLENDER_WORKBENCH", "-P", -- cgit v1.2.3