From 93901e7f0a05ba471f3b4c0201500d9dfcd68c2c Mon Sep 17 00:00:00 2001 From: Brecht Van Lommel Date: Fri, 10 May 2019 23:00:35 +0200 Subject: Tests: speed up render tests by running multiple in the same process Blender startup time and shader compilation is a big factor when running hundreds of tests, so now all renders in the same ctest run in the same process. If a test crashes, the remaining tests in the same category will be marked as skipped. Benchmarked on a quad core with ctest -j8. cycles: 118.1s -> 94.3s eevee: 66.2s -> 29.2s workbench: 31.7s -> 8.6s --- tests/python/cycles_render_tests.py | 97 +++++++++++++++++----------------- tests/python/eevee_render_tests.py | 66 ++++++++++++----------- tests/python/modules/render_report.py | 73 +++++++++++++------------ tests/python/opengl_draw_tests.py | 79 ++++++++++++++------------- tests/python/workbench_render_tests.py | 66 ++++++++++++----------- 5 files changed, 201 insertions(+), 180 deletions(-) (limited to 'tests/python') diff --git a/tests/python/cycles_render_tests.py b/tests/python/cycles_render_tests.py index 36f5459c2f7..a66409d85c3 100755 --- a/tests/python/cycles_render_tests.py +++ b/tests/python/cycles_render_tests.py @@ -9,77 +9,76 @@ import subprocess import sys -def render_file(filepath, output_filepath): - dirname = os.path.dirname(filepath) - basedir = os.path.dirname(dirname) - subject = os.path.basename(dirname) - - frame_filepath = output_filepath + '0001.png' - - common_args = [ - "-noaudio", - "--factory-startup", - "--enable-autoexec", - filepath, - "-E", "CYCLES", - "-o", output_filepath, - "-F", "PNG"] +def render_files(filepaths, output_filepaths): + command = [BLENDER, "--background"] # OSL and GPU examples # custom_args += ["--python-expr", "import bpy; bpy.context.scene.cycles.shading_system = True"] # custom_args += ["--python-expr", "import bpy; bpy.context.scene.cycles.device = 'GPU'"] custom_args = os.getenv('CYCLESTEST_ARGS') custom_args = shlex.split(custom_args) if custom_args else [] - common_args += custom_args - - if subject == 'opengl': - command = [BLENDER, "--window-geometry", "0", "0", "1", "1"] - command += common_args - command += ['--python', os.path.join(basedir, "util", "render_opengl.py")] - elif subject == 'bake': - command = [BLENDER, "--background"] - command += common_args - command += ['--python', os.path.join(basedir, "util", "render_bake.py")] - elif subject == 'denoise_animation': - command = [BLENDER, "--background"] - command += common_args - command += ['--python', os.path.join(basedir, "util", "render_denoise.py")] - else: - command = [BLENDER, "--background"] - command += common_args - command += ["-f", "1"] + for filepath, output_filepath in zip(filepaths, output_filepaths): + dirname = os.path.dirname(filepath) + basedir = os.path.dirname(dirname) + subject = os.path.basename(dirname) + + frame_filepath = output_filepath + '0001.png' + + common_args = [ + "-noaudio", + "--factory-startup", + "--enable-autoexec", + filepath, + "-E", "CYCLES", + "-o", output_filepath, + "-F", "PNG"] + + common_args += custom_args + + if subject == 'bake': + command.extend(common_args) + command.extend(['--python', os.path.join(basedir, "util", "render_bake.py")]) + elif subject == 'denoise_animation': + command.extend(common_args) + command.extend(['--python', os.path.join(basedir, "util", "render_denoise.py")]) + else: + command.extend(common_args) + command.extend(["-f", "1"]) + + error = None try: # Success output = subprocess.check_output(command) - if os.path.exists(frame_filepath): - shutil.copy(frame_filepath, output_filepath) - os.remove(frame_filepath) if VERBOSE: print(" ".join(command)) print(output.decode("utf-8")) - return None except subprocess.CalledProcessError as e: # Error - if os.path.exists(frame_filepath): - os.remove(frame_filepath) if VERBOSE: print(" ".join(command)) print(e.output.decode("utf-8")) - if b"Error: engine not found" in e.output: - return "NO_ENGINE" - elif b"blender probably wont start" in e.output: - return "NO_START" - return "CRASH" + error = "CRASH" except BaseException as e: # Crash - if os.path.exists(frame_filepath): - os.remove(frame_filepath) if VERBOSE: print(" ".join(command)) - print(e) - return "CRASH" + print(e.decode("utf-8")) + error = "CRASH" + + # Detect missing filepaths and consider those errors + errors = [] + for output_filepath in output_filepaths: + frame_filepath = output_filepath + '0001.png' + if os.path.exists(frame_filepath): + shutil.copy(frame_filepath, output_filepath) + os.remove(frame_filepath) + errors.append(None) + else: + errors.append(error) + error = 'SKIPPED' + return errors def create_argparse(): parser = argparse.ArgumentParser() @@ -108,7 +107,7 @@ def main(): report.set_pixelated(True) report.set_reference_dir("cycles_renders") report.set_compare_engines('cycles', 'eevee') - ok = report.run(test_dir, render_file) + ok = report.run(test_dir, render_files) sys.exit(not ok) diff --git a/tests/python/eevee_render_tests.py b/tests/python/eevee_render_tests.py index c0536e05164..d2c37dbcb2e 100755 --- a/tests/python/eevee_render_tests.py +++ b/tests/python/eevee_render_tests.py @@ -49,57 +49,61 @@ if inside_blender: sys.exit(1) -def render_file(filepath, output_filepath): - dirname = os.path.dirname(filepath) - basedir = os.path.dirname(dirname) - subject = os.path.basename(dirname) - - frame_filepath = output_filepath + '0001.png' - +def render_files(filepaths, output_filepaths): command = [ BLENDER, "--background", "-noaudio", "--factory-startup", - "--enable-autoexec", - filepath, - "-E", "BLENDER_EEVEE", - "-P", - os.path.realpath(__file__), - "-o", output_filepath, - "-F", "PNG", - "-f", "1"] + "--enable-autoexec"] + + for filepath, output_filepath in zip(filepaths, output_filepaths): + frame_filepath = output_filepath + '0001.png' + if os.path.exists(frame_filepath): + os.remove(frame_filepath) + command.extend([ + filepath, + "-E", "BLENDER_EEVEE", + "-P", + os.path.realpath(__file__), + "-o", output_filepath, + "-F", "PNG", + "-f", "1"]) + + error = None try: # Success output = subprocess.check_output(command) - if os.path.exists(frame_filepath): - shutil.copy(frame_filepath, output_filepath) - os.remove(frame_filepath) if VERBOSE: print(" ".join(command)) print(output.decode("utf-8")) - return None except subprocess.CalledProcessError as e: # Error - if os.path.exists(frame_filepath): - os.remove(frame_filepath) if VERBOSE: print(" ".join(command)) print(e.output.decode("utf-8")) - if b"Error: engine not found" in e.output: - return "NO_ENGINE" - elif b"blender probably wont start" in e.output: - return "NO_START" - return "CRASH" + error = "CRASH" except BaseException as e: # Crash - if os.path.exists(frame_filepath): - os.remove(frame_filepath) if VERBOSE: print(" ".join(command)) - print(e) - return "CRASH" + print(e.decode("utf-8")) + error = "CRASH" + + # Detect missing filepaths and consider those errors + errors = [] + for output_filepath in output_filepaths: + frame_filepath = output_filepath + '0001.png' + if os.path.exists(frame_filepath): + shutil.copy(frame_filepath, output_filepath) + os.remove(frame_filepath) + errors.append(None) + else: + errors.append(error) + error = 'SKIPPED' + + return errors def create_argparse(): @@ -129,7 +133,7 @@ def main(): report.set_pixelated(True) report.set_reference_dir("eevee_renders") report.set_compare_engines('eevee', 'cycles') - ok = report.run(test_dir, render_file) + ok = report.run(test_dir, render_files) sys.exit(not ok) diff --git a/tests/python/modules/render_report.py b/tests/python/modules/render_report.py index 5a2baa354c7..7dfc74904a7 100755 --- a/tests/python/modules/render_report.py +++ b/tests/python/modules/render_report.py @@ -374,41 +374,48 @@ class Report: return not failed - def _run_test(self, filepath, render_cb): - testname = test_get_name(filepath) - print_message(testname, 'SUCCESS', 'RUN') - time_start = time.time() - tmp_filepath = os.path.join(self.output_dir, "tmp_" + testname) + def _run_tests(self, filepaths, render_cb): + # Run all tests together for performance, since Blender + # startup time is a significant factor. + tmp_filepaths = [] + for filepath in filepaths: + testname = test_get_name(filepath) + print_message(testname, 'SUCCESS', 'RUN') + tmp_filepaths.append(os.path.join(self.output_dir, "tmp_" + testname)) - error = render_cb(filepath, tmp_filepath) - status = "FAIL" - if not error: - if not self._diff_output(filepath, tmp_filepath): - error = "VERIFY" + run_errors = render_cb(filepaths, tmp_filepaths) + errors = [] - if os.path.exists(tmp_filepath): - os.remove(tmp_filepath) + for error, filepath, tmp_filepath in zip(run_errors, filepaths, tmp_filepaths): + if not error: + if os.path.getsize(tmp_filepath) == 0: + error = "VERIFY" + elif not self._diff_output(filepath, tmp_filepath): + error = "VERIFY" - time_end = time.time() - elapsed_ms = int((time_end - time_start) * 1000) - if not error: - print_message("{} ({} ms)" . format(testname, elapsed_ms), - 'SUCCESS', 'OK') - else: - if error == "NO_ENGINE": - print_message("Can't perform tests because the render engine failed to load!") - return error - elif error == "NO_START": - print_message('Can not perform tests because blender fails to start.', - 'Make sure INSTALL target was run.') - return error - elif error == 'VERIFY': - print_message("Rendered result is different from reference image") + if os.path.exists(tmp_filepath): + os.remove(tmp_filepath) + + errors.append(error) + + testname = test_get_name(filepath) + if not error: + print_message(testname, 'SUCCESS', 'OK') else: - print_message("Unknown error %r" % error) - print_message("{} ({} ms)" . format(testname, elapsed_ms), - 'FAILURE', 'FAILED') - return error + if error == "SKIPPED": + print_message("Skipped after previous render caused error") + elif error == "NO_ENGINE": + print_message("Can't perform tests because the render engine failed to load!") + elif error == "NO_START": + print_message('Can not perform tests because blender fails to start.', + 'Make sure INSTALL target was run.') + elif error == 'VERIFY': + print_message("Rendered result is different from reference image") + else: + print_message("Unknown error %r" % error) + print_message(testname, 'FAILURE', 'FAILED') + + return errors def _run_all_tests(self, dirname, dirpath, render_cb): passed_tests = [] @@ -419,8 +426,8 @@ class Report: format(len(all_files)), 'SUCCESS', "==========") time_start = time.time() - for filepath in all_files: - error = self._run_test(filepath, render_cb) + errors = self._run_tests(all_files, render_cb) + for filepath, error in zip(all_files, errors): testname = test_get_name(filepath) if error: if error == "NO_ENGINE": diff --git a/tests/python/opengl_draw_tests.py b/tests/python/opengl_draw_tests.py index 9913f875689..1cc5ddecf1d 100755 --- a/tests/python/opengl_draw_tests.py +++ b/tests/python/opengl_draw_tests.py @@ -31,41 +31,48 @@ if inside_blender: sys.exit(0) -def render_file(filepath, output_filepath): - command = ( - BLENDER, - "--no-window-focus", - "--window-geometry", - "0", "0", "1024", "768", - "-noaudio", - "--factory-startup", - "--enable-autoexec", - filepath, - "-P", - os.path.realpath(__file__), - "--", - output_filepath) - - try: - # Success - output = subprocess.check_output(command) - if VERBOSE: - print(output.decode("utf-8")) - return None - except subprocess.CalledProcessError as e: - # Error - if os.path.exists(output_filepath): - os.remove(output_filepath) - if VERBOSE: - print(e.output.decode("utf-8")) - return "CRASH" - except BaseException as e: - # Crash - if os.path.exists(output_filepath): - os.remove(output_filepath) - if VERBOSE: - print(e) - return "CRASH" +def render_files(filepaths, output_filepaths): + errors = [] + + for filepath, output_filepath in zip(filepaths, output_filepaths): + command = ( + BLENDER, + "--no-window-focus", + "--window-geometry", + "0", "0", "1024", "768", + "-noaudio", + "--factory-startup", + "--enable-autoexec", + filepath, + "-P", + os.path.realpath(__file__), + "--", + output_filepath) + + error = None + try: + # Success + output = subprocess.check_output(command) + if VERBOSE: + print(output.decode("utf-8")) + except subprocess.CalledProcessError as e: + # Error + if os.path.exists(output_filepath): + os.remove(output_filepath) + if VERBOSE: + print(e.output.decode("utf-8")) + error = "CRASH" + except BaseException as e: + # Crash + if os.path.exists(output_filepath): + os.remove(output_filepath) + if VERBOSE: + print(e) + error = "CRASH" + + errors.append(error) + + return errors def create_argparse(): @@ -92,7 +99,7 @@ def main(): from modules import render_report report = render_report.Report("OpenGL Draw Test Report", output_dir, idiff) - ok = report.run(test_dir, render_file) + ok = report.run(test_dir, render_files) sys.exit(not ok) diff --git a/tests/python/workbench_render_tests.py b/tests/python/workbench_render_tests.py index 1a0d639bccd..7514443ed57 100755 --- a/tests/python/workbench_render_tests.py +++ b/tests/python/workbench_render_tests.py @@ -34,57 +34,61 @@ if inside_blender: sys.exit(1) -def render_file(filepath, output_filepath): - dirname = os.path.dirname(filepath) - basedir = os.path.dirname(dirname) - subject = os.path.basename(dirname) - - frame_filepath = output_filepath + '0001.png' - +def render_files(filepaths, output_filepaths): command = [ BLENDER, "--background", "-noaudio", "--factory-startup", - "--enable-autoexec", - filepath, - "-E", "BLENDER_WORKBENCH", - "-P", - os.path.realpath(__file__), - "-o", output_filepath, - "-F", "PNG", - "-f", "1"] + "--enable-autoexec"] + + for filepath, output_filepath in zip(filepaths, output_filepaths): + frame_filepath = output_filepath + '0001.png' + if os.path.exists(frame_filepath): + os.remove(frame_filepath) + command.extend([ + filepath, + "-E", "BLENDER_WORKBENCH", + "-P", + os.path.realpath(__file__), + "-o", output_filepath, + "-F", "PNG", + "-f", "1"]) + + error = None try: # Success output = subprocess.check_output(command) - if os.path.exists(frame_filepath): - shutil.copy(frame_filepath, output_filepath) - os.remove(frame_filepath) if VERBOSE: print(" ".join(command)) print(output.decode("utf-8")) - return None except subprocess.CalledProcessError as e: # Error - if os.path.exists(frame_filepath): - os.remove(frame_filepath) if VERBOSE: print(" ".join(command)) print(e.output.decode("utf-8")) - if b"Error: engine not found" in e.output: - return "NO_ENGINE" - elif b"blender probably wont start" in e.output: - return "NO_START" - return "CRASH" + error = "CRASH" except BaseException as e: # Crash - if os.path.exists(frame_filepath): - os.remove(frame_filepath) if VERBOSE: print(" ".join(command)) - print(e) - return "CRASH" + print(e.decode("utf-8")) + error = "CRASH" + + # Detect missing filepaths and consider those errors + errors = [] + for output_filepath in output_filepaths: + frame_filepath = output_filepath + '0001.png' + if os.path.exists(frame_filepath): + shutil.copy(frame_filepath, output_filepath) + os.remove(frame_filepath) + errors.append(None) + else: + errors.append(error) + error = 'SKIPPED' + + return errors def create_argparse(): @@ -114,7 +118,7 @@ def main(): report.set_pixelated(True) report.set_reference_dir("workbench_renders") report.set_compare_engines('workbench', 'eevee') - ok = report.run(test_dir, render_file) + ok = report.run(test_dir, render_files) sys.exit(not ok) -- cgit v1.2.3