From 3b5126038785ef2333627fad07e498d0eb718ac8 Mon Sep 17 00:00:00 2001 From: Brecht Van Lommel Date: Thu, 16 May 2019 15:48:30 +0200 Subject: Revert "Tests: speed up render tests by running multiple in the same process" This makes finding the crashing tests harder, reverting until there is a better solution. This reverts commit 93901e7f0a05ba471f3b4c0201500d9dfcd68c2c. --- tests/python/cycles_render_tests.py | 97 +++++++++++++++++----------------- tests/python/eevee_render_tests.py | 66 +++++++++++------------ tests/python/modules/render_report.py | 73 ++++++++++++------------- tests/python/opengl_draw_tests.py | 79 +++++++++++++-------------- tests/python/workbench_render_tests.py | 66 +++++++++++------------ 5 files changed, 180 insertions(+), 201 deletions(-) (limited to 'tests') diff --git a/tests/python/cycles_render_tests.py b/tests/python/cycles_render_tests.py index a66409d85c3..36f5459c2f7 100755 --- a/tests/python/cycles_render_tests.py +++ b/tests/python/cycles_render_tests.py @@ -9,76 +9,77 @@ import subprocess import sys -def render_files(filepaths, output_filepaths): - command = [BLENDER, "--background"] +def render_file(filepath, output_filepath): + dirname = os.path.dirname(filepath) + basedir = os.path.dirname(dirname) + subject = os.path.basename(dirname) + + frame_filepath = output_filepath + '0001.png' + + common_args = [ + "-noaudio", + "--factory-startup", + "--enable-autoexec", + filepath, + "-E", "CYCLES", + "-o", output_filepath, + "-F", "PNG"] # OSL and GPU examples # custom_args += ["--python-expr", "import bpy; bpy.context.scene.cycles.shading_system = True"] # custom_args += ["--python-expr", "import bpy; bpy.context.scene.cycles.device = 'GPU'"] custom_args = os.getenv('CYCLESTEST_ARGS') custom_args = shlex.split(custom_args) if custom_args else [] + common_args += custom_args + + if subject == 'opengl': + command = [BLENDER, "--window-geometry", "0", "0", "1", "1"] + command += common_args + command += ['--python', os.path.join(basedir, "util", "render_opengl.py")] + elif subject == 'bake': + command = [BLENDER, "--background"] + command += common_args + command += ['--python', os.path.join(basedir, "util", "render_bake.py")] + elif subject == 'denoise_animation': + command = [BLENDER, "--background"] + command += common_args + command += ['--python', os.path.join(basedir, "util", "render_denoise.py")] + else: + command = [BLENDER, "--background"] + command += common_args + command += ["-f", "1"] - for filepath, output_filepath in zip(filepaths, output_filepaths): - dirname = os.path.dirname(filepath) - basedir = os.path.dirname(dirname) - subject = os.path.basename(dirname) - - frame_filepath = output_filepath + '0001.png' - - common_args = [ - "-noaudio", - "--factory-startup", - "--enable-autoexec", - filepath, - "-E", "CYCLES", - "-o", output_filepath, - "-F", "PNG"] - - common_args += custom_args - - if subject == 'bake': - command.extend(common_args) - command.extend(['--python', os.path.join(basedir, "util", "render_bake.py")]) - elif subject == 'denoise_animation': - command.extend(common_args) - command.extend(['--python', os.path.join(basedir, "util", "render_denoise.py")]) - else: - command.extend(common_args) - command.extend(["-f", "1"]) - - error = None try: # Success output = subprocess.check_output(command) + if os.path.exists(frame_filepath): + shutil.copy(frame_filepath, output_filepath) + os.remove(frame_filepath) if VERBOSE: print(" ".join(command)) print(output.decode("utf-8")) + return None except subprocess.CalledProcessError as e: # Error + if os.path.exists(frame_filepath): + os.remove(frame_filepath) if VERBOSE: print(" ".join(command)) print(e.output.decode("utf-8")) - error = "CRASH" + if b"Error: engine not found" in e.output: + return "NO_ENGINE" + elif b"blender probably wont start" in e.output: + return "NO_START" + return "CRASH" except BaseException as e: # Crash - if VERBOSE: - print(" ".join(command)) - print(e.decode("utf-8")) - error = "CRASH" - - # Detect missing filepaths and consider those errors - errors = [] - for output_filepath in output_filepaths: - frame_filepath = output_filepath + '0001.png' if os.path.exists(frame_filepath): - shutil.copy(frame_filepath, output_filepath) os.remove(frame_filepath) - errors.append(None) - else: - errors.append(error) - error = 'SKIPPED' + if VERBOSE: + print(" ".join(command)) + print(e) + return "CRASH" - return errors def create_argparse(): parser = argparse.ArgumentParser() @@ -107,7 +108,7 @@ def main(): report.set_pixelated(True) report.set_reference_dir("cycles_renders") report.set_compare_engines('cycles', 'eevee') - ok = report.run(test_dir, render_files) + ok = report.run(test_dir, render_file) sys.exit(not ok) diff --git a/tests/python/eevee_render_tests.py b/tests/python/eevee_render_tests.py index f324b22b682..8eaf04f8bea 100755 --- a/tests/python/eevee_render_tests.py +++ b/tests/python/eevee_render_tests.py @@ -49,61 +49,57 @@ if inside_blender: sys.exit(1) -def render_files(filepaths, output_filepaths): +def render_file(filepath, output_filepath): + dirname = os.path.dirname(filepath) + basedir = os.path.dirname(dirname) + subject = os.path.basename(dirname) + + frame_filepath = output_filepath + '0001.png' + command = [ BLENDER, "--background", "-noaudio", "--factory-startup", - "--enable-autoexec"] - - for filepath, output_filepath in zip(filepaths, output_filepaths): - frame_filepath = output_filepath + '0001.png' - if os.path.exists(frame_filepath): - os.remove(frame_filepath) + "--enable-autoexec", + filepath, + "-E", "BLENDER_EEVEE", + "-P", + os.path.realpath(__file__), + "-o", output_filepath, + "-F", "PNG", + "-f", "1"] - command.extend([ - filepath, - "-E", "BLENDER_EEVEE", - "-P", - os.path.realpath(__file__), - "-o", output_filepath, - "-F", "PNG", - "-f", "1"]) - - error = None try: # Success output = subprocess.check_output(command) + if os.path.exists(frame_filepath): + shutil.copy(frame_filepath, output_filepath) + os.remove(frame_filepath) if VERBOSE: print(" ".join(command)) print(output.decode("utf-8")) + return None except subprocess.CalledProcessError as e: # Error + if os.path.exists(frame_filepath): + os.remove(frame_filepath) if VERBOSE: print(" ".join(command)) print(e.output.decode("utf-8")) - error = "CRASH" + if b"Error: engine not found" in e.output: + return "NO_ENGINE" + elif b"blender probably wont start" in e.output: + return "NO_START" + return "CRASH" except BaseException as e: # Crash - if VERBOSE: - print(" ".join(command)) - print(e.decode("utf-8")) - error = "CRASH" - - # Detect missing filepaths and consider those errors - errors = [] - for output_filepath in output_filepaths: - frame_filepath = output_filepath + '0001.png' if os.path.exists(frame_filepath): - shutil.copy(frame_filepath, output_filepath) os.remove(frame_filepath) - errors.append(None) - else: - errors.append(error) - error = 'SKIPPED' - - return errors + if VERBOSE: + print(" ".join(command)) + print(e) + return "CRASH" def create_argparse(): @@ -133,7 +129,7 @@ def main(): report.set_pixelated(True) report.set_reference_dir("eevee_renders") report.set_compare_engines('eevee', 'cycles') - ok = report.run(test_dir, render_files) + ok = report.run(test_dir, render_file) sys.exit(not ok) diff --git a/tests/python/modules/render_report.py b/tests/python/modules/render_report.py index 7dfc74904a7..5a2baa354c7 100755 --- a/tests/python/modules/render_report.py +++ b/tests/python/modules/render_report.py @@ -374,48 +374,41 @@ class Report: return not failed - def _run_tests(self, filepaths, render_cb): - # Run all tests together for performance, since Blender - # startup time is a significant factor. - tmp_filepaths = [] - for filepath in filepaths: - testname = test_get_name(filepath) - print_message(testname, 'SUCCESS', 'RUN') - tmp_filepaths.append(os.path.join(self.output_dir, "tmp_" + testname)) - - run_errors = render_cb(filepaths, tmp_filepaths) - errors = [] - - for error, filepath, tmp_filepath in zip(run_errors, filepaths, tmp_filepaths): - if not error: - if os.path.getsize(tmp_filepath) == 0: - error = "VERIFY" - elif not self._diff_output(filepath, tmp_filepath): - error = "VERIFY" + def _run_test(self, filepath, render_cb): + testname = test_get_name(filepath) + print_message(testname, 'SUCCESS', 'RUN') + time_start = time.time() + tmp_filepath = os.path.join(self.output_dir, "tmp_" + testname) - if os.path.exists(tmp_filepath): - os.remove(tmp_filepath) + error = render_cb(filepath, tmp_filepath) + status = "FAIL" + if not error: + if not self._diff_output(filepath, tmp_filepath): + error = "VERIFY" - errors.append(error) + if os.path.exists(tmp_filepath): + os.remove(tmp_filepath) - testname = test_get_name(filepath) - if not error: - print_message(testname, 'SUCCESS', 'OK') + time_end = time.time() + elapsed_ms = int((time_end - time_start) * 1000) + if not error: + print_message("{} ({} ms)" . format(testname, elapsed_ms), + 'SUCCESS', 'OK') + else: + if error == "NO_ENGINE": + print_message("Can't perform tests because the render engine failed to load!") + return error + elif error == "NO_START": + print_message('Can not perform tests because blender fails to start.', + 'Make sure INSTALL target was run.') + return error + elif error == 'VERIFY': + print_message("Rendered result is different from reference image") else: - if error == "SKIPPED": - print_message("Skipped after previous render caused error") - elif error == "NO_ENGINE": - print_message("Can't perform tests because the render engine failed to load!") - elif error == "NO_START": - print_message('Can not perform tests because blender fails to start.', - 'Make sure INSTALL target was run.') - elif error == 'VERIFY': - print_message("Rendered result is different from reference image") - else: - print_message("Unknown error %r" % error) - print_message(testname, 'FAILURE', 'FAILED') - - return errors + print_message("Unknown error %r" % error) + print_message("{} ({} ms)" . format(testname, elapsed_ms), + 'FAILURE', 'FAILED') + return error def _run_all_tests(self, dirname, dirpath, render_cb): passed_tests = [] @@ -426,8 +419,8 @@ class Report: format(len(all_files)), 'SUCCESS', "==========") time_start = time.time() - errors = self._run_tests(all_files, render_cb) - for filepath, error in zip(all_files, errors): + for filepath in all_files: + error = self._run_test(filepath, render_cb) testname = test_get_name(filepath) if error: if error == "NO_ENGINE": diff --git a/tests/python/opengl_draw_tests.py b/tests/python/opengl_draw_tests.py index 1cc5ddecf1d..9913f875689 100755 --- a/tests/python/opengl_draw_tests.py +++ b/tests/python/opengl_draw_tests.py @@ -31,48 +31,41 @@ if inside_blender: sys.exit(0) -def render_files(filepaths, output_filepaths): - errors = [] - - for filepath, output_filepath in zip(filepaths, output_filepaths): - command = ( - BLENDER, - "--no-window-focus", - "--window-geometry", - "0", "0", "1024", "768", - "-noaudio", - "--factory-startup", - "--enable-autoexec", - filepath, - "-P", - os.path.realpath(__file__), - "--", - output_filepath) - - error = None - try: - # Success - output = subprocess.check_output(command) - if VERBOSE: - print(output.decode("utf-8")) - except subprocess.CalledProcessError as e: - # Error - if os.path.exists(output_filepath): - os.remove(output_filepath) - if VERBOSE: - print(e.output.decode("utf-8")) - error = "CRASH" - except BaseException as e: - # Crash - if os.path.exists(output_filepath): - os.remove(output_filepath) - if VERBOSE: - print(e) - error = "CRASH" - - errors.append(error) - - return errors +def render_file(filepath, output_filepath): + command = ( + BLENDER, + "--no-window-focus", + "--window-geometry", + "0", "0", "1024", "768", + "-noaudio", + "--factory-startup", + "--enable-autoexec", + filepath, + "-P", + os.path.realpath(__file__), + "--", + output_filepath) + + try: + # Success + output = subprocess.check_output(command) + if VERBOSE: + print(output.decode("utf-8")) + return None + except subprocess.CalledProcessError as e: + # Error + if os.path.exists(output_filepath): + os.remove(output_filepath) + if VERBOSE: + print(e.output.decode("utf-8")) + return "CRASH" + except BaseException as e: + # Crash + if os.path.exists(output_filepath): + os.remove(output_filepath) + if VERBOSE: + print(e) + return "CRASH" def create_argparse(): @@ -99,7 +92,7 @@ def main(): from modules import render_report report = render_report.Report("OpenGL Draw Test Report", output_dir, idiff) - ok = report.run(test_dir, render_files) + ok = report.run(test_dir, render_file) sys.exit(not ok) diff --git a/tests/python/workbench_render_tests.py b/tests/python/workbench_render_tests.py index 7514443ed57..1a0d639bccd 100755 --- a/tests/python/workbench_render_tests.py +++ b/tests/python/workbench_render_tests.py @@ -34,61 +34,57 @@ if inside_blender: sys.exit(1) -def render_files(filepaths, output_filepaths): +def render_file(filepath, output_filepath): + dirname = os.path.dirname(filepath) + basedir = os.path.dirname(dirname) + subject = os.path.basename(dirname) + + frame_filepath = output_filepath + '0001.png' + command = [ BLENDER, "--background", "-noaudio", "--factory-startup", - "--enable-autoexec"] - - for filepath, output_filepath in zip(filepaths, output_filepaths): - frame_filepath = output_filepath + '0001.png' - if os.path.exists(frame_filepath): - os.remove(frame_filepath) + "--enable-autoexec", + filepath, + "-E", "BLENDER_WORKBENCH", + "-P", + os.path.realpath(__file__), + "-o", output_filepath, + "-F", "PNG", + "-f", "1"] - command.extend([ - filepath, - "-E", "BLENDER_WORKBENCH", - "-P", - os.path.realpath(__file__), - "-o", output_filepath, - "-F", "PNG", - "-f", "1"]) - - error = None try: # Success output = subprocess.check_output(command) + if os.path.exists(frame_filepath): + shutil.copy(frame_filepath, output_filepath) + os.remove(frame_filepath) if VERBOSE: print(" ".join(command)) print(output.decode("utf-8")) + return None except subprocess.CalledProcessError as e: # Error + if os.path.exists(frame_filepath): + os.remove(frame_filepath) if VERBOSE: print(" ".join(command)) print(e.output.decode("utf-8")) - error = "CRASH" + if b"Error: engine not found" in e.output: + return "NO_ENGINE" + elif b"blender probably wont start" in e.output: + return "NO_START" + return "CRASH" except BaseException as e: # Crash - if VERBOSE: - print(" ".join(command)) - print(e.decode("utf-8")) - error = "CRASH" - - # Detect missing filepaths and consider those errors - errors = [] - for output_filepath in output_filepaths: - frame_filepath = output_filepath + '0001.png' if os.path.exists(frame_filepath): - shutil.copy(frame_filepath, output_filepath) os.remove(frame_filepath) - errors.append(None) - else: - errors.append(error) - error = 'SKIPPED' - - return errors + if VERBOSE: + print(" ".join(command)) + print(e) + return "CRASH" def create_argparse(): @@ -118,7 +114,7 @@ def main(): report.set_pixelated(True) report.set_reference_dir("workbench_renders") report.set_compare_engines('workbench', 'eevee') - ok = report.run(test_dir, render_files) + ok = report.run(test_dir, render_file) sys.exit(not ok) -- cgit v1.2.3