diff options
| -rwxr-xr-x | tests/python/cycles_render_tests.py | 69 | ||||
| -rwxr-xr-x | tests/python/eevee_render_tests.py | 51 | ||||
| -rwxr-xr-x | tests/python/modules/render_report.py | 112 | ||||
| -rwxr-xr-x | tests/python/opengl_draw_tests.py | 36 | ||||
| -rwxr-xr-x | tests/python/workbench_render_tests.py | 50 |
5 files changed, 100 insertions, 218 deletions
diff --git a/tests/python/cycles_render_tests.py b/tests/python/cycles_render_tests.py index 349ba83f88a..79ba11fdd44 100755 --- a/tests/python/cycles_render_tests.py +++ b/tests/python/cycles_render_tests.py @@ -9,14 +9,13 @@ import subprocess import sys -def render_file(filepath, output_filepath): +def get_arguments(filepath, output_filepath): dirname = os.path.dirname(filepath) basedir = os.path.dirname(dirname) subject = os.path.basename(dirname) - frame_filepath = output_filepath + '0001.png' - - common_args = [ + args = [ + "--background", "-noaudio", "--factory-startup", "--enable-autoexec", @@ -29,57 +28,17 @@ def render_file(filepath, output_filepath): # custom_args += ["--python-expr", "import bpy; bpy.context.scene.cycles.shading_system = True"] # custom_args += ["--python-expr", "import bpy; bpy.context.scene.cycles.device = 'GPU'"] custom_args = os.getenv('CYCLESTEST_ARGS') - custom_args = shlex.split(custom_args) if custom_args else [] - common_args += custom_args + if custom_args: + args.extend(shlex.split(custom_args)) - if subject == 'opengl': - command = [BLENDER, "--window-geometry", "0", "0", "1", "1"] - command += common_args - command += ['--python', os.path.join(basedir, "util", "render_opengl.py")] - elif subject == 'bake': - command = [BLENDER, "--background"] - command += common_args - command += ['--python', os.path.join(basedir, "util", "render_bake.py")] + if subject == 'bake': + args.extend(['--python', os.path.join(basedir, "util", "render_bake.py")]) elif subject == 'denoise_animation': - command = [BLENDER, "--background"] - command += common_args - command += ['--python', os.path.join(basedir, "util", "render_denoise.py")] + args.extend(['--python', os.path.join(basedir, "util", "render_denoise.py")]) else: - command = [BLENDER, "--background"] - command += common_args - command += ["-f", "1"] - - try: - # Success - output = subprocess.check_output(command) - if os.path.exists(frame_filepath): - shutil.copy(frame_filepath, output_filepath) - os.remove(frame_filepath) - if VERBOSE: - print(" ".join(command)) - print(output.decode("utf-8")) - return None - except subprocess.CalledProcessError as e: - # Error - if os.path.exists(frame_filepath): - os.remove(frame_filepath) - if VERBOSE: - print(" ".join(command)) - print(e.output.decode("utf-8")) - if b"Error: engine not found" in e.output: - return "NO_ENGINE" - elif b"blender probably wont start" in e.output: - return "NO_START" - return "CRASH" - except BaseException as e: - # Crash - if os.path.exists(frame_filepath): - os.remove(frame_filepath) - if VERBOSE: - print(" ".join(command)) - print(e) - return "CRASH" + args.extend(["-f", "1"]) + return args def create_argparse(): parser = argparse.ArgumentParser() @@ -94,11 +53,7 @@ def main(): parser = create_argparse() args = parser.parse_args() - global BLENDER, VERBOSE - - BLENDER = args.blender[0] - VERBOSE = os.environ.get("BLENDER_VERBOSE") is not None - + blender = args.blender[0] test_dir = args.testdir[0] idiff = args.idiff[0] output_dir = args.outdir[0] @@ -108,7 +63,7 @@ def main(): report.set_pixelated(True) report.set_reference_dir("cycles_renders") report.set_compare_engines('cycles', 'eevee') - ok = report.run(test_dir, render_file) + ok = report.run(test_dir, blender, get_arguments, batch=True) sys.exit(not ok) diff --git a/tests/python/eevee_render_tests.py b/tests/python/eevee_render_tests.py index 9fb8a5f41a8..0875e4b171a 100755 --- a/tests/python/eevee_render_tests.py +++ b/tests/python/eevee_render_tests.py @@ -8,7 +8,6 @@ import shutil import subprocess import sys - def setup(): import bpy @@ -46,15 +45,8 @@ if inside_blender: sys.exit(1) -def render_file(filepath, output_filepath): - dirname = os.path.dirname(filepath) - basedir = os.path.dirname(dirname) - subject = os.path.basename(dirname) - - frame_filepath = output_filepath + '0001.png' - - command = [ - BLENDER, +def get_arguments(filepath, output_filepath): + return [ "--background", "-noaudio", "--factory-startup", @@ -67,37 +59,6 @@ def render_file(filepath, output_filepath): "-F", "PNG", "-f", "1"] - try: - # Success - output = subprocess.check_output(command) - if os.path.exists(frame_filepath): - shutil.copy(frame_filepath, output_filepath) - os.remove(frame_filepath) - if VERBOSE: - print(" ".join(command)) - print(output.decode("utf-8")) - return None - except subprocess.CalledProcessError as e: - # Error - if os.path.exists(frame_filepath): - os.remove(frame_filepath) - if VERBOSE: - print(" ".join(command)) - print(e.output.decode("utf-8")) - if b"Error: engine not found" in e.output: - return "NO_ENGINE" - elif b"blender probably wont start" in e.output: - return "NO_START" - return "CRASH" - except BaseException as e: - # Crash - if os.path.exists(frame_filepath): - os.remove(frame_filepath) - if VERBOSE: - print(" ".join(command)) - print(e) - return "CRASH" - def create_argparse(): parser = argparse.ArgumentParser() @@ -112,11 +73,7 @@ def main(): parser = create_argparse() args = parser.parse_args() - global BLENDER, VERBOSE - - BLENDER = args.blender[0] - VERBOSE = os.environ.get("BLENDER_VERBOSE") is not None - + blender = args.blender[0] test_dir = args.testdir[0] idiff = args.idiff[0] output_dir = args.outdir[0] @@ -126,7 +83,7 @@ def main(): report.set_pixelated(True) report.set_reference_dir("eevee_renders") report.set_compare_engines('eevee', 'cycles') - ok = report.run(test_dir, render_file) + ok = report.run(test_dir, blender, get_arguments, batch=True) sys.exit(not ok) diff --git a/tests/python/modules/render_report.py b/tests/python/modules/render_report.py index 2f99a3b6292..15826f97400 100755 --- a/tests/python/modules/render_report.py +++ b/tests/python/modules/render_report.py @@ -134,10 +134,10 @@ class Report: def set_compare_engines(self, engine, other_engine): self.compare_engines = (engine, other_engine) - def run(self, dirpath, render_cb): + def run(self, dirpath, blender, arguments_cb, batch=False): # Run tests and output report. dirname = os.path.basename(dirpath) - ok = self._run_all_tests(dirname, dirpath, render_cb) + ok = self._run_all_tests(dirname, dirpath, blender, arguments_cb, batch) self._write_data(dirname) self._write_html() if self.compare_engines: @@ -399,43 +399,81 @@ class Report: return not failed - def _run_test(self, filepath, render_cb): - testname = test_get_name(filepath) - print_message(testname, 'SUCCESS', 'RUN') - time_start = time.time() - tmp_filepath = os.path.join(self.output_dir, "tmp_" + testname) + def _run_tests(self, filepaths, blender, arguments_cb, batch): + # Run multiple tests in a single Blender process since startup can be + # a significant factor. In case of crashes, re-run the remaining tests. + verbose = os.environ.get("BLENDER_VERBOSE") is not None - error = render_cb(filepath, tmp_filepath) - status = "FAIL" - if not error: - if not self._diff_output(filepath, tmp_filepath): - error = "VERIFY" + remaining_filepaths = filepaths[:] + errors = [] - if os.path.exists(tmp_filepath): - os.remove(tmp_filepath) + while len(remaining_filepaths) > 0: + command = [blender] + output_filepaths = [] - time_end = time.time() - elapsed_ms = int((time_end - time_start) * 1000) - if not error: - print_message("{} ({} ms)" . format(testname, elapsed_ms), - 'SUCCESS', 'OK') - else: - if error == "NO_ENGINE": - print_message("Can't perform tests because the render engine failed to load!") - return error - elif error == "NO_START": - print_message('Can not perform tests because blender fails to start.', - 'Make sure INSTALL target was run.') - return error - elif error == 'VERIFY': - print_message("Rendered result is different from reference image") - else: - print_message("Unknown error %r" % error) - print_message("{} ({} ms)" . format(testname, elapsed_ms), - 'FAILURE', 'FAILED') - return error + # Construct output filepaths and command to run + for filepath in remaining_filepaths: + testname = test_get_name(filepath) + print_message(testname, 'SUCCESS', 'RUN') - def _run_all_tests(self, dirname, dirpath, render_cb): + base_output_filepath = os.path.join(self.output_dir, "tmp_" + testname) + output_filepath = base_output_filepath + '0001.png' + output_filepaths.append(output_filepath) + + if os.path.exists(output_filepath): + os.remove(output_filepath) + + command.extend(arguments_cb(filepath, base_output_filepath)) + + # Only chain multiple commands for batch + if not batch: + break + + # Run process + crash = False + try: + output = subprocess.check_output(command) + except subprocess.CalledProcessError as e: + crash = True + except BaseException as e: + crash = True + + if verbose: + print(" ".join(command)) + print(output.decode("utf-8")) + + # Detect missing filepaths and consider those errors + for filepath, output_filepath in zip(remaining_filepaths[:], output_filepaths): + remaining_filepaths.pop(0) + + if crash: + # In case of crash, stop after missing files and re-render remaing + if not os.path.exists(output_filepath): + errors.append("CRASH") + print_message("Crash running Blender") + print_message(testname, 'FAILURE', 'FAILED') + break + + testname = test_get_name(filepath) + + if not os.path.exists(output_filepath) or os.path.getsize(output_filepath) == 0: + errors.append("NO OUTPUT") + print_message("No render result file found") + print_message(testname, 'FAILURE', 'FAILED') + elif not self._diff_output(filepath, output_filepath): + errors.append("VERIFY") + print_message("Render result is different from reference image") + print_message(testname, 'FAILURE', 'FAILED') + else: + errors.append(None) + print_message(testname, 'SUCCESS', 'OK') + + if os.path.exists(output_filepath): + os.remove(output_filepath) + + return errors + + def _run_all_tests(self, dirname, dirpath, blender, arguments_cb, batch): passed_tests = [] failed_tests = [] all_files = list(blend_list(dirpath)) @@ -444,8 +482,8 @@ class Report: format(len(all_files)), 'SUCCESS', "==========") time_start = time.time() - for filepath in all_files: - error = self._run_test(filepath, render_cb) + errors = self._run_tests(all_files, blender, arguments_cb, batch) + for filepath, error in zip(all_files, errors): testname = test_get_name(filepath) if error: if error == "NO_ENGINE": diff --git a/tests/python/opengl_draw_tests.py b/tests/python/opengl_draw_tests.py index b70111c042f..ab4df63afd9 100755 --- a/tests/python/opengl_draw_tests.py +++ b/tests/python/opengl_draw_tests.py @@ -31,9 +31,8 @@ if inside_blender: sys.exit(0) -def render_file(filepath, output_filepath): - command = ( - BLENDER, +def get_arguments(filepath, output_filepath): + return [ "--no-window-focus", "--window-geometry", "0", "0", "1024", "768", @@ -44,28 +43,7 @@ def render_file(filepath, output_filepath): "-P", os.path.realpath(__file__), "--", - output_filepath) - - try: - # Success - output = subprocess.check_output(command) - if VERBOSE: - print(output.decode("utf-8")) - return None - except subprocess.CalledProcessError as e: - # Error - if os.path.exists(output_filepath): - os.remove(output_filepath) - if VERBOSE: - print(e.output.decode("utf-8")) - return "CRASH" - except BaseException as e: - # Crash - if os.path.exists(output_filepath): - os.remove(output_filepath) - if VERBOSE: - print(e) - return "CRASH" + output_filepath + '0001.png'] def create_argparse(): @@ -81,18 +59,14 @@ def main(): parser = create_argparse() args = parser.parse_args() - global BLENDER, VERBOSE - - BLENDER = args.blender[0] - VERBOSE = os.environ.get("BLENDER_VERBOSE") is not None - + blender = args.blender[0] test_dir = args.testdir[0] idiff = args.idiff[0] output_dir = args.outdir[0] from modules import render_report report = render_report.Report("OpenGL Draw", output_dir, idiff) - ok = report.run(test_dir, render_file) + ok = report.run(test_dir, blender, get_arguments) sys.exit(not ok) diff --git a/tests/python/workbench_render_tests.py b/tests/python/workbench_render_tests.py index e43aac175b8..155b54098a8 100755 --- a/tests/python/workbench_render_tests.py +++ b/tests/python/workbench_render_tests.py @@ -33,15 +33,8 @@ if inside_blender: sys.exit(1) -def render_file(filepath, output_filepath): - dirname = os.path.dirname(filepath) - basedir = os.path.dirname(dirname) - subject = os.path.basename(dirname) - - frame_filepath = output_filepath + '0001.png' - - command = [ - BLENDER, +def get_arguments(filepath, output_filepath): + return [ "--background", "-noaudio", "--factory-startup", @@ -54,37 +47,6 @@ def render_file(filepath, output_filepath): "-F", "PNG", "-f", "1"] - try: - # Success - output = subprocess.check_output(command) - if os.path.exists(frame_filepath): - shutil.copy(frame_filepath, output_filepath) - os.remove(frame_filepath) - if VERBOSE: - print(" ".join(command)) - print(output.decode("utf-8")) - return None - except subprocess.CalledProcessError as e: - # Error - if os.path.exists(frame_filepath): - os.remove(frame_filepath) - if VERBOSE: - print(" ".join(command)) - print(e.output.decode("utf-8")) - if b"Error: engine not found" in e.output: - return "NO_ENGINE" - elif b"blender probably wont start" in e.output: - return "NO_START" - return "CRASH" - except BaseException as e: - # Crash - if os.path.exists(frame_filepath): - os.remove(frame_filepath) - if VERBOSE: - print(" ".join(command)) - print(e) - return "CRASH" - def create_argparse(): parser = argparse.ArgumentParser() @@ -99,11 +61,7 @@ def main(): parser = create_argparse() args = parser.parse_args() - global BLENDER, VERBOSE - - BLENDER = args.blender[0] - VERBOSE = os.environ.get("BLENDER_VERBOSE") is not None - + blender = args.blender[0] test_dir = args.testdir[0] idiff = args.idiff[0] output_dir = args.outdir[0] @@ -113,7 +71,7 @@ def main(): report.set_pixelated(True) report.set_reference_dir("workbench_renders") report.set_compare_engines('workbench', 'eevee') - ok = report.run(test_dir, render_file) + ok = report.run(test_dir, blender, get_arguments, batch=True) sys.exit(not ok) |