diff options
Diffstat (limited to 'tests/python/cycles_render_tests.py')
| -rwxr-xr-x | tests/python/cycles_render_tests.py | 298 |
1 files changed, 241 insertions, 57 deletions
diff --git a/tests/python/cycles_render_tests.py b/tests/python/cycles_render_tests.py index a030cc5e0de..ba4c04f7cf3 100755 --- a/tests/python/cycles_render_tests.py +++ b/tests/python/cycles_render_tests.py @@ -2,7 +2,10 @@ # Apache License, Version 2.0 import argparse +import glob import os +import pathlib +import shlex import shutil import subprocess import sys @@ -24,7 +27,7 @@ class COLORS_DUMMY: COLORS = COLORS_DUMMY -def printMessage(type, status, message): +def print_message(message, type=None, status=''): if type == 'SUCCESS': print(COLORS.GREEN, end="") elif type == 'FAILURE': @@ -50,38 +53,44 @@ def render_file(filepath): dirname = os.path.dirname(filepath) basedir = os.path.dirname(dirname) subject = os.path.basename(dirname) + + custom_args = os.getenv('CYCLESTEST_ARGS') + custom_args = shlex.split(custom_args) if custom_args else [] + + # OSL and GPU examples + # custom_args += ["--python-expr", "import bpy; bpy.context.scene.cycles.shading_system = True"] + # custom_args += ["--python-expr", "import bpy; bpy.context.scene.cycles.device = 'GPU'"] + if subject == 'opengl': - command = ( + command = [ BLENDER, "--window-geometry", "0", "0", "1", "1", "-noaudio", "--factory-startup", "--enable-autoexec", filepath, - "-E", "CYCLES", - # Run with OSL enabled - # "--python-expr", "import bpy; bpy.context.scene.cycles.shading_system = True", + "-E", "CYCLES"] + command += custom_args + command += [ "-o", TEMP_FILE_MASK, "-F", "PNG", '--python', os.path.join(basedir, "util", - "render_opengl.py") - ) + "render_opengl.py")] else: - command = ( + command = [ BLENDER, "--background", "-noaudio", "--factory-startup", "--enable-autoexec", filepath, - "-E", "CYCLES", - # Run with OSL enabled - # "--python-expr", "import bpy; bpy.context.scene.cycles.shading_system = True", + "-E", "CYCLES"] + command += custom_args + command += [ "-o", TEMP_FILE_MASK, "-F", "PNG", - "-f", "1", - ) + "-f", "1"] try: output = subprocess.check_output(command) if VERBOSE: @@ -109,20 +118,172 @@ def test_get_name(filepath): filename = os.path.basename(filepath) return os.path.splitext(filename)[0] - -def verify_output(filepath): +def test_get_images(filepath): testname = test_get_name(filepath) dirpath = os.path.dirname(filepath) - reference_dirpath = os.path.join(dirpath, "reference_renders") - reference_image = os.path.join(reference_dirpath, testname + ".png") - failed_image = os.path.join(reference_dirpath, testname + ".fail.png") - if not os.path.exists(reference_image): + + old_dirpath = os.path.join(dirpath, "reference_renders") + old_img = os.path.join(old_dirpath, testname + ".png") + + ref_dirpath = os.path.join(OUTDIR, os.path.basename(dirpath), "ref") + ref_img = os.path.join(ref_dirpath, testname + ".png") + if not os.path.exists(ref_dirpath): + os.makedirs(ref_dirpath) + if os.path.exists(old_img): + shutil.copy(old_img, ref_img) + + new_dirpath = os.path.join(OUTDIR, os.path.basename(dirpath)) + if not os.path.exists(new_dirpath): + os.makedirs(new_dirpath) + new_img = os.path.join(new_dirpath, testname + ".png") + + diff_dirpath = os.path.join(OUTDIR, os.path.basename(dirpath), "diff") + if not os.path.exists(diff_dirpath): + os.makedirs(diff_dirpath) + diff_img = os.path.join(diff_dirpath, testname + ".diff.png") + + return ref_img, new_img, diff_img + + +class Report: + def __init__(self, testname): + self.failed_tests = "" + self.passed_tests = "" + self.testname = testname + + def output(self): + # write intermediate data for single test + outdir = os.path.join(OUTDIR, self.testname) + if not os.path.exists(outdir): + os.makedirs(outdir) + + f = open(os.path.join(outdir, "failed.data"), "w") + f.write(self.failed_tests) + f.close() + + f = open(os.path.join(outdir, "passed.data"), "w") + f.write(self.passed_tests) + f.close() + + # gather intermediate data for all tests + failed_data = sorted(glob.glob(os.path.join(OUTDIR, "*/failed.data"))) + passed_data = sorted(glob.glob(os.path.join(OUTDIR, "*/passed.data"))) + + failed_tests = "" + passed_tests = "" + + for filename in failed_data: + failed_tests += open(os.path.join(OUTDIR, filename), "r").read() + for filename in passed_data: + passed_tests += open(os.path.join(OUTDIR, filename), "r").read() + + # write html for all tests + self.html = """ +<html> +<head> + <title>Cycles Test Report</title> + <style> + img {{ image-rendering: pixelated; width: 256px; background-color: #000; }} + img.render {{ + background-color: #fff; + background-image: + -moz-linear-gradient(45deg, #eee 25%, transparent 25%), + -moz-linear-gradient(-45deg, #eee 25%, transparent 25%), + -moz-linear-gradient(45deg, transparent 75%, #eee 75%), + -moz-linear-gradient(-45deg, transparent 75%, #eee 75%); + background-image: + -webkit-gradient(linear, 0 100%, 100% 0, color-stop(.25, #eee), color-stop(.25, transparent)), + -webkit-gradient(linear, 0 0, 100% 100%, color-stop(.25, #eee), color-stop(.25, transparent)), + -webkit-gradient(linear, 0 100%, 100% 0, color-stop(.75, transparent), color-stop(.75, #eee)), + -webkit-gradient(linear, 0 0, 100% 100%, color-stop(.75, transparent), color-stop(.75, #eee)); + + -moz-background-size:50px 50px; + background-size:50px 50px; + -webkit-background-size:50px 51px; /* override value for shitty webkit */ + + background-position:0 0, 25px 0, 25px -25px, 0px 25px; + }} + table td:first-child {{ width: 256px; }} + </style> + <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0-alpha.6/css/bootstrap.min.css"> +</head> +<body> + <div class="container"> + <br/> + <h1>Cycles Test Report</h1> + <br/> + <table class="table table-striped"> + <thead class="thead-default"> + <tr><th>Name</th><th>New</th><th>Reference</th><th>Diff</th> + </thead> + {}{} + </table> + <br/> + </div> +</body> +</html> + """ . format(failed_tests, passed_tests) + + filepath = os.path.join(OUTDIR, "report.html") + f = open(filepath, "w") + f.write(self.html) + f.close() + + print_message("Report saved to: " + pathlib.Path(filepath).as_uri()) + + def relative_url(self, filepath): + relpath = os.path.relpath(filepath, OUTDIR) + return pathlib.Path(relpath).as_posix() + + def add_test(self, filepath, error): + name = test_get_name(filepath) + name = name.replace('_', ' ') + + ref_img, new_img, diff_img = test_get_images(filepath) + + status = error if error else "" + style = """ style="background-color: #f99;" """ if error else "" + + new_url = self.relative_url(new_img) + ref_url = self.relative_url(ref_img) + diff_url = self.relative_url(diff_img) + + test_html = """ + <tr{}> + <td><b>{}</b><br/>{}<br/>{}</td> + <td><img src="{}" onmouseover="this.src='{}';" onmouseout="this.src='{}';" class="render"></td> + <td><img src="{}" onmouseover="this.src='{}';" onmouseout="this.src='{}';" class="render"></td> + <td><img src="{}"></td> + </tr>""" . format(style, name, self.testname, status, + new_url, ref_url, new_url, + ref_url, new_url, ref_url, + diff_url) + + if error: + self.failed_tests += test_html + else: + self.passed_tests += test_html + + +def verify_output(report, filepath): + ref_img, new_img, diff_img = test_get_images(filepath) + + # copy new image + if os.path.exists(new_img): + os.remove(new_img) + if os.path.exists(TEMP_FILE): + shutil.copy(TEMP_FILE, new_img) + + + if not os.path.exists(ref_img): return False + + # diff test with threshold command = ( IDIFF, - "-fail", "0.015", + "-fail", "0.016", "-failpercent", "1", - reference_image, + ref_img, TEMP_FILE, ) try: @@ -130,47 +291,60 @@ def verify_output(filepath): failed = False except subprocess.CalledProcessError as e: if VERBOSE: - print(e.output.decode("utf-8")) + print_message(e.output.decode("utf-8")) failed = e.returncode != 1 - if failed: - shutil.copy(TEMP_FILE, failed_image) - elif os.path.exists(failed_image): - os.remove(failed_image) + + # generate diff image + command = ( + IDIFF, + "-o", diff_img, + "-abs", "-scale", "16", + ref_img, + TEMP_FILE + ) + + try: + subprocess.check_output(command) + except subprocess.CalledProcessError as e: + if VERBOSE: + print_message(e.output.decode("utf-8")) + return not failed -def run_test(filepath): +def run_test(report, filepath): testname = test_get_name(filepath) spacer = "." * (32 - len(testname)) - printMessage('SUCCESS', 'RUN', testname) + print_message(testname, 'SUCCESS', 'RUN') time_start = time.time() error = render_file(filepath) status = "FAIL" if not error: - if not verify_output(filepath): + if not verify_output(report, filepath): error = "VERIFY" time_end = time.time() elapsed_ms = int((time_end - time_start) * 1000) if not error: - printMessage('SUCCESS', 'OK', "{} ({} ms)" . - format(testname, elapsed_ms)) + print_message("{} ({} ms)" . format(testname, elapsed_ms), + 'SUCCESS', 'OK') else: if error == "NO_CYCLES": - print("Can't perform tests because Cycles failed to load!") - return False + print_message("Can't perform tests because Cycles failed to load!") + return error elif error == "NO_START": - print('Can not perform tests because blender fails to start.', + print_message('Can not perform tests because blender fails to start.', 'Make sure INSTALL target was run.') - return False + return error elif error == 'VERIFY': - print("Rendered result is different from reference image") + print_message("Rendered result is different from reference image") else: - print("Unknown error %r" % error) - printMessage('FAILURE', 'FAILED', "{} ({} ms)" . - format(testname, elapsed_ms)) + print_message("Unknown error %r" % error) + print_message("{} ({} ms)" . format(testname, elapsed_ms), + 'FAILURE', 'FAILED') return error + def blend_list(path): for dirpath, dirnames, filenames in os.walk(path): for filename in filenames: @@ -178,17 +352,18 @@ def blend_list(path): filepath = os.path.join(dirpath, filename) yield filepath - def run_all_tests(dirpath): passed_tests = [] failed_tests = [] all_files = list(blend_list(dirpath)) all_files.sort() - printMessage('SUCCESS', "==========", - "Running {} tests from 1 test case." . format(len(all_files))) + report = Report(os.path.basename(dirpath)) + print_message("Running {} tests from 1 test case." . + format(len(all_files)), + 'SUCCESS', "==========") time_start = time.time() for filepath in all_files: - error = run_test(filepath) + error = run_test(report, filepath) testname = test_get_name(filepath) if error: if error == "NO_CYCLES": @@ -198,28 +373,33 @@ def run_all_tests(dirpath): failed_tests.append(testname) else: passed_tests.append(testname) + report.add_test(filepath, error) time_end = time.time() elapsed_ms = int((time_end - time_start) * 1000) - print("") - printMessage('SUCCESS', "==========", - "{} tests from 1 test case ran. ({} ms total)" . - format(len(all_files), elapsed_ms)) - printMessage('SUCCESS', 'PASSED', "{} tests." . - format(len(passed_tests))) + print_message("") + print_message("{} tests from 1 test case ran. ({} ms total)" . + format(len(all_files), elapsed_ms), + 'SUCCESS', "==========") + print_message("{} tests." . + format(len(passed_tests)), + 'SUCCESS', 'PASSED') if failed_tests: - printMessage('FAILURE', 'FAILED', "{} tests, listed below:" . - format(len(failed_tests))) + print_message("{} tests, listed below:" . + format(len(failed_tests)), + 'FAILURE', 'FAILED') failed_tests.sort() for test in failed_tests: - printMessage('FAILURE', "FAILED", "{}" . format(test)) - return False - return True + print_message("{}" . format(test), 'FAILURE', "FAILED") + + report.output() + return not bool(failed_tests) def create_argparse(): parser = argparse.ArgumentParser() parser.add_argument("-blender", nargs="+") parser.add_argument("-testdir", nargs=1) + parser.add_argument("-outdir", nargs=1) parser.add_argument("-idiff", nargs=1) return parser @@ -229,7 +409,7 @@ def main(): args = parser.parse_args() global COLORS - global BLENDER, ROOT, IDIFF + global BLENDER, TESTDIR, IDIFF, OUTDIR global TEMP_FILE, TEMP_FILE_MASK, TEST_SCRIPT global VERBOSE @@ -237,8 +417,12 @@ def main(): COLORS = COLORS_ANSI BLENDER = args.blender[0] - ROOT = args.testdir[0] + TESTDIR = args.testdir[0] IDIFF = args.idiff[0] + OUTDIR = args.outdir[0] + + if not os.path.exists(OUTDIR): + os.makedirs(OUTDIR) TEMP = tempfile.mkdtemp() TEMP_FILE_MASK = os.path.join(TEMP, "test") @@ -248,7 +432,7 @@ def main(): VERBOSE = os.environ.get("BLENDER_VERBOSE") is not None - ok = run_all_tests(ROOT) + ok = run_all_tests(TESTDIR) # Cleanup temp files and folders if os.path.exists(TEMP_FILE): |