From 0f23f618f36a7472d1c67b36344ef87a31eb586c Mon Sep 17 00:00:00 2001 From: Brecht Van Lommel Date: Wed, 14 Feb 2018 17:33:06 +0100 Subject: Tests: split off render report test code from Cycles tests. This renames test environment variables from CYCLESTEST_* to BLENDER_TEST_*. Differential Revision: https://developer.blender.org/D3064 --- tests/python/cycles_render_tests.py | 390 +++------------------------------ tests/python/modules/render_report.py | 398 ++++++++++++++++++++++++++++++++++ 2 files changed, 424 insertions(+), 364 deletions(-) create mode 100755 tests/python/modules/render_report.py (limited to 'tests') diff --git a/tests/python/cycles_render_tests.py b/tests/python/cycles_render_tests.py index 731996df8ef..a01a6f74e15 100755 --- a/tests/python/cycles_render_tests.py +++ b/tests/python/cycles_render_tests.py @@ -2,55 +2,14 @@ # Apache License, Version 2.0 import argparse -import glob import os -import pathlib import shlex import shutil import subprocess import sys -import time -import tempfile -class COLORS_ANSI: - RED = '\033[00;31m' - GREEN = '\033[00;32m' - ENDC = '\033[0m' - - -class COLORS_DUMMY: - RED = '' - GREEN = '' - ENDC = '' - -COLORS = COLORS_DUMMY - - -def print_message(message, type=None, status=''): - if type == 'SUCCESS': - print(COLORS.GREEN, end="") - elif type == 'FAILURE': - print(COLORS.RED, end="") - status_text = ... - if status == 'RUN': - status_text = " RUN " - elif status == 'OK': - status_text = " OK " - elif status == 'PASSED': - status_text = " PASSED " - elif status == 'FAILED': - status_text = " FAILED " - else: - status_text = status - if status_text: - print("[{}]" . format(status_text), end="") - print(COLORS.ENDC, end="") - print(" {}" . format(message)) - sys.stdout.flush() - - -def render_file(filepath): +def render_file(filepath, output_filepath): dirname = os.path.dirname(filepath) basedir = os.path.dirname(dirname) subject = os.path.basename(dirname) @@ -62,6 +21,8 @@ def render_file(filepath): # custom_args += ["--python-expr", "import bpy; bpy.context.scene.cycles.shading_system = True"] # custom_args += ["--python-expr", "import bpy; bpy.context.scene.cycles.device = 'GPU'"] + frame_filepath = output_filepath + '0001.png' + if subject == 'opengl': command = [ BLENDER, @@ -73,7 +34,7 @@ def render_file(filepath): "-E", "CYCLES"] command += custom_args command += [ - "-o", TEMP_FILE_MASK, + "-o", output_filepath, "-F", "PNG", '--python', os.path.join(basedir, "util", @@ -89,7 +50,7 @@ def render_file(filepath): "-E", "CYCLES"] command += custom_args command += [ - "-o", TEMP_FILE_MASK, + "-o", output_filepath, "-F", "PNG", '--python', os.path.join(basedir, "util", @@ -105,321 +66,39 @@ def render_file(filepath): "-E", "CYCLES"] command += custom_args command += [ - "-o", TEMP_FILE_MASK, + "-o", output_filepath, "-F", "PNG", "-f", "1"] + try: + # Success output = subprocess.check_output(command) + if os.path.exists(frame_filepath): + shutil.copy(frame_filepath, output_filepath) + os.remove(frame_filepath) if VERBOSE: print(output.decode("utf-8")) return None except subprocess.CalledProcessError as e: - if os.path.exists(TEMP_FILE): - os.remove(TEMP_FILE) + # Error + if os.path.exists(frame_filepath): + os.remove(frame_filepath) if VERBOSE: print(e.output.decode("utf-8")) if b"Error: engine not found" in e.output: - return "NO_CYCLES" + return "NO_ENGINE" elif b"blender probably wont start" in e.output: return "NO_START" return "CRASH" except BaseException as e: - if os.path.exists(TEMP_FILE): - os.remove(TEMP_FILE) + # Crash + if os.path.exists(frame_filepath): + os.remove(frame_filepath) if VERBOSE: print(e) return "CRASH" -def test_get_name(filepath): - filename = os.path.basename(filepath) - return os.path.splitext(filename)[0] - -def test_get_images(filepath): - testname = test_get_name(filepath) - dirpath = os.path.dirname(filepath) - - old_dirpath = os.path.join(dirpath, "reference_renders") - old_img = os.path.join(old_dirpath, testname + ".png") - - ref_dirpath = os.path.join(OUTDIR, os.path.basename(dirpath), "ref") - ref_img = os.path.join(ref_dirpath, testname + ".png") - if not os.path.exists(ref_dirpath): - os.makedirs(ref_dirpath) - if os.path.exists(old_img): - shutil.copy(old_img, ref_img) - - new_dirpath = os.path.join(OUTDIR, os.path.basename(dirpath)) - if not os.path.exists(new_dirpath): - os.makedirs(new_dirpath) - new_img = os.path.join(new_dirpath, testname + ".png") - - diff_dirpath = os.path.join(OUTDIR, os.path.basename(dirpath), "diff") - if not os.path.exists(diff_dirpath): - os.makedirs(diff_dirpath) - diff_img = os.path.join(diff_dirpath, testname + ".diff.png") - - return old_img, ref_img, new_img, diff_img - - -class Report: - def __init__(self, testname): - self.failed_tests = "" - self.passed_tests = "" - self.testname = testname - - def output(self): - # write intermediate data for single test - outdir = os.path.join(OUTDIR, self.testname) - if not os.path.exists(outdir): - os.makedirs(outdir) - - filepath = os.path.join(outdir, "failed.data") - pathlib.Path(filepath).write_text(self.failed_tests) - - filepath = os.path.join(outdir, "passed.data") - pathlib.Path(filepath).write_text(self.passed_tests) - - # gather intermediate data for all tests - failed_data = sorted(glob.glob(os.path.join(OUTDIR, "*/failed.data"))) - passed_data = sorted(glob.glob(os.path.join(OUTDIR, "*/passed.data"))) - - failed_tests = "" - passed_tests = "" - - for filename in failed_data: - filepath = os.path.join(OUTDIR, filename) - failed_tests += pathlib.Path(filepath).read_text() - for filename in passed_data: - filepath = os.path.join(OUTDIR, filename) - passed_tests += pathlib.Path(filepath).read_text() - - # write html for all tests - self.html = """ - - - Cycles Test Report - - - - -
-
-

Cycles Test Report

-
- - - - - {}{} -
NameNewReferenceDiff
-
-
- - - """ . format(failed_tests, passed_tests) - - filepath = os.path.join(OUTDIR, "report.html") - pathlib.Path(filepath).write_text(self.html) - - print_message("Report saved to: " + pathlib.Path(filepath).as_uri()) - - def relative_url(self, filepath): - relpath = os.path.relpath(filepath, OUTDIR) - return pathlib.Path(relpath).as_posix() - - def add_test(self, filepath, error): - name = test_get_name(filepath) - name = name.replace('_', ' ') - - old_img, ref_img, new_img, diff_img = test_get_images(filepath) - - status = error if error else "" - style = """ style="background-color: #f99;" """ if error else "" - - new_url = self.relative_url(new_img) - ref_url = self.relative_url(ref_img) - diff_url = self.relative_url(diff_img) - - test_html = """ - - {}
{}
{} - - - - """ . format(style, name, self.testname, status, - new_url, ref_url, new_url, - ref_url, new_url, ref_url, - diff_url) - - if error: - self.failed_tests += test_html - else: - self.passed_tests += test_html - - -def verify_output(report, filepath): - old_img, ref_img, new_img, diff_img = test_get_images(filepath) - - # copy new image - if os.path.exists(new_img): - os.remove(new_img) - if os.path.exists(TEMP_FILE): - shutil.copy(TEMP_FILE, new_img) - - update = os.getenv('CYCLESTEST_UPDATE') - - if os.path.exists(ref_img): - # diff test with threshold - command = ( - IDIFF, - "-fail", "0.016", - "-failpercent", "1", - ref_img, - TEMP_FILE, - ) - try: - subprocess.check_output(command) - failed = False - except subprocess.CalledProcessError as e: - if VERBOSE: - print_message(e.output.decode("utf-8")) - failed = e.returncode != 1 - else: - if not update: - return False - - failed = True - - if failed and update: - # update reference - shutil.copy(new_img, ref_img) - shutil.copy(new_img, old_img) - failed = False - - # generate diff image - command = ( - IDIFF, - "-o", diff_img, - "-abs", "-scale", "16", - ref_img, - TEMP_FILE - ) - - try: - subprocess.check_output(command) - except subprocess.CalledProcessError as e: - if VERBOSE: - print_message(e.output.decode("utf-8")) - - return not failed - - -def run_test(report, filepath): - testname = test_get_name(filepath) - spacer = "." * (32 - len(testname)) - print_message(testname, 'SUCCESS', 'RUN') - time_start = time.time() - error = render_file(filepath) - status = "FAIL" - if not error: - if not verify_output(report, filepath): - error = "VERIFY" - time_end = time.time() - elapsed_ms = int((time_end - time_start) * 1000) - if not error: - print_message("{} ({} ms)" . format(testname, elapsed_ms), - 'SUCCESS', 'OK') - else: - if error == "NO_CYCLES": - print_message("Can't perform tests because Cycles failed to load!") - return error - elif error == "NO_START": - print_message('Can not perform tests because blender fails to start.', - 'Make sure INSTALL target was run.') - return error - elif error == 'VERIFY': - print_message("Rendered result is different from reference image") - else: - print_message("Unknown error %r" % error) - print_message("{} ({} ms)" . format(testname, elapsed_ms), - 'FAILURE', 'FAILED') - return error - - - -def blend_list(path): - for dirpath, dirnames, filenames in os.walk(path): - for filename in filenames: - if filename.lower().endswith(".blend"): - filepath = os.path.join(dirpath, filename) - yield filepath - -def run_all_tests(dirpath): - passed_tests = [] - failed_tests = [] - all_files = list(blend_list(dirpath)) - all_files.sort() - report = Report(os.path.basename(dirpath)) - print_message("Running {} tests from 1 test case." . - format(len(all_files)), - 'SUCCESS', "==========") - time_start = time.time() - for filepath in all_files: - error = run_test(report, filepath) - testname = test_get_name(filepath) - if error: - if error == "NO_CYCLES": - return False - elif error == "NO_START": - return False - failed_tests.append(testname) - else: - passed_tests.append(testname) - report.add_test(filepath, error) - time_end = time.time() - elapsed_ms = int((time_end - time_start) * 1000) - print_message("") - print_message("{} tests from 1 test case ran. ({} ms total)" . - format(len(all_files), elapsed_ms), - 'SUCCESS', "==========") - print_message("{} tests." . - format(len(passed_tests)), - 'SUCCESS', 'PASSED') - if failed_tests: - print_message("{} tests, listed below:" . - format(len(failed_tests)), - 'FAILURE', 'FAILED') - failed_tests.sort() - for test in failed_tests: - print_message("{}" . format(test), 'FAILURE', "FAILED") - - report.output() - return not bool(failed_tests) - - def create_argparse(): parser = argparse.ArgumentParser() parser.add_argument("-blender", nargs="+") @@ -433,36 +112,19 @@ def main(): parser = create_argparse() args = parser.parse_args() - global COLORS - global BLENDER, TESTDIR, IDIFF, OUTDIR - global TEMP_FILE, TEMP_FILE_MASK, TEST_SCRIPT - global VERBOSE - - if os.environ.get("CYCLESTEST_COLOR") is not None: - COLORS = COLORS_ANSI + global BLENDER, VERBOSE BLENDER = args.blender[0] - TESTDIR = args.testdir[0] - IDIFF = args.idiff[0] - OUTDIR = args.outdir[0] - - if not os.path.exists(OUTDIR): - os.makedirs(OUTDIR) - - TEMP = tempfile.mkdtemp() - TEMP_FILE_MASK = os.path.join(TEMP, "test") - TEMP_FILE = TEMP_FILE_MASK + "0001.png" - - TEST_SCRIPT = os.path.join(os.path.dirname(__file__), "runtime_check.py") - VERBOSE = os.environ.get("BLENDER_VERBOSE") is not None - ok = run_all_tests(TESTDIR) + test_dir = args.testdir[0] + idiff = args.idiff[0] + output_dir = args.outdir[0] - # Cleanup temp files and folders - if os.path.exists(TEMP_FILE): - os.remove(TEMP_FILE) - os.rmdir(TEMP) + from modules import render_report + report = render_report.Report("Cycles Test Report", output_dir, idiff) + report.set_pixelated(True) + ok = report.run(test_dir, render_file) sys.exit(not ok) diff --git a/tests/python/modules/render_report.py b/tests/python/modules/render_report.py new file mode 100755 index 00000000000..930a08282e8 --- /dev/null +++ b/tests/python/modules/render_report.py @@ -0,0 +1,398 @@ +# Apache License, Version 2.0 +# +# Compare renders or screenshots against reference versions and generate +# a HTML report showing the differences, for regression testing. + +import glob +import os +import pathlib +import shutil +import subprocess +import sys +import time + + +class COLORS_ANSI: + RED = '\033[00;31m' + GREEN = '\033[00;32m' + ENDC = '\033[0m' + + +class COLORS_DUMMY: + RED = '' + GREEN = '' + ENDC = '' + +COLORS = COLORS_DUMMY + + +def print_message(message, type=None, status=''): + if type == 'SUCCESS': + print(COLORS.GREEN, end="") + elif type == 'FAILURE': + print(COLORS.RED, end="") + status_text = ... + if status == 'RUN': + status_text = " RUN " + elif status == 'OK': + status_text = " OK " + elif status == 'PASSED': + status_text = " PASSED " + elif status == 'FAILED': + status_text = " FAILED " + else: + status_text = status + if status_text: + print("[{}]" . format(status_text), end="") + print(COLORS.ENDC, end="") + print(" {}" . format(message)) + sys.stdout.flush() + + +def blend_list(path): + for dirpath, dirnames, filenames in os.walk(path): + for filename in filenames: + if filename.lower().endswith(".blend"): + filepath = os.path.join(dirpath, filename) + yield filepath + +def test_get_name(filepath): + filename = os.path.basename(filepath) + return os.path.splitext(filename)[0] + +def test_get_images(output_dir, filepath): + testname = test_get_name(filepath) + dirpath = os.path.dirname(filepath) + + old_dirpath = os.path.join(dirpath, "reference_renders") + old_img = os.path.join(old_dirpath, testname + ".png") + + ref_dirpath = os.path.join(output_dir, os.path.basename(dirpath), "ref") + ref_img = os.path.join(ref_dirpath, testname + ".png") + if not os.path.exists(ref_dirpath): + os.makedirs(ref_dirpath) + if os.path.exists(old_img): + shutil.copy(old_img, ref_img) + + new_dirpath = os.path.join(output_dir, os.path.basename(dirpath)) + if not os.path.exists(new_dirpath): + os.makedirs(new_dirpath) + new_img = os.path.join(new_dirpath, testname + ".png") + + diff_dirpath = os.path.join(output_dir, os.path.basename(dirpath), "diff") + if not os.path.exists(diff_dirpath): + os.makedirs(diff_dirpath) + diff_img = os.path.join(diff_dirpath, testname + ".diff.png") + + return old_img, ref_img, new_img, diff_img + + +class Report: + __slots__ = ( + 'title', + 'output_dir', + 'idiff', + 'pixelated', + 'verbose', + 'update', + 'failed_tests', + 'passed_tests' + ) + + def __init__(self, title, output_dir, idiff): + self.title = title + self.output_dir = output_dir + self.idiff = idiff + + self.pixelated = False + self.verbose = os.environ.get("BLENDER_VERBOSE") is not None + self.update = os.getenv('BLENDER_TEST_UPDATE') is not None + + if os.environ.get("BLENDER_TEST_COLOR") is not None: + global COLORS, COLORS_ANSI + COLORS = COLORS_ANSI + + self.failed_tests = "" + self.passed_tests = "" + + if not os.path.exists(output_dir): + os.makedirs(output_dir) + + def set_pixelated(self, pixelated): + self.pixelated = pixelated + + def run(self, dirpath, render_cb): + # Run tests and output report. + dirname = os.path.basename(dirpath) + ok = self._run_all_tests(dirname, dirpath, render_cb) + self._write_html(dirname) + return ok + + def _write_html(self, dirname): + # Write intermediate data for single test. + outdir = os.path.join(self.output_dir, dirname) + if not os.path.exists(outdir): + os.makedirs(outdir) + + filepath = os.path.join(outdir, "failed.data") + pathlib.Path(filepath).write_text(self.failed_tests) + + filepath = os.path.join(outdir, "passed.data") + pathlib.Path(filepath).write_text(self.passed_tests) + + # Gather intermediate data for all tests. + failed_data = sorted(glob.glob(os.path.join(self.output_dir, "*/failed.data"))) + passed_data = sorted(glob.glob(os.path.join(self.output_dir, "*/passed.data"))) + + failed_tests = "" + passed_tests = "" + + for filename in failed_data: + filepath = os.path.join(self.output_dir, filename) + failed_tests += pathlib.Path(filepath).read_text() + for filename in passed_data: + filepath = os.path.join(self.output_dir, filename) + passed_tests += pathlib.Path(filepath).read_text() + + tests_html = failed_tests + passed_tests + + # Write html for all tests. + if self.pixelated: + image_rendering = 'pixelated' + else: + image_rendering = 'auto' + + if len(failed_tests) > 0: + message = "

Run BLENDER_TEST_UPDATE=1 ctest to create or update reference images for failed tests.

" + else: + message = "" + + html = """ + + + {title} + + + + +
+
+

{title}

+ {message} +
+ + + + + {tests_html} +
NameNewReferenceDiff
+
+
+ + + """ . format(title=self.title, + message=message, + image_rendering=image_rendering, + tests_html=tests_html) + + filepath = os.path.join(self.output_dir, "report.html") + pathlib.Path(filepath).write_text(html) + + print_message("Report saved to: " + pathlib.Path(filepath).as_uri()) + + def _relative_url(self, filepath): + relpath = os.path.relpath(filepath, self.output_dir) + return pathlib.Path(relpath).as_posix() + + def _write_test_html(self, testname, filepath, error): + name = test_get_name(filepath) + name = name.replace('_', ' ') + + old_img, ref_img, new_img, diff_img = test_get_images(self.output_dir, filepath) + + status = error if error else "" + tr_style = """ style="background-color: #f99;" """ if error else "" + + new_url = self._relative_url(new_img) + ref_url = self._relative_url(ref_img) + diff_url = self._relative_url(diff_img) + + test_html = """ + + {name}
{testname}
{status} + + + + """ . format(tr_style=tr_style, + name=name, + testname=testname, + status=status, + new_url=new_url, + ref_url=ref_url, + diff_url=diff_url) + + if error: + self.failed_tests += test_html + else: + self.passed_tests += test_html + + + def _diff_output(self, filepath, tmp_filepath): + old_img, ref_img, new_img, diff_img = test_get_images(self.output_dir, filepath) + + # Create reference render directory. + old_dirpath = os.path.dirname(old_img) + if not os.path.exists(old_dirpath): + os.makedirs(old_dirpath) + + # Copy temporary to new image. + if os.path.exists(new_img): + os.remove(new_img) + if os.path.exists(tmp_filepath): + shutil.copy(tmp_filepath, new_img) + + if os.path.exists(ref_img): + # Diff images test with threshold. + command = ( + self.idiff, + "-fail", "0.016", + "-failpercent", "1", + ref_img, + tmp_filepath, + ) + try: + subprocess.check_output(command) + failed = False + except subprocess.CalledProcessError as e: + if self.verbose: + print_message(e.output.decode("utf-8")) + failed = e.returncode != 1 + else: + if not self.update: + return False + + failed = True + + if failed and self.update: + # Update reference image if requested. + shutil.copy(new_img, ref_img) + shutil.copy(new_img, old_img) + failed = False + + # Generate diff image. + command = ( + self.idiff, + "-o", diff_img, + "-abs", "-scale", "16", + ref_img, + tmp_filepath + ) + + try: + subprocess.check_output(command) + except subprocess.CalledProcessError as e: + if self.verbose: + print_message(e.output.decode("utf-8")) + + return not failed + + + def _run_test(self, filepath, render_cb): + testname = test_get_name(filepath) + print_message(testname, 'SUCCESS', 'RUN') + time_start = time.time() + tmp_filepath = os.path.join(self.output_dir, "tmp") + + error = render_cb(filepath, tmp_filepath) + status = "FAIL" + if not error: + if not self._diff_output(filepath, tmp_filepath): + error = "VERIFY" + + if os.path.exists(tmp_filepath): + os.remove(tmp_filepath) + + time_end = time.time() + elapsed_ms = int((time_end - time_start) * 1000) + if not error: + print_message("{} ({} ms)" . format(testname, elapsed_ms), + 'SUCCESS', 'OK') + else: + if error == "NO_ENGINE": + print_message("Can't perform tests because the render engine failed to load!") + return error + elif error == "NO_START": + print_message('Can not perform tests because blender fails to start.', + 'Make sure INSTALL target was run.') + return error + elif error == 'VERIFY': + print_message("Rendered result is different from reference image") + else: + print_message("Unknown error %r" % error) + print_message("{} ({} ms)" . format(testname, elapsed_ms), + 'FAILURE', 'FAILED') + return error + + + def _run_all_tests(self, dirname, dirpath, render_cb): + passed_tests = [] + failed_tests = [] + all_files = list(blend_list(dirpath)) + all_files.sort() + print_message("Running {} tests from 1 test case." . + format(len(all_files)), + 'SUCCESS', "==========") + time_start = time.time() + for filepath in all_files: + error = self._run_test(filepath, render_cb) + testname = test_get_name(filepath) + if error: + if error == "NO_ENGINE": + return False + elif error == "NO_START": + return False + failed_tests.append(testname) + else: + passed_tests.append(testname) + self._write_test_html(dirname, filepath, error) + time_end = time.time() + elapsed_ms = int((time_end - time_start) * 1000) + print_message("") + print_message("{} tests from 1 test case ran. ({} ms total)" . + format(len(all_files), elapsed_ms), + 'SUCCESS', "==========") + print_message("{} tests." . + format(len(passed_tests)), + 'SUCCESS', 'PASSED') + if failed_tests: + print_message("{} tests, listed below:" . + format(len(failed_tests)), + 'FAILURE', 'FAILED') + failed_tests.sort() + for test in failed_tests: + print_message("{}" . format(test), 'FAILURE', "FAILED") + + return not bool(failed_tests) + -- cgit v1.2.3