diff options
Diffstat (limited to 'tests')
-rw-r--r-- | tests/python/CMakeLists.txt | 100 | ||||
-rwxr-xr-x | tests/python/cycles_render_tests.py | 390 | ||||
-rwxr-xr-x | tests/python/modules/render_report.py | 397 | ||||
-rwxr-xr-x | tests/python/opengl_draw_tests.py | 97 |
4 files changed, 578 insertions, 406 deletions
diff --git a/tests/python/CMakeLists.txt b/tests/python/CMakeLists.txt index 79031824b83..f973488d657 100644 --- a/tests/python/CMakeLists.txt +++ b/tests/python/CMakeLists.txt @@ -513,32 +513,36 @@ add_test( ) endif() +# Run Python script outside Blender. +function(add_python_test testname testscript) + if(MSVC) + add_test( + NAME ${testname} + COMMAND + "$<TARGET_FILE_DIR:blender>/${BLENDER_VERSION_MAJOR}.${BLENDER_VERSION_MINOR}/python/bin/python$<$<CONFIG:Debug>:_d>" + ${testscript} ${ARGN} + ) + else() + add_test( + NAME ${testname} + COMMAND ${testscript} ${ARGN} + ) + endif() +endfunction() + if(WITH_CYCLES) if(OPENIMAGEIO_IDIFF AND EXISTS "${TEST_SRC_DIR}/cycles/ctests/shader") macro(add_cycles_render_test subject) - if(MSVC) - add_test( - NAME cycles_${subject}_test - COMMAND - "$<TARGET_FILE_DIR:blender>/${BLENDER_VERSION_MAJOR}.${BLENDER_VERSION_MINOR}/python/bin/python$<$<CONFIG:Debug>:_d>" - ${CMAKE_CURRENT_LIST_DIR}/cycles_render_tests.py - -blender "$<TARGET_FILE:blender>" - -testdir "${TEST_SRC_DIR}/cycles/ctests/${subject}" - -idiff "${OPENIMAGEIO_IDIFF}" - -outdir "${TEST_OUT_DIR}/cycles" - ) - else() - add_test( - NAME cycles_${subject}_test - COMMAND ${CMAKE_CURRENT_LIST_DIR}/cycles_render_tests.py - -blender "$<TARGET_FILE:blender>" - -testdir "${TEST_SRC_DIR}/cycles/ctests/${subject}" - -idiff "${OPENIMAGEIO_IDIFF}" - -outdir "${TEST_OUT_DIR}/cycles" - ) - endif() + add_python_test( + cycles_${subject}_test + ${CMAKE_CURRENT_LIST_DIR}/cycles_render_tests.py + -blender "$<TARGET_FILE:blender>" + -testdir "${TEST_SRC_DIR}/cycles/ctests/${subject}" + -idiff "${OPENIMAGEIO_IDIFF}" + -outdir "${TEST_OUT_DIR}/cycles" + ) endmacro() - if(WITH_OPENGL_TESTS) + if(WITH_OPENGL_RENDER_TESTS) add_cycles_render_test(opengl) endif() add_cycles_render_test(bake) @@ -562,6 +566,31 @@ if(WITH_CYCLES) endif() endif() +if(WITH_OPENGL_DRAW_TESTS) + if(OPENIMAGEIO_IDIFF AND EXISTS "${TEST_SRC_DIR}/opengl") + # Use all test folders + file(GLOB children RELATIVE ${TEST_SRC_DIR} ${TEST_SRC_DIR}/*) + foreach(child ${children}) + if(IS_DIRECTORY ${TEST_SRC_DIR}/${child}) + file(GLOB blends ${TEST_SRC_DIR}/${child}/*.blend) + if(blends) + add_python_test( + opengl_draw_${child}_test + ${CMAKE_CURRENT_LIST_DIR}/opengl_draw_tests.py + -blender "$<TARGET_FILE:blender>" + -testdir "${TEST_SRC_DIR}/${child}" + -idiff "${OPENIMAGEIO_IDIFF}" + -outdir "${TEST_OUT_DIR}/opengl_draw" + ) + endif() + endif() + endforeach() + else() + MESSAGE(STATUS "Disabling OpenGL tests because tests folder does not exist") + endif() +endif() + + if(WITH_ALEMBIC) find_package_wrapper(Alembic) if(NOT ALEMBIC_FOUND) @@ -570,26 +599,13 @@ if(WITH_ALEMBIC) get_filename_component(real_include_dir ${ALEMBIC_INCLUDE_DIR} REALPATH) get_filename_component(ALEMBIC_ROOT_DIR ${real_include_dir} DIRECTORY) - if(MSVC) - # FIXME, de-duplicate. - add_test( - NAME alembic_tests - COMMAND - "$<TARGET_FILE_DIR:blender>/${BLENDER_VERSION_MAJOR}.${BLENDER_VERSION_MINOR}/python/bin/python$<$<CONFIG:Debug>:_d>" - ${CMAKE_CURRENT_LIST_DIR}/alembic_tests.py - --blender "$<TARGET_FILE:blender>" - --testdir "${TEST_SRC_DIR}/alembic" - --alembic-root "${ALEMBIC_ROOT_DIR}" - ) - else() - add_test( - NAME alembic_tests - COMMAND ${CMAKE_CURRENT_LIST_DIR}/alembic_tests.py - --blender "$<TARGET_FILE:blender>" - --testdir "${TEST_SRC_DIR}/alembic" - --alembic-root "${ALEMBIC_ROOT_DIR}" - ) - endif() + add_python_test( + alembic_tests + ${CMAKE_CURRENT_LIST_DIR}/alembic_tests.py + --blender "$<TARGET_FILE:blender>" + --testdir "${TEST_SRC_DIR}/alembic" + --alembic-root "${ALEMBIC_ROOT_DIR}" + ) add_test( NAME script_alembic_import diff --git a/tests/python/cycles_render_tests.py b/tests/python/cycles_render_tests.py index 731996df8ef..a01a6f74e15 100755 --- a/tests/python/cycles_render_tests.py +++ b/tests/python/cycles_render_tests.py @@ -2,55 +2,14 @@ # Apache License, Version 2.0 import argparse -import glob import os -import pathlib import shlex import shutil import subprocess import sys -import time -import tempfile -class COLORS_ANSI: - RED = '\033[00;31m' - GREEN = '\033[00;32m' - ENDC = '\033[0m' - - -class COLORS_DUMMY: - RED = '' - GREEN = '' - ENDC = '' - -COLORS = COLORS_DUMMY - - -def print_message(message, type=None, status=''): - if type == 'SUCCESS': - print(COLORS.GREEN, end="") - elif type == 'FAILURE': - print(COLORS.RED, end="") - status_text = ... - if status == 'RUN': - status_text = " RUN " - elif status == 'OK': - status_text = " OK " - elif status == 'PASSED': - status_text = " PASSED " - elif status == 'FAILED': - status_text = " FAILED " - else: - status_text = status - if status_text: - print("[{}]" . format(status_text), end="") - print(COLORS.ENDC, end="") - print(" {}" . format(message)) - sys.stdout.flush() - - -def render_file(filepath): +def render_file(filepath, output_filepath): dirname = os.path.dirname(filepath) basedir = os.path.dirname(dirname) subject = os.path.basename(dirname) @@ -62,6 +21,8 @@ def render_file(filepath): # custom_args += ["--python-expr", "import bpy; bpy.context.scene.cycles.shading_system = True"] # custom_args += ["--python-expr", "import bpy; bpy.context.scene.cycles.device = 'GPU'"] + frame_filepath = output_filepath + '0001.png' + if subject == 'opengl': command = [ BLENDER, @@ -73,7 +34,7 @@ def render_file(filepath): "-E", "CYCLES"] command += custom_args command += [ - "-o", TEMP_FILE_MASK, + "-o", output_filepath, "-F", "PNG", '--python', os.path.join(basedir, "util", @@ -89,7 +50,7 @@ def render_file(filepath): "-E", "CYCLES"] command += custom_args command += [ - "-o", TEMP_FILE_MASK, + "-o", output_filepath, "-F", "PNG", '--python', os.path.join(basedir, "util", @@ -105,321 +66,39 @@ def render_file(filepath): "-E", "CYCLES"] command += custom_args command += [ - "-o", TEMP_FILE_MASK, + "-o", output_filepath, "-F", "PNG", "-f", "1"] + try: + # Success output = subprocess.check_output(command) + if os.path.exists(frame_filepath): + shutil.copy(frame_filepath, output_filepath) + os.remove(frame_filepath) if VERBOSE: print(output.decode("utf-8")) return None except subprocess.CalledProcessError as e: - if os.path.exists(TEMP_FILE): - os.remove(TEMP_FILE) + # Error + if os.path.exists(frame_filepath): + os.remove(frame_filepath) if VERBOSE: print(e.output.decode("utf-8")) if b"Error: engine not found" in e.output: - return "NO_CYCLES" + return "NO_ENGINE" elif b"blender probably wont start" in e.output: return "NO_START" return "CRASH" except BaseException as e: - if os.path.exists(TEMP_FILE): - os.remove(TEMP_FILE) + # Crash + if os.path.exists(frame_filepath): + os.remove(frame_filepath) if VERBOSE: print(e) return "CRASH" -def test_get_name(filepath): - filename = os.path.basename(filepath) - return os.path.splitext(filename)[0] - -def test_get_images(filepath): - testname = test_get_name(filepath) - dirpath = os.path.dirname(filepath) - - old_dirpath = os.path.join(dirpath, "reference_renders") - old_img = os.path.join(old_dirpath, testname + ".png") - - ref_dirpath = os.path.join(OUTDIR, os.path.basename(dirpath), "ref") - ref_img = os.path.join(ref_dirpath, testname + ".png") - if not os.path.exists(ref_dirpath): - os.makedirs(ref_dirpath) - if os.path.exists(old_img): - shutil.copy(old_img, ref_img) - - new_dirpath = os.path.join(OUTDIR, os.path.basename(dirpath)) - if not os.path.exists(new_dirpath): - os.makedirs(new_dirpath) - new_img = os.path.join(new_dirpath, testname + ".png") - - diff_dirpath = os.path.join(OUTDIR, os.path.basename(dirpath), "diff") - if not os.path.exists(diff_dirpath): - os.makedirs(diff_dirpath) - diff_img = os.path.join(diff_dirpath, testname + ".diff.png") - - return old_img, ref_img, new_img, diff_img - - -class Report: - def __init__(self, testname): - self.failed_tests = "" - self.passed_tests = "" - self.testname = testname - - def output(self): - # write intermediate data for single test - outdir = os.path.join(OUTDIR, self.testname) - if not os.path.exists(outdir): - os.makedirs(outdir) - - filepath = os.path.join(outdir, "failed.data") - pathlib.Path(filepath).write_text(self.failed_tests) - - filepath = os.path.join(outdir, "passed.data") - pathlib.Path(filepath).write_text(self.passed_tests) - - # gather intermediate data for all tests - failed_data = sorted(glob.glob(os.path.join(OUTDIR, "*/failed.data"))) - passed_data = sorted(glob.glob(os.path.join(OUTDIR, "*/passed.data"))) - - failed_tests = "" - passed_tests = "" - - for filename in failed_data: - filepath = os.path.join(OUTDIR, filename) - failed_tests += pathlib.Path(filepath).read_text() - for filename in passed_data: - filepath = os.path.join(OUTDIR, filename) - passed_tests += pathlib.Path(filepath).read_text() - - # write html for all tests - self.html = """ -<html> -<head> - <title>Cycles Test Report</title> - <style> - img {{ image-rendering: pixelated; width: 256px; background-color: #000; }} - img.render {{ - background-color: #fff; - background-image: - -moz-linear-gradient(45deg, #eee 25%, transparent 25%), - -moz-linear-gradient(-45deg, #eee 25%, transparent 25%), - -moz-linear-gradient(45deg, transparent 75%, #eee 75%), - -moz-linear-gradient(-45deg, transparent 75%, #eee 75%); - background-image: - -webkit-gradient(linear, 0 100%, 100% 0, color-stop(.25, #eee), color-stop(.25, transparent)), - -webkit-gradient(linear, 0 0, 100% 100%, color-stop(.25, #eee), color-stop(.25, transparent)), - -webkit-gradient(linear, 0 100%, 100% 0, color-stop(.75, transparent), color-stop(.75, #eee)), - -webkit-gradient(linear, 0 0, 100% 100%, color-stop(.75, transparent), color-stop(.75, #eee)); - - -moz-background-size:50px 50px; - background-size:50px 50px; - -webkit-background-size:50px 51px; /* override value for shitty webkit */ - - background-position:0 0, 25px 0, 25px -25px, 0px 25px; - }} - table td:first-child {{ width: 256px; }} - </style> - <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0-alpha.6/css/bootstrap.min.css"> -</head> -<body> - <div class="container"> - <br/> - <h1>Cycles Test Report</h1> - <br/> - <table class="table table-striped"> - <thead class="thead-default"> - <tr><th>Name</th><th>New</th><th>Reference</th><th>Diff</th> - </thead> - {}{} - </table> - <br/> - </div> -</body> -</html> - """ . format(failed_tests, passed_tests) - - filepath = os.path.join(OUTDIR, "report.html") - pathlib.Path(filepath).write_text(self.html) - - print_message("Report saved to: " + pathlib.Path(filepath).as_uri()) - - def relative_url(self, filepath): - relpath = os.path.relpath(filepath, OUTDIR) - return pathlib.Path(relpath).as_posix() - - def add_test(self, filepath, error): - name = test_get_name(filepath) - name = name.replace('_', ' ') - - old_img, ref_img, new_img, diff_img = test_get_images(filepath) - - status = error if error else "" - style = """ style="background-color: #f99;" """ if error else "" - - new_url = self.relative_url(new_img) - ref_url = self.relative_url(ref_img) - diff_url = self.relative_url(diff_img) - - test_html = """ - <tr{}> - <td><b>{}</b><br/>{}<br/>{}</td> - <td><img src="{}" onmouseover="this.src='{}';" onmouseout="this.src='{}';" class="render"></td> - <td><img src="{}" onmouseover="this.src='{}';" onmouseout="this.src='{}';" class="render"></td> - <td><img src="{}"></td> - </tr>""" . format(style, name, self.testname, status, - new_url, ref_url, new_url, - ref_url, new_url, ref_url, - diff_url) - - if error: - self.failed_tests += test_html - else: - self.passed_tests += test_html - - -def verify_output(report, filepath): - old_img, ref_img, new_img, diff_img = test_get_images(filepath) - - # copy new image - if os.path.exists(new_img): - os.remove(new_img) - if os.path.exists(TEMP_FILE): - shutil.copy(TEMP_FILE, new_img) - - update = os.getenv('CYCLESTEST_UPDATE') - - if os.path.exists(ref_img): - # diff test with threshold - command = ( - IDIFF, - "-fail", "0.016", - "-failpercent", "1", - ref_img, - TEMP_FILE, - ) - try: - subprocess.check_output(command) - failed = False - except subprocess.CalledProcessError as e: - if VERBOSE: - print_message(e.output.decode("utf-8")) - failed = e.returncode != 1 - else: - if not update: - return False - - failed = True - - if failed and update: - # update reference - shutil.copy(new_img, ref_img) - shutil.copy(new_img, old_img) - failed = False - - # generate diff image - command = ( - IDIFF, - "-o", diff_img, - "-abs", "-scale", "16", - ref_img, - TEMP_FILE - ) - - try: - subprocess.check_output(command) - except subprocess.CalledProcessError as e: - if VERBOSE: - print_message(e.output.decode("utf-8")) - - return not failed - - -def run_test(report, filepath): - testname = test_get_name(filepath) - spacer = "." * (32 - len(testname)) - print_message(testname, 'SUCCESS', 'RUN') - time_start = time.time() - error = render_file(filepath) - status = "FAIL" - if not error: - if not verify_output(report, filepath): - error = "VERIFY" - time_end = time.time() - elapsed_ms = int((time_end - time_start) * 1000) - if not error: - print_message("{} ({} ms)" . format(testname, elapsed_ms), - 'SUCCESS', 'OK') - else: - if error == "NO_CYCLES": - print_message("Can't perform tests because Cycles failed to load!") - return error - elif error == "NO_START": - print_message('Can not perform tests because blender fails to start.', - 'Make sure INSTALL target was run.') - return error - elif error == 'VERIFY': - print_message("Rendered result is different from reference image") - else: - print_message("Unknown error %r" % error) - print_message("{} ({} ms)" . format(testname, elapsed_ms), - 'FAILURE', 'FAILED') - return error - - - -def blend_list(path): - for dirpath, dirnames, filenames in os.walk(path): - for filename in filenames: - if filename.lower().endswith(".blend"): - filepath = os.path.join(dirpath, filename) - yield filepath - -def run_all_tests(dirpath): - passed_tests = [] - failed_tests = [] - all_files = list(blend_list(dirpath)) - all_files.sort() - report = Report(os.path.basename(dirpath)) - print_message("Running {} tests from 1 test case." . - format(len(all_files)), - 'SUCCESS', "==========") - time_start = time.time() - for filepath in all_files: - error = run_test(report, filepath) - testname = test_get_name(filepath) - if error: - if error == "NO_CYCLES": - return False - elif error == "NO_START": - return False - failed_tests.append(testname) - else: - passed_tests.append(testname) - report.add_test(filepath, error) - time_end = time.time() - elapsed_ms = int((time_end - time_start) * 1000) - print_message("") - print_message("{} tests from 1 test case ran. ({} ms total)" . - format(len(all_files), elapsed_ms), - 'SUCCESS', "==========") - print_message("{} tests." . - format(len(passed_tests)), - 'SUCCESS', 'PASSED') - if failed_tests: - print_message("{} tests, listed below:" . - format(len(failed_tests)), - 'FAILURE', 'FAILED') - failed_tests.sort() - for test in failed_tests: - print_message("{}" . format(test), 'FAILURE', "FAILED") - - report.output() - return not bool(failed_tests) - - def create_argparse(): parser = argparse.ArgumentParser() parser.add_argument("-blender", nargs="+") @@ -433,36 +112,19 @@ def main(): parser = create_argparse() args = parser.parse_args() - global COLORS - global BLENDER, TESTDIR, IDIFF, OUTDIR - global TEMP_FILE, TEMP_FILE_MASK, TEST_SCRIPT - global VERBOSE - - if os.environ.get("CYCLESTEST_COLOR") is not None: - COLORS = COLORS_ANSI + global BLENDER, VERBOSE BLENDER = args.blender[0] - TESTDIR = args.testdir[0] - IDIFF = args.idiff[0] - OUTDIR = args.outdir[0] - - if not os.path.exists(OUTDIR): - os.makedirs(OUTDIR) - - TEMP = tempfile.mkdtemp() - TEMP_FILE_MASK = os.path.join(TEMP, "test") - TEMP_FILE = TEMP_FILE_MASK + "0001.png" - - TEST_SCRIPT = os.path.join(os.path.dirname(__file__), "runtime_check.py") - VERBOSE = os.environ.get("BLENDER_VERBOSE") is not None - ok = run_all_tests(TESTDIR) + test_dir = args.testdir[0] + idiff = args.idiff[0] + output_dir = args.outdir[0] - # Cleanup temp files and folders - if os.path.exists(TEMP_FILE): - os.remove(TEMP_FILE) - os.rmdir(TEMP) + from modules import render_report + report = render_report.Report("Cycles Test Report", output_dir, idiff) + report.set_pixelated(True) + ok = report.run(test_dir, render_file) sys.exit(not ok) diff --git a/tests/python/modules/render_report.py b/tests/python/modules/render_report.py new file mode 100755 index 00000000000..5ccd5076fbc --- /dev/null +++ b/tests/python/modules/render_report.py @@ -0,0 +1,397 @@ +# Apache License, Version 2.0 +# +# Compare renders or screenshots against reference versions and generate +# a HTML report showing the differences, for regression testing. + +import glob +import os +import pathlib +import shutil +import subprocess +import sys +import time + + +class COLORS_ANSI: + RED = '\033[00;31m' + GREEN = '\033[00;32m' + ENDC = '\033[0m' + + +class COLORS_DUMMY: + RED = '' + GREEN = '' + ENDC = '' + +COLORS = COLORS_DUMMY + + +def print_message(message, type=None, status=''): + if type == 'SUCCESS': + print(COLORS.GREEN, end="") + elif type == 'FAILURE': + print(COLORS.RED, end="") + status_text = ... + if status == 'RUN': + status_text = " RUN " + elif status == 'OK': + status_text = " OK " + elif status == 'PASSED': + status_text = " PASSED " + elif status == 'FAILED': + status_text = " FAILED " + else: + status_text = status + if status_text: + print("[{}]" . format(status_text), end="") + print(COLORS.ENDC, end="") + print(" {}" . format(message)) + sys.stdout.flush() + + +def blend_list(dirpath): + for filename in os.listdir(dirpath): + if filename.lower().endswith(".blend"): + filepath = os.path.join(dirpath, filename) + yield filepath + +def test_get_name(filepath): + filename = os.path.basename(filepath) + return os.path.splitext(filename)[0] + +def test_get_images(output_dir, filepath): + testname = test_get_name(filepath) + dirpath = os.path.dirname(filepath) + + old_dirpath = os.path.join(dirpath, "reference_renders") + old_img = os.path.join(old_dirpath, testname + ".png") + + ref_dirpath = os.path.join(output_dir, os.path.basename(dirpath), "ref") + ref_img = os.path.join(ref_dirpath, testname + ".png") + if not os.path.exists(ref_dirpath): + os.makedirs(ref_dirpath) + if os.path.exists(old_img): + shutil.copy(old_img, ref_img) + + new_dirpath = os.path.join(output_dir, os.path.basename(dirpath)) + if not os.path.exists(new_dirpath): + os.makedirs(new_dirpath) + new_img = os.path.join(new_dirpath, testname + ".png") + + diff_dirpath = os.path.join(output_dir, os.path.basename(dirpath), "diff") + if not os.path.exists(diff_dirpath): + os.makedirs(diff_dirpath) + diff_img = os.path.join(diff_dirpath, testname + ".diff.png") + + return old_img, ref_img, new_img, diff_img + + +class Report: + __slots__ = ( + 'title', + 'output_dir', + 'idiff', + 'pixelated', + 'verbose', + 'update', + 'failed_tests', + 'passed_tests' + ) + + def __init__(self, title, output_dir, idiff): + self.title = title + self.output_dir = output_dir + self.idiff = idiff + + self.pixelated = False + self.verbose = os.environ.get("BLENDER_VERBOSE") is not None + self.update = os.getenv('BLENDER_TEST_UPDATE') is not None + + if os.environ.get("BLENDER_TEST_COLOR") is not None: + global COLORS, COLORS_ANSI + COLORS = COLORS_ANSI + + self.failed_tests = "" + self.passed_tests = "" + + if not os.path.exists(output_dir): + os.makedirs(output_dir) + + def set_pixelated(self, pixelated): + self.pixelated = pixelated + + def run(self, dirpath, render_cb): + # Run tests and output report. + dirname = os.path.basename(dirpath) + ok = self._run_all_tests(dirname, dirpath, render_cb) + self._write_html(dirname) + return ok + + def _write_html(self, dirname): + # Write intermediate data for single test. + outdir = os.path.join(self.output_dir, dirname) + if not os.path.exists(outdir): + os.makedirs(outdir) + + filepath = os.path.join(outdir, "failed.data") + pathlib.Path(filepath).write_text(self.failed_tests) + + filepath = os.path.join(outdir, "passed.data") + pathlib.Path(filepath).write_text(self.passed_tests) + + # Gather intermediate data for all tests. + failed_data = sorted(glob.glob(os.path.join(self.output_dir, "*/failed.data"))) + passed_data = sorted(glob.glob(os.path.join(self.output_dir, "*/passed.data"))) + + failed_tests = "" + passed_tests = "" + + for filename in failed_data: + filepath = os.path.join(self.output_dir, filename) + failed_tests += pathlib.Path(filepath).read_text() + for filename in passed_data: + filepath = os.path.join(self.output_dir, filename) + passed_tests += pathlib.Path(filepath).read_text() + + tests_html = failed_tests + passed_tests + + # Write html for all tests. + if self.pixelated: + image_rendering = 'pixelated' + else: + image_rendering = 'auto' + + if len(failed_tests) > 0: + message = "<p>Run <tt>BLENDER_TEST_UPDATE=1 ctest</tt> to create or update reference images for failed tests.</p>" + else: + message = "" + + html = """ +<html> +<head> + <title>{title}</title> + <style> + img {{ image-rendering: {image_rendering}; width: 256px; background-color: #000; }} + img.render {{ + background-color: #fff; + background-image: + -moz-linear-gradient(45deg, #eee 25%, transparent 25%), + -moz-linear-gradient(-45deg, #eee 25%, transparent 25%), + -moz-linear-gradient(45deg, transparent 75%, #eee 75%), + -moz-linear-gradient(-45deg, transparent 75%, #eee 75%); + background-image: + -webkit-gradient(linear, 0 100%, 100% 0, color-stop(.25, #eee), color-stop(.25, transparent)), + -webkit-gradient(linear, 0 0, 100% 100%, color-stop(.25, #eee), color-stop(.25, transparent)), + -webkit-gradient(linear, 0 100%, 100% 0, color-stop(.75, transparent), color-stop(.75, #eee)), + -webkit-gradient(linear, 0 0, 100% 100%, color-stop(.75, transparent), color-stop(.75, #eee)); + + -moz-background-size:50px 50px; + background-size:50px 50px; + -webkit-background-size:50px 51px; /* override value for shitty webkit */ + + background-position:0 0, 25px 0, 25px -25px, 0px 25px; + }} + table td:first-child {{ width: 256px; }} + </style> + <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0-alpha.6/css/bootstrap.min.css"> +</head> +<body> + <div class="container"> + <br/> + <h1>{title}</h1> + {message} + <br/> + <table class="table table-striped"> + <thead class="thead-default"> + <tr><th>Name</th><th>New</th><th>Reference</th><th>Diff</th> + </thead> + {tests_html} + </table> + <br/> + </div> +</body> +</html> + """ . format(title=self.title, + message=message, + image_rendering=image_rendering, + tests_html=tests_html) + + filepath = os.path.join(self.output_dir, "report.html") + pathlib.Path(filepath).write_text(html) + + print_message("Report saved to: " + pathlib.Path(filepath).as_uri()) + + def _relative_url(self, filepath): + relpath = os.path.relpath(filepath, self.output_dir) + return pathlib.Path(relpath).as_posix() + + def _write_test_html(self, testname, filepath, error): + name = test_get_name(filepath) + name = name.replace('_', ' ') + + old_img, ref_img, new_img, diff_img = test_get_images(self.output_dir, filepath) + + status = error if error else "" + tr_style = """ style="background-color: #f99;" """ if error else "" + + new_url = self._relative_url(new_img) + ref_url = self._relative_url(ref_img) + diff_url = self._relative_url(diff_img) + + test_html = """ + <tr{tr_style}> + <td><b>{name}</b><br/>{testname}<br/>{status}</td> + <td><img src="{new_url}" onmouseover="this.src='{ref_url}';" onmouseout="this.src='{new_url}';" class="render"></td> + <td><img src="{ref_url}" onmouseover="this.src='{new_url}';" onmouseout="this.src='{ref_url}';" class="render"></td> + <td><img src="{diff_url}"></td> + </tr>""" . format(tr_style=tr_style, + name=name, + testname=testname, + status=status, + new_url=new_url, + ref_url=ref_url, + diff_url=diff_url) + + if error: + self.failed_tests += test_html + else: + self.passed_tests += test_html + + + def _diff_output(self, filepath, tmp_filepath): + old_img, ref_img, new_img, diff_img = test_get_images(self.output_dir, filepath) + + # Create reference render directory. + old_dirpath = os.path.dirname(old_img) + if not os.path.exists(old_dirpath): + os.makedirs(old_dirpath) + + # Copy temporary to new image. + if os.path.exists(new_img): + os.remove(new_img) + if os.path.exists(tmp_filepath): + shutil.copy(tmp_filepath, new_img) + + if os.path.exists(ref_img): + # Diff images test with threshold. + command = ( + self.idiff, + "-fail", "0.016", + "-failpercent", "1", + ref_img, + tmp_filepath, + ) + try: + subprocess.check_output(command) + failed = False + except subprocess.CalledProcessError as e: + if self.verbose: + print_message(e.output.decode("utf-8")) + failed = e.returncode != 1 + else: + if not self.update: + return False + + failed = True + + if failed and self.update: + # Update reference image if requested. + shutil.copy(new_img, ref_img) + shutil.copy(new_img, old_img) + failed = False + + # Generate diff image. + command = ( + self.idiff, + "-o", diff_img, + "-abs", "-scale", "16", + ref_img, + tmp_filepath + ) + + try: + subprocess.check_output(command) + except subprocess.CalledProcessError as e: + if self.verbose: + print_message(e.output.decode("utf-8")) + + return not failed + + + def _run_test(self, filepath, render_cb): + testname = test_get_name(filepath) + print_message(testname, 'SUCCESS', 'RUN') + time_start = time.time() + tmp_filepath = os.path.join(self.output_dir, "tmp") + + error = render_cb(filepath, tmp_filepath) + status = "FAIL" + if not error: + if not self._diff_output(filepath, tmp_filepath): + error = "VERIFY" + + if os.path.exists(tmp_filepath): + os.remove(tmp_filepath) + + time_end = time.time() + elapsed_ms = int((time_end - time_start) * 1000) + if not error: + print_message("{} ({} ms)" . format(testname, elapsed_ms), + 'SUCCESS', 'OK') + else: + if error == "NO_ENGINE": + print_message("Can't perform tests because the render engine failed to load!") + return error + elif error == "NO_START": + print_message('Can not perform tests because blender fails to start.', + 'Make sure INSTALL target was run.') + return error + elif error == 'VERIFY': + print_message("Rendered result is different from reference image") + else: + print_message("Unknown error %r" % error) + print_message("{} ({} ms)" . format(testname, elapsed_ms), + 'FAILURE', 'FAILED') + return error + + + def _run_all_tests(self, dirname, dirpath, render_cb): + passed_tests = [] + failed_tests = [] + all_files = list(blend_list(dirpath)) + all_files.sort() + print_message("Running {} tests from 1 test case." . + format(len(all_files)), + 'SUCCESS', "==========") + time_start = time.time() + for filepath in all_files: + error = self._run_test(filepath, render_cb) + testname = test_get_name(filepath) + if error: + if error == "NO_ENGINE": + return False + elif error == "NO_START": + return False + failed_tests.append(testname) + else: + passed_tests.append(testname) + self._write_test_html(dirname, filepath, error) + time_end = time.time() + elapsed_ms = int((time_end - time_start) * 1000) + print_message("") + print_message("{} tests from 1 test case ran. ({} ms total)" . + format(len(all_files), elapsed_ms), + 'SUCCESS', "==========") + print_message("{} tests." . + format(len(passed_tests)), + 'SUCCESS', 'PASSED') + if failed_tests: + print_message("{} tests, listed below:" . + format(len(failed_tests)), + 'FAILURE', 'FAILED') + failed_tests.sort() + for test in failed_tests: + print_message("{}" . format(test), 'FAILURE', "FAILED") + + return not bool(failed_tests) + diff --git a/tests/python/opengl_draw_tests.py b/tests/python/opengl_draw_tests.py new file mode 100755 index 00000000000..999304570df --- /dev/null +++ b/tests/python/opengl_draw_tests.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +# Apache License, Version 2.0 + +import argparse +import os +import shlex +import shutil +import subprocess +import sys + +def screenshot(): + import bpy + + output_path = sys.argv[-1] + + # Force redraw and take screenshot. + bpy.ops.wm.redraw_timer(type='DRAW_WIN_SWAP', iterations=1) + bpy.ops.screen.screenshot(filepath=output_path, full=True) + + bpy.ops.wm.quit_blender() + +# When run from inside Blender, take screenshot and exit. +try: + import bpy + inside_blender = True +except ImportError: + inside_blender = False + +if inside_blender: + screenshot() + sys.exit(0) + + +def render_file(filepath, output_filepath): + command = ( + BLENDER, + "-noaudio", + "--factory-startup", + "--enable-autoexec", + filepath, + "-P", + os.path.realpath(__file__), + "--", + output_filepath) + + try: + # Success + output = subprocess.check_output(command) + if VERBOSE: + print(output.decode("utf-8")) + return None + except subprocess.CalledProcessError as e: + # Error + if os.path.exists(output_filepath): + os.remove(output_filepath) + if VERBOSE: + print(e.output.decode("utf-8")) + return "CRASH" + except BaseException as e: + # Crash + if os.path.exists(output_filepath): + os.remove(output_filepath) + if VERBOSE: + print(e) + return "CRASH" + + +def create_argparse(): + parser = argparse.ArgumentParser() + parser.add_argument("-blender", nargs="+") + parser.add_argument("-testdir", nargs=1) + parser.add_argument("-outdir", nargs=1) + parser.add_argument("-idiff", nargs=1) + return parser + + +def main(): + parser = create_argparse() + args = parser.parse_args() + + global BLENDER, VERBOSE + + BLENDER = args.blender[0] + VERBOSE = os.environ.get("BLENDER_VERBOSE") is not None + + test_dir = args.testdir[0] + idiff = args.idiff[0] + output_dir = args.outdir[0] + + from modules import render_report + report = render_report.Report("OpenGL Draw Test Report", output_dir, idiff) + ok = report.run(test_dir, render_file) + + sys.exit(not ok) + +if __name__ == "__main__": + main() |