diff options
author | Brecht Van Lommel <brechtvanlommel@gmail.com> | 2018-05-18 17:40:41 +0300 |
---|---|---|
committer | Brecht Van Lommel <brechtvanlommel@gmail.com> | 2018-05-18 18:11:24 +0300 |
commit | 6a782ed767f993404cd5ba1cd233f49f9fffaaf2 (patch) | |
tree | dbc3bbd1f67b1295d7e1c53121b575e7019c9ed0 | |
parent | 00071d78bcfbe8ae0c813c769af075f069d2cb8d (diff) |
Eevee: add regression tests.
This adds Eevee render tests using the Cycles files. Currently it must
be enabled by setting WITH_OPENGL_RENDER_TESTS=ON. Once we have reference
images we can enable it by default.
Some of the Cycles and Eevee tests are also currently broken due to
modifier and particle changes.
Differential Revision: https://developer.blender.org/D3182
-rw-r--r-- | tests/python/CMakeLists.txt | 62 | ||||
-rwxr-xr-x | tests/python/cycles_render_tests.py | 3 | ||||
-rwxr-xr-x | tests/python/eevee_render_tests.py | 132 | ||||
-rwxr-xr-x | tests/python/modules/render_report.py | 13 |
4 files changed, 179 insertions, 31 deletions
diff --git a/tests/python/CMakeLists.txt b/tests/python/CMakeLists.txt index 50e2927f54f..d5ba68b9b93 100644 --- a/tests/python/CMakeLists.txt +++ b/tests/python/CMakeLists.txt @@ -531,9 +531,9 @@ function(add_python_test testname testscript) endif() endfunction() -if(WITH_CYCLES) - if(OPENIMAGEIO_IDIFF AND EXISTS "${TEST_SRC_DIR}/render/ctests/shader") - macro(add_cycles_render_test subject) +if(OPENIMAGEIO_IDIFF AND EXISTS "${TEST_SRC_DIR}/render/ctests/shader") + macro(add_cycles_render_test subject) + if(WITH_CYCLES) add_python_test( cycles_${subject}_test ${CMAKE_CURRENT_LIST_DIR}/cycles_render_tests.py @@ -542,31 +542,39 @@ if(WITH_CYCLES) -idiff "${OPENIMAGEIO_IDIFF}" -outdir "${TEST_OUT_DIR}/cycles" ) - endmacro() - if(WITH_OPENGL_RENDER_TESTS) - add_cycles_render_test(opengl) endif() - add_cycles_render_test(bake) - add_cycles_render_test(bsdf) - add_cycles_render_test(denoise) - add_cycles_render_test(displacement) - add_cycles_render_test(hair) - add_cycles_render_test(image_data_types) - add_cycles_render_test(image_mapping) - add_cycles_render_test(image_texture_limit) - add_cycles_render_test(integrator) - add_cycles_render_test(light) - add_cycles_render_test(mesh) - add_cycles_render_test(motion_blur) - add_cycles_render_test(render_layer) - add_cycles_render_test(reports) - add_cycles_render_test(shader) - add_cycles_render_test(shadow_catcher) - add_cycles_render_test(sss) - add_cycles_render_test(volume) - else() - MESSAGE(STATUS "Disabling Cycles tests because tests folder does not exist") - endif() + + if(WITH_OPENGL_RENDER_TESTS AND (NOT ${subject} MATCHES "bake")) + add_python_test( + eevee_${subject}_test + ${CMAKE_CURRENT_LIST_DIR}/eevee_render_tests.py + -blender "$<TARGET_FILE:blender>" + -testdir "${TEST_SRC_DIR}/render/ctests/${subject}" + -idiff "${OPENIMAGEIO_IDIFF}" + -outdir "${TEST_OUT_DIR}/eevee" + ) + endif() + endmacro() + add_cycles_render_test(bake) + add_cycles_render_test(bsdf) + add_cycles_render_test(denoise) + add_cycles_render_test(displacement) + add_cycles_render_test(hair) + add_cycles_render_test(image_data_types) + add_cycles_render_test(image_mapping) + add_cycles_render_test(image_texture_limit) + add_cycles_render_test(integrator) + add_cycles_render_test(light) + add_cycles_render_test(mesh) + add_cycles_render_test(motion_blur) + add_cycles_render_test(render_layer) + add_cycles_render_test(reports) + add_cycles_render_test(shader) + add_cycles_render_test(shadow_catcher) + add_cycles_render_test(sss) + add_cycles_render_test(volume) +elseif(WITH_CYCLES) + MESSAGE(STATUS "Disabling Cycles tests because tests folder does not exist") endif() if(WITH_OPENGL_DRAW_TESTS) diff --git a/tests/python/cycles_render_tests.py b/tests/python/cycles_render_tests.py index a01a6f74e15..eb7a1d96d24 100755 --- a/tests/python/cycles_render_tests.py +++ b/tests/python/cycles_render_tests.py @@ -77,6 +77,7 @@ def render_file(filepath, output_filepath): shutil.copy(frame_filepath, output_filepath) os.remove(frame_filepath) if VERBOSE: + print(" ".join(command)) print(output.decode("utf-8")) return None except subprocess.CalledProcessError as e: @@ -84,6 +85,7 @@ def render_file(filepath, output_filepath): if os.path.exists(frame_filepath): os.remove(frame_filepath) if VERBOSE: + print(" ".join(command)) print(e.output.decode("utf-8")) if b"Error: engine not found" in e.output: return "NO_ENGINE" @@ -95,6 +97,7 @@ def render_file(filepath, output_filepath): if os.path.exists(frame_filepath): os.remove(frame_filepath) if VERBOSE: + print(" ".join(command)) print(e) return "CRASH" diff --git a/tests/python/eevee_render_tests.py b/tests/python/eevee_render_tests.py new file mode 100755 index 00000000000..176bb470e9a --- /dev/null +++ b/tests/python/eevee_render_tests.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +# Apache License, Version 2.0 + +import argparse +import os +import shlex +import shutil +import subprocess +import sys + +def setup(): + import bpy + + # Enable Eevee features + scene = bpy.context.scene + eevee = scene.eevee + + eevee.sss_enable = True + eevee.ssr_enable = True + eevee.ssr_refraction = True + eevee.gtao_enable = True + eevee.dof_enable = True + + eevee.volumetric_enable = True + eevee.volumetric_shadows = True + eevee.volumetric_tile_size = '2' + + for mat in bpy.data.materials: + mat.use_screen_refraction = True + mat.use_screen_subsurface = True + +# When run from inside Blender, render and exit. +try: + import bpy + inside_blender = True +except ImportError: + inside_blender = False + +if inside_blender: + try: + setup() + except Exception as e: + print(e) + sys.exit(1) + + +def render_file(filepath, output_filepath): + dirname = os.path.dirname(filepath) + basedir = os.path.dirname(dirname) + subject = os.path.basename(dirname) + + frame_filepath = output_filepath + '0001.png' + + command = [ + BLENDER, + "--background", + "-noaudio", + "--factory-startup", + "--enable-autoexec", + filepath, + "-E", "BLENDER_EEVEE", + "-P", + os.path.realpath(__file__), + "-o", output_filepath, + "-F", "PNG", + "-f", "1"] + + try: + # Success + output = subprocess.check_output(command) + if os.path.exists(frame_filepath): + shutil.copy(frame_filepath, output_filepath) + os.remove(frame_filepath) + if VERBOSE: + print(" ".join(command)) + print(output.decode("utf-8")) + return None + except subprocess.CalledProcessError as e: + # Error + if os.path.exists(frame_filepath): + os.remove(frame_filepath) + if VERBOSE: + print(" ".join(command)) + print(e.output.decode("utf-8")) + if b"Error: engine not found" in e.output: + return "NO_ENGINE" + elif b"blender probably wont start" in e.output: + return "NO_START" + return "CRASH" + except BaseException as e: + # Crash + if os.path.exists(frame_filepath): + os.remove(frame_filepath) + if VERBOSE: + print(" ".join(command)) + print(e) + return "CRASH" + + +def create_argparse(): + parser = argparse.ArgumentParser() + parser.add_argument("-blender", nargs="+") + parser.add_argument("-testdir", nargs=1) + parser.add_argument("-outdir", nargs=1) + parser.add_argument("-idiff", nargs=1) + return parser + + +def main(): + parser = create_argparse() + args = parser.parse_args() + + global BLENDER, VERBOSE + + BLENDER = args.blender[0] + VERBOSE = os.environ.get("BLENDER_VERBOSE") is not None + + test_dir = args.testdir[0] + idiff = args.idiff[0] + output_dir = args.outdir[0] + + from modules import render_report + report = render_report.Report("Eevee Test Report", output_dir, idiff) + report.set_pixelated(True) + report.set_reference_dir("eevee_renders") + ok = report.run(test_dir, render_file) + + sys.exit(not ok) + + +if not inside_blender and __name__ == "__main__": + main() diff --git a/tests/python/modules/render_report.py b/tests/python/modules/render_report.py index 5ccd5076fbc..ec54ba4e823 100755 --- a/tests/python/modules/render_report.py +++ b/tests/python/modules/render_report.py @@ -59,11 +59,11 @@ def test_get_name(filepath): filename = os.path.basename(filepath) return os.path.splitext(filename)[0] -def test_get_images(output_dir, filepath): +def test_get_images(output_dir, filepath, reference_dir): testname = test_get_name(filepath) dirpath = os.path.dirname(filepath) - old_dirpath = os.path.join(dirpath, "reference_renders") + old_dirpath = os.path.join(dirpath, reference_dir) old_img = os.path.join(old_dirpath, testname + ".png") ref_dirpath = os.path.join(output_dir, os.path.basename(dirpath), "ref") @@ -90,6 +90,7 @@ class Report: __slots__ = ( 'title', 'output_dir', + 'reference_dir', 'idiff', 'pixelated', 'verbose', @@ -101,6 +102,7 @@ class Report: def __init__(self, title, output_dir, idiff): self.title = title self.output_dir = output_dir + self.reference_dir = 'reference_renders' self.idiff = idiff self.pixelated = False @@ -120,6 +122,9 @@ class Report: def set_pixelated(self, pixelated): self.pixelated = pixelated + def set_reference_dir(self, reference_dir): + self.reference_dir = reference_dir + def run(self, dirpath, render_cb): # Run tests and output report. dirname = os.path.basename(dirpath) @@ -229,7 +234,7 @@ class Report: name = test_get_name(filepath) name = name.replace('_', ' ') - old_img, ref_img, new_img, diff_img = test_get_images(self.output_dir, filepath) + old_img, ref_img, new_img, diff_img = test_get_images(self.output_dir, filepath, self.reference_dir) status = error if error else "" tr_style = """ style="background-color: #f99;" """ if error else "" @@ -259,7 +264,7 @@ class Report: def _diff_output(self, filepath, tmp_filepath): - old_img, ref_img, new_img, diff_img = test_get_images(self.output_dir, filepath) + old_img, ref_img, new_img, diff_img = test_get_images(self.output_dir, filepath, self.reference_dir) # Create reference render directory. old_dirpath = os.path.dirname(old_img) |