Tests: add Eevee reference images, and add workbench tests

Being able to compare Eevee reference images is useful for refactoring I'm
working on so might as well add them now, even if we can still improve them.

Workbench tests are just rendering the same files as Cycles and Eevee. This
doesn't really tests many workbench settings until we add tests specifically
for them, but does cover how it it handles the different object types.

3 files changed, 133 insertions, 3 deletions

diff --git a/tests/python/CMakeLists.txt b/tests/python/CMakeLists.txt
index 2d9fe8c4f37..28bdd48229e 100644
--- a/tests/python/CMakeLists.txt
+++ b/tests/python/CMakeLists.txt
@@ -528,14 +528,14 @@ function(add_python_test testname testscript)
   endif()
 endfunction()
 
-if(OPENIMAGEIO_IDIFF AND EXISTS "${TEST_SRC_DIR}/render/ctests/shader")
+if(OPENIMAGEIO_IDIFF AND EXISTS "${TEST_SRC_DIR}/render/shader")
   macro(add_cycles_render_test subject)
     if(WITH_CYCLES)
       add_python_test(
         cycles_${subject}
         ${CMAKE_CURRENT_LIST_DIR}/cycles_render_tests.py
         -blender "$<TARGET_FILE:blender>"
-        -testdir "${TEST_SRC_DIR}/render/ctests/${subject}"
+        -testdir "${TEST_SRC_DIR}/render/${subject}"
         -idiff "${OPENIMAGEIO_IDIFF}"
         -outdir "${TEST_OUT_DIR}/cycles"
       )
@@ -546,10 +546,19 @@ if(OPENIMAGEIO_IDIFF AND EXISTS "${TEST_SRC_DIR}/render/ctests/shader")
         eevee_${subject}_test
         ${CMAKE_CURRENT_LIST_DIR}/eevee_render_tests.py
         -blender "$<TARGET_FILE:blender>"
-        -testdir "${TEST_SRC_DIR}/render/ctests/${subject}"
+        -testdir "${TEST_SRC_DIR}/render/${subject}"
         -idiff "${OPENIMAGEIO_IDIFF}"
         -outdir "${TEST_OUT_DIR}/eevee"
       )
+
+      add_python_test(
+        workbench_${subject}_test
+        ${CMAKE_CURRENT_LIST_DIR}/workbench_render_tests.py
+        -blender "$<TARGET_FILE:blender>"
+        -testdir "${TEST_SRC_DIR}/render/${subject}"
+        -idiff "${OPENIMAGEIO_IDIFF}"
+        -outdir "${TEST_OUT_DIR}/workbench"
+      )
     endif()
   endmacro()
   add_cycles_render_test(bake)
diff --git a/tests/python/cycles_render_tests.py b/tests/python/cycles_render_tests.py
index 381c04d9b7f..36f5459c2f7 100755
--- a/tests/python/cycles_render_tests.py
+++ b/tests/python/cycles_render_tests.py
@@ -106,6 +106,7 @@ def main():
     from modules import render_report
     report = render_report.Report("Cycles Test Report", output_dir, idiff)
     report.set_pixelated(True)
+    report.set_reference_dir("cycles_renders")
     report.set_compare_engines('cycles', 'eevee')
     ok = report.run(test_dir, render_file)
 
diff --git a/tests/python/workbench_render_tests.py b/tests/python/workbench_render_tests.py
new file mode 100755
index 00000000000..5b759214140
--- /dev/null
+++ b/tests/python/workbench_render_tests.py
@@ -0,0 +1,120 @@
+#!/usr/bin/env python3
+# Apache License, Version 2.0
+
+import argparse
+import os
+import shlex
+import shutil
+import subprocess
+import sys
+
+
+def setup():
+    import bpy
+
+    scene = bpy.context.scene
+    scene.display.shading.color_type = 'TEXTURE'
+
+
+# When run from inside Blender, render and exit.
+try:
+    import bpy
+    inside_blender = True
+except ImportError:
+    inside_blender = False
+
+if inside_blender:
+    try:
+        setup()
+    except Exception as e:
+        print(e)
+        sys.exit(1)
+
+
+def render_file(filepath, output_filepath):
+    dirname = os.path.dirname(filepath)
+    basedir = os.path.dirname(dirname)
+    subject = os.path.basename(dirname)
+
+    frame_filepath = output_filepath + '0001.png'
+
+    command = [
+        BLENDER,
+        "--background",
+        "-noaudio",
+        "--factory-startup",
+        "--enable-autoexec",
+        filepath,
+        "-E", "BLENDER_WORKBENCH",
+        "-P",
+        os.path.realpath(__file__),
+        "-o", output_filepath,
+        "-F", "PNG",
+        "-f", "1"]
+
+    try:
+        # Success
+        output = subprocess.check_output(command)
+        if os.path.exists(frame_filepath):
+            shutil.copy(frame_filepath, output_filepath)
+            os.remove(frame_filepath)
+        if VERBOSE:
+            print(" ".join(command))
+            print(output.decode("utf-8"))
+        return None
+    except subprocess.CalledProcessError as e:
+        # Error
+        if os.path.exists(frame_filepath):
+            os.remove(frame_filepath)
+        if VERBOSE:
+            print(" ".join(command))
+            print(e.output.decode("utf-8"))
+        if b"Error: engine not found" in e.output:
+            return "NO_ENGINE"
+        elif b"blender probably wont start" in e.output:
+            return "NO_START"
+        return "CRASH"
+    except BaseException as e:
+        # Crash
+        if os.path.exists(frame_filepath):
+            os.remove(frame_filepath)
+        if VERBOSE:
+            print(" ".join(command))
+            print(e)
+        return "CRASH"
+
+
+def create_argparse():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-blender", nargs="+")
+    parser.add_argument("-testdir", nargs=1)
+    parser.add_argument("-outdir", nargs=1)
+    parser.add_argument("-idiff", nargs=1)
+    return parser
+
+
+def main():
+    parser = create_argparse()
+    args = parser.parse_args()
+
+    global BLENDER, VERBOSE
+
+    BLENDER = args.blender[0]
+    VERBOSE = os.environ.get("BLENDER_VERBOSE") is not None
+
+    test_dir = args.testdir[0]
+    idiff = args.idiff[0]
+    output_dir = args.outdir[0]
+
+    from modules import render_report
+    report = render_report.Report("Workbench Test Report", output_dir, idiff)
+    report.set_pixelated(True)
+    report.set_reference_dir("workbench_renders")
+    report.set_compare_engines('workbench', 'eevee')
+    ok = report.run(test_dir, render_file)
+
+    sys.exit(not ok)
+
+
+if not inside_blender and __name__ == "__main__":
+    main()