Tests: speed up render tests by running multiple in the same process

Blender startup time and shader compilation is a big factor when running
hundreds of tests, so now all renders in the same ctest run in the same
process.

This was previously reverted due to skipping other tests when one test
crashed. Now if a test crashes, Blender is re-run with the remaining
tests so we get results from them still.

5 files changed, 100 insertions, 218 deletions

diff --git a/tests/python/cycles_render_tests.py b/tests/python/cycles_render_tests.py
index 349ba83f88a..79ba11fdd44 100755
--- a/tests/python/cycles_render_tests.py
+++ b/tests/python/cycles_render_tests.py
@@ -9,14 +9,13 @@ import subprocess
 import sys
 
 
-def render_file(filepath, output_filepath):
+def get_arguments(filepath, output_filepath):
     dirname = os.path.dirname(filepath)
     basedir = os.path.dirname(dirname)
     subject = os.path.basename(dirname)
 
-    frame_filepath = output_filepath + '0001.png'
-
-    common_args = [
+    args = [
+        "--background",
         "-noaudio",
         "--factory-startup",
         "--enable-autoexec",
@@ -29,57 +28,17 @@ def render_file(filepath, output_filepath):
     # custom_args += ["--python-expr", "import bpy; bpy.context.scene.cycles.shading_system = True"]
     # custom_args += ["--python-expr", "import bpy; bpy.context.scene.cycles.device = 'GPU'"]
     custom_args = os.getenv('CYCLESTEST_ARGS')
-    custom_args = shlex.split(custom_args) if custom_args else []
-    common_args += custom_args
+    if custom_args:
+        args.extend(shlex.split(custom_args))
 
-    if subject == 'opengl':
-        command = [BLENDER, "--window-geometry", "0", "0", "1", "1"]
-        command += common_args
-        command += ['--python', os.path.join(basedir, "util", "render_opengl.py")]
-    elif subject == 'bake':
-        command = [BLENDER, "--background"]
-        command += common_args
-        command += ['--python', os.path.join(basedir, "util", "render_bake.py")]
+    if subject == 'bake':
+        args.extend(['--python', os.path.join(basedir, "util", "render_bake.py")])
     elif subject == 'denoise_animation':
-        command = [BLENDER, "--background"]
-        command += common_args
-        command += ['--python', os.path.join(basedir, "util", "render_denoise.py")]
+        args.extend(['--python', os.path.join(basedir, "util", "render_denoise.py")])
     else:
-        command = [BLENDER, "--background"]
-        command += common_args
-        command += ["-f", "1"]
-
-    try:
-        # Success
-        output = subprocess.check_output(command)
-        if os.path.exists(frame_filepath):
-            shutil.copy(frame_filepath, output_filepath)
-            os.remove(frame_filepath)
-        if VERBOSE:
-            print(" ".join(command))
-            print(output.decode("utf-8"))
-        return None
-    except subprocess.CalledProcessError as e:
-        # Error
-        if os.path.exists(frame_filepath):
-            os.remove(frame_filepath)
-        if VERBOSE:
-            print(" ".join(command))
-            print(e.output.decode("utf-8"))
-        if b"Error: engine not found" in e.output:
-            return "NO_ENGINE"
-        elif b"blender probably wont start" in e.output:
-            return "NO_START"
-        return "CRASH"
-    except BaseException as e:
-        # Crash
-        if os.path.exists(frame_filepath):
-            os.remove(frame_filepath)
-        if VERBOSE:
-            print(" ".join(command))
-            print(e)
-        return "CRASH"
+        args.extend(["-f", "1"])
 
+    return args
 
 def create_argparse():
     parser = argparse.ArgumentParser()
@@ -94,11 +53,7 @@ def main():
     parser = create_argparse()
     args = parser.parse_args()
 
-    global BLENDER, VERBOSE
-
-    BLENDER = args.blender[0]
-    VERBOSE = os.environ.get("BLENDER_VERBOSE") is not None
-
+    blender = args.blender[0]
     test_dir = args.testdir[0]
     idiff = args.idiff[0]
     output_dir = args.outdir[0]
@@ -108,7 +63,7 @@ def main():
     report.set_pixelated(True)
     report.set_reference_dir("cycles_renders")
     report.set_compare_engines('cycles', 'eevee')
-    ok = report.run(test_dir, render_file)
+    ok = report.run(test_dir, blender, get_arguments, batch=True)
 
     sys.exit(not ok)
 
diff --git a/tests/python/eevee_render_tests.py b/tests/python/eevee_render_tests.py
index 9fb8a5f41a8..0875e4b171a 100755
--- a/tests/python/eevee_render_tests.py
+++ b/tests/python/eevee_render_tests.py
@@ -8,7 +8,6 @@ import shutil
 import subprocess
 import sys
 
-
 def setup():
     import bpy
 
@@ -46,15 +45,8 @@ if inside_blender:
         sys.exit(1)
 
 
-def render_file(filepath, output_filepath):
-    dirname = os.path.dirname(filepath)
-    basedir = os.path.dirname(dirname)
-    subject = os.path.basename(dirname)
-
-    frame_filepath = output_filepath + '0001.png'
-
-    command = [
-        BLENDER,
+def get_arguments(filepath, output_filepath):
+    return [
         "--background",
         "-noaudio",
         "--factory-startup",
@@ -67,37 +59,6 @@ def render_file(filepath, output_filepath):
         "-F", "PNG",
         "-f", "1"]
 
-    try:
-        # Success
-        output = subprocess.check_output(command)
-        if os.path.exists(frame_filepath):
-            shutil.copy(frame_filepath, output_filepath)
-            os.remove(frame_filepath)
-        if VERBOSE:
-            print(" ".join(command))
-            print(output.decode("utf-8"))
-        return None
-    except subprocess.CalledProcessError as e:
-        # Error
-        if os.path.exists(frame_filepath):
-            os.remove(frame_filepath)
-        if VERBOSE:
-            print(" ".join(command))
-            print(e.output.decode("utf-8"))
-        if b"Error: engine not found" in e.output:
-            return "NO_ENGINE"
-        elif b"blender probably wont start" in e.output:
-            return "NO_START"
-        return "CRASH"
-    except BaseException as e:
-        # Crash
-        if os.path.exists(frame_filepath):
-            os.remove(frame_filepath)
-        if VERBOSE:
-            print(" ".join(command))
-            print(e)
-        return "CRASH"
-
 
 def create_argparse():
     parser = argparse.ArgumentParser()
@@ -112,11 +73,7 @@ def main():
     parser = create_argparse()
     args = parser.parse_args()
 
-    global BLENDER, VERBOSE
-
-    BLENDER = args.blender[0]
-    VERBOSE = os.environ.get("BLENDER_VERBOSE") is not None
-
+    blender = args.blender[0]
     test_dir = args.testdir[0]
     idiff = args.idiff[0]
     output_dir = args.outdir[0]
@@ -126,7 +83,7 @@ def main():
     report.set_pixelated(True)
     report.set_reference_dir("eevee_renders")
     report.set_compare_engines('eevee', 'cycles')
-    ok = report.run(test_dir, render_file)
+    ok = report.run(test_dir, blender, get_arguments, batch=True)
 
     sys.exit(not ok)
 
diff --git a/tests/python/modules/render_report.py b/tests/python/modules/render_report.py
index 2f99a3b6292..15826f97400 100755
--- a/tests/python/modules/render_report.py
+++ b/tests/python/modules/render_report.py
@@ -134,10 +134,10 @@ class Report:
     def set_compare_engines(self, engine, other_engine):
         self.compare_engines = (engine, other_engine)
 
-    def run(self, dirpath, render_cb):
+    def run(self, dirpath, blender, arguments_cb, batch=False):
         # Run tests and output report.
         dirname = os.path.basename(dirpath)
-        ok = self._run_all_tests(dirname, dirpath, render_cb)
+        ok = self._run_all_tests(dirname, dirpath, blender, arguments_cb, batch)
         self._write_data(dirname)
         self._write_html()
         if self.compare_engines:
@@ -399,43 +399,81 @@ class Report:
 
         return not failed
 
-    def _run_test(self, filepath, render_cb):
-        testname = test_get_name(filepath)
-        print_message(testname, 'SUCCESS', 'RUN')
-        time_start = time.time()
-        tmp_filepath = os.path.join(self.output_dir, "tmp_" + testname)
+    def _run_tests(self, filepaths, blender, arguments_cb, batch):
+        # Run multiple tests in a single Blender process since startup can be
+        # a significant factor. In case of crashes, re-run the remaining tests.
+        verbose = os.environ.get("BLENDER_VERBOSE") is not None
 
-        error = render_cb(filepath, tmp_filepath)
-        status = "FAIL"
-        if not error:
-            if not self._diff_output(filepath, tmp_filepath):
-                error = "VERIFY"
+        remaining_filepaths = filepaths[:]
+        errors = []
 
-        if os.path.exists(tmp_filepath):
-            os.remove(tmp_filepath)
+        while len(remaining_filepaths) > 0:
+            command = [blender]
+            output_filepaths = []
 
-        time_end = time.time()
-        elapsed_ms = int((time_end - time_start) * 1000)
-        if not error:
-            print_message("{} ({} ms)" . format(testname, elapsed_ms),
-                          'SUCCESS', 'OK')
-        else:
-            if error == "NO_ENGINE":
-                print_message("Can't perform tests because the render engine failed to load!")
-                return error
-            elif error == "NO_START":
-                print_message('Can not perform tests because blender fails to start.',
-                              'Make sure INSTALL target was run.')
-                return error
-            elif error == 'VERIFY':
-                print_message("Rendered result is different from reference image")
-            else:
-                print_message("Unknown error %r" % error)
-            print_message("{} ({} ms)" . format(testname, elapsed_ms),
-                          'FAILURE', 'FAILED')
-        return error
+            # Construct output filepaths and command to run
+            for filepath in remaining_filepaths:
+                testname = test_get_name(filepath)
+                print_message(testname, 'SUCCESS', 'RUN')
 
-    def _run_all_tests(self, dirname, dirpath, render_cb):
+                base_output_filepath = os.path.join(self.output_dir, "tmp_" + testname)
+                output_filepath = base_output_filepath + '0001.png'
+                output_filepaths.append(output_filepath)
+
+                if os.path.exists(output_filepath):
+                    os.remove(output_filepath)
+
+                command.extend(arguments_cb(filepath, base_output_filepath))
+
+                # Only chain multiple commands for batch
+                if not batch:
+                    break
+
+            # Run process
+            crash = False
+            try:
+                output = subprocess.check_output(command)
+            except subprocess.CalledProcessError as e:
+                crash = True
+            except BaseException as e:
+                crash = True
+
+            if verbose:
+                print(" ".join(command))
+                print(output.decode("utf-8"))
+
+            # Detect missing filepaths and consider those errors
+            for filepath, output_filepath in zip(remaining_filepaths[:], output_filepaths):
+                remaining_filepaths.pop(0)
+
+                if crash:
+                    # In case of crash, stop after missing files and re-render remaing
+                    if not os.path.exists(output_filepath):
+                        errors.append("CRASH")
+                        print_message("Crash running Blender")
+                        print_message(testname, 'FAILURE', 'FAILED')
+                        break
+
+                testname = test_get_name(filepath)
+
+                if not os.path.exists(output_filepath) or os.path.getsize(output_filepath) == 0:
+                    errors.append("NO OUTPUT")
+                    print_message("No render result file found")
+                    print_message(testname, 'FAILURE', 'FAILED')
+                elif not self._diff_output(filepath, output_filepath):
+                    errors.append("VERIFY")
+                    print_message("Render result is different from reference image")
+                    print_message(testname, 'FAILURE', 'FAILED')
+                else:
+                    errors.append(None)
+                    print_message(testname, 'SUCCESS', 'OK')
+
+                if os.path.exists(output_filepath):
+                    os.remove(output_filepath)
+
+        return errors
+
+    def _run_all_tests(self, dirname, dirpath, blender, arguments_cb, batch):
         passed_tests = []
         failed_tests = []
         all_files = list(blend_list(dirpath))
@@ -444,8 +482,8 @@ class Report:
                       format(len(all_files)),
                       'SUCCESS', "==========")
         time_start = time.time()
-        for filepath in all_files:
-            error = self._run_test(filepath, render_cb)
+        errors = self._run_tests(all_files, blender, arguments_cb, batch)
+        for filepath, error in zip(all_files, errors):
             testname = test_get_name(filepath)
             if error:
                 if error == "NO_ENGINE":
diff --git a/tests/python/opengl_draw_tests.py b/tests/python/opengl_draw_tests.py
index b70111c042f..ab4df63afd9 100755
--- a/tests/python/opengl_draw_tests.py
+++ b/tests/python/opengl_draw_tests.py
@@ -31,9 +31,8 @@ if inside_blender:
     sys.exit(0)
 
 
-def render_file(filepath, output_filepath):
-    command = (
-        BLENDER,
+def get_arguments(filepath, output_filepath):
+    return [
         "--no-window-focus",
         "--window-geometry",
         "0", "0", "1024", "768",
@@ -44,28 +43,7 @@ def render_file(filepath, output_filepath):
         "-P",
         os.path.realpath(__file__),
         "--",
-        output_filepath)
-
-    try:
-        # Success
-        output = subprocess.check_output(command)
-        if VERBOSE:
-            print(output.decode("utf-8"))
-        return None
-    except subprocess.CalledProcessError as e:
-        # Error
-        if os.path.exists(output_filepath):
-            os.remove(output_filepath)
-        if VERBOSE:
-            print(e.output.decode("utf-8"))
-        return "CRASH"
-    except BaseException as e:
-        # Crash
-        if os.path.exists(output_filepath):
-            os.remove(output_filepath)
-        if VERBOSE:
-            print(e)
-        return "CRASH"
+        output_filepath + '0001.png']
 
 
 def create_argparse():
@@ -81,18 +59,14 @@ def main():
     parser = create_argparse()
     args = parser.parse_args()
 
-    global BLENDER, VERBOSE
-
-    BLENDER = args.blender[0]
-    VERBOSE = os.environ.get("BLENDER_VERBOSE") is not None
-
+    blender = args.blender[0]
     test_dir = args.testdir[0]
     idiff = args.idiff[0]
     output_dir = args.outdir[0]
 
     from modules import render_report
     report = render_report.Report("OpenGL Draw", output_dir, idiff)
-    ok = report.run(test_dir, render_file)
+    ok = report.run(test_dir, blender, get_arguments)
 
     sys.exit(not ok)
 
diff --git a/tests/python/workbench_render_tests.py b/tests/python/workbench_render_tests.py
index e43aac175b8..155b54098a8 100755
--- a/tests/python/workbench_render_tests.py
+++ b/tests/python/workbench_render_tests.py
@@ -33,15 +33,8 @@ if inside_blender:
         sys.exit(1)
 
 
-def render_file(filepath, output_filepath):
-    dirname = os.path.dirname(filepath)
-    basedir = os.path.dirname(dirname)
-    subject = os.path.basename(dirname)
-
-    frame_filepath = output_filepath + '0001.png'
-
-    command = [
-        BLENDER,
+def get_arguments(filepath, output_filepath):
+    return [
         "--background",
         "-noaudio",
         "--factory-startup",
@@ -54,37 +47,6 @@ def render_file(filepath, output_filepath):
         "-F", "PNG",
         "-f", "1"]
 
-    try:
-        # Success
-        output = subprocess.check_output(command)
-        if os.path.exists(frame_filepath):
-            shutil.copy(frame_filepath, output_filepath)
-            os.remove(frame_filepath)
-        if VERBOSE:
-            print(" ".join(command))
-            print(output.decode("utf-8"))
-        return None
-    except subprocess.CalledProcessError as e:
-        # Error
-        if os.path.exists(frame_filepath):
-            os.remove(frame_filepath)
-        if VERBOSE:
-            print(" ".join(command))
-            print(e.output.decode("utf-8"))
-        if b"Error: engine not found" in e.output:
-            return "NO_ENGINE"
-        elif b"blender probably wont start" in e.output:
-            return "NO_START"
-        return "CRASH"
-    except BaseException as e:
-        # Crash
-        if os.path.exists(frame_filepath):
-            os.remove(frame_filepath)
-        if VERBOSE:
-            print(" ".join(command))
-            print(e)
-        return "CRASH"
-
 
 def create_argparse():
     parser = argparse.ArgumentParser()
@@ -99,11 +61,7 @@ def main():
     parser = create_argparse()
     args = parser.parse_args()
 
-    global BLENDER, VERBOSE
-
-    BLENDER = args.blender[0]
-    VERBOSE = os.environ.get("BLENDER_VERBOSE") is not None
-
+    blender = args.blender[0]
     test_dir = args.testdir[0]
     idiff = args.idiff[0]
     output_dir = args.outdir[0]
@@ -113,7 +71,7 @@ def main():
     report.set_pixelated(True)
     report.set_reference_dir("workbench_renders")
     report.set_compare_engines('workbench', 'eevee')
-    ok = report.run(test_dir, render_file)
+    ok = report.run(test_dir, blender, get_arguments, batch=True)
 
     sys.exit(not ok)