Revert "Tests: speed up render tests by running multiple in the same process"

This makes finding the crashing tests harder, reverting until there is a
better solution.

This reverts commit 93901e7f0a05ba471f3b4c0201500d9dfcd68c2c.

5 files changed, 180 insertions, 201 deletions

diff --git a/tests/python/cycles_render_tests.py b/tests/python/cycles_render_tests.py
index a66409d85c3..36f5459c2f7 100755
--- a/tests/python/cycles_render_tests.py
+++ b/tests/python/cycles_render_tests.py
@@ -9,76 +9,77 @@ import subprocess
 import sys
 
 
-def render_files(filepaths, output_filepaths):
-    command = [BLENDER, "--background"]
+def render_file(filepath, output_filepath):
+    dirname = os.path.dirname(filepath)
+    basedir = os.path.dirname(dirname)
+    subject = os.path.basename(dirname)
+
+    frame_filepath = output_filepath + '0001.png'
+
+    common_args = [
+        "-noaudio",
+        "--factory-startup",
+        "--enable-autoexec",
+        filepath,
+        "-E", "CYCLES",
+        "-o", output_filepath,
+        "-F", "PNG"]
 
     # OSL and GPU examples
     # custom_args += ["--python-expr", "import bpy; bpy.context.scene.cycles.shading_system = True"]
     # custom_args += ["--python-expr", "import bpy; bpy.context.scene.cycles.device = 'GPU'"]
     custom_args = os.getenv('CYCLESTEST_ARGS')
     custom_args = shlex.split(custom_args) if custom_args else []
+    common_args += custom_args
+
+    if subject == 'opengl':
+        command = [BLENDER, "--window-geometry", "0", "0", "1", "1"]
+        command += common_args
+        command += ['--python', os.path.join(basedir, "util", "render_opengl.py")]
+    elif subject == 'bake':
+        command = [BLENDER, "--background"]
+        command += common_args
+        command += ['--python', os.path.join(basedir, "util", "render_bake.py")]
+    elif subject == 'denoise_animation':
+        command = [BLENDER, "--background"]
+        command += common_args
+        command += ['--python', os.path.join(basedir, "util", "render_denoise.py")]
+    else:
+        command = [BLENDER, "--background"]
+        command += common_args
+        command += ["-f", "1"]
 
-    for filepath, output_filepath in zip(filepaths, output_filepaths):
-        dirname = os.path.dirname(filepath)
-        basedir = os.path.dirname(dirname)
-        subject = os.path.basename(dirname)
-
-        frame_filepath = output_filepath + '0001.png'
-
-        common_args = [
-            "-noaudio",
-            "--factory-startup",
-            "--enable-autoexec",
-            filepath,
-            "-E", "CYCLES",
-            "-o", output_filepath,
-            "-F", "PNG"]
-
-        common_args += custom_args
-
-        if subject == 'bake':
-            command.extend(common_args)
-            command.extend(['--python', os.path.join(basedir, "util", "render_bake.py")])
-        elif subject == 'denoise_animation':
-            command.extend(common_args)
-            command.extend(['--python', os.path.join(basedir, "util", "render_denoise.py")])
-        else:
-            command.extend(common_args)
-            command.extend(["-f", "1"])
-
-    error = None
     try:
         # Success
         output = subprocess.check_output(command)
+        if os.path.exists(frame_filepath):
+            shutil.copy(frame_filepath, output_filepath)
+            os.remove(frame_filepath)
         if VERBOSE:
             print(" ".join(command))
             print(output.decode("utf-8"))
+        return None
     except subprocess.CalledProcessError as e:
         # Error
+        if os.path.exists(frame_filepath):
+            os.remove(frame_filepath)
         if VERBOSE:
             print(" ".join(command))
             print(e.output.decode("utf-8"))
-        error = "CRASH"
+        if b"Error: engine not found" in e.output:
+            return "NO_ENGINE"
+        elif b"blender probably wont start" in e.output:
+            return "NO_START"
+        return "CRASH"
     except BaseException as e:
         # Crash
-        if VERBOSE:
-            print(" ".join(command))
-            print(e.decode("utf-8"))
-        error = "CRASH"
-
-    # Detect missing filepaths and consider those errors
-    errors = []
-    for output_filepath in output_filepaths:
-        frame_filepath = output_filepath + '0001.png'
         if os.path.exists(frame_filepath):
-            shutil.copy(frame_filepath, output_filepath)
             os.remove(frame_filepath)
-            errors.append(None)
-        else:
-            errors.append(error)
-            error = 'SKIPPED'
+        if VERBOSE:
+            print(" ".join(command))
+            print(e)
+        return "CRASH"
 
-    return errors
 
 def create_argparse():
     parser = argparse.ArgumentParser()
@@ -107,7 +108,7 @@ def main():
     report.set_pixelated(True)
     report.set_reference_dir("cycles_renders")
     report.set_compare_engines('cycles', 'eevee')
-    ok = report.run(test_dir, render_files)
+    ok = report.run(test_dir, render_file)
 
     sys.exit(not ok)
 
diff --git a/tests/python/eevee_render_tests.py b/tests/python/eevee_render_tests.py
index f324b22b682..8eaf04f8bea 100755
--- a/tests/python/eevee_render_tests.py
+++ b/tests/python/eevee_render_tests.py
@@ -49,61 +49,57 @@ if inside_blender:
         sys.exit(1)
 
 
-def render_files(filepaths, output_filepaths):
+def render_file(filepath, output_filepath):
+    dirname = os.path.dirname(filepath)
+    basedir = os.path.dirname(dirname)
+    subject = os.path.basename(dirname)
+
+    frame_filepath = output_filepath + '0001.png'
+
     command = [
         BLENDER,
         "--background",
         "-noaudio",
         "--factory-startup",
-        "--enable-autoexec"]
-
-    for filepath, output_filepath in zip(filepaths, output_filepaths):
-        frame_filepath = output_filepath + '0001.png'
-        if os.path.exists(frame_filepath):
-            os.remove(frame_filepath)
+        "--enable-autoexec",
+        filepath,
+        "-E", "BLENDER_EEVEE",
+        "-P",
+        os.path.realpath(__file__),
+        "-o", output_filepath,
+        "-F", "PNG",
+        "-f", "1"]
 
-        command.extend([
-            filepath,
-            "-E", "BLENDER_EEVEE",
-            "-P",
-            os.path.realpath(__file__),
-            "-o", output_filepath,
-            "-F", "PNG",
-            "-f", "1"])
-
-    error = None
     try:
         # Success
         output = subprocess.check_output(command)
+        if os.path.exists(frame_filepath):
+            shutil.copy(frame_filepath, output_filepath)
+            os.remove(frame_filepath)
         if VERBOSE:
             print(" ".join(command))
             print(output.decode("utf-8"))
+        return None
     except subprocess.CalledProcessError as e:
         # Error
+        if os.path.exists(frame_filepath):
+            os.remove(frame_filepath)
         if VERBOSE:
             print(" ".join(command))
             print(e.output.decode("utf-8"))
-        error = "CRASH"
+        if b"Error: engine not found" in e.output:
+            return "NO_ENGINE"
+        elif b"blender probably wont start" in e.output:
+            return "NO_START"
+        return "CRASH"
     except BaseException as e:
         # Crash
-        if VERBOSE:
-            print(" ".join(command))
-            print(e.decode("utf-8"))
-        error = "CRASH"
-
-    # Detect missing filepaths and consider those errors
-    errors = []
-    for output_filepath in output_filepaths:
-        frame_filepath = output_filepath + '0001.png'
         if os.path.exists(frame_filepath):
-            shutil.copy(frame_filepath, output_filepath)
             os.remove(frame_filepath)
-            errors.append(None)
-        else:
-            errors.append(error)
-            error = 'SKIPPED'
-
-    return errors
+        if VERBOSE:
+            print(" ".join(command))
+            print(e)
+        return "CRASH"
 
 
 def create_argparse():
@@ -133,7 +129,7 @@ def main():
     report.set_pixelated(True)
     report.set_reference_dir("eevee_renders")
     report.set_compare_engines('eevee', 'cycles')
-    ok = report.run(test_dir, render_files)
+    ok = report.run(test_dir, render_file)
 
     sys.exit(not ok)
 
diff --git a/tests/python/modules/render_report.py b/tests/python/modules/render_report.py
index 7dfc74904a7..5a2baa354c7 100755
--- a/tests/python/modules/render_report.py
+++ b/tests/python/modules/render_report.py
@@ -374,48 +374,41 @@ class Report:
 
         return not failed
 
-    def _run_tests(self, filepaths, render_cb):
-        # Run all tests together for performance, since Blender
-        # startup time is a significant factor.
-        tmp_filepaths = []
-        for filepath in filepaths:
-            testname = test_get_name(filepath)
-            print_message(testname, 'SUCCESS', 'RUN')
-            tmp_filepaths.append(os.path.join(self.output_dir, "tmp_" + testname))
-
-        run_errors = render_cb(filepaths, tmp_filepaths)
-        errors = []
-
-        for error, filepath, tmp_filepath in zip(run_errors, filepaths, tmp_filepaths):
-            if not error:
-                if os.path.getsize(tmp_filepath) == 0:
-                    error = "VERIFY"
-                elif not self._diff_output(filepath, tmp_filepath):
-                    error = "VERIFY"
+    def _run_test(self, filepath, render_cb):
+        testname = test_get_name(filepath)
+        print_message(testname, 'SUCCESS', 'RUN')
+        time_start = time.time()
+        tmp_filepath = os.path.join(self.output_dir, "tmp_" + testname)
 
-            if os.path.exists(tmp_filepath):
-                os.remove(tmp_filepath)
+        error = render_cb(filepath, tmp_filepath)
+        status = "FAIL"
+        if not error:
+            if not self._diff_output(filepath, tmp_filepath):
+                error = "VERIFY"
 
-            errors.append(error)
+        if os.path.exists(tmp_filepath):
+            os.remove(tmp_filepath)
 
-            testname = test_get_name(filepath)
-            if not error:
-                print_message(testname, 'SUCCESS', 'OK')
+        time_end = time.time()
+        elapsed_ms = int((time_end - time_start) * 1000)
+        if not error:
+            print_message("{} ({} ms)" . format(testname, elapsed_ms),
+                          'SUCCESS', 'OK')
+        else:
+            if error == "NO_ENGINE":
+                print_message("Can't perform tests because the render engine failed to load!")
+                return error
+            elif error == "NO_START":
+                print_message('Can not perform tests because blender fails to start.',
+                              'Make sure INSTALL target was run.')
+                return error
+            elif error == 'VERIFY':
+                print_message("Rendered result is different from reference image")
             else:
-                if error == "SKIPPED":
-                    print_message("Skipped after previous render caused error")
-                elif error == "NO_ENGINE":
-                    print_message("Can't perform tests because the render engine failed to load!")
-                elif error == "NO_START":
-                    print_message('Can not perform tests because blender fails to start.',
-                                  'Make sure INSTALL target was run.')
-                elif error == 'VERIFY':
-                    print_message("Rendered result is different from reference image")
-                else:
-                    print_message("Unknown error %r" % error)
-                print_message(testname, 'FAILURE', 'FAILED')
-
-        return errors
+                print_message("Unknown error %r" % error)
+            print_message("{} ({} ms)" . format(testname, elapsed_ms),
+                          'FAILURE', 'FAILED')
+        return error
 
     def _run_all_tests(self, dirname, dirpath, render_cb):
         passed_tests = []
@@ -426,8 +419,8 @@ class Report:
                       format(len(all_files)),
                       'SUCCESS', "==========")
         time_start = time.time()
-        errors = self._run_tests(all_files, render_cb)
-        for filepath, error in zip(all_files, errors):
+        for filepath in all_files:
+            error = self._run_test(filepath, render_cb)
             testname = test_get_name(filepath)
             if error:
                 if error == "NO_ENGINE":
diff --git a/tests/python/opengl_draw_tests.py b/tests/python/opengl_draw_tests.py
index 1cc5ddecf1d..9913f875689 100755
--- a/tests/python/opengl_draw_tests.py
+++ b/tests/python/opengl_draw_tests.py
@@ -31,48 +31,41 @@ if inside_blender:
     sys.exit(0)
 
 
-def render_files(filepaths, output_filepaths):
-    errors = []
-
-    for filepath, output_filepath in zip(filepaths, output_filepaths):
-        command = (
-            BLENDER,
-            "--no-window-focus",
-            "--window-geometry",
-            "0", "0", "1024", "768",
-            "-noaudio",
-            "--factory-startup",
-            "--enable-autoexec",
-            filepath,
-            "-P",
-            os.path.realpath(__file__),
-            "--",
-            output_filepath)
-
-        error = None
-        try:
-            # Success
-            output = subprocess.check_output(command)
-            if VERBOSE:
-                print(output.decode("utf-8"))
-        except subprocess.CalledProcessError as e:
-            # Error
-            if os.path.exists(output_filepath):
-                os.remove(output_filepath)
-            if VERBOSE:
-                print(e.output.decode("utf-8"))
-            error = "CRASH"
-        except BaseException as e:
-            # Crash
-            if os.path.exists(output_filepath):
-                os.remove(output_filepath)
-            if VERBOSE:
-                print(e)
-            error = "CRASH"
-
-        errors.append(error)
-
-    return errors
+def render_file(filepath, output_filepath):
+    command = (
+        BLENDER,
+        "--no-window-focus",
+        "--window-geometry",
+        "0", "0", "1024", "768",
+        "-noaudio",
+        "--factory-startup",
+        "--enable-autoexec",
+        filepath,
+        "-P",
+        os.path.realpath(__file__),
+        "--",
+        output_filepath)
+
+    try:
+        # Success
+        output = subprocess.check_output(command)
+        if VERBOSE:
+            print(output.decode("utf-8"))
+        return None
+    except subprocess.CalledProcessError as e:
+        # Error
+        if os.path.exists(output_filepath):
+            os.remove(output_filepath)
+        if VERBOSE:
+            print(e.output.decode("utf-8"))
+        return "CRASH"
+    except BaseException as e:
+        # Crash
+        if os.path.exists(output_filepath):
+            os.remove(output_filepath)
+        if VERBOSE:
+            print(e)
+        return "CRASH"
 
 
 def create_argparse():
@@ -99,7 +92,7 @@ def main():
 
     from modules import render_report
     report = render_report.Report("OpenGL Draw Test Report", output_dir, idiff)
-    ok = report.run(test_dir, render_files)
+    ok = report.run(test_dir, render_file)
 
     sys.exit(not ok)
 
diff --git a/tests/python/workbench_render_tests.py b/tests/python/workbench_render_tests.py
index 7514443ed57..1a0d639bccd 100755
--- a/tests/python/workbench_render_tests.py
+++ b/tests/python/workbench_render_tests.py
@@ -34,61 +34,57 @@ if inside_blender:
         sys.exit(1)
 
 
-def render_files(filepaths, output_filepaths):
+def render_file(filepath, output_filepath):
+    dirname = os.path.dirname(filepath)
+    basedir = os.path.dirname(dirname)
+    subject = os.path.basename(dirname)
+
+    frame_filepath = output_filepath + '0001.png'
+
     command = [
         BLENDER,
         "--background",
         "-noaudio",
         "--factory-startup",
-        "--enable-autoexec"]
-
-    for filepath, output_filepath in zip(filepaths, output_filepaths):
-        frame_filepath = output_filepath + '0001.png'
-        if os.path.exists(frame_filepath):
-            os.remove(frame_filepath)
+        "--enable-autoexec",
+        filepath,
+        "-E", "BLENDER_WORKBENCH",
+        "-P",
+        os.path.realpath(__file__),
+        "-o", output_filepath,
+        "-F", "PNG",
+        "-f", "1"]
 
-        command.extend([
-            filepath,
-            "-E", "BLENDER_WORKBENCH",
-            "-P",
-            os.path.realpath(__file__),
-            "-o", output_filepath,
-            "-F", "PNG",
-            "-f", "1"])
-
-    error = None
     try:
         # Success
         output = subprocess.check_output(command)
+        if os.path.exists(frame_filepath):
+            shutil.copy(frame_filepath, output_filepath)
+            os.remove(frame_filepath)
         if VERBOSE:
             print(" ".join(command))
             print(output.decode("utf-8"))
+        return None
     except subprocess.CalledProcessError as e:
         # Error
+        if os.path.exists(frame_filepath):
+            os.remove(frame_filepath)
         if VERBOSE:
             print(" ".join(command))
             print(e.output.decode("utf-8"))
-        error = "CRASH"
+        if b"Error: engine not found" in e.output:
+            return "NO_ENGINE"
+        elif b"blender probably wont start" in e.output:
+            return "NO_START"
+        return "CRASH"
     except BaseException as e:
         # Crash
-        if VERBOSE:
-            print(" ".join(command))
-            print(e.decode("utf-8"))
-        error = "CRASH"
-
-    # Detect missing filepaths and consider those errors
-    errors = []
-    for output_filepath in output_filepaths:
-        frame_filepath = output_filepath + '0001.png'
         if os.path.exists(frame_filepath):
-            shutil.copy(frame_filepath, output_filepath)
             os.remove(frame_filepath)
-            errors.append(None)
-        else:
-            errors.append(error)
-            error = 'SKIPPED'
-
-    return errors
+        if VERBOSE:
+            print(" ".join(command))
+            print(e)
+        return "CRASH"
 
 
 def create_argparse():
@@ -118,7 +114,7 @@ def main():
     report.set_pixelated(True)
     report.set_reference_dir("workbench_renders")
     report.set_compare_engines('workbench', 'eevee')
-    ok = report.run(test_dir, render_files)
+    ok = report.run(test_dir, render_file)
 
     sys.exit(not ok)