Tests: updates for performance benchmarking

* Make "run" command (re-)run all tests, add "update" command to only
  run queued and outdated tests equivalent to the old "run" command.
* Support specifying environment variables for revisions, to easily
  compare multiple parameter values.
* Better sorting of revisions in graph.

1 files changed, 18 insertions, 8 deletions

diff --git a/tests/performance/benchmark b/tests/performance/benchmark
index ad1e07d0ef3..343af3be7d1 100755
--- a/tests/performance/benchmark
+++ b/tests/performance/benchmark
@@ -83,15 +83,20 @@ def match_entry(entry: api.TestEntry, args: argparse.Namespace):
            entry.test.find(args.test) != -1 or \
            entry.category.find(args.test) != -1
 
-def run_entry(env: api.TestEnvironment, config: api.TestConfig, row: List, entry: api.TestEntry):
+def run_entry(env: api.TestEnvironment,
+              config: api.TestConfig,
+              row: List,
+              entry: api.TestEntry,
+              update_only: bool):
     # Check if entry needs to be run.
-    if entry.status not in ('queued', 'outdated'):
+    if update_only and entry.status not in ('queued', 'outdated'):
         print_row(config, row, end='\r')
         return False
 
     # Run test entry.
     revision = entry.revision
     git_hash = entry.git_hash
+    environment = entry.environment
     testname = entry.test
     testcategory = entry.category
     device_type = entry.device_type
@@ -116,13 +121,15 @@ def run_entry(env: api.TestEnvironment, config: api.TestConfig, row: List, entry
     print_row(config, row, end='\r')
     executable_ok = True
     if len(entry.executable):
-        env.set_blender_executable(pathlib.Path(entry.executable))
+        env.set_blender_executable(pathlib.Path(entry.executable), environment)
     else:
         env.checkout(git_hash)
         executable_ok = env.build()
         if not executable_ok:
             entry.status = 'failed'
             entry.error_msg = 'Failed to build'
+        else:
+            env.set_blender_executable(env.blender_executable, environment)
 
     # Run test and update output and status.
     if executable_ok:
@@ -219,7 +226,7 @@ def cmd_reset(env: api.TestEnvironment, argv: List):
 
         config.queue.write()
 
-def cmd_run(env: api.TestEnvironment, argv: List):
+def cmd_run(env: api.TestEnvironment, argv: List, update_only: bool):
     # Run tests.
     parser = argparse.ArgumentParser()
     parser.add_argument('config', nargs='?', default=None)
@@ -233,7 +240,7 @@ def cmd_run(env: api.TestEnvironment, argv: List):
         for row in config.queue.rows(use_revision_columns(config)):
             if match_entry(row[0], args):
                 for entry in row:
-                    if run_entry(env, config, row, entry):
+                    if run_entry(env, config, row, entry, update_only):
                         updated = True
                         # Write queue every time in case running gets interrupted,
                         # so it can be resumed.
@@ -268,8 +275,9 @@ def main():
              '  \n'
              '  list                                 List available tests, devices and configurations\n'
              '  \n'
-             '  run [<config>] [<test>]              Execute tests for configuration\n'
-             '  reset [<config>] [<test>]            Clear tests results from config, for re-running\n'
+             '  run [<config>] [<test>]              Execute all tests in configuration\n'
+             '  update [<config>] [<test>]           Execute only queued and outdated tests\n'
+             '  reset [<config>] [<test>]            Clear tests results in configuration\n'
              '  status [<config>] [<test>]           List configurations and their tests\n'
              '  \n'
              '  graph a.json b.json... -o out.html   Create graph from results in JSON files\n')
@@ -304,7 +312,9 @@ def main():
     if args.command == 'list':
         cmd_list(env, argv)
     elif args.command == 'run':
-        cmd_run(env, argv)
+        cmd_run(env, argv, update_only=False)
+    elif args.command == 'update':
+        cmd_run(env, argv, update_only=True)
     elif args.command == 'reset':
         cmd_reset(env, argv)
     elif args.command == 'status':