Tests: updates for performance benchmarking

* Make "run" command (re-)run all tests, add "update" command to only
  run queued and outdated tests equivalent to the old "run" command.
* Support specifying environment variables for revisions, to easily
  compare multiple parameter values.
* Better sorting of revisions in graph.

4 files changed, 47 insertions, 17 deletions

diff --git a/tests/performance/api/config.py b/tests/performance/api/config.py
index d3a79eede14..aa991e7d7d8 100644
--- a/tests/performance/api/config.py
+++ b/tests/performance/api/config.py
@@ -25,6 +25,7 @@ class TestEntry:
     category: str = ''
     revision: str = ''
     git_hash: str = ''
+    environment: Dict = field(default_factory=dict)
     executable: str = ''
     date: int = 0
     device_type: str = 'CPU'
@@ -191,9 +192,10 @@ class TestConfig:
 
         # Get entries for specified commits, tags and branches.
         for revision_name, revision_commit in self.revisions.items():
+            revision_commit, environment = self._split_environment_variables(revision_commit)
             git_hash = env.resolve_git_hash(revision_commit)
             date = env.git_hash_date(git_hash)
-            entries += self._get_entries(revision_name, git_hash, '', date)
+            entries += self._get_entries(revision_name, git_hash, '', environment, date)
 
         # Optimization to avoid rebuilds.
         revisions_to_build = set()
@@ -204,6 +206,7 @@ class TestConfig:
 
         # Get entries for revisions based on existing builds.
         for revision_name, executable in self.builds.items():
+            executable, environment = self._split_environment_variables(executable)
             executable_path = env._blender_executable_from_path(pathlib.Path(executable))
             if not executable_path:
                 sys.stderr.write(f'Error: build {executable} not found\n')
@@ -214,7 +217,7 @@ class TestConfig:
             env.set_default_blender_executable()
 
             mtime = executable_path.stat().st_mtime
-            entries += self._get_entries(revision_name, git_hash, executable, mtime)
+            entries += self._get_entries(revision_name, git_hash, executable, environment, mtime)
 
         # Detect number of categories for more compact printing.
         categories = set()
@@ -229,6 +232,7 @@ class TestConfig:
                      revision_name: str,
                      git_hash: str,
                      executable: pathlib.Path,
+                     environment: str,
                      date: int) -> None:
         entries = []
         for test in self.tests.tests:
@@ -241,10 +245,12 @@ class TestConfig:
                     # Test if revision hash or executable changed.
                     if entry.git_hash != git_hash or \
                        entry.executable != executable or \
+                       entry.environment != environment or \
                        entry.benchmark_type != self.benchmark_type or \
                        entry.date != date:
                         # Update existing entry.
                         entry.git_hash = git_hash
+                        entry.environment = environment
                         entry.executable = executable
                         entry.benchmark_type = self.benchmark_type
                         entry.date = date
@@ -256,6 +262,7 @@ class TestConfig:
                         revision=revision_name,
                         git_hash=git_hash,
                         executable=executable,
+                        environment=environment,
                         date=date,
                         test=test_name,
                         category=test_category,
@@ -266,3 +273,9 @@ class TestConfig:
                 entries.append(entry)
 
         return entries
+
+    def _split_environment_variables(self, revision):
+        if isinstance(revision, str):
+            return revision, {}
+        else:
+            return revision[0], revision[1]
diff --git a/tests/performance/api/environment.py b/tests/performance/api/environment.py
index 76c731b6118..750d991ebc8 100644
--- a/tests/performance/api/environment.py
+++ b/tests/performance/api/environment.py
@@ -104,9 +104,10 @@ class TestEnvironment:
         self._init_default_blender_executable()
         return True
 
-    def set_blender_executable(self, executable_path: pathlib.Path) -> None:
+    def set_blender_executable(self, executable_path: pathlib.Path, environment: Dict = {}) -> None:
         # Run all Blender commands with this executable.
         self.blender_executable = executable_path
+        self.blender_executable_environment = environment
 
     def _blender_executable_name(self) -> pathlib.Path:
         if platform.system() == "Windows":
@@ -150,6 +151,7 @@ class TestEnvironment:
 
     def set_default_blender_executable(self) -> None:
         self.blender_executable = self.default_blender_executable
+        self.blender_executable_environment = {}
 
     def set_log_file(self, filepath: pathlib.Path, clear=True) -> None:
         # Log all commands and output to this file.
@@ -161,7 +163,7 @@ class TestEnvironment:
     def unset_log_file(self) -> None:
         self.log_file = None
 
-    def call(self, args: List[str], cwd: pathlib.Path, silent=False) -> List[str]:
+    def call(self, args: List[str], cwd: pathlib.Path, silent: bool=False, environment: Dict={}) -> List[str]:
         # Execute command with arguments in specified directory,
         # and return combined stdout and stderr output.
 
@@ -173,7 +175,13 @@ class TestEnvironment:
             f = open(self.log_file, 'a')
             f.write('\n' + ' '.join([str(arg) for arg in args]) + '\n\n')
 
-        proc = subprocess.Popen(args, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+        env = os.environ
+        if len(environment):
+            env = env.copy()
+            for key, value in environment.items():
+                env[key] = value
+
+        proc = subprocess.Popen(args, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=env)
 
         # Read line by line
         lines = []
@@ -208,7 +216,8 @@ class TestEnvironment:
         else:
             common_args += ['--background']
 
-        return self.call([self.blender_executable] + common_args + args, cwd=self.base_dir)
+        return self.call([self.blender_executable] + common_args + args, cwd=self.base_dir,
+                         environment=self.blender_executable_environment)
 
     def run_in_blender(self,
                        function: Callable[[Dict], Dict],
diff --git a/tests/performance/api/graph.py b/tests/performance/api/graph.py
index 4ee5ae7cf0e..fe4d4800894 100644
--- a/tests/performance/api/graph.py
+++ b/tests/performance/api/graph.py
@@ -42,7 +42,7 @@ class TestGraph:
 
             # Generate one graph for every device x category x result key combination.
             for category, category_entries in categories.items():
-                entries = sorted(category_entries, key=lambda entry: (entry.revision, entry.test))
+                entries = sorted(category_entries, key=lambda entry: (entry.revision, entry.test, entry.date))
 
                 outputs = set()
                 for entry in entries:
@@ -58,8 +58,6 @@ class TestGraph:
         self.json = json.dumps(data, indent=2)
 
     def chart(self, device_name: str, chart_name: str, entries: List, chart_type: str, output: str) -> Dict:
-        entries = sorted(entries, key=lambda entry: entry.date)
-
         # Gather used tests.
         tests = {}
         for entry in entries:
diff --git a/tests/performance/benchmark b/tests/performance/benchmark
index ad1e07d0ef3..343af3be7d1 100755
--- a/tests/performance/benchmark
+++ b/tests/performance/benchmark
@@ -83,15 +83,20 @@ def match_entry(entry: api.TestEntry, args: argparse.Namespace):
            entry.test.find(args.test) != -1 or \
            entry.category.find(args.test) != -1
 
-def run_entry(env: api.TestEnvironment, config: api.TestConfig, row: List, entry: api.TestEntry):
+def run_entry(env: api.TestEnvironment,
+              config: api.TestConfig,
+              row: List,
+              entry: api.TestEntry,
+              update_only: bool):
     # Check if entry needs to be run.
-    if entry.status not in ('queued', 'outdated'):
+    if update_only and entry.status not in ('queued', 'outdated'):
         print_row(config, row, end='\r')
         return False
 
     # Run test entry.
     revision = entry.revision
     git_hash = entry.git_hash
+    environment = entry.environment
     testname = entry.test
     testcategory = entry.category
     device_type = entry.device_type
@@ -116,13 +121,15 @@ def run_entry(env: api.TestEnvironment, config: api.TestConfig, row: List, entry
     print_row(config, row, end='\r')
     executable_ok = True
     if len(entry.executable):
-        env.set_blender_executable(pathlib.Path(entry.executable))
+        env.set_blender_executable(pathlib.Path(entry.executable), environment)
     else:
         env.checkout(git_hash)
         executable_ok = env.build()
         if not executable_ok:
             entry.status = 'failed'
             entry.error_msg = 'Failed to build'
+        else:
+            env.set_blender_executable(env.blender_executable, environment)
 
     # Run test and update output and status.
     if executable_ok:
@@ -219,7 +226,7 @@ def cmd_reset(env: api.TestEnvironment, argv: List):
 
         config.queue.write()
 
-def cmd_run(env: api.TestEnvironment, argv: List):
+def cmd_run(env: api.TestEnvironment, argv: List, update_only: bool):
     # Run tests.
     parser = argparse.ArgumentParser()
     parser.add_argument('config', nargs='?', default=None)
@@ -233,7 +240,7 @@ def cmd_run(env: api.TestEnvironment, argv: List):
         for row in config.queue.rows(use_revision_columns(config)):
             if match_entry(row[0], args):
                 for entry in row:
-                    if run_entry(env, config, row, entry):
+                    if run_entry(env, config, row, entry, update_only):
                         updated = True
                         # Write queue every time in case running gets interrupted,
                         # so it can be resumed.
@@ -268,8 +275,9 @@ def main():
              '  \n'
              '  list                                 List available tests, devices and configurations\n'
              '  \n'
-             '  run [<config>] [<test>]              Execute tests for configuration\n'
-             '  reset [<config>] [<test>]            Clear tests results from config, for re-running\n'
+             '  run [<config>] [<test>]              Execute all tests in configuration\n'
+             '  update [<config>] [<test>]           Execute only queued and outdated tests\n'
+             '  reset [<config>] [<test>]            Clear tests results in configuration\n'
              '  status [<config>] [<test>]           List configurations and their tests\n'
              '  \n'
              '  graph a.json b.json... -o out.html   Create graph from results in JSON files\n')
@@ -304,7 +312,9 @@ def main():
     if args.command == 'list':
         cmd_list(env, argv)
     elif args.command == 'run':
-        cmd_run(env, argv)
+        cmd_run(env, argv, update_only=False)
+    elif args.command == 'update':
+        cmd_run(env, argv, update_only=True)
     elif args.command == 'reset':
         cmd_reset(env, argv)
     elif args.command == 'status':