diff options
author | Brecht Van Lommel <brecht@blender.org> | 2021-09-08 16:56:50 +0300 |
---|---|---|
committer | Brecht Van Lommel <brecht@blender.org> | 2021-09-08 17:40:58 +0300 |
commit | 6fc94d18485a57d262a3301ad6255b60abcfd883 (patch) | |
tree | c034a64e373cd4e8201c053265adb6773a18cae9 /tests/performance/api/config.py | |
parent | 6bc6ffc35c58ca4ac1ac1a55367e20fc06f5fe3a (diff) |
Tests: updates for performance benchmarking
* Make "run" command (re-)run all tests, add "update" command to only
run queued and outdated tests equivalent to the old "run" command.
* Support specifying environment variables for revisions, to easily
compare multiple parameter values.
* Better sorting of revisions in graph.
Diffstat (limited to 'tests/performance/api/config.py')
-rw-r--r-- | tests/performance/api/config.py | 17 |
1 files changed, 15 insertions, 2 deletions
diff --git a/tests/performance/api/config.py b/tests/performance/api/config.py index d3a79eede14..aa991e7d7d8 100644 --- a/tests/performance/api/config.py +++ b/tests/performance/api/config.py @@ -25,6 +25,7 @@ class TestEntry: category: str = '' revision: str = '' git_hash: str = '' + environment: Dict = field(default_factory=dict) executable: str = '' date: int = 0 device_type: str = 'CPU' @@ -191,9 +192,10 @@ class TestConfig: # Get entries for specified commits, tags and branches. for revision_name, revision_commit in self.revisions.items(): + revision_commit, environment = self._split_environment_variables(revision_commit) git_hash = env.resolve_git_hash(revision_commit) date = env.git_hash_date(git_hash) - entries += self._get_entries(revision_name, git_hash, '', date) + entries += self._get_entries(revision_name, git_hash, '', environment, date) # Optimization to avoid rebuilds. revisions_to_build = set() @@ -204,6 +206,7 @@ class TestConfig: # Get entries for revisions based on existing builds. for revision_name, executable in self.builds.items(): + executable, environment = self._split_environment_variables(executable) executable_path = env._blender_executable_from_path(pathlib.Path(executable)) if not executable_path: sys.stderr.write(f'Error: build {executable} not found\n') @@ -214,7 +217,7 @@ class TestConfig: env.set_default_blender_executable() mtime = executable_path.stat().st_mtime - entries += self._get_entries(revision_name, git_hash, executable, mtime) + entries += self._get_entries(revision_name, git_hash, executable, environment, mtime) # Detect number of categories for more compact printing. categories = set() @@ -229,6 +232,7 @@ class TestConfig: revision_name: str, git_hash: str, executable: pathlib.Path, + environment: str, date: int) -> None: entries = [] for test in self.tests.tests: @@ -241,10 +245,12 @@ class TestConfig: # Test if revision hash or executable changed. if entry.git_hash != git_hash or \ entry.executable != executable or \ + entry.environment != environment or \ entry.benchmark_type != self.benchmark_type or \ entry.date != date: # Update existing entry. entry.git_hash = git_hash + entry.environment = environment entry.executable = executable entry.benchmark_type = self.benchmark_type entry.date = date @@ -256,6 +262,7 @@ class TestConfig: revision=revision_name, git_hash=git_hash, executable=executable, + environment=environment, date=date, test=test_name, category=test_category, @@ -266,3 +273,9 @@ class TestConfig: entries.append(entry) return entries + + def _split_environment_variables(self, revision): + if isinstance(revision, str): + return revision, {} + else: + return revision[0], revision[1] |