diff options
author | Brecht Van Lommel <brecht@blender.org> | 2021-07-20 17:52:22 +0300 |
---|---|---|
committer | Brecht Van Lommel <brecht@blender.org> | 2021-07-22 17:35:00 +0300 |
commit | 320f34af86eb4ad4466fb4ef40baa1cf3fd0c617 (patch) | |
tree | ad4b581ce542056174df011f391793d39a9d885e /tests/performance/api | |
parent | 19b597c55d5d6cd6eb6cecdcff6db3e3fb5525ef (diff) |
Tests: continue running benchmarks if some tests fail to build or run
Convenient when testing many revisions where some might be broken.
Diffstat (limited to 'tests/performance/api')
-rw-r--r-- | tests/performance/api/config.py | 4 | ||||
-rw-r--r-- | tests/performance/api/environment.py | 17 | ||||
-rw-r--r-- | tests/performance/api/graph.py | 3 |
3 files changed, 18 insertions, 6 deletions
diff --git a/tests/performance/api/config.py b/tests/performance/api/config.py index 283d1ff16ec..d3a79eede14 100644 --- a/tests/performance/api/config.py +++ b/tests/performance/api/config.py @@ -31,6 +31,7 @@ class TestEntry: device_id: str = 'CPU' device_name: str = 'Unknown CPU' status: str = 'queued' + error_msg: str = '' output: Dict = field(default_factory=dict) benchmark_type: str = 'comparison' @@ -42,7 +43,8 @@ class TestEntry: def from_json(self, json_dict): for field in self.__dataclass_fields__: - setattr(self, field, json_dict[field]) + if field in json_dict: + setattr(self, field, json_dict[field]) class TestQueue: diff --git a/tests/performance/api/environment.py b/tests/performance/api/environment.py index 1a8f5ceab51..3a9b3eaf936 100644 --- a/tests/performance/api/environment.py +++ b/tests/performance/api/environment.py @@ -88,16 +88,21 @@ class TestEnvironment: self.call([self.git_executable, 'reset', '--hard', 'HEAD'], self.blender_dir) self.call([self.git_executable, 'checkout', '--detach', git_hash], self.blender_dir) - def build(self) -> None: + def build(self) -> bool: # Build Blender revision if not self.build_dir.exists(): sys.stderr.write('\n\nError: no build set up, run `./benchmark init --build` first\n') sys.exit(1) jobs = str(multiprocessing.cpu_count()) - self.call([self.cmake_executable, '.'] + self.cmake_options, self.build_dir) - self.call([self.cmake_executable, '--build', '.', '-j', jobs, '--target', 'install'], self.build_dir) + try: + self.call([self.cmake_executable, '.'] + self.cmake_options, self.build_dir) + self.call([self.cmake_executable, '--build', '.', '-j', jobs, '--target', 'install'], self.build_dir) + except: + return False + self._init_default_blender_executable() + return True def set_blender_executable(self, executable_path: pathlib.Path) -> None: # Run all Blender commands with this executable. @@ -277,8 +282,10 @@ class TestEnvironment: def resolve_git_hash(self, revision): # Get git hash for a tag or branch. - return self.call([self.git_executable, 'rev-parse', revision], self.blender_git_dir)[0].strip() + lines = self.call([self.git_executable, 'rev-parse', revision], self.blender_git_dir) + return lines[0].strip() if len(lines) else revision def git_hash_date(self, git_hash): # Get commit data for a git hash. - return int(self.call([self.git_executable, 'log', '-n1', git_hash, '--format=%at'], self.blender_git_dir)[0].strip()) + lines = self.call([self.git_executable, 'log', '-n1', git_hash, '--format=%at'], self.blender_git_dir) + return int(lines[0].strip()) if len(lines) else 0 diff --git a/tests/performance/api/graph.py b/tests/performance/api/graph.py index b3c8329ff27..4ee5ae7cf0e 100644 --- a/tests/performance/api/graph.py +++ b/tests/performance/api/graph.py @@ -30,6 +30,7 @@ class TestGraph: data = [] for device_name, device_entries in devices.items(): + # Gather used categories. categories = {} for entry in device_entries: @@ -57,6 +58,8 @@ class TestGraph: self.json = json.dumps(data, indent=2) def chart(self, device_name: str, chart_name: str, entries: List, chart_type: str, output: str) -> Dict: + entries = sorted(entries, key=lambda entry: entry.date) + # Gather used tests. tests = {} for entry in entries: |