Tests: more graceful handling of keyboard interrupting benchmarks

Leave current test result unchanged and stop executing immediately,
so it can be continued.

3 files changed, 22 insertions, 13 deletions

diff --git a/tests/performance/api/environment.py b/tests/performance/api/environment.py
index 750d991ebc8..eec92cc7b6b 100644
--- a/tests/performance/api/environment.py
+++ b/tests/performance/api/environment.py
@@ -98,6 +98,8 @@ class TestEnvironment:
         try:
             self.call([self.cmake_executable, '.'] + self.cmake_options, self.build_dir)
             self.call([self.cmake_executable, '--build', '.', '-j', jobs, '--target', 'install'], self.build_dir)
+        except KeyboardInterrupt as e:
+            raise e
         except:
             return False
 
@@ -193,17 +195,13 @@ class TestEnvironment:
                     lines.append(line_str)
                     if f:
                         f.write(line_str)
-        except KeyboardInterrupt:
+        except KeyboardInterrupt as e:
             # Avoid processes that keep running when interrupting.
             proc.terminate()
+            raise e
 
-        if f:
-            f.close()
-
-        # Print command output on error
+        # Raise error on failure
         if proc.returncode != 0 and not silent:
-            for line in lines:
-                print(line.rstrip())
             raise Exception("Error executing command")
 
         return lines
diff --git a/tests/performance/api/graph.py b/tests/performance/api/graph.py
index fe4d4800894..e54adc194de 100644
--- a/tests/performance/api/graph.py
+++ b/tests/performance/api/graph.py
@@ -42,7 +42,7 @@ class TestGraph:
 
             # Generate one graph for every device x category x result key combination.
             for category, category_entries in categories.items():
-                entries = sorted(category_entries, key=lambda entry: (entry.revision, entry.test, entry.date))
+                entries = sorted(category_entries, key=lambda entry: (entry.date, entry.revision, entry.test))
 
                 outputs = set()
                 for entry in entries:
diff --git a/tests/performance/benchmark b/tests/performance/benchmark
index 343af3be7d1..a58c339e9f8 100755
--- a/tests/performance/benchmark
+++ b/tests/performance/benchmark
@@ -141,6 +141,8 @@ def run_entry(env: api.TestEnvironment,
             if not entry.output:
                 raise Exception("Test produced no output")
             entry.status = 'done'
+        except KeyboardInterrupt as e:
+            raise e
         except Exception as e:
             entry.status = 'failed'
             entry.error_msg = str(e)
@@ -236,17 +238,26 @@ def cmd_run(env: api.TestEnvironment, argv: List, update_only: bool):
     configs = env.get_configs(args.config)
     for config in configs:
         updated = False
+        cancel = False
         print_header(config)
         for row in config.queue.rows(use_revision_columns(config)):
             if match_entry(row[0], args):
                 for entry in row:
-                    if run_entry(env, config, row, entry, update_only):
-                        updated = True
-                        # Write queue every time in case running gets interrupted,
-                        # so it can be resumed.
-                        config.queue.write()
+                    try:
+                        if run_entry(env, config, row, entry, update_only):
+                            updated = True
+                            # Write queue every time in case running gets interrupted,
+                            # so it can be resumed.
+                            config.queue.write()
+                    except KeyboardInterrupt as e:
+                        cancel = True
+                        break
+
                 print_row(config, row)
 
+            if cancel:
+                break
+
         if updated:
             # Generate graph if test were run.
             json_filepath = config.base_dir / "results.json"