From 61455b6191bb5da9d8b799f4b7056570d3df8820 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Fri, 19 Aug 2022 18:57:55 -0500 Subject: Added back heuristic-based power-loss testing The main change here from the previous test framework design is: 1. Powerloss testing remains in-process, speeding up testing. 2. The state of a test, included all powerlosses, is encoded in the test id + leb16 encoded powerloss string. This means exhaustive testing can be run in CI, but then easily reproduced locally with full debugger support. For example: ./scripts/test.py test_dirs#reentrant_many_dir#10#1248g1g2 --gdb Will run the test test_dir, case reentrant_many_dir, permutation #10, with powerlosses at 1, 2, 4, 8, 16, and 32 cycles. Dropping into gdb if an assert fails. The changes to the block-device are a work-in-progress for a lazily-allocated/copy-on-write block device that I'm hoping will keep exhaustive testing relatively low-cost. --- scripts/test.py | 106 ++++++++++++++++++++++++++++++++------------------------ 1 file changed, 60 insertions(+), 46 deletions(-) (limited to 'scripts') diff --git a/scripts/test.py b/scripts/test.py index 281265e..cbc7ab9 100755 --- a/scripts/test.py +++ b/scripts/test.py @@ -73,8 +73,6 @@ class TestCase: self.in_ = config.pop('in', config.pop('suite_in', None)) - self.normal = config.pop('normal', - config.pop('suite_normal', True)) self.reentrant = config.pop('reentrant', config.pop('suite_reentrant', False)) @@ -159,7 +157,6 @@ class TestSuite: # a couple of these we just forward to all cases defines = config.pop('defines', {}) in_ = config.pop('in', None) - normal = config.pop('normal', True) reentrant = config.pop('reentrant', False) self.cases = [] @@ -172,7 +169,6 @@ class TestSuite: 'suite': self.name, 'suite_defines': defines, 'suite_in': in_, - 'suite_normal': normal, 'suite_reentrant': reentrant, **case})) @@ -181,7 +177,6 @@ class TestSuite: set(case.defines) for case in self.cases)) # combine other per-case things - self.normal = any(case.normal for case in self.cases) self.reentrant = any(case.reentrant for case in self.cases) for k in config.keys(): @@ -236,6 +231,12 @@ def compile(**args): f.write = write f.writeln = writeln + f.writeln("// Generated by %s:" % sys.argv[0]) + f.writeln("//") + f.writeln("// %s" % ' '.join(sys.argv)) + f.writeln("//") + f.writeln() + # redirect littlefs tracing f.writeln('#define LFS_TRACE_(fmt, ...) do { \\') f.writeln(8*' '+'extern FILE *test_trace; \\') @@ -366,10 +367,10 @@ def compile(**args): f.writeln(4*' '+'.id = "%s",' % suite.id()) f.writeln(4*' '+'.name = "%s",' % suite.name) f.writeln(4*' '+'.path = "%s",' % suite.path) - f.writeln(4*' '+'.types = %s,' - % ' | '.join(filter(None, [ - 'TEST_NORMAL' if suite.normal else None, - 'TEST_REENTRANT' if suite.reentrant else None]))) + f.writeln(4*' '+'.flags = %s,' + % (' | '.join(filter(None, [ + 'TEST_REENTRANT' if suite.reentrant else None])) + or 0)) if suite.defines: # create suite define names f.writeln(4*' '+'.define_names = (const char *const[]){') @@ -384,10 +385,10 @@ def compile(**args): f.writeln(12*' '+'.id = "%s",' % case.id()) f.writeln(12*' '+'.name = "%s",' % case.name) f.writeln(12*' '+'.path = "%s",' % case.path) - f.writeln(12*' '+'.types = %s,' - % ' | '.join(filter(None, [ - 'TEST_NORMAL' if case.normal else None, - 'TEST_REENTRANT' if case.reentrant else None]))) + f.writeln(12*' '+'.flags = %s,' + % (' | '.join(filter(None, [ + 'TEST_REENTRANT' if case.reentrant else None])) + or 0)) f.writeln(12*' '+'.permutations = %d,' % len(case.permutations)) if case.defines: @@ -461,12 +462,13 @@ def runner(**args): '--error-exitcode=4', '-q']) - # filter tests? - if args.get('normal'): cmd.append('-n') - if args.get('reentrant'): cmd.append('-r') + # other context if args.get('geometry'): cmd.append('-G%s' % args.get('geometry')) + if args.get('powerloss'): + cmd.append('-p%s' % args.get('powerloss')) + # defines? if args.get('define'): for define in args.get('define'): @@ -476,12 +478,13 @@ def runner(**args): def list_(**args): cmd = runner(**args) - if args.get('summary'): cmd.append('--summary') - if args.get('list_suites'): cmd.append('--list-suites') - if args.get('list_cases'): cmd.append('--list-cases') - if args.get('list_paths'): cmd.append('--list-paths') - if args.get('list_defines'): cmd.append('--list-defines') - if args.get('list_geometries'): cmd.append('--list-geometries') + if args.get('summary'): cmd.append('--summary') + if args.get('list_suites'): cmd.append('--list-suites') + if args.get('list_cases'): cmd.append('--list-cases') + if args.get('list_paths'): cmd.append('--list-paths') + if args.get('list_defines'): cmd.append('--list-defines') + if args.get('list_geometries'): cmd.append('--list-geometries') + if args.get('list_powerlosses'): cmd.append('--list-powerlosses') if args.get('verbose'): print(' '.join(shlex.quote(c) for c in cmd)) @@ -598,11 +601,12 @@ def run_stage(name, runner_, **args): passed_suite_perms = co.defaultdict(lambda: 0) passed_case_perms = co.defaultdict(lambda: 0) passed_perms = 0 + powerlosses = 0 failures = [] killed = False pattern = re.compile('^(?:' - '(?Prunning|finished|skipped) ' + '(?Prunning|finished|skipped|powerloss) ' '(?P(?P(?P[^#]+)#[^\s#]+)[^\s]*)' '|' '(?P[^:]+):(?P\d+):(?Passert):' ' *(?P.*)' ')$') @@ -613,6 +617,7 @@ def run_stage(name, runner_, **args): nonlocal passed_suite_perms nonlocal passed_case_perms nonlocal passed_perms + nonlocal powerlosses nonlocal locals # run the tests! @@ -659,6 +664,9 @@ def run_stage(name, runner_, **args): last_id = m.group('id') last_output = [] last_assert = None + elif op == 'powerloss': + last_id = m.group('id') + powerlosses += 1 elif op == 'finished': passed_suite_perms[m.group('suite')] += 1 passed_case_perms[m.group('case')] += 1 @@ -766,6 +774,8 @@ def run_stage(name, runner_, **args): len(expected_case_perms)) if not args.get('by_cases') else None, '%d/%d perms' % (passed_perms, expected_perms), + '%dpls!' % powerlosses + if powerlosses else None, '\x1b[31m%d/%d failures\x1b[m' % (len(failures), expected_perms) if failures else None])))) @@ -785,6 +795,7 @@ def run_stage(name, runner_, **args): return ( expected_perms, passed_perms, + powerlosses, failures, killed) @@ -806,33 +817,34 @@ def run(**args): expected = 0 passed = 0 + powerlosses = 0 failures = [] - for type, by in it.product( - ['normal', 'reentrant'], - expected_case_perms.keys() if args.get('by_cases') - else expected_suite_perms.keys() if args.get('by_suites') - else [None]): + for by in (expected_case_perms.keys() if args.get('by_cases') + else expected_suite_perms.keys() if args.get('by_suites') + else [None]): # rebuild runner for each stage to override test identifier if needed stage_runner = runner(**args | { - 'test_ids': [by] if by is not None else args.get('test_ids', []), - 'normal': type == 'normal', - 'reentrant': type == 'reentrant'}) + 'test_ids': [by] if by is not None else args.get('test_ids', [])}) # spawn jobs for stage - expected_, passed_, failures_, killed = run_stage( - '%s %s' % (type, by or 'tests'), stage_runner, **args) + expected_, passed_, powerlosses_, failures_, killed = run_stage( + by or 'tests', stage_runner, **args) expected += expected_ passed += passed_ + powerlosses += powerlosses_ failures.extend(failures_) if (failures and not args.get('keep_going')) or killed: break # show summary print() - print('\x1b[%dmdone:\x1b[m %d/%d passed, %d/%d failed, in %.2fs' + print('\x1b[%dmdone:\x1b[m %s' # %d/%d passed, %d/%d failed%s, in %.2fs' % (32 if not failures else 31, - passed, expected, len(failures), expected, - time.time()-start)) + ', '.join(filter(None, [ + '%d/%d passed' % (passed, expected), + '%d/%d failed' % (len(failures), expected), + '%dpls!' % powerlosses if powerlosses else None, + 'in %.2fs' % (time.time()-start)])))) print() # print each failure @@ -844,7 +856,7 @@ def run(**args): for failure in failures: # show summary of failure path, lineno = runner_paths[testcase(failure.id)] - defines = runner_defines[failure.id] + defines = runner_defines.get(failure.id, {}) print('\x1b[01m%s:%d:\x1b[01;31mfailure:\x1b[m %s%s failed' % (path, lineno, failure.id, @@ -913,8 +925,9 @@ def main(**args): or args.get('list_cases') or args.get('list_paths') or args.get('list_defines') + or args.get('list_defaults') or args.get('list_geometries') - or args.get('list_defaults')): + or args.get('list_powerlosses')): list_(**args) else: run(**args) @@ -930,7 +943,7 @@ if __name__ == "__main__": help="Description of testis to run. May be a directory, path, or \ test identifier. Test identifiers are of the form \ ##, but suffixes can be \ - dropped to run any matching tests. Defaults to %r." % TEST_PATHS) + dropped to run any matching tests. Defaults to %s." % TEST_PATHS) parser.add_argument('-v', '--verbose', action='store_true', help="Output commands that run behind the scenes.") # test flags @@ -945,20 +958,21 @@ if __name__ == "__main__": help="List the path for each test case.") test_parser.add_argument('--list-defines', action='store_true', help="List the defines for each test permutation.") - test_parser.add_argument('--list-geometries', action='store_true', - help="List the disk geometries used for testing.") test_parser.add_argument('--list-defaults', action='store_true', help="List the default defines in this test-runner.") + test_parser.add_argument('--list-geometries', action='store_true', + help="List the disk geometries used for testing.") + test_parser.add_argument('--list-powerlosses', action='store_true', + help="List the available power-loss scenarios.") test_parser.add_argument('-D', '--define', action='append', help="Override a test define.") test_parser.add_argument('-G', '--geometry', help="Filter by geometry.") - test_parser.add_argument('-n', '--normal', action='store_true', - help="Filter for normal tests. Can be combined.") - test_parser.add_argument('-r', '--reentrant', action='store_true', - help="Filter for reentrant tests. Can be combined.") + test_parser.add_argument('-p', '--powerloss', + help="Comma-separated list of power-loss scenarios to test. \ + Defaults to 0,l.") test_parser.add_argument('-d', '--disk', - help="Use this file as the disk.") + help="Redirect block device operations to this file.") test_parser.add_argument('-t', '--trace', help="Redirect trace output to this file.") test_parser.add_argument('-o', '--output', -- cgit v1.2.3