Restructured block devices again for better test exploitation

Also finished migrating tests with test_relocations and test_exhaustion.

The issue I was running into when migrating these tests was a lack of
flexibility with what you could do with the block devices. It was
possible to hack in some hooks for things like bad blocks and power
loss, but it wasn't clean or easily extendable.

The solution here was to just put all of these test extensions into a
third block device, testbd, that uses the other two example block
devices internally.

testbd has several useful features for testing. Note this makes it a
pretty terrible block device _example_ since these hooks look more
complicated than a block device needs to be.

- testbd can simulate different erase values, supporting 1s, 0s, other byte
  patterns, or no erases at all (which can cause surprising bugs). This
  actually depends on the simulated erase values in ramdb and filebd.

  I did try to move this out of rambd/filebd, but it's not possible to
  simulate erases in testbd without buffering entire blocks and creating
  an excessive amount of extra write operations.

- testbd also helps simulate power-loss by containing a "power cycles"
  counter that is decremented every write operation until it calls exit.

  This is notably faster than the previous gdb approach, which is
  valuable since the reentrant tests tend to take a while to resolve.

- testbd also tracks wear, which can be manually set and read. This is
  very useful for testing things like bad block handling, wear leveling,
  or even changing the effective size of the block device at runtime.

2 files changed, 87 insertions, 146 deletions

diff --git a/scripts/explode_asserts.py b/scripts/explode_asserts.py
index 7c24c63..ff3f260 100755
--- a/scripts/explode_asserts.py
+++ b/scripts/explode_asserts.py
@@ -146,7 +146,7 @@ def pnested():
 
 pexpr = (
     # shortcut for a bit better performance
-    p.regex('[^%s/#\'"();{}=><,&|-]+' % ASSERT_CHARS) |
+    p.regex('[^%s/#\'"():;{}=><,&|-]+' % ASSERT_CHARS) |
     pws |
     passert |
     pstring |
@@ -157,7 +157,7 @@ pexpr = (
 @p.generate
 def pstmt():
     ws = yield pws.many()
-    lh = yield pexpr.until(p.string('=>') | p.regex('[;{}]'))
+    lh = yield pexpr.until(p.string('=>') | p.regex('[:;{}]'))
     op = yield p.string('=>').optional()
     if op == '=>':
         rh = yield pstmt
@@ -168,7 +168,7 @@ def pstmt():
 @p.generate
 def pstmts():
     a = yield pstmt
-    b = yield (p.regex('[;{}]') + pstmt).many()
+    b = yield (p.regex('[:;{}]') + pstmt).many()
     return [a] + b
 
 def main(args):
diff --git a/scripts/test_.py b/scripts/test_.py
index 2b78510..c4e44a1 100755
--- a/scripts/test_.py
+++ b/scripts/test_.py
@@ -19,7 +19,7 @@
 # x config chaining correct
 # - why can't gdb see my defines?
 # - say no to internal?
-# - buffering stdout issues?
+# x buffering stdout issues?
 
 import toml
 import glob
@@ -33,6 +33,9 @@ import base64
 import sys
 import copy
 import shlex
+import pty
+import errno
+import signal
 
 TESTDIR = 'tests_'
 RULES = """
@@ -45,98 +48,31 @@ $(foreach target,$(SRC),$(eval $(FLATTEN)))
 -include tests_/*.d
 
 .SECONDARY:
-%.test: override CFLAGS += -fdiagnostics-color=always
-%.test: override CFLAGS += -ggdb
+%.test: override CFLAGS += -gdwarf-2
+%.test: override CFLAGS += -ggdb3
+%.test: override CFLAGS += -g3
 %.test: %.test.o $(foreach f,$(subst /,.,$(SRC:.c=.o)),%.$f)
     $(CC) $(CFLAGS) $^ $(LFLAGS) -o $@
 """
 GLOBALS = """
 //////////////// AUTOGENERATED TEST ////////////////
 #include "lfs.h"
-#include "filebd/lfs_filebd.h"
-#include "rambd/lfs_rambd.h"
+#include "testbd/lfs_testbd.h"
 #include <stdio.h>
-
-extern const char *lfs_testbd_disk;
-typedef union {
-    lfs_filebd_t filebd;
-    lfs_rambd_t rambd;
-} lfs_testbd_t; 
-struct lfs_testbd_config {
-    struct lfs_filebd_config filecfg;
-    struct lfs_rambd_config ramcfg;
-};
-
-__attribute__((unused))
-static int lfs_testbd_createcfg(const struct lfs_config *cfg,
-        const struct lfs_testbd_config *bdcfg) {
-    if (lfs_testbd_disk) {
-        return lfs_filebd_createcfg(cfg, lfs_testbd_disk, &bdcfg->filecfg);
-    } else {
-        return lfs_rambd_createcfg(cfg, &bdcfg->ramcfg);
-    }
-}
-
-__attribute__((unused))
-static void lfs_testbd_destroy(const struct lfs_config *cfg) {
-    if (lfs_testbd_disk) {
-        lfs_filebd_destroy(cfg);
-    } else {
-        lfs_rambd_destroy(cfg);
-    }
-}
-
-__attribute__((unused))
-static int lfs_testbd_read(const struct lfs_config *cfg, lfs_block_t block,
-        lfs_off_t off, void *buffer, lfs_size_t size) {
-    if (lfs_testbd_disk) {
-        return lfs_filebd_read(cfg, block, off, buffer, size);
-    } else {
-        return lfs_rambd_read(cfg, block, off, buffer, size);
-    }
-}
-
-__attribute__((unused))
-static int lfs_testbd_prog(const struct lfs_config *cfg, lfs_block_t block,
-        lfs_off_t off, const void *buffer, lfs_size_t size) {
-    if (lfs_testbd_disk) {
-        return lfs_filebd_prog(cfg, block, off, buffer, size);
-    } else {
-        return lfs_rambd_prog(cfg, block, off, buffer, size);
-    }
-}
-
-__attribute__((unused))
-static int lfs_testbd_erase(const struct lfs_config *cfg, lfs_block_t block) {
-    if (lfs_testbd_disk) {
-        return lfs_filebd_erase(cfg, block);
-    } else {
-        return lfs_rambd_erase(cfg, block);
-    }
-}
-
-__attribute__((unused))
-static int lfs_testbd_sync(const struct lfs_config *cfg) {
-    if (lfs_testbd_disk) {
-        return lfs_filebd_sync(cfg);
-    } else {
-        return lfs_rambd_sync(cfg);
-    }
-}
+extern const char *lfs_testbd_path;
+extern uint32_t lfs_testbd_cycles;
 """
 DEFINES = {
-    "LFS_BD_READ": "lfs_testbd_read",
-    "LFS_BD_PROG": "lfs_testbd_prog",
-    "LFS_BD_ERASE": "lfs_testbd_erase",
-    "LFS_BD_SYNC": "lfs_testbd_sync",
-    "LFS_READ_SIZE": 16,
-    "LFS_PROG_SIZE": "LFS_READ_SIZE",
-    "LFS_BLOCK_SIZE": 512,
-    "LFS_BLOCK_COUNT": 1024,
-    "LFS_BLOCK_CYCLES": -1,
-    "LFS_CACHE_SIZE": "(64 % LFS_PROG_SIZE == 0 ? 64 : LFS_PROG_SIZE)",
-    "LFS_LOOKAHEAD_SIZE": 16,
-    "LFS_ERASE_VALUE": 0xff,
+    'LFS_READ_SIZE': 16,
+    'LFS_PROG_SIZE': 'LFS_READ_SIZE',
+    'LFS_BLOCK_SIZE': 512,
+    'LFS_BLOCK_COUNT': 1024,
+    'LFS_BLOCK_CYCLES': -1,
+    'LFS_CACHE_SIZE': '(64 % LFS_PROG_SIZE == 0 ? 64 : LFS_PROG_SIZE)',
+    'LFS_LOOKAHEAD_SIZE': 16,
+    'LFS_ERASE_VALUE': 0xff,
+    'LFS_ERASE_CYCLES': 0,
+    'LFS_BADBLOCK_BEHAVIOR': 'LFS_TESTBD_BADBLOCK_NOPROG',
 }
 PROLOGUE = """
     // prologue
@@ -152,10 +88,10 @@ PROLOGUE = """
     
     __attribute__((unused)) const struct lfs_config cfg = {
         .context        = &bd,
-        .read           = LFS_BD_READ,
-        .prog           = LFS_BD_PROG,
-        .erase          = LFS_BD_ERASE,
-        .sync           = LFS_BD_SYNC,
+        .read           = lfs_testbd_read,
+        .prog           = lfs_testbd_prog,
+        .erase          = lfs_testbd_erase,
+        .sync           = lfs_testbd_sync,
         .read_size      = LFS_READ_SIZE,
         .prog_size      = LFS_PROG_SIZE,
         .block_size     = LFS_BLOCK_SIZE,
@@ -166,15 +102,17 @@ PROLOGUE = """
     };
 
     __attribute__((unused)) const struct lfs_testbd_config bdcfg = {
-        .filecfg.erase_value = LFS_ERASE_VALUE,
-        .ramcfg.erase_value = LFS_ERASE_VALUE,
+        .erase_value        = LFS_ERASE_VALUE,
+        .erase_cycles       = LFS_ERASE_CYCLES,
+        .badblock_behavior  = LFS_BADBLOCK_BEHAVIOR,
+        .power_cycles       = lfs_testbd_cycles,
     };
 
-    lfs_testbd_createcfg(&cfg, &bdcfg) => 0;
+    lfs_testbd_createcfg(&cfg, lfs_testbd_path, &bdcfg) => 0;
 """
 EPILOGUE = """
     // epilogue
-    lfs_testbd_destroy(&cfg);
+    lfs_testbd_destroy(&cfg) => 0;
 """
 PASS = '\033[32m✓\033[0m'
 FAIL = '\033[31m✗\033[0m'
@@ -224,15 +162,15 @@ class TestCase:
 
     def build(self, f, **_):
         # prologue
-        f.write('void test_case%d(%s) {\n' % (self.caseno, ','.join(
+        for k, v in sorted(self.defines.items()):
+            if k not in self.suite.defines:
+                f.write('#define %s %s\n' % (k, v))
+
+        f.write('void test_case%d(%s) {' % (self.caseno, ','.join(
             '\n'+8*' '+'__attribute__((unused)) intmax_t %s' % k
             for k in sorted(self.perms[0].defines)
             if k not in self.defines)))
 
-        for k, v in sorted(self.defines.items()):
-            if k not in self.suite.defines:
-                f.write(4*' '+'#define %s %s\n' % (k, v))
-
         f.write(PROLOGUE)
         f.write('\n')
         f.write(4*' '+'// test case %d\n' % self.caseno)
@@ -243,13 +181,11 @@ class TestCase:
 
         # epilogue
         f.write(EPILOGUE)
-        f.write('\n')
+        f.write('}\n')
 
         for k, v in sorted(self.defines.items()):
             if k not in self.suite.defines:
-                f.write(4*' '+'#undef %s\n' % k)
-
-        f.write('}\n')
+                f.write('#undef %s\n' % k)
 
     def shouldtest(self, **args):
         if (self.filter is not None and
@@ -265,7 +201,8 @@ class TestCase:
         else:
             return True
 
-    def test(self, exec=[], persist=False, gdb=False, failure=None, **args):
+    def test(self, exec=[], persist=False, cycles=None,
+            gdb=False, failure=None, **args):
         # build command
         cmd = exec + ['./%s.test' % self.suite.path,
             repr(self.caseno), repr(self.permno)]
@@ -280,6 +217,10 @@ class TestCase:
 
             cmd.append(self.suite.path + '.disk')
 
+        # simulate power-loss after n cycles?
+        if cycles:
+            cmd.append(str(cycles))
+
         # failed? drop into debugger?
         if gdb and failure:
             ncmd = ['gdb']
@@ -295,19 +236,25 @@ class TestCase:
 
             if args.get('verbose', False):
                 print(' '.join(shlex.quote(c) for c in ncmd))
+            signal.signal(signal.SIGINT, signal.SIG_IGN)
             sys.exit(sp.call(ncmd))
 
         # run test case!
-        stdout = []
-        assert_ = None
+        mpty, spty = pty.openpty()
         if args.get('verbose', False):
             print(' '.join(shlex.quote(c) for c in cmd))
-        proc = sp.Popen(cmd,
-            universal_newlines=True,
-            bufsize=1,
-            stdout=sp.PIPE,
-            stderr=sp.STDOUT)
-        for line in iter(proc.stdout.readline, ''):
+        proc = sp.Popen(cmd, stdout=spty, stderr=spty)
+        os.close(spty)
+        mpty = os.fdopen(mpty, 'r', 1)
+        stdout = []
+        assert_ = None
+        while True:
+            try:
+                line = mpty.readline()
+            except OSError as e:
+                if e.errno == errno.EIO:
+                    break
+                raise
             stdout.append(line)
             if args.get('verbose', False):
                 sys.stdout.write(line)
@@ -361,36 +308,23 @@ class ReentrantTestCase(TestCase):
         return self.reentrant and super().shouldtest(**args)
 
     def test(self, exec=[], persist=False, gdb=False, failure=None, **args):
-        # clear disk first?
-        if persist != 'noerase':
-            try:
-                os.remove(self.suite.path + '.disk')
-            except FileNotFoundError:
-                pass
-
         for cycles in it.count(1):
+            # clear disk first?
+            if cycles == 1 and persist != 'noerase':
+                persist = 'erase'
+            else:
+                persist = 'noerase'
+
             # exact cycle we should drop into debugger?
             if gdb and failure and failure.cycleno == cycles:
-                return super().test(exec=exec, persist='noerase',
-                    gdb=gdb, failure=failure, **args)
+                return super().test(gdb=gdb,
+                    persist=persist, failure=failure, **args)
 
             # run tests, but kill the program after prog/erase has
             # been hit n cycles. We exit with a special return code if the
             # program has not finished, since this isn't a test failure.
-            nexec = exec + [
-                'gdb', '-batch-silent',
-                '-ex', 'handle all nostop',
-                '-ex', 'b lfs_filebd_prog',
-                '-ex', 'b lfs_filebd_erase',
-                '-ex', 'r',
-                ] + cycles*['-ex', 'c'] + [
-                '-ex', 'q '
-                    '!$_isvoid($_exitsignal) ? $_exitsignal : '
-                    '!$_isvoid($_exitcode) ? $_exitcode : '
-                    '33',
-                '--args']
             try:
-                return super().test(exec=nexec, persist='noerase', **args)
+                return super().test(persist=persist, cycles=cycles, **args)
             except TestFailure as nfailure:
                 if nfailure.returncode == 33:
                     continue
@@ -535,11 +469,13 @@ class TestSuite:
             case.build(tfs[case.in_], **args)
 
         tf.write('\n')
-        tf.write('const char *lfs_testbd_disk;\n')
+        tf.write('const char *lfs_testbd_path;\n')
+        tf.write('uint32_t lfs_testbd_cycles;\n')
         tf.write('int main(int argc, char **argv) {\n')
-        tf.write(4*' '+'int case_ = (argc >= 2) ? atoi(argv[1]) : 0;\n')
-        tf.write(4*' '+'int perm = (argc >= 3) ? atoi(argv[2]) : 0;\n')
-        tf.write(4*' '+'lfs_testbd_disk = (argc >= 4) ? argv[3] : NULL;\n')
+        tf.write(4*' '+'int case_         = (argc > 1) ? atoi(argv[1]) : 0;\n')
+        tf.write(4*' '+'int perm          = (argc > 2) ? atoi(argv[2]) : 0;\n')
+        tf.write(4*' '+'lfs_testbd_path   = (argc > 3) ? argv[3] : NULL;\n')
+        tf.write(4*' '+'lfs_testbd_cycles = (argc > 4) ? atoi(argv[4]) : 0;\n')
         for perm in self.perms:
             # test declaration
             tf.write(4*' '+'extern void test_case%d(%s);\n' % (
@@ -671,15 +607,20 @@ def main(**args):
     cmd = (['make', '-f', 'Makefile'] +
         list(it.chain.from_iterable(['-f', m] for m in makefiles)) +
         [target for target in targets])
-    stdout = []
+    mpty, spty = pty.openpty()
     if args.get('verbose', False):
         print(' '.join(shlex.quote(c) for c in cmd))
-    proc = sp.Popen(cmd,
-        universal_newlines=True,
-        bufsize=1,
-        stdout=sp.PIPE,
-        stderr=sp.STDOUT)
-    for line in iter(proc.stdout.readline, ''):
+    proc = sp.Popen(cmd, stdout=spty, stderr=spty)
+    os.close(spty)
+    mpty = os.fdopen(mpty, 'r', 1)
+    stdout = []
+    while True:
+        try:
+            line = mpty.readline()
+        except OSError as e:
+            if e.errno == errno.EIO:
+                break
+            raise
         stdout.append(line)
         if args.get('verbose', False):
             sys.stdout.write(line)