test_runner: add initial CLI runner

This commit introduces an initial version of a CLI-based
test runner.

PR-URL: https://github.com/nodejs/node/pull/42658
Reviewed-By: Antoine du Hamel <duhamelantoine1995@gmail.com>

5 files changed, 310 insertions, 86 deletions

diff --git a/lib/internal/errors.js b/lib/internal/errors.js
index a5c64080a59..434576bde16 100644
--- a/lib/internal/errors.js
+++ b/lib/internal/errors.js
@@ -207,6 +207,16 @@ function isErrorStackTraceLimitWritable() {
     desc.set !== undefined;
 }
 
+function inspectWithNoCustomRetry(obj, options) {
+  const utilInspect = lazyInternalUtilInspect();
+
+  try {
+    return utilInspect.inspect(obj, options);
+  } catch {
+    return utilInspect.inspect(obj, { ...options, customInspect: false });
+  }
+}
+
 // A specialized Error that includes an additional info property with
 // additional information about the error condition.
 // It has the properties present in a UVException but with a custom error
@@ -862,6 +872,7 @@ module.exports = {
   getMessage,
   hideInternalStackFrames,
   hideStackFrames,
+  inspectWithNoCustomRetry,
   isErrorStackTraceLimitWritable,
   isStackOverflowError,
   kEnhanceStackBeforeInspector,
@@ -1549,11 +1560,14 @@ E('ERR_TEST_FAILURE', function(error, failureType) {
   assert(typeof failureType === 'string',
          "The 'failureType' argument must be of type string.");
 
-  const msg = error?.message ?? lazyInternalUtilInspect().inspect(error);
+  let msg = error?.message ?? error;
 
-  this.failureType = error?.failureType ?? failureType;
-  this.cause = error;
+  if (typeof msg !== 'string') {
+    msg = inspectWithNoCustomRetry(msg);
+  }
 
+  this.failureType = failureType;
+  this.cause = error;
   return msg;
 }, Error);
 E('ERR_TLS_CERT_ALTNAME_FORMAT', 'Invalid subject alternative name string',
diff --git a/lib/internal/main/test_runner.js b/lib/internal/main/test_runner.js
new file mode 100644
index 00000000000..71bf21782f3
--- /dev/null
+++ b/lib/internal/main/test_runner.js
@@ -0,0 +1,157 @@
+'use strict';
+const {
+  ArrayFrom,
+  ArrayPrototypeFilter,
+  ArrayPrototypeIncludes,
+  ArrayPrototypePush,
+  ArrayPrototypeSlice,
+  ArrayPrototypeSort,
+  Promise,
+  SafeSet,
+} = primordials;
+const {
+  prepareMainThreadExecution,
+} = require('internal/bootstrap/pre_execution');
+const { spawn } = require('child_process');
+const { readdirSync, statSync } = require('fs');
+const console = require('internal/console/global');
+const {
+  codes: {
+    ERR_TEST_FAILURE,
+  },
+} = require('internal/errors');
+const test = require('internal/test_runner/harness');
+const { kSubtestsFailed } = require('internal/test_runner/test');
+const {
+  isSupportedFileType,
+  doesPathMatchFilter,
+} = require('internal/test_runner/utils');
+const { basename, join, resolve } = require('path');
+const kFilterArgs = ['--test'];
+
+prepareMainThreadExecution(false);
+markBootstrapComplete();
+
+// TODO(cjihrig): Replace this with recursive readdir once it lands.
+function processPath(path, testFiles, options) {
+  const stats = statSync(path);
+
+  if (stats.isFile()) {
+    if (options.userSupplied ||
+        (options.underTestDir && isSupportedFileType(path)) ||
+        doesPathMatchFilter(path)) {
+      testFiles.add(path);
+    }
+  } else if (stats.isDirectory()) {
+    const name = basename(path);
+
+    if (!options.userSupplied && name === 'node_modules') {
+      return;
+    }
+
+    // 'test' directories get special treatment. Recursively add all .js,
+    // .cjs, and .mjs files in the 'test' directory.
+    const isTestDir = name === 'test';
+    const { underTestDir } = options;
+    const entries = readdirSync(path);
+
+    if (isTestDir) {
+      options.underTestDir = true;
+    }
+
+    options.userSupplied = false;
+
+    for (let i = 0; i < entries.length; i++) {
+      processPath(join(path, entries[i]), testFiles, options);
+    }
+
+    options.underTestDir = underTestDir;
+  }
+}
+
+function createTestFileList() {
+  const cwd = process.cwd();
+  const hasUserSuppliedPaths = process.argv.length > 1;
+  const testPaths = hasUserSuppliedPaths ?
+    ArrayPrototypeSlice(process.argv, 1) : [cwd];
+  const testFiles = new SafeSet();
+
+  try {
+    for (let i = 0; i < testPaths.length; i++) {
+      const absolutePath = resolve(testPaths[i]);
+
+      processPath(absolutePath, testFiles, { userSupplied: true });
+    }
+  } catch (err) {
+    if (err?.code === 'ENOENT') {
+      console.error(`Could not find '${err.path}'`);
+      process.exit(1);
+    }
+
+    throw err;
+  }
+
+  return ArrayPrototypeSort(ArrayFrom(testFiles));
+}
+
+function filterExecArgv(arg) {
+  return !ArrayPrototypeIncludes(kFilterArgs, arg);
+}
+
+function runTestFile(path) {
+  return test(path, () => {
+    return new Promise((resolve, reject) => {
+      const args = ArrayPrototypeFilter(process.execArgv, filterExecArgv);
+      ArrayPrototypePush(args, path);
+
+      const child = spawn(process.execPath, args);
+      // TODO(cjihrig): Implement a TAP parser to read the child's stdout
+      // instead of just displaying it all if the child fails.
+      let stdout = '';
+      let stderr = '';
+      let err;
+
+      child.on('error', (error) => {
+        err = error;
+      });
+
+      child.stdout.setEncoding('utf8');
+      child.stderr.setEncoding('utf8');
+
+      child.stdout.on('data', (chunk) => {
+        stdout += chunk;
+      });
+
+      child.stderr.on('data', (chunk) => {
+        stderr += chunk;
+      });
+
+      child.once('exit', (code, signal) => {
+        if (code !== 0 || signal !== null) {
+          if (!err) {
+            err = new ERR_TEST_FAILURE('test failed', kSubtestsFailed);
+            err.exitCode = code;
+            err.signal = signal;
+            err.stdout = stdout;
+            err.stderr = stderr;
+            // The stack will not be useful since the failures came from tests
+            // in a child process.
+            err.stack = undefined;
+          }
+
+          return reject(err);
+        }
+
+        resolve();
+      });
+    });
+  });
+}
+
+(async function main() {
+  const testFiles = createTestFileList();
+
+  for (let i = 0; i < testFiles.length; i++) {
+    runTestFile(testFiles[i]);
+  }
+})();
diff --git a/lib/internal/test_runner/tap_stream.js b/lib/internal/test_runner/tap_stream.js
index d5e095991c7..a6bfbb3367c 100644
--- a/lib/internal/test_runner/tap_stream.js
+++ b/lib/internal/test_runner/tap_stream.js
@@ -5,13 +5,13 @@ const {
   ArrayPrototypePush,
   ArrayPrototypeShift,
   ObjectEntries,
-  StringPrototypeReplace,
   StringPrototypeReplaceAll,
   StringPrototypeSplit,
+  RegExpPrototypeSymbolReplace,
 } = primordials;
+const { inspectWithNoCustomRetry } = require('internal/errors');
 const Readable = require('internal/streams/readable');
 const { isError } = require('internal/util');
-const { inspect } = require('internal/util/inspect');
 const kFrameStartRegExp = /^ {4}at /;
 const kLineBreakRegExp = /\n|\r\n/;
 const inspectOptions = { colors: false, breakLength: Infinity };
@@ -74,77 +74,8 @@ class TapStream extends Readable {
   details(indent, duration, error) {
     let details = `${indent}  ---\n`;
 
-    details += `${indent}  duration_ms: ${duration}\n`;
-
-    if (error !== null && typeof error === 'object') {
-      const entries = ObjectEntries(error);
-      const isErrorObj = isError(error);
-
-      for (let i = 0; i < entries.length; i++) {
-        const { 0: key, 1: value } = entries[i];
-
-        if (isError && (key === 'cause' || key === 'code')) {
-          continue;
-        }
-
-        details += `${indent}  ${key}: ${inspect(value, inspectOptions)}\n`;
-      }
-
-      if (isErrorObj) {
-        const { kTestCodeFailure } = lazyLoadTest();
-        const {
-          cause,
-          code,
-          failureType,
-          message,
-          stack,
-        } = error;
-        let errMsg = message ?? '<unknown error>';
-        let errStack = stack;
-        let errCode = code;
-
-        // If the ERR_TEST_FAILURE came from an error provided by user code,
-        // then try to unwrap the original error message and stack.
-        if (code === 'ERR_TEST_FAILURE' && failureType === kTestCodeFailure) {
-          errMsg = cause?.message ?? errMsg;
-          errStack = cause?.stack ?? errStack;
-          errCode = cause?.code ?? errCode;
-        }
-
-        details += `${indent}  error: ${inspect(errMsg, inspectOptions)}\n`;
-
-        if (errCode) {
-          details += `${indent}  code: ${errCode}\n`;
-        }
-
-        if (typeof errStack === 'string') {
-          const frames = [];
-
-          ArrayPrototypeForEach(
-            StringPrototypeSplit(errStack, kLineBreakRegExp),
-            (frame) => {
-              const processed = StringPrototypeReplace(
-                frame, kFrameStartRegExp, ''
-              );
-
-              if (processed.length > 0 && processed.length !== frame.length) {
-                ArrayPrototypePush(frames, processed);
-              }
-            }
-          );
-
-          if (frames.length > 0) {
-            const frameDelimiter = `\n${indent}    `;
-
-            details += `${indent}  stack: |-${frameDelimiter}`;
-            details += `${ArrayPrototypeJoin(frames, `${frameDelimiter}`)}\n`;
-          }
-        }
-      }
-    } else if (error !== null && error !== undefined) {
-      details += `${indent}  error: ${inspect(error, inspectOptions)}\n`;
-    }
-
+    details += jsToYaml(indent, 'duration_ms', duration);
+    details += jsToYaml(indent, null, error);
     details += `${indent}  ...\n`;
     this.#tryPush(details);
   }
@@ -190,4 +121,100 @@ function tapEscape(input) {
   );
 }
 
+function jsToYaml(indent, name, value) {
+  if (value === null || value === undefined) {
+    return '';
+  }
+
+  if (typeof value !== 'object') {
+    const prefix = `${indent}  ${name}: `;
+
+    if (typeof value !== 'string') {
+      return `${prefix}${inspectWithNoCustomRetry(value, inspectOptions)}\n`;
+    }
+
+    const lines = StringPrototypeSplit(value, kLineBreakRegExp);
+
+    if (lines.length === 1) {
+      return `${prefix}${inspectWithNoCustomRetry(value, inspectOptions)}\n`;
+    }
+
+    let str = `${prefix}|-\n`;
+
+    for (let i = 0; i < lines.length; i++) {
+      str += `${indent}    ${lines[i]}\n`;
+    }
+
+    return str;
+  }
+
+  const entries = ObjectEntries(value);
+  const isErrorObj = isError(value);
+  let result = '';
+
+  for (let i = 0; i < entries.length; i++) {
+    const { 0: key, 1: value } = entries[i];
+
+    if (isErrorObj && (key === 'cause' || key === 'code')) {
+      continue;
+    }
+
+    result += jsToYaml(indent, key, value);
+  }
+
+  if (isErrorObj) {
+    const { kTestCodeFailure } = lazyLoadTest();
+    const {
+      cause,
+      code,
+      failureType,
+      message,
+      stack,
+    } = value;
+    let errMsg = message ?? '<unknown error>';
+    let errStack = stack;
+    let errCode = code;
+
+    // If the ERR_TEST_FAILURE came from an error provided by user code,
+    // then try to unwrap the original error message and stack.
+    if (code === 'ERR_TEST_FAILURE' && failureType === kTestCodeFailure) {
+      errMsg = cause?.message ?? errMsg;
+      errStack = cause?.stack ?? errStack;
+      errCode = cause?.code ?? errCode;
+    }
+
+    result += jsToYaml(indent, 'error', errMsg);
+
+    if (errCode) {
+      result += jsToYaml(indent, 'code', errCode);
+    }
+
+    if (typeof errStack === 'string') {
+      const frames = [];
+
+      ArrayPrototypeForEach(
+        StringPrototypeSplit(errStack, kLineBreakRegExp),
+        (frame) => {
+          const processed = RegExpPrototypeSymbolReplace(
+            kFrameStartRegExp, frame, ''
+          );
+
+          if (processed.length > 0 && processed.length !== frame.length) {
+            ArrayPrototypePush(frames, processed);
+          }
+        }
+      );
+
+      if (frames.length > 0) {
+        const frameDelimiter = `\n${indent}    `;
+
+        result += `${indent}  stack: |-${frameDelimiter}`;
+        result += `${ArrayPrototypeJoin(frames, `${frameDelimiter}`)}\n`;
+      }
+    }
+  }
+
+  return result;
+}
+
 module.exports = { TapStream };
diff --git a/lib/internal/test_runner/test.js b/lib/internal/test_runner/test.js
index 3e632cc1c52..d35bf6307bf 100644
--- a/lib/internal/test_runner/test.js
+++ b/lib/internal/test_runner/test.js
@@ -12,12 +12,14 @@ const {
   codes: {
     ERR_TEST_FAILURE,
   },
+  kIsNodeError,
 } = require('internal/errors');
 const { getOptionValue } = require('internal/options');
 const { TapStream } = require('internal/test_runner/tap_stream');
 const { createDeferredPromise } = require('internal/util');
 const { isPromise } = require('internal/util/types');
 const { isUint32 } = require('internal/validators');
+const { cpus } = require('os');
 const { bigint: hrtime } = process.hrtime;
 const kCallbackAndPromisePresent = 'callbackAndPromisePresent';
 const kCancelledByParent = 'cancelledByParent';
@@ -27,7 +29,10 @@ const kSubtestsFailed = 'subtestsFailed';
 const kTestCodeFailure = 'testCodeFailure';
 const kDefaultIndent = '    ';
 const noop = FunctionPrototype;
-const testOnlyFlag = getOptionValue('--test-only');
+const isTestRunner = getOptionValue('--test');
+const testOnlyFlag = !isTestRunner && getOptionValue('--test-only');
+// TODO(cjihrig): Use uv_available_parallelism() once it lands.
+const rootConcurrency = isTestRunner ? cpus().length : 1;
 
 class TestContext {
   #test;
@@ -79,7 +84,7 @@ class Test extends AsyncResource {
     }
 
     if (parent === null) {
-      this.concurrency = 1;
+      this.concurrency = rootConcurrency;
       this.indent = '';
       this.indentString = kDefaultIndent;
       this.only = testOnlyFlag;
@@ -303,12 +308,14 @@ class Test extends AsyncResource {
 
           // If the callback is called a second time, let the user know, but
           // don't let them know more than once.
-          if (calledCount === 2) {
-            throw new ERR_TEST_FAILURE(
-              'callback invoked multiple times',
-              kMultipleCallbackInvocations
-            );
-          } else if (calledCount > 2) {
+          if (calledCount > 1) {
+            if (calledCount === 2) {
+              throw new ERR_TEST_FAILURE(
+                'callback invoked multiple times',
+                kMultipleCallbackInvocations
+              );
+            }
+
             return;
           }
 
@@ -335,7 +342,11 @@ class Test extends AsyncResource {
 
       this.pass();
     } catch (err) {
-      this.fail(new ERR_TEST_FAILURE(err, kTestCodeFailure));
+      if (err?.code === 'ERR_TEST_FAILURE' && kIsNodeError in err) {
+        this.fail(err);
+      } else {
+        this.fail(new ERR_TEST_FAILURE(err, kTestCodeFailure));
+      }
     }
 
     // Clean up the test. Then, try to report the results and execute any
@@ -436,4 +447,4 @@ class Test extends AsyncResource {
   }
 }
 
-module.exports = { kDefaultIndent, kTestCodeFailure, Test };
+module.exports = { kDefaultIndent, kSubtestsFailed, kTestCodeFailure, Test };
diff --git a/lib/internal/test_runner/utils.js b/lib/internal/test_runner/utils.js
new file mode 100644
index 00000000000..09803d33aeb
--- /dev/null
+++ b/lib/internal/test_runner/utils.js
@@ -0,0 +1,15 @@
+'use strict';
+const { RegExpPrototypeExec } = primordials;
+const { basename } = require('path');
+const kSupportedFileExtensions = /\.[cm]?js$/;
+const kTestFilePattern = /((^test(-.+)?)|(.+[.\-_]test))\.[cm]?js$/;
+
+function doesPathMatchFilter(p) {
+  return RegExpPrototypeExec(kTestFilePattern, basename(p)) !== null;
+}
+
+function isSupportedFileType(p) {
+  return RegExpPrototypeExec(kSupportedFileExtensions, p) !== null;
+}
+
+module.exports = { isSupportedFileType, doesPathMatchFilter };