test: add initial test module

This commit adds a new 'test' module that exposes an API
for creating JavaScript tests. As the tests execute, TAP
output is written to standard output. This commit only supports
executing individual test files, and does not implement
command line functionality for a full test runner.

PR-URL: https://github.com/nodejs/node/pull/42325
Refs: https://github.com/nodejs/node/issues/40954
Reviewed-By: Matteo Collina <matteo.collina@gmail.com>
Reviewed-By: Antoine du Hamel <duhamelantoine1995@gmail.com>

5 files changed, 770 insertions, 0 deletions

diff --git a/lib/internal/errors.js b/lib/internal/errors.js
index dce159b94cc..5f75c0290b3 100644
--- a/lib/internal/errors.js
+++ b/lib/internal/errors.js
@@ -1544,6 +1544,18 @@ E('ERR_STREAM_WRAP', 'Stream has StringDecoder set or is in objectMode', Error);
 E('ERR_STREAM_WRITE_AFTER_END', 'write after end', Error);
 E('ERR_SYNTHETIC', 'JavaScript Callstack', Error);
 E('ERR_SYSTEM_ERROR', 'A system error occurred', SystemError);
+E('ERR_TEST_FAILURE', function(error, failureType) {
+  hideInternalStackFrames(this);
+  assert(typeof failureType === 'string',
+         "The 'failureType' argument must be of type string.");
+
+  const msg = error?.message ?? lazyInternalUtilInspect().inspect(error);
+
+  this.failureType = error?.failureType ?? failureType;
+  this.cause = error;
+
+  return msg;
+}, Error);
 E('ERR_TLS_CERT_ALTNAME_FORMAT', 'Invalid subject alternative name string',
   SyntaxError);
 E('ERR_TLS_CERT_ALTNAME_INVALID', function(reason, host, cert) {
diff --git a/lib/internal/test_runner/harness.js b/lib/internal/test_runner/harness.js
new file mode 100644
index 00000000000..66544d91522
--- /dev/null
+++ b/lib/internal/test_runner/harness.js
@@ -0,0 +1,131 @@
+'use strict';
+const { FunctionPrototypeBind, SafeMap } = primordials;
+const {
+  createHook,
+  executionAsyncId,
+} = require('async_hooks');
+const {
+  codes: {
+    ERR_TEST_FAILURE,
+  },
+} = require('internal/errors');
+const { Test } = require('internal/test_runner/test');
+
+function createProcessEventHandler(eventName, rootTest, testResources) {
+  return (err) => {
+    // Check if this error is coming from a test. If it is, fail the test.
+    const test = testResources.get(executionAsyncId());
+
+    if (test !== undefined) {
+      if (test.finished) {
+        // If the test is already finished, report this as a top level
+        // diagnostic since this is a malformed test.
+        const msg = `Warning: Test "${test.name}" generated asynchronous ` +
+          'activity after the test ended. This activity created the error ' +
+          `"${err}" and would have caused the test to fail, but instead ` +
+          `triggered an ${eventName} event.`;
+
+        rootTest.diagnostic(msg);
+        return;
+      }
+
+      test.fail(new ERR_TEST_FAILURE(err, eventName));
+      test.postRun();
+    }
+  };
+}
+
+function setup(root) {
+  const testResources = new SafeMap();
+  const hook = createHook({
+    init(asyncId, type, triggerAsyncId, resource) {
+      if (resource instanceof Test) {
+        testResources.set(asyncId, resource);
+        return;
+      }
+
+      const parent = testResources.get(triggerAsyncId);
+
+      if (parent !== undefined) {
+        testResources.set(asyncId, parent);
+      }
+    },
+    destroy(asyncId) {
+      testResources.delete(asyncId);
+    }
+  });
+
+  hook.enable();
+
+  const exceptionHandler =
+    createProcessEventHandler('uncaughtException', root, testResources);
+  const rejectionHandler =
+    createProcessEventHandler('unhandledRejection', root, testResources);
+
+  process.on('uncaughtException', exceptionHandler);
+  process.on('unhandledRejection', rejectionHandler);
+  process.on('beforeExit', () => {
+    root.postRun();
+
+    let passCount = 0;
+    let failCount = 0;
+    let skipCount = 0;
+    let todoCount = 0;
+
+    for (let i = 0; i < root.subtests.length; i++) {
+      const test = root.subtests[i];
+
+      // Check SKIP and TODO tests first, as those should not be counted as
+      // failures.
+      if (test.skipped) {
+        skipCount++;
+      } else if (test.isTodo) {
+        todoCount++;
+      } else if (!test.passed) {
+        failCount++;
+      } else {
+        passCount++;
+      }
+    }
+
+    root.reporter.plan(root.indent, root.subtests.length);
+
+    for (let i = 0; i < root.diagnostics.length; i++) {
+      root.reporter.diagnostic(root.indent, root.diagnostics[i]);
+    }
+
+    root.reporter.diagnostic(root.indent, `tests ${root.subtests.length}`);
+    root.reporter.diagnostic(root.indent, `pass ${passCount}`);
+    root.reporter.diagnostic(root.indent, `fail ${failCount}`);
+    root.reporter.diagnostic(root.indent, `skipped ${skipCount}`);
+    root.reporter.diagnostic(root.indent, `todo ${todoCount}`);
+    root.reporter.diagnostic(root.indent, `duration_ms ${process.uptime()}`);
+
+    root.reporter.push(null);
+    hook.disable();
+    process.removeListener('unhandledRejection', rejectionHandler);
+    process.removeListener('uncaughtException', exceptionHandler);
+
+    if (failCount > 0) {
+      process.exitCode = 1;
+    }
+  });
+
+  root.reporter.pipe(process.stdout);
+  root.reporter.version();
+}
+
+function test(name, options, fn) {
+  // If this is the first test encountered, bootstrap the test harness.
+  if (this.subtests.length === 0) {
+    setup(this);
+  }
+
+  const subtest = this.createSubtest(name, options, fn);
+
+  return subtest.start();
+}
+
+const root = new Test({ name: '<root>' });
+
+module.exports = FunctionPrototypeBind(test, root);
diff --git a/lib/internal/test_runner/tap_stream.js b/lib/internal/test_runner/tap_stream.js
new file mode 100644
index 00000000000..d5e095991c7
--- /dev/null
+++ b/lib/internal/test_runner/tap_stream.js
@@ -0,0 +1,193 @@
+'use strict';
+const {
+  ArrayPrototypeForEach,
+  ArrayPrototypeJoin,
+  ArrayPrototypePush,
+  ArrayPrototypeShift,
+  ObjectEntries,
+  StringPrototypeReplace,
+  StringPrototypeReplaceAll,
+  StringPrototypeSplit,
+} = primordials;
+const Readable = require('internal/streams/readable');
+const { isError } = require('internal/util');
+const { inspect } = require('internal/util/inspect');
+const kFrameStartRegExp = /^ {4}at /;
+const kLineBreakRegExp = /\n|\r\n/;
+const inspectOptions = { colors: false, breakLength: Infinity };
+let testModule; // Lazy loaded due to circular dependency.
+
+function lazyLoadTest() {
+  testModule ??= require('internal/test_runner/test');
+
+  return testModule;
+}
+
+class TapStream extends Readable {
+  #buffer;
+  #canPush;
+
+  constructor() {
+    super();
+    this.#buffer = [];
+    this.#canPush = true;
+  }
+
+  _read() {
+    this.#canPush = true;
+
+    while (this.#buffer.length > 0) {
+      const line = ArrayPrototypeShift(this.#buffer);
+
+      if (!this.#tryPush(line)) {
+        return;
+      }
+    }
+  }
+
+  bail(message) {
+    this.#tryPush(`Bail out!${message ? ` ${tapEscape(message)}` : ''}\n`);
+  }
+
+  fail(indent, testNumber, description, directive) {
+    this.#test(indent, testNumber, 'not ok', description, directive);
+  }
+
+  ok(indent, testNumber, description, directive) {
+    this.#test(indent, testNumber, 'ok', description, directive);
+  }
+
+  plan(indent, count, explanation) {
+    const exp = `${explanation ? ` # ${tapEscape(explanation)}` : ''}`;
+
+    this.#tryPush(`${indent}1..${count}${exp}\n`);
+  }
+
+  getSkip(reason) {
+    return `SKIP${reason ? ` ${tapEscape(reason)}` : ''}`;
+  }
+
+  getTodo(reason) {
+    return `TODO${reason ? ` ${tapEscape(reason)}` : ''}`;
+  }
+
+  details(indent, duration, error) {
+    let details = `${indent}  ---\n`;
+
+    details += `${indent}  duration_ms: ${duration}\n`;
+
+    if (error !== null && typeof error === 'object') {
+      const entries = ObjectEntries(error);
+      const isErrorObj = isError(error);
+
+      for (let i = 0; i < entries.length; i++) {
+        const { 0: key, 1: value } = entries[i];
+
+        if (isError && (key === 'cause' || key === 'code')) {
+          continue;
+        }
+
+        details += `${indent}  ${key}: ${inspect(value, inspectOptions)}\n`;
+      }
+
+      if (isErrorObj) {
+        const { kTestCodeFailure } = lazyLoadTest();
+        const {
+          cause,
+          code,
+          failureType,
+          message,
+          stack,
+        } = error;
+        let errMsg = message ?? '<unknown error>';
+        let errStack = stack;
+        let errCode = code;
+
+        // If the ERR_TEST_FAILURE came from an error provided by user code,
+        // then try to unwrap the original error message and stack.
+        if (code === 'ERR_TEST_FAILURE' && failureType === kTestCodeFailure) {
+          errMsg = cause?.message ?? errMsg;
+          errStack = cause?.stack ?? errStack;
+          errCode = cause?.code ?? errCode;
+        }
+
+        details += `${indent}  error: ${inspect(errMsg, inspectOptions)}\n`;
+
+        if (errCode) {
+          details += `${indent}  code: ${errCode}\n`;
+        }
+
+        if (typeof errStack === 'string') {
+          const frames = [];
+
+          ArrayPrototypeForEach(
+            StringPrototypeSplit(errStack, kLineBreakRegExp),
+            (frame) => {
+              const processed = StringPrototypeReplace(
+                frame, kFrameStartRegExp, ''
+              );
+
+              if (processed.length > 0 && processed.length !== frame.length) {
+                ArrayPrototypePush(frames, processed);
+              }
+            }
+          );
+
+          if (frames.length > 0) {
+            const frameDelimiter = `\n${indent}    `;
+
+            details += `${indent}  stack: |-${frameDelimiter}`;
+            details += `${ArrayPrototypeJoin(frames, `${frameDelimiter}`)}\n`;
+          }
+        }
+      }
+    } else if (error !== null && error !== undefined) {
+      details += `${indent}  error: ${inspect(error, inspectOptions)}\n`;
+    }
+
+    details += `${indent}  ...\n`;
+    this.#tryPush(details);
+  }
+
+  diagnostic(indent, message) {
+    this.#tryPush(`${indent}# ${tapEscape(message)}\n`);
+  }
+
+  version() {
+    this.#tryPush('TAP version 13\n');
+  }
+
+  #test(indent, testNumber, status, description, directive) {
+    let line = `${indent}${status} ${testNumber}`;
+
+    if (description) {
+      line += ` ${tapEscape(description)}`;
+    }
+
+    if (directive) {
+      line += ` # ${directive}`;
+    }
+
+    line += '\n';
+    this.#tryPush(line);
+  }
+
+  #tryPush(message) {
+    if (this.#canPush) {
+      this.#canPush = this.push(message);
+    } else {
+      ArrayPrototypePush(this.#buffer, message);
+    }
+
+    return this.#canPush;
+  }
+}
+
+// In certain places, # and \ need to be escaped as \# and \\.
+function tapEscape(input) {
+  return StringPrototypeReplaceAll(
+    StringPrototypeReplaceAll(input, '\\', '\\\\'), '#', '\\#'
+  );
+}
+
+module.exports = { TapStream };
diff --git a/lib/internal/test_runner/test.js b/lib/internal/test_runner/test.js
new file mode 100644
index 00000000000..e2dddf5cc20
--- /dev/null
+++ b/lib/internal/test_runner/test.js
@@ -0,0 +1,426 @@
+'use strict';
+const {
+  ArrayPrototypePush,
+  ArrayPrototypeShift,
+  FunctionPrototype,
+  Number,
+  ObjectCreate,
+  SafeMap,
+} = primordials;
+const { AsyncResource } = require('async_hooks');
+const {
+  codes: {
+    ERR_TEST_FAILURE,
+  },
+} = require('internal/errors');
+const { TapStream } = require('internal/test_runner/tap_stream');
+const { createDeferredPromise } = require('internal/util');
+const { isPromise } = require('internal/util/types');
+const { isUint32 } = require('internal/validators');
+const { bigint: hrtime } = process.hrtime;
+const kCallbackAndPromisePresent = 'callbackAndPromisePresent';
+const kCancelledByParent = 'cancelledByParent';
+const kMultipleCallbackInvocations = 'multipleCallbackInvocations';
+const kParentAlreadyFinished = 'parentAlreadyFinished';
+const kSubtestsFailed = 'subtestsFailed';
+const kTestCodeFailure = 'testCodeFailure';
+const kDefaultIndent = '    ';
+const noop = FunctionPrototype;
+
+class TestContext {
+  #test;
+
+  constructor(test) {
+    this.#test = test;
+  }
+
+  diagnostic(message) {
+    this.#test.diagnostic(message);
+  }
+
+  skip(message) {
+    this.#test.skip(message);
+  }
+
+  todo(message) {
+    this.#test.todo(message);
+  }
+
+  test(name, options, fn) {
+    const subtest = this.#test.createSubtest(name, options, fn);
+
+    return subtest.start();
+  }
+}
+
+class Test extends AsyncResource {
+  constructor(options) {
+    super('Test');
+
+    let { fn, name, parent } = options;
+    const { concurrency, skip, todo } = options;
+
+    if (typeof fn !== 'function') {
+      fn = noop;
+    }
+
+    if (typeof name !== 'string' || name === '') {
+      name = fn.name || '<anonymous>';
+    }
+
+    if (!(parent instanceof Test)) {
+      parent = null;
+    }
+
+    if (skip) {
+      fn = noop;
+    }
+
+    this.fn = fn;
+    this.name = name;
+    this.parent = parent;
+
+    if (parent === null) {
+      this.concurrency = 1;
+      this.indent = '';
+      this.indentString = kDefaultIndent;
+      this.reporter = new TapStream();
+      this.testNumber = 0;
+    } else {
+      const indent = parent.parent === null ? parent.indent :
+        parent.indent + parent.indentString;
+
+      this.concurrency = parent.concurrency;
+      this.indent = indent;
+      this.indentString = parent.indentString;
+      this.reporter = parent.reporter;
+      this.testNumber = parent.subtests.length + 1;
+    }
+
+    if (isUint32(concurrency) && concurrency !== 0) {
+      this.concurrency = concurrency;
+    }
+
+    this.cancelled = false;
+    this.skipped = !!skip;
+    this.isTodo = !!todo;
+    this.startTime = null;
+    this.endTime = null;
+    this.passed = false;
+    this.error = null;
+    this.diagnostics = [];
+    this.message = typeof skip === 'string' ? skip :
+      typeof todo === 'string' ? todo : null;
+    this.activeSubtests = 0;
+    this.pendingSubtests = [];
+    this.readySubtests = new SafeMap();
+    this.subtests = [];
+    this.waitingOn = 0;
+    this.finished = false;
+  }
+
+  hasConcurrency() {
+    return this.concurrency > this.activeSubtests;
+  }
+
+  addPendingSubtest(deferred) {
+    this.pendingSubtests.push(deferred);
+  }
+
+  async processPendingSubtests() {
+    while (this.pendingSubtests.length > 0 && this.hasConcurrency()) {
+      const deferred = ArrayPrototypeShift(this.pendingSubtests);
+      await deferred.test.run();
+      deferred.resolve();
+    }
+  }
+
+  addReadySubtest(subtest) {
+    this.readySubtests.set(subtest.testNumber, subtest);
+  }
+
+  processReadySubtestRange(canSend) {
+    const start = this.waitingOn;
+    const end = start + this.readySubtests.size;
+
+    for (let i = start; i < end; i++) {
+      const subtest = this.readySubtests.get(i);
+
+      // Check if the specified subtest is in the map. If it is not, return
+      // early to avoid trying to process any more tests since they would be
+      // out of order.
+      if (subtest === undefined) {
+        return;
+      }
+
+      // Call isClearToSend() in the loop so that it is:
+      // - Only called if there are results to report in the correct order.
+      // - Guaranteed to only be called a maximum of once per call to
+      //   processReadySubtestRange().
+      canSend = canSend || this.isClearToSend();
+
+      if (!canSend) {
+        return;
+      }
+
+      // Report the subtest's results and remove it from the ready map.
+      subtest.finalize();
+      this.readySubtests.delete(i);
+    }
+  }
+
+  createSubtest(name, options, fn) {
+    if (typeof name === 'function') {
+      fn = name;
+    } else if (name !== null && typeof name === 'object') {
+      fn = options;
+      options = name;
+    } else if (typeof options === 'function') {
+      fn = options;
+    }
+
+    if (options === null || typeof options !== 'object') {
+      options = ObjectCreate(null);
+    }
+
+    let parent = this;
+
+    // If this test has already ended, attach this test to the root test so
+    // that the error can be properly reported.
+    if (this.finished) {
+      while (parent.parent !== null) {
+        parent = parent.parent;
+      }
+    }
+
+    const test = new Test({ fn, name, parent, ...options });
+
+    if (parent.waitingOn === 0) {
+      parent.waitingOn = test.testNumber;
+    }
+
+    if (this.finished) {
+      test.fail(
+        new ERR_TEST_FAILURE(
+          'test could not be started because its parent finished',
+          kParentAlreadyFinished
+        )
+      );
+    }
+
+    ArrayPrototypePush(parent.subtests, test);
+    return test;
+  }
+
+  cancel() {
+    if (this.endTime !== null) {
+      return;
+    }
+
+    this.fail(
+      new ERR_TEST_FAILURE(
+        'test did not finish before its parent and was cancelled',
+        kCancelledByParent
+      )
+    );
+    this.cancelled = true;
+  }
+
+  fail(err) {
+    if (this.error !== null) {
+      return;
+    }
+
+    this.endTime = hrtime();
+    this.passed = false;
+    this.error = err;
+  }
+
+  pass() {
+    if (this.endTime !== null) {
+      return;
+    }
+
+    this.endTime = hrtime();
+    this.passed = true;
+  }
+
+  skip(message) {
+    this.skipped = true;
+    this.message = message;
+  }
+
+  todo(message) {
+    this.isTodo = true;
+    this.message = message;
+  }
+
+  diagnostic(message) {
+    ArrayPrototypePush(this.diagnostics, message);
+  }
+
+  start() {
+    // If there is enough available concurrency to run the test now, then do
+    // it. Otherwise, return a Promise to the caller and mark the test as
+    // pending for later execution.
+    if (!this.parent.hasConcurrency()) {
+      const deferred = createDeferredPromise();
+
+      deferred.test = this;
+      this.parent.addPendingSubtest(deferred);
+      return deferred.promise;
+    }
+
+    return this.run();
+  }
+
+  async run() {
+    this.parent.activeSubtests++;
+    this.startTime = hrtime();
+
+    try {
+      const ctx = new TestContext(this);
+
+      if (this.fn.length === 2) {
+        // This test is using legacy Node.js error first callbacks.
+        const { promise, resolve, reject } = createDeferredPromise();
+        let calledCount = 0;
+        const ret = this.runInAsyncScope(this.fn, ctx, ctx, (err) => {
+          calledCount++;
+
+          // If the callback is called a second time, let the user know, but
+          // don't let them know more than once.
+          if (calledCount === 2) {
+            throw new ERR_TEST_FAILURE(
+              'callback invoked multiple times',
+              kMultipleCallbackInvocations
+            );
+          } else if (calledCount > 2) {
+            return;
+          }
+
+          if (err) {
+            return reject(err);
+          }
+
+          resolve();
+        });
+
+        if (isPromise(ret)) {
+          this.fail(new ERR_TEST_FAILURE(
+            'passed a callback but also returned a Promise',
+            kCallbackAndPromisePresent
+          ));
+          await ret;
+        } else {
+          await promise;
+        }
+      } else {
+        // This test is synchronous or using Promises.
+        await this.runInAsyncScope(this.fn, ctx, ctx);
+      }
+
+      this.pass();
+    } catch (err) {
+      this.fail(new ERR_TEST_FAILURE(err, kTestCodeFailure));
+    }
+
+    // Clean up the test. Then, try to report the results and execute any
+    // tests that were pending due to available concurrency.
+    this.postRun();
+  }
+
+  postRun() {
+    let failedSubtests = 0;
+
+    // If the test was failed before it even started, then the end time will
+    // be earlier than the start time. Correct that here.
+    if (this.endTime < this.startTime) {
+      this.endTime = hrtime();
+    }
+
+    // The test has run, so recursively cancel any outstanding subtests and
+    // mark this test as failed if any subtests failed.
+    for (let i = 0; i < this.subtests.length; i++) {
+      const subtest = this.subtests[i];
+
+      if (!subtest.finished) {
+        subtest.cancel();
+        subtest.postRun();
+      }
+
+      if (!subtest.passed) {
+        failedSubtests++;
+      }
+    }
+
+    if (this.passed && failedSubtests > 0) {
+      const subtestString = `subtest${failedSubtests > 1 ? 's' : ''}`;
+      const msg = `${failedSubtests} ${subtestString} failed`;
+
+      this.fail(new ERR_TEST_FAILURE(msg, kSubtestsFailed));
+    }
+
+    if (this.parent !== null) {
+      this.parent.activeSubtests--;
+      this.parent.addReadySubtest(this);
+      this.parent.processReadySubtestRange(false);
+      this.parent.processPendingSubtests();
+    }
+  }
+
+  isClearToSend() {
+    return this.parent === null ||
+      (
+        this.parent.waitingOn === this.testNumber && this.parent.isClearToSend()
+      );
+  }
+
+  finalize() {
+    // By the time this function is called, the following can be relied on:
+    // - The current test has completed or been cancelled.
+    // - All of this test's subtests have completed or been cancelled.
+    // - It is the current test's turn to report its results.
+
+    // Report any subtests that have not been reported yet. Since all of the
+    // subtests have finished, it's safe to pass true to
+    // processReadySubtestRange(), which will finalize all remaining subtests.
+    this.processReadySubtestRange(true);
+
+    // Output this test's results and update the parent's waiting counter.
+    if (this.subtests.length > 0) {
+      this.reporter.plan(this.subtests[0].indent, this.subtests.length);
+    }
+
+    this.report();
+    this.parent.waitingOn++;
+    this.finished = true;
+  }
+
+  report() {
+    // Duration is recorded in BigInt nanoseconds. Convert to seconds.
+    const duration = Number(this.endTime - this.startTime) / 1_000_000_000;
+    const message = `- ${this.name}`;
+    let directive;
+
+    if (this.skipped) {
+      directive = this.reporter.getSkip(this.message);
+    } else if (this.isTodo) {
+      directive = this.reporter.getTodo(this.message);
+    }
+
+    if (this.passed) {
+      this.reporter.ok(this.indent, this.testNumber, message, directive);
+    } else {
+      this.reporter.fail(this.indent, this.testNumber, message, directive);
+    }
+
+    this.reporter.details(this.indent, duration, this.error);
+
+    for (let i = 0; i < this.diagnostics.length; i++) {
+      this.reporter.diagnostic(this.indent, this.diagnostics[i]);
+    }
+  }
+}
+
+module.exports = { kDefaultIndent, kTestCodeFailure, Test };
diff --git a/lib/test.js b/lib/test.js
new file mode 100644
index 00000000000..fa319fa17b3
--- /dev/null
+++ b/lib/test.js
@@ -0,0 +1,8 @@
+'use strict';
+const test = require('internal/test_runner/harness');
+const { emitExperimentalWarning } = require('internal/util');
+
+emitExperimentalWarning('The test runner');
+
+module.exports = test;
+module.exports.test = test;