test_runner: add initial CLI runner

This commit introduces an initial version of a CLI-based
test runner.

PR-URL: https://github.com/nodejs/node/pull/42658
Reviewed-By: Antoine du Hamel <duhamelantoine1995@gmail.com>

12 files changed, 256 insertions, 42 deletions

diff --git a/test/fixtures/test-runner/index.js b/test/fixtures/test-runner/index.js
new file mode 100644
index 00000000000..fcf4b4d8eaa
--- /dev/null
+++ b/test/fixtures/test-runner/index.js
@@ -0,0 +1,2 @@
+'use strict';
+throw new Error('thrown from index.js');
diff --git a/test/fixtures/test-runner/index.test.js b/test/fixtures/test-runner/index.test.js
new file mode 100644
index 00000000000..2a722c504b9
--- /dev/null
+++ b/test/fixtures/test-runner/index.test.js
@@ -0,0 +1,4 @@
+'use strict';
+const test = require('node:test');
+
+test('this should pass');
diff --git a/test/fixtures/test-runner/node_modules/test-nm.js b/test/fixtures/test-runner/node_modules/test-nm.js
new file mode 100644
index 00000000000..30024eab1f1
--- /dev/null
+++ b/test/fixtures/test-runner/node_modules/test-nm.js
@@ -0,0 +1,2 @@
+'use strict';
+throw new Error('thrown from node_modules');
diff --git a/test/fixtures/test-runner/random.test.mjs b/test/fixtures/test-runner/random.test.mjs
new file mode 100644
index 00000000000..a87a671d006
--- /dev/null
+++ b/test/fixtures/test-runner/random.test.mjs
@@ -0,0 +1,5 @@
+import test from 'node:test';
+
+test('this should fail', () => {
+  throw new Error('this is a failing test');
+});
diff --git a/test/fixtures/test-runner/subdir/subdir_test.js b/test/fixtures/test-runner/subdir/subdir_test.js
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/test/fixtures/test-runner/subdir/subdir_test.js
diff --git a/test/fixtures/test-runner/test/random.cjs b/test/fixtures/test-runner/test/random.cjs
new file mode 100644
index 00000000000..2a722c504b9
--- /dev/null
+++ b/test/fixtures/test-runner/test/random.cjs
@@ -0,0 +1,4 @@
+'use strict';
+const test = require('node:test');
+
+test('this should pass');
diff --git a/test/message/test_runner_no_refs.out b/test/message/test_runner_no_refs.out
index 0379ff8ca74..c5407e3bd33 100644
--- a/test/message/test_runner_no_refs.out
+++ b/test/message/test_runner_no_refs.out
@@ -3,8 +3,8 @@ TAP version 13
       ---
       duration_ms: *
       failureType: 'cancelledByParent'
-      error: "'test did not finish before its parent and was cancelled'"
-      code: ERR_TEST_FAILURE
+      error: 'test did not finish before its parent and was cancelled'
+      code: 'ERR_TEST_FAILURE'
       stack: |-
         *
       ...
@@ -13,8 +13,8 @@ not ok 1 - does not keep event loop alive
   ---
   duration_ms: *
   failureType: 'cancelledByParent'
-  error: "'test did not finish before its parent and was cancelled'"
-  code: ERR_TEST_FAILURE
+  error: 'test did not finish before its parent and was cancelled'
+  code: 'ERR_TEST_FAILURE'
   stack: |-
     *
   ...
diff --git a/test/message/test_runner_output.js b/test/message/test_runner_output.js
index 2639c327966..d397f1ee7a2 100644
--- a/test/message/test_runner_output.js
+++ b/test/message/test_runner_output.js
@@ -3,6 +3,7 @@
 require('../common');
 const assert = require('node:assert');
 const test = require('node:test');
+const util = require('util');
 
 test('sync pass todo', (t) => {
   t.todo();
@@ -296,3 +297,25 @@ test('only is set but not in only mode', { only: true }, async (t) => {
   t.runOnly(false);
   await t.test('running subtest 4');
 });
+
+test('custom inspect symbol fail', () => {
+  const obj = {
+    [util.inspect.custom]() {
+      return 'customized';
+    },
+    foo: 1
+  };
+
+  throw obj;
+});
+
+test('custom inspect symbol that throws fail', () => {
+  const obj = {
+    [util.inspect.custom]() {
+      throw new Error('bad-inspect');
+    },
+    foo: 1
+  };
+
+  throw obj;
+});
diff --git a/test/message/test_runner_output.out b/test/message/test_runner_output.out
index bf42b4035f2..866b498deab 100644
--- a/test/message/test_runner_output.out
+++ b/test/message/test_runner_output.out
@@ -12,7 +12,7 @@ not ok 3 - sync fail todo # TODO
   duration_ms: *
   failureType: 'testCodeFailure'
   error: 'thrown from sync fail todo'
-  code: ERR_TEST_FAILURE
+  code: 'ERR_TEST_FAILURE'
   stack: |-
     *
     *
@@ -28,7 +28,7 @@ not ok 4 - sync fail todo with message # TODO this is a failing todo
   duration_ms: *
   failureType: 'testCodeFailure'
   error: 'thrown from sync fail todo with message'
-  code: ERR_TEST_FAILURE
+  code: 'ERR_TEST_FAILURE'
   stack: |-
     *
     *
@@ -59,7 +59,7 @@ not ok 8 - sync throw fail
   duration_ms: *
   failureType: 'testCodeFailure'
   error: 'thrown from sync throw fail'
-  code: ERR_TEST_FAILURE
+  code: 'ERR_TEST_FAILURE'
   stack: |-
     *
     *
@@ -83,7 +83,7 @@ not ok 11 - async throw fail
   duration_ms: *
   failureType: 'testCodeFailure'
   error: 'thrown from async throw fail'
-  code: ERR_TEST_FAILURE
+  code: 'ERR_TEST_FAILURE'
   stack: |-
     *
     *
@@ -99,7 +99,7 @@ not ok 12 - async skip fail # SKIP
   duration_ms: *
   failureType: 'testCodeFailure'
   error: 'thrown from async throw fail'
-  code: ERR_TEST_FAILURE
+  code: 'ERR_TEST_FAILURE'
   stack: |-
     *
     *
@@ -114,8 +114,12 @@ not ok 13 - async assertion fail
   ---
   duration_ms: *
   failureType: 'testCodeFailure'
-  error: 'Expected values to be strictly equal:\n\ntrue !== false\n'
-  code: ERR_ASSERTION
+  error: |-
+    Expected values to be strictly equal:
+
+    true !== false
+
+  code: 'ERR_ASSERTION'
   stack: |-
     *
     *
@@ -135,7 +139,7 @@ not ok 15 - reject fail
   duration_ms: *
   failureType: 'testCodeFailure'
   error: 'rejected from reject fail'
-  code: ERR_TEST_FAILURE
+  code: 'ERR_TEST_FAILURE'
   stack: |-
     *
     *
@@ -171,7 +175,7 @@ ok 20 - immediate resolve pass
       duration_ms: *
       failureType: 'testCodeFailure'
       error: 'thrown from subtest sync throw fail'
-      code: ERR_TEST_FAILURE
+      code: 'ERR_TEST_FAILURE'
       stack: |-
         *
         *
@@ -190,15 +194,15 @@ not ok 21 - subtest sync throw fail
   ---
   duration_ms: *
   failureType: 'subtestsFailed'
-  error: "'1 subtest failed'"
-  code: ERR_TEST_FAILURE
+  error: '1 subtest failed'
+  code: 'ERR_TEST_FAILURE'
   ...
 not ok 22 - sync throw non-error fail
   ---
   duration_ms: *
   failureType: 'testCodeFailure'
   error: 'Symbol(thrown symbol from sync throw non-error fail)'
-  code: ERR_TEST_FAILURE
+  code: 'ERR_TEST_FAILURE'
   ...
     ok 1 - level 1a
       ---
@@ -225,8 +229,8 @@ ok 23 - level 0a
       ---
       duration_ms: *
       failureType: 'cancelledByParent'
-      error: "'test did not finish before its parent and was cancelled'"
-      code: ERR_TEST_FAILURE
+      error: 'test did not finish before its parent and was cancelled'
+      code: 'ERR_TEST_FAILURE'
       ...
         ok 1 - ++short running
           ---
@@ -242,8 +246,8 @@ not ok 24 - top level
   ---
   duration_ms: *
   failureType: 'subtestsFailed'
-  error: "'1 subtest failed'"
-  code: ERR_TEST_FAILURE
+  error: '1 subtest failed'
+  code: 'ERR_TEST_FAILURE'
   ...
 ok 25 - invalid subtest - pass but subtest fails
   ---
@@ -262,7 +266,7 @@ not ok 28 - sync skip option is false fail
   duration_ms: *
   failureType: 'testCodeFailure'
   error: 'this should be executed'
-  code: ERR_TEST_FAILURE
+  code: 'ERR_TEST_FAILURE'
   stack: |-
     *
     *
@@ -330,7 +334,7 @@ not ok 42 - callback fail
   duration_ms: *
   failureType: 'testCodeFailure'
   error: 'callback failure'
-  code: ERR_TEST_FAILURE
+  code: 'ERR_TEST_FAILURE'
   stack: |-
     *
     *
@@ -351,15 +355,15 @@ not ok 46 - callback also returns a Promise
   ---
   duration_ms: *
   failureType: 'callbackAndPromisePresent'
-  error: "'passed a callback but also returned a Promise'"
-  code: ERR_TEST_FAILURE
+  error: 'passed a callback but also returned a Promise'
+  code: 'ERR_TEST_FAILURE'
   ...
 not ok 47 - callback throw
   ---
   duration_ms: *
   failureType: 'testCodeFailure'
   error: 'thrown from callback throw'
-  code: ERR_TEST_FAILURE
+  code: 'ERR_TEST_FAILURE'
   stack: |-
     *
     *
@@ -373,8 +377,11 @@ not ok 48 - callback called twice
   ---
   duration_ms: *
   failureType: 'multipleCallbackInvocations'
-  error: "'callback invoked multiple times'"
-  code: ERR_TEST_FAILURE
+  error: 'callback invoked multiple times'
+  code: 'ERR_TEST_FAILURE'
+  stack: |-
+    *
+    *
   ...
 ok 49 - callback called twice in different ticks
   ---
@@ -383,9 +390,9 @@ ok 49 - callback called twice in different ticks
 not ok 50 - callback called twice in future tick
   ---
   duration_ms: *
-  failureType: 'multipleCallbackInvocations'
-  error: "'callback invoked multiple times'"
-  code: ERR_TEST_FAILURE
+  failureType: 'uncaughtException'
+  error: 'callback invoked multiple times'
+  code: 'ERR_TEST_FAILURE'
   stack: |-
     *
   ...
@@ -394,7 +401,7 @@ not ok 51 - callback async throw
   duration_ms: *
   failureType: 'uncaughtException'
   error: 'thrown from callback async throw'
-  code: ERR_TEST_FAILURE
+  code: 'ERR_TEST_FAILURE'
   stack: |-
     *
   ...
@@ -423,25 +430,43 @@ ok 53 - only is set but not in only mode
   ---
   duration_ms: *
   ...
-not ok 54 - invalid subtest fail
+not ok 54 - custom inspect symbol fail
+  ---
+  duration_ms: *
+  failureType: 'testCodeFailure'
+  error: 'customized'
+  code: 'ERR_TEST_FAILURE'
+  ...
+not ok 55 - custom inspect symbol that throws fail
+  ---
+  duration_ms: *
+  failureType: 'testCodeFailure'
+  error: |-
+    {
+      foo: 1,
+      [Symbol(nodejs.util.inspect.custom)]: [Function: [nodejs.util.inspect.custom]]
+    }
+  code: 'ERR_TEST_FAILURE'
+  ...
+not ok 56 - invalid subtest fail
   ---
   duration_ms: *
   failureType: 'parentAlreadyFinished'
-  error: "'test could not be started because its parent finished'"
-  code: ERR_TEST_FAILURE
+  error: 'test could not be started because its parent finished'
+  code: 'ERR_TEST_FAILURE'
   stack: |-
     *
   ...
-1..54
+1..56
 # Warning: Test "unhandled rejection - passes but warns" generated asynchronous activity after the test ended. This activity created the error "Error: rejected from unhandled rejection fail" and would have caused the test to fail, but instead triggered an unhandledRejection event.
 # Warning: Test "async unhandled rejection - passes but warns" generated asynchronous activity after the test ended. This activity created the error "Error: rejected from async unhandled rejection fail" and would have caused the test to fail, but instead triggered an unhandledRejection event.
 # Warning: Test "immediate throw - passes but warns" generated asynchronous activity after the test ended. This activity created the error "Error: thrown from immediate throw fail" and would have caused the test to fail, but instead triggered an uncaughtException event.
 # Warning: Test "immediate reject - passes but warns" generated asynchronous activity after the test ended. This activity created the error "Error: rejected from immediate reject fail" and would have caused the test to fail, but instead triggered an unhandledRejection event.
-# Warning: Test "callback called twice in different ticks" generated asynchronous activity after the test ended. This activity created the error "Error [ERR_TEST_FAILURE]: 'callback invoked multiple times'" and would have caused the test to fail, but instead triggered an uncaughtException event.
+# Warning: Test "callback called twice in different ticks" generated asynchronous activity after the test ended. This activity created the error "Error [ERR_TEST_FAILURE]: callback invoked multiple times" and would have caused the test to fail, but instead triggered an uncaughtException event.
 # Warning: Test "callback async throw after done" generated asynchronous activity after the test ended. This activity created the error "Error: thrown from callback async throw after done" and would have caused the test to fail, but instead triggered an uncaughtException event.
-# tests 54
+# tests 56
 # pass 24
-# fail 15
+# fail 17
 # skipped 10
 # todo 5
 # duration_ms *
diff --git a/test/message/test_runner_unresolved_promise.out b/test/message/test_runner_unresolved_promise.out
index 263b2411c85..98f52966c33 100644
--- a/test/message/test_runner_unresolved_promise.out
+++ b/test/message/test_runner_unresolved_promise.out
@@ -7,8 +7,8 @@ not ok 2 - never resolving promise
   ---
   duration_ms: *
   failureType: 'cancelledByParent'
-  error: "'test did not finish before its parent and was cancelled'"
-  code: ERR_TEST_FAILURE
+  error: 'test did not finish before its parent and was cancelled'
+  code: 'ERR_TEST_FAILURE'
   stack: |-
     *
   ...
@@ -16,8 +16,8 @@ not ok 3 - fail
   ---
   duration_ms: *
   failureType: 'cancelledByParent'
-  error: "'test did not finish before its parent and was cancelled'"
-  code: ERR_TEST_FAILURE
+  error: 'test did not finish before its parent and was cancelled'
+  code: 'ERR_TEST_FAILURE'
   stack: |-
     *
   ...
diff --git a/test/parallel/test-runner-cli.js b/test/parallel/test-runner-cli.js
new file mode 100644
index 00000000000..8d16205cdaf
--- /dev/null
+++ b/test/parallel/test-runner-cli.js
@@ -0,0 +1,107 @@
+'use strict';
+require('../common');
+const assert = require('assert');
+const { spawnSync } = require('child_process');
+const { join } = require('path');
+const fixtures = require('../common/fixtures');
+const testFixtures = fixtures.path('test-runner');
+
+{
+  // File not found.
+  const args = ['--test', 'a-random-file-that-does-not-exist.js'];
+  const child = spawnSync(process.execPath, args);
+
+  assert.strictEqual(child.status, 1);
+  assert.strictEqual(child.signal, null);
+  assert.strictEqual(child.stdout.toString(), '');
+  assert(/^Could not find/.test(child.stderr.toString()));
+}
+
+{
+  // Default behavior. node_modules is ignored. Files that don't match the
+  // pattern are ignored except in test/ directories.
+  const args = ['--test', testFixtures];
+  const child = spawnSync(process.execPath, args);
+
+  assert.strictEqual(child.status, 1);
+  assert.strictEqual(child.signal, null);
+  assert.strictEqual(child.stderr.toString(), '');
+  const stdout = child.stdout.toString();
+  assert(/ok 1 - .+index\.test\.js/.test(stdout));
+  assert(/not ok 2 - .+random\.test\.mjs/.test(stdout));
+  assert(/not ok 1 - this should fail/.test(stdout));
+  assert(/ok 3 - .+subdir.+subdir_test\.js/.test(stdout));
+  assert(/ok 4 - .+random\.cjs/.test(stdout));
+}
+
+{
+  // User specified files that don't match the pattern are still run.
+  const args = ['--test', testFixtures, join(testFixtures, 'index.js')];
+  const child = spawnSync(process.execPath, args);
+
+  assert.strictEqual(child.status, 1);
+  assert.strictEqual(child.signal, null);
+  assert.strictEqual(child.stderr.toString(), '');
+  const stdout = child.stdout.toString();
+  assert(/not ok 1 - .+index\.js/.test(stdout));
+  assert(/ok 2 - .+index\.test\.js/.test(stdout));
+  assert(/not ok 3 - .+random\.test\.mjs/.test(stdout));
+  assert(/not ok 1 - this should fail/.test(stdout));
+  assert(/ok 4 - .+subdir.+subdir_test\.js/.test(stdout));
+}
+
+{
+  // Searches node_modules if specified.
+  const args = ['--test', join(testFixtures, 'node_modules')];
+  const child = spawnSync(process.execPath, args);
+
+  assert.strictEqual(child.status, 1);
+  assert.strictEqual(child.signal, null);
+  assert.strictEqual(child.stderr.toString(), '');
+  const stdout = child.stdout.toString();
+  assert(/not ok 1 - .+test-nm\.js/.test(stdout));
+}
+
+{
+  // The current directory is used by default.
+  const args = ['--test'];
+  const options = { cwd: testFixtures };
+  const child = spawnSync(process.execPath, args, options);
+
+  assert.strictEqual(child.status, 1);
+  assert.strictEqual(child.signal, null);
+  assert.strictEqual(child.stderr.toString(), '');
+  const stdout = child.stdout.toString();
+  assert(/ok 1 - .+index\.test\.js/.test(stdout));
+  assert(/not ok 2 - .+random\.test\.mjs/.test(stdout));
+  assert(/not ok 1 - this should fail/.test(stdout));
+  assert(/ok 3 - .+subdir.+subdir_test\.js/.test(stdout));
+  assert(/ok 4 - .+random\.cjs/.test(stdout));
+}
+
+{
+  // Flags that cannot be combined with --test.
+  const flags = [
+    ['--check', '--test'],
+    ['--interactive', '--test'],
+    ['--eval', 'console.log("should not print")', '--test'],
+    ['--print', 'console.log("should not print")', '--test'],
+  ];
+
+  if (process.features.inspector) {
+    flags.push(
+      ['--inspect', '--test'],
+      ['--inspect-brk', '--test'],
+    );
+  }
+
+  flags.forEach((args) => {
+    const child = spawnSync(process.execPath, args);
+
+    assert.notStrictEqual(child.status, 0);
+    assert.strictEqual(child.signal, null);
+    assert.strictEqual(child.stdout.toString(), '');
+    const stderr = child.stderr.toString();
+    assert(/--test/.test(stderr));
+  });
+}
diff --git a/test/parallel/test-runner-test-filter.js b/test/parallel/test-runner-test-filter.js
new file mode 100644
index 00000000000..b6afba22a2e
--- /dev/null
+++ b/test/parallel/test-runner-test-filter.js
@@ -0,0 +1,42 @@
+// Flags: --expose-internals
+'use strict';
+require('../common');
+const assert = require('assert');
+const { doesPathMatchFilter } = require('internal/test_runner/utils');
+
+// Paths expected to match
+[
+  'test.js',
+  'test.cjs',
+  'test.mjs',
+  'test-foo.js',
+  'test-foo.cjs',
+  'test-foo.mjs',
+  'foo.test.js',
+  'foo.test.cjs',
+  'foo.test.mjs',
+  'foo-test.js',
+  'foo-test.cjs',
+  'foo-test.mjs',
+  'foo_test.js',
+  'foo_test.cjs',
+  'foo_test.mjs',
+].forEach((p) => {
+  assert.strictEqual(doesPathMatchFilter(p), true);
+});
+
+// Paths expected not to match
+[
+  'test',
+  'test.djs',
+  'test.cs',
+  'test.mj',
+  'foo.js',
+  'test-foo.sj',
+  'test.foo.js',
+  'test_foo.js',
+  'testfoo.js',
+  'foo-test1.mjs',
+].forEach((p) => {
+  assert.strictEqual(doesPathMatchFilter(p), false);
+});