diff options
author | Colin Ihrig <cjihrig@gmail.com> | 2022-04-15 20:37:28 +0300 |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-04-15 20:37:28 +0300 |
commit | adaf60240559ffb58636130950262ee3237b7a41 (patch) | |
tree | 0fb45518c5a7a2c8232197a10d52db1de95cec46 | |
parent | 24adba675179ebba363d46f5dd30685d58cdb7f4 (diff) |
test_runner: add initial CLI runner
This commit introduces an initial version of a CLI-based
test runner.
PR-URL: https://github.com/nodejs/node/pull/42658
Reviewed-By: Antoine du Hamel <duhamelantoine1995@gmail.com>
23 files changed, 669 insertions, 128 deletions
diff --git a/doc/api/cli.md b/doc/api/cli.md index 92acc87e648..2e3ef59d3d2 100644 --- a/doc/api/cli.md +++ b/doc/api/cli.md @@ -1052,6 +1052,16 @@ minimum allocation from the secure heap. The minimum value is `2`. The maximum value is the lesser of `--secure-heap` or `2147483647`. The value given must be a power of two. +### `--test` + +<!-- YAML +added: REPLACEME +--> + +Starts the Node.js command line test runner. This flag cannot be combined with +`--check`, `--eval`, `--interactive`, or the inspector. See the documentation +on [running tests from the command line][] for more details. + ### `--test-only` <!-- YAML @@ -2033,6 +2043,7 @@ $ node --max-old-space-size=1536 index.js [jitless]: https://v8.dev/blog/jitless [libuv threadpool documentation]: https://docs.libuv.org/en/latest/threadpool.html [remote code execution]: https://www.owasp.org/index.php/Code_Injection +[running tests from the command line]: test.md#running-tests-from-the-command-line [security warning]: #warning-binding-inspector-to-a-public-ipport-combination-is-insecure [timezone IDs]: https://en.wikipedia.org/wiki/List_of_tz_database_time_zones [ways that `TZ` is handled in other environments]: https://www.gnu.org/software/libc/manual/html_node/TZ-Variable.html diff --git a/doc/api/test.md b/doc/api/test.md index 9ce8b731a46..38e74aaa01d 100644 --- a/doc/api/test.md +++ b/doc/api/test.md @@ -219,6 +219,67 @@ test('a test that creates asynchronous activity', (t) => { }); ``` +## Running tests from the command line + +The Node.js test runner can be invoked from the command line by passing the +[`--test`][] flag: + +```bash +node --test +``` + +By default, Node.js will recursively search the current directory for +JavaScript source files matching a specific naming convention. Matching files +are executed as test files. More information on the expected test file naming +convention and behavior can be found in the [test runner execution model][] +section. + +Alternatively, one or more paths can be provided as the final argument(s) to +the Node.js command, as shown below. + +```bash +node --test test1.js test2.mjs custom_test_dir/ +``` + +In this example, the test runner will execute the files `test1.js` and +`test2.mjs`. The test runner will also recursively search the +`custom_test_dir/` directory for test files to execute. + +### Test runner execution model + +When searching for test files to execute, the test runner behaves as follows: + +* Any files explicitly provided by the user are executed. +* If the user did not explicitly specify any paths, the current working + directory is recursively searched for files as specified in the following + steps. +* `node_modules` directories are skipped unless explicitly provided by the + user. +* If a directory named `test` is encountered, the test runner will search it + recursively for all all `.js`, `.cjs`, and `.mjs` files. All of these files + are treated as test files, and do not need to match the specific naming + convention detailed below. This is to accommodate projects that place all of + their tests in a single `test` directory. +* In all other directories, `.js`, `.cjs`, and `.mjs` files matching the + following patterns are treated as test files: + * `^test$` - Files whose basename is the string `'test'`. Examples: + `test.js`, `test.cjs`, `test.mjs`. + * `^test-.+` - Files whose basename starts with the string `'test-'` + followed by one or more characters. Examples: `test-example.js`, + `test-another-example.mjs`. + * `.+[\.\-\_]test$` - Files whose basename ends with `.test`, `-test`, or + `_test`, preceded by one or more characters. Examples: `example.test.js`, + `example-test.cjs`, `example_test.mjs`. + * Other file types understood by Node.js such as `.node` and `.json` are not + automatically executed by the test runner, but are supported if explicitly + provided on the command line. + +Each matching test file is executed in a separate child process. If the child +process finishes with an exit code of 0, the test is considered passing. +Otherwise, the test is considered to be a failure. Test files must be +executable by Node.js, but are not required to use the `node:test` module +internally. + ## `test([name][, options][, fn])` <!-- YAML @@ -368,5 +429,7 @@ behaves in the same fashion as the top level [`test()`][] function. [TAP]: https://testanything.org/ [`--test-only`]: cli.md#--test-only +[`--test`]: cli.md#--test [`TestContext`]: #class-testcontext [`test()`]: #testname-options-fn +[test runner execution model]: #test-runner-execution-model diff --git a/doc/node.1 b/doc/node.1 index bd3550625a7..f404747dcb8 100644 --- a/doc/node.1 +++ b/doc/node.1 @@ -381,6 +381,9 @@ the secure heap. The default is 0. The value must be a power of two. .It Fl -secure-heap-min Ns = Ns Ar n Specify the minimum allocation from the OpenSSL secure heap. The default is 2. The value must be a power of two. . +.It Fl -test +Starts the Node.js command line test runner. +. .It Fl -test-only Configures the test runner to only execute top level tests that have the `only` option set. diff --git a/lib/internal/errors.js b/lib/internal/errors.js index a5c64080a59..434576bde16 100644 --- a/lib/internal/errors.js +++ b/lib/internal/errors.js @@ -207,6 +207,16 @@ function isErrorStackTraceLimitWritable() { desc.set !== undefined; } +function inspectWithNoCustomRetry(obj, options) { + const utilInspect = lazyInternalUtilInspect(); + + try { + return utilInspect.inspect(obj, options); + } catch { + return utilInspect.inspect(obj, { ...options, customInspect: false }); + } +} + // A specialized Error that includes an additional info property with // additional information about the error condition. // It has the properties present in a UVException but with a custom error @@ -862,6 +872,7 @@ module.exports = { getMessage, hideInternalStackFrames, hideStackFrames, + inspectWithNoCustomRetry, isErrorStackTraceLimitWritable, isStackOverflowError, kEnhanceStackBeforeInspector, @@ -1549,11 +1560,14 @@ E('ERR_TEST_FAILURE', function(error, failureType) { assert(typeof failureType === 'string', "The 'failureType' argument must be of type string."); - const msg = error?.message ?? lazyInternalUtilInspect().inspect(error); + let msg = error?.message ?? error; - this.failureType = error?.failureType ?? failureType; - this.cause = error; + if (typeof msg !== 'string') { + msg = inspectWithNoCustomRetry(msg); + } + this.failureType = failureType; + this.cause = error; return msg; }, Error); E('ERR_TLS_CERT_ALTNAME_FORMAT', 'Invalid subject alternative name string', diff --git a/lib/internal/main/test_runner.js b/lib/internal/main/test_runner.js new file mode 100644 index 00000000000..71bf21782f3 --- /dev/null +++ b/lib/internal/main/test_runner.js @@ -0,0 +1,157 @@ +'use strict'; +const { + ArrayFrom, + ArrayPrototypeFilter, + ArrayPrototypeIncludes, + ArrayPrototypePush, + ArrayPrototypeSlice, + ArrayPrototypeSort, + Promise, + SafeSet, +} = primordials; +const { + prepareMainThreadExecution, +} = require('internal/bootstrap/pre_execution'); +const { spawn } = require('child_process'); +const { readdirSync, statSync } = require('fs'); +const console = require('internal/console/global'); +const { + codes: { + ERR_TEST_FAILURE, + }, +} = require('internal/errors'); +const test = require('internal/test_runner/harness'); +const { kSubtestsFailed } = require('internal/test_runner/test'); +const { + isSupportedFileType, + doesPathMatchFilter, +} = require('internal/test_runner/utils'); +const { basename, join, resolve } = require('path'); +const kFilterArgs = ['--test']; + +prepareMainThreadExecution(false); +markBootstrapComplete(); + +// TODO(cjihrig): Replace this with recursive readdir once it lands. +function processPath(path, testFiles, options) { + const stats = statSync(path); + + if (stats.isFile()) { + if (options.userSupplied || + (options.underTestDir && isSupportedFileType(path)) || + doesPathMatchFilter(path)) { + testFiles.add(path); + } + } else if (stats.isDirectory()) { + const name = basename(path); + + if (!options.userSupplied && name === 'node_modules') { + return; + } + + // 'test' directories get special treatment. Recursively add all .js, + // .cjs, and .mjs files in the 'test' directory. + const isTestDir = name === 'test'; + const { underTestDir } = options; + const entries = readdirSync(path); + + if (isTestDir) { + options.underTestDir = true; + } + + options.userSupplied = false; + + for (let i = 0; i < entries.length; i++) { + processPath(join(path, entries[i]), testFiles, options); + } + + options.underTestDir = underTestDir; + } +} + +function createTestFileList() { + const cwd = process.cwd(); + const hasUserSuppliedPaths = process.argv.length > 1; + const testPaths = hasUserSuppliedPaths ? + ArrayPrototypeSlice(process.argv, 1) : [cwd]; + const testFiles = new SafeSet(); + + try { + for (let i = 0; i < testPaths.length; i++) { + const absolutePath = resolve(testPaths[i]); + + processPath(absolutePath, testFiles, { userSupplied: true }); + } + } catch (err) { + if (err?.code === 'ENOENT') { + console.error(`Could not find '${err.path}'`); + process.exit(1); + } + + throw err; + } + + return ArrayPrototypeSort(ArrayFrom(testFiles)); +} + +function filterExecArgv(arg) { + return !ArrayPrototypeIncludes(kFilterArgs, arg); +} + +function runTestFile(path) { + return test(path, () => { + return new Promise((resolve, reject) => { + const args = ArrayPrototypeFilter(process.execArgv, filterExecArgv); + ArrayPrototypePush(args, path); + + const child = spawn(process.execPath, args); + // TODO(cjihrig): Implement a TAP parser to read the child's stdout + // instead of just displaying it all if the child fails. + let stdout = ''; + let stderr = ''; + let err; + + child.on('error', (error) => { + err = error; + }); + + child.stdout.setEncoding('utf8'); + child.stderr.setEncoding('utf8'); + + child.stdout.on('data', (chunk) => { + stdout += chunk; + }); + + child.stderr.on('data', (chunk) => { + stderr += chunk; + }); + + child.once('exit', (code, signal) => { + if (code !== 0 || signal !== null) { + if (!err) { + err = new ERR_TEST_FAILURE('test failed', kSubtestsFailed); + err.exitCode = code; + err.signal = signal; + err.stdout = stdout; + err.stderr = stderr; + // The stack will not be useful since the failures came from tests + // in a child process. + err.stack = undefined; + } + + return reject(err); + } + + resolve(); + }); + }); + }); +} + +(async function main() { + const testFiles = createTestFileList(); + + for (let i = 0; i < testFiles.length; i++) { + runTestFile(testFiles[i]); + } +})(); diff --git a/lib/internal/test_runner/tap_stream.js b/lib/internal/test_runner/tap_stream.js index d5e095991c7..a6bfbb3367c 100644 --- a/lib/internal/test_runner/tap_stream.js +++ b/lib/internal/test_runner/tap_stream.js @@ -5,13 +5,13 @@ const { ArrayPrototypePush, ArrayPrototypeShift, ObjectEntries, - StringPrototypeReplace, StringPrototypeReplaceAll, StringPrototypeSplit, + RegExpPrototypeSymbolReplace, } = primordials; +const { inspectWithNoCustomRetry } = require('internal/errors'); const Readable = require('internal/streams/readable'); const { isError } = require('internal/util'); -const { inspect } = require('internal/util/inspect'); const kFrameStartRegExp = /^ {4}at /; const kLineBreakRegExp = /\n|\r\n/; const inspectOptions = { colors: false, breakLength: Infinity }; @@ -74,77 +74,8 @@ class TapStream extends Readable { details(indent, duration, error) { let details = `${indent} ---\n`; - details += `${indent} duration_ms: ${duration}\n`; - - if (error !== null && typeof error === 'object') { - const entries = ObjectEntries(error); - const isErrorObj = isError(error); - - for (let i = 0; i < entries.length; i++) { - const { 0: key, 1: value } = entries[i]; - - if (isError && (key === 'cause' || key === 'code')) { - continue; - } - - details += `${indent} ${key}: ${inspect(value, inspectOptions)}\n`; - } - - if (isErrorObj) { - const { kTestCodeFailure } = lazyLoadTest(); - const { - cause, - code, - failureType, - message, - stack, - } = error; - let errMsg = message ?? '<unknown error>'; - let errStack = stack; - let errCode = code; - - // If the ERR_TEST_FAILURE came from an error provided by user code, - // then try to unwrap the original error message and stack. - if (code === 'ERR_TEST_FAILURE' && failureType === kTestCodeFailure) { - errMsg = cause?.message ?? errMsg; - errStack = cause?.stack ?? errStack; - errCode = cause?.code ?? errCode; - } - - details += `${indent} error: ${inspect(errMsg, inspectOptions)}\n`; - - if (errCode) { - details += `${indent} code: ${errCode}\n`; - } - - if (typeof errStack === 'string') { - const frames = []; - - ArrayPrototypeForEach( - StringPrototypeSplit(errStack, kLineBreakRegExp), - (frame) => { - const processed = StringPrototypeReplace( - frame, kFrameStartRegExp, '' - ); - - if (processed.length > 0 && processed.length !== frame.length) { - ArrayPrototypePush(frames, processed); - } - } - ); - - if (frames.length > 0) { - const frameDelimiter = `\n${indent} `; - - details += `${indent} stack: |-${frameDelimiter}`; - details += `${ArrayPrototypeJoin(frames, `${frameDelimiter}`)}\n`; - } - } - } - } else if (error !== null && error !== undefined) { - details += `${indent} error: ${inspect(error, inspectOptions)}\n`; - } - + details += jsToYaml(indent, 'duration_ms', duration); + details += jsToYaml(indent, null, error); details += `${indent} ...\n`; this.#tryPush(details); } @@ -190,4 +121,100 @@ function tapEscape(input) { ); } +function jsToYaml(indent, name, value) { + if (value === null || value === undefined) { + return ''; + } + + if (typeof value !== 'object') { + const prefix = `${indent} ${name}: `; + + if (typeof value !== 'string') { + return `${prefix}${inspectWithNoCustomRetry(value, inspectOptions)}\n`; + } + + const lines = StringPrototypeSplit(value, kLineBreakRegExp); + + if (lines.length === 1) { + return `${prefix}${inspectWithNoCustomRetry(value, inspectOptions)}\n`; + } + + let str = `${prefix}|-\n`; + + for (let i = 0; i < lines.length; i++) { + str += `${indent} ${lines[i]}\n`; + } + + return str; + } + + const entries = ObjectEntries(value); + const isErrorObj = isError(value); + let result = ''; + + for (let i = 0; i < entries.length; i++) { + const { 0: key, 1: value } = entries[i]; + + if (isErrorObj && (key === 'cause' || key === 'code')) { + continue; + } + + result += jsToYaml(indent, key, value); + } + + if (isErrorObj) { + const { kTestCodeFailure } = lazyLoadTest(); + const { + cause, + code, + failureType, + message, + stack, + } = value; + let errMsg = message ?? '<unknown error>'; + let errStack = stack; + let errCode = code; + + // If the ERR_TEST_FAILURE came from an error provided by user code, + // then try to unwrap the original error message and stack. + if (code === 'ERR_TEST_FAILURE' && failureType === kTestCodeFailure) { + errMsg = cause?.message ?? errMsg; + errStack = cause?.stack ?? errStack; + errCode = cause?.code ?? errCode; + } + + result += jsToYaml(indent, 'error', errMsg); + + if (errCode) { + result += jsToYaml(indent, 'code', errCode); + } + + if (typeof errStack === 'string') { + const frames = []; + + ArrayPrototypeForEach( + StringPrototypeSplit(errStack, kLineBreakRegExp), + (frame) => { + const processed = RegExpPrototypeSymbolReplace( + kFrameStartRegExp, frame, '' + ); + + if (processed.length > 0 && processed.length !== frame.length) { + ArrayPrototypePush(frames, processed); + } + } + ); + + if (frames.length > 0) { + const frameDelimiter = `\n${indent} `; + + result += `${indent} stack: |-${frameDelimiter}`; + result += `${ArrayPrototypeJoin(frames, `${frameDelimiter}`)}\n`; + } + } + } + + return result; +} + module.exports = { TapStream }; diff --git a/lib/internal/test_runner/test.js b/lib/internal/test_runner/test.js index 3e632cc1c52..d35bf6307bf 100644 --- a/lib/internal/test_runner/test.js +++ b/lib/internal/test_runner/test.js @@ -12,12 +12,14 @@ const { codes: { ERR_TEST_FAILURE, }, + kIsNodeError, } = require('internal/errors'); const { getOptionValue } = require('internal/options'); const { TapStream } = require('internal/test_runner/tap_stream'); const { createDeferredPromise } = require('internal/util'); const { isPromise } = require('internal/util/types'); const { isUint32 } = require('internal/validators'); +const { cpus } = require('os'); const { bigint: hrtime } = process.hrtime; const kCallbackAndPromisePresent = 'callbackAndPromisePresent'; const kCancelledByParent = 'cancelledByParent'; @@ -27,7 +29,10 @@ const kSubtestsFailed = 'subtestsFailed'; const kTestCodeFailure = 'testCodeFailure'; const kDefaultIndent = ' '; const noop = FunctionPrototype; -const testOnlyFlag = getOptionValue('--test-only'); +const isTestRunner = getOptionValue('--test'); +const testOnlyFlag = !isTestRunner && getOptionValue('--test-only'); +// TODO(cjihrig): Use uv_available_parallelism() once it lands. +const rootConcurrency = isTestRunner ? cpus().length : 1; class TestContext { #test; @@ -79,7 +84,7 @@ class Test extends AsyncResource { } if (parent === null) { - this.concurrency = 1; + this.concurrency = rootConcurrency; this.indent = ''; this.indentString = kDefaultIndent; this.only = testOnlyFlag; @@ -303,12 +308,14 @@ class Test extends AsyncResource { // If the callback is called a second time, let the user know, but // don't let them know more than once. - if (calledCount === 2) { - throw new ERR_TEST_FAILURE( - 'callback invoked multiple times', - kMultipleCallbackInvocations - ); - } else if (calledCount > 2) { + if (calledCount > 1) { + if (calledCount === 2) { + throw new ERR_TEST_FAILURE( + 'callback invoked multiple times', + kMultipleCallbackInvocations + ); + } + return; } @@ -335,7 +342,11 @@ class Test extends AsyncResource { this.pass(); } catch (err) { - this.fail(new ERR_TEST_FAILURE(err, kTestCodeFailure)); + if (err?.code === 'ERR_TEST_FAILURE' && kIsNodeError in err) { + this.fail(err); + } else { + this.fail(new ERR_TEST_FAILURE(err, kTestCodeFailure)); + } } // Clean up the test. Then, try to report the results and execute any @@ -436,4 +447,4 @@ class Test extends AsyncResource { } } -module.exports = { kDefaultIndent, kTestCodeFailure, Test }; +module.exports = { kDefaultIndent, kSubtestsFailed, kTestCodeFailure, Test }; diff --git a/lib/internal/test_runner/utils.js b/lib/internal/test_runner/utils.js new file mode 100644 index 00000000000..09803d33aeb --- /dev/null +++ b/lib/internal/test_runner/utils.js @@ -0,0 +1,15 @@ +'use strict'; +const { RegExpPrototypeExec } = primordials; +const { basename } = require('path'); +const kSupportedFileExtensions = /\.[cm]?js$/; +const kTestFilePattern = /((^test(-.+)?)|(.+[.\-_]test))\.[cm]?js$/; + +function doesPathMatchFilter(p) { + return RegExpPrototypeExec(kTestFilePattern, basename(p)) !== null; +} + +function isSupportedFileType(p) { + return RegExpPrototypeExec(kSupportedFileExtensions, p) !== null; +} + +module.exports = { isSupportedFileType, doesPathMatchFilter }; diff --git a/src/node.cc b/src/node.cc index 5ba03b75407..b43f915c5d5 100644 --- a/src/node.cc +++ b/src/node.cc @@ -517,6 +517,10 @@ MaybeLocal<Value> StartExecution(Environment* env, StartExecutionCallback cb) { return StartExecution(env, "internal/main/check_syntax"); } + if (env->options()->test_runner) { + return StartExecution(env, "internal/main/test_runner"); + } + if (!first_argv.empty() && first_argv != "-") { return StartExecution(env, "internal/main/run_main_module"); } diff --git a/src/node_options.cc b/src/node_options.cc index 313fb219294..b2b4c2c18c7 100644 --- a/src/node_options.cc +++ b/src/node_options.cc @@ -143,6 +143,24 @@ void EnvironmentOptions::CheckOptions(std::vector<std::string>* errors) { errors->push_back("--heap-snapshot-near-heap-limit must not be negative"); } + if (test_runner) { + if (syntax_check_only) { + errors->push_back("either --test or --check can be used, not both"); + } + + if (has_eval_string) { + errors->push_back("either --test or --eval can be used, not both"); + } + + if (force_repl) { + errors->push_back("either --test or --interactive can be used, not both"); + } + + if (debug_options_.inspector_enabled) { + errors->push_back("the inspector cannot be used with --test"); + } + } + #if HAVE_INSPECTOR if (!cpu_prof) { if (!cpu_prof_name.empty()) { @@ -498,6 +516,9 @@ EnvironmentOptionsParser::EnvironmentOptionsParser() { "write warnings to file instead of stderr", &EnvironmentOptions::redirect_warnings, kAllowedInEnvironment); + AddOption("--test", + "launch test runner on startup", + &EnvironmentOptions::test_runner); AddOption("--test-only", "run tests with 'only' option set", &EnvironmentOptions::test_only, diff --git a/src/node_options.h b/src/node_options.h index 97e01a9435b..0757a767a16 100644 --- a/src/node_options.h +++ b/src/node_options.h @@ -148,6 +148,7 @@ class EnvironmentOptions : public Options { #endif // HAVE_INSPECTOR std::string redirect_warnings; std::string diagnostic_dir; + bool test_runner = false; bool test_only = false; bool test_udp_no_try_send = false; bool throw_deprecation = false; diff --git a/test/fixtures/test-runner/index.js b/test/fixtures/test-runner/index.js new file mode 100644 index 00000000000..fcf4b4d8eaa --- /dev/null +++ b/test/fixtures/test-runner/index.js @@ -0,0 +1,2 @@ +'use strict'; +throw new Error('thrown from index.js'); diff --git a/test/fixtures/test-runner/index.test.js b/test/fixtures/test-runner/index.test.js new file mode 100644 index 00000000000..2a722c504b9 --- /dev/null +++ b/test/fixtures/test-runner/index.test.js @@ -0,0 +1,4 @@ +'use strict'; +const test = require('node:test'); + +test('this should pass'); diff --git a/test/fixtures/test-runner/node_modules/test-nm.js b/test/fixtures/test-runner/node_modules/test-nm.js new file mode 100644 index 00000000000..30024eab1f1 --- /dev/null +++ b/test/fixtures/test-runner/node_modules/test-nm.js @@ -0,0 +1,2 @@ +'use strict'; +throw new Error('thrown from node_modules'); diff --git a/test/fixtures/test-runner/random.test.mjs b/test/fixtures/test-runner/random.test.mjs new file mode 100644 index 00000000000..a87a671d006 --- /dev/null +++ b/test/fixtures/test-runner/random.test.mjs @@ -0,0 +1,5 @@ +import test from 'node:test'; + +test('this should fail', () => { + throw new Error('this is a failing test'); +}); diff --git a/test/fixtures/test-runner/subdir/subdir_test.js b/test/fixtures/test-runner/subdir/subdir_test.js new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/test/fixtures/test-runner/subdir/subdir_test.js diff --git a/test/fixtures/test-runner/test/random.cjs b/test/fixtures/test-runner/test/random.cjs new file mode 100644 index 00000000000..2a722c504b9 --- /dev/null +++ b/test/fixtures/test-runner/test/random.cjs @@ -0,0 +1,4 @@ +'use strict'; +const test = require('node:test'); + +test('this should pass'); diff --git a/test/message/test_runner_no_refs.out b/test/message/test_runner_no_refs.out index 0379ff8ca74..c5407e3bd33 100644 --- a/test/message/test_runner_no_refs.out +++ b/test/message/test_runner_no_refs.out @@ -3,8 +3,8 @@ TAP version 13 --- duration_ms: * failureType: 'cancelledByParent' - error: "'test did not finish before its parent and was cancelled'" - code: ERR_TEST_FAILURE + error: 'test did not finish before its parent and was cancelled' + code: 'ERR_TEST_FAILURE' stack: |- * ... @@ -13,8 +13,8 @@ not ok 1 - does not keep event loop alive --- duration_ms: * failureType: 'cancelledByParent' - error: "'test did not finish before its parent and was cancelled'" - code: ERR_TEST_FAILURE + error: 'test did not finish before its parent and was cancelled' + code: 'ERR_TEST_FAILURE' stack: |- * ... diff --git a/test/message/test_runner_output.js b/test/message/test_runner_output.js index 2639c327966..d397f1ee7a2 100644 --- a/test/message/test_runner_output.js +++ b/test/message/test_runner_output.js @@ -3,6 +3,7 @@ require('../common'); const assert = require('node:assert'); const test = require('node:test'); +const util = require('util'); test('sync pass todo', (t) => { t.todo(); @@ -296,3 +297,25 @@ test('only is set but not in only mode', { only: true }, async (t) => { t.runOnly(false); await t.test('running subtest 4'); }); + +test('custom inspect symbol fail', () => { + const obj = { + [util.inspect.custom]() { + return 'customized'; + }, + foo: 1 + }; + + throw obj; +}); + +test('custom inspect symbol that throws fail', () => { + const obj = { + [util.inspect.custom]() { + throw new Error('bad-inspect'); + }, + foo: 1 + }; + + throw obj; +}); diff --git a/test/message/test_runner_output.out b/test/message/test_runner_output.out index bf42b4035f2..866b498deab 100644 --- a/test/message/test_runner_output.out +++ b/test/message/test_runner_output.out @@ -12,7 +12,7 @@ not ok 3 - sync fail todo # TODO duration_ms: * failureType: 'testCodeFailure' error: 'thrown from sync fail todo' - code: ERR_TEST_FAILURE + code: 'ERR_TEST_FAILURE' stack: |- * * @@ -28,7 +28,7 @@ not ok 4 - sync fail todo with message # TODO this is a failing todo duration_ms: * failureType: 'testCodeFailure' error: 'thrown from sync fail todo with message' - code: ERR_TEST_FAILURE + code: 'ERR_TEST_FAILURE' stack: |- * * @@ -59,7 +59,7 @@ not ok 8 - sync throw fail duration_ms: * failureType: 'testCodeFailure' error: 'thrown from sync throw fail' - code: ERR_TEST_FAILURE + code: 'ERR_TEST_FAILURE' stack: |- * * @@ -83,7 +83,7 @@ not ok 11 - async throw fail duration_ms: * failureType: 'testCodeFailure' error: 'thrown from async throw fail' - code: ERR_TEST_FAILURE + code: 'ERR_TEST_FAILURE' stack: |- * * @@ -99,7 +99,7 @@ not ok 12 - async skip fail # SKIP duration_ms: * failureType: 'testCodeFailure' error: 'thrown from async throw fail' - code: ERR_TEST_FAILURE + code: 'ERR_TEST_FAILURE' stack: |- * * @@ -114,8 +114,12 @@ not ok 13 - async assertion fail --- duration_ms: * failureType: 'testCodeFailure' - error: 'Expected values to be strictly equal:\n\ntrue !== false\n' - code: ERR_ASSERTION + error: |- + Expected values to be strictly equal: + + true !== false + + code: 'ERR_ASSERTION' stack: |- * * @@ -135,7 +139,7 @@ not ok 15 - reject fail duration_ms: * failureType: 'testCodeFailure' error: 'rejected from reject fail' - code: ERR_TEST_FAILURE + code: 'ERR_TEST_FAILURE' stack: |- * * @@ -171,7 +175,7 @@ ok 20 - immediate resolve pass duration_ms: * failureType: 'testCodeFailure' error: 'thrown from subtest sync throw fail' - code: ERR_TEST_FAILURE + code: 'ERR_TEST_FAILURE' stack: |- * * @@ -190,15 +194,15 @@ not ok 21 - subtest sync throw fail --- duration_ms: * failureType: 'subtestsFailed' - error: "'1 subtest failed'" - code: ERR_TEST_FAILURE + error: '1 subtest failed' + code: 'ERR_TEST_FAILURE' ... not ok 22 - sync throw non-error fail --- duration_ms: * failureType: 'testCodeFailure' error: 'Symbol(thrown symbol from sync throw non-error fail)' - code: ERR_TEST_FAILURE + code: 'ERR_TEST_FAILURE' ... ok 1 - level 1a --- @@ -225,8 +229,8 @@ ok 23 - level 0a --- duration_ms: * failureType: 'cancelledByParent' - error: "'test did not finish before its parent and was cancelled'" - code: ERR_TEST_FAILURE + error: 'test did not finish before its parent and was cancelled' + code: 'ERR_TEST_FAILURE' ... ok 1 - ++short running --- @@ -242,8 +246,8 @@ not ok 24 - top level --- duration_ms: * failureType: 'subtestsFailed' - error: "'1 subtest failed'" - code: ERR_TEST_FAILURE + error: '1 subtest failed' + code: 'ERR_TEST_FAILURE' ... ok 25 - invalid subtest - pass but subtest fails --- @@ -262,7 +266,7 @@ not ok 28 - sync skip option is false fail duration_ms: * failureType: 'testCodeFailure' error: 'this should be executed' - code: ERR_TEST_FAILURE + code: 'ERR_TEST_FAILURE' stack: |- * * @@ -330,7 +334,7 @@ not ok 42 - callback fail duration_ms: * failureType: 'testCodeFailure' error: 'callback failure' - code: ERR_TEST_FAILURE + code: 'ERR_TEST_FAILURE' stack: |- * * @@ -351,15 +355,15 @@ not ok 46 - callback also returns a Promise --- duration_ms: * failureType: 'callbackAndPromisePresent' - error: "'passed a callback but also returned a Promise'" - code: ERR_TEST_FAILURE + error: 'passed a callback but also returned a Promise' + code: 'ERR_TEST_FAILURE' ... not ok 47 - callback throw --- duration_ms: * failureType: 'testCodeFailure' error: 'thrown from callback throw' - code: ERR_TEST_FAILURE + code: 'ERR_TEST_FAILURE' stack: |- * * @@ -373,8 +377,11 @@ not ok 48 - callback called twice --- duration_ms: * failureType: 'multipleCallbackInvocations' - error: "'callback invoked multiple times'" - code: ERR_TEST_FAILURE + error: 'callback invoked multiple times' + code: 'ERR_TEST_FAILURE' + stack: |- + * + * ... ok 49 - callback called twice in different ticks --- @@ -383,9 +390,9 @@ ok 49 - callback called twice in different ticks not ok 50 - callback called twice in future tick --- duration_ms: * - failureType: 'multipleCallbackInvocations' - error: "'callback invoked multiple times'" - code: ERR_TEST_FAILURE + failureType: 'uncaughtException' + error: 'callback invoked multiple times' + code: 'ERR_TEST_FAILURE' stack: |- * ... @@ -394,7 +401,7 @@ not ok 51 - callback async throw duration_ms: * failureType: 'uncaughtException' error: 'thrown from callback async throw' - code: ERR_TEST_FAILURE + code: 'ERR_TEST_FAILURE' stack: |- * ... @@ -423,25 +430,43 @@ ok 53 - only is set but not in only mode --- duration_ms: * ... -not ok 54 - invalid subtest fail +not ok 54 - custom inspect symbol fail + --- + duration_ms: * + failureType: 'testCodeFailure' + error: 'customized' + code: 'ERR_TEST_FAILURE' + ... +not ok 55 - custom inspect symbol that throws fail + --- + duration_ms: * + failureType: 'testCodeFailure' + error: |- + { + foo: 1, + [Symbol(nodejs.util.inspect.custom)]: [Function: [nodejs.util.inspect.custom]] + } + code: 'ERR_TEST_FAILURE' + ... +not ok 56 - invalid subtest fail --- duration_ms: * failureType: 'parentAlreadyFinished' - error: "'test could not be started because its parent finished'" - code: ERR_TEST_FAILURE + error: 'test could not be started because its parent finished' + code: 'ERR_TEST_FAILURE' stack: |- * ... -1..54 +1..56 # Warning: Test "unhandled rejection - passes but warns" generated asynchronous activity after the test ended. This activity created the error "Error: rejected from unhandled rejection fail" and would have caused the test to fail, but instead triggered an unhandledRejection event. # Warning: Test "async unhandled rejection - passes but warns" generated asynchronous activity after the test ended. This activity created the error "Error: rejected from async unhandled rejection fail" and would have caused the test to fail, but instead triggered an unhandledRejection event. # Warning: Test "immediate throw - passes but warns" generated asynchronous activity after the test ended. This activity created the error "Error: thrown from immediate throw fail" and would have caused the test to fail, but instead triggered an uncaughtException event. # Warning: Test "immediate reject - passes but warns" generated asynchronous activity after the test ended. This activity created the error "Error: rejected from immediate reject fail" and would have caused the test to fail, but instead triggered an unhandledRejection event. -# Warning: Test "callback called twice in different ticks" generated asynchronous activity after the test ended. This activity created the error "Error [ERR_TEST_FAILURE]: 'callback invoked multiple times'" and would have caused the test to fail, but instead triggered an uncaughtException event. +# Warning: Test "callback called twice in different ticks" generated asynchronous activity after the test ended. This activity created the error "Error [ERR_TEST_FAILURE]: callback invoked multiple times" and would have caused the test to fail, but instead triggered an uncaughtException event. # Warning: Test "callback async throw after done" generated asynchronous activity after the test ended. This activity created the error "Error: thrown from callback async throw after done" and would have caused the test to fail, but instead triggered an uncaughtException event. -# tests 54 +# tests 56 # pass 24 -# fail 15 +# fail 17 # skipped 10 # todo 5 # duration_ms * diff --git a/test/message/test_runner_unresolved_promise.out b/test/message/test_runner_unresolved_promise.out index 263b2411c85..98f52966c33 100644 --- a/test/message/test_runner_unresolved_promise.out +++ b/test/message/test_runner_unresolved_promise.out @@ -7,8 +7,8 @@ not ok 2 - never resolving promise --- duration_ms: * failureType: 'cancelledByParent' - error: "'test did not finish before its parent and was cancelled'" - code: ERR_TEST_FAILURE + error: 'test did not finish before its parent and was cancelled' + code: 'ERR_TEST_FAILURE' stack: |- * ... @@ -16,8 +16,8 @@ not ok 3 - fail --- duration_ms: * failureType: 'cancelledByParent' - error: "'test did not finish before its parent and was cancelled'" - code: ERR_TEST_FAILURE + error: 'test did not finish before its parent and was cancelled' + code: 'ERR_TEST_FAILURE' stack: |- * ... diff --git a/test/parallel/test-runner-cli.js b/test/parallel/test-runner-cli.js new file mode 100644 index 00000000000..8d16205cdaf --- /dev/null +++ b/test/parallel/test-runner-cli.js @@ -0,0 +1,107 @@ +'use strict'; +require('../common'); +const assert = require('assert'); +const { spawnSync } = require('child_process'); +const { join } = require('path'); +const fixtures = require('../common/fixtures'); +const testFixtures = fixtures.path('test-runner'); + +{ + // File not found. + const args = ['--test', 'a-random-file-that-does-not-exist.js']; + const child = spawnSync(process.execPath, args); + + assert.strictEqual(child.status, 1); + assert.strictEqual(child.signal, null); + assert.strictEqual(child.stdout.toString(), ''); + assert(/^Could not find/.test(child.stderr.toString())); +} + +{ + // Default behavior. node_modules is ignored. Files that don't match the + // pattern are ignored except in test/ directories. + const args = ['--test', testFixtures]; + const child = spawnSync(process.execPath, args); + + assert.strictEqual(child.status, 1); + assert.strictEqual(child.signal, null); + assert.strictEqual(child.stderr.toString(), ''); + const stdout = child.stdout.toString(); + assert(/ok 1 - .+index\.test\.js/.test(stdout)); + assert(/not ok 2 - .+random\.test\.mjs/.test(stdout)); + assert(/not ok 1 - this should fail/.test(stdout)); + assert(/ok 3 - .+subdir.+subdir_test\.js/.test(stdout)); + assert(/ok 4 - .+random\.cjs/.test(stdout)); +} + +{ + // User specified files that don't match the pattern are still run. + const args = ['--test', testFixtures, join(testFixtures, 'index.js')]; + const child = spawnSync(process.execPath, args); + + assert.strictEqual(child.status, 1); + assert.strictEqual(child.signal, null); + assert.strictEqual(child.stderr.toString(), ''); + const stdout = child.stdout.toString(); + assert(/not ok 1 - .+index\.js/.test(stdout)); + assert(/ok 2 - .+index\.test\.js/.test(stdout)); + assert(/not ok 3 - .+random\.test\.mjs/.test(stdout)); + assert(/not ok 1 - this should fail/.test(stdout)); + assert(/ok 4 - .+subdir.+subdir_test\.js/.test(stdout)); +} + +{ + // Searches node_modules if specified. + const args = ['--test', join(testFixtures, 'node_modules')]; + const child = spawnSync(process.execPath, args); + + assert.strictEqual(child.status, 1); + assert.strictEqual(child.signal, null); + assert.strictEqual(child.stderr.toString(), ''); + const stdout = child.stdout.toString(); + assert(/not ok 1 - .+test-nm\.js/.test(stdout)); +} + +{ + // The current directory is used by default. + const args = ['--test']; + const options = { cwd: testFixtures }; + const child = spawnSync(process.execPath, args, options); + + assert.strictEqual(child.status, 1); + assert.strictEqual(child.signal, null); + assert.strictEqual(child.stderr.toString(), ''); + const stdout = child.stdout.toString(); + assert(/ok 1 - .+index\.test\.js/.test(stdout)); + assert(/not ok 2 - .+random\.test\.mjs/.test(stdout)); + assert(/not ok 1 - this should fail/.test(stdout)); + assert(/ok 3 - .+subdir.+subdir_test\.js/.test(stdout)); + assert(/ok 4 - .+random\.cjs/.test(stdout)); +} + +{ + // Flags that cannot be combined with --test. + const flags = [ + ['--check', '--test'], + ['--interactive', '--test'], + ['--eval', 'console.log("should not print")', '--test'], + ['--print', 'console.log("should not print")', '--test'], + ]; + + if (process.features.inspector) { + flags.push( + ['--inspect', '--test'], + ['--inspect-brk', '--test'], + ); + } + + flags.forEach((args) => { + const child = spawnSync(process.execPath, args); + + assert.notStrictEqual(child.status, 0); + assert.strictEqual(child.signal, null); + assert.strictEqual(child.stdout.toString(), ''); + const stderr = child.stderr.toString(); + assert(/--test/.test(stderr)); + }); +} diff --git a/test/parallel/test-runner-test-filter.js b/test/parallel/test-runner-test-filter.js new file mode 100644 index 00000000000..b6afba22a2e --- /dev/null +++ b/test/parallel/test-runner-test-filter.js @@ -0,0 +1,42 @@ +// Flags: --expose-internals +'use strict'; +require('../common'); +const assert = require('assert'); +const { doesPathMatchFilter } = require('internal/test_runner/utils'); + +// Paths expected to match +[ + 'test.js', + 'test.cjs', + 'test.mjs', + 'test-foo.js', + 'test-foo.cjs', + 'test-foo.mjs', + 'foo.test.js', + 'foo.test.cjs', + 'foo.test.mjs', + 'foo-test.js', + 'foo-test.cjs', + 'foo-test.mjs', + 'foo_test.js', + 'foo_test.cjs', + 'foo_test.mjs', +].forEach((p) => { + assert.strictEqual(doesPathMatchFilter(p), true); +}); + +// Paths expected not to match +[ + 'test', + 'test.djs', + 'test.cs', + 'test.mj', + 'foo.js', + 'test-foo.sj', + 'test.foo.js', + 'test_foo.js', + 'testfoo.js', + 'foo-test1.mjs', +].forEach((p) => { + assert.strictEqual(doesPathMatchFilter(p), false); +}); |