test_runner: add --test-name-pattern CLI flag

This commit adds support for running tests that match a regular expression. Fixes: #42984

Author: cjihrig

doc/api/cli.md

@@ -1187,6 +1187,16 @@ Starts the Node.js command line test runner. This flag cannot be combined with
 `--check`, `--eval`, `--interactive`, or the inspector. See the documentation
 on [running tests from the command line][] for more details.
 
+### `--test-name-pattern`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+A regular expression that configures the test runner to only execute tests
+whose name matches the provided pattern. See the documentation on
+[filtering tests by name][] for more details.
+
 ### `--test-only`
 
 <!-- YAML
@@ -2262,6 +2272,7 @@ done
 [debugger]: debugger.md
 [debugging security implications]: https://nodejs.org/en/docs/guides/debugging-getting-started/#security-implications
 [emit_warning]: process.md#processemitwarningwarning-options
+[filtering tests by name]: test.md#filtering-tests-by-name
 [jitless]: https://v8.dev/blog/jitless
 [libuv threadpool documentation]: https://docs.libuv.org/en/latest/threadpool.html
 [remote code execution]: https://www.owasp.org/index.php/Code_Injection

doc/api/test.md

@@ -220,6 +220,42 @@ test('this test is not run', () => {
 });
 ```
 
+## Filtering tests by name
+
+The [`--test-name-pattern`][] command-line option can be used to only run tests
+whose name matches the provided pattern. Test name patterns are interpreted as
+JavaScript regular expressions. The `--test-name-pattern` option can be
+specified multiple times in order to run nested tests. For each test that is
+executed, any corresponding test hooks, such as `beforeEach()`, are also
+run.
+
+Given the following test file, starting Node.js with the
+`--test-name-pattern="test [1-3]"` option would cause the test runner to execute
+`test 1`, `test 2`, and `test 3`. If `test 1` did not match the test name
+pattern, then its subtests would not execute, despite matching the pattern. The
+same set of tests could also be executed by passing `--test-name-pattern`
+multiple times (e.g. `--test-name-pattern="test 1"`,
+`--test-name-pattern="test 2"`, etc.).
+
+```js
+test('test 1', async (t) => {
+ await t.test('test 2');
+ await t.test('test 3');
+});
+
+test('Test 4', async (t) => {
+ await t.test('Test 5');
+ await t.test('test 6');
+});
+```
+
+Test name patterns can also be specified using regular expression literals. This
+allows regular expression flags to be used. In the previous example, starting
+Node.js with `--test-name-pattern="/test [4-5]/i"` would match `Test 4` and
+`Test 5` because the pattern is case-insensitive.
+
+Test name patterns do not change the set of files that the test runner executes.
+
 ## Extraneous asynchronous activity
 
 Once a test function finishes executing, the TAP results are output as quickly
@@ -896,6 +932,7 @@ added: v18.7.0
  aborted.
 
 [TAP]: https://testanything.org/
+[`--test-name-pattern`]: cli.md#--test-name-pattern
 [`--test-only`]: cli.md#--test-only
 [`--test`]: cli.md#--test
 [`SuiteContext`]: #class-suitecontext

doc/node.1

@@ -387,6 +387,10 @@ Specify the minimum allocation from the OpenSSL secure heap. The default is 2. T
 .It Fl -test
 Starts the Node.js command line test runner.
 .
+.It Fl -test-name-pattern
+A regular expression that configures the test runner to only execute tests
+whose name matches the provided pattern.
+.
 .It Fl -test-only
 Configures the test runner to only execute top level tests that have the `only`
 option set.

lib/internal/test_runner/test.js

@@ -1,9 +1,11 @@
 'use strict';
 const {
+ ArrayPrototypeMap,
  ArrayPrototypePush,
  ArrayPrototypeReduce,
  ArrayPrototypeShift,
  ArrayPrototypeSlice,
+ ArrayPrototypeSome,
  ArrayPrototypeUnshift,
  FunctionPrototype,
  MathMax,
@@ -12,6 +14,7 @@ const {
  PromisePrototypeThen,
  PromiseResolve,
  ReflectApply,
+ RegExpPrototypeExec,
  SafeMap,
  SafeSet,
  SafePromiseAll,
@@ -30,7 +33,11 @@ const {
 } = require('internal/errors');
 const { getOptionValue } = require('internal/options');
 const { TapStream } = require('internal/test_runner/tap_stream');
-const { createDeferredCallback, isTestFailureError } = require('internal/test_runner/utils');
+const {
+ convertStringToRegExp,
+ createDeferredCallback,
+ isTestFailureError,
+} = require('internal/test_runner/utils');
 const {
  createDeferredPromise,
  kEmptyObject,
@@ -58,6 +65,13 @@ const kDefaultTimeout = null;
 const noop = FunctionPrototype;
 const isTestRunner = getOptionValue('--test');
 const testOnlyFlag = !isTestRunner && getOptionValue('--test-only');
+const testNamePatternFlag = isTestRunner ? null :
+ getOptionValue('--test-name-pattern');
+const testNamePatterns = testNamePatternFlag?.length > 0 ?
+ ArrayPrototypeMap(
+ testNamePatternFlag,
+ (re) => convertStringToRegExp(re, '--test-name-pattern')
+ ) : null;
 const kShouldAbort = Symbol('kShouldAbort');
 const kRunHook = Symbol('kRunHook');
 const kHookNames = ObjectSeal(['before', 'after', 'beforeEach', 'afterEach']);
@@ -195,6 +209,18 @@ class Test extends AsyncResource {
  this.timeout = timeout;
  }
 
+ if (testNamePatterns !== null) {
+ // eslint-disable-next-line no-use-before-define
+ const match = this instanceof TestHook || ArrayPrototypeSome(
+ testNamePatterns,
+ (re) => RegExpPrototypeExec(re, name) !== null
+ );
+
+ if (!match) {
+ skip = 'test name does not match pattern';
+ }
+ }
+
  if (testOnlyFlag && !this.only) {
  skip = '\'only\' option not set';
  }
@@ -210,7 +236,6 @@ class Test extends AsyncResource {
  validateAbortSignal(signal, 'options.signal');
  this.#outerSignal?.addEventListener('abort', this.#abortHandler);
 
-
  this.fn = fn;
  this.name = name;
  this.parent = parent;
@@ -669,6 +694,7 @@ class ItTest extends Test {
  return { ctx: { signal: this.signal, name: this.name }, args: [] };
  }
 }
+
 class Suite extends Test {
  constructor(options) {
  super(options);
@@ -704,7 +730,6 @@ class Suite extends Test {
  return;
  }
 
-
  const hookArgs = this.getRunArgs();
  await this[kRunHook]('before', hookArgs);
  const stopPromise = stopTest(this.timeout, this.signal);

lib/internal/test_runner/utils.js

@@ -1,15 +1,17 @@
 'use strict';
-const { RegExpPrototypeExec } = primordials;
+const { RegExp, RegExpPrototypeExec } = primordials;
 const { basename } = require('path');
 const { createDeferredPromise } = require('internal/util');
 const {
  codes: {
+ ERR_INVALID_ARG_VALUE,
  ERR_TEST_FAILURE,
  },
  kIsNodeError,
 } = require('internal/errors');
 
 const kMultipleCallbackInvocations = 'multipleCallbackInvocations';
+const kRegExpPattern = /^\/(.*)\/([a-z]*)$/;
 const kSupportedFileExtensions = /\.[cm]?js$/;
 const kTestFilePattern = /((^test(-.+)?)|(.+[.\-_]test))\.[cm]?js$/;
 
@@ -54,7 +56,26 @@ function isTestFailureError(err) {
  return err?.code === 'ERR_TEST_FAILURE' && kIsNodeError in err;
 }
 
+function convertStringToRegExp(str, name) {
+ const match = RegExpPrototypeExec(kRegExpPattern, str);
+ const pattern = match?.[1] ?? str;
+ const flags = match?.[2] || '';
+
+ try {
+ return new RegExp(pattern, flags);
+ } catch (err) {
+ const msg = err?.message;
+
+ throw new ERR_INVALID_ARG_VALUE(
+ name,
+ str,
+ `is an invalid regular expression.${msg ? ` ${msg}` : ''}`
+ );
+ }
+}
+
 module.exports = {
+ convertStringToRegExp,
  createDeferredCallback,
  doesPathMatchFilter,
  isSupportedFileType,

src/node_options.cc

@@ -553,6 +553,9 @@ EnvironmentOptionsParser::EnvironmentOptionsParser() {
  AddOption("--test",
  "launch test runner on startup",
  &EnvironmentOptions::test_runner);
+ AddOption("--test-name-pattern",
+ "run tests whose name matches this regular expression",
+ &EnvironmentOptions::test_name_pattern);
  AddOption("--test-only",
  "run tests with 'only' option set",
  &EnvironmentOptions::test_only,

src/node_options.h

@@ -153,6 +153,7 @@ class EnvironmentOptions : public Options {
  std::string redirect_warnings;
  std::string diagnostic_dir;
  bool test_runner = false;
+ std::vector<std::string> test_name_pattern;
  bool test_only = false;
  bool test_udp_no_try_send = false;
  bool throw_deprecation = false;

test/message/test_runner_test_name_pattern.js

@@ -0,0 +1,47 @@
+// Flags: --no-warnings --test-name-pattern=enabled --test-name-pattern=/pattern/i
+'use strict';
+const common = require('../common');
+const {
+ after,
+ afterEach,
+ before,
+ beforeEach,
+ describe,
+ it,
+ test,
+} = require('node:test');
+
+test('top level test disabled', common.mustNotCall());
+test('top level skipped test disabled', { skip: true }, common.mustNotCall());
+test('top level skipped test enabled', { skip: true }, common.mustNotCall());
+it('top level it enabled', common.mustCall());
+it('top level it disabled', common.mustNotCall());
+it.skip('top level skipped it disabled', common.mustNotCall());
+it.skip('top level skipped it enabled', common.mustNotCall());
+describe('top level describe disabled', common.mustNotCall());
+describe.skip('top level skipped describe disabled', common.mustNotCall());
+describe.skip('top level skipped describe enabled', common.mustNotCall());
+test('top level runs because name includes PaTtErN', common.mustCall());
+
+test('top level test enabled', common.mustCall(async (t) => {
+ t.beforeEach(common.mustCall());
+ t.afterEach(common.mustCall());
+ await t.test(
+ 'nested test runs because name includes PATTERN',
+ common.mustCall()
+ );
+}));
+
+describe('top level describe enabled', () => {
+ before(common.mustCall());
+ beforeEach(common.mustCall(2));
+ afterEach(common.mustCall(2));
+ after(common.mustCall());
+
+ it('nested it disabled', common.mustNotCall());
+ it('nested it enabled', common.mustCall());
+ describe('nested describe disabled', common.mustNotCall());
+ describe('nested describe enabled', common.mustCall(() => {
+ it('is enabled', common.mustCall());
+ }));
+});

test/message/test_runner_test_name_pattern.out

@@ -0,0 +1,107 @@
+TAP version 13
+# Subtest: top level test disabled
+ok 1 - top level test disabled # SKIP test name does not match pattern
+ ---
+ duration_ms: *
+ ...
+# Subtest: top level skipped test disabled
+ok 2 - top level skipped test disabled # SKIP test name does not match pattern
+ ---
+ duration_ms: *
+ ...
+# Subtest: top level skipped test enabled
+ok 3 - top level skipped test enabled # SKIP
+ ---
+ duration_ms: *
+ ...
+# Subtest: top level it enabled
+ok 4 - top level it enabled
+ ---
+ duration_ms: *
+ ...
+# Subtest: top level it disabled
+ok 5 - top level it disabled # SKIP test name does not match pattern
+ ---
+ duration_ms: *
+ ...
+# Subtest: top level skipped it disabled
+ok 6 - top level skipped it disabled # SKIP test name does not match pattern
+ ---
+ duration_ms: *
+ ...
+# Subtest: top level skipped it enabled
+ok 7 - top level skipped it enabled # SKIP
+ ---
+ duration_ms: *
+ ...
+# Subtest: top level describe disabled
+ok 8 - top level describe disabled # SKIP test name does not match pattern
+ ---
+ duration_ms: *
+ ...
+# Subtest: top level skipped describe disabled
+ok 9 - top level skipped describe disabled # SKIP test name does not match pattern
+ ---
+ duration_ms: *
+ ...
+# Subtest: top level skipped describe enabled
+ok 10 - top level skipped describe enabled # SKIP
+ ---
+ duration_ms: *
+ ...
+# Subtest: top level runs because name includes PaTtErN
+ok 11 - top level runs because name includes PaTtErN
+ ---
+ duration_ms: *
+ ...
+# Subtest: top level test enabled
+ # Subtest: nested test runs because name includes PATTERN
+ ok 1 - nested test runs because name includes PATTERN
+ ---
+ duration_ms: *
+ ...
+ 1..1
+ok 12 - top level test enabled
+ ---
+ duration_ms: *
+ ...
+# Subtest: top level describe enabled
+ # Subtest: nested it disabled
+ ok 1 - nested it disabled # SKIP test name does not match pattern
+ ---
+ duration_ms: *
+ ...
+ # Subtest: nested it enabled
+ ok 2 - nested it enabled
+ ---
+ duration_ms: *
+ ...
+ # Subtest: nested describe disabled
+ ok 3 - nested describe disabled # SKIP test name does not match pattern
+ ---
+ duration_ms: *
+ ...
+ # Subtest: nested describe enabled
+ # Subtest: is enabled
+ ok 1 - is enabled
+ ---
+ duration_ms: *
+ ...
+ 1..1
+ ok 4 - nested describe enabled
+ ---
+ duration_ms: *
+ ...
+ 1..4
+ok 13 - top level describe enabled
+ ---
+ duration_ms: *
+ ...
+1..13
+# tests 13
+# pass 4
+# fail 0
+# cancelled 0
+# skipped 9
+# todo 0
+# duration_ms *