test_runner: support test order randomization · nodejs/node@d14029b
@@ -59,6 +59,7 @@ const {
5959 validateObject,
6060 validateOneOf,
6161 validateInteger,
62+ validateUint32,
6263 validateString,
6364 validateStringArray,
6465} = require('internal/validators');
@@ -84,6 +85,7 @@ const {
8485const { FastBuffer } = require('internal/buffer');
85868687const {
88+ createRandomSeed,
8789 convertStringToRegExp,
8890 countCompletedTest,
8991 kDefaultPattern,
@@ -105,12 +107,14 @@ const kIsolatedProcessName = Symbol('kIsolatedProcessName');
105107const kFilterArgs = [
106108'--test',
107109'--experimental-test-coverage',
110+'--test-randomize',
108111'--watch',
109112'--experimental-default-config-file',
110113];
111114const kFilterArgValues = [
112115'--test-reporter',
113116'--test-reporter-destination',
117+'--test-random-seed',
114118'--experimental-config-file',
115119];
116120const kDiagnosticsFilterArgs = ['tests', 'suites', 'pass', 'fail', 'cancelled', 'skipped', 'todo', 'duration_ms'];
@@ -168,6 +172,8 @@ function getRunArgs(path, { forceExit,
168172argv: suppliedArgs,
169173 execArgv,
170174 rerunFailuresFilePath,
175+ randomize,
176+ randomSeed,
171177root: { timeout },
172178 cwd }) {
173179const processNodeOptions = getOptionsAsFlagsFromBinding();
@@ -209,6 +215,12 @@ function getRunArgs(path, { forceExit,
209215if (rerunFailuresFilePath) {
210216ArrayPrototypePush(runArgs, `--test-rerun-failures=${rerunFailuresFilePath}`);
211217}
218+if (randomize) {
219+ArrayPrototypePush(runArgs, '--test-randomize');
220+}
221+if (randomSeed != null) {
222+ArrayPrototypePush(runArgs, `--test-random-seed=${randomSeed}`);
223+}
212224213225ArrayPrototypePushApply(runArgs, execArgv);
214226@@ -646,6 +658,8 @@ function run(options = kEmptyObject) {
646658 lineCoverage = 0,
647659 branchCoverage = 0,
648660 functionCoverage = 0,
661+randomize: suppliedRandomize,
662+randomSeed: suppliedRandomSeed,
649663 execArgv = [],
650664 argv = [],
651665 cwd = process.cwd(),
@@ -674,6 +688,56 @@ function run(options = kEmptyObject) {
674688if (globPatterns != null) {
675689validateArray(globPatterns, 'options.globPatterns');
676690}
691+if (suppliedRandomize != null) {
692+validateBoolean(suppliedRandomize, 'options.randomize');
693+}
694+if (suppliedRandomSeed != null) {
695+validateUint32(suppliedRandomSeed, 'options.randomSeed');
696+}
697+let randomize = suppliedRandomize;
698+let randomSeed = suppliedRandomSeed;
699+700+if (randomSeed != null) {
701+randomize = true;
702+}
703+if (watch) {
704+if (randomSeed != null) {
705+throw new ERR_INVALID_ARG_VALUE(
706+'options.randomSeed',
707+randomSeed,
708+'is not supported with watch mode',
709+);
710+}
711+if (randomize) {
712+throw new ERR_INVALID_ARG_VALUE(
713+'options.randomize',
714+randomize,
715+'is not supported with watch mode',
716+);
717+}
718+}
719+if (rerunFailuresFilePath) {
720+validatePath(rerunFailuresFilePath, 'options.rerunFailuresFilePath');
721+// TODO(pmarchini): Support rerun-failures with randomization by
722+// persisting the randomization seed in the rerun state file.
723+if (randomSeed != null) {
724+throw new ERR_INVALID_ARG_VALUE(
725+'options.randomSeed',
726+randomSeed,
727+'is not supported with rerun failures mode',
728+);
729+}
730+if (randomize) {
731+throw new ERR_INVALID_ARG_VALUE(
732+'options.randomize',
733+randomize,
734+'is not supported with rerun failures mode',
735+);
736+}
737+}
738+if (randomize) {
739+randomSeed ??= createRandomSeed();
740+}
677741678742validateString(cwd, 'options.cwd');
679743@@ -683,10 +747,6 @@ function run(options = kEmptyObject) {
683747);
684748}
685749686-if (rerunFailuresFilePath) {
687-validatePath(rerunFailuresFilePath, 'options.rerunFailuresFilePath');
688-}
689-690750if (shard != null) {
691751validateObject(shard, 'options.shard');
692752// Avoid re-evaluating the shard object in case it's a getter
@@ -783,11 +843,18 @@ function run(options = kEmptyObject) {
783843functionCoverage: functionCoverage,
784844 cwd,
785845 globalSetupPath,
846+ randomize,
847+ randomSeed,
786848};
849+787850const root = createTestTree(rootTestOptions, globalOptions);
788851let testFiles = files ?? createTestFileList(globPatterns, cwd);
789852const { isTestRunner } = globalOptions;
790853854+if (randomize) {
855+root.diagnostic(`Randomized test order seed: ${randomSeed}`);
856+}
857+791858if (shard) {
792859testFiles = ArrayPrototypeFilter(testFiles, (_, index) => index % shard.total === shard.index - 1);
793860}
@@ -833,6 +900,8 @@ function run(options = kEmptyObject) {
833900 rerunFailuresFilePath,
834901 env,
835902workerIdPool: isolation === 'process' ? workerIdPool : null,
903+ randomize,
904+ randomSeed,
836905};
837906838907if (isolation === 'process') {