From 69a578034f8f317dd88e63ebce2bc810eec5eadf Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Tue, 14 Jul 2026 00:16:49 -0700 Subject: [PATCH] test: isolate provider auth parity suite (#107167) --- scripts/lib/ci-node-test-plan.mjs | 4 +-- scripts/run-vitest.mjs | 3 +- scripts/test-projects.test-support.mjs | 31 +++++++++++-------- test/scripts/run-vitest.test.ts | 7 +++-- test/scripts/test-projects.test.ts | 13 ++++++++ test/vitest-scoped-config.test.ts | 7 +++-- .../vitest.tooling-isolated-paths.d.mts | 2 ++ test/vitest/vitest.tooling-isolated-paths.mjs | 11 +++++++ test/vitest/vitest.tooling-isolated.config.ts | 3 +- test/vitest/vitest.tooling.config.ts | 7 ++--- 10 files changed, 61 insertions(+), 27 deletions(-) create mode 100644 test/vitest/vitest.tooling-isolated-paths.d.mts create mode 100644 test/vitest/vitest.tooling-isolated-paths.mjs diff --git a/scripts/lib/ci-node-test-plan.mjs b/scripts/lib/ci-node-test-plan.mjs index 129f4fc6e5c3..554f05b63412 100644 --- a/scripts/lib/ci-node-test-plan.mjs +++ b/scripts/lib/ci-node-test-plan.mjs @@ -2,6 +2,7 @@ import { relative } from "node:path"; import { commandsLightTestFiles } from "../../test/vitest/vitest.commands-light-paths.mjs"; import { fullSuiteVitestShards } from "../../test/vitest/vitest.test-shards.mjs"; +import { toolingIsolatedTestFiles } from "../../test/vitest/vitest.tooling-isolated-paths.mjs"; import { getUnitFastTestFilesForIncludePatterns } from "../../test/vitest/vitest.unit-fast-paths.mjs"; import { boundaryTestFiles } from "../../test/vitest/vitest.unit-paths.mjs"; import { listTrackedTestFiles } from "./list-test-files.mjs"; @@ -34,7 +35,6 @@ const COMPACT_WHOLE_NODE_TEST_TIMEOUT_MINUTES = 120; const TOOLING_CONFIG = "test/vitest/vitest.tooling.config.ts"; const TOOLING_DOCKER_TEST_FILE = "test/scripts/docker-build-helper.test.ts"; const TOOLING_ISOLATED_CONFIG = "test/vitest/vitest.tooling-isolated.config.ts"; -const TOOLING_ISOLATED_TEST_FILE = "test/scripts/openclaw-e2e-instance.test.ts"; // The full matrix is capped at 28 jobs. Admit the consistently slow serial // shards first so short alphabetical groups cannot leave them on the tail. const FULL_NODE_TEST_ADMISSION_PRIORITY = new Map([ @@ -1045,7 +1045,7 @@ function listCompactToolingTestFiles() { ...boundaryTestFiles, ...unitFastFiles, TOOLING_DOCKER_TEST_FILE, - TOOLING_ISOLATED_TEST_FILE, + ...toolingIsolatedTestFiles, ]); return [...listTestFiles("test"), ...listTestFiles("src/scripts")].filter( (file) => diff --git a/scripts/run-vitest.mjs b/scripts/run-vitest.mjs index 44e6123b11dd..949c6320b93c 100644 --- a/scripts/run-vitest.mjs +++ b/scripts/run-vitest.mjs @@ -5,6 +5,7 @@ import fs from "node:fs"; import { createRequire } from "node:module"; import path from "node:path"; import { fileURLToPath } from "node:url"; +import { toolingIsolatedTestFiles } from "../test/vitest/vitest.tooling-isolated-paths.mjs"; import { isUiTestTarget } from "../test/vitest/vitest.ui-paths.mjs"; import { boundaryTestFiles } from "../test/vitest/vitest.unit-paths.mjs"; import { resolveLocalVitestEnv } from "./lib/vitest-local-scheduling.mjs"; @@ -56,7 +57,7 @@ export const VITEST_CONFIG_NO_OUTPUT_TIMEOUT_MS = new Map([ export const TOOLING_EXCLUDED_TESTS = new Set([ ...boundaryTestFiles, "test/scripts/docker-build-helper.test.ts", - "test/scripts/openclaw-e2e-instance.test.ts", + ...toolingIsolatedTestFiles, ]); const EXPLICIT_FILE_TARGET_RE = /\.(?:[cm]?[jt]sx?)$/u; const EXPLICIT_TEST_FILE_RE = /\.(?:test|e2e|live)\.(?:[cm]?[jt]sx?)$/u; diff --git a/scripts/test-projects.test-support.mjs b/scripts/test-projects.test-support.mjs index 7b68d559460e..fb55a9a9d965 100644 --- a/scripts/test-projects.test-support.mjs +++ b/scripts/test-projects.test-support.mjs @@ -42,6 +42,10 @@ import { resolvePluginSdkLightIncludePattern, } from "../test/vitest/vitest.plugin-sdk-paths.mjs"; import { fullSuiteVitestShards } from "../test/vitest/vitest.test-shards.mjs"; +import { + isToolingIsolatedTestFile, + toolingIsolatedTestFiles, +} from "../test/vitest/vitest.tooling-isolated-paths.mjs"; import { getUnitFastTestFiles, getUnitFastTimerTestFiles, @@ -285,7 +289,6 @@ const TOOLING_DOCKER_VITEST_CONFIG = "test/vitest/vitest.tooling-docker.config.t const TOOLING_ISOLATED_VITEST_CONFIG = "test/vitest/vitest.tooling-isolated.config.ts"; const TOOLING_VITEST_CONFIG = "test/vitest/vitest.tooling.config.ts"; const TOOLING_DOCKER_TEST_TARGET = "test/scripts/docker-build-helper.test.ts"; -const TOOLING_ISOLATED_TEST_TARGET = "test/scripts/openclaw-e2e-instance.test.ts"; const BROAD_TOOLING_SCRIPT_TEST_PATTERNS = new Set([ "test/scripts/**/*.test.ts", "test/scripts/*.test.ts", @@ -3864,7 +3867,7 @@ function classifyTarget(arg, cwd) { if (isBoundaryTestFile(relative)) { return "boundary"; } - if (relative === TOOLING_ISOLATED_TEST_TARGET) { + if (isToolingIsolatedTestFile(relative)) { return "toolingIsolated"; } if (relative === TOOLING_DOCKER_TEST_TARGET) { @@ -4128,19 +4131,21 @@ export function buildVitestRunPlans( groupedTargets.set("toolingDocker", current); } } - if ( - !watchMode && - toolingTargets.some((targetArg) => - includePatternMatchesAnyFile(toScopedIncludePattern(targetArg, cwd), [ - TOOLING_ISOLATED_TEST_TARGET, - ]), - ) - ) { + const impliedToolingIsolatedTargets = !watchMode + ? toolingIsolatedTestFiles.filter((file) => + toolingTargets.some((targetArg) => + includePatternMatchesAnyFile(toScopedIncludePattern(targetArg, cwd), [file]), + ), + ) + : []; + if (impliedToolingIsolatedTargets.length > 0) { const current = groupedTargets.get("toolingIsolated") ?? []; - if (!current.includes(TOOLING_ISOLATED_TEST_TARGET)) { - current.push(TOOLING_ISOLATED_TEST_TARGET); - groupedTargets.set("toolingIsolated", current); + for (const target of impliedToolingIsolatedTargets) { + if (!current.includes(target)) { + current.push(target); + } } + groupedTargets.set("toolingIsolated", current); } if (watchMode && groupedTargets.size > 1) { diff --git a/test/scripts/run-vitest.test.ts b/test/scripts/run-vitest.test.ts index 7a26d46cb298..263a413eb015 100644 --- a/test/scripts/run-vitest.test.ts +++ b/test/scripts/run-vitest.test.ts @@ -187,8 +187,11 @@ describe("scripts/run-vitest", () => { ]); }); - it("keeps tooling-excluded explicit tests on existing routing", () => { - const argv = ["run", "test/scripts/openclaw-e2e-instance.test.ts"]; + it.each([ + "test/plugins/bundled-provider-auth-literal-parity.test.ts", + "test/scripts/openclaw-e2e-instance.test.ts", + ])("keeps tooling-excluded explicit test %s on existing routing", (testFile) => { + const argv = ["run", testFile]; expect(resolveImplicitVitestArgs(argv)).toBe(argv); }); diff --git a/test/scripts/test-projects.test.ts b/test/scripts/test-projects.test.ts index 7491c96c6f5e..e626434ee2e7 100644 --- a/test/scripts/test-projects.test.ts +++ b/test/scripts/test-projects.test.ts @@ -2453,6 +2453,19 @@ describe("scripts/test-projects changed-target routing", () => { ]); }); + it("routes the bundled provider auth parity test to the isolated tooling shard", () => { + expect( + buildVitestRunPlans(["test/plugins/bundled-provider-auth-literal-parity.test.ts"]), + ).toEqual([ + { + config: "test/vitest/vitest.tooling-isolated.config.ts", + forwardedArgs: [], + includePatterns: ["test/plugins/bundled-provider-auth-literal-parity.test.ts"], + watchMode: false, + }, + ]); + }); + it("routes Docker E2E script targets to their owner tooling tests", () => { const targets = [ "scripts/e2e/kitchen-sink-plugin-docker.sh", diff --git a/test/vitest-scoped-config.test.ts b/test/vitest-scoped-config.test.ts index 9e2f4abeb342..6a09800a458c 100644 --- a/test/vitest-scoped-config.test.ts +++ b/test/vitest-scoped-config.test.ts @@ -69,6 +69,7 @@ import { createToolingDockerVitestConfig, toolingDockerTestFiles, } from "./vitest/vitest.tooling-docker.config.ts"; +import { toolingIsolatedTestFiles } from "./vitest/vitest.tooling-isolated-paths.mjs"; import { createToolingIsolatedVitestConfig } from "./vitest/vitest.tooling-isolated.config.ts"; import { createToolingVitestConfig } from "./vitest/vitest.tooling.config.ts"; import { createTuiVitestConfig } from "./vitest/vitest.tui.config.ts"; @@ -1094,7 +1095,7 @@ describe("scoped vitest configs", () => { const testConfig = requireTestConfig(defaultToolingConfig); expect(testConfig.include).toEqual(["test/**/*.test.ts", "src/scripts/**/*.test.ts"]); expect(testConfig.exclude).toEqual(expect.arrayContaining(toolingDockerTestFiles)); - expect(testConfig.exclude).toContain("test/scripts/openclaw-e2e-instance.test.ts"); + expect(testConfig.exclude).toEqual(expect.arrayContaining(toolingIsolatedTestFiles)); expect(testConfig.include).not.toContain("src/config/doc-baseline.integration.test.ts"); }); @@ -1104,9 +1105,9 @@ describe("scoped vitest configs", () => { expect(testConfig.fileParallelism).toBe(false); }); - it("runs shell helper tooling tests isolated from shared mocks", () => { + it("runs state-sensitive tooling tests isolated from shared mocks", () => { const testConfig = requireTestConfig(createToolingIsolatedVitestConfig({})); - expect(testConfig.include).toEqual(["test/scripts/openclaw-e2e-instance.test.ts"]); + expect(testConfig.include).toEqual(toolingIsolatedTestFiles); expect(testConfig.isolate).toBe(true); expect(testConfig.runner).toBeUndefined(); }); diff --git a/test/vitest/vitest.tooling-isolated-paths.d.mts b/test/vitest/vitest.tooling-isolated-paths.d.mts new file mode 100644 index 000000000000..7e6aab8d6edd --- /dev/null +++ b/test/vitest/vitest.tooling-isolated-paths.d.mts @@ -0,0 +1,2 @@ +export const toolingIsolatedTestFiles: string[]; +export function isToolingIsolatedTestFile(file: string): boolean; diff --git a/test/vitest/vitest.tooling-isolated-paths.mjs b/test/vitest/vitest.tooling-isolated-paths.mjs new file mode 100644 index 000000000000..8123f05638e0 --- /dev/null +++ b/test/vitest/vitest.tooling-isolated-paths.mjs @@ -0,0 +1,11 @@ +// Tooling tests that need fresh module or process state instead of the shared serial worker. +export const toolingIsolatedTestFiles = [ + "test/plugins/bundled-provider-auth-literal-parity.test.ts", + "test/scripts/openclaw-e2e-instance.test.ts", +]; + +const toolingIsolatedTestFileSet = new Set(toolingIsolatedTestFiles); + +export function isToolingIsolatedTestFile(value) { + return toolingIsolatedTestFileSet.has(value.replaceAll("\\", "/")); +} diff --git a/test/vitest/vitest.tooling-isolated.config.ts b/test/vitest/vitest.tooling-isolated.config.ts index fd132e0d28c9..a67b771bc406 100644 --- a/test/vitest/vitest.tooling-isolated.config.ts +++ b/test/vitest/vitest.tooling-isolated.config.ts @@ -1,8 +1,9 @@ // Vitest tooling isolated config wires the tooling isolated test shard. import { createScopedVitestConfig } from "./vitest.scoped-config.ts"; +import { toolingIsolatedTestFiles } from "./vitest.tooling-isolated-paths.mjs"; export function createToolingIsolatedVitestConfig(env?: Record) { - return createScopedVitestConfig(["test/scripts/openclaw-e2e-instance.test.ts"], { + return createScopedVitestConfig(toolingIsolatedTestFiles, { env, isolate: true, name: "tooling-isolated", diff --git a/test/vitest/vitest.tooling.config.ts b/test/vitest/vitest.tooling.config.ts index a4858623bb1e..e931c1d01ff7 100644 --- a/test/vitest/vitest.tooling.config.ts +++ b/test/vitest/vitest.tooling.config.ts @@ -2,6 +2,7 @@ import { loadPatternListFromEnv } from "./vitest.pattern-file.ts"; import { createScopedVitestConfig } from "./vitest.scoped-config.ts"; import { toolingDockerTestFiles } from "./vitest.tooling-docker.config.ts"; +import { toolingIsolatedTestFiles } from "./vitest.tooling-isolated-paths.mjs"; import { boundaryTestFiles } from "./vitest.unit-paths.mjs"; export function loadIncludePatternsFromEnv( @@ -15,11 +16,7 @@ export function createToolingVitestConfig(env?: Record