From 663501206f1f897f7779213fae56f51800e38622 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Tue, 21 Apr 2026 06:09:42 +0100 Subject: [PATCH] test: speed up channel contract CI --- scripts/lib/channel-contract-test-plan.mjs | 8 +- scripts/test-projects.test-support.mjs | 84 ++++++++++++- src/scripts/test-projects.test.ts | 37 ++++++ .../channels/surface-contract-registry.ts | 110 +++++++++++++----- .../channel-contract-test-plan.test.ts | 11 ++ test/vitest-projects-config.test.ts | 28 ++++- test/vitest/vitest.contracts-shared.ts | 26 ++++- 7 files changed, 269 insertions(+), 35 deletions(-) diff --git a/scripts/lib/channel-contract-test-plan.mjs b/scripts/lib/channel-contract-test-plan.mjs index 80a45a69178..e02cc5f29b7 100644 --- a/scripts/lib/channel-contract-test-plan.mjs +++ b/scripts/lib/channel-contract-test-plan.mjs @@ -23,16 +23,16 @@ const CONTRACT_FILE_WEIGHTS = new Map([ function resolveContractFileWeight(file) { const name = file.replaceAll("\\", "/").split("/").pop(); if (name.startsWith("plugin.registry-backed-shard-")) { - return 5; + return 40; } if (name.startsWith("surfaces-only.registry-backed-shard-")) { - return 5; + return 40; } if (name.startsWith("directory.registry-backed-shard-")) { - return 4; + return 24; } if (name.startsWith("threading.registry-backed-shard-")) { - return 4; + return 18; } return CONTRACT_FILE_WEIGHTS.get(name) ?? 8; } diff --git a/scripts/test-projects.test-support.mjs b/scripts/test-projects.test-support.mjs index 8bd67948856..e3b30715561 100644 --- a/scripts/test-projects.test-support.mjs +++ b/scripts/test-projects.test-support.mjs @@ -208,6 +208,49 @@ const GENERATED_CHANGED_TEST_TARGETS = new Set([ const VITEST_CONFIG_TARGET_KIND_BY_PATH = new Map( Object.entries(VITEST_CONFIG_BY_KIND).map(([kind, config]) => [config, kind]), ); +const CHANNEL_CONTRACT_CONFIG_PATTERNS = new Map([ + [ + CONTRACTS_CHANNEL_SURFACE_VITEST_CONFIG, + [ + "src/channels/plugins/contracts/channel-catalog.contract.test.ts", + "src/channels/plugins/contracts/channel-import-guardrails.test.ts", + "src/channels/plugins/contracts/group-policy.fallback.contract.test.ts", + "src/channels/plugins/contracts/outbound-payload.contract.test.ts", + "src/channels/plugins/contracts/*-shard-a.contract.test.ts", + "src/channels/plugins/contracts/*-shard-e.contract.test.ts", + ], + ], + [ + CONTRACTS_CHANNEL_CONFIG_VITEST_CONFIG, + [ + "src/channels/plugins/contracts/plugins-core.authorize-config-write.policy.contract.test.ts", + "src/channels/plugins/contracts/plugins-core.authorize-config-write.targets.contract.test.ts", + "src/channels/plugins/contracts/plugins-core.catalog.entries.contract.test.ts", + "src/channels/plugins/contracts/*-shard-b.contract.test.ts", + "src/channels/plugins/contracts/*-shard-f.contract.test.ts", + ], + ], + [ + CONTRACTS_CHANNEL_REGISTRY_VITEST_CONFIG, + [ + "src/channels/plugins/contracts/plugins-core.catalog.paths.contract.test.ts", + "src/channels/plugins/contracts/plugins-core.loader.contract.test.ts", + "src/channels/plugins/contracts/plugins-core.registry.contract.test.ts", + "src/channels/plugins/contracts/*-shard-c.contract.test.ts", + "src/channels/plugins/contracts/*-shard-g.contract.test.ts", + ], + ], + [ + CONTRACTS_CHANNEL_SESSION_VITEST_CONFIG, + [ + "src/channels/plugins/contracts/plugins-core.resolve-config-writes.contract.test.ts", + "src/channels/plugins/contracts/registry.contract.test.ts", + "src/channels/plugins/contracts/session-binding.registry-backed.contract.test.ts", + "src/channels/plugins/contracts/*-shard-d.contract.test.ts", + "src/channels/plugins/contracts/*-shard-h.contract.test.ts", + ], + ], +]); function normalizePathPattern(value) { return value.replaceAll("\\", "/"); @@ -960,7 +1003,8 @@ export function applyParallelVitestCachePaths(specs, params = {}) { export function createVitestRunSpecs(args, params = {}) { const cwd = params.cwd ?? process.cwd(); - const plans = buildVitestRunPlans(args, cwd); + const baseEnv = params.baseEnv ?? process.env; + const plans = filterPlansForContractIncludeFile(buildVitestRunPlans(args, cwd), baseEnv); return plans.map((plan, index) => { const includeFilePath = plan.includePatterns ? path.join( @@ -972,10 +1016,10 @@ export function createVitestRunSpecs(args, params = {}) { config: plan.config, env: includeFilePath ? { - ...(params.baseEnv ?? process.env), + ...baseEnv, [INCLUDE_FILE_ENV_KEY]: includeFilePath, } - : (params.baseEnv ?? process.env), + : baseEnv, includeFilePath, includePatterns: plan.includePatterns, pnpmArgs: createVitestArgs(plan), @@ -984,6 +1028,40 @@ export function createVitestRunSpecs(args, params = {}) { }); } +function loadIncludePatternsForSpecFilter(env) { + const filePath = env[INCLUDE_FILE_ENV_KEY]?.trim(); + if (!filePath) { + return null; + } + const parsed = JSON.parse(fs.readFileSync(filePath, "utf8")); + if (!Array.isArray(parsed)) { + return []; + } + return parsed.filter((value) => typeof value === "string" && value.length > 0); +} + +function includePatternMatchesConfig(candidate, configPatterns) { + return configPatterns.some( + (pattern) => path.matchesGlob(candidate, pattern) || path.matchesGlob(pattern, candidate), + ); +} + +function filterPlansForContractIncludeFile(plans, env) { + const includePatterns = loadIncludePatternsForSpecFilter(env); + if (!includePatterns) { + return plans; + } + return plans.filter((plan) => { + const configPatterns = CHANNEL_CONTRACT_CONFIG_PATTERNS.get(plan.config); + if (!configPatterns) { + return true; + } + return includePatterns.some((candidate) => + includePatternMatchesConfig(candidate, configPatterns), + ); + }); +} + export function shouldAcquireLocalHeavyCheckLock(runSpecs, env = process.env) { if (env.OPENCLAW_TEST_PROJECTS_FORCE_LOCK === "1") { return true; diff --git a/src/scripts/test-projects.test.ts b/src/scripts/test-projects.test.ts index bbe4ad44bc9..68a18c4b0ee 100644 --- a/src/scripts/test-projects.test.ts +++ b/src/scripts/test-projects.test.ts @@ -1,3 +1,6 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; import { describe, expect, it } from "vitest"; const { @@ -975,6 +978,40 @@ describe("test-projects args", () => { expect(spec?.env.OPENCLAW_VITEST_INCLUDE_FILE).toBe(spec?.includeFilePath); }); + it("skips channel contract configs with no matching external include patterns", () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-contract-include-")); + try { + const includeFile = path.join(tempDir, "include.json"); + fs.writeFileSync( + includeFile, + JSON.stringify([ + "src/channels/plugins/contracts/surfaces-only.registry-backed-shard-b.contract.test.ts", + ]), + "utf8", + ); + + const specs = createVitestRunSpecs( + [ + "test/vitest/vitest.contracts-channel-surface.config.ts", + "test/vitest/vitest.contracts-channel-config.config.ts", + "test/vitest/vitest.contracts-channel-registry.config.ts", + "test/vitest/vitest.contracts-channel-session.config.ts", + ], + { + baseEnv: { + OPENCLAW_VITEST_INCLUDE_FILE: includeFile, + } as NodeJS.ProcessEnv, + }, + ); + + expect(specs.map((spec) => spec.config)).toEqual([ + "test/vitest/vitest.contracts-channel-config.config.ts", + ]); + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + } + }); + it("rejects watch mode when a command spans multiple suites", () => { expect(() => buildVitestRunPlans([ diff --git a/test/helpers/channels/surface-contract-registry.ts b/test/helpers/channels/surface-contract-registry.ts index cebd12acd2c..1a441a65345 100644 --- a/test/helpers/channels/surface-contract-registry.ts +++ b/test/helpers/channels/surface-contract-registry.ts @@ -43,6 +43,36 @@ const surfaceContractEntryCache = new Map([ + "bluebubbles", + "discord", + "googlechat", + "matrix", + "mattermost", + "msteams", + "slack", + "telegram", + "zalo", + "zalouser", +]); + +const directoryContractPluginIds = new Set([ + "discord", + "feishu", + "googlechat", + "irc", + "line", + "matrix", + "mattermost", + "msteams", + "slack", + "synology-chat", + "telegram", + "whatsapp", + "zalo", + "zalouser", +]); + function toSurfaceContractEntry(plugin: ChannelPlugin): SurfaceContractEntry { return { id: plugin.id, @@ -86,12 +116,19 @@ export function getSurfaceContractRegistryShard(params: { } export function getThreadingContractRegistry(): ThreadingContractEntry[] { - threadingContractRegistryCache ??= getSurfaceContractRegistry() - .filter((entry) => entry.surfaces.includes("threading")) - .map((entry) => ({ - id: entry.id, - plugin: entry.plugin, - })); + threadingContractRegistryCache ??= listBundledChannelPluginIds() + .filter((id) => threadingContractPluginIds.has(id)) + .flatMap((id) => { + const entry = getSurfaceContractEntry(id); + return entry && entry.surfaces.includes("threading") + ? [ + { + id: entry.id, + plugin: entry.plugin, + }, + ] + : []; + }); return threadingContractRegistryCache; } @@ -99,24 +136,38 @@ export function getThreadingContractRegistryShard(params: { shardIndex: number; shardCount: number; }): ThreadingContractEntry[] { - return getSurfaceContractRegistryShard(params) - .filter((entry) => entry.surfaces.includes("threading")) - .map((entry) => ({ - id: entry.id, - plugin: entry.plugin, - })); + return getBundledChannelPluginIdsForShard(params) + .filter((id) => threadingContractPluginIds.has(id)) + .flatMap((id) => { + const entry = getSurfaceContractEntry(id); + return entry && entry.surfaces.includes("threading") + ? [ + { + id: entry.id, + plugin: entry.plugin, + }, + ] + : []; + }); } const directoryPresenceOnlyIds = new Set(["whatsapp", "zalouser"]); export function getDirectoryContractRegistry(): DirectoryContractEntry[] { - directoryContractRegistryCache ??= getSurfaceContractRegistry() - .filter((entry) => entry.surfaces.includes("directory")) - .map((entry) => ({ - id: entry.id, - plugin: entry.plugin, - coverage: directoryPresenceOnlyIds.has(entry.id) ? "presence" : "lookups", - })); + directoryContractRegistryCache ??= listBundledChannelPluginIds() + .filter((id) => directoryContractPluginIds.has(id)) + .flatMap((id) => { + const entry = getSurfaceContractEntry(id); + return entry && entry.surfaces.includes("directory") + ? [ + { + id: entry.id, + plugin: entry.plugin, + coverage: directoryPresenceOnlyIds.has(entry.id) ? "presence" : "lookups", + }, + ] + : []; + }); return directoryContractRegistryCache; } @@ -124,11 +175,18 @@ export function getDirectoryContractRegistryShard(params: { shardIndex: number; shardCount: number; }): DirectoryContractEntry[] { - return getSurfaceContractRegistryShard(params) - .filter((entry) => entry.surfaces.includes("directory")) - .map((entry) => ({ - id: entry.id, - plugin: entry.plugin, - coverage: directoryPresenceOnlyIds.has(entry.id) ? "presence" : "lookups", - })); + return getBundledChannelPluginIdsForShard(params) + .filter((id) => directoryContractPluginIds.has(id)) + .flatMap((id) => { + const entry = getSurfaceContractEntry(id); + return entry && entry.surfaces.includes("directory") + ? [ + { + id: entry.id, + plugin: entry.plugin, + coverage: directoryPresenceOnlyIds.has(entry.id) ? "presence" : "lookups", + }, + ] + : []; + }); } diff --git a/test/scripts/channel-contract-test-plan.test.ts b/test/scripts/channel-contract-test-plan.test.ts index 81b845c2410..9b18fd770e1 100644 --- a/test/scripts/channel-contract-test-plan.test.ts +++ b/test/scripts/channel-contract-test-plan.test.ts @@ -43,4 +43,15 @@ describe("scripts/lib/channel-contract-test-plan.mjs", () => { expect(actual).toEqual(listContractTests()); expect(new Set(actual).size).toBe(actual.length); }); + + it("keeps registry-backed surface shards spread across checks", () => { + for (const shard of createChannelContractTestShards().filter((entry) => + entry.checkName.includes("-registry-"), + )) { + const surfaceRegistryFiles = shard.includePatterns.filter((pattern) => + pattern.includes("/surfaces-only.registry-backed-shard-"), + ); + expect(surfaceRegistryFiles.length).toBeLessThanOrEqual(1); + } + }); }); diff --git a/test/vitest-projects-config.test.ts b/test/vitest-projects-config.test.ts index 1283c091d27..d584fdbc476 100644 --- a/test/vitest-projects-config.test.ts +++ b/test/vitest-projects-config.test.ts @@ -1,4 +1,5 @@ -import { describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it } from "vitest"; +import { createPatternFileHelper } from "./helpers/pattern-file.js"; import { normalizeConfigPath, normalizeConfigPaths } from "./helpers/vitest-config-paths.js"; import { createAgentsVitestConfig } from "./vitest/vitest.agents.config.ts"; import bundledConfig from "./vitest/vitest.bundled.config.ts"; @@ -17,6 +18,12 @@ import { createUnitFastVitestConfig } from "./vitest/vitest.unit-fast.config.ts" import unitUiConfig from "./vitest/vitest.unit-ui.config.ts"; import { createUnitVitestConfig } from "./vitest/vitest.unit.config.ts"; +const patternFiles = createPatternFileHelper("openclaw-vitest-projects-config-"); + +afterEach(() => { + patternFiles.cleanup(); +}); + describe("projects vitest config", () => { it("defines the native root project list for all non-live Vitest lanes", () => { expect(baseConfig.test?.projects).toEqual([...rootVitestProjects]); @@ -57,6 +64,25 @@ describe("projects vitest config", () => { ]); }); + it("intersects contract include-file shards with the config family", () => { + const includeFile = patternFiles.writePatternFile("include.json", [ + "src/channels/plugins/contracts/surfaces-only.registry-backed-shard-b.contract.test.ts", + "src/channels/plugins/contracts/surfaces-only.registry-backed-shard-d.contract.test.ts", + "src/channels/plugins/contracts/directory.registry-backed-shard-a.contract.test.ts", + ]); + + const config = createContractsVitestConfig( + ["src/channels/plugins/contracts/*-shard-a.contract.test.ts"], + { + OPENCLAW_VITEST_INCLUDE_FILE: includeFile, + }, + ); + + expect(config.test.include).toEqual([ + "src/channels/plugins/contracts/directory.registry-backed-shard-a.contract.test.ts", + ]); + }); + it("keeps the root ui lane aligned with the isolated jsdom setup", () => { const config = createUiVitestConfig(); expect(config.test.environment).toBe("jsdom"); diff --git a/test/vitest/vitest.contracts-shared.ts b/test/vitest/vitest.contracts-shared.ts index c071c2eb11d..d588974e741 100644 --- a/test/vitest/vitest.contracts-shared.ts +++ b/test/vitest/vitest.contracts-shared.ts @@ -1,3 +1,4 @@ +import path from "node:path"; import { defineConfig } from "vitest/config"; import { loadPatternListFromEnv, narrowIncludePatternsForCli } from "./vitest.pattern-file.ts"; import { nonIsolatedRunnerPath, sharedVitestConfig } from "./vitest.shared.config.ts"; @@ -46,12 +47,35 @@ export function loadContractsIncludePatternsFromEnv( return loadPatternListFromEnv("OPENCLAW_VITEST_INCLUDE_FILE", env); } +function narrowContractIncludePatterns( + includePatterns: string[], + candidatePatterns: string[] | null, +): string[] | null { + if (!candidatePatterns) { + return null; + } + + return [ + ...new Set( + candidatePatterns.filter((candidate) => + includePatterns.some( + (pattern) => path.matchesGlob(candidate, pattern) || path.matchesGlob(pattern, candidate), + ), + ), + ), + ]; +} + export function createContractsVitestConfig( includePatterns: string[], env: Record = process.env, argv: string[] = process.argv, ) { const cliIncludePatterns = narrowIncludePatternsForCli(includePatterns, argv); + const envIncludePatterns = narrowContractIncludePatterns( + includePatterns, + loadContractsIncludePatternsFromEnv(env), + ); return defineConfig({ ...base, test: { @@ -62,7 +86,7 @@ export function createContractsVitestConfig( pool: "forks", runner: nonIsolatedRunnerPath, setupFiles: baseTest.setupFiles ?? [], - include: loadContractsIncludePatternsFromEnv(env) ?? cliIncludePatterns ?? includePatterns, + include: envIncludePatterns ?? cliIncludePatterns ?? includePatterns, passWithNoTests: true, }, });