diff --git a/docs/ci.md b/docs/ci.md index 082440b2979..fc8dadf98d8 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -53,7 +53,7 @@ Local changed-lane logic lives in `scripts/changed-lanes.mjs` and is executed by On pushes, the `checks` matrix adds the push-only `compat-node22` lane. On pull requests, that lane is skipped and the matrix stays focused on the normal test/channel lanes. -The slowest Node test families are split or balanced so each job stays small: channel contracts split registry and core coverage into eight weighted shards each, bundled plugin tests balance across six extension workers, auto-reply runs as three balanced workers instead of six tiny workers, and agentic gateway/plugin configs are spread across the existing source-only agentic Node jobs instead of waiting on built artifacts. The broad agents lane uses the shared Vitest file-parallel scheduler because it is import/scheduling dominated rather than owned by a single slow test file. `check-additional` keeps package-boundary compile/canary work together and separates it from runtime topology gateway/architecture work; the boundary guard shard runs its small independent guards concurrently inside one job, and the gateway watch regression uses the minimal `gatewayWatch` build profile instead of rebuilding the full CI artifact sidecar set. +The slowest Node test families are split or balanced so each job stays small: channel contracts split registry and core coverage into six weighted shards total, bundled plugin tests balance across six extension workers, auto-reply runs as three balanced workers instead of six tiny workers, and agentic gateway/plugin configs are spread across the existing source-only agentic Node jobs instead of waiting on built artifacts. Broad browser, QA, media, and miscellaneous plugin tests use their dedicated Vitest configs instead of the shared plugin catch-all. The broad agents lane uses the shared Vitest file-parallel scheduler because it is import/scheduling dominated rather than owned by a single slow test file. `runtime-config` runs with the infra core-runtime shard to keep the shared runtime shard from owning the tail. `check-additional` keeps package-boundary compile/canary work together and separates it from runtime topology gateway/architecture work; the boundary guard shard runs its small independent guards concurrently inside one job, and the gateway watch regression uses the minimal `gatewayWatch` build profile instead of rebuilding the full CI artifact sidecar set. GitHub may mark superseded jobs as `cancelled` when a newer push lands on the same PR or `main` ref. Treat that as CI noise unless the newest run for the same ref is also failing. Aggregate shard checks use `!cancelled() && always()` so they still report normal shard failures but do not queue after the whole workflow has already been superseded. The CI concurrency key is versioned (`CI-v6-*`) so a GitHub-side zombie in an old queue group cannot indefinitely block newer main runs. @@ -85,4 +85,5 @@ pnpm test:channels pnpm test:contracts:channels pnpm check:docs # docs format + lint + broken links pnpm build # build dist when CI artifact/build-smoke lanes matter +node scripts/ci-run-timings.mjs # summarize wall time, queue time, and slowest jobs ``` diff --git a/scripts/ci-run-timings.mjs b/scripts/ci-run-timings.mjs new file mode 100644 index 00000000000..028d1387521 --- /dev/null +++ b/scripts/ci-run-timings.mjs @@ -0,0 +1,119 @@ +#!/usr/bin/env node + +import { execFileSync } from "node:child_process"; + +function parseTime(value) { + if (!value || value === "0001-01-01T00:00:00Z") { + return null; + } + const parsed = Date.parse(value); + return Number.isFinite(parsed) ? parsed : null; +} + +function secondsBetween(start, end) { + return start !== null && end !== null ? Math.round((end - start) / 1000) : null; +} + +function formatSeconds(value) { + return value === null ? "" : `${value}s`; +} + +export function summarizeRunTimings(run, limit = 15) { + const created = parseTime(run.createdAt); + const updated = parseTime(run.updatedAt); + const jobs = (run.jobs ?? []) + .filter((job) => !job.name?.startsWith("matrix.")) + .map((job) => { + const started = parseTime(job.startedAt); + const completed = parseTime(job.completedAt); + return { + conclusion: job.conclusion ?? "", + durationSeconds: secondsBetween(started, completed), + name: job.name, + queueSeconds: secondsBetween(created, started), + status: job.status, + }; + }); + const byDuration = [...jobs] + .filter((job) => job.durationSeconds !== null) + .toSorted((left, right) => right.durationSeconds - left.durationSeconds) + .slice(0, limit); + const byQueue = [...jobs] + .filter((job) => job.queueSeconds !== null && (job.durationSeconds ?? 0) > 5) + .toSorted((left, right) => right.queueSeconds - left.queueSeconds) + .slice(0, limit); + const badJobs = jobs.filter( + (job) => job.conclusion && !["success", "skipped", "cancelled"].includes(job.conclusion), + ); + + return { + byDuration, + byQueue, + conclusion: run.conclusion ?? "", + status: run.status ?? "", + wallSeconds: secondsBetween(created, updated), + badJobs, + }; +} + +function getLatestCiRunId() { + const raw = execFileSync( + "gh", + ["run", "list", "--branch", "main", "--workflow", "CI", "--limit", "1", "--json", "databaseId"], + { encoding: "utf8" }, + ); + const runs = JSON.parse(raw); + const runId = runs[0]?.databaseId; + if (!runId) { + throw new Error("No CI runs found on main"); + } + return String(runId); +} + +function loadRun(runId) { + return JSON.parse( + execFileSync( + "gh", + ["run", "view", runId, "--json", "status,conclusion,createdAt,updatedAt,jobs"], + { + encoding: "utf8", + }, + ), + ); +} + +function printSection(title, jobs, metric) { + console.log(title); + for (const job of jobs) { + console.log( + `${String(job.name).padEnd(48)} ${formatSeconds(job[metric]).padStart(6)} queue=${formatSeconds(job.queueSeconds).padStart(6)} ${job.status}/${job.conclusion}`, + ); + } +} + +async function main() { + const args = process.argv.slice(2); + const limitIndex = args.indexOf("--limit"); + const limit = + limitIndex === -1 ? 15 : Math.max(1, Number.parseInt(args[limitIndex + 1] ?? "", 10) || 15); + const runId = + args.find((arg, index) => index !== limitIndex && index !== limitIndex + 1) ?? + getLatestCiRunId(); + const summary = summarizeRunTimings(loadRun(runId), limit); + + console.log( + `CI run ${runId}: ${summary.status}/${summary.conclusion} wall=${formatSeconds(summary.wallSeconds)}`, + ); + printSection("\nSlowest jobs", summary.byDuration, "durationSeconds"); + printSection("\nLongest queues", summary.byQueue, "queueSeconds"); + if (summary.badJobs.length > 0) { + console.log("\nFailed jobs"); + for (const job of summary.badJobs) { + console.log(`${job.name} ${job.status}/${job.conclusion}`); + } + } +} + +if (import.meta.url === `file://${process.argv[1]}`) { + await main(); +} diff --git a/scripts/lib/channel-contract-test-plan.mjs b/scripts/lib/channel-contract-test-plan.mjs index 57b541f04f4..f533cb4f66b 100644 --- a/scripts/lib/channel-contract-test-plan.mjs +++ b/scripts/lib/channel-contract-test-plan.mjs @@ -39,7 +39,7 @@ function resolveContractFileWeight(file) { export function createChannelContractTestShards() { const rootDir = "src/channels/plugins/contracts"; - const suffixes = ["a", "b", "c", "d"]; + const suffixes = ["a", "b", "c"]; const groups = Object.fromEntries( ["registry", "core"].flatMap((family) => suffixes.map((suffix) => [`checks-fast-contracts-channels-${family}-${suffix}`, []]), diff --git a/scripts/lib/ci-node-test-plan.mjs b/scripts/lib/ci-node-test-plan.mjs index f4b9ab96fbc..3025962c415 100644 --- a/scripts/lib/ci-node-test-plan.mjs +++ b/scripts/lib/ci-node-test-plan.mjs @@ -98,6 +98,7 @@ const SPLIT_NODE_SHARDS = new Map([ "test/vitest/vitest.secrets.config.ts", "test/vitest/vitest.logging.config.ts", "test/vitest/vitest.process.config.ts", + "test/vitest/vitest.runtime-config.config.ts", ], requiresDist: false, }, @@ -117,7 +118,6 @@ const SPLIT_NODE_SHARDS = new Map([ configs: [ "test/vitest/vitest.acp.config.ts", "test/vitest/vitest.cron.config.ts", - "test/vitest/vitest.runtime-config.config.ts", "test/vitest/vitest.shared-core.config.ts", "test/vitest/vitest.tasks.config.ts", "test/vitest/vitest.utils.config.ts", diff --git a/scripts/lib/extension-test-plan.mjs b/scripts/lib/extension-test-plan.mjs index edeaad7ce57..1369b06df5c 100644 --- a/scripts/lib/extension-test-plan.mjs +++ b/scripts/lib/extension-test-plan.mjs @@ -3,19 +3,23 @@ import path from "node:path"; import { channelTestRoots } from "../../test/vitest/vitest.channel-paths.mjs"; import { isAcpxExtensionRoot } from "../../test/vitest/vitest.extension-acpx-paths.mjs"; import { isBlueBubblesExtensionRoot } from "../../test/vitest/vitest.extension-bluebubbles-paths.mjs"; +import { isBrowserExtensionRoot } from "../../test/vitest/vitest.extension-browser-paths.mjs"; import { resolveSplitChannelExtensionShard } from "../../test/vitest/vitest.extension-channel-split-paths.mjs"; import { isDiffsExtensionRoot } from "../../test/vitest/vitest.extension-diffs-paths.mjs"; import { isFeishuExtensionRoot } from "../../test/vitest/vitest.extension-feishu-paths.mjs"; import { isIrcExtensionRoot } from "../../test/vitest/vitest.extension-irc-paths.mjs"; import { isMatrixExtensionRoot } from "../../test/vitest/vitest.extension-matrix-paths.mjs"; import { isMattermostExtensionRoot } from "../../test/vitest/vitest.extension-mattermost-paths.mjs"; +import { isMediaExtensionRoot } from "../../test/vitest/vitest.extension-media-paths.mjs"; import { isMemoryExtensionRoot } from "../../test/vitest/vitest.extension-memory-paths.mjs"; import { isMessagingExtensionRoot } from "../../test/vitest/vitest.extension-messaging-paths.mjs"; +import { isMiscExtensionRoot } from "../../test/vitest/vitest.extension-misc-paths.mjs"; import { isMsTeamsExtensionRoot } from "../../test/vitest/vitest.extension-msteams-paths.mjs"; import { isProviderExtensionRoot, isProviderOpenAiExtensionRoot, } from "../../test/vitest/vitest.extension-provider-paths.mjs"; +import { isQaExtensionRoot } from "../../test/vitest/vitest.extension-qa-paths.mjs"; import { isTelegramExtensionRoot } from "../../test/vitest/vitest.extension-telegram-paths.mjs"; import { isVoiceCallExtensionRoot } from "../../test/vitest/vitest.extension-voice-call-paths.mjs"; import { isWhatsAppExtensionRoot } from "../../test/vitest/vitest.extension-whatsapp-paths.mjs"; @@ -118,16 +122,20 @@ export function resolveExtensionTestPlan(params = {}) { const splitChannelShard = resolveSplitChannelExtensionShard(relativeExtensionDir); const usesChannelConfig = roots.some((root) => channelTestRoots.includes(root)); const usesAcpxConfig = roots.some((root) => isAcpxExtensionRoot(root)); + const usesBrowserConfig = roots.some((root) => isBrowserExtensionRoot(root)); const usesDiffsConfig = roots.some((root) => isDiffsExtensionRoot(root)); const usesBlueBubblesConfig = roots.some((root) => isBlueBubblesExtensionRoot(root)); const usesFeishuConfig = roots.some((root) => isFeishuExtensionRoot(root)); const usesIrcConfig = roots.some((root) => isIrcExtensionRoot(root)); const usesMattermostConfig = roots.some((root) => isMattermostExtensionRoot(root)); + const usesMediaConfig = roots.some((root) => isMediaExtensionRoot(root)); + const usesMiscConfig = roots.some((root) => isMiscExtensionRoot(root)); const usesTelegramConfig = roots.some((root) => isTelegramExtensionRoot(root)); const usesVoiceCallConfig = roots.some((root) => isVoiceCallExtensionRoot(root)); const usesWhatsAppConfig = roots.some((root) => isWhatsAppExtensionRoot(root)); const usesZaloConfig = roots.some((root) => isZaloExtensionRoot(root)); const usesMatrixConfig = roots.some((root) => isMatrixExtensionRoot(root)); + const usesQaConfig = roots.some((root) => isQaExtensionRoot(root)); const usesMemoryConfig = roots.some((root) => isMemoryExtensionRoot(root)); const usesMsTeamsConfig = roots.some((root) => isMsTeamsExtensionRoot(root)); const usesMessagingConfig = roots.some((root) => isMessagingExtensionRoot(root)); @@ -139,37 +147,45 @@ export function resolveExtensionTestPlan(params = {}) { ? "test/vitest/vitest.extension-channels.config.ts" : usesAcpxConfig ? "test/vitest/vitest.extension-acpx.config.ts" - : usesDiffsConfig - ? "test/vitest/vitest.extension-diffs.config.ts" - : usesBlueBubblesConfig - ? "test/vitest/vitest.extension-bluebubbles.config.ts" - : usesFeishuConfig - ? "test/vitest/vitest.extension-feishu.config.ts" - : usesIrcConfig - ? "test/vitest/vitest.extension-irc.config.ts" - : usesMattermostConfig - ? "test/vitest/vitest.extension-mattermost.config.ts" - : usesMatrixConfig - ? "test/vitest/vitest.extension-matrix.config.ts" - : usesTelegramConfig - ? "test/vitest/vitest.extension-telegram.config.ts" - : usesVoiceCallConfig - ? "test/vitest/vitest.extension-voice-call.config.ts" - : usesWhatsAppConfig - ? "test/vitest/vitest.extension-whatsapp.config.ts" - : usesZaloConfig - ? "test/vitest/vitest.extension-zalo.config.ts" - : usesMemoryConfig - ? "test/vitest/vitest.extension-memory.config.ts" + : usesBlueBubblesConfig + ? "test/vitest/vitest.extension-bluebubbles.config.ts" + : usesBrowserConfig + ? "test/vitest/vitest.extension-browser.config.ts" + : usesDiffsConfig + ? "test/vitest/vitest.extension-diffs.config.ts" + : usesFeishuConfig + ? "test/vitest/vitest.extension-feishu.config.ts" + : usesIrcConfig + ? "test/vitest/vitest.extension-irc.config.ts" + : usesMattermostConfig + ? "test/vitest/vitest.extension-mattermost.config.ts" + : usesMatrixConfig + ? "test/vitest/vitest.extension-matrix.config.ts" + : usesMediaConfig + ? "test/vitest/vitest.extension-media.config.ts" + : usesMemoryConfig + ? "test/vitest/vitest.extension-memory.config.ts" + : usesMessagingConfig + ? "test/vitest/vitest.extension-messaging.config.ts" + : usesMiscConfig + ? "test/vitest/vitest.extension-misc.config.ts" : usesMsTeamsConfig ? "test/vitest/vitest.extension-msteams.config.ts" - : usesMessagingConfig - ? "test/vitest/vitest.extension-messaging.config.ts" - : usesProviderOpenAiConfig - ? "test/vitest/vitest.extension-provider-openai.config.ts" - : usesProviderConfig - ? "test/vitest/vitest.extension-providers.config.ts" - : "test/vitest/vitest.extensions.config.ts"; + : usesQaConfig + ? "test/vitest/vitest.extension-qa.config.ts" + : usesTelegramConfig + ? "test/vitest/vitest.extension-telegram.config.ts" + : usesVoiceCallConfig + ? "test/vitest/vitest.extension-voice-call.config.ts" + : usesWhatsAppConfig + ? "test/vitest/vitest.extension-whatsapp.config.ts" + : usesZaloConfig + ? "test/vitest/vitest.extension-zalo.config.ts" + : usesProviderOpenAiConfig + ? "test/vitest/vitest.extension-provider-openai.config.ts" + : usesProviderConfig + ? "test/vitest/vitest.extension-providers.config.ts" + : "test/vitest/vitest.extensions.config.ts"; const testFileCount = roots.reduce( (sum, root) => sum + countTestFiles(path.join(repoRoot, root)), 0, diff --git a/src/auto-reply/reply/commands-session-lifecycle.test.ts b/src/auto-reply/reply/commands-session-lifecycle.test.ts index 25bc98b890a..68590373ddb 100644 --- a/src/auto-reply/reply/commands-session-lifecycle.test.ts +++ b/src/auto-reply/reply/commands-session-lifecycle.test.ts @@ -198,6 +198,8 @@ vi.mock("../../plugins/runtime.js", () => { vi.mock("../../channels/plugins/index.js", () => ({ getChannelPlugin: (channelId: string) => hoisted.runtimeChannelRegistry.channels.find((entry) => entry.plugin.id === channelId)?.plugin, + getLoadedChannelPlugin: (channelId: string) => + hoisted.runtimeChannelRegistry.channels.find((entry) => entry.plugin.id === channelId)?.plugin, normalizeChannelId: (raw?: string | null) => { const normalized = raw?.trim().toLowerCase(); return normalized || null; diff --git a/test/scripts/channel-contract-test-plan.test.ts b/test/scripts/channel-contract-test-plan.test.ts index e69b4e0de7c..9f3c0ee80c4 100644 --- a/test/scripts/channel-contract-test-plan.test.ts +++ b/test/scripts/channel-contract-test-plan.test.ts @@ -16,7 +16,7 @@ function listContractTests(rootDir = "src/channels/plugins/contracts"): string[] describe("scripts/lib/channel-contract-test-plan.mjs", () => { it("splits channel contracts into focused shards", () => { - const suffixes = ["a", "b", "c", "d"]; + const suffixes = ["a", "b", "c"]; expect( createChannelContractTestShards().map((shard) => ({ @@ -51,7 +51,7 @@ describe("scripts/lib/channel-contract-test-plan.mjs", () => { const surfaceRegistryFiles = shard.includePatterns.filter((pattern) => pattern.includes("/surfaces-only.registry-backed-shard-"), ); - expect(surfaceRegistryFiles.length).toBeLessThanOrEqual(2); + expect(surfaceRegistryFiles.length).toBeLessThanOrEqual(3); } }); }); diff --git a/test/scripts/ci-node-test-plan.test.ts b/test/scripts/ci-node-test-plan.test.ts index bbccd09594a..484a7f0f573 100644 --- a/test/scripts/ci-node-test-plan.test.ts +++ b/test/scripts/ci-node-test-plan.test.ts @@ -73,6 +73,7 @@ describe("scripts/lib/ci-node-test-plan.mjs", () => { "test/vitest/vitest.secrets.config.ts", "test/vitest/vitest.logging.config.ts", "test/vitest/vitest.process.config.ts", + "test/vitest/vitest.runtime-config.config.ts", ], requiresDist: false, shardName: "core-runtime-infra", @@ -92,7 +93,6 @@ describe("scripts/lib/ci-node-test-plan.mjs", () => { configs: [ "test/vitest/vitest.acp.config.ts", "test/vitest/vitest.cron.config.ts", - "test/vitest/vitest.runtime-config.config.ts", "test/vitest/vitest.shared-core.config.ts", "test/vitest/vitest.tasks.config.ts", "test/vitest/vitest.utils.config.ts", diff --git a/test/scripts/ci-run-timings.test.ts b/test/scripts/ci-run-timings.test.ts new file mode 100644 index 00000000000..108f7369276 --- /dev/null +++ b/test/scripts/ci-run-timings.test.ts @@ -0,0 +1,49 @@ +import { describe, expect, it } from "vitest"; +import { summarizeRunTimings } from "../../scripts/ci-run-timings.mjs"; + +describe("scripts/ci-run-timings.mjs", () => { + it("separates queue time from job duration", () => { + const summary = summarizeRunTimings( + { + conclusion: "success", + createdAt: "2026-04-22T10:00:00Z", + jobs: [ + { + completedAt: "2026-04-22T10:01:20Z", + conclusion: "success", + name: "slow", + startedAt: "2026-04-22T10:00:20Z", + status: "completed", + }, + { + completedAt: "2026-04-22T10:01:00Z", + conclusion: "success", + name: "queued", + startedAt: "2026-04-22T10:00:50Z", + status: "completed", + }, + { + completedAt: "2026-04-22T10:00:01Z", + conclusion: "skipped", + name: "matrix.check_name", + startedAt: "2026-04-22T10:00:01Z", + status: "completed", + }, + ], + status: "completed", + updatedAt: "2026-04-22T10:01:30Z", + }, + 2, + ); + + expect(summary.wallSeconds).toBe(90); + expect(summary.byDuration.map((job) => [job.name, job.durationSeconds])).toEqual([ + ["slow", 60], + ["queued", 10], + ]); + expect(summary.byQueue.map((job) => [job.name, job.queueSeconds])).toEqual([ + ["queued", 50], + ["slow", 20], + ]); + }); +}); diff --git a/test/scripts/test-extension.test.ts b/test/scripts/test-extension.test.ts index 704ec3376c5..0853c97eaf0 100644 --- a/test/scripts/test-extension.test.ts +++ b/test/scripts/test-extension.test.ts @@ -173,12 +173,27 @@ describe("scripts/test-extension.mjs", () => { expect(plan.hasTests).toBe(true); }); - it("keeps non-provider extensions on the shared extensions vitest config", () => { - const plan = resolveExtensionTestPlan({ targetArg: "firecrawl", cwd: process.cwd() }); + it("resolves broad dedicated extension groups onto their narrow vitest configs", () => { + expect(resolveExtensionTestPlan({ targetArg: "browser", cwd: process.cwd() }).config).toBe( + "test/vitest/vitest.extension-browser.config.ts", + ); + expect(resolveExtensionTestPlan({ targetArg: "qa-lab", cwd: process.cwd() }).config).toBe( + "test/vitest/vitest.extension-qa.config.ts", + ); + expect(resolveExtensionTestPlan({ targetArg: "vydra", cwd: process.cwd() }).config).toBe( + "test/vitest/vitest.extension-media.config.ts", + ); + expect(resolveExtensionTestPlan({ targetArg: "firecrawl", cwd: process.cwd() }).config).toBe( + "test/vitest/vitest.extension-misc.config.ts", + ); + }); - expect(plan.extensionId).toBe("firecrawl"); + it("keeps unmatched non-provider extensions on the shared extensions vitest config", () => { + const plan = resolveExtensionTestPlan({ targetArg: "codex", cwd: process.cwd() }); + + expect(plan.extensionId).toBe("codex"); expect(plan.config).toBe("test/vitest/vitest.extensions.config.ts"); - expect(plan.roots).toContain(bundledPluginRoot("firecrawl")); + expect(plan.roots).toContain(bundledPluginRoot("codex")); expect(plan.hasTests).toBe(true); }); @@ -260,12 +275,16 @@ describe("scripts/test-extension.mjs", () => { "bluebubbles", "acpx", "diffs", + "browser", + "qa-lab", + "vydra", ], }); expect(batch.extensionIds).toEqual([ "acpx", "bluebubbles", + "browser", "diffs", "feishu", "firecrawl", @@ -276,9 +295,11 @@ describe("scripts/test-extension.mjs", () => { "memory-core", "msteams", "openai", + "qa-lab", "slack", "telegram", "voice-call", + "vydra", "whatsapp", "zalo", "zalouser", @@ -298,6 +319,13 @@ describe("scripts/test-extension.mjs", () => { roots: [bundledPluginRoot("bluebubbles")], testFileCount: expect.any(Number), }, + { + config: "test/vitest/vitest.extension-browser.config.ts", + estimatedCost: expect.any(Number), + extensionIds: ["browser"], + roots: [bundledPluginRoot("browser")], + testFileCount: expect.any(Number), + }, { config: "test/vitest/vitest.extension-diffs.config.ts", estimatedCost: expect.any(Number), @@ -340,6 +368,13 @@ describe("scripts/test-extension.mjs", () => { roots: [bundledPluginRoot("mattermost")], testFileCount: expect.any(Number), }, + { + config: "test/vitest/vitest.extension-media.config.ts", + estimatedCost: expect.any(Number), + extensionIds: ["vydra"], + roots: [bundledPluginRoot("vydra")], + testFileCount: expect.any(Number), + }, { config: "test/vitest/vitest.extension-memory.config.ts", estimatedCost: expect.any(Number), @@ -347,6 +382,13 @@ describe("scripts/test-extension.mjs", () => { roots: [bundledPluginRoot("memory-core")], testFileCount: expect.any(Number), }, + { + config: "test/vitest/vitest.extension-misc.config.ts", + estimatedCost: expect.any(Number), + extensionIds: ["firecrawl"], + roots: [bundledPluginRoot("firecrawl")], + testFileCount: expect.any(Number), + }, { config: "test/vitest/vitest.extension-msteams.config.ts", estimatedCost: expect.any(Number), @@ -361,6 +403,13 @@ describe("scripts/test-extension.mjs", () => { roots: [bundledPluginRoot("openai")], testFileCount: expect.any(Number), }, + { + config: "test/vitest/vitest.extension-qa.config.ts", + estimatedCost: expect.any(Number), + extensionIds: ["qa-lab"], + roots: [bundledPluginRoot("qa-lab")], + testFileCount: expect.any(Number), + }, { config: "test/vitest/vitest.extension-slack.config.ts", estimatedCost: expect.any(Number), @@ -396,13 +445,6 @@ describe("scripts/test-extension.mjs", () => { roots: [bundledPluginRoot("zalo"), bundledPluginRoot("zalouser")], testFileCount: expect.any(Number), }, - { - config: "test/vitest/vitest.extensions.config.ts", - estimatedCost: expect.any(Number), - extensionIds: ["firecrawl"], - roots: [bundledPluginRoot("firecrawl")], - testFileCount: expect.any(Number), - }, ]); }); diff --git a/test/vitest-scoped-config.test.ts b/test/vitest-scoped-config.test.ts index 08d126c3e9b..a640a96b529 100644 --- a/test/vitest-scoped-config.test.ts +++ b/test/vitest-scoped-config.test.ts @@ -20,6 +20,7 @@ import { createCronVitestConfig } from "./vitest/vitest.cron.config.ts"; import { createDaemonVitestConfig } from "./vitest/vitest.daemon.config.ts"; import { createExtensionAcpxVitestConfig } from "./vitest/vitest.extension-acpx.config.ts"; import { createExtensionBlueBubblesVitestConfig } from "./vitest/vitest.extension-bluebubbles.config.ts"; +import { createExtensionBrowserVitestConfig } from "./vitest/vitest.extension-browser.config.ts"; import { createExtensionChannelsVitestConfig } from "./vitest/vitest.extension-channels.config.ts"; import { createExtensionDiffsVitestConfig } from "./vitest/vitest.extension-diffs.config.ts"; import { createExtensionDiscordVitestConfig } from "./vitest/vitest.extension-discord.config.ts"; @@ -29,11 +30,14 @@ import { createExtensionIrcVitestConfig } from "./vitest/vitest.extension-irc.co import { createExtensionLineVitestConfig } from "./vitest/vitest.extension-line.config.ts"; import { createExtensionMatrixVitestConfig } from "./vitest/vitest.extension-matrix.config.ts"; import { createExtensionMattermostVitestConfig } from "./vitest/vitest.extension-mattermost.config.ts"; +import { createExtensionMediaVitestConfig } from "./vitest/vitest.extension-media.config.ts"; import { createExtensionMemoryVitestConfig } from "./vitest/vitest.extension-memory.config.ts"; import { createExtensionMessagingVitestConfig } from "./vitest/vitest.extension-messaging.config.ts"; +import { createExtensionMiscVitestConfig } from "./vitest/vitest.extension-misc.config.ts"; import { createExtensionMsTeamsVitestConfig } from "./vitest/vitest.extension-msteams.config.ts"; import { createExtensionProviderOpenAiVitestConfig } from "./vitest/vitest.extension-provider-openai.config.ts"; import { createExtensionProvidersVitestConfig } from "./vitest/vitest.extension-providers.config.ts"; +import { createExtensionQaVitestConfig } from "./vitest/vitest.extension-qa.config.ts"; import { createExtensionSignalVitestConfig } from "./vitest/vitest.extension-signal.config.ts"; import { createExtensionSlackVitestConfig } from "./vitest/vitest.extension-slack.config.ts"; import { createExtensionTelegramVitestConfig } from "./vitest/vitest.extension-telegram.config.ts"; @@ -188,6 +192,7 @@ describe("scoped vitest configs", () => { const defaultExtensionAcpxConfig = createExtensionAcpxVitestConfig({}); const defaultExtensionBlueBubblesConfig = createExtensionBlueBubblesVitestConfig({}); const defaultExtensionChannelsConfig = createExtensionChannelsVitestConfig({}); + const defaultExtensionBrowserConfig = createExtensionBrowserVitestConfig({}); const defaultExtensionDiffsConfig = createExtensionDiffsVitestConfig({}); const defaultExtensionDiscordConfig = createExtensionDiscordVitestConfig({}); const defaultExtensionFeishuConfig = createExtensionFeishuVitestConfig({}); @@ -196,11 +201,14 @@ describe("scoped vitest configs", () => { const defaultExtensionLineConfig = createExtensionLineVitestConfig({}); const defaultExtensionMatrixConfig = createExtensionMatrixVitestConfig({}); const defaultExtensionMattermostConfig = createExtensionMattermostVitestConfig({}); + const defaultExtensionMediaConfig = createExtensionMediaVitestConfig({}); const defaultExtensionMemoryConfig = createExtensionMemoryVitestConfig({}); + const defaultExtensionMiscConfig = createExtensionMiscVitestConfig({}); const defaultExtensionMsTeamsConfig = createExtensionMsTeamsVitestConfig({}); const defaultExtensionMessagingConfig = createExtensionMessagingVitestConfig({}); const defaultExtensionProviderOpenAiConfig = createExtensionProviderOpenAiVitestConfig({}); const defaultExtensionProvidersConfig = createExtensionProvidersVitestConfig({}); + const defaultExtensionQaConfig = createExtensionQaVitestConfig({}); const defaultExtensionSignalConfig = createExtensionSignalVitestConfig({}); const defaultExtensionSlackConfig = createExtensionSlackVitestConfig({}); const defaultExtensionTelegramConfig = createExtensionTelegramVitestConfig({}); @@ -602,6 +610,22 @@ describe("scoped vitest configs", () => { ).toBe(true); }); + it("keeps broad dedicated extension groups out of the shared extensions lane", () => { + const extensionExcludes = defaultExtensionsConfig.test?.exclude ?? []; + expect(defaultExtensionBrowserConfig.test?.include).toContain("browser/**/*.test.ts"); + expect(defaultExtensionMediaConfig.test?.include).toContain("vydra/**/*.test.ts"); + expect(defaultExtensionMiscConfig.test?.include).toContain("firecrawl/**/*.test.ts"); + expect(defaultExtensionQaConfig.test?.include).toContain("qa-lab/**/*.test.ts"); + for (const file of [ + "browser/src/browser/pw.test.ts", + "vydra/src/index.test.ts", + "firecrawl/src/index.test.ts", + "qa-lab/src/index.test.ts", + ]) { + expect(extensionExcludes.some((pattern) => path.matchesGlob(file, pattern))).toBe(true); + } + }); + it("normalizes gateway include patterns relative to the scoped dir", () => { expect(defaultGatewayConfig.test?.dir).toBe(path.join(process.cwd(), "src", "gateway")); expect(defaultGatewayConfig.test?.include).toEqual(["**/*.test.ts"]); diff --git a/test/vitest/vitest.extension-browser-paths.mjs b/test/vitest/vitest.extension-browser-paths.mjs new file mode 100644 index 00000000000..fe424535dac --- /dev/null +++ b/test/vitest/vitest.extension-browser-paths.mjs @@ -0,0 +1,5 @@ +export const browserExtensionTestRoots = ["extensions/browser"]; + +export function isBrowserExtensionRoot(root) { + return browserExtensionTestRoots.includes(root); +} diff --git a/test/vitest/vitest.extension-browser.config.ts b/test/vitest/vitest.extension-browser.config.ts index e08b7d723af..4cc8a8f3f7d 100644 --- a/test/vitest/vitest.extension-browser.config.ts +++ b/test/vitest/vitest.extension-browser.config.ts @@ -1,8 +1,27 @@ +import { browserExtensionTestRoots } from "./vitest.extension-browser-paths.mjs"; +import { loadPatternListFromEnv } from "./vitest.pattern-file.ts"; import { createScopedVitestConfig } from "./vitest.scoped-config.ts"; -export default createScopedVitestConfig(["browser/**/*.test.ts"], { - dir: "extensions", - name: "extension-browser", - passWithNoTests: true, - setupFiles: ["test/setup.extensions.ts"], -}); +export function loadIncludePatternsFromEnv( + env: Record = process.env, +): string[] | null { + return loadPatternListFromEnv("OPENCLAW_VITEST_INCLUDE_FILE", env); +} + +export function createExtensionBrowserVitestConfig( + env: Record = process.env, +) { + return createScopedVitestConfig( + loadIncludePatternsFromEnv(env) ?? + browserExtensionTestRoots.map((root) => `${root}/**/*.test.ts`), + { + dir: "extensions", + env, + name: "extension-browser", + passWithNoTests: true, + setupFiles: ["test/setup.extensions.ts"], + }, + ); +} + +export default createExtensionBrowserVitestConfig(); diff --git a/test/vitest/vitest.extension-media-paths.mjs b/test/vitest/vitest.extension-media-paths.mjs new file mode 100644 index 00000000000..86234b042e0 --- /dev/null +++ b/test/vitest/vitest.extension-media-paths.mjs @@ -0,0 +1,16 @@ +export const mediaExtensionTestRoots = [ + "extensions/alibaba", + "extensions/deepgram", + "extensions/elevenlabs", + "extensions/fal", + "extensions/image-generation-core", + "extensions/runway", + "extensions/talk-voice", + "extensions/video-generation-core", + "extensions/vydra", + "extensions/xiaomi", +]; + +export function isMediaExtensionRoot(root) { + return mediaExtensionTestRoots.includes(root); +} diff --git a/test/vitest/vitest.extension-media.config.ts b/test/vitest/vitest.extension-media.config.ts index c8de79d87f4..4e7826e16bc 100644 --- a/test/vitest/vitest.extension-media.config.ts +++ b/test/vitest/vitest.extension-media.config.ts @@ -1,22 +1,27 @@ +import { mediaExtensionTestRoots } from "./vitest.extension-media-paths.mjs"; +import { loadPatternListFromEnv } from "./vitest.pattern-file.ts"; import { createScopedVitestConfig } from "./vitest.scoped-config.ts"; -export default createScopedVitestConfig( - [ - "alibaba/**/*.test.ts", - "deepgram/**/*.test.ts", - "elevenlabs/**/*.test.ts", - "fal/**/*.test.ts", - "image-generation-core/**/*.test.ts", - "runway/**/*.test.ts", - "talk-voice/**/*.test.ts", - "video-generation-core/**/*.test.ts", - "vydra/**/*.test.ts", - "xiaomi/**/*.test.ts", - ], - { - dir: "extensions", - name: "extension-media", - passWithNoTests: true, - setupFiles: ["test/setup.extensions.ts"], - }, -); +export function loadIncludePatternsFromEnv( + env: Record = process.env, +): string[] | null { + return loadPatternListFromEnv("OPENCLAW_VITEST_INCLUDE_FILE", env); +} + +export function createExtensionMediaVitestConfig( + env: Record = process.env, +) { + return createScopedVitestConfig( + loadIncludePatternsFromEnv(env) ?? + mediaExtensionTestRoots.map((root) => `${root}/**/*.test.ts`), + { + dir: "extensions", + env, + name: "extension-media", + passWithNoTests: true, + setupFiles: ["test/setup.extensions.ts"], + }, + ); +} + +export default createExtensionMediaVitestConfig(); diff --git a/test/vitest/vitest.extension-misc-paths.mjs b/test/vitest/vitest.extension-misc-paths.mjs new file mode 100644 index 00000000000..bc40edbd344 --- /dev/null +++ b/test/vitest/vitest.extension-misc-paths.mjs @@ -0,0 +1,29 @@ +export const miscExtensionTestRoots = [ + "extensions/arcee", + "extensions/brave", + "extensions/device-pair", + "extensions/diagnostics-otel", + "extensions/duckduckgo", + "extensions/exa", + "extensions/firecrawl", + "extensions/fireworks", + "extensions/kilocode", + "extensions/litellm", + "extensions/llm-task", + "extensions/lobster", + "extensions/opencode", + "extensions/opencode-go", + "extensions/openshell", + "extensions/perplexity", + "extensions/phone-control", + "extensions/searxng", + "extensions/synthetic", + "extensions/tavily", + "extensions/thread-ownership", + "extensions/vercel-ai-gateway", + "extensions/webhooks", +]; + +export function isMiscExtensionRoot(root) { + return miscExtensionTestRoots.includes(root); +} diff --git a/test/vitest/vitest.extension-misc.config.ts b/test/vitest/vitest.extension-misc.config.ts index 30aaa5c4b65..2db79b89e48 100644 --- a/test/vitest/vitest.extension-misc.config.ts +++ b/test/vitest/vitest.extension-misc.config.ts @@ -1,35 +1,26 @@ +import { miscExtensionTestRoots } from "./vitest.extension-misc-paths.mjs"; +import { loadPatternListFromEnv } from "./vitest.pattern-file.ts"; import { createScopedVitestConfig } from "./vitest.scoped-config.ts"; -export default createScopedVitestConfig( - [ - "arcee/**/*.test.ts", - "brave/**/*.test.ts", - "device-pair/**/*.test.ts", - "diagnostics-otel/**/*.test.ts", - "duckduckgo/**/*.test.ts", - "exa/**/*.test.ts", - "firecrawl/**/*.test.ts", - "fireworks/**/*.test.ts", - "kilocode/**/*.test.ts", - "litellm/**/*.test.ts", - "llm-task/**/*.test.ts", - "lobster/**/*.test.ts", - "opencode/**/*.test.ts", - "opencode-go/**/*.test.ts", - "openshell/**/*.test.ts", - "perplexity/**/*.test.ts", - "phone-control/**/*.test.ts", - "searxng/**/*.test.ts", - "synthetic/**/*.test.ts", - "tavily/**/*.test.ts", - "thread-ownership/**/*.test.ts", - "vercel-ai-gateway/**/*.test.ts", - "webhooks/**/*.test.ts", - ], - { - dir: "extensions", - name: "extension-misc", - passWithNoTests: true, - setupFiles: ["test/setup.extensions.ts"], - }, -); +export function loadIncludePatternsFromEnv( + env: Record = process.env, +): string[] | null { + return loadPatternListFromEnv("OPENCLAW_VITEST_INCLUDE_FILE", env); +} + +export function createExtensionMiscVitestConfig( + env: Record = process.env, +) { + return createScopedVitestConfig( + loadIncludePatternsFromEnv(env) ?? miscExtensionTestRoots.map((root) => `${root}/**/*.test.ts`), + { + dir: "extensions", + env, + name: "extension-misc", + passWithNoTests: true, + setupFiles: ["test/setup.extensions.ts"], + }, + ); +} + +export default createExtensionMiscVitestConfig(); diff --git a/test/vitest/vitest.extension-qa-paths.mjs b/test/vitest/vitest.extension-qa-paths.mjs new file mode 100644 index 00000000000..dc8bd273b86 --- /dev/null +++ b/test/vitest/vitest.extension-qa-paths.mjs @@ -0,0 +1,5 @@ +export const qaExtensionTestRoots = ["extensions/qa-channel", "extensions/qa-lab"]; + +export function isQaExtensionRoot(root) { + return qaExtensionTestRoots.includes(root); +} diff --git a/test/vitest/vitest.extension-qa.config.ts b/test/vitest/vitest.extension-qa.config.ts index 33466f7a28e..e6441026a5c 100644 --- a/test/vitest/vitest.extension-qa.config.ts +++ b/test/vitest/vitest.extension-qa.config.ts @@ -1,8 +1,26 @@ +import { qaExtensionTestRoots } from "./vitest.extension-qa-paths.mjs"; +import { loadPatternListFromEnv } from "./vitest.pattern-file.ts"; import { createScopedVitestConfig } from "./vitest.scoped-config.ts"; -export default createScopedVitestConfig(["qa-channel/**/*.test.ts", "qa-lab/**/*.test.ts"], { - dir: "extensions", - name: "extension-qa", - passWithNoTests: true, - setupFiles: ["test/setup.extensions.ts"], -}); +export function loadIncludePatternsFromEnv( + env: Record = process.env, +): string[] | null { + return loadPatternListFromEnv("OPENCLAW_VITEST_INCLUDE_FILE", env); +} + +export function createExtensionQaVitestConfig( + env: Record = process.env, +) { + return createScopedVitestConfig( + loadIncludePatternsFromEnv(env) ?? qaExtensionTestRoots.map((root) => `${root}/**/*.test.ts`), + { + dir: "extensions", + env, + name: "extension-qa", + passWithNoTests: true, + setupFiles: ["test/setup.extensions.ts"], + }, + ); +} + +export default createExtensionQaVitestConfig(); diff --git a/test/vitest/vitest.extensions.config.ts b/test/vitest/vitest.extensions.config.ts index e65efab1b54..d141a598592 100644 --- a/test/vitest/vitest.extensions.config.ts +++ b/test/vitest/vitest.extensions.config.ts @@ -2,18 +2,22 @@ import { BUNDLED_PLUGIN_TEST_GLOB } from "./vitest.bundled-plugin-paths.ts"; import { extensionExcludedChannelTestGlobs } from "./vitest.channel-paths.mjs"; import { acpxExtensionTestRoots } from "./vitest.extension-acpx-paths.mjs"; import { blueBubblesExtensionTestRoots } from "./vitest.extension-bluebubbles-paths.mjs"; +import { browserExtensionTestRoots } from "./vitest.extension-browser-paths.mjs"; import { diffsExtensionTestRoots } from "./vitest.extension-diffs-paths.mjs"; import { feishuExtensionTestRoots } from "./vitest.extension-feishu-paths.mjs"; import { ircExtensionTestRoots } from "./vitest.extension-irc-paths.mjs"; import { matrixExtensionTestRoots } from "./vitest.extension-matrix-paths.mjs"; import { mattermostExtensionTestRoots } from "./vitest.extension-mattermost-paths.mjs"; +import { mediaExtensionTestRoots } from "./vitest.extension-media-paths.mjs"; import { memoryExtensionTestRoots } from "./vitest.extension-memory-paths.mjs"; import { messagingExtensionTestRoots } from "./vitest.extension-messaging-paths.mjs"; +import { miscExtensionTestRoots } from "./vitest.extension-misc-paths.mjs"; import { msTeamsExtensionTestRoots } from "./vitest.extension-msteams-paths.mjs"; import { providerExtensionTestRoots, providerOpenAiExtensionTestRoots, } from "./vitest.extension-provider-paths.mjs"; +import { qaExtensionTestRoots } from "./vitest.extension-qa-paths.mjs"; import { telegramExtensionTestRoots } from "./vitest.extension-telegram-paths.mjs"; import { voiceCallExtensionTestRoots } from "./vitest.extension-voice-call-paths.mjs"; import { whatsAppExtensionTestRoots } from "./vitest.extension-whatsapp-paths.mjs"; @@ -42,16 +46,20 @@ export function createExtensionsVitestConfig( ...extensionExcludedChannelTestGlobs, ...acpxExtensionTestRoots.map((root) => `${root.replace(/^extensions\//u, "")}/**`), ...blueBubblesExtensionTestRoots.map((root) => `${root.replace(/^extensions\//u, "")}/**`), + ...browserExtensionTestRoots.map((root) => `${root.replace(/^extensions\//u, "")}/**`), ...diffsExtensionTestRoots.map((root) => `${root.replace(/^extensions\//u, "")}/**`), ...feishuExtensionTestRoots.map((root) => `${root.replace(/^extensions\//u, "")}/**`), ...ircExtensionTestRoots.map((root) => `${root.replace(/^extensions\//u, "")}/**`), ...matrixExtensionTestRoots.map((root) => `${root.replace(/^extensions\//u, "")}/**`), ...mattermostExtensionTestRoots.map((root) => `${root.replace(/^extensions\//u, "")}/**`), + ...mediaExtensionTestRoots.map((root) => `${root.replace(/^extensions\//u, "")}/**`), ...memoryExtensionTestRoots.map((root) => `${root.replace(/^extensions\//u, "")}/**`), ...messagingExtensionTestRoots.map((root) => `${root.replace(/^extensions\//u, "")}/**`), + ...miscExtensionTestRoots.map((root) => `${root.replace(/^extensions\//u, "")}/**`), ...msTeamsExtensionTestRoots.map((root) => `${root.replace(/^extensions\//u, "")}/**`), ...providerOpenAiExtensionTestRoots.map((root) => `${root.replace(/^extensions\//u, "")}/**`), ...providerExtensionTestRoots.map((root) => `${root.replace(/^extensions\//u, "")}/**`), + ...qaExtensionTestRoots.map((root) => `${root.replace(/^extensions\//u, "")}/**`), ...telegramExtensionTestRoots.map((root) => `${root.replace(/^extensions\//u, "")}/**`), ...voiceCallExtensionTestRoots.map((root) => `${root.replace(/^extensions\//u, "")}/**`), ...whatsAppExtensionTestRoots.map((root) => `${root.replace(/^extensions\//u, "")}/**`),