From 87079d025e8305936f86bf37a01db663c4e6c86e Mon Sep 17 00:00:00 2001 From: Dallin Romney Date: Wed, 15 Jul 2026 17:20:07 -0700 Subject: [PATCH] improve(qa): consolidate live transport selector contracts (#108465) * refactor(qa): consolidate live transport selectors * refactor(qa): remove live transport indirection --- .../qa-lab/src/live-transports/cli.test.ts | 45 ++++--- .../discord/cli.runtime.test.ts | 71 ----------- .../live-transports/discord/cli.runtime.ts | 26 ---- .../qa-lab/src/live-transports/discord/cli.ts | 31 ++--- .../discord/legacy-scenario-parity.test.ts | 28 ----- .../{profiles.ts => scenario-selection.ts} | 2 +- .../live-transport-adapters.test.ts | 32 ++++- .../matrix/cli.runtime.test.ts | 93 --------------- .../src/live-transports/matrix/cli.runtime.ts | 37 ------ .../src/live-transports/matrix/cli.test.ts | 28 +++-- .../qa-lab/src/live-transports/matrix/cli.ts | 34 ++++-- .../live-transports/matrix/profiles.test.ts | 47 +++++--- .../scenario-module-parity.test.ts | 84 +++++++++++++ .../shared/live-credential-source.ts | 10 -- .../shared/live-transport-cli.ts | 16 +++ .../shared/live-transport-result.ts | 23 ---- .../live-transport-suite.runtime.test.ts | 111 ++++++++++++++++++ .../shared/live-transport-suite.runtime.ts | 65 ++++++++++ .../live-transports/slack/cli.runtime.test.ts | 71 ----------- .../src/live-transports/slack/cli.runtime.ts | 26 ---- .../qa-lab/src/live-transports/slack/cli.ts | 34 +++--- .../slack/legacy-scenario-parity.test.ts | 28 ----- .../{profiles.ts => scenario-selection.ts} | 2 +- .../slack/slack-live.runtime.test.ts | 4 +- .../telegram/adapter.runtime.test.ts | 3 +- .../telegram/adapter.runtime.ts | 21 ++-- .../live-transports/telegram/cli.runtime.ts | 24 ++-- .../src/live-transports/telegram/cli.ts | 35 +++--- .../live-transports/telegram/profiles.test.ts | 8 +- .../run-options.runtime.test.ts} | 6 +- .../run-options.runtime.ts} | 12 +- .../whatsapp/cli.runtime.test.ts | 69 ----------- .../live-transports/whatsapp/cli.runtime.ts | 29 ----- .../src/live-transports/whatsapp/cli.ts | 38 +++--- .../whatsapp/legacy-scenario-parity.test.ts | 28 ----- .../{profiles.ts => scenario-selection.ts} | 0 .../whatsapp/whatsapp-live.runtime.test.ts | 2 +- 37 files changed, 513 insertions(+), 710 deletions(-) delete mode 100644 extensions/qa-lab/src/live-transports/discord/cli.runtime.test.ts delete mode 100644 extensions/qa-lab/src/live-transports/discord/cli.runtime.ts delete mode 100644 extensions/qa-lab/src/live-transports/discord/legacy-scenario-parity.test.ts rename extensions/qa-lab/src/live-transports/discord/{profiles.ts => scenario-selection.ts} (69%) delete mode 100644 extensions/qa-lab/src/live-transports/matrix/cli.runtime.test.ts delete mode 100644 extensions/qa-lab/src/live-transports/matrix/cli.runtime.ts create mode 100644 extensions/qa-lab/src/live-transports/scenario-module-parity.test.ts delete mode 100644 extensions/qa-lab/src/live-transports/shared/live-credential-source.ts delete mode 100644 extensions/qa-lab/src/live-transports/shared/live-transport-result.ts create mode 100644 extensions/qa-lab/src/live-transports/shared/live-transport-suite.runtime.test.ts create mode 100644 extensions/qa-lab/src/live-transports/shared/live-transport-suite.runtime.ts delete mode 100644 extensions/qa-lab/src/live-transports/slack/cli.runtime.test.ts delete mode 100644 extensions/qa-lab/src/live-transports/slack/cli.runtime.ts delete mode 100644 extensions/qa-lab/src/live-transports/slack/legacy-scenario-parity.test.ts rename extensions/qa-lab/src/live-transports/slack/{profiles.ts => scenario-selection.ts} (69%) rename extensions/qa-lab/src/live-transports/{shared/live-transport-cli.runtime.test.ts => telegram/run-options.runtime.test.ts} (77%) rename extensions/qa-lab/src/live-transports/{shared/live-transport-cli.runtime.ts => telegram/run-options.runtime.ts} (74%) delete mode 100644 extensions/qa-lab/src/live-transports/whatsapp/cli.runtime.test.ts delete mode 100644 extensions/qa-lab/src/live-transports/whatsapp/cli.runtime.ts delete mode 100644 extensions/qa-lab/src/live-transports/whatsapp/legacy-scenario-parity.test.ts rename extensions/qa-lab/src/live-transports/whatsapp/{profiles.ts => scenario-selection.ts} (100%) diff --git a/extensions/qa-lab/src/live-transports/cli.test.ts b/extensions/qa-lab/src/live-transports/cli.test.ts index 2d902826f4aa..65f6e3c38ca3 100644 --- a/extensions/qa-lab/src/live-transports/cli.test.ts +++ b/extensions/qa-lab/src/live-transports/cli.test.ts @@ -3,21 +3,17 @@ import { Command } from "commander"; import type { QaRunnerCliContribution } from "openclaw/plugin-sdk/qa-runner-runtime"; import { beforeEach, describe, expect, it, vi } from "vitest"; -const { listQaRunnerCliContributions, runDiscord, runSlack, runTelegram, runWhatsApp } = vi.hoisted( +const { listQaRunnerCliContributions, runLiveTransportQaSuiteCommand, runTelegram } = vi.hoisted( () => ({ listQaRunnerCliContributions: vi.fn<() => QaRunnerCliContribution[]>(() => []), - runDiscord: vi.fn(), - runSlack: vi.fn(), + runLiveTransportQaSuiteCommand: vi.fn(), runTelegram: vi.fn(), - runWhatsApp: vi.fn(), }), ); vi.mock("openclaw/plugin-sdk/qa-runner-runtime", () => ({ listQaRunnerCliContributions })); -vi.mock("./discord/cli.runtime.js", () => ({ runQaDiscordCommand: runDiscord })); -vi.mock("./slack/cli.runtime.js", () => ({ runQaSlackCommand: runSlack })); +vi.mock("./shared/live-transport-suite.runtime.js", () => ({ runLiveTransportQaSuiteCommand })); vi.mock("./telegram/cli.runtime.js", () => ({ runQaTelegramCommand: runTelegram })); -vi.mock("./whatsapp/cli.runtime.js", () => ({ runQaWhatsAppCommand: runWhatsApp })); import { listLiveTransportQaAdapterFactories, listLiveTransportQaCliRegistrations } from "./cli.js"; @@ -37,22 +33,37 @@ describe("live transport QA contributions", () => { ]); }); - it.each([ - ["telegram", runTelegram], - ["discord", runDiscord], - ["slack", runSlack], - ["whatsapp", runWhatsApp], - ] as const)("keeps the shipped %s command runner", async (commandName, runCommand) => { + it.each(["discord", "slack", "whatsapp"] as const)( + "routes the shipped %s command through the shared suite host", + async (commandName) => { + const registration = listLiveTransportQaCliRegistrations().find( + (candidate) => candidate.commandName === commandName, + ); + const qa = new Command(); + registration?.register(qa); + + await qa.parseAsync(["node", "openclaw", commandName, "--scenario", `${commandName}-canary`]); + + expect(runLiveTransportQaSuiteCommand).toHaveBeenCalledWith( + expect.objectContaining({ + channelId: commandName, + options: expect.objectContaining({ scenarioIds: [`${commandName}-canary`] }), + }), + ); + }, + ); + + it("keeps the specialized Telegram command runner", async () => { const registration = listLiveTransportQaCliRegistrations().find( - (candidate) => candidate.commandName === commandName, + (candidate) => candidate.commandName === "telegram", ); const qa = new Command(); registration?.register(qa); - await qa.parseAsync(["node", "openclaw", commandName, "--scenario", `${commandName}-canary`]); + await qa.parseAsync(["node", "openclaw", "telegram", "--scenario", "telegram-canary"]); - expect(runCommand).toHaveBeenCalledWith( - expect.objectContaining({ scenarioIds: [`${commandName}-canary`] }), + expect(runTelegram).toHaveBeenCalledWith( + expect.objectContaining({ scenarioIds: ["telegram-canary"] }), ); }); }); diff --git a/extensions/qa-lab/src/live-transports/discord/cli.runtime.test.ts b/extensions/qa-lab/src/live-transports/discord/cli.runtime.test.ts deleted file mode 100644 index 4aae1e4a2cca..000000000000 --- a/extensions/qa-lab/src/live-transports/discord/cli.runtime.test.ts +++ /dev/null @@ -1,71 +0,0 @@ -// QA Lab Discord tests cover CLI delegation into the shared suite host. -import { beforeEach, describe, expect, it, vi } from "vitest"; - -const runQaSuiteCommand = vi.hoisted(() => vi.fn()); - -vi.mock("../../cli.runtime.js", () => ({ runQaSuiteCommand })); -vi.mock("../shared/live-transport-cli.runtime.js", () => ({ - resolveLiveTransportQaRunOptions: (opts: Record) => ({ - ...opts, - repoRoot: "/resolved-repo", - outputDir: "/resolved-repo/.artifacts/resolved", - providerMode: opts.providerMode ?? "mock-openai", - }), -})); - -import { listQaScenariosForExecutionProfile } from "../../scenario-catalog.js"; -import { runQaDiscordCommand } from "./cli.runtime.js"; - -describe("QA Lab Discord CLI runtime", () => { - beforeEach(() => { - vi.clearAllMocks(); - }); - - it("delegates the default profile into the Discord adapter host", async () => { - await runQaDiscordCommand({ - repoRoot: "/repo", - outputDir: ".artifacts/discord", - providerMode: "mock-openai", - primaryModel: "mock-openai/gpt-5.5", - alternateModel: "mock-openai/gpt-5.5-alt", - fastMode: true, - allowFailures: true, - failFast: true, - credentialSource: "convex", - credentialRole: "ci", - sutAccountId: "discord-sut", - }); - - expect(runQaSuiteCommand).toHaveBeenCalledWith({ - repoRoot: "/repo", - outputDir: ".artifacts/discord", - providerMode: "mock-openai", - primaryModel: "mock-openai/gpt-5.5", - alternateModel: "mock-openai/gpt-5.5-alt", - fastMode: true, - allowFailures: true, - failFast: true, - channelDriver: "live", - channel: "discord", - concurrency: 1, - scenarioIds: listQaScenariosForExecutionProfile("discord:adapter").map( - (scenario) => scenario.id, - ), - sutAccountId: "discord-sut", - credentialSource: "convex", - credentialRole: "ci", - explicitScenarioSelection: false, - }); - }); - - it("lets explicit scenarios override profile selection", async () => { - await runQaDiscordCommand({ scenarioIds: ["discord-voice-autojoin"] }); - - expect(runQaSuiteCommand).toHaveBeenCalledWith( - expect.objectContaining({ - explicitScenarioSelection: true, - scenarioIds: ["discord-voice-autojoin"], - }), - ); - }); -}); diff --git a/extensions/qa-lab/src/live-transports/discord/cli.runtime.ts b/extensions/qa-lab/src/live-transports/discord/cli.runtime.ts deleted file mode 100644 index c676bbb350f9..000000000000 --- a/extensions/qa-lab/src/live-transports/discord/cli.runtime.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { runQaSuiteCommand } from "../../cli.runtime.js"; -import type { LiveTransportQaCommandOptions } from "../shared/live-transport-cli.js"; -import { resolveLiveTransportQaRunOptions } from "../shared/live-transport-cli.runtime.js"; -import { resolveDiscordQaScenarioIds } from "./profiles.js"; - -export async function runQaDiscordCommand(opts: LiveTransportQaCommandOptions) { - const runOptions = resolveLiveTransportQaRunOptions(opts); - return await runQaSuiteCommand({ - repoRoot: opts.repoRoot, - outputDir: opts.outputDir, - providerMode: runOptions.providerMode, - primaryModel: runOptions.primaryModel, - alternateModel: runOptions.alternateModel, - fastMode: runOptions.fastMode, - allowFailures: runOptions.allowFailures, - failFast: runOptions.failFast, - channelDriver: "live", - channel: "discord", - concurrency: 1, - scenarioIds: resolveDiscordQaScenarioIds(runOptions.scenarioIds), - sutAccountId: runOptions.sutAccountId, - credentialSource: runOptions.credentialSource, - credentialRole: runOptions.credentialRole, - explicitScenarioSelection: Boolean(runOptions.scenarioIds?.length), - }); -} diff --git a/extensions/qa-lab/src/live-transports/discord/cli.ts b/extensions/qa-lab/src/live-transports/discord/cli.ts index be933b57a76f..8118dec881f1 100644 --- a/extensions/qa-lab/src/live-transports/discord/cli.ts +++ b/extensions/qa-lab/src/live-transports/discord/cli.ts @@ -1,36 +1,37 @@ // Qa Lab plugin module implements cli behavior. import { + createLiveTransportQaAdapterFactory, createLazyCliRuntimeLoader, createLiveTransportQaCliRegistration, + loadLiveTransportQaSuiteRuntime, type LiveTransportQaCliRegistration, type LiveTransportQaCommandOptions, } from "../shared/live-transport-cli.js"; +import { resolveDiscordQaScenarioIds } from "./scenario-selection.js"; -type DiscordQaCliRuntime = typeof import("./cli.runtime.js"); -type DiscordQaAdapterRuntime = typeof import("./adapter.runtime.js"); - -const loadDiscordQaCliRuntime = createLazyCliRuntimeLoader( - () => import("./cli.runtime.js"), -); -const loadDiscordQaAdapterRuntime = createLazyCliRuntimeLoader( - () => import("./adapter.runtime.js"), -); +const loadDiscordQaAdapterRuntime = createLazyCliRuntimeLoader< + typeof import("./adapter.runtime.js") +>(() => import("./adapter.runtime.js")); async function runQaDiscord(opts: LiveTransportQaCommandOptions) { - const runtime = await loadDiscordQaCliRuntime(); - await runtime.runQaDiscordCommand(opts); + const runtime = await loadLiveTransportQaSuiteRuntime(); + await runtime.runLiveTransportQaSuiteCommand({ + channelId: "discord", + defaultProviderMode: "live-frontier", + options: opts, + selectScenarioIds: resolveDiscordQaScenarioIds, + }); } export const discordQaCliRegistration: LiveTransportQaCliRegistration = createLiveTransportQaCliRegistration({ commandName: "discord", - adapterFactory: { + adapterFactory: createLiveTransportQaAdapterFactory({ id: "discord", - matches: ({ channelId, driver }) => driver === "live" && channelId === "discord", async create(context) { - return await (await loadDiscordQaAdapterRuntime()).createDiscordQaTransportAdapter(context); + return (await loadDiscordQaAdapterRuntime()).createDiscordQaTransportAdapter(context); }, - }, + }), credentialOptions: { sourceDescription: "Credential source for Discord QA: env or convex (default: env)", roleDescription: diff --git a/extensions/qa-lab/src/live-transports/discord/legacy-scenario-parity.test.ts b/extensions/qa-lab/src/live-transports/discord/legacy-scenario-parity.test.ts deleted file mode 100644 index ca251292142c..000000000000 --- a/extensions/qa-lab/src/live-transports/discord/legacy-scenario-parity.test.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { readQaScenarioPack } from "../../scenario-catalog.js"; -import * as scenarioRuntime from "./scenario-runtime.js"; - -describe("legacy Discord scenario migration", () => { - it("keeps every retired runner id as a Discord module scenario", () => { - const scenarios = readQaScenarioPack().scenarios.filter( - (scenario) => - scenario.execution.kind === "flow" && - scenario.execution.channel === "discord" && - JSON.stringify(scenario.execution.flow).includes( - "./live-transports/discord/scenario-runtime.js", - ), - ); - expect(scenarios).toHaveLength(6); - for (const scenario of scenarios) { - expect(scenario.execution).toMatchObject({ kind: "flow", channel: "discord" }); - expect(JSON.stringify(scenario.execution.flow)).toContain( - "./live-transports/discord/scenario-runtime.js", - ); - const flowText = JSON.stringify(scenario.execution.flow); - const callName = flowText.match(/scenarioModule\.([A-Za-z0-9]+)/u)?.[1]; - expect(typeof scenarioRuntime[callName as keyof typeof scenarioRuntime], scenario.id).toBe( - "function", - ); - } - }); -}); diff --git a/extensions/qa-lab/src/live-transports/discord/profiles.ts b/extensions/qa-lab/src/live-transports/discord/scenario-selection.ts similarity index 69% rename from extensions/qa-lab/src/live-transports/discord/profiles.ts rename to extensions/qa-lab/src/live-transports/discord/scenario-selection.ts index c396a28a90dc..1ec43b320f37 100644 --- a/extensions/qa-lab/src/live-transports/discord/profiles.ts +++ b/extensions/qa-lab/src/live-transports/discord/scenario-selection.ts @@ -1,6 +1,6 @@ import { listQaScenariosForExecutionProfile } from "../../scenario-catalog.js"; -export function resolveDiscordQaScenarioIds(scenarioIds?: readonly string[]) { +export function resolveDiscordQaScenarioIds({ scenarioIds }: { scenarioIds?: readonly string[] }) { return scenarioIds?.length ? [...scenarioIds] : listQaScenariosForExecutionProfile("discord:adapter").map((scenario) => scenario.id); diff --git a/extensions/qa-lab/src/live-transports/live-transport-adapters.test.ts b/extensions/qa-lab/src/live-transports/live-transport-adapters.test.ts index 8ae3b03439a2..b63d0d1db457 100644 --- a/extensions/qa-lab/src/live-transports/live-transport-adapters.test.ts +++ b/extensions/qa-lab/src/live-transports/live-transport-adapters.test.ts @@ -5,12 +5,18 @@ import { createQaChannelTransport } from "../qa-channel-transport.js"; import { createQaTransportAdapter } from "../qa-transport-registry.js"; import { listQaScenariosForExecutionProfile } from "../scenario-catalog.js"; -const { createSlack, createTelegram, createWhatsApp } = vi.hoisted(() => ({ - createSlack: vi.fn(), - createTelegram: vi.fn(), - createWhatsApp: vi.fn(), -})); +const { createDiscord, createMatrix, createSlack, createTelegram, createWhatsApp } = vi.hoisted( + () => ({ + createDiscord: vi.fn(), + createMatrix: vi.fn(), + createSlack: vi.fn(), + createTelegram: vi.fn(), + createWhatsApp: vi.fn(), + }), +); +vi.mock("./discord/adapter.runtime.js", () => ({ createDiscordQaTransportAdapter: createDiscord })); +vi.mock("./matrix/adapter.runtime.js", () => ({ createMatrixQaTransportAdapter: createMatrix })); vi.mock("./slack/adapter.runtime.js", () => ({ createSlackQaTransportAdapter: createSlack })); vi.mock("./telegram/adapter.runtime.js", () => ({ createTelegramQaTransportAdapter: createTelegram, @@ -19,19 +25,31 @@ vi.mock("./whatsapp/adapter.runtime.js", () => ({ createWhatsAppQaTransportAdapter: createWhatsApp, })); +import { discordQaCliRegistration } from "./discord/cli.js"; +import { matrixQaCliRegistration } from "./matrix/cli.js"; import { slackQaCliRegistration } from "./slack/cli.js"; import { telegramQaCliRegistration } from "./telegram/cli.js"; import { whatsappQaCliRegistration } from "./whatsapp/cli.js"; +const discordQaAdapterFactory = discordQaCliRegistration.adapterFactory; +const matrixQaAdapterFactory = matrixQaCliRegistration.adapterFactory; const slackQaAdapterFactory = slackQaCliRegistration.adapterFactory; const telegramQaAdapterFactory = telegramQaCliRegistration.adapterFactory; const whatsappQaAdapterFactory = whatsappQaCliRegistration.adapterFactory; -if (!slackQaAdapterFactory || !telegramQaAdapterFactory || !whatsappQaAdapterFactory) { +if ( + !discordQaAdapterFactory || + !matrixQaAdapterFactory || + !slackQaAdapterFactory || + !telegramQaAdapterFactory || + !whatsappQaAdapterFactory +) { throw new Error("expected live transport adapter factories"); } const factories = [ telegramQaAdapterFactory, + discordQaAdapterFactory, + matrixQaAdapterFactory, slackQaAdapterFactory, whatsappQaAdapterFactory, ] as const; @@ -64,6 +82,8 @@ describe("live transport adapter factories", () => { }); it.each([ + ["discord", createDiscord], + ["matrix", createMatrix], ["telegram", createTelegram], ["slack", createSlack], ["whatsapp", createWhatsApp], diff --git a/extensions/qa-lab/src/live-transports/matrix/cli.runtime.test.ts b/extensions/qa-lab/src/live-transports/matrix/cli.runtime.test.ts deleted file mode 100644 index 13e2d93b5d11..000000000000 --- a/extensions/qa-lab/src/live-transports/matrix/cli.runtime.test.ts +++ /dev/null @@ -1,93 +0,0 @@ -// QA Lab Matrix tests cover CLI delegation into the shared suite host. -import { beforeEach, describe, expect, it, vi } from "vitest"; - -const runQaSuiteCommand = vi.hoisted(() => vi.fn()); - -vi.mock("../../cli.runtime.js", () => ({ runQaSuiteCommand })); - -import { runQaMatrixCommand } from "./cli.runtime.js"; -import { resolveMatrixQaScenarioIds } from "./profiles.js"; - -describe("QA Lab Matrix CLI runtime", () => { - beforeEach(() => { - vi.clearAllMocks(); - }); - - it("delegates the release profile into the live Matrix adapter host", async () => { - await runQaMatrixCommand({ - repoRoot: "/repo", - outputDir: ".artifacts/matrix", - providerMode: "mock-openai", - primaryModel: "mock-openai/gpt-5.5", - alternateModel: "mock-openai/gpt-5.5-alt", - fastMode: true, - failFast: true, - profile: "release", - sutAccountId: "matrix-sut", - }); - - expect(runQaSuiteCommand).toHaveBeenCalledWith({ - repoRoot: "/repo", - outputDir: ".artifacts/matrix", - providerMode: "mock-openai", - primaryModel: "mock-openai/gpt-5.5", - alternateModel: "mock-openai/gpt-5.5-alt", - fastMode: true, - allowFailures: undefined, - failFast: true, - channelDriver: "live", - channel: "matrix", - concurrency: 1, - scenarioIds: ["channel-chat-baseline", "matrix-allowlist-hot-reload"], - sutAccountId: "matrix-sut", - }); - }); - - it("lets explicit scenarios override profile selection", async () => { - await runQaMatrixCommand({ - profile: "all", - scenarioIds: ["matrix-restart-resume"], - }); - - expect(runQaSuiteCommand).toHaveBeenCalledWith( - expect.objectContaining({ scenarioIds: ["matrix-restart-resume"] }), - ); - }); - - it("keeps the dedicated Matrix command default on all profile scenarios", async () => { - await runQaMatrixCommand({}); - - const call = runQaSuiteCommand.mock.calls.at(-1)?.[0]; - expect(call?.scenarioIds).toEqual(resolveMatrixQaScenarioIds({})); - expect(call?.scenarioIds).toContain("matrix-e2ee-cli-encryption-setup"); - }); - - it("delegates restored E2EE profiles through the same live Matrix adapter host", async () => { - await runQaMatrixCommand({ profile: "e2ee-deep" }); - - expect(runQaSuiteCommand).toHaveBeenCalledWith( - expect.objectContaining({ - channel: "matrix", - channelDriver: "live", - scenarioIds: expect.arrayContaining([ - "matrix-e2ee-state-loss-external-recovery-key", - "matrix-e2ee-server-device-deleted-relogin-recovers", - ]), - }), - ); - }); - - it("rejects unknown provider modes before suite dispatch", async () => { - await expect(runQaMatrixCommand({ providerMode: "unknown" })).rejects.toThrow( - "unknown QA provider mode: unknown", - ); - expect(runQaSuiteCommand).not.toHaveBeenCalled(); - }); - - it("rejects shared credential leases", async () => { - await expect(runQaMatrixCommand({ credentialSource: "convex" })).rejects.toThrow( - "supports only --credential-source env", - ); - expect(runQaSuiteCommand).not.toHaveBeenCalled(); - }); -}); diff --git a/extensions/qa-lab/src/live-transports/matrix/cli.runtime.ts b/extensions/qa-lab/src/live-transports/matrix/cli.runtime.ts deleted file mode 100644 index e31f2ab284cb..000000000000 --- a/extensions/qa-lab/src/live-transports/matrix/cli.runtime.ts +++ /dev/null @@ -1,37 +0,0 @@ -// QA Lab Matrix delegates CLI profile selection into the shared suite host. -import { runQaSuiteCommand } from "../../cli.runtime.js"; -import { normalizeQaProviderMode } from "../../run-config.js"; -import type { LiveTransportQaCommandOptions } from "../shared/live-transport-cli.js"; -import { resolveMatrixQaScenarioIds } from "./profiles.js"; - -export async function runQaMatrixCommand(opts: LiveTransportQaCommandOptions) { - const credentialSource = opts.credentialSource?.trim().toLowerCase(); - if (credentialSource && credentialSource !== "env") { - throw new Error( - "QA Lab Matrix supports only --credential-source env because its homeserver is disposable and local.", - ); - } - if (opts.credentialRole?.trim()) { - throw new Error("QA Lab Matrix does not use credential roles."); - } - - return await runQaSuiteCommand({ - repoRoot: opts.repoRoot, - outputDir: opts.outputDir, - providerMode: - opts.providerMode === undefined ? undefined : normalizeQaProviderMode(opts.providerMode), - primaryModel: opts.primaryModel, - alternateModel: opts.alternateModel, - fastMode: opts.fastMode, - allowFailures: opts.allowFailures, - failFast: opts.failFast, - channelDriver: "live", - channel: "matrix", - concurrency: 1, - scenarioIds: resolveMatrixQaScenarioIds({ - profile: opts.profile, - scenarioIds: opts.scenarioIds, - }), - sutAccountId: opts.sutAccountId, - }); -} diff --git a/extensions/qa-lab/src/live-transports/matrix/cli.test.ts b/extensions/qa-lab/src/live-transports/matrix/cli.test.ts index c4ed575d36e7..a962347ccced 100644 --- a/extensions/qa-lab/src/live-transports/matrix/cli.test.ts +++ b/extensions/qa-lab/src/live-transports/matrix/cli.test.ts @@ -2,9 +2,9 @@ import { Command } from "commander"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -const runQaMatrixCommand = vi.hoisted(() => vi.fn()); +const runLiveTransportQaSuiteCommand = vi.hoisted(() => vi.fn()); -vi.mock("./cli.runtime.js", () => ({ runQaMatrixCommand })); +vi.mock("../shared/live-transport-suite.runtime.js", () => ({ runLiveTransportQaSuiteCommand })); import { matrixQaCliRegistration } from "./cli.js"; @@ -30,7 +30,7 @@ describe("QA Lab Matrix CLI registration", () => { beforeEach(() => { process.exitCode = undefined; - runQaMatrixCommand.mockReset(); + runLiveTransportQaSuiteCommand.mockReset(); exitSpy = vi.spyOn(process, "exit").mockImplementation((code?: string | number | null) => { throw new Error(`process.exit(${String(code)})`); }); @@ -89,11 +89,15 @@ describe("QA Lab Matrix CLI registration", () => { "matrix-allowlist-hot-reload", ]); - expect(runQaMatrixCommand).toHaveBeenCalledWith( + expect(runLiveTransportQaSuiteCommand).toHaveBeenCalledWith( expect.objectContaining({ - profile: "release", - providerMode: "live-frontier", - scenarioIds: ["matrix-allowlist-hot-reload"], + channelId: "matrix", + credentialMode: "env-only", + options: expect.objectContaining({ + profile: "release", + providerMode: "live-frontier", + scenarioIds: ["matrix-allowlist-hot-reload"], + }), }), ); }); @@ -101,7 +105,7 @@ describe("QA Lab Matrix CLI registration", () => { it("exits successfully after Matrix artifacts are written", async () => { const qa = new Command(); matrixQaCliRegistration.register(qa); - runQaMatrixCommand.mockResolvedValue(undefined); + runLiveTransportQaSuiteCommand.mockResolvedValue(undefined); await expect(qa.parseAsync(["node", "openclaw", "matrix"])).rejects.toThrow("process.exit(0)"); @@ -111,7 +115,9 @@ describe("QA Lab Matrix CLI registration", () => { it("prints a failed run and exits after its artifacts are written", async () => { const qa = new Command(); matrixQaCliRegistration.register(qa); - runQaMatrixCommand.mockRejectedValue(new Error("Matrix QA failed.\nreport: /tmp/report.md")); + runLiveTransportQaSuiteCommand.mockRejectedValue( + new Error("Matrix QA failed.\nreport: /tmp/report.md"), + ); await expect(qa.parseAsync(["node", "openclaw", "matrix"])).rejects.toThrow("process.exit(1)"); @@ -122,7 +128,7 @@ describe("QA Lab Matrix CLI registration", () => { it("preserves a failed suite exit code after the runtime returns", async () => { const qa = new Command(); matrixQaCliRegistration.register(qa); - runQaMatrixCommand.mockImplementation(async () => { + runLiveTransportQaSuiteCommand.mockImplementation(async () => { process.exitCode = 1; }); @@ -135,7 +141,7 @@ describe("QA Lab Matrix CLI registration", () => { process.env.OPENCLAW_QA_MATRIX_DISABLE_FORCE_EXIT = "1"; const qa = new Command(); matrixQaCliRegistration.register(qa); - runQaMatrixCommand.mockRejectedValue(new Error("scenario failed")); + runLiveTransportQaSuiteCommand.mockRejectedValue(new Error("scenario failed")); await expect(qa.parseAsync(["node", "openclaw", "matrix"])).rejects.toThrow("scenario failed"); diff --git a/extensions/qa-lab/src/live-transports/matrix/cli.ts b/extensions/qa-lab/src/live-transports/matrix/cli.ts index 8fd0b873efcb..cec1855f620d 100644 --- a/extensions/qa-lab/src/live-transports/matrix/cli.ts +++ b/extensions/qa-lab/src/live-transports/matrix/cli.ts @@ -1,17 +1,17 @@ // Qa Lab plugin module implements Matrix live transport CLI behavior. import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime"; import { + createLiveTransportQaAdapterFactory, createLazyCliRuntimeLoader, createLiveTransportQaCliRegistration, + loadLiveTransportQaSuiteRuntime, type LiveTransportQaCliRegistration, type LiveTransportQaCommandOptions, } from "../shared/live-transport-cli.js"; +import { resolveMatrixQaScenarioIds } from "./profiles.js"; const DISABLE_MATRIX_QA_FORCE_EXIT_ENV = "OPENCLAW_QA_MATRIX_DISABLE_FORCE_EXIT"; -const loadMatrixQaCliRuntime = createLazyCliRuntimeLoader( - () => import("./cli.runtime.js"), -); const loadMatrixQaAdapterRuntime = createLazyCliRuntimeLoader< typeof import("./adapter.runtime.js") >(() => import("./adapter.runtime.js")); @@ -37,7 +37,18 @@ async function exitMatrixQaCommand(code: number): Promise { } async function runQaMatrix(opts: LiveTransportQaCommandOptions) { - const run = async () => await (await loadMatrixQaCliRuntime()).runQaMatrixCommand(opts); + const run = async () => { + const runtime = await loadLiveTransportQaSuiteRuntime(); + await runtime.runLiveTransportQaSuiteCommand({ + channelId: "matrix", + credentialMode: "env-only", + defaultProviderMode: "live-frontier", + envCredentialReason: "its homeserver is disposable and local.", + laneLabel: "Matrix", + options: opts, + selectScenarioIds: resolveMatrixQaScenarioIds, + }); + }; if (process.env[DISABLE_MATRIX_QA_FORCE_EXIT_ENV] === "1") { await run(); return; @@ -54,18 +65,15 @@ async function runQaMatrix(opts: LiveTransportQaCommandOptions) { await exitMatrixQaCommand(exitCode); } -const matrixQaAdapterFactory: NonNullable = { - id: "matrix", - matches: ({ channelId, driver }) => driver === "live" && channelId === "matrix", - async create(context) { - return await (await loadMatrixQaAdapterRuntime()).createMatrixQaTransportAdapter(context); - }, -}; - export const matrixQaCliRegistration: LiveTransportQaCliRegistration = createLiveTransportQaCliRegistration({ commandName: "matrix", - adapterFactory: matrixQaAdapterFactory, + adapterFactory: createLiveTransportQaAdapterFactory({ + id: "matrix", + async create(context) { + return (await loadMatrixQaAdapterRuntime()).createMatrixQaTransportAdapter(context); + }, + }), defaultProviderMode: "live-frontier", description: "Run the Docker-backed Matrix live QA lane against a disposable homeserver", outputDirHelp: "Matrix QA artifact directory", diff --git a/extensions/qa-lab/src/live-transports/matrix/profiles.test.ts b/extensions/qa-lab/src/live-transports/matrix/profiles.test.ts index dd65ff73b084..ffbe006ed215 100644 --- a/extensions/qa-lab/src/live-transports/matrix/profiles.test.ts +++ b/extensions/qa-lab/src/live-transports/matrix/profiles.test.ts @@ -1,8 +1,20 @@ import { describe, expect, it } from "vitest"; +import { readQaScenarioById } from "../../scenario-catalog.js"; import { resolveMatrixQaScenarioIds } from "./profiles.js"; +const MATRIX_QA_PROFILE_NAMES = [ + "all", + "fast", + "release", + "transport", + "media", + "e2ee-smoke", + "e2ee-deep", + "e2ee-cli", +] as const; + describe("QA Lab Matrix profiles", () => { - it("preserves the legacy profile sizes and default selection", () => { + it("preserves the profile sizes and default selection", () => { const allScenarioIds = resolveMatrixQaScenarioIds({ profile: "all" }); expect(allScenarioIds).toHaveLength(93); expect(resolveMatrixQaScenarioIds({ profile: "fast" })).toHaveLength(12); @@ -20,22 +32,27 @@ describe("QA Lab Matrix profiles", () => { expect(allScenarioIds).toContain("channel-chat-baseline"); }); - it("keeps profile ids unique and excludes the legacy explicit-only scenarios", () => { - for (const profile of [ - "all", - "fast", - "release", - "transport", - "media", - "e2ee-smoke", - "e2ee-deep", - "e2ee-cli", - ]) { + it("keeps every profile unique, catalog-backed, and contained by all", () => { + const allScenarioIds = resolveMatrixQaScenarioIds({ profile: "all" }); + const allScenarioIdSet = new Set(allScenarioIds); + for (const profile of MATRIX_QA_PROFILE_NAMES) { const scenarioIds = resolveMatrixQaScenarioIds({ profile }); expect(new Set(scenarioIds).size).toBe(scenarioIds.length); + for (const scenarioId of scenarioIds) { + expect(readQaScenarioById(scenarioId).id).toBe(scenarioId); + if (profile !== "all") { + expect(allScenarioIdSet.has(scenarioId), `${profile}:${scenarioId}`).toBe(true); + } + } } - const allScenarioIds = resolveMatrixQaScenarioIds({ profile: "all" }); - expect(allScenarioIds).not.toContain("matrix-room-block-streaming"); - expect(allScenarioIds).not.toContain("subagent-thread-spawn"); + }); + + it("keeps explicit-only scenarios out of every named profile", () => { + const profileScenarioIds = MATRIX_QA_PROFILE_NAMES.flatMap((profile) => + resolveMatrixQaScenarioIds({ profile }), + ); + + expect(profileScenarioIds).not.toContain("matrix-room-block-streaming"); + expect(profileScenarioIds).not.toContain("subagent-thread-spawn"); }); }); diff --git a/extensions/qa-lab/src/live-transports/scenario-module-parity.test.ts b/extensions/qa-lab/src/live-transports/scenario-module-parity.test.ts new file mode 100644 index 000000000000..94b93a3db541 --- /dev/null +++ b/extensions/qa-lab/src/live-transports/scenario-module-parity.test.ts @@ -0,0 +1,84 @@ +import { describe, expect, it } from "vitest"; +import { readQaScenarioPack, type QaSeedScenarioWithSource } from "../scenario-catalog.js"; +import * as discordScenarioRuntime from "./discord/scenario-runtime.js"; +import * as slackScenarioRuntime from "./slack/scenario-runtime.js"; +import * as whatsappScenarioRuntime from "./whatsapp/scenario-runtime.js"; + +const LANES = [ + { + channel: "discord", + modulePath: "./live-transports/discord/scenario-runtime.js", + runtime: discordScenarioRuntime, + }, + { + channel: "slack", + modulePath: "./live-transports/slack/scenario-runtime.js", + runtime: slackScenarioRuntime, + }, + { + channel: "whatsapp", + modulePath: "./live-transports/whatsapp/scenario-runtime.js", + runtime: whatsappScenarioRuntime, + }, +] as const; + +function readScenarioModuleCallName( + scenario: QaSeedScenarioWithSource, + modulePath: string, +): string | undefined { + if (scenario.execution.kind !== "flow" || !scenario.execution.flow) { + return undefined; + } + const actions = scenario.execution.flow.steps.flatMap((step) => step.actions); + const importExpression = `await qaImport(${JSON.stringify(modulePath)})`; + const importsModule = actions.some( + (action) => + typeof action === "object" && + action !== null && + "set" in action && + action.set === "scenarioModule" && + "value" in action && + typeof action.value === "object" && + action.value !== null && + "expr" in action.value && + action.value.expr === importExpression, + ); + if (!importsModule) { + return undefined; + } + const callPrefix = "scenarioModule."; + const callAction = actions.find( + (action): action is { call: string } => + typeof action === "object" && + action !== null && + "call" in action && + typeof action.call === "string" && + action.call.startsWith(callPrefix), + ); + if (!callAction) { + throw new Error(`scenario module flow has no call: ${scenario.id}`); + } + return callAction.call.slice(callPrefix.length); +} + +describe("live transport scenario module parity", () => { + it.each(LANES)( + "keeps $channel scenario definitions and runtime exports in one-to-one parity", + ({ channel, modulePath, runtime }) => { + const bindings = readQaScenarioPack().scenarios.flatMap((scenario) => { + if (scenario.execution.kind !== "flow" || scenario.execution.channel !== channel) { + return []; + } + const callName = readScenarioModuleCallName(scenario, modulePath); + return callName ? [{ callName, scenarioId: scenario.id }] : []; + }); + const callNames = bindings.map(({ callName, scenarioId }) => { + expect(Reflect.get(runtime, callName), scenarioId).toBeTypeOf("function"); + return callName; + }); + + expect(new Set(callNames).size).toBe(callNames.length); + expect(callNames.toSorted()).toEqual(Object.keys(runtime).toSorted()); + }, + ); +}); diff --git a/extensions/qa-lab/src/live-transports/shared/live-credential-source.ts b/extensions/qa-lab/src/live-transports/shared/live-credential-source.ts deleted file mode 100644 index 537f351d2872..000000000000 --- a/extensions/qa-lab/src/live-transports/shared/live-credential-source.ts +++ /dev/null @@ -1,10 +0,0 @@ -type QaInferredCredentialSource = "convex" | "env"; - -export function inferQaCredentialSource( - value: string | undefined, - env: NodeJS.ProcessEnv = process.env, -): QaInferredCredentialSource { - const normalized = - value?.trim().toLowerCase() || env.OPENCLAW_QA_CREDENTIAL_SOURCE?.trim().toLowerCase(); - return normalized === "convex" ? "convex" : "env"; -} diff --git a/extensions/qa-lab/src/live-transports/shared/live-transport-cli.ts b/extensions/qa-lab/src/live-transports/shared/live-transport-cli.ts index 7c893b183baa..d8a2d8f87a82 100644 --- a/extensions/qa-lab/src/live-transports/shared/live-transport-cli.ts +++ b/extensions/qa-lab/src/live-transports/shared/live-transport-cli.ts @@ -139,6 +139,11 @@ function createSharedLiveTransportQaCliRegistration( }; } +// All dedicated commands share one memoized import of the consolidated suite host. +export const loadLiveTransportQaSuiteRuntime = createLazyCliRuntimeLoader< + typeof import("./live-transport-suite.runtime.js") +>(() => import("./live-transport-suite.runtime.js")); + type QaLabLiveTransportQaCliRegistrationOptions = Omit< LiveTransportQaCliRegistrationOptions, "allowFailuresHelp" | "defaultProviderMode" | "providerModeHelp" @@ -156,3 +161,14 @@ export function createLiveTransportQaCliRegistration( providerModeHelp: formatQaProviderModeHelp(), }); } + +export function createLiveTransportQaAdapterFactory(params: { + create: NonNullable["create"]; + id: string; +}): NonNullable { + return { + id: params.id, + matches: ({ channelId, driver }) => driver === "live" && channelId === params.id, + create: params.create, + }; +} diff --git a/extensions/qa-lab/src/live-transports/shared/live-transport-result.ts b/extensions/qa-lab/src/live-transports/shared/live-transport-result.ts deleted file mode 100644 index 351fa0006059..000000000000 --- a/extensions/qa-lab/src/live-transports/shared/live-transport-result.ts +++ /dev/null @@ -1,23 +0,0 @@ -// Qa Lab plugin module implements shared live-transport result shapes. -import type { QaEvidenceTiming } from "../../evidence-summary.js"; - -type LiveTransportRttMeasurement = { - finalMatchedReplyRttMs: number; - requestStartedAt: string; - responseObservedAt: string; - source: "request-to-observed-message"; -}; - -export type LiveTransportCheckResult = { - id: string; - title: string; - status: "pass" | "fail"; - details: string; - timing?: QaEvidenceTiming; - rttMs?: number; - requestStartedAt?: string; - responseObservedAt?: string; - rttMeasurement?: LiveTransportRttMeasurement; - sentMessageId?: number; - responseMessageId?: number; -}; diff --git a/extensions/qa-lab/src/live-transports/shared/live-transport-suite.runtime.test.ts b/extensions/qa-lab/src/live-transports/shared/live-transport-suite.runtime.test.ts new file mode 100644 index 000000000000..cbffbd0c2ce0 --- /dev/null +++ b/extensions/qa-lab/src/live-transports/shared/live-transport-suite.runtime.test.ts @@ -0,0 +1,111 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const runQaSuiteCommand = vi.hoisted(() => vi.fn()); + +vi.mock("../../cli.runtime.js", () => ({ runQaSuiteCommand })); + +import { runLiveTransportQaSuiteCommand } from "./live-transport-suite.runtime.js"; + +describe("live transport suite runtime", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("normalizes one live command into the shared suite host", async () => { + await runLiveTransportQaSuiteCommand({ + channelId: "slack", + defaultProviderMode: "live-frontier", + options: { + repoRoot: "/repo", + outputDir: ".artifacts/slack", + primaryModel: "openai/gpt-5.5", + alternateModel: "openai/gpt-5.5-alt", + fastMode: true, + allowFailures: true, + failFast: true, + credentialSource: " convex ", + credentialRole: " ci ", + sutAccountId: "slack-sut", + }, + selectScenarioIds: ({ providerMode, scenarioIds }) => { + expect(providerMode).toBe("live-frontier"); + expect(scenarioIds).toBeUndefined(); + return ["slack-canary"]; + }, + }); + + expect(runQaSuiteCommand).toHaveBeenCalledWith({ + repoRoot: "/repo", + outputDir: ".artifacts/slack", + providerMode: "live-frontier", + primaryModel: "openai/gpt-5.5", + alternateModel: "openai/gpt-5.5-alt", + fastMode: true, + allowFailures: true, + failFast: true, + channelDriver: "live", + channel: "slack", + concurrency: 1, + scenarioIds: ["slack-canary"], + sutAccountId: "slack-sut", + credentialSource: "convex", + credentialRole: "ci", + explicitScenarioSelection: false, + }); + }); + + it("preserves explicit scenario selection after resolving defaults", async () => { + await runLiveTransportQaSuiteCommand({ + channelId: "whatsapp", + defaultProviderMode: "live-frontier", + options: { scenarioIds: ["whatsapp-help-command"] }, + selectScenarioIds: ({ scenarioIds }) => [...(scenarioIds ?? [])], + }); + + expect(runQaSuiteCommand).toHaveBeenCalledWith( + expect.objectContaining({ + explicitScenarioSelection: true, + scenarioIds: ["whatsapp-help-command"], + }), + ); + }); + + it("rejects shared credentials for disposable transports", async () => { + await expect( + runLiveTransportQaSuiteCommand({ + channelId: "matrix", + credentialMode: "env-only", + defaultProviderMode: "live-frontier", + envCredentialReason: "its homeserver is disposable and local.", + laneLabel: "Matrix", + options: { credentialSource: "convex" }, + selectScenarioIds: () => ["channel-chat-baseline"], + }), + ).rejects.toThrow( + "QA Lab Matrix supports only --credential-source env because its homeserver is disposable and local.", + ); + await expect( + runLiveTransportQaSuiteCommand({ + channelId: "matrix", + credentialMode: "env-only", + defaultProviderMode: "live-frontier", + laneLabel: "Matrix", + options: { credentialRole: "ci" }, + selectScenarioIds: () => ["channel-chat-baseline"], + }), + ).rejects.toThrow("QA Lab Matrix does not use credential roles."); + expect(runQaSuiteCommand).not.toHaveBeenCalled(); + }); + + it("rejects unknown provider modes before suite dispatch", async () => { + await expect( + runLiveTransportQaSuiteCommand({ + channelId: "discord", + defaultProviderMode: "live-frontier", + options: { providerMode: "unknown" }, + selectScenarioIds: () => ["discord-canary"], + }), + ).rejects.toThrow("unknown QA provider mode: unknown"); + expect(runQaSuiteCommand).not.toHaveBeenCalled(); + }); +}); diff --git a/extensions/qa-lab/src/live-transports/shared/live-transport-suite.runtime.ts b/extensions/qa-lab/src/live-transports/shared/live-transport-suite.runtime.ts new file mode 100644 index 000000000000..4a52c4db175b --- /dev/null +++ b/extensions/qa-lab/src/live-transports/shared/live-transport-suite.runtime.ts @@ -0,0 +1,65 @@ +import type { LiveTransportQaCommandOptions } from "openclaw/plugin-sdk/qa-runtime"; +import { runQaSuiteCommand } from "../../cli.runtime.js"; +import type { QaProviderMode } from "../../providers/index.js"; +import { normalizeQaProviderMode } from "../../run-config.js"; + +type LiveTransportScenarioSelection = (params: { + profile?: string; + providerMode: QaProviderMode; + scenarioIds?: readonly string[]; +}) => string[]; + +export async function runLiveTransportQaSuiteCommand(params: { + channelId: string; + credentialMode?: "env-only" | "shared-lease"; + defaultProviderMode: QaProviderMode; + envCredentialReason?: string; + laneLabel?: string; + options: LiveTransportQaCommandOptions; + selectScenarioIds: LiveTransportScenarioSelection; +}) { + const options = params.options; + if (params.credentialMode === "env-only") { + const laneLabel = params.laneLabel ?? params.channelId; + const credentialSource = options.credentialSource?.trim().toLowerCase(); + if (credentialSource && credentialSource !== "env") { + throw new Error( + `QA Lab ${laneLabel} supports only --credential-source env${params.envCredentialReason ? ` because ${params.envCredentialReason}` : "."}`, + ); + } + if (options.credentialRole?.trim()) { + throw new Error(`QA Lab ${laneLabel} does not use credential roles.`); + } + } + + const providerMode = + options.providerMode === undefined + ? params.defaultProviderMode + : normalizeQaProviderMode(options.providerMode); + return runQaSuiteCommand({ + repoRoot: options.repoRoot, + outputDir: options.outputDir, + providerMode, + primaryModel: options.primaryModel, + alternateModel: options.alternateModel, + fastMode: options.fastMode, + allowFailures: options.allowFailures, + failFast: options.failFast, + channelDriver: "live", + channel: params.channelId, + concurrency: 1, + scenarioIds: params.selectScenarioIds({ + profile: options.profile, + providerMode, + scenarioIds: options.scenarioIds, + }), + sutAccountId: options.sutAccountId, + ...(params.credentialMode === "env-only" + ? {} + : { + credentialSource: options.credentialSource?.trim(), + credentialRole: options.credentialRole?.trim(), + }), + explicitScenarioSelection: Boolean(options.scenarioIds?.length), + }); +} diff --git a/extensions/qa-lab/src/live-transports/slack/cli.runtime.test.ts b/extensions/qa-lab/src/live-transports/slack/cli.runtime.test.ts deleted file mode 100644 index 08839801c7fe..000000000000 --- a/extensions/qa-lab/src/live-transports/slack/cli.runtime.test.ts +++ /dev/null @@ -1,71 +0,0 @@ -// QA Lab Slack tests cover CLI delegation into the shared suite host. -import { beforeEach, describe, expect, it, vi } from "vitest"; - -const runQaSuiteCommand = vi.hoisted(() => vi.fn()); - -vi.mock("../../cli.runtime.js", () => ({ runQaSuiteCommand })); -vi.mock("../shared/live-transport-cli.runtime.js", () => ({ - resolveLiveTransportQaRunOptions: (opts: Record) => ({ - ...opts, - repoRoot: "/resolved-repo", - outputDir: "/resolved-repo/.artifacts/resolved", - providerMode: opts.providerMode ?? "mock-openai", - }), -})); - -import { listQaScenariosForExecutionProfile } from "../../scenario-catalog.js"; -import { runQaSlackCommand } from "./cli.runtime.js"; - -describe("QA Lab Slack CLI runtime", () => { - beforeEach(() => { - vi.clearAllMocks(); - }); - - it("delegates the default profile into the Slack adapter host", async () => { - await runQaSlackCommand({ - repoRoot: "/repo", - outputDir: ".artifacts/slack", - providerMode: "mock-openai", - primaryModel: "mock-openai/gpt-5.5", - alternateModel: "mock-openai/gpt-5.5-alt", - fastMode: true, - allowFailures: true, - failFast: true, - credentialSource: "convex", - credentialRole: "ci", - sutAccountId: "slack-sut", - }); - - expect(runQaSuiteCommand).toHaveBeenCalledWith({ - repoRoot: "/repo", - outputDir: ".artifacts/slack", - providerMode: "mock-openai", - primaryModel: "mock-openai/gpt-5.5", - alternateModel: "mock-openai/gpt-5.5-alt", - fastMode: true, - allowFailures: true, - failFast: true, - channelDriver: "live", - channel: "slack", - concurrency: 1, - scenarioIds: listQaScenariosForExecutionProfile("slack:default").map( - (scenario) => scenario.id, - ), - sutAccountId: "slack-sut", - credentialSource: "convex", - credentialRole: "ci", - explicitScenarioSelection: false, - }); - }); - - it("lets explicit scenarios override profile selection", async () => { - await runQaSlackCommand({ scenarioIds: ["slack-codex-approval-exec-native"] }); - - expect(runQaSuiteCommand).toHaveBeenCalledWith( - expect.objectContaining({ - explicitScenarioSelection: true, - scenarioIds: ["slack-codex-approval-exec-native"], - }), - ); - }); -}); diff --git a/extensions/qa-lab/src/live-transports/slack/cli.runtime.ts b/extensions/qa-lab/src/live-transports/slack/cli.runtime.ts deleted file mode 100644 index 4b3d80a24af4..000000000000 --- a/extensions/qa-lab/src/live-transports/slack/cli.runtime.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { runQaSuiteCommand } from "../../cli.runtime.js"; -import type { LiveTransportQaCommandOptions } from "../shared/live-transport-cli.js"; -import { resolveLiveTransportQaRunOptions } from "../shared/live-transport-cli.runtime.js"; -import { resolveSlackQaScenarioIds } from "./profiles.js"; - -export async function runQaSlackCommand(opts: LiveTransportQaCommandOptions) { - const runOptions = resolveLiveTransportQaRunOptions(opts); - return await runQaSuiteCommand({ - repoRoot: opts.repoRoot, - outputDir: opts.outputDir, - providerMode: runOptions.providerMode, - primaryModel: runOptions.primaryModel, - alternateModel: runOptions.alternateModel, - fastMode: runOptions.fastMode, - allowFailures: runOptions.allowFailures, - failFast: runOptions.failFast, - channelDriver: "live", - channel: "slack", - concurrency: 1, - scenarioIds: resolveSlackQaScenarioIds(runOptions.scenarioIds), - sutAccountId: runOptions.sutAccountId, - credentialSource: runOptions.credentialSource, - credentialRole: runOptions.credentialRole, - explicitScenarioSelection: Boolean(runOptions.scenarioIds?.length), - }); -} diff --git a/extensions/qa-lab/src/live-transports/slack/cli.ts b/extensions/qa-lab/src/live-transports/slack/cli.ts index f12b43b60071..a1967931e53a 100644 --- a/extensions/qa-lab/src/live-transports/slack/cli.ts +++ b/extensions/qa-lab/src/live-transports/slack/cli.ts @@ -1,37 +1,37 @@ // Qa Lab plugin module implements cli behavior. import { + createLiveTransportQaAdapterFactory, createLazyCliRuntimeLoader, createLiveTransportQaCliRegistration, + loadLiveTransportQaSuiteRuntime, type LiveTransportQaCliRegistration, type LiveTransportQaCommandOptions, } from "../shared/live-transport-cli.js"; +import { resolveSlackQaScenarioIds } from "./scenario-selection.js"; -type SlackQaAdapterRuntime = typeof import("./adapter.runtime.js"); -type SlackQaCliRuntime = typeof import("./cli.runtime.js"); - -const loadSlackQaAdapterRuntime = createLazyCliRuntimeLoader( +const loadSlackQaAdapterRuntime = createLazyCliRuntimeLoader( () => import("./adapter.runtime.js"), ); -const loadSlackQaCliRuntime = createLazyCliRuntimeLoader( - () => import("./cli.runtime.js"), -); async function runQaSlack(opts: LiveTransportQaCommandOptions) { - await (await loadSlackQaCliRuntime()).runQaSlackCommand(opts); + const runtime = await loadLiveTransportQaSuiteRuntime(); + await runtime.runLiveTransportQaSuiteCommand({ + channelId: "slack", + defaultProviderMode: "live-frontier", + options: opts, + selectScenarioIds: resolveSlackQaScenarioIds, + }); } -const slackQaAdapterFactory: NonNullable = { - id: "slack", - matches: ({ channelId, driver }) => driver === "live" && channelId === "slack", - async create(context) { - return await (await loadSlackQaAdapterRuntime()).createSlackQaTransportAdapter(context); - }, -}; - export const slackQaCliRegistration: LiveTransportQaCliRegistration = createLiveTransportQaCliRegistration({ commandName: "slack", - adapterFactory: slackQaAdapterFactory, + adapterFactory: createLiveTransportQaAdapterFactory({ + id: "slack", + async create(context) { + return (await loadSlackQaAdapterRuntime()).createSlackQaTransportAdapter(context); + }, + }), credentialOptions: { sourceDescription: "Credential source for Slack QA: env or convex (default: env)", roleDescription: diff --git a/extensions/qa-lab/src/live-transports/slack/legacy-scenario-parity.test.ts b/extensions/qa-lab/src/live-transports/slack/legacy-scenario-parity.test.ts deleted file mode 100644 index 79b4de9f9041..000000000000 --- a/extensions/qa-lab/src/live-transports/slack/legacy-scenario-parity.test.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { readQaScenarioPack } from "../../scenario-catalog.js"; -import * as scenarioRuntime from "./scenario-runtime.js"; - -describe("legacy Slack scenario migration", () => { - it("keeps every retired runner id as a Slack module scenario", () => { - const scenarios = readQaScenarioPack().scenarios.filter( - (scenario) => - scenario.execution.kind === "flow" && - scenario.execution.channel === "slack" && - JSON.stringify(scenario.execution.flow).includes( - "./live-transports/slack/scenario-runtime.js", - ), - ); - expect(scenarios).toHaveLength(17); - for (const scenario of scenarios) { - expect(scenario.execution).toMatchObject({ kind: "flow", channel: "slack" }); - expect(JSON.stringify(scenario.execution.flow)).toContain( - "./live-transports/slack/scenario-runtime.js", - ); - const flowText = JSON.stringify(scenario.execution.flow); - const callName = flowText.match(/scenarioModule\.([A-Za-z0-9]+)/u)?.[1]; - expect(typeof scenarioRuntime[callName as keyof typeof scenarioRuntime], scenario.id).toBe( - "function", - ); - } - }); -}); diff --git a/extensions/qa-lab/src/live-transports/slack/profiles.ts b/extensions/qa-lab/src/live-transports/slack/scenario-selection.ts similarity index 69% rename from extensions/qa-lab/src/live-transports/slack/profiles.ts rename to extensions/qa-lab/src/live-transports/slack/scenario-selection.ts index 3fe8cbdb14e9..24f88b07638d 100644 --- a/extensions/qa-lab/src/live-transports/slack/profiles.ts +++ b/extensions/qa-lab/src/live-transports/slack/scenario-selection.ts @@ -1,6 +1,6 @@ import { listQaScenariosForExecutionProfile } from "../../scenario-catalog.js"; -export function resolveSlackQaScenarioIds(scenarioIds?: readonly string[]) { +export function resolveSlackQaScenarioIds({ scenarioIds }: { scenarioIds?: readonly string[] }) { return scenarioIds?.length ? [...scenarioIds] : listQaScenariosForExecutionProfile("slack:default").map((scenario) => scenario.id); diff --git a/extensions/qa-lab/src/live-transports/slack/slack-live.runtime.test.ts b/extensions/qa-lab/src/live-transports/slack/slack-live.runtime.test.ts index cae3edf49e14..1ffb64adb00a 100644 --- a/extensions/qa-lab/src/live-transports/slack/slack-live.runtime.test.ts +++ b/extensions/qa-lab/src/live-transports/slack/slack-live.runtime.test.ts @@ -1,6 +1,6 @@ // Qa Lab tests cover slack live plugin behavior. import { beforeEach, describe, expect, it, vi } from "vitest"; -import { resolveSlackQaScenarioIds } from "./profiles.js"; +import { resolveSlackQaScenarioIds } from "./scenario-selection.js"; import { resolveApprovalDecision } from "./slack-live.approvals.js"; import { quiesceCodexApprovalAgentRun, @@ -30,7 +30,7 @@ import { } from "./slack-live.scenarios.js"; function findScenario(ids?: string[]) { - const requestedIds = new Set(ids?.length ? ids : resolveSlackQaScenarioIds()); + const requestedIds = new Set(ids?.length ? ids : resolveSlackQaScenarioIds({})); return listSlackQaScenarioCatalog() .filter(({ id }) => requestedIds.has(id)) .map(({ id }) => getSlackQaScenarioDefinition(id)); diff --git a/extensions/qa-lab/src/live-transports/telegram/adapter.runtime.test.ts b/extensions/qa-lab/src/live-transports/telegram/adapter.runtime.test.ts index a09a91cbc16c..b664afbc3e4c 100644 --- a/extensions/qa-lab/src/live-transports/telegram/adapter.runtime.test.ts +++ b/extensions/qa-lab/src/live-transports/telegram/adapter.runtime.test.ts @@ -172,9 +172,10 @@ describe("Telegram QA transport adapter", () => { ); await vi.waitFor(() => expect(pollResolvers).toHaveLength(3)); + mocks.heartbeatStop.mockRejectedValueOnce(new Error("heartbeat stop failed")); const cleanup = adapter.cleanup?.(); pollResolvers[2]?.([]); - await cleanup; + await expect(cleanup).rejects.toThrow("heartbeat stop failed"); expect(mocks.heartbeatStop).toHaveBeenCalledOnce(); expect(mocks.leaseRelease).toHaveBeenCalledOnce(); }); diff --git a/extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts b/extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts index cb52c79b379a..8771ab144c87 100644 --- a/extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts +++ b/extensions/qa-lab/src/live-transports/telegram/adapter.runtime.ts @@ -45,6 +45,14 @@ export async function createTelegramQaTransportAdapter( throw error; } const heartbeat = startQaCredentialLeaseHeartbeat(credentialLease); + const releaseCredentialLease = async () => { + // Lease release must still run when heartbeat shutdown reports an error. + try { + await heartbeat.stop(); + } finally { + await credentialLease.release(); + } + }; const runtimeEnv = credentialLease.payload; let driverIdentity: TelegramBotIdentity; let sutIdentity: TelegramBotIdentity; @@ -68,10 +76,10 @@ export async function createTelegramQaTransportAdapter( flushTelegramUpdates(runtimeEnv.sutToken), ]); } catch (error) { - await heartbeat.stop(); - await credentialLease.release(); + await releaseCredentialLease(); throw error; } + const accountId = options.sutAccountId?.trim() || "sut"; let stopped = false; let pollingError: Error | undefined; let logicalConversationId = runtimeEnv.groupId; @@ -112,7 +120,7 @@ export async function createTelegramQaTransportAdapter( if (update.edited_message) { if (existingMessageId) { await context.messages.editMessage({ - accountId: options.sutAccountId?.trim() || "sut", + accountId, messageId: existingMessageId, text: message.text, timestamp: message.timestamp, @@ -121,7 +129,7 @@ export async function createTelegramQaTransportAdapter( continue; } const outbound = await context.messages.addOutboundMessage({ - accountId: options.sutAccountId?.trim() || "sut", + accountId, to: `${logicalConversationKind}:${logicalConversationId}`, senderId: String(message.senderId), senderName: message.senderUsername, @@ -141,8 +149,6 @@ export async function createTelegramQaTransportAdapter( pollingError = error instanceof Error ? error : new Error(String(error)); } }); - const accountId = options.sutAccountId?.trim() || "sut"; - return { id: "telegram", label: "Telegram live", @@ -238,8 +244,7 @@ export async function createTelegramQaTransportAdapter( quarantineErrors.length > 0 ? { cause: new AggregateError(quarantineErrors) } : undefined, ); } - await heartbeat.stop(); - await credentialLease.release(); + await releaseCredentialLease(); }, }; } diff --git a/extensions/qa-lab/src/live-transports/telegram/cli.runtime.ts b/extensions/qa-lab/src/live-transports/telegram/cli.runtime.ts index bc6702193da1..12ce1839715e 100644 --- a/extensions/qa-lab/src/live-transports/telegram/cli.runtime.ts +++ b/extensions/qa-lab/src/live-transports/telegram/cli.runtime.ts @@ -1,16 +1,16 @@ import { constants as fsConstants } from "node:fs"; import fs from "node:fs/promises"; import path from "node:path"; +import type { LiveTransportQaCommandOptions } from "openclaw/plugin-sdk/qa-runtime"; import type { QaGatewayChildCommand } from "../../gateway-child.js"; import { runQaFlowSuiteFromRuntime } from "../../suite-launch.runtime.js"; import type { QaSuiteRoundTripProbe } from "../../suite-round-trip.js"; import { readQaSuiteFailedScenarioCountFromFile } from "../../suite-summary.js"; // Qa Lab plugin module implements cli behavior. import { printLiveTransportQaArtifacts } from "../shared/live-artifacts.js"; -import type { LiveTransportQaCommandOptions } from "../shared/live-transport-cli.js"; -import { resolveLiveTransportQaRunOptions } from "../shared/live-transport-cli.runtime.js"; import { createTelegramQaTransportAdapter } from "./adapter.runtime.js"; import { listTelegramQaScenarios, resolveTelegramQaScenarioIds } from "./profiles.js"; +import { resolveTelegramQaRunOptions } from "./run-options.runtime.js"; const TELEGRAM_QA_SUT_OPENCLAW_COMMAND_ENV = "OPENCLAW_QA_TELEGRAM_SUT_OPENCLAW_COMMAND"; const TELEGRAM_QA_SUT_UID_ENV = "OPENCLAW_QA_TELEGRAM_SUT_UID"; @@ -136,12 +136,13 @@ async function resolveTelegramQaSutOpenClawCommand( } type TelegramQaSuiteOptions = LiveTransportQaCommandOptions & { + resolvedScenarioIds?: readonly string[]; roundTripProbe?: QaSuiteRoundTripProbe; sutOpenClawCommand?: QaGatewayChildCommand; }; export async function runQaTelegramSuite(opts: TelegramQaSuiteOptions) { - const runOptions = resolveLiveTransportQaRunOptions(opts); + const runOptions = resolveTelegramQaRunOptions(opts); if (runOptions.listScenarios) { for (const scenario of listTelegramQaScenarios(runOptions.providerMode)) { const defaultLabel = scenario.defaultEnabled ? "default" : "optional"; @@ -153,11 +154,13 @@ export async function runQaTelegramSuite(opts: TelegramQaSuiteOptions) { } return undefined; } - const scenarioIds = resolveTelegramQaScenarioIds({ - profile: opts.profile, - providerMode: runOptions.providerMode, - scenarioIds: runOptions.scenarioIds, - }); + const scenarioIds = opts.resolvedScenarioIds + ? [...opts.resolvedScenarioIds] + : resolveTelegramQaScenarioIds({ + profile: opts.profile, + providerMode: runOptions.providerMode, + scenarioIds: runOptions.scenarioIds, + }); const result = await runQaFlowSuiteFromRuntime({ adapterFactories: [ { @@ -200,11 +203,11 @@ export async function runQaTelegramSuite(opts: TelegramQaSuiteOptions) { } export async function runQaTelegramCommand(opts: LiveTransportQaCommandOptions) { - const runOptions = resolveLiveTransportQaRunOptions(opts); + const runOptions = resolveTelegramQaRunOptions(opts); if (runOptions.listScenarios) { return await runQaTelegramSuite(opts); } - resolveTelegramQaScenarioIds({ + const resolvedScenarioIds = resolveTelegramQaScenarioIds({ profile: opts.profile, providerMode: runOptions.providerMode, scenarioIds: runOptions.scenarioIds, @@ -212,6 +215,7 @@ export async function runQaTelegramCommand(opts: LiveTransportQaCommandOptions) const sutOpenClawCommand = await resolveTelegramQaSutOpenClawCommand(runOptions.repoRoot); return await runQaTelegramSuite({ ...opts, + resolvedScenarioIds, ...(sutOpenClawCommand ? { sutOpenClawCommand } : {}), }); } diff --git a/extensions/qa-lab/src/live-transports/telegram/cli.ts b/extensions/qa-lab/src/live-transports/telegram/cli.ts index 8837ba288aea..b5e83e64b6fc 100644 --- a/extensions/qa-lab/src/live-transports/telegram/cli.ts +++ b/extensions/qa-lab/src/live-transports/telegram/cli.ts @@ -1,37 +1,28 @@ // Qa Lab plugin module implements cli behavior. import { + createLiveTransportQaAdapterFactory, createLazyCliRuntimeLoader, createLiveTransportQaCliRegistration, type LiveTransportQaCliRegistration, type LiveTransportQaCommandOptions, } from "../shared/live-transport-cli.js"; -type TelegramQaAdapterRuntime = typeof import("./adapter.runtime.js"); -type TelegramQaCliRuntime = typeof import("./cli.runtime.js"); - -const loadTelegramQaAdapterRuntime = createLazyCliRuntimeLoader( - () => import("./adapter.runtime.js"), -); -const loadTelegramQaCliRuntime = createLazyCliRuntimeLoader( +const loadTelegramQaAdapterRuntime = createLazyCliRuntimeLoader< + typeof import("./adapter.runtime.js") +>(() => import("./adapter.runtime.js")); +const loadTelegramQaCliRuntime = createLazyCliRuntimeLoader( () => import("./cli.runtime.js"), ); -async function runQaTelegram(opts: LiveTransportQaCommandOptions) { - await (await loadTelegramQaCliRuntime()).runQaTelegramCommand(opts); -} - -const telegramQaAdapterFactory: NonNullable = { - id: "telegram", - matches: ({ channelId, driver }) => driver === "live" && channelId === "telegram", - async create(context) { - return await (await loadTelegramQaAdapterRuntime()).createTelegramQaTransportAdapter(context); - }, -}; - export const telegramQaCliRegistration: LiveTransportQaCliRegistration = createLiveTransportQaCliRegistration({ commandName: "telegram", - adapterFactory: telegramQaAdapterFactory, + adapterFactory: createLiveTransportQaAdapterFactory({ + id: "telegram", + async create(context) { + return (await loadTelegramQaAdapterRuntime()).createTelegramQaTransportAdapter(context); + }, + }), credentialOptions: { sourceDescription: "Credential source for Telegram QA: env or convex (default: env)", roleDescription: @@ -41,7 +32,9 @@ export const telegramQaCliRegistration: LiveTransportQaCliRegistration = listScenariosHelp: "Print available Telegram scenario ids and exit", outputDirHelp: "Telegram QA artifact directory", profileHelp: "QA Lab Telegram profile: release or all (default: release)", - run: runQaTelegram, + async run(opts: LiveTransportQaCommandOptions) { + await (await loadTelegramQaCliRuntime()).runQaTelegramCommand(opts); + }, scenarioHelp: "Run only the named Telegram QA scenario (repeatable)", sutAccountHelp: "Temporary Telegram account id inside the QA gateway config", }); diff --git a/extensions/qa-lab/src/live-transports/telegram/profiles.test.ts b/extensions/qa-lab/src/live-transports/telegram/profiles.test.ts index f4e2479b879e..6fc1ebcb84fe 100644 --- a/extensions/qa-lab/src/live-transports/telegram/profiles.test.ts +++ b/extensions/qa-lab/src/live-transports/telegram/profiles.test.ts @@ -28,16 +28,16 @@ describe("Telegram QA profiles", () => { ).toEqual(["thread-follow-up"]); }); - it("rejects unknown profiles and scenarios before gateway startup", () => { + it("rejects unknown profiles and leaves explicit scenario validation to the suite catalog", () => { expect(() => resolveTelegramQaScenarioIds({ providerMode: "live-frontier", profile: "transport" }), ).toThrow('Unknown QA Lab Telegram profile "transport"'); - expect(() => + expect( resolveTelegramQaScenarioIds({ providerMode: "live-frontier", - scenarioIds: ["telegram-missing"], + scenarioIds: ["channel-chat-baseline"], }), - ).toThrow("unknown Telegram QA scenario id(s): telegram-missing"); + ).toEqual(["channel-chat-baseline"]); }); it("lists the YAML catalog with provider-specific release defaults", () => { diff --git a/extensions/qa-lab/src/live-transports/shared/live-transport-cli.runtime.test.ts b/extensions/qa-lab/src/live-transports/telegram/run-options.runtime.test.ts similarity index 77% rename from extensions/qa-lab/src/live-transports/shared/live-transport-cli.runtime.test.ts rename to extensions/qa-lab/src/live-transports/telegram/run-options.runtime.test.ts index 9fdad094a3ad..10a6eef7111a 100644 --- a/extensions/qa-lab/src/live-transports/shared/live-transport-cli.runtime.test.ts +++ b/extensions/qa-lab/src/live-transports/telegram/run-options.runtime.test.ts @@ -1,11 +1,11 @@ // Qa Lab tests cover live transport cli plugin behavior. import path from "node:path"; import { describe, expect, it } from "vitest"; -import { resolveLiveTransportQaRunOptions } from "./live-transport-cli.runtime.js"; +import { resolveTelegramQaRunOptions } from "./run-options.runtime.js"; -describe("resolveLiveTransportQaRunOptions", () => { +describe("resolveTelegramQaRunOptions", () => { it("drops blank model refs so live transports can use provider defaults", () => { - const options = resolveLiveTransportQaRunOptions({ + const options = resolveTelegramQaRunOptions({ repoRoot: "/tmp/openclaw-repo", providerMode: "live-frontier", primaryModel: " ", diff --git a/extensions/qa-lab/src/live-transports/shared/live-transport-cli.runtime.ts b/extensions/qa-lab/src/live-transports/telegram/run-options.runtime.ts similarity index 74% rename from extensions/qa-lab/src/live-transports/shared/live-transport-cli.runtime.ts rename to extensions/qa-lab/src/live-transports/telegram/run-options.runtime.ts index f1238883d52b..f26e9f64bd97 100644 --- a/extensions/qa-lab/src/live-transports/shared/live-transport-cli.runtime.ts +++ b/extensions/qa-lab/src/live-transports/telegram/run-options.runtime.ts @@ -1,17 +1,17 @@ -// Qa Lab plugin module implements live transport cli behavior. +// Qa Lab plugin module normalizes Telegram live-run options. import path from "node:path"; +import type { LiveTransportQaCommandOptions } from "openclaw/plugin-sdk/qa-runtime"; import { resolveRepoRelativeOutputDir } from "../../cli-paths.js"; import { DEFAULT_QA_LIVE_PROVIDER_MODE } from "../../providers/index.js"; import type { QaProviderMode } from "../../run-config.js"; import { normalizeQaProviderMode } from "../../run-config.js"; -import type { LiveTransportQaCommandOptions } from "./live-transport-cli.js"; -function normalizeLiveTransportModelRef(input: string | undefined) { +function normalizeTelegramModelRef(input: string | undefined) { const model = input?.trim(); return model && model.length > 0 ? model : undefined; } -export function resolveLiveTransportQaRunOptions( +export function resolveTelegramQaRunOptions( opts: LiveTransportQaCommandOptions, ): LiveTransportQaCommandOptions & { repoRoot: string; @@ -27,8 +27,8 @@ export function resolveLiveTransportQaRunOptions( opts.providerMode === undefined ? DEFAULT_QA_LIVE_PROVIDER_MODE : normalizeQaProviderMode(opts.providerMode), - primaryModel: normalizeLiveTransportModelRef(opts.primaryModel), - alternateModel: normalizeLiveTransportModelRef(opts.alternateModel), + primaryModel: normalizeTelegramModelRef(opts.primaryModel), + alternateModel: normalizeTelegramModelRef(opts.alternateModel), fastMode: opts.fastMode, allowFailures: opts.allowFailures, failFast: opts.failFast, diff --git a/extensions/qa-lab/src/live-transports/whatsapp/cli.runtime.test.ts b/extensions/qa-lab/src/live-transports/whatsapp/cli.runtime.test.ts deleted file mode 100644 index b383b4243b06..000000000000 --- a/extensions/qa-lab/src/live-transports/whatsapp/cli.runtime.test.ts +++ /dev/null @@ -1,69 +0,0 @@ -// QA Lab WhatsApp tests cover CLI delegation into the shared suite host. -import { beforeEach, describe, expect, it, vi } from "vitest"; - -const runQaSuiteCommand = vi.hoisted(() => vi.fn()); - -vi.mock("../../cli.runtime.js", () => ({ runQaSuiteCommand })); -vi.mock("../shared/live-transport-cli.runtime.js", () => ({ - resolveLiveTransportQaRunOptions: (opts: Record) => ({ - ...opts, - repoRoot: "/resolved-repo", - outputDir: "/resolved-repo/.artifacts/resolved", - providerMode: opts.providerMode ?? "live-frontier", - }), -})); - -import { runQaWhatsAppCommand } from "./cli.runtime.js"; -import { resolveWhatsAppQaScenarioIds } from "./profiles.js"; - -describe("QA Lab WhatsApp CLI runtime", () => { - beforeEach(() => { - vi.clearAllMocks(); - }); - - it("delegates the live profile into the WhatsApp adapter host", async () => { - await runQaWhatsAppCommand({ - repoRoot: "/repo", - outputDir: ".artifacts/whatsapp", - providerMode: "live-frontier", - primaryModel: "openai/gpt-5.5", - alternateModel: "openai/gpt-5.5-alt", - fastMode: true, - allowFailures: true, - failFast: true, - credentialSource: "convex", - credentialRole: "ci", - sutAccountId: "whatsapp-sut", - }); - - expect(runQaSuiteCommand).toHaveBeenCalledWith({ - repoRoot: "/repo", - outputDir: ".artifacts/whatsapp", - providerMode: "live-frontier", - primaryModel: "openai/gpt-5.5", - alternateModel: "openai/gpt-5.5-alt", - fastMode: true, - allowFailures: true, - failFast: true, - channelDriver: "live", - channel: "whatsapp", - concurrency: 1, - scenarioIds: resolveWhatsAppQaScenarioIds({ providerMode: "live-frontier" }), - sutAccountId: "whatsapp-sut", - credentialSource: "convex", - credentialRole: "ci", - explicitScenarioSelection: false, - }); - }); - - it("lets explicit scenarios override profile selection", async () => { - await runQaWhatsAppCommand({ scenarioIds: ["whatsapp-help-command"] }); - - expect(runQaSuiteCommand).toHaveBeenCalledWith( - expect.objectContaining({ - explicitScenarioSelection: true, - scenarioIds: ["whatsapp-help-command"], - }), - ); - }); -}); diff --git a/extensions/qa-lab/src/live-transports/whatsapp/cli.runtime.ts b/extensions/qa-lab/src/live-transports/whatsapp/cli.runtime.ts deleted file mode 100644 index e794426af50f..000000000000 --- a/extensions/qa-lab/src/live-transports/whatsapp/cli.runtime.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { runQaSuiteCommand } from "../../cli.runtime.js"; -import type { LiveTransportQaCommandOptions } from "../shared/live-transport-cli.js"; -import { resolveLiveTransportQaRunOptions } from "../shared/live-transport-cli.runtime.js"; -import { resolveWhatsAppQaScenarioIds } from "./profiles.js"; - -export async function runQaWhatsAppCommand(opts: LiveTransportQaCommandOptions) { - const runOptions = resolveLiveTransportQaRunOptions(opts); - return await runQaSuiteCommand({ - repoRoot: opts.repoRoot, - outputDir: opts.outputDir, - providerMode: runOptions.providerMode, - primaryModel: runOptions.primaryModel, - alternateModel: runOptions.alternateModel, - fastMode: runOptions.fastMode, - allowFailures: runOptions.allowFailures, - failFast: runOptions.failFast, - channelDriver: "live", - channel: "whatsapp", - concurrency: 1, - scenarioIds: resolveWhatsAppQaScenarioIds({ - providerMode: runOptions.providerMode, - scenarioIds: runOptions.scenarioIds, - }), - sutAccountId: runOptions.sutAccountId, - credentialSource: runOptions.credentialSource, - credentialRole: runOptions.credentialRole, - explicitScenarioSelection: Boolean(runOptions.scenarioIds?.length), - }); -} diff --git a/extensions/qa-lab/src/live-transports/whatsapp/cli.ts b/extensions/qa-lab/src/live-transports/whatsapp/cli.ts index 3b5f304e45ce..71810618ce39 100644 --- a/extensions/qa-lab/src/live-transports/whatsapp/cli.ts +++ b/extensions/qa-lab/src/live-transports/whatsapp/cli.ts @@ -1,37 +1,37 @@ // Qa Lab plugin module implements cli behavior. import { + createLiveTransportQaAdapterFactory, createLazyCliRuntimeLoader, createLiveTransportQaCliRegistration, + loadLiveTransportQaSuiteRuntime, type LiveTransportQaCliRegistration, type LiveTransportQaCommandOptions, } from "../shared/live-transport-cli.js"; +import { resolveWhatsAppQaScenarioIds } from "./scenario-selection.js"; -type WhatsAppQaAdapterRuntime = typeof import("./adapter.runtime.js"); -type WhatsAppQaCliRuntime = typeof import("./cli.runtime.js"); - -const loadWhatsAppQaAdapterRuntime = createLazyCliRuntimeLoader( - () => import("./adapter.runtime.js"), -); -const loadWhatsAppQaCliRuntime = createLazyCliRuntimeLoader( - () => import("./cli.runtime.js"), -); +const loadWhatsAppQaAdapterRuntime = createLazyCliRuntimeLoader< + typeof import("./adapter.runtime.js") +>(() => import("./adapter.runtime.js")); async function runQaWhatsApp(opts: LiveTransportQaCommandOptions) { - await (await loadWhatsAppQaCliRuntime()).runQaWhatsAppCommand(opts); + const runtime = await loadLiveTransportQaSuiteRuntime(); + await runtime.runLiveTransportQaSuiteCommand({ + channelId: "whatsapp", + defaultProviderMode: "live-frontier", + options: opts, + selectScenarioIds: resolveWhatsAppQaScenarioIds, + }); } -const whatsappQaAdapterFactory: NonNullable = { - id: "whatsapp", - matches: ({ channelId, driver }) => driver === "live" && channelId === "whatsapp", - async create(context) { - return await (await loadWhatsAppQaAdapterRuntime()).createWhatsAppQaTransportAdapter(context); - }, -}; - export const whatsappQaCliRegistration: LiveTransportQaCliRegistration = createLiveTransportQaCliRegistration({ commandName: "whatsapp", - adapterFactory: whatsappQaAdapterFactory, + adapterFactory: createLiveTransportQaAdapterFactory({ + id: "whatsapp", + async create(context) { + return (await loadWhatsAppQaAdapterRuntime()).createWhatsAppQaTransportAdapter(context); + }, + }), credentialOptions: { sourceDescription: "Credential source for WhatsApp QA: env or convex (default: env)", roleDescription: diff --git a/extensions/qa-lab/src/live-transports/whatsapp/legacy-scenario-parity.test.ts b/extensions/qa-lab/src/live-transports/whatsapp/legacy-scenario-parity.test.ts deleted file mode 100644 index 33514727edf1..000000000000 --- a/extensions/qa-lab/src/live-transports/whatsapp/legacy-scenario-parity.test.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { readQaScenarioPack } from "../../scenario-catalog.js"; -import * as scenarioRuntime from "./scenario-runtime.js"; - -describe("legacy WhatsApp scenario migration", () => { - it("keeps every retired runner id as a WhatsApp module scenario", () => { - const scenarios = readQaScenarioPack().scenarios.filter( - (scenario) => - scenario.execution.kind === "flow" && - scenario.execution.channel === "whatsapp" && - JSON.stringify(scenario.execution.flow).includes( - "./live-transports/whatsapp/scenario-runtime.js", - ), - ); - expect(scenarios).toHaveLength(38); - for (const scenario of scenarios) { - expect(scenario.execution).toMatchObject({ kind: "flow", channel: "whatsapp" }); - expect(JSON.stringify(scenario.execution.flow)).toContain( - "./live-transports/whatsapp/scenario-runtime.js", - ); - const flowText = JSON.stringify(scenario.execution.flow); - const callName = flowText.match(/scenarioModule\.([A-Za-z0-9]+)/u)?.[1]; - expect(typeof scenarioRuntime[callName as keyof typeof scenarioRuntime], scenario.id).toBe( - "function", - ); - } - }); -}); diff --git a/extensions/qa-lab/src/live-transports/whatsapp/profiles.ts b/extensions/qa-lab/src/live-transports/whatsapp/scenario-selection.ts similarity index 100% rename from extensions/qa-lab/src/live-transports/whatsapp/profiles.ts rename to extensions/qa-lab/src/live-transports/whatsapp/scenario-selection.ts diff --git a/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.runtime.test.ts b/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.runtime.test.ts index 840ed7ac8f74..0b02f9c79ae0 100644 --- a/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.runtime.test.ts +++ b/extensions/qa-lab/src/live-transports/whatsapp/whatsapp-live.runtime.test.ts @@ -11,7 +11,7 @@ import type { } from "@openclaw/whatsapp/api.js"; import { describe, expect, it, vi } from "vitest"; import { fingerprintQaCredentialId } from "../../qa-credentials-fingerprint.runtime.js"; -import { resolveWhatsAppQaScenarioIds } from "./profiles.js"; +import { resolveWhatsAppQaScenarioIds } from "./scenario-selection.js"; import { runWhatsAppApprovalScenario } from "./whatsapp-live.approvals.js"; import { buildWhatsAppQaConfig, parseWhatsAppQaCredentialPayload } from "./whatsapp-live.config.js"; import { resolveWhatsAppQaMessageTargets } from "./whatsapp-live.contracts.js";