From 5cd4996205c6a333926daeb2d85e57412fe77026 Mon Sep 17 00:00:00 2001 From: Ayaan Zaidi Date: Fri, 8 May 2026 15:47:29 +0530 Subject: [PATCH] feat(qa-lab): list telegram live scenarios --- extensions/qa-lab/src/cli.runtime.test.ts | 29 +++++++++++++++++++ extensions/qa-lab/src/cli.test.ts | 13 ++++++++- .../shared/live-transport-cli.runtime.test.ts | 2 ++ .../shared/live-transport-cli.runtime.ts | 1 + .../shared/live-transport-cli.ts | 10 +++++++ .../live-transports/telegram/cli.runtime.ts | 13 ++++++++- .../src/live-transports/telegram/cli.ts | 1 + 7 files changed, 67 insertions(+), 2 deletions(-) diff --git a/extensions/qa-lab/src/cli.runtime.test.ts b/extensions/qa-lab/src/cli.runtime.test.ts index 1a1af099647..45ee7b56a71 100644 --- a/extensions/qa-lab/src/cli.runtime.test.ts +++ b/extensions/qa-lab/src/cli.runtime.test.ts @@ -8,6 +8,7 @@ const { runQaSuiteFromRuntime, runQaCharacterEval, runQaMultipass, + listTelegramQaScenarioCatalog, runTelegramQaLive, startQaLabServer, writeQaDockerHarnessFiles, @@ -19,6 +20,7 @@ const { runQaSuiteFromRuntime: vi.fn(), runQaCharacterEval: vi.fn(), runQaMultipass: vi.fn(), + listTelegramQaScenarioCatalog: vi.fn(), runTelegramQaLive: vi.fn(), startQaLabServer: vi.fn(), writeQaDockerHarnessFiles: vi.fn(), @@ -45,6 +47,7 @@ vi.mock("./multipass.runtime.js", () => ({ })); vi.mock("./live-transports/telegram/telegram-live.runtime.js", () => ({ + listTelegramQaScenarioCatalog, runTelegramQaLive, })); @@ -111,6 +114,7 @@ describe("qa cli runtime", () => { runQaCharacterEval.mockReset(); runQaManualLane.mockReset(); runQaMultipass.mockReset(); + listTelegramQaScenarioCatalog.mockReset(); runTelegramQaLive.mockReset(); startQaLabServer.mockReset(); writeQaDockerHarnessFiles.mockReset(); @@ -153,6 +157,15 @@ describe("qa cli runtime", () => { observedMessagesPath: "/tmp/telegram/observed.json", scenarios: [], }); + listTelegramQaScenarioCatalog.mockReturnValue([ + { + id: "telegram-status-command", + title: "Telegram status command reply", + defaultEnabled: true, + rationale: "status rationale", + regressionRefs: ["openclaw/openclaw#74698"], + }, + ]); startQaLabServer.mockResolvedValue({ baseUrl: "http://127.0.0.1:58000", runSelfCheck: vi.fn().mockResolvedValue({ @@ -297,6 +310,22 @@ describe("qa cli runtime", () => { ); }); + it("prints telegram scenario catalog without starting the live lane", async () => { + await runQaTelegramCommand({ + repoRoot: "/tmp/openclaw-repo", + providerMode: "mock-openai", + listScenarios: true, + }); + + expect(listTelegramQaScenarioCatalog).toHaveBeenCalledWith("mock-openai"); + expect(runTelegramQaLive).not.toHaveBeenCalled(); + expect(stdoutWrite).toHaveBeenCalledWith( + expect.stringContaining( + "telegram-status-command\tdefault\tTelegram status command reply\tstatus rationale refs=openclaw/openclaw#74698", + ), + ); + }); + it("sets a failing exit code when telegram scenarios fail", async () => { const priorExitCode = process.exitCode; process.exitCode = undefined; diff --git a/extensions/qa-lab/src/cli.test.ts b/extensions/qa-lab/src/cli.test.ts index 760f803d55b..a28dafee5f0 100644 --- a/extensions/qa-lab/src/cli.test.ts +++ b/extensions/qa-lab/src/cli.test.ts @@ -384,7 +384,7 @@ describe("qa cli registration", () => { const optionNames = telegram?.options.map((option) => option.long) ?? []; expect(optionNames).toEqual( - expect.arrayContaining(["--credential-source", "--credential-role"]), + expect.arrayContaining(["--credential-source", "--credential-role", "--list-scenarios"]), ); }); @@ -434,12 +434,23 @@ describe("qa cli registration", () => { fastMode: false, allowFailures: false, scenarioIds: [], + listScenarios: false, sutAccountId: "sut", credentialSource: undefined, credentialRole: undefined, }); }); + it("forwards --list-scenarios for telegram runs", async () => { + await program.parseAsync(["node", "openclaw", "qa", "telegram", "--list-scenarios"]); + + expect(runQaTelegramCommand).toHaveBeenCalledWith( + expect.objectContaining({ + listScenarios: true, + }), + ); + }); + it("forwards --allow-failures for telegram runs", async () => { await program.parseAsync(["node", "openclaw", "qa", "telegram", "--allow-failures"]); diff --git a/extensions/qa-lab/src/live-transports/shared/live-transport-cli.runtime.test.ts b/extensions/qa-lab/src/live-transports/shared/live-transport-cli.runtime.test.ts index f2bcf0fd072..16a65c2480e 100644 --- a/extensions/qa-lab/src/live-transports/shared/live-transport-cli.runtime.test.ts +++ b/extensions/qa-lab/src/live-transports/shared/live-transport-cli.runtime.test.ts @@ -10,12 +10,14 @@ describe("resolveLiveTransportQaRunOptions", () => { providerMode: "live-frontier", primaryModel: " ", alternateModel: "", + listScenarios: true, }), ).toMatchObject({ repoRoot: path.resolve("/tmp/openclaw-repo"), providerMode: "live-frontier", primaryModel: undefined, alternateModel: undefined, + listScenarios: true, }); }); }); diff --git a/extensions/qa-lab/src/live-transports/shared/live-transport-cli.runtime.ts b/extensions/qa-lab/src/live-transports/shared/live-transport-cli.runtime.ts index ed8eed64e73..348f33119b8 100644 --- a/extensions/qa-lab/src/live-transports/shared/live-transport-cli.runtime.ts +++ b/extensions/qa-lab/src/live-transports/shared/live-transport-cli.runtime.ts @@ -31,6 +31,7 @@ export function resolveLiveTransportQaRunOptions( fastMode: opts.fastMode, allowFailures: opts.allowFailures, scenarioIds: opts.scenarioIds, + listScenarios: opts.listScenarios, sutAccountId: opts.sutAccountId, credentialSource: opts.credentialSource?.trim(), credentialRole: opts.credentialRole?.trim(), diff --git a/extensions/qa-lab/src/live-transports/shared/live-transport-cli.ts b/extensions/qa-lab/src/live-transports/shared/live-transport-cli.ts index a0d9737dcdc..bdde85766f6 100644 --- a/extensions/qa-lab/src/live-transports/shared/live-transport-cli.ts +++ b/extensions/qa-lab/src/live-transports/shared/live-transport-cli.ts @@ -12,6 +12,7 @@ export type LiveTransportQaCommandOptions = { fastMode?: boolean; allowFailures?: boolean; scenarioIds?: string[]; + listScenarios?: boolean; sutAccountId?: string; credentialSource?: string; credentialRole?: string; @@ -24,6 +25,7 @@ type LiveTransportQaCommanderOptions = { model?: string; altModel?: string; scenario?: string[]; + listScenarios?: boolean; fast?: boolean; allowFailures?: boolean; sutAccount?: string; @@ -61,6 +63,7 @@ function mapLiveTransportQaCommanderOptions( fastMode: opts.fast, allowFailures: opts.allowFailures, scenarioIds: opts.scenario, + listScenarios: opts.listScenarios, sutAccountId: opts.sutAccount, credentialSource: opts.credentialSource, credentialRole: opts.credentialRole, @@ -72,6 +75,7 @@ function registerLiveTransportQaCli(params: { commandName: string; credentialOptions?: LiveTransportQaCredentialCliOptions; description: string; + listScenariosHelp?: string; outputDirHelp: string; scenarioHelp: string; sutAccountHelp: string; @@ -94,6 +98,10 @@ function registerLiveTransportQaCli(params: { ) .option("--sut-account ", params.sutAccountHelp, "sut"); + if (params.listScenariosHelp) { + command.option("--list-scenarios", params.listScenariosHelp, false); + } + if (params.credentialOptions) { command.option( "--credential-source ", @@ -114,6 +122,7 @@ export function createLiveTransportQaCliRegistration(params: { commandName: string; credentialOptions?: LiveTransportQaCredentialCliOptions; description: string; + listScenariosHelp?: string; outputDirHelp: string; scenarioHelp: string; sutAccountHelp: string; @@ -127,6 +136,7 @@ export function createLiveTransportQaCliRegistration(params: { commandName: params.commandName, credentialOptions: params.credentialOptions, description: params.description, + listScenariosHelp: params.listScenariosHelp, outputDirHelp: params.outputDirHelp, scenarioHelp: params.scenarioHelp, sutAccountHelp: params.sutAccountHelp, diff --git a/extensions/qa-lab/src/live-transports/telegram/cli.runtime.ts b/extensions/qa-lab/src/live-transports/telegram/cli.runtime.ts index c07e4ea8d04..a18f9d3a18b 100644 --- a/extensions/qa-lab/src/live-transports/telegram/cli.runtime.ts +++ b/extensions/qa-lab/src/live-transports/telegram/cli.runtime.ts @@ -3,10 +3,21 @@ import { printLiveTransportQaArtifacts, resolveLiveTransportQaRunOptions, } from "../shared/live-transport-cli.runtime.js"; -import { runTelegramQaLive } from "./telegram-live.runtime.js"; +import { listTelegramQaScenarioCatalog, runTelegramQaLive } from "./telegram-live.runtime.js"; export async function runQaTelegramCommand(opts: LiveTransportQaCommandOptions) { const runOptions = resolveLiveTransportQaRunOptions(opts); + if (runOptions.listScenarios) { + for (const scenario of listTelegramQaScenarioCatalog(runOptions.providerMode)) { + const defaultLabel = scenario.defaultEnabled ? "default" : "optional"; + const refs = + scenario.regressionRefs.length > 0 ? ` refs=${scenario.regressionRefs.join(",")}` : ""; + process.stdout.write( + `${scenario.id}\t${defaultLabel}\t${scenario.title}\t${scenario.rationale}${refs}\n`, + ); + } + return; + } const result = await runTelegramQaLive(runOptions); printLiveTransportQaArtifacts("Telegram QA", { report: result.reportPath, diff --git a/extensions/qa-lab/src/live-transports/telegram/cli.ts b/extensions/qa-lab/src/live-transports/telegram/cli.ts index b0f2c0de177..d7638a25945 100644 --- a/extensions/qa-lab/src/live-transports/telegram/cli.ts +++ b/extensions/qa-lab/src/live-transports/telegram/cli.ts @@ -25,6 +25,7 @@ export const telegramQaCliRegistration: LiveTransportQaCliRegistration = "Credential role for convex auth: maintainer or ci (default: ci in CI, maintainer otherwise)", }, description: "Run the manual Telegram live QA lane against a private bot-to-bot group harness", + listScenariosHelp: "Print available Telegram scenario ids and exit", outputDirHelp: "Telegram QA artifact directory", scenarioHelp: "Run only the named Telegram QA scenario (repeatable)", sutAccountHelp: "Temporary Telegram account id inside the QA gateway config",