feat(qa-lab): list telegram live scenarios

This commit is contained in:
Ayaan Zaidi
2026-05-08 15:47:29 +05:30
parent ec54642581
commit 5cd4996205
7 changed files with 67 additions and 2 deletions

View File

@@ -8,6 +8,7 @@ const {
runQaSuiteFromRuntime,
runQaCharacterEval,
runQaMultipass,
listTelegramQaScenarioCatalog,
runTelegramQaLive,
startQaLabServer,
writeQaDockerHarnessFiles,
@@ -19,6 +20,7 @@ const {
runQaSuiteFromRuntime: vi.fn(),
runQaCharacterEval: vi.fn(),
runQaMultipass: vi.fn(),
listTelegramQaScenarioCatalog: vi.fn(),
runTelegramQaLive: vi.fn(),
startQaLabServer: vi.fn(),
writeQaDockerHarnessFiles: vi.fn(),
@@ -45,6 +47,7 @@ vi.mock("./multipass.runtime.js", () => ({
}));
vi.mock("./live-transports/telegram/telegram-live.runtime.js", () => ({
listTelegramQaScenarioCatalog,
runTelegramQaLive,
}));
@@ -111,6 +114,7 @@ describe("qa cli runtime", () => {
runQaCharacterEval.mockReset();
runQaManualLane.mockReset();
runQaMultipass.mockReset();
listTelegramQaScenarioCatalog.mockReset();
runTelegramQaLive.mockReset();
startQaLabServer.mockReset();
writeQaDockerHarnessFiles.mockReset();
@@ -153,6 +157,15 @@ describe("qa cli runtime", () => {
observedMessagesPath: "/tmp/telegram/observed.json",
scenarios: [],
});
listTelegramQaScenarioCatalog.mockReturnValue([
{
id: "telegram-status-command",
title: "Telegram status command reply",
defaultEnabled: true,
rationale: "status rationale",
regressionRefs: ["openclaw/openclaw#74698"],
},
]);
startQaLabServer.mockResolvedValue({
baseUrl: "http://127.0.0.1:58000",
runSelfCheck: vi.fn().mockResolvedValue({
@@ -297,6 +310,22 @@ describe("qa cli runtime", () => {
);
});
it("prints telegram scenario catalog without starting the live lane", async () => {
await runQaTelegramCommand({
repoRoot: "/tmp/openclaw-repo",
providerMode: "mock-openai",
listScenarios: true,
});
expect(listTelegramQaScenarioCatalog).toHaveBeenCalledWith("mock-openai");
expect(runTelegramQaLive).not.toHaveBeenCalled();
expect(stdoutWrite).toHaveBeenCalledWith(
expect.stringContaining(
"telegram-status-command\tdefault\tTelegram status command reply\tstatus rationale refs=openclaw/openclaw#74698",
),
);
});
it("sets a failing exit code when telegram scenarios fail", async () => {
const priorExitCode = process.exitCode;
process.exitCode = undefined;

View File

@@ -384,7 +384,7 @@ describe("qa cli registration", () => {
const optionNames = telegram?.options.map((option) => option.long) ?? [];
expect(optionNames).toEqual(
expect.arrayContaining(["--credential-source", "--credential-role"]),
expect.arrayContaining(["--credential-source", "--credential-role", "--list-scenarios"]),
);
});
@@ -434,12 +434,23 @@ describe("qa cli registration", () => {
fastMode: false,
allowFailures: false,
scenarioIds: [],
listScenarios: false,
sutAccountId: "sut",
credentialSource: undefined,
credentialRole: undefined,
});
});
it("forwards --list-scenarios for telegram runs", async () => {
await program.parseAsync(["node", "openclaw", "qa", "telegram", "--list-scenarios"]);
expect(runQaTelegramCommand).toHaveBeenCalledWith(
expect.objectContaining({
listScenarios: true,
}),
);
});
it("forwards --allow-failures for telegram runs", async () => {
await program.parseAsync(["node", "openclaw", "qa", "telegram", "--allow-failures"]);

View File

@@ -10,12 +10,14 @@ describe("resolveLiveTransportQaRunOptions", () => {
providerMode: "live-frontier",
primaryModel: " ",
alternateModel: "",
listScenarios: true,
}),
).toMatchObject({
repoRoot: path.resolve("/tmp/openclaw-repo"),
providerMode: "live-frontier",
primaryModel: undefined,
alternateModel: undefined,
listScenarios: true,
});
});
});

View File

@@ -31,6 +31,7 @@ export function resolveLiveTransportQaRunOptions(
fastMode: opts.fastMode,
allowFailures: opts.allowFailures,
scenarioIds: opts.scenarioIds,
listScenarios: opts.listScenarios,
sutAccountId: opts.sutAccountId,
credentialSource: opts.credentialSource?.trim(),
credentialRole: opts.credentialRole?.trim(),

View File

@@ -12,6 +12,7 @@ export type LiveTransportQaCommandOptions = {
fastMode?: boolean;
allowFailures?: boolean;
scenarioIds?: string[];
listScenarios?: boolean;
sutAccountId?: string;
credentialSource?: string;
credentialRole?: string;
@@ -24,6 +25,7 @@ type LiveTransportQaCommanderOptions = {
model?: string;
altModel?: string;
scenario?: string[];
listScenarios?: boolean;
fast?: boolean;
allowFailures?: boolean;
sutAccount?: string;
@@ -61,6 +63,7 @@ function mapLiveTransportQaCommanderOptions(
fastMode: opts.fast,
allowFailures: opts.allowFailures,
scenarioIds: opts.scenario,
listScenarios: opts.listScenarios,
sutAccountId: opts.sutAccount,
credentialSource: opts.credentialSource,
credentialRole: opts.credentialRole,
@@ -72,6 +75,7 @@ function registerLiveTransportQaCli(params: {
commandName: string;
credentialOptions?: LiveTransportQaCredentialCliOptions;
description: string;
listScenariosHelp?: string;
outputDirHelp: string;
scenarioHelp: string;
sutAccountHelp: string;
@@ -94,6 +98,10 @@ function registerLiveTransportQaCli(params: {
)
.option("--sut-account <id>", params.sutAccountHelp, "sut");
if (params.listScenariosHelp) {
command.option("--list-scenarios", params.listScenariosHelp, false);
}
if (params.credentialOptions) {
command.option(
"--credential-source <source>",
@@ -114,6 +122,7 @@ export function createLiveTransportQaCliRegistration(params: {
commandName: string;
credentialOptions?: LiveTransportQaCredentialCliOptions;
description: string;
listScenariosHelp?: string;
outputDirHelp: string;
scenarioHelp: string;
sutAccountHelp: string;
@@ -127,6 +136,7 @@ export function createLiveTransportQaCliRegistration(params: {
commandName: params.commandName,
credentialOptions: params.credentialOptions,
description: params.description,
listScenariosHelp: params.listScenariosHelp,
outputDirHelp: params.outputDirHelp,
scenarioHelp: params.scenarioHelp,
sutAccountHelp: params.sutAccountHelp,

View File

@@ -3,10 +3,21 @@ import {
printLiveTransportQaArtifacts,
resolveLiveTransportQaRunOptions,
} from "../shared/live-transport-cli.runtime.js";
import { runTelegramQaLive } from "./telegram-live.runtime.js";
import { listTelegramQaScenarioCatalog, runTelegramQaLive } from "./telegram-live.runtime.js";
export async function runQaTelegramCommand(opts: LiveTransportQaCommandOptions) {
const runOptions = resolveLiveTransportQaRunOptions(opts);
if (runOptions.listScenarios) {
for (const scenario of listTelegramQaScenarioCatalog(runOptions.providerMode)) {
const defaultLabel = scenario.defaultEnabled ? "default" : "optional";
const refs =
scenario.regressionRefs.length > 0 ? ` refs=${scenario.regressionRefs.join(",")}` : "";
process.stdout.write(
`${scenario.id}\t${defaultLabel}\t${scenario.title}\t${scenario.rationale}${refs}\n`,
);
}
return;
}
const result = await runTelegramQaLive(runOptions);
printLiveTransportQaArtifacts("Telegram QA", {
report: result.reportPath,

View File

@@ -25,6 +25,7 @@ export const telegramQaCliRegistration: LiveTransportQaCliRegistration =
"Credential role for convex auth: maintainer or ci (default: ci in CI, maintainer otherwise)",
},
description: "Run the manual Telegram live QA lane against a private bot-to-bot group harness",
listScenariosHelp: "Print available Telegram scenario ids and exit",
outputDirHelp: "Telegram QA artifact directory",
scenarioHelp: "Run only the named Telegram QA scenario (repeatable)",
sutAccountHelp: "Temporary Telegram account id inside the QA gateway config",