mirror of
https://github.com/openclaw/openclaw.git
synced 2026-05-09 02:40:46 +00:00
feat(qa-lab): list telegram live scenarios
This commit is contained in:
@@ -8,6 +8,7 @@ const {
|
||||
runQaSuiteFromRuntime,
|
||||
runQaCharacterEval,
|
||||
runQaMultipass,
|
||||
listTelegramQaScenarioCatalog,
|
||||
runTelegramQaLive,
|
||||
startQaLabServer,
|
||||
writeQaDockerHarnessFiles,
|
||||
@@ -19,6 +20,7 @@ const {
|
||||
runQaSuiteFromRuntime: vi.fn(),
|
||||
runQaCharacterEval: vi.fn(),
|
||||
runQaMultipass: vi.fn(),
|
||||
listTelegramQaScenarioCatalog: vi.fn(),
|
||||
runTelegramQaLive: vi.fn(),
|
||||
startQaLabServer: vi.fn(),
|
||||
writeQaDockerHarnessFiles: vi.fn(),
|
||||
@@ -45,6 +47,7 @@ vi.mock("./multipass.runtime.js", () => ({
|
||||
}));
|
||||
|
||||
vi.mock("./live-transports/telegram/telegram-live.runtime.js", () => ({
|
||||
listTelegramQaScenarioCatalog,
|
||||
runTelegramQaLive,
|
||||
}));
|
||||
|
||||
@@ -111,6 +114,7 @@ describe("qa cli runtime", () => {
|
||||
runQaCharacterEval.mockReset();
|
||||
runQaManualLane.mockReset();
|
||||
runQaMultipass.mockReset();
|
||||
listTelegramQaScenarioCatalog.mockReset();
|
||||
runTelegramQaLive.mockReset();
|
||||
startQaLabServer.mockReset();
|
||||
writeQaDockerHarnessFiles.mockReset();
|
||||
@@ -153,6 +157,15 @@ describe("qa cli runtime", () => {
|
||||
observedMessagesPath: "/tmp/telegram/observed.json",
|
||||
scenarios: [],
|
||||
});
|
||||
listTelegramQaScenarioCatalog.mockReturnValue([
|
||||
{
|
||||
id: "telegram-status-command",
|
||||
title: "Telegram status command reply",
|
||||
defaultEnabled: true,
|
||||
rationale: "status rationale",
|
||||
regressionRefs: ["openclaw/openclaw#74698"],
|
||||
},
|
||||
]);
|
||||
startQaLabServer.mockResolvedValue({
|
||||
baseUrl: "http://127.0.0.1:58000",
|
||||
runSelfCheck: vi.fn().mockResolvedValue({
|
||||
@@ -297,6 +310,22 @@ describe("qa cli runtime", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("prints telegram scenario catalog without starting the live lane", async () => {
|
||||
await runQaTelegramCommand({
|
||||
repoRoot: "/tmp/openclaw-repo",
|
||||
providerMode: "mock-openai",
|
||||
listScenarios: true,
|
||||
});
|
||||
|
||||
expect(listTelegramQaScenarioCatalog).toHaveBeenCalledWith("mock-openai");
|
||||
expect(runTelegramQaLive).not.toHaveBeenCalled();
|
||||
expect(stdoutWrite).toHaveBeenCalledWith(
|
||||
expect.stringContaining(
|
||||
"telegram-status-command\tdefault\tTelegram status command reply\tstatus rationale refs=openclaw/openclaw#74698",
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
it("sets a failing exit code when telegram scenarios fail", async () => {
|
||||
const priorExitCode = process.exitCode;
|
||||
process.exitCode = undefined;
|
||||
|
||||
@@ -384,7 +384,7 @@ describe("qa cli registration", () => {
|
||||
const optionNames = telegram?.options.map((option) => option.long) ?? [];
|
||||
|
||||
expect(optionNames).toEqual(
|
||||
expect.arrayContaining(["--credential-source", "--credential-role"]),
|
||||
expect.arrayContaining(["--credential-source", "--credential-role", "--list-scenarios"]),
|
||||
);
|
||||
});
|
||||
|
||||
@@ -434,12 +434,23 @@ describe("qa cli registration", () => {
|
||||
fastMode: false,
|
||||
allowFailures: false,
|
||||
scenarioIds: [],
|
||||
listScenarios: false,
|
||||
sutAccountId: "sut",
|
||||
credentialSource: undefined,
|
||||
credentialRole: undefined,
|
||||
});
|
||||
});
|
||||
|
||||
it("forwards --list-scenarios for telegram runs", async () => {
|
||||
await program.parseAsync(["node", "openclaw", "qa", "telegram", "--list-scenarios"]);
|
||||
|
||||
expect(runQaTelegramCommand).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
listScenarios: true,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("forwards --allow-failures for telegram runs", async () => {
|
||||
await program.parseAsync(["node", "openclaw", "qa", "telegram", "--allow-failures"]);
|
||||
|
||||
|
||||
@@ -10,12 +10,14 @@ describe("resolveLiveTransportQaRunOptions", () => {
|
||||
providerMode: "live-frontier",
|
||||
primaryModel: " ",
|
||||
alternateModel: "",
|
||||
listScenarios: true,
|
||||
}),
|
||||
).toMatchObject({
|
||||
repoRoot: path.resolve("/tmp/openclaw-repo"),
|
||||
providerMode: "live-frontier",
|
||||
primaryModel: undefined,
|
||||
alternateModel: undefined,
|
||||
listScenarios: true,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -31,6 +31,7 @@ export function resolveLiveTransportQaRunOptions(
|
||||
fastMode: opts.fastMode,
|
||||
allowFailures: opts.allowFailures,
|
||||
scenarioIds: opts.scenarioIds,
|
||||
listScenarios: opts.listScenarios,
|
||||
sutAccountId: opts.sutAccountId,
|
||||
credentialSource: opts.credentialSource?.trim(),
|
||||
credentialRole: opts.credentialRole?.trim(),
|
||||
|
||||
@@ -12,6 +12,7 @@ export type LiveTransportQaCommandOptions = {
|
||||
fastMode?: boolean;
|
||||
allowFailures?: boolean;
|
||||
scenarioIds?: string[];
|
||||
listScenarios?: boolean;
|
||||
sutAccountId?: string;
|
||||
credentialSource?: string;
|
||||
credentialRole?: string;
|
||||
@@ -24,6 +25,7 @@ type LiveTransportQaCommanderOptions = {
|
||||
model?: string;
|
||||
altModel?: string;
|
||||
scenario?: string[];
|
||||
listScenarios?: boolean;
|
||||
fast?: boolean;
|
||||
allowFailures?: boolean;
|
||||
sutAccount?: string;
|
||||
@@ -61,6 +63,7 @@ function mapLiveTransportQaCommanderOptions(
|
||||
fastMode: opts.fast,
|
||||
allowFailures: opts.allowFailures,
|
||||
scenarioIds: opts.scenario,
|
||||
listScenarios: opts.listScenarios,
|
||||
sutAccountId: opts.sutAccount,
|
||||
credentialSource: opts.credentialSource,
|
||||
credentialRole: opts.credentialRole,
|
||||
@@ -72,6 +75,7 @@ function registerLiveTransportQaCli(params: {
|
||||
commandName: string;
|
||||
credentialOptions?: LiveTransportQaCredentialCliOptions;
|
||||
description: string;
|
||||
listScenariosHelp?: string;
|
||||
outputDirHelp: string;
|
||||
scenarioHelp: string;
|
||||
sutAccountHelp: string;
|
||||
@@ -94,6 +98,10 @@ function registerLiveTransportQaCli(params: {
|
||||
)
|
||||
.option("--sut-account <id>", params.sutAccountHelp, "sut");
|
||||
|
||||
if (params.listScenariosHelp) {
|
||||
command.option("--list-scenarios", params.listScenariosHelp, false);
|
||||
}
|
||||
|
||||
if (params.credentialOptions) {
|
||||
command.option(
|
||||
"--credential-source <source>",
|
||||
@@ -114,6 +122,7 @@ export function createLiveTransportQaCliRegistration(params: {
|
||||
commandName: string;
|
||||
credentialOptions?: LiveTransportQaCredentialCliOptions;
|
||||
description: string;
|
||||
listScenariosHelp?: string;
|
||||
outputDirHelp: string;
|
||||
scenarioHelp: string;
|
||||
sutAccountHelp: string;
|
||||
@@ -127,6 +136,7 @@ export function createLiveTransportQaCliRegistration(params: {
|
||||
commandName: params.commandName,
|
||||
credentialOptions: params.credentialOptions,
|
||||
description: params.description,
|
||||
listScenariosHelp: params.listScenariosHelp,
|
||||
outputDirHelp: params.outputDirHelp,
|
||||
scenarioHelp: params.scenarioHelp,
|
||||
sutAccountHelp: params.sutAccountHelp,
|
||||
|
||||
@@ -3,10 +3,21 @@ import {
|
||||
printLiveTransportQaArtifacts,
|
||||
resolveLiveTransportQaRunOptions,
|
||||
} from "../shared/live-transport-cli.runtime.js";
|
||||
import { runTelegramQaLive } from "./telegram-live.runtime.js";
|
||||
import { listTelegramQaScenarioCatalog, runTelegramQaLive } from "./telegram-live.runtime.js";
|
||||
|
||||
export async function runQaTelegramCommand(opts: LiveTransportQaCommandOptions) {
|
||||
const runOptions = resolveLiveTransportQaRunOptions(opts);
|
||||
if (runOptions.listScenarios) {
|
||||
for (const scenario of listTelegramQaScenarioCatalog(runOptions.providerMode)) {
|
||||
const defaultLabel = scenario.defaultEnabled ? "default" : "optional";
|
||||
const refs =
|
||||
scenario.regressionRefs.length > 0 ? ` refs=${scenario.regressionRefs.join(",")}` : "";
|
||||
process.stdout.write(
|
||||
`${scenario.id}\t${defaultLabel}\t${scenario.title}\t${scenario.rationale}${refs}\n`,
|
||||
);
|
||||
}
|
||||
return;
|
||||
}
|
||||
const result = await runTelegramQaLive(runOptions);
|
||||
printLiveTransportQaArtifacts("Telegram QA", {
|
||||
report: result.reportPath,
|
||||
|
||||
@@ -25,6 +25,7 @@ export const telegramQaCliRegistration: LiveTransportQaCliRegistration =
|
||||
"Credential role for convex auth: maintainer or ci (default: ci in CI, maintainer otherwise)",
|
||||
},
|
||||
description: "Run the manual Telegram live QA lane against a private bot-to-bot group harness",
|
||||
listScenariosHelp: "Print available Telegram scenario ids and exit",
|
||||
outputDirHelp: "Telegram QA artifact directory",
|
||||
scenarioHelp: "Run only the named Telegram QA scenario (repeatable)",
|
||||
sutAccountHelp: "Temporary Telegram account id inside the QA gateway config",
|
||||
|
||||
Reference in New Issue
Block a user