diff --git a/extensions/qa-lab/src/live-transports/shared/live-transport-cli.ts b/extensions/qa-lab/src/live-transports/shared/live-transport-cli.ts index bdde85766f6..2c7ea30a07b 100644 --- a/extensions/qa-lab/src/live-transports/shared/live-transport-cli.ts +++ b/extensions/qa-lab/src/live-transports/shared/live-transport-cli.ts @@ -1,147 +1,27 @@ -import type { Command } from "commander"; -import { collectString } from "../../cli-options.js"; +import { + createLiveTransportQaCliRegistration as createSharedLiveTransportQaCliRegistration, + type LiveTransportQaCliRegistrationOptions, +} from "openclaw/plugin-sdk/qa-runtime"; import { DEFAULT_QA_LIVE_PROVIDER_MODE, formatQaProviderModeHelp } from "../../providers/index.js"; -import type { QaProviderModeInput } from "../../run-config.js"; -export type LiveTransportQaCommandOptions = { - repoRoot?: string; - outputDir?: string; - providerMode?: QaProviderModeInput; - primaryModel?: string; - alternateModel?: string; - fastMode?: boolean; - allowFailures?: boolean; - scenarioIds?: string[]; - listScenarios?: boolean; - sutAccountId?: string; - credentialSource?: string; - credentialRole?: string; -}; +export { + createLazyCliRuntimeLoader, + type LiveTransportQaCliRegistration, + type LiveTransportQaCommandOptions, +} from "openclaw/plugin-sdk/qa-runtime"; -type LiveTransportQaCommanderOptions = { - repoRoot?: string; - outputDir?: string; - providerMode?: QaProviderModeInput; - model?: string; - altModel?: string; - scenario?: string[]; - listScenarios?: boolean; - fast?: boolean; - allowFailures?: boolean; - sutAccount?: string; - credentialSource?: string; - credentialRole?: string; -}; +type QaLabLiveTransportQaCliRegistrationOptions = Omit< + LiveTransportQaCliRegistrationOptions, + "allowFailuresHelp" | "defaultProviderMode" | "providerModeHelp" +>; -export type LiveTransportQaCliRegistration = { - commandName: string; - register(qa: Command): void; -}; - -type LiveTransportQaCredentialCliOptions = { - sourceDescription?: string; - roleDescription?: string; -}; - -export function createLazyCliRuntimeLoader(load: () => Promise) { - let promise: Promise | null = null; - return async () => { - promise ??= load(); - return await promise; - }; -} - -function mapLiveTransportQaCommanderOptions( - opts: LiveTransportQaCommanderOptions, -): LiveTransportQaCommandOptions { - return { - repoRoot: opts.repoRoot, - outputDir: opts.outputDir, - providerMode: opts.providerMode, - primaryModel: opts.model, - alternateModel: opts.altModel, - fastMode: opts.fast, - allowFailures: opts.allowFailures, - scenarioIds: opts.scenario, - listScenarios: opts.listScenarios, - sutAccountId: opts.sutAccount, - credentialSource: opts.credentialSource, - credentialRole: opts.credentialRole, - }; -} - -function registerLiveTransportQaCli(params: { - qa: Command; - commandName: string; - credentialOptions?: LiveTransportQaCredentialCliOptions; - description: string; - listScenariosHelp?: string; - outputDirHelp: string; - scenarioHelp: string; - sutAccountHelp: string; - run: (opts: LiveTransportQaCommandOptions) => Promise; -}) { - const command = params.qa - .command(params.commandName) - .description(params.description) - .option("--repo-root ", "Repository root to target when running from a neutral cwd") - .option("--output-dir ", params.outputDirHelp) - .option("--provider-mode ", formatQaProviderModeHelp(), DEFAULT_QA_LIVE_PROVIDER_MODE) - .option("--model ", "Primary provider/model ref") - .option("--alt-model ", "Alternate provider/model ref") - .option("--scenario ", params.scenarioHelp, collectString, []) - .option("--fast", "Enable provider fast mode where supported", false) - .option( - "--allow-failures", - "Write artifacts without setting a failing exit code when scenarios fail", - false, - ) - .option("--sut-account ", params.sutAccountHelp, "sut"); - - if (params.listScenariosHelp) { - command.option("--list-scenarios", params.listScenariosHelp, false); - } - - if (params.credentialOptions) { - command.option( - "--credential-source ", - params.credentialOptions.sourceDescription ?? - "Credential source for live lanes: env or convex (default: env)", - ); - if (params.credentialOptions.roleDescription) { - command.option("--credential-role ", params.credentialOptions.roleDescription); - } - } - - command.action(async (opts: LiveTransportQaCommanderOptions) => { - await params.run(mapLiveTransportQaCommanderOptions(opts)); +export function createLiveTransportQaCliRegistration( + params: QaLabLiveTransportQaCliRegistrationOptions, +) { + return createSharedLiveTransportQaCliRegistration({ + ...params, + allowFailuresHelp: "Write artifacts without setting a failing exit code when scenarios fail", + defaultProviderMode: DEFAULT_QA_LIVE_PROVIDER_MODE, + providerModeHelp: formatQaProviderModeHelp(), }); } - -export function createLiveTransportQaCliRegistration(params: { - commandName: string; - credentialOptions?: LiveTransportQaCredentialCliOptions; - description: string; - listScenariosHelp?: string; - outputDirHelp: string; - scenarioHelp: string; - sutAccountHelp: string; - run: (opts: LiveTransportQaCommandOptions) => Promise; -}): LiveTransportQaCliRegistration { - return { - commandName: params.commandName, - register(qa: Command) { - registerLiveTransportQaCli({ - qa, - commandName: params.commandName, - credentialOptions: params.credentialOptions, - description: params.description, - listScenariosHelp: params.listScenariosHelp, - outputDirHelp: params.outputDirHelp, - scenarioHelp: params.scenarioHelp, - sutAccountHelp: params.sutAccountHelp, - run: params.run, - }); - }, - }; -} diff --git a/extensions/qa-matrix/src/cli-options.ts b/extensions/qa-matrix/src/cli-options.ts deleted file mode 100644 index 1576d43291f..00000000000 --- a/extensions/qa-matrix/src/cli-options.ts +++ /dev/null @@ -1,4 +0,0 @@ -export function collectString(value: string, previous: string[]) { - const trimmed = value.trim(); - return trimmed ? [...previous, trimmed] : previous; -} diff --git a/extensions/qa-matrix/src/shared/live-transport-cli.ts b/extensions/qa-matrix/src/shared/live-transport-cli.ts index 59ce1f4d3e0..62b0abcb771 100644 --- a/extensions/qa-matrix/src/shared/live-transport-cli.ts +++ b/extensions/qa-matrix/src/shared/live-transport-cli.ts @@ -1,152 +1,26 @@ -import type { Command } from "commander"; -import { collectString } from "../cli-options.js"; -import type { QaProviderModeInput } from "../run-config.js"; +import { + createLiveTransportQaCliRegistration as createSharedLiveTransportQaCliRegistration, + type LiveTransportQaCliRegistrationOptions, +} from "openclaw/plugin-sdk/qa-runtime"; -export type LiveTransportQaCommandOptions = { - repoRoot?: string; - outputDir?: string; - providerMode?: QaProviderModeInput; - primaryModel?: string; - alternateModel?: string; - fastMode?: boolean; - failFast?: boolean; - profile?: string; - scenarioIds?: string[]; - sutAccountId?: string; - credentialSource?: string; - credentialRole?: string; -}; +export { + createLazyCliRuntimeLoader, + type LiveTransportQaCliRegistration, + type LiveTransportQaCommandOptions, +} from "openclaw/plugin-sdk/qa-runtime"; -type LiveTransportQaCommanderOptions = { - repoRoot?: string; - outputDir?: string; - providerMode?: QaProviderModeInput; - model?: string; - altModel?: string; - scenario?: string[]; - fast?: boolean; - failFast?: boolean; - profile?: string; - sutAccount?: string; - credentialSource?: string; - credentialRole?: string; -}; +type MatrixLiveTransportQaCliRegistrationOptions = Omit< + LiveTransportQaCliRegistrationOptions, + "defaultProviderMode" | "providerModeHelp" +>; -export type LiveTransportQaCliRegistration = { - commandName: string; - register(qa: Command): void; -}; - -type LiveTransportQaCredentialCliOptions = { - sourceDescription?: string; - roleDescription?: string; -}; - -export function createLazyCliRuntimeLoader(load: () => Promise) { - let promise: Promise | null = null; - return async () => { - promise ??= load(); - return await promise; - }; -} - -function mapLiveTransportQaCommanderOptions( - opts: LiveTransportQaCommanderOptions, -): LiveTransportQaCommandOptions { - return { - repoRoot: opts.repoRoot, - outputDir: opts.outputDir, - providerMode: opts.providerMode, - primaryModel: opts.model, - alternateModel: opts.altModel, - fastMode: opts.fast, - failFast: opts.failFast, - profile: opts.profile, - scenarioIds: opts.scenario, - sutAccountId: opts.sutAccount, - credentialSource: opts.credentialSource, - credentialRole: opts.credentialRole, - }; -} - -function registerLiveTransportQaCli(params: { - qa: Command; - commandName: string; - credentialOptions?: LiveTransportQaCredentialCliOptions; - description: string; - outputDirHelp: string; - profileHelp?: string; - failFastHelp?: string; - scenarioHelp: string; - sutAccountHelp: string; - run: (opts: LiveTransportQaCommandOptions) => Promise; -}) { - const command = params.qa - .command(params.commandName) - .description(params.description) - .option("--repo-root ", "Repository root to target when running from a neutral cwd") - .option("--output-dir ", params.outputDirHelp) - .option( - "--provider-mode ", +export function createLiveTransportQaCliRegistration( + params: MatrixLiveTransportQaCliRegistrationOptions, +) { + return createSharedLiveTransportQaCliRegistration({ + ...params, + defaultProviderMode: "live-frontier", + providerModeHelp: "Provider mode: mock-openai or live-frontier (legacy live-openai still works)", - "live-frontier", - ) - .option("--model ", "Primary provider/model ref") - .option("--alt-model ", "Alternate provider/model ref") - .option("--scenario ", params.scenarioHelp, collectString, []) - .option("--fast", "Enable provider fast mode where supported", false) - .option("--sut-account ", params.sutAccountHelp, "sut"); - - if (params.profileHelp) { - command.option("--profile ", params.profileHelp); - } - - if (params.failFastHelp) { - command.option("--fail-fast", params.failFastHelp, false); - } - - if (params.credentialOptions) { - command.option( - "--credential-source ", - params.credentialOptions.sourceDescription ?? - "Credential source for live lanes: env or convex (default: env)", - ); - if (params.credentialOptions.roleDescription) { - command.option("--credential-role ", params.credentialOptions.roleDescription); - } - } - - command.action(async (opts: LiveTransportQaCommanderOptions) => { - await params.run(mapLiveTransportQaCommanderOptions(opts)); }); } - -export function createLiveTransportQaCliRegistration(params: { - commandName: string; - credentialOptions?: LiveTransportQaCredentialCliOptions; - description: string; - outputDirHelp: string; - profileHelp?: string; - failFastHelp?: string; - scenarioHelp: string; - sutAccountHelp: string; - run: (opts: LiveTransportQaCommandOptions) => Promise; -}): LiveTransportQaCliRegistration { - return { - commandName: params.commandName, - register(qa: Command) { - registerLiveTransportQaCli({ - qa, - commandName: params.commandName, - credentialOptions: params.credentialOptions, - description: params.description, - outputDirHelp: params.outputDirHelp, - profileHelp: params.profileHelp, - failFastHelp: params.failFastHelp, - scenarioHelp: params.scenarioHelp, - sutAccountHelp: params.sutAccountHelp, - run: params.run, - }); - }, - }; -} diff --git a/src/plugin-sdk/qa-runtime.test.ts b/src/plugin-sdk/qa-runtime.test.ts index 40c4decf2a1..5db2d9970ba 100644 --- a/src/plugin-sdk/qa-runtime.test.ts +++ b/src/plugin-sdk/qa-runtime.test.ts @@ -1,3 +1,4 @@ +import { Command } from "commander"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { cleanupTempDirs, @@ -147,6 +148,84 @@ describe("plugin-sdk qa-runtime", () => { ).toEqual(["allowlist-block", "top-level-reply-shape"]); }); + it("registers shared live transport QA CLI options", async () => { + const module = await import("./qa-runtime.js"); + const run = vi.fn(async () => {}); + const qa = new Command(); + + module + .createLiveTransportQaCliRegistration({ + commandName: "telegram", + credentialOptions: { + sourceDescription: "Credential source for Telegram QA", + roleDescription: "Credential role for Telegram QA", + }, + defaultProviderMode: "live-frontier", + description: "Run Telegram QA", + providerModeHelp: "Provider mode", + listScenariosHelp: "List Telegram scenarios", + outputDirHelp: "Telegram output directory", + profileHelp: "QA profile", + failFastHelp: "Stop after first failure", + allowFailuresHelp: "Allow failures", + scenarioHelp: "Run only the named scenario", + sutAccountHelp: "Temporary SUT account", + run, + }) + .register(qa); + + await qa.parseAsync([ + "node", + "openclaw", + "telegram", + "--repo-root", + "/tmp/repo", + "--output-dir", + ".artifacts/qa", + "--provider-mode", + "mock-openai", + "--model", + "primary", + "--alt-model", + "alternate", + "--scenario", + "alpha", + "--scenario", + " ", + "--scenario", + "beta", + "--fast", + "--allow-failures", + "--list-scenarios", + "--profile", + "fast", + "--fail-fast", + "--sut-account", + "sut-2", + "--credential-source", + "convex", + "--credential-role", + "maintainer", + ]); + + expect(run).toHaveBeenCalledWith({ + repoRoot: "/tmp/repo", + outputDir: ".artifacts/qa", + providerMode: "mock-openai", + primaryModel: "primary", + alternateModel: "alternate", + fastMode: true, + allowFailures: true, + failFast: true, + profile: "fast", + scenarioIds: ["alpha", "beta"], + listScenarios: true, + sutAccountId: "sut-2", + credentialSource: "convex", + credentialRole: "maintainer", + }); + }); + it("builds shared live-lane artifact errors", async () => { const module = await import("./qa-runtime.js"); diff --git a/src/plugin-sdk/qa-runtime.ts b/src/plugin-sdk/qa-runtime.ts index 915536d8a55..d63ef053e3a 100644 --- a/src/plugin-sdk/qa-runtime.ts +++ b/src/plugin-sdk/qa-runtime.ts @@ -2,6 +2,7 @@ import fs from "node:fs"; import fsp from "node:fs/promises"; import { createServer } from "node:net"; import path from "node:path"; +import type { Command } from "commander"; import { formatErrorMessage } from "./error-runtime.js"; import { loadBundledPluginPublicSurfaceModuleSync } from "./facade-runtime.js"; import { resolvePrivateQaBundledPluginsEnv } from "./private-qa-bundled-env.js"; @@ -49,6 +50,164 @@ export function isQaRuntimeAvailable(): boolean { } } +export type LiveTransportQaCommandOptions = { + repoRoot?: string; + outputDir?: string; + providerMode?: string; + primaryModel?: string; + alternateModel?: string; + fastMode?: boolean; + allowFailures?: boolean; + failFast?: boolean; + profile?: string; + scenarioIds?: string[]; + listScenarios?: boolean; + sutAccountId?: string; + credentialSource?: string; + credentialRole?: string; +}; + +type LiveTransportQaCommanderOptions = { + repoRoot?: string; + outputDir?: string; + providerMode?: string; + model?: string; + altModel?: string; + scenario?: string[]; + listScenarios?: boolean; + fast?: boolean; + allowFailures?: boolean; + failFast?: boolean; + profile?: string; + sutAccount?: string; + credentialSource?: string; + credentialRole?: string; +}; + +export type LiveTransportQaCliRegistration = { + commandName: string; + register(qa: Command): void; +}; + +export type LiveTransportQaCredentialCliOptions = { + sourceDescription?: string; + roleDescription?: string; +}; + +export type LiveTransportQaCliRegistrationOptions = { + commandName: string; + credentialOptions?: LiveTransportQaCredentialCliOptions; + defaultProviderMode: string; + description: string; + providerModeHelp: string; + listScenariosHelp?: string; + outputDirHelp: string; + profileHelp?: string; + failFastHelp?: string; + allowFailuresHelp?: string; + scenarioHelp: string; + sutAccountHelp: string; + run: (opts: LiveTransportQaCommandOptions) => Promise; +}; + +export function createLazyCliRuntimeLoader(load: () => Promise) { + let promise: Promise | null = null; + return async () => { + promise ??= load(); + return await promise; + }; +} + +function collectLiveTransportQaStringOption(value: string, previous: string[]) { + const trimmed = value.trim(); + return trimmed ? [...previous, trimmed] : previous; +} + +function mapLiveTransportQaCommanderOptions( + opts: LiveTransportQaCommanderOptions, +): LiveTransportQaCommandOptions { + return { + repoRoot: opts.repoRoot, + outputDir: opts.outputDir, + providerMode: opts.providerMode, + primaryModel: opts.model, + alternateModel: opts.altModel, + fastMode: opts.fast, + allowFailures: opts.allowFailures, + failFast: opts.failFast, + profile: opts.profile, + scenarioIds: opts.scenario, + listScenarios: opts.listScenarios, + sutAccountId: opts.sutAccount, + credentialSource: opts.credentialSource, + credentialRole: opts.credentialRole, + }; +} + +function registerLiveTransportQaCli( + params: LiveTransportQaCliRegistrationOptions & { + qa: Command; + }, +) { + const command = params.qa + .command(params.commandName) + .description(params.description) + .option("--repo-root ", "Repository root to target when running from a neutral cwd") + .option("--output-dir ", params.outputDirHelp) + .option("--provider-mode ", params.providerModeHelp, params.defaultProviderMode) + .option("--model ", "Primary provider/model ref") + .option("--alt-model ", "Alternate provider/model ref") + .option("--scenario ", params.scenarioHelp, collectLiveTransportQaStringOption, []) + .option("--fast", "Enable provider fast mode where supported", false); + + if (params.allowFailuresHelp) { + command.option("--allow-failures", params.allowFailuresHelp, false); + } + + command.option("--sut-account ", params.sutAccountHelp, "sut"); + + if (params.listScenariosHelp) { + command.option("--list-scenarios", params.listScenariosHelp, false); + } + + if (params.profileHelp) { + command.option("--profile ", params.profileHelp); + } + + if (params.failFastHelp) { + command.option("--fail-fast", params.failFastHelp, false); + } + + if (params.credentialOptions) { + command.option( + "--credential-source ", + params.credentialOptions.sourceDescription ?? + "Credential source for live lanes: env or convex (default: env)", + ); + if (params.credentialOptions.roleDescription) { + command.option("--credential-role ", params.credentialOptions.roleDescription); + } + } + + command.action(async (opts: LiveTransportQaCommanderOptions) => { + await params.run(mapLiveTransportQaCommanderOptions(opts)); + }); +} + +export function createLiveTransportQaCliRegistration( + params: LiveTransportQaCliRegistrationOptions, +): LiveTransportQaCliRegistration { + return { + commandName: params.commandName, + register(qa: Command) { + registerLiveTransportQaCli({ + ...params, + qa, + }); + }, + }; +} + export type QaReportCheck = { name: string; status: "pass" | "fail" | "skip";