From bb7e9823a8869a6d01d071d939822489a32e2479 Mon Sep 17 00:00:00 2001 From: Gustavo Madeira Santana Date: Thu, 16 Apr 2026 21:23:23 -0400 Subject: [PATCH] QA Lab: add provider registry Move mock and live provider behavior behind provider-owned definitions so suite, manual, Matrix, and transport lanes share defaults, auth staging, model config, and standalone server startup. Add AIMock as a first-class local provider mode while keeping mock-openai as the scenario-aware deterministic lane. --- docs/concepts/qa-e2e-automation.md | 16 + docs/help/testing.md | 7 + extensions/qa-lab/api.ts | 1 - extensions/qa-lab/src/character-eval.ts | 30 +- extensions/qa-lab/src/cli.runtime.test.ts | 75 ++-- extensions/qa-lab/src/cli.runtime.ts | 32 +- extensions/qa-lab/src/cli.test.ts | 18 + extensions/qa-lab/src/cli.ts | 54 +-- extensions/qa-lab/src/gateway-child.ts | 345 ++---------------- .../qa-lab/src/gateway-log-redaction.ts | 21 +- extensions/qa-lab/src/live-timeout.ts | 40 +- .../shared/live-gateway.runtime.test.ts | 29 +- .../shared/live-gateway.runtime.ts | 15 +- .../shared/live-transport-cli.runtime.ts | 3 +- .../shared/live-transport-cli.ts | 7 +- .../telegram/telegram-live.runtime.ts | 5 +- .../qa-lab/src/manual-lane.runtime.test.ts | 29 +- extensions/qa-lab/src/manual-lane.runtime.ts | 15 +- .../qa-lab/src/model-catalog.runtime.ts | 46 +-- .../src/model-selection.runtime.test.ts | 4 +- .../qa-lab/src/model-selection.runtime.ts | 38 +- extensions/qa-lab/src/model-selection.ts | 21 +- extensions/qa-lab/src/multipass.runtime.ts | 121 +----- extensions/qa-lab/src/providers/README.md | 21 ++ .../qa-lab/src/providers/aimock/index.ts | 9 + .../src/providers/aimock/server.test.ts | 82 +++++ .../qa-lab/src/providers/aimock/server.ts | 203 +++++++++++ extensions/qa-lab/src/providers/env.ts | 229 ++++++++++++ .../src/providers/image-generation.test.ts | 46 +++ .../qa-lab/src/providers/image-generation.ts | 73 ++++ extensions/qa-lab/src/providers/index.ts | 62 ++++ .../src/providers/live-frontier/auth.ts | 60 +++ .../src/providers/live-frontier/catalog.ts | 7 + .../providers/live-frontier/character-eval.ts | 36 ++ .../src/providers/live-frontier/index.ts | 61 ++++ .../live-frontier/model-selection.runtime.ts | 27 ++ .../src/providers/live-frontier/parity.ts | 2 + .../qa-lab/src/providers/mock-openai/index.ts | 9 + .../mock-openai/server.test.ts} | 144 +++++++- .../mock-openai/server.ts} | 104 +++++- .../qa-lab/src/providers/server-runtime.ts | 36 ++ .../qa-lab/src/providers/shared/mock-auth.ts | 70 ++++ .../src/providers/shared/mock-model-config.ts | 104 ++++++ .../shared/mock-provider-definition.ts | 45 +++ .../qa-lab/src/providers/shared/types.ts | 56 +++ extensions/qa-lab/src/qa-channel-transport.ts | 6 +- .../qa-lab/src/qa-gateway-config.test.ts | 23 ++ extensions/qa-lab/src/qa-gateway-config.ts | 254 +++---------- extensions/qa-lab/src/qa-thinking.ts | 28 ++ extensions/qa-lab/src/qa-transport.ts | 3 +- extensions/qa-lab/src/run-config.test.ts | 55 ++- extensions/qa-lab/src/run-config.ts | 26 +- extensions/qa-lab/src/suite-planning.ts | 8 +- .../qa-lab/src/suite-runtime-agent-common.ts | 3 +- .../src/suite-runtime-agent-media.test.ts | 4 +- .../qa-lab/src/suite-runtime-agent-media.ts | 61 +--- extensions/qa-lab/src/suite-runtime-types.ts | 3 +- extensions/qa-lab/src/suite.ts | 19 +- package.json | 1 + pnpm-lock.yaml | 10 + 60 files changed, 1964 insertions(+), 998 deletions(-) create mode 100644 extensions/qa-lab/src/providers/README.md create mode 100644 extensions/qa-lab/src/providers/aimock/index.ts create mode 100644 extensions/qa-lab/src/providers/aimock/server.test.ts create mode 100644 extensions/qa-lab/src/providers/aimock/server.ts create mode 100644 extensions/qa-lab/src/providers/env.ts create mode 100644 extensions/qa-lab/src/providers/image-generation.test.ts create mode 100644 extensions/qa-lab/src/providers/image-generation.ts create mode 100644 extensions/qa-lab/src/providers/index.ts create mode 100644 extensions/qa-lab/src/providers/live-frontier/auth.ts create mode 100644 extensions/qa-lab/src/providers/live-frontier/catalog.ts create mode 100644 extensions/qa-lab/src/providers/live-frontier/character-eval.ts create mode 100644 extensions/qa-lab/src/providers/live-frontier/index.ts create mode 100644 extensions/qa-lab/src/providers/live-frontier/model-selection.runtime.ts create mode 100644 extensions/qa-lab/src/providers/live-frontier/parity.ts create mode 100644 extensions/qa-lab/src/providers/mock-openai/index.ts rename extensions/qa-lab/src/{mock-openai-server.test.ts => providers/mock-openai/server.test.ts} (93%) rename extensions/qa-lab/src/{mock-openai-server.ts => providers/mock-openai/server.ts} (94%) create mode 100644 extensions/qa-lab/src/providers/server-runtime.ts create mode 100644 extensions/qa-lab/src/providers/shared/mock-auth.ts create mode 100644 extensions/qa-lab/src/providers/shared/mock-model-config.ts create mode 100644 extensions/qa-lab/src/providers/shared/mock-provider-definition.ts create mode 100644 extensions/qa-lab/src/providers/shared/types.ts create mode 100644 extensions/qa-lab/src/qa-thinking.ts diff --git a/docs/concepts/qa-e2e-automation.md b/docs/concepts/qa-e2e-automation.md index 8943e47653d..94ad02848af 100644 --- a/docs/concepts/qa-e2e-automation.md +++ b/docs/concepts/qa-e2e-automation.md @@ -151,6 +151,22 @@ The baseline list should stay broad enough to cover: - repo-reading and docs-reading - one small build task such as Lobster Invaders +## Provider mock lanes + +`qa suite` has two local provider mock lanes: + +- `mock-openai` is the scenario-aware OpenClaw mock. It remains the default + deterministic mock lane for repo-backed QA and parity gates. +- `aimock` starts an AIMock-backed provider server for experimental protocol, + fixture, record/replay, and chaos coverage. It is additive and does not + replace the `mock-openai` scenario dispatcher. + +Provider-lane implementation lives under `extensions/qa-lab/src/providers/`. +Each provider owns its defaults, local server startup, gateway model config, +auth-profile staging needs, and live/mock capability flags. Shared suite and +gateway code should route through the provider registry instead of branching on +provider names. + ## Transport adapters `qa-lab` owns a generic transport seam for markdown QA scenarios. diff --git a/docs/help/testing.md b/docs/help/testing.md index 2242c30c79f..09b31f876b4 100644 --- a/docs/help/testing.md +++ b/docs/help/testing.md @@ -52,6 +52,10 @@ These commands sit beside the main test suites when you need QA-lab realism: gateway workers, up to 64 workers or the selected scenario count. Use `--concurrency ` to tune the worker count, or `--concurrency 1` for the older serial lane. + - Supports provider modes `live-frontier`, `mock-openai`, and `aimock`. + `aimock` starts a local AIMock-backed provider server for experimental + fixture and protocol-mock coverage without replacing the scenario-aware + `mock-openai` lane. - `pnpm openclaw qa suite --runner multipass` - Runs the same QA suite inside a disposable Multipass Linux VM. - Keeps the same scenario-selection behavior as `qa suite` on the host. @@ -65,6 +69,9 @@ These commands sit beside the main test suites when you need QA-lab realism: `.artifacts/qa-e2e/...`. - `pnpm qa:lab:up` - Starts the Docker-backed QA site for operator-style QA work. +- `pnpm openclaw qa aimock` + - Starts only the local AIMock provider server for direct protocol smoke + testing. - `pnpm openclaw qa matrix` - Runs the Matrix live QA lane against a disposable Docker-backed Tuwunel homeserver. - This QA host is repo/dev-only today. Packaged OpenClaw installs do not ship diff --git a/extensions/qa-lab/api.ts b/extensions/qa-lab/api.ts index 140ea5d1190..47f510f55b7 100644 --- a/extensions/qa-lab/api.ts +++ b/extensions/qa-lab/api.ts @@ -6,7 +6,6 @@ export * from "./src/cli.js"; export * from "./src/harness-runtime.js"; export * from "./src/lab-server.js"; export * from "./src/docker-harness.js"; -export * from "./src/mock-openai-server.js"; export * from "./src/qa-agent-bootstrap.js"; export * from "./src/qa-agent-workspace.js"; export * from "./src/qa-gateway-config.js"; diff --git a/extensions/qa-lab/src/character-eval.ts b/extensions/qa-lab/src/character-eval.ts index 0d53f33448f..6309d874d29 100644 --- a/extensions/qa-lab/src/character-eval.ts +++ b/extensions/qa-lab/src/character-eval.ts @@ -3,38 +3,28 @@ import path from "node:path"; import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime"; import { runQaManualLane } from "./manual-lane.runtime.js"; import { isQaFastModeModelRef, type QaProviderMode } from "./model-selection.js"; +import { + QA_FRONTIER_CHARACTER_EVAL_MODELS, + QA_FRONTIER_CHARACTER_JUDGE_MODEL_OPTIONS, + QA_FRONTIER_CHARACTER_JUDGE_MODELS, + QA_FRONTIER_CHARACTER_THINKING_BY_MODEL, +} from "./providers/live-frontier/character-eval.js"; import { type QaThinkingLevel } from "./qa-gateway-config.js"; import { extractQaVisibleReplyLeakText } from "./reply-failure.js"; import { runQaSuiteFromRuntime } from "./suite-launch.runtime.js"; import type { QaSuiteResult } from "./suite.js"; const DEFAULT_CHARACTER_SCENARIO_ID = "character-vibes-gollum"; -const DEFAULT_CHARACTER_EVAL_MODELS = Object.freeze([ - "openai/gpt-5.4", - "openai/gpt-5.2", - "openai/gpt-5", - "anthropic/claude-opus-4-6", - "anthropic/claude-sonnet-4-6", - "zai/glm-5.1", - "moonshot/kimi-k2.5", - "google/gemini-3.1-pro-preview", -]); +const DEFAULT_CHARACTER_EVAL_MODELS = QA_FRONTIER_CHARACTER_EVAL_MODELS; const DEFAULT_CHARACTER_THINKING: QaThinkingLevel = "high"; const DEFAULT_CHARACTER_EVAL_CONCURRENCY = 16; const DEFAULT_CHARACTER_THINKING_BY_MODEL: Readonly> = - Object.freeze({ - "openai/gpt-5.4": "xhigh", - "openai/gpt-5.2": "xhigh", - "openai/gpt-5": "xhigh", - }); -const DEFAULT_JUDGE_MODELS = Object.freeze(["openai/gpt-5.4", "anthropic/claude-opus-4-6"]); + QA_FRONTIER_CHARACTER_THINKING_BY_MODEL; +const DEFAULT_JUDGE_MODELS = QA_FRONTIER_CHARACTER_JUDGE_MODELS; const DEFAULT_JUDGE_THINKING: QaThinkingLevel = "xhigh"; const DEFAULT_JUDGE_TIMEOUT_MS = 300_000; const DEFAULT_JUDGE_MODEL_OPTIONS: Readonly> = - Object.freeze({ - "openai/gpt-5.4": { thinkingDefault: "xhigh" }, - "anthropic/claude-opus-4-6": { thinkingDefault: "high" }, - }); + QA_FRONTIER_CHARACTER_JUDGE_MODEL_OPTIONS; type QaCharacterRunStatus = "pass" | "fail"; diff --git a/extensions/qa-lab/src/cli.runtime.test.ts b/extensions/qa-lab/src/cli.runtime.test.ts index 984d6b0d401..a87d055b068 100644 --- a/extensions/qa-lab/src/cli.runtime.test.ts +++ b/extensions/qa-lab/src/cli.runtime.test.ts @@ -24,14 +24,8 @@ const { writeQaDockerHarnessFiles: vi.fn(), buildQaDockerHarnessImage: vi.fn(), runQaDockerUp: vi.fn(), - defaultQaRuntimeModelForMode: vi.fn<(mode: string, options?: { alternate?: boolean }) => string>( - (mode, options) => - mode === "live-frontier" - ? "openai/gpt-5.4" - : options?.alternate - ? "mock-openai/gpt-5.4-alt" - : "mock-openai/gpt-5.4", - ), + defaultQaRuntimeModelForMode: + vi.fn<(mode: string, options?: { alternate?: boolean }) => string>(), })); vi.mock("./manual-lane.runtime.js", () => ({ @@ -83,6 +77,8 @@ import { runQaSuiteCommand, } from "./cli.runtime.js"; import { runQaTelegramCommand } from "./live-transports/telegram/cli.runtime.js"; +import { defaultQaModelForMode as defaultQaProviderModelForMode } from "./model-selection.js"; +import type { QaProviderModeInput } from "./run-config.js"; describe("qa cli runtime", () => { let stdoutWrite: ReturnType; @@ -100,11 +96,7 @@ describe("qa cli runtime", () => { runQaDockerUp.mockReset(); defaultQaRuntimeModelForMode.mockImplementation( (mode: string, options?: { alternate?: boolean }) => - mode === "live-frontier" - ? "openai/gpt-5.4" - : options?.alternate - ? "mock-openai/gpt-5.4-alt" - : "mock-openai/gpt-5.4", + defaultQaProviderModelForMode(mode as QaProviderModeInput, options), ); runQaSuiteFromRuntime.mockResolvedValue({ watchUrl: "http://127.0.0.1:43124", @@ -234,22 +226,6 @@ describe("qa cli runtime", () => { ); }); - it("normalizes legacy live-openai suite runs onto the frontier provider mode", async () => { - await runQaSuiteCommand({ - repoRoot: "/tmp/openclaw-repo", - providerMode: "live-openai", - scenarioIds: ["approval-turn-tool-followthrough"], - }); - - expect(runQaSuiteFromRuntime).toHaveBeenCalledWith( - expect.objectContaining({ - repoRoot: path.resolve("/tmp/openclaw-repo"), - transportId: "qa-channel", - providerMode: "live-frontier", - }), - ); - }); - it("passes host suite concurrency through", async () => { await runQaSuiteCommand({ repoRoot: "/tmp/openclaw-repo", @@ -602,6 +578,25 @@ describe("qa cli runtime", () => { }); }); + it("defaults manual aimock runs onto the aimock model lane", async () => { + await runQaManualLaneCommand({ + repoRoot: "/tmp/openclaw-repo", + providerMode: "aimock", + message: "read qa kickoff and reply short", + }); + + expect(runQaManualLane).toHaveBeenCalledWith({ + repoRoot: path.resolve("/tmp/openclaw-repo"), + transportId: "qa-channel", + providerMode: "aimock", + primaryModel: "aimock/gpt-5.4", + alternateModel: "aimock/gpt-5.4-alt", + fastMode: undefined, + message: "read qa kickoff and reply short", + timeoutMs: undefined, + }); + }); + it("defaults manual frontier runs onto the frontier model lane", async () => { await runQaManualLaneCommand({ repoRoot: "/tmp/openclaw-repo", @@ -640,31 +635,11 @@ describe("qa cli runtime", () => { }); }); - it("normalizes legacy live-openai manual runs onto the frontier provider mode", async () => { - await runQaManualLaneCommand({ - repoRoot: "/tmp/openclaw-repo", - providerMode: "live-openai", - message: "read qa kickoff and reply short", - }); - - expect(runQaManualLane).toHaveBeenCalledWith( - expect.objectContaining({ - repoRoot: path.resolve("/tmp/openclaw-repo"), - transportId: "qa-channel", - providerMode: "live-frontier", - primaryModel: "openai/gpt-5.4", - alternateModel: "openai/gpt-5.4", - }), - ); - }); - it("defaults manual frontier runs onto Codex OAuth when the runtime resolver prefers it", async () => { defaultQaRuntimeModelForMode.mockImplementation((mode, options) => mode === "live-frontier" ? "openai-codex/gpt-5.4" - : options?.alternate - ? "mock-openai/gpt-5.4-alt" - : "mock-openai/gpt-5.4", + : defaultQaProviderModelForMode(mode as QaProviderModeInput, options), ); await runQaManualLaneCommand({ diff --git a/extensions/qa-lab/src/cli.runtime.ts b/extensions/qa-lab/src/cli.runtime.ts index f0001ffe7be..6b2f78a86ce 100644 --- a/extensions/qa-lab/src/cli.runtime.ts +++ b/extensions/qa-lab/src/cli.runtime.ts @@ -14,8 +14,13 @@ import { runQaDockerUp } from "./docker-up.runtime.js"; import type { QaCliBackendAuthMode } from "./gateway-child.js"; import { startQaLabServer } from "./lab-server.js"; import { runQaManualLane } from "./manual-lane.runtime.js"; -import { startQaMockOpenAiServer } from "./mock-openai-server.js"; import { runQaMultipass } from "./multipass.runtime.js"; +import { DEFAULT_QA_LIVE_PROVIDER_MODE, getQaProvider } from "./providers/index.js"; +import { + QA_FRONTIER_PARITY_BASELINE_LABEL, + QA_FRONTIER_PARITY_CANDIDATE_LABEL, +} from "./providers/live-frontier/parity.js"; +import { startQaProviderServer } from "./providers/server-runtime.js"; import { addQaCredentialSet, listQaCredentialSets, @@ -419,8 +424,8 @@ export async function runQaParityReportCommand(opts: { ) as QaParitySuiteSummary; const comparison = buildQaAgenticParityComparison({ - candidateLabel: opts.candidateLabel?.trim() || "openai/gpt-5.4", - baselineLabel: opts.baselineLabel?.trim() || "anthropic/claude-opus-4-6", + candidateLabel: opts.candidateLabel?.trim() || QA_FRONTIER_PARITY_CANDIDATE_LABEL, + baselineLabel: opts.baselineLabel?.trim() || QA_FRONTIER_PARITY_BASELINE_LABEL, candidateSummary, baselineSummary, }); @@ -488,7 +493,9 @@ export async function runQaManualLaneCommand(opts: { const repoRoot = path.resolve(opts.repoRoot ?? process.cwd()); const transportId = normalizeQaTransportId(opts.transportId); const providerMode: QaProviderMode = - opts.providerMode === undefined ? "live-frontier" : normalizeQaProviderMode(opts.providerMode); + opts.providerMode === undefined + ? DEFAULT_QA_LIVE_PROVIDER_MODE + : normalizeQaProviderMode(opts.providerMode); const models = resolveQaManualLaneModels({ providerMode, primaryModel: opts.primaryModel, @@ -748,12 +755,23 @@ export async function runQaDockerUpCommand(opts: { process.stdout.write(`Stop: ${result.stopCommand}\n`); } -export async function runQaMockOpenAiCommand(opts: { host?: string; port?: number }) { - const server = await startQaMockOpenAiServer({ +export async function runQaProviderServerCommand( + providerMode: QaProviderMode, + opts: { host?: string; port?: number }, +) { + const provider = getQaProvider(providerMode); + const standaloneCommand = provider.standaloneCommand; + if (!standaloneCommand) { + throw new Error(`QA provider "${providerMode}" does not expose a standalone server command.`); + } + const server = await startQaProviderServer(providerMode, { host: opts.host, port: Number.isFinite(opts.port) ? opts.port : undefined, }); - await runInterruptibleServer("QA mock OpenAI", server); + if (!server) { + throw new Error(`QA provider "${providerMode}" does not expose a standalone server command.`); + } + await runInterruptibleServer(standaloneCommand.serverLabel, server); } export const __testing = { diff --git a/extensions/qa-lab/src/cli.test.ts b/extensions/qa-lab/src/cli.test.ts index 0c287f1e51d..f71ba5ca5f7 100644 --- a/extensions/qa-lab/src/cli.test.ts +++ b/extensions/qa-lab/src/cli.test.ts @@ -44,11 +44,13 @@ const { runQaCredentialsAddCommand, runQaCredentialsListCommand, runQaCredentialsRemoveCommand, + runQaProviderServerCommand, runQaTelegramCommand, } = vi.hoisted(() => ({ runQaCredentialsAddCommand: vi.fn(), runQaCredentialsListCommand: vi.fn(), runQaCredentialsRemoveCommand: vi.fn(), + runQaProviderServerCommand: vi.fn(), runQaTelegramCommand: vi.fn(), })); @@ -70,6 +72,7 @@ vi.mock("./cli.runtime.js", () => ({ runQaCredentialsAddCommand, runQaCredentialsListCommand, runQaCredentialsRemoveCommand, + runQaProviderServerCommand, })); import { registerQaLabCli } from "./cli.js"; @@ -82,6 +85,7 @@ describe("qa cli registration", () => { runQaCredentialsAddCommand.mockReset(); runQaCredentialsListCommand.mockReset(); runQaCredentialsRemoveCommand.mockReset(); + runQaProviderServerCommand.mockReset(); runQaTelegramCommand.mockReset(); listQaRunnerCliContributions .mockReset() @@ -116,6 +120,20 @@ describe("qa cli registration", () => { ); }); + it("registers standalone provider server commands from the provider registry", async () => { + const qa = program.commands.find((command) => command.name() === "qa"); + expect(qa?.commands.map((command) => command.name())).toEqual( + expect.arrayContaining(["mock-openai", "aimock"]), + ); + + await program.parseAsync(["node", "openclaw", "qa", "aimock", "--port", "44080"]); + + expect(runQaProviderServerCommand).toHaveBeenCalledWith("aimock", { + host: "127.0.0.1", + port: 44080, + }); + }); + it("shows an enable hint when a discovered runner plugin is installed but blocked", async () => { listQaRunnerCliContributions.mockReset().mockReturnValue([createBlockedQaRunnerContribution()]); const blockedProgram = new Command(); diff --git a/extensions/qa-lab/src/cli.ts b/extensions/qa-lab/src/cli.ts index 435a08b9486..0abba6901d8 100644 --- a/extensions/qa-lab/src/cli.ts +++ b/extensions/qa-lab/src/cli.ts @@ -1,7 +1,16 @@ import type { Command } from "commander"; import { collectString } from "./cli-options.js"; import { listLiveTransportQaCliRegistrations } from "./live-transports/cli.js"; -import type { QaProviderModeInput } from "./run-config.js"; +import { + DEFAULT_QA_LIVE_PROVIDER_MODE, + formatQaProviderModeHelp, + listQaStandaloneProviderCommands, +} from "./providers/index.js"; +import { + QA_FRONTIER_PARITY_BASELINE_LABEL, + QA_FRONTIER_PARITY_CANDIDATE_LABEL, +} from "./providers/live-frontier/parity.js"; +import type { QaProviderMode, QaProviderModeInput } from "./run-config.js"; import { hasQaScenarioPack } from "./scenario-catalog.js"; type QaLabCliRuntime = typeof import("./cli.runtime.js"); @@ -174,9 +183,12 @@ async function runQaDockerUp(opts: { await runtime.runQaDockerUpCommand(opts); } -async function runQaMockOpenAi(opts: { host?: string; port?: number }) { +async function runQaProviderServer( + providerMode: QaProviderMode, + opts: { host?: string; port?: number }, +) { const runtime = await loadQaLabCliRuntime(); - await runtime.runQaMockOpenAiCommand(opts); + await runtime.runQaProviderServerCommand(providerMode, opts); } export function isQaLabCliAvailable(): boolean { @@ -208,11 +220,7 @@ export function registerQaLabCli(program: Command) { .option("--output-dir ", "Suite artifact directory") .option("--runner ", "Execution runner: host or multipass", "host") .option("--transport ", "QA transport id", "qa-channel") - .option( - "--provider-mode ", - "Provider mode: mock-openai or live-frontier (legacy live-openai still works)", - "live-frontier", - ) + .option("--provider-mode ", formatQaProviderModeHelp(), DEFAULT_QA_LIVE_PROVIDER_MODE) .option("--model ", "Primary provider/model ref") .option("--alt-model ", "Alternate provider/model ref") .option( @@ -274,8 +282,12 @@ export function registerQaLabCli(program: Command) { .requiredOption("--candidate-summary ", "Candidate qa-suite-summary.json path") .requiredOption("--baseline-summary ", "Baseline qa-suite-summary.json path") .option("--repo-root ", "Repository root to target when running from a neutral cwd") - .option("--candidate-label