From eb64a8a60d7be07904158cd20fc2cf1d20065620 Mon Sep 17 00:00:00 2001 From: Gustavo Madeira Santana Date: Tue, 14 Apr 2026 13:39:42 -0400 Subject: [PATCH] QA: simplify runner registration seams --- docs/help/testing.md | 5 ++- docs/plugins/manifest.md | 11 ++--- extensions/qa-lab/src/cli.test.ts | 10 +++++ extensions/qa-lab/src/live-transports/cli.ts | 40 ++++++++++-------- .../shared/live-transport-cli.ts | 36 ++++++++++------ .../src/live-transports/telegram/cli.ts | 4 ++ extensions/qa-matrix/index.ts | 2 +- extensions/qa-matrix/runtime-api.ts | 5 +-- extensions/qa-matrix/src/cli.test.ts | 29 +++++++++++++ extensions/qa-matrix/src/runtime-api.test.ts | 9 ++++ .../src/shared/live-transport-cli.ts | 36 ++++++++++------ src/plugin-sdk/qa-runner-runtime.ts | 41 ++++++++----------- 12 files changed, 152 insertions(+), 76 deletions(-) create mode 100644 extensions/qa-matrix/src/cli.test.ts create mode 100644 extensions/qa-matrix/src/runtime-api.test.ts diff --git a/docs/help/testing.md b/docs/help/testing.md index a3ca92ef7b1..c3d5ce7cced 100644 --- a/docs/help/testing.md +++ b/docs/help/testing.md @@ -73,7 +73,7 @@ These commands sit beside the main test suites when you need QA-lab realism: `openclaw plugins install -l ./extensions/qa-matrix`. - Provisions three temporary Matrix users (`driver`, `sut`, `observer`) plus one private room, then starts a QA gateway child with the real Matrix plugin as the SUT transport. - Uses the pinned stable Tuwunel image `ghcr.io/matrix-construct/tuwunel:v1.5.1` by default. Override with `OPENCLAW_QA_MATRIX_TUWUNEL_IMAGE` when you need to test a different image. - - Matrix currently supports only `--credential-source env` because the lane provisions disposable users locally. + - Matrix does not expose shared credential-source flags because the lane provisions disposable users locally. - Writes a Matrix QA report, summary, and observed-events artifact under `.artifacts/qa-e2e/...`. - `pnpm openclaw qa telegram` - Runs the Telegram live QA lane against a real private group using the driver and SUT bot tokens from env. @@ -204,7 +204,8 @@ The minimum adoption bar for a new channel is: 2. Implement the transport runner on the shared `qa-lab` host seam. 3. Keep transport-specific mechanics inside the runner plugin or channel harness. 4. Mount the runner as `openclaw qa ` instead of registering a competing root command. - Runner plugins should declare `qaRunners` in `openclaw.plugin.json` and export matching `qaRunnerCliRegistrations` from `runtime-api.ts`. + Runner plugins should declare `qaRunners` in `openclaw.plugin.json` and export a matching `qaRunnerCliRegistrations` array from `runtime-api.ts`. + Keep `runtime-api.ts` light; lazy CLI and runner execution should stay behind separate entrypoints. 5. Author or adapt markdown scenarios under `qa/scenarios/`. 6. Use the generic scenario helpers for new scenarios. 7. Keep existing compatibility aliases working unless the repo is doing an intentional migration. diff --git a/docs/plugins/manifest.md b/docs/plugins/manifest.md index b009ae44255..c48fc64af79 100644 --- a/docs/plugins/manifest.md +++ b/docs/plugins/manifest.md @@ -160,7 +160,7 @@ Those belong in your plugin code and `package.json`. | `providerAuthChoices` | No | `object[]` | Cheap auth-choice metadata for onboarding pickers, preferred-provider resolution, and simple CLI flag wiring. | | `activation` | No | `object` | Cheap activation hints for provider, command, channel, route, and capability-triggered loading. Metadata only; plugin runtime still owns actual behavior. | | `setup` | No | `object` | Cheap setup/onboarding descriptors that discovery and setup surfaces can inspect without loading plugin runtime. | -| `qaRunners` | No | `object[]` | Cheap QA runner descriptors used by the shared `openclaw qa` host before plugin runtime loads. | +| `qaRunners` | No | `object[]` | Cheap QA runner descriptors used by the shared `openclaw qa` host before plugin runtime loads. | | `contracts` | No | `object` | Static bundled capability snapshot for speech, realtime transcription, realtime voice, media-understanding, image-generation, music-generation, video-generation, web-fetch, web search, and tool ownership. | | `channelConfigs` | No | `Record` | Manifest-owned channel config metadata merged into discovery and validation surfaces before runtime loads. | | `skills` | No | `string[]` | Skill directories to load, relative to the plugin root. | @@ -226,7 +226,8 @@ should activate it later. Use `qaRunners` when a plugin contributes one or more transport runners beneath the shared `openclaw qa` root. Keep this metadata cheap and static; the plugin -runtime still owns actual CLI registration through `runtime-api.ts`. +runtime still owns actual CLI registration through a lightweight +`runtime-api.ts` surface that exports `qaRunnerCliRegistrations`. ```json { @@ -239,9 +240,9 @@ runtime still owns actual CLI registration through `runtime-api.ts`. } ``` -| Field | Required | Type | What it means | -| ------------- | -------- | -------- | ----------------------------------------------------------------- | -| `commandName` | Yes | `string` | Subcommand mounted beneath `openclaw qa`, for example `matrix`. | +| Field | Required | Type | What it means | +| ------------- | -------- | -------- | ------------------------------------------------------------------ | +| `commandName` | Yes | `string` | Subcommand mounted beneath `openclaw qa`, for example `matrix`. | | `description` | No | `string` | Fallback help text used when the shared host needs a stub command. | This block is metadata only. It does not register runtime behavior, and it does diff --git a/extensions/qa-lab/src/cli.test.ts b/extensions/qa-lab/src/cli.test.ts index 618577b68d4..7be488376c9 100644 --- a/extensions/qa-lab/src/cli.test.ts +++ b/extensions/qa-lab/src/cli.test.ts @@ -86,6 +86,16 @@ describe("qa cli registration", () => { expect(registration.register).toHaveBeenCalledTimes(1); }); + it("keeps Telegram credential flags on the shared host CLI", () => { + const qa = program.commands.find((command) => command.name() === "qa"); + const telegram = qa?.commands.find((command) => command.name() === "telegram"); + const optionNames = telegram?.options.map((option) => option.long) ?? []; + + expect(optionNames).toEqual( + expect.arrayContaining(["--credential-source", "--credential-role"]), + ); + }); + it("shows an install hint when a discovered runner plugin is unavailable", async () => { listQaRunnerCliContributions.mockReset().mockReturnValue([ { diff --git a/extensions/qa-lab/src/live-transports/cli.ts b/extensions/qa-lab/src/live-transports/cli.ts index ea2c1b7b6ef..9d2275ae6b4 100644 --- a/extensions/qa-lab/src/live-transports/cli.ts +++ b/extensions/qa-lab/src/live-transports/cli.ts @@ -40,6 +40,28 @@ function createBlockedQaRunnerCliRegistration(params: { }; } +function createQaRunnerCliRegistration( + runner: ReturnType[number], +): LiveTransportQaCliRegistration { + if (runner.status === "available") { + return runner.registration; + } + if (runner.status === "blocked") { + return createBlockedQaRunnerCliRegistration({ + commandName: runner.commandName, + description: runner.description, + pluginId: runner.pluginId, + }); + } + return createMissingQaRunnerCliRegistration({ + commandName: runner.commandName, + description: + runner.description ?? + `Run the ${runner.commandName} live QA lane (install ${runner.npmSpec} first)`, + npmSpec: runner.npmSpec, + }); +} + export const LIVE_TRANSPORT_QA_CLI_REGISTRATIONS: readonly LiveTransportQaCliRegistration[] = [ telegramQaCliRegistration, ]; @@ -49,23 +71,7 @@ export function listLiveTransportQaCliRegistrations(): readonly LiveTransportQaC const discoveredRunners = listQaRunnerCliContributions(); for (const runner of discoveredRunners) { - liveRegistrations.push( - runner.status === "available" - ? runner.registration - : runner.status === "blocked" - ? createBlockedQaRunnerCliRegistration({ - commandName: runner.commandName, - description: runner.description, - pluginId: runner.pluginId, - }) - : createMissingQaRunnerCliRegistration({ - commandName: runner.commandName, - description: - runner.description ?? - `Run the ${runner.commandName} live QA lane (install ${runner.npmSpec} first)`, - npmSpec: runner.npmSpec, - }), - ); + liveRegistrations.push(createQaRunnerCliRegistration(runner)); } return liveRegistrations; diff --git a/extensions/qa-lab/src/live-transports/shared/live-transport-cli.ts b/extensions/qa-lab/src/live-transports/shared/live-transport-cli.ts index fe389ebc500..65ab2cf4e18 100644 --- a/extensions/qa-lab/src/live-transports/shared/live-transport-cli.ts +++ b/extensions/qa-lab/src/live-transports/shared/live-transport-cli.ts @@ -33,6 +33,11 @@ export type LiveTransportQaCliRegistration = { register(qa: Command): void; }; +export type LiveTransportQaCredentialCliOptions = { + sourceDescription?: string; + roleDescription?: string; +}; + export function createLazyCliRuntimeLoader(load: () => Promise) { let promise: Promise | null = null; return async () => { @@ -61,13 +66,14 @@ export function mapLiveTransportQaCommanderOptions( export function registerLiveTransportQaCli(params: { qa: Command; commandName: string; + credentialOptions?: LiveTransportQaCredentialCliOptions; description: string; outputDirHelp: string; scenarioHelp: string; sutAccountHelp: string; run: (opts: LiveTransportQaCommandOptions) => Promise; }) { - params.qa + const command = params.qa .command(params.commandName) .description(params.description) .option("--repo-root ", "Repository root to target when running from a neutral cwd") @@ -81,22 +87,27 @@ export function registerLiveTransportQaCli(params: { .option("--alt-model ", "Alternate provider/model ref") .option("--scenario ", params.scenarioHelp, collectString, []) .option("--fast", "Enable provider fast mode where supported", false) - .option("--sut-account ", params.sutAccountHelp, "sut") - .option( + .option("--sut-account ", params.sutAccountHelp, "sut"); + + if (params.credentialOptions) { + command.option( "--credential-source ", - "Credential source for live lanes: env or convex (default: env)", - ) - .option( - "--credential-role ", - "Credential role for convex auth: maintainer or ci (default: maintainer)", - ) - .action(async (opts: LiveTransportQaCommanderOptions) => { - await params.run(mapLiveTransportQaCommanderOptions(opts)); - }); + params.credentialOptions.sourceDescription ?? + "Credential source for live lanes: env or convex (default: env)", + ); + if (params.credentialOptions.roleDescription) { + command.option("--credential-role ", params.credentialOptions.roleDescription); + } + } + + command.action(async (opts: LiveTransportQaCommanderOptions) => { + await params.run(mapLiveTransportQaCommanderOptions(opts)); + }); } export function createLiveTransportQaCliRegistration(params: { commandName: string; + credentialOptions?: LiveTransportQaCredentialCliOptions; description: string; outputDirHelp: string; scenarioHelp: string; @@ -109,6 +120,7 @@ export function createLiveTransportQaCliRegistration(params: { registerLiveTransportQaCli({ qa, commandName: params.commandName, + credentialOptions: params.credentialOptions, description: params.description, outputDirHelp: params.outputDirHelp, scenarioHelp: params.scenarioHelp, diff --git a/extensions/qa-lab/src/live-transports/telegram/cli.ts b/extensions/qa-lab/src/live-transports/telegram/cli.ts index e957f229ebb..6237476b16f 100644 --- a/extensions/qa-lab/src/live-transports/telegram/cli.ts +++ b/extensions/qa-lab/src/live-transports/telegram/cli.ts @@ -20,6 +20,10 @@ async function runQaTelegram(opts: LiveTransportQaCommandOptions) { export const telegramQaCliRegistration: LiveTransportQaCliRegistration = createLiveTransportQaCliRegistration({ commandName: "telegram", + credentialOptions: { + sourceDescription: "Credential source for Telegram QA: env or convex (default: env)", + roleDescription: "Credential role for convex auth: maintainer or ci (default: maintainer)", + }, description: "Run the manual Telegram live QA lane against a private bot-to-bot group harness", outputDirHelp: "Telegram QA artifact directory", scenarioHelp: "Run only the named Telegram QA scenario (repeatable)", diff --git a/extensions/qa-matrix/index.ts b/extensions/qa-matrix/index.ts index 6c0d8629ff0..f584ecb7ed6 100644 --- a/extensions/qa-matrix/index.ts +++ b/extensions/qa-matrix/index.ts @@ -1,4 +1,4 @@ -import { definePluginEntry } from "./runtime-api.js"; +import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry"; export default definePluginEntry({ id: "qa-matrix", diff --git a/extensions/qa-matrix/runtime-api.ts b/extensions/qa-matrix/runtime-api.ts index 7862b296d81..127fa5b033e 100644 --- a/extensions/qa-matrix/runtime-api.ts +++ b/extensions/qa-matrix/runtime-api.ts @@ -1,4 +1 @@ -export { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry"; -export { qaRunnerCliRegistrations, registerMatrixQaCli } from "./cli.js"; -export { runQaMatrixCommand } from "./cli.runtime.js"; -export { runMatrixQaLive } from "./runtime.js"; +export { qaRunnerCliRegistrations } from "./cli.js"; diff --git a/extensions/qa-matrix/src/cli.test.ts b/extensions/qa-matrix/src/cli.test.ts new file mode 100644 index 00000000000..4a5dd704f6a --- /dev/null +++ b/extensions/qa-matrix/src/cli.test.ts @@ -0,0 +1,29 @@ +import { Command } from "commander"; +import { describe, expect, it } from "vitest"; +import { matrixQaCliRegistration } from "./cli.js"; + +describe("matrix qa cli registration", () => { + it("keeps disposable Matrix lane flags focused", () => { + const qa = new Command(); + + matrixQaCliRegistration.register(qa); + + const matrix = qa.commands.find((command) => command.name() === "matrix"); + const optionNames = matrix?.options.map((option) => option.long) ?? []; + + expect(optionNames).toEqual( + expect.arrayContaining([ + "--repo-root", + "--output-dir", + "--provider-mode", + "--model", + "--alt-model", + "--scenario", + "--fast", + "--sut-account", + ]), + ); + expect(optionNames).not.toContain("--credential-source"); + expect(optionNames).not.toContain("--credential-role"); + }); +}); diff --git a/extensions/qa-matrix/src/runtime-api.test.ts b/extensions/qa-matrix/src/runtime-api.test.ts new file mode 100644 index 00000000000..87a3796b68d --- /dev/null +++ b/extensions/qa-matrix/src/runtime-api.test.ts @@ -0,0 +1,9 @@ +import { describe, expect, it } from "vitest"; + +describe("matrix qa runtime api surface", () => { + it("keeps runner discovery lightweight", async () => { + const runtimeApi = await import("../runtime-api.js"); + + expect(Object.keys(runtimeApi).toSorted()).toEqual(["qaRunnerCliRegistrations"]); + }); +}); diff --git a/extensions/qa-matrix/src/shared/live-transport-cli.ts b/extensions/qa-matrix/src/shared/live-transport-cli.ts index f8a91a7ee25..92f320b4eb4 100644 --- a/extensions/qa-matrix/src/shared/live-transport-cli.ts +++ b/extensions/qa-matrix/src/shared/live-transport-cli.ts @@ -33,6 +33,11 @@ export type LiveTransportQaCliRegistration = { register(qa: Command): void; }; +export type LiveTransportQaCredentialCliOptions = { + sourceDescription?: string; + roleDescription?: string; +}; + export function createLazyCliRuntimeLoader(load: () => Promise) { let promise: Promise | null = null; return async () => { @@ -61,13 +66,14 @@ export function mapLiveTransportQaCommanderOptions( export function registerLiveTransportQaCli(params: { qa: Command; commandName: string; + credentialOptions?: LiveTransportQaCredentialCliOptions; description: string; outputDirHelp: string; scenarioHelp: string; sutAccountHelp: string; run: (opts: LiveTransportQaCommandOptions) => Promise; }) { - params.qa + const command = params.qa .command(params.commandName) .description(params.description) .option("--repo-root ", "Repository root to target when running from a neutral cwd") @@ -81,22 +87,27 @@ export function registerLiveTransportQaCli(params: { .option("--alt-model ", "Alternate provider/model ref") .option("--scenario ", params.scenarioHelp, collectString, []) .option("--fast", "Enable provider fast mode where supported", false) - .option("--sut-account ", params.sutAccountHelp, "sut") - .option( + .option("--sut-account ", params.sutAccountHelp, "sut"); + + if (params.credentialOptions) { + command.option( "--credential-source ", - "Credential source for live lanes: env or convex (default: env)", - ) - .option( - "--credential-role ", - "Credential role for convex auth: maintainer or ci (default: maintainer)", - ) - .action(async (opts: LiveTransportQaCommanderOptions) => { - await params.run(mapLiveTransportQaCommanderOptions(opts)); - }); + params.credentialOptions.sourceDescription ?? + "Credential source for live lanes: env or convex (default: env)", + ); + if (params.credentialOptions.roleDescription) { + command.option("--credential-role ", params.credentialOptions.roleDescription); + } + } + + command.action(async (opts: LiveTransportQaCommanderOptions) => { + await params.run(mapLiveTransportQaCommanderOptions(opts)); + }); } export function createLiveTransportQaCliRegistration(params: { commandName: string; + credentialOptions?: LiveTransportQaCredentialCliOptions; description: string; outputDirHelp: string; scenarioHelp: string; @@ -109,6 +120,7 @@ export function createLiveTransportQaCliRegistration(params: { registerLiveTransportQaCli({ qa, commandName: params.commandName, + credentialOptions: params.credentialOptions, description: params.description, outputDirHelp: params.outputDirHelp, scenarioHelp: params.scenarioHelp, diff --git a/src/plugin-sdk/qa-runner-runtime.ts b/src/plugin-sdk/qa-runner-runtime.ts index e4ce8787e5a..65e49b0aed9 100644 --- a/src/plugin-sdk/qa-runner-runtime.ts +++ b/src/plugin-sdk/qa-runner-runtime.ts @@ -10,7 +10,6 @@ export type QaRunnerCliRegistration = { }; type QaRunnerRuntimeSurface = { - listQaRunnerCliRegistrations?: () => readonly QaRunnerCliRegistration[]; qaRunnerCliRegistrations?: readonly QaRunnerCliRegistration[]; }; @@ -39,8 +38,8 @@ export type QaRunnerCliContribution = function listDeclaredQaRunnerPlugins(): Array< Pick > { - return loadPluginManifestRegistry({ cache: true }).plugins - .filter( + return loadPluginManifestRegistry({ cache: true }) + .plugins.filter( ( plugin, ): plugin is Pick & { @@ -56,25 +55,24 @@ function listDeclaredQaRunnerPlugins(): Array< }); } -function listRuntimeRegistrations( +function indexRuntimeRegistrations( pluginId: string, surface: QaRunnerRuntimeSurface, -): readonly QaRunnerCliRegistration[] { - const registrations = - surface.listQaRunnerCliRegistrations?.() ?? surface.qaRunnerCliRegistrations ?? []; - const seen = new Set(); +): ReadonlyMap { + const registrations = surface.qaRunnerCliRegistrations ?? []; + const registrationByCommandName = new Map(); for (const registration of registrations) { if (!registration?.commandName || typeof registration.register !== "function") { throw new Error(`QA runner plugin "${pluginId}" exported an invalid CLI registration`); } - if (seen.has(registration.commandName)) { + if (registrationByCommandName.has(registration.commandName)) { throw new Error( `QA runner plugin "${pluginId}" exported duplicate CLI registration "${registration.commandName}"`, ); } - seen.add(registration.commandName); + registrationByCommandName.set(registration.commandName, registration); } - return registrations; + return registrationByCommandName; } function buildKnownQaRunnerCatalog(): readonly QaRunnerCliContribution[] { @@ -106,14 +104,13 @@ export function listQaRunnerCliContributions(): readonly QaRunnerCliContribution } for (const plugin of listDeclaredQaRunnerPlugins()) { - const runtimeSurface = tryLoadActivatedBundledPluginPublicSurfaceModuleSync( - { + const runtimeSurface = + tryLoadActivatedBundledPluginPublicSurfaceModuleSync({ dirName: plugin.id, artifactBasename: "runtime-api.js", - }, - ); - const runtimeRegistrations = runtimeSurface - ? listRuntimeRegistrations(plugin.id, runtimeSurface) + }); + const runtimeRegistrationByCommandName = runtimeSurface + ? indexRuntimeRegistrations(plugin.id, runtimeSurface) : null; const declaredCommandNames = new Set(plugin.qaRunners.map((runner) => runner.commandName)); @@ -125,9 +122,7 @@ export function listQaRunnerCliContributions(): readonly QaRunnerCliContribution ); } - const registration = runtimeRegistrations?.find( - (entry) => entry.commandName === runner.commandName, - ); + const registration = runtimeRegistrationByCommandName?.get(runner.commandName); if (!runtimeSurface) { contributions.set(runner.commandName, { pluginId: plugin.id, @@ -151,10 +146,10 @@ export function listQaRunnerCliContributions(): readonly QaRunnerCliContribution }); } - for (const registration of runtimeRegistrations ?? []) { - if (!declaredCommandNames.has(registration.commandName)) { + for (const commandName of runtimeRegistrationByCommandName?.keys() ?? []) { + if (!declaredCommandNames.has(commandName)) { throw new Error( - `QA runner plugin "${plugin.id}" exported "${registration.commandName}" from runtime-api.js but did not declare it in openclaw.plugin.json`, + `QA runner plugin "${plugin.id}" exported "${commandName}" from runtime-api.js but did not declare it in openclaw.plugin.json`, ); } }