From 2a20f8261f03cbf25687fd9ef012f3d88f6ef07d Mon Sep 17 00:00:00 2001 From: Gustavo Madeira Santana Date: Tue, 14 Apr 2026 13:07:02 -0400 Subject: [PATCH] QA: add generic runner discovery seam Teach qa-lab to discover transport runners from manifest metadata plus a shared runtime facade instead of hardcoding qa-matrix. The host now mounts activated runners generically, shows enable guidance for blocked plugins, and keeps the explicit install hint for missing optional runners. This also promotes the runner contract into the public SDK, replaces the matrix-specific host seam, and documents the new manifest/runtime exports so future QA transports can reuse the same path. --- docs/help/testing.md | 1 + docs/plugins/manifest.md | 25 ++++ extensions/qa-lab/src/cli.test.ts | 65 +++++++--- extensions/qa-lab/src/live-transports/cli.ts | 77 +++++++++--- extensions/qa-matrix/openclaw.plugin.json | 6 + extensions/qa-matrix/runtime-api.ts | 2 +- extensions/qa-matrix/src/cli.ts | 2 + package.json | 6 +- scripts/lib/plugin-sdk-entrypoints.json | 2 +- src/plugin-sdk/qa-matrix.test.ts | 48 ------- src/plugin-sdk/qa-matrix.ts | 36 ------ src/plugin-sdk/qa-runner-runtime.test.ts | 120 ++++++++++++++++++ src/plugin-sdk/qa-runner-runtime.ts | 124 +++++++++++++++++++ src/plugins/manifest-registry.test.ts | 27 ++++ src/plugins/manifest-registry.ts | 3 + src/plugins/manifest.ts | 33 +++++ 16 files changed, 458 insertions(+), 119 deletions(-) delete mode 100644 src/plugin-sdk/qa-matrix.test.ts delete mode 100644 src/plugin-sdk/qa-matrix.ts create mode 100644 src/plugin-sdk/qa-runner-runtime.test.ts create mode 100644 src/plugin-sdk/qa-runner-runtime.ts diff --git a/docs/help/testing.md b/docs/help/testing.md index 746ad03cea4..a3ca92ef7b1 100644 --- a/docs/help/testing.md +++ b/docs/help/testing.md @@ -204,6 +204,7 @@ The minimum adoption bar for a new channel is: 2. Implement the transport runner on the shared `qa-lab` host seam. 3. Keep transport-specific mechanics inside the runner plugin or channel harness. 4. Mount the runner as `openclaw qa ` instead of registering a competing root command. + Runner plugins should declare `qaRunners` in `openclaw.plugin.json` and export matching `qaRunnerCliRegistrations` from `runtime-api.ts`. 5. Author or adapt markdown scenarios under `qa/scenarios/`. 6. Use the generic scenario helpers for new scenarios. 7. Keep existing compatibility aliases working unless the repo is doing an intentional migration. diff --git a/docs/plugins/manifest.md b/docs/plugins/manifest.md index 26c8fd98edc..b009ae44255 100644 --- a/docs/plugins/manifest.md +++ b/docs/plugins/manifest.md @@ -56,6 +56,8 @@ Use it for: plugin before runtime loads - static capability ownership snapshots used for bundled compat wiring and contract coverage +- cheap QA runner metadata that the shared `openclaw qa` host can inspect + before plugin runtime loads - channel-specific config metadata that should merge into catalog and validation surfaces without loading runtime - config UI hints @@ -158,6 +160,7 @@ Those belong in your plugin code and `package.json`. | `providerAuthChoices` | No | `object[]` | Cheap auth-choice metadata for onboarding pickers, preferred-provider resolution, and simple CLI flag wiring. | | `activation` | No | `object` | Cheap activation hints for provider, command, channel, route, and capability-triggered loading. Metadata only; plugin runtime still owns actual behavior. | | `setup` | No | `object` | Cheap setup/onboarding descriptors that discovery and setup surfaces can inspect without loading plugin runtime. | +| `qaRunners` | No | `object[]` | Cheap QA runner descriptors used by the shared `openclaw qa` host before plugin runtime loads. | | `contracts` | No | `object` | Static bundled capability snapshot for speech, realtime transcription, realtime voice, media-understanding, image-generation, music-generation, video-generation, web-fetch, web search, and tool ownership. | | `channelConfigs` | No | `Record` | Manifest-owned channel config metadata merged into discovery and validation surfaces before runtime loads. | | `skills` | No | `string[]` | Skill directories to load, relative to the plugin root. | @@ -219,6 +222,28 @@ uses this metadata for diagnostics without importing plugin runtime code. Use `activation` when the plugin can cheaply declare which control-plane events should activate it later. +## qaRunners reference + +Use `qaRunners` when a plugin contributes one or more transport runners beneath +the shared `openclaw qa` root. Keep this metadata cheap and static; the plugin +runtime still owns actual CLI registration through `runtime-api.ts`. + +```json +{ + "qaRunners": [ + { + "commandName": "matrix", + "description": "Run the Docker-backed Matrix live QA lane against a disposable homeserver" + } + ] +} +``` + +| Field | Required | Type | What it means | +| ------------- | -------- | -------- | ----------------------------------------------------------------- | +| `commandName` | Yes | `string` | Subcommand mounted beneath `openclaw qa`, for example `matrix`. | +| `description` | No | `string` | Fallback help text used when the shared host needs a stub command. | + This block is metadata only. It does not register runtime behavior, and it does not replace `register(...)`, `setupEntry`, or other runtime/plugin entrypoints. Current consumers use it as a narrowing hint before broader plugin loading, so diff --git a/extensions/qa-lab/src/cli.test.ts b/extensions/qa-lab/src/cli.test.ts index e9d31a22429..afd802ee99a 100644 --- a/extensions/qa-lab/src/cli.test.ts +++ b/extensions/qa-lab/src/cli.test.ts @@ -13,16 +13,24 @@ const { runQaTelegramCommand: vi.fn(), })); -const { isMatrixQaCliAvailable, registerMatrixQaCli } = vi.hoisted(() => ({ - isMatrixQaCliAvailable: vi.fn(() => true), - registerMatrixQaCli: vi.fn((qa: Command) => { - qa.command("matrix").action(() => undefined); - }), +const { listQaRunnerCliContributions } = vi.hoisted(() => ({ + listQaRunnerCliContributions: vi.fn(() => [ + { + pluginId: "qa-matrix", + commandName: "matrix", + status: "available" as const, + registration: { + commandName: "matrix", + register: vi.fn((qa: Command) => { + qa.command("matrix").action(() => undefined); + }), + }, + }, + ]), })); -vi.mock("openclaw/plugin-sdk/qa-matrix", () => ({ - isMatrixQaCliAvailable, - registerMatrixQaCli, +vi.mock("openclaw/plugin-sdk/qa-runner-runtime", () => ({ + listQaRunnerCliContributions, })); vi.mock("./live-transports/telegram/cli.runtime.js", () => ({ @@ -46,8 +54,19 @@ describe("qa cli registration", () => { runQaCredentialsListCommand.mockReset(); runQaCredentialsRemoveCommand.mockReset(); runQaTelegramCommand.mockReset(); - isMatrixQaCliAvailable.mockClear().mockReturnValue(true); - registerMatrixQaCli.mockClear(); + listQaRunnerCliContributions.mockReset().mockReturnValue([ + { + pluginId: "qa-matrix", + commandName: "matrix", + status: "available", + registration: { + commandName: "matrix", + register: vi.fn((qa: Command) => { + qa.command("matrix").action(() => undefined); + }), + }, + }, + ]); registerQaLabCli(program); }); @@ -63,20 +82,36 @@ describe("qa cli registration", () => { ); }); - it("delegates matrix command registration to the qa-matrix facade", () => { - expect(registerMatrixQaCli).toHaveBeenCalledTimes(1); + it("delegates discovered qa runner registration through the generic host seam", () => { + const [{ registration }] = listQaRunnerCliContributions.mock.results[0]?.value; + expect(registration.register).toHaveBeenCalledTimes(1); }); it("shows an install hint when the matrix runner plugin is unavailable", async () => { - isMatrixQaCliAvailable.mockReset().mockReturnValue(false); - registerMatrixQaCli.mockReset(); + listQaRunnerCliContributions.mockReset().mockReturnValue([]); const missingProgram = new Command(); registerQaLabCli(missingProgram); await expect(missingProgram.parseAsync(["node", "openclaw", "qa", "matrix"])).rejects.toThrow( "openclaw plugins install @openclaw/qa-matrix", ); - expect(registerMatrixQaCli).not.toHaveBeenCalled(); + }); + + it("shows an enable hint when the matrix runner plugin is installed but blocked", async () => { + listQaRunnerCliContributions.mockReset().mockReturnValue([ + { + pluginId: "qa-matrix", + commandName: "matrix", + description: "Run the Matrix live QA lane", + status: "blocked", + }, + ]); + const blockedProgram = new Command(); + registerQaLabCli(blockedProgram); + + await expect(blockedProgram.parseAsync(["node", "openclaw", "qa", "matrix"])).rejects.toThrow( + 'Enable or allow plugin "qa-matrix"', + ); }); it("routes telegram CLI defaults into the lane runtime", async () => { diff --git a/extensions/qa-lab/src/live-transports/cli.ts b/extensions/qa-lab/src/live-transports/cli.ts index 940b20d236d..1b2feb59d72 100644 --- a/extensions/qa-lab/src/live-transports/cli.ts +++ b/extensions/qa-lab/src/live-transports/cli.ts @@ -1,16 +1,47 @@ -import { isMatrixQaCliAvailable, registerMatrixQaCli } from "openclaw/plugin-sdk/qa-matrix"; +import { listQaRunnerCliContributions } from "openclaw/plugin-sdk/qa-runner-runtime"; import type { LiveTransportQaCliRegistration } from "./shared/live-transport-cli.js"; import { telegramQaCliRegistration } from "./telegram/cli.js"; -function createUnavailableMatrixQaCliRegistration(): LiveTransportQaCliRegistration { - return { +const OPTIONAL_QA_RUNNER_INSTALLS = [ + { commandName: "matrix", + description: "Run the Matrix live QA lane (install @openclaw/qa-matrix first)", + npmSpec: "@openclaw/qa-matrix", + }, +] as const; + +function createMissingQaRunnerCliRegistration(params: { + commandName: string; + description: string; + npmSpec: string; +}): LiveTransportQaCliRegistration { + return { + commandName: params.commandName, register(qa) { - qa.command("matrix") - .description("Run the Matrix live QA lane (install @openclaw/qa-matrix first)") + qa.command(params.commandName) + .description(params.description) .action(() => { throw new Error( - 'Matrix QA runner not installed. Install it with "openclaw plugins install @openclaw/qa-matrix".', + `QA runner "${params.commandName}" not installed. Install it with "openclaw plugins install ${params.npmSpec}".`, + ); + }); + }, + }; +} + +function createBlockedQaRunnerCliRegistration(params: { + commandName: string; + description?: string; + pluginId: string; +}): LiveTransportQaCliRegistration { + return { + commandName: params.commandName, + register(qa) { + qa.command(params.commandName) + .description(params.description ?? `Run the ${params.commandName} live QA lane`) + .action(() => { + throw new Error( + `QA runner "${params.commandName}" is installed but not active. Enable or allow plugin "${params.pluginId}" in your OpenClaw config, then try again.`, ); }); }, @@ -22,13 +53,29 @@ export const LIVE_TRANSPORT_QA_CLI_REGISTRATIONS: readonly LiveTransportQaCliReg ]; export function listLiveTransportQaCliRegistrations(): readonly LiveTransportQaCliRegistration[] { - return [ - ...LIVE_TRANSPORT_QA_CLI_REGISTRATIONS, - isMatrixQaCliAvailable() - ? { - commandName: "matrix", - register: registerMatrixQaCli, - } - : createUnavailableMatrixQaCliRegistration(), - ]; + const liveRegistrations = [...LIVE_TRANSPORT_QA_CLI_REGISTRATIONS]; + const discoveredRunners = listQaRunnerCliContributions(); + const seenCommandNames = new Set(liveRegistrations.map((registration) => registration.commandName)); + + for (const runner of discoveredRunners) { + seenCommandNames.add(runner.commandName); + liveRegistrations.push( + runner.status === "available" + ? runner.registration + : createBlockedQaRunnerCliRegistration({ + commandName: runner.commandName, + description: runner.description, + pluginId: runner.pluginId, + }), + ); + } + + for (const runner of OPTIONAL_QA_RUNNER_INSTALLS) { + if (seenCommandNames.has(runner.commandName)) { + continue; + } + liveRegistrations.push(createMissingQaRunnerCliRegistration(runner)); + } + + return liveRegistrations; } diff --git a/extensions/qa-matrix/openclaw.plugin.json b/extensions/qa-matrix/openclaw.plugin.json index 0a40ce77732..718ad710609 100644 --- a/extensions/qa-matrix/openclaw.plugin.json +++ b/extensions/qa-matrix/openclaw.plugin.json @@ -2,6 +2,12 @@ "id": "qa-matrix", "name": "QA Matrix", "description": "Matrix QA transport runner and substrate", + "qaRunners": [ + { + "commandName": "matrix", + "description": "Run the Docker-backed Matrix live QA lane against a disposable homeserver" + } + ], "configSchema": { "type": "object", "additionalProperties": false, diff --git a/extensions/qa-matrix/runtime-api.ts b/extensions/qa-matrix/runtime-api.ts index 57f2ea533d5..7862b296d81 100644 --- a/extensions/qa-matrix/runtime-api.ts +++ b/extensions/qa-matrix/runtime-api.ts @@ -1,4 +1,4 @@ export { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry"; -export { registerMatrixQaCli } from "./cli.js"; +export { qaRunnerCliRegistrations, registerMatrixQaCli } from "./cli.js"; export { runQaMatrixCommand } from "./cli.runtime.js"; export { runMatrixQaLive } from "./runtime.js"; diff --git a/extensions/qa-matrix/src/cli.ts b/extensions/qa-matrix/src/cli.ts index 836c443c3b9..6a81d8a5502 100644 --- a/extensions/qa-matrix/src/cli.ts +++ b/extensions/qa-matrix/src/cli.ts @@ -27,6 +27,8 @@ export const matrixQaCliRegistration: LiveTransportQaCliRegistration = run: runQaMatrix, }); +export const qaRunnerCliRegistrations = [matrixQaCliRegistration] as const; + export function registerMatrixQaCli(qa: Command) { matrixQaCliRegistration.register(qa); } diff --git a/package.json b/package.json index 4b03af65077..aeebb76f4a5 100644 --- a/package.json +++ b/package.json @@ -769,9 +769,9 @@ "types": "./dist/plugin-sdk/qa-lab-runtime.d.ts", "default": "./dist/plugin-sdk/qa-lab-runtime.js" }, - "./plugin-sdk/qa-matrix": { - "types": "./dist/plugin-sdk/qa-matrix.d.ts", - "default": "./dist/plugin-sdk/qa-matrix.js" + "./plugin-sdk/qa-runner-runtime": { + "types": "./dist/plugin-sdk/qa-runner-runtime.d.ts", + "default": "./dist/plugin-sdk/qa-runner-runtime.js" }, "./plugin-sdk/mattermost": { "types": "./dist/plugin-sdk/mattermost.d.ts", diff --git a/scripts/lib/plugin-sdk-entrypoints.json b/scripts/lib/plugin-sdk-entrypoints.json index c37027265f0..85274c6ccc3 100644 --- a/scripts/lib/plugin-sdk-entrypoints.json +++ b/scripts/lib/plugin-sdk-entrypoints.json @@ -180,7 +180,7 @@ "matrix-surface", "matrix-thread-bindings", "qa-lab-runtime", - "qa-matrix", + "qa-runner-runtime", "mattermost", "mattermost-policy", "memory-core", diff --git a/src/plugin-sdk/qa-matrix.test.ts b/src/plugin-sdk/qa-matrix.test.ts deleted file mode 100644 index 7bca0e90423..00000000000 --- a/src/plugin-sdk/qa-matrix.test.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { beforeEach, describe, expect, it, vi } from "vitest"; - -const loadBundledPluginPublicSurfaceModuleSync = vi.hoisted(() => vi.fn()); -const registerMatrixQaCliImpl = vi.hoisted(() => vi.fn()); - -vi.mock("./facade-runtime.js", async () => { - const actual = await vi.importActual("./facade-runtime.js"); - return { - ...actual, - loadBundledPluginPublicSurfaceModuleSync, - }; -}); - -describe("plugin-sdk qa-matrix", () => { - beforeEach(() => { - registerMatrixQaCliImpl.mockReset(); - loadBundledPluginPublicSurfaceModuleSync.mockReset().mockReturnValue({ - registerMatrixQaCli: registerMatrixQaCliImpl, - }); - }); - - it("keeps the qa-matrix facade cold until used", async () => { - const module = await import("./qa-matrix.js"); - - expect(loadBundledPluginPublicSurfaceModuleSync).not.toHaveBeenCalled(); - module.registerMatrixQaCli({} as never); - expect(loadBundledPluginPublicSurfaceModuleSync).toHaveBeenCalledWith({ - dirName: "qa-matrix", - artifactBasename: "cli.js", - }); - }); - - it("delegates matrix qa cli registration through the public surface", async () => { - const module = await import("./qa-matrix.js"); - - module.registerMatrixQaCli({} as never); - expect(registerMatrixQaCliImpl).toHaveBeenCalledWith({} as never); - }); - - it("reports qa-matrix unavailable when the public facade is missing", async () => { - loadBundledPluginPublicSurfaceModuleSync.mockImplementation(() => { - throw new Error("Unable to resolve bundled plugin public surface qa-matrix/cli.js"); - }); - const module = await import("./qa-matrix.js"); - - expect(module.isMatrixQaCliAvailable()).toBe(false); - }); -}); diff --git a/src/plugin-sdk/qa-matrix.ts b/src/plugin-sdk/qa-matrix.ts deleted file mode 100644 index aa7218fcdea..00000000000 --- a/src/plugin-sdk/qa-matrix.ts +++ /dev/null @@ -1,36 +0,0 @@ -import type { Command } from "commander"; -import { loadBundledPluginPublicSurfaceModuleSync } from "./facade-runtime.js"; - -type MatrixQaCliSurface = { - registerMatrixQaCli: (qa: Command) => void; -}; - -function isMissingMatrixQaFacadeError(error: unknown) { - return ( - error instanceof Error && - (error.message === "Unable to resolve bundled plugin public surface qa-matrix/cli.js" || - error.message.startsWith("Unable to open bundled plugin public surface ")) - ); -} - -function loadFacadeModule(): MatrixQaCliSurface { - return loadBundledPluginPublicSurfaceModuleSync({ - dirName: "qa-matrix", - artifactBasename: "cli.js", - }); -} - -export const registerMatrixQaCli: MatrixQaCliSurface["registerMatrixQaCli"] = ((...args) => - loadFacadeModule().registerMatrixQaCli(...args)) as MatrixQaCliSurface["registerMatrixQaCli"]; - -export function isMatrixQaCliAvailable(): boolean { - try { - loadFacadeModule(); - return true; - } catch (error) { - if (isMissingMatrixQaFacadeError(error)) { - return false; - } - throw error; - } -} diff --git a/src/plugin-sdk/qa-runner-runtime.test.ts b/src/plugin-sdk/qa-runner-runtime.test.ts new file mode 100644 index 00000000000..13cc4f6ee0f --- /dev/null +++ b/src/plugin-sdk/qa-runner-runtime.test.ts @@ -0,0 +1,120 @@ +import type { Command } from "commander"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const loadPluginManifestRegistry = vi.hoisted(() => vi.fn()); +const tryLoadActivatedBundledPluginPublicSurfaceModuleSync = vi.hoisted(() => vi.fn()); + +vi.mock("../plugins/manifest-registry.js", () => ({ + loadPluginManifestRegistry, +})); + +vi.mock("./facade-runtime.js", () => ({ + tryLoadActivatedBundledPluginPublicSurfaceModuleSync, +})); + +describe("plugin-sdk qa-runner-runtime", () => { + beforeEach(() => { + loadPluginManifestRegistry.mockReset().mockReturnValue({ + plugins: [], + diagnostics: [], + }); + tryLoadActivatedBundledPluginPublicSurfaceModuleSync.mockReset(); + }); + + it("stays cold until runner discovery is requested", async () => { + await import("./qa-runner-runtime.js"); + + expect(loadPluginManifestRegistry).not.toHaveBeenCalled(); + expect(tryLoadActivatedBundledPluginPublicSurfaceModuleSync).not.toHaveBeenCalled(); + }); + + it("returns activated runner registrations declared in plugin manifests", async () => { + const register = vi.fn((qa: Command) => qa); + loadPluginManifestRegistry.mockReturnValue({ + plugins: [ + { + id: "qa-matrix", + qaRunners: [ + { + commandName: "matrix", + description: "Run the Matrix live QA lane", + }, + ], + rootDir: "/tmp/qa-matrix", + }, + ], + diagnostics: [], + }); + tryLoadActivatedBundledPluginPublicSurfaceModuleSync.mockReturnValue({ + qaRunnerCliRegistrations: [{ commandName: "matrix", register }], + }); + + const module = await import("./qa-runner-runtime.js"); + + expect(module.listQaRunnerCliContributions()).toEqual([ + { + pluginId: "qa-matrix", + commandName: "matrix", + description: "Run the Matrix live QA lane", + status: "available", + registration: { + commandName: "matrix", + register, + }, + }, + ]); + expect(tryLoadActivatedBundledPluginPublicSurfaceModuleSync).toHaveBeenCalledWith({ + dirName: "qa-matrix", + artifactBasename: "runtime-api.js", + }); + }); + + it("reports declared runners as blocked when the plugin is present but not activated", async () => { + loadPluginManifestRegistry.mockReturnValue({ + plugins: [ + { + id: "qa-matrix", + qaRunners: [{ commandName: "matrix" }], + rootDir: "/tmp/qa-matrix", + }, + ], + diagnostics: [], + }); + tryLoadActivatedBundledPluginPublicSurfaceModuleSync.mockReturnValue(null); + + const module = await import("./qa-runner-runtime.js"); + + expect(module.listQaRunnerCliContributions()).toEqual([ + { + pluginId: "qa-matrix", + commandName: "matrix", + status: "blocked", + }, + ]); + }); + + it("fails fast when two plugins declare the same qa runner command", async () => { + loadPluginManifestRegistry.mockReturnValue({ + plugins: [ + { + id: "alpha", + qaRunners: [{ commandName: "matrix" }], + rootDir: "/tmp/alpha", + }, + { + id: "beta", + qaRunners: [{ commandName: "matrix" }], + rootDir: "/tmp/beta", + }, + ], + diagnostics: [], + }); + tryLoadActivatedBundledPluginPublicSurfaceModuleSync.mockReturnValue(null); + + const module = await import("./qa-runner-runtime.js"); + + expect(() => module.listQaRunnerCliContributions()).toThrow( + 'QA runner command "matrix" declared by both "alpha" and "beta"', + ); + }); +}); diff --git a/src/plugin-sdk/qa-runner-runtime.ts b/src/plugin-sdk/qa-runner-runtime.ts new file mode 100644 index 00000000000..6b7bf7769ce --- /dev/null +++ b/src/plugin-sdk/qa-runner-runtime.ts @@ -0,0 +1,124 @@ +import type { Command } from "commander"; +import type { PluginManifestRecord } from "../plugins/manifest-registry.js"; +import { loadPluginManifestRegistry } from "../plugins/manifest-registry.js"; +import { tryLoadActivatedBundledPluginPublicSurfaceModuleSync } from "./facade-runtime.js"; + +export type QaRunnerCliRegistration = { + commandName: string; + register(qa: Command): void; +}; + +type QaRunnerRuntimeSurface = { + listQaRunnerCliRegistrations?: () => readonly QaRunnerCliRegistration[]; + qaRunnerCliRegistrations?: readonly QaRunnerCliRegistration[]; +}; + +export type QaRunnerCliContribution = + | { + pluginId: string; + commandName: string; + description?: string; + status: "available"; + registration: QaRunnerCliRegistration; + } + | { + pluginId: string; + commandName: string; + description?: string; + status: "blocked"; + }; + +function listDeclaredQaRunnerPlugins(): Array< + Pick +> { + return loadPluginManifestRegistry({ cache: true }).plugins + .filter( + ( + plugin, + ): plugin is Pick & { + qaRunners: NonNullable; + } => Array.isArray(plugin.qaRunners) && plugin.qaRunners.length > 0, + ) + .toSorted((left, right) => { + const idCompare = left.id.localeCompare(right.id); + if (idCompare !== 0) { + return idCompare; + } + return left.rootDir.localeCompare(right.rootDir); + }); +} + +function listRuntimeRegistrations( + pluginId: string, + surface: QaRunnerRuntimeSurface, +): readonly QaRunnerCliRegistration[] { + const registrations = + surface.listQaRunnerCliRegistrations?.() ?? surface.qaRunnerCliRegistrations ?? []; + const seen = new Set(); + for (const registration of registrations) { + if (!registration?.commandName || typeof registration.register !== "function") { + throw new Error(`QA runner plugin "${pluginId}" exported an invalid CLI registration`); + } + if (seen.has(registration.commandName)) { + throw new Error( + `QA runner plugin "${pluginId}" exported duplicate CLI registration "${registration.commandName}"`, + ); + } + seen.add(registration.commandName); + } + return registrations; +} + +export function listQaRunnerCliContributions(): readonly QaRunnerCliContribution[] { + const contributions: QaRunnerCliContribution[] = []; + const seenCommandNames = new Map(); + + for (const plugin of listDeclaredQaRunnerPlugins()) { + const runtimeSurface = tryLoadActivatedBundledPluginPublicSurfaceModuleSync( + { + dirName: plugin.id, + artifactBasename: "runtime-api.js", + }, + ); + const runtimeRegistrations = runtimeSurface + ? listRuntimeRegistrations(plugin.id, runtimeSurface) + : null; + + for (const runner of plugin.qaRunners) { + const previousOwner = seenCommandNames.get(runner.commandName); + if (previousOwner) { + throw new Error( + `QA runner command "${runner.commandName}" declared by both "${previousOwner}" and "${plugin.id}"`, + ); + } + seenCommandNames.set(runner.commandName, plugin.id); + + const registration = runtimeRegistrations?.find( + (entry) => entry.commandName === runner.commandName, + ); + if (!runtimeSurface) { + contributions.push({ + pluginId: plugin.id, + commandName: runner.commandName, + ...(runner.description ? { description: runner.description } : {}), + status: "blocked", + }); + continue; + } + if (!registration) { + throw new Error( + `QA runner plugin "${plugin.id}" declared "${runner.commandName}" in openclaw.plugin.json but did not export a matching CLI registration`, + ); + } + contributions.push({ + pluginId: plugin.id, + commandName: runner.commandName, + ...(runner.description ? { description: runner.description } : {}), + status: "available", + registration, + }); + } + } + + return contributions; +} diff --git a/src/plugins/manifest-registry.test.ts b/src/plugins/manifest-registry.test.ts index a2497b94860..2b1d2c94af4 100644 --- a/src/plugins/manifest-registry.test.ts +++ b/src/plugins/manifest-registry.test.ts @@ -499,6 +499,33 @@ describe("loadPluginManifestRegistry", () => { }); }); + it("preserves qa runner descriptors from plugin manifests", () => { + const dir = makeTempDir(); + writeManifest(dir, { + id: "qa-matrix", + qaRunners: [ + { + commandName: "matrix", + description: "Run the Matrix live QA lane", + }, + ], + configSchema: { type: "object" }, + }); + + const registry = loadSingleCandidateRegistry({ + idHint: "qa-matrix", + rootDir: dir, + origin: "bundled", + }); + + expect(registry.plugins[0]?.qaRunners).toEqual([ + { + commandName: "matrix", + description: "Run the Matrix live QA lane", + }, + ]); + }); + it("preserves channel config metadata from plugin manifests", () => { const dir = makeTempDir(); writeManifest(dir, { diff --git a/src/plugins/manifest-registry.ts b/src/plugins/manifest-registry.ts index e46aa114154..c3bca9edd3c 100644 --- a/src/plugins/manifest-registry.ts +++ b/src/plugins/manifest-registry.ts @@ -34,6 +34,7 @@ import { type PluginManifestChannelConfig, type PluginManifestContracts, type PluginManifestModelSupport, + type PluginManifestQaRunner, type PluginManifestSetup, } from "./manifest.js"; import { checkMinHostVersion } from "./min-host-version.js"; @@ -92,6 +93,7 @@ export type PluginManifestRecord = { providerAuthChoices?: PluginManifest["providerAuthChoices"]; activation?: PluginManifestActivation; setup?: PluginManifestSetup; + qaRunners?: PluginManifestQaRunner[]; skills: string[]; settingsFiles?: string[]; hooks: string[]; @@ -333,6 +335,7 @@ function buildRecord(params: { providerAuthChoices: params.manifest.providerAuthChoices, activation: params.manifest.activation, setup: params.manifest.setup, + qaRunners: params.manifest.qaRunners, skills: params.manifest.skills ?? [], settingsFiles: [], hooks: [], diff --git a/src/plugins/manifest.ts b/src/plugins/manifest.ts index 3dc39b0d941..f05eb7e4e06 100644 --- a/src/plugins/manifest.ts +++ b/src/plugins/manifest.ts @@ -80,6 +80,13 @@ export type PluginManifestSetup = { requiresRuntime?: boolean; }; +export type PluginManifestQaRunner = { + /** Subcommand mounted beneath `openclaw qa`, for example `matrix`. */ + commandName: string; + /** Optional user-facing help text for fallback host stubs. */ + description?: string; +}; + export type PluginManifestConfigLiteral = string | number | boolean | null; export type PluginManifestDangerousConfigFlag = { @@ -174,6 +181,8 @@ export type PluginManifest = { activation?: PluginManifestActivation; /** Cheap setup/onboarding metadata exposed before plugin runtime loads. */ setup?: PluginManifestSetup; + /** Cheap QA runner metadata exposed before plugin runtime loads. */ + qaRunners?: PluginManifestQaRunner[]; skills?: string[]; name?: string; description?: string; @@ -484,6 +493,28 @@ function normalizeManifestSetup(value: unknown): PluginManifestSetup | undefined return Object.keys(setup).length > 0 ? setup : undefined; } +function normalizeManifestQaRunners(value: unknown): PluginManifestQaRunner[] | undefined { + if (!Array.isArray(value)) { + return undefined; + } + const normalized: PluginManifestQaRunner[] = []; + for (const entry of value) { + if (!isRecord(entry)) { + continue; + } + const commandName = normalizeOptionalString(entry.commandName) ?? ""; + if (!commandName) { + continue; + } + const description = normalizeOptionalString(entry.description) ?? ""; + normalized.push({ + commandName, + ...(description ? { description } : {}), + }); + } + return normalized.length > 0 ? normalized : undefined; +} + function normalizeProviderAuthChoices( value: unknown, ): PluginManifestProviderAuthChoice[] | undefined { @@ -673,6 +704,7 @@ export function loadPluginManifest( const providerAuthChoices = normalizeProviderAuthChoices(raw.providerAuthChoices); const activation = normalizeManifestActivation(raw.activation); const setup = normalizeManifestSetup(raw.setup); + const qaRunners = normalizeManifestQaRunners(raw.qaRunners); const skills = normalizeTrimmedStringList(raw.skills); const contracts = normalizeManifestContracts(raw.contracts); const configContracts = normalizeManifestConfigContracts(raw.configContracts); @@ -706,6 +738,7 @@ export function loadPluginManifest( providerAuthChoices, activation, setup, + qaRunners, skills, name, description,