vultr/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-05-06 07:00:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

QA: simplify runner registration seams

Browse Source
This commit is contained in:
Gustavo Madeira Santana
2026-04-14 13:39:42 -04:00
parent d969c4f57e
commit eb64a8a60d
12 changed files with 152 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
docs/help/testing.md
View File

@@ -73,7 +73,7 @@ These commands sit beside the main test suites when you need QA-lab realism:
`openclaw plugins install -l ./extensions/qa-matrix`.
- Provisions three temporary Matrix users (`driver`, `sut`, `observer`) plus one private room, then starts a QA gateway child with the real Matrix plugin as the SUT transport.
- Uses the pinned stable Tuwunel image `ghcr.io/matrix-construct/tuwunel:v1.5.1` by default. Override with `OPENCLAW_QA_MATRIX_TUWUNEL_IMAGE` when you need to test a different image.
- Matrix currently supports only `--credential-source env` because the lane provisions disposable users locally.
- Matrix does not expose shared credential-source flags because the lane provisions disposable users locally.
- Writes a Matrix QA report, summary, and observed-events artifact under `.artifacts/qa-e2e/...`.
- `pnpm openclaw qa telegram`
- Runs the Telegram live QA lane against a real private group using the driver and SUT bot tokens from env.
@@ -204,7 +204,8 @@ The minimum adoption bar for a new channel is:
2. Implement the transport runner on the shared `qa-lab` host seam.
3. Keep transport-specific mechanics inside the runner plugin or channel harness.
4. Mount the runner as `openclaw qa <runner>` instead of registering a competing root command.
Runner plugins should declare `qaRunners` in `openclaw.plugin.json` and export matching `qaRunnerCliRegistrations` from `runtime-api.ts`.
Runner plugins should declare `qaRunners` in `openclaw.plugin.json` and export a matching `qaRunnerCliRegistrations` array from `runtime-api.ts`.
Keep `runtime-api.ts` light; lazy CLI and runner execution should stay behind separate entrypoints.
5. Author or adapt markdown scenarios under `qa/scenarios/`.
6. Use the generic scenario helpers for new scenarios.
7. Keep existing compatibility aliases working unless the repo is doing an intentional migration.

11
docs/plugins/manifest.md
View File

@@ -160,7 +160,7 @@ Those belong in your plugin code and `package.json`.
| `providerAuthChoices` | No | `object[]` | Cheap auth-choice metadata for onboarding pickers, preferred-provider resolution, and simple CLI flag wiring. |
| `activation` | No | `object` | Cheap activation hints for provider, command, channel, route, and capability-triggered loading. Metadata only; plugin runtime still owns actual behavior. |
| `setup` | No | `object` | Cheap setup/onboarding descriptors that discovery and setup surfaces can inspect without loading plugin runtime. |
| `qaRunners` | No | `object[]` | Cheap QA runner descriptors used by the shared `openclaw qa` host before plugin runtime loads. |
| `qaRunners` | No | `object[]` | Cheap QA runner descriptors used by the shared `openclaw qa` host before plugin runtime loads. |
| `contracts` | No | `object` | Static bundled capability snapshot for speech, realtime transcription, realtime voice, media-understanding, image-generation, music-generation, video-generation, web-fetch, web search, and tool ownership. |
| `channelConfigs` | No | `Record<string, object>` | Manifest-owned channel config metadata merged into discovery and validation surfaces before runtime loads. |
| `skills` | No | `string[]` | Skill directories to load, relative to the plugin root. |
@@ -226,7 +226,8 @@ should activate it later.
Use `qaRunners` when a plugin contributes one or more transport runners beneath
the shared `openclaw qa` root. Keep this metadata cheap and static; the plugin
runtime still owns actual CLI registration through `runtime-api.ts`.
runtime still owns actual CLI registration through a lightweight
`runtime-api.ts` surface that exports `qaRunnerCliRegistrations`.
```json
{
@@ -239,9 +240,9 @@ runtime still owns actual CLI registration through `runtime-api.ts`.
}
```
| Field | Required | Type | What it means |
| ------------- | -------- | -------- | ----------------------------------------------------------------- |
| `commandName` | Yes | `string` | Subcommand mounted beneath `openclaw qa`, for example `matrix`. |
| Field | Required | Type | What it means |
| ------------- | -------- | -------- | ------------------------------------------------------------------ |
| `commandName` | Yes | `string` | Subcommand mounted beneath `openclaw qa`, for example `matrix`. |
| `description` | No | `string` | Fallback help text used when the shared host needs a stub command. |
This block is metadata only. It does not register runtime behavior, and it does

10
extensions/qa-lab/src/cli.test.ts
View File

@@ -86,6 +86,16 @@ describe("qa cli registration", () => {
expect(registration.register).toHaveBeenCalledTimes(1);
});
it("keeps Telegram credential flags on the shared host CLI", () => {
const qa = program.commands.find((command) => command.name() === "qa");
const telegram = qa?.commands.find((command) => command.name() === "telegram");
const optionNames = telegram?.options.map((option) => option.long) ?? [];
expect(optionNames).toEqual(
expect.arrayContaining(["--credential-source", "--credential-role"]),
);
});
it("shows an install hint when a discovered runner plugin is unavailable", async () => {
listQaRunnerCliContributions.mockReset().mockReturnValue([
{

40
extensions/qa-lab/src/live-transports/cli.ts
View File

@@ -40,6 +40,28 @@ function createBlockedQaRunnerCliRegistration(params: {
};
}
function createQaRunnerCliRegistration(
runner: ReturnType<typeof listQaRunnerCliContributions>[number],
): LiveTransportQaCliRegistration {
if (runner.status === "available") {
return runner.registration;
}
if (runner.status === "blocked") {
return createBlockedQaRunnerCliRegistration({
commandName: runner.commandName,
description: runner.description,
pluginId: runner.pluginId,
});
}
return createMissingQaRunnerCliRegistration({
commandName: runner.commandName,
description:
runner.description ??
`Run the ${runner.commandName} live QA lane (install ${runner.npmSpec} first)`,
npmSpec: runner.npmSpec,
});
}
export const LIVE_TRANSPORT_QA_CLI_REGISTRATIONS: readonly LiveTransportQaCliRegistration[] = [
telegramQaCliRegistration,
];
@@ -49,23 +71,7 @@ export function listLiveTransportQaCliRegistrations(): readonly LiveTransportQaC
const discoveredRunners = listQaRunnerCliContributions();
for (const runner of discoveredRunners) {
liveRegistrations.push(
runner.status === "available"
? runner.registration
: runner.status === "blocked"
? createBlockedQaRunnerCliRegistration({
commandName: runner.commandName,
description: runner.description,
pluginId: runner.pluginId,
})
: createMissingQaRunnerCliRegistration({
commandName: runner.commandName,
description:
runner.description ??
`Run the ${runner.commandName} live QA lane (install ${runner.npmSpec} first)`,
npmSpec: runner.npmSpec,
}),
);
liveRegistrations.push(createQaRunnerCliRegistration(runner));
}
return liveRegistrations;

36
extensions/qa-lab/src/live-transports/shared/live-transport-cli.ts
View File

@@ -33,6 +33,11 @@ export type LiveTransportQaCliRegistration = {
register(qa: Command): void;
};
export type LiveTransportQaCredentialCliOptions = {
sourceDescription?: string;
roleDescription?: string;
};
export function createLazyCliRuntimeLoader<T>(load: () => Promise<T>) {
let promise: Promise<T> | null = null;
return async () => {
@@ -61,13 +66,14 @@ export function mapLiveTransportQaCommanderOptions(
export function registerLiveTransportQaCli(params: {
qa: Command;
commandName: string;
credentialOptions?: LiveTransportQaCredentialCliOptions;
description: string;
outputDirHelp: string;
scenarioHelp: string;
sutAccountHelp: string;
run: (opts: LiveTransportQaCommandOptions) => Promise<void>;
}) {
params.qa
const command = params.qa
.command(params.commandName)
.description(params.description)
.option("--repo-root <path>", "Repository root to target when running from a neutral cwd")
@@ -81,22 +87,27 @@ export function registerLiveTransportQaCli(params: {
.option("--alt-model <ref>", "Alternate provider/model ref")
.option("--scenario <id>", params.scenarioHelp, collectString, [])
.option("--fast", "Enable provider fast mode where supported", false)
.option("--sut-account <id>", params.sutAccountHelp, "sut")
.option(
.option("--sut-account <id>", params.sutAccountHelp, "sut");
if (params.credentialOptions) {
command.option(
"--credential-source <source>",
"Credential source for live lanes: env or convex (default: env)",
)
.option(
"--credential-role <role>",
"Credential role for convex auth: maintainer or ci (default: maintainer)",
)
.action(async (opts: LiveTransportQaCommanderOptions) => {
await params.run(mapLiveTransportQaCommanderOptions(opts));
});
params.credentialOptions.sourceDescription ??
"Credential source for live lanes: env or convex (default: env)",
);
if (params.credentialOptions.roleDescription) {
command.option("--credential-role <role>", params.credentialOptions.roleDescription);
}
}
command.action(async (opts: LiveTransportQaCommanderOptions) => {
await params.run(mapLiveTransportQaCommanderOptions(opts));
});
}
export function createLiveTransportQaCliRegistration(params: {
commandName: string;
credentialOptions?: LiveTransportQaCredentialCliOptions;
description: string;
outputDirHelp: string;
scenarioHelp: string;
@@ -109,6 +120,7 @@ export function createLiveTransportQaCliRegistration(params: {
registerLiveTransportQaCli({
qa,
commandName: params.commandName,
credentialOptions: params.credentialOptions,
description: params.description,
outputDirHelp: params.outputDirHelp,
scenarioHelp: params.scenarioHelp,

4
extensions/qa-lab/src/live-transports/telegram/cli.ts
View File

@@ -20,6 +20,10 @@ async function runQaTelegram(opts: LiveTransportQaCommandOptions) {
export const telegramQaCliRegistration: LiveTransportQaCliRegistration =
createLiveTransportQaCliRegistration({
commandName: "telegram",
credentialOptions: {
sourceDescription: "Credential source for Telegram QA: env or convex (default: env)",
roleDescription: "Credential role for convex auth: maintainer or ci (default: maintainer)",
},
description: "Run the manual Telegram live QA lane against a private bot-to-bot group harness",
outputDirHelp: "Telegram QA artifact directory",
scenarioHelp: "Run only the named Telegram QA scenario (repeatable)",

2
extensions/qa-matrix/index.ts
View File

@@ -1,4 +1,4 @@
import { definePluginEntry } from "./runtime-api.js";
import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";
export default definePluginEntry({
id: "qa-matrix",

5
extensions/qa-matrix/runtime-api.ts
View File

@@ -1,4 +1 @@
export { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";
export { qaRunnerCliRegistrations, registerMatrixQaCli } from "./cli.js";
export { runQaMatrixCommand } from "./cli.runtime.js";
export { runMatrixQaLive } from "./runtime.js";
export { qaRunnerCliRegistrations } from "./cli.js";

29
extensions/qa-matrix/src/cli.test.ts Normal file
View File

@@ -0,0 +1,29 @@
import { Command } from "commander";
import { describe, expect, it } from "vitest";
import { matrixQaCliRegistration } from "./cli.js";
describe("matrix qa cli registration", () => {
it("keeps disposable Matrix lane flags focused", () => {
const qa = new Command();
matrixQaCliRegistration.register(qa);
const matrix = qa.commands.find((command) => command.name() === "matrix");
const optionNames = matrix?.options.map((option) => option.long) ?? [];
expect(optionNames).toEqual(
expect.arrayContaining([
"--repo-root",
"--output-dir",
"--provider-mode",
"--model",
"--alt-model",
"--scenario",
"--fast",
"--sut-account",
]),
);
expect(optionNames).not.toContain("--credential-source");
expect(optionNames).not.toContain("--credential-role");
});
});

9
extensions/qa-matrix/src/runtime-api.test.ts Normal file
View File

@@ -0,0 +1,9 @@
import { describe, expect, it } from "vitest";
describe("matrix qa runtime api surface", () => {
it("keeps runner discovery lightweight", async () => {
const runtimeApi = await import("../runtime-api.js");
expect(Object.keys(runtimeApi).toSorted()).toEqual(["qaRunnerCliRegistrations"]);
});
});

36
extensions/qa-matrix/src/shared/live-transport-cli.ts
View File

@@ -33,6 +33,11 @@ export type LiveTransportQaCliRegistration = {
register(qa: Command): void;
};
export type LiveTransportQaCredentialCliOptions = {
sourceDescription?: string;
roleDescription?: string;
};
export function createLazyCliRuntimeLoader<T>(load: () => Promise<T>) {
let promise: Promise<T> | null = null;
return async () => {
@@ -61,13 +66,14 @@ export function mapLiveTransportQaCommanderOptions(
export function registerLiveTransportQaCli(params: {
qa: Command;
commandName: string;
credentialOptions?: LiveTransportQaCredentialCliOptions;
description: string;
outputDirHelp: string;
scenarioHelp: string;
sutAccountHelp: string;
run: (opts: LiveTransportQaCommandOptions) => Promise<void>;
}) {
params.qa
const command = params.qa
.command(params.commandName)
.description(params.description)
.option("--repo-root <path>", "Repository root to target when running from a neutral cwd")
@@ -81,22 +87,27 @@ export function registerLiveTransportQaCli(params: {
.option("--alt-model <ref>", "Alternate provider/model ref")
.option("--scenario <id>", params.scenarioHelp, collectString, [])
.option("--fast", "Enable provider fast mode where supported", false)
.option("--sut-account <id>", params.sutAccountHelp, "sut")
.option(
.option("--sut-account <id>", params.sutAccountHelp, "sut");
if (params.credentialOptions) {
command.option(
"--credential-source <source>",
"Credential source for live lanes: env or convex (default: env)",
)
.option(
"--credential-role <role>",
"Credential role for convex auth: maintainer or ci (default: maintainer)",
)
.action(async (opts: LiveTransportQaCommanderOptions) => {
await params.run(mapLiveTransportQaCommanderOptions(opts));
});
params.credentialOptions.sourceDescription ??
"Credential source for live lanes: env or convex (default: env)",
);
if (params.credentialOptions.roleDescription) {
command.option("--credential-role <role>", params.credentialOptions.roleDescription);
}
}
command.action(async (opts: LiveTransportQaCommanderOptions) => {
await params.run(mapLiveTransportQaCommanderOptions(opts));
});
}
export function createLiveTransportQaCliRegistration(params: {
commandName: string;
credentialOptions?: LiveTransportQaCredentialCliOptions;
description: string;
outputDirHelp: string;
scenarioHelp: string;
@@ -109,6 +120,7 @@ export function createLiveTransportQaCliRegistration(params: {
registerLiveTransportQaCli({
qa,
commandName: params.commandName,
credentialOptions: params.credentialOptions,
description: params.description,
outputDirHelp: params.outputDirHelp,
scenarioHelp: params.scenarioHelp,

41
src/plugin-sdk/qa-runner-runtime.ts
View File

@@ -10,7 +10,6 @@ export type QaRunnerCliRegistration = {
};
type QaRunnerRuntimeSurface = {
listQaRunnerCliRegistrations?: () => readonly QaRunnerCliRegistration[];
qaRunnerCliRegistrations?: readonly QaRunnerCliRegistration[];
};
@@ -39,8 +38,8 @@ export type QaRunnerCliContribution =
function listDeclaredQaRunnerPlugins(): Array<
Pick<PluginManifestRecord, "id" | "qaRunners" | "rootDir">
> {
return loadPluginManifestRegistry({ cache: true }).plugins
.filter(
return loadPluginManifestRegistry({ cache: true })
.plugins.filter(
(
plugin,
): plugin is Pick<PluginManifestRecord, "id" | "qaRunners" | "rootDir"> & {
@@ -56,25 +55,24 @@ function listDeclaredQaRunnerPlugins(): Array<
});
}
function listRuntimeRegistrations(
function indexRuntimeRegistrations(
pluginId: string,
surface: QaRunnerRuntimeSurface,
): readonly QaRunnerCliRegistration[] {
const registrations =
surface.listQaRunnerCliRegistrations?.() ?? surface.qaRunnerCliRegistrations ?? [];
const seen = new Set<string>();
): ReadonlyMap<string, QaRunnerCliRegistration> {
const registrations = surface.qaRunnerCliRegistrations ?? [];
const registrationByCommandName = new Map<string, QaRunnerCliRegistration>();
for (const registration of registrations) {
if (!registration?.commandName || typeof registration.register !== "function") {
throw new Error(`QA runner plugin "${pluginId}" exported an invalid CLI registration`);
}
if (seen.has(registration.commandName)) {
if (registrationByCommandName.has(registration.commandName)) {
throw new Error(
`QA runner plugin "${pluginId}" exported duplicate CLI registration "${registration.commandName}"`,
);
}
seen.add(registration.commandName);
registrationByCommandName.set(registration.commandName, registration);
}
return registrations;
return registrationByCommandName;
}
function buildKnownQaRunnerCatalog(): readonly QaRunnerCliContribution[] {
@@ -106,14 +104,13 @@ export function listQaRunnerCliContributions(): readonly QaRunnerCliContribution
}
for (const plugin of listDeclaredQaRunnerPlugins()) {
const runtimeSurface = tryLoadActivatedBundledPluginPublicSurfaceModuleSync<QaRunnerRuntimeSurface>(
{
const runtimeSurface =
tryLoadActivatedBundledPluginPublicSurfaceModuleSync<QaRunnerRuntimeSurface>({
dirName: plugin.id,
artifactBasename: "runtime-api.js",
},
);
const runtimeRegistrations = runtimeSurface
? listRuntimeRegistrations(plugin.id, runtimeSurface)
});
const runtimeRegistrationByCommandName = runtimeSurface
? indexRuntimeRegistrations(plugin.id, runtimeSurface)
: null;
const declaredCommandNames = new Set(plugin.qaRunners.map((runner) => runner.commandName));
@@ -125,9 +122,7 @@ export function listQaRunnerCliContributions(): readonly QaRunnerCliContribution
);
}
const registration = runtimeRegistrations?.find(
(entry) => entry.commandName === runner.commandName,
);
const registration = runtimeRegistrationByCommandName?.get(runner.commandName);
if (!runtimeSurface) {
contributions.set(runner.commandName, {
pluginId: plugin.id,
@@ -151,10 +146,10 @@ export function listQaRunnerCliContributions(): readonly QaRunnerCliContribution
});
}
for (const registration of runtimeRegistrations ?? []) {
if (!declaredCommandNames.has(registration.commandName)) {
for (const commandName of runtimeRegistrationByCommandName?.keys() ?? []) {
if (!declaredCommandNames.has(commandName)) {
throw new Error(
`QA runner plugin "${plugin.id}" exported "${registration.commandName}" from runtime-api.js but did not declare it in openclaw.plugin.json`,
`QA runner plugin "${plugin.id}" exported "${commandName}" from runtime-api.js but did not declare it in openclaw.plugin.json`,
);
}
}