// QA runner runtime helpers expose plugin QA scenarios through the CLI command surface. import type { Command } from "commander"; import type { PluginManifestRecord } from "../plugins/manifest-registry.js"; import { loadPluginManifestRegistry } from "../plugins/manifest-registry.js"; import type { OpenClawConfig } from "./config-contracts.js"; import { loadBundledPluginPublicSurfaceModuleSync, tryLoadActivatedBundledPluginPublicSurfaceModuleSync, } from "./facade-runtime.js"; import { resolvePrivateQaBundledPluginsEnv } from "./private-qa-bundled-env.js"; import type { QaBusEditMessageInput, QaBusInboundMessageInput, QaBusMessage, QaBusOutboundMessageInput, } from "./qa-channel-protocol.js"; type QaRunnerTransportPolicy = { requireGroupMention?: true; senderAllowlist?: readonly string[]; topLevelReplies?: true; }; type QaRunnerAdapterOptions = { repoRoot?: string; sutAccountId?: string; credentialSource?: string; credentialRole?: string; transportPolicy?: QaRunnerTransportPolicy; }; type QaRunnerMessageRecorder = { addInboundMessage: (input: QaBusInboundMessageInput) => QaBusMessage | Promise; addOutboundMessage: (input: QaBusOutboundMessageInput) => QaBusMessage | Promise; editMessage: (input: QaBusEditMessageInput) => QaBusMessage | Promise; }; type QaRunnerTransportAdapterDefinition = { id: string; label: string; accountId: string; requiredPluginIds: readonly string[]; supportedActions: readonly ("delete" | "edit" | "react" | "thread-create")[]; assertTransportHealthy?: () => void; resetTransport?: () => void | Promise; sendInbound: (input: QaBusInboundMessageInput) => Promise; sendNativeCommand?: ( input: Omit & { command: string }, ) => Promise; waitForOutboundSequence?: (input: { conversationId?: string; finalSettleMs?: number; finalTextIncludes: string; minimumPreviewEvents?: number; sinceCursor?: number; threadId?: string; timeoutMs?: number; }) => Promise<{ events: Array<{ cursor: number; kind: "sent" | "edited" | "deleted"; message: QaBusMessage }>; final: QaBusMessage; }>; createGatewayConfig: (params: { baseUrl: string; }) => Pick; waitReady: (params: { gateway: { call: ( method: string, params?: unknown, options?: { timeoutMs?: number }, ) => Promise; }; timeoutMs?: number; pollIntervalMs?: number; }) => Promise; buildAgentDelivery: (params: { target: string }) => { channel: string; to?: string; replyChannel: string; replyTo: string; }; createRuntimeEnvPatch?: () => NodeJS.ProcessEnv; handleAction: (params: { action: "delete" | "edit" | "react" | "thread-create"; args: Record; cfg: OpenClawConfig; accountId?: string | null; }) => Promise; createReportNotes: (params: { providerMode: "mock-openai" | "aimock" | "live-frontier"; primaryModel: string; alternateModel: string; fastMode: boolean; concurrency: number; isolatedWorkers?: boolean; }) => string[]; cleanup?: () => Promise; }; type QaRunnerTransportFactory = { id: string; scenarioIds?: readonly string[]; matches: (context: { channelId: string; driver: string }) => boolean; create: (context: { adapterOptions?: QaRunnerAdapterOptions; channelId: string; driver: string; messages: QaRunnerMessageRecorder; outputDir: string; }) => Promise; }; /** CLI registration and optional transport adapter factory exported by a QA runner plugin. */ export type QaRunnerCliRegistration = { commandName: string; adapterFactory?: QaRunnerTransportFactory; register(qa: Command): void; }; type QaRunnerRuntimeSurface = { qaRunnerCliRegistrations?: readonly QaRunnerCliRegistration[]; }; type QaRuntimeSurface = { defaultQaRuntimeModelForMode: ( mode: string, options?: { alternate?: boolean; preferredLiveModel?: string; }, ) => string; startQaLiveLaneGateway: (...args: unknown[]) => Promise; }; /** Resolved QA runner CLI contribution declared by plugin manifest metadata. */ export type QaRunnerCliContribution = | { pluginId: string; commandName: string; description?: string; status: "available"; registration: QaRunnerCliRegistration; } | { pluginId: string; commandName: string; description?: string; status: "blocked"; }; function isMissingQaRuntimeError(error: unknown) { if (!(error instanceof Error)) { return false; } return ( error.message.includes("qa-lab") && (error.message.includes("runtime-api.js") || error.message.startsWith("Unable to open bundled plugin public surface ")) ); } /** Load the private QA Lab runtime facade used by QA runner commands. */ export function loadQaRuntimeModule(): QaRuntimeSurface { const env = resolvePrivateQaBundledPluginsEnv(); return loadBundledPluginPublicSurfaceModuleSync({ dirName: ["qa", "lab"].join("-"), artifactBasename: ["runtime-api", "js"].join("."), ...(env ? { env } : {}), }); } /** Load a bundled QA runner plugin test API facade by plugin id. */ // oxlint-disable-next-line typescript/no-unnecessary-type-parameters -- QA runtime loader uses caller-supplied test API surface type. export function loadQaRunnerBundledPluginTestApi(pluginId: string): T { const env = resolvePrivateQaBundledPluginsEnv(); return loadBundledPluginPublicSurfaceModuleSync({ dirName: pluginId, artifactBasename: "test-api.js", ...(env ? { env } : {}), }); } /** Returns whether the private QA Lab runtime facade is available in this build. */ export function isQaRuntimeAvailable(): boolean { try { loadQaRuntimeModule(); return true; } catch (error) { if (isMissingQaRuntimeError(error)) { return false; } throw error; } } function listDeclaredQaRunnerPlugins( env: NodeJS.ProcessEnv | undefined = resolvePrivateQaBundledPluginsEnv(), ): Array< PluginManifestRecord & { qaRunners: NonNullable; } > { return loadPluginManifestRegistry(env ? { env } : {}) .plugins.filter( ( plugin, ): plugin is PluginManifestRecord & { qaRunners: NonNullable; } => Array.isArray(plugin.qaRunners) && plugin.qaRunners.length > 0, ) .toSorted((left, right) => { const idCompare = left.id.localeCompare(right.id); if (idCompare !== 0) { return idCompare; } return left.rootDir.localeCompare(right.rootDir); }); } function indexRuntimeRegistrations( pluginId: string, surface: QaRunnerRuntimeSurface, ): ReadonlyMap { const registrations = surface.qaRunnerCliRegistrations ?? []; const registrationByCommandName = new Map(); for (const registration of registrations) { if (!registration?.commandName || typeof registration.register !== "function") { throw new Error(`QA runner plugin "${pluginId}" exported an invalid CLI registration`); } if (registrationByCommandName.has(registration.commandName)) { throw new Error( `QA runner plugin "${pluginId}" exported duplicate CLI registration "${registration.commandName}"`, ); } registrationByCommandName.set(registration.commandName, registration); } return registrationByCommandName; } function loadQaRunnerRuntimeSurface( plugin: PluginManifestRecord, env?: NodeJS.ProcessEnv, ): QaRunnerRuntimeSurface | null { if (plugin.origin === "bundled") { return loadBundledPluginPublicSurfaceModuleSync({ dirName: plugin.id, artifactBasename: "runtime-api.js", ...(env ? { env } : {}), }); } return tryLoadActivatedBundledPluginPublicSurfaceModuleSync({ dirName: plugin.id, artifactBasename: "runtime-api.js", ...(env ? { env } : {}), }); } /** List QA runner CLI contributions declared by manifests and backed by runtime registrations. */ export function listQaRunnerCliContributions(): readonly QaRunnerCliContribution[] { const env = resolvePrivateQaBundledPluginsEnv(); const contributions = new Map(); for (const plugin of listDeclaredQaRunnerPlugins(env)) { const runtimeSurface = loadQaRunnerRuntimeSurface(plugin, env); const runtimeRegistrationByCommandName = runtimeSurface ? indexRuntimeRegistrations(plugin.id, runtimeSurface) : null; const declaredCommandNames = new Set(plugin.qaRunners.map((runner) => runner.commandName)); for (const runner of plugin.qaRunners) { const previous = contributions.get(runner.commandName); if (previous && previous.pluginId !== plugin.id) { throw new Error( `QA runner command "${runner.commandName}" declared by both "${previous.pluginId}" and "${plugin.id}"`, ); } const registration = runtimeRegistrationByCommandName?.get(runner.commandName); if (!runtimeSurface) { contributions.set(runner.commandName, { pluginId: plugin.id, commandName: runner.commandName, ...(runner.description ? { description: runner.description } : {}), status: "blocked", }); continue; } if (!registration) { throw new Error( `QA runner plugin "${plugin.id}" declared "${runner.commandName}" in openclaw.plugin.json but did not export a matching CLI registration`, ); } const adapterFactory = registration.adapterFactory; if ( adapterFactory && (adapterFactory.id !== runner.commandName || typeof adapterFactory.matches !== "function" || typeof adapterFactory.create !== "function") ) { throw new Error( `QA runner plugin "${plugin.id}" exported an invalid transport factory for "${runner.commandName}"`, ); } contributions.set(runner.commandName, { pluginId: plugin.id, commandName: runner.commandName, ...(runner.description ? { description: runner.description } : {}), status: "available", registration, }); } for (const commandName of runtimeRegistrationByCommandName?.keys() ?? []) { if (!declaredCommandNames.has(commandName)) { throw new Error( `QA runner plugin "${plugin.id}" exported "${commandName}" from runtime-api.js but did not declare it in openclaw.plugin.json`, ); } } } return [...contributions.values()]; }