diff --git a/extensions/qa-lab/src/cli.runtime.test.ts b/extensions/qa-lab/src/cli.runtime.test.ts index 4753a7ed824..e87ed9e2fbe 100644 --- a/extensions/qa-lab/src/cli.runtime.test.ts +++ b/extensions/qa-lab/src/cli.runtime.test.ts @@ -206,6 +206,27 @@ describe("qa cli runtime", () => { }); }); + it("passes explicit suite plugin enablements into the host gateway run", async () => { + await runQaSuiteCommand({ + repoRoot: "/tmp/openclaw-repo", + providerMode: "mock-openai", + scenarioIds: ["channel-chat-baseline"], + enabledPluginIds: ["browser", "memory-core"], + }); + + expect(runQaSuiteFromRuntime).toHaveBeenCalledWith({ + repoRoot: path.resolve("/tmp/openclaw-repo"), + outputDir: undefined, + transportId: "qa-channel", + providerMode: "mock-openai", + primaryModel: undefined, + alternateModel: undefined, + fastMode: undefined, + scenarioIds: ["channel-chat-baseline"], + enabledPluginIds: ["browser", "memory-core"], + }); + }); + it("drops blank suite model refs so provider defaults apply", async () => { await runQaSuiteCommand({ repoRoot: "/tmp/openclaw-repo", diff --git a/extensions/qa-lab/src/cli.runtime.ts b/extensions/qa-lab/src/cli.runtime.ts index 9d695e1239f..7dc60fb9a23 100644 --- a/extensions/qa-lab/src/cli.runtime.ts +++ b/extensions/qa-lab/src/cli.runtime.ts @@ -474,6 +474,7 @@ export async function runQaSuiteCommand(opts: { scenarioIds?: string[]; concurrency?: number; allowFailures?: boolean; + enabledPluginIds?: string[]; image?: string; cpus?: number; memory?: string; @@ -567,6 +568,7 @@ export async function runQaSuiteCommand(opts: { ...(thinkingDefault ? { thinkingDefault } : {}), ...(claudeCliAuthMode ? { claudeCliAuthMode } : {}), scenarioIds, + ...(opts.enabledPluginIds !== undefined ? { enabledPluginIds: opts.enabledPluginIds } : {}), ...(opts.concurrency !== undefined ? { concurrency: parseQaPositiveIntegerOption("--concurrency", opts.concurrency) } : {}), diff --git a/extensions/qa-lab/src/cli.ts b/extensions/qa-lab/src/cli.ts index ba60648a288..582cd39bf82 100644 --- a/extensions/qa-lab/src/cli.ts +++ b/extensions/qa-lab/src/cli.ts @@ -37,6 +37,7 @@ async function runQaSuite(opts: { fastMode?: boolean; thinking?: string; allowFailures?: boolean; + enabledPluginIds?: string[]; cliAuthMode?: string; parityPack?: string; scenarioIds?: string[]; @@ -248,6 +249,12 @@ export function registerQaLabCli(program: Command) { ) .option("--parity-pack ", 'Preset scenario pack; currently only "agentic" is supported') .option("--scenario ", "Run only the named QA scenario (repeatable)", collectString, []) + .option( + "--enable-plugin ", + "Enable an extra bundled plugin in the QA gateway config (repeatable)", + collectString, + [], + ) .option("--concurrency ", "Scenario worker concurrency", (value: string) => Number(value), ) @@ -278,6 +285,7 @@ export function registerQaLabCli(program: Command) { cliAuthMode?: string; parityPack?: string; scenario?: string[]; + enablePlugin?: string[]; concurrency?: number; allowFailures?: boolean; fast?: boolean; @@ -301,6 +309,7 @@ export function registerQaLabCli(program: Command) { cliAuthMode: opts.cliAuthMode, parityPack: opts.parityPack, scenarioIds: opts.scenario, + enabledPluginIds: opts.enablePlugin, concurrency: opts.concurrency, allowFailures: opts.allowFailures, image: opts.image, diff --git a/extensions/qa-lab/src/suite.ts b/extensions/qa-lab/src/suite.ts index bab656bcea5..784878ad035 100644 --- a/extensions/qa-lab/src/suite.ts +++ b/extensions/qa-lab/src/suite.ts @@ -83,6 +83,7 @@ export type QaSuiteRunParams = { lab?: QaLabServerHandle; startLab?: QaSuiteStartLabFn; concurrency?: number; + enabledPluginIds?: string[]; controlUiEnabled?: boolean; transportReadyTimeoutMs?: number; }; @@ -433,7 +434,12 @@ export async function runQaSuite(params?: QaSuiteRunParams): Promise pluginId.trim()).filter(Boolean), + ]), + ]; const gatewayConfigPatch = collectQaSuiteGatewayConfigPatch(selectedCatalogScenarios); const gatewayRuntimeOptions = collectQaSuiteGatewayRuntimeOptions(selectedCatalogScenarios); const concurrency = normalizeQaSuiteConcurrency( @@ -553,6 +559,7 @@ export async function runQaSuite(params?: QaSuiteRunParams): Promise { + const value = argv[index + 1]; + if (!value) { + throw new Error(`Missing value for ${arg}`); + } + index += 1; + return value; + }; + switch (arg) { + case "--": + break; + case "--repo-root": + options.repoRoot = path.resolve(readValue()); + break; + case "--output-dir": + options.outputDir = path.resolve(readValue()); + break; + case "--plugin": + options.pluginIds.push(readValue()); + break; + case "--shard-total": + options.shardTotal = parsePositiveInt(readValue(), "--shard-total"); + break; + case "--shard-index": + options.shardIndex = parseNonNegativeInt(readValue(), "--shard-index"); + break; + case "--limit": + options.limit = parsePositiveInt(readValue(), "--limit"); + break; + case "--qa-scenario": + options.qaScenarios.push(readValue()); + break; + case "--qa-plugin-chunk-size": + options.qaPluginChunkSize = parsePositiveInt(readValue(), "--qa-plugin-chunk-size"); + break; + case "--cpu-core-warn": + options.cpuCoreWarn = parsePositiveNumber(readValue(), "--cpu-core-warn"); + break; + case "--hot-wall-warn-ms": + options.hotWallWarnMs = parsePositiveInt(readValue(), "--hot-wall-warn-ms"); + break; + case "--max-rss-warn-mb": + options.maxRssWarnMb = parsePositiveNumber(readValue(), "--max-rss-warn-mb"); + break; + case "--wall-anomaly-multiplier": + options.wallAnomalyMultiplier = parsePositiveNumber( + readValue(), + "--wall-anomaly-multiplier", + ); + break; + case "--rss-anomaly-multiplier": + options.rssAnomalyMultiplier = parsePositiveNumber(readValue(), "--rss-anomaly-multiplier"); + break; + case "--command-timeout-ms": + options.commandTimeoutMs = parsePositiveInt(readValue(), "--command-timeout-ms"); + break; + case "--build-timeout-ms": + options.buildTimeoutMs = parsePositiveInt(readValue(), "--build-timeout-ms"); + break; + case "--qa-timeout-ms": + options.qaTimeoutMs = parsePositiveInt(readValue(), "--qa-timeout-ms"); + break; + case "--skip-prebuild": + options.skipPrebuild = true; + break; + case "--skip-lifecycle": + options.skipLifecycle = true; + break; + case "--skip-qa": + options.skipQa = true; + break; + case "--skip-slash-help": + options.skipSlashHelp = true; + break; + case "--help": + printHelp(); + process.exit(0); + break; + default: + throw new Error(`Unknown argument: ${arg}`); + } + } + if (options.qaScenarios.length === 0) { + options.qaScenarios = [...DEFAULT_QA_SCENARIOS]; + } + return options; +} + +function printHelp() { + console.log(`Usage: pnpm test:plugins:gateway-gauntlet [options] + +Runs a shardable bundled-plugin lifecycle, slash inventory, and QA gateway perf gauntlet. + +Options: + --plugin Plugin id to include, repeatable + --shard-total Total plugin shards (default: env or 1) + --shard-index Zero-based shard index (default: env or 0) + --limit Limit selected plugins after sharding + --output-dir Artifact directory + --qa-scenario QA Lab scenario id, repeatable + --qa-plugin-chunk-size Plugins enabled per QA run (default: 12) + --cpu-core-warn Hot CPU threshold (default: 0.9) + --hot-wall-warn-ms Minimum wall time for hot CPU observations (default: 30000) + --max-rss-warn-mb Maximum RSS warning threshold (default: 1536) + --skip-prebuild Skip the upfront build used to avoid per-command rebuild noise + --skip-lifecycle Skip plugin install/inspect/disable/enable/doctor/uninstall + --skip-qa Skip QA Lab RPC conversation runs + --skip-slash-help Skip CLI help probes for plugin-declared command aliases +`); +} + +function normalizeCsv(raw) { + return raw + ? raw + .split(",") + .map((entry) => entry.trim()) + .filter((entry) => entry.length > 0) + : []; +} + +function readOptionalPositiveIntEnv(name) { + const raw = process.env[name]; + return raw ? parsePositiveInt(raw, name) : undefined; +} + +function readOptionalNonNegativeIntEnv(name) { + const raw = process.env[name]; + return raw ? parseNonNegativeInt(raw, name) : undefined; +} + +function parsePositiveInt(raw, label) { + const value = Number(raw); + if (!Number.isInteger(value) || value < 1) { + throw new Error(`${label} must be a positive integer`); + } + return value; +} + +function parseNonNegativeInt(raw, label) { + const value = Number(raw); + if (!Number.isInteger(value) || value < 0) { + throw new Error(`${label} must be a non-negative integer`); + } + return value; +} + +function parsePositiveNumber(raw, label) { + const value = Number(raw); + if (!Number.isFinite(value) || value <= 0) { + throw new Error(`${label} must be a positive number`); + } + return value; +} + +function pnpmCommand() { + return process.platform === "win32" ? "pnpm.cmd" : "pnpm"; +} + +function openclawCommand(repoRoot, args) { + return { + command: process.execPath, + args: [path.join(repoRoot, "scripts", "run-node.mjs"), ...args], + }; +} + +function chunkArray(values, chunkSize) { + const chunks = []; + for (let index = 0; index < values.length; index += chunkSize) { + chunks.push(values.slice(index, index + chunkSize)); + } + return chunks; +} + +function toRepoRelativePath(repoRoot, absolutePath) { + const relativePath = path.relative(repoRoot, absolutePath); + if (!relativePath || relativePath.startsWith("..") || path.isAbsolute(relativePath)) { + throw new Error(`Output path must stay inside repo root: ${absolutePath}`); + } + return relativePath; +} + +function createIsolatedEnv(repoRoot, runRoot) { + const home = path.join(runRoot, "home"); + const stateDir = path.join(runRoot, "state"); + fs.mkdirSync(home, { recursive: true }); + fs.mkdirSync(stateDir, { recursive: true }); + return { + ...process.env, + HOME: home, + XDG_CONFIG_HOME: path.join(home, ".config"), + XDG_CACHE_HOME: path.join(home, ".cache"), + XDG_DATA_HOME: path.join(home, ".local", "share"), + OPENCLAW_STATE_DIR: stateDir, + OPENCLAW_CONFIG_PATH: path.join(stateDir, "openclaw.json"), + OPENCLAW_LOG_DIR: path.join(runRoot, "logs"), + OPENCLAW_QA_SUITE_PROGRESS: process.env.OPENCLAW_QA_SUITE_PROGRESS ?? "1", + PATH: process.env.PATH, + PWD: repoRoot, + }; +} + +function hasUsrBinTime() { + return fs.existsSync("/usr/bin/time"); +} + +function timeWrapperArgs(command, args) { + if (!hasUsrBinTime()) { + return { command, args, mode: "none" }; + } + if (process.platform === "darwin") { + return { command: "/usr/bin/time", args: ["-l", command, ...args], mode: "bsd" }; + } + return { command: "/usr/bin/time", args: ["-v", command, ...args], mode: "gnu" }; +} + +function parseTimedMetrics(stderr, wallMs, mode) { + let userSeconds = null; + let systemSeconds = null; + let maxRssMb = null; + if (mode === "gnu") { + userSeconds = parseFirstFloat(stderr, /User time \(seconds\):\s*([0-9.]+)/u); + systemSeconds = parseFirstFloat(stderr, /System time \(seconds\):\s*([0-9.]+)/u); + const maxRssKb = parseFirstFloat(stderr, /Maximum resident set size \(kbytes\):\s*([0-9.]+)/u); + maxRssMb = maxRssKb == null ? null : maxRssKb / 1024; + } else if (mode === "bsd") { + userSeconds = parseFirstFloat(stderr, /[0-9.]+\s+real\s+([0-9.]+)\s+user/u); + systemSeconds = parseFirstFloat(stderr, /([0-9.]+)\s+sys/u); + const maxRssBytes = parseFirstFloat(stderr, /([0-9]+)\s+maximum resident set size/u); + maxRssMb = maxRssBytes == null ? null : maxRssBytes / 1024 / 1024; + } + const cpuMs = + userSeconds == null && systemSeconds == null + ? null + : ((userSeconds ?? 0) + (systemSeconds ?? 0)) * 1000; + return { + wallMs, + cpuMs, + cpuCoreRatio: cpuMs == null || wallMs <= 0 ? null : cpuMs / wallMs, + maxRssMb, + }; +} + +function parseFirstFloat(value, pattern) { + const match = value.match(pattern); + if (!match) { + return null; + } + const parsed = Number(match[1]); + return Number.isFinite(parsed) ? parsed : null; +} + +function stripAnsi(value) { + return value.replace(/\u001B\[[0-9;]*m/gu, ""); +} + +function writeCommandLog(params) { + const { logDir, label, stdout, stderr } = params; + fs.mkdirSync(logDir, { recursive: true }); + const safeLabel = label.replace(/[^a-zA-Z0-9_.-]+/gu, "_"); + const logPath = path.join(logDir, `${safeLabel}.log`); + fs.writeFileSync( + logPath, + [`$ ${params.command.join(" ")}`, "", stripAnsi(stdout), stripAnsi(stderr)].join("\n"), + "utf8", + ); + return logPath; +} + +function runMeasuredCommand(params) { + const { command, args, mode } = timeWrapperArgs(params.command, params.args); + const started = process.hrtime.bigint(); + const result = spawnSync(command, args, { + cwd: params.cwd, + env: params.env, + encoding: "utf8", + timeout: params.timeoutMs, + maxBuffer: 16 * 1024 * 1024, + }); + const wallMs = Number(process.hrtime.bigint() - started) / 1_000_000; + const status = result.status ?? (result.signal ? 1 : 0); + const stdout = result.stdout ?? ""; + const stderr = result.stderr ?? ""; + const logPath = writeCommandLog({ + logDir: params.logDir, + label: params.label, + command: [params.command, ...params.args], + stdout, + stderr, + }); + return { + label: params.label, + phase: params.phase, + pluginId: params.pluginId ?? null, + status, + signal: result.signal ?? null, + timedOut: result.error?.code === "ETIMEDOUT", + logPath, + ...parseTimedMetrics(stderr, wallMs, mode), + }; +} + +function runPluginLifecycle(params) { + for (const plugin of params.plugins) { + const commands = [ + ["install", ["install", plugin.id]], + ["inspect", ["inspect", plugin.id, "--json"]], + ["disable", ["disable", plugin.id]], + ["enable", ["enable", plugin.id]], + ["doctor", ["doctor"]], + ["uninstall", ["uninstall", plugin.id, "--force"]], + ]; + for (const [phase, args] of commands) { + process.stderr.write(`[plugin-gauntlet] ${plugin.id} ${phase}\n`); + params.rows.push( + runMeasuredCommand({ + cwd: params.repoRoot, + env: params.env, + logDir: path.join(params.outputDir, "logs", "lifecycle"), + ...openclawCommand(params.repoRoot, ["plugins", ...args]), + label: `${plugin.id}-${phase}`, + phase: `lifecycle:${phase}`, + pluginId: plugin.id, + timeoutMs: params.commandTimeoutMs, + }), + ); + } + } +} + +function runSlashHelpProbes(params) { + for (const plugin of params.plugins) { + for (const alias of plugin.cliCommandAliases) { + const command = alias.cliCommand ?? alias.name; + process.stderr.write(`[plugin-gauntlet] ${plugin.id} slash-help /${alias.name}\n`); + params.rows.push( + runMeasuredCommand({ + cwd: params.repoRoot, + env: params.env, + logDir: path.join(params.outputDir, "logs", "slash-help"), + ...openclawCommand(params.repoRoot, [command, "--help"]), + label: `${plugin.id}-slash-${alias.name}`, + phase: "slash:help", + pluginId: plugin.id, + timeoutMs: params.commandTimeoutMs, + }), + ); + } + } +} + +function runQaChunks(params) { + const chunks = chunkArray(params.plugins, params.qaPluginChunkSize); + const summaries = []; + for (let index = 0; index < chunks.length; index += 1) { + const chunk = chunks[index]; + const outputDir = path.join( + params.outputDir, + "qa-suite", + `chunk-${String(index).padStart(2, "0")}`, + ); + const outputArg = toRepoRelativePath(params.repoRoot, outputDir); + const pluginIds = chunk.map((plugin) => plugin.id); + process.stderr.write( + `[plugin-gauntlet] qa chunk ${index + 1}/${chunks.length}: ${pluginIds.join(",")}\n`, + ); + const row = runMeasuredCommand({ + cwd: params.repoRoot, + env: params.env, + logDir: path.join(params.outputDir, "logs", "qa-suite"), + ...openclawCommand(params.repoRoot, [ + "qa", + "suite", + "--provider-mode", + "mock-openai", + "--concurrency", + "1", + "--output-dir", + outputArg, + ...params.qaScenarios.flatMap((scenario) => ["--scenario", scenario]), + ...pluginIds.flatMap((pluginId) => ["--enable-plugin", pluginId]), + ]), + label: `qa-chunk-${String(index).padStart(2, "0")}`, + phase: "qa:rpc", + timeoutMs: params.qaTimeoutMs, + }); + params.rows.push({ ...row, pluginId: pluginIds.join(",") }); + const summaryPath = path.join(outputDir, "qa-suite-summary.json"); + if (fs.existsSync(summaryPath)) { + summaries.push(JSON.parse(fs.readFileSync(summaryPath, "utf8"))); + } + } + return summaries; +} + +async function main() { + const options = parseArgs(process.argv.slice(2)); + const repoRoot = path.resolve(options.repoRoot); + fs.mkdirSync(options.outputDir, { recursive: true }); + const runRoot = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-plugin-gauntlet-")); + const env = createIsolatedEnv(repoRoot, runRoot); + const matrix = discoverBundledPluginManifests(repoRoot); + const selectedPlugins = selectPluginEntries(matrix, { + ids: options.pluginIds, + shardTotal: options.shardTotal, + shardIndex: options.shardIndex, + limit: options.limit, + }); + const rows = []; + if (!options.skipPrebuild && (selectedPlugins.length > 0 || !options.skipQa)) { + process.stderr.write("[plugin-gauntlet] prebuild\n"); + rows.push( + runMeasuredCommand({ + cwd: repoRoot, + env, + logDir: path.join(options.outputDir, "logs", "prebuild"), + command: pnpmCommand(), + args: ["build"], + label: "prebuild", + phase: "prebuild", + timeoutMs: options.buildTimeoutMs, + }), + ); + } + const prebuildFailed = rows.some( + (row) => row.phase === "prebuild" && (row.status !== 0 || row.timedOut), + ); + if (!prebuildFailed && !options.skipLifecycle) { + runPluginLifecycle({ + repoRoot, + outputDir: options.outputDir, + env, + plugins: selectedPlugins, + rows, + commandTimeoutMs: options.commandTimeoutMs, + }); + } + if (!prebuildFailed && !options.skipSlashHelp) { + runSlashHelpProbes({ + repoRoot, + outputDir: options.outputDir, + env, + plugins: selectedPlugins, + rows, + commandTimeoutMs: options.commandTimeoutMs, + }); + } + const qaSummaries = + options.skipQa || prebuildFailed + ? [] + : runQaChunks({ + repoRoot, + outputDir: options.outputDir, + env, + plugins: selectedPlugins, + rows, + qaScenarios: options.qaScenarios, + qaPluginChunkSize: options.qaPluginChunkSize, + qaTimeoutMs: options.qaTimeoutMs, + }); + const metricObservations = collectMetricObservations(rows, { + cpuCoreWarn: options.cpuCoreWarn, + hotWallWarnMs: options.hotWallWarnMs, + maxRssWarnMb: options.maxRssWarnMb, + wallAnomalyMultiplier: options.wallAnomalyMultiplier, + rssAnomalyMultiplier: options.rssAnomalyMultiplier, + }); + const gatewayObservations = qaSummaries.flatMap((qa) => + collectGatewayCpuObservations({ + startup: null, + qa, + cpuCoreWarn: options.cpuCoreWarn, + hotWallWarnMs: options.hotWallWarnMs, + }), + ); + const failures = rows.filter((row) => row.status !== 0 || row.timedOut); + const summary = { + generatedAt: new Date().toISOString(), + repoRoot, + outputDir: options.outputDir, + isolatedRunRoot: runRoot, + selectedPluginCount: selectedPlugins.length, + totalPluginCount: matrix.length, + options: { + pluginIds: options.pluginIds, + shardTotal: options.shardTotal, + shardIndex: options.shardIndex, + limit: options.limit ?? null, + qaScenarios: options.qaScenarios, + qaPluginChunkSize: options.qaPluginChunkSize, + skipLifecycle: options.skipLifecycle, + skipQa: options.skipQa, + skipSlashHelp: options.skipSlashHelp, + skipPrebuild: options.skipPrebuild, + thresholds: { + cpuCoreWarn: options.cpuCoreWarn, + hotWallWarnMs: options.hotWallWarnMs, + maxRssWarnMb: options.maxRssWarnMb, + wallAnomalyMultiplier: options.wallAnomalyMultiplier, + rssAnomalyMultiplier: options.rssAnomalyMultiplier, + }, + }, + matrix, + selectedPlugins, + rows, + observations: [...metricObservations, ...gatewayObservations], + failures, + }; + const summaryPath = path.join(options.outputDir, "plugin-gateway-gauntlet-summary.json"); + fs.writeFileSync(summaryPath, `${JSON.stringify(summary, null, 2)}\n`, "utf8"); + process.stdout.write(`[plugin-gauntlet] summary: ${summaryPath}\n`); + process.stdout.write( + `[plugin-gauntlet] plugins=${selectedPlugins.length}/${matrix.length} rows=${rows.length} failures=${failures.length} observations=${summary.observations.length}\n`, + ); + if (failures.length > 0) { + process.exitCode = 1; + } +} + +main().catch((error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; +}); diff --git a/scripts/lib/plugin-gateway-gauntlet.mjs b/scripts/lib/plugin-gateway-gauntlet.mjs new file mode 100644 index 00000000000..dbfce9258e5 --- /dev/null +++ b/scripts/lib/plugin-gateway-gauntlet.mjs @@ -0,0 +1,326 @@ +import fs from "node:fs"; +import path from "node:path"; +import JSON5 from "json5"; + +const MANIFEST_NAMES = ["openclaw.plugin.json", "openclaw.plugin.json5"]; + +function isPlainObject(value) { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +function normalizeString(value) { + return typeof value === "string" ? value.trim() : ""; +} + +function normalizeStringArray(value) { + return Array.isArray(value) + ? value.map((entry) => normalizeString(entry)).filter((entry) => entry.length > 0) + : []; +} + +function readPluginManifest(manifestPath) { + const raw = fs.readFileSync(manifestPath, "utf8"); + const parsed = manifestPath.endsWith(".json5") ? JSON5.parse(raw) : JSON.parse(raw); + if (!isPlainObject(parsed)) { + throw new Error(`Plugin manifest must be an object: ${manifestPath}`); + } + const id = normalizeString(parsed.id); + if (!id) { + throw new Error(`Plugin manifest is missing id: ${manifestPath}`); + } + return parsed; +} + +function schemaHasRequiredFields(schema, seen = new Set()) { + if (!isPlainObject(schema) || seen.has(schema)) { + return false; + } + seen.add(schema); + if (Array.isArray(schema.required) && schema.required.length > 0) { + return true; + } + for (const key of ["properties", "patternProperties", "$defs", "definitions"]) { + const children = schema[key]; + if (!isPlainObject(children)) { + continue; + } + for (const child of Object.values(children)) { + if (schemaHasRequiredFields(child, seen)) { + return true; + } + } + } + for (const key of ["items", "additionalProperties", "contains", "not", "if", "then", "else"]) { + if (schemaHasRequiredFields(schema[key], seen)) { + return true; + } + } + for (const key of ["allOf", "anyOf", "oneOf", "prefixItems"]) { + const children = schema[key]; + if (!Array.isArray(children)) { + continue; + } + if (children.some((child) => schemaHasRequiredFields(child, seen))) { + return true; + } + } + return false; +} + +function collectCommandAliasRecords(manifest) { + const aliases = Array.isArray(manifest.commandAliases) ? manifest.commandAliases : []; + return aliases + .map((alias) => { + if (typeof alias === "string") { + const name = normalizeString(alias); + return name ? { name, kind: "runtime-slash", cliCommand: null } : null; + } + if (!isPlainObject(alias)) { + return null; + } + const name = normalizeString(alias.name); + if (!name) { + return null; + } + return { + name, + kind: normalizeString(alias.kind) || "runtime-slash", + cliCommand: normalizeString(alias.cliCommand) || null, + }; + }) + .filter(Boolean); +} + +function collectAuthMethods(manifest) { + const auth = Array.isArray(manifest.auth) ? manifest.auth : []; + return auth + .map((entry) => (isPlainObject(entry) ? normalizeString(entry.method) : "")) + .filter((method) => method.length > 0); +} + +function collectOnboardingScopes(manifest) { + const scopes = new Set(); + const addScopes = (value) => { + for (const scope of normalizeStringArray(value)) { + scopes.add(scope); + } + }; + addScopes(manifest.onboardingScopes); + if (Array.isArray(manifest.auth)) { + for (const entry of manifest.auth) { + if (isPlainObject(entry)) { + addScopes(entry.onboardingScopes); + } + } + } + return [...scopes]; +} + +function buildPluginMatrixEntry(params) { + const { repoRoot, manifestPath, manifest } = params; + const relativeManifestPath = path.relative(repoRoot, manifestPath); + const commandAliases = collectCommandAliasRecords(manifest); + return { + id: manifest.id, + name: normalizeString(manifest.name) || manifest.id, + dir: path.relative(repoRoot, path.dirname(manifestPath)), + manifestPath: relativeManifestPath, + enabledByDefault: manifest.enabledByDefault === true, + activation: isPlainObject(manifest.activation) ? manifest.activation : {}, + providers: normalizeStringArray(manifest.providers), + channels: normalizeStringArray(manifest.channels), + skills: normalizeStringArray(manifest.skills), + authMethods: collectAuthMethods(manifest), + onboardingScopes: collectOnboardingScopes(manifest), + hasConfigSchema: isPlainObject(manifest.configSchema), + hasRequiredConfigFields: schemaHasRequiredFields(manifest.configSchema), + commandAliases, + cliCommandAliases: commandAliases.filter((alias) => alias.cliCommand), + runtimeSlashAliases: commandAliases.filter((alias) => alias.kind === "runtime-slash"), + }; +} + +function discoverBundledPluginManifests(repoRoot) { + const extensionsDir = path.join(repoRoot, "extensions"); + const entries = fs + .readdirSync(extensionsDir, { withFileTypes: true }) + .filter((entry) => entry.isDirectory()) + .flatMap((entry) => { + const pluginDir = path.join(extensionsDir, entry.name); + const manifestName = MANIFEST_NAMES.find((name) => fs.existsSync(path.join(pluginDir, name))); + if (!manifestName) { + return []; + } + const manifestPath = path.join(pluginDir, manifestName); + const manifest = readPluginManifest(manifestPath); + return [buildPluginMatrixEntry({ repoRoot, manifestPath, manifest })]; + }); + return entries.sort((left, right) => left.id.localeCompare(right.id)); +} + +function selectPluginEntries(entries, options = {}) { + const ids = new Set(normalizeStringArray(options.ids)); + let selected = ids.size > 0 ? entries.filter((entry) => ids.has(entry.id)) : [...entries]; + const missingIds = [...ids].filter((id) => !entries.some((entry) => entry.id === id)); + if (missingIds.length > 0) { + throw new Error(`Unknown bundled plugin id(s): ${missingIds.join(", ")}`); + } + const shardTotal = options.shardTotal ?? 1; + const shardIndex = options.shardIndex ?? 0; + if (!Number.isInteger(shardTotal) || shardTotal < 1) { + throw new Error("--shard-total must be a positive integer"); + } + if (!Number.isInteger(shardIndex) || shardIndex < 0 || shardIndex >= shardTotal) { + throw new Error("--shard-index must be in range [0, shard-total)"); + } + selected = selected.filter((_, index) => index % shardTotal === shardIndex); + if (options.limit !== undefined) { + if (!Number.isInteger(options.limit) || options.limit < 1) { + throw new Error("--limit must be a positive integer"); + } + selected = selected.slice(0, options.limit); + } + return selected; +} + +function median(values) { + const sorted = values + .filter((value) => typeof value === "number" && Number.isFinite(value)) + .sort((left, right) => left - right); + if (sorted.length === 0) { + return null; + } + const midpoint = Math.floor(sorted.length / 2); + return sorted.length % 2 === 1 ? sorted[midpoint] : (sorted[midpoint - 1] + sorted[midpoint]) / 2; +} + +function groupByPhase(rows) { + const phases = new Map(); + for (const row of rows) { + const phase = normalizeString(row.phase) || "unknown"; + const current = phases.get(phase) ?? []; + current.push(row); + phases.set(phase, current); + } + return phases; +} + +function collectMetricObservations(rows, thresholds = {}) { + const cpuCoreWarn = thresholds.cpuCoreWarn ?? 0.9; + const hotWallWarnMs = thresholds.hotWallWarnMs ?? 30_000; + const wallAnomalyMultiplier = thresholds.wallAnomalyMultiplier ?? 3; + const maxRssWarnMb = thresholds.maxRssWarnMb ?? null; + const rssAnomalyMultiplier = thresholds.rssAnomalyMultiplier ?? 2.5; + const observations = []; + for (const [phase, phaseRows] of groupByPhase(rows)) { + const wallMedianMs = median(phaseRows.map((row) => row.wallMs)); + const rssMedianMb = median(phaseRows.map((row) => row.maxRssMb)); + for (const row of phaseRows) { + if ( + typeof row.cpuCoreRatio === "number" && + typeof row.wallMs === "number" && + row.cpuCoreRatio >= cpuCoreWarn && + row.wallMs >= hotWallWarnMs + ) { + observations.push({ + kind: "phase-cpu-hot", + pluginId: row.pluginId ?? null, + phase, + cpuCoreRatio: row.cpuCoreRatio, + wallMs: row.wallMs, + }); + } + if ( + wallMedianMs !== null && + phaseRows.length >= 3 && + typeof row.wallMs === "number" && + row.wallMs >= wallMedianMs * wallAnomalyMultiplier + ) { + observations.push({ + kind: "phase-wall-anomaly", + pluginId: row.pluginId ?? null, + phase, + wallMs: row.wallMs, + medianWallMs: wallMedianMs, + multiplier: wallAnomalyMultiplier, + }); + } + if ( + typeof maxRssWarnMb === "number" && + typeof row.maxRssMb === "number" && + row.maxRssMb >= maxRssWarnMb + ) { + observations.push({ + kind: "phase-rss-high", + pluginId: row.pluginId ?? null, + phase, + maxRssMb: row.maxRssMb, + thresholdMb: maxRssWarnMb, + }); + } + if ( + rssMedianMb !== null && + rssMedianMb > 0 && + phaseRows.length >= 3 && + typeof row.maxRssMb === "number" && + row.maxRssMb >= rssMedianMb * rssAnomalyMultiplier + ) { + observations.push({ + kind: "phase-rss-anomaly", + pluginId: row.pluginId ?? null, + phase, + maxRssMb: row.maxRssMb, + medianRssMb: rssMedianMb, + multiplier: rssAnomalyMultiplier, + }); + } + } + } + return observations; +} + +function collectGatewayCpuObservations(params) { + const observations = []; + for (const result of params.startup?.results ?? []) { + const cpuCoreMax = result.summary?.cpuCoreRatio?.max; + const wallMax = result.summary?.readyzMs?.max ?? result.summary?.healthzMs?.max; + if ( + typeof cpuCoreMax === "number" && + typeof wallMax === "number" && + cpuCoreMax >= params.cpuCoreWarn && + wallMax >= params.hotWallWarnMs + ) { + observations.push({ + kind: "startup-cpu-hot", + id: result.id, + cpuCoreRatioMax: cpuCoreMax, + wallMsMax: wallMax, + }); + } + } + const qaCpuCoreRatio = params.qa?.metrics?.gatewayCpuCoreRatio; + const qaWallMs = params.qa?.metrics?.wallMs; + if ( + typeof qaCpuCoreRatio === "number" && + typeof qaWallMs === "number" && + qaCpuCoreRatio >= params.cpuCoreWarn && + qaWallMs >= params.hotWallWarnMs + ) { + observations.push({ + kind: "qa-cpu-hot", + id: "qa-suite", + cpuCoreRatio: qaCpuCoreRatio, + wallMs: qaWallMs, + }); + } + return observations; +} + +export { + collectCommandAliasRecords, + collectGatewayCpuObservations, + collectMetricObservations, + discoverBundledPluginManifests, + schemaHasRequiredFields, + selectPluginEntries, +}; diff --git a/test/scripts/plugin-gateway-gauntlet.test.ts b/test/scripts/plugin-gateway-gauntlet.test.ts new file mode 100644 index 00000000000..b7bcb8c9a5a --- /dev/null +++ b/test/scripts/plugin-gateway-gauntlet.test.ts @@ -0,0 +1,173 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { + collectGatewayCpuObservations, + collectMetricObservations, + discoverBundledPluginManifests, + schemaHasRequiredFields, + selectPluginEntries, +} from "../../scripts/lib/plugin-gateway-gauntlet.mjs"; + +describe("plugin gateway gauntlet helpers", () => { + let repoRoot: string; + + beforeEach(async () => { + repoRoot = await fs.mkdtemp(path.join(os.tmpdir(), "plugin-gauntlet-")); + await fs.mkdir(path.join(repoRoot, "extensions"), { recursive: true }); + }); + + afterEach(async () => { + await fs.rm(repoRoot, { recursive: true, force: true }); + }); + + async function writeManifest(pluginDir: string, fileName: string, source: string) { + const dir = path.join(repoRoot, "extensions", pluginDir); + await fs.mkdir(dir, { recursive: true }); + await fs.writeFile(path.join(dir, fileName), source, "utf8"); + } + + it("discovers bundled plugin manifests into lifecycle matrix rows", async () => { + await writeManifest( + "alpha", + "openclaw.plugin.json", + JSON.stringify({ + id: "alpha", + enabledByDefault: true, + providers: ["openai"], + commandAliases: [{ name: "alpha", kind: "runtime-slash", cliCommand: "plugins" }], + auth: [{ method: "oauth", onboardingScopes: ["models"] }], + configSchema: { + type: "object", + properties: { + nested: { + type: "object", + required: ["token"], + }, + }, + }, + }), + ); + await writeManifest( + "beta", + "openclaw.plugin.json5", + `{ id: "beta", commandAliases: ["dreaming"], onboardingScopes: ["memory"] }`, + ); + + const matrix = discoverBundledPluginManifests(repoRoot); + + expect(matrix.map((entry) => entry.id)).toEqual(["alpha", "beta"]); + expect(matrix[0]).toMatchObject({ + id: "alpha", + dir: path.join("extensions", "alpha"), + manifestPath: path.join("extensions", "alpha", "openclaw.plugin.json"), + enabledByDefault: true, + providers: ["openai"], + authMethods: ["oauth"], + onboardingScopes: ["models"], + hasConfigSchema: true, + hasRequiredConfigFields: true, + cliCommandAliases: [{ name: "alpha", kind: "runtime-slash", cliCommand: "plugins" }], + }); + expect(matrix[1].runtimeSlashAliases).toEqual([ + { name: "dreaming", kind: "runtime-slash", cliCommand: null }, + ]); + }); + + it("selects plugin shards after explicit id filtering", () => { + const entries = ["a", "b", "c", "d"].map((id) => ({ id })); + + expect(selectPluginEntries(entries, { ids: ["d", "b"], shardTotal: 2, shardIndex: 0 })).toEqual( + [{ id: "b" }], + ); + expect(() => selectPluginEntries(entries, { ids: ["missing"] })).toThrow( + "Unknown bundled plugin id(s): missing", + ); + }); + + it("detects required schema fields recursively", () => { + expect( + schemaHasRequiredFields({ + type: "object", + properties: { + auth: { + oneOf: [{ type: "object" }, { type: "object", required: ["token"] }], + }, + }, + }), + ).toBe(true); + expect( + schemaHasRequiredFields({ type: "object", properties: { enabled: { type: "boolean" } } }), + ).toBe(false); + }); + + it("flags gateway startup CPU observations using bench summary keys", () => { + expect( + collectGatewayCpuObservations({ + startup: { + results: [ + { + id: "default", + summary: { + cpuCoreRatio: { max: 1.1 }, + readyzMs: { max: 45_000 }, + }, + }, + ], + }, + qa: { + metrics: { + gatewayCpuCoreRatio: 1.2, + wallMs: 60_000, + }, + }, + cpuCoreWarn: 0.9, + hotWallWarnMs: 30_000, + }), + ).toEqual([ + { + kind: "startup-cpu-hot", + id: "default", + cpuCoreRatioMax: 1.1, + wallMsMax: 45_000, + }, + { + kind: "qa-cpu-hot", + id: "qa-suite", + cpuCoreRatio: 1.2, + wallMs: 60_000, + }, + ]); + }); + + it("flags absolute peaks and phase-relative anomalies", () => { + const observations = collectMetricObservations( + [ + { pluginId: "a", phase: "lifecycle:install", wallMs: 100, maxRssMb: 100 }, + { pluginId: "b", phase: "lifecycle:install", wallMs: 110, maxRssMb: 110 }, + { + pluginId: "c", + phase: "lifecycle:install", + wallMs: 1_000, + cpuCoreRatio: 1.2, + maxRssMb: 500, + }, + ], + { + cpuCoreWarn: 0.9, + hotWallWarnMs: 900, + maxRssWarnMb: 450, + wallAnomalyMultiplier: 3, + rssAnomalyMultiplier: 2.5, + }, + ); + + expect(observations.map((observation) => observation.kind)).toEqual([ + "phase-cpu-hot", + "phase-wall-anomaly", + "phase-rss-high", + "phase-rss-anomaly", + ]); + }); +});