diff --git a/scripts/check-plugin-gateway-gauntlet.mjs b/scripts/check-plugin-gateway-gauntlet.mjs index 3e87fb9236e..b9e0cf2f28c 100644 --- a/scripts/check-plugin-gateway-gauntlet.mjs +++ b/scripts/check-plugin-gateway-gauntlet.mjs @@ -10,6 +10,7 @@ import { collectGatewayCpuObservations, collectMetricObservations, collectQaBaselineRegressionObservations, + detectCommandDiagnosticFailure, discoverBundledPluginManifests, selectPluginEntries, } from "./lib/plugin-gateway-gauntlet.mjs"; @@ -365,6 +366,7 @@ function runMeasuredCommand(params) { const status = result.status ?? (result.signal ? 1 : 0); const stdout = result.stdout ?? ""; const stderr = result.stderr ?? ""; + const diagnosticFailure = detectCommandDiagnosticFailure(stdout, stderr); const logPath = writeCommandLog({ logDir: params.logDir, label: params.label, @@ -377,6 +379,7 @@ function runMeasuredCommand(params) { phase: params.phase, pluginId: params.pluginId ?? null, status, + diagnosticFailure, signal: result.signal ?? null, timedOut: result.error?.code === "ETIMEDOUT", logPath, @@ -575,7 +578,7 @@ async function main() { hotWallWarnMs: options.hotWallWarnMs, }), ); - const failures = rows.filter((row) => row.status !== 0 || row.timedOut); + const failures = rows.filter((row) => row.status !== 0 || row.timedOut || row.diagnosticFailure); const summary = { generatedAt: new Date().toISOString(), repoRoot, @@ -619,7 +622,7 @@ async function main() { ); for (const failure of failures) { process.stdout.write( - `[plugin-gauntlet] failure phase=${failure.phase} plugin=${failure.pluginId ?? ""} status=${failure.status} timedOut=${failure.timedOut} wallMs=${Math.round(failure.wallMs)} log=${failure.logPath}\n`, + `[plugin-gauntlet] failure phase=${failure.phase} plugin=${failure.pluginId ?? ""} status=${failure.status} timedOut=${failure.timedOut} diagnostic=${failure.diagnosticFailure ?? ""} wallMs=${Math.round(failure.wallMs)} log=${failure.logPath}\n`, ); } for (const observation of summary.observations.slice(0, 20)) { diff --git a/scripts/lib/plugin-gateway-gauntlet.mjs b/scripts/lib/plugin-gateway-gauntlet.mjs index 2910aeb1749..d0c066debcb 100644 --- a/scripts/lib/plugin-gateway-gauntlet.mjs +++ b/scripts/lib/plugin-gateway-gauntlet.mjs @@ -1,9 +1,13 @@ import fs from "node:fs"; import path from "node:path"; import JSON5 from "json5"; -import { collectBundledPluginBuildEntries } from "./bundled-plugin-build-entries.mjs"; +import { + NON_PACKAGED_BUNDLED_PLUGIN_DIRS, + collectBundledPluginBuildEntries, +} from "./bundled-plugin-build-entries.mjs"; const MANIFEST_NAMES = ["openclaw.plugin.json", "openclaw.plugin.json5"]; +const ANSI_PATTERN = new RegExp(String.raw`\u001B\[[0-9;]*m`, "gu"); function isPlainObject(value) { return value !== null && typeof value === "object" && !Array.isArray(value); @@ -156,6 +160,9 @@ function discoverBundledPluginManifests(repoRoot) { if (!manifestName) { return []; } + if (NON_PACKAGED_BUNDLED_PLUGIN_DIRS.has(entry.name)) { + return []; + } const manifestPath = path.join(pluginDir, manifestName); const manifest = readPluginManifest(manifestPath); return [buildPluginMatrixEntry({ repoRoot, manifestPath, manifest })]; @@ -350,6 +357,14 @@ function buildGauntletPrebuildEnv(env, options = {}) { }; } +function detectCommandDiagnosticFailure(stdout, stderr) { + const output = `${stdout}\n${stderr}`.replace(ANSI_PATTERN, ""); + if (/^\[plugins\]\s+\S+\s+failed to load from\s+/mu.test(output)) { + return "plugin-load-failure"; + } + return null; +} + function collectGatewayCpuObservations(params) { const observations = []; for (const result of params.startup?.results ?? []) { @@ -392,6 +407,7 @@ export { collectGatewayCpuObservations, collectMetricObservations, buildGauntletPrebuildEnv, + detectCommandDiagnosticFailure, discoverBundledPluginManifests, schemaHasRequiredFields, selectPluginEntries, diff --git a/test/scripts/plugin-gateway-gauntlet.test.ts b/test/scripts/plugin-gateway-gauntlet.test.ts index 9c1c9e8fa57..29192eb838c 100644 --- a/test/scripts/plugin-gateway-gauntlet.test.ts +++ b/test/scripts/plugin-gateway-gauntlet.test.ts @@ -7,6 +7,7 @@ import { collectGatewayCpuObservations, collectMetricObservations, collectQaBaselineRegressionObservations, + detectCommandDiagnosticFailure, discoverBundledPluginManifests, schemaHasRequiredFields, selectPluginEntries, @@ -84,6 +85,7 @@ describe("plugin gateway gauntlet helpers", () => { }); it("skips source-only plugin dirs that are excluded from the built runtime", async () => { + await writeManifest("qa-lab", "openclaw.plugin.json", JSON.stringify({ id: "qa-lab" })); await writeManifest("qqbot", "openclaw.plugin.json", JSON.stringify({ id: "qqbot" })); await writeManifest("telegram", "openclaw.plugin.json", JSON.stringify({ id: "telegram" })); @@ -92,6 +94,22 @@ describe("plugin gateway gauntlet helpers", () => { expect(matrix.map((entry) => entry.id)).toEqual(["telegram"]); }); + it("detects plugin load failures in successful command output", () => { + expect( + detectCommandDiagnosticFailure( + "Installed plugin: qa-lab\n", + "[plugins] qa-lab failed to load from /repo/extensions/qa-lab/index.ts: Error: nope\n", + ), + ).toBe("plugin-load-failure"); + expect( + detectCommandDiagnosticFailure( + "", + "\u001B[36m[plugins]\u001B[39m qa-lab failed to load from /repo/extensions/qa-lab/index.ts: Error: nope\n", + ), + ).toBe("plugin-load-failure"); + expect(detectCommandDiagnosticFailure("Installed plugin: qa-lab\n", "")).toBeNull(); + }); + it("selects plugin shards after explicit id filtering", () => { const entries = ["a", "b", "c", "d"].map((id) => ({ id }));