test(plugins): fail gauntlet on load diagnostics

scripts/check-plugin-gateway-gauntlet.mjs

@@ -10,6 +10,7 @@ import {
   collectGatewayCpuObservations,
   collectMetricObservations,
   collectQaBaselineRegressionObservations,
+  detectCommandDiagnosticFailure,
   discoverBundledPluginManifests,
   selectPluginEntries,
 } from "./lib/plugin-gateway-gauntlet.mjs";
@@ -365,6 +366,7 @@ function runMeasuredCommand(params) {
   const status = result.status ?? (result.signal ? 1 : 0);
   const stdout = result.stdout ?? "";
   const stderr = result.stderr ?? "";
+  const diagnosticFailure = detectCommandDiagnosticFailure(stdout, stderr);
   const logPath = writeCommandLog({
     logDir: params.logDir,
     label: params.label,
@@ -377,6 +379,7 @@ function runMeasuredCommand(params) {
     phase: params.phase,
     pluginId: params.pluginId ?? null,
     status,
+    diagnosticFailure,
     signal: result.signal ?? null,
     timedOut: result.error?.code === "ETIMEDOUT",
     logPath,
@@ -575,7 +578,7 @@ async function main() {
       hotWallWarnMs: options.hotWallWarnMs,
     }),
   );
-  const failures = rows.filter((row) => row.status !== 0 || row.timedOut);
+  const failures = rows.filter((row) => row.status !== 0 || row.timedOut || row.diagnosticFailure);
   const summary = {
     generatedAt: new Date().toISOString(),
     repoRoot,
@@ -619,7 +622,7 @@ async function main() {
   );
   for (const failure of failures) {
     process.stdout.write(
-      `[plugin-gauntlet] failure phase=${failure.phase} plugin=${failure.pluginId ?? "<none>"} status=${failure.status} timedOut=${failure.timedOut} wallMs=${Math.round(failure.wallMs)} log=${failure.logPath}\n`,
+      `[plugin-gauntlet] failure phase=${failure.phase} plugin=${failure.pluginId ?? "<none>"} status=${failure.status} timedOut=${failure.timedOut} diagnostic=${failure.diagnosticFailure ?? ""} wallMs=${Math.round(failure.wallMs)} log=${failure.logPath}\n`,
     );
   }
   for (const observation of summary.observations.slice(0, 20)) {

scripts/lib/plugin-gateway-gauntlet.mjs

@@ -1,9 +1,13 @@
 import fs from "node:fs";
 import path from "node:path";
 import JSON5 from "json5";
-import { collectBundledPluginBuildEntries } from "./bundled-plugin-build-entries.mjs";
+import {
+  NON_PACKAGED_BUNDLED_PLUGIN_DIRS,
+  collectBundledPluginBuildEntries,
+} from "./bundled-plugin-build-entries.mjs";
 
 const MANIFEST_NAMES = ["openclaw.plugin.json", "openclaw.plugin.json5"];
+const ANSI_PATTERN = new RegExp(String.raw`\u001B\[[0-9;]*m`, "gu");
 
 function isPlainObject(value) {
   return value !== null && typeof value === "object" && !Array.isArray(value);
@@ -156,6 +160,9 @@ function discoverBundledPluginManifests(repoRoot) {
       if (!manifestName) {
         return [];
       }
+      if (NON_PACKAGED_BUNDLED_PLUGIN_DIRS.has(entry.name)) {
+        return [];
+      }
       const manifestPath = path.join(pluginDir, manifestName);
       const manifest = readPluginManifest(manifestPath);
       return [buildPluginMatrixEntry({ repoRoot, manifestPath, manifest })];
@@ -350,6 +357,14 @@ function buildGauntletPrebuildEnv(env, options = {}) {
   };
 }
 
+function detectCommandDiagnosticFailure(stdout, stderr) {
+  const output = `${stdout}\n${stderr}`.replace(ANSI_PATTERN, "");
+  if (/^\[plugins\]\s+\S+\s+failed to load from\s+/mu.test(output)) {
+    return "plugin-load-failure";
+  }
+  return null;
+}
+
 function collectGatewayCpuObservations(params) {
   const observations = [];
   for (const result of params.startup?.results ?? []) {
@@ -392,6 +407,7 @@ export {
   collectGatewayCpuObservations,
   collectMetricObservations,
   buildGauntletPrebuildEnv,
+  detectCommandDiagnosticFailure,
   discoverBundledPluginManifests,
   schemaHasRequiredFields,
   selectPluginEntries,

test/scripts/plugin-gateway-gauntlet.test.ts

@@ -7,6 +7,7 @@ import {
   collectGatewayCpuObservations,
   collectMetricObservations,
   collectQaBaselineRegressionObservations,
+  detectCommandDiagnosticFailure,
   discoverBundledPluginManifests,
   schemaHasRequiredFields,
   selectPluginEntries,
@@ -84,6 +85,7 @@ describe("plugin gateway gauntlet helpers", () => {
   });
 
   it("skips source-only plugin dirs that are excluded from the built runtime", async () => {
+    await writeManifest("qa-lab", "openclaw.plugin.json", JSON.stringify({ id: "qa-lab" }));
     await writeManifest("qqbot", "openclaw.plugin.json", JSON.stringify({ id: "qqbot" }));
     await writeManifest("telegram", "openclaw.plugin.json", JSON.stringify({ id: "telegram" }));
 
@@ -92,6 +94,22 @@ describe("plugin gateway gauntlet helpers", () => {
     expect(matrix.map((entry) => entry.id)).toEqual(["telegram"]);
   });
 
+  it("detects plugin load failures in successful command output", () => {
+    expect(
+      detectCommandDiagnosticFailure(
+        "Installed plugin: qa-lab\n",
+        "[plugins] qa-lab failed to load from /repo/extensions/qa-lab/index.ts: Error: nope\n",
+      ),
+    ).toBe("plugin-load-failure");
+    expect(
+      detectCommandDiagnosticFailure(
+        "",
+        "\u001B[36m[plugins]\u001B[39m qa-lab failed to load from /repo/extensions/qa-lab/index.ts: Error: nope\n",
+      ),
+    ).toBe("plugin-load-failure");
+    expect(detectCommandDiagnosticFailure("Installed plugin: qa-lab\n", "")).toBeNull();
+  });
+
   it("selects plugin shards after explicit id filtering", () => {
     const entries = ["a", "b", "c", "d"].map((id) => ({ id }));