fix: gateway boots when a configured plugin payload is broken (#110239)

* fix: quarantine broken plugins during gateway startup

* fix(plugins): preserve degraded boot on package read errors

* fix(gateway): emit quarantine diagnostic once

* fix(gateway): refresh plugin quarantine every boot

* fix(gateway): harden plugin payload quarantine

Co-authored-by: VACInc <3279061+VACInc@users.noreply.github.com>

* fix(ci): satisfy plugin quarantine checks

Co-authored-by: VACInc <3279061+VACInc@users.noreply.github.com>

---------

Co-authored-by: VACInc <3279061+VACInc@users.noreply.github.com>
Co-authored-by: Peter Steinberger <steipete@gmail.com>

src/cli/update-cli/active-plugin-payload-validation.ts

@@ -0,0 +1,56 @@
+// Boot-local plugin payload verification without repair, install, or catalog imports.
+import type { OpenClawConfig } from "../../config/types.openclaw.js";
+import type { PluginInstallRecord } from "../../config/types.plugins.js";
+import { normalizePluginsConfig, resolveEffectiveEnableState } from "../../plugins/config-state.js";
+import {
+  resolveTrustedSourceLinkedOfficialClawHubSpec,
+  resolveTrustedSourceLinkedOfficialNpmSpec,
+} from "../../plugins/official-external-install-records.js";
+import {
+  runPluginPayloadSmokeCheck,
+  type PluginPayloadSmokeResult,
+} from "./plugin-payload-validation.js";
+
+/** Runs the static payload check without repair, installs, or network access. */
+export async function runActivePluginPayloadSmokeCheck(params: {
+  cfg: OpenClawConfig;
+  records: Record<string, PluginInstallRecord>;
+  env: NodeJS.ProcessEnv;
+}): Promise<PluginPayloadSmokeResult> {
+  return await runPluginPayloadSmokeCheck({
+    records: filterRecordsToActive({ cfg: params.cfg, records: params.records }),
+    env: params.env,
+  });
+}
+
+/** Selects the installed records covered by update/startup payload verification. */
+export function filterRecordsToActive(params: {
+  cfg: OpenClawConfig;
+  records: Record<string, PluginInstallRecord>;
+}): Record<string, PluginInstallRecord> {
+  const normalizedPluginConfig = normalizePluginsConfig(params.cfg.plugins);
+  const filtered: Record<string, PluginInstallRecord> = {};
+  for (const [pluginId, record] of Object.entries(params.records)) {
+    if (!record || typeof record !== "object") {
+      continue;
+    }
+    const enableState = resolveEffectiveEnableState({
+      id: pluginId,
+      origin: "global",
+      config: normalizedPluginConfig,
+      rootConfig: params.cfg,
+    });
+    if (enableState.enabled) {
+      filtered[pluginId] = record;
+      continue;
+    }
+    // Trusted-source-linked official installs remain authoritative sync targets
+    // even when their plugin entry is disabled.
+    const officialNpm = resolveTrustedSourceLinkedOfficialNpmSpec({ pluginId, record });
+    const officialClawHub = resolveTrustedSourceLinkedOfficialClawHubSpec({ pluginId, record });
+    if (officialNpm || officialClawHub) {
+      filtered[pluginId] = record;
+    }
+  }
+  return filtered;
+}

src/cli/update-cli/plugin-payload-validation.ts

@@ -7,28 +7,17 @@ import type { PluginBundleFormat } from "../../plugins/manifest-types.js";
 import { resolvePackageExtensionEntries, type PackageManifest } from "../../plugins/manifest.js";
 import { validatePackageExtensionEntriesForInstall } from "../../plugins/package-entry-resolution.js";
 import { auditOpenClawPeerDependencyLink } from "../../plugins/plugin-peer-link.js";
+import type { PluginVerificationFailureReason } from "../../plugins/runtime-degraded-state.js";
 import { resolveUserPath } from "../../utils.js";
 
-type PluginPayloadSmokeFailureReason =
-  | "missing-install-path"
-  | "missing-package-dir"
-  | "missing-package-json"
-  | "unreadable-package-json"
-  | "invalid-package-json"
-  | "missing-bundle-manifest"
-  | "invalid-bundle-manifest"
-  | "missing-main-entry"
-  | "missing-extension-entry"
-  | "missing-openclaw-peer-link";
-
 export type PluginPayloadSmokeFailure = {
   pluginId: string;
   installPath?: string;
-  reason: PluginPayloadSmokeFailureReason;
+  reason: PluginVerificationFailureReason;
   detail: string;
 };
 
-type PluginPayloadSmokeResult = {
+export type PluginPayloadSmokeResult = {
   checked: string[];
   failures: PluginPayloadSmokeFailure[];
 };

src/cli/update-cli/post-core-plugin-convergence.test.ts

@@ -29,8 +29,11 @@ vi.mock("./plugin-payload-validation.js", () => ({
 import type { OpenClawConfig } from "../../config/types.openclaw.js";
 import { VERSION } from "../../version.js";
 import {
-  convergenceWarningsToOutcomes,
   filterRecordsToActive,
+  runActivePluginPayloadSmokeCheck,
+} from "./active-plugin-payload-validation.js";
+import {
+  convergenceWarningsToOutcomes,
   runPostCorePluginConvergence,
 } from "./post-core-plugin-convergence.js";
 
@@ -110,6 +113,28 @@ describe("runPostCorePluginConvergence", () => {
     });
   });
 
+  it("checks active payloads without running repair or peer-link convergence", async () => {
+    const cfg = {
+      plugins: {
+        deny: ["disabled"],
+        entries: { active: { enabled: true }, disabled: { enabled: true } },
+      },
+    } as unknown as OpenClawConfig;
+    const records = {
+      active: { source: "npm" as const, installPath: "/p/active" },
+      disabled: { source: "npm" as const, installPath: "/p/disabled" },
+    };
+
+    await runActivePluginPayloadSmokeCheck({ cfg, records, env: { OPENCLAW_STATE_DIR: "/state" } });
+
+    expect(mocks.runPluginPayloadSmokeCheck).toHaveBeenCalledWith({
+      records: { active: records.active },
+      env: { OPENCLAW_STATE_DIR: "/state" },
+    });
+    expect(mocks.repairMissingConfiguredPluginInstalls).not.toHaveBeenCalled();
+    expect(mocks.relinkOpenClawPeerDependenciesInManagedNpmRoot).not.toHaveBeenCalled();
+  });
+
   it("uses the candidate runtime version over a stale inherited host version", async () => {
     const cfg = { plugins: { entries: {} } } as unknown as OpenClawConfig;
     await runPostCorePluginConvergence({
@@ -189,10 +214,12 @@ describe("runPostCorePluginConvergence", () => {
     expect(mocks.relinkOpenClawPeerDependenciesInManagedNpmRoot).toHaveBeenNthCalledWith(1, {
       npmRoot: "/tmp/openclaw-state/npm",
       logger: {},
+      onPackageReadError: expect.any(Function),
     });
     expect(mocks.relinkOpenClawPeerDependenciesInManagedNpmRoot).toHaveBeenNthCalledWith(2, {
       npmRoot: "/tmp/openclaw-state/npm/projects/codex",
       logger: {},
+      onPackageReadError: expect.any(Function),
     });
     expect(result.changes).toEqual([
       "Repaired OpenClaw host peer link(s) for 1 managed npm plugin package(s).",
@@ -549,6 +576,105 @@ describe("runPostCorePluginConvergence", () => {
     });
   });
 
+  it("does not duplicate a package-scoped repair error owned by a smoke failure", async () => {
+    const installPath = "/tmp/openclaw-state/npm/projects/brave/node_modules/brave";
+    mocks.repairMissingConfiguredPluginInstalls.mockResolvedValue({
+      changes: [],
+      warnings: [],
+      records: { brave: { source: "npm", installPath } },
+    });
+    mocks.relinkOpenClawPeerDependenciesInManagedNpmRoot.mockImplementation(
+      async (params: { onPackageReadError?: (error: unknown, packageDir: string) => void }) => {
+        params.onPackageReadError?.(
+          new Error(`EACCES: permission denied, open '${installPath}/package.json'`),
+          installPath,
+        );
+        return { checked: 0, attempted: 0, repaired: 0, skipped: 1 };
+      },
+    );
+    mocks.runPluginPayloadSmokeCheck.mockResolvedValue({
+      checked: ["brave"],
+      failures: [
+        {
+          pluginId: "brave",
+          installPath,
+          reason: "unreadable-package-json",
+          detail: `Could not read package.json at ${installPath}/package.json: EACCES: permission denied`,
+        },
+      ],
+    });
+
+    const result = await runPostCorePluginConvergence({
+      cfg: {
+        plugins: { entries: { brave: { enabled: true } } },
+      } as unknown as OpenClawConfig,
+      env: { OPENCLAW_STATE_DIR: "/tmp/openclaw-state" },
+    });
+
+    expect(result.warnings).toHaveLength(1);
+    expect(result.warnings[0]).toMatchObject({
+      pluginId: "brave",
+      reason: expect.stringContaining("unreadable-package-json"),
+    });
+    expect(result.errored).toBe(true);
+  });
+
+  it("does not promote an inactive package read error into an ownerless blocker", async () => {
+    const installPath = "/tmp/openclaw-state/npm/projects/brave/node_modules/brave";
+    mocks.repairMissingConfiguredPluginInstalls.mockResolvedValue({
+      changes: [],
+      warnings: [],
+      records: { brave: { source: "npm", installPath } },
+    });
+    mocks.relinkOpenClawPeerDependenciesInManagedNpmRoot.mockImplementation(
+      async (params: { onPackageReadError?: (error: unknown, packageDir: string) => void }) => {
+        params.onPackageReadError?.(
+          new Error(`EACCES: permission denied, open '${installPath}/package.json'`),
+          installPath,
+        );
+        return { checked: 0, attempted: 0, repaired: 0, skipped: 1 };
+      },
+    );
+
+    const result = await runPostCorePluginConvergence({
+      cfg: {
+        plugins: { entries: { brave: { enabled: false } } },
+      } as unknown as OpenClawConfig,
+      env: { OPENCLAW_STATE_DIR: "/tmp/openclaw-state" },
+    });
+
+    expect(mocks.runPluginPayloadSmokeCheck).toHaveBeenCalledWith({
+      records: {},
+      env: expect.any(Object),
+    });
+    expect(result.warnings).toEqual([]);
+    expect(result.errored).toBe(false);
+  });
+
+  it("keeps an unowned package read error visible for startup to block", async () => {
+    const packageDir = "/tmp/openclaw-state/npm/node_modules/untracked";
+    mocks.relinkOpenClawPeerDependenciesInManagedNpmRoot.mockImplementation(
+      async (params: { onPackageReadError?: (error: unknown, packageDir: string) => void }) => {
+        params.onPackageReadError?.(new Error("EACCES: permission denied"), packageDir);
+        return { checked: 0, attempted: 0, repaired: 0, skipped: 1 };
+      },
+    );
+
+    const result = await runPostCorePluginConvergence({
+      cfg: { plugins: { entries: {} } } as unknown as OpenClawConfig,
+      env: { OPENCLAW_STATE_DIR: "/tmp/openclaw-state" },
+    });
+
+    expect(result.warnings).toStrictEqual([
+      {
+        reason: "Failed to repair managed npm OpenClaw host peer links: EACCES: permission denied",
+        message: "Failed to repair managed npm OpenClaw host peer links: EACCES: permission denied",
+        guidance: ["Run `openclaw update repair` to retry plugin repair."],
+      },
+    ]);
+    expect(result.errored).toBe(false);
+  });
+
   it("hands repair's post-mutation records straight to the smoke check (no second disk read)", async () => {
     const records = { brave: { source: "npm" as const, installPath: "/p/brave" } };
     mocks.repairMissingConfiguredPluginInstalls.mockResolvedValue({

src/cli/update-cli/post-core-plugin-convergence.ts

@@ -5,20 +5,17 @@ import { UPDATE_POST_CORE_CONVERGENCE_ENV } from "../../commands/doctor/shared/u
 import type { OpenClawConfig } from "../../config/types.openclaw.js";
 import type { PluginInstallRecord } from "../../config/types.plugins.js";
 import type { ClawHubRiskAcknowledgementRequest } from "../../infra/clawhub-install-trust.js";
-import { normalizePluginsConfig, resolveEffectiveEnableState } from "../../plugins/config-state.js";
 import { resolveDefaultPluginNpmDir } from "../../plugins/install-paths.js";
 import { listManagedPluginNpmRoots } from "../../plugins/npm-project-roots.js";
-import {
-  resolveTrustedSourceLinkedOfficialClawHubSpec,
-  resolveTrustedSourceLinkedOfficialNpmSpec,
-} from "../../plugins/official-external-install-records.js";
 import { relinkOpenClawPeerDependenciesInManagedNpmRoot } from "../../plugins/plugin-peer-link.js";
 import { pruneStaleLocalBundledPluginInstallRecords } from "../../plugins/stale-local-bundled-plugin-install-records.js";
+import { resolveUserPath } from "../../utils.js";
 import { VERSION } from "../../version.js";
 import {
-  runPluginPayloadSmokeCheck,
-  type PluginPayloadSmokeFailure,
-} from "./plugin-payload-validation.js";
+  filterRecordsToActive,
+  runActivePluginPayloadSmokeCheck,
+} from "./active-plugin-payload-validation.js";
+import type { PluginPayloadSmokeFailure } from "./plugin-payload-validation.js";
 
 type PostCoreConvergenceWarning = {
   pluginId?: string;
@@ -63,9 +60,12 @@ function smokeFailureGuidance(failure: PluginPayloadSmokeFailure): string[] {
   ];
 }
 
-async function repairManagedNpmOpenClawPeerLinks(params: {
-  env: NodeJS.ProcessEnv;
-}): Promise<{ changes: string[]; warnings: PostCoreConvergenceWarning[] }> {
+async function repairManagedNpmOpenClawPeerLinks(params: { env: NodeJS.ProcessEnv }): Promise<{
+  changes: string[];
+  warnings: PostCoreConvergenceWarning[];
+  packageReadFailures: Array<{ error: unknown; packageDir: string }>;
+}> {
+  const packageReadFailures: Array<{ error: unknown; packageDir: string }> = [];
   try {
     const npmRoots = await listManagedPluginNpmRoots(resolveDefaultPluginNpmDir(params.env));
     const results = await Promise.all(
@@ -73,6 +73,9 @@ async function repairManagedNpmOpenClawPeerLinks(params: {
         relinkOpenClawPeerDependenciesInManagedNpmRoot({
           npmRoot,
           logger: {},
+          onPackageReadError: (error, packageDir) => {
+            packageReadFailures.push({ error, packageDir });
+          },
         }),
       ),
     );
@@ -83,6 +86,7 @@ async function repairManagedNpmOpenClawPeerLinks(params: {
           ? [`Repaired OpenClaw host peer link(s) for ${repaired} managed npm plugin package(s).`]
           : [],
       warnings: [],
+      packageReadFailures,
     };
   } catch (err) {
     const message = `Failed to repair managed npm OpenClaw host peer links: ${err instanceof Error ? err.message : String(err)}`;
@@ -95,17 +99,29 @@ async function repairManagedNpmOpenClawPeerLinks(params: {
           guidance: [REPAIR_GUIDANCE],
         },
       ],
+      packageReadFailures,
     };
   }
 }
 
+function formatPeerLinkPackageReadWarning(failure: { error: unknown }): PostCoreConvergenceWarning {
+  const message = `Failed to repair managed npm OpenClaw host peer links: ${failure.error instanceof Error ? failure.error.message : String(failure.error)}`;
+  return {
+    reason: message,
+    message,
+    guidance: [REPAIR_GUIDANCE],
+  };
+}
+
 /**
  * Mandatory post-core convergence pass. Runs AFTER the core package files
  * are swapped and the in-update doctor pass has already returned, but BEFORE
  * the gateway is restarted. Missing-plugin repair failures stay nonblocking:
  * an external package fetch may be transient, and failing the core update
- * would strand the user. Payload smoke failures still block the restart so we
- * never restart with an installed active plugin whose payload is unloadable.
+ * would strand the user. Explicit `openclaw update` callers keep reporting
+ * payload smoke failures as errors. Gateway startup consumes the same typed
+ * failures by quarantining each known plugin owner before any module import,
+ * then boots with that plugin marked configured-unavailable.
  */
 export async function runPostCorePluginConvergence(params: {
   cfg: OpenClawConfig;
@@ -163,8 +179,42 @@ export async function runPostCorePluginConvergence(params: {
   // this filter, a stale install record for a disabled or no-longer-
   // configured plugin whose payload was deleted on disk would block the
   // entire update — even though the gateway will never load that plugin.
+  const smoke = await runActivePluginPayloadSmokeCheck({
+    cfg: params.cfg,
+    records,
+    env,
+  });
   const smokeRecords = filterRecordsToActive({ cfg: params.cfg, records });
-  const smoke = await runPluginPayloadSmokeCheck({ records: smokeRecords, env });
+  const resolveInstallRecordPaths = (
+    installRecords: Record<string, PluginInstallRecord>,
+  ): Set<string> =>
+    new Set(
+      Object.values(installRecords).flatMap((record) => {
+        const installPath = record.installPath?.trim();
+        return installPath ? [path.resolve(resolveUserPath(installPath, env))] : [];
+      }),
+    );
+  const knownInstallPaths = resolveInstallRecordPaths(records);
+  const activeInstallPaths = resolveInstallRecordPaths(smokeRecords);
+  const smokeFailureInstallPaths = new Set(
+    smoke.failures.flatMap((failure) =>
+      failure.installPath ? [path.resolve(failure.installPath)] : [],
+    ),
+  );
+  for (const failure of peerLinkRepair.packageReadFailures.toSorted((left, right) =>
+    left.packageDir.localeCompare(right.packageDir),
+  )) {
+    // A typed smoke failure owns this exact package and startup quarantines it.
+    // Re-emitting the repair error without that owner would turn it back into
+    // an unknown warning and incorrectly block gateway readiness.
+    const packageDir = path.resolve(failure.packageDir);
+    const hasTypedFailure = smokeFailureInstallPaths.has(packageDir);
+    const belongsToInactivePlugin =
+      knownInstallPaths.has(packageDir) && !activeInstallPaths.has(packageDir);
+    if (!hasTypedFailure && !belongsToInactivePlugin) {
+      warnings.push(formatPeerLinkPackageReadWarning(failure));
+    }
+  }
   for (const failure of smoke.failures) {
     warnings.push({
       pluginId: failure.pluginId,
@@ -202,38 +252,6 @@ export async function runPostCorePluginConvergence(params: {
  * that are enabled implicitly via auth profiles or model refs. Effective
  * enable state is the right precision boundary.
  */
-export function filterRecordsToActive(params: {
-  cfg: OpenClawConfig;
-  records: Record<string, PluginInstallRecord>;
-}): Record<string, PluginInstallRecord> {
-  const normalizedPluginConfig = normalizePluginsConfig(params.cfg.plugins);
-  const filtered: Record<string, PluginInstallRecord> = {};
-  for (const [pluginId, record] of Object.entries(params.records)) {
-    if (!record || typeof record !== "object") {
-      continue;
-    }
-    const enableState = resolveEffectiveEnableState({
-      id: pluginId,
-      origin: "global",
-      config: normalizedPluginConfig,
-      rootConfig: params.cfg,
-    });
-    if (enableState.enabled) {
-      filtered[pluginId] = record;
-      continue;
-    }
-    // Even when disabled, retain trusted-source-linked official installs
-    // because the existing post-update sync path treats them as
-    // authoritative regardless of the entry's enable flag.
-    const officialNpm = resolveTrustedSourceLinkedOfficialNpmSpec({ pluginId, record });
-    const officialClawHub = resolveTrustedSourceLinkedOfficialClawHubSpec({ pluginId, record });
-    if (officialNpm || officialClawHub) {
-      filtered[pluginId] = record;
-    }
-  }
-  return filtered;
-}
-
 /**
  * Pure helper used by `updatePluginsAfterCoreUpdate` to fold a convergence
  * result into the existing `PluginUpdateOutcome[]` / warning shape that the
@@ -244,8 +262,8 @@ export function filterRecordsToActive(params: {
  *    warnings that name a `pluginId` produce per-plugin error outcomes; the
  *    rest are surfaced via `warnings`.
  *  - `errored` boolean that callers translate into `status: "error"`.
- *    Repair warnings are nonblocking; smoke failures remain blocking
- *    because they prove an active installed payload is unloadable.
+ *    Repair warnings are nonblocking; smoke failures remain errors on the
+ *    explicit update path even though Gateway startup can quarantine them.
  */
 export function convergenceWarningsToOutcomes(convergence: PostCoreConvergenceResult): {
   warnings: PostCoreConvergenceWarning[];