fix: recover suspicious gateway startup configs (#89480)

2026-06-22 11:08:10 +00:00 · 2026-06-02 10:12:35 -04:00
parent 00d846daf7
commit c8d21fe7f0
4 changed files with 352 additions and 9 deletions
--- a/src/cli/gateway-cli/run.ts
+++ b/src/cli/gateway-cli/run.ts
@@ -294,7 +294,7 @@ async function readGatewayStartupConfig(params: {
  const { readConfigFileSnapshotWithPluginMetadata } = await import("../../config/config.js");
  const snapshotRead: ReadConfigFileSnapshotWithPluginMetadataResult | null =
    await params.startupTrace.measure("cli.config-snapshot", () =>
-      readConfigFileSnapshotWithPluginMetadata().catch(() => null),
+      readConfigFileSnapshotWithPluginMetadata({ recoverSuspicious: true }).catch(() => null),
    );
  const snapshot: ConfigFileSnapshot | null = snapshotRead?.snapshot ?? null;
  const cfg = snapshot?.config ?? {};
--- a/src/config/io.observe-recovery.test.ts
+++ b/src/config/io.observe-recovery.test.ts
@@ -5,6 +5,7 @@ import path from "node:path";
 import JSON5 from "json5";
 import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
 import { CONFIG_CLOBBER_SNAPSHOT_LIMIT } from "./io.clobber-snapshot.js";
+import { createConfigIO } from "./io.js";
 import {
  maybeRecoverSuspiciousConfigRead,
  maybeRecoverSuspiciousConfigReadSync,
@@ -135,6 +136,29 @@ describe("config observe recovery", () => {
    return (await readObserveEvents(auditPath)).at(-1);
  }

+  function createTestConfigIO(
+    home: string,
+    warn = vi.fn(),
+    options: { env?: NodeJS.ProcessEnv; observe?: boolean } = {},
+  ) {
+    const configPath = path.join(home, ".openclaw", "openclaw.json");
+    const error = vi.fn();
+    return {
+      configPath,
+      warn,
+      error,
+      io: createConfigIO({
+        fs,
+        json5: JSON5,
+        env: options.env ?? ({} as NodeJS.ProcessEnv),
+        homedir: () => home,
+        configPath,
+        logger: { warn, error },
+        ...(options.observe === false ? { observe: false } : {}),
+      }),
+    };
+  }
+
  async function recoverClobberedUpdateChannel(params: {
    deps: ObserveRecoveryDeps;
    configPath: string;
@@ -368,6 +392,180 @@ describe("config observe recovery", () => {
    });
  });

+  it("read snapshots auto-restore tiny valid clobbers before recording them observed", async () => {
+    await withSuiteHome(async (home) => {
+      const { io, configPath, warn } = createTestConfigIO(home);
+      const auditPath = path.join(home, ".openclaw", "logs", "config-audit.jsonl");
+      await seedConfigBackup(configPath, {
+        ...recoverableTelegramConfig,
+        channels: {
+          telegram: {
+            enabled: true,
+            dmPolicy: "pairing",
+            groupPolicy: "allowlist",
+            allowFrom: Array.from({ length: 60 }, (_, index) => `telegram-user-${index}`),
+          },
+        },
+      });
+      const clobbered = await writeConfigRaw(configPath, {
+        meta: { lastTouchedVersion: "2026.5.28" },
+      });
+
+      const snapshot = await io.readConfigFileSnapshot({ recoverSuspicious: true });
+
+      expect(snapshot.valid).toBe(true);
+      expect(snapshot.config.gateway?.mode).toBe("local");
+      await expect(fsp.readFile(configPath, "utf-8")).resolves.not.toBe(clobbered.raw);
+      expectWarnContaining(warn, "Config auto-restored from backup:");
+      const observeEvents = await readObserveEvents(auditPath);
+      expect(observeEvents).toHaveLength(1);
+      expect(observeEvents[0]?.restoredFromBackup).toBe(true);
+      expectSuspiciousMatching(observeEvents[0], /^size-drop-vs-last-good:/);
+      expectSuspiciousIncludes(observeEvents[0], "gateway-mode-missing-vs-last-good");
+      await expect(listClobberFiles(configPath)).resolves.toHaveLength(1);
+    });
+  });
+
+  it("loadConfig auto-restores tiny valid clobbers before using defaults", async () => {
+    await withSuiteHome(async (home) => {
+      const { io, configPath, warn } = createTestConfigIO(home);
+      await seedConfigBackup(configPath, recoverableTelegramConfig);
+      await writeConfigRaw(configPath, {
+        meta: { lastTouchedVersion: "2026.5.28" },
+      });
+
+      const config = io.loadConfig();
+
+      expect(config.gateway?.mode).toBe("local");
+      expectWarnContaining(warn, "Config auto-restored from backup:");
+    });
+  });
+
+  it("loadConfig clears env vars from the discarded clobbered config before rereading backup", async () => {
+    await withSuiteHome(async (home) => {
+      const env = {} as NodeJS.ProcessEnv;
+      const { io, configPath } = createTestConfigIO(home, vi.fn(), { env });
+      await seedConfigBackup(configPath, recoverableTelegramConfig);
+      await writeConfigRaw(configPath, {
+        meta: { lastTouchedVersion: "2026.5.28" },
+        env: { vars: { OPENCLAW_CLOBBER_ONLY: "bad" } },
+      });
+
+      const config = io.loadConfig();
+
+      expect(config.gateway?.mode).toBe("local");
+      expect(env.OPENCLAW_CLOBBER_ONLY).toBeUndefined();
+    });
+  });
+
+  it("read snapshot recovery clears env vars from the discarded clobbered config", async () => {
+    await withSuiteHome(async (home) => {
+      const env = {} as NodeJS.ProcessEnv;
+      const { io, configPath } = createTestConfigIO(home, vi.fn(), { env });
+      await seedConfigBackup(configPath, recoverableTelegramConfig);
+      await writeConfigRaw(configPath, {
+        meta: { lastTouchedVersion: "2026.5.28" },
+        env: { vars: { OPENCLAW_CLOBBER_ONLY: "bad" } },
+      });
+
+      const snapshot = await io.readConfigFileSnapshot({ recoverSuspicious: true });
+
+      expect(snapshot.config.gateway?.mode).toBe("local");
+      expect(env.OPENCLAW_CLOBBER_ONLY).toBeUndefined();
+    });
+  });
+
+  it("does not auto-restore read snapshots when observation is disabled", async () => {
+    await withSuiteHome(async (home) => {
+      const { io, configPath } = createTestConfigIO(home, vi.fn(), { observe: false });
+      const auditPath = path.join(home, ".openclaw", "logs", "config-audit.jsonl");
+      await seedConfigBackup(configPath, recoverableTelegramConfig);
+      const clobbered = await writeConfigRaw(configPath, {
+        meta: { lastTouchedVersion: "2026.5.28" },
+      });
+
+      const snapshot = await io.readConfigFileSnapshot({ recoverSuspicious: true });
+
+      expect(snapshot.valid).toBe(true);
+      expect(snapshot.config.gateway?.mode).toBeUndefined();
+      await expect(fsp.readFile(configPath, "utf-8")).resolves.toBe(clobbered.raw);
+      await expectPathMissing(auditPath);
+    });
+  });
+
+  it("does not auto-restore include-authored roots from stale full-file backups", async () => {
+    await withSuiteHome(async (home) => {
+      const { io, configPath } = createTestConfigIO(home);
+      const auditPath = path.join(home, ".openclaw", "logs", "config-audit.jsonl");
+      const includedConfig = {
+        ...recoverableTelegramConfig,
+        channels: {
+          telegram: {
+            enabled: true,
+            dmPolicy: "pairing",
+            groupPolicy: "allowlist",
+            allowFrom: Array.from({ length: 60 }, (_, index) => `telegram-user-${index}`),
+          },
+        },
+      };
+      await seedConfigBackup(configPath, includedConfig);
+      await fsp.writeFile(
+        path.join(path.dirname(configPath), "base.json5"),
+        `${JSON.stringify(includedConfig, null, 2)}\n`,
+        "utf-8",
+      );
+      const includeRootRaw = `{\n  "$include": "./base.json5"\n}\n`;
+      await fsp.writeFile(configPath, includeRootRaw, "utf-8");
+
+      const snapshot = await io.readConfigFileSnapshot({ recoverSuspicious: true });
+
+      expect(snapshot.valid).toBe(true);
+      expect(snapshot.config.gateway?.mode).toBe("local");
+      await expect(fsp.readFile(configPath, "utf-8")).resolves.toBe(includeRootRaw);
+      const observe = await readLastObserveEvent(auditPath);
+      expect(observe?.restoredFromBackup).toBe(false);
+    });
+  });
+
+  it("does not auto-restore invalid backup candidates during opted-in reads", async () => {
+    await withSuiteHome(async (home) => {
+      const { io, configPath } = createTestConfigIO(home);
+      await seedConfigBackup(configPath, {
+        gateway: { mode: "local" },
+        agents: { defaults: { model: 123 } },
+      });
+      const clobbered = await writeConfigRaw(configPath, {
+        meta: { lastTouchedVersion: "2026.5.28" },
+      });
+
+      const snapshot = await io.readConfigFileSnapshot({ recoverSuspicious: true });
+
+      expect(snapshot.valid).toBe(true);
+      expect(snapshot.config.gateway?.mode).toBeUndefined();
+      await expect(fsp.readFile(configPath, "utf-8")).resolves.toBe(clobbered.raw);
+      await expect(listClobberFiles(configPath)).resolves.toHaveLength(0);
+    });
+  });
+
+  it("validates backup candidates without leaking their env into live state", async () => {
+    await withSuiteHome(async (home) => {
+      const env = {} as NodeJS.ProcessEnv;
+      const { io, configPath } = createTestConfigIO(home, vi.fn(), { env });
+      await seedConfigBackup(configPath, {
+        gateway: { mode: "local" },
+        env: { vars: { OPENCLAW_BACKUP_ONLY: "stale" } },
+        agents: { defaults: { model: 123 } },
+      });
+      await writeConfigRaw(configPath, {
+        meta: { lastTouchedVersion: "2026.5.28" },
+      });
+
+      await io.readConfigFileSnapshot({ recoverSuspicious: true });
+
+      expect(env.OPENCLAW_BACKUP_ONLY).toBeUndefined();
+    });
+  });
+
  it("does not restore noncritical config edits", async () => {
    await withSuiteHome(async (home) => {
      const { deps, configPath, auditPath } = makeDeps(home);
--- a/src/config/io.observe-recovery.ts
+++ b/src/config/io.observe-recovery.ts
@@ -131,6 +131,8 @@ type ConfigReadRecoveryParams = {
  configPath: string;
  raw: string;
  parsed: unknown;
+  validateBackup?: (backup: { raw: string; parsed: unknown }) => Promise<boolean>;
+  validateBackupSync?: (backup: { raw: string; parsed: unknown }) => boolean;
 };

 type ConfigReadRecoveryResult = {
@@ -710,6 +712,12 @@ export async function maybeRecoverSuspiciousConfigRead(
  if (!backupParse) {
    return returnOriginalConfigRead(params);
  }
+  if (
+    params.validateBackup &&
+    !(await params.validateBackup({ raw: backupRaw, parsed: backupParse.parsed }))
+  ) {
+    return returnOriginalConfigRead(params);
+  }
  const backup = backupBaseline ?? (await readConfigFingerprintForPath(params.deps, backupPath));
  if (!backup?.gatewayMode) {
    return returnOriginalConfigRead(params);
@@ -811,6 +819,12 @@ export function maybeRecoverSuspiciousConfigReadSync(
  if (!backupParse) {
    return returnOriginalConfigRead(params);
  }
+  if (
+    params.validateBackupSync &&
+    !params.validateBackupSync({ raw: backupRaw, parsed: backupParse.parsed })
+  ) {
+    return returnOriginalConfigRead(params);
+  }
  const backup = backupBaseline ?? readConfigFingerprintForPathSync(params.deps, backupPath);
  if (!backup?.gatewayMode) {
    return returnOriginalConfigRead(params);
--- a/src/config/io.ts
+++ b/src/config/io.ts
@@ -41,6 +41,7 @@ import {
 import { applyConfigEnvVars } from "./env-vars.js";
 import {
  ConfigIncludeError,
+  INCLUDE_KEY,
  readConfigIncludeFileWithGuards,
  resolveConfigIncludes,
 } from "./includes.js";
@@ -57,6 +58,8 @@ import { persistBoundedClobberedConfigSnapshot } from "./io.clobber-snapshot.js"
 import { throwInvalidConfig } from "./io.invalid-config.js";
 import { stampConfigWriteMetadata } from "./io.meta.js";
 import {
+  maybeRecoverSuspiciousConfigRead as maybeRecoverSuspiciousConfigReadWithDeps,
+  maybeRecoverSuspiciousConfigReadSync as maybeRecoverSuspiciousConfigReadSyncWithDeps,
  promoteConfigSnapshotToLastKnownGood as promoteConfigSnapshotToLastKnownGoodWithDeps,
  recoverConfigFromLastKnownGood as recoverConfigFromLastKnownGoodWithDeps,
 } from "./io.observe-recovery.js";
@@ -422,6 +425,19 @@ function collectEnvRefPaths(value: unknown, pathLocal: string, output: Map<strin
  }
 }

+function containsConfigIncludeDirective(value: unknown): boolean {
+  if (Array.isArray(value)) {
+    return value.some((item) => containsConfigIncludeDirective(item));
+  }
+  if (!isRecord(value)) {
+    return false;
+  }
+  if (INCLUDE_KEY in value) {
+    return true;
+  }
+  return Object.values(value).some((item) => containsConfigIncludeDirective(item));
+}
+
 function resolveConfigHealthStatePath(env: NodeJS.ProcessEnv, homedir: () => string): string {
  return path.join(resolveStateDir(env, homedir), "logs", CONFIG_HEALTH_STATE_FILENAME);
 }
@@ -950,6 +966,7 @@ export type ConfigIoDeps = {
 export type ConfigSnapshotReadOptions = {
  measure?: ConfigSnapshotReadMeasure;
  observe?: boolean;
+  recoverSuspicious?: boolean;
  skipPluginValidation?: boolean;
  preservedLegacyRootKeys?: readonly string[];
  suppressFutureVersionWarning?: boolean;
@@ -1550,9 +1567,59 @@ export function createConfigIO(
    return false;
  }

-  function loadConfigLocal(): OpenClawConfig {
+  function validateSuspiciousRecoveryBackup(parsed: unknown): boolean {
+    try {
+      const candidateEnv = { ...deps.env } as NodeJS.ProcessEnv;
+      const candidateDeps = { ...deps, env: candidateEnv };
+      const resolved = resolveConfigIncludesForRead(parsed, configPath, candidateDeps);
+      const readResolution = resolveConfigForRead(resolved, candidateEnv);
+      const installMigration = migrateAndStripShippedPluginInstallConfigRecords(
+        readResolution.resolvedConfigRaw,
+        {
+          persist: false,
+          rootConfigRaw: parsed,
+        },
+      );
+      const effectiveConfigRaw = installMigration.config;
+      const validationConfigRaw = installMigration.validationConfig ?? effectiveConfigRaw;
+      let pluginMetadataSnapshot: PluginMetadataSnapshot | undefined;
+      const loadValidationPluginMetadataSnapshot = (config: OpenClawConfig) => {
+        if (pluginMetadataSnapshot) {
+          return pluginMetadataSnapshot;
+        }
+        const metadataConfig = retainRuntimeOnlyShippedPluginInstallConfigRecords(
+          config,
+          effectiveConfigRaw,
+        );
+        const defaultAgentId = resolveDefaultAgentId(metadataConfig);
+        pluginMetadataSnapshot = resolvePluginMetadataSnapshot({
+          config: metadataConfig,
+          workspaceDir: resolveAgentWorkspaceDir(metadataConfig, defaultAgentId),
+          env: candidateEnv,
+          allowWorkspaceScopedCurrent: true,
+          pluginIdScope: createConfigValidationMetadataPluginIdScope({
+            config: metadataConfig,
+            env: candidateEnv,
+          }),
+        });
+        return pluginMetadataSnapshot;
+      };
+      return validateConfigObjectWithPlugins(validationConfigRaw, {
+        env: candidateEnv,
+        pluginValidation: overrides.pluginValidation,
+        loadPluginMetadataSnapshot: loadValidationPluginMetadataSnapshot,
+        sourceRaw: parsed,
+        preservedLegacyRootKeys: overrides.preservedLegacyRootKeys,
+      }).ok;
+    } catch {
+      return false;
+    }
+  }
+
+  function loadConfigLocal(options: { skipSuspiciousRecovery?: boolean } = {}): OpenClawConfig {
    try {
      maybeLoadDotEnvForConfig(deps.env);
+      const envBeforeRead = snapshotEnv(deps.env);
      if (!deps.fs.existsSync(configPath)) {
        if (
          overrides.shellEnvFallback !== "defer" &&
@@ -1683,6 +1750,27 @@ export function createConfigIO(
      if (!deps.suppressFutureVersionWarning) {
        warnIfConfigFromFuture(validated.config, deps.logger);
      }
+      if (
+        deps.observe &&
+        !options.skipSuspiciousRecovery &&
+        !containsConfigIncludeDirective(parsed)
+      ) {
+        const recovery = maybeRecoverSuspiciousConfigReadSyncWithDeps({
+          deps,
+          configPath,
+          raw,
+          parsed,
+          validateBackupSync: (backup) => validateSuspiciousRecoveryBackup(backup.parsed),
+        });
+        if (recovery.raw !== raw) {
+          restoreEnvChangesIfUnchanged({
+            env: deps.env,
+            before: envBeforeRead,
+            after: snapshotEnv(deps.env),
+          });
+          return loadConfigLocal({ skipSuspiciousRecovery: true });
+        }
+      }
      const cfg = retainRuntimeOnlyShippedPluginInstallConfigRecords(
        materializeRuntimeConfig(validated.config, "load", {
          manifestRegistry: pluginMetadataSnapshot?.manifestRegistry,
@@ -1720,8 +1808,11 @@ export function createConfigIO(
    }
  }

-  async function readConfigFileSnapshotInternal(): Promise<ReadConfigFileSnapshotInternalResult> {
+  async function readConfigFileSnapshotInternal(
+    options: { recoverSuspicious?: boolean; skipSuspiciousRecovery?: boolean } = {},
+  ): Promise<ReadConfigFileSnapshotInternalResult> {
    maybeLoadDotEnvForConfig(deps.env);
+    const envBeforeRead = snapshotEnv(deps.env);
    const exists = deps.fs.existsSync(configPath);
    if (!exists) {
      const hash = hashConfigRaw(null);
@@ -1898,6 +1989,33 @@ export function createConfigIO(
      if (!deps.suppressFutureVersionWarning) {
        warnIfConfigFromFuture(validated.config, deps.logger);
      }
+      if (
+        options.recoverSuspicious === true &&
+        deps.observe &&
+        !options.skipSuspiciousRecovery &&
+        !containsConfigIncludeDirective(effectiveParsed)
+      ) {
+        const recovery = await deps.measure("config.snapshot.read.recover-suspicious", () =>
+          maybeRecoverSuspiciousConfigReadWithDeps({
+            deps,
+            configPath,
+            raw,
+            parsed: effectiveParsed,
+            validateBackup: async (backup) => validateSuspiciousRecoveryBackup(backup.parsed),
+          }),
+        );
+        if (recovery.raw !== raw) {
+          restoreEnvChangesIfUnchanged({
+            env: deps.env,
+            before: envBeforeRead,
+            after: snapshotEnv(deps.env),
+          });
+          return await readConfigFileSnapshotInternal({
+            recoverSuspicious: options.recoverSuspicious,
+            skipSuspiciousRecovery: true,
+          });
+        }
+      }
      const snapshotConfig = await deps.measure("config.snapshot.read.materialize", () =>
        retainRuntimeOnlyShippedPluginInstallConfigRecords(
          materializeRuntimeConfig(validated.config, "snapshot", {
@@ -1965,13 +2083,21 @@ export function createConfigIO(
    }
  }

-  async function readConfigFileSnapshotLocal(): Promise<ConfigFileSnapshot> {
-    const result = await readConfigFileSnapshotInternal();
+  async function readConfigFileSnapshotLocal(
+    options: ConfigSnapshotReadOptions = {},
+  ): Promise<ConfigFileSnapshot> {
+    const result = await readConfigFileSnapshotInternal({
+      recoverSuspicious: options.recoverSuspicious === true,
+    });
    return result.snapshot;
  }

-  async function readConfigFileSnapshotWithPluginMetadataLocal(): Promise<ReadConfigFileSnapshotWithPluginMetadataResult> {
-    const result = await readConfigFileSnapshotInternal();
+  async function readConfigFileSnapshotWithPluginMetadataLocal(
+    options: ConfigSnapshotReadOptions = {},
+  ): Promise<ReadConfigFileSnapshotWithPluginMetadataResult> {
+    const result = await readConfigFileSnapshotInternal({
+      recoverSuspicious: options.recoverSuspicious === true,
+    });
    return {
      snapshot: result.snapshot,
      ...(result.pluginMetadataSnapshot
@@ -2525,15 +2651,20 @@ export async function readConfigFileSnapshot(
    ...(options.preservedLegacyRootKeys
      ? { preservedLegacyRootKeys: options.preservedLegacyRootKeys }
      : {}),
-  }).readConfigFileSnapshot();
+  }).readConfigFileSnapshot({
+    recoverSuspicious: options.recoverSuspicious === true,
+  });
 }

 export async function readConfigFileSnapshotWithPluginMetadata(options?: {
  measure?: ConfigSnapshotReadMeasure;
+  recoverSuspicious?: boolean;
 }): Promise<ReadConfigFileSnapshotWithPluginMetadataResult> {
  return await createConfigIO(
    options?.measure ? { measure: options.measure } : {},
-  ).readConfigFileSnapshotWithPluginMetadata();
+  ).readConfigFileSnapshotWithPluginMetadata({
+    recoverSuspicious: options?.recoverSuspicious === true,
+  });
 }

 export async function promoteConfigSnapshotToLastKnownGood(