fix(secrets): skip optional web fetch discovery before bind

2026-05-06 07:00:43 +00:00 · 2026-04-30 14:42:18 +01:00
parent 3766bbb674
commit afb17eade9
5 changed files with 320 additions and 37 deletions
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,7 @@ Docs: https://docs.openclaw.ai
 ### Fixes

 - Agents/subagents: bound automatic orphan recovery with persisted recovery attempts and a wedged-session tombstone, and teach task maintenance/doctor to reconcile those sessions so restart loops no longer require manual `sessions.json` surgery. Fixes #74864. Thanks @solosage1.
+- Gateway/startup: skip pre-bind web-fetch provider discovery for credential-free `tools.web.fetch` config, so Docker/Kubernetes gateways bind even when optional fetch limits are present. Fixes #74896. Thanks @KoykL.
 - CLI/progress: suppress nested progress spinners and line clears while TUI input owns raw stdin, so Crestodian `/status` no longer disturbs the active input row. (#75003) Thanks @velvet-shark.
 - Models/OpenAI Codex: restore `openai-codex/gpt-5.4-mini` for ChatGPT/Codex OAuth PI runs after live OAuth proof, and align the manifest, forward-compat metadata, docs, and regression tests so stale cron and heartbeat configs resolve again. Fixes #74451. Thanks @0xCyda, @hclsys, and @Marvae.
 - Telegram: use durable message edits for streaming previews instead of native draft state, so generated replies no longer flicker through draft-to-message transitions that look like duplicates. (#75073) Thanks @obviyus.
--- a/src/secrets/runtime-web-tools.test.ts
+++ b/src/secrets/runtime-web-tools.test.ts
@@ -300,24 +300,6 @@ function readProviderKey(config: OpenClawConfig, provider: ProviderUnderTest): u
  return pluginConfig?.webSearch?.apiKey;
 }

-function expectInactiveWebFetchProviderSecretRef(params: {
-  resolveSpy: ReturnType<typeof vi.spyOn>;
-  metadata: Awaited<ReturnType<typeof runRuntimeWebTools>>["metadata"];
-  context: Awaited<ReturnType<typeof runRuntimeWebTools>>["context"];
-}) {
-  expect(params.resolveSpy).not.toHaveBeenCalled();
-  expect(params.metadata.fetch.selectedProvider).toBeUndefined();
-  expect(params.metadata.fetch.selectedProviderKeySource).toBeUndefined();
-  expect(params.context.warnings).toEqual(
-    expect.arrayContaining([
-      expect.objectContaining({
-        code: "SECRETS_REF_IGNORED_INACTIVE_SURFACE",
-        path: "plugins.entries.firecrawl.config.webFetch.apiKey",
-      }),
-    ]),
-  );
-}
-
 describe("runtime web tools resolution", () => {
  beforeAll(async () => {
    secretResolve = await import("./resolve.js");
@@ -416,6 +398,105 @@ describe("runtime web tools resolution", () => {
    expect(resolvePluginWebFetchProvidersMock).not.toHaveBeenCalled();
  });

+  it("skips fetch provider discovery when web fetch only configures runtime limits", async () => {
+    const { metadata } = await runRuntimeWebTools({
+      config: asConfig({
+        tools: {
+          web: {
+            fetch: {
+              enabled: true,
+              maxChars: 200_000,
+              maxCharsCap: 2_000_000,
+            },
+          },
+        },
+        plugins: {
+          enabled: true,
+          allow: [],
+          entries: {},
+        },
+      }),
+      env: {
+        FIRECRAWL_API_KEY: "firecrawl-key-should-not-resolve", // pragma: allowlist secret
+      },
+    });
+
+    expect(metadata.fetch.providerSource).toBe("none");
+    expect(metadata.fetch.selectedProvider).toBeUndefined();
+    expect(resolveBundledExplicitWebFetchProvidersFromPublicArtifactsMock).not.toHaveBeenCalled();
+    expect(resolveBundledWebFetchProvidersFromPublicArtifactsMock).not.toHaveBeenCalled();
+    expect(resolvePluginWebFetchProvidersMock).not.toHaveBeenCalled();
+  });
+
+  it("skips fetch provider discovery when web fetch is explicitly disabled", async () => {
+    const { metadata } = await runRuntimeWebTools({
+      config: asConfig({
+        tools: {
+          web: {
+            fetch: {
+              enabled: false,
+              provider: "firecrawl",
+            },
+          },
+        },
+        plugins: {
+          entries: {
+            firecrawl: {
+              config: {
+                webFetch: {
+                  apiKey: { source: "env", provider: "default", id: "FIRECRAWL_API_KEY" },
+                },
+              },
+            },
+          },
+        },
+      }),
+      env: {
+        FIRECRAWL_API_KEY: "firecrawl-key-should-not-resolve", // pragma: allowlist secret
+      },
+    });
+
+    expect(metadata.fetch.providerSource).toBe("none");
+    expect(metadata.fetch.selectedProvider).toBeUndefined();
+    expect(resolveBundledExplicitWebFetchProvidersFromPublicArtifactsMock).not.toHaveBeenCalled();
+    expect(resolveBundledWebFetchProvidersFromPublicArtifactsMock).not.toHaveBeenCalled();
+    expect(resolvePluginWebFetchProvidersMock).not.toHaveBeenCalled();
+  });
+
+  it("keeps active fetch provider SecretRefs on the discovery path", async () => {
+    const { metadata } = await runRuntimeWebTools({
+      config: asConfig({
+        tools: {
+          web: {
+            fetch: {
+              provider: "firecrawl",
+            },
+          },
+        },
+        plugins: {
+          entries: {
+            firecrawl: {
+              config: {
+                webFetch: {
+                  apiKey: { source: "env", provider: "default", id: "FIRECRAWL_API_KEY" },
+                },
+              },
+            },
+          },
+        },
+      }),
+      env: {
+        FIRECRAWL_API_KEY: "firecrawl-key", // pragma: allowlist secret
+      },
+    });
+
+    expect(metadata.fetch.providerSource).toBe("configured");
+    expect(metadata.fetch.selectedProvider).toBe("firecrawl");
+    expect(resolveBundledExplicitWebFetchProvidersFromPublicArtifactsMock).toHaveBeenCalledWith({
+      onlyPluginIds: ["firecrawl"],
+    });
+  });
+
  it("auto-selects a keyless provider when no credentials are configured", async () => {
    const { metadata } = await runRuntimeWebTools({
      config: asConfig({
@@ -969,7 +1050,13 @@ describe("runtime web tools resolution", () => {
      }),
    });

-    expectInactiveWebFetchProviderSecretRef({ resolveSpy, metadata, context });
+    expect(resolveSpy).not.toHaveBeenCalled();
+    expect(metadata.fetch.selectedProvider).toBeUndefined();
+    expect(metadata.fetch.selectedProviderKeySource).toBeUndefined();
+    expect(context.warnings).toEqual([]);
+    expect(resolveBundledExplicitWebFetchProvidersFromPublicArtifactsMock).not.toHaveBeenCalled();
+    expect(resolveBundledWebFetchProvidersFromPublicArtifactsMock).not.toHaveBeenCalled();
+    expect(resolvePluginWebFetchProvidersMock).not.toHaveBeenCalled();
  });

  it("keeps configured provider metadata and inactive warnings when search is disabled", async () => {
@@ -1151,17 +1238,18 @@ describe("runtime web tools resolution", () => {

    const { metadata } = await runRuntimeWebTools({
      config: asConfig({
-        tools: {
-          web: {
-            fetch: {
-              enabled: true,
+        plugins: {
+          entries: {
+            firecrawl: {
+              config: {
+                webFetch: {
+                  apiKey: "firecrawl-config-key",
+                },
+              },
            },
          },
        },
      }),
-      env: {
-        FIRECRAWL_API_KEY: "firecrawl-key", // pragma: allowlist secret
-      },
    });

    expect(metadata.fetch.selectedProvider).toBe("firecrawl");
--- a/src/secrets/runtime-web-tools.ts
+++ b/src/secrets/runtime-web-tools.ts
@@ -66,6 +66,56 @@ type SecretResolutionSource =
  | WebSearchCredentialResolutionSource
  | WebFetchCredentialResolutionSource;

+const WEB_FETCH_CREDENTIAL_FIELD_NAMES = new Set(["apikey", "key", "token", "secret", "password"]);
+
+function hasCredentialBearingWebFetchValue(
+  value: unknown,
+  defaults: SecretDefaults | undefined,
+  seen = new WeakSet<object>(),
+): boolean {
+  if (hasConfiguredSecretRef(value, defaults)) {
+    return true;
+  }
+  if (!value || typeof value !== "object") {
+    return false;
+  }
+  if (seen.has(value)) {
+    return false;
+  }
+  seen.add(value);
+  if (Array.isArray(value)) {
+    return value.some((entry) => hasCredentialBearingWebFetchValue(entry, defaults, seen));
+  }
+  return Object.entries(value as Record<string, unknown>).some(([rawKey, entry]) => {
+    const key = rawKey.toLowerCase();
+    if (WEB_FETCH_CREDENTIAL_FIELD_NAMES.has(key) && entry != null && entry !== "") {
+      return true;
+    }
+    return hasCredentialBearingWebFetchValue(entry, defaults, seen);
+  });
+}
+
+function needsRuntimeWebFetchProviderDiscovery(params: {
+  fetch: FetchConfig;
+  rawProvider: string;
+  hasPluginWebFetchConfig: boolean;
+  defaults: SecretDefaults | undefined;
+}): boolean {
+  if (isRecord(params.fetch) && params.fetch.enabled === false) {
+    return false;
+  }
+  if (params.hasPluginWebFetchConfig) {
+    return true;
+  }
+  if (!isRecord(params.fetch)) {
+    return false;
+  }
+  if (params.rawProvider) {
+    return true;
+  }
+  return hasCredentialBearingWebFetchValue(params.fetch, params.defaults);
+}
+
 function hasPluginScopedWebToolConfig(
  config: OpenClawConfig,
  key: "webSearch" | "webFetch",
@@ -679,7 +729,13 @@ export async function resolveRuntimeWebTools(params: {
    providerSource: "none",
    diagnostics: [],
  };
-  if (fetch || hasPluginWebFetchConfig) {
+  const discoverFetchProviders = needsRuntimeWebFetchProviderDiscovery({
+    fetch,
+    rawProvider: rawFetchProvider,
+    hasPluginWebFetchConfig,
+    defaults,
+  });
+  if (discoverFetchProviders) {
    const fetchSurface = await resolveRuntimeWebProviderSurface({
      contract: "webFetchProviders",
      rawProvider: rawFetchProvider,
--- a/src/secrets/runtime.fast-path.test.ts
+++ b/src/secrets/runtime.fast-path.test.ts
@@ -6,7 +6,14 @@ import { setActivePluginRegistry } from "../plugins/runtime.js";
 import { clearSecretsRuntimeSnapshot } from "./runtime.js";
 import { asConfig } from "./runtime.test-support.js";

-const runtimePrepareImportMock = vi.hoisted(() => vi.fn());
+const { resolveRuntimeWebToolsMock, runtimePrepareImportMock } = vi.hoisted(() => ({
+  resolveRuntimeWebToolsMock: vi.fn(async () => ({
+    search: { providerSource: "none", diagnostics: [] },
+    fetch: { providerSource: "none", diagnostics: [] },
+    diagnostics: [],
+  })),
+  runtimePrepareImportMock: vi.fn(),
+}));

 vi.mock("./runtime-prepare.runtime.js", () => {
  runtimePrepareImportMock();
@@ -23,11 +30,7 @@ vi.mock("./runtime-prepare.runtime.js", () => {
    collectAuthStoreAssignments: () => undefined,
    resolveSecretRefValues: async () => new Map(),
    applyResolvedAssignments: () => undefined,
-    resolveRuntimeWebTools: async () => ({
-      search: { providerSource: "none", diagnostics: [] },
-      fetch: { providerSource: "none", diagnostics: [] },
-      diagnostics: [],
-    }),
+    resolveRuntimeWebTools: resolveRuntimeWebToolsMock,
  };
 });

@@ -38,6 +41,7 @@ function emptyAuthStore(): AuthProfileStore {
 describe("secrets runtime fast path", () => {
  afterEach(() => {
    runtimePrepareImportMock.mockClear();
+    resolveRuntimeWebToolsMock.mockClear();
    setActivePluginRegistry(createEmptyPluginRegistry());
    clearSecretsRuntimeSnapshot();
    clearRuntimeConfigSnapshot();
@@ -72,6 +76,57 @@ describe("secrets runtime fast path", () => {
    ]);
  });

+  it("uses the fast path when web fetch only configures runtime limits", async () => {
+    const { prepareSecretsRuntimeSnapshot } = await import("./runtime.js");
+
+    const snapshot = await prepareSecretsRuntimeSnapshot({
+      config: asConfig({
+        tools: {
+          web: {
+            fetch: {
+              enabled: true,
+              maxChars: 200_000,
+              maxCharsCap: 2_000_000,
+            },
+          },
+        },
+        plugins: {
+          enabled: true,
+          allow: [],
+          entries: {},
+        },
+      }),
+      env: {},
+      agentDirs: ["/tmp/openclaw-agent-main"],
+      loadAuthStore: emptyAuthStore,
+    });
+
+    expect(runtimePrepareImportMock).not.toHaveBeenCalled();
+    expect(snapshot.webTools.fetch.providerSource).toBe("none");
+  });
+
+  it("uses the fast path when web fetch is explicitly disabled", async () => {
+    const { prepareSecretsRuntimeSnapshot } = await import("./runtime.js");
+
+    await prepareSecretsRuntimeSnapshot({
+      config: asConfig({
+        tools: {
+          web: {
+            fetch: {
+              enabled: false,
+              maxChars: 200_000,
+            },
+          },
+        },
+      }),
+      env: {},
+      agentDirs: ["/tmp/openclaw-agent-main"],
+      loadAuthStore: emptyAuthStore,
+    });
+
+    expect(runtimePrepareImportMock).not.toHaveBeenCalled();
+  });
+
  it("uses the resolver path when an auth profile store contains a SecretRef", async () => {
    const { prepareSecretsRuntimeSnapshot } = await import("./runtime.js");

@@ -91,6 +146,27 @@ describe("secrets runtime fast path", () => {
      }),
    });

-    expect(runtimePrepareImportMock).toHaveBeenCalledTimes(1);
+    expect(resolveRuntimeWebToolsMock).toHaveBeenCalledTimes(1);
+  });
+
+  it("keeps explicit web fetch provider config on the resolver path", async () => {
+    const { prepareSecretsRuntimeSnapshot } = await import("./runtime.js");
+
+    await prepareSecretsRuntimeSnapshot({
+      config: asConfig({
+        tools: {
+          web: {
+            fetch: {
+              provider: "firecrawl",
+            },
+          },
+        },
+      }),
+      env: {},
+      agentDirs: ["/tmp/openclaw-agent-main"],
+      loadAuthStore: emptyAuthStore,
+    });
+
+    expect(resolveRuntimeWebToolsMock).toHaveBeenCalledTimes(1);
  });
 });
--- a/src/secrets/runtime.ts
+++ b/src/secrets/runtime.ts
@@ -197,10 +197,72 @@ function createEmptyRuntimeWebToolsMetadata(): RuntimeWebToolsMetadata {
  };
 }

+const WEB_FETCH_CREDENTIAL_FIELD_NAMES = new Set(["apikey", "key", "token", "secret", "password"]);
+
+function hasCredentialBearingWebFetchValue(
+  value: unknown,
+  defaults: Parameters<typeof coerceSecretRef>[1],
+  seen = new WeakSet<object>(),
+): boolean {
+  if (coerceSecretRef(value, defaults)) {
+    return true;
+  }
+  if (!value || typeof value !== "object") {
+    return false;
+  }
+  if (seen.has(value)) {
+    return false;
+  }
+  seen.add(value);
+  if (Array.isArray(value)) {
+    return value.some((entry) => hasCredentialBearingWebFetchValue(entry, defaults, seen));
+  }
+  return Object.entries(value as Record<string, unknown>).some(([rawKey, entry]) => {
+    const key = rawKey.toLowerCase();
+    if (WEB_FETCH_CREDENTIAL_FIELD_NAMES.has(key) && entry != null && entry !== "") {
+      return true;
+    }
+    return hasCredentialBearingWebFetchValue(entry, defaults, seen);
+  });
+}
+
+function hasActiveRuntimeWebFetchProviderSurface(
+  fetch: unknown,
+  defaults: Parameters<typeof coerceSecretRef>[1],
+): boolean {
+  if (!fetch || typeof fetch !== "object" || Array.isArray(fetch)) {
+    return false;
+  }
+  const fetchConfig = fetch as Record<string, unknown>;
+  if (fetchConfig.enabled === false) {
+    return false;
+  }
+  if (typeof fetchConfig.provider === "string" && fetchConfig.provider.trim()) {
+    return true;
+  }
+  return hasCredentialBearingWebFetchValue(fetchConfig, defaults);
+}
+
 function hasRuntimeWebToolConfigSurface(config: OpenClawConfig): boolean {
  const web = config.tools?.web;
-  if (web && typeof web === "object" && ("search" in web || "fetch" in web || "x_search" in web)) {
-    return true;
+  const defaults = config.secrets?.defaults;
+  const fetchExplicitlyDisabled =
+    web &&
+    typeof web === "object" &&
+    !Array.isArray(web) &&
+    typeof (web as Record<string, unknown>).fetch === "object" &&
+    (web as { fetch?: { enabled?: unknown } }).fetch?.enabled === false;
+  if (web && typeof web === "object" && !Array.isArray(web)) {
+    const webRecord = web as Record<string, unknown>;
+    if ("search" in webRecord || "x_search" in webRecord) {
+      return true;
+    }
+    if (
+      "fetch" in webRecord &&
+      hasActiveRuntimeWebFetchProviderSurface(webRecord.fetch, defaults)
+    ) {
+      return true;
+    }
  }
  const entries = config.plugins?.entries;
  if (!entries || typeof entries !== "object" || Array.isArray(entries)) {
@@ -215,7 +277,7 @@ function hasRuntimeWebToolConfigSurface(config: OpenClawConfig): boolean {
      !!pluginConfig &&
      typeof pluginConfig === "object" &&
      !Array.isArray(pluginConfig) &&
-      ("webSearch" in pluginConfig || "webFetch" in pluginConfig)
+      ("webSearch" in pluginConfig || (!fetchExplicitlyDisabled && "webFetch" in pluginConfig))
    );
  });
 }