From afb17eade9cc1321d84f3f85e81318ab927af6bc Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Thu, 30 Apr 2026 14:42:18 +0100 Subject: [PATCH] fix(secrets): skip optional web fetch discovery before bind --- CHANGELOG.md | 1 + src/secrets/runtime-web-tools.test.ts | 140 +++++++++++++++++++++----- src/secrets/runtime-web-tools.ts | 58 ++++++++++- src/secrets/runtime.fast-path.test.ts | 90 +++++++++++++++-- src/secrets/runtime.ts | 68 ++++++++++++- 5 files changed, 320 insertions(+), 37 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 453f8a1d175..fbade503071 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ Docs: https://docs.openclaw.ai ### Fixes - Agents/subagents: bound automatic orphan recovery with persisted recovery attempts and a wedged-session tombstone, and teach task maintenance/doctor to reconcile those sessions so restart loops no longer require manual `sessions.json` surgery. Fixes #74864. Thanks @solosage1. +- Gateway/startup: skip pre-bind web-fetch provider discovery for credential-free `tools.web.fetch` config, so Docker/Kubernetes gateways bind even when optional fetch limits are present. Fixes #74896. Thanks @KoykL. - CLI/progress: suppress nested progress spinners and line clears while TUI input owns raw stdin, so Crestodian `/status` no longer disturbs the active input row. (#75003) Thanks @velvet-shark. - Models/OpenAI Codex: restore `openai-codex/gpt-5.4-mini` for ChatGPT/Codex OAuth PI runs after live OAuth proof, and align the manifest, forward-compat metadata, docs, and regression tests so stale cron and heartbeat configs resolve again. Fixes #74451. Thanks @0xCyda, @hclsys, and @Marvae. - Telegram: use durable message edits for streaming previews instead of native draft state, so generated replies no longer flicker through draft-to-message transitions that look like duplicates. (#75073) Thanks @obviyus. diff --git a/src/secrets/runtime-web-tools.test.ts b/src/secrets/runtime-web-tools.test.ts index dce54b6aa68..95b43cdce13 100644 --- a/src/secrets/runtime-web-tools.test.ts +++ b/src/secrets/runtime-web-tools.test.ts @@ -300,24 +300,6 @@ function readProviderKey(config: OpenClawConfig, provider: ProviderUnderTest): u return pluginConfig?.webSearch?.apiKey; } -function expectInactiveWebFetchProviderSecretRef(params: { - resolveSpy: ReturnType; - metadata: Awaited>["metadata"]; - context: Awaited>["context"]; -}) { - expect(params.resolveSpy).not.toHaveBeenCalled(); - expect(params.metadata.fetch.selectedProvider).toBeUndefined(); - expect(params.metadata.fetch.selectedProviderKeySource).toBeUndefined(); - expect(params.context.warnings).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - code: "SECRETS_REF_IGNORED_INACTIVE_SURFACE", - path: "plugins.entries.firecrawl.config.webFetch.apiKey", - }), - ]), - ); -} - describe("runtime web tools resolution", () => { beforeAll(async () => { secretResolve = await import("./resolve.js"); @@ -416,6 +398,105 @@ describe("runtime web tools resolution", () => { expect(resolvePluginWebFetchProvidersMock).not.toHaveBeenCalled(); }); + it("skips fetch provider discovery when web fetch only configures runtime limits", async () => { + const { metadata } = await runRuntimeWebTools({ + config: asConfig({ + tools: { + web: { + fetch: { + enabled: true, + maxChars: 200_000, + maxCharsCap: 2_000_000, + }, + }, + }, + plugins: { + enabled: true, + allow: [], + entries: {}, + }, + }), + env: { + FIRECRAWL_API_KEY: "firecrawl-key-should-not-resolve", // pragma: allowlist secret + }, + }); + + expect(metadata.fetch.providerSource).toBe("none"); + expect(metadata.fetch.selectedProvider).toBeUndefined(); + expect(resolveBundledExplicitWebFetchProvidersFromPublicArtifactsMock).not.toHaveBeenCalled(); + expect(resolveBundledWebFetchProvidersFromPublicArtifactsMock).not.toHaveBeenCalled(); + expect(resolvePluginWebFetchProvidersMock).not.toHaveBeenCalled(); + }); + + it("skips fetch provider discovery when web fetch is explicitly disabled", async () => { + const { metadata } = await runRuntimeWebTools({ + config: asConfig({ + tools: { + web: { + fetch: { + enabled: false, + provider: "firecrawl", + }, + }, + }, + plugins: { + entries: { + firecrawl: { + config: { + webFetch: { + apiKey: { source: "env", provider: "default", id: "FIRECRAWL_API_KEY" }, + }, + }, + }, + }, + }, + }), + env: { + FIRECRAWL_API_KEY: "firecrawl-key-should-not-resolve", // pragma: allowlist secret + }, + }); + + expect(metadata.fetch.providerSource).toBe("none"); + expect(metadata.fetch.selectedProvider).toBeUndefined(); + expect(resolveBundledExplicitWebFetchProvidersFromPublicArtifactsMock).not.toHaveBeenCalled(); + expect(resolveBundledWebFetchProvidersFromPublicArtifactsMock).not.toHaveBeenCalled(); + expect(resolvePluginWebFetchProvidersMock).not.toHaveBeenCalled(); + }); + + it("keeps active fetch provider SecretRefs on the discovery path", async () => { + const { metadata } = await runRuntimeWebTools({ + config: asConfig({ + tools: { + web: { + fetch: { + provider: "firecrawl", + }, + }, + }, + plugins: { + entries: { + firecrawl: { + config: { + webFetch: { + apiKey: { source: "env", provider: "default", id: "FIRECRAWL_API_KEY" }, + }, + }, + }, + }, + }, + }), + env: { + FIRECRAWL_API_KEY: "firecrawl-key", // pragma: allowlist secret + }, + }); + + expect(metadata.fetch.providerSource).toBe("configured"); + expect(metadata.fetch.selectedProvider).toBe("firecrawl"); + expect(resolveBundledExplicitWebFetchProvidersFromPublicArtifactsMock).toHaveBeenCalledWith({ + onlyPluginIds: ["firecrawl"], + }); + }); + it("auto-selects a keyless provider when no credentials are configured", async () => { const { metadata } = await runRuntimeWebTools({ config: asConfig({ @@ -969,7 +1050,13 @@ describe("runtime web tools resolution", () => { }), }); - expectInactiveWebFetchProviderSecretRef({ resolveSpy, metadata, context }); + expect(resolveSpy).not.toHaveBeenCalled(); + expect(metadata.fetch.selectedProvider).toBeUndefined(); + expect(metadata.fetch.selectedProviderKeySource).toBeUndefined(); + expect(context.warnings).toEqual([]); + expect(resolveBundledExplicitWebFetchProvidersFromPublicArtifactsMock).not.toHaveBeenCalled(); + expect(resolveBundledWebFetchProvidersFromPublicArtifactsMock).not.toHaveBeenCalled(); + expect(resolvePluginWebFetchProvidersMock).not.toHaveBeenCalled(); }); it("keeps configured provider metadata and inactive warnings when search is disabled", async () => { @@ -1151,17 +1238,18 @@ describe("runtime web tools resolution", () => { const { metadata } = await runRuntimeWebTools({ config: asConfig({ - tools: { - web: { - fetch: { - enabled: true, + plugins: { + entries: { + firecrawl: { + config: { + webFetch: { + apiKey: "firecrawl-config-key", + }, + }, }, }, }, }), - env: { - FIRECRAWL_API_KEY: "firecrawl-key", // pragma: allowlist secret - }, }); expect(metadata.fetch.selectedProvider).toBe("firecrawl"); diff --git a/src/secrets/runtime-web-tools.ts b/src/secrets/runtime-web-tools.ts index 6378d5f2777..1b24859c9be 100644 --- a/src/secrets/runtime-web-tools.ts +++ b/src/secrets/runtime-web-tools.ts @@ -66,6 +66,56 @@ type SecretResolutionSource = | WebSearchCredentialResolutionSource | WebFetchCredentialResolutionSource; +const WEB_FETCH_CREDENTIAL_FIELD_NAMES = new Set(["apikey", "key", "token", "secret", "password"]); + +function hasCredentialBearingWebFetchValue( + value: unknown, + defaults: SecretDefaults | undefined, + seen = new WeakSet(), +): boolean { + if (hasConfiguredSecretRef(value, defaults)) { + return true; + } + if (!value || typeof value !== "object") { + return false; + } + if (seen.has(value)) { + return false; + } + seen.add(value); + if (Array.isArray(value)) { + return value.some((entry) => hasCredentialBearingWebFetchValue(entry, defaults, seen)); + } + return Object.entries(value as Record).some(([rawKey, entry]) => { + const key = rawKey.toLowerCase(); + if (WEB_FETCH_CREDENTIAL_FIELD_NAMES.has(key) && entry != null && entry !== "") { + return true; + } + return hasCredentialBearingWebFetchValue(entry, defaults, seen); + }); +} + +function needsRuntimeWebFetchProviderDiscovery(params: { + fetch: FetchConfig; + rawProvider: string; + hasPluginWebFetchConfig: boolean; + defaults: SecretDefaults | undefined; +}): boolean { + if (isRecord(params.fetch) && params.fetch.enabled === false) { + return false; + } + if (params.hasPluginWebFetchConfig) { + return true; + } + if (!isRecord(params.fetch)) { + return false; + } + if (params.rawProvider) { + return true; + } + return hasCredentialBearingWebFetchValue(params.fetch, params.defaults); +} + function hasPluginScopedWebToolConfig( config: OpenClawConfig, key: "webSearch" | "webFetch", @@ -679,7 +729,13 @@ export async function resolveRuntimeWebTools(params: { providerSource: "none", diagnostics: [], }; - if (fetch || hasPluginWebFetchConfig) { + const discoverFetchProviders = needsRuntimeWebFetchProviderDiscovery({ + fetch, + rawProvider: rawFetchProvider, + hasPluginWebFetchConfig, + defaults, + }); + if (discoverFetchProviders) { const fetchSurface = await resolveRuntimeWebProviderSurface({ contract: "webFetchProviders", rawProvider: rawFetchProvider, diff --git a/src/secrets/runtime.fast-path.test.ts b/src/secrets/runtime.fast-path.test.ts index 52569300b4f..b8fae79cc30 100644 --- a/src/secrets/runtime.fast-path.test.ts +++ b/src/secrets/runtime.fast-path.test.ts @@ -6,7 +6,14 @@ import { setActivePluginRegistry } from "../plugins/runtime.js"; import { clearSecretsRuntimeSnapshot } from "./runtime.js"; import { asConfig } from "./runtime.test-support.js"; -const runtimePrepareImportMock = vi.hoisted(() => vi.fn()); +const { resolveRuntimeWebToolsMock, runtimePrepareImportMock } = vi.hoisted(() => ({ + resolveRuntimeWebToolsMock: vi.fn(async () => ({ + search: { providerSource: "none", diagnostics: [] }, + fetch: { providerSource: "none", diagnostics: [] }, + diagnostics: [], + })), + runtimePrepareImportMock: vi.fn(), +})); vi.mock("./runtime-prepare.runtime.js", () => { runtimePrepareImportMock(); @@ -23,11 +30,7 @@ vi.mock("./runtime-prepare.runtime.js", () => { collectAuthStoreAssignments: () => undefined, resolveSecretRefValues: async () => new Map(), applyResolvedAssignments: () => undefined, - resolveRuntimeWebTools: async () => ({ - search: { providerSource: "none", diagnostics: [] }, - fetch: { providerSource: "none", diagnostics: [] }, - diagnostics: [], - }), + resolveRuntimeWebTools: resolveRuntimeWebToolsMock, }; }); @@ -38,6 +41,7 @@ function emptyAuthStore(): AuthProfileStore { describe("secrets runtime fast path", () => { afterEach(() => { runtimePrepareImportMock.mockClear(); + resolveRuntimeWebToolsMock.mockClear(); setActivePluginRegistry(createEmptyPluginRegistry()); clearSecretsRuntimeSnapshot(); clearRuntimeConfigSnapshot(); @@ -72,6 +76,57 @@ describe("secrets runtime fast path", () => { ]); }); + it("uses the fast path when web fetch only configures runtime limits", async () => { + const { prepareSecretsRuntimeSnapshot } = await import("./runtime.js"); + + const snapshot = await prepareSecretsRuntimeSnapshot({ + config: asConfig({ + tools: { + web: { + fetch: { + enabled: true, + maxChars: 200_000, + maxCharsCap: 2_000_000, + }, + }, + }, + plugins: { + enabled: true, + allow: [], + entries: {}, + }, + }), + env: {}, + agentDirs: ["/tmp/openclaw-agent-main"], + loadAuthStore: emptyAuthStore, + }); + + expect(runtimePrepareImportMock).not.toHaveBeenCalled(); + expect(snapshot.webTools.fetch.providerSource).toBe("none"); + }); + + it("uses the fast path when web fetch is explicitly disabled", async () => { + const { prepareSecretsRuntimeSnapshot } = await import("./runtime.js"); + + await prepareSecretsRuntimeSnapshot({ + config: asConfig({ + tools: { + web: { + fetch: { + enabled: false, + maxChars: 200_000, + }, + }, + }, + }), + env: {}, + agentDirs: ["/tmp/openclaw-agent-main"], + loadAuthStore: emptyAuthStore, + }); + + expect(runtimePrepareImportMock).not.toHaveBeenCalled(); + }); + it("uses the resolver path when an auth profile store contains a SecretRef", async () => { const { prepareSecretsRuntimeSnapshot } = await import("./runtime.js"); @@ -91,6 +146,27 @@ describe("secrets runtime fast path", () => { }), }); - expect(runtimePrepareImportMock).toHaveBeenCalledTimes(1); + expect(resolveRuntimeWebToolsMock).toHaveBeenCalledTimes(1); + }); + + it("keeps explicit web fetch provider config on the resolver path", async () => { + const { prepareSecretsRuntimeSnapshot } = await import("./runtime.js"); + + await prepareSecretsRuntimeSnapshot({ + config: asConfig({ + tools: { + web: { + fetch: { + provider: "firecrawl", + }, + }, + }, + }), + env: {}, + agentDirs: ["/tmp/openclaw-agent-main"], + loadAuthStore: emptyAuthStore, + }); + + expect(resolveRuntimeWebToolsMock).toHaveBeenCalledTimes(1); }); }); diff --git a/src/secrets/runtime.ts b/src/secrets/runtime.ts index 3f04672502c..c07cdb27ebd 100644 --- a/src/secrets/runtime.ts +++ b/src/secrets/runtime.ts @@ -197,10 +197,72 @@ function createEmptyRuntimeWebToolsMetadata(): RuntimeWebToolsMetadata { }; } +const WEB_FETCH_CREDENTIAL_FIELD_NAMES = new Set(["apikey", "key", "token", "secret", "password"]); + +function hasCredentialBearingWebFetchValue( + value: unknown, + defaults: Parameters[1], + seen = new WeakSet(), +): boolean { + if (coerceSecretRef(value, defaults)) { + return true; + } + if (!value || typeof value !== "object") { + return false; + } + if (seen.has(value)) { + return false; + } + seen.add(value); + if (Array.isArray(value)) { + return value.some((entry) => hasCredentialBearingWebFetchValue(entry, defaults, seen)); + } + return Object.entries(value as Record).some(([rawKey, entry]) => { + const key = rawKey.toLowerCase(); + if (WEB_FETCH_CREDENTIAL_FIELD_NAMES.has(key) && entry != null && entry !== "") { + return true; + } + return hasCredentialBearingWebFetchValue(entry, defaults, seen); + }); +} + +function hasActiveRuntimeWebFetchProviderSurface( + fetch: unknown, + defaults: Parameters[1], +): boolean { + if (!fetch || typeof fetch !== "object" || Array.isArray(fetch)) { + return false; + } + const fetchConfig = fetch as Record; + if (fetchConfig.enabled === false) { + return false; + } + if (typeof fetchConfig.provider === "string" && fetchConfig.provider.trim()) { + return true; + } + return hasCredentialBearingWebFetchValue(fetchConfig, defaults); +} + function hasRuntimeWebToolConfigSurface(config: OpenClawConfig): boolean { const web = config.tools?.web; - if (web && typeof web === "object" && ("search" in web || "fetch" in web || "x_search" in web)) { - return true; + const defaults = config.secrets?.defaults; + const fetchExplicitlyDisabled = + web && + typeof web === "object" && + !Array.isArray(web) && + typeof (web as Record).fetch === "object" && + (web as { fetch?: { enabled?: unknown } }).fetch?.enabled === false; + if (web && typeof web === "object" && !Array.isArray(web)) { + const webRecord = web as Record; + if ("search" in webRecord || "x_search" in webRecord) { + return true; + } + if ( + "fetch" in webRecord && + hasActiveRuntimeWebFetchProviderSurface(webRecord.fetch, defaults) + ) { + return true; + } } const entries = config.plugins?.entries; if (!entries || typeof entries !== "object" || Array.isArray(entries)) { @@ -215,7 +277,7 @@ function hasRuntimeWebToolConfigSurface(config: OpenClawConfig): boolean { !!pluginConfig && typeof pluginConfig === "object" && !Array.isArray(pluginConfig) && - ("webSearch" in pluginConfig || "webFetch" in pluginConfig) + ("webSearch" in pluginConfig || (!fetchExplicitlyDisabled && "webFetch" in pluginConfig)) ); }); }