From e4c127e678fca8c96f47e0f002072de7c1ffb08d Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 2 May 2026 04:08:59 +0100 Subject: [PATCH] fix(web-fetch): resolve external providers --- CHANGELOG.md | 1 + docs/tools/web-fetch.md | 5 +++- docs/tools/web.md | 2 ++ src/web-fetch/runtime.test.ts | 46 +++++++++++++++++++++++++++++++-- src/web-fetch/runtime.ts | 48 ++++++++++++++++++++++++++++------- 5 files changed, 90 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bc1c922326a..3609c1b3288 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,7 @@ Docs: https://docs.openclaw.ai - Web search: point missing-key errors to `web_fetch` for known URLs and the browser tool for interactive pages. Thanks @zhaoyang97. - Web search: late-bind managed agent `web_search` calls to the current runtime config snapshot, so existing sessions do not keep stale unresolved SecretRefs after secrets reload. Fixes #75420. Thanks @richardmqq. - Web search: honor `baseUrl` overrides for Gemini, Grok, and x_search provider-owned config, so proxy-backed search tools no longer dial hardcoded public endpoints. Supersedes #61972. Thanks @Lanfei. +- Web fetch: resolve external plugin `webFetchProviders` for non-sandboxed `web_fetch`, while keeping sandboxed fetches limited to bundled providers. Fixes #74915. Thanks @ultrahighsuper and @mingmingtsao. - Heartbeat: strip legacy `[TOOL_CALL]...[/TOOL_CALL]` and `[TOOL_RESULT]...[/TOOL_RESULT]` pseudo-call blocks from heartbeat replies before channel delivery. Fixes #54138. Thanks @Deniable9570. - macOS/Voice Wake: send wake-word and Push-to-Talk transcripts through the selected macOS session target instead of always falling back to main WebChat. Fixes #51040. Thanks @carl-jeffrolc. - Providers/xAI: give Grok `web_search` a 60s default timeout, harden malformed xAI Responses parsing, and return structured timeout errors instead of aborting the tool call. Fixes #58063 and #58733. Thanks @dnishimura, @marvcasasola-svg, and @Nanako0129. diff --git a/docs/tools/web-fetch.md b/docs/tools/web-fetch.md index 6dbf8d4f3a1..9b4a1475421 100644 --- a/docs/tools/web-fetch.md +++ b/docs/tools/web-fetch.md @@ -134,7 +134,10 @@ Current runtime behavior: - `tools.web.fetch.provider` selects the fetch fallback provider explicitly. - If `provider` is omitted, OpenClaw auto-detects the first ready web-fetch - provider from available credentials. Today the bundled provider is Firecrawl. + provider from available credentials. Non-sandboxed `web_fetch` can use + installed plugins that declare `contracts.webFetchProviders` and register a + matching provider at runtime. Today the bundled provider is Firecrawl. +- Sandboxed `web_fetch` calls stay limited to bundled providers. - If Readability is disabled, `web_fetch` skips straight to the selected provider fallback. If no provider is available, it fails closed. diff --git a/docs/tools/web.md b/docs/tools/web.md index 32453114e46..7d04df4cd60 100644 --- a/docs/tools/web.md +++ b/docs/tools/web.md @@ -221,6 +221,8 @@ examples. - choose it with `tools.web.fetch.provider` - or omit that field and let OpenClaw auto-detect the first ready web-fetch provider from available credentials +- non-sandboxed `web_fetch` can use installed plugin providers that declare + `contracts.webFetchProviders`; sandboxed fetches stay bundled-only - today the bundled web-fetch provider is Firecrawl, configured under `plugins.entries.firecrawl.config.webFetch.*` diff --git a/src/web-fetch/runtime.test.ts b/src/web-fetch/runtime.test.ts index baa7b83075c..e413929986f 100644 --- a/src/web-fetch/runtime.test.ts +++ b/src/web-fetch/runtime.test.ts @@ -201,7 +201,7 @@ describe("web fetch runtime", () => { expect(resolved?.provider.id).toBe("firecrawl"); }); - it("keeps non-sandboxed web fetch on bundled providers even when runtime providers are preferred", () => { + it("uses runtime providers for non-sandboxed web fetch when runtime providers are preferred", () => { const bundled = createFirecrawlProvider({ getConfiguredCredentialValue: () => "bundled-key", }); @@ -215,6 +215,48 @@ describe("web fetch runtime", () => { preferRuntimeProviders: true, }); - expect(resolved?.provider.id).toBe("firecrawl"); + expect(resolved?.provider.id).toBe("thirdparty"); + }); + + it("resolves an explicitly configured non-bundled provider from plugin providers", () => { + const bundled = createFirecrawlProvider({ + getConfiguredCredentialValue: () => "bundled-key", + }); + const external = createThirdPartyFetchProvider(); + resolvePluginWebFetchProvidersMock.mockReturnValue([bundled, external]); + + const resolved = resolveWebFetchDefinition({ + config: { + tools: { web: { fetch: { provider: "thirdparty" } } }, + } as OpenClawConfig, + sandboxed: false, + preferRuntimeProviders: false, + }); + + expect(resolved?.provider.id).toBe("thirdparty"); + }); + + it("prefers an explicitly configured non-bundled provider over runtime metadata", () => { + const bundled = createFirecrawlProvider({ + getConfiguredCredentialValue: () => "bundled-key", + }); + const external = createThirdPartyFetchProvider(); + resolveRuntimeWebFetchProvidersMock.mockReturnValue([bundled, external]); + + const resolved = resolveWebFetchDefinition({ + config: { + tools: { web: { fetch: { provider: "thirdparty" } } }, + } as OpenClawConfig, + runtimeWebFetch: { + providerSource: "auto-detect", + selectedProvider: "firecrawl", + selectedProviderKeySource: "env", + diagnostics: [], + }, + sandboxed: false, + preferRuntimeProviders: true, + }); + + expect(resolved?.provider.id).toBe("thirdparty"); }); }); diff --git a/src/web-fetch/runtime.ts b/src/web-fetch/runtime.ts index 02056669308..4313a5ed130 100644 --- a/src/web-fetch/runtime.ts +++ b/src/web-fetch/runtime.ts @@ -4,7 +4,10 @@ import type { PluginWebFetchProviderEntry, WebFetchProviderToolDefinition, } from "../plugins/types.js"; -import { resolvePluginWebFetchProviders } from "../plugins/web-fetch-providers.runtime.js"; +import { + resolvePluginWebFetchProviders, + resolveRuntimeWebFetchProviders, +} from "../plugins/web-fetch-providers.runtime.js"; import { sortWebFetchProvidersForAutoDetect } from "../plugins/web-fetch-providers.shared.js"; import { getActiveRuntimeWebToolsMetadata } from "../secrets/runtime-web-tools-state.js"; import type { RuntimeWebFetchMetadata } from "../secrets/runtime-web-tools.types.js"; @@ -81,7 +84,6 @@ export function listWebFetchProviders(params?: { return resolvePluginWebFetchProviders({ config: params?.config, bundledAllowlistCompat: true, - origin: "bundled", }); } @@ -104,7 +106,6 @@ export function resolveWebFetchProviderId(params: { resolvePluginWebFetchProviders({ config: params.config, bundledAllowlistCompat: true, - origin: "bundled", }), ); const raw = @@ -138,6 +139,20 @@ export function resolveWebFetchProviderId(params: { return ""; } +function resolveConfiguredWebFetchProviderId(params: { + fetch?: WebFetchConfig; + providers: PluginWebFetchProviderEntry[]; +}): string | undefined { + const raw = + params.fetch && "provider" in params.fetch + ? normalizeLowercaseStringOrEmpty(params.fetch.provider) + : ""; + if (!raw) { + return undefined; + } + return params.providers.find((provider) => provider.id === raw)?.id; +} + export function resolveWebFetchDefinition( options?: ResolveWebFetchDefinitionParams, ): { provider: PluginWebFetchProviderEntry; definition: WebFetchProviderToolDefinition } | null { @@ -146,18 +161,33 @@ export function resolveWebFetchDefinition( | undefined; const runtimeWebFetch = options?.runtimeWebFetch ?? getActiveRuntimeWebToolsMetadata()?.fetch; const providers = sortWebFetchProvidersForAutoDetect( - resolvePluginWebFetchProviders({ - config: options?.config, - bundledAllowlistCompat: true, - origin: "bundled", - }), + options?.sandboxed + ? resolvePluginWebFetchProviders({ + config: options?.config, + bundledAllowlistCompat: true, + origin: "bundled", + }) + : options?.preferRuntimeProviders + ? resolveRuntimeWebFetchProviders({ + config: options?.config, + bundledAllowlistCompat: true, + }) + : resolvePluginWebFetchProviders({ + config: options?.config, + bundledAllowlistCompat: true, + }), ); return resolveWebProviderDefinition({ config: options?.config, toolConfig: fetch as Record | undefined, runtimeMetadata: runtimeWebFetch, sandboxed: options?.sandboxed, - providerId: options?.providerId, + providerId: + options?.providerId ?? + resolveConfiguredWebFetchProviderId({ + fetch, + providers, + }), providers, resolveEnabled: ({ toolConfig, sandboxed }) => resolveWebFetchEnabled({