From fbf9132b32dad759fe8af0f0286db34fc70f0376 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Mon, 4 May 2026 00:02:17 -0700 Subject: [PATCH] fix(web-fetch): late-bind runtime config --- CHANGELOG.md | 1 + src/agents/openclaw-tools.ts | 1 + .../tools/web-fetch.provider-fallback.test.ts | 97 +++++++++++++++++++ src/agents/tools/web-fetch.ts | 83 +++++++++------- 4 files changed, 148 insertions(+), 34 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e4cca2b144..3d06682602e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,6 +56,7 @@ Docs: https://docs.openclaw.ai ### Fixes +- Web fetch: late-bind `web_fetch` config and provider fallback metadata from the active runtime snapshot, matching `web_search` so long-lived tools do not use stale fetch provider settings. Thanks @vincentkoc. - Discord: clear stale startup probe bot/application status when the async bot probe throws, not just when it returns a degraded probe result. Thanks @vincentkoc. - Web search: scope explicit bundled `web_search` provider runtime loading through manifest ownership, so selecting DuckDuckGo/Gemini/etc. does not import unrelated bundled providers or log their optional dependency failures. Thanks @vincentkoc. - Plugins/discovery: demote the source-only TypeScript runtime check on already-installed `origin: "global"` plugin packages from a config-blocking error to a warning and let the runtime fall through to the TypeScript source via jiti, so a single broken installed package no longer blocks `plugins install` for unrelated plugins; install-time rejection of newly-installed source-only packages is unchanged. Thanks @romneyda. diff --git a/src/agents/openclaw-tools.ts b/src/agents/openclaw-tools.ts index 79d9629983b..c9df46a34a5 100644 --- a/src/agents/openclaw-tools.ts +++ b/src/agents/openclaw-tools.ts @@ -455,6 +455,7 @@ export function createOpenClawTools( config: options?.config, sandboxed: options?.sandboxed, runtimeWebFetch: runtimeWebTools?.fetch, + lateBindRuntimeConfig: true, }); options?.recordToolPrepStage?.("openclaw-tools:web-fetch-tool"); const messageTool = options?.disableMessageTool diff --git a/src/agents/tools/web-fetch.provider-fallback.test.ts b/src/agents/tools/web-fetch.provider-fallback.test.ts index 92a90b34c93..80ad644efbb 100644 --- a/src/agents/tools/web-fetch.provider-fallback.test.ts +++ b/src/agents/tools/web-fetch.provider-fallback.test.ts @@ -6,21 +6,35 @@ import { createWebFetchTool } from "./web-fetch.js"; const { resolveWebFetchDefinitionMock } = vi.hoisted(() => ({ resolveWebFetchDefinitionMock: vi.fn(), })); +const runtimeState = vi.hoisted(() => ({ + activeSecretsRuntimeSnapshot: null as null | { config: unknown }, + activeRuntimeWebToolsMetadata: null as null | Record, +})); vi.mock("../../web-fetch/runtime.js", () => ({ resolveWebFetchDefinition: resolveWebFetchDefinitionMock, })); +vi.mock("../../secrets/runtime.js", () => ({ + getActiveSecretsRuntimeSnapshot: () => runtimeState.activeSecretsRuntimeSnapshot, +})); +vi.mock("../../secrets/runtime-web-tools-state.js", () => ({ + getActiveRuntimeWebToolsMetadata: () => runtimeState.activeRuntimeWebToolsMetadata, +})); describe("web_fetch provider fallback normalization", () => { const priorFetch = global.fetch; beforeEach(() => { resolveWebFetchDefinitionMock.mockReset(); + runtimeState.activeSecretsRuntimeSnapshot = null; + runtimeState.activeRuntimeWebToolsMetadata = null; }); afterEach(() => { global.fetch = priorFetch; vi.restoreAllMocks(); + runtimeState.activeSecretsRuntimeSnapshot = null; + runtimeState.activeRuntimeWebToolsMetadata = null; }); it("re-wraps and truncates provider fallback payloads before caching or returning", async () => { @@ -124,4 +138,87 @@ describe("web_fetch provider fallback normalization", () => { expect(details.url).toBe("https://example.com/fallback"); expect(details.finalUrl).toBe("https://example.com/fallback"); }); + + it("late-binds provider fallback config and runtime metadata from the active runtime snapshot", async () => { + global.fetch = withFetchPreconnect( + vi.fn(async () => { + throw new Error("network failed"); + }), + ); + const runtimeConfig = { + tools: { + web: { + fetch: { + provider: "firecrawl", + maxChars: 640, + }, + }, + }, + } as OpenClawConfig; + runtimeState.activeSecretsRuntimeSnapshot = { config: runtimeConfig }; + runtimeState.activeRuntimeWebToolsMetadata = { + fetch: { + providerConfigured: "firecrawl", + providerSource: "configured", + selectedProvider: "firecrawl", + selectedProviderKeySource: "config", + diagnostics: [], + }, + diagnostics: [], + }; + resolveWebFetchDefinitionMock.mockReturnValue({ + provider: { id: "firecrawl" }, + definition: { + description: "firecrawl", + parameters: {}, + execute: async () => ({ + text: "runtime fallback body ".repeat(200), + }), + }, + }); + + const tool = createWebFetchTool({ + config: { + tools: { + web: { + fetch: { + provider: "stale", + maxChars: 200, + }, + }, + }, + } as OpenClawConfig, + sandboxed: false, + runtimeWebFetch: { + providerConfigured: "stale", + providerSource: "configured", + selectedProvider: "stale", + selectedProviderKeySource: "config", + diagnostics: [], + }, + lateBindRuntimeConfig: true, + }); + + const result = await tool?.execute?.("call-provider-fallback", { + url: "https://example.com/fallback", + }); + const details = result?.details as { + wrappedLength?: number; + externalContent?: Record; + }; + + expect(details.wrappedLength).toBeGreaterThan(200); + expect(details.wrappedLength).toBeLessThanOrEqual(640); + expect(details.externalContent).toMatchObject({ + provider: "firecrawl", + }); + expect(resolveWebFetchDefinitionMock).toHaveBeenCalledWith( + expect.objectContaining({ + config: runtimeConfig, + runtimeWebFetch: expect.objectContaining({ + selectedProvider: "firecrawl", + }), + }), + ); + }); }); diff --git a/src/agents/tools/web-fetch.ts b/src/agents/tools/web-fetch.ts index ffea858032b..2a9f7419701 100644 --- a/src/agents/tools/web-fetch.ts +++ b/src/agents/tools/web-fetch.ts @@ -616,38 +616,13 @@ export function createWebFetchTool(options?: { config?: OpenClawConfig; sandboxed?: boolean; runtimeWebFetch?: RuntimeWebFetchMetadata; + lateBindRuntimeConfig?: boolean; lookupFn?: LookupFn; }): AnyAgentTool | null { const fetch = resolveFetchConfig(options?.config); if (!resolveFetchEnabled({ fetch, sandboxed: options?.sandboxed })) { return null; } - const readabilityEnabled = resolveFetchReadabilityEnabled(fetch); - const userAgent = - (fetch && "userAgent" in fetch && typeof fetch.userAgent === "string" && fetch.userAgent) || - DEFAULT_FETCH_USER_AGENT; - const maxResponseBytes = resolveFetchMaxResponseBytes(fetch); - let providerFallbackResolved = false; - let providerFallbackCache: WebFetchProviderFallback; - const resolveProviderFallback = async () => { - if (!providerFallbackResolved) { - const { resolveWebFetchDefinition } = await loadWebFetchRuntime(); - const { config, preferRuntimeProviders, runtimeWebFetch } = resolveWebFetchToolRuntimeContext( - { - config: options?.config, - runtimeWebFetch: options?.runtimeWebFetch, - }, - ); - providerFallbackCache = resolveWebFetchDefinition({ - config, - sandboxed: options?.sandboxed, - runtimeWebFetch, - preferRuntimeProviders, - }); - providerFallbackResolved = true; - } - return providerFallbackCache; - }; return { label: "Web Fetch", name: "web_fetch", @@ -655,28 +630,68 @@ export function createWebFetchTool(options?: { "Fetch and extract readable content from a URL (HTML → markdown/text). Use for lightweight page access without browser automation.", parameters: WebFetchSchema, execute: async (_toolCallId, args) => { + const { config, preferRuntimeProviders, runtimeWebFetch } = resolveWebFetchToolRuntimeContext( + { + config: options?.config, + lateBindRuntimeConfig: options?.lateBindRuntimeConfig, + runtimeWebFetch: options?.runtimeWebFetch, + }, + ); + const executionFetch = resolveFetchConfig(config); + if (!resolveFetchEnabled({ fetch: executionFetch, sandboxed: options?.sandboxed })) { + throw new Error("web_fetch is disabled."); + } + const readabilityEnabled = resolveFetchReadabilityEnabled(executionFetch); + const userAgent = + (executionFetch && + "userAgent" in executionFetch && + typeof executionFetch.userAgent === "string" && + executionFetch.userAgent) || + DEFAULT_FETCH_USER_AGENT; + const maxResponseBytes = resolveFetchMaxResponseBytes(executionFetch); + let providerFallbackResolved = false; + let providerFallbackCache: WebFetchProviderFallback; + const resolveProviderFallback = async () => { + if (!providerFallbackResolved) { + const { resolveWebFetchDefinition } = await loadWebFetchRuntime(); + providerFallbackCache = resolveWebFetchDefinition({ + config, + sandboxed: options?.sandboxed, + runtimeWebFetch, + preferRuntimeProviders, + }); + providerFallbackResolved = true; + } + return providerFallbackCache; + }; const params = args as Record; const url = readStringParam(params, "url", { required: true }); const extractMode = readStringParam(params, "extractMode") === "text" ? "text" : "markdown"; const maxChars = readNumberParam(params, "maxChars", { integer: true }); - const maxCharsCap = resolveFetchMaxCharsCap(fetch); + const maxCharsCap = resolveFetchMaxCharsCap(executionFetch); const result = await runWebFetch({ url, extractMode, maxChars: resolveMaxChars( - maxChars ?? fetch?.maxChars, + maxChars ?? executionFetch?.maxChars, DEFAULT_FETCH_MAX_CHARS, maxCharsCap, ), maxResponseBytes, - maxRedirects: resolveMaxRedirects(fetch?.maxRedirects, DEFAULT_FETCH_MAX_REDIRECTS), - timeoutSeconds: resolveTimeoutSeconds(fetch?.timeoutSeconds, DEFAULT_TIMEOUT_SECONDS), - cacheTtlMs: resolveCacheTtlMs(fetch?.cacheTtlMinutes, DEFAULT_CACHE_TTL_MINUTES), + maxRedirects: resolveMaxRedirects( + executionFetch?.maxRedirects, + DEFAULT_FETCH_MAX_REDIRECTS, + ), + timeoutSeconds: resolveTimeoutSeconds( + executionFetch?.timeoutSeconds, + DEFAULT_TIMEOUT_SECONDS, + ), + cacheTtlMs: resolveCacheTtlMs(executionFetch?.cacheTtlMinutes, DEFAULT_CACHE_TTL_MINUTES), userAgent, readabilityEnabled, - config: options?.config, - useTrustedEnvProxy: resolveFetchUseTrustedEnvProxy(fetch), - ssrfPolicy: fetch?.ssrfPolicy, + config, + useTrustedEnvProxy: resolveFetchUseTrustedEnvProxy(executionFetch), + ssrfPolicy: executionFetch?.ssrfPolicy, lookupFn: options?.lookupFn, resolveProviderFallback, });