From edb7e00721fd4e35f48389aa94c6aa39e6339a4e Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sun, 3 May 2026 19:52:34 +0100 Subject: [PATCH] fix(network): scope fake-ip SSRF policy to provider hosts --- CHANGELOG.md | 1 + docs/concepts/model-providers.md | 1 + docs/tools/web.md | 12 ++++ src/agents/provider-transport-fetch.test.ts | 61 +++++++++++++++++++++ src/agents/provider-transport-fetch.ts | 49 ++++++++++++++++- src/agents/tools/web-guarded-fetch.test.ts | 9 ++- src/agents/tools/web-guarded-fetch.ts | 10 +++- src/infra/net/fetch-guard.ssrf.test.ts | 37 +++++++++++++ src/infra/net/fetch-guard.ts | 25 ++++----- src/infra/net/ssrf.test.ts | 21 +++++++ src/infra/net/ssrf.ts | 22 ++++++++ 11 files changed, 229 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c57909ccfe6..094bcd182e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,7 @@ Docs: https://docs.openclaw.ai - Plugins/hooks: let `plugins.entries..hooks.timeoutMs` and `plugins.entries..hooks.timeouts` bound plugin typed hooks from operator config, so slow hooks can be tuned without patching installed plugin code. Fixes #76778. Thanks @vincentkoc. - Telegram: add `channels.telegram.mediaGroupFlushMs` at the top level and per account so operators can tune album buffering instead of being stuck with the hard-coded 500ms media-group flush window. Fixes #76149. Thanks @vincentkoc. - Config/messages: coerce boolean `messages.visibleReplies` and `messages.groupChat.visibleReplies` values to the documented enum modes so an intuitive toggle no longer invalidates config and drops channel startup. Fixes #75390. Thanks @scottgl9. +- Agents/network: allow trusted web-search providers and configured model-provider hosts to work behind Surge/Clash/sing-box fake-IP DNS by accepting RFC 2544 and IPv6 ULA synthetic answers only for the request's scoped hostname, without broad private-network access. Refs #76530 and #76549. Thanks @zqchris. - Feishu: accept and honor `channels.feishu.blockStreaming` at the top level and per account, while keeping the legacy default off so Feishu cards no longer reject documented config or silently drop block replies. Fixes #75555. Thanks @vincentkoc. - Google Chat: normalize custom Google auth transport headers before google-auth/gaxios interceptors run, restoring webhook token verification when certificate retrieval expects Fetch `Headers`. Fixes #76742. Thanks @donbowman. - Doctor/plugins: reset stale `plugins.slots.memory` and `plugins.slots.contextEngine` references during `doctor --fix`, so cleanup of missing plugin config does not leave unrecoverable slot owners behind. Fixes #76550 and #76551. Thanks @vincentkoc. diff --git a/docs/concepts/model-providers.md b/docs/concepts/model-providers.md index 2e758bc751d..1864f3a0907 100644 --- a/docs/concepts/model-providers.md +++ b/docs/concepts/model-providers.md @@ -688,6 +688,7 @@ Example (OpenAI‑compatible): - For OpenAI-compatible Completions proxies that need vendor-specific fields, set `agents.defaults.models["provider/model"].params.extra_body` (or `extraBody`) to merge extra JSON into the outbound request body. - For vLLM chat-template controls, set `agents.defaults.models["provider/model"].params.chat_template_kwargs`. The bundled vLLM plugin automatically sends `enable_thinking: false` and `force_nonempty_content: true` for `vllm/nemotron-3-*` when the session thinking level is off. - For slow local models or remote LAN/tailnet hosts, set `models.providers..timeoutSeconds`. This extends provider model HTTP request handling, including connect, headers, body streaming, and the total guarded-fetch abort, without increasing the whole agent runtime timeout. + - Model provider HTTP calls allow Surge, Clash, and sing-box fake-IP DNS answers in `198.18.0.0/15` and `fc00::/7` only for the configured provider `baseUrl` hostname. Other private, loopback, link-local, and metadata destinations still require an explicit `models.providers..request.allowPrivateNetwork: true` opt-in. - If `baseUrl` is empty/omitted, OpenClaw keeps the default OpenAI behavior (which resolves to `api.openai.com`). - For safety, an explicit `compat.supportsDeveloperRole: true` is still overridden on non-native `openai-completions` endpoints. - For `api: "anthropic-messages"` on non-direct endpoints (any provider other than canonical `anthropic`, or a custom `models.providers.anthropic.baseUrl` whose host is not a public `api.anthropic.com` endpoint), OpenClaw suppresses implicit Anthropic beta headers such as `claude-code-20250219`, `interleaved-thinking-2025-05-14`, and OAuth markers, so custom Anthropic-compatible proxies do not reject unsupported beta flags. Set `models.providers..headers["anthropic-beta"]` explicitly if your proxy needs specific beta features. diff --git a/docs/tools/web.md b/docs/tools/web.md index e1c508c89c5..208074028ae 100644 --- a/docs/tools/web.md +++ b/docs/tools/web.md @@ -153,6 +153,18 @@ Codex-capable models can optionally use the provider-native Responses `web_searc If native Codex search is enabled but the current model is not Codex-capable, OpenClaw keeps the normal managed `web_search` behavior. +## Network safety + +Managed `web_search` provider calls use OpenClaw's guarded fetch path. For +trusted provider API hosts, OpenClaw allows Surge, Clash, and sing-box fake-IP +DNS answers in `198.18.0.0/15` and `fc00::/7` only for that provider hostname. +Other private, loopback, link-local, and metadata destinations remain blocked. + +This automatic allowance does not apply to arbitrary `web_fetch` URLs. For +`web_fetch`, enable `tools.web.fetch.ssrfPolicy.allowRfc2544BenchmarkRange` and +`tools.web.fetch.ssrfPolicy.allowIpv6UniqueLocalRange` explicitly only when your +trusted proxy owns those synthetic ranges. + ## Setting up web search Provider lists in docs and setup flows are alphabetical. Auto-detection keeps a diff --git a/src/agents/provider-transport-fetch.test.ts b/src/agents/provider-transport-fetch.test.ts index 38302047920..5043888b43e 100644 --- a/src/agents/provider-transport-fetch.test.ts +++ b/src/agents/provider-transport-fetch.test.ts @@ -76,6 +76,67 @@ describe("buildGuardedModelFetch", () => { ); }); + it("scopes fake-IP DNS exemptions to the configured provider host", async () => { + const { buildGuardedModelFetch } = await import("./provider-transport-fetch.js"); + const model = { + id: "gpt-5.4", + provider: "openai", + api: "openai-responses", + baseUrl: "https://api.openai.com/v1", + } as unknown as Model<"openai-responses">; + + const fetcher = buildGuardedModelFetch(model); + await fetcher("https://api.openai.com/v1/responses", { method: "POST" }); + + const policy = fetchWithSsrFGuardMock.mock.calls[0]?.[0]?.policy; + expect(policy).toEqual({ + allowRfc2544BenchmarkRange: true, + allowIpv6UniqueLocalRange: true, + hostnameAllowlist: ["api.openai.com"], + }); + expect(policy?.allowedHostnames).toBeUndefined(); + expect(policy?.allowPrivateNetwork).toBeUndefined(); + expect(policy?.dangerouslyAllowPrivateNetwork).toBeUndefined(); + }); + + it("does not apply fake-IP exemptions to non-provider hosts", async () => { + const { buildGuardedModelFetch } = await import("./provider-transport-fetch.js"); + const model = { + id: "gpt-5.4", + provider: "openai", + api: "openai-responses", + baseUrl: "https://api.openai.com/v1", + } as unknown as Model<"openai-responses">; + + const fetcher = buildGuardedModelFetch(model); + await fetcher("https://uploads.openai.com/v1/files", { method: "POST" }); + + const policy = fetchWithSsrFGuardMock.mock.calls[0]?.[0]?.policy; + expect(policy).toBeUndefined(); + }); + + it("merges explicit private-network opt-in into the provider-host fake-IP policy", async () => { + resolveProviderRequestPolicyConfigMock.mockReturnValueOnce({ allowPrivateNetwork: true }); + const { buildGuardedModelFetch } = await import("./provider-transport-fetch.js"); + const model = { + id: "qwen3:32b", + provider: "ollama", + api: "ollama", + baseUrl: "http://10.0.0.5:11434", + } as unknown as Model<"ollama">; + + const fetcher = buildGuardedModelFetch(model); + await fetcher("http://10.0.0.5:11434/api/chat", { method: "POST" }); + + const policy = fetchWithSsrFGuardMock.mock.calls[0]?.[0]?.policy; + expect(policy).toEqual({ + allowRfc2544BenchmarkRange: true, + allowIpv6UniqueLocalRange: true, + hostnameAllowlist: ["10.0.0.5"], + allowPrivateNetwork: true, + }); + }); + it("threads explicit transport timeouts into the shared guarded fetch seam", async () => { const { buildGuardedModelFetch } = await import("./provider-transport-fetch.js"); const model = { diff --git a/src/agents/provider-transport-fetch.ts b/src/agents/provider-transport-fetch.ts index 78d9a8b2e9f..df4a4627c3b 100644 --- a/src/agents/provider-transport-fetch.ts +++ b/src/agents/provider-transport-fetch.ts @@ -1,5 +1,9 @@ import type { Api, Model } from "@mariozechner/pi-ai"; import { fetchWithSsrFGuard } from "../infra/net/fetch-guard.js"; +import { + ssrfPolicyFromHttpBaseUrlFakeIpHostnameAllowlist, + type SsrFPolicy, +} from "../infra/net/ssrf.js"; import { resolveDebugProxySettings } from "../proxy-capture/env.js"; import { buildProviderRequestDispatcherPolicy, @@ -268,6 +272,44 @@ export function resolveModelRequestTimeoutMs( : undefined; } +function resolveHttpHostname(value: unknown): string | undefined { + if (typeof value !== "string" || !value.trim()) { + return undefined; + } + try { + const parsed = new URL(value); + if (parsed.protocol !== "http:" && parsed.protocol !== "https:") { + return undefined; + } + return parsed.hostname.toLowerCase(); + } catch { + return undefined; + } +} + +function resolveModelTransportSsrFPolicy(params: { + model: Model; + url: string; + allowPrivateNetwork?: boolean; +}): SsrFPolicy | undefined { + const baseUrl = (params.model as { baseUrl?: unknown }).baseUrl; + const baseHostname = resolveHttpHostname(baseUrl); + const requestHostname = resolveHttpHostname(params.url); + const fakeIpPolicy = + typeof baseUrl === "string" && baseHostname && requestHostname === baseHostname + ? ssrfPolicyFromHttpBaseUrlFakeIpHostnameAllowlist(baseUrl) + : undefined; + + if (fakeIpPolicy) { + return { + ...fakeIpPolicy, + ...(params.allowPrivateNetwork ? { allowPrivateNetwork: true } : {}), + }; + } + + return params.allowPrivateNetwork ? { allowPrivateNetwork: true } : undefined; +} + export function buildGuardedModelFetch(model: Model, timeoutMs?: number): typeof fetch { const requestConfig = resolveModelRequestPolicy(model); const dispatcherPolicy = buildProviderRequestDispatcherPolicy(requestConfig); @@ -283,6 +325,11 @@ export function buildGuardedModelFetch(model: Model, timeoutMs?: number): t : (() => { throw new Error("Unsupported fetch input for transport-aware model request"); })()); + const policy = resolveModelTransportSsrFPolicy({ + model, + url, + allowPrivateNetwork: requestConfig.allowPrivateNetwork, + }); const requestInit = request && ({ @@ -308,7 +355,7 @@ export function buildGuardedModelFetch(model: Model, timeoutMs?: number): t // Provider transport intentionally keeps the secure default and never // replays unsafe request bodies across cross-origin redirects. allowCrossOriginUnsafeRedirectReplay: false, - ...(requestConfig.allowPrivateNetwork ? { policy: { allowPrivateNetwork: true } } : {}), + ...(policy ? { policy } : {}), }); let response = result.response; if (shouldBypassLongSdkRetry(response)) { diff --git a/src/agents/tools/web-guarded-fetch.test.ts b/src/agents/tools/web-guarded-fetch.test.ts index 179a8151e43..7dd874bd0c8 100644 --- a/src/agents/tools/web-guarded-fetch.test.ts +++ b/src/agents/tools/web-guarded-fetch.test.ts @@ -30,7 +30,7 @@ describe("web-guarded-fetch", () => { vi.clearAllMocks(); }); - it("uses strict SSRF policy for trusted web tools endpoints", async () => { + it("uses a host-scoped fake-IP SSRF policy for trusted web tools endpoints", async () => { vi.mocked(fetchWithSsrFGuard).mockResolvedValue({ response: new Response("ok", { status: 200 }), finalUrl: "https://example.com", @@ -42,7 +42,11 @@ describe("web-guarded-fetch", () => { expect(fetchWithSsrFGuard).toHaveBeenCalledWith( expect.objectContaining({ url: "https://example.com", - policy: {}, + policy: { + allowRfc2544BenchmarkRange: true, + allowIpv6UniqueLocalRange: true, + hostnameAllowlist: ["example.com"], + }, mode: GUARDED_FETCH_MODE.TRUSTED_ENV_PROXY, }), ); @@ -63,6 +67,7 @@ describe("web-guarded-fetch", () => { policy: expect.objectContaining({ dangerouslyAllowPrivateNetwork: true, allowRfc2544BenchmarkRange: true, + allowIpv6UniqueLocalRange: true, }), mode: GUARDED_FETCH_MODE.TRUSTED_ENV_PROXY, }), diff --git a/src/agents/tools/web-guarded-fetch.ts b/src/agents/tools/web-guarded-fetch.ts index f1d542fdcf4..1d868df9b71 100644 --- a/src/agents/tools/web-guarded-fetch.ts +++ b/src/agents/tools/web-guarded-fetch.ts @@ -5,12 +5,15 @@ import { withStrictGuardedFetchMode, withTrustedEnvProxyGuardedFetchMode, } from "../../infra/net/fetch-guard.js"; -import type { SsrFPolicy } from "../../infra/net/ssrf.js"; +import { + ssrfPolicyFromHttpBaseUrlFakeIpHostnameAllowlist, + type SsrFPolicy, +} from "../../infra/net/ssrf.js"; -const WEB_TOOLS_TRUSTED_NETWORK_SSRF_POLICY: SsrFPolicy = {}; const WEB_TOOLS_SELF_HOSTED_NETWORK_SSRF_POLICY: SsrFPolicy = { dangerouslyAllowPrivateNetwork: true, allowRfc2544BenchmarkRange: true, + allowIpv6UniqueLocalRange: true, }; type WebToolGuardedFetchOptions = Omit< @@ -66,10 +69,11 @@ export async function withTrustedWebToolsEndpoint( params: WebToolEndpointFetchOptions, run: (result: { response: Response; finalUrl: string }) => Promise, ): Promise { + const trustedPolicy = ssrfPolicyFromHttpBaseUrlFakeIpHostnameAllowlist(params.url) ?? {}; return await withWebToolsNetworkGuard( { ...params, - policy: WEB_TOOLS_TRUSTED_NETWORK_SSRF_POLICY, + policy: trustedPolicy, useEnvProxy: true, }, run, diff --git a/src/infra/net/fetch-guard.ssrf.test.ts b/src/infra/net/fetch-guard.ssrf.test.ts index eeec7181248..7bbbe5ebc54 100644 --- a/src/infra/net/fetch-guard.ssrf.test.ts +++ b/src/infra/net/fetch-guard.ssrf.test.ts @@ -1176,6 +1176,43 @@ describe("fetchWithSsrFGuard hardening", () => { await result.release(); }); + it("does not apply target hostname allowlists to public explicit proxy hosts", async () => { + (globalThis as Record)[TEST_UNDICI_RUNTIME_DEPS_KEY] = { + Agent: agentCtor, + EnvHttpProxyAgent: envHttpProxyAgentCtor, + ProxyAgent: proxyAgentCtor, + fetch: vi.fn(async () => okResponse()), + }; + const lookupFn: LookupFn = vi.fn(async (hostname: string) => { + if (hostname === "proxy.example.net") { + return [{ address: "93.184.216.34", family: 4 }]; + } + return [{ address: "149.154.167.220", family: 4 }]; + }) as unknown as LookupFn; + const fetchImpl = vi.fn(async () => okResponse()); + + const result = await fetchWithSsrFGuard({ + url: "https://api.telegram.org/file/bot123/photos/test.jpg", + fetchImpl, + lookupFn, + policy: { + allowRfc2544BenchmarkRange: true, + allowIpv6UniqueLocalRange: true, + hostnameAllowlist: ["api.telegram.org"], + }, + dispatcherPolicy: { + mode: "explicit-proxy", + proxyUrl: "http://proxy.example.net:6152", + }, + }); + + expect(fetchImpl).toHaveBeenCalledTimes(1); + expect(lookupFn).toHaveBeenCalledWith("proxy.example.net", { all: true }); + expect(lookupFn).toHaveBeenCalledWith("api.telegram.org", { all: true }); + expect(proxyAgentCtor).toHaveBeenCalled(); + await result.release(); + }); + it("skips target DNS pinning in trusted explicit-proxy mode after hostname-policy checks", async () => { (globalThis as Record)[TEST_UNDICI_RUNTIME_DEPS_KEY] = { Agent: agentCtor, diff --git a/src/infra/net/fetch-guard.ts b/src/infra/net/fetch-guard.ts index 9acea3c5092..fad52009660 100644 --- a/src/infra/net/fetch-guard.ts +++ b/src/infra/net/fetch-guard.ts @@ -194,21 +194,20 @@ async function assertExplicitProxyAllowed( if (!["http:", "https:"].includes(parsedProxyUrl.protocol)) { throw new Error("Explicit proxy URL must use http or https"); } + const proxyPolicy: SsrFPolicy | undefined = + policy || dispatcherPolicy.allowPrivateProxy === true + ? { + ...policy, + // The proxy hostname is operator-configured, not user input. Target-scoped + // allowlists must not reject a configured proxy host before the request + // target gets checked against that same allowlist below. + hostnameAllowlist: undefined, + ...(dispatcherPolicy.allowPrivateProxy === true ? { allowPrivateNetwork: true } : {}), + } + : undefined; await resolvePinnedHostnameWithPolicy(parsedProxyUrl.hostname, { lookupFn, - policy: - dispatcherPolicy.allowPrivateProxy === true - ? { - // The proxy hostname is operator-configured, not user input. - // Clear the target-scoped hostnameAllowlist so configured proxies - // like localhost or internal hosts aren't rejected by an allowlist - // that was built for the target URL (for example api.example.test). - // Private-network IP checks still apply via allowPrivateNetwork. - ...policy, - allowPrivateNetwork: true, - hostnameAllowlist: undefined, - } - : policy, + policy: proxyPolicy, }); } diff --git a/src/infra/net/ssrf.test.ts b/src/infra/net/ssrf.test.ts index ffeca962b2a..e6d3a02f6b3 100644 --- a/src/infra/net/ssrf.test.ts +++ b/src/infra/net/ssrf.test.ts @@ -5,6 +5,7 @@ import { isPrivateIpAddress, isSameSsrFPolicy, ssrfPolicyFromHttpBaseUrlAllowedHostname, + ssrfPolicyFromHttpBaseUrlFakeIpHostnameAllowlist, } from "./ssrf.js"; const privateIpCases = [ @@ -125,6 +126,26 @@ describe("ssrfPolicyFromHttpBaseUrlAllowedHostname", () => { }); }); +describe("ssrfPolicyFromHttpBaseUrlFakeIpHostnameAllowlist", () => { + it("builds a host-scoped fake-IP policy from HTTP base URLs", () => { + expect( + ssrfPolicyFromHttpBaseUrlFakeIpHostnameAllowlist(" https://api.example.com/v1 "), + ).toEqual({ + allowRfc2544BenchmarkRange: true, + allowIpv6UniqueLocalRange: true, + hostnameAllowlist: ["api.example.com"], + }); + }); + + it("ignores empty, invalid, and non-HTTP URLs", () => { + expect(ssrfPolicyFromHttpBaseUrlFakeIpHostnameAllowlist("")).toBeUndefined(); + expect(ssrfPolicyFromHttpBaseUrlFakeIpHostnameAllowlist("not-a-url")).toBeUndefined(); + expect( + ssrfPolicyFromHttpBaseUrlFakeIpHostnameAllowlist("ftp://api.example.com"), + ).toBeUndefined(); + }); +}); + describe("isBlockedHostnameOrIp", () => { it.each([ "localhost.localdomain", diff --git a/src/infra/net/ssrf.ts b/src/infra/net/ssrf.ts index f36e2f69c15..862901574b2 100644 --- a/src/infra/net/ssrf.ts +++ b/src/infra/net/ssrf.ts @@ -99,6 +99,28 @@ export function ssrfPolicyFromHttpBaseUrlAllowedHostname(baseUrl: string): SsrFP } } +export function ssrfPolicyFromHttpBaseUrlFakeIpHostnameAllowlist( + baseUrl: string, +): SsrFPolicy | undefined { + const trimmed = baseUrl.trim(); + if (!trimmed) { + return undefined; + } + try { + const parsed = new URL(trimmed); + if (parsed.protocol !== "http:" && parsed.protocol !== "https:") { + return undefined; + } + return { + allowRfc2544BenchmarkRange: true, + allowIpv6UniqueLocalRange: true, + hostnameAllowlist: [parsed.hostname], + }; + } catch { + return undefined; + } +} + const BLOCKED_HOSTNAMES = new Set([ "localhost", "localhost.localdomain",