diff --git a/CHANGELOG.md b/CHANGELOG.md index 445c77856e7..1b43282e291 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,7 @@ Docs: https://docs.openclaw.ai - Providers/OpenAI Codex: synthesize the `openai-codex/gpt-5.5` OAuth model row when Codex catalog discovery omits it, so cron and subagent runs do not fail with `Unknown model` while the account is authenticated. - Models/CLI: keep `openclaw models list` read-only while still showing eligible configured-provider rows, so listing models no longer rewrites per-agent `models.json`. (#70847) Thanks @shakkernerd. - Providers/Google: honor the private-network SSRF opt-in for Gemini image generation requests, so trusted proxy setups that resolve Google API hosts to private addresses can use `image_generate`. Fixes #67216. +- Agents/transport: propagate configured attempt timeouts into guarded per-request dispatchers, so slow local LLM calls such as Ollama no longer fail at Undici's default 60-second body timeout. Fixes #70829. (#70831) Thanks @DranboFieldston. - Agents/transport: stop embedded runs from lowering the process-wide undici stream timeouts, so slow Gemini image generation and other long-running provider requests no longer inherit short run-attempt headers timeouts. Fixes #70423. Thanks @giangthb. - Providers/OpenRouter: send image-understanding prompts as user text before image parts, restoring non-empty vision responses for OpenRouter multimodal models. Fixes #70410. - Plugins/providers: mirror runtime auth choices in bundled provider manifests and detect `KIMI_API_KEY` for Moonshot/Kimi web search before plugin runtime loads. Thanks @vincentkoc. diff --git a/src/agents/provider-transport-fetch.test.ts b/src/agents/provider-transport-fetch.test.ts index ab026ee6a34..5777f0bf622 100644 --- a/src/agents/provider-transport-fetch.test.ts +++ b/src/agents/provider-transport-fetch.test.ts @@ -75,6 +75,25 @@ describe("buildGuardedModelFetch", () => { ); }); + it("threads explicit transport timeouts into the shared guarded fetch seam", async () => { + const { buildGuardedModelFetch } = await import("./provider-transport-fetch.js"); + const model = { + id: "gpt-5.4", + provider: "openai", + api: "openai-responses", + baseUrl: "https://api.openai.com/v1", + } as unknown as Model<"openai-responses">; + + const fetcher = buildGuardedModelFetch(model, 123_456); + await fetcher("https://api.openai.com/v1/responses", { method: "POST" }); + + expect(fetchWithSsrFGuardMock).toHaveBeenCalledWith( + expect.objectContaining({ + timeoutMs: 123_456, + }), + ); + }); + it("does not force explicit debug proxy overrides onto plain HTTP model transports", async () => { process.env.OPENCLAW_DEBUG_PROXY_ENABLED = "1"; process.env.OPENCLAW_DEBUG_PROXY_URL = "http://127.0.0.1:7799"; diff --git a/src/agents/provider-transport-fetch.ts b/src/agents/provider-transport-fetch.ts index 815f703de63..5842e553915 100644 --- a/src/agents/provider-transport-fetch.ts +++ b/src/agents/provider-transport-fetch.ts @@ -150,7 +150,7 @@ function resolveModelRequestPolicy(model: Model) { }); } -export function buildGuardedModelFetch(model: Model): typeof fetch { +export function buildGuardedModelFetch(model: Model, timeoutMs?: number): typeof fetch { const requestConfig = resolveModelRequestPolicy(model); const dispatcherPolicy = buildProviderRequestDispatcherPolicy(requestConfig); return async (input, init) => { @@ -185,6 +185,7 @@ export function buildGuardedModelFetch(model: Model): typeof fetch { }, }, dispatcherPolicy, + timeoutMs, // Provider transport intentionally keeps the secure default and never // replays unsafe request bodies across cross-origin redirects. allowCrossOriginUnsafeRedirectReplay: false, diff --git a/src/infra/net/fetch-guard.ssrf.test.ts b/src/infra/net/fetch-guard.ssrf.test.ts index ee44c0793c3..a991c639d2f 100644 --- a/src/infra/net/fetch-guard.ssrf.test.ts +++ b/src/infra/net/fetch-guard.ssrf.test.ts @@ -4,6 +4,10 @@ import { GUARDED_FETCH_MODE, retainSafeHeadersForCrossOriginRedirectHeaders, } from "./fetch-guard.js"; +import { + ensureGlobalUndiciStreamTimeouts, + resetGlobalUndiciStreamTimeoutsForTests, +} from "./undici-global-dispatcher.js"; import { TEST_UNDICI_RUNTIME_DEPS_KEY } from "./undici-runtime.js"; const { agentCtor, envHttpProxyAgentCtor, proxyAgentCtor } = vi.hoisted(() => ({ @@ -171,6 +175,7 @@ describe("fetchWithSsrFGuard hardening", () => { envHttpProxyAgentCtor.mockClear(); proxyAgentCtor.mockClear(); logWarnMock.mockClear(); + resetGlobalUndiciStreamTimeoutsForTests(); Reflect.deleteProperty(globalThis as object, TEST_UNDICI_RUNTIME_DEPS_KEY); }); @@ -1013,6 +1018,69 @@ describe("fetchWithSsrFGuard hardening", () => { }); }); + it("applies explicit timeoutMs to guarded direct dispatchers", async () => { + (globalThis as Record)[TEST_UNDICI_RUNTIME_DEPS_KEY] = { + Agent: agentCtor, + EnvHttpProxyAgent: envHttpProxyAgentCtor, + ProxyAgent: proxyAgentCtor, + fetch: vi.fn(async () => okResponse()), + }; + const fetchImpl = vi.fn(async () => okResponse()); + + const result = await fetchWithSsrFGuard({ + url: "https://public.example/resource", + fetchImpl, + lookupFn: createPublicLookup(), + timeoutMs: 123_456, + }); + + expect(fetchImpl).toHaveBeenCalledTimes(1); + expect(agentCtor).toHaveBeenCalledWith({ + connect: expect.objectContaining({ + lookup: expect.any(Function), + }), + allowH2: false, + bodyTimeout: 123_456, + headersTimeout: 123_456, + }); + await result.release(); + }); + + it("inherits the configured global stream timeout for guarded direct dispatchers", async () => { + const { getGlobalDispatcher, setGlobalDispatcher } = await import("undici"); + const previousDispatcher = getGlobalDispatcher(); + try { + ensureGlobalUndiciStreamTimeouts({ timeoutMs: 1_900_000 }); + (globalThis as Record)[TEST_UNDICI_RUNTIME_DEPS_KEY] = { + Agent: agentCtor, + EnvHttpProxyAgent: envHttpProxyAgentCtor, + ProxyAgent: proxyAgentCtor, + fetch: vi.fn(async () => okResponse()), + }; + const fetchImpl = vi.fn(async () => okResponse()); + + const result = await fetchWithSsrFGuard({ + url: "https://public.example/resource", + fetchImpl, + lookupFn: createPublicLookup(), + }); + + expect(fetchImpl).toHaveBeenCalledTimes(1); + expect(agentCtor).toHaveBeenCalledWith({ + connect: expect.objectContaining({ + lookup: expect.any(Function), + }), + allowH2: false, + bodyTimeout: 1_900_000, + headersTimeout: 1_900_000, + }); + await result.release(); + } finally { + setGlobalDispatcher(previousDispatcher); + resetGlobalUndiciStreamTimeoutsForTests(); + } + }); + it("allows explicit proxy on localhost when allowPrivateProxy is true even with restrictive hostnameAllowlist", async () => { // Reproduces #61906: Telegram media downloads fail because the SSRF guard // checks the proxy hostname (localhost) against a target-scoped allowlist diff --git a/src/infra/net/fetch-guard.ts b/src/infra/net/fetch-guard.ts index 0f053fa8d92..a8c195f4902 100644 --- a/src/infra/net/fetch-guard.ts +++ b/src/infra/net/fetch-guard.ts @@ -19,12 +19,25 @@ import { SsrFBlockedError, type SsrFPolicy, } from "./ssrf.js"; +import { _globalUndiciStreamTimeoutMs } from "./undici-global-dispatcher.js"; import { createHttp1Agent, createHttp1EnvHttpProxyAgent, createHttp1ProxyAgent, } from "./undici-runtime.js"; +function resolveDispatcherTimeoutMs(fromParams: number | undefined): number | undefined { + if (fromParams !== undefined) { + return fromParams; + } + // Fall back to module-level bridge set by ensureGlobalUndiciStreamTimeouts + // (avoids reading Undici's non-public `.options` field) + if (_globalUndiciStreamTimeoutMs !== undefined) { + return _globalUndiciStreamTimeoutMs; + } + return undefined; +} + type FetchLike = (input: RequestInfo | URL, init?: RequestInit) => Promise; export const GUARDED_FETCH_MODE = { @@ -125,6 +138,7 @@ function assertExplicitProxySupportsPinnedDns( function createPolicyDispatcherWithoutPinnedDns( dispatcherPolicy?: PinnedDispatcherPolicy, + timeoutMs?: number, ): Dispatcher | null { if (!dispatcherPolicy) { return null; @@ -133,23 +147,28 @@ function createPolicyDispatcherWithoutPinnedDns( if (dispatcherPolicy.mode === "direct") { return createHttp1Agent( dispatcherPolicy.connect ? { connect: { ...dispatcherPolicy.connect } } : undefined, + timeoutMs, ); } if (dispatcherPolicy.mode === "env-proxy") { - return createHttp1EnvHttpProxyAgent({ - ...(dispatcherPolicy.connect ? { connect: { ...dispatcherPolicy.connect } } : {}), - ...(dispatcherPolicy.proxyTls ? { proxyTls: { ...dispatcherPolicy.proxyTls } } : {}), - }); + return createHttp1EnvHttpProxyAgent( + { + ...(dispatcherPolicy.connect ? { connect: { ...dispatcherPolicy.connect } } : {}), + ...(dispatcherPolicy.proxyTls ? { proxyTls: { ...dispatcherPolicy.proxyTls } } : {}), + }, + timeoutMs, + ); } const proxyUrl = dispatcherPolicy.proxyUrl.trim(); - return dispatcherPolicy.proxyTls - ? createHttp1ProxyAgent({ - uri: proxyUrl, - requestTls: { ...dispatcherPolicy.proxyTls }, - }) - : createHttp1ProxyAgent({ uri: proxyUrl }); + if (dispatcherPolicy.proxyTls) { + return createHttp1ProxyAgent( + { uri: proxyUrl, requestTls: { ...dispatcherPolicy.proxyTls } }, + timeoutMs, + ); + } + return createHttp1ProxyAgent({ uri: proxyUrl }, timeoutMs); } async function assertExplicitProxyAllowed( @@ -337,25 +356,31 @@ export async function fetchWithSsrFGuard(params: GuardedFetchOptions): Promise { }); }); + it("applies stream timeouts to pinned direct dispatchers", () => { + const lookup = vi.fn() as unknown as PinnedHostname["lookup"]; + const pinned: PinnedHostname = { + hostname: "api.telegram.org", + addresses: ["149.154.167.220"], + lookup, + }; + + createPinnedDispatcher(pinned, undefined, undefined, 123_456); + + expect(agentCtor).toHaveBeenCalledWith({ + connect: { + lookup, + }, + allowH2: false, + bodyTimeout: 123_456, + headersTimeout: 123_456, + }); + }); + it("replaces the pinned lookup when a dispatcher override hostname is provided", () => { const originalLookup = vi.fn() as unknown as PinnedHostname["lookup"]; const lookup = createDispatcherWithPinnedOverride(originalLookup); @@ -217,4 +237,37 @@ describe("createPinnedDispatcher", () => { }, }); }); + + it("applies stream timeouts to explicit proxy dispatchers", () => { + const lookup = vi.fn() as unknown as PinnedHostname["lookup"]; + const pinned: PinnedHostname = { + hostname: "api.telegram.org", + addresses: ["149.154.167.220"], + lookup, + }; + + createPinnedDispatcher( + pinned, + { + mode: "explicit-proxy", + proxyUrl: "http://127.0.0.1:7890", + proxyTls: { + autoSelectFamily: false, + }, + }, + undefined, + 654_321, + ); + + expect(proxyAgentCtor).toHaveBeenCalledWith({ + uri: "http://127.0.0.1:7890", + requestTls: { + autoSelectFamily: false, + lookup, + }, + allowH2: false, + bodyTimeout: 654_321, + headersTimeout: 654_321, + }); + }); }); diff --git a/src/infra/net/ssrf.ts b/src/infra/net/ssrf.ts index 0df8fb0096e..a240adf5b02 100644 --- a/src/infra/net/ssrf.ts +++ b/src/infra/net/ssrf.ts @@ -448,34 +448,39 @@ export function createPinnedDispatcher( pinned: PinnedHostname, policy?: PinnedDispatcherPolicy, ssrfPolicy?: SsrFPolicy, + timeoutMs?: number, ): Dispatcher { const lookup = resolvePinnedDispatcherLookup(pinned, policy?.pinnedHostname, ssrfPolicy); if (!policy || policy.mode === "direct") { - return createHttp1Agent({ - connect: withPinnedLookup(lookup, policy?.connect), - }); + return createHttp1Agent({ connect: withPinnedLookup(lookup, policy?.connect) }, timeoutMs); } if (policy.mode === "env-proxy") { - return createHttp1EnvHttpProxyAgent({ - connect: withPinnedLookup(lookup, policy.connect), - ...(policy.proxyTls ? { proxyTls: { ...policy.proxyTls } } : {}), - }); + return createHttp1EnvHttpProxyAgent( + { + connect: withPinnedLookup(lookup, policy.connect), + ...(policy.proxyTls ? { proxyTls: { ...policy.proxyTls } } : {}), + }, + timeoutMs, + ); } const proxyUrl = policy.proxyUrl.trim(); const requestTls = withPinnedLookup(lookup, policy.proxyTls); if (!requestTls) { - return createHttp1ProxyAgent({ uri: proxyUrl }); + return createHttp1ProxyAgent({ uri: proxyUrl }, timeoutMs); } - return createHttp1ProxyAgent({ - uri: proxyUrl, - // `PinnedDispatcherPolicy.proxyTls` historically carried target-hop - // transport hints for explicit proxies. Translate that to undici's - // `requestTls` so HTTPS proxy tunnels keep the pinned DNS lookup. - requestTls, - }); + return createHttp1ProxyAgent( + { + uri: proxyUrl, + // `PinnedDispatcherPolicy.proxyTls` historically carried target-hop + // transport hints for explicit proxies. Translate that to undici's + // `requestTls` so HTTPS proxy tunnels keep the pinned DNS lookup. + requestTls, + }, + timeoutMs, + ); } export async function closeDispatcher(dispatcher?: Dispatcher | null): Promise { diff --git a/src/infra/net/undici-global-dispatcher.test.ts b/src/infra/net/undici-global-dispatcher.test.ts index 0dcfc2f2f43..41092c34e2b 100644 --- a/src/infra/net/undici-global-dispatcher.test.ts +++ b/src/infra/net/undici-global-dispatcher.test.ts @@ -73,15 +73,17 @@ let DEFAULT_UNDICI_STREAM_TIMEOUT_MS: typeof import("./undici-global-dispatcher. let ensureGlobalUndiciEnvProxyDispatcher: typeof import("./undici-global-dispatcher.js").ensureGlobalUndiciEnvProxyDispatcher; let ensureGlobalUndiciStreamTimeouts: typeof import("./undici-global-dispatcher.js").ensureGlobalUndiciStreamTimeouts; let resetGlobalUndiciStreamTimeoutsForTests: typeof import("./undici-global-dispatcher.js").resetGlobalUndiciStreamTimeoutsForTests; +let undiciGlobalDispatcherModule: typeof import("./undici-global-dispatcher.js"); describe("ensureGlobalUndiciStreamTimeouts", () => { beforeAll(async () => { + undiciGlobalDispatcherModule = await import("./undici-global-dispatcher.js"); ({ DEFAULT_UNDICI_STREAM_TIMEOUT_MS, ensureGlobalUndiciEnvProxyDispatcher, ensureGlobalUndiciStreamTimeouts, resetGlobalUndiciStreamTimeoutsForTests, - } = await import("./undici-global-dispatcher.js")); + } = undiciGlobalDispatcherModule); }); beforeEach(() => { @@ -125,12 +127,13 @@ describe("ensureGlobalUndiciStreamTimeouts", () => { }); }); - it("does not override unsupported custom proxy dispatcher types", () => { + it("records timeout bridge but does not override unsupported custom proxy dispatcher types", () => { setCurrentDispatcher(new ProxyAgent("http://proxy.test:8080")); - ensureGlobalUndiciStreamTimeouts(); + ensureGlobalUndiciStreamTimeouts({ timeoutMs: 1_900_000 }); expect(setGlobalDispatcher).not.toHaveBeenCalled(); + expect(undiciGlobalDispatcherModule._globalUndiciStreamTimeoutMs).toBe(1_900_000); }); it("is idempotent for unchanged dispatcher kind and network policy", () => { diff --git a/src/infra/net/undici-global-dispatcher.ts b/src/infra/net/undici-global-dispatcher.ts index 6eac302e968..0bef2b84511 100644 --- a/src/infra/net/undici-global-dispatcher.ts +++ b/src/infra/net/undici-global-dispatcher.ts @@ -5,6 +5,13 @@ import { hasEnvHttpProxyConfigured } from "./proxy-env.js"; export const DEFAULT_UNDICI_STREAM_TIMEOUT_MS = 30 * 60 * 1000; +/** + * Module-level bridge so `resolveDispatcherTimeoutMs` in fetch-guard.ts + * can read the global dispatcher timeout without relying on Undici's + * non-public `.options` field. + */ +export let _globalUndiciStreamTimeoutMs: number | undefined; + const AUTO_SELECT_FAMILY_ATTEMPT_TIMEOUT_MS = 300; let lastAppliedTimeoutKey: string | null = null; @@ -114,6 +121,7 @@ export function ensureGlobalUndiciStreamTimeouts(opts?: { timeoutMs?: number }): return; } const timeoutMs = Math.max(DEFAULT_UNDICI_STREAM_TIMEOUT_MS, Math.floor(timeoutMsRaw)); + _globalUndiciStreamTimeoutMs = timeoutMs; const kind = resolveCurrentDispatcherKind(); if (kind === null) { return; @@ -152,4 +160,5 @@ export function ensureGlobalUndiciStreamTimeouts(opts?: { timeoutMs?: number }): export function resetGlobalUndiciStreamTimeoutsForTests(): void { lastAppliedTimeoutKey = null; lastAppliedProxyBootstrap = false; + _globalUndiciStreamTimeoutMs = undefined; } diff --git a/src/infra/net/undici-runtime.ts b/src/infra/net/undici-runtime.ts index 5cf8434c948..c6e7c23b0b6 100644 --- a/src/infra/net/undici-runtime.ts +++ b/src/infra/net/undici-runtime.ts @@ -53,36 +53,47 @@ export function loadUndiciRuntimeDeps(): UndiciRuntimeDeps { function withHttp1OnlyDispatcherOptions( options?: T, + timeoutMs?: number, ): (T extends object ? T : Record) & { allowH2: false } { - if (!options) { - return { ...HTTP1_ONLY_DISPATCHER_OPTIONS } as (T extends object ? T : Record) & { - allowH2: false; - }; + const base = {} as (T extends object ? T : Record) & { allowH2: false }; + if (options) { + Object.assign(base, options); } - return { - ...options, - ...HTTP1_ONLY_DISPATCHER_OPTIONS, - } as (T extends object ? T : Record) & { allowH2: false }; + // Enforce HTTP/1.1-only — must come after options to prevent accidental override + Object.assign(base, HTTP1_ONLY_DISPATCHER_OPTIONS); + if (timeoutMs !== undefined && Number.isFinite(timeoutMs) && timeoutMs > 0) { + (base as Record).bodyTimeout = timeoutMs; + (base as Record).headersTimeout = timeoutMs; + } + return base; } -export function createHttp1Agent(options?: UndiciAgentOptions): import("undici").Agent { +export function createHttp1Agent( + options?: UndiciAgentOptions, + timeoutMs?: number, +): import("undici").Agent { const { Agent } = loadUndiciRuntimeDeps(); - return new Agent(withHttp1OnlyDispatcherOptions(options)); + return new Agent(withHttp1OnlyDispatcherOptions(options, timeoutMs)); } export function createHttp1EnvHttpProxyAgent( options?: UndiciEnvHttpProxyAgentOptions, + timeoutMs?: number, ): import("undici").EnvHttpProxyAgent { const { EnvHttpProxyAgent } = loadUndiciRuntimeDeps(); - return new EnvHttpProxyAgent(withHttp1OnlyDispatcherOptions(options)); + return new EnvHttpProxyAgent(withHttp1OnlyDispatcherOptions(options, timeoutMs)); } export function createHttp1ProxyAgent( options: UndiciProxyAgentOptions, + timeoutMs?: number, ): import("undici").ProxyAgent { const { ProxyAgent } = loadUndiciRuntimeDeps(); - if (typeof options === "string" || options instanceof URL) { - return new ProxyAgent(withHttp1OnlyDispatcherOptions({ uri: options.toString() })); - } - return new ProxyAgent(withHttp1OnlyDispatcherOptions(options)); + const normalized = + typeof options === "string" || options instanceof URL + ? { uri: options.toString() } + : { ...options }; + return new ProxyAgent( + withHttp1OnlyDispatcherOptions(normalized as object, timeoutMs) as UndiciProxyAgentOptions, + ); }