diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c775dbee0b..1182205ee12 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ Docs: https://docs.openclaw.ai ### Fixes +- Memory-host SDK: use trusted env-proxy mode for remote embedding and batch HTTP calls only when Undici will proxy that target, preserving SSRF DNS pinning for `ALL_PROXY`-only and `NO_PROXY` bypass cases. Fixes #52162. (#71506) Thanks @DhtIsCoding. - Gateway/dashboard: render Control UI and WebSocket links with `https://`/`wss://` when `gateway.tls.enabled=true`, including `openclaw gateway status`. Fixes #71494. (#71499) Thanks @deepkilo. - Agents/OpenAI-compatible: default proxy/local completions tool requests to `tool_choice: "auto"` when tools are present, so providers enter native tool-calling mode instead of replying with plain-text tool directives. (#71472) Thanks @Speed-maker. - OpenAI image generation: use `gpt-5.5` for the Codex OAuth responses transport instead of the retired `gpt-5.4` model, fixing 500s from ChatGPT Codex image generation. Fixes #71513. Thanks @baolongl. diff --git a/packages/memory-host-sdk/src/host/remote-http.test.ts b/packages/memory-host-sdk/src/host/remote-http.test.ts new file mode 100644 index 00000000000..a7a7b0bb1f4 --- /dev/null +++ b/packages/memory-host-sdk/src/host/remote-http.test.ts @@ -0,0 +1,68 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const { fetchWithSsrFGuardMock, shouldUseEnvHttpProxyForUrlMock } = vi.hoisted(() => ({ + fetchWithSsrFGuardMock: vi.fn(), + shouldUseEnvHttpProxyForUrlMock: vi.fn(() => false), +})); + +vi.mock("../../../../src/infra/net/fetch-guard.js", async () => { + const actual = await vi.importActual( + "../../../../src/infra/net/fetch-guard.js", + ); + return { + ...actual, + fetchWithSsrFGuard: fetchWithSsrFGuardMock, + }; +}); + +vi.mock("../../../../src/infra/net/proxy-env.js", async () => { + const actual = await vi.importActual( + "../../../../src/infra/net/proxy-env.js", + ); + return { + ...actual, + shouldUseEnvHttpProxyForUrl: shouldUseEnvHttpProxyForUrlMock, + }; +}); + +import { GUARDED_FETCH_MODE } from "../../../../src/infra/net/fetch-guard.js"; +import { withRemoteHttpResponse } from "./remote-http.js"; + +describe("package withRemoteHttpResponse", () => { + beforeEach(() => { + vi.clearAllMocks(); + shouldUseEnvHttpProxyForUrlMock.mockReturnValue(false); + fetchWithSsrFGuardMock.mockResolvedValue({ + response: new Response("ok", { status: 200 }), + finalUrl: "https://memory.example/v1", + release: vi.fn(async () => {}), + }); + }); + + it("uses trusted env proxy mode when the target will use EnvHttpProxyAgent", async () => { + shouldUseEnvHttpProxyForUrlMock.mockReturnValue(true); + + await withRemoteHttpResponse({ + url: "https://memory.example/v1/embeddings", + onResponse: async () => undefined, + }); + + expect(fetchWithSsrFGuardMock).toHaveBeenCalledWith( + expect.objectContaining({ + url: "https://memory.example/v1/embeddings", + mode: GUARDED_FETCH_MODE.TRUSTED_ENV_PROXY, + }), + ); + }); + + it("keeps strict guarded fetch mode when proxy env would not proxy the target", async () => { + await withRemoteHttpResponse({ + url: "https://internal.corp.example/v1/embeddings", + onResponse: async () => undefined, + }); + + const call = fetchWithSsrFGuardMock.mock.calls[0]?.[0]; + expect(call).toBeDefined(); + expect(call).not.toHaveProperty("mode"); + }); +}); diff --git a/packages/memory-host-sdk/src/host/remote-http.ts b/packages/memory-host-sdk/src/host/remote-http.ts index f591830ff62..919e4aa7815 100644 --- a/packages/memory-host-sdk/src/host/remote-http.ts +++ b/packages/memory-host-sdk/src/host/remote-http.ts @@ -1,4 +1,5 @@ -import { fetchWithSsrFGuard } from "../../../../src/infra/net/fetch-guard.js"; +import { fetchWithSsrFGuard, GUARDED_FETCH_MODE } from "../../../../src/infra/net/fetch-guard.js"; +import { shouldUseEnvHttpProxyForUrl } from "../../../../src/infra/net/proxy-env.js"; import type { SsrFPolicy } from "../../../../src/infra/net/ssrf.js"; export function buildRemoteBaseUrlPolicy(baseUrl: string): SsrFPolicy | undefined { @@ -31,6 +32,9 @@ export async function withRemoteHttpResponse(params: { init: params.init, policy: params.ssrfPolicy, auditContext: params.auditContext ?? "memory-remote", + ...(shouldUseEnvHttpProxyForUrl(params.url) + ? { mode: GUARDED_FETCH_MODE.TRUSTED_ENV_PROXY } + : {}), }); try { return await params.onResponse(response); diff --git a/src/infra/net/fetch-guard.ssrf.test.ts b/src/infra/net/fetch-guard.ssrf.test.ts index a991c639d2f..0b0d3c5c8d1 100644 --- a/src/infra/net/fetch-guard.ssrf.test.ts +++ b/src/infra/net/fetch-guard.ssrf.test.ts @@ -133,7 +133,7 @@ describe("fetchWithSsrFGuard hardening", () => { expectEnvProxy: boolean; }): Promise { clearProxyEnv(); - vi.stubEnv("HTTP_PROXY", "http://127.0.0.1:7890"); + vi.stubEnv("http_proxy", "http://127.0.0.1:7890"); (globalThis as Record)[TEST_UNDICI_RUNTIME_DEPS_KEY] = { Agent: agentCtor, EnvHttpProxyAgent: envHttpProxyAgentCtor, @@ -1018,6 +1018,57 @@ describe("fetchWithSsrFGuard hardening", () => { }); }); + it("keeps DNS pinning in trusted proxy mode when only ALL_PROXY is configured", async () => { + clearProxyEnv(); + vi.stubEnv("ALL_PROXY", "http://127.0.0.1:7890"); + (globalThis as Record)[TEST_UNDICI_RUNTIME_DEPS_KEY] = { + Agent: agentCtor, + EnvHttpProxyAgent: envHttpProxyAgentCtor, + ProxyAgent: proxyAgentCtor, + fetch: vi.fn(async () => okResponse()), + }; + const lookupFn = createPublicLookup(); + const fetchImpl = vi.fn(async () => okResponse()); + + const result = await fetchWithSsrFGuard({ + url: "https://public.example/resource", + fetchImpl, + lookupFn, + mode: GUARDED_FETCH_MODE.TRUSTED_ENV_PROXY, + }); + + expect(envHttpProxyAgentCtor).not.toHaveBeenCalled(); + expect(agentCtor).toHaveBeenCalled(); + expect(lookupFn).toHaveBeenCalledWith("public.example", { all: true }); + await result.release(); + }); + + it("keeps DNS pinning in trusted proxy mode for NO_PROXY targets", async () => { + clearProxyEnv(); + vi.stubEnv("HTTPS_PROXY", "http://127.0.0.1:7890"); + vi.stubEnv("NO_PROXY", "public.example"); + (globalThis as Record)[TEST_UNDICI_RUNTIME_DEPS_KEY] = { + Agent: agentCtor, + EnvHttpProxyAgent: envHttpProxyAgentCtor, + ProxyAgent: proxyAgentCtor, + fetch: vi.fn(async () => okResponse()), + }; + const lookupFn = createPublicLookup(); + const fetchImpl = vi.fn(async () => okResponse()); + + const result = await fetchWithSsrFGuard({ + url: "https://public.example/resource", + fetchImpl, + lookupFn, + mode: GUARDED_FETCH_MODE.TRUSTED_ENV_PROXY, + }); + + expect(envHttpProxyAgentCtor).not.toHaveBeenCalled(); + expect(agentCtor).toHaveBeenCalled(); + expect(lookupFn).toHaveBeenCalledWith("public.example", { all: true }); + await result.release(); + }); + it("applies explicit timeoutMs to guarded direct dispatchers", async () => { (globalThis as Record)[TEST_UNDICI_RUNTIME_DEPS_KEY] = { Agent: agentCtor, diff --git a/src/infra/net/fetch-guard.ts b/src/infra/net/fetch-guard.ts index a8c195f4902..170ec72aaab 100644 --- a/src/infra/net/fetch-guard.ts +++ b/src/infra/net/fetch-guard.ts @@ -2,7 +2,7 @@ import type { Dispatcher } from "undici"; import { logWarn } from "../../logger.js"; import { captureHttpExchange } from "../../proxy-capture/runtime.js"; import { buildTimeoutAbortSignal } from "../../utils/fetch-timeout.js"; -import { hasProxyEnvConfigured } from "./proxy-env.js"; +import { shouldUseEnvHttpProxyForUrl } from "./proxy-env.js"; import { retainSafeHeadersForCrossOriginRedirect as retainSafeRedirectHeaders } from "./redirect-headers.js"; import { fetchWithRuntimeDispatcher, @@ -355,7 +355,8 @@ export async function fetchWithSsrFGuard(params: GuardedFetchOptions): Promise { @@ -233,3 +234,55 @@ describe("matchesNoProxy", () => { expect(matchesNoProxy(url, env)).toBe(expected); }); }); + +describe("shouldUseEnvHttpProxyForUrl", () => { + it.each([ + { + name: "uses HTTPS_PROXY for https URLs", + url: "https://api.example.com/v1", + env: { HTTPS_PROXY: "http://proxy.test:8080" } as NodeJS.ProcessEnv, + expected: true, + }, + { + name: "falls back to HTTP_PROXY for https URLs", + url: "https://api.example.com/v1", + env: { HTTP_PROXY: "http://proxy.test:8080" } as NodeJS.ProcessEnv, + expected: true, + }, + { + name: "uses HTTP_PROXY for http URLs", + url: "http://api.example.com/v1", + env: { HTTP_PROXY: "http://proxy.test:8080" } as NodeJS.ProcessEnv, + expected: true, + }, + { + name: "ignores ALL_PROXY-only environments", + url: "https://api.example.com/v1", + env: { ALL_PROXY: "http://proxy.test:8080" } as NodeJS.ProcessEnv, + expected: false, + }, + { + name: "keeps strict mode for NO_PROXY matches", + url: "https://internal.corp.example/v1", + env: { + HTTPS_PROXY: "http://proxy.test:8080", + NO_PROXY: "corp.example", + } as NodeJS.ProcessEnv, + expected: false, + }, + { + name: "keeps strict mode for non-http URLs", + url: "file:///tmp/input.txt", + env: { HTTPS_PROXY: "http://proxy.test:8080" } as NodeJS.ProcessEnv, + expected: false, + }, + { + name: "keeps strict mode for malformed URLs", + url: "not-a-url", + env: { HTTPS_PROXY: "http://proxy.test:8080" } as NodeJS.ProcessEnv, + expected: false, + }, + ])("$name", ({ url, env, expected }) => { + expect(shouldUseEnvHttpProxyForUrl(url, env)).toBe(expected); + }); +}); diff --git a/src/infra/net/proxy-env.ts b/src/infra/net/proxy-env.ts index 1f154ac4ae5..283fe60d94a 100644 --- a/src/infra/net/proxy-env.ts +++ b/src/infra/net/proxy-env.ts @@ -54,6 +54,27 @@ export function hasEnvHttpProxyConfigured( return resolveEnvHttpProxyUrl(protocol, env) !== undefined; } +export function shouldUseEnvHttpProxyForUrl( + targetUrl: string, + env: NodeJS.ProcessEnv = process.env, +): boolean { + let protocol: "http" | "https"; + try { + const parsed = new URL(targetUrl); + if (parsed.protocol === "http:") { + protocol = "http"; + } else if (parsed.protocol === "https:") { + protocol = "https"; + } else { + return false; + } + } catch { + return false; + } + + return hasEnvHttpProxyConfigured(protocol, env) && !matchesNoProxy(targetUrl, env); +} + /** * Check whether a target URL should bypass the HTTP proxy per NO_PROXY env var. * diff --git a/src/media-understanding/shared.test.ts b/src/media-understanding/shared.test.ts index 6b5b214c657..66e9b052efb 100644 --- a/src/media-understanding/shared.test.ts +++ b/src/media-understanding/shared.test.ts @@ -1,12 +1,9 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -const { fetchWithSsrFGuardMock, hasEnvHttpProxyConfiguredMock, matchesNoProxyMock } = vi.hoisted( - () => ({ - fetchWithSsrFGuardMock: vi.fn(), - hasEnvHttpProxyConfiguredMock: vi.fn(() => false), - matchesNoProxyMock: vi.fn(() => false), - }), -); +const { fetchWithSsrFGuardMock, shouldUseEnvHttpProxyForUrlMock } = vi.hoisted(() => ({ + fetchWithSsrFGuardMock: vi.fn(), + shouldUseEnvHttpProxyForUrlMock: vi.fn(() => false), +})); vi.mock("../infra/net/fetch-guard.js", async () => { const actual = await vi.importActual( @@ -24,8 +21,7 @@ vi.mock("../infra/net/proxy-env.js", async () => { ); return { ...actual, - hasEnvHttpProxyConfigured: hasEnvHttpProxyConfiguredMock, - matchesNoProxy: matchesNoProxyMock, + shouldUseEnvHttpProxyForUrl: shouldUseEnvHttpProxyForUrlMock, }; }); @@ -42,8 +38,7 @@ import { } from "./shared.js"; beforeEach(() => { - hasEnvHttpProxyConfiguredMock.mockReturnValue(false); - matchesNoProxyMock.mockReturnValue(false); + shouldUseEnvHttpProxyForUrlMock.mockReturnValue(false); }); afterEach(() => { @@ -417,7 +412,7 @@ describe("fetchWithTimeoutGuarded", () => { }); it("does not set a guarded fetch mode when no HTTP proxy env is configured", async () => { - hasEnvHttpProxyConfiguredMock.mockReturnValue(false); + shouldUseEnvHttpProxyForUrlMock.mockReturnValue(false); fetchWithSsrFGuardMock.mockResolvedValue({ response: new Response(null, { status: 200 }), finalUrl: "https://example.com", @@ -432,7 +427,7 @@ describe("fetchWithTimeoutGuarded", () => { }); it("auto-selects trusted env proxy mode when HTTP proxy env is configured", async () => { - hasEnvHttpProxyConfiguredMock.mockReturnValue(true); + shouldUseEnvHttpProxyForUrlMock.mockReturnValue(true); fetchWithSsrFGuardMock.mockResolvedValue({ response: new Response(null, { status: 200 }), finalUrl: "https://api.minimax.io", @@ -454,7 +449,7 @@ describe("fetchWithTimeoutGuarded", () => { }); it("respects an explicit mode from the caller when HTTP proxy env is configured", async () => { - hasEnvHttpProxyConfiguredMock.mockReturnValue(true); + shouldUseEnvHttpProxyForUrlMock.mockReturnValue(true); fetchWithSsrFGuardMock.mockResolvedValue({ response: new Response(null, { status: 200 }), finalUrl: "https://api.example.com", @@ -473,7 +468,7 @@ describe("fetchWithTimeoutGuarded", () => { }); it("auto-upgrades transcription requests to trusted env proxy when proxy env is configured", async () => { - hasEnvHttpProxyConfiguredMock.mockReturnValue(true); + shouldUseEnvHttpProxyForUrlMock.mockReturnValue(true); fetchWithSsrFGuardMock.mockResolvedValue({ response: new Response(null, { status: 200 }), finalUrl: "https://api.openai.com", @@ -495,7 +490,7 @@ describe("fetchWithTimeoutGuarded", () => { }); it("forwards an explicit mode override through postJsonRequest even when proxy env is configured", async () => { - hasEnvHttpProxyConfiguredMock.mockReturnValue(true); + shouldUseEnvHttpProxyForUrlMock.mockReturnValue(true); fetchWithSsrFGuardMock.mockResolvedValue({ response: new Response(null, { status: 200 }), finalUrl: "https://api.example.com", @@ -518,7 +513,7 @@ describe("fetchWithTimeoutGuarded", () => { }); it("forwards an explicit mode override through postTranscriptionRequest even when proxy env is configured", async () => { - hasEnvHttpProxyConfiguredMock.mockReturnValue(true); + shouldUseEnvHttpProxyForUrlMock.mockReturnValue(true); fetchWithSsrFGuardMock.mockResolvedValue({ response: new Response(null, { status: 200 }), finalUrl: "https://api.example.com", @@ -541,11 +536,11 @@ describe("fetchWithTimeoutGuarded", () => { }); it("does not auto-upgrade when only ALL_PROXY is configured (HTTP(S) proxy gate)", async () => { - // ALL_PROXY is ignored by EnvHttpProxyAgent; `hasEnvHttpProxyConfigured` + // ALL_PROXY is ignored by EnvHttpProxyAgent; the shared proxy URL helper // reflects that by returning false when only ALL_PROXY is set. Auto-upgrade // must NOT fire, otherwise the request would skip pinned-DNS/SSRF checks // and then be dispatched directly. - hasEnvHttpProxyConfiguredMock.mockReturnValue(false); + shouldUseEnvHttpProxyForUrlMock.mockReturnValue(false); fetchWithSsrFGuardMock.mockResolvedValue({ response: new Response(null, { status: 200 }), finalUrl: "https://api.example.com", @@ -568,7 +563,7 @@ describe("fetchWithTimeoutGuarded", () => { // Callers with custom proxy URL / proxyTls / connect options must keep // control over the dispatcher. Auto-upgrade would build an // EnvHttpProxyAgent that silently drops those overrides. - hasEnvHttpProxyConfiguredMock.mockReturnValue(true); + shouldUseEnvHttpProxyForUrlMock.mockReturnValue(true); fetchWithSsrFGuardMock.mockResolvedValue({ response: new Response(null, { status: 200 }), finalUrl: "https://api.example.com", @@ -595,8 +590,7 @@ describe("fetchWithTimeoutGuarded", () => { // for NO_PROXY matches, but in TRUSTED_ENV_PROXY mode fetchWithSsrFGuard // skips pinned-DNS checks — so auto-upgrading those targets would bypass // SSRF protection. Keep strict mode for NO_PROXY matches. - hasEnvHttpProxyConfiguredMock.mockReturnValue(true); - matchesNoProxyMock.mockReturnValue(true); + shouldUseEnvHttpProxyForUrlMock.mockReturnValue(false); fetchWithSsrFGuardMock.mockResolvedValue({ response: new Response(null, { status: 200 }), finalUrl: "https://internal.corp.example", diff --git a/src/media-understanding/shared.ts b/src/media-understanding/shared.ts index 409c70d1113..a57601da8f0 100644 --- a/src/media-understanding/shared.ts +++ b/src/media-understanding/shared.ts @@ -15,7 +15,7 @@ import { } from "../agents/provider-request-config.js"; import type { GuardedFetchMode, GuardedFetchResult } from "../infra/net/fetch-guard.js"; import { fetchWithSsrFGuard, GUARDED_FETCH_MODE } from "../infra/net/fetch-guard.js"; -import { hasEnvHttpProxyConfigured, matchesNoProxy } from "../infra/net/proxy-env.js"; +import { shouldUseEnvHttpProxyForUrl } from "../infra/net/proxy-env.js"; import type { LookupFn, PinnedDispatcherPolicy, SsrFPolicy } from "../infra/net/ssrf.js"; import { fetchWithTimeout } from "../utils/fetch-timeout.js"; export { fetchWithTimeout }; @@ -258,29 +258,7 @@ function shouldAutoUpgradeToTrustedEnvProxy(params: { return false; } - let protocol: "http" | "https"; - try { - const parsed = new URL(params.url); - if (parsed.protocol === "http:") { - protocol = "http"; - } else if (parsed.protocol === "https:") { - protocol = "https"; - } else { - return false; - } - } catch { - return false; - } - - if (!hasEnvHttpProxyConfigured(protocol)) { - return false; - } - - if (matchesNoProxy(params.url)) { - return false; - } - - return true; + return shouldUseEnvHttpProxyForUrl(params.url); } export async function fetchWithTimeoutGuarded( diff --git a/src/memory-host-sdk/host/remote-http.test.ts b/src/memory-host-sdk/host/remote-http.test.ts new file mode 100644 index 00000000000..4f7ae2febfb --- /dev/null +++ b/src/memory-host-sdk/host/remote-http.test.ts @@ -0,0 +1,68 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const { fetchWithSsrFGuardMock, shouldUseEnvHttpProxyForUrlMock } = vi.hoisted(() => ({ + fetchWithSsrFGuardMock: vi.fn(), + shouldUseEnvHttpProxyForUrlMock: vi.fn(() => false), +})); + +vi.mock("../../infra/net/fetch-guard.js", async () => { + const actual = await vi.importActual( + "../../infra/net/fetch-guard.js", + ); + return { + ...actual, + fetchWithSsrFGuard: fetchWithSsrFGuardMock, + }; +}); + +vi.mock("../../infra/net/proxy-env.js", async () => { + const actual = await vi.importActual( + "../../infra/net/proxy-env.js", + ); + return { + ...actual, + shouldUseEnvHttpProxyForUrl: shouldUseEnvHttpProxyForUrlMock, + }; +}); + +import { GUARDED_FETCH_MODE } from "../../infra/net/fetch-guard.js"; +import { withRemoteHttpResponse } from "./remote-http.js"; + +describe("withRemoteHttpResponse", () => { + beforeEach(() => { + vi.clearAllMocks(); + shouldUseEnvHttpProxyForUrlMock.mockReturnValue(false); + fetchWithSsrFGuardMock.mockResolvedValue({ + response: new Response("ok", { status: 200 }), + finalUrl: "https://memory.example/v1", + release: vi.fn(async () => {}), + }); + }); + + it("uses trusted env proxy mode when the target will use EnvHttpProxyAgent", async () => { + shouldUseEnvHttpProxyForUrlMock.mockReturnValue(true); + + await withRemoteHttpResponse({ + url: "https://memory.example/v1/embeddings", + onResponse: async () => undefined, + }); + + expect(fetchWithSsrFGuardMock).toHaveBeenCalledWith( + expect.objectContaining({ + url: "https://memory.example/v1/embeddings", + mode: GUARDED_FETCH_MODE.TRUSTED_ENV_PROXY, + }), + ); + }); + + it("keeps strict guarded fetch mode when proxy env would not proxy the target", async () => { + await withRemoteHttpResponse({ + url: "https://internal.corp.example/v1/embeddings", + onResponse: async () => undefined, + }); + + const call = fetchWithSsrFGuardMock.mock.calls[0]?.[0]; + expect(call).toBeDefined(); + expect(call).not.toHaveProperty("mode"); + }); +}); diff --git a/src/memory-host-sdk/host/remote-http.ts b/src/memory-host-sdk/host/remote-http.ts index 7385125beda..cf5f91643ed 100644 --- a/src/memory-host-sdk/host/remote-http.ts +++ b/src/memory-host-sdk/host/remote-http.ts @@ -1,4 +1,5 @@ -import { fetchWithSsrFGuard } from "../../infra/net/fetch-guard.js"; +import { fetchWithSsrFGuard, GUARDED_FETCH_MODE } from "../../infra/net/fetch-guard.js"; +import { shouldUseEnvHttpProxyForUrl } from "../../infra/net/proxy-env.js"; import { ssrfPolicyFromHttpBaseUrlAllowedHostname, type SsrFPolicy } from "../../infra/net/ssrf.js"; export const buildRemoteBaseUrlPolicy = ssrfPolicyFromHttpBaseUrlAllowedHostname; @@ -17,6 +18,9 @@ export async function withRemoteHttpResponse(params: { init: params.init, policy: params.ssrfPolicy, auditContext: params.auditContext ?? "memory-remote", + ...(shouldUseEnvHttpProxyForUrl(params.url) + ? { mode: GUARDED_FETCH_MODE.TRUSTED_ENV_PROXY } + : {}), }); try { return await params.onResponse(response);