fix(media-understanding): auto-upgrade provider HTTP helper to trusted env proxy mode (#66458)

* fix(media-understanding): auto-upgrade provider HTTP helper to trusted env proxy mode

* Update CHANGELOG.md
This commit is contained in:
Vincent Koc
2026-04-14 10:29:09 +01:00
committed by GitHub
parent dfed74b254
commit 6ee8e194c0
5 changed files with 576 additions and 7 deletions

View File

@@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest";
import {
hasEnvHttpProxyConfigured,
hasProxyEnvConfigured,
matchesNoProxy,
resolveEnvHttpProxyUrl,
} from "./proxy-env.js";
@@ -93,3 +94,142 @@ describe("resolveEnvHttpProxyUrl", () => {
expect(hasEnvHttpProxyConfigured(protocol, env)).toBe(expectedConfigured);
});
});
describe("matchesNoProxy", () => {
it.each([
{
name: "returns false when no NO_PROXY is set",
url: "https://api.openai.com/v1/chat",
env: {} as NodeJS.ProcessEnv,
expected: false,
},
{
name: "returns false for blank NO_PROXY",
url: "https://api.openai.com",
env: { NO_PROXY: " " } as NodeJS.ProcessEnv,
expected: false,
},
{
name: "matches wildcard",
url: "https://api.openai.com/v1/chat",
env: { NO_PROXY: "*" } as NodeJS.ProcessEnv,
expected: true,
},
{
name: "matches exact hostname",
url: "https://api.openai.com/v1/chat",
env: { NO_PROXY: "api.openai.com" } as NodeJS.ProcessEnv,
expected: true,
},
{
name: "matches subdomain via leading-dot normalization",
url: "https://api.openai.com/v1/chat",
env: { NO_PROXY: ".openai.com" } as NodeJS.ProcessEnv,
expected: true,
},
{
name: "matches subdomain suffix without leading dot",
url: "https://api.openai.com/v1/chat",
env: { NO_PROXY: "openai.com" } as NodeJS.ProcessEnv,
expected: true,
},
{
name: "does not match unrelated hostname",
url: "https://api.example.org/v1/chat",
env: { NO_PROXY: "openai.com" } as NodeJS.ProcessEnv,
expected: false,
},
{
name: "does not match when suffix is not a domain boundary",
url: "https://notopenai.com/v1",
env: { NO_PROXY: "openai.com" } as NodeJS.ProcessEnv,
expected: false,
},
{
name: "respects port in NO_PROXY entry",
url: "https://api.internal:8443/v1",
env: { NO_PROXY: "api.internal:8443" } as NodeJS.ProcessEnv,
expected: true,
},
{
name: "does not match when port differs",
url: "https://api.internal:9000/v1",
env: { NO_PROXY: "api.internal:8443" } as NodeJS.ProcessEnv,
expected: false,
},
{
name: "is case-insensitive",
url: "https://API.OpenAI.COM/v1",
env: { no_proxy: "api.openai.com" } as NodeJS.ProcessEnv,
expected: true,
},
{
name: "parses comma-separated list",
url: "https://internal.corp.example",
env: { NO_PROXY: "localhost,127.0.0.1,internal.corp.example" } as NodeJS.ProcessEnv,
expected: true,
},
{
name: "parses whitespace-separated list (undici tokenizes on [,\\s])",
url: "https://foo.corp.internal",
env: { NO_PROXY: "localhost *.corp.internal" } as NodeJS.ProcessEnv,
expected: true,
},
{
name: "parses mixed comma-and-whitespace list",
url: "https://api.openai.com",
env: { NO_PROXY: "localhost, 127.0.0.1\tapi.openai.com" } as NodeJS.ProcessEnv,
expected: true,
},
{
name: "tab and newline act as delimiters",
url: "https://internal.example",
env: { NO_PROXY: "localhost\n127.0.0.1\tinternal.example" } as NodeJS.ProcessEnv,
expected: true,
},
{
name: "matches subdomain via *. wildcard normalization",
url: "https://foo.example.com/v1",
env: { NO_PROXY: "*.example.com" } as NodeJS.ProcessEnv,
expected: true,
},
{
name: "wildcard *.example.com matches bare example.com (undici normalizes to base domain)",
url: "https://example.com/v1",
env: { NO_PROXY: "*.example.com" } as NodeJS.ProcessEnv,
expected: true,
},
{
name: "*. wildcard respects port",
url: "https://api.corp.internal:8443",
env: { NO_PROXY: "*.corp.internal:8443" } as NodeJS.ProcessEnv,
expected: true,
},
{
name: "*. wildcard does not match unrelated suffix",
url: "https://api.example.org",
env: { NO_PROXY: "*.example.com" } as NodeJS.ProcessEnv,
expected: false,
},
{
name: "lower-case no_proxy is honored",
url: "https://corp.local",
env: { no_proxy: "corp.local" } as NodeJS.ProcessEnv,
expected: true,
},
{
name: "matches bracketed IPv6 literal",
url: "http://[::1]:8080/health",
env: { NO_PROXY: "[::1]:8080" } as NodeJS.ProcessEnv,
expected: true,
},
{
name: "returns false for malformed target URL",
url: "not-a-url",
env: { NO_PROXY: "*" } as NodeJS.ProcessEnv,
expected: false,
},
])("$name", ({ url, env, expected }) => {
expect(matchesNoProxy(url, env)).toBe(expected);
});
});

View File

@@ -53,3 +53,107 @@ export function hasEnvHttpProxyConfigured(
): boolean {
return resolveEnvHttpProxyUrl(protocol, env) !== undefined;
}
/**
* Check whether a target URL should bypass the HTTP proxy per NO_PROXY env var.
*
* Mirrors undici EnvHttpProxyAgent semantics
* (`undici/lib/dispatcher/env-http-proxy-agent.js`):
* - Entries separated by commas OR whitespace (undici splits on `/[,\s]/`)
* - Case-insensitive
* - Empty or missing → no bypass
* - `*` → bypass everything
* - Exact hostname match
* - Leading-dot match (`.example.com` matches `foo.example.com`)
* - Leading `*.` wildcard match (`*.example.com` matches `foo.example.com`);
* undici normalizes via `.replace(/^\*?\./, '')`, so the bare domain also
* matches (kept in sync with that behavior)
* - Subdomain suffix match (`openai.com` matches `api.openai.com`)
* - Optional `:port` suffix; when present, must match target port
* - IPv6 literals in bracketed form (`[::1]`)
*
* Undici does not export its matcher, so this is a targeted reimplementation
* kept in sync with the upstream file above. Paired with
* `hasEnvHttpProxyConfigured` this gates the trusted-env-proxy auto-upgrade
* in provider HTTP helpers; see openclaw#64974 review thread on NO_PROXY
* SSRF bypass.
*/
export function matchesNoProxy(targetUrl: string, env: NodeJS.ProcessEnv = process.env): boolean {
const raw = normalizeProxyEnvValue(env.no_proxy) ?? normalizeProxyEnvValue(env.NO_PROXY);
if (!raw) {
return false;
}
let parsed: URL;
try {
parsed = new URL(targetUrl);
} catch {
return false;
}
const targetHost = parsed.hostname.toLowerCase().replace(/^\[|\]$/g, "");
if (!targetHost) {
return false;
}
const targetPort =
parsed.port !== ""
? parsed.port
: parsed.protocol === "https:"
? "443"
: parsed.protocol === "http:"
? "80"
: "";
// Undici tokenizes NO_PROXY on BOTH commas and whitespace (single-char
// class, empty entries filtered below). Values like `"localhost *.corp"`
// or `"a, b\tc"` must all parse correctly.
for (const rawEntry of raw.split(/[,\s]/)) {
const entry = rawEntry.trim().toLowerCase();
if (!entry) {
continue;
}
if (entry === "*") {
return true;
}
let entryHost: string;
let entryPort: string | undefined;
if (entry.startsWith("[")) {
const m = entry.match(/^\[([^\]]+)\](?::(\d+))?$/);
if (!m) {
continue;
}
entryHost = m[1];
entryPort = m[2];
} else {
const colonIdx = entry.lastIndexOf(":");
if (colonIdx > 0 && /^\d+$/.test(entry.slice(colonIdx + 1))) {
entryHost = entry.slice(0, colonIdx);
entryPort = entry.slice(colonIdx + 1);
} else {
entryHost = entry;
}
}
if (entryPort && entryPort !== targetPort) {
continue;
}
// Mirror undici: strip optional leading `*` followed by `.` so both
// `.example.com` and `*.example.com` normalize to `example.com`.
const normalizedEntry = entryHost.replace(/^\*?\./, "");
if (!normalizedEntry) {
continue;
}
if (targetHost === normalizedEntry) {
return true;
}
if (targetHost.endsWith("." + normalizedEntry)) {
return true;
}
}
return false;
}

View File

@@ -1,8 +1,12 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
const { fetchWithSsrFGuardMock } = vi.hoisted(() => ({
fetchWithSsrFGuardMock: vi.fn(),
}));
const { fetchWithSsrFGuardMock, hasEnvHttpProxyConfiguredMock, matchesNoProxyMock } = vi.hoisted(
() => ({
fetchWithSsrFGuardMock: vi.fn(),
hasEnvHttpProxyConfiguredMock: vi.fn(() => false),
matchesNoProxyMock: vi.fn(() => false),
}),
);
vi.mock("../infra/net/fetch-guard.js", async () => {
const actual = await vi.importActual<typeof import("../infra/net/fetch-guard.js")>(
@@ -14,6 +18,17 @@ vi.mock("../infra/net/fetch-guard.js", async () => {
};
});
vi.mock("../infra/net/proxy-env.js", async () => {
const actual = await vi.importActual<typeof import("../infra/net/proxy-env.js")>(
"../infra/net/proxy-env.js",
);
return {
...actual,
hasEnvHttpProxyConfigured: hasEnvHttpProxyConfiguredMock,
matchesNoProxy: matchesNoProxyMock,
};
});
import {
fetchWithTimeoutGuarded,
postJsonRequest,
@@ -22,6 +37,11 @@ import {
resolveProviderHttpRequestConfig,
} from "./shared.js";
beforeEach(() => {
hasEnvHttpProxyConfiguredMock.mockReturnValue(false);
matchesNoProxyMock.mockReturnValue(false);
});
afterEach(() => {
vi.clearAllMocks();
});
@@ -268,4 +288,203 @@ describe("fetchWithTimeoutGuarded", () => {
}),
);
});
it("does not set a guarded fetch mode when no HTTP proxy env is configured", async () => {
hasEnvHttpProxyConfiguredMock.mockReturnValue(false);
fetchWithSsrFGuardMock.mockResolvedValue({
response: new Response(null, { status: 200 }),
finalUrl: "https://example.com",
release: async () => {},
});
await fetchWithTimeoutGuarded("https://example.com", {}, undefined, fetch);
const call = fetchWithSsrFGuardMock.mock.calls[0]?.[0];
expect(call).toBeDefined();
expect(call).not.toHaveProperty("mode");
});
it("auto-selects trusted env proxy mode when HTTP proxy env is configured", async () => {
hasEnvHttpProxyConfiguredMock.mockReturnValue(true);
fetchWithSsrFGuardMock.mockResolvedValue({
response: new Response(null, { status: 200 }),
finalUrl: "https://api.minimax.io",
release: async () => {},
});
await postJsonRequest({
url: "https://api.minimax.io/v1/image_generation",
headers: new Headers({ authorization: "Bearer test" }),
body: { model: "image-01", prompt: "a red cube" },
fetchFn: fetch,
});
expect(fetchWithSsrFGuardMock).toHaveBeenCalledWith(
expect.objectContaining({
mode: "trusted_env_proxy",
}),
);
});
it("respects an explicit mode from the caller when HTTP proxy env is configured", async () => {
hasEnvHttpProxyConfiguredMock.mockReturnValue(true);
fetchWithSsrFGuardMock.mockResolvedValue({
response: new Response(null, { status: 200 }),
finalUrl: "https://api.example.com",
release: async () => {},
});
await fetchWithTimeoutGuarded("https://api.example.com", {}, undefined, fetch, {
mode: "strict",
});
expect(fetchWithSsrFGuardMock).toHaveBeenCalledWith(
expect.objectContaining({
mode: "strict",
}),
);
});
it("auto-upgrades transcription requests to trusted env proxy when proxy env is configured", async () => {
hasEnvHttpProxyConfiguredMock.mockReturnValue(true);
fetchWithSsrFGuardMock.mockResolvedValue({
response: new Response(null, { status: 200 }),
finalUrl: "https://api.openai.com",
release: async () => {},
});
await postTranscriptionRequest({
url: "https://api.openai.com/v1/audio/transcriptions",
headers: new Headers({ authorization: "Bearer test" }),
body: "audio-bytes",
fetchFn: fetch,
});
expect(fetchWithSsrFGuardMock).toHaveBeenCalledWith(
expect.objectContaining({
mode: "trusted_env_proxy",
}),
);
});
it("forwards an explicit mode override through postJsonRequest even when proxy env is configured", async () => {
hasEnvHttpProxyConfiguredMock.mockReturnValue(true);
fetchWithSsrFGuardMock.mockResolvedValue({
response: new Response(null, { status: 200 }),
finalUrl: "https://api.example.com",
release: async () => {},
});
await postJsonRequest({
url: "https://api.example.com/v1/strict",
headers: new Headers(),
body: { ok: true },
fetchFn: fetch,
mode: "strict",
});
expect(fetchWithSsrFGuardMock).toHaveBeenCalledWith(
expect.objectContaining({
mode: "strict",
}),
);
});
it("forwards an explicit mode override through postTranscriptionRequest even when proxy env is configured", async () => {
hasEnvHttpProxyConfiguredMock.mockReturnValue(true);
fetchWithSsrFGuardMock.mockResolvedValue({
response: new Response(null, { status: 200 }),
finalUrl: "https://api.example.com",
release: async () => {},
});
await postTranscriptionRequest({
url: "https://api.example.com/v1/transcriptions",
headers: new Headers(),
body: "audio-bytes",
fetchFn: fetch,
mode: "strict",
});
expect(fetchWithSsrFGuardMock).toHaveBeenCalledWith(
expect.objectContaining({
mode: "strict",
}),
);
});
it("does not auto-upgrade when only ALL_PROXY is configured (HTTP(S) proxy gate)", async () => {
// ALL_PROXY is ignored by EnvHttpProxyAgent; `hasEnvHttpProxyConfigured`
// reflects that by returning false when only ALL_PROXY is set. Auto-upgrade
// must NOT fire, otherwise the request would skip pinned-DNS/SSRF checks
// and then be dispatched directly.
hasEnvHttpProxyConfiguredMock.mockReturnValue(false);
fetchWithSsrFGuardMock.mockResolvedValue({
response: new Response(null, { status: 200 }),
finalUrl: "https://api.example.com",
release: async () => {},
});
await postJsonRequest({
url: "https://api.example.com/v1/image",
headers: new Headers(),
body: { ok: true },
fetchFn: fetch,
});
const call = fetchWithSsrFGuardMock.mock.calls[0]?.[0];
expect(call).toBeDefined();
expect(call).not.toHaveProperty("mode");
});
it("does not auto-upgrade when caller passes explicit dispatcherPolicy", async () => {
// Callers with custom proxy URL / proxyTls / connect options must keep
// control over the dispatcher. Auto-upgrade would build an
// EnvHttpProxyAgent that silently drops those overrides.
hasEnvHttpProxyConfiguredMock.mockReturnValue(true);
fetchWithSsrFGuardMock.mockResolvedValue({
response: new Response(null, { status: 200 }),
finalUrl: "https://api.example.com",
release: async () => {},
});
const explicitPolicy = {
mode: "explicit-proxy" as const,
proxyUrl: "http://corp-proxy.internal:3128",
};
await fetchWithTimeoutGuarded("https://api.example.com/v1/image", {}, undefined, fetch, {
dispatcherPolicy: explicitPolicy,
});
const call = fetchWithSsrFGuardMock.mock.calls[0]?.[0];
expect(call).toBeDefined();
expect(call).not.toHaveProperty("mode");
expect(call).toHaveProperty("dispatcherPolicy", explicitPolicy);
});
it("does not auto-upgrade when target URL matches NO_PROXY", async () => {
// With HTTP_PROXY + NO_PROXY, EnvHttpProxyAgent makes direct connections
// for NO_PROXY matches, but in TRUSTED_ENV_PROXY mode fetchWithSsrFGuard
// skips pinned-DNS checks — so auto-upgrading those targets would bypass
// SSRF protection. Keep strict mode for NO_PROXY matches.
hasEnvHttpProxyConfiguredMock.mockReturnValue(true);
matchesNoProxyMock.mockReturnValue(true);
fetchWithSsrFGuardMock.mockResolvedValue({
response: new Response(null, { status: 200 }),
finalUrl: "https://internal.corp.example",
release: async () => {},
});
await postJsonRequest({
url: "https://internal.corp.example/v1/image",
headers: new Headers(),
body: { ok: true },
fetchFn: fetch,
});
const call = fetchWithSsrFGuardMock.mock.calls[0]?.[0];
expect(call).toBeDefined();
expect(call).not.toHaveProperty("mode");
});
});

View File

@@ -9,8 +9,9 @@ import {
type ProviderRequestTransportOverrides,
type ResolvedProviderRequestConfig,
} from "../agents/provider-request-config.js";
import type { GuardedFetchResult } from "../infra/net/fetch-guard.js";
import { fetchWithSsrFGuard } from "../infra/net/fetch-guard.js";
import type { GuardedFetchMode, GuardedFetchResult } from "../infra/net/fetch-guard.js";
import { fetchWithSsrFGuard, GUARDED_FETCH_MODE } from "../infra/net/fetch-guard.js";
import { hasEnvHttpProxyConfigured, matchesNoProxy } from "../infra/net/proxy-env.js";
import type { LookupFn, PinnedDispatcherPolicy, SsrFPolicy } from "../infra/net/ssrf.js";
export { fetchWithTimeout } from "../utils/fetch-timeout.js";
export { normalizeBaseUrl } from "../agents/provider-request-config.js";
@@ -85,6 +86,63 @@ export function resolveProviderHttpRequestConfig(params: {
};
}
/**
* Decide whether to auto-upgrade a provider HTTP request into
* `TRUSTED_ENV_PROXY` mode based on the runtime environment.
*
* This is gated conservatively to avoid the SSRF bypasses the initial
* auto-upgrade path exposed (see openclaw#64974 review threads):
*
* 1. If the caller supplied an explicit `dispatcherPolicy` — custom proxy URL,
* `proxyTls`, or `connect` options — do NOT override it. Trusted-env mode
* builds an `EnvHttpProxyAgent` that would silently drop those overrides,
* breaking enterprise proxy/mTLS configs.
*
* 2. Only auto-upgrade when `HTTP_PROXY` or `HTTPS_PROXY` (lower- or
* upper-case) is configured for the target protocol. `ALL_PROXY` is
* explicitly ignored by `EnvHttpProxyAgent`, so counting it would
* auto-upgrade requests that then make direct connections while skipping
* pinned-DNS/SSRF hostname checks.
*
* 3. If `NO_PROXY` would bypass the proxy for this target, do NOT auto-upgrade.
* `EnvHttpProxyAgent` makes direct connections for `NO_PROXY` matches, but
* in `TRUSTED_ENV_PROXY` mode `fetchWithSsrFGuard` skips
* `resolvePinnedHostnameWithPolicy` — so those direct connections would
* bypass SSRF protection. Keep strict mode for `NO_PROXY` matches.
*/
function shouldAutoUpgradeToTrustedEnvProxy(params: {
url: string;
dispatcherPolicy: PinnedDispatcherPolicy | undefined;
}): boolean {
if (params.dispatcherPolicy) {
return false;
}
let protocol: "http" | "https";
try {
const parsed = new URL(params.url);
if (parsed.protocol === "http:") {
protocol = "http";
} else if (parsed.protocol === "https:") {
protocol = "https";
} else {
return false;
}
} catch {
return false;
}
if (!hasEnvHttpProxyConfigured(protocol)) {
return false;
}
if (matchesNoProxy(params.url)) {
return false;
}
return true;
}
export async function fetchWithTimeoutGuarded(
url: string,
init: RequestInit,
@@ -96,8 +154,39 @@ export async function fetchWithTimeoutGuarded(
pinDns?: boolean;
dispatcherPolicy?: PinnedDispatcherPolicy;
auditContext?: string;
mode?: GuardedFetchMode;
},
): Promise<GuardedFetchResult> {
// Provider HTTP helpers (image/music/video generation, transcription, etc.)
// call this function from every provider that talks to a remote API. When
// the host has HTTP_PROXY/HTTPS_PROXY configured, the lower-level strict
// mode would force Node-level `dns.lookup()` on the target hostname before
// dialing the proxy — which fails with EAI_AGAIN in proxy-only environments
// (containers, restricted sandboxes, corporate networks with DNS-over-proxy,
// Clash TUN fake-IP, etc.). Auto-upgrade to trusted env proxy mode in that
// case so the request goes through the configured proxy agent instead of
// doing a local DNS pre-resolution.
//
// This does not weaken SSRF protection when the auto-upgrade fires: an HTTP
// CONNECT proxy on the egress path performs hostname resolution itself and
// client-side DNS pinning cannot meaningfully constrain the target IP. But
// the auto-upgrade is gated (see `shouldAutoUpgradeToTrustedEnvProxy`) to
// avoid three SSRF-bypass edge cases: caller-provided `dispatcherPolicy`,
// `ALL_PROXY`-only envs, and `NO_PROXY` target matches. Callers that
// explicitly need strict pinned-DNS can still opt in by passing
// `mode: GUARDED_FETCH_MODE.STRICT` here or by using `fetchWithSsrFGuard`
// directly.
//
// See openclaw#52162 for the reported failure mode on memory embeddings,
// which shares this code path with image/music/video/audio generation.
const resolvedMode =
options?.mode ??
(shouldAutoUpgradeToTrustedEnvProxy({
url,
dispatcherPolicy: options?.dispatcherPolicy,
})
? GUARDED_FETCH_MODE.TRUSTED_ENV_PROXY
: undefined);
return await fetchWithSsrFGuard({
url,
fetchImpl: fetchFn,
@@ -108,6 +197,7 @@ export async function fetchWithTimeoutGuarded(
pinDns: options?.pinDns,
dispatcherPolicy: options?.dispatcherPolicy,
auditContext: sanitizeAuditContext(options?.auditContext),
...(resolvedMode ? { mode: resolvedMode } : {}),
});
}
@@ -118,12 +208,14 @@ function resolveGuardedPostRequestOptions(params: {
allowPrivateNetwork?: boolean;
dispatcherPolicy?: PinnedDispatcherPolicy;
auditContext?: string;
mode?: GuardedFetchMode;
}): GuardedPostRequestOptions | undefined {
if (
!params.allowPrivateNetwork &&
!params.dispatcherPolicy &&
params.pinDns === undefined &&
!params.auditContext
!params.auditContext &&
params.mode === undefined
) {
return undefined;
}
@@ -132,6 +224,7 @@ function resolveGuardedPostRequestOptions(params: {
...(params.pinDns !== undefined ? { pinDns: params.pinDns } : {}),
...(params.dispatcherPolicy ? { dispatcherPolicy: params.dispatcherPolicy } : {}),
...(params.auditContext ? { auditContext: params.auditContext } : {}),
...(params.mode !== undefined ? { mode: params.mode } : {}),
};
}
@@ -145,6 +238,12 @@ export async function postTranscriptionRequest(params: {
allowPrivateNetwork?: boolean;
dispatcherPolicy?: PinnedDispatcherPolicy;
auditContext?: string;
/**
* Override the guarded-fetch mode. Defaults to an auto-upgrade to
* `TRUSTED_ENV_PROXY` when `HTTP_PROXY`/`HTTPS_PROXY` is configured in the
* environment; pass `"strict"` to force pinned-DNS even inside a proxy.
*/
mode?: GuardedFetchMode;
}) {
return fetchWithTimeoutGuarded(
params.url,
@@ -169,6 +268,12 @@ export async function postJsonRequest(params: {
allowPrivateNetwork?: boolean;
dispatcherPolicy?: PinnedDispatcherPolicy;
auditContext?: string;
/**
* Override the guarded-fetch mode. Defaults to an auto-upgrade to
* `TRUSTED_ENV_PROXY` when `HTTP_PROXY`/`HTTPS_PROXY` is configured in the
* environment; pass `"strict"` to force pinned-DNS even inside a proxy.
*/
mode?: GuardedFetchMode;
}) {
return fetchWithTimeoutGuarded(
params.url,