From e4825a0f93856f6417cabe77bb9aed16fe027dc2 Mon Sep 17 00:00:00 2001 From: Ayaan Zaidi Date: Tue, 17 Mar 2026 22:44:15 +0530 Subject: [PATCH] fix(telegram): unify transport fallback chain (#49148) * fix(telegram): unify transport fallback chain * fix: address telegram fallback review comments * fix: validate pinned SSRF overrides * fix: unify telegram fallback retries (#49148) --- CHANGELOG.md | 1 + .../src/bot/delivery.resolve-media.ts | 7 +- extensions/telegram/src/fetch.test.ts | 62 ++++- extensions/telegram/src/fetch.ts | 237 ++++++++++++------ src/infra/net/fetch-guard.ts | 2 +- src/infra/net/ssrf.dispatcher.test.ts | 52 ++++ src/infra/net/ssrf.pinning.test.ts | 9 + src/infra/net/ssrf.ts | 58 ++++- src/media/fetch.telegram-network.test.ts | 94 ++++++- src/media/fetch.ts | 72 ++++-- 10 files changed, 459 insertions(+), 135 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 53114cb9d75..8930840332c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -136,6 +136,7 @@ Docs: https://docs.openclaw.ai - ACP/gateway startup: use direct Telegram and Discord startup/status helpers instead of routing probes through the plugin runtime, and prepend the selected daemon Node bin dir to service PATH so plugin-local installs can still find `npm` and `pnpm`. - ACP/configured bindings: reinitialize configured ACP sessions that are stuck in `error` state instead of reusing the failed runtime. - Mattermost/DM send: retry transient direct-channel creation failures for DM deliveries, with configurable backoff and per-request timeout. (#42398) Thanks @JonathanJing. +- Telegram/network: unify API and media fetches under the same sticky IPv4 and pinned-IP fallback chain, and re-validate pinned override addresses against SSRF policy. (#49148) Thanks @obviyus. ## 2026.3.13 diff --git a/extensions/telegram/src/bot/delivery.resolve-media.ts b/extensions/telegram/src/bot/delivery.resolve-media.ts index 36b3bb50be9..52f6eef966c 100644 --- a/extensions/telegram/src/bot/delivery.resolve-media.ts +++ b/extensions/telegram/src/bot/delivery.resolve-media.ts @@ -4,7 +4,7 @@ import { retryAsync } from "openclaw/plugin-sdk/infra-runtime"; import { fetchRemoteMedia } from "openclaw/plugin-sdk/media-runtime"; import { saveMediaBuffer } from "openclaw/plugin-sdk/media-runtime"; import { logVerbose, warn } from "openclaw/plugin-sdk/runtime-env"; -import { shouldRetryTelegramIpv4Fallback, type TelegramTransport } from "../fetch.js"; +import { shouldRetryTelegramTransportFallback, type TelegramTransport } from "../fetch.js"; import { cacheSticker, getCachedSticker } from "../sticker-cache.js"; import { resolveTelegramMediaPlaceholder } from "./helpers.js"; import type { StickerMetadata, TelegramContext } from "./types.js"; @@ -129,9 +129,8 @@ async function downloadAndSaveTelegramFile(params: { const fetched = await fetchRemoteMedia({ url, fetchImpl: params.transport.sourceFetch, - dispatcherPolicy: params.transport.pinnedDispatcherPolicy, - fallbackDispatcherPolicy: params.transport.fallbackPinnedDispatcherPolicy, - shouldRetryFetchError: shouldRetryTelegramIpv4Fallback, + dispatcherAttempts: params.transport.dispatcherAttempts, + shouldRetryFetchError: shouldRetryTelegramTransportFallback, filePathHint: params.filePath, maxBytes: params.maxBytes, readIdleTimeoutMs: TELEGRAM_DOWNLOAD_IDLE_TIMEOUT_MS, diff --git a/extensions/telegram/src/fetch.test.ts b/extensions/telegram/src/fetch.test.ts index 7681d0c8701..4afdacf0568 100644 --- a/extensions/telegram/src/fetch.test.ts +++ b/extensions/telegram/src/fetch.test.ts @@ -1,6 +1,4 @@ -import { afterEach, describe, expect, it, vi } from "vitest"; -import { resolveFetch } from "../../../src/infra/fetch.js"; -import { resolveTelegramFetch, resolveTelegramTransport } from "./fetch.js"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; const setDefaultResultOrder = vi.hoisted(() => vi.fn()); const setDefaultAutoSelectFamily = vi.hoisted(() => vi.fn()); @@ -56,6 +54,16 @@ vi.mock("undici", () => ({ setGlobalDispatcher, })); +let resolveFetch: typeof import("../../../src/infra/fetch.js").resolveFetch; +let resolveTelegramFetch: typeof import("./fetch.js").resolveTelegramFetch; +let resolveTelegramTransport: typeof import("./fetch.js").resolveTelegramTransport; + +beforeEach(async () => { + vi.resetModules(); + ({ resolveFetch } = await import("../../../src/infra/fetch.js")); + ({ resolveTelegramFetch, resolveTelegramTransport } = await import("./fetch.js")); +}); + function resolveTelegramFetchOrThrow( proxyFetch?: typeof fetch, options?: { network?: { autoSelectFamily?: boolean; dnsResultOrder?: "ipv4first" | "verbatim" } }, @@ -152,6 +160,24 @@ function expectPinnedIpv4ConnectDispatcher(args: { } } +function expectPinnedFallbackIpDispatcher(callIndex: number) { + const dispatcher = getDispatcherFromUndiciCall(callIndex); + expect(dispatcher?.options?.connect).toEqual( + expect.objectContaining({ + family: 4, + autoSelectFamily: false, + lookup: expect.any(Function), + }), + ); + const callback = vi.fn(); + ( + dispatcher?.options?.connect?.lookup as + | ((hostname: string, callback: (err: null, address: string, family: number) => void) => void) + | undefined + )?.("api.telegram.org", callback); + expect(callback).toHaveBeenCalledWith(null, "149.154.167.220", 4); +} + function expectCallerDispatcherPreserved(callIndexes: number[], dispatcher: unknown) { for (const callIndex of callIndexes) { const callInit = undiciFetch.mock.calls[callIndex - 1]?.[1] as @@ -395,7 +421,7 @@ describe("resolveTelegramFetch", () => { pinnedCall: 2, followupCall: 3, }); - expect(transport.pinnedDispatcherPolicy).toEqual( + expect(transport.dispatcherAttempts?.[0]?.dispatcherPolicy).toEqual( expect.objectContaining({ mode: "direct", }), @@ -533,6 +559,34 @@ describe("resolveTelegramFetch", () => { ); }); + it("escalates from IPv4 fallback to pinned Telegram IP and keeps it sticky", async () => { + undiciFetch + .mockRejectedValueOnce(buildFetchFallbackError("ETIMEDOUT")) + .mockRejectedValueOnce(buildFetchFallbackError("EHOSTUNREACH")) + .mockResolvedValueOnce({ ok: true } as Response) + .mockResolvedValueOnce({ ok: true } as Response); + + const resolved = resolveTelegramFetchOrThrow(undefined, { + network: { + autoSelectFamily: true, + dnsResultOrder: "ipv4first", + }, + }); + + await resolved("https://api.telegram.org/botx/sendMessage"); + await resolved("https://api.telegram.org/botx/sendChatAction"); + + expect(undiciFetch).toHaveBeenCalledTimes(4); + + const secondDispatcher = getDispatcherFromUndiciCall(2); + const thirdDispatcher = getDispatcherFromUndiciCall(3); + const fourthDispatcher = getDispatcherFromUndiciCall(4); + + expect(secondDispatcher).not.toBe(thirdDispatcher); + expect(thirdDispatcher).toBe(fourthDispatcher); + expectPinnedFallbackIpDispatcher(3); + }); + it("preserves caller-provided dispatcher across fallback retry", async () => { const fetchError = buildFetchFallbackError("EHOSTUNREACH"); undiciFetch.mockRejectedValueOnce(fetchError).mockResolvedValueOnce({ ok: true } as Response); diff --git a/extensions/telegram/src/fetch.ts b/extensions/telegram/src/fetch.ts index 962d0256af1..ad60faab13b 100644 --- a/extensions/telegram/src/fetch.ts +++ b/extensions/telegram/src/fetch.ts @@ -1,8 +1,11 @@ import * as dns from "node:dns"; import type { TelegramNetworkConfig } from "openclaw/plugin-sdk/config-runtime"; -import { resolveFetch } from "openclaw/plugin-sdk/infra-runtime"; -import { hasEnvHttpProxyConfigured } from "openclaw/plugin-sdk/infra-runtime"; -import type { PinnedDispatcherPolicy } from "openclaw/plugin-sdk/infra-runtime"; +import { + createPinnedLookup, + hasEnvHttpProxyConfigured, + resolveFetch, + type PinnedDispatcherPolicy, +} from "openclaw/plugin-sdk/infra-runtime"; import { createSubsystemLogger } from "openclaw/plugin-sdk/runtime-env"; import { Agent, EnvHttpProxyAgent, ProxyAgent, fetch as undiciFetch } from "undici"; import { @@ -15,6 +18,7 @@ const log = createSubsystemLogger("telegram/network"); const TELEGRAM_AUTO_SELECT_FAMILY_ATTEMPT_TIMEOUT_MS = 300; const TELEGRAM_API_HOSTNAME = "api.telegram.org"; +const TELEGRAM_FALLBACK_IPS: readonly string[] = ["149.154.167.220"]; type RequestInitWithDispatcher = RequestInit & { dispatcher?: unknown; @@ -24,6 +28,16 @@ type TelegramDispatcher = Agent | EnvHttpProxyAgent | ProxyAgent; type TelegramDispatcherMode = "direct" | "env-proxy" | "explicit-proxy"; +type TelegramDispatcherAttempt = { + dispatcherPolicy?: PinnedDispatcherPolicy; +}; + +type TelegramTransportAttempt = { + createDispatcher: () => TelegramDispatcher; + exportAttempt: TelegramDispatcherAttempt; + logMessage?: string; +}; + type TelegramDnsResultOrder = "ipv4first" | "verbatim"; type LookupCallback = @@ -49,17 +63,17 @@ const FALLBACK_RETRY_ERROR_CODES = [ "UND_ERR_SOCKET", ] as const; -type Ipv4FallbackContext = { +type TelegramTransportFallbackContext = { message: string; codes: Set; }; -type Ipv4FallbackRule = { +type TelegramTransportFallbackRule = { name: string; - matches: (ctx: Ipv4FallbackContext) => boolean; + matches: (ctx: TelegramTransportFallbackContext) => boolean; }; -const IPV4_FALLBACK_RULES: readonly Ipv4FallbackRule[] = [ +const TELEGRAM_TRANSPORT_FALLBACK_RULES: readonly TelegramTransportFallbackRule[] = [ { name: "fetch-failed-envelope", matches: ({ message }) => message.includes("fetch failed"), @@ -98,7 +112,6 @@ function createDnsResultOrderLookup( const lookupOptions: LookupOptions = { ...baseOptions, order, - // Keep `verbatim` for compatibility with Node runtimes that ignore `order`. verbatim: order === "verbatim", }; lookup(hostname, lookupOptions, callback); @@ -139,14 +152,6 @@ function buildTelegramConnectOptions(params: { } function shouldBypassEnvProxyForTelegramApi(env: NodeJS.ProcessEnv = process.env): boolean { - // We need this classification before dispatch to decide whether sticky IPv4 fallback - // can safely arm. EnvHttpProxyAgent does not expose route decisions (proxy vs direct - // NO_PROXY bypass), so we mirror undici's parsing/matching behavior for this host. - // Match EnvHttpProxyAgent behavior (undici): - // - lower-case no_proxy takes precedence over NO_PROXY - // - entries split by comma or whitespace - // - wildcard handling is exact-string "*" only - // - leading "." and "*." are normalized the same way const noProxyValue = env.no_proxy ?? env.NO_PROXY ?? ""; if (!noProxyValue) { return false; @@ -228,16 +233,32 @@ function resolveTelegramDispatcherPolicy(params: { }; } +function withPinnedLookup( + options: Record | undefined, + pinnedHostname: PinnedDispatcherPolicy["pinnedHostname"], +): Record | undefined { + if (!pinnedHostname) { + return options ? { ...options } : undefined; + } + const lookup = createPinnedLookup({ + hostname: pinnedHostname.hostname, + addresses: [...pinnedHostname.addresses], + fallback: dns.lookup, + }); + return options ? { ...options, lookup } : { lookup }; +} + function createTelegramDispatcher(policy: PinnedDispatcherPolicy): { dispatcher: TelegramDispatcher; mode: TelegramDispatcherMode; effectivePolicy: PinnedDispatcherPolicy; } { if (policy.mode === "explicit-proxy") { - const proxyOptions = policy.proxyTls + const proxyTlsOptions = withPinnedLookup(policy.proxyTls, policy.pinnedHostname); + const proxyOptions = proxyTlsOptions ? ({ uri: policy.proxyUrl, - proxyTls: { ...policy.proxyTls }, + proxyTls: proxyTlsOptions, } satisfies ConstructorParameters[0]) : policy.proxyUrl; try { @@ -253,13 +274,13 @@ function createTelegramDispatcher(policy: PinnedDispatcherPolicy): { } if (policy.mode === "env-proxy") { + const connectOptions = withPinnedLookup(policy.connect, policy.pinnedHostname); + const proxyTlsOptions = withPinnedLookup(policy.proxyTls, policy.pinnedHostname); const proxyOptions = - policy.connect || policy.proxyTls + connectOptions || proxyTlsOptions ? ({ - ...(policy.connect ? { connect: { ...policy.connect } } : {}), - // undici's EnvHttpProxyAgent passes `connect` only to the no-proxy Agent. - // Real proxied HTTPS traffic reads transport settings from ProxyAgent.proxyTls. - ...(policy.proxyTls ? { proxyTls: { ...policy.proxyTls } } : {}), + ...(connectOptions ? { connect: connectOptions } : {}), + ...(proxyTlsOptions ? { proxyTls: proxyTlsOptions } : {}), } satisfies ConstructorParameters[0]) : undefined; try { @@ -276,14 +297,12 @@ function createTelegramDispatcher(policy: PinnedDispatcherPolicy): { ); const directPolicy: PinnedDispatcherPolicy = { mode: "direct", - ...(policy.connect ? { connect: { ...policy.connect } } : {}), + ...(connectOptions ? { connect: connectOptions } : {}), }; return { dispatcher: new Agent( directPolicy.connect - ? ({ - connect: { ...directPolicy.connect }, - } satisfies ConstructorParameters[0]) + ? ({ connect: directPolicy.connect } satisfies ConstructorParameters[0]) : undefined, ), mode: "direct", @@ -292,11 +311,12 @@ function createTelegramDispatcher(policy: PinnedDispatcherPolicy): { } } + const connectOptions = withPinnedLookup(policy.connect, policy.pinnedHostname); return { dispatcher: new Agent( - policy.connect + connectOptions ? ({ - connect: { ...policy.connect }, + connect: connectOptions, } satisfies ConstructorParameters[0]) : undefined, ), @@ -375,13 +395,13 @@ function formatErrorCodes(err: unknown): string { return codes.length > 0 ? codes.join(",") : "none"; } -function shouldRetryWithIpv4Fallback(err: unknown): boolean { - const ctx: Ipv4FallbackContext = { +function shouldUseTelegramTransportFallback(err: unknown): boolean { + const ctx: TelegramTransportFallbackContext = { message: err && typeof err === "object" && "message" in err ? String(err.message).toLowerCase() : "", codes: collectErrorCodes(err), }; - for (const rule of IPV4_FALLBACK_RULES) { + for (const rule of TELEGRAM_TRANSPORT_FALLBACK_RULES) { if (!rule.matches(ctx)) { return false; } @@ -389,18 +409,71 @@ function shouldRetryWithIpv4Fallback(err: unknown): boolean { return true; } -export function shouldRetryTelegramIpv4Fallback(err: unknown): boolean { - return shouldRetryWithIpv4Fallback(err); +export function shouldRetryTelegramTransportFallback(err: unknown): boolean { + return shouldUseTelegramTransportFallback(err); } -// Prefer wrapped fetch when available to normalize AbortSignal across runtimes. export type TelegramTransport = { fetch: typeof fetch; sourceFetch: typeof fetch; - pinnedDispatcherPolicy?: PinnedDispatcherPolicy; - fallbackPinnedDispatcherPolicy?: PinnedDispatcherPolicy; + dispatcherAttempts?: TelegramDispatcherAttempt[]; }; +function createTelegramTransportAttempts(params: { + defaultDispatcher: ReturnType; + allowFallback: boolean; + fallbackPolicy?: PinnedDispatcherPolicy; +}): TelegramTransportAttempt[] { + const attempts: TelegramTransportAttempt[] = [ + { + createDispatcher: () => params.defaultDispatcher.dispatcher, + exportAttempt: { dispatcherPolicy: params.defaultDispatcher.effectivePolicy }, + }, + ]; + + if (!params.allowFallback || !params.fallbackPolicy) { + return attempts; + } + const fallbackPolicy = params.fallbackPolicy; + + let ipv4Dispatcher: TelegramDispatcher | null = null; + attempts.push({ + createDispatcher: () => { + if (!ipv4Dispatcher) { + ipv4Dispatcher = createTelegramDispatcher(fallbackPolicy).dispatcher; + } + return ipv4Dispatcher; + }, + exportAttempt: { dispatcherPolicy: fallbackPolicy }, + logMessage: "fetch fallback: enabling sticky IPv4-only dispatcher", + }); + + if (TELEGRAM_FALLBACK_IPS.length === 0) { + return attempts; + } + + const fallbackIpPolicy: PinnedDispatcherPolicy = { + ...fallbackPolicy, + pinnedHostname: { + hostname: TELEGRAM_API_HOSTNAME, + addresses: [...TELEGRAM_FALLBACK_IPS], + }, + }; + let fallbackIpDispatcher: TelegramDispatcher | null = null; + attempts.push({ + createDispatcher: () => { + if (!fallbackIpDispatcher) { + fallbackIpDispatcher = createTelegramDispatcher(fallbackIpPolicy).dispatcher; + } + return fallbackIpDispatcher; + }, + exportAttempt: { dispatcherPolicy: fallbackIpPolicy }, + logMessage: "fetch fallback: DNS-resolved IP unreachable; trying alternative Telegram API IP", + }); + + return attempts; +} + export function resolveTelegramTransport( proxyFetch?: typeof fetch, options?: { network?: TelegramNetworkConfig }, @@ -424,7 +497,6 @@ export function resolveTelegramTransport( ? resolveWrappedFetch(proxyFetch) : undiciSourceFetch; const dnsResultOrder = normalizeDnsResultOrder(dnsDecision.value); - // Preserve fully caller-owned custom fetch implementations. if (proxyFetch && !explicitProxyUrl) { return { fetch: sourceFetch, sourceFetch }; } @@ -439,70 +511,75 @@ export function resolveTelegramTransport( }); const defaultDispatcher = createTelegramDispatcher(defaultDispatcherResolution.policy); const shouldBypassEnvProxy = shouldBypassEnvProxyForTelegramApi(); - const allowStickyIpv4Fallback = + const allowStickyFallback = defaultDispatcher.mode === "direct" || (defaultDispatcher.mode === "env-proxy" && shouldBypassEnvProxy); - const stickyShouldUseEnvProxy = defaultDispatcher.mode === "env-proxy"; - const fallbackPinnedDispatcherPolicy = allowStickyIpv4Fallback + const fallbackDispatcherPolicy = allowStickyFallback ? resolveTelegramDispatcherPolicy({ autoSelectFamily: false, dnsResultOrder: "ipv4first", - useEnvProxy: stickyShouldUseEnvProxy, + useEnvProxy: defaultDispatcher.mode === "env-proxy", forceIpv4: true, proxyUrl: explicitProxyUrl, }).policy : undefined; + const transportAttempts = createTelegramTransportAttempts({ + defaultDispatcher, + allowFallback: allowStickyFallback, + fallbackPolicy: fallbackDispatcherPolicy, + }); - let stickyIpv4FallbackEnabled = false; - let stickyIpv4Dispatcher: TelegramDispatcher | null = null; - const resolveStickyIpv4Dispatcher = () => { - if (!stickyIpv4Dispatcher) { - if (!fallbackPinnedDispatcherPolicy) { - return defaultDispatcher.dispatcher; - } - stickyIpv4Dispatcher = createTelegramDispatcher(fallbackPinnedDispatcherPolicy).dispatcher; - } - return stickyIpv4Dispatcher; - }; - + let stickyAttemptIndex = 0; const resolvedFetch = (async (input: RequestInfo | URL, init?: RequestInit) => { const callerProvidedDispatcher = Boolean( (init as RequestInitWithDispatcher | undefined)?.dispatcher, ); - const initialInit = withDispatcherIfMissing( - init, - stickyIpv4FallbackEnabled ? resolveStickyIpv4Dispatcher() : defaultDispatcher.dispatcher, - ); + const startIndex = Math.min(stickyAttemptIndex, transportAttempts.length - 1); + let err: unknown; + try { - return await sourceFetch(input, initialInit); - } catch (err) { - if (shouldRetryWithIpv4Fallback(err)) { - // Preserve caller-owned dispatchers on retry. - if (callerProvidedDispatcher) { - return sourceFetch(input, init ?? {}); - } - // Proxy routes should not arm sticky IPv4 mode; `family=4` would constrain - // proxy-connect behavior instead of Telegram endpoint selection. - if (!allowStickyIpv4Fallback) { - throw err; - } - if (!stickyIpv4FallbackEnabled) { - stickyIpv4FallbackEnabled = true; - log.warn( - `fetch fallback: enabling sticky IPv4-only dispatcher (codes=${formatErrorCodes(err)})`, - ); - } - return sourceFetch(input, withDispatcherIfMissing(init, resolveStickyIpv4Dispatcher())); - } + return await sourceFetch( + input, + withDispatcherIfMissing(init, transportAttempts[startIndex].createDispatcher()), + ); + } catch (caught) { + err = caught; + } + + if (!shouldUseTelegramTransportFallback(err)) { throw err; } + if (callerProvidedDispatcher) { + return sourceFetch(input, init ?? {}); + } + + for (let nextIndex = startIndex + 1; nextIndex < transportAttempts.length; nextIndex += 1) { + const nextAttempt = transportAttempts[nextIndex]; + if (nextAttempt.logMessage) { + log.warn(`${nextAttempt.logMessage} (codes=${formatErrorCodes(err)})`); + } + try { + const response = await sourceFetch( + input, + withDispatcherIfMissing(init, nextAttempt.createDispatcher()), + ); + stickyAttemptIndex = nextIndex; + return response; + } catch (caught) { + err = caught; + if (!shouldUseTelegramTransportFallback(err)) { + throw err; + } + } + } + + throw err; }) as typeof fetch; return { fetch: resolvedFetch, sourceFetch, - pinnedDispatcherPolicy: defaultDispatcher.effectivePolicy, - fallbackPinnedDispatcherPolicy, + dispatcherAttempts: transportAttempts.map((attempt) => attempt.exportAttempt), }; } diff --git a/src/infra/net/fetch-guard.ts b/src/infra/net/fetch-guard.ts index ed082e92fb9..8aec91a62ef 100644 --- a/src/infra/net/fetch-guard.ts +++ b/src/infra/net/fetch-guard.ts @@ -198,7 +198,7 @@ export async function fetchWithSsrFGuard(params: GuardedFetchOptions): Promise { }); }); + it("replaces the pinned lookup when a dispatcher override hostname is provided", () => { + const originalLookup = vi.fn() as unknown as PinnedHostname["lookup"]; + const pinned: PinnedHostname = { + hostname: "api.telegram.org", + addresses: ["149.154.167.221"], + lookup: originalLookup, + }; + + createPinnedDispatcher(pinned, { + mode: "direct", + pinnedHostname: { + hostname: "api.telegram.org", + addresses: ["149.154.167.220"], + }, + }); + + const firstCallArg = agentCtor.mock.calls.at(-1)?.[0] as + | { connect?: { lookup?: PinnedHostname["lookup"] } } + | undefined; + expect(firstCallArg?.connect?.lookup).toBeTypeOf("function"); + + const lookup = firstCallArg?.connect?.lookup; + const callback = vi.fn(); + lookup?.("api.telegram.org", callback); + + expect(callback).toHaveBeenCalledWith(null, "149.154.167.220", 4); + expect(originalLookup).not.toHaveBeenCalled(); + }); + + it("rejects pinned override addresses that violate SSRF policy", () => { + const originalLookup = vi.fn() as unknown as PinnedHostname["lookup"]; + const pinned: PinnedHostname = { + hostname: "api.telegram.org", + addresses: ["149.154.167.221"], + lookup: originalLookup, + }; + + expect(() => + createPinnedDispatcher( + pinned, + { + mode: "direct", + pinnedHostname: { + hostname: "api.telegram.org", + addresses: ["127.0.0.1"], + }, + }, + undefined, + ), + ).toThrow(/private|internal|blocked/i); + }); + it("keeps env proxy route while pinning the direct no-proxy path", () => { const lookup = vi.fn() as unknown as PinnedHostname["lookup"]; const pinned: PinnedHostname = { diff --git a/src/infra/net/ssrf.pinning.test.ts b/src/infra/net/ssrf.pinning.test.ts index 28420ea373f..a8847c26642 100644 --- a/src/infra/net/ssrf.pinning.test.ts +++ b/src/infra/net/ssrf.pinning.test.ts @@ -99,6 +99,15 @@ describe("ssrf pinning", () => { expect(result.address).toBe("1.2.3.4"); }); + it("fails loud when a pinned lookup is created without any addresses", () => { + expect(() => + createPinnedLookup({ + hostname: "example.com", + addresses: [], + }), + ).toThrow("Pinned lookup requires at least one address for example.com"); + }); + it("enforces hostname allowlist when configured", async () => { const lookup = vi.fn(async () => [ { address: "93.184.216.34", family: 4 }, diff --git a/src/infra/net/ssrf.ts b/src/infra/net/ssrf.ts index db70664a43f..fd633fcb20d 100644 --- a/src/infra/net/ssrf.ts +++ b/src/infra/net/ssrf.ts @@ -67,6 +67,13 @@ export function isPrivateNetworkAllowedByPolicy(policy?: SsrFPolicy): boolean { return policy?.dangerouslyAllowPrivateNetwork === true || policy?.allowPrivateNetwork === true; } +function shouldSkipPrivateNetworkChecks(hostname: string, policy?: SsrFPolicy): boolean { + return ( + isPrivateNetworkAllowedByPolicy(policy) || + normalizeHostnameSet(policy?.allowedHostnames).has(hostname) + ); +} + function resolveIpv4SpecialUseBlockOptions(policy?: SsrFPolicy): Ipv4SpecialUseBlockOptions { return { allowRfc2544BenchmarkRange: policy?.allowRfc2544BenchmarkRange === true, @@ -198,6 +205,9 @@ export function createPinnedLookup(params: { fallback?: typeof dnsLookupCb; }): typeof dnsLookupCb { const normalizedHost = normalizeHostname(params.hostname); + if (params.addresses.length === 0) { + throw new Error(`Pinned lookup requires at least one address for ${params.hostname}`); + } const fallback = params.fallback ?? dnsLookupCb; const fallbackLookup = fallback as unknown as ( hostname: string, @@ -255,20 +265,28 @@ export type PinnedHostname = { lookup: typeof dnsLookupCb; }; +export type PinnedHostnameOverride = { + hostname: string; + addresses: string[]; +}; + export type PinnedDispatcherPolicy = | { mode: "direct"; connect?: Record; + pinnedHostname?: PinnedHostnameOverride; } | { mode: "env-proxy"; connect?: Record; proxyTls?: Record; + pinnedHostname?: PinnedHostnameOverride; } | { mode: "explicit-proxy"; proxyUrl: string; proxyTls?: Record; + pinnedHostname?: PinnedHostnameOverride; }; function dedupeAndPreferIpv4(results: readonly LookupAddress[]): string[] { @@ -298,11 +316,8 @@ export async function resolvePinnedHostnameWithPolicy( throw new Error("Invalid hostname"); } - const allowPrivateNetwork = isPrivateNetworkAllowedByPolicy(params.policy); - const allowedHostnames = normalizeHostnameSet(params.policy?.allowedHostnames); const hostnameAllowlist = normalizeHostnameAllowlist(params.policy?.hostnameAllowlist); - const isExplicitAllowed = allowedHostnames.has(normalized); - const skipPrivateNetworkChecks = allowPrivateNetwork || isExplicitAllowed; + const skipPrivateNetworkChecks = shouldSkipPrivateNetworkChecks(normalized, params.policy); if (!matchesHostnameAllowlist(normalized, hostnameAllowlist)) { throw new SsrFBlockedError(`Blocked hostname (not in allowlist): ${hostname}`); @@ -352,19 +367,50 @@ function withPinnedLookup( return connect ? { ...connect, lookup } : { lookup }; } +function resolvePinnedDispatcherLookup( + pinned: PinnedHostname, + override?: PinnedHostnameOverride, + policy?: SsrFPolicy, +): PinnedHostname["lookup"] { + if (!override) { + return pinned.lookup; + } + const normalizedOverrideHost = normalizeHostname(override.hostname); + if (!normalizedOverrideHost || normalizedOverrideHost !== pinned.hostname) { + throw new Error( + `Pinned dispatcher override hostname mismatch: expected ${pinned.hostname}, got ${override.hostname}`, + ); + } + const records = override.addresses.map((address) => ({ + address, + family: address.includes(":") ? 6 : 4, + })); + if (!shouldSkipPrivateNetworkChecks(pinned.hostname, policy)) { + assertAllowedResolvedAddressesOrThrow(records, policy); + } + return createPinnedLookup({ + hostname: pinned.hostname, + addresses: [...override.addresses], + fallback: pinned.lookup, + }); +} + export function createPinnedDispatcher( pinned: PinnedHostname, policy?: PinnedDispatcherPolicy, + ssrfPolicy?: SsrFPolicy, ): Dispatcher { + const lookup = resolvePinnedDispatcherLookup(pinned, policy?.pinnedHostname, ssrfPolicy); + if (!policy || policy.mode === "direct") { return new Agent({ - connect: withPinnedLookup(pinned.lookup, policy?.connect), + connect: withPinnedLookup(lookup, policy?.connect), }); } if (policy.mode === "env-proxy") { return new EnvHttpProxyAgent({ - connect: withPinnedLookup(pinned.lookup, policy.connect), + connect: withPinnedLookup(lookup, policy.connect), ...(policy.proxyTls ? { proxyTls: { ...policy.proxyTls } } : {}), }); } diff --git a/src/media/fetch.telegram-network.test.ts b/src/media/fetch.telegram-network.test.ts index 60e60f1c48c..faf16314d98 100644 --- a/src/media/fetch.telegram-network.test.ts +++ b/src/media/fetch.telegram-network.test.ts @@ -22,7 +22,7 @@ vi.mock("undici", () => ({ })); let resolveTelegramTransport: typeof import("../../extensions/telegram/src/fetch.js").resolveTelegramTransport; -let shouldRetryTelegramIpv4Fallback: typeof import("../../extensions/telegram/src/fetch.js").shouldRetryTelegramIpv4Fallback; +let shouldRetryTelegramTransportFallback: typeof import("../../extensions/telegram/src/fetch.js").shouldRetryTelegramTransportFallback; let fetchRemoteMedia: typeof import("./fetch.js").fetchRemoteMedia; describe("fetchRemoteMedia telegram network policy", () => { @@ -30,7 +30,7 @@ describe("fetchRemoteMedia telegram network policy", () => { beforeEach(async () => { vi.resetModules(); - ({ resolveTelegramTransport, shouldRetryTelegramIpv4Fallback } = + ({ resolveTelegramTransport, shouldRetryTelegramTransportFallback } = await import("../../extensions/telegram/src/fetch.js")); ({ fetchRemoteMedia } = await import("./fetch.js")); }); @@ -70,7 +70,7 @@ describe("fetchRemoteMedia telegram network policy", () => { await fetchRemoteMedia({ url: "https://api.telegram.org/file/bottok/photos/1.jpg", fetchImpl: telegramTransport.sourceFetch, - dispatcherPolicy: telegramTransport.pinnedDispatcherPolicy, + dispatcherAttempts: telegramTransport.dispatcherAttempts, lookupFn, maxBytes: 1024, ssrfPolicy: { @@ -120,7 +120,7 @@ describe("fetchRemoteMedia telegram network policy", () => { await fetchRemoteMedia({ url: "https://api.telegram.org/file/bottok/files/1.pdf", fetchImpl: telegramTransport.sourceFetch, - dispatcherPolicy: telegramTransport.pinnedDispatcherPolicy, + dispatcherAttempts: telegramTransport.dispatcherAttempts, lookupFn, maxBytes: 1024, ssrfPolicy: { @@ -167,9 +167,8 @@ describe("fetchRemoteMedia telegram network policy", () => { await fetchRemoteMedia({ url: "https://api.telegram.org/file/bottok/photos/2.jpg", fetchImpl: telegramTransport.sourceFetch, - dispatcherPolicy: telegramTransport.pinnedDispatcherPolicy, - fallbackDispatcherPolicy: telegramTransport.fallbackPinnedDispatcherPolicy, - shouldRetryFetchError: shouldRetryTelegramIpv4Fallback, + dispatcherAttempts: telegramTransport.dispatcherAttempts, + shouldRetryFetchError: shouldRetryTelegramTransportFallback, lookupFn, maxBytes: 1024, ssrfPolicy: { @@ -214,14 +213,83 @@ describe("fetchRemoteMedia telegram network policy", () => { ); }); - it("preserves both primary and fallback errors when Telegram media retry fails twice", async () => { + it("retries Telegram file downloads with pinned Telegram IP after IPv4 fallback fails", async () => { + const lookupFn = vi.fn(async () => [ + { address: "149.154.167.221", family: 4 }, + { address: "2001:67c:4e8:f004::9", family: 6 }, + ]) as unknown as LookupFn; + undiciMocks.fetch + .mockRejectedValueOnce(createTelegramFetchFailedError("EHOSTUNREACH")) + .mockRejectedValueOnce(createTelegramFetchFailedError("ETIMEDOUT")) + .mockResolvedValueOnce( + new Response(new Uint8Array([0xff, 0xd8, 0xff, 0x00]), { + status: 200, + headers: { "content-type": "image/jpeg" }, + }), + ); + + const telegramTransport = resolveTelegramTransport(undefined, { + network: { + autoSelectFamily: true, + dnsResultOrder: "ipv4first", + }, + }); + + await fetchRemoteMedia({ + url: "https://api.telegram.org/file/bottok/photos/3.jpg", + fetchImpl: telegramTransport.sourceFetch, + dispatcherAttempts: telegramTransport.dispatcherAttempts, + shouldRetryFetchError: shouldRetryTelegramTransportFallback, + lookupFn, + maxBytes: 1024, + ssrfPolicy: { + allowedHostnames: ["api.telegram.org"], + allowRfc2544BenchmarkRange: true, + }, + }); + + const thirdInit = undiciMocks.fetch.mock.calls[2]?.[1] as + | (RequestInit & { + dispatcher?: { + options?: { + connect?: Record; + }; + }; + }) + | undefined; + const callback = vi.fn(); + ( + thirdInit?.dispatcher?.options?.connect?.lookup as + | (( + hostname: string, + callback: (err: null, address: string, family: number) => void, + ) => void) + | undefined + )?.("api.telegram.org", callback); + + expect(undiciMocks.fetch).toHaveBeenCalledTimes(3); + expect(thirdInit?.dispatcher?.options?.connect).toEqual( + expect.objectContaining({ + family: 4, + autoSelectFamily: false, + lookup: expect.any(Function), + }), + ); + expect(callback).toHaveBeenCalledWith(null, "149.154.167.220", 4); + }); + + it("preserves both primary and final fallback errors when Telegram media retry chain fails", async () => { const lookupFn = vi.fn(async () => [ { address: "149.154.167.220", family: 4 }, { address: "2001:67c:4e8:f004::9", family: 6 }, ]) as unknown as LookupFn; const primaryError = createTelegramFetchFailedError("EHOSTUNREACH"); + const ipv4Error = createTelegramFetchFailedError("ETIMEDOUT"); const fallbackError = createTelegramFetchFailedError("ETIMEDOUT"); - undiciMocks.fetch.mockRejectedValueOnce(primaryError).mockRejectedValueOnce(fallbackError); + undiciMocks.fetch + .mockRejectedValueOnce(primaryError) + .mockRejectedValueOnce(ipv4Error) + .mockRejectedValueOnce(fallbackError); const telegramTransport = resolveTelegramTransport(undefined, { network: { @@ -232,11 +300,10 @@ describe("fetchRemoteMedia telegram network policy", () => { await expect( fetchRemoteMedia({ - url: "https://api.telegram.org/file/bottok/photos/3.jpg", + url: "https://api.telegram.org/file/bottok/photos/4.jpg", fetchImpl: telegramTransport.sourceFetch, - dispatcherPolicy: telegramTransport.pinnedDispatcherPolicy, - fallbackDispatcherPolicy: telegramTransport.fallbackPinnedDispatcherPolicy, - shouldRetryFetchError: shouldRetryTelegramIpv4Fallback, + dispatcherAttempts: telegramTransport.dispatcherAttempts, + shouldRetryFetchError: shouldRetryTelegramTransportFallback, lookupFn, maxBytes: 1024, ssrfPolicy: { @@ -250,6 +317,7 @@ describe("fetchRemoteMedia telegram network policy", () => { cause: expect.objectContaining({ name: "Error", cause: fallbackError, + attemptErrors: [primaryError, ipv4Error, fallbackError], primaryError, }), }); diff --git a/src/media/fetch.ts b/src/media/fetch.ts index 020ac8040bd..3893b1366d4 100644 --- a/src/media/fetch.ts +++ b/src/media/fetch.ts @@ -26,6 +26,11 @@ export class MediaFetchError extends Error { export type FetchLike = (input: RequestInfo | URL, init?: RequestInit) => Promise; +export type FetchDispatcherAttempt = { + dispatcherPolicy?: PinnedDispatcherPolicy; + lookupFn?: LookupFn; +}; + type FetchMediaOptions = { url: string; fetchImpl?: FetchLike; @@ -37,8 +42,7 @@ type FetchMediaOptions = { readIdleTimeoutMs?: number; ssrfPolicy?: SsrFPolicy; lookupFn?: LookupFn; - dispatcherPolicy?: PinnedDispatcherPolicy; - fallbackDispatcherPolicy?: PinnedDispatcherPolicy; + dispatcherAttempts?: FetchDispatcherAttempt[]; shouldRetryFetchError?: (error: unknown) => boolean; }; @@ -101,8 +105,7 @@ export async function fetchRemoteMedia(options: FetchMediaOptions): Promise Promise) | null = null; - const runGuardedFetch = async (policy?: PinnedDispatcherPolicy) => + const attempts = + dispatcherAttempts && dispatcherAttempts.length > 0 + ? dispatcherAttempts + : [{ dispatcherPolicy: undefined, lookupFn }]; + const runGuardedFetch = async (attempt: FetchDispatcherAttempt) => await fetchWithSsrFGuard( withStrictGuardedFetchMode({ url, @@ -118,32 +125,43 @@ export async function fetchRemoteMedia(options: FetchMediaOptions): Promise>; + const attemptErrors: unknown[] = []; + for (let i = 0; i < attempts.length; i += 1) { + try { + result = await runGuardedFetch(attempts[i]); + break; + } catch (err) { + if ( + typeof shouldRetryFetchError !== "function" || + !shouldRetryFetchError(err) || + i === attempts.length - 1 + ) { + if (attemptErrors.length > 0) { + const combined = new Error( + `Primary fetch failed and fallback fetch also failed for ${sourceUrl}`, + { cause: err }, + ); + ( + combined as Error & { + primaryError?: unknown; + attemptErrors?: unknown[]; + } + ).primaryError = attemptErrors[0]; + (combined as Error & { attemptErrors?: unknown[] }).attemptErrors = [ + ...attemptErrors, + err, + ]; + throw combined; + } + throw err; } - } else { - throw err; + attemptErrors.push(err); } } res = result.response;