diff --git a/src/agents/tools/web-fetch.cf-markdown.test.ts b/src/agents/tools/web-fetch.cf-markdown.test.ts index ab46d136afeb..8be80256eaa5 100644 --- a/src/agents/tools/web-fetch.cf-markdown.test.ts +++ b/src/agents/tools/web-fetch.cf-markdown.test.ts @@ -6,7 +6,7 @@ import * as logger from "../../logger.js"; import { withFetchPreconnect } from "../../test-utils/fetch-mock.js"; import "./web-fetch.test-mocks.js"; import { createWebFetchTool } from "./web-fetch.js"; -import { createBaseWebFetchToolConfig, makeFetchHeaders } from "./web-fetch.test-harness.js"; +import { createBaseWebFetchToolConfig } from "./web-fetch.test-harness.js"; const lookupMock = vi.fn(); const baseToolConfig = createBaseWebFetchToolConfig({ @@ -14,24 +14,20 @@ const baseToolConfig = createBaseWebFetchToolConfig({ }); function markdownResponse(body: string, extraHeaders: Record = {}): Response { - return { - ok: true, + return new Response(body, { status: 200, - headers: makeFetchHeaders({ + headers: { "content-type": "text/markdown; charset=utf-8", ...extraHeaders, - }), - text: async () => body, - } as Response; + }, + }); } function htmlResponse(body: string): Response { - return { - ok: true, + return new Response(body, { status: 200, - headers: makeFetchHeaders({ "content-type": "text/html; charset=utf-8" }), - text: async () => body, - } as Response; + headers: { "content-type": "text/html; charset=utf-8" }, + }); } describe("web_fetch Cloudflare Markdown for Agents", () => { diff --git a/src/agents/tools/web-shared.test.ts b/src/agents/tools/web-shared.test.ts index 85c82001ab19..358588ba44a4 100644 --- a/src/agents/tools/web-shared.test.ts +++ b/src/agents/tools/web-shared.test.ts @@ -146,4 +146,48 @@ describe("readResponseText", () => { expect(cancel).toHaveBeenCalledTimes(1); expect(releaseLock).toHaveBeenCalledTimes(1); }); + + it("does not invoke whole-body fallbacks when maxBytes is set", async () => { + const arrayBuffer = vi.fn(async () => new TextEncoder().encode("hello").buffer); + const text = vi.fn(async () => "hello"); + const response = { + arrayBuffer, + headers: new Headers(), + text, + } as unknown as Response; + + await expect(readResponseText(response, { maxBytes: 4 })).resolves.toEqual({ + text: "", + truncated: true, + bytesRead: 0, + }); + expect(arrayBuffer).not.toHaveBeenCalled(); + expect(text).not.toHaveBeenCalled(); + }); + + it("treats a native bodyless response as empty when maxBytes is set", async () => { + await expect( + readResponseText(new Response(null, { status: 204 }), { maxBytes: 4 }), + ).resolves.toEqual({ + text: "", + truncated: false, + bytesRead: 0, + }); + }); + + it("preserves uncapped text-only fallback byte accounting", async () => { + const value = "中文🔥"; + const text = vi.fn(async () => value); + const response = { + headers: new Headers(), + text, + } as unknown as Response; + + await expect(readResponseText(response)).resolves.toEqual({ + text: value, + truncated: false, + bytesRead: new TextEncoder().encode(value).byteLength, + }); + expect(text).toHaveBeenCalledTimes(1); + }); }); diff --git a/src/agents/tools/web-shared.ts b/src/agents/tools/web-shared.ts index ea41f49cd298..523cc6694ae5 100644 --- a/src/agents/tools/web-shared.ts +++ b/src/agents/tools/web-shared.ts @@ -297,6 +297,15 @@ export async function readResponseText( return { text: decodeResponseBytes(res, bytes), truncated, bytesRead }; } + if (maxBytes) { + if (res instanceof Response && res.body === null) { + return { text: "", truncated: false, bytesRead: 0 }; + } + // Whole-body fallbacks allocate before returning, so they cannot honor a byte cap. + // Fail closed instead of making maxBytes a returned-text limit only. + return { text: "", truncated: true, bytesRead: 0 }; + } + const readBytes = (res as { arrayBuffer?: () => Promise }).arrayBuffer; if (typeof readBytes === "function") { try { @@ -313,7 +322,8 @@ export async function readResponseText( try { const text = await res.text(); - return { text, truncated: false, bytesRead: text.length }; + const bytes = new TextEncoder().encode(text); + return { text, truncated: false, bytesRead: bytes.byteLength }; } catch { return { text: "", truncated: false, bytesRead: 0 }; } diff --git a/src/agents/tools/web-tools.fetch.test.ts b/src/agents/tools/web-tools.fetch.test.ts index b0b0ef9bfdb6..d5bd448fde03 100644 --- a/src/agents/tools/web-tools.fetch.test.ts +++ b/src/agents/tools/web-tools.fetch.test.ts @@ -6,7 +6,6 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import type { LookupFn } from "../../infra/net/ssrf.js"; import { resolveRequestUrl } from "../../plugin-sdk/request-url.js"; import { withFetchPreconnect } from "../../test-utils/fetch-mock.js"; -import { makeFetchHeaders } from "./web-fetch.test-harness.js"; const { extractReadableContentMock, resolveWebFetchDefinitionMock } = vi.hoisted(() => ({ extractReadableContentMock: vi.fn(), resolveWebFetchDefinitionMock: vi.fn(), @@ -29,37 +28,36 @@ import { WEB_FETCH_SPILL_MAX_CHARS } from "./web-fetch.js"; const lookupMock = vi.fn(); -type MockResponse = { - ok: boolean; - status: number; - url?: string; - headers?: { get: (key: string) => string | null }; - text?: () => Promise; - json?: () => Promise; -}; +function responseWithUrl(body: BodyInit, init: ResponseInit, url: string): Response { + const response = new Response(body, init); + Object.defineProperty(response, "url", { value: url }); + return response; +} -function htmlResponse(html: string, url = "https://example.com/"): MockResponse { - return { - ok: true, - status: 200, +function htmlResponse(html: string, url = "https://example.com/"): Response { + return responseWithUrl( + html, + { + status: 200, + headers: { "content-type": "text/html; charset=utf-8" }, + }, url, - headers: makeFetchHeaders({ "content-type": "text/html; charset=utf-8" }), - text: async () => html, - }; + ); } function textResponse( text: string, url = "https://example.com/", contentType = "text/plain; charset=utf-8", -): MockResponse { - return { - ok: true, - status: 200, +): Response { + return responseWithUrl( + text, + { + status: 200, + headers: { "content-type": contentType }, + }, url, - headers: makeFetchHeaders({ "content-type": contentType }), - text: async () => text, - }; + ); } function errorHtmlResponse( @@ -67,14 +65,16 @@ function errorHtmlResponse( status = 404, url = "https://example.com/", contentType: string | null = "text/html; charset=utf-8", -): MockResponse { - return { - ok: false, - status, +): Response { + const body = contentType ? html : new TextEncoder().encode(html); + return responseWithUrl( + body, + { + status, + headers: contentType ? { "content-type": contentType } : undefined, + }, url, - headers: contentType ? makeFetchHeaders({ "content-type": contentType }) : makeFetchHeaders({}), - text: async () => html, - }; + ); } function installMockFetch( impl: (input: RequestInfo | URL, init?: RequestInit) => Promise, @@ -111,13 +111,7 @@ function createFetchTool(fetchOverrides: Record = {}) { function installPlainTextFetch(text: string) { installMockFetch((input: RequestInfo | URL) => - Promise.resolve({ - ok: true, - status: 200, - headers: makeFetchHeaders({ "content-type": "text/plain" }), - text: async () => text, - url: resolveRequestUrl(input), - } as Response), + Promise.resolve(textResponse(text, resolveRequestUrl(input))), ); } @@ -415,13 +409,7 @@ describe("web_fetch extraction fallbacks", () => { it("enforces maxChars after wrapping", async () => { const longText = "x".repeat(5_000); installMockFetch((input: RequestInfo | URL) => - Promise.resolve({ - ok: true, - status: 200, - headers: makeFetchHeaders({ "content-type": "text/plain" }), - text: async () => longText, - url: resolveRequestUrl(input), - } as Response), + Promise.resolve(textResponse(longText, resolveRequestUrl(input))), ); const tool = createFetchTool({ @@ -655,13 +643,7 @@ describe("web_fetch extraction fallbacks", () => { it("keeps DNS pinning for web_fetch by default even when HTTP_PROXY is configured", async () => { vi.stubEnv("HTTP_PROXY", "http://127.0.0.1:7890"); const mockFetch = installMockFetch((input: RequestInfo | URL) => - Promise.resolve({ - ok: true, - status: 200, - headers: makeFetchHeaders({ "content-type": "text/plain" }), - text: async () => "proxy body", - url: resolveRequestUrl(input), - } as Response), + Promise.resolve(textResponse("proxy body", resolveRequestUrl(input))), ); const tool = createFetchTool({ firecrawl: { enabled: false } }); @@ -678,13 +660,7 @@ describe("web_fetch extraction fallbacks", () => { it("uses env proxy dispatch for web_fetch when trusted env proxy is explicitly enabled", async () => { vi.stubEnv("HTTP_PROXY", "http://127.0.0.1:7890"); const mockFetch = installMockFetch((input: RequestInfo | URL) => - Promise.resolve({ - ok: true, - status: 200, - headers: makeFetchHeaders({ "content-type": "text/plain" }), - text: async () => "proxy body", - url: resolveRequestUrl(input), - } as Response), + Promise.resolve(textResponse("proxy body", resolveRequestUrl(input))), ); const tool = createFetchTool({ firecrawl: { enabled: false }, @@ -800,13 +776,14 @@ describe("web_fetch extraction fallbacks", () => { }); it("uses the provider fallback when direct fetch fails", async () => { - installMockFetch((_input: RequestInfo | URL) => { - return Promise.resolve({ - ok: false, - status: 403, - headers: makeFetchHeaders({ "content-type": "text/html" }), - text: async () => "blocked", - } as Response); + installMockFetch((input: RequestInfo | URL) => { + return Promise.resolve( + responseWithUrl( + "blocked", + { status: 403, headers: { "content-type": "text/html" } }, + resolveRequestUrl(input), + ), + ); }); resolveWebFetchDefinitionMock.mockReturnValue({