diff --git a/scripts/deadcode-exports.baseline.mjs b/scripts/deadcode-exports.baseline.mjs index a34ce224e4f4..b0a0ede95e89 100644 --- a/scripts/deadcode-exports.baseline.mjs +++ b/scripts/deadcode-exports.baseline.mjs @@ -753,8 +753,6 @@ export const KNIP_UNUSED_EXPORT_BASELINE = [ "src/agents/tools/web-fetch.ts: sanitizeWebFetchUrl", "src/agents/tools/web-fetch.ts: WEB_FETCH_SPILL_MAX_CHARS", "src/agents/utility-model.ts: resolveProviderDefaultUtilityModelRef", - "src/agents/utils/image-resize.ts: formatDimensionNote", - "src/agents/utils/image-resize.ts: resizeImage", "src/agents/utils/tools-manager.ts: getToolPath", "src/agents/utils/tools-manager.ts: testing", "src/agents/workspace-templates.ts: resetWorkspaceTemplateDirCache", diff --git a/src/agents/utils/image-resize.test.ts b/src/agents/utils/image-resize.test.ts index b4431b423676..5f13e72ce8c1 100644 --- a/src/agents/utils/image-resize.test.ts +++ b/src/agents/utils/image-resize.test.ts @@ -3,34 +3,32 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; const mocks = vi.hoisted(() => ({ + convertImageToPng: vi.fn(), encode: vi.fn(), probe: vi.fn(), })); vi.mock("../../media/image-ops.js", () => ({ + convertImageToPng: mocks.convertImageToPng, createImageProcessor: () => ({ encode: mocks.encode, probe: mocks.probe, }), - isImageProcessorUnavailableError: (error: unknown) => - typeof error === "object" && - error !== null && - "code" in error && - error.code === "IMAGE_PROCESSOR_UNAVAILABLE", })); -import { formatDimensionNote, resizeImage } from "./image-resize.js"; +import { processImage } from "./image-resize.js"; describe("image resize utility", () => { beforeEach(() => { + mocks.convertImageToPng.mockReset(); mocks.encode.mockReset(); mocks.probe.mockReset(); }); - it("keeps images that already fit the inline limits", async () => { - const input = Buffer.from("small image").toString("base64"); + it("keeps images that exactly fit the inline limits", async () => { + const input = "a".repeat(4.5 * 1024 * 1024); mocks.probe.mockResolvedValue({ - bytes: 11, + bytes: Buffer.byteLength(input, "base64"), format: "png", hasAlpha: false, height: 20, @@ -38,19 +36,15 @@ describe("image resize utility", () => { width: 10, }); - const resized = await resizeImage( + const result = await processImage( { type: "image", data: input, mimeType: "image/png" }, - { maxWidth: 100, maxHeight: 100, maxBytes: 1_000 }, + { autoResizeImages: true }, ); - expect(resized).toMatchObject({ - data: input, - height: 20, - mimeType: "image/png", - originalHeight: 20, - originalWidth: 10, - wasResized: false, - width: 10, + expect(result).toStrictEqual({ + ok: true, + image: { type: "image", data: input, mimeType: "image/png" }, + hints: [], }); expect(mocks.encode).not.toHaveBeenCalled(); }); @@ -82,9 +76,9 @@ describe("image resize utility", () => { withinBudget: true, }); - const resized = await resizeImage( + const result = await processImage( { type: "image", data: inputBuffer.toString("base64"), mimeType: "image/jpeg" }, - { maxWidth: 2_000, maxHeight: 2_000, maxBytes: 4_000, jpegQuality: 70 }, + { autoResizeImages: true }, ); expect(mocks.encode).toHaveBeenCalledWith(inputBuffer, { @@ -93,29 +87,28 @@ describe("image resize utility", () => { maxHeight: 2_000, maxWidth: 2_000, }, - maxBytes: 3_000, - opaque: { format: "jpeg", quality: 70 }, + maxBase64Bytes: 4.5 * 1024 * 1024, + opaque: { format: "jpeg", quality: 80 }, search: { compressionLevel: [6, 9], - quality: [70, 85, 55, 40, 35], + quality: [80, 85, 70, 55, 40, 35], }, transparent: { format: "png" }, }); - expect(resized).toMatchObject({ - data: outputBuffer.toString("base64"), - height: 1600, - mimeType: "image/jpeg", - originalHeight: 3000, - originalWidth: 1200, - wasResized: true, - width: 640, + expect(result).toStrictEqual({ + ok: true, + image: { + type: "image", + data: outputBuffer.toString("base64"), + mimeType: "image/jpeg", + }, + hints: [ + "[Image: original 1200x3000, displayed at 640x1600. Multiply coordinates by 1.88 to map to original image.]", + ], }); - expect(formatDimensionNote(resized!)).toBe( - "[Image: original 1200x3000, displayed at 640x1600. Multiply coordinates by 1.88 to map to original image.]", - ); }); - it("returns null when Rastermill cannot satisfy the base64 budget", async () => { + it("omits images when Rastermill cannot satisfy the base64 budget", async () => { const inputBuffer = Buffer.from("too large"); mocks.probe.mockResolvedValue({ bytes: inputBuffer.byteLength, @@ -140,10 +133,54 @@ describe("image resize utility", () => { }); await expect( - resizeImage( + processImage( { type: "image", data: inputBuffer.toString("base64"), mimeType: "image/png" }, - { maxWidth: 100, maxHeight: 100, maxBytes: 50 }, + { autoResizeImages: true }, ), - ).resolves.toBeNull(); + ).resolves.toStrictEqual({ + ok: false, + message: "[Image omitted: could not be resized below the inline image size limit.]", + }); + }); + + it("does not add coordinate hints when Rastermill only re-encodes the image", async () => { + const input = "a".repeat(4.5 * 1024 * 1024 + 1); + const outputBuffer = Buffer.from("re-encoded"); + mocks.probe.mockResolvedValue({ + bytes: Buffer.byteLength(input, "base64"), + format: "png", + hasAlpha: false, + height: 20, + orientation: null, + width: 10, + }); + mocks.encode.mockResolvedValue({ + base64Bytes: Buffer.byteLength(outputBuffer.toString("base64"), "utf8"), + bytes: outputBuffer.byteLength, + chosen: { format: "jpeg", quality: 80 }, + data: outputBuffer, + format: "jpeg", + height: 20, + metadata: "stripped", + mimeType: "image/jpeg", + resized: false, + width: 10, + withinBudget: true, + }); + + await expect( + processImage( + { type: "image", data: input, mimeType: "image/png" }, + { autoResizeImages: true }, + ), + ).resolves.toStrictEqual({ + ok: true, + image: { + type: "image", + data: outputBuffer.toString("base64"), + mimeType: "image/jpeg", + }, + hints: [], + }); }); }); diff --git a/src/agents/utils/image-resize.ts b/src/agents/utils/image-resize.ts index 77e36dceca9c..c91f20c7c9e3 100644 --- a/src/agents/utils/image-resize.ts +++ b/src/agents/utils/image-resize.ts @@ -4,19 +4,7 @@ * Downscales base64 image content for provider payload limits using the configured image processor. */ import type { ImageContent } from "../../llm/types.js"; -import { - convertImageToPng, - createImageProcessor, - isImageProcessorUnavailableError, - type ImageProbe, -} from "../../media/image-ops.js"; - -interface ImageResizeOptions { - maxWidth?: number; // Default: 2000 - maxHeight?: number; // Default: 2000 - maxBytes?: number; // Default: 4.5MB of base64 payload (below Anthropic's 5MB limit) - jpegQuality?: number; // Default: 80 -} +import { convertImageToPng, createImageProcessor, type ImageProbe } from "../../media/image-ops.js"; interface ResizedImage { data: string; // base64 @@ -96,34 +84,11 @@ export async function processImage( }; } -// 4.5MB of base64 payload. Provides headroom below Anthropic's 5MB limit. -const DEFAULT_MAX_BYTES = 4.5 * 1024 * 1024; - -const DEFAULT_OPTIONS: Required = { - maxWidth: 2000, - maxHeight: 2000, - maxBytes: DEFAULT_MAX_BYTES, - jpegQuality: 80, -}; - -function maxBinaryBytesForBase64Budget(maxBase64Bytes: number): number { - return Math.floor(maxBase64Bytes / 4) * 3; -} - -interface EncodedCandidate { - data: string; - encodedSize: number; - mimeType: string; -} - -function encodeCandidate(buffer: Buffer, mimeType: string): EncodedCandidate { - const data = buffer.toString("base64"); - return { - data, - encodedSize: Buffer.byteLength(data, "utf-8"), - mimeType, - }; -} +const MAX_IMAGE_WIDTH = 2000; +const MAX_IMAGE_HEIGHT = 2000; +// 4.5MB of base64 payload leaves headroom below Anthropic's 5MB limit. +const MAX_IMAGE_BASE64_BYTES = 4.5 * 1024 * 1024; +const JPEG_QUALITY = 80; function orientedDimensions(probe: ImageProbe): { width: number; height: number } { return probe.orientation && probe.orientation >= 5 && probe.orientation <= 8 @@ -132,23 +97,19 @@ function orientedDimensions(probe: ImageProbe): { width: number; height: number } /** - * Resize an image to fit within the specified max dimensions and encoded file size. - * Returns null if the image cannot be resized below maxBytes. + * Resize an image to fit within the inline dimensions and base64 payload limit. + * Returns null if Rastermill cannot produce output within those limits. * * Uses Rastermill for image processing. If no Rastermill backend is available, * returns null. * - * Strategy for staying under maxBytes: - * 1. First resize to maxWidth/maxHeight + * Strategy for staying under the inline limits: + * 1. First resize to the maximum dimensions * 2. Let Rastermill choose JPEG or PNG for the image transparency profile * 3. If still too large, search decreasing quality/compression settings * 4. If still too large, progressively reduce dimensions */ -export async function resizeImage( - img: ImageContent, - options?: ImageResizeOptions, -): Promise { - const opts = { ...DEFAULT_OPTIONS, ...options }; +async function resizeImage(img: ImageContent): Promise { const inputBuffer = Buffer.from(img.data, "base64"); const inputBase64Size = Buffer.byteLength(img.data, "utf-8"); const processor = createImageProcessor(); @@ -159,17 +120,16 @@ export async function resizeImage( return null; } const { width: originalWidth, height: originalHeight } = orientedDimensions(probe); - const format = img.mimeType?.split("/")[1] ?? "png"; // Check if already within all limits (dimensions AND encoded size) if ( - originalWidth <= opts.maxWidth && - originalHeight <= opts.maxHeight && - inputBase64Size < opts.maxBytes + originalWidth <= MAX_IMAGE_WIDTH && + originalHeight <= MAX_IMAGE_HEIGHT && + inputBase64Size <= MAX_IMAGE_BASE64_BYTES ) { return { data: img.data, - mimeType: img.mimeType ?? `image/${format}`, + mimeType: img.mimeType, originalWidth, originalHeight, width: originalWidth, @@ -178,39 +138,35 @@ export async function resizeImage( }; } - const qualitySteps = Array.from(new Set([opts.jpegQuality, 85, 70, 55, 40, 35])); + const qualitySteps = [JPEG_QUALITY, 85, 70, 55, 40, 35]; const output = await processor.encode(inputBuffer, { format: "auto", limits: { - maxWidth: opts.maxWidth, - maxHeight: opts.maxHeight, + maxWidth: MAX_IMAGE_WIDTH, + maxHeight: MAX_IMAGE_HEIGHT, }, - maxBytes: maxBinaryBytesForBase64Budget(opts.maxBytes), - opaque: { format: "jpeg", quality: opts.jpegQuality }, + maxBase64Bytes: MAX_IMAGE_BASE64_BYTES, + opaque: { format: "jpeg", quality: JPEG_QUALITY }, transparent: { format: "png" }, search: { quality: qualitySteps, compressionLevel: [6, 9], }, }); - const candidate = encodeCandidate(output.data, output.mimeType); - if (candidate.encodedSize > opts.maxBytes || output.withinBudget === false) { + if (output.withinBudget !== true) { return null; } return { - data: candidate.data, - mimeType: candidate.mimeType, + data: output.data.toString("base64"), + mimeType: output.mimeType, originalWidth, originalHeight, width: output.width, height: output.height, - wasResized: !output.data.equals(inputBuffer), + wasResized: output.resized, }; - } catch (error) { - if (isImageProcessorUnavailableError(error)) { - return null; - } + } catch { return null; } } @@ -219,7 +175,7 @@ export async function resizeImage( * Format a dimension note for resized images. * This helps the model understand the coordinate mapping. */ -export function formatDimensionNote(result: ResizedImage): string | undefined { +function formatDimensionNote(result: ResizedImage): string | undefined { if (!result.wasResized) { return undefined; }