vultr/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-07-26 00:01:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor(agents): simplify image resize policy (#107027)

Browse Source
This commit is contained in:
Vincent Koc
2026-07-14 10:46:01 +08:00
committed by GitHub
parent 9b823c6108
commit d4ee32185c
3 changed files with 102 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
scripts/deadcode-exports.baseline.mjs
View File

@@ -753,8 +753,6 @@ export const KNIP_UNUSED_EXPORT_BASELINE = [
"src/agents/tools/web-fetch.ts: sanitizeWebFetchUrl",
"src/agents/tools/web-fetch.ts: WEB_FETCH_SPILL_MAX_CHARS",
"src/agents/utility-model.ts: resolveProviderDefaultUtilityModelRef",
"src/agents/utils/image-resize.ts: formatDimensionNote",
"src/agents/utils/image-resize.ts: resizeImage",
"src/agents/utils/tools-manager.ts: getToolPath",
"src/agents/utils/tools-manager.ts: testing",
"src/agents/workspace-templates.ts: resetWorkspaceTemplateDirCache",

115
src/agents/utils/image-resize.test.ts
View File

@@ -3,34 +3,32 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
const mocks = vi.hoisted(() => ({
convertImageToPng: vi.fn(),
encode: vi.fn(),
probe: vi.fn(),
}));
vi.mock("../../media/image-ops.js", () => ({
convertImageToPng: mocks.convertImageToPng,
createImageProcessor: () => ({
encode: mocks.encode,
probe: mocks.probe,
}),
isImageProcessorUnavailableError: (error: unknown) =>
typeof error === "object" &&
error !== null &&
"code" in error &&
error.code === "IMAGE_PROCESSOR_UNAVAILABLE",
}));
import { formatDimensionNote, resizeImage } from "./image-resize.js";
import { processImage } from "./image-resize.js";
describe("image resize utility", () => {
beforeEach(() => {
mocks.convertImageToPng.mockReset();
mocks.encode.mockReset();
mocks.probe.mockReset();
});
it("keeps images that already fit the inline limits", async () => {
const input = Buffer.from("small image").toString("base64");
it("keeps images that exactly fit the inline limits", async () => {
const input = "a".repeat(4.5 * 1024 * 1024);
mocks.probe.mockResolvedValue({
bytes: 11,
bytes: Buffer.byteLength(input, "base64"),
format: "png",
hasAlpha: false,
height: 20,
@@ -38,19 +36,15 @@ describe("image resize utility", () => {
width: 10,
});
const resized = await resizeImage(
const result = await processImage(
{ type: "image", data: input, mimeType: "image/png" },
{ maxWidth: 100, maxHeight: 100, maxBytes: 1_000 },
{ autoResizeImages: true },
);
expect(resized).toMatchObject({
data: input,
height: 20,
mimeType: "image/png",
originalHeight: 20,
originalWidth: 10,
wasResized: false,
width: 10,
expect(result).toStrictEqual({
ok: true,
image: { type: "image", data: input, mimeType: "image/png" },
hints: [],
});
expect(mocks.encode).not.toHaveBeenCalled();
});
@@ -82,9 +76,9 @@ describe("image resize utility", () => {
withinBudget: true,
});
const resized = await resizeImage(
const result = await processImage(
{ type: "image", data: inputBuffer.toString("base64"), mimeType: "image/jpeg" },
{ maxWidth: 2_000, maxHeight: 2_000, maxBytes: 4_000, jpegQuality: 70 },
{ autoResizeImages: true },
);
expect(mocks.encode).toHaveBeenCalledWith(inputBuffer, {
@@ -93,29 +87,28 @@ describe("image resize utility", () => {
maxHeight: 2_000,
maxWidth: 2_000,
},
maxBytes: 3_000,
opaque: { format: "jpeg", quality: 70 },
maxBase64Bytes: 4.5 * 1024 * 1024,
opaque: { format: "jpeg", quality: 80 },
search: {
compressionLevel: [6, 9],
quality: [70, 85, 55, 40, 35],
quality: [80, 85, 70, 55, 40, 35],
},
transparent: { format: "png" },
});
expect(resized).toMatchObject({
data: outputBuffer.toString("base64"),
height: 1600,
mimeType: "image/jpeg",
originalHeight: 3000,
originalWidth: 1200,
wasResized: true,
width: 640,
expect(result).toStrictEqual({
ok: true,
image: {
type: "image",
data: outputBuffer.toString("base64"),
mimeType: "image/jpeg",
},
hints: [
"[Image: original 1200x3000, displayed at 640x1600. Multiply coordinates by 1.88 to map to original image.]",
],
});
expect(formatDimensionNote(resized!)).toBe(
"[Image: original 1200x3000, displayed at 640x1600. Multiply coordinates by 1.88 to map to original image.]",
);
});
it("returns null when Rastermill cannot satisfy the base64 budget", async () => {
it("omits images when Rastermill cannot satisfy the base64 budget", async () => {
const inputBuffer = Buffer.from("too large");
mocks.probe.mockResolvedValue({
bytes: inputBuffer.byteLength,
@@ -140,10 +133,54 @@ describe("image resize utility", () => {
});
await expect(
resizeImage(
processImage(
{ type: "image", data: inputBuffer.toString("base64"), mimeType: "image/png" },
{ maxWidth: 100, maxHeight: 100, maxBytes: 50 },
{ autoResizeImages: true },
),
).resolves.toBeNull();
).resolves.toStrictEqual({
ok: false,
message: "[Image omitted: could not be resized below the inline image size limit.]",
});
});
it("does not add coordinate hints when Rastermill only re-encodes the image", async () => {
const input = "a".repeat(4.5 * 1024 * 1024 + 1);
const outputBuffer = Buffer.from("re-encoded");
mocks.probe.mockResolvedValue({
bytes: Buffer.byteLength(input, "base64"),
format: "png",
hasAlpha: false,
height: 20,
orientation: null,
width: 10,
});
mocks.encode.mockResolvedValue({
base64Bytes: Buffer.byteLength(outputBuffer.toString("base64"), "utf8"),
bytes: outputBuffer.byteLength,
chosen: { format: "jpeg", quality: 80 },
data: outputBuffer,
format: "jpeg",
height: 20,
metadata: "stripped",
mimeType: "image/jpeg",
resized: false,
width: 10,
withinBudget: true,
});
await expect(
processImage(
{ type: "image", data: input, mimeType: "image/png" },
{ autoResizeImages: true },
),
).resolves.toStrictEqual({
ok: true,
image: {
type: "image",
data: outputBuffer.toString("base64"),
mimeType: "image/jpeg",
},
hints: [],
});
});
});

96
src/agents/utils/image-resize.ts
View File

@@ -4,19 +4,7 @@
* Downscales base64 image content for provider payload limits using the configured image processor.
*/
import type { ImageContent } from "../../llm/types.js";
import {
convertImageToPng,
createImageProcessor,
isImageProcessorUnavailableError,
type ImageProbe,
} from "../../media/image-ops.js";
interface ImageResizeOptions {
maxWidth?: number; // Default: 2000
maxHeight?: number; // Default: 2000
maxBytes?: number; // Default: 4.5MB of base64 payload (below Anthropic's 5MB limit)
jpegQuality?: number; // Default: 80
}
import { convertImageToPng, createImageProcessor, type ImageProbe } from "../../media/image-ops.js";
interface ResizedImage {
data: string; // base64
@@ -96,34 +84,11 @@ export async function processImage(
};
}
// 4.5MB of base64 payload. Provides headroom below Anthropic's 5MB limit.
const DEFAULT_MAX_BYTES = 4.5 * 1024 * 1024;
const DEFAULT_OPTIONS: Required<ImageResizeOptions> = {
maxWidth: 2000,
maxHeight: 2000,
maxBytes: DEFAULT_MAX_BYTES,
jpegQuality: 80,
};
function maxBinaryBytesForBase64Budget(maxBase64Bytes: number): number {
return Math.floor(maxBase64Bytes / 4) * 3;
}
interface EncodedCandidate {
data: string;
encodedSize: number;
mimeType: string;
}
function encodeCandidate(buffer: Buffer, mimeType: string): EncodedCandidate {
const data = buffer.toString("base64");
return {
data,
encodedSize: Buffer.byteLength(data, "utf-8"),
mimeType,
};
}
const MAX_IMAGE_WIDTH = 2000;
const MAX_IMAGE_HEIGHT = 2000;
// 4.5MB of base64 payload leaves headroom below Anthropic's 5MB limit.
const MAX_IMAGE_BASE64_BYTES = 4.5 * 1024 * 1024;
const JPEG_QUALITY = 80;
function orientedDimensions(probe: ImageProbe): { width: number; height: number } {
return probe.orientation && probe.orientation >= 5 && probe.orientation <= 8
@@ -132,23 +97,19 @@ function orientedDimensions(probe: ImageProbe): { width: number; height: number
}
/**
* Resize an image to fit within the specified max dimensions and encoded file size.
* Returns null if the image cannot be resized below maxBytes.
* Resize an image to fit within the inline dimensions and base64 payload limit.
* Returns null if Rastermill cannot produce output within those limits.
*
* Uses Rastermill for image processing. If no Rastermill backend is available,
* returns null.
*
* Strategy for staying under maxBytes:
* 1. First resize to maxWidth/maxHeight
* Strategy for staying under the inline limits:
* 1. First resize to the maximum dimensions
* 2. Let Rastermill choose JPEG or PNG for the image transparency profile
* 3. If still too large, search decreasing quality/compression settings
* 4. If still too large, progressively reduce dimensions
*/
export async function resizeImage(
img: ImageContent,
options?: ImageResizeOptions,
): Promise<ResizedImage | null> {
const opts = { ...DEFAULT_OPTIONS, ...options };
async function resizeImage(img: ImageContent): Promise<ResizedImage | null> {
const inputBuffer = Buffer.from(img.data, "base64");
const inputBase64Size = Buffer.byteLength(img.data, "utf-8");
const processor = createImageProcessor();
@@ -159,17 +120,16 @@ export async function resizeImage(
return null;
}
const { width: originalWidth, height: originalHeight } = orientedDimensions(probe);
const format = img.mimeType?.split("/")[1] ?? "png";
// Check if already within all limits (dimensions AND encoded size)
if (
originalWidth <= opts.maxWidth &&
originalHeight <= opts.maxHeight &&
inputBase64Size < opts.maxBytes
originalWidth <= MAX_IMAGE_WIDTH &&
originalHeight <= MAX_IMAGE_HEIGHT &&
inputBase64Size <= MAX_IMAGE_BASE64_BYTES
) {
return {
data: img.data,
mimeType: img.mimeType ?? `image/${format}`,
mimeType: img.mimeType,
originalWidth,
originalHeight,
width: originalWidth,
@@ -178,39 +138,35 @@ export async function resizeImage(
};
}
const qualitySteps = Array.from(new Set([opts.jpegQuality, 85, 70, 55, 40, 35]));
const qualitySteps = [JPEG_QUALITY, 85, 70, 55, 40, 35];
const output = await processor.encode(inputBuffer, {
format: "auto",
limits: {
maxWidth: opts.maxWidth,
maxHeight: opts.maxHeight,
maxWidth: MAX_IMAGE_WIDTH,
maxHeight: MAX_IMAGE_HEIGHT,
},
maxBytes: maxBinaryBytesForBase64Budget(opts.maxBytes),
opaque: { format: "jpeg", quality: opts.jpegQuality },
maxBase64Bytes: MAX_IMAGE_BASE64_BYTES,
opaque: { format: "jpeg", quality: JPEG_QUALITY },
transparent: { format: "png" },
search: {
quality: qualitySteps,
compressionLevel: [6, 9],
},
});
const candidate = encodeCandidate(output.data, output.mimeType);
if (candidate.encodedSize > opts.maxBytes || output.withinBudget === false) {
if (output.withinBudget !== true) {
return null;
}
return {
data: candidate.data,
mimeType: candidate.mimeType,
data: output.data.toString("base64"),
mimeType: output.mimeType,
originalWidth,
originalHeight,
width: output.width,
height: output.height,
wasResized: !output.data.equals(inputBuffer),
wasResized: output.resized,
};
} catch (error) {
if (isImageProcessorUnavailableError(error)) {
return null;
}
} catch {
return null;
}
}
@@ -219,7 +175,7 @@ export async function resizeImage(
* Format a dimension note for resized images.
* This helps the model understand the coordinate mapping.
*/
export function formatDimensionNote(result: ResizedImage): string | undefined {
function formatDimensionNote(result: ResizedImage): string | undefined {
if (!result.wasResized) {
return undefined;
}