mirror of
https://github.com/openclaw/openclaw.git
synced 2026-05-06 05:50:43 +00:00
fix: bound default media response reads
This commit is contained in:
@@ -11,9 +11,11 @@ vi.mock("../infra/net/fetch-guard.js", () => ({
|
||||
}),
|
||||
}));
|
||||
|
||||
type FetchRemoteMedia = typeof import("./fetch.js").fetchRemoteMedia;
|
||||
type FetchModule = typeof import("./fetch.js");
|
||||
type FetchRemoteMedia = FetchModule["fetchRemoteMedia"];
|
||||
type LookupFn = NonNullable<Parameters<FetchRemoteMedia>[0]["lookupFn"]>;
|
||||
let fetchRemoteMedia: FetchRemoteMedia;
|
||||
let defaultFetchMediaMaxBytes: number;
|
||||
|
||||
function makeStream(chunks: Uint8Array[]) {
|
||||
return new ReadableStream<Uint8Array>({
|
||||
@@ -177,7 +179,9 @@ describe("fetchRemoteMedia", () => {
|
||||
const botFileUrl = `https://files.example.test/file/bot${botToken}/photos/1.jpg`;
|
||||
|
||||
beforeAll(async () => {
|
||||
({ fetchRemoteMedia } = await import("./fetch.js"));
|
||||
const fetchModule = await import("./fetch.js");
|
||||
fetchRemoteMedia = fetchModule.fetchRemoteMedia;
|
||||
defaultFetchMediaMaxBytes = fetchModule.DEFAULT_FETCH_MEDIA_MAX_BYTES;
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
@@ -223,6 +227,24 @@ describe("fetchRemoteMedia", () => {
|
||||
await expectRemoteMediaMaxBytesError({ fetchImpl, maxBytes: 4 });
|
||||
});
|
||||
|
||||
it("applies a default stream limit when maxBytes is omitted", async () => {
|
||||
const fetchImpl = vi.fn(
|
||||
async () =>
|
||||
new Response(makeStream([new Uint8Array([1])]), {
|
||||
status: 200,
|
||||
headers: { "content-length": String(defaultFetchMediaMaxBytes + 1) },
|
||||
}),
|
||||
);
|
||||
|
||||
await expect(
|
||||
fetchRemoteMedia({
|
||||
url: "https://example.com/file.bin",
|
||||
fetchImpl,
|
||||
lookupFn: makeLookupFn(),
|
||||
}),
|
||||
).rejects.toThrow(`exceeds maxBytes ${defaultFetchMediaMaxBytes}`);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{
|
||||
name: "redacts bot tokens from fetch failure messages",
|
||||
|
||||
@@ -7,9 +7,12 @@ import {
|
||||
} from "../infra/net/fetch-guard.js";
|
||||
import type { LookupFn, PinnedDispatcherPolicy, SsrFPolicy } from "../infra/net/ssrf.js";
|
||||
import { redactSensitiveText } from "../logging/redact.js";
|
||||
import { MAX_DOCUMENT_BYTES } from "./constants.js";
|
||||
import { detectMime, extensionForMime } from "./mime.js";
|
||||
import { readResponseTextSnippet, readResponseWithLimit } from "./read-response-with-limit.js";
|
||||
|
||||
export const DEFAULT_FETCH_MEDIA_MAX_BYTES = MAX_DOCUMENT_BYTES;
|
||||
|
||||
type FetchMediaResult = {
|
||||
buffer: Buffer;
|
||||
contentType?: string;
|
||||
@@ -209,29 +212,28 @@ export async function fetchRemoteMedia(options: FetchMediaOptions): Promise<Fetc
|
||||
);
|
||||
}
|
||||
|
||||
const effectiveMaxBytes = maxBytes ?? DEFAULT_FETCH_MEDIA_MAX_BYTES;
|
||||
const contentLength = res.headers.get("content-length");
|
||||
if (maxBytes && contentLength) {
|
||||
if (contentLength) {
|
||||
const length = Number(contentLength);
|
||||
if (Number.isFinite(length) && length > maxBytes) {
|
||||
if (Number.isFinite(length) && length > effectiveMaxBytes) {
|
||||
throw new MediaFetchError(
|
||||
"max_bytes",
|
||||
`Failed to fetch media from ${sourceUrl}: content length ${length} exceeds maxBytes ${maxBytes}`,
|
||||
`Failed to fetch media from ${sourceUrl}: content length ${length} exceeds maxBytes ${effectiveMaxBytes}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let buffer: Buffer;
|
||||
try {
|
||||
buffer = maxBytes
|
||||
? await readResponseWithLimit(res, maxBytes, {
|
||||
onOverflow: ({ maxBytes, res }) =>
|
||||
new MediaFetchError(
|
||||
"max_bytes",
|
||||
`Failed to fetch media from ${redactMediaUrl(res.url || url)}: payload exceeds maxBytes ${maxBytes}`,
|
||||
),
|
||||
chunkTimeoutMs: readIdleTimeoutMs,
|
||||
})
|
||||
: Buffer.from(await res.arrayBuffer());
|
||||
buffer = await readResponseWithLimit(res, effectiveMaxBytes, {
|
||||
onOverflow: ({ maxBytes, res }) =>
|
||||
new MediaFetchError(
|
||||
"max_bytes",
|
||||
`Failed to fetch media from ${redactMediaUrl(res.url || url)}: payload exceeds maxBytes ${maxBytes}`,
|
||||
),
|
||||
chunkTimeoutMs: readIdleTimeoutMs,
|
||||
});
|
||||
} catch (err) {
|
||||
if (err instanceof MediaFetchError) {
|
||||
throw err;
|
||||
|
||||
Reference in New Issue
Block a user