mirror of
https://github.com/openclaw/openclaw.git
synced 2026-06-28 13:13:39 +00:00
fix(infra): bound ClawHub fetchJson and error response bodies
ClawHub is an external marketplace (untrusted source); fetchJson read the success body via response.json() and readErrorBody read the error body via response.text(), both without a byte cap, so a hostile or malfunctioning host could exhaust memory with an unbounded response. Read both through the existing read-response-with-limit helpers (16 MiB cap for JSON, 8 KiB / 400 chars for the error snippet), cancelling the stream on overflow/idle. Symmetric counterpart to the Anthropic error-stream hardening in #95108.
This commit is contained in:
committed by
Peter Steinberger
parent
3a93d7fd68
commit
48853df18c
@@ -801,6 +801,59 @@ describe("clawhub helpers", () => {
|
||||
).rejects.toThrow("ClawHub /api/v1/search returned malformed JSON");
|
||||
});
|
||||
|
||||
it("bounds oversized successful ClawHub JSON responses and cancels the stream", async () => {
|
||||
const cancel = vi.fn();
|
||||
const chunk = new Uint8Array(512 * 1024).fill("x".charCodeAt(0));
|
||||
const overshootChunks = 34; // 34 * 512 KiB = 17 MiB > 16 MiB cap
|
||||
let emitted = 0;
|
||||
const body = new ReadableStream<Uint8Array>({
|
||||
pull(controller) {
|
||||
if (emitted >= overshootChunks) {
|
||||
controller.close();
|
||||
return;
|
||||
}
|
||||
emitted += 1;
|
||||
controller.enqueue(chunk);
|
||||
},
|
||||
cancel() {
|
||||
cancel();
|
||||
},
|
||||
});
|
||||
|
||||
await expect(
|
||||
searchClawHubSkills({
|
||||
query: "calendar",
|
||||
fetchImpl: async () =>
|
||||
new Response(body, {
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" },
|
||||
}),
|
||||
}),
|
||||
).rejects.toThrow(/ClawHub \/api\/v1\/search response exceeded 16777216 bytes/);
|
||||
// The reader is cancelled at the cap so the oversized stream releases its
|
||||
// socket/buffer instead of being drained into memory.
|
||||
expect(cancel).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("bounds oversized ClawHub error bodies to a short collapsed snippet", async () => {
|
||||
const oversized = "boom ".repeat(64 * 1024); // ~320 KiB error body
|
||||
let error: unknown;
|
||||
try {
|
||||
await searchClawHubSkills({
|
||||
query: "calendar",
|
||||
fetchImpl: async () => new Response(oversized, { status: 500 }),
|
||||
});
|
||||
} catch (caught) {
|
||||
error = caught;
|
||||
}
|
||||
expect(error).toBeInstanceOf(Error);
|
||||
const message = (error as Error).message;
|
||||
expect(message.startsWith("ClawHub /api/v1/search failed (500): ")).toBe(true);
|
||||
expect(message.endsWith("…")).toBe(true);
|
||||
// prefix + 400-char snippet + "…" stays far below the raw ~320 KiB body.
|
||||
expect(message.length).toBeLessThanOrEqual(500);
|
||||
});
|
||||
|
||||
it("annotates 429 errors with the reset hint but no sign-in hint when authenticated", async () => {
|
||||
process.env.OPENCLAW_CLAWHUB_TOKEN = "env-token-123";
|
||||
await expect(
|
||||
|
||||
@@ -3,7 +3,10 @@ import { createHash } from "node:crypto";
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { readResponseWithLimit } from "@openclaw/media-core/read-response-with-limit";
|
||||
import {
|
||||
readResponseTextSnippet,
|
||||
readResponseWithLimit,
|
||||
} from "@openclaw/media-core/read-response-with-limit";
|
||||
import { resolveTimerTimeoutMs } from "@openclaw/normalization-core/number-coercion";
|
||||
import {
|
||||
normalizeLowercaseStringOrEmpty,
|
||||
@@ -20,6 +23,12 @@ const DEFAULT_CLAWHUB_URL = "https://clawhub.ai";
|
||||
const DEFAULT_GITHUB_CODELOAD_URL = "https://codeload.github.com";
|
||||
const DEFAULT_FETCH_TIMEOUT_MS = 30_000;
|
||||
const SKILL_CARD_MAX_BYTES = 256 * 1024;
|
||||
// ClawHub is an external marketplace (untrusted source): bound JSON and error
|
||||
// bodies so a hostile or malfunctioning host cannot exhaust memory by streaming
|
||||
// an unbounded response. Mirrors the error-stream hardening landed in #95108.
|
||||
const CLAWHUB_JSON_MAX_BYTES = 16 * 1024 * 1024;
|
||||
const CLAWHUB_ERROR_BODY_MAX_BYTES = 8 * 1024;
|
||||
const CLAWHUB_ERROR_BODY_MAX_CHARS = 400;
|
||||
|
||||
export type ClawHubPackageFamily = "skill" | "code-plugin" | "bundle-plugin";
|
||||
export type ClawHubPackageChannel = "official" | "community" | "private";
|
||||
@@ -675,8 +684,12 @@ async function clawhubRequest(
|
||||
|
||||
async function readErrorBody(response: Response): Promise<string> {
|
||||
try {
|
||||
const text = (await response.text()).trim();
|
||||
return text || response.statusText || `HTTP ${response.status}`;
|
||||
const snippet = await readResponseTextSnippet(response, {
|
||||
maxBytes: CLAWHUB_ERROR_BODY_MAX_BYTES,
|
||||
maxChars: CLAWHUB_ERROR_BODY_MAX_CHARS,
|
||||
chunkTimeoutMs: DEFAULT_FETCH_TIMEOUT_MS,
|
||||
});
|
||||
return snippet || response.statusText || `HTTP ${response.status}`;
|
||||
} catch {
|
||||
return response.statusText || `HTTP ${response.status}`;
|
||||
}
|
||||
@@ -720,8 +733,21 @@ async function fetchJson<T>(params: ClawHubRequestParams): Promise<T> {
|
||||
if (!response.ok) {
|
||||
throw await buildClawHubError(response, url, hasToken);
|
||||
}
|
||||
return parseClawHubJsonBody<T>(response, url);
|
||||
}
|
||||
|
||||
async function parseClawHubJsonBody<T>(response: Response, url: URL): Promise<T> {
|
||||
const buffer = await readResponseWithLimit(response, CLAWHUB_JSON_MAX_BYTES, {
|
||||
chunkTimeoutMs: DEFAULT_FETCH_TIMEOUT_MS,
|
||||
onOverflow: ({ size, maxBytes }) =>
|
||||
new Error(
|
||||
`ClawHub ${url.pathname} response exceeded ${maxBytes} bytes (${size} bytes received)`,
|
||||
),
|
||||
onIdleTimeout: ({ chunkTimeoutMs }) =>
|
||||
new Error(`ClawHub ${url.pathname} response stalled after ${chunkTimeoutMs}ms`),
|
||||
});
|
||||
try {
|
||||
return (await response.json()) as T;
|
||||
return JSON.parse(new TextDecoder().decode(buffer)) as T;
|
||||
} catch (cause) {
|
||||
throw new Error(`ClawHub ${url.pathname} returned malformed JSON`, { cause });
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user