fix(infra): bound ClawHub fetchJson and error response bodies

ClawHub is an external marketplace (untrusted source); fetchJson read the
success body via response.json() and readErrorBody read the error body via
response.text(), both without a byte cap, so a hostile or malfunctioning host
could exhaust memory with an unbounded response. Read both through the existing
read-response-with-limit helpers (16 MiB cap for JSON, 8 KiB / 400 chars for the
error snippet), cancelling the stream on overflow/idle. Symmetric counterpart to
the Anthropic error-stream hardening in #95108.
This commit is contained in:
Alix-007
2026-06-20 12:39:29 +08:00
committed by Peter Steinberger
parent 3a93d7fd68
commit 48853df18c
2 changed files with 83 additions and 4 deletions

View File

@@ -801,6 +801,59 @@ describe("clawhub helpers", () => {
).rejects.toThrow("ClawHub /api/v1/search returned malformed JSON");
});
it("bounds oversized successful ClawHub JSON responses and cancels the stream", async () => {
const cancel = vi.fn();
const chunk = new Uint8Array(512 * 1024).fill("x".charCodeAt(0));
const overshootChunks = 34; // 34 * 512 KiB = 17 MiB > 16 MiB cap
let emitted = 0;
const body = new ReadableStream<Uint8Array>({
pull(controller) {
if (emitted >= overshootChunks) {
controller.close();
return;
}
emitted += 1;
controller.enqueue(chunk);
},
cancel() {
cancel();
},
});
await expect(
searchClawHubSkills({
query: "calendar",
fetchImpl: async () =>
new Response(body, {
status: 200,
headers: { "content-type": "application/json" },
}),
}),
).rejects.toThrow(/ClawHub \/api\/v1\/search response exceeded 16777216 bytes/);
// The reader is cancelled at the cap so the oversized stream releases its
// socket/buffer instead of being drained into memory.
expect(cancel).toHaveBeenCalledTimes(1);
});
it("bounds oversized ClawHub error bodies to a short collapsed snippet", async () => {
const oversized = "boom ".repeat(64 * 1024); // ~320 KiB error body
let error: unknown;
try {
await searchClawHubSkills({
query: "calendar",
fetchImpl: async () => new Response(oversized, { status: 500 }),
});
} catch (caught) {
error = caught;
}
expect(error).toBeInstanceOf(Error);
const message = (error as Error).message;
expect(message.startsWith("ClawHub /api/v1/search failed (500): ")).toBe(true);
expect(message.endsWith("…")).toBe(true);
// prefix + 400-char snippet + "…" stays far below the raw ~320 KiB body.
expect(message.length).toBeLessThanOrEqual(500);
});
it("annotates 429 errors with the reset hint but no sign-in hint when authenticated", async () => {
process.env.OPENCLAW_CLAWHUB_TOKEN = "env-token-123";
await expect(

View File

@@ -3,7 +3,10 @@ import { createHash } from "node:crypto";
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { readResponseWithLimit } from "@openclaw/media-core/read-response-with-limit";
import {
readResponseTextSnippet,
readResponseWithLimit,
} from "@openclaw/media-core/read-response-with-limit";
import { resolveTimerTimeoutMs } from "@openclaw/normalization-core/number-coercion";
import {
normalizeLowercaseStringOrEmpty,
@@ -20,6 +23,12 @@ const DEFAULT_CLAWHUB_URL = "https://clawhub.ai";
const DEFAULT_GITHUB_CODELOAD_URL = "https://codeload.github.com";
const DEFAULT_FETCH_TIMEOUT_MS = 30_000;
const SKILL_CARD_MAX_BYTES = 256 * 1024;
// ClawHub is an external marketplace (untrusted source): bound JSON and error
// bodies so a hostile or malfunctioning host cannot exhaust memory by streaming
// an unbounded response. Mirrors the error-stream hardening landed in #95108.
const CLAWHUB_JSON_MAX_BYTES = 16 * 1024 * 1024;
const CLAWHUB_ERROR_BODY_MAX_BYTES = 8 * 1024;
const CLAWHUB_ERROR_BODY_MAX_CHARS = 400;
export type ClawHubPackageFamily = "skill" | "code-plugin" | "bundle-plugin";
export type ClawHubPackageChannel = "official" | "community" | "private";
@@ -675,8 +684,12 @@ async function clawhubRequest(
async function readErrorBody(response: Response): Promise<string> {
try {
const text = (await response.text()).trim();
return text || response.statusText || `HTTP ${response.status}`;
const snippet = await readResponseTextSnippet(response, {
maxBytes: CLAWHUB_ERROR_BODY_MAX_BYTES,
maxChars: CLAWHUB_ERROR_BODY_MAX_CHARS,
chunkTimeoutMs: DEFAULT_FETCH_TIMEOUT_MS,
});
return snippet || response.statusText || `HTTP ${response.status}`;
} catch {
return response.statusText || `HTTP ${response.status}`;
}
@@ -720,8 +733,21 @@ async function fetchJson<T>(params: ClawHubRequestParams): Promise<T> {
if (!response.ok) {
throw await buildClawHubError(response, url, hasToken);
}
return parseClawHubJsonBody<T>(response, url);
}
async function parseClawHubJsonBody<T>(response: Response, url: URL): Promise<T> {
const buffer = await readResponseWithLimit(response, CLAWHUB_JSON_MAX_BYTES, {
chunkTimeoutMs: DEFAULT_FETCH_TIMEOUT_MS,
onOverflow: ({ size, maxBytes }) =>
new Error(
`ClawHub ${url.pathname} response exceeded ${maxBytes} bytes (${size} bytes received)`,
),
onIdleTimeout: ({ chunkTimeoutMs }) =>
new Error(`ClawHub ${url.pathname} response stalled after ${chunkTimeoutMs}ms`),
});
try {
return (await response.json()) as T;
return JSON.parse(new TextDecoder().decode(buffer)) as T;
} catch (cause) {
throw new Error(`ClawHub ${url.pathname} returned malformed JSON`, { cause });
}