mirror of
https://github.com/openclaw/openclaw.git
synced 2026-06-12 06:22:57 +00:00
fix: harden qqbot direct media uploads
Harden QQBot direct media URL uploads by downloading through the local SSRF guard before QQ upload, disabling redirects, bounding fetch/setup and body reads, and routing downloaded buffers through the existing one-shot/chunked size gate. Co-authored-by: Agustin Rivera <agustin@rivera-web.com>
This commit is contained in:
18
.github/workflows/ci.yml
vendored
18
.github/workflows/ci.yml
vendored
@@ -585,6 +585,20 @@ jobs:
|
||||
pids+=("$!")
|
||||
}
|
||||
|
||||
if [ "$RUN_GATEWAY_WATCH" = "true" ]; then
|
||||
gateway_watch_log="${RUNNER_TEMP}/gateway-watch.log"
|
||||
echo "starting gateway-watch: node scripts/check-gateway-watch-regression.mjs --skip-build --ready-timeout-ms 5000"
|
||||
if node scripts/check-gateway-watch-regression.mjs --skip-build --ready-timeout-ms 5000 >"$gateway_watch_log" 2>&1; then
|
||||
result="success"
|
||||
else
|
||||
result="failure"
|
||||
fi
|
||||
echo "::group::gateway-watch log"
|
||||
cat "$gateway_watch_log"
|
||||
echo "::endgroup::"
|
||||
results["gateway-watch"]="$result"
|
||||
fi
|
||||
|
||||
if [ "$RUN_CHANNELS" = "true" ]; then
|
||||
start_check "channels" env \
|
||||
NODE_OPTIONS=--max-old-space-size=8192 \
|
||||
@@ -599,10 +613,6 @@ jobs:
|
||||
node scripts/run-vitest.mjs run --config test/vitest/vitest.full-core-support-boundary.config.ts
|
||||
fi
|
||||
|
||||
if [ "$RUN_GATEWAY_WATCH" = "true" ]; then
|
||||
start_check "gateway-watch" node scripts/check-gateway-watch-regression.mjs --skip-build --ready-timeout-ms 5000
|
||||
fi
|
||||
|
||||
for index in "${!pids[@]}"; do
|
||||
name="${names[$index]}"
|
||||
log="${logs[$index]}"
|
||||
|
||||
@@ -108,7 +108,7 @@ vi.mock("./reply-delivery.js", () => ({
|
||||
}));
|
||||
|
||||
type DispatchInboundParams = {
|
||||
ctx?: unknown;
|
||||
ctx?: Record<string, unknown>;
|
||||
dispatcher: {
|
||||
sendBlockReply: (payload: ReplyPayload) => boolean | Promise<boolean>;
|
||||
sendFinalReply: (payload: ReplyPayload) => boolean | Promise<boolean>;
|
||||
@@ -240,7 +240,7 @@ vi.mock("openclaw/plugin-sdk/reply-runtime", () => ({
|
||||
onSettled?: () => unknown;
|
||||
onFreshSettledDelivery?: () => unknown;
|
||||
};
|
||||
ctx?: unknown;
|
||||
ctx?: Record<string, unknown>;
|
||||
replyOptions?: DispatchInboundParams["replyOptions"];
|
||||
}) => {
|
||||
const pendingDeliveries: Promise<void>[] = [];
|
||||
|
||||
@@ -250,17 +250,10 @@ describe("media-chunked: ChunkedMediaApi.uploadChunked", () => {
|
||||
]),
|
||||
);
|
||||
|
||||
// Cache populated with the complete result.
|
||||
const expectedMd5 = crypto.createHash("md5").update(FIXTURE_BUFFER).digest("hex");
|
||||
expect(cache.setSpy).toHaveBeenCalledWith(
|
||||
expectedMd5,
|
||||
"group",
|
||||
"g1",
|
||||
MediaFileType.FILE,
|
||||
"final-file-info",
|
||||
"uuid-final",
|
||||
3600,
|
||||
);
|
||||
// FILE uploads carry filename metadata in upload_prepare, so the content-only
|
||||
// cache is bypassed to avoid reusing file_info with a stale name.
|
||||
expect(cache.getSpy).not.toHaveBeenCalled();
|
||||
expect(cache.setSpy).not.toHaveBeenCalled();
|
||||
|
||||
// Progress callback hit 3 times with monotonically-increasing counts.
|
||||
expect(onProgress).toHaveBeenCalledTimes(3);
|
||||
|
||||
@@ -202,7 +202,8 @@ export class ChunkedMediaApi {
|
||||
|
||||
// 3. Upload-cache fast path: the md5 hash is already a strong content
|
||||
// identifier, so we can short-circuit before even calling upload_prepare.
|
||||
if (this.cache) {
|
||||
const canUseUploadCache = opts.fileType !== MediaFileType.FILE;
|
||||
if (this.cache && canUseUploadCache) {
|
||||
const cached = this.cache.get(hashes.md5, opts.scope, opts.targetId, opts.fileType);
|
||||
if (cached) {
|
||||
this.logger?.info?.(
|
||||
@@ -293,7 +294,7 @@ export class ChunkedMediaApi {
|
||||
this.logger?.info?.(`${prefix} completed: file_uuid=${result.file_uuid} ttl=${result.ttl}s`);
|
||||
|
||||
// 7. Populate the shared upload cache so subsequent sends skip re-uploading.
|
||||
if (this.cache && result.file_info && result.ttl > 0) {
|
||||
if (this.cache && canUseUploadCache && result.file_info && result.ttl > 0) {
|
||||
this.cache.set(
|
||||
hashes.md5,
|
||||
opts.scope,
|
||||
|
||||
448
extensions/qqbot/src/engine/api/media.test.ts
Normal file
448
extensions/qqbot/src/engine/api/media.test.ts
Normal file
@@ -0,0 +1,448 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { MediaFileType, type UploadMediaResponse } from "../types.js";
|
||||
import { MAX_UPLOAD_SIZE } from "../utils/file-utils.js";
|
||||
import { ApiClient } from "./api-client.js";
|
||||
import { MediaApi } from "./media.js";
|
||||
import { TokenManager } from "./token.js";
|
||||
|
||||
const fetchWithSsrFGuardMock = vi.hoisted(() => vi.fn());
|
||||
const readResponseWithLimitMock = vi.hoisted(() => vi.fn());
|
||||
|
||||
vi.mock("openclaw/plugin-sdk/response-limit-runtime", async (importOriginal) => {
|
||||
const actual =
|
||||
await importOriginal<typeof import("openclaw/plugin-sdk/response-limit-runtime")>();
|
||||
return {
|
||||
...actual,
|
||||
readResponseWithLimit: readResponseWithLimitMock,
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock("openclaw/plugin-sdk/ssrf-runtime", async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import("openclaw/plugin-sdk/ssrf-runtime")>();
|
||||
return {
|
||||
...actual,
|
||||
fetchWithSsrFGuard: fetchWithSsrFGuardMock,
|
||||
};
|
||||
});
|
||||
|
||||
const UPLOAD_RESPONSE: UploadMediaResponse = {
|
||||
file_uuid: "uuid-1",
|
||||
file_info: "file-info-1",
|
||||
ttl: 3600,
|
||||
};
|
||||
|
||||
const MEDIA_BYTES = Buffer.from("downloaded-media");
|
||||
const MEDIA_BASE64 = MEDIA_BYTES.toString("base64");
|
||||
|
||||
function mockGuardedResponse(
|
||||
body: BodyInit = MEDIA_BYTES,
|
||||
init?: ResponseInit,
|
||||
): {
|
||||
release: ReturnType<typeof vi.fn>;
|
||||
} {
|
||||
const release = vi.fn(async () => {});
|
||||
fetchWithSsrFGuardMock.mockResolvedValueOnce({
|
||||
response: new Response(body, init),
|
||||
release,
|
||||
});
|
||||
return { release };
|
||||
}
|
||||
|
||||
function mockApiClient(): ApiClient {
|
||||
const client = new ApiClient();
|
||||
vi.spyOn(client, "request").mockResolvedValue(UPLOAD_RESPONSE);
|
||||
return client;
|
||||
}
|
||||
|
||||
function mockTokenManager(): TokenManager {
|
||||
const tokenManager = new TokenManager();
|
||||
vi.spyOn(tokenManager, "getAccessToken").mockResolvedValue("token-1");
|
||||
return tokenManager;
|
||||
}
|
||||
|
||||
function expectGuardedDownload(url: string): void {
|
||||
expect(fetchWithSsrFGuardMock).toHaveBeenCalledWith({
|
||||
url,
|
||||
maxRedirects: 0,
|
||||
signal: expect.any(AbortSignal),
|
||||
});
|
||||
expect(fetchWithSsrFGuardMock).not.toHaveBeenCalledWith(
|
||||
expect.objectContaining({ timeoutMs: expect.any(Number) }),
|
||||
);
|
||||
const signal = fetchWithSsrFGuardMock.mock.calls.at(-1)?.[0]?.signal;
|
||||
expect(signal).toBeInstanceOf(AbortSignal);
|
||||
}
|
||||
|
||||
describe("MediaApi.uploadMedia direct URL uploads", () => {
|
||||
beforeEach(() => {
|
||||
fetchWithSsrFGuardMock.mockReset();
|
||||
readResponseWithLimitMock.mockReset();
|
||||
readResponseWithLimitMock.mockResolvedValue(MEDIA_BYTES);
|
||||
mockGuardedResponse();
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ fileType: MediaFileType.IMAGE, url: "https://cdn.example.com/assets/photo.png" },
|
||||
{ fileType: MediaFileType.VIDEO, url: "http://cdn.example.com/assets/video.mp4" },
|
||||
{ fileType: MediaFileType.FILE, url: "http://cdn.example.com/assets/report.pdf" },
|
||||
])(
|
||||
"downloads public HTTP(S) $fileType URLs through the pinned SSRF guard",
|
||||
async ({ fileType, url }) => {
|
||||
const client = mockApiClient();
|
||||
const tokenManager = mockTokenManager();
|
||||
const api = new MediaApi(client, tokenManager);
|
||||
|
||||
const result = await api.uploadMedia(
|
||||
"c2c",
|
||||
"user-openid",
|
||||
fileType,
|
||||
{ appId: "app-id", clientSecret: "client-secret" },
|
||||
{ url },
|
||||
);
|
||||
|
||||
expect(result).toBe(UPLOAD_RESPONSE);
|
||||
expectGuardedDownload(url);
|
||||
expect(readResponseWithLimitMock).toHaveBeenCalledWith(
|
||||
expect.any(Response),
|
||||
MAX_UPLOAD_SIZE,
|
||||
{ chunkTimeoutMs: 10_000 },
|
||||
);
|
||||
expect(tokenManager.getAccessToken).toHaveBeenCalledWith("app-id", "client-secret");
|
||||
expect(client.request).toHaveBeenCalledWith(
|
||||
"token-1",
|
||||
"POST",
|
||||
expect.any(String),
|
||||
{
|
||||
file_type: fileType,
|
||||
srv_send_msg: false,
|
||||
file_data: MEDIA_BASE64,
|
||||
},
|
||||
{
|
||||
redactBodyKeys: ["file_data"],
|
||||
uploadRequest: true,
|
||||
},
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
it("releases the pinned SSRF dispatcher after downloading media", async () => {
|
||||
fetchWithSsrFGuardMock.mockReset();
|
||||
const { release } = mockGuardedResponse();
|
||||
const client = mockApiClient();
|
||||
const tokenManager = mockTokenManager();
|
||||
const api = new MediaApi(client, tokenManager);
|
||||
|
||||
await api.uploadMedia(
|
||||
"c2c",
|
||||
"user-openid",
|
||||
MediaFileType.IMAGE,
|
||||
{ appId: "app-id", clientSecret: "client-secret" },
|
||||
{ url: "https://cdn.example.com/assets/photo.png" },
|
||||
);
|
||||
|
||||
expect(release).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("bounds stalled guarded fetch setup before reading URL bodies", async () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
fetchWithSsrFGuardMock.mockReset();
|
||||
fetchWithSsrFGuardMock.mockImplementationOnce(() => new Promise(() => {}));
|
||||
const client = mockApiClient();
|
||||
const tokenManager = mockTokenManager();
|
||||
const api = new MediaApi(client, tokenManager);
|
||||
|
||||
const uploadPromise = api.uploadMedia(
|
||||
"c2c",
|
||||
"user-openid",
|
||||
MediaFileType.IMAGE,
|
||||
{ appId: "app-id", clientSecret: "client-secret" },
|
||||
{ url: "https://slow-dns.example.com/assets/photo.png" },
|
||||
);
|
||||
const rejection = expect(uploadPromise).rejects.toThrow(
|
||||
"Direct-upload media URL fetch timed out",
|
||||
);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(30_000);
|
||||
await rejection;
|
||||
expect(readResponseWithLimitMock).not.toHaveBeenCalled();
|
||||
expect(tokenManager.getAccessToken).not.toHaveBeenCalled();
|
||||
expect(client.request).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects URL bodies that keep trickling under the idle timeout", async () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
fetchWithSsrFGuardMock.mockReset();
|
||||
const { release } = mockGuardedResponse();
|
||||
readResponseWithLimitMock.mockReset();
|
||||
readResponseWithLimitMock.mockImplementationOnce(() => new Promise<Buffer>(() => {}));
|
||||
const client = mockApiClient();
|
||||
const tokenManager = mockTokenManager();
|
||||
const api = new MediaApi(client, tokenManager);
|
||||
|
||||
const uploadPromise = api.uploadMedia(
|
||||
"c2c",
|
||||
"user-openid",
|
||||
MediaFileType.IMAGE,
|
||||
{ appId: "app-id", clientSecret: "client-secret" },
|
||||
{ url: "https://cdn.example.com/assets/slow.bin" },
|
||||
);
|
||||
|
||||
for (let i = 0; i < 5 && readResponseWithLimitMock.mock.calls.length === 0; i += 1) {
|
||||
await Promise.resolve();
|
||||
}
|
||||
expect(readResponseWithLimitMock).toHaveBeenCalledOnce();
|
||||
|
||||
const rejection = expect(uploadPromise).rejects.toThrow(
|
||||
"Direct-upload media URL body timed out",
|
||||
);
|
||||
await vi.advanceTimersByTimeAsync(8 * 60_000);
|
||||
await rejection;
|
||||
expect(release).toHaveBeenCalledTimes(1);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it("dedupes downloaded URL media through the base64 upload cache", async () => {
|
||||
const cache = {
|
||||
computeHash: vi.fn(() => "hash-1"),
|
||||
get: vi.fn(() => "cached-file-info"),
|
||||
set: vi.fn(),
|
||||
};
|
||||
const client = mockApiClient();
|
||||
const tokenManager = mockTokenManager();
|
||||
const api = new MediaApi(client, tokenManager, { uploadCache: cache });
|
||||
|
||||
const result = await api.uploadMedia(
|
||||
"c2c",
|
||||
"user-openid",
|
||||
MediaFileType.IMAGE,
|
||||
{ appId: "app-id", clientSecret: "client-secret" },
|
||||
{ url: "https://cdn.example.com/assets/photo.png" },
|
||||
);
|
||||
|
||||
expect(result).toEqual({ file_uuid: "", file_info: "cached-file-info", ttl: 0 });
|
||||
expect(cache.computeHash).toHaveBeenCalledWith(MEDIA_BASE64);
|
||||
expect(cache.get).toHaveBeenCalledWith("hash-1", "c2c", "user-openid", MediaFileType.IMAGE);
|
||||
expect(tokenManager.getAccessToken).not.toHaveBeenCalled();
|
||||
expect(client.request).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does not reuse cached FILE uploads when the requested filename differs", async () => {
|
||||
const cache = {
|
||||
computeHash: vi.fn(() => "hash-1"),
|
||||
get: vi.fn(() => "cached-file-info"),
|
||||
set: vi.fn(),
|
||||
};
|
||||
const client = mockApiClient();
|
||||
const tokenManager = mockTokenManager();
|
||||
const api = new MediaApi(client, tokenManager, {
|
||||
uploadCache: cache,
|
||||
sanitizeFileName: (name) => `safe-${name}`,
|
||||
});
|
||||
|
||||
await api.uploadMedia(
|
||||
"c2c",
|
||||
"user-openid",
|
||||
MediaFileType.FILE,
|
||||
{ appId: "app-id", clientSecret: "client-secret" },
|
||||
{ url: "https://cdn.example.com/report.pdf", fileName: "report.pdf" },
|
||||
);
|
||||
|
||||
expect(cache.computeHash).not.toHaveBeenCalled();
|
||||
expect(cache.get).not.toHaveBeenCalled();
|
||||
expect(cache.set).not.toHaveBeenCalled();
|
||||
expect(client.request).toHaveBeenCalledWith(
|
||||
"token-1",
|
||||
"POST",
|
||||
expect.any(String),
|
||||
expect.objectContaining({
|
||||
file_data: MEDIA_BASE64,
|
||||
file_name: "safe-report.pdf",
|
||||
}),
|
||||
expect.any(Object),
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects invalid direct-upload URLs before downloading media or calling the QQ API", async () => {
|
||||
const client = mockApiClient();
|
||||
const tokenManager = mockTokenManager();
|
||||
const api = new MediaApi(client, tokenManager);
|
||||
|
||||
await expect(
|
||||
api.uploadMedia(
|
||||
"c2c",
|
||||
"user-openid",
|
||||
MediaFileType.IMAGE,
|
||||
{ appId: "app-id", clientSecret: "client-secret" },
|
||||
{ url: "not a url" },
|
||||
),
|
||||
).rejects.toThrow("Direct-upload media URL must be a valid URL");
|
||||
|
||||
expect(fetchWithSsrFGuardMock).not.toHaveBeenCalled();
|
||||
expect(tokenManager.getAccessToken).not.toHaveBeenCalled();
|
||||
expect(client.request).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("rejects non-HTTP direct-upload URLs before downloading media or calling the QQ API", async () => {
|
||||
const client = mockApiClient();
|
||||
const tokenManager = mockTokenManager();
|
||||
const api = new MediaApi(client, tokenManager);
|
||||
|
||||
await expect(
|
||||
api.uploadMedia(
|
||||
"c2c",
|
||||
"user-openid",
|
||||
MediaFileType.IMAGE,
|
||||
{ appId: "app-id", clientSecret: "client-secret" },
|
||||
{ url: "ftp://media.qq.com/assets/photo.png" },
|
||||
),
|
||||
).rejects.toThrow("Direct-upload media URL must use HTTP or HTTPS");
|
||||
|
||||
expect(fetchWithSsrFGuardMock).not.toHaveBeenCalled();
|
||||
expect(tokenManager.getAccessToken).not.toHaveBeenCalled();
|
||||
expect(client.request).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each(["127.0.0.1", "169.254.169.254", "10.0.0.1", "192.168.1.1"])(
|
||||
"does not upload direct URLs rejected by the SSRF guard: %s",
|
||||
async (host) => {
|
||||
fetchWithSsrFGuardMock.mockReset();
|
||||
const client = mockApiClient();
|
||||
const tokenManager = mockTokenManager();
|
||||
const api = new MediaApi(client, tokenManager);
|
||||
|
||||
await expect(
|
||||
api.uploadMedia(
|
||||
"group",
|
||||
"group-openid",
|
||||
MediaFileType.IMAGE,
|
||||
{ appId: "app-id", clientSecret: "client-secret" },
|
||||
{ url: `https://${host}/latest/meta-data/` },
|
||||
),
|
||||
).rejects.toThrow("Blocked hostname");
|
||||
|
||||
expect(fetchWithSsrFGuardMock).not.toHaveBeenCalled();
|
||||
expect(tokenManager.getAccessToken).not.toHaveBeenCalled();
|
||||
expect(client.request).not.toHaveBeenCalled();
|
||||
},
|
||||
);
|
||||
|
||||
it("does not forward URLs when the guarded download fails", async () => {
|
||||
fetchWithSsrFGuardMock.mockReset();
|
||||
fetchWithSsrFGuardMock.mockRejectedValueOnce(
|
||||
new Error("Blocked: resolves to private/internal/special-use IP address"),
|
||||
);
|
||||
const client = mockApiClient();
|
||||
const tokenManager = mockTokenManager();
|
||||
const api = new MediaApi(client, tokenManager);
|
||||
|
||||
await expect(
|
||||
api.uploadMedia(
|
||||
"group",
|
||||
"group-openid",
|
||||
MediaFileType.IMAGE,
|
||||
{ appId: "app-id", clientSecret: "client-secret" },
|
||||
{ url: "https://attacker.example/latest/meta-data/" },
|
||||
),
|
||||
).rejects.toThrow("resolves to private");
|
||||
|
||||
expect(tokenManager.getAccessToken).not.toHaveBeenCalled();
|
||||
expect(client.request).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("rejects literal RFC 2544 special-use URL hosts through the guarded download", async () => {
|
||||
fetchWithSsrFGuardMock.mockReset();
|
||||
const client = mockApiClient();
|
||||
const tokenManager = mockTokenManager();
|
||||
const api = new MediaApi(client, tokenManager);
|
||||
|
||||
await expect(
|
||||
api.uploadMedia(
|
||||
"c2c",
|
||||
"user-openid",
|
||||
MediaFileType.IMAGE,
|
||||
{ appId: "app-id", clientSecret: "client-secret" },
|
||||
{ url: "https://198.18.0.42/assets/photo.png" },
|
||||
),
|
||||
).rejects.toThrow("Blocked hostname");
|
||||
|
||||
expect(fetchWithSsrFGuardMock).not.toHaveBeenCalled();
|
||||
expect(tokenManager.getAccessToken).not.toHaveBeenCalled();
|
||||
expect(client.request).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("keeps public literal IP URLs on the default SSRF policy", async () => {
|
||||
const client = mockApiClient();
|
||||
const tokenManager = mockTokenManager();
|
||||
const api = new MediaApi(client, tokenManager);
|
||||
|
||||
await api.uploadMedia(
|
||||
"c2c",
|
||||
"user-openid",
|
||||
MediaFileType.IMAGE,
|
||||
{ appId: "app-id", clientSecret: "client-secret" },
|
||||
{ url: "http://93.184.216.34/assets/photo.png" },
|
||||
);
|
||||
|
||||
expectGuardedDownload("http://93.184.216.34/assets/photo.png");
|
||||
});
|
||||
|
||||
it("does not pass URL or fake-IP DNS policy to the QQ upload body", async () => {
|
||||
const client = mockApiClient();
|
||||
const tokenManager = mockTokenManager();
|
||||
const api = new MediaApi(client, tokenManager);
|
||||
|
||||
await api.uploadMedia(
|
||||
"c2c",
|
||||
"user-openid",
|
||||
MediaFileType.IMAGE,
|
||||
{ appId: "app-id", clientSecret: "client-secret" },
|
||||
{ url: "https://cdn.example.com/assets/photo.png" },
|
||||
);
|
||||
|
||||
expectGuardedDownload("https://cdn.example.com/assets/photo.png");
|
||||
expect(client.request).toHaveBeenCalledWith(
|
||||
"token-1",
|
||||
"POST",
|
||||
expect.any(String),
|
||||
expect.objectContaining({
|
||||
file_data: MEDIA_BASE64,
|
||||
}),
|
||||
expect.any(Object),
|
||||
);
|
||||
expect(client.request).not.toHaveBeenCalledWith(
|
||||
expect.any(String),
|
||||
expect.any(String),
|
||||
expect.any(String),
|
||||
expect.objectContaining({ url: expect.any(String) }),
|
||||
expect.any(Object),
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects HTTP errors from guarded direct-upload downloads before calling the QQ API", async () => {
|
||||
fetchWithSsrFGuardMock.mockReset();
|
||||
mockGuardedResponse("not found", { status: 404 });
|
||||
const client = mockApiClient();
|
||||
const tokenManager = mockTokenManager();
|
||||
const api = new MediaApi(client, tokenManager);
|
||||
|
||||
await expect(
|
||||
api.uploadMedia(
|
||||
"c2c",
|
||||
"user-openid",
|
||||
MediaFileType.IMAGE,
|
||||
{ appId: "app-id", clientSecret: "client-secret" },
|
||||
{ url: "https://cdn.example.com/missing.png" },
|
||||
),
|
||||
).rejects.toThrow("Direct-upload media URL returned HTTP 404");
|
||||
|
||||
expect(tokenManager.getAccessToken).not.toHaveBeenCalled();
|
||||
expect(client.request).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -12,6 +12,8 @@
|
||||
*/
|
||||
|
||||
import * as fs from "node:fs";
|
||||
import { readResponseWithLimit } from "openclaw/plugin-sdk/response-limit-runtime";
|
||||
import { fetchWithSsrFGuard, isBlockedHostnameOrIp } from "openclaw/plugin-sdk/ssrf-runtime";
|
||||
import {
|
||||
MediaFileType,
|
||||
type ChatScope,
|
||||
@@ -19,6 +21,7 @@ import {
|
||||
type MessageResponse,
|
||||
type EngineLogger,
|
||||
} from "../types.js";
|
||||
import { MAX_UPLOAD_SIZE } from "../utils/file-utils.js";
|
||||
import { ApiClient } from "./api-client.js";
|
||||
import { withRetry, UPLOAD_RETRY_POLICY } from "./retry.js";
|
||||
import { mediaUploadPath, messagePath, getNextMsgSeq } from "./routes.js";
|
||||
@@ -50,6 +53,122 @@ interface MediaApiConfig {
|
||||
sanitizeFileName?: SanitizeFileNameFn;
|
||||
}
|
||||
|
||||
const DIRECT_UPLOAD_DOWNLOAD_TIMEOUT_MS = 30_000;
|
||||
const DIRECT_UPLOAD_READ_IDLE_TIMEOUT_MS = 10_000;
|
||||
const DIRECT_UPLOAD_BODY_GRACE_TIMEOUT_MS = 30_000;
|
||||
const DIRECT_UPLOAD_MIN_DOWNLOAD_BYTES_PER_SECOND = 256 * 1024;
|
||||
const DIRECT_UPLOAD_MAX_BODY_TIMEOUT_MS = 8 * 60_000;
|
||||
|
||||
function assertDirectUploadDownloadHostAllowed(hostname: string): void {
|
||||
if (isBlockedHostnameOrIp(hostname)) {
|
||||
throw new Error("Blocked hostname or private/internal/special-use IP address");
|
||||
}
|
||||
}
|
||||
|
||||
async function fetchDirectUploadDownload(url: string) {
|
||||
const controller = new AbortController();
|
||||
const timeoutError = new Error("Direct-upload media URL fetch timed out");
|
||||
let timedOut = false;
|
||||
let timeout: ReturnType<typeof setTimeout> | undefined;
|
||||
const timeoutPromise = new Promise<never>((_, reject) => {
|
||||
timeout = setTimeout(() => {
|
||||
timedOut = true;
|
||||
controller.abort(timeoutError);
|
||||
reject(timeoutError);
|
||||
}, DIRECT_UPLOAD_DOWNLOAD_TIMEOUT_MS);
|
||||
unrefTimer(timeout);
|
||||
});
|
||||
const guardedFetch = fetchWithSsrFGuard({
|
||||
url,
|
||||
maxRedirects: 0,
|
||||
signal: controller.signal,
|
||||
});
|
||||
void guardedFetch.then(
|
||||
(result) => {
|
||||
if (timedOut) {
|
||||
void result.release().catch(() => undefined);
|
||||
}
|
||||
},
|
||||
() => undefined,
|
||||
);
|
||||
try {
|
||||
return await Promise.race([guardedFetch, timeoutPromise]);
|
||||
} finally {
|
||||
if (timeout) {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function unrefTimer(timeout: ReturnType<typeof setTimeout>): void {
|
||||
if (typeof timeout === "object" && "unref" in timeout) {
|
||||
(timeout as { unref: () => void }).unref();
|
||||
}
|
||||
}
|
||||
|
||||
function resolveDirectUploadBodyTimeoutMs(maxBytes: number): number {
|
||||
const transferTimeoutMs = Math.ceil(
|
||||
(maxBytes / DIRECT_UPLOAD_MIN_DOWNLOAD_BYTES_PER_SECOND) * 1000,
|
||||
);
|
||||
return Math.min(
|
||||
DIRECT_UPLOAD_BODY_GRACE_TIMEOUT_MS + transferTimeoutMs,
|
||||
DIRECT_UPLOAD_MAX_BODY_TIMEOUT_MS,
|
||||
);
|
||||
}
|
||||
|
||||
async function readDirectUploadResponse(response: Response, maxBytes: number): Promise<Buffer> {
|
||||
const timeoutMs = resolveDirectUploadBodyTimeoutMs(maxBytes);
|
||||
const timeoutError = new Error(`Direct-upload media URL body timed out after ${timeoutMs}ms`);
|
||||
let timeout: ReturnType<typeof setTimeout> | undefined;
|
||||
const timeoutPromise = new Promise<never>((_, reject) => {
|
||||
timeout = setTimeout(() => {
|
||||
void response.body?.cancel(timeoutError).catch(() => undefined);
|
||||
reject(timeoutError);
|
||||
}, timeoutMs);
|
||||
unrefTimer(timeout);
|
||||
});
|
||||
|
||||
try {
|
||||
return await Promise.race([
|
||||
readResponseWithLimit(response, maxBytes, {
|
||||
chunkTimeoutMs: DIRECT_UPLOAD_READ_IDLE_TIMEOUT_MS,
|
||||
}),
|
||||
timeoutPromise,
|
||||
]);
|
||||
} finally {
|
||||
if (timeout) {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function downloadDirectUploadUrl(
|
||||
url: string,
|
||||
opts: { maxBytes?: number } = {},
|
||||
): Promise<Buffer> {
|
||||
let parsed: URL;
|
||||
try {
|
||||
parsed = new URL(url);
|
||||
} catch {
|
||||
throw new Error("Direct-upload media URL must be a valid URL");
|
||||
}
|
||||
|
||||
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
|
||||
throw new Error("Direct-upload media URL must use HTTP or HTTPS");
|
||||
}
|
||||
|
||||
assertDirectUploadDownloadHostAllowed(parsed.hostname);
|
||||
const { response, release } = await fetchDirectUploadDownload(parsed.toString());
|
||||
try {
|
||||
if (!response.ok) {
|
||||
throw new Error(`Direct-upload media URL returned HTTP ${response.status}`);
|
||||
}
|
||||
return await readDirectUploadResponse(response, opts.maxBytes ?? MAX_UPLOAD_SIZE);
|
||||
} finally {
|
||||
await release?.();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Small-file media upload module.
|
||||
*
|
||||
@@ -129,12 +248,19 @@ export class MediaApi {
|
||||
} else if (opts.localPath) {
|
||||
const buf = await fs.promises.readFile(opts.localPath);
|
||||
fileData = buf.toString("base64");
|
||||
} else if (opts.url !== undefined) {
|
||||
const buf = await downloadDirectUploadUrl(opts.url);
|
||||
fileData = buf.toString("base64");
|
||||
}
|
||||
|
||||
// Check cache for base64 uploads.
|
||||
if (fileData && this.cache) {
|
||||
const hash = this.cache.computeHash(fileData);
|
||||
const cached = this.cache.get(hash, scope, targetId, fileType);
|
||||
const uploadCache =
|
||||
fileData !== undefined && !(fileType === MediaFileType.FILE && opts.fileName)
|
||||
? this.cache
|
||||
: undefined;
|
||||
if (fileData !== undefined && uploadCache) {
|
||||
const hash = uploadCache.computeHash(fileData);
|
||||
const cached = uploadCache.get(hash, scope, targetId, fileType);
|
||||
if (cached) {
|
||||
return { file_uuid: "", file_info: cached, ttl: 0 };
|
||||
}
|
||||
@@ -144,9 +270,7 @@ export class MediaApi {
|
||||
file_type: fileType,
|
||||
srv_send_msg: opts.srvSendMsg ?? false,
|
||||
};
|
||||
if (opts.url) {
|
||||
body.url = opts.url;
|
||||
} else if (fileData) {
|
||||
if (fileData !== undefined) {
|
||||
body.file_data = fileData;
|
||||
}
|
||||
if (fileType === MediaFileType.FILE && opts.fileName) {
|
||||
@@ -168,9 +292,9 @@ export class MediaApi {
|
||||
);
|
||||
|
||||
// Cache the result for future dedup.
|
||||
if (fileData && result.file_info && result.ttl > 0 && this.cache) {
|
||||
const hash = this.cache.computeHash(fileData);
|
||||
this.cache.set(
|
||||
if (fileData !== undefined && uploadCache && result.file_info && result.ttl > 0) {
|
||||
const hash = uploadCache.computeHash(fileData);
|
||||
uploadCache.set(
|
||||
hash,
|
||||
scope,
|
||||
targetId,
|
||||
|
||||
@@ -27,7 +27,7 @@
|
||||
import os from "node:os";
|
||||
import { ApiClient } from "../api/api-client.js";
|
||||
import { ChunkedMediaApi as ChunkedMediaApiClass } from "../api/media-chunked.js";
|
||||
import { MediaApi as MediaApiClass } from "../api/media.js";
|
||||
import { downloadDirectUploadUrl, MediaApi as MediaApiClass } from "../api/media.js";
|
||||
import type { Credentials } from "../api/messages.js";
|
||||
import { MessageApi as MessageApiClass } from "../api/messages.js";
|
||||
import { getNextMsgSeq } from "../api/routes.js";
|
||||
@@ -41,7 +41,7 @@ import {
|
||||
type OutboundMeta,
|
||||
type UploadMediaResponse,
|
||||
} from "../types.js";
|
||||
import { LARGE_FILE_THRESHOLD } from "../utils/file-utils.js";
|
||||
import { getMaxUploadSize, LARGE_FILE_THRESHOLD } from "../utils/file-utils.js";
|
||||
import { formatErrorMessage } from "../utils/format.js";
|
||||
import { debugLog, debugError, debugWarn } from "../utils/log.js";
|
||||
import { sanitizeFileName } from "../utils/string-normalize.js";
|
||||
@@ -653,11 +653,25 @@ async function dispatchUpload(
|
||||
fileName?: string,
|
||||
): Promise<UploadMediaResponse> {
|
||||
switch (source.kind) {
|
||||
case "url":
|
||||
case "url": {
|
||||
const buffer = await downloadDirectUploadUrl(source.url, {
|
||||
maxBytes: getMaxUploadSize(fileType),
|
||||
});
|
||||
if (buffer.length >= LARGE_FILE_THRESHOLD) {
|
||||
return ctx.chunkedMediaApi.uploadChunked({
|
||||
scope,
|
||||
targetId,
|
||||
fileType,
|
||||
source: { kind: "buffer", buffer, fileName },
|
||||
creds,
|
||||
fileName,
|
||||
});
|
||||
}
|
||||
return ctx.mediaApi.uploadMedia(scope, targetId, fileType, creds, {
|
||||
url: source.url,
|
||||
buffer,
|
||||
fileName,
|
||||
});
|
||||
}
|
||||
case "base64":
|
||||
return ctx.mediaApi.uploadMedia(scope, targetId, fileType, creds, {
|
||||
fileData: source.data,
|
||||
|
||||
@@ -27,7 +27,7 @@ function readPositiveNumberEnv(name, fallback, env = process.env) {
|
||||
if (raw === undefined || raw === "") {
|
||||
return fallback;
|
||||
}
|
||||
const text = String(raw).trim();
|
||||
const text = raw.trim();
|
||||
if (!/^(?:\d+(?:\.\d+)?|\.\d+)$/u.test(text)) {
|
||||
return fallback;
|
||||
}
|
||||
|
||||
@@ -62,9 +62,9 @@ function dispatchWithDeliveries(
|
||||
deliveries: Delivery[],
|
||||
dispatcherOptions: {
|
||||
beforeDeliver?: ReplyDispatchBeforeDeliver;
|
||||
deliver?: (payload: ReplyPayload, info: { kind: Delivery["kind"] }) => Promise<unknown>;
|
||||
onSettled?: () => unknown;
|
||||
onFreshSettledDelivery?: () => unknown;
|
||||
deliver?: (payload: ReplyPayload, info: { kind: Delivery["kind"] }) => Promise<object | void>;
|
||||
onSettled?: () => object | void | Promise<object | void>;
|
||||
onFreshSettledDelivery?: () => object | void | Promise<object | void>;
|
||||
} = {},
|
||||
) {
|
||||
return dispatchInboundMessageWithBufferedDispatcher({
|
||||
|
||||
Reference in New Issue
Block a user