From b860a0d4d0d91d07ef4506b1ff03e213b743512e Mon Sep 17 00:00:00 2001 From: Agustin Rivera <31522568+eleqtrizit@users.noreply.github.com> Date: Wed, 27 May 2026 20:21:46 -0700 Subject: [PATCH] fix: harden qqbot direct media uploads Harden QQBot direct media URL uploads by downloading through the local SSRF guard before QQ upload, disabling redirects, bounding fetch/setup and body reads, and routing downloaded buffers through the existing one-shot/chunked size gate. Co-authored-by: Agustin Rivera --- .github/workflows/ci.yml | 18 +- .../monitor/message-handler.process.test.ts | 4 +- .../src/engine/api/media-chunked.test.ts | 15 +- .../qqbot/src/engine/api/media-chunked.ts | 5 +- extensions/qqbot/src/engine/api/media.test.ts | 448 ++++++++++++++++++ extensions/qqbot/src/engine/api/media.ts | 142 +++++- .../qqbot/src/engine/messaging/sender.ts | 22 +- scripts/check-cli-startup-memory.mjs | 2 +- src/auto-reply/dispatch.freshness.test.ts | 6 +- 9 files changed, 626 insertions(+), 36 deletions(-) create mode 100644 extensions/qqbot/src/engine/api/media.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 52b6c57493d..8b571cc5231 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -585,6 +585,20 @@ jobs: pids+=("$!") } + if [ "$RUN_GATEWAY_WATCH" = "true" ]; then + gateway_watch_log="${RUNNER_TEMP}/gateway-watch.log" + echo "starting gateway-watch: node scripts/check-gateway-watch-regression.mjs --skip-build --ready-timeout-ms 5000" + if node scripts/check-gateway-watch-regression.mjs --skip-build --ready-timeout-ms 5000 >"$gateway_watch_log" 2>&1; then + result="success" + else + result="failure" + fi + echo "::group::gateway-watch log" + cat "$gateway_watch_log" + echo "::endgroup::" + results["gateway-watch"]="$result" + fi + if [ "$RUN_CHANNELS" = "true" ]; then start_check "channels" env \ NODE_OPTIONS=--max-old-space-size=8192 \ @@ -599,10 +613,6 @@ jobs: node scripts/run-vitest.mjs run --config test/vitest/vitest.full-core-support-boundary.config.ts fi - if [ "$RUN_GATEWAY_WATCH" = "true" ]; then - start_check "gateway-watch" node scripts/check-gateway-watch-regression.mjs --skip-build --ready-timeout-ms 5000 - fi - for index in "${!pids[@]}"; do name="${names[$index]}" log="${logs[$index]}" diff --git a/extensions/discord/src/monitor/message-handler.process.test.ts b/extensions/discord/src/monitor/message-handler.process.test.ts index 8486e1b9cce..815a7dadea5 100644 --- a/extensions/discord/src/monitor/message-handler.process.test.ts +++ b/extensions/discord/src/monitor/message-handler.process.test.ts @@ -108,7 +108,7 @@ vi.mock("./reply-delivery.js", () => ({ })); type DispatchInboundParams = { - ctx?: unknown; + ctx?: Record; dispatcher: { sendBlockReply: (payload: ReplyPayload) => boolean | Promise; sendFinalReply: (payload: ReplyPayload) => boolean | Promise; @@ -240,7 +240,7 @@ vi.mock("openclaw/plugin-sdk/reply-runtime", () => ({ onSettled?: () => unknown; onFreshSettledDelivery?: () => unknown; }; - ctx?: unknown; + ctx?: Record; replyOptions?: DispatchInboundParams["replyOptions"]; }) => { const pendingDeliveries: Promise[] = []; diff --git a/extensions/qqbot/src/engine/api/media-chunked.test.ts b/extensions/qqbot/src/engine/api/media-chunked.test.ts index 838dfa3c3ed..b16b9b19a44 100644 --- a/extensions/qqbot/src/engine/api/media-chunked.test.ts +++ b/extensions/qqbot/src/engine/api/media-chunked.test.ts @@ -250,17 +250,10 @@ describe("media-chunked: ChunkedMediaApi.uploadChunked", () => { ]), ); - // Cache populated with the complete result. - const expectedMd5 = crypto.createHash("md5").update(FIXTURE_BUFFER).digest("hex"); - expect(cache.setSpy).toHaveBeenCalledWith( - expectedMd5, - "group", - "g1", - MediaFileType.FILE, - "final-file-info", - "uuid-final", - 3600, - ); + // FILE uploads carry filename metadata in upload_prepare, so the content-only + // cache is bypassed to avoid reusing file_info with a stale name. + expect(cache.getSpy).not.toHaveBeenCalled(); + expect(cache.setSpy).not.toHaveBeenCalled(); // Progress callback hit 3 times with monotonically-increasing counts. expect(onProgress).toHaveBeenCalledTimes(3); diff --git a/extensions/qqbot/src/engine/api/media-chunked.ts b/extensions/qqbot/src/engine/api/media-chunked.ts index 9d030869273..cfd57955567 100644 --- a/extensions/qqbot/src/engine/api/media-chunked.ts +++ b/extensions/qqbot/src/engine/api/media-chunked.ts @@ -202,7 +202,8 @@ export class ChunkedMediaApi { // 3. Upload-cache fast path: the md5 hash is already a strong content // identifier, so we can short-circuit before even calling upload_prepare. - if (this.cache) { + const canUseUploadCache = opts.fileType !== MediaFileType.FILE; + if (this.cache && canUseUploadCache) { const cached = this.cache.get(hashes.md5, opts.scope, opts.targetId, opts.fileType); if (cached) { this.logger?.info?.( @@ -293,7 +294,7 @@ export class ChunkedMediaApi { this.logger?.info?.(`${prefix} completed: file_uuid=${result.file_uuid} ttl=${result.ttl}s`); // 7. Populate the shared upload cache so subsequent sends skip re-uploading. - if (this.cache && result.file_info && result.ttl > 0) { + if (this.cache && canUseUploadCache && result.file_info && result.ttl > 0) { this.cache.set( hashes.md5, opts.scope, diff --git a/extensions/qqbot/src/engine/api/media.test.ts b/extensions/qqbot/src/engine/api/media.test.ts new file mode 100644 index 00000000000..785787eeeea --- /dev/null +++ b/extensions/qqbot/src/engine/api/media.test.ts @@ -0,0 +1,448 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { MediaFileType, type UploadMediaResponse } from "../types.js"; +import { MAX_UPLOAD_SIZE } from "../utils/file-utils.js"; +import { ApiClient } from "./api-client.js"; +import { MediaApi } from "./media.js"; +import { TokenManager } from "./token.js"; + +const fetchWithSsrFGuardMock = vi.hoisted(() => vi.fn()); +const readResponseWithLimitMock = vi.hoisted(() => vi.fn()); + +vi.mock("openclaw/plugin-sdk/response-limit-runtime", async (importOriginal) => { + const actual = + await importOriginal(); + return { + ...actual, + readResponseWithLimit: readResponseWithLimitMock, + }; +}); + +vi.mock("openclaw/plugin-sdk/ssrf-runtime", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + fetchWithSsrFGuard: fetchWithSsrFGuardMock, + }; +}); + +const UPLOAD_RESPONSE: UploadMediaResponse = { + file_uuid: "uuid-1", + file_info: "file-info-1", + ttl: 3600, +}; + +const MEDIA_BYTES = Buffer.from("downloaded-media"); +const MEDIA_BASE64 = MEDIA_BYTES.toString("base64"); + +function mockGuardedResponse( + body: BodyInit = MEDIA_BYTES, + init?: ResponseInit, +): { + release: ReturnType; +} { + const release = vi.fn(async () => {}); + fetchWithSsrFGuardMock.mockResolvedValueOnce({ + response: new Response(body, init), + release, + }); + return { release }; +} + +function mockApiClient(): ApiClient { + const client = new ApiClient(); + vi.spyOn(client, "request").mockResolvedValue(UPLOAD_RESPONSE); + return client; +} + +function mockTokenManager(): TokenManager { + const tokenManager = new TokenManager(); + vi.spyOn(tokenManager, "getAccessToken").mockResolvedValue("token-1"); + return tokenManager; +} + +function expectGuardedDownload(url: string): void { + expect(fetchWithSsrFGuardMock).toHaveBeenCalledWith({ + url, + maxRedirects: 0, + signal: expect.any(AbortSignal), + }); + expect(fetchWithSsrFGuardMock).not.toHaveBeenCalledWith( + expect.objectContaining({ timeoutMs: expect.any(Number) }), + ); + const signal = fetchWithSsrFGuardMock.mock.calls.at(-1)?.[0]?.signal; + expect(signal).toBeInstanceOf(AbortSignal); +} + +describe("MediaApi.uploadMedia direct URL uploads", () => { + beforeEach(() => { + fetchWithSsrFGuardMock.mockReset(); + readResponseWithLimitMock.mockReset(); + readResponseWithLimitMock.mockResolvedValue(MEDIA_BYTES); + mockGuardedResponse(); + }); + + it.each([ + { fileType: MediaFileType.IMAGE, url: "https://cdn.example.com/assets/photo.png" }, + { fileType: MediaFileType.VIDEO, url: "http://cdn.example.com/assets/video.mp4" }, + { fileType: MediaFileType.FILE, url: "http://cdn.example.com/assets/report.pdf" }, + ])( + "downloads public HTTP(S) $fileType URLs through the pinned SSRF guard", + async ({ fileType, url }) => { + const client = mockApiClient(); + const tokenManager = mockTokenManager(); + const api = new MediaApi(client, tokenManager); + + const result = await api.uploadMedia( + "c2c", + "user-openid", + fileType, + { appId: "app-id", clientSecret: "client-secret" }, + { url }, + ); + + expect(result).toBe(UPLOAD_RESPONSE); + expectGuardedDownload(url); + expect(readResponseWithLimitMock).toHaveBeenCalledWith( + expect.any(Response), + MAX_UPLOAD_SIZE, + { chunkTimeoutMs: 10_000 }, + ); + expect(tokenManager.getAccessToken).toHaveBeenCalledWith("app-id", "client-secret"); + expect(client.request).toHaveBeenCalledWith( + "token-1", + "POST", + expect.any(String), + { + file_type: fileType, + srv_send_msg: false, + file_data: MEDIA_BASE64, + }, + { + redactBodyKeys: ["file_data"], + uploadRequest: true, + }, + ); + }, + ); + + it("releases the pinned SSRF dispatcher after downloading media", async () => { + fetchWithSsrFGuardMock.mockReset(); + const { release } = mockGuardedResponse(); + const client = mockApiClient(); + const tokenManager = mockTokenManager(); + const api = new MediaApi(client, tokenManager); + + await api.uploadMedia( + "c2c", + "user-openid", + MediaFileType.IMAGE, + { appId: "app-id", clientSecret: "client-secret" }, + { url: "https://cdn.example.com/assets/photo.png" }, + ); + + expect(release).toHaveBeenCalledTimes(1); + }); + + it("bounds stalled guarded fetch setup before reading URL bodies", async () => { + vi.useFakeTimers(); + try { + fetchWithSsrFGuardMock.mockReset(); + fetchWithSsrFGuardMock.mockImplementationOnce(() => new Promise(() => {})); + const client = mockApiClient(); + const tokenManager = mockTokenManager(); + const api = new MediaApi(client, tokenManager); + + const uploadPromise = api.uploadMedia( + "c2c", + "user-openid", + MediaFileType.IMAGE, + { appId: "app-id", clientSecret: "client-secret" }, + { url: "https://slow-dns.example.com/assets/photo.png" }, + ); + const rejection = expect(uploadPromise).rejects.toThrow( + "Direct-upload media URL fetch timed out", + ); + + await vi.advanceTimersByTimeAsync(30_000); + await rejection; + expect(readResponseWithLimitMock).not.toHaveBeenCalled(); + expect(tokenManager.getAccessToken).not.toHaveBeenCalled(); + expect(client.request).not.toHaveBeenCalled(); + } finally { + vi.useRealTimers(); + } + }); + + it("rejects URL bodies that keep trickling under the idle timeout", async () => { + vi.useFakeTimers(); + try { + fetchWithSsrFGuardMock.mockReset(); + const { release } = mockGuardedResponse(); + readResponseWithLimitMock.mockReset(); + readResponseWithLimitMock.mockImplementationOnce(() => new Promise(() => {})); + const client = mockApiClient(); + const tokenManager = mockTokenManager(); + const api = new MediaApi(client, tokenManager); + + const uploadPromise = api.uploadMedia( + "c2c", + "user-openid", + MediaFileType.IMAGE, + { appId: "app-id", clientSecret: "client-secret" }, + { url: "https://cdn.example.com/assets/slow.bin" }, + ); + + for (let i = 0; i < 5 && readResponseWithLimitMock.mock.calls.length === 0; i += 1) { + await Promise.resolve(); + } + expect(readResponseWithLimitMock).toHaveBeenCalledOnce(); + + const rejection = expect(uploadPromise).rejects.toThrow( + "Direct-upload media URL body timed out", + ); + await vi.advanceTimersByTimeAsync(8 * 60_000); + await rejection; + expect(release).toHaveBeenCalledTimes(1); + } finally { + vi.useRealTimers(); + } + }); + + it("dedupes downloaded URL media through the base64 upload cache", async () => { + const cache = { + computeHash: vi.fn(() => "hash-1"), + get: vi.fn(() => "cached-file-info"), + set: vi.fn(), + }; + const client = mockApiClient(); + const tokenManager = mockTokenManager(); + const api = new MediaApi(client, tokenManager, { uploadCache: cache }); + + const result = await api.uploadMedia( + "c2c", + "user-openid", + MediaFileType.IMAGE, + { appId: "app-id", clientSecret: "client-secret" }, + { url: "https://cdn.example.com/assets/photo.png" }, + ); + + expect(result).toEqual({ file_uuid: "", file_info: "cached-file-info", ttl: 0 }); + expect(cache.computeHash).toHaveBeenCalledWith(MEDIA_BASE64); + expect(cache.get).toHaveBeenCalledWith("hash-1", "c2c", "user-openid", MediaFileType.IMAGE); + expect(tokenManager.getAccessToken).not.toHaveBeenCalled(); + expect(client.request).not.toHaveBeenCalled(); + }); + + it("does not reuse cached FILE uploads when the requested filename differs", async () => { + const cache = { + computeHash: vi.fn(() => "hash-1"), + get: vi.fn(() => "cached-file-info"), + set: vi.fn(), + }; + const client = mockApiClient(); + const tokenManager = mockTokenManager(); + const api = new MediaApi(client, tokenManager, { + uploadCache: cache, + sanitizeFileName: (name) => `safe-${name}`, + }); + + await api.uploadMedia( + "c2c", + "user-openid", + MediaFileType.FILE, + { appId: "app-id", clientSecret: "client-secret" }, + { url: "https://cdn.example.com/report.pdf", fileName: "report.pdf" }, + ); + + expect(cache.computeHash).not.toHaveBeenCalled(); + expect(cache.get).not.toHaveBeenCalled(); + expect(cache.set).not.toHaveBeenCalled(); + expect(client.request).toHaveBeenCalledWith( + "token-1", + "POST", + expect.any(String), + expect.objectContaining({ + file_data: MEDIA_BASE64, + file_name: "safe-report.pdf", + }), + expect.any(Object), + ); + }); + + it("rejects invalid direct-upload URLs before downloading media or calling the QQ API", async () => { + const client = mockApiClient(); + const tokenManager = mockTokenManager(); + const api = new MediaApi(client, tokenManager); + + await expect( + api.uploadMedia( + "c2c", + "user-openid", + MediaFileType.IMAGE, + { appId: "app-id", clientSecret: "client-secret" }, + { url: "not a url" }, + ), + ).rejects.toThrow("Direct-upload media URL must be a valid URL"); + + expect(fetchWithSsrFGuardMock).not.toHaveBeenCalled(); + expect(tokenManager.getAccessToken).not.toHaveBeenCalled(); + expect(client.request).not.toHaveBeenCalled(); + }); + + it("rejects non-HTTP direct-upload URLs before downloading media or calling the QQ API", async () => { + const client = mockApiClient(); + const tokenManager = mockTokenManager(); + const api = new MediaApi(client, tokenManager); + + await expect( + api.uploadMedia( + "c2c", + "user-openid", + MediaFileType.IMAGE, + { appId: "app-id", clientSecret: "client-secret" }, + { url: "ftp://media.qq.com/assets/photo.png" }, + ), + ).rejects.toThrow("Direct-upload media URL must use HTTP or HTTPS"); + + expect(fetchWithSsrFGuardMock).not.toHaveBeenCalled(); + expect(tokenManager.getAccessToken).not.toHaveBeenCalled(); + expect(client.request).not.toHaveBeenCalled(); + }); + + it.each(["127.0.0.1", "169.254.169.254", "10.0.0.1", "192.168.1.1"])( + "does not upload direct URLs rejected by the SSRF guard: %s", + async (host) => { + fetchWithSsrFGuardMock.mockReset(); + const client = mockApiClient(); + const tokenManager = mockTokenManager(); + const api = new MediaApi(client, tokenManager); + + await expect( + api.uploadMedia( + "group", + "group-openid", + MediaFileType.IMAGE, + { appId: "app-id", clientSecret: "client-secret" }, + { url: `https://${host}/latest/meta-data/` }, + ), + ).rejects.toThrow("Blocked hostname"); + + expect(fetchWithSsrFGuardMock).not.toHaveBeenCalled(); + expect(tokenManager.getAccessToken).not.toHaveBeenCalled(); + expect(client.request).not.toHaveBeenCalled(); + }, + ); + + it("does not forward URLs when the guarded download fails", async () => { + fetchWithSsrFGuardMock.mockReset(); + fetchWithSsrFGuardMock.mockRejectedValueOnce( + new Error("Blocked: resolves to private/internal/special-use IP address"), + ); + const client = mockApiClient(); + const tokenManager = mockTokenManager(); + const api = new MediaApi(client, tokenManager); + + await expect( + api.uploadMedia( + "group", + "group-openid", + MediaFileType.IMAGE, + { appId: "app-id", clientSecret: "client-secret" }, + { url: "https://attacker.example/latest/meta-data/" }, + ), + ).rejects.toThrow("resolves to private"); + + expect(tokenManager.getAccessToken).not.toHaveBeenCalled(); + expect(client.request).not.toHaveBeenCalled(); + }); + + it("rejects literal RFC 2544 special-use URL hosts through the guarded download", async () => { + fetchWithSsrFGuardMock.mockReset(); + const client = mockApiClient(); + const tokenManager = mockTokenManager(); + const api = new MediaApi(client, tokenManager); + + await expect( + api.uploadMedia( + "c2c", + "user-openid", + MediaFileType.IMAGE, + { appId: "app-id", clientSecret: "client-secret" }, + { url: "https://198.18.0.42/assets/photo.png" }, + ), + ).rejects.toThrow("Blocked hostname"); + + expect(fetchWithSsrFGuardMock).not.toHaveBeenCalled(); + expect(tokenManager.getAccessToken).not.toHaveBeenCalled(); + expect(client.request).not.toHaveBeenCalled(); + }); + + it("keeps public literal IP URLs on the default SSRF policy", async () => { + const client = mockApiClient(); + const tokenManager = mockTokenManager(); + const api = new MediaApi(client, tokenManager); + + await api.uploadMedia( + "c2c", + "user-openid", + MediaFileType.IMAGE, + { appId: "app-id", clientSecret: "client-secret" }, + { url: "http://93.184.216.34/assets/photo.png" }, + ); + + expectGuardedDownload("http://93.184.216.34/assets/photo.png"); + }); + + it("does not pass URL or fake-IP DNS policy to the QQ upload body", async () => { + const client = mockApiClient(); + const tokenManager = mockTokenManager(); + const api = new MediaApi(client, tokenManager); + + await api.uploadMedia( + "c2c", + "user-openid", + MediaFileType.IMAGE, + { appId: "app-id", clientSecret: "client-secret" }, + { url: "https://cdn.example.com/assets/photo.png" }, + ); + + expectGuardedDownload("https://cdn.example.com/assets/photo.png"); + expect(client.request).toHaveBeenCalledWith( + "token-1", + "POST", + expect.any(String), + expect.objectContaining({ + file_data: MEDIA_BASE64, + }), + expect.any(Object), + ); + expect(client.request).not.toHaveBeenCalledWith( + expect.any(String), + expect.any(String), + expect.any(String), + expect.objectContaining({ url: expect.any(String) }), + expect.any(Object), + ); + }); + + it("rejects HTTP errors from guarded direct-upload downloads before calling the QQ API", async () => { + fetchWithSsrFGuardMock.mockReset(); + mockGuardedResponse("not found", { status: 404 }); + const client = mockApiClient(); + const tokenManager = mockTokenManager(); + const api = new MediaApi(client, tokenManager); + + await expect( + api.uploadMedia( + "c2c", + "user-openid", + MediaFileType.IMAGE, + { appId: "app-id", clientSecret: "client-secret" }, + { url: "https://cdn.example.com/missing.png" }, + ), + ).rejects.toThrow("Direct-upload media URL returned HTTP 404"); + + expect(tokenManager.getAccessToken).not.toHaveBeenCalled(); + expect(client.request).not.toHaveBeenCalled(); + }); +}); diff --git a/extensions/qqbot/src/engine/api/media.ts b/extensions/qqbot/src/engine/api/media.ts index 93534aa5c67..c06ba5a6ee9 100644 --- a/extensions/qqbot/src/engine/api/media.ts +++ b/extensions/qqbot/src/engine/api/media.ts @@ -12,6 +12,8 @@ */ import * as fs from "node:fs"; +import { readResponseWithLimit } from "openclaw/plugin-sdk/response-limit-runtime"; +import { fetchWithSsrFGuard, isBlockedHostnameOrIp } from "openclaw/plugin-sdk/ssrf-runtime"; import { MediaFileType, type ChatScope, @@ -19,6 +21,7 @@ import { type MessageResponse, type EngineLogger, } from "../types.js"; +import { MAX_UPLOAD_SIZE } from "../utils/file-utils.js"; import { ApiClient } from "./api-client.js"; import { withRetry, UPLOAD_RETRY_POLICY } from "./retry.js"; import { mediaUploadPath, messagePath, getNextMsgSeq } from "./routes.js"; @@ -50,6 +53,122 @@ interface MediaApiConfig { sanitizeFileName?: SanitizeFileNameFn; } +const DIRECT_UPLOAD_DOWNLOAD_TIMEOUT_MS = 30_000; +const DIRECT_UPLOAD_READ_IDLE_TIMEOUT_MS = 10_000; +const DIRECT_UPLOAD_BODY_GRACE_TIMEOUT_MS = 30_000; +const DIRECT_UPLOAD_MIN_DOWNLOAD_BYTES_PER_SECOND = 256 * 1024; +const DIRECT_UPLOAD_MAX_BODY_TIMEOUT_MS = 8 * 60_000; + +function assertDirectUploadDownloadHostAllowed(hostname: string): void { + if (isBlockedHostnameOrIp(hostname)) { + throw new Error("Blocked hostname or private/internal/special-use IP address"); + } +} + +async function fetchDirectUploadDownload(url: string) { + const controller = new AbortController(); + const timeoutError = new Error("Direct-upload media URL fetch timed out"); + let timedOut = false; + let timeout: ReturnType | undefined; + const timeoutPromise = new Promise((_, reject) => { + timeout = setTimeout(() => { + timedOut = true; + controller.abort(timeoutError); + reject(timeoutError); + }, DIRECT_UPLOAD_DOWNLOAD_TIMEOUT_MS); + unrefTimer(timeout); + }); + const guardedFetch = fetchWithSsrFGuard({ + url, + maxRedirects: 0, + signal: controller.signal, + }); + void guardedFetch.then( + (result) => { + if (timedOut) { + void result.release().catch(() => undefined); + } + }, + () => undefined, + ); + try { + return await Promise.race([guardedFetch, timeoutPromise]); + } finally { + if (timeout) { + clearTimeout(timeout); + } + } +} + +function unrefTimer(timeout: ReturnType): void { + if (typeof timeout === "object" && "unref" in timeout) { + (timeout as { unref: () => void }).unref(); + } +} + +function resolveDirectUploadBodyTimeoutMs(maxBytes: number): number { + const transferTimeoutMs = Math.ceil( + (maxBytes / DIRECT_UPLOAD_MIN_DOWNLOAD_BYTES_PER_SECOND) * 1000, + ); + return Math.min( + DIRECT_UPLOAD_BODY_GRACE_TIMEOUT_MS + transferTimeoutMs, + DIRECT_UPLOAD_MAX_BODY_TIMEOUT_MS, + ); +} + +async function readDirectUploadResponse(response: Response, maxBytes: number): Promise { + const timeoutMs = resolveDirectUploadBodyTimeoutMs(maxBytes); + const timeoutError = new Error(`Direct-upload media URL body timed out after ${timeoutMs}ms`); + let timeout: ReturnType | undefined; + const timeoutPromise = new Promise((_, reject) => { + timeout = setTimeout(() => { + void response.body?.cancel(timeoutError).catch(() => undefined); + reject(timeoutError); + }, timeoutMs); + unrefTimer(timeout); + }); + + try { + return await Promise.race([ + readResponseWithLimit(response, maxBytes, { + chunkTimeoutMs: DIRECT_UPLOAD_READ_IDLE_TIMEOUT_MS, + }), + timeoutPromise, + ]); + } finally { + if (timeout) { + clearTimeout(timeout); + } + } +} + +export async function downloadDirectUploadUrl( + url: string, + opts: { maxBytes?: number } = {}, +): Promise { + let parsed: URL; + try { + parsed = new URL(url); + } catch { + throw new Error("Direct-upload media URL must be a valid URL"); + } + + if (parsed.protocol !== "http:" && parsed.protocol !== "https:") { + throw new Error("Direct-upload media URL must use HTTP or HTTPS"); + } + + assertDirectUploadDownloadHostAllowed(parsed.hostname); + const { response, release } = await fetchDirectUploadDownload(parsed.toString()); + try { + if (!response.ok) { + throw new Error(`Direct-upload media URL returned HTTP ${response.status}`); + } + return await readDirectUploadResponse(response, opts.maxBytes ?? MAX_UPLOAD_SIZE); + } finally { + await release?.(); + } +} + /** * Small-file media upload module. * @@ -129,12 +248,19 @@ export class MediaApi { } else if (opts.localPath) { const buf = await fs.promises.readFile(opts.localPath); fileData = buf.toString("base64"); + } else if (opts.url !== undefined) { + const buf = await downloadDirectUploadUrl(opts.url); + fileData = buf.toString("base64"); } // Check cache for base64 uploads. - if (fileData && this.cache) { - const hash = this.cache.computeHash(fileData); - const cached = this.cache.get(hash, scope, targetId, fileType); + const uploadCache = + fileData !== undefined && !(fileType === MediaFileType.FILE && opts.fileName) + ? this.cache + : undefined; + if (fileData !== undefined && uploadCache) { + const hash = uploadCache.computeHash(fileData); + const cached = uploadCache.get(hash, scope, targetId, fileType); if (cached) { return { file_uuid: "", file_info: cached, ttl: 0 }; } @@ -144,9 +270,7 @@ export class MediaApi { file_type: fileType, srv_send_msg: opts.srvSendMsg ?? false, }; - if (opts.url) { - body.url = opts.url; - } else if (fileData) { + if (fileData !== undefined) { body.file_data = fileData; } if (fileType === MediaFileType.FILE && opts.fileName) { @@ -168,9 +292,9 @@ export class MediaApi { ); // Cache the result for future dedup. - if (fileData && result.file_info && result.ttl > 0 && this.cache) { - const hash = this.cache.computeHash(fileData); - this.cache.set( + if (fileData !== undefined && uploadCache && result.file_info && result.ttl > 0) { + const hash = uploadCache.computeHash(fileData); + uploadCache.set( hash, scope, targetId, diff --git a/extensions/qqbot/src/engine/messaging/sender.ts b/extensions/qqbot/src/engine/messaging/sender.ts index 75b8b997415..88b0decc29c 100644 --- a/extensions/qqbot/src/engine/messaging/sender.ts +++ b/extensions/qqbot/src/engine/messaging/sender.ts @@ -27,7 +27,7 @@ import os from "node:os"; import { ApiClient } from "../api/api-client.js"; import { ChunkedMediaApi as ChunkedMediaApiClass } from "../api/media-chunked.js"; -import { MediaApi as MediaApiClass } from "../api/media.js"; +import { downloadDirectUploadUrl, MediaApi as MediaApiClass } from "../api/media.js"; import type { Credentials } from "../api/messages.js"; import { MessageApi as MessageApiClass } from "../api/messages.js"; import { getNextMsgSeq } from "../api/routes.js"; @@ -41,7 +41,7 @@ import { type OutboundMeta, type UploadMediaResponse, } from "../types.js"; -import { LARGE_FILE_THRESHOLD } from "../utils/file-utils.js"; +import { getMaxUploadSize, LARGE_FILE_THRESHOLD } from "../utils/file-utils.js"; import { formatErrorMessage } from "../utils/format.js"; import { debugLog, debugError, debugWarn } from "../utils/log.js"; import { sanitizeFileName } from "../utils/string-normalize.js"; @@ -653,11 +653,25 @@ async function dispatchUpload( fileName?: string, ): Promise { switch (source.kind) { - case "url": + case "url": { + const buffer = await downloadDirectUploadUrl(source.url, { + maxBytes: getMaxUploadSize(fileType), + }); + if (buffer.length >= LARGE_FILE_THRESHOLD) { + return ctx.chunkedMediaApi.uploadChunked({ + scope, + targetId, + fileType, + source: { kind: "buffer", buffer, fileName }, + creds, + fileName, + }); + } return ctx.mediaApi.uploadMedia(scope, targetId, fileType, creds, { - url: source.url, + buffer, fileName, }); + } case "base64": return ctx.mediaApi.uploadMedia(scope, targetId, fileType, creds, { fileData: source.data, diff --git a/scripts/check-cli-startup-memory.mjs b/scripts/check-cli-startup-memory.mjs index 866f408ec17..63287243636 100644 --- a/scripts/check-cli-startup-memory.mjs +++ b/scripts/check-cli-startup-memory.mjs @@ -27,7 +27,7 @@ function readPositiveNumberEnv(name, fallback, env = process.env) { if (raw === undefined || raw === "") { return fallback; } - const text = String(raw).trim(); + const text = raw.trim(); if (!/^(?:\d+(?:\.\d+)?|\.\d+)$/u.test(text)) { return fallback; } diff --git a/src/auto-reply/dispatch.freshness.test.ts b/src/auto-reply/dispatch.freshness.test.ts index dfb921895c3..4acad08437e 100644 --- a/src/auto-reply/dispatch.freshness.test.ts +++ b/src/auto-reply/dispatch.freshness.test.ts @@ -62,9 +62,9 @@ function dispatchWithDeliveries( deliveries: Delivery[], dispatcherOptions: { beforeDeliver?: ReplyDispatchBeforeDeliver; - deliver?: (payload: ReplyPayload, info: { kind: Delivery["kind"] }) => Promise; - onSettled?: () => unknown; - onFreshSettledDelivery?: () => unknown; + deliver?: (payload: ReplyPayload, info: { kind: Delivery["kind"] }) => Promise; + onSettled?: () => object | void | Promise; + onFreshSettledDelivery?: () => object | void | Promise; } = {}, ) { return dispatchInboundMessageWithBufferedDispatcher({