diff --git a/docs/.generated/plugin-sdk-api-baseline.sha256 b/docs/.generated/plugin-sdk-api-baseline.sha256 index d1e94f5dc8ac..84f39904e287 100644 --- a/docs/.generated/plugin-sdk-api-baseline.sha256 +++ b/docs/.generated/plugin-sdk-api-baseline.sha256 @@ -78,7 +78,7 @@ d8ca27a737f235e09f1a9f8cd4b82ac0cde96f10c662347b6ac82e27b70b4a40 module/lazy-ru fd00374a91ab5b393152bbbaf90010f9140cf15827bde360d0c6c8724ed34e2e module/logging-core 7670808d8c6227a52b2cddb72121aed9767313dd674f294a1f426dc185c13046 module/matrix a5f59c9acbcaa3f82247bf806eb5ba08032373fb853719f0ec9457690f16fc70 module/media-mime -7b30827ec2f616ae9de0c1d81348a2112a41e9b4a8aeb8f072cfb1480d2eb864 module/media-runtime +0c3d926d16e44d200018330b708065e9190f12ff8bce8321ae7bae184fe9de59 module/media-runtime 7a5a1de743fff9d5a86515b501ffd88e7fa49970c9f46a5153816e2ebaef7bb0 module/media-store c5e3eb1a584f4b8126d9d6c177a840ec9103671e8d1242634ee67db9b5b9e573 module/media-understanding c0ffaed532578cf33493992e1ff806b2268b8e3774a92edbaede5cf5bda162a6 module/media-understanding-runtime diff --git a/extensions/whatsapp/src/auto-reply/deliver-reply.test.ts b/extensions/whatsapp/src/auto-reply/deliver-reply.test.ts index d8b8bd800084..75ad797fd759 100644 --- a/extensions/whatsapp/src/auto-reply/deliver-reply.test.ts +++ b/extensions/whatsapp/src/auto-reply/deliver-reply.test.ts @@ -1,9 +1,9 @@ // Whatsapp tests cover deliver reply plugin behavior. -import fsSync from "node:fs"; import { createMessageReceiptFromOutboundResults, listMessageReceiptPlatformIds, } from "openclaw/plugin-sdk/channel-outbound"; +import { MEDIA_FFMPEG_MAX_AUDIO_DURATION_SECS } from "openclaw/plugin-sdk/media-runtime"; import { logVerbose } from "openclaw/plugin-sdk/runtime-env"; import { beforeAll, describe, expect, it, vi } from "vitest"; import { createAcceptedWhatsAppSendResult } from "../inbound/send-result.test-helper.js"; @@ -14,7 +14,7 @@ import { cacheInboundMessageMeta } from "../quoted-message.js"; import { withWhatsAppSocketOperationTimeout } from "../socket-timing.js"; const hoisted = vi.hoisted(() => ({ - runFfmpeg: vi.fn(), + transcodeAudioBufferToOpus: vi.fn(), })); vi.mock("openclaw/plugin-sdk/media-runtime", async () => { @@ -23,7 +23,7 @@ vi.mock("openclaw/plugin-sdk/media-runtime", async () => { ); return { ...actual, - runFfmpeg: hoisted.runFfmpeg, + transcodeAudioBufferToOpus: hoisted.transcodeAudioBufferToOpus, }; }); @@ -927,10 +927,7 @@ describe("deliverWebReply", () => { it("transcodes mp3 audio media before sending a ptt voice note", async () => { vi.clearAllMocks(); - hoisted.runFfmpeg.mockImplementation(async (args: string[]) => { - fsSync.writeFileSync(args.at(-1) ?? "", Buffer.from("opus-output")); - return ""; - }); + hoisted.transcodeAudioBufferToOpus.mockResolvedValue(Buffer.from("opus-output")); const msg = makeMsg(); ( loadWebMedia as unknown as { mockResolvedValueOnce: (v: unknown) => void } @@ -950,16 +947,16 @@ describe("deliverWebReply", () => { skipLog: true, }); - const ffmpegArgs = mockCallArg(hoisted.runFfmpeg, 0, 0, "runFfmpeg"); - expect(Array.isArray(ffmpegArgs)).toBe(true); - const ffmpegArgList = ffmpegArgs as unknown[]; - expect(ffmpegArgList).toContain("-c:a"); - expect(ffmpegArgList).toContain("libopus"); - expect(ffmpegArgList).toContain("-ar"); - expect(ffmpegArgList).toContain("48000"); - expect(ffmpegArgList).toContain("-b:a"); - expect(ffmpegArgList).toContain("64k"); - expect(ffmpegArgList.slice(-3, -1)).toEqual(["-f", "ogg"]); + expect(hoisted.transcodeAudioBufferToOpus).toHaveBeenCalledWith({ + audioBuffer: Buffer.from("mp3"), + inputFileName: "voice.mp3", + tempPrefix: "whatsapp-voice-", + outputFileName: "voice.ogg", + maxDurationSeconds: MEDIA_FFMPEG_MAX_AUDIO_DURATION_SECS, + sampleRateHz: 48000, + channels: 1, + bitrate: "64k", + }); const mediaPayload = requireRecord( mockCallArg(msg.platform.sendMedia, 0, 0, "sendMedia"), "sendMedia payload", diff --git a/extensions/whatsapp/src/channel-react-action.test.ts b/extensions/whatsapp/src/channel-react-action.test.ts index c49c259b2ec3..ca6e48ae0b46 100644 --- a/extensions/whatsapp/src/channel-react-action.test.ts +++ b/extensions/whatsapp/src/channel-react-action.test.ts @@ -206,12 +206,12 @@ describe("whatsapp react action messageId resolution", () => { expect(hoisted.sendMessageWhatsApp).not.toHaveBeenCalled(); }); - it("sends upload-file from the hydrated buffer payload", async () => { + it("sends upload-file from a whitespace-heavy base64 data URL", async () => { await handleWhatsAppMessageAction({ action: "upload-file", params: { to: "+1555", - buffer: Buffer.from("hello").toString("base64"), + buffer: " \n DATA:text/plain;BASE64, aG Vs\nbG8= \n ", contentType: "text/plain", filename: "hello.txt", filePath: "/tmp/hello.txt", @@ -240,23 +240,46 @@ describe("whatsapp react action messageId resolution", () => { }); }); + it.each(["SGVsbG8=!", "data:text/plain,hello", "data:text/plain;base64"])( + "rejects malformed upload-file buffer %s", + async (buffer) => { + await expect( + handleWhatsAppMessageAction({ + action: "upload-file", + params: { to: "+1555", buffer }, + cfg: baseCfg, + accountId: "default", + }), + ).rejects.toThrow("must be valid base64 or a base64 data URL"); + expect(hoisted.sendMessageWhatsApp).not.toHaveBeenCalled(); + }, + ); + it("rejects upload-file buffers above the WhatsApp media limit", async () => { hoisted.resolveWhatsAppMediaMaxBytes.mockReturnValueOnce(4); + const encoded = Buffer.from("hello").toString("base64"); + const bufferFromSpy = vi.spyOn(Buffer, "from"); - await expect( - handleWhatsAppMessageAction({ - action: "upload-file", - params: { - to: "+1555", - buffer: Buffer.from("hello").toString("base64"), - contentType: "text/plain", - filename: "hello.txt", - }, - cfg: baseCfg, - accountId: "default", - }), - ).rejects.toThrow("WhatsApp upload-file buffer exceeds configured media limit"); - expect(hoisted.sendMessageWhatsApp).not.toHaveBeenCalled(); + try { + await expect( + handleWhatsAppMessageAction({ + action: "upload-file", + params: { + to: "+1555", + buffer: encoded, + contentType: "text/plain", + filename: "hello.txt", + }, + cfg: baseCfg, + accountId: "default", + }), + ).rejects.toThrow("WhatsApp upload-file buffer exceeds configured media limit"); + const bufferFromCalls = bufferFromSpy.mock.calls as unknown[][]; + expect(bufferFromCalls.some((call) => call[1] === "base64")).toBe(false); + expect(hoisted.sendMessageWhatsApp).not.toHaveBeenCalled(); + } finally { + bufferFromSpy.mockRestore(); + } }); it("requires upload-file media path input", async () => { diff --git a/extensions/whatsapp/src/channel-react-action.ts b/extensions/whatsapp/src/channel-react-action.ts index c2869d09ddb2..9b255627865c 100644 --- a/extensions/whatsapp/src/channel-react-action.ts +++ b/extensions/whatsapp/src/channel-react-action.ts @@ -1,6 +1,7 @@ // Whatsapp plugin module implements channel react action behavior. import { readBooleanParam } from "openclaw/plugin-sdk/boolean-param"; import { jsonResult } from "openclaw/plugin-sdk/channel-actions"; +import { canonicalizeBase64, estimateBase64DecodedBytes } from "openclaw/plugin-sdk/media-runtime"; import { isWhatsAppGroupJid, resolveAuthorizedWhatsAppOutboundTarget, @@ -75,18 +76,8 @@ function readWhatsAppActionChatJid(params: WhatsAppMessageActionParams): string } function extractBase64Payload(encoded: string): string { - const match = /^data:[^;]+;base64,(.*)$/i.exec(encoded.trim()); - const payload = match?.[1]; - return payload !== undefined ? payload : encoded; -} - -function estimateBase64DecodedBytes(encoded: string): number { - const compact = extractBase64Payload(encoded).replace(/\s/g, ""); - if (!compact) { - return 0; - } - const padding = compact.endsWith("==") ? 2 : compact.endsWith("=") ? 1 : 0; - return Math.max(0, Math.floor((compact.length * 3) / 4) - padding); + const match = /^data:[^;]+;base64,(.*)$/is.exec(encoded.trim()); + return match?.[1] ?? encoded; } function decodeUploadFileMediaPayload(params: { @@ -100,8 +91,11 @@ function decodeUploadFileMediaPayload(params: { fileName?: string; } | undefined { + const payload = extractBase64Payload(params.encoded); if (params.maxBytes !== undefined) { - const estimatedBytes = estimateBase64DecodedBytes(params.encoded); + // Enforce the budget before canonicalization and decode so hostile input cannot force an + // oversized Buffer allocation before rejection. + const estimatedBytes = estimateBase64DecodedBytes(payload); if (estimatedBytes > params.maxBytes) { throw new Error( `WhatsApp upload-file buffer exceeds configured media limit (${estimatedBytes} bytes > ${params.maxBytes} bytes).`, @@ -112,7 +106,11 @@ function decodeUploadFileMediaPayload(params: { readStringParam(params.args, "contentType") ?? readStringParam(params.args, "mimeType"); const fileName = readStringParam(params.args, "filename") ?? readStringParam(params.args, "fileName"); - const buffer = Buffer.from(extractBase64Payload(params.encoded), "base64"); + const canonicalPayload = canonicalizeBase64(payload); + if (!canonicalPayload) { + throw new Error("WhatsApp upload-file buffer must be valid base64 or a base64 data URL."); + } + const buffer = Buffer.from(canonicalPayload, "base64"); if (params.maxBytes !== undefined && buffer.byteLength > params.maxBytes) { throw new Error( `WhatsApp upload-file buffer exceeds configured media limit (${buffer.byteLength} bytes > ${params.maxBytes} bytes).`, diff --git a/extensions/whatsapp/src/outbound-media-contract.ts b/extensions/whatsapp/src/outbound-media-contract.ts index 7bfe146b8373..e206dc195a2c 100644 --- a/extensions/whatsapp/src/outbound-media-contract.ts +++ b/extensions/whatsapp/src/outbound-media-contract.ts @@ -1,11 +1,13 @@ // Whatsapp plugin module implements outbound media contract behavior. import path from "node:path"; import { sanitizeForPlainText } from "openclaw/plugin-sdk/channel-outbound"; -import { MEDIA_FFMPEG_MAX_AUDIO_DURATION_SECS, runFfmpeg } from "openclaw/plugin-sdk/media-runtime"; +import { mediaKindFromMime, normalizeMimeType } from "openclaw/plugin-sdk/media-mime"; +import { + MEDIA_FFMPEG_MAX_AUDIO_DURATION_SECS, + transcodeAudioBufferToOpus, +} from "openclaw/plugin-sdk/media-runtime"; import { resolveOutboundMediaUrls } from "openclaw/plugin-sdk/reply-payload"; -import { writeExternalFileWithinRoot } from "openclaw/plugin-sdk/security-runtime"; import { normalizeUniqueStringEntries } from "openclaw/plugin-sdk/string-coerce-runtime"; -import { resolvePreferredOpenClawTmpDir, withTempWorkspace } from "openclaw/plugin-sdk/temp-path"; import { resolveWhatsAppDocumentFileName } from "./document-filename.js"; import { sanitizeAssistantVisibleText, @@ -123,17 +125,7 @@ function inferWhatsAppMediaKind( ) { return media.kind; } - const contentType = normalizeContentType(media.contentType); - if (contentType.startsWith("image/")) { - return "image"; - } - if (contentType.startsWith("audio/")) { - return "audio"; - } - if (contentType.startsWith("video/")) { - return "video"; - } - return "document"; + return mediaKindFromMime(normalizeMimeType(media.contentType)) ?? "document"; } function normalizeWhatsAppLoadedMedia( @@ -189,16 +181,12 @@ export async function prepareWhatsAppOutboundMedia( }; } -function normalizeContentType(value: string | undefined): string { - return value?.split(";", 1)[0]?.trim().toLowerCase() ?? ""; -} - function isWhatsAppNativeVoiceAudio(params: { contentType?: string; fileName?: string; mediaUrl?: string; }): boolean { - const contentType = normalizeContentType(params.contentType); + const contentType = normalizeMimeType(params.contentType); if (contentType === "audio/ogg" || contentType === "audio/opus") { return true; } @@ -211,45 +199,16 @@ async function transcodeToWhatsAppVoiceOpus(params: { buffer: Buffer; fileName: string; }): Promise { - return await withTempWorkspace( - { rootDir: resolvePreferredOpenClawTmpDir(), prefix: "whatsapp-voice-" }, - async (workspace) => { - const ext = path.extname(params.fileName).toLowerCase(); - const inputExt = ext && ext.length <= 12 ? ext : ".audio"; - const inputPath = await workspace.write(`input${inputExt}`, params.buffer); - await writeExternalFileWithinRoot({ - rootDir: workspace.dir, - path: WHATSAPP_VOICE_FILE_NAME, - write: async (outputPath) => { - await runFfmpeg([ - "-hide_banner", - "-loglevel", - "error", - "-y", - "-i", - inputPath, - "-vn", - "-sn", - "-dn", - "-t", - String(MEDIA_FFMPEG_MAX_AUDIO_DURATION_SECS), - "-ar", - String(WHATSAPP_VOICE_SAMPLE_RATE_HZ), - "-ac", - "1", - "-c:a", - "libopus", - "-b:a", - WHATSAPP_VOICE_BITRATE, - "-f", - "ogg", - outputPath, - ]); - }, - }); - return await workspace.read(WHATSAPP_VOICE_FILE_NAME); - }, - ); + return await transcodeAudioBufferToOpus({ + audioBuffer: params.buffer, + inputFileName: params.fileName, + tempPrefix: "whatsapp-voice-", + outputFileName: WHATSAPP_VOICE_FILE_NAME, + maxDurationSeconds: MEDIA_FFMPEG_MAX_AUDIO_DURATION_SECS, + sampleRateHz: WHATSAPP_VOICE_SAMPLE_RATE_HZ, + channels: 1, + bitrate: WHATSAPP_VOICE_BITRATE, + }); } function deriveWhatsAppDocumentFileName(mediaUrl: string | undefined): string | undefined { diff --git a/extensions/whatsapp/src/outbound-media.runtime.ts b/extensions/whatsapp/src/outbound-media.runtime.ts deleted file mode 100644 index e042fdf2371c..000000000000 --- a/extensions/whatsapp/src/outbound-media.runtime.ts +++ /dev/null @@ -1,42 +0,0 @@ -// Whatsapp plugin module implements outbound media behavior. -import { loadWebMedia } from "openclaw/plugin-sdk/web-media"; - -export async function loadOutboundMediaFromUrl( - mediaUrl: string, - options: { - maxBytes?: number; - mediaAccess?: { - localRoots?: readonly string[]; - readFile?: (filePath: string) => Promise; - }; - mediaLocalRoots?: readonly string[]; - mediaReadFile?: (filePath: string) => Promise; - optimizeImages?: boolean; - } = {}, -) { - const readFile = options.mediaAccess?.readFile ?? options.mediaReadFile; - const localRoots = - options.mediaAccess?.localRoots?.length && options.mediaAccess.localRoots.length > 0 - ? options.mediaAccess.localRoots - : options.mediaLocalRoots && options.mediaLocalRoots.length > 0 - ? options.mediaLocalRoots - : undefined; - const sharedOptions = { - ...(options.maxBytes !== undefined ? { maxBytes: options.maxBytes } : {}), - ...(options.optimizeImages !== undefined ? { optimizeImages: options.optimizeImages } : {}), - }; - return await loadWebMedia( - mediaUrl, - readFile - ? { - ...sharedOptions, - localRoots: "any", - readFile, - hostReadCapability: true, - } - : { - ...sharedOptions, - ...(localRoots ? { localRoots } : {}), - }, - ); -} diff --git a/extensions/whatsapp/src/runtime-api.ts b/extensions/whatsapp/src/runtime-api.ts index 4f784d4456a9..279be0958dfb 100644 --- a/extensions/whatsapp/src/runtime-api.ts +++ b/extensions/whatsapp/src/runtime-api.ts @@ -20,7 +20,6 @@ import type { OpenClawConfig as RuntimeOpenClawConfig } from "openclaw/plugin-sd import { loadWhatsAppChannelRuntime } from "./channel-runtime-loader.js"; export { type ChannelMessageActionName } from "openclaw/plugin-sdk/channel-contract"; -export { loadOutboundMediaFromUrl } from "./outbound-media.runtime.js"; export { resolveWhatsAppGroupRequireMention, resolveWhatsAppGroupToolPolicy, diff --git a/extensions/whatsapp/src/send.test.ts b/extensions/whatsapp/src/send.test.ts index aa57648f13c5..36df13069ec6 100644 --- a/extensions/whatsapp/src/send.test.ts +++ b/extensions/whatsapp/src/send.test.ts @@ -14,7 +14,7 @@ import type { ActiveWebListener } from "./inbound/types.js"; const hoisted = vi.hoisted(() => ({ loadOutboundMediaFromUrl: vi.fn(), controllerListeners: new Map(), - runFfmpeg: vi.fn(), + transcodeAudioBufferToOpus: vi.fn(), })); const loadWebMediaMock = vi.fn(); let sendMessageWhatsApp: typeof import("./send.js").sendMessageWhatsApp; @@ -45,9 +45,9 @@ vi.mock("./connection-controller-runtime-context.js", async () => { }; }); -vi.mock("./outbound-media.runtime.js", async () => { - const actual = await vi.importActual( - "./outbound-media.runtime.js", +vi.mock("openclaw/plugin-sdk/outbound-media", async () => { + const actual = await vi.importActual( + "openclaw/plugin-sdk/outbound-media", ); return { ...actual, @@ -61,7 +61,7 @@ vi.mock("openclaw/plugin-sdk/media-runtime", async () => { ); return { ...actual, - runFfmpeg: hoisted.runFfmpeg, + transcodeAudioBufferToOpus: hoisted.transcodeAudioBufferToOpus, }; }); @@ -92,10 +92,7 @@ describe("web outbound", () => { beforeEach(() => { vi.clearAllMocks(); - hoisted.runFfmpeg.mockReset().mockImplementation(async (args: string[]) => { - fsSync.writeFileSync(args.at(-1) ?? "", Buffer.from("opus-output")); - return ""; - }); + hoisted.transcodeAudioBufferToOpus.mockReset().mockResolvedValue(Buffer.from("opus-output")); hoisted.loadOutboundMediaFromUrl.mockReset().mockImplementation( async ( mediaUrl: string, @@ -361,6 +358,27 @@ describe("web outbound", () => { expect(sendMessage).toHaveBeenNthCalledWith(2, "+1555", "voice note", undefined, undefined); }); + it("normalizes MIME parameters when inferring media kind", async () => { + const buf = Buffer.from("image"); + loadWebMediaMock.mockResolvedValueOnce({ + buffer: buf, + contentType: " Image/PNG; charset=binary ", + }); + + await sendMessageWhatsApp("+1555", "caption", { + verbose: false, + cfg: WHATSAPP_TEST_CFG, + mediaUrl: "/tmp/image.png", + }); + + expect(sendMessage).toHaveBeenLastCalledWith( + "+1555", + "caption", + buf, + " Image/PNG; charset=binary ", + ); + }); + it("reports the accepted voice send before a caption failure", async () => { const buf = Buffer.from("audio"); loadWebMediaMock.mockResolvedValueOnce({ @@ -390,6 +408,7 @@ describe("web outbound", () => { it.each([ { name: "mp3", contentType: "audio/mpeg", fileName: "voice.mp3" }, + { name: "m4a", contentType: "audio/mp4; codecs=mp4a.40.2", fileName: "voice.m4a" }, { name: "webm", contentType: "audio/webm", fileName: "voice.webm" }, ])("transcodes $name audio to Ogg Opus before sending a PTT voice note", async (media) => { const buf = Buffer.from(media.name); @@ -406,30 +425,16 @@ describe("web outbound", () => { mediaUrl: `/tmp/${media.fileName}`, }); - expect(hoisted.runFfmpeg).toHaveBeenCalledTimes(1); - const ffmpegArgs = hoisted.runFfmpeg.mock.calls.at(0)?.[0] as string[] | undefined; - expect(ffmpegArgs?.slice(0, 5)).toEqual(["-hide_banner", "-loglevel", "error", "-y", "-i"]); - expect(ffmpegArgs?.[5]).toContain(`/input.${media.name}`); - expect(ffmpegArgs?.slice(6, -1)).toEqual([ - "-vn", - "-sn", - "-dn", - "-t", - String(MEDIA_FFMPEG_MAX_AUDIO_DURATION_SECS), - "-ar", - "48000", - "-ac", - "1", - "-c:a", - "libopus", - "-b:a", - "64k", - "-f", - "ogg", - ]); - const outputPath = ffmpegArgs?.at(-1); - expect(outputPath).toContain("/fs-safe-output-"); - expect(outputPath).toContain("-voice.ogg.part"); + expect(hoisted.transcodeAudioBufferToOpus).toHaveBeenCalledWith({ + audioBuffer: buf, + inputFileName: media.fileName, + tempPrefix: "whatsapp-voice-", + outputFileName: "voice.ogg", + maxDurationSeconds: MEDIA_FFMPEG_MAX_AUDIO_DURATION_SECS, + sampleRateHz: 48000, + channels: 1, + bitrate: "64k", + }); expect(sendMessage).toHaveBeenNthCalledWith( 1, "+1555", diff --git a/extensions/whatsapp/src/send.ts b/extensions/whatsapp/src/send.ts index f004e0cddbfe..71de26ee7425 100644 --- a/extensions/whatsapp/src/send.ts +++ b/extensions/whatsapp/src/send.ts @@ -8,6 +8,7 @@ import { convertMarkdownTables, resolveMarkdownTableMode, } from "openclaw/plugin-sdk/markdown-table-runtime"; +import { loadOutboundMediaFromUrl } from "openclaw/plugin-sdk/outbound-media"; import { requireRuntimeConfig } from "openclaw/plugin-sdk/plugin-config-runtime"; import { normalizePollInput, type PollInput } from "openclaw/plugin-sdk/poll-runtime"; import { createSubsystemLogger, getChildLogger } from "openclaw/plugin-sdk/runtime-env"; @@ -25,7 +26,6 @@ import { prepareWhatsAppOutboundMedia, resolveAdditiveWhatsAppMediaUrls, } from "./outbound-media-contract.js"; -import { loadOutboundMediaFromUrl } from "./outbound-media.runtime.js"; import { markdownToWhatsApp, toWhatsappJid } from "./text-runtime.js"; const outboundLog = createSubsystemLogger("gateway/channels/whatsapp").child("outbound"); @@ -198,6 +198,8 @@ export async function sendMessageWhatsApp( if (mediaPayload) { media = await prepareWhatsAppOutboundMedia(mediaPayload, primaryMediaUrl); } else if (primaryMediaUrl) { + // Injected readers must carry an explicit local-root boundary. The shared loader enforces + // that contract; never restore the former implicit `localRoots: "any"` widening here. media = await prepareWhatsAppOutboundMedia( await loadOutboundMediaFromUrl(primaryMediaUrl, { maxBytes: resolveWhatsAppMediaMaxBytes(account), diff --git a/src/media/audio-transcode.test.ts b/src/media/audio-transcode.test.ts index a7c3af9e8049..571ede0fa777 100644 --- a/src/media/audio-transcode.test.ts +++ b/src/media/audio-transcode.test.ts @@ -1,6 +1,6 @@ // Audio transcode tests cover ffmpeg-backed audio conversion behavior. import { existsSync, realpathSync } from "node:fs"; -import { readFile } from "node:fs/promises"; +import { readFile, writeFile } from "node:fs/promises"; import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; import { resolvePreferredOpenClawTmpDir } from "../infra/tmp-openclaw-dir.js"; @@ -105,6 +105,24 @@ describe("transcodeAudioBufferToOpus", () => { }); }); + it("passes the maximum duration to ffmpeg when requested", async () => { + runFfmpegMock.mockImplementationOnce(async (args: string[]) => { + const outputPath = args.at(-1); + if (!outputPath) { + throw new Error("missing ffmpeg output path"); + } + await writeFile(outputPath, Buffer.from("ogg-opus-output")); + }); + + await transcodeAudioBufferToOpus({ + audioBuffer: Buffer.from("source-m4a"), + maxDurationSeconds: 300, + }); + + const ffmpegArgs = firstMockCall(runFfmpegMock, "runFfmpeg")[0] as string[]; + expect(ffmpegArgs).toEqual(expect.arrayContaining(["-t", "300", "-c:a", "libopus"])); + }); + it("keeps temp prefixes and output names inside the preferred temp root", async () => { let capturedInputPath: string | undefined; let capturedOutputPath: string | undefined; diff --git a/src/media/audio-transcode.ts b/src/media/audio-transcode.ts index ec2e07a7cb78..191960db2eae 100644 --- a/src/media/audio-transcode.ts +++ b/src/media/audio-transcode.ts @@ -43,6 +43,16 @@ function normalizeOutputFileName(value?: string): string { return DEFAULT_OUTPUT_FILE_NAME; } +function resolveMaxDurationSeconds(value?: number): number | undefined { + if (value === undefined) { + return undefined; + } + if (!Number.isFinite(value) || value <= 0) { + throw new Error("maxDurationSeconds must be a positive finite number"); + } + return value; +} + /** Transcodes arbitrary audio input into mono Opus using a scoped temp workspace. */ export async function transcodeAudioBufferToOpus(params: { audioBuffer: Buffer; @@ -54,7 +64,10 @@ export async function transcodeAudioBufferToOpus(params: { sampleRateHz?: number; bitrate?: string; channels?: number; + /** Maximum output duration passed to ffmpeg's `-t` option. */ + maxDurationSeconds?: number; }): Promise { + const maxDurationSeconds = resolveMaxDurationSeconds(params.maxDurationSeconds); return await withTempWorkspace( { rootDir: resolvePreferredOpenClawTmpDir(), @@ -81,6 +94,7 @@ export async function transcodeAudioBufferToOpus(params: { "-vn", "-sn", "-dn", + ...(maxDurationSeconds === undefined ? [] : ["-t", String(maxDurationSeconds)]), "-c:a", "libopus", "-b:a", diff --git a/src/plugin-sdk/outbound-media.test.ts b/src/plugin-sdk/outbound-media.test.ts index d94e0522df88..0aee92f5e96c 100644 --- a/src/plugin-sdk/outbound-media.test.ts +++ b/src/plugin-sdk/outbound-media.test.ts @@ -34,7 +34,7 @@ beforeEach(() => { }); describe("loadOutboundMediaFromUrl", () => { - it("forwards maxBytes and mediaLocalRoots to loadWebMedia", async () => { + it("forwards media limits, local roots, and image optimization to loadWebMedia", async () => { loadWebMediaMock.mockResolvedValueOnce({ buffer: Buffer.from("x"), kind: "image", @@ -44,11 +44,13 @@ describe("loadOutboundMediaFromUrl", () => { await loadOutboundMediaFromUrl("file:///tmp/image.png", { maxBytes: 1024, mediaLocalRoots: ["/tmp/workspace-agent"], + optimizeImages: false, }); expect(loadWebMediaMock).toHaveBeenCalledWith("file:///tmp/image.png", { maxBytes: 1024, localRoots: ["/tmp/workspace-agent"], + optimizeImages: false, }); }); diff --git a/src/plugin-sdk/outbound-media.ts b/src/plugin-sdk/outbound-media.ts index c7cb3591af71..b941267f4c44 100644 --- a/src/plugin-sdk/outbound-media.ts +++ b/src/plugin-sdk/outbound-media.ts @@ -22,6 +22,8 @@ export type OutboundMediaLoadOptions = { fetchImpl?: (input: RequestInfo | URL, init?: RequestInit) => Promise; /** Extra fetch options merged into remote media requests. */ requestInit?: RequestInit; + /** Whether shared media loading may optimize image payloads. */ + optimizeImages?: boolean; /** Allows explicit proxy DNS behavior to be trusted by the media fetch guard. */ trustExplicitProxyDns?: boolean; }; @@ -42,6 +44,7 @@ export async function loadOutboundMediaFromUrl( proxyUrl: options.proxyUrl, fetchImpl: options.fetchImpl, requestInit: options.requestInit, + optimizeImages: options.optimizeImages, trustExplicitProxyDns: options.trustExplicitProxyDns, }), );