diff --git a/extensions/telegram/src/bot/delivery.resolve-media-retry.test.ts b/extensions/telegram/src/bot/delivery.resolve-media-retry.test.ts index 063bc2a3975d..28ad0688d9c0 100644 --- a/extensions/telegram/src/bot/delivery.resolve-media-retry.test.ts +++ b/extensions/telegram/src/bot/delivery.resolve-media-retry.test.ts @@ -1011,6 +1011,27 @@ describe("resolveMedia original filename preservation", () => { }); }); + it("classifies an audio document from the saved MIME type", async () => { + const getFile = vi.fn().mockResolvedValue({ file_path: "documents/recording.m2a" }); + saveRemoteMedia.mockResolvedValueOnce({ + path: "/tmp/inbound/recording.m2a", + contentType: "audio/mpeg", + }); + + const result = await resolveMediaWithDefaults( + makeCtx("document", getFile, { + file_name: "recording.m2a", + mime_type: "application/octet-stream", + }), + ); + + expectResolvedMediaFields(result, "MPEG-2 audio document", { + path: "/tmp/inbound/recording.m2a", + contentType: "audio/mpeg", + placeholder: "", + }); + }); + it("passes audio.file_name to saveMediaBuffer", async () => { const getFile = vi.fn().mockResolvedValue({ file_path: "music/file_99.mp3" }); readRemoteMediaBuffer.mockResolvedValueOnce({ diff --git a/extensions/telegram/src/bot/delivery.resolve-media.ts b/extensions/telegram/src/bot/delivery.resolve-media.ts index b92cf9b391e4..1af89640b62c 100644 --- a/extensions/telegram/src/bot/delivery.resolve-media.ts +++ b/extensions/telegram/src/bot/delivery.resolve-media.ts @@ -517,6 +517,8 @@ export async function resolveMedia(params: { trustedLocalFileRoots, dangerouslyAllowPrivateNetwork, }); - const placeholder = resolveTelegramMediaPlaceholder(msg) ?? ""; + const placeholder = saved.contentType?.startsWith("audio/") + ? "" + : (resolveTelegramMediaPlaceholder(msg) ?? ""); return { path: saved.path, contentType: saved.contentType, placeholder }; } diff --git a/packages/media-core/src/mime.test.ts b/packages/media-core/src/mime.test.ts index 6de70cb99fd1..c10a6d3718de 100644 --- a/packages/media-core/src/mime.test.ts +++ b/packages/media-core/src/mime.test.ts @@ -191,6 +191,8 @@ describe("mimeTypeFromFilePath", () => { { filePath: "photo.jpg", expected: "image/jpeg" }, { filePath: "photo.JPG", expected: "image/jpeg" }, { filePath: "voice.mp3", expected: "audio/mpeg" }, + { filePath: "voice.m2a", expected: "audio/mpeg" }, + { filePath: "voice.oga", expected: "audio/ogg" }, { filePath: "voice.wav", expected: "audio/wav" }, { filePath: "clip.avi", expected: "video/x-msvideo" }, { filePath: "clip.mkv", expected: "video/x-matroska" }, @@ -266,6 +268,9 @@ describe("isAudioFileName", () => { it.each([ { fileName: "voice.mp3", expected: true }, { fileName: "voice.caf", expected: true }, + { fileName: "voice.M2A", expected: true }, + { fileName: "voice.oga", expected: true }, + { fileName: "voice.webm", expected: false }, { fileName: "voice.bin", expected: false }, ] as const)("matches audio extension for $fileName", ({ fileName, expected }) => { expectAudioFileNameCase(fileName, expected); diff --git a/packages/media-core/src/mime.ts b/packages/media-core/src/mime.ts index 8542acba3047..2ca219d35ecb 100644 --- a/packages/media-core/src/mime.ts +++ b/packages/media-core/src/mime.ts @@ -72,7 +72,9 @@ const MIME_BY_EXT: Record = { ...buildMimeByExt(), // Canonical extension mappings for common MIME aliases ".jpg": "image/jpeg", + ".m2a": "audio/mpeg", ".mp3": "audio/mpeg", + ".oga": "audio/ogg", ".wav": "audio/wav", ".webm": "video/webm", // Additional extension aliases @@ -84,18 +86,6 @@ const MIME_BY_EXT: Record = { ".yml": "application/yaml", }; -const AUDIO_FILE_EXTENSIONS = new Set([ - ".aac", - ".caf", - ".flac", - ".m4a", - ".mp3", - ".oga", - ".ogg", - ".opus", - ".wav", -]); - const fileTypeModuleLoader = createLazyImportLoader(() => import("file-type")); /** Normalizes MIME strings by dropping parameters, lowercasing, and folding APNG to PNG. */ @@ -174,11 +164,7 @@ export function mimeTypeFromFilePath(filePath?: string | null): string | undefin /** Returns true when a filename extension is a supported audio container. */ export function isAudioFileName(fileName?: string | null): boolean { - const ext = getFileExtension(fileName); - if (!ext) { - return false; - } - return AUDIO_FILE_EXTENSIONS.has(ext); + return mediaKindFromMime(mimeTypeFromFilePath(fileName)) === "audio"; } /** Detects the best MIME type from bytes, file path, and header metadata. */ diff --git a/packages/memory-host-sdk/src/host/internal.test.ts b/packages/memory-host-sdk/src/host/internal.test.ts index 0a5489f5c542..84499f66aaf1 100644 --- a/packages/memory-host-sdk/src/host/internal.test.ts +++ b/packages/memory-host-sdk/src/host/internal.test.ts @@ -124,6 +124,7 @@ describe("memory host SDK package internals", () => { fsSync.mkdirSync(extraDir, { recursive: true }); fsSync.writeFileSync(path.join(extraDir, "note.md"), "# Note"); fsSync.writeFileSync(path.join(extraDir, "diagram.png"), Buffer.from("png")); + fsSync.writeFileSync(path.join(extraDir, "recording.m2a"), Buffer.from("audio")); fsSync.writeFileSync(path.join(extraDir, "ignore.txt"), "ignored"); const files = await listMemoryFiles( @@ -136,6 +137,7 @@ describe("memory host SDK package internals", () => { "MEMORY.md", path.join("extra", "diagram.png"), path.join("extra", "note.md"), + path.join("extra", "recording.m2a"), ]); }); diff --git a/packages/memory-host-sdk/src/host/multimodal.ts b/packages/memory-host-sdk/src/host/multimodal.ts index 0060302976ec..42cb6fd6dc6d 100644 --- a/packages/memory-host-sdk/src/host/multimodal.ts +++ b/packages/memory-host-sdk/src/host/multimodal.ts @@ -10,7 +10,7 @@ const MEMORY_MULTIMODAL_SPECS = { }, audio: { labelPrefix: "Audio file", - extensions: [".mp3", ".wav", ".ogg", ".opus", ".m4a", ".aac", ".flac"], + extensions: [".mp3", ".wav", ".ogg", ".opus", ".m4a", ".m2a", ".aac", ".flac"], }, } as const; diff --git a/src/auto-reply/media-note.test.ts b/src/auto-reply/media-note.test.ts index 1f4b63920c90..0b90a6bf1218 100644 --- a/src/auto-reply/media-note.test.ts +++ b/src/auto-reply/media-note.test.ts @@ -317,6 +317,21 @@ describe("buildInboundMediaNote", () => { expect(note).toBe("[media attached: /tmp/document.pdf]"); }); + it("strips transcribed MPEG-2 audio by extension", () => { + const note = buildInboundMediaNote({ + MediaPaths: ["/tmp/recording.m2a", "/tmp/document.pdf"], + MediaUnderstanding: [ + { + kind: "audio.transcription", + attachmentIndex: 0, + text: "Transcribed audio content", + provider: "whisper", + }, + ], + }); + expect(note).toBe("[media attached: /tmp/document.pdf]"); + }); + it("keeps audio attachments when no transcription is available", () => { const note = buildInboundMediaNote({ MediaPaths: ["/tmp/voice.ogg"], diff --git a/src/auto-reply/media-note.ts b/src/auto-reply/media-note.ts index 34f97b83e593..bff5c9acbe04 100644 --- a/src/auto-reply/media-note.ts +++ b/src/auto-reply/media-note.ts @@ -66,6 +66,7 @@ const AUDIO_EXTENSIONS = new Set([ ".opus", ".mp3", ".m4a", + ".m2a", ".wav", ".webm", ".flac", diff --git a/src/gateway/server-methods/chat-webchat-media.test.ts b/src/gateway/server-methods/chat-webchat-media.test.ts index 29f5f617a968..dc655fedcea3 100644 --- a/src/gateway/server-methods/chat-webchat-media.test.ts +++ b/src/gateway/server-methods/chat-webchat-media.test.ts @@ -22,9 +22,9 @@ describe("buildWebchatAudioContentBlocksFromReplyPayloads", () => { tmpDir = undefined; }); - function writeAudioFixture(bytes = [0xff, 0xfb, 0x90, 0x00]) { + function writeAudioFixture(bytes = [0xff, 0xfb, 0x90, 0x00], extension = ".mp3") { tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-webchat-audio-")); - const audioPath = path.join(tmpDir, "clip.mp3"); + const audioPath = path.join(tmpDir, `clip${extension}`); fs.writeFileSync(audioPath, Buffer.from(bytes)); return { audioPath, localRoot: tmpDir }; } @@ -51,6 +51,19 @@ describe("buildWebchatAudioContentBlocksFromReplyPayloads", () => { }); }); + it("exposes MPEG-2 audio files with their canonical MIME type", async () => { + const { audioPath, localRoot } = writeAudioFixture([0xff, 0xfd, 0x80, 0x00], ".m2a"); + + const blocks = await buildWebchatAudioContentBlocksFromReplyPayloads( + [{ mediaUrl: audioPath, trustedLocalMedia: true }], + { localRoots: [localRoot] }, + ); + + expect(blocks[0]).toMatchObject({ + attachment: { label: "clip.m2a", kind: "audio", mimeType: "audio/mpeg" }, + }); + }); + it("preserves voice-note metadata on local audio attachments", async () => { const { audioPath, localRoot } = writeAudioFixture(); diff --git a/src/gateway/server-methods/chat-webchat-media.ts b/src/gateway/server-methods/chat-webchat-media.ts index 9c5d83b247b3..94547f842a79 100644 --- a/src/gateway/server-methods/chat-webchat-media.ts +++ b/src/gateway/server-methods/chat-webchat-media.ts @@ -2,7 +2,7 @@ // blocks that the control UI can render without unsafe file exposure. import path from "node:path"; import { estimateBase64DecodedBytes } from "@openclaw/media-core/base64"; -import { isAudioFileName } from "@openclaw/media-core/mime"; +import { isAudioFileName, mimeTypeFromFilePath } from "@openclaw/media-core/mime"; import { normalizeLowercaseStringOrEmpty } from "@openclaw/normalization-core/string-coerce"; import type { ReplyPayload } from "../../auto-reply/reply-payload.js"; import { openLocalFileSafely } from "../../infra/fs-safe.js"; @@ -26,17 +26,6 @@ const ALLOWED_WEBCHAT_DATA_IMAGE_MEDIA_TYPES = new Set([ "image/webp", ]); -const MIME_BY_EXT: Record = { - ".aac": "audio/aac", - ".m4a": "audio/mp4", - ".mp3": "audio/mpeg", - ".oga": "audio/ogg", - ".ogg": "audio/ogg", - ".opus": "audio/opus", - ".wav": "audio/wav", - ".webm": "audio/webm", -}; - type WebchatAudioEmbeddingOptions = { localRoots?: readonly string[]; onLocalAudioAccessDenied?: (err: LocalMediaAccessError) => void; @@ -152,8 +141,7 @@ async function resolveReplyMediaAudioEmbedding( } function mimeTypeForPath(filePath: string): string { - const ext = normalizeLowercaseStringOrEmpty(path.extname(filePath)); - return MIME_BY_EXT[ext] ?? "audio/mpeg"; + return mimeTypeFromFilePath(filePath) ?? "audio/mpeg"; } function isBase64DataPayload(value: string): boolean { diff --git a/ui/src/lib/chat/message-normalizer.test.ts b/ui/src/lib/chat/message-normalizer.test.ts index be2e841a9f63..d2aa14ddd3d8 100644 --- a/ui/src/lib/chat/message-normalizer.test.ts +++ b/ui/src/lib/chat/message-normalizer.test.ts @@ -335,6 +335,25 @@ describe("message-normalizer", () => { ]); }); + it("classifies MPEG-2 audio attachments", () => { + const result = normalizeMessage({ + role: "assistant", + content: "MEDIA:https://example.com/recording.m2a", + }); + + expect(result.content).toEqual([ + { + type: "attachment", + attachment: { + url: "https://example.com/recording.m2a", + kind: "audio", + label: "recording.m2a", + mimeType: "audio/mpeg", + }, + }, + ]); + }); + it("keeps valid local MEDIA paths as assistant attachments", () => { const result = normalizeMessage({ role: "assistant", diff --git a/ui/src/lib/chat/message-normalizer.ts b/ui/src/lib/chat/message-normalizer.ts index bd2592e61b62..c5dd56acd1b2 100644 --- a/ui/src/lib/chat/message-normalizer.ts +++ b/ui/src/lib/chat/message-normalizer.ts @@ -144,6 +144,7 @@ const MIME_BY_EXT: Record = { aac: "audio/aac", opus: "audio/opus", m4a: "audio/mp4", + m2a: "audio/mpeg", mp4: "video/mp4", mov: "video/quicktime", pdf: "application/pdf", diff --git a/ui/src/pages/chat/components/chat-message.ts b/ui/src/pages/chat/components/chat-message.ts index d478d0338f50..d6bd6a937be2 100644 --- a/ui/src/pages/chat/components/chat-message.ts +++ b/ui/src/pages/chat/components/chat-message.ts @@ -290,7 +290,8 @@ function isAudioTranscriptMediaPath(path: string, mediaType: unknown): boolean { } const ext = getFileExtension(path); return ( - ext !== undefined && ["aac", "flac", "m4a", "mp3", "oga", "ogg", "opus", "wav"].includes(ext) + ext !== undefined && + ["aac", "flac", "m2a", "m4a", "mp3", "oga", "ogg", "opus", "wav"].includes(ext) ); }