fix(media): recognize m2a as MPEG audio (#92167)

Co-authored-by: Peter Steinberger <steipete@gmail.com>
This commit is contained in:
ZZIPP
2026-07-07 00:48:37 +08:00
committed by GitHub
parent 077e35f3b2
commit 979d198fd1
13 changed files with 90 additions and 36 deletions

View File

@@ -1011,6 +1011,27 @@ describe("resolveMedia original filename preservation", () => {
});
});
it("classifies an audio document from the saved MIME type", async () => {
const getFile = vi.fn().mockResolvedValue({ file_path: "documents/recording.m2a" });
saveRemoteMedia.mockResolvedValueOnce({
path: "/tmp/inbound/recording.m2a",
contentType: "audio/mpeg",
});
const result = await resolveMediaWithDefaults(
makeCtx("document", getFile, {
file_name: "recording.m2a",
mime_type: "application/octet-stream",
}),
);
expectResolvedMediaFields(result, "MPEG-2 audio document", {
path: "/tmp/inbound/recording.m2a",
contentType: "audio/mpeg",
placeholder: "<media:audio>",
});
});
it("passes audio.file_name to saveMediaBuffer", async () => {
const getFile = vi.fn().mockResolvedValue({ file_path: "music/file_99.mp3" });
readRemoteMediaBuffer.mockResolvedValueOnce({

View File

@@ -517,6 +517,8 @@ export async function resolveMedia(params: {
trustedLocalFileRoots,
dangerouslyAllowPrivateNetwork,
});
const placeholder = resolveTelegramMediaPlaceholder(msg) ?? "<media:document>";
const placeholder = saved.contentType?.startsWith("audio/")
? "<media:audio>"
: (resolveTelegramMediaPlaceholder(msg) ?? "<media:document>");
return { path: saved.path, contentType: saved.contentType, placeholder };
}

View File

@@ -191,6 +191,8 @@ describe("mimeTypeFromFilePath", () => {
{ filePath: "photo.jpg", expected: "image/jpeg" },
{ filePath: "photo.JPG", expected: "image/jpeg" },
{ filePath: "voice.mp3", expected: "audio/mpeg" },
{ filePath: "voice.m2a", expected: "audio/mpeg" },
{ filePath: "voice.oga", expected: "audio/ogg" },
{ filePath: "voice.wav", expected: "audio/wav" },
{ filePath: "clip.avi", expected: "video/x-msvideo" },
{ filePath: "clip.mkv", expected: "video/x-matroska" },
@@ -266,6 +268,9 @@ describe("isAudioFileName", () => {
it.each([
{ fileName: "voice.mp3", expected: true },
{ fileName: "voice.caf", expected: true },
{ fileName: "voice.M2A", expected: true },
{ fileName: "voice.oga", expected: true },
{ fileName: "voice.webm", expected: false },
{ fileName: "voice.bin", expected: false },
] as const)("matches audio extension for $fileName", ({ fileName, expected }) => {
expectAudioFileNameCase(fileName, expected);

View File

@@ -72,7 +72,9 @@ const MIME_BY_EXT: Record<string, string> = {
...buildMimeByExt(),
// Canonical extension mappings for common MIME aliases
".jpg": "image/jpeg",
".m2a": "audio/mpeg",
".mp3": "audio/mpeg",
".oga": "audio/ogg",
".wav": "audio/wav",
".webm": "video/webm",
// Additional extension aliases
@@ -84,18 +86,6 @@ const MIME_BY_EXT: Record<string, string> = {
".yml": "application/yaml",
};
const AUDIO_FILE_EXTENSIONS = new Set([
".aac",
".caf",
".flac",
".m4a",
".mp3",
".oga",
".ogg",
".opus",
".wav",
]);
const fileTypeModuleLoader = createLazyImportLoader(() => import("file-type"));
/** Normalizes MIME strings by dropping parameters, lowercasing, and folding APNG to PNG. */
@@ -174,11 +164,7 @@ export function mimeTypeFromFilePath(filePath?: string | null): string | undefin
/** Returns true when a filename extension is a supported audio container. */
export function isAudioFileName(fileName?: string | null): boolean {
const ext = getFileExtension(fileName);
if (!ext) {
return false;
}
return AUDIO_FILE_EXTENSIONS.has(ext);
return mediaKindFromMime(mimeTypeFromFilePath(fileName)) === "audio";
}
/** Detects the best MIME type from bytes, file path, and header metadata. */

View File

@@ -124,6 +124,7 @@ describe("memory host SDK package internals", () => {
fsSync.mkdirSync(extraDir, { recursive: true });
fsSync.writeFileSync(path.join(extraDir, "note.md"), "# Note");
fsSync.writeFileSync(path.join(extraDir, "diagram.png"), Buffer.from("png"));
fsSync.writeFileSync(path.join(extraDir, "recording.m2a"), Buffer.from("audio"));
fsSync.writeFileSync(path.join(extraDir, "ignore.txt"), "ignored");
const files = await listMemoryFiles(
@@ -136,6 +137,7 @@ describe("memory host SDK package internals", () => {
"MEMORY.md",
path.join("extra", "diagram.png"),
path.join("extra", "note.md"),
path.join("extra", "recording.m2a"),
]);
});

View File

@@ -10,7 +10,7 @@ const MEMORY_MULTIMODAL_SPECS = {
},
audio: {
labelPrefix: "Audio file",
extensions: [".mp3", ".wav", ".ogg", ".opus", ".m4a", ".aac", ".flac"],
extensions: [".mp3", ".wav", ".ogg", ".opus", ".m4a", ".m2a", ".aac", ".flac"],
},
} as const;

View File

@@ -317,6 +317,21 @@ describe("buildInboundMediaNote", () => {
expect(note).toBe("[media attached: /tmp/document.pdf]");
});
it("strips transcribed MPEG-2 audio by extension", () => {
const note = buildInboundMediaNote({
MediaPaths: ["/tmp/recording.m2a", "/tmp/document.pdf"],
MediaUnderstanding: [
{
kind: "audio.transcription",
attachmentIndex: 0,
text: "Transcribed audio content",
provider: "whisper",
},
],
});
expect(note).toBe("[media attached: /tmp/document.pdf]");
});
it("keeps audio attachments when no transcription is available", () => {
const note = buildInboundMediaNote({
MediaPaths: ["/tmp/voice.ogg"],

View File

@@ -66,6 +66,7 @@ const AUDIO_EXTENSIONS = new Set([
".opus",
".mp3",
".m4a",
".m2a",
".wav",
".webm",
".flac",

View File

@@ -22,9 +22,9 @@ describe("buildWebchatAudioContentBlocksFromReplyPayloads", () => {
tmpDir = undefined;
});
function writeAudioFixture(bytes = [0xff, 0xfb, 0x90, 0x00]) {
function writeAudioFixture(bytes = [0xff, 0xfb, 0x90, 0x00], extension = ".mp3") {
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-webchat-audio-"));
const audioPath = path.join(tmpDir, "clip.mp3");
const audioPath = path.join(tmpDir, `clip${extension}`);
fs.writeFileSync(audioPath, Buffer.from(bytes));
return { audioPath, localRoot: tmpDir };
}
@@ -51,6 +51,19 @@ describe("buildWebchatAudioContentBlocksFromReplyPayloads", () => {
});
});
it("exposes MPEG-2 audio files with their canonical MIME type", async () => {
const { audioPath, localRoot } = writeAudioFixture([0xff, 0xfd, 0x80, 0x00], ".m2a");
const blocks = await buildWebchatAudioContentBlocksFromReplyPayloads(
[{ mediaUrl: audioPath, trustedLocalMedia: true }],
{ localRoots: [localRoot] },
);
expect(blocks[0]).toMatchObject({
attachment: { label: "clip.m2a", kind: "audio", mimeType: "audio/mpeg" },
});
});
it("preserves voice-note metadata on local audio attachments", async () => {
const { audioPath, localRoot } = writeAudioFixture();

View File

@@ -2,7 +2,7 @@
// blocks that the control UI can render without unsafe file exposure.
import path from "node:path";
import { estimateBase64DecodedBytes } from "@openclaw/media-core/base64";
import { isAudioFileName } from "@openclaw/media-core/mime";
import { isAudioFileName, mimeTypeFromFilePath } from "@openclaw/media-core/mime";
import { normalizeLowercaseStringOrEmpty } from "@openclaw/normalization-core/string-coerce";
import type { ReplyPayload } from "../../auto-reply/reply-payload.js";
import { openLocalFileSafely } from "../../infra/fs-safe.js";
@@ -26,17 +26,6 @@ const ALLOWED_WEBCHAT_DATA_IMAGE_MEDIA_TYPES = new Set([
"image/webp",
]);
const MIME_BY_EXT: Record<string, string> = {
".aac": "audio/aac",
".m4a": "audio/mp4",
".mp3": "audio/mpeg",
".oga": "audio/ogg",
".ogg": "audio/ogg",
".opus": "audio/opus",
".wav": "audio/wav",
".webm": "audio/webm",
};
type WebchatAudioEmbeddingOptions = {
localRoots?: readonly string[];
onLocalAudioAccessDenied?: (err: LocalMediaAccessError) => void;
@@ -152,8 +141,7 @@ async function resolveReplyMediaAudioEmbedding(
}
function mimeTypeForPath(filePath: string): string {
const ext = normalizeLowercaseStringOrEmpty(path.extname(filePath));
return MIME_BY_EXT[ext] ?? "audio/mpeg";
return mimeTypeFromFilePath(filePath) ?? "audio/mpeg";
}
function isBase64DataPayload(value: string): boolean {

View File

@@ -335,6 +335,25 @@ describe("message-normalizer", () => {
]);
});
it("classifies MPEG-2 audio attachments", () => {
const result = normalizeMessage({
role: "assistant",
content: "MEDIA:https://example.com/recording.m2a",
});
expect(result.content).toEqual([
{
type: "attachment",
attachment: {
url: "https://example.com/recording.m2a",
kind: "audio",
label: "recording.m2a",
mimeType: "audio/mpeg",
},
},
]);
});
it("keeps valid local MEDIA paths as assistant attachments", () => {
const result = normalizeMessage({
role: "assistant",

View File

@@ -144,6 +144,7 @@ const MIME_BY_EXT: Record<string, string> = {
aac: "audio/aac",
opus: "audio/opus",
m4a: "audio/mp4",
m2a: "audio/mpeg",
mp4: "video/mp4",
mov: "video/quicktime",
pdf: "application/pdf",

View File

@@ -290,7 +290,8 @@ function isAudioTranscriptMediaPath(path: string, mediaType: unknown): boolean {
}
const ext = getFileExtension(path);
return (
ext !== undefined && ["aac", "flac", "m4a", "mp3", "oga", "ogg", "opus", "wav"].includes(ext)
ext !== undefined &&
["aac", "flac", "m2a", "m4a", "mp3", "oga", "ogg", "opus", "wav"].includes(ext)
);
}