mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-09 03:24:04 +00:00
fix(media): recognize m2a as MPEG audio (#92167)
Co-authored-by: Peter Steinberger <steipete@gmail.com>
This commit is contained in:
@@ -1011,6 +1011,27 @@ describe("resolveMedia original filename preservation", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("classifies an audio document from the saved MIME type", async () => {
|
||||
const getFile = vi.fn().mockResolvedValue({ file_path: "documents/recording.m2a" });
|
||||
saveRemoteMedia.mockResolvedValueOnce({
|
||||
path: "/tmp/inbound/recording.m2a",
|
||||
contentType: "audio/mpeg",
|
||||
});
|
||||
|
||||
const result = await resolveMediaWithDefaults(
|
||||
makeCtx("document", getFile, {
|
||||
file_name: "recording.m2a",
|
||||
mime_type: "application/octet-stream",
|
||||
}),
|
||||
);
|
||||
|
||||
expectResolvedMediaFields(result, "MPEG-2 audio document", {
|
||||
path: "/tmp/inbound/recording.m2a",
|
||||
contentType: "audio/mpeg",
|
||||
placeholder: "<media:audio>",
|
||||
});
|
||||
});
|
||||
|
||||
it("passes audio.file_name to saveMediaBuffer", async () => {
|
||||
const getFile = vi.fn().mockResolvedValue({ file_path: "music/file_99.mp3" });
|
||||
readRemoteMediaBuffer.mockResolvedValueOnce({
|
||||
|
||||
@@ -517,6 +517,8 @@ export async function resolveMedia(params: {
|
||||
trustedLocalFileRoots,
|
||||
dangerouslyAllowPrivateNetwork,
|
||||
});
|
||||
const placeholder = resolveTelegramMediaPlaceholder(msg) ?? "<media:document>";
|
||||
const placeholder = saved.contentType?.startsWith("audio/")
|
||||
? "<media:audio>"
|
||||
: (resolveTelegramMediaPlaceholder(msg) ?? "<media:document>");
|
||||
return { path: saved.path, contentType: saved.contentType, placeholder };
|
||||
}
|
||||
|
||||
@@ -191,6 +191,8 @@ describe("mimeTypeFromFilePath", () => {
|
||||
{ filePath: "photo.jpg", expected: "image/jpeg" },
|
||||
{ filePath: "photo.JPG", expected: "image/jpeg" },
|
||||
{ filePath: "voice.mp3", expected: "audio/mpeg" },
|
||||
{ filePath: "voice.m2a", expected: "audio/mpeg" },
|
||||
{ filePath: "voice.oga", expected: "audio/ogg" },
|
||||
{ filePath: "voice.wav", expected: "audio/wav" },
|
||||
{ filePath: "clip.avi", expected: "video/x-msvideo" },
|
||||
{ filePath: "clip.mkv", expected: "video/x-matroska" },
|
||||
@@ -266,6 +268,9 @@ describe("isAudioFileName", () => {
|
||||
it.each([
|
||||
{ fileName: "voice.mp3", expected: true },
|
||||
{ fileName: "voice.caf", expected: true },
|
||||
{ fileName: "voice.M2A", expected: true },
|
||||
{ fileName: "voice.oga", expected: true },
|
||||
{ fileName: "voice.webm", expected: false },
|
||||
{ fileName: "voice.bin", expected: false },
|
||||
] as const)("matches audio extension for $fileName", ({ fileName, expected }) => {
|
||||
expectAudioFileNameCase(fileName, expected);
|
||||
|
||||
@@ -72,7 +72,9 @@ const MIME_BY_EXT: Record<string, string> = {
|
||||
...buildMimeByExt(),
|
||||
// Canonical extension mappings for common MIME aliases
|
||||
".jpg": "image/jpeg",
|
||||
".m2a": "audio/mpeg",
|
||||
".mp3": "audio/mpeg",
|
||||
".oga": "audio/ogg",
|
||||
".wav": "audio/wav",
|
||||
".webm": "video/webm",
|
||||
// Additional extension aliases
|
||||
@@ -84,18 +86,6 @@ const MIME_BY_EXT: Record<string, string> = {
|
||||
".yml": "application/yaml",
|
||||
};
|
||||
|
||||
const AUDIO_FILE_EXTENSIONS = new Set([
|
||||
".aac",
|
||||
".caf",
|
||||
".flac",
|
||||
".m4a",
|
||||
".mp3",
|
||||
".oga",
|
||||
".ogg",
|
||||
".opus",
|
||||
".wav",
|
||||
]);
|
||||
|
||||
const fileTypeModuleLoader = createLazyImportLoader(() => import("file-type"));
|
||||
|
||||
/** Normalizes MIME strings by dropping parameters, lowercasing, and folding APNG to PNG. */
|
||||
@@ -174,11 +164,7 @@ export function mimeTypeFromFilePath(filePath?: string | null): string | undefin
|
||||
|
||||
/** Returns true when a filename extension is a supported audio container. */
|
||||
export function isAudioFileName(fileName?: string | null): boolean {
|
||||
const ext = getFileExtension(fileName);
|
||||
if (!ext) {
|
||||
return false;
|
||||
}
|
||||
return AUDIO_FILE_EXTENSIONS.has(ext);
|
||||
return mediaKindFromMime(mimeTypeFromFilePath(fileName)) === "audio";
|
||||
}
|
||||
|
||||
/** Detects the best MIME type from bytes, file path, and header metadata. */
|
||||
|
||||
@@ -124,6 +124,7 @@ describe("memory host SDK package internals", () => {
|
||||
fsSync.mkdirSync(extraDir, { recursive: true });
|
||||
fsSync.writeFileSync(path.join(extraDir, "note.md"), "# Note");
|
||||
fsSync.writeFileSync(path.join(extraDir, "diagram.png"), Buffer.from("png"));
|
||||
fsSync.writeFileSync(path.join(extraDir, "recording.m2a"), Buffer.from("audio"));
|
||||
fsSync.writeFileSync(path.join(extraDir, "ignore.txt"), "ignored");
|
||||
|
||||
const files = await listMemoryFiles(
|
||||
@@ -136,6 +137,7 @@ describe("memory host SDK package internals", () => {
|
||||
"MEMORY.md",
|
||||
path.join("extra", "diagram.png"),
|
||||
path.join("extra", "note.md"),
|
||||
path.join("extra", "recording.m2a"),
|
||||
]);
|
||||
});
|
||||
|
||||
|
||||
@@ -10,7 +10,7 @@ const MEMORY_MULTIMODAL_SPECS = {
|
||||
},
|
||||
audio: {
|
||||
labelPrefix: "Audio file",
|
||||
extensions: [".mp3", ".wav", ".ogg", ".opus", ".m4a", ".aac", ".flac"],
|
||||
extensions: [".mp3", ".wav", ".ogg", ".opus", ".m4a", ".m2a", ".aac", ".flac"],
|
||||
},
|
||||
} as const;
|
||||
|
||||
|
||||
@@ -317,6 +317,21 @@ describe("buildInboundMediaNote", () => {
|
||||
expect(note).toBe("[media attached: /tmp/document.pdf]");
|
||||
});
|
||||
|
||||
it("strips transcribed MPEG-2 audio by extension", () => {
|
||||
const note = buildInboundMediaNote({
|
||||
MediaPaths: ["/tmp/recording.m2a", "/tmp/document.pdf"],
|
||||
MediaUnderstanding: [
|
||||
{
|
||||
kind: "audio.transcription",
|
||||
attachmentIndex: 0,
|
||||
text: "Transcribed audio content",
|
||||
provider: "whisper",
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(note).toBe("[media attached: /tmp/document.pdf]");
|
||||
});
|
||||
|
||||
it("keeps audio attachments when no transcription is available", () => {
|
||||
const note = buildInboundMediaNote({
|
||||
MediaPaths: ["/tmp/voice.ogg"],
|
||||
|
||||
@@ -66,6 +66,7 @@ const AUDIO_EXTENSIONS = new Set([
|
||||
".opus",
|
||||
".mp3",
|
||||
".m4a",
|
||||
".m2a",
|
||||
".wav",
|
||||
".webm",
|
||||
".flac",
|
||||
|
||||
@@ -22,9 +22,9 @@ describe("buildWebchatAudioContentBlocksFromReplyPayloads", () => {
|
||||
tmpDir = undefined;
|
||||
});
|
||||
|
||||
function writeAudioFixture(bytes = [0xff, 0xfb, 0x90, 0x00]) {
|
||||
function writeAudioFixture(bytes = [0xff, 0xfb, 0x90, 0x00], extension = ".mp3") {
|
||||
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-webchat-audio-"));
|
||||
const audioPath = path.join(tmpDir, "clip.mp3");
|
||||
const audioPath = path.join(tmpDir, `clip${extension}`);
|
||||
fs.writeFileSync(audioPath, Buffer.from(bytes));
|
||||
return { audioPath, localRoot: tmpDir };
|
||||
}
|
||||
@@ -51,6 +51,19 @@ describe("buildWebchatAudioContentBlocksFromReplyPayloads", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("exposes MPEG-2 audio files with their canonical MIME type", async () => {
|
||||
const { audioPath, localRoot } = writeAudioFixture([0xff, 0xfd, 0x80, 0x00], ".m2a");
|
||||
|
||||
const blocks = await buildWebchatAudioContentBlocksFromReplyPayloads(
|
||||
[{ mediaUrl: audioPath, trustedLocalMedia: true }],
|
||||
{ localRoots: [localRoot] },
|
||||
);
|
||||
|
||||
expect(blocks[0]).toMatchObject({
|
||||
attachment: { label: "clip.m2a", kind: "audio", mimeType: "audio/mpeg" },
|
||||
});
|
||||
});
|
||||
|
||||
it("preserves voice-note metadata on local audio attachments", async () => {
|
||||
const { audioPath, localRoot } = writeAudioFixture();
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
// blocks that the control UI can render without unsafe file exposure.
|
||||
import path from "node:path";
|
||||
import { estimateBase64DecodedBytes } from "@openclaw/media-core/base64";
|
||||
import { isAudioFileName } from "@openclaw/media-core/mime";
|
||||
import { isAudioFileName, mimeTypeFromFilePath } from "@openclaw/media-core/mime";
|
||||
import { normalizeLowercaseStringOrEmpty } from "@openclaw/normalization-core/string-coerce";
|
||||
import type { ReplyPayload } from "../../auto-reply/reply-payload.js";
|
||||
import { openLocalFileSafely } from "../../infra/fs-safe.js";
|
||||
@@ -26,17 +26,6 @@ const ALLOWED_WEBCHAT_DATA_IMAGE_MEDIA_TYPES = new Set([
|
||||
"image/webp",
|
||||
]);
|
||||
|
||||
const MIME_BY_EXT: Record<string, string> = {
|
||||
".aac": "audio/aac",
|
||||
".m4a": "audio/mp4",
|
||||
".mp3": "audio/mpeg",
|
||||
".oga": "audio/ogg",
|
||||
".ogg": "audio/ogg",
|
||||
".opus": "audio/opus",
|
||||
".wav": "audio/wav",
|
||||
".webm": "audio/webm",
|
||||
};
|
||||
|
||||
type WebchatAudioEmbeddingOptions = {
|
||||
localRoots?: readonly string[];
|
||||
onLocalAudioAccessDenied?: (err: LocalMediaAccessError) => void;
|
||||
@@ -152,8 +141,7 @@ async function resolveReplyMediaAudioEmbedding(
|
||||
}
|
||||
|
||||
function mimeTypeForPath(filePath: string): string {
|
||||
const ext = normalizeLowercaseStringOrEmpty(path.extname(filePath));
|
||||
return MIME_BY_EXT[ext] ?? "audio/mpeg";
|
||||
return mimeTypeFromFilePath(filePath) ?? "audio/mpeg";
|
||||
}
|
||||
|
||||
function isBase64DataPayload(value: string): boolean {
|
||||
|
||||
@@ -335,6 +335,25 @@ describe("message-normalizer", () => {
|
||||
]);
|
||||
});
|
||||
|
||||
it("classifies MPEG-2 audio attachments", () => {
|
||||
const result = normalizeMessage({
|
||||
role: "assistant",
|
||||
content: "MEDIA:https://example.com/recording.m2a",
|
||||
});
|
||||
|
||||
expect(result.content).toEqual([
|
||||
{
|
||||
type: "attachment",
|
||||
attachment: {
|
||||
url: "https://example.com/recording.m2a",
|
||||
kind: "audio",
|
||||
label: "recording.m2a",
|
||||
mimeType: "audio/mpeg",
|
||||
},
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("keeps valid local MEDIA paths as assistant attachments", () => {
|
||||
const result = normalizeMessage({
|
||||
role: "assistant",
|
||||
|
||||
@@ -144,6 +144,7 @@ const MIME_BY_EXT: Record<string, string> = {
|
||||
aac: "audio/aac",
|
||||
opus: "audio/opus",
|
||||
m4a: "audio/mp4",
|
||||
m2a: "audio/mpeg",
|
||||
mp4: "video/mp4",
|
||||
mov: "video/quicktime",
|
||||
pdf: "application/pdf",
|
||||
|
||||
@@ -290,7 +290,8 @@ function isAudioTranscriptMediaPath(path: string, mediaType: unknown): boolean {
|
||||
}
|
||||
const ext = getFileExtension(path);
|
||||
return (
|
||||
ext !== undefined && ["aac", "flac", "m4a", "mp3", "oga", "ogg", "opus", "wav"].includes(ext)
|
||||
ext !== undefined &&
|
||||
["aac", "flac", "m2a", "m4a", "mp3", "oga", "ogg", "opus", "wav"].includes(ext)
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user