diff --git a/src/media-understanding/openai-compatible-audio.test.ts b/src/media-understanding/openai-compatible-audio.test.ts index fa39a2d4c7a7..1989c9e98c2b 100644 --- a/src/media-understanding/openai-compatible-audio.test.ts +++ b/src/media-understanding/openai-compatible-audio.test.ts @@ -75,6 +75,29 @@ describe("transcribeOpenAiCompatibleAudio", () => { expect((file as File).name).toBe("voice-note.m4a"); }); + it("omits the optional prompt field while preserving language hints", async () => { + const { fetchFn, getRequest } = createRequestCaptureJsonFetch({ text: "ok" }); + + await transcribeOpenAiCompatibleAudio({ + buffer: Buffer.from("audio"), + fileName: "note.ogg", + mime: "audio/ogg", + apiKey: "test-key", + timeoutMs: 1000, + fetchFn, + provider: "groq", + baseUrl: "https://api.groq.com/openai/v1", + defaultBaseUrl: "https://api.groq.com/openai/v1", + defaultModel: "whisper-large-v3-turbo", + language: "ru", + }); + + const form = getRequest().init?.body; + expect(form).toBeInstanceOf(FormData); + expect((form as FormData).get("language")).toBe("ru"); + expect((form as FormData).get("prompt")).toBeNull(); + }); + it("omits bearer auth for explicit no-auth requests", async () => { const { fetchFn, getRequest } = createRequestCaptureJsonFetch({ text: "ok" }); diff --git a/src/media-understanding/runner.auto-audio.test.ts b/src/media-understanding/runner.auto-audio.test.ts index 0dae19d77a96..452fb66962cf 100644 --- a/src/media-understanding/runner.auto-audio.test.ts +++ b/src/media-understanding/runner.auto-audio.test.ts @@ -5,6 +5,7 @@ import os from "node:os"; import path from "node:path"; import { describe, expect, it, vi } from "vitest"; import type { OpenClawConfig } from "../config/types.js"; +import type { MediaUnderstandingConfig } from "../config/types.tools.js"; import { withEnvAsync } from "../test-utils/env.js"; import { clearMediaUnderstandingBinaryCacheForTests, runCapability } from "./runner.js"; import { withAudioFixture } from "./runner.test-utils.js"; @@ -400,6 +401,78 @@ describe("runCapability auto audio entries", () => { expect(seenPrompt).toBe("Focus on names"); }); + it("omits the implicit English audio prompt when a non-English language is configured", async () => { + let seenLanguage: string | undefined; + let seenPrompt: string | undefined; + const result = await runAutoAudioCase({ + transcribeAudio: async (req) => { + seenLanguage = req.language; + seenPrompt = req.prompt; + return { text: "ok", model: req.model ?? "unknown" }; + }, + cfgExtra: { + tools: { + media: { + audio: { + enabled: true, + language: "ru", + models: [{ provider: "openai", model: "whisper-1" }], + }, + }, + }, + } as Partial, + }); + + expect(requireCapabilityOutput(result, 0).text).toBe("ok"); + expect(seenLanguage).toBe("ru"); + expect(seenPrompt).toBeUndefined(); + }); + + it("keeps explicit and English-compatible audio prompts", async () => { + const seenPrompts: Array = []; + const runCase = async (audio: MediaUnderstandingConfig) => { + await runAutoAudioCase({ + transcribeAudio: async (req) => { + seenPrompts.push(req.prompt); + return { text: "ok", model: req.model ?? "unknown" }; + }, + cfgExtra: { + tools: { + media: { + audio, + }, + }, + } as Partial, + }); + }; + + await runCase({ + enabled: true, + language: "ru", + prompt: "Transcribe in Russian.", + models: [{ provider: "openai", model: "whisper-1" }], + }); + for (const language of ["en-US", "eng", "english"]) { + await runCase({ + enabled: true, + language, + models: [{ provider: "openai", model: "whisper-1" }], + }); + } + await runCase({ + enabled: true, + models: [{ provider: "openai", model: "whisper-1" }], + }); + + expect(seenPrompts).toEqual([ + "Transcribe in Russian.", + "Transcribe the audio.", + "Transcribe the audio.", + "Transcribe the audio.", + "Transcribe the audio.", + ]); + }); + it("uses mistral when only mistral key is configured", async () => { const isolatedAgentDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-audio-agent-")); let runResult: Awaited> | undefined; diff --git a/src/media-understanding/runner.entries.ts b/src/media-understanding/runner.entries.ts index fd0f4b3e8eb3..dab99aa7cf09 100644 --- a/src/media-understanding/runner.entries.ts +++ b/src/media-understanding/runner.entries.ts @@ -397,7 +397,13 @@ function resolveEntryRunOptions(params: { entry: MediaUnderstandingModelConfig; cfg: OpenClawConfig; config?: MediaUnderstandingConfig; -}): { maxBytes: number; maxChars?: number; timeoutMs: number; prompt: string } { +}): { + maxBytes: number; + maxChars?: number; + timeoutMs: number; + prompt: string; + hasConfiguredPrompt: boolean; +} { const { capability, entry, cfg } = params; const maxBytes = resolveMaxBytes({ capability, entry, cfg, config: params.config }); const maxChars = resolveMaxChars({ capability, entry, cfg, config: params.config }); @@ -407,12 +413,16 @@ function resolveEntryRunOptions(params: { cfg.tools?.media?.[capability]?.timeoutSeconds, DEFAULT_TIMEOUT_SECONDS[capability], ); - const prompt = resolvePrompt( - capability, - entry.prompt ?? params.config?.prompt ?? cfg.tools?.media?.[capability]?.prompt, + const configuredPrompt = + entry.prompt ?? params.config?.prompt ?? cfg.tools?.media?.[capability]?.prompt; + const prompt = resolvePrompt(capability, configuredPrompt, maxChars); + return { + maxBytes, maxChars, - ); - return { maxBytes, maxChars, timeoutMs, prompt }; + timeoutMs, + prompt, + hasConfiguredPrompt: Boolean(configuredPrompt?.trim()), + }; } function resolveMediaRequestOverrides(config: MediaUnderstandingConfig | undefined): { @@ -429,6 +439,28 @@ function resolveMediaRequestOverrides(config: MediaUnderstandingConfig | undefin }; } +function resolveAudioProviderPrompt(params: { + prompt: string; + hasConfiguredPrompt: boolean; + language?: string; +}): string | undefined { + const language = params.language?.trim().toLowerCase(); + const isEnglish = + !language || + language === "en" || + language === "eng" || + language === "english" || + language.startsWith("en-") || + language.startsWith("en_"); + if (params.hasConfiguredPrompt || isEnglish) { + return params.prompt; + } + // OpenAI-compatible transcription prompts guide style/context and should + // match the audio language; omit OpenClaw's English default for non-English + // language hints unless the user supplied an explicit prompt. + return undefined; +} + type ProviderExecutionAuth = | { kind: "api-key"; @@ -727,7 +759,7 @@ export async function runProviderEntry(params: { } const providerId = normalizeMediaProviderId(providerIdRaw); const requestProviderId = normalizeMediaExecutionProviderId(providerIdRaw); - const { maxBytes, maxChars, timeoutMs, prompt } = resolveEntryRunOptions({ + const { maxBytes, maxChars, timeoutMs, prompt, hasConfiguredPrompt } = resolveEntryRunOptions({ capability, entry, cfg, @@ -803,6 +835,18 @@ export async function runProviderEntry(params: { timeoutMs, }); assertMinAudioSize({ size: media.size, attachmentIndex: params.attachmentIndex }); + const audioLanguage = + requestOverrides.language ?? + entry.language ?? + params.config?.language ?? + cfg.tools?.media?.audio?.language; + const audioPrompt = + requestOverrides.prompt ?? + resolveAudioProviderPrompt({ + prompt, + hasConfiguredPrompt, + language: audioLanguage, + }); const { auth, baseUrl, headers, request } = await resolveProviderExecutionContext({ capability, providerId, @@ -841,12 +885,8 @@ export async function runProviderEntry(params: { headers, request, model, - language: - requestOverrides.language ?? - entry.language ?? - params.config?.language ?? - cfg.tools?.media?.audio?.language, - prompt: requestOverrides.prompt ?? prompt, + language: audioLanguage, + prompt: audioPrompt, query: providerQuery, timeoutMs, fetchFn,