From fcd2a784f45fef63468465148953e6b6f55e778d Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Thu, 16 Jul 2026 00:46:24 -0700 Subject: [PATCH] fix(auto-reply): transcribe locked harness audio (#108714) --- .../reply/get-reply.message-hooks.test.ts | 90 +++++++++++++++++-- src/auto-reply/reply/get-reply.ts | 23 ++++- src/media-understanding/apply.test.ts | 49 ++++++++++ src/media-understanding/apply.ts | 23 +++-- 4 files changed, 165 insertions(+), 20 deletions(-) diff --git a/src/auto-reply/reply/get-reply.message-hooks.test.ts b/src/auto-reply/reply/get-reply.message-hooks.test.ts index f0ad0539b971..677fc8d621b1 100644 --- a/src/auto-reply/reply/get-reply.message-hooks.test.ts +++ b/src/auto-reply/reply/get-reply.message-hooks.test.ts @@ -249,12 +249,12 @@ describe("getReplyFromConfig message hooks", () => { ); }); - it("skips utility media understanding for a model-locked harness session", async () => { - const sessionKey = "agent:main:harness:codex:supervision:locked-media"; + it("runs configured audio transcription for a model-locked harness voice note", async () => { + const sessionKey = "agent:main:harness:claude-cli:locked-audio"; const sessionEntry = { sessionId: "locked-session", updatedAt: 1, - agentHarnessId: "codex", + agentHarnessId: "claude-cli", modelSelectionLocked: true, }; mocks.resolveReplySessionPreprocessingState.mockReturnValueOnce({ @@ -276,26 +276,100 @@ describe("getReplyFromConfig message hooks", () => { await getReplyFromConfig( buildCtx({ SessionKey: sessionKey }), undefined, - withFastReplyConfig({}), + withFastReplyConfig({ + tools: { + media: { + audio: { + enabled: true, + models: [ + { + type: "cli", + command: "/usr/local/bin/stt-transcribe", + args: ["{{MediaPath}}"], + }, + ], + }, + }, + }, + }), ); expect(mocks.resolveReplySessionPreprocessingState).toHaveBeenCalledOnce(); expect(mocks.initSessionState).toHaveBeenCalledOnce(); - expect(mocks.applyMediaUnderstanding).not.toHaveBeenCalled(); + expect(mocks.applyMediaUnderstanding).toHaveBeenCalledOnce(); + expect(mocks.applyMediaUnderstanding.mock.calls[0]?.[0]).toEqual( + expect.objectContaining({ processingMode: "audio-only" }), + ); expect(mocks.resolveReplyDirectives.mock.calls[0]?.[0]).toEqual( expect.objectContaining({ sessionEntry: expect.objectContaining({ - agentHarnessId: "codex", + agentHarnessId: "claude-cli", modelSelectionLocked: true, }), - sessionCtx: expect.objectContaining({ - BodyForAgent: "", + ctx: expect.objectContaining({ + BodyForAgent: "[Audio]\nTranscript:\nvoice transcript", SessionKey: sessionKey, }), }), ); }); + it("runs normal media understanding for an unlocked voice note", async () => { + await getReplyFromConfig( + buildCtx(), + undefined, + withFastReplyConfig({ + tools: { + media: { + audio: { + enabled: true, + models: [ + { + type: "cli", + command: "/usr/local/bin/stt-transcribe", + args: ["{{MediaPath}}"], + }, + ], + }, + }, + }, + }), + ); + + expect(mocks.applyMediaUnderstanding).toHaveBeenCalledOnce(); + expect(mocks.applyMediaUnderstanding.mock.calls[0]?.[0]).not.toHaveProperty("processingMode"); + expect(mocks.resolveReplyDirectives.mock.calls[0]?.[0]).toEqual( + expect.objectContaining({ + ctx: expect.objectContaining({ + BodyForAgent: "[Audio]\nTranscript:\nvoice transcript", + }), + }), + ); + }); + + it("keeps unconfigured audio with a model-locked harness", async () => { + const sessionKey = "agent:main:harness:claude-cli:locked-unconfigured-audio"; + const sessionEntry = { + sessionId: "locked-unconfigured-session", + updatedAt: 1, + agentHarnessId: "claude-cli", + modelSelectionLocked: true, + }; + mocks.resolveReplySessionPreprocessingState.mockReturnValueOnce({ + sessionEntry, + sessionKey, + storePath: "/tmp/sessions.json", + }); + + await getReplyFromConfig( + buildCtx({ SessionKey: sessionKey }), + undefined, + withFastReplyConfig({}), + ); + + expect(mocks.applyMediaUnderstanding).not.toHaveBeenCalled(); + }); + it("skips utility link understanding for a model-locked harness session", async () => { const sessionKey = "agent:main:harness:codex:supervision:locked-link"; const body = "read https://example.test/page"; diff --git a/src/auto-reply/reply/get-reply.ts b/src/auto-reply/reply/get-reply.ts index dec37baab0e0..c08fb36c7c30 100644 --- a/src/auto-reply/reply/get-reply.ts +++ b/src/auto-reply/reply/get-reply.ts @@ -59,7 +59,11 @@ import type { ReplySessionBinding, } from "./get-reply.types.js"; import { finalizeInboundContext } from "./inbound-context.js"; -import { hasInboundMedia, hasInboundMediaForUnderstanding } from "./inbound-media.js"; +import { + hasInboundAudio, + hasInboundMedia, + hasInboundMediaForUnderstanding, +} from "./inbound-media.js"; import { emitPreAgentMessageHooks } from "./message-preprocess-hooks.js"; import { createFastTestModelSelectionState, createModelSelectionState } from "./model-selection.js"; import { sanitizePendingFinalDeliveryText } from "./pending-final-delivery.js"; @@ -168,6 +172,7 @@ async function applyMediaUnderstandingIfNeeded(params: { agentDir?: string; workspaceDir?: string; activeModel: { provider: string; model: string }; + processingMode?: "audio-only"; }): Promise { if (!hasInboundMediaForUnderstanding(params.ctx)) { return undefined; @@ -184,6 +189,11 @@ async function applyMediaUnderstandingIfNeeded(params: { } } +function hasExplicitAudioUnderstandingConfig(cfg: OpenClawConfig): boolean { + const audio = cfg.tools?.media?.audio; + return audio !== undefined && audio.enabled !== false; +} + function withExtractedFileImages( opts: RuntimeInternalGetReplyOptions | undefined, extractedFileImages: ExtractedFileImage[] | undefined, @@ -434,9 +444,13 @@ export async function getReplyFromConfig( const utilityModelSelectionLocked = isModelSelectionLocked(preprocessingState?.sessionEntry); if (mediaUnderstandingRequested) { - // A durable native-harness lock owns attachment and link interpretation. The - // harness receives raw inputs, so unrelated utility models stay outside the turn. - if (!utilityModelSelectionLocked) { + const shouldApplyLockedAudio = + utilityModelSelectionLocked && + hasInboundAudio(finalized) && + hasExplicitAudioUnderstandingConfig(cfg); + // Native harnesses own image, video, and file interpretation. They cannot + // transcribe audio, so an explicitly configured STT pipeline still runs alone. + if (!utilityModelSelectionLocked || shouldApplyLockedAudio) { const mediaResult = await traceGetReplyPhase("reply.apply_media_understanding", () => applyMediaUnderstandingIfNeeded({ ctx: finalized, @@ -445,6 +459,7 @@ export async function getReplyFromConfig( agentDir, workspaceDir, activeModel: { provider, model }, + ...(shouldApplyLockedAudio ? { processingMode: "audio-only" as const } : {}), }), ); if (mediaResult?.extractedFileImages.length) { diff --git a/src/media-understanding/apply.test.ts b/src/media-understanding/apply.test.ts index 4406ad5524b1..ad0a146e0d62 100644 --- a/src/media-understanding/apply.test.ts +++ b/src/media-understanding/apply.test.ts @@ -1501,6 +1501,55 @@ describe("applyMediaUnderstanding", () => { expect(ctx.BodyForCommands).toBe("audio ok"); }); + it("limits native-harness preprocessing to audio", async () => { + const dir = await createTempMediaDir(); + const imagePath = path.join(dir, "photo.jpg"); + const audioPath = path.join(dir, "note.ogg"); + const filePath = path.join(dir, "notes.txt"); + await fs.writeFile(imagePath, "image-bytes"); + await fs.writeFile(audioPath, createSafeAudioFixtureBuffer(2048)); + await fs.writeFile(filePath, "file text"); + + const describeImage = vi.fn(async () => ({ text: "image ok" })); + const transcribeAudio = vi.fn(async () => ({ text: "audio ok" })); + const ctx: MsgContext = { + Body: "", + MediaPaths: [imagePath, audioPath, filePath], + MediaTypes: ["image/jpeg", "audio/ogg", "text/plain"], + }; + const cfg: OpenClawConfig = { + tools: { + media: { + image: { enabled: true, models: [{ provider: "openai", model: "gpt-5.4" }] }, + audio: { enabled: true, models: [{ provider: "groq" }] }, + }, + }, + }; + + const result = await applyMediaUnderstanding({ + ctx, + cfg, + processingMode: "audio-only", + providers: { + openai: { id: "openai", describeImage }, + groq: { id: "groq", transcribeAudio }, + }, + }); + + expect(describeImage).not.toHaveBeenCalled(); + expect(transcribeAudio).toHaveBeenCalledOnce(); + expect(result).toEqual( + expect.objectContaining({ + appliedImage: false, + appliedAudio: true, + appliedVideo: false, + appliedFile: false, + extractedFileImages: [], + }), + ); + expect(ctx.Body).toBe("[Audio]\nTranscript:\naudio ok"); + }); + it("orders synthetic too-small audio output between image and video", async () => { const dir = await createTempMediaDir(); const imagePath = path.join(dir, "photo.jpg"); diff --git a/src/media-understanding/apply.ts b/src/media-understanding/apply.ts index ba46cc7e5580..f7bb0bc01aa7 100644 --- a/src/media-understanding/apply.ts +++ b/src/media-understanding/apply.ts @@ -53,6 +53,7 @@ export type ApplyMediaUnderstandingResult = { }; const CAPABILITY_ORDER: MediaUnderstandingCapability[] = ["image", "audio", "video"]; +const AUDIO_ONLY_CAPABILITY_ORDER: MediaUnderstandingCapability[] = ["audio"]; const EMPTY_VOICE_NOTE_PLACEHOLDER = "[Voice note could not be transcribed because the audio attachment was too small]"; const EXTRA_TEXT_MIMES = [ @@ -538,6 +539,8 @@ export async function applyMediaUnderstanding(params: { workspaceDir?: string; providers?: Record; activeModel?: ActiveMediaModel; + /** Preserve native-harness ownership of image, video, and file inputs while applying STT. */ + processingMode?: "audio-only"; }): Promise { const { ctx, cfg } = params; const mediaWorkspaceDir = ctx.MediaWorkspaceDir ?? params.workspaceDir; @@ -561,7 +564,7 @@ export async function applyMediaUnderstanding(params: { try { const results = await pMap( - CAPABILITY_ORDER, + params.processingMode === "audio-only" ? AUDIO_ONLY_CAPABILITY_ORDER : CAPABILITY_ORDER, async (capability) => await runMediaCapability({ capability, @@ -686,13 +689,17 @@ export async function applyMediaUnderstanding(params: { ) .map((output) => output.attachmentIndex), ); - const fileContext = await extractFileContext({ - attachments, - cache, - cfg, - limits: resolveFileExtractionLimits(cfg), - skipAttachmentIndexes: audioAttachmentIndexes.size > 0 ? audioAttachmentIndexes : undefined, - }); + const fileContext = + params.processingMode === "audio-only" + ? { blocks: [], images: [] } + : await extractFileContext({ + attachments, + cache, + cfg, + limits: resolveFileExtractionLimits(cfg), + skipAttachmentIndexes: + audioAttachmentIndexes.size > 0 ? audioAttachmentIndexes : undefined, + }); if (fileContext.blocks.length > 0) { ctx.Body = appendFileBlocks(ctx.Body, fileContext.blocks); }