fix(auto-reply): transcribe locked harness audio (#108714)

src/auto-reply/reply/get-reply.message-hooks.test.ts

@@ -249,12 +249,12 @@ describe("getReplyFromConfig message hooks", () => {
     );
   });
 
-  it("skips utility media understanding for a model-locked harness session", async () => {
-    const sessionKey = "agent:main:harness:codex:supervision:locked-media";
+  it("runs configured audio transcription for a model-locked harness voice note", async () => {
+    const sessionKey = "agent:main:harness:claude-cli:locked-audio";
     const sessionEntry = {
       sessionId: "locked-session",
       updatedAt: 1,
-      agentHarnessId: "codex",
+      agentHarnessId: "claude-cli",
       modelSelectionLocked: true,
     };
     mocks.resolveReplySessionPreprocessingState.mockReturnValueOnce({
@@ -276,26 +276,100 @@ describe("getReplyFromConfig message hooks", () => {
     await getReplyFromConfig(
       buildCtx({ SessionKey: sessionKey }),
       undefined,
-      withFastReplyConfig({}),
+      withFastReplyConfig({
+        tools: {
+          media: {
+            audio: {
+              enabled: true,
+              models: [
+                {
+                  type: "cli",
+                  command: "/usr/local/bin/stt-transcribe",
+                  args: ["{{MediaPath}}"],
+                },
+              ],
+            },
+          },
+        },
+      }),
     );
 
     expect(mocks.resolveReplySessionPreprocessingState).toHaveBeenCalledOnce();
     expect(mocks.initSessionState).toHaveBeenCalledOnce();
-    expect(mocks.applyMediaUnderstanding).not.toHaveBeenCalled();
+    expect(mocks.applyMediaUnderstanding).toHaveBeenCalledOnce();
+    expect(mocks.applyMediaUnderstanding.mock.calls[0]?.[0]).toEqual(
+      expect.objectContaining({ processingMode: "audio-only" }),
+    );
     expect(mocks.resolveReplyDirectives.mock.calls[0]?.[0]).toEqual(
       expect.objectContaining({
         sessionEntry: expect.objectContaining({
-          agentHarnessId: "codex",
+          agentHarnessId: "claude-cli",
           modelSelectionLocked: true,
         }),
-        sessionCtx: expect.objectContaining({
-          BodyForAgent: "<media:audio>",
+        ctx: expect.objectContaining({
+          BodyForAgent: "[Audio]\nTranscript:\nvoice transcript",
           SessionKey: sessionKey,
         }),
       }),
     );
   });
 
+  it("runs normal media understanding for an unlocked voice note", async () => {
+    await getReplyFromConfig(
+      buildCtx(),
+      undefined,
+      withFastReplyConfig({
+        tools: {
+          media: {
+            audio: {
+              enabled: true,
+              models: [
+                {
+                  type: "cli",
+                  command: "/usr/local/bin/stt-transcribe",
+                  args: ["{{MediaPath}}"],
+                },
+              ],
+            },
+          },
+        },
+      }),
+    );
+
+    expect(mocks.applyMediaUnderstanding).toHaveBeenCalledOnce();
+    expect(mocks.applyMediaUnderstanding.mock.calls[0]?.[0]).not.toHaveProperty("processingMode");
+    expect(mocks.resolveReplyDirectives.mock.calls[0]?.[0]).toEqual(
+      expect.objectContaining({
+        ctx: expect.objectContaining({
+          BodyForAgent: "[Audio]\nTranscript:\nvoice transcript",
+        }),
+      }),
+    );
+  });
+
+  it("keeps unconfigured audio with a model-locked harness", async () => {
+    const sessionKey = "agent:main:harness:claude-cli:locked-unconfigured-audio";
+    const sessionEntry = {
+      sessionId: "locked-unconfigured-session",
+      updatedAt: 1,
+      agentHarnessId: "claude-cli",
+      modelSelectionLocked: true,
+    };
+    mocks.resolveReplySessionPreprocessingState.mockReturnValueOnce({
+      sessionEntry,
+      sessionKey,
+      storePath: "/tmp/sessions.json",
+    });
+
+    await getReplyFromConfig(
+      buildCtx({ SessionKey: sessionKey }),
+      undefined,
+      withFastReplyConfig({}),
+    );
+
+    expect(mocks.applyMediaUnderstanding).not.toHaveBeenCalled();
+  });
+
   it("skips utility link understanding for a model-locked harness session", async () => {
     const sessionKey = "agent:main:harness:codex:supervision:locked-link";
     const body = "read https://example.test/page";

src/auto-reply/reply/get-reply.ts

@@ -59,7 +59,11 @@ import type {
   ReplySessionBinding,
 } from "./get-reply.types.js";
 import { finalizeInboundContext } from "./inbound-context.js";
-import { hasInboundMedia, hasInboundMediaForUnderstanding } from "./inbound-media.js";
+import {
+  hasInboundAudio,
+  hasInboundMedia,
+  hasInboundMediaForUnderstanding,
+} from "./inbound-media.js";
 import { emitPreAgentMessageHooks } from "./message-preprocess-hooks.js";
 import { createFastTestModelSelectionState, createModelSelectionState } from "./model-selection.js";
 import { sanitizePendingFinalDeliveryText } from "./pending-final-delivery.js";
@@ -168,6 +172,7 @@ async function applyMediaUnderstandingIfNeeded(params: {
   agentDir?: string;
   workspaceDir?: string;
   activeModel: { provider: string; model: string };
+  processingMode?: "audio-only";
 }): Promise<ApplyMediaUnderstandingResult | undefined> {
   if (!hasInboundMediaForUnderstanding(params.ctx)) {
     return undefined;
@@ -184,6 +189,11 @@ async function applyMediaUnderstandingIfNeeded(params: {
   }
 }
 
+function hasExplicitAudioUnderstandingConfig(cfg: OpenClawConfig): boolean {
+  const audio = cfg.tools?.media?.audio;
+  return audio !== undefined && audio.enabled !== false;
+}
+
 function withExtractedFileImages(
   opts: RuntimeInternalGetReplyOptions | undefined,
   extractedFileImages: ExtractedFileImage[] | undefined,
@@ -434,9 +444,13 @@ export async function getReplyFromConfig(
   const utilityModelSelectionLocked = isModelSelectionLocked(preprocessingState?.sessionEntry);
 
   if (mediaUnderstandingRequested) {
-    // A durable native-harness lock owns attachment and link interpretation. The
-    // harness receives raw inputs, so unrelated utility models stay outside the turn.
-    if (!utilityModelSelectionLocked) {
+    const shouldApplyLockedAudio =
+      utilityModelSelectionLocked &&
+      hasInboundAudio(finalized) &&
+      hasExplicitAudioUnderstandingConfig(cfg);
+    // Native harnesses own image, video, and file interpretation. They cannot
+    // transcribe audio, so an explicitly configured STT pipeline still runs alone.
+    if (!utilityModelSelectionLocked || shouldApplyLockedAudio) {
       const mediaResult = await traceGetReplyPhase("reply.apply_media_understanding", () =>
         applyMediaUnderstandingIfNeeded({
           ctx: finalized,
@@ -445,6 +459,7 @@ export async function getReplyFromConfig(
           agentDir,
           workspaceDir,
           activeModel: { provider, model },
+          ...(shouldApplyLockedAudio ? { processingMode: "audio-only" as const } : {}),
         }),
       );
       if (mediaResult?.extractedFileImages.length) {

src/media-understanding/apply.test.ts

@@ -1501,6 +1501,55 @@ describe("applyMediaUnderstanding", () => {
     expect(ctx.BodyForCommands).toBe("audio ok");
   });
 
+  it("limits native-harness preprocessing to audio", async () => {
+    const dir = await createTempMediaDir();
+    const imagePath = path.join(dir, "photo.jpg");
+    const audioPath = path.join(dir, "note.ogg");
+    const filePath = path.join(dir, "notes.txt");
+    await fs.writeFile(imagePath, "image-bytes");
+    await fs.writeFile(audioPath, createSafeAudioFixtureBuffer(2048));
+    await fs.writeFile(filePath, "file text");
+
+    const describeImage = vi.fn(async () => ({ text: "image ok" }));
+    const transcribeAudio = vi.fn(async () => ({ text: "audio ok" }));
+    const ctx: MsgContext = {
+      Body: "<media:mixed>",
+      MediaPaths: [imagePath, audioPath, filePath],
+      MediaTypes: ["image/jpeg", "audio/ogg", "text/plain"],
+    };
+    const cfg: OpenClawConfig = {
+      tools: {
+        media: {
+          image: { enabled: true, models: [{ provider: "openai", model: "gpt-5.4" }] },
+          audio: { enabled: true, models: [{ provider: "groq" }] },
+        },
+      },
+    };
+
+    const result = await applyMediaUnderstanding({
+      ctx,
+      cfg,
+      processingMode: "audio-only",
+      providers: {
+        openai: { id: "openai", describeImage },
+        groq: { id: "groq", transcribeAudio },
+      },
+    });
+
+    expect(describeImage).not.toHaveBeenCalled();
+    expect(transcribeAudio).toHaveBeenCalledOnce();
+    expect(result).toEqual(
+      expect.objectContaining({
+        appliedImage: false,
+        appliedAudio: true,
+        appliedVideo: false,
+        appliedFile: false,
+        extractedFileImages: [],
+      }),
+    );
+    expect(ctx.Body).toBe("[Audio]\nTranscript:\naudio ok");
+  });
+
   it("orders synthetic too-small audio output between image and video", async () => {
     const dir = await createTempMediaDir();
     const imagePath = path.join(dir, "photo.jpg");

src/media-understanding/apply.ts

@@ -53,6 +53,7 @@ export type ApplyMediaUnderstandingResult = {
 };
 
 const CAPABILITY_ORDER: MediaUnderstandingCapability[] = ["image", "audio", "video"];
+const AUDIO_ONLY_CAPABILITY_ORDER: MediaUnderstandingCapability[] = ["audio"];
 const EMPTY_VOICE_NOTE_PLACEHOLDER =
   "[Voice note could not be transcribed because the audio attachment was too small]";
 const EXTRA_TEXT_MIMES = [
@@ -538,6 +539,8 @@ export async function applyMediaUnderstanding(params: {
   workspaceDir?: string;
   providers?: Record<string, MediaUnderstandingProvider>;
   activeModel?: ActiveMediaModel;
+  /** Preserve native-harness ownership of image, video, and file inputs while applying STT. */
+  processingMode?: "audio-only";
 }): Promise<ApplyMediaUnderstandingResult> {
   const { ctx, cfg } = params;
   const mediaWorkspaceDir = ctx.MediaWorkspaceDir ?? params.workspaceDir;
@@ -561,7 +564,7 @@ export async function applyMediaUnderstanding(params: {
 
   try {
     const results = await pMap(
-      CAPABILITY_ORDER,
+      params.processingMode === "audio-only" ? AUDIO_ONLY_CAPABILITY_ORDER : CAPABILITY_ORDER,
       async (capability) =>
         await runMediaCapability({
           capability,
@@ -686,13 +689,17 @@ export async function applyMediaUnderstanding(params: {
         )
         .map((output) => output.attachmentIndex),
     );
-    const fileContext = await extractFileContext({
-      attachments,
-      cache,
-      cfg,
-      limits: resolveFileExtractionLimits(cfg),
-      skipAttachmentIndexes: audioAttachmentIndexes.size > 0 ? audioAttachmentIndexes : undefined,
-    });
+    const fileContext =
+      params.processingMode === "audio-only"
+        ? { blocks: [], images: [] }
+        : await extractFileContext({
+            attachments,
+            cache,
+            cfg,
+            limits: resolveFileExtractionLimits(cfg),
+            skipAttachmentIndexes:
+              audioAttachmentIndexes.size > 0 ? audioAttachmentIndexes : undefined,
+          });
     if (fileContext.blocks.length > 0) {
       ctx.Body = appendFileBlocks(ctx.Body, fileContext.blocks);
     }