[AI] fix(feishu): send text and voice separately for Auto-TTS replies (#103781)

* fix(feishu): use TTS supplement metadata for precise text suppression * refactor(feishu): centralize TTS supplement delivery * style(feishu): format TTS delivery callback * fix(feishu): preserve streamed TTS text --------- Co-authored-by: Peter Steinberger <steipete@gmail.com>
2026-07-23 11:41:10 +00:00 · 2026-07-17 04:18:28 +08:00
parent 9edd662741
commit 1bff4eecb2
6 changed files with 297 additions and 6 deletions
--- a/extensions/feishu/src/media.test.ts
+++ b/extensions/feishu/src/media.test.ts
@@ -208,6 +208,29 @@ describe("sendMediaFeishu msg_type routing", () => {
    ).toBe(false);
  });

+  it("respects ttsSupplement.visibleTextAlreadyDelivered over audioAsVoice", () => {
+    expect(
+      shouldSuppressFeishuTextForVoiceMedia({
+        mediaUrl: "https://example.com/tts.mp3",
+        audioAsVoice: true,
+        ttsSupplement: {
+          spokenText: "Hello world",
+        },
+      }),
+    ).toBe(false);
+
+    expect(
+      shouldSuppressFeishuTextForVoiceMedia({
+        mediaUrl: "https://example.com/tts.mp3",
+        audioAsVoice: true,
+        ttsSupplement: {
+          spokenText: "Hello world",
+          visibleTextAlreadyDelivered: true,
+        },
+      }),
+    ).toBe(true);
+  });
+
  it("uses msg_type=media for mp4 video", async () => {
    runFfprobeMock.mockResolvedValueOnce("4.2\n");

--- a/extensions/feishu/src/media.ts
+++ b/extensions/feishu/src/media.ts
@@ -11,6 +11,7 @@ import {
  runFfprobe,
 } from "openclaw/plugin-sdk/media-runtime";
 import { saveMediaBuffer, type SavedMedia } from "openclaw/plugin-sdk/media-store";
+import type { ReplyPayloadTtsSupplement } from "openclaw/plugin-sdk/reply-payload";
 import { readRegularFile, writeExternalFileWithinRoot } from "openclaw/plugin-sdk/security-runtime";
 import { normalizeLowercaseStringOrEmpty } from "openclaw/plugin-sdk/string-coerce-runtime";
 import {
@@ -711,10 +712,18 @@ export function shouldSuppressFeishuTextForVoiceMedia(params: {
  fileName?: string;
  contentType?: string;
  audioAsVoice?: boolean;
+  ttsSupplement?: ReplyPayloadTtsSupplement;
 }): boolean {
+  // TTS metadata owns visibility; voice-bubble inference must not hide text
+  // that has not been delivered yet.
+  if (params.ttsSupplement) {
+    return params.ttsSupplement.visibleTextAlreadyDelivered === true;
+  }
+
  if (params.audioAsVoice === true) {
    return true;
  }
+
  if (
    params.fileName &&
    isFeishuNativeVoiceAudio({
--- a/extensions/feishu/src/outbound.test.ts
+++ b/extensions/feishu/src/outbound.test.ts
@@ -20,8 +20,15 @@ const sendStructuredCardFeishuMock = vi.hoisted(() => vi.fn());
 const deliverCommentThreadTextMock = vi.hoisted(() => vi.fn());
 const cleanupAmbientCommentTypingReactionMock = vi.hoisted(() => vi.fn(async () => false));
 const shouldSuppressFeishuTextForVoiceMediaMock = vi.hoisted(
-  () => (params: { mediaUrl?: string; audioAsVoice?: boolean }) =>
-    params.audioAsVoice === true || /\.(?:ogg|opus)(?:[?#]|$)/i.test(params.mediaUrl ?? ""),
+  () =>
+    (params: {
+      mediaUrl?: string;
+      audioAsVoice?: boolean;
+      ttsSupplement?: { visibleTextAlreadyDelivered?: boolean };
+    }) =>
+      params.ttsSupplement
+        ? params.ttsSupplement.visibleTextAlreadyDelivered === true
+        : params.audioAsVoice === true || /\.(?:ogg|opus)(?:[?#]|$)/i.test(params.mediaUrl ?? ""),
 );

 vi.mock("./media.js", () => ({
@@ -349,6 +356,82 @@ describe("feishuOutbound.sendText local-image auto-convert", () => {
    return { dir, file };
  }

+  it("sends missing TTS text before its voice supplement", async () => {
+    const payload = {
+      text: "Readable answer",
+      mediaUrl: "https://example.com/reply.ogg",
+      audioAsVoice: true,
+      ttsSupplement: { spokenText: "Readable answer" },
+    };
+
+    await feishuOutbound.sendPayload?.({
+      cfg: emptyConfig,
+      to: "chat_1",
+      text: payload.text,
+      accountId: "main",
+      payload,
+    });
+
+    expect(sendMessageCall()?.text).toBe("Readable answer");
+    expect(sendMediaCall()?.mediaUrl).toBe("https://example.com/reply.ogg");
+    expect(sendMediaCall()?.audioAsVoice).toBe(true);
+    expect(sendMessageFeishuMock.mock.invocationCallOrder[0]).toBeLessThan(
+      sendMediaFeishuMock.mock.invocationCallOrder[0] ?? 0,
+    );
+  });
+
+  it("sends only TTS media when its text is already visible", async () => {
+    const payload = {
+      text: "Readable answer",
+      mediaUrl: "https://example.com/reply.ogg",
+      audioAsVoice: true,
+      ttsSupplement: {
+        spokenText: "Readable answer",
+        visibleTextAlreadyDelivered: true,
+      },
+    };
+
+    await feishuOutbound.sendPayload?.({
+      cfg: emptyConfig,
+      to: "chat_1",
+      text: payload.text,
+      accountId: "main",
+      payload,
+    });
+
+    expect(sendMessageFeishuMock).not.toHaveBeenCalled();
+    expect(sendStructuredCardFeishuMock).not.toHaveBeenCalled();
+    expect(sendMediaCall()?.mediaUrl).toBe("https://example.com/reply.ogg");
+  });
+
+  it("preserves a structured card before its TTS supplement", async () => {
+    const card = {
+      schema: "2.0",
+      body: { elements: [{ tag: "markdown", content: "Readable answer" }] },
+    };
+    const payload = {
+      text: "Readable answer",
+      mediaUrl: "https://example.com/reply.ogg",
+      audioAsVoice: true,
+      ttsSupplement: { spokenText: "Readable answer" },
+      channelData: { feishu: { card } },
+    };
+
+    await feishuOutbound.sendPayload?.({
+      cfg: emptyConfig,
+      to: "chat_1",
+      text: payload.text,
+      accountId: "main",
+      payload,
+    });
+
+    expect(sendCardCall()?.card).toMatchObject(card);
+    expect(sendMediaCall()?.mediaUrl).toBe("https://example.com/reply.ogg");
+    expect(sendCardFeishuMock.mock.invocationCallOrder[0]).toBeLessThan(
+      sendMediaFeishuMock.mock.invocationCallOrder[0] ?? 0,
+    );
+  });
+
  it("sends an absolute existing local image path as media", async () => {
    const { dir, file } = await createTmpImage();
    try {
@@ -1731,6 +1814,25 @@ describe("feishuOutbound.sendPayload native cards", () => {
    expectFeishuResult(result, "reply_msg");
  });

+  it("keeps TTS supplements on the document-comment delivery path", async () => {
+    await feishuOutbound.sendPayload?.({
+      cfg: emptyConfig,
+      to: "comment:docx:doxcn123:7623358762119646411",
+      text: "Readable answer",
+      accountId: "main",
+      payload: {
+        text: "Readable answer",
+        mediaUrl: "https://example.com/reply.ogg",
+        audioAsVoice: true,
+        ttsSupplement: { spokenText: "Readable answer" },
+      },
+    });
+
+    expect(deliverCommentThreadTextMock).toHaveBeenCalled();
+    expect(sendMessageFeishuMock).not.toHaveBeenCalled();
+    expect(sendMediaFeishuMock).not.toHaveBeenCalled();
+  });
+
  it("rejects card-only document comments instead of reporting an empty delivery", async () => {
    const text = JSON.stringify({
      header: { title: { tag: "plain_text", content: "Raw card" } },
--- a/extensions/feishu/src/outbound.ts
+++ b/extensions/feishu/src/outbound.ts
@@ -16,6 +16,7 @@ import {
 import { resolveMarkdownTableMode } from "openclaw/plugin-sdk/markdown-table-runtime";
 import { resolveChunkMode, resolveTextChunkLimit } from "openclaw/plugin-sdk/reply-chunking";
 import {
+  getReplyPayloadTtsSupplement,
  resolvePayloadMediaUrls,
  sendPayloadMediaSequenceAndFinalize,
  sendTextMediaPayload,
@@ -512,6 +513,62 @@ async function sendFeishuFallbackPayload(params: {
  return lastResult!;
 }

+async function sendFeishuTtsSupplementPayload(params: {
+  ctx: FeishuSendPayloadContext;
+  payload: FeishuOutboundPayload;
+  supplement: NonNullable<ReturnType<typeof getReplyPayloadTtsSupplement>>;
+  sendVisiblePayload?: (
+    replyToId: string | undefined,
+  ) => ReturnType<NonNullable<ChannelOutboundAdapter["sendText"]>>;
+}) {
+  const sendMedia = feishuOutbound.sendMedia;
+  const sendText = feishuOutbound.sendText;
+  if (!sendMedia || !sendText) {
+    throw new Error("Feishu TTS supplement delivery is not available.");
+  }
+
+  const { normalizedReplyToId } = resolveFeishuReplyMode({
+    replyToId: params.ctx.replyToId,
+    threadId: params.ctx.threadId,
+  });
+  const nextReplyToId = createReplyToFanout({
+    replyToId: normalizedReplyToId,
+    replyToIdSource: params.ctx.replyToIdSource,
+    replyToMode: params.ctx.replyToMode,
+  });
+  const ctx = { ...params.ctx, payload: params.payload };
+  let lastResult: Awaited<ReturnType<typeof sendText>> | undefined;
+
+  // Structured payloads still need their actions. Plain text follows the TTS
+  // visibility marker so an existing streamed reply is not duplicated.
+  if (params.sendVisiblePayload) {
+    lastResult = await params.sendVisiblePayload(nextReplyToId());
+    await ctx.onDeliveryResult?.(lastResult);
+  } else if (params.supplement.visibleTextAlreadyDelivered !== true) {
+    const text = params.payload.text?.trim() ? params.payload.text : params.supplement.spokenText;
+    for (const chunk of chunkFeishuMarkdown(text, FEISHU_TEXT_CHUNK_LIMIT)) {
+      lastResult = await sendText({
+        ...ctx,
+        text: chunk,
+        replyToId: nextReplyToId(),
+        onDeliveryResult: undefined,
+      });
+      await ctx.onDeliveryResult?.(lastResult);
+    }
+  }
+
+  for (const mediaUrl of normalizeStringEntries(resolvePayloadMediaUrls(params.payload))) {
+    lastResult = await sendMedia({
+      ...ctx,
+      text: "",
+      mediaUrl,
+      replyToId: nextReplyToId(),
+      audioAsVoice: params.payload.audioAsVoice ?? ctx.audioAsVoice,
+    });
+  }
+  return lastResult ?? { channel: "feishu", messageId: "" };
+}
+
 export const feishuOutbound: ChannelOutboundAdapter = {
  deliveryMode: "direct",
  chunker: chunkFeishuMarkdown,
@@ -540,6 +597,7 @@ export const feishuOutbound: ChannelOutboundAdapter = {
  renderPresentation: renderFeishuPresentationPayload,
  sendPayload: async (ctx) => {
    const { payload, presentationFallback } = consumeFeishuPresentationFallbackMarker(ctx.payload);
+    const ttsSupplement = getReplyPayloadTtsSupplement(payload);
    if (parseFeishuCommentTarget(ctx.to)) {
      const interactive = normalizeInteractiveReply(payload.interactive);
      const normalizedPresentation =
@@ -584,13 +642,15 @@ export const feishuOutbound: ChannelOutboundAdapter = {
        separateMediaAndText: true,
      });
    }
-
    const card = buildFeishuPayloadCard({
      payload,
      text: ctx.text,
      identity: ctx.identity,
    });
    if (!card) {
+      if (ttsSupplement) {
+        return await sendFeishuTtsSupplementPayload({ ctx, payload, supplement: ttsSupplement });
+      }
      const interactive = normalizeInteractiveReply(payload.interactive);
      const presentation =
        normalizeMessagePresentation(payload.presentation) ??
@@ -613,6 +673,31 @@ export const feishuOutbound: ChannelOutboundAdapter = {
      });
    }

+    if (ttsSupplement) {
+      return await sendFeishuTtsSupplementPayload({
+        ctx,
+        payload,
+        supplement: ttsSupplement,
+        sendVisiblePayload: async (replyToId) => {
+          const { replyToMessageId, replyInThread } = resolveFeishuReplyMode({
+            replyToId,
+            threadId: ctx.threadId,
+          });
+          return attachChannelToResult(
+            "feishu",
+            await sendCardFeishu({
+              cfg: ctx.cfg,
+              to: ctx.to,
+              card,
+              replyToMessageId,
+              replyInThread,
+              accountId: ctx.accountId ?? undefined,
+            }),
+          );
+        },
+      });
+    }
+
    const { normalizedReplyToId } = resolveFeishuReplyMode({
      replyToId: ctx.replyToId,
      threadId: ctx.threadId,
--- a/extensions/feishu/src/reply-dispatcher.test.ts
+++ b/extensions/feishu/src/reply-dispatcher.test.ts
@@ -24,8 +24,15 @@ const addTypingIndicatorMock = vi.hoisted(() => vi.fn(async () => ({ messageId:
 const removeTypingIndicatorMock = vi.hoisted(() => vi.fn(async () => {}));
 const streamingInstances = vi.hoisted((): StreamingSessionStub[] => []);
 const shouldSuppressFeishuTextForVoiceMediaMock = vi.hoisted(
-  () => (params: { mediaUrl?: string; audioAsVoice?: boolean }) =>
-    params.audioAsVoice === true || /\.(?:ogg|opus)(?:[?#]|$)/i.test(params.mediaUrl ?? ""),
+  () =>
+    (params: {
+      mediaUrl?: string;
+      audioAsVoice?: boolean;
+      ttsSupplement?: { visibleTextAlreadyDelivered?: boolean };
+    }) =>
+      params.ttsSupplement
+        ? params.ttsSupplement.visibleTextAlreadyDelivered === true
+        : params.audioAsVoice === true || /\.(?:ogg|opus)(?:[?#]|$)/i.test(params.mediaUrl ?? ""),
 );

 function mergeStreamingText(
@@ -129,6 +136,7 @@ describe("createFeishuReplyDispatcher streaming behavior", () => {
        mediaUrl?: string;
        mediaUrls?: string[];
        audioAsVoice?: boolean;
+        ttsSupplement?: { spokenText: string; visibleTextAlreadyDelivered?: boolean };
        isError?: boolean;
      },
      meta: { kind: string },
@@ -1316,6 +1324,64 @@ describe("createFeishuReplyDispatcher streaming behavior", () => {
    });
  });

+  it("sends TTS text before voice media when it is not already visible", async () => {
+    useNonStreamingAutoAccount();
+    const { options } = createDispatcherHarness();
+    await options.deliver(
+      {
+        text: "Readable answer",
+        mediaUrl: "https://example.com/reply.ogg",
+        audioAsVoice: true,
+        ttsSupplement: { spokenText: "Readable answer" },
+      },
+      { kind: "final" },
+    );
+
+    expectMockArgFields(sendMessageFeishuMock, "message send params", {
+      text: "Readable answer",
+    });
+    expectMockArgFields(sendMediaFeishuMock, "media send params", {
+      mediaUrl: "https://example.com/reply.ogg",
+      audioAsVoice: true,
+    });
+    expect(sendMessageFeishuMock.mock.invocationCallOrder[0]).toBeLessThan(
+      sendMediaFeishuMock.mock.invocationCallOrder[0] ?? 0,
+    );
+  });
+
+  it("keeps streamed text visible before its TTS supplement", async () => {
+    const account = resolveFeishuAccountMock();
+    resolveFeishuAccountMock.mockReturnValue({
+      ...account,
+      config: {
+        ...account.config,
+        streaming: { ...account.config.streaming, block: { enabled: true } },
+      },
+    });
+    const { options } = createDispatcherHarness();
+    await options.deliver({ text: "Readable answer" }, { kind: "block" });
+    await options.deliver(
+      {
+        mediaUrl: "https://example.com/reply.ogg",
+        audioAsVoice: true,
+        ttsSupplement: {
+          spokenText: "Readable answer",
+          visibleTextAlreadyDelivered: true,
+        },
+      },
+      { kind: "final" },
+    );
+    await options.onIdle?.();
+
+    expect(streamingInstances).toHaveLength(1);
+    expect(requireStreamingInstance(0).discard).not.toHaveBeenCalled();
+    expect(requireStreamingInstance(0).close).toHaveBeenCalledWith("Readable answer", {
+      note: "Agent: agent",
+    });
+    expect(sendMessageFeishuMock).not.toHaveBeenCalled();
+    expect(sendMediaFeishuMock).toHaveBeenCalledTimes(1);
+  });
+
  it("discards partial streaming text when final replies send voice media", async () => {
    const { result, options } = createDispatcherHarness({
      runtime: createRuntimeLogger(),
--- a/extensions/feishu/src/reply-dispatcher.ts
+++ b/extensions/feishu/src/reply-dispatcher.ts
@@ -9,6 +9,7 @@ import {
  resolveChannelStreamingBlockEnabled,
 } from "openclaw/plugin-sdk/channel-outbound";
 import {
+  getReplyPayloadTtsSupplement,
  resolveSendableOutboundReplyParts,
  resolveTextChunksWithFallback,
  sendMediaWithLeadingCaption,
@@ -689,12 +690,15 @@ export function createFeishuReplyDispatcher(params: CreateFeishuReplyDispatcherP
            : reply.text;
        const hasText = reply.hasText;
        const hasMedia = reply.hasMedia;
+        const ttsSupplement = getReplyPayloadTtsSupplement(payload);
+        const ttsTextAlreadyVisible = ttsSupplement?.visibleTextAlreadyDelivered === true;
        const hasVoiceMedia =
          hasMedia &&
          reply.mediaUrls.some((mediaUrl) =>
            shouldSuppressFeishuTextForVoiceMedia({
              mediaUrl,
              ...(payload.audioAsVoice === true ? { audioAsVoice: true } : {}),
+              ttsSupplement,
            }),
          );
        const finalTextExceedsStreamingLimit =
@@ -728,7 +732,9 @@ export function createFeishuReplyDispatcher(params: CreateFeishuReplyDispatcherP
        const shouldDiscardStreamingPreview =
          info?.kind === "final" &&
          (finalTextExceedsStreamingLimit ||
-            (hasMedia && ((hasVoiceMedia && !shouldDeliverText) || skipTextForDuplicateFinal)));
+            (hasMedia &&
+              ((hasVoiceMedia && !shouldDeliverText && !ttsTextAlreadyVisible) ||
+                skipTextForDuplicateFinal)));

        if (!shouldDeliverText && !hasMedia) {
          return;