diff --git a/extensions/feishu/src/media.test.ts b/extensions/feishu/src/media.test.ts index af3643d8ecb6..de2670a01601 100644 --- a/extensions/feishu/src/media.test.ts +++ b/extensions/feishu/src/media.test.ts @@ -208,6 +208,29 @@ describe("sendMediaFeishu msg_type routing", () => { ).toBe(false); }); + it("respects ttsSupplement.visibleTextAlreadyDelivered over audioAsVoice", () => { + expect( + shouldSuppressFeishuTextForVoiceMedia({ + mediaUrl: "https://example.com/tts.mp3", + audioAsVoice: true, + ttsSupplement: { + spokenText: "Hello world", + }, + }), + ).toBe(false); + + expect( + shouldSuppressFeishuTextForVoiceMedia({ + mediaUrl: "https://example.com/tts.mp3", + audioAsVoice: true, + ttsSupplement: { + spokenText: "Hello world", + visibleTextAlreadyDelivered: true, + }, + }), + ).toBe(true); + }); + it("uses msg_type=media for mp4 video", async () => { runFfprobeMock.mockResolvedValueOnce("4.2\n"); diff --git a/extensions/feishu/src/media.ts b/extensions/feishu/src/media.ts index b001848671ad..97012fda5fef 100644 --- a/extensions/feishu/src/media.ts +++ b/extensions/feishu/src/media.ts @@ -11,6 +11,7 @@ import { runFfprobe, } from "openclaw/plugin-sdk/media-runtime"; import { saveMediaBuffer, type SavedMedia } from "openclaw/plugin-sdk/media-store"; +import type { ReplyPayloadTtsSupplement } from "openclaw/plugin-sdk/reply-payload"; import { readRegularFile, writeExternalFileWithinRoot } from "openclaw/plugin-sdk/security-runtime"; import { normalizeLowercaseStringOrEmpty } from "openclaw/plugin-sdk/string-coerce-runtime"; import { @@ -711,10 +712,18 @@ export function shouldSuppressFeishuTextForVoiceMedia(params: { fileName?: string; contentType?: string; audioAsVoice?: boolean; + ttsSupplement?: ReplyPayloadTtsSupplement; }): boolean { + // TTS metadata owns visibility; voice-bubble inference must not hide text + // that has not been delivered yet. + if (params.ttsSupplement) { + return params.ttsSupplement.visibleTextAlreadyDelivered === true; + } + if (params.audioAsVoice === true) { return true; } + if ( params.fileName && isFeishuNativeVoiceAudio({ diff --git a/extensions/feishu/src/outbound.test.ts b/extensions/feishu/src/outbound.test.ts index edad6e100165..a2a7a5d965dd 100644 --- a/extensions/feishu/src/outbound.test.ts +++ b/extensions/feishu/src/outbound.test.ts @@ -20,8 +20,15 @@ const sendStructuredCardFeishuMock = vi.hoisted(() => vi.fn()); const deliverCommentThreadTextMock = vi.hoisted(() => vi.fn()); const cleanupAmbientCommentTypingReactionMock = vi.hoisted(() => vi.fn(async () => false)); const shouldSuppressFeishuTextForVoiceMediaMock = vi.hoisted( - () => (params: { mediaUrl?: string; audioAsVoice?: boolean }) => - params.audioAsVoice === true || /\.(?:ogg|opus)(?:[?#]|$)/i.test(params.mediaUrl ?? ""), + () => + (params: { + mediaUrl?: string; + audioAsVoice?: boolean; + ttsSupplement?: { visibleTextAlreadyDelivered?: boolean }; + }) => + params.ttsSupplement + ? params.ttsSupplement.visibleTextAlreadyDelivered === true + : params.audioAsVoice === true || /\.(?:ogg|opus)(?:[?#]|$)/i.test(params.mediaUrl ?? ""), ); vi.mock("./media.js", () => ({ @@ -349,6 +356,82 @@ describe("feishuOutbound.sendText local-image auto-convert", () => { return { dir, file }; } + it("sends missing TTS text before its voice supplement", async () => { + const payload = { + text: "Readable answer", + mediaUrl: "https://example.com/reply.ogg", + audioAsVoice: true, + ttsSupplement: { spokenText: "Readable answer" }, + }; + + await feishuOutbound.sendPayload?.({ + cfg: emptyConfig, + to: "chat_1", + text: payload.text, + accountId: "main", + payload, + }); + + expect(sendMessageCall()?.text).toBe("Readable answer"); + expect(sendMediaCall()?.mediaUrl).toBe("https://example.com/reply.ogg"); + expect(sendMediaCall()?.audioAsVoice).toBe(true); + expect(sendMessageFeishuMock.mock.invocationCallOrder[0]).toBeLessThan( + sendMediaFeishuMock.mock.invocationCallOrder[0] ?? 0, + ); + }); + + it("sends only TTS media when its text is already visible", async () => { + const payload = { + text: "Readable answer", + mediaUrl: "https://example.com/reply.ogg", + audioAsVoice: true, + ttsSupplement: { + spokenText: "Readable answer", + visibleTextAlreadyDelivered: true, + }, + }; + + await feishuOutbound.sendPayload?.({ + cfg: emptyConfig, + to: "chat_1", + text: payload.text, + accountId: "main", + payload, + }); + + expect(sendMessageFeishuMock).not.toHaveBeenCalled(); + expect(sendStructuredCardFeishuMock).not.toHaveBeenCalled(); + expect(sendMediaCall()?.mediaUrl).toBe("https://example.com/reply.ogg"); + }); + + it("preserves a structured card before its TTS supplement", async () => { + const card = { + schema: "2.0", + body: { elements: [{ tag: "markdown", content: "Readable answer" }] }, + }; + const payload = { + text: "Readable answer", + mediaUrl: "https://example.com/reply.ogg", + audioAsVoice: true, + ttsSupplement: { spokenText: "Readable answer" }, + channelData: { feishu: { card } }, + }; + + await feishuOutbound.sendPayload?.({ + cfg: emptyConfig, + to: "chat_1", + text: payload.text, + accountId: "main", + payload, + }); + + expect(sendCardCall()?.card).toMatchObject(card); + expect(sendMediaCall()?.mediaUrl).toBe("https://example.com/reply.ogg"); + expect(sendCardFeishuMock.mock.invocationCallOrder[0]).toBeLessThan( + sendMediaFeishuMock.mock.invocationCallOrder[0] ?? 0, + ); + }); + it("sends an absolute existing local image path as media", async () => { const { dir, file } = await createTmpImage(); try { @@ -1731,6 +1814,25 @@ describe("feishuOutbound.sendPayload native cards", () => { expectFeishuResult(result, "reply_msg"); }); + it("keeps TTS supplements on the document-comment delivery path", async () => { + await feishuOutbound.sendPayload?.({ + cfg: emptyConfig, + to: "comment:docx:doxcn123:7623358762119646411", + text: "Readable answer", + accountId: "main", + payload: { + text: "Readable answer", + mediaUrl: "https://example.com/reply.ogg", + audioAsVoice: true, + ttsSupplement: { spokenText: "Readable answer" }, + }, + }); + + expect(deliverCommentThreadTextMock).toHaveBeenCalled(); + expect(sendMessageFeishuMock).not.toHaveBeenCalled(); + expect(sendMediaFeishuMock).not.toHaveBeenCalled(); + }); + it("rejects card-only document comments instead of reporting an empty delivery", async () => { const text = JSON.stringify({ header: { title: { tag: "plain_text", content: "Raw card" } }, diff --git a/extensions/feishu/src/outbound.ts b/extensions/feishu/src/outbound.ts index 5109731a8da4..e09d2d5caa75 100644 --- a/extensions/feishu/src/outbound.ts +++ b/extensions/feishu/src/outbound.ts @@ -16,6 +16,7 @@ import { import { resolveMarkdownTableMode } from "openclaw/plugin-sdk/markdown-table-runtime"; import { resolveChunkMode, resolveTextChunkLimit } from "openclaw/plugin-sdk/reply-chunking"; import { + getReplyPayloadTtsSupplement, resolvePayloadMediaUrls, sendPayloadMediaSequenceAndFinalize, sendTextMediaPayload, @@ -512,6 +513,62 @@ async function sendFeishuFallbackPayload(params: { return lastResult!; } +async function sendFeishuTtsSupplementPayload(params: { + ctx: FeishuSendPayloadContext; + payload: FeishuOutboundPayload; + supplement: NonNullable>; + sendVisiblePayload?: ( + replyToId: string | undefined, + ) => ReturnType>; +}) { + const sendMedia = feishuOutbound.sendMedia; + const sendText = feishuOutbound.sendText; + if (!sendMedia || !sendText) { + throw new Error("Feishu TTS supplement delivery is not available."); + } + + const { normalizedReplyToId } = resolveFeishuReplyMode({ + replyToId: params.ctx.replyToId, + threadId: params.ctx.threadId, + }); + const nextReplyToId = createReplyToFanout({ + replyToId: normalizedReplyToId, + replyToIdSource: params.ctx.replyToIdSource, + replyToMode: params.ctx.replyToMode, + }); + const ctx = { ...params.ctx, payload: params.payload }; + let lastResult: Awaited> | undefined; + + // Structured payloads still need their actions. Plain text follows the TTS + // visibility marker so an existing streamed reply is not duplicated. + if (params.sendVisiblePayload) { + lastResult = await params.sendVisiblePayload(nextReplyToId()); + await ctx.onDeliveryResult?.(lastResult); + } else if (params.supplement.visibleTextAlreadyDelivered !== true) { + const text = params.payload.text?.trim() ? params.payload.text : params.supplement.spokenText; + for (const chunk of chunkFeishuMarkdown(text, FEISHU_TEXT_CHUNK_LIMIT)) { + lastResult = await sendText({ + ...ctx, + text: chunk, + replyToId: nextReplyToId(), + onDeliveryResult: undefined, + }); + await ctx.onDeliveryResult?.(lastResult); + } + } + + for (const mediaUrl of normalizeStringEntries(resolvePayloadMediaUrls(params.payload))) { + lastResult = await sendMedia({ + ...ctx, + text: "", + mediaUrl, + replyToId: nextReplyToId(), + audioAsVoice: params.payload.audioAsVoice ?? ctx.audioAsVoice, + }); + } + return lastResult ?? { channel: "feishu", messageId: "" }; +} + export const feishuOutbound: ChannelOutboundAdapter = { deliveryMode: "direct", chunker: chunkFeishuMarkdown, @@ -540,6 +597,7 @@ export const feishuOutbound: ChannelOutboundAdapter = { renderPresentation: renderFeishuPresentationPayload, sendPayload: async (ctx) => { const { payload, presentationFallback } = consumeFeishuPresentationFallbackMarker(ctx.payload); + const ttsSupplement = getReplyPayloadTtsSupplement(payload); if (parseFeishuCommentTarget(ctx.to)) { const interactive = normalizeInteractiveReply(payload.interactive); const normalizedPresentation = @@ -584,13 +642,15 @@ export const feishuOutbound: ChannelOutboundAdapter = { separateMediaAndText: true, }); } - const card = buildFeishuPayloadCard({ payload, text: ctx.text, identity: ctx.identity, }); if (!card) { + if (ttsSupplement) { + return await sendFeishuTtsSupplementPayload({ ctx, payload, supplement: ttsSupplement }); + } const interactive = normalizeInteractiveReply(payload.interactive); const presentation = normalizeMessagePresentation(payload.presentation) ?? @@ -613,6 +673,31 @@ export const feishuOutbound: ChannelOutboundAdapter = { }); } + if (ttsSupplement) { + return await sendFeishuTtsSupplementPayload({ + ctx, + payload, + supplement: ttsSupplement, + sendVisiblePayload: async (replyToId) => { + const { replyToMessageId, replyInThread } = resolveFeishuReplyMode({ + replyToId, + threadId: ctx.threadId, + }); + return attachChannelToResult( + "feishu", + await sendCardFeishu({ + cfg: ctx.cfg, + to: ctx.to, + card, + replyToMessageId, + replyInThread, + accountId: ctx.accountId ?? undefined, + }), + ); + }, + }); + } + const { normalizedReplyToId } = resolveFeishuReplyMode({ replyToId: ctx.replyToId, threadId: ctx.threadId, diff --git a/extensions/feishu/src/reply-dispatcher.test.ts b/extensions/feishu/src/reply-dispatcher.test.ts index bbdbac164a48..e578ea519bb1 100644 --- a/extensions/feishu/src/reply-dispatcher.test.ts +++ b/extensions/feishu/src/reply-dispatcher.test.ts @@ -24,8 +24,15 @@ const addTypingIndicatorMock = vi.hoisted(() => vi.fn(async () => ({ messageId: const removeTypingIndicatorMock = vi.hoisted(() => vi.fn(async () => {})); const streamingInstances = vi.hoisted((): StreamingSessionStub[] => []); const shouldSuppressFeishuTextForVoiceMediaMock = vi.hoisted( - () => (params: { mediaUrl?: string; audioAsVoice?: boolean }) => - params.audioAsVoice === true || /\.(?:ogg|opus)(?:[?#]|$)/i.test(params.mediaUrl ?? ""), + () => + (params: { + mediaUrl?: string; + audioAsVoice?: boolean; + ttsSupplement?: { visibleTextAlreadyDelivered?: boolean }; + }) => + params.ttsSupplement + ? params.ttsSupplement.visibleTextAlreadyDelivered === true + : params.audioAsVoice === true || /\.(?:ogg|opus)(?:[?#]|$)/i.test(params.mediaUrl ?? ""), ); function mergeStreamingText( @@ -129,6 +136,7 @@ describe("createFeishuReplyDispatcher streaming behavior", () => { mediaUrl?: string; mediaUrls?: string[]; audioAsVoice?: boolean; + ttsSupplement?: { spokenText: string; visibleTextAlreadyDelivered?: boolean }; isError?: boolean; }, meta: { kind: string }, @@ -1316,6 +1324,64 @@ describe("createFeishuReplyDispatcher streaming behavior", () => { }); }); + it("sends TTS text before voice media when it is not already visible", async () => { + useNonStreamingAutoAccount(); + const { options } = createDispatcherHarness(); + await options.deliver( + { + text: "Readable answer", + mediaUrl: "https://example.com/reply.ogg", + audioAsVoice: true, + ttsSupplement: { spokenText: "Readable answer" }, + }, + { kind: "final" }, + ); + + expectMockArgFields(sendMessageFeishuMock, "message send params", { + text: "Readable answer", + }); + expectMockArgFields(sendMediaFeishuMock, "media send params", { + mediaUrl: "https://example.com/reply.ogg", + audioAsVoice: true, + }); + expect(sendMessageFeishuMock.mock.invocationCallOrder[0]).toBeLessThan( + sendMediaFeishuMock.mock.invocationCallOrder[0] ?? 0, + ); + }); + + it("keeps streamed text visible before its TTS supplement", async () => { + const account = resolveFeishuAccountMock(); + resolveFeishuAccountMock.mockReturnValue({ + ...account, + config: { + ...account.config, + streaming: { ...account.config.streaming, block: { enabled: true } }, + }, + }); + const { options } = createDispatcherHarness(); + await options.deliver({ text: "Readable answer" }, { kind: "block" }); + await options.deliver( + { + mediaUrl: "https://example.com/reply.ogg", + audioAsVoice: true, + ttsSupplement: { + spokenText: "Readable answer", + visibleTextAlreadyDelivered: true, + }, + }, + { kind: "final" }, + ); + await options.onIdle?.(); + + expect(streamingInstances).toHaveLength(1); + expect(requireStreamingInstance(0).discard).not.toHaveBeenCalled(); + expect(requireStreamingInstance(0).close).toHaveBeenCalledWith("Readable answer", { + note: "Agent: agent", + }); + expect(sendMessageFeishuMock).not.toHaveBeenCalled(); + expect(sendMediaFeishuMock).toHaveBeenCalledTimes(1); + }); + it("discards partial streaming text when final replies send voice media", async () => { const { result, options } = createDispatcherHarness({ runtime: createRuntimeLogger(), diff --git a/extensions/feishu/src/reply-dispatcher.ts b/extensions/feishu/src/reply-dispatcher.ts index 43d955963390..57c9cc1e55ad 100644 --- a/extensions/feishu/src/reply-dispatcher.ts +++ b/extensions/feishu/src/reply-dispatcher.ts @@ -9,6 +9,7 @@ import { resolveChannelStreamingBlockEnabled, } from "openclaw/plugin-sdk/channel-outbound"; import { + getReplyPayloadTtsSupplement, resolveSendableOutboundReplyParts, resolveTextChunksWithFallback, sendMediaWithLeadingCaption, @@ -689,12 +690,15 @@ export function createFeishuReplyDispatcher(params: CreateFeishuReplyDispatcherP : reply.text; const hasText = reply.hasText; const hasMedia = reply.hasMedia; + const ttsSupplement = getReplyPayloadTtsSupplement(payload); + const ttsTextAlreadyVisible = ttsSupplement?.visibleTextAlreadyDelivered === true; const hasVoiceMedia = hasMedia && reply.mediaUrls.some((mediaUrl) => shouldSuppressFeishuTextForVoiceMedia({ mediaUrl, ...(payload.audioAsVoice === true ? { audioAsVoice: true } : {}), + ttsSupplement, }), ); const finalTextExceedsStreamingLimit = @@ -728,7 +732,9 @@ export function createFeishuReplyDispatcher(params: CreateFeishuReplyDispatcherP const shouldDiscardStreamingPreview = info?.kind === "final" && (finalTextExceedsStreamingLimit || - (hasMedia && ((hasVoiceMedia && !shouldDeliverText) || skipTextForDuplicateFinal))); + (hasMedia && + ((hasVoiceMedia && !shouldDeliverText && !ttsTextAlreadyVisible) || + skipTextForDuplicateFinal))); if (!shouldDeliverText && !hasMedia) { return;