From fa08ab28ca3ee75f097d65d002c83ef9aefb4f1f Mon Sep 17 00:00:00 2001 From: DaigoSoup Date: Fri, 17 Jul 2026 02:18:17 +0800 Subject: [PATCH] fix(telegram): preserve word boundaries in proactive sends (#108801) * fix(telegram): preserve proactive message word boundaries * fix(markdown): retain whitespace across rendered chunks * fix(telegram): preserve links in plain fallbacks * fix(channels): preserve chunk boundary semantics * test(sms): verify lossless chunk whitespace --------- Co-authored-by: Peter Steinberger --- extensions/sms/src/send.test.ts | 4 +- .../telegram/src/format.wrap-md.test.ts | 3 +- extensions/telegram/src/send.test.ts | 110 +++++++++++++----- extensions/telegram/src/send.ts | 11 +- .../src/render-aware-chunking.test.ts | 13 +++ .../src/render-aware-chunking.ts | 3 +- 6 files changed, 110 insertions(+), 34 deletions(-) diff --git a/extensions/sms/src/send.test.ts b/extensions/sms/src/send.test.ts index 38892deb9a8e..8ad1877a8903 100644 --- a/extensions/sms/src/send.test.ts +++ b/extensions/sms/src/send.test.ts @@ -49,7 +49,9 @@ describe("sendSmsTextChunks", () => { }); expect(sendSmsViaTwilio).toHaveBeenCalledTimes(2); - expect(sendSmsViaTwilio.mock.calls.map(([call]) => call.text)).toEqual(["alpha", "beta"]); + const texts = sendSmsViaTwilio.mock.calls.map(([call]) => call.text); + expect(texts).toEqual(["alpha", " beta"]); + expect(texts.join("")).toBe("alpha beta"); }); it("labels transcript-role headers promoted to an SMS chunk boundary", async () => { diff --git a/extensions/telegram/src/format.wrap-md.test.ts b/extensions/telegram/src/format.wrap-md.test.ts index ab7e8c74a9ac..05fec4d4bf3f 100644 --- a/extensions/telegram/src/format.wrap-md.test.ts +++ b/extensions/telegram/src/format.wrap-md.test.ts @@ -244,7 +244,8 @@ describe("markdownToTelegramChunks - file reference wrapping", () => { it("falls back to in-paren word boundaries when the parenthesis is unbalanced", () => { const input = "**foo (bar baz qux quux**"; const chunks = markdownToTelegramChunks(input, 20); - expect(chunks.map((chunk) => chunk.text)).toEqual(["foo", "(bar baz qux ", "quux"]); + expect(chunks.map((chunk) => chunk.text)).toEqual(["foo ", "(bar baz qux ", "quux"]); + expect(chunks.map((chunk) => chunk.text).join("")).toBe("foo (bar baz qux quux"); expectHtmlChunkLengthsAtMost(chunks, 20); }); diff --git a/extensions/telegram/src/send.test.ts b/extensions/telegram/src/send.test.ts index 6160e4775ab8..d8e8dc5839b6 100644 --- a/extensions/telegram/src/send.test.ts +++ b/extensions/telegram/src/send.test.ts @@ -1668,6 +1668,29 @@ describe("sendMessageTelegram", () => { expect(cursor.nextPartIndex).toBe(1); }); + it("preserves markdown link targets when Telegram rejects HTML", async () => { + botApi.sendMessage + .mockRejectedValueOnce(createHtmlParseError()) + .mockResolvedValueOnce({ message_id: 157, chat: { id: "123" } }); + + await sendMessageTelegram("123", "Read [docs](https://example.com/guide)", { + cfg: TELEGRAM_TEST_CFG, + token: "tok", + }); + + expect(botApi.sendMessage).toHaveBeenNthCalledWith( + 1, + "123", + 'Read docs', + { parse_mode: "HTML" }, + ); + expect(botApi.sendMessage).toHaveBeenNthCalledWith( + 2, + "123", + "Read docs (https://example.com/guide)", + ); + }); + it("reports the first Telegram chunk before a later chunk fails", async () => { const storePath = `/tmp/openclaw-telegram-projection-partial-${process.pid}-${Date.now()}.json`; const cursor = createTelegramPromptContextProjectionCursor({ @@ -1735,6 +1758,28 @@ describe("sendMessageTelegram", () => { expect(chunks.every((chunk) => chunk.length <= 4000)).toBe(true); }); + it("preserves word boundaries when rendered markdown exceeds the text limit", async () => { + botApi.sendMessage.mockResolvedValue({ message_id: 53, chat: { id: "123" } }); + const visibleText = Array.from({ length: 260 }, () => "alpha beta gamma").join(" "); + const markdown = `**${visibleText}**`; + + await sendMessageTelegram("123", markdown, { + cfg: TELEGRAM_TEST_CFG, + token: "tok", + }); + + const chunks = sendMessageTexts(botApi.sendMessage); + const visibleChunks = chunks.map((chunk) => telegramHtmlToPlainTextFallback(chunk)); + expect(chunks.length).toBeGreaterThan(1); + expect(chunks.every((chunk) => chunk.length <= 4000)).toBe(true); + expect(visibleChunks.join("")).toBe(visibleText); + for (let index = 0; index < visibleChunks.length - 1; index += 1) { + const left = visibleChunks[index] ?? ""; + const right = visibleChunks[index + 1] ?? ""; + expect(`${left.at(-1) ?? ""}${right.at(0) ?? ""}`).not.toMatch(/^[A-Za-z]{2}$/); + } + }); + it("chunks long markdown headings on the text path", async () => { botApi.sendMessage.mockResolvedValue({ message_id: 54, chat: { id: "123" } }); const markdown = Array.from({ length: 600 }, (_, index) => `# Heading ${index + 1}`).join("\n"); @@ -2131,7 +2176,8 @@ describe("sendMessageTelegram", () => { const sendMessage = vi .fn() .mockResolvedValueOnce({ message_id: 73, chat: { id: chatId } }) - .mockResolvedValueOnce({ message_id: 74, chat: { id: chatId } }); + .mockResolvedValueOnce({ message_id: 74, chat: { id: chatId } }) + .mockResolvedValueOnce({ message_id: 75, chat: { id: chatId } }); const api = { sendPhoto, sendMessage } as unknown as { sendPhoto: typeof sendPhoto; sendMessage: typeof sendMessage; @@ -2154,18 +2200,18 @@ describe("sendMessageTelegram", () => { expectMediaSendCall(firstMockCall(sendPhoto, "send photo call"), "send photo call", chatId, { caption: undefined, }); - expect(sendMessage).toHaveBeenCalledTimes(2); + expect(sendMessage).toHaveBeenCalledTimes(3); expect(sendMessage.mock.calls.every((call) => call[2]?.parse_mode === "HTML")).toBe(true); expect(sendMessage.mock.calls.map((call) => String(call[1] ?? "")).join("")).toContain("A"); - expect(res.messageId).toBe("74"); + expect(res.messageId).toBe("75"); expect(res.receipt?.primaryPlatformMessageId).toBe("73"); - expect(res.receipt?.platformMessageIds).toEqual(["73", "74"]); - expect(res.receipt?.parts.map((part) => part.kind)).toEqual(["text", "text"]); + expect(res.receipt?.platformMessageIds).toEqual(["73", "74", "75"]); + expect(res.receipt?.parts.map((part) => part.kind)).toEqual(["text", "text", "text"]); const cache = createTelegramMessageCache({ scope: resolveTelegramMessageCacheScope(storePath), }); const projections = await Promise.all( - ["72", "73", "74"].map( + ["72", "73", "74", "75"].map( async (messageId) => (await cache.get({ accountId: "default", chatId, messageId })) ?.promptContextProjectionMarker, @@ -2180,9 +2226,13 @@ describe("sendMessageTelegram", () => { kind: "valid", projection: { ...cursor.source, partIndex: 1, finalPart: false }, }, - { kind: "valid", projection: { ...cursor.source, partIndex: 2, finalPart: true } }, + { + kind: "valid", + projection: { ...cursor.source, partIndex: 2, finalPart: false }, + }, + { kind: "valid", projection: { ...cursor.source, partIndex: 3, finalPart: true } }, ]); - expect(cursor.nextPartIndex).toBe(3); + expect(cursor.nextPartIndex).toBe(4); }); it("uses caption when text is within 1024 char limit", async () => { @@ -3178,7 +3228,8 @@ describe("sendMessageTelegram", () => { const sendMessage = vi .fn() .mockResolvedValueOnce({ message_id: 101, chat: { id: "-1001234567890" } }) - .mockResolvedValueOnce({ message_id: 102, chat: { id: "-1001234567890" } }); + .mockResolvedValueOnce({ message_id: 102, chat: { id: "-1001234567890" } }) + .mockResolvedValueOnce({ message_id: 103, chat: { id: "-1001234567890" } }); const api = { sendMessage } as unknown as { sendMessage: typeof sendMessage; }; @@ -3193,18 +3244,12 @@ describe("sendMessageTelegram", () => { replyToMode: "first", }); - expect(sendMessage).toHaveBeenCalledTimes(2); - expect(sendMessage.mock.calls[0]?.[2]).toEqual({ - parse_mode: "HTML", - message_thread_id: 271, - }); - expect(sendMessage.mock.calls[1]?.[2]).toEqual({ - parse_mode: "HTML", - message_thread_id: 271, - }); - expect(result.messageId).toBe("102"); + expect(sendMessage).toHaveBeenCalledTimes(3); + expect(sendMessage.mock.calls.every((call) => call[2]?.parse_mode === "HTML")).toBe(true); + expect(sendMessage.mock.calls.every((call) => call[2]?.message_thread_id === 271)).toBe(true); + expect(result.messageId).toBe("103"); expect(result.receipt?.primaryPlatformMessageId).toBe("101"); - expect(result.receipt?.platformMessageIds).toEqual(["101", "102"]); + expect(result.receipt?.platformMessageIds).toEqual(["101", "102", "103"]); expect(result.receipt?.threadId).toBe("271"); expect(result.receipt?.replyToId).toBeUndefined(); expect( @@ -3230,6 +3275,13 @@ describe("sendMessageTelegram", () => { threadId: "271", replyToId: undefined, }, + { + platformMessageId: "103", + kind: "text", + index: 2, + threadId: "271", + replyToId: undefined, + }, ]); }); @@ -3237,7 +3289,8 @@ describe("sendMessageTelegram", () => { const sendMessage = vi .fn() .mockResolvedValueOnce({ message_id: 101, chat: { id: "-1001234567890" } }) - .mockResolvedValueOnce({ message_id: 102, chat: { id: "-1001234567890" } }); + .mockResolvedValueOnce({ message_id: 102, chat: { id: "-1001234567890" } }) + .mockResolvedValueOnce({ message_id: 103, chat: { id: "-1001234567890" } }); const api = { sendMessage } as unknown as { sendMessage: typeof sendMessage; }; @@ -3251,14 +3304,13 @@ describe("sendMessageTelegram", () => { replyToMode: "first", }); - expect(sendMessage.mock.calls[0]?.[2]).toMatchObject({ - reply_to_message_id: 500, - allow_sending_without_reply: true, - }); - expect(sendMessage.mock.calls[1]?.[2]).toMatchObject({ - reply_to_message_id: 500, - allow_sending_without_reply: true, - }); + expect(sendMessage).toHaveBeenCalledTimes(3); + for (const call of sendMessage.mock.calls) { + expect(call[2]).toMatchObject({ + reply_to_message_id: 500, + allow_sending_without_reply: true, + }); + } }); it("fails topic sends instead of retrying without message_thread_id", async () => { diff --git a/extensions/telegram/src/send.ts b/extensions/telegram/src/send.ts index 8c677a9c80ca..4f4cec96e130 100644 --- a/extensions/telegram/src/send.ts +++ b/extensions/telegram/src/send.ts @@ -33,6 +33,7 @@ import { splitTelegramCaption } from "./caption.js"; import { asTelegramClientFetch, createTelegramClientFetch } from "./client-fetch.js"; import { resolveTelegramTransport, type TelegramTransport } from "./fetch.js"; import { + markdownToTelegramChunks, renderTelegramHtmlText, splitTelegramHtmlChunks, telegramHtmlToPlainTextFallback, @@ -1094,8 +1095,16 @@ async function sendMessageTelegramWithContext( }; const buildChunkedTextPlan = (rawText: string, context: string): TelegramTextChunk[] => { + if (textMode === "markdown") { + // Chunk Markdown before rendering so HTML expansion cannot introduce a + // second mid-word split. Caller-authored HTML keeps its safe splitter below. + return markdownToTelegramChunks(rawText, 4000, { tableMode }).map((chunk) => ({ + htmlText: chunk.html, + plainText: telegramHtmlToPlainTextFallback(chunk.html), + })); + } const htmlText = renderHtmlText(rawText); - const fallbackText = textMode === "html" ? telegramHtmlToPlainTextFallback(htmlText) : rawText; + const fallbackText = telegramHtmlToPlainTextFallback(htmlText); let htmlChunks: string[]; try { htmlChunks = splitTelegramHtmlChunks(htmlText, 4000); diff --git a/packages/markdown-core/src/render-aware-chunking.test.ts b/packages/markdown-core/src/render-aware-chunking.test.ts index bfb57231220c..d1075519b74b 100644 --- a/packages/markdown-core/src/render-aware-chunking.test.ts +++ b/packages/markdown-core/src/render-aware-chunking.test.ts @@ -72,6 +72,19 @@ describe("renderMarkdownIRChunksWithinLimit", () => { expect(chunks.map((chunk) => chunk.source.text)).toEqual(["README.md", "<"]); }); + it("preserves separator whitespace in the initial rendered-size split", () => { + const ir = markdownToIR("alpha beta gamma"); + const chunks = renderMarkdownIRChunksWithinLimit({ + ir, + limit: 10, + renderChunk: (chunk) => chunk.text, + measureRendered: (rendered) => rendered.length, + }); + + expect(chunks.map((chunk) => chunk.source.text)).toEqual(["alpha ", "beta gamma"]); + expect(chunks.map((chunk) => chunk.source.text).join("")).toBe(ir.text); + }); + it("normalizes non-finite limits before chunking", () => { const ir = markdownToIR("abc"); const chunks = renderMarkdownIRChunksWithinLimit({ diff --git a/packages/markdown-core/src/render-aware-chunking.ts b/packages/markdown-core/src/render-aware-chunking.ts index 06a816f0d9d5..8e566df73910 100644 --- a/packages/markdown-core/src/render-aware-chunking.ts +++ b/packages/markdown-core/src/render-aware-chunking.ts @@ -3,7 +3,6 @@ import { avoidTrailingHighSurrogateBreak } from "./chunk-text.js"; import { annotateAssistantTranscriptRoleMessageBoundary } from "./ir-annotations.js"; import { mergeAnnotationSpans, type MarkdownAnnotationSpan } from "./ir-spans.js"; import { - chunkMarkdownIR, sliceMarkdownIR, type MarkdownIR, type MarkdownLinkSpan, @@ -77,7 +76,7 @@ export function renderMarkdownIRChunksWithinLimit( // Treat the pending worklist as a stack so each dequeue/enqueue stays O(1). // The initial reverse keeps the final order stable while avoiding shift/unshift // moving every remaining chunk for long messages. - const pending = chunkMarkdownIR(options.ir, normalizedLimit).toReversed(); + const pending = splitMarkdownIRPreserveWhitespace(options.ir, normalizedLimit).toReversed(); const finalized: MarkdownIR[] = []; while (pending.length > 0) {