diff --git a/src/auto-reply/chunk.test.ts b/src/auto-reply/chunk.test.ts index 4bd5dc4b65b2..33731b8fca27 100644 --- a/src/auto-reply/chunk.test.ts +++ b/src/auto-reply/chunk.test.ts @@ -6,6 +6,7 @@ import * as fences from "../../packages/markdown-core/src/fences.js"; import { hasBalancedFences } from "../test-utils/chunk-test-helpers.js"; import { chunkByNewline, + chunkByParagraph, chunkMarkdownText, chunkMarkdownTextWithMode, chunkText, @@ -230,6 +231,37 @@ describe("chunkText", () => { ]); }); +describe("chunkByParagraph Unicode line/paragraph separators", () => { + it.each([ + { + name: "treats lone U+2029 as a standalone paragraph boundary", + text: "paragraph one\u2029paragraph two starts here", + normalized: "paragraph one\n\nparagraph two starts here", + limit: 39, + expected: ["paragraph one", "paragraph two starts here"], + }, + { + name: "treats lone U+2028 as a line break within one paragraph", + text: "paragraph one line\u2028still same paragraph", + normalized: "paragraph one line\nstill same paragraph", + limit: 50, + expected: ["paragraph one line\nstill same paragraph"], + }, + { + name: "treats consecutive U+2028 and U+2029 as a paragraph boundary", + text: "paragraph one line\u2028\u2029paragraph two starts here", + normalized: "paragraph one line\n\nparagraph two starts here", + limit: 40, + expected: ["paragraph one line", "paragraph two starts here"], + }, + ] as const)("$name", ({ text, normalized, limit, expected }) => { + const chunks = chunkByParagraph(text, limit); + + expect(chunks).toEqual(expected); + expect(chunks).toEqual(chunkByParagraph(normalized, limit)); + }); +}); + describe("resolveTextChunkLimit", () => { it.each([ ...(["whatsapp", "telegram", "slack", "signal", "imessage", "discord"] as const).map( diff --git a/src/auto-reply/chunk.ts b/src/auto-reply/chunk.ts index 2b365765cc21..157d9d8872d1 100644 --- a/src/auto-reply/chunk.ts +++ b/src/auto-reply/chunk.ts @@ -207,8 +207,10 @@ export function chunkByParagraph( } const splitLongParagraphs = opts?.splitLongParagraphs !== false; - // Normalize to \n so blank line detection is consistent. - const normalized = text.replace(/\r\n?/g, "\n"); + // U+2029 PARAGRAPH SEPARATOR maps to a blank-line boundary; U+2028 LINE + // SEPARATOR and CR/CRLF map to a single newline. + // Normalize in two steps so consecutive U+2028\u2029 also produces a blank line. + const normalized = text.replace(/\u2029/g, "\n\n").replace(/\r\n?|\u2028/g, "\n"); // Fast-path: if there are no blank-line paragraph separators, do not split. // (We *do not* early-return based on `limit` — newline mode is about paragraph