From 3a4e2ef65f418d3fdc79cebcbbd698a7aa8b9344 Mon Sep 17 00:00:00 2001
From: maweibin <ma.weibin@xydigit.com>
Date: Sat, 18 Jul 2026 08:53:25 +0800
Subject: [PATCH] fix(auto-reply): treat U+2028/U+2029 as paragraph boundaries
 when chunking (#103518)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* fix(auto-reply): treat U+2028/U+2029 as paragraph boundaries when chunking

chunkByParagraph normalized only CR/CRLF before blank-line paragraph detection,
so model output using Unicode LINE/PARAGRAPH SEPARATOR (U+2028/U+2029) instead
of a blank line was not split at those boundaries and fell back to length-based
splitting. Normalize U+2028/U+2029 to \n alongside CR/CRLF, matching how the
Control UI markdown renderer handles them.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

* fix(auto-reply): normalize U+2028 as line break, U+2029 as blank-line paragraph boundary

Distinguish U+2028 (LINE SEPARATOR) from U+2029 (PARAGRAPH SEPARATOR):
U+2029 becomes \n\n (blank line — paragraph boundary) while U+2028
becomes \n (single newline — intra-paragraph line break).

The original fix mapped both to \n, so standalone U+2029 still
produced single-line text without a blank-line gap — paragraph
detection failed. The combined U+2028
 input accidentally
produced the right blank-line sequence, which masked the bug.

Adds individual tests for lone U+2029 (splits at paragraph boundary),
lone U+2028 (stays within paragraph), and consecutive U+2028

(combined blank line — matches \n\n behavior).

* test(auto-reply): simplify Unicode separator cases

---------

Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
Co-authored-by: Peter Steinberger <steipete@gmail.com>
---
 src/auto-reply/chunk.test.ts | 32 ++++++++++++++++++++++++++++++++
 src/auto-reply/chunk.ts      |  6 ++++--
 2 files changed, 36 insertions(+), 2 deletions(-)

diff --git a/src/auto-reply/chunk.test.ts b/src/auto-reply/chunk.test.ts
index 4bd5dc4b65b2..33731b8fca27 100644
--- a/src/auto-reply/chunk.test.ts
+++ b/src/auto-reply/chunk.test.ts
@@ -6,6 +6,7 @@ import * as fences from "../../packages/markdown-core/src/fences.js";
 import { hasBalancedFences } from "../test-utils/chunk-test-helpers.js";
 import {
   chunkByNewline,
+  chunkByParagraph,
   chunkMarkdownText,
   chunkMarkdownTextWithMode,
   chunkText,
@@ -230,6 +231,37 @@ describe("chunkText", () => {
   ]);
 });
 
+describe("chunkByParagraph Unicode line/paragraph separators", () => {
+  it.each([
+    {
+      name: "treats lone U+2029 as a standalone paragraph boundary",
+      text: "paragraph one\u2029paragraph two starts here",
+      normalized: "paragraph one\n\nparagraph two starts here",
+      limit: 39,
+      expected: ["paragraph one", "paragraph two starts here"],
+    },
+    {
+      name: "treats lone U+2028 as a line break within one paragraph",
+      text: "paragraph one line\u2028still same paragraph",
+      normalized: "paragraph one line\nstill same paragraph",
+      limit: 50,
+      expected: ["paragraph one line\nstill same paragraph"],
+    },
+    {
+      name: "treats consecutive U+2028 and U+2029 as a paragraph boundary",
+      text: "paragraph one line\u2028\u2029paragraph two starts here",
+      normalized: "paragraph one line\n\nparagraph two starts here",
+      limit: 40,
+      expected: ["paragraph one line", "paragraph two starts here"],
+    },
+  ] as const)("$name", ({ text, normalized, limit, expected }) => {
+    const chunks = chunkByParagraph(text, limit);
+
+    expect(chunks).toEqual(expected);
+    expect(chunks).toEqual(chunkByParagraph(normalized, limit));
+  });
+});
+
 describe("resolveTextChunkLimit", () => {
   it.each([
     ...(["whatsapp", "telegram", "slack", "signal", "imessage", "discord"] as const).map(
diff --git a/src/auto-reply/chunk.ts b/src/auto-reply/chunk.ts
index 2b365765cc21..157d9d8872d1 100644
--- a/src/auto-reply/chunk.ts
+++ b/src/auto-reply/chunk.ts
@@ -207,8 +207,10 @@ export function chunkByParagraph(
   }
   const splitLongParagraphs = opts?.splitLongParagraphs !== false;
 
-  // Normalize to \n so blank line detection is consistent.
-  const normalized = text.replace(/\r\n?/g, "\n");
+  // U+2029 PARAGRAPH SEPARATOR maps to a blank-line boundary; U+2028 LINE
+  // SEPARATOR and CR/CRLF map to a single newline.
+  // Normalize in two steps so consecutive U+2028\u2029 also produces a blank line.
+  const normalized = text.replace(/\u2029/g, "\n\n").replace(/\r\n?|\u2028/g, "\n");
 
   // Fast-path: if there are no blank-line paragraph separators, do not split.
   // (We *do not* early-return based on `limit` — newline mode is about paragraph