fix(markdown): keep IR text slicing UTF-16 safe (#110643)

* fix(markdown): keep IR text slicing UTF-16 safe

* fix(markdown): preserve UTF-16 IR boundaries

* chore(markdown): reconcile original merge-base

* fix(markdown): preserve UTF-16 IR boundaries

* test(markdown): verify portable UTF-16 boundaries

* fix(markdown): normalize safe slice offsets once

* fix(markdown): preserve safe slice SDK contract

* fix(markdown): preserve native UTF-16 slice semantics

---------

Co-authored-by: lizeyu-xydt <li.zeyu@xydigit.com>
Co-authored-by: Peter Steinberger <steipete@gmail.com>

packages/markdown-core/src/ir.test.ts

@@ -0,0 +1,201 @@
+import { describe, expect, it } from "vitest";
+import { chunkTextRanges } from "./chunk-text.js";
+import { markdownToIR, sliceMarkdownIR } from "./ir.js";
+
+// U+1F600 (😀) = 😀 in UTF-16.
+const EMOJI = "\u{1F600}";
+const LEAD_HIGH = "\uD83D"; // High surrogate for U+1F600
+const LEAD_LOW = "\uDE00"; // Low surrogate for U+1F600
+
+function expectWellFormedUtf16(text: string): void {
+  expect(new TextDecoder().decode(new TextEncoder().encode(text))).toBe(text);
+}
+
+describe("sliceMarkdownIR surrogate pair boundaries", () => {
+  it("expands start boundary backward when it lands on a low surrogate", () => {
+    // "a😀b" — UTF-16: [a] [\uD83D] [\uDE00] [b], indices 0-3
+    const ir = markdownToIR(`a${EMOJI}b`);
+    expect(ir.text[1]).toBe(LEAD_HIGH);
+    expect(ir.text[2]).toBe(LEAD_LOW);
+    // from=2 points at \uDE00 (LS); should expand to from=1 to include 😀
+    const sliced = sliceMarkdownIR(ir, 2, 4);
+    expect(sliced.text).toBe(`${EMOJI}b`);
+  });
+
+  it("expands end boundary forward when it splits between HS and LS", () => {
+    // "a😀b" — UTF-16: [a] [\uD83D] [\uDE00] [b], indices 0-3
+    const ir = markdownToIR(`a${EMOJI}b`);
+    // to=2 splits between \uD83D (HS) and \uDE00 (LS); should expand to to=3
+    const sliced = sliceMarkdownIR(ir, 0, 2);
+    expect(sliced.text).toBe(`a${EMOJI}`);
+  });
+
+  it("preserves full text when boundaries are already clean", () => {
+    const ir = markdownToIR(`a${EMOJI}b`);
+    // Clean boundaries that don't split any surrogate pair
+    const sliced = sliceMarkdownIR(ir, 0, 4);
+    expect(sliced.text).toBe(`a${EMOJI}b`);
+  });
+
+  it("leaves boundaries unchanged when start is on a high surrogate", () => {
+    const ir = markdownToIR(`a${EMOJI}b`);
+    // from=1 points at \uD83D (HS) — start of pair, no adjustment needed
+    const sliced = sliceMarkdownIR(ir, 1, 4);
+    expect(sliced.text).toBe(`${EMOJI}b`);
+  });
+
+  it("handles multiple consecutive surrogate pairs", () => {
+    // U+1F600 😀 (😀) + U+1F431 🐱 (🐱)
+    // Indices: 0=HS😀, 1=LS😀, 2=HS🐱, 3=LS🐱, len=4
+    const cat = "\u{1F431}";
+    const ir = markdownToIR(`${EMOJI}${cat}`);
+    // from=1 is LS of 😀 → expand backward to 0; to=4 is past end → stays 4
+    const sliced = sliceMarkdownIR(ir, 1, 4);
+    expect(sliced.text).toBe(`${EMOJI}${cat}`);
+  });
+
+  it("preserves empty slice when start === end lands inside a surrogate pair", () => {
+    // "a😀b" — UTF-16: [a] [\uD83D] [\uDE00] [b], indices 0-3
+    // start=end=2 lands on \uDE00 (LS); must remain empty, not expand to "😀"
+    const ir = markdownToIR(`a${EMOJI}b`);
+    const sliced = sliceMarkdownIR(ir, 2, 2);
+    expect(sliced.text).toBe("");
+  });
+
+  it("preserves empty slice when start === end lands on a high surrogate", () => {
+    // start=end=1 lands on \uD83D (HS); must remain empty
+    const ir = markdownToIR(`a${EMOJI}b`);
+    const sliced = sliceMarkdownIR(ir, 1, 1);
+    expect(sliced.text).toBe("");
+  });
+
+  it("handles negative start index", () => {
+    const ir = markdownToIR(`a${EMOJI}b`);
+    // from=-2 => len-2 = 2, which is LS; should expand backward to 1
+    const sliced = sliceMarkdownIR(ir, -2, 4);
+    expect(sliced.text).toBe(`${EMOJI}b`);
+  });
+
+  it("preserves native fractional and non-finite slice index semantics", () => {
+    const ir = markdownToIR("[**abcd**](https://example.com)");
+    const ranges = [
+      [-1.5, 4],
+      [0, -1.5],
+      [1.5, 3.9],
+      [Number.NaN, 2],
+      [Number.NEGATIVE_INFINITY, Number.POSITIVE_INFINITY],
+    ] as const;
+
+    for (const [start, end] of ranges) {
+      const sliced = sliceMarkdownIR(ir, start, end);
+      const expected = ir.text.slice(start, end);
+
+      expect(sliced.text).toBe(expected);
+      expect(sliced.styles).toEqual([
+        expect.objectContaining({ start: 0, end: expected.length, style: "bold" }),
+      ]);
+      expect(sliced.links).toEqual([
+        { start: 0, end: expected.length, href: "https://example.com" },
+      ]);
+    }
+  });
+
+  it("propagates adjusted boundaries to link spans", () => {
+    const ir = markdownToIR(`a[${EMOJI}b](https://example.com)`);
+    // from=2 is LS, should expand to 1
+    const sliced = sliceMarkdownIR(ir, 2, ir.text.length);
+    expect(sliced.text).toContain(EMOJI);
+    expect(sliced.links.length).toBeGreaterThan(0);
+  });
+
+  it("keeps nested link and style spans aligned with the expanded start", () => {
+    const href = "https://example.com";
+    const ir = markdownToIR(`a[**${EMOJI}b**](${href})`);
+    const sliced = sliceMarkdownIR(ir, 2, ir.text.length);
+
+    expect(sliced.text).toBe(`${EMOJI}b`);
+    expectWellFormedUtf16(sliced.text);
+    expect(sliced.links).toEqual([{ start: 0, end: 3, href }]);
+    expect(sliced.styles).toEqual([expect.objectContaining({ start: 0, end: 3, style: "bold" })]);
+  });
+
+  it("keeps transcript annotations and links aligned with the expanded end", () => {
+    const cat = "\u{1F431}";
+    const href = "https://example.com";
+    const ir = markdownToIR(`user[Thu 2026-07-02] **A${EMOJI}B** [C${cat}D](${href})`, {
+      assistantTranscriptRoleHeaders: true,
+    });
+    const catStart = ir.text.indexOf(cat);
+    const sliced = sliceMarkdownIR(ir, 0, catStart + 1);
+
+    expect(catStart).toBeGreaterThan(0);
+    expect(sliced.text).toBe(ir.text.slice(0, catStart + cat.length));
+    expectWellFormedUtf16(sliced.text);
+    expect(sliced.annotations).toEqual(ir.annotations);
+    expect(sliced.styles).toEqual(ir.styles);
+    expect(sliced.links).toEqual([
+      expect.objectContaining({
+        start: ir.links[0]?.start,
+        end: sliced.text.length,
+        href,
+      }),
+    ]);
+  });
+
+  it("preserves nested list markers when the final item ends inside a surrogate pair", () => {
+    const cat = "\u{1F431}";
+    const ir = markdownToIR(`- **A${EMOJI}B**\n  - [x] **C${cat}D**`);
+    const catStart = ir.text.indexOf(cat);
+    const sliced = sliceMarkdownIR(ir, 0, catStart + 1);
+
+    expect(catStart).toBeGreaterThan(0);
+    expect(sliced.text).toBe(ir.text.slice(0, catStart + cat.length));
+    expectWellFormedUtf16(sliced.text);
+    expect(sliced.listItems).toHaveLength(ir.listItems?.length ?? 0);
+    expect(sliced.listItems).toEqual(
+      expect.arrayContaining([
+        expect.objectContaining({ kind: "bullet", depth: 0 }),
+        expect.objectContaining({ kind: "bullet", depth: 1 }),
+      ]),
+    );
+    expect(sliced.styles).toEqual(
+      expect.arrayContaining([expect.objectContaining({ style: "bold", end: sliced.text.length })]),
+    );
+
+    const blocks = Reflect.get(ir, "blocks") as Array<{ start: number; end: number }>;
+    const slicedBlocks = Reflect.get(sliced, "blocks") as
+      | Array<{ start: number; end: number }>
+      | undefined;
+    if (blocks?.length) {
+      expect(slicedBlocks).toBeDefined();
+      expect(
+        slicedBlocks?.every((block) => block.start >= 0 && block.end <= sliced.text.length),
+      ).toBe(true);
+    }
+  });
+
+  it("rejoins the existing surrogate-safe transport chunks without duplicating emoji", () => {
+    const ir = markdownToIR(`a${EMOJI}b\u{1F431}c`);
+
+    for (const mode of ["hard", "preferred"] as const) {
+      for (const limit of [1, 2, 3, 4]) {
+        const chunks = chunkTextRanges(ir.text, { limit, mode }).map(({ start, end }) =>
+          sliceMarkdownIR(ir, start, end),
+        );
+
+        for (const chunk of chunks) {
+          expectWellFormedUtf16(chunk.text);
+        }
+        expect(chunks.map((chunk) => chunk.text).join("")).toBe(ir.text);
+      }
+    }
+  });
+
+  it("preserves normalized empty slice with mixed positive/negative indices", () => {
+    // After normalization: start=-1 → from=len-1 which is > to=0 = normalized empty.
+    // Surrogate adjustment must not expand this into a non-empty slice.
+    const ir = markdownToIR(`a${EMOJI}b`);
+    const sliced = sliceMarkdownIR(ir, -1, 0);
+    expect(sliced.text).toBe("");
+  });
+});

packages/markdown-core/src/ir.ts

@@ -1,4 +1,5 @@
 // Markdown Core module implements ir behavior.
+import { avoidTrailingHighSurrogateBreak } from "@openclaw/normalization-core/utf16-slice";
 import MarkdownIt from "markdown-it";
 import markdownItCjkFriendly from "markdown-it-cjk-friendly";
 import { HTML_TAG_RE } from "markdown-it/lib/common/html_re.mjs";
@@ -1447,14 +1448,44 @@ function sliceListMarker(
 }
 
 export function sliceMarkdownIR(ir: MarkdownIR, start: number, end: number): MarkdownIR {
+  const textLength = ir.text.length;
+  const integerStart = Math.trunc(start) || 0;
+  const integerEnd = Math.trunc(end) || 0;
+  let normalizedStart =
+    integerStart < 0 ? Math.max(textLength + integerStart, 0) : Math.min(integerStart, textLength);
+  let normalizedEnd =
+    integerEnd < 0 ? Math.max(textLength + integerEnd, 0) : Math.min(integerEnd, textLength);
+
+  if (normalizedStart < normalizedEnd) {
+    // Normalize once so text, formatting, links, and structural metadata share
+    // the same complete-code-point boundaries.
+    const safeStart = avoidTrailingHighSurrogateBreak(ir.text, 0, normalizedStart);
+    if (safeStart !== normalizedStart) {
+      normalizedStart = safeStart < normalizedStart ? safeStart : normalizedStart - 1;
+    }
+
+    const safeEnd = avoidTrailingHighSurrogateBreak(ir.text, 0, normalizedEnd);
+    if (safeEnd !== normalizedEnd) {
+      normalizedEnd = safeEnd > normalizedEnd ? safeEnd : normalizedEnd + 1;
+    }
+  }
+
   const metadataIR = ir as MarkdownIRWithMetadata;
-  const annotations = sliceAnnotationSpans(ir.annotations ?? [], start, end);
+  const annotations = sliceAnnotationSpans(ir.annotations ?? [], normalizedStart, normalizedEnd);
   const listItems = ((ir.listItems ?? []) as MarkdownListItemWithMetadata[]).flatMap((item) => {
-    const listMarker = item.listMarker ? sliceListMarker(item.listMarker, start, end) : undefined;
-    const taskMarker = item.taskMarker ? sliceListMarker(item.taskMarker, start, end) : undefined;
+    const listMarker = item.listMarker
+      ? sliceListMarker(item.listMarker, normalizedStart, normalizedEnd)
+      : undefined;
+    const taskMarker = item.taskMarker
+      ? sliceListMarker(item.taskMarker, normalizedStart, normalizedEnd)
+      : undefined;
     const content =
       item.contentStart !== undefined && item.contentEnd !== undefined
-        ? sliceListMarker({ start: item.contentStart, end: item.contentEnd }, start, end)
+        ? sliceListMarker(
+            { start: item.contentStart, end: item.contentEnd },
+            normalizedStart,
+            normalizedEnd,
+          )
         : undefined;
     return listMarker || taskMarker
       ? [
@@ -1467,8 +1498,12 @@ export function sliceMarkdownIR(ir: MarkdownIR, start: number, end: number): Mar
               ...(item.listId !== undefined ? { listId: item.listId } : {}),
               ...(item.parentListId !== undefined ? { parentListId: item.parentListId } : {}),
               ...(item.depth !== undefined ? { depth: item.depth } : {}),
-              ...(item.start !== undefined ? { start: Math.max(item.start, start) - start } : {}),
-              ...(item.end !== undefined ? { end: Math.min(item.end, end) - start } : {}),
+              ...(item.start !== undefined
+                ? { start: Math.max(item.start, normalizedStart) - normalizedStart }
+                : {}),
+              ...(item.end !== undefined
+                ? { end: Math.min(item.end, normalizedEnd) - normalizedStart }
+                : {}),
             },
             {
               ...(content ? { contentStart: content.start, contentEnd: content.end } : {}),
@@ -1486,18 +1521,20 @@ export function sliceMarkdownIR(ir: MarkdownIR, start: number, end: number): Mar
   const blocks = (metadataIR.blocks ?? []).flatMap((block) => {
     if (block.start === block.end) {
       const containsPoint =
-        start === end ? block.start === start : block.start >= start && block.start < end;
+        normalizedStart === normalizedEnd
+          ? block.start === normalizedStart
+          : block.start >= normalizedStart && block.start < normalizedEnd;
       return containsPoint
-        ? [{ ...block, start: block.start - start, end: block.end - start }]
+        ? [{ ...block, start: block.start - normalizedStart, end: block.end - normalizedStart }]
         : [];
     }
-    const sliced = sliceListMarker(block, start, end);
+    const sliced = sliceListMarker(block, normalizedStart, normalizedEnd);
     return sliced ? [{ ...block, ...sliced }] : [];
   });
   const sliced: MarkdownIR = {
-    text: ir.text.slice(start, end),
-    styles: sliceStyleSpans(ir.styles, start, end),
-    links: sliceLinkSpans(ir.links, start, end),
+    text: ir.text.slice(normalizedStart, normalizedEnd),
+    styles: sliceStyleSpans(ir.styles, normalizedStart, normalizedEnd),
+    links: sliceLinkSpans(ir.links, normalizedStart, normalizedEnd),
     ...(annotations.length > 0 ? { annotations } : {}),
     ...(listItems.length > 0 ? { listItems } : {}),
   };