diff --git a/packages/markdown-core/src/ir.test.ts b/packages/markdown-core/src/ir.test.ts new file mode 100644 index 000000000000..ee9b2c76259d --- /dev/null +++ b/packages/markdown-core/src/ir.test.ts @@ -0,0 +1,201 @@ +import { describe, expect, it } from "vitest"; +import { chunkTextRanges } from "./chunk-text.js"; +import { markdownToIR, sliceMarkdownIR } from "./ir.js"; + +// U+1F600 (😀) = 😀 in UTF-16. +const EMOJI = "\u{1F600}"; +const LEAD_HIGH = "\uD83D"; // High surrogate for U+1F600 +const LEAD_LOW = "\uDE00"; // Low surrogate for U+1F600 + +function expectWellFormedUtf16(text: string): void { + expect(new TextDecoder().decode(new TextEncoder().encode(text))).toBe(text); +} + +describe("sliceMarkdownIR surrogate pair boundaries", () => { + it("expands start boundary backward when it lands on a low surrogate", () => { + // "a😀b" — UTF-16: [a] [\uD83D] [\uDE00] [b], indices 0-3 + const ir = markdownToIR(`a${EMOJI}b`); + expect(ir.text[1]).toBe(LEAD_HIGH); + expect(ir.text[2]).toBe(LEAD_LOW); + // from=2 points at \uDE00 (LS); should expand to from=1 to include 😀 + const sliced = sliceMarkdownIR(ir, 2, 4); + expect(sliced.text).toBe(`${EMOJI}b`); + }); + + it("expands end boundary forward when it splits between HS and LS", () => { + // "a😀b" — UTF-16: [a] [\uD83D] [\uDE00] [b], indices 0-3 + const ir = markdownToIR(`a${EMOJI}b`); + // to=2 splits between \uD83D (HS) and \uDE00 (LS); should expand to to=3 + const sliced = sliceMarkdownIR(ir, 0, 2); + expect(sliced.text).toBe(`a${EMOJI}`); + }); + + it("preserves full text when boundaries are already clean", () => { + const ir = markdownToIR(`a${EMOJI}b`); + // Clean boundaries that don't split any surrogate pair + const sliced = sliceMarkdownIR(ir, 0, 4); + expect(sliced.text).toBe(`a${EMOJI}b`); + }); + + it("leaves boundaries unchanged when start is on a high surrogate", () => { + const ir = markdownToIR(`a${EMOJI}b`); + // from=1 points at \uD83D (HS) — start of pair, no adjustment needed + const sliced = sliceMarkdownIR(ir, 1, 4); + expect(sliced.text).toBe(`${EMOJI}b`); + }); + + it("handles multiple consecutive surrogate pairs", () => { + // U+1F600 😀 (😀) + U+1F431 🐱 (🐱) + // Indices: 0=HS😀, 1=LS😀, 2=HS🐱, 3=LS🐱, len=4 + const cat = "\u{1F431}"; + const ir = markdownToIR(`${EMOJI}${cat}`); + // from=1 is LS of 😀 → expand backward to 0; to=4 is past end → stays 4 + const sliced = sliceMarkdownIR(ir, 1, 4); + expect(sliced.text).toBe(`${EMOJI}${cat}`); + }); + + it("preserves empty slice when start === end lands inside a surrogate pair", () => { + // "a😀b" — UTF-16: [a] [\uD83D] [\uDE00] [b], indices 0-3 + // start=end=2 lands on \uDE00 (LS); must remain empty, not expand to "😀" + const ir = markdownToIR(`a${EMOJI}b`); + const sliced = sliceMarkdownIR(ir, 2, 2); + expect(sliced.text).toBe(""); + }); + + it("preserves empty slice when start === end lands on a high surrogate", () => { + // start=end=1 lands on \uD83D (HS); must remain empty + const ir = markdownToIR(`a${EMOJI}b`); + const sliced = sliceMarkdownIR(ir, 1, 1); + expect(sliced.text).toBe(""); + }); + + it("handles negative start index", () => { + const ir = markdownToIR(`a${EMOJI}b`); + // from=-2 => len-2 = 2, which is LS; should expand backward to 1 + const sliced = sliceMarkdownIR(ir, -2, 4); + expect(sliced.text).toBe(`${EMOJI}b`); + }); + + it("preserves native fractional and non-finite slice index semantics", () => { + const ir = markdownToIR("[**abcd**](https://example.com)"); + const ranges = [ + [-1.5, 4], + [0, -1.5], + [1.5, 3.9], + [Number.NaN, 2], + [Number.NEGATIVE_INFINITY, Number.POSITIVE_INFINITY], + ] as const; + + for (const [start, end] of ranges) { + const sliced = sliceMarkdownIR(ir, start, end); + const expected = ir.text.slice(start, end); + + expect(sliced.text).toBe(expected); + expect(sliced.styles).toEqual([ + expect.objectContaining({ start: 0, end: expected.length, style: "bold" }), + ]); + expect(sliced.links).toEqual([ + { start: 0, end: expected.length, href: "https://example.com" }, + ]); + } + }); + + it("propagates adjusted boundaries to link spans", () => { + const ir = markdownToIR(`a[${EMOJI}b](https://example.com)`); + // from=2 is LS, should expand to 1 + const sliced = sliceMarkdownIR(ir, 2, ir.text.length); + expect(sliced.text).toContain(EMOJI); + expect(sliced.links.length).toBeGreaterThan(0); + }); + + it("keeps nested link and style spans aligned with the expanded start", () => { + const href = "https://example.com"; + const ir = markdownToIR(`a[**${EMOJI}b**](${href})`); + const sliced = sliceMarkdownIR(ir, 2, ir.text.length); + + expect(sliced.text).toBe(`${EMOJI}b`); + expectWellFormedUtf16(sliced.text); + expect(sliced.links).toEqual([{ start: 0, end: 3, href }]); + expect(sliced.styles).toEqual([expect.objectContaining({ start: 0, end: 3, style: "bold" })]); + }); + + it("keeps transcript annotations and links aligned with the expanded end", () => { + const cat = "\u{1F431}"; + const href = "https://example.com"; + const ir = markdownToIR(`user[Thu 2026-07-02] **A${EMOJI}B** [C${cat}D](${href})`, { + assistantTranscriptRoleHeaders: true, + }); + const catStart = ir.text.indexOf(cat); + const sliced = sliceMarkdownIR(ir, 0, catStart + 1); + + expect(catStart).toBeGreaterThan(0); + expect(sliced.text).toBe(ir.text.slice(0, catStart + cat.length)); + expectWellFormedUtf16(sliced.text); + expect(sliced.annotations).toEqual(ir.annotations); + expect(sliced.styles).toEqual(ir.styles); + expect(sliced.links).toEqual([ + expect.objectContaining({ + start: ir.links[0]?.start, + end: sliced.text.length, + href, + }), + ]); + }); + + it("preserves nested list markers when the final item ends inside a surrogate pair", () => { + const cat = "\u{1F431}"; + const ir = markdownToIR(`- **A${EMOJI}B**\n - [x] **C${cat}D**`); + const catStart = ir.text.indexOf(cat); + const sliced = sliceMarkdownIR(ir, 0, catStart + 1); + + expect(catStart).toBeGreaterThan(0); + expect(sliced.text).toBe(ir.text.slice(0, catStart + cat.length)); + expectWellFormedUtf16(sliced.text); + expect(sliced.listItems).toHaveLength(ir.listItems?.length ?? 0); + expect(sliced.listItems).toEqual( + expect.arrayContaining([ + expect.objectContaining({ kind: "bullet", depth: 0 }), + expect.objectContaining({ kind: "bullet", depth: 1 }), + ]), + ); + expect(sliced.styles).toEqual( + expect.arrayContaining([expect.objectContaining({ style: "bold", end: sliced.text.length })]), + ); + + const blocks = Reflect.get(ir, "blocks") as Array<{ start: number; end: number }>; + const slicedBlocks = Reflect.get(sliced, "blocks") as + | Array<{ start: number; end: number }> + | undefined; + if (blocks?.length) { + expect(slicedBlocks).toBeDefined(); + expect( + slicedBlocks?.every((block) => block.start >= 0 && block.end <= sliced.text.length), + ).toBe(true); + } + }); + + it("rejoins the existing surrogate-safe transport chunks without duplicating emoji", () => { + const ir = markdownToIR(`a${EMOJI}b\u{1F431}c`); + + for (const mode of ["hard", "preferred"] as const) { + for (const limit of [1, 2, 3, 4]) { + const chunks = chunkTextRanges(ir.text, { limit, mode }).map(({ start, end }) => + sliceMarkdownIR(ir, start, end), + ); + + for (const chunk of chunks) { + expectWellFormedUtf16(chunk.text); + } + expect(chunks.map((chunk) => chunk.text).join("")).toBe(ir.text); + } + } + }); + + it("preserves normalized empty slice with mixed positive/negative indices", () => { + // After normalization: start=-1 → from=len-1 which is > to=0 = normalized empty. + // Surrogate adjustment must not expand this into a non-empty slice. + const ir = markdownToIR(`a${EMOJI}b`); + const sliced = sliceMarkdownIR(ir, -1, 0); + expect(sliced.text).toBe(""); + }); +}); diff --git a/packages/markdown-core/src/ir.ts b/packages/markdown-core/src/ir.ts index 97e8ca616fb2..6e9b5f8ca922 100644 --- a/packages/markdown-core/src/ir.ts +++ b/packages/markdown-core/src/ir.ts @@ -1,4 +1,5 @@ // Markdown Core module implements ir behavior. +import { avoidTrailingHighSurrogateBreak } from "@openclaw/normalization-core/utf16-slice"; import MarkdownIt from "markdown-it"; import markdownItCjkFriendly from "markdown-it-cjk-friendly"; import { HTML_TAG_RE } from "markdown-it/lib/common/html_re.mjs"; @@ -1447,14 +1448,44 @@ function sliceListMarker( } export function sliceMarkdownIR(ir: MarkdownIR, start: number, end: number): MarkdownIR { + const textLength = ir.text.length; + const integerStart = Math.trunc(start) || 0; + const integerEnd = Math.trunc(end) || 0; + let normalizedStart = + integerStart < 0 ? Math.max(textLength + integerStart, 0) : Math.min(integerStart, textLength); + let normalizedEnd = + integerEnd < 0 ? Math.max(textLength + integerEnd, 0) : Math.min(integerEnd, textLength); + + if (normalizedStart < normalizedEnd) { + // Normalize once so text, formatting, links, and structural metadata share + // the same complete-code-point boundaries. + const safeStart = avoidTrailingHighSurrogateBreak(ir.text, 0, normalizedStart); + if (safeStart !== normalizedStart) { + normalizedStart = safeStart < normalizedStart ? safeStart : normalizedStart - 1; + } + + const safeEnd = avoidTrailingHighSurrogateBreak(ir.text, 0, normalizedEnd); + if (safeEnd !== normalizedEnd) { + normalizedEnd = safeEnd > normalizedEnd ? safeEnd : normalizedEnd + 1; + } + } + const metadataIR = ir as MarkdownIRWithMetadata; - const annotations = sliceAnnotationSpans(ir.annotations ?? [], start, end); + const annotations = sliceAnnotationSpans(ir.annotations ?? [], normalizedStart, normalizedEnd); const listItems = ((ir.listItems ?? []) as MarkdownListItemWithMetadata[]).flatMap((item) => { - const listMarker = item.listMarker ? sliceListMarker(item.listMarker, start, end) : undefined; - const taskMarker = item.taskMarker ? sliceListMarker(item.taskMarker, start, end) : undefined; + const listMarker = item.listMarker + ? sliceListMarker(item.listMarker, normalizedStart, normalizedEnd) + : undefined; + const taskMarker = item.taskMarker + ? sliceListMarker(item.taskMarker, normalizedStart, normalizedEnd) + : undefined; const content = item.contentStart !== undefined && item.contentEnd !== undefined - ? sliceListMarker({ start: item.contentStart, end: item.contentEnd }, start, end) + ? sliceListMarker( + { start: item.contentStart, end: item.contentEnd }, + normalizedStart, + normalizedEnd, + ) : undefined; return listMarker || taskMarker ? [ @@ -1467,8 +1498,12 @@ export function sliceMarkdownIR(ir: MarkdownIR, start: number, end: number): Mar ...(item.listId !== undefined ? { listId: item.listId } : {}), ...(item.parentListId !== undefined ? { parentListId: item.parentListId } : {}), ...(item.depth !== undefined ? { depth: item.depth } : {}), - ...(item.start !== undefined ? { start: Math.max(item.start, start) - start } : {}), - ...(item.end !== undefined ? { end: Math.min(item.end, end) - start } : {}), + ...(item.start !== undefined + ? { start: Math.max(item.start, normalizedStart) - normalizedStart } + : {}), + ...(item.end !== undefined + ? { end: Math.min(item.end, normalizedEnd) - normalizedStart } + : {}), }, { ...(content ? { contentStart: content.start, contentEnd: content.end } : {}), @@ -1486,18 +1521,20 @@ export function sliceMarkdownIR(ir: MarkdownIR, start: number, end: number): Mar const blocks = (metadataIR.blocks ?? []).flatMap((block) => { if (block.start === block.end) { const containsPoint = - start === end ? block.start === start : block.start >= start && block.start < end; + normalizedStart === normalizedEnd + ? block.start === normalizedStart + : block.start >= normalizedStart && block.start < normalizedEnd; return containsPoint - ? [{ ...block, start: block.start - start, end: block.end - start }] + ? [{ ...block, start: block.start - normalizedStart, end: block.end - normalizedStart }] : []; } - const sliced = sliceListMarker(block, start, end); + const sliced = sliceListMarker(block, normalizedStart, normalizedEnd); return sliced ? [{ ...block, ...sliced }] : []; }); const sliced: MarkdownIR = { - text: ir.text.slice(start, end), - styles: sliceStyleSpans(ir.styles, start, end), - links: sliceLinkSpans(ir.links, start, end), + text: ir.text.slice(normalizedStart, normalizedEnd), + styles: sliceStyleSpans(ir.styles, normalizedStart, normalizedEnd), + links: sliceLinkSpans(ir.links, normalizedStart, normalizedEnd), ...(annotations.length > 0 ? { annotations } : {}), ...(listItems.length > 0 ? { listItems } : {}), };