mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-20 18:51:38 +00:00
1b6557dfa2
* fix(markdown): a fenced-code line with trailing text is content, not a closing fence scanFenceSpans accepted any line starting with >=3 matching fence markers as a closing fence, ignoring trailing text after the marker. Per CommonMark a closing fence may be followed only by whitespace, so a code-content line such as "``` not a close" was wrongly treated as a close: the block ended early, the following lines were reported as outside any fence, and the trailing marker line became a new unclosed opener. That made isSafeFenceBreak() return true for offsets inside the real code block and findFenceSpanAt() return undefined, so chunkers (chunkMarkdownText, the embedded-agent block chunker) could split inside a fenced code block — the exact thing this module exists to prevent. Require the closing fence's trailing text to be whitespace-only. Opening info strings, bare closes, and longer same-marker closes are unaffected. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * fix(markdown): honor fence suffix whitespace rules Co-authored-by: ly-wang19 <ly-wang19@users.noreply.github.com> --------- Co-authored-by: ly-wang19 <ly-wang19@users.noreply.github.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Co-authored-by: Vincent Koc <vincentkoc@ieee.org>
46 lines
2.0 KiB
TypeScript
46 lines
2.0 KiB
TypeScript
// Tests fenced-code-block span scanning used to keep chunk breaks out of code blocks.
|
|
import { describe, expect, it } from "vitest";
|
|
import { isSafeFenceBreak, parseFenceSpans } from "./fences.js";
|
|
|
|
describe("parseFenceSpans closing-fence rules", () => {
|
|
it("treats a marker line with trailing text as code content, not a closing fence", () => {
|
|
// CommonMark: a closing fence may be followed only by whitespace, so "``` not a close" is code
|
|
// content and the block stays open until the real closing fence. Reporting an interior offset
|
|
// as a safe break would let a chunker split inside the code block.
|
|
const text = "```\ncode\n``` not a close\nmore code\n```\n";
|
|
const spans = parseFenceSpans(text);
|
|
|
|
expect(spans).toHaveLength(1);
|
|
expect(isSafeFenceBreak(spans, text.indexOf("more code") + 1)).toBe(false);
|
|
});
|
|
|
|
it("does not close on non-space/tab whitespace", () => {
|
|
for (const suffix of ["\u00a0", "\v", "\f"]) {
|
|
const text = `\`\`\`\ncode\n\`\`\`${suffix}\nmore code\n`;
|
|
const spans = parseFenceSpans(text);
|
|
|
|
expect(spans).toHaveLength(1);
|
|
expect(spans[0]?.end).toBe(text.length);
|
|
expect(isSafeFenceBreak(spans, text.indexOf("more code") + 1)).toBe(false);
|
|
}
|
|
});
|
|
|
|
it("closes fences with CRLF line endings", () => {
|
|
const text = "```\r\ncode\r\n```\r\nafter\r\n";
|
|
const spans = parseFenceSpans(text);
|
|
|
|
expect(spans).toHaveLength(1);
|
|
expect(isSafeFenceBreak(spans, text.indexOf("after") + 1)).toBe(true);
|
|
});
|
|
|
|
it("still closes on a bare fence, a longer same-marker fence, and keeps an opener info string", () => {
|
|
expect(parseFenceSpans("```\ncode\n```\nafter\n")).toHaveLength(1);
|
|
expect(parseFenceSpans("```\ncode\n````` \nafter\n")).toHaveLength(1);
|
|
expect(parseFenceSpans("```python\nx = 1\n```\n")).toHaveLength(1);
|
|
|
|
const closed = "```\ncode\n```\nafter\n";
|
|
const spans = parseFenceSpans(closed);
|
|
expect(isSafeFenceBreak(spans, closed.indexOf("after") + 1)).toBe(true);
|
|
});
|
|
});
|