mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-25 15:21:12 +00:00
e8ad0466ff
* fix: render assistant transcript headers safely Co-authored-by: snowzlmbot <293528334+snowzlmbot@users.noreply.github.com> * fix(markdown): nest crossing annotation spans * fix(markdown): protect final transport projections * refactor(markdown): split transcript render ownership * test(ui): cover assistant transcript render flag * refactor(markdown): keep transcript helpers private * docs(changelog): note assistant transcript headers * chore(plugin-sdk): refresh transcript annotation baseline * fix(markdown): harden final transcript projections * docs(changelog): defer transcript note to release * refactor(markdown): centralize HTML tokenization * fix(markdown): satisfy lint gates --------- Co-authored-by: Peter Steinberger <steipete@gmail.com> Co-authored-by: snowzlmbot <293528334+snowzlmbot@users.noreply.github.com>
85 lines
2.6 KiB
TypeScript
85 lines
2.6 KiB
TypeScript
/**
|
|
* Tests text and Markdown chunking helpers exported by the plugin SDK.
|
|
*/
|
|
import { describe, expect, it } from "vitest";
|
|
import { chunkTextForOutbound, chunkTextRanges, tokenizeHtmlTags } from "./text-chunking.js";
|
|
|
|
describe("tokenizeHtmlTags", () => {
|
|
it("keeps quoted attribute delimiters inside one tag token", () => {
|
|
expect([...tokenizeHtmlTags('<a href="https://example.com/?q=>">label</a>')]).toEqual([
|
|
expect.objectContaining({
|
|
raw: '<a href="https://example.com/?q=>">',
|
|
name: "a",
|
|
closing: false,
|
|
}),
|
|
expect.objectContaining({ raw: "</a>", name: "a", closing: true }),
|
|
]);
|
|
});
|
|
});
|
|
|
|
describe("chunkTextForOutbound", () => {
|
|
it.each([
|
|
{
|
|
name: "returns empty for empty input",
|
|
text: "",
|
|
maxLen: 10,
|
|
expected: [],
|
|
},
|
|
{
|
|
name: "splits on newline or whitespace boundaries",
|
|
text: "alpha\nbeta gamma",
|
|
maxLen: 8,
|
|
expected: ["alpha", "beta", "gamma"],
|
|
},
|
|
{
|
|
name: "falls back to hard limit when no separator exists",
|
|
text: "abcdefghij",
|
|
maxLen: 4,
|
|
expected: ["abcd", "efgh", "ij"],
|
|
},
|
|
])("$name", ({ text, maxLen, expected }) => {
|
|
expect(chunkTextForOutbound(text, maxLen)).toEqual(expected);
|
|
});
|
|
});
|
|
|
|
describe("chunkTextRanges", () => {
|
|
it("returns contiguous hard ranges without dropping whitespace", () => {
|
|
const text = "alpha beta\n\ngamma delta";
|
|
const ranges = chunkTextRanges(text, { limit: 12, mode: "hard" });
|
|
|
|
expect(ranges).toEqual([
|
|
{ start: 0, end: 12 },
|
|
{ start: 12, end: 24 },
|
|
]);
|
|
expect(ranges.map(({ start, end }) => text.slice(start, end))).toEqual([
|
|
"alpha beta\n",
|
|
"\ngamma delta",
|
|
]);
|
|
});
|
|
|
|
it("prefers paragraph, newline, then whitespace boundaries", () => {
|
|
const text = "a\n\nb\nc xyz";
|
|
const ranges = chunkTextRanges(text, { limit: 8, mode: "preferred" });
|
|
|
|
expect(ranges.map(({ start, end }) => text.slice(start, end))).toEqual(["a\n\n", "b\nc xyz"]);
|
|
});
|
|
|
|
it("falls back to hard ranges and handles empty or non-positive limits", () => {
|
|
expect(chunkTextRanges("abcdefgh", { limit: 3, mode: "preferred" })).toEqual([
|
|
{ start: 0, end: 3 },
|
|
{ start: 3, end: 6 },
|
|
{ start: 6, end: 8 },
|
|
]);
|
|
expect(chunkTextRanges("", { limit: 3 })).toEqual([]);
|
|
expect(chunkTextRanges("abc", { limit: 0 })).toEqual([{ start: 0, end: 3 }]);
|
|
});
|
|
|
|
it.each(["hard", "preferred"] as const)("keeps surrogate pairs intact in %s mode", (mode) => {
|
|
expect(chunkTextRanges("a😀b", { limit: 2, mode })).toEqual([
|
|
{ start: 0, end: 1 },
|
|
{ start: 1, end: 3 },
|
|
{ start: 3, end: 4 },
|
|
]);
|
|
});
|
|
});
|