mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-06 06:11:42 +00:00
3f9b4519a9
* fix: normalize fractional text chunk limits * fix: normalize markdown chunk limits * fix: normalize direct newline chunk limits * fix(matrix): reuse progress-safe text chunker * test(matrix): align runtime API guard * test(matrix): keep outbound shard topology stable * fix(matrix): preserve facade chunk compatibility * test(matrix): keep runtime export guard stable * fix(matrix): normalize render-aware chunk limits * test(matrix): type real-send assertions * fix(matrix): preserve one-unit event limits --------- Co-authored-by: Tak Hoffman <781889+Takhoffman@users.noreply.github.com> Co-authored-by: Vincent Koc <vincentkoc@ieee.org>
67 lines
2.9 KiB
TypeScript
67 lines
2.9 KiB
TypeScript
// Text chunking tests cover splitting text into bounded model-safe chunks.
|
|
import { describe, expect, it } from "vitest";
|
|
import { chunkTextByBreakResolver, splitLongTextLine } from "./text-chunking.js";
|
|
|
|
describe("shared/text-chunking", () => {
|
|
it("returns empty for blank input and the full text when under limit", () => {
|
|
expect(chunkTextByBreakResolver("", 10, () => 5)).toStrictEqual([]);
|
|
expect(chunkTextByBreakResolver("hello", 10, () => 2)).toEqual(["hello"]);
|
|
expect(chunkTextByBreakResolver("hello", 0, () => 2)).toEqual(["hello"]);
|
|
expect(chunkTextByBreakResolver("hello ", 10, () => 2)).toEqual(["hello "]);
|
|
expect(chunkTextByBreakResolver("hello ", 0, () => 2)).toEqual(["hello "]);
|
|
});
|
|
|
|
it("splits at resolver-provided breakpoints and trims separator boundaries", () => {
|
|
expect(
|
|
chunkTextByBreakResolver("alpha beta gamma", 10, (window) => window.lastIndexOf(" ")),
|
|
).toEqual(["alpha", "beta gamma"]);
|
|
expect(chunkTextByBreakResolver("abcd efgh", 4, () => 4)).toEqual(["abcd", "efgh"]);
|
|
});
|
|
|
|
it("falls back to hard limits for invalid break indexes", () => {
|
|
expect(chunkTextByBreakResolver("abcdefghij", 4, () => Number.NaN)).toEqual([
|
|
"abcd",
|
|
"efgh",
|
|
"ij",
|
|
]);
|
|
expect(chunkTextByBreakResolver("abcdefghij", 4, () => 99)).toEqual(["abcd", "efgh", "ij"]);
|
|
expect(chunkTextByBreakResolver("abcdefghij", 4, () => 0)).toEqual(["abcd", "efgh", "ij"]);
|
|
expect(chunkTextByBreakResolver("abcdefghij", 4, () => 0.5)).toEqual(["abcd", "efgh", "ij"]);
|
|
});
|
|
|
|
it("normalizes positive fractional limits before splitting", () => {
|
|
expect(chunkTextByBreakResolver("abc", 0.5, (window) => window.lastIndexOf(" "))).toEqual([
|
|
"a",
|
|
"b",
|
|
"c",
|
|
]);
|
|
expect(splitLongTextLine("abc", 0.5, { preserveWhitespace: true })).toEqual(["a", "b", "c"]);
|
|
expect(chunkTextByBreakResolver("😀😀", 0.5, () => -1)).toEqual(["😀", "😀"]);
|
|
expect(splitLongTextLine("😀😀", 0.5, { preserveWhitespace: true })).toEqual(["😀", "😀"]);
|
|
expect(splitLongTextLine("😀😀", 0.5, { preserveWhitespace: false })).toEqual(["😀", "😀"]);
|
|
});
|
|
|
|
it("skips empty chunks created by whitespace-only segments", () => {
|
|
expect(
|
|
chunkTextByBreakResolver("word next", 5, (window) => window.lastIndexOf(" ")),
|
|
).toEqual(["word", "next"]);
|
|
});
|
|
|
|
it("trims trailing whitespace from emitted chunks before continuing", () => {
|
|
expect(chunkTextByBreakResolver("abc def", 6, (window) => window.lastIndexOf(" "))).toEqual([
|
|
"abc",
|
|
"def",
|
|
]);
|
|
});
|
|
|
|
it.each([
|
|
{ text: " ! ", limit: 2, expected: ["!"] },
|
|
{ text: "a b ", limit: 2, expected: ["a", "b"] },
|
|
{ text: "alpha beta ", limit: 8, expected: ["alpha", "beta"] },
|
|
])("trims trailing whitespace from the final chunk: $text", ({ text, limit, expected }) => {
|
|
expect(chunkTextByBreakResolver(text, limit, (window) => window.lastIndexOf(" "))).toEqual(
|
|
expected,
|
|
);
|
|
});
|
|
});
|