mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-05 17:31:34 +00:00
fix: prevent fractional chunk limits from stalling text splitting (#117727)
* fix: normalize fractional text chunk limits * fix: normalize markdown chunk limits * fix: normalize direct newline chunk limits * fix(matrix): reuse progress-safe text chunker * test(matrix): align runtime API guard * test(matrix): keep outbound shard topology stable * fix(matrix): preserve facade chunk compatibility * test(matrix): keep runtime export guard stable * fix(matrix): normalize render-aware chunk limits * test(matrix): type real-send assertions * fix(matrix): preserve one-unit event limits --------- Co-authored-by: Tak Hoffman <781889+Takhoffman@users.noreply.github.com> Co-authored-by: Vincent Koc <vincentkoc@ieee.org>
This commit is contained in:
@@ -1,4 +1,6 @@
|
||||
// Matrix API module exposes the plugin public contract.
|
||||
import { chunkTextForOutbound as chunkTextForOutboundSdk } from "openclaw/plugin-sdk/text-chunking";
|
||||
|
||||
export {
|
||||
type MatrixResolvedStringField,
|
||||
type MatrixResolvedStringValues,
|
||||
@@ -53,7 +55,15 @@ export type { PluginRuntime, RuntimeLogger } from "openclaw/plugin-sdk/plugin-ru
|
||||
export type { RuntimeEnv } from "openclaw/plugin-sdk/runtime-env";
|
||||
export type { WizardPrompter } from "openclaw/plugin-sdk/setup";
|
||||
|
||||
// This facade shipped distinct empty and whitespace behavior. Preserve that
|
||||
// contract while delegating fractional limits to the progress-safe SDK owner.
|
||||
export function chunkTextForOutbound(text: string, limit: number): string[] {
|
||||
if (text.length === 0) {
|
||||
return [""];
|
||||
}
|
||||
if (Number.isFinite(limit) && limit > 0 && !Number.isInteger(limit)) {
|
||||
return chunkTextForOutboundSdk(text, limit);
|
||||
}
|
||||
const chunks: string[] = [];
|
||||
let remaining = text;
|
||||
while (remaining.length > limit) {
|
||||
@@ -63,7 +73,7 @@ export function chunkTextForOutbound(text: string, limit: number): string[] {
|
||||
chunks.push(remaining.slice(0, breakAt).trimEnd());
|
||||
remaining = remaining.slice(breakAt).trimStart();
|
||||
}
|
||||
if (remaining.length > 0 || text.length === 0) {
|
||||
if (remaining.length > 0) {
|
||||
chunks.push(remaining);
|
||||
}
|
||||
return chunks;
|
||||
|
||||
@@ -441,6 +441,52 @@ describe("sendMessageMatrix durable delivery", () => {
|
||||
fs.rmSync(stateDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("dispatches fractional BMP and astral limits through the real send path", async () => {
|
||||
chunkMarkdownTextWithModeMock.mockImplementation((text) => Array.from(text));
|
||||
|
||||
resolveTextChunkLimitMock.mockReturnValue(0.5);
|
||||
const bmp = makeClient();
|
||||
await sendMessageMatrix("room:!room:example", "ABCD", {
|
||||
client: bmp.client,
|
||||
cfg: {} as never,
|
||||
});
|
||||
expect(bmp.sendMessage).toHaveBeenCalledTimes(4);
|
||||
expect(
|
||||
bmp.sendMessage.mock.calls.map((call) => requireRecord(call[1], "BMP content").body),
|
||||
).toEqual(["A", "B", "C", "D"]);
|
||||
|
||||
resolveTextChunkLimitMock.mockReturnValue(1.5);
|
||||
const astral = makeClient();
|
||||
await sendMessageMatrix("room:!room:example", "😀😀", {
|
||||
client: astral.client,
|
||||
cfg: {} as never,
|
||||
});
|
||||
expect(astral.sendMessage).toHaveBeenCalledTimes(2);
|
||||
expect(
|
||||
astral.sendMessage.mock.calls.map((call) => requireRecord(call[1], "astral content").body),
|
||||
).toEqual(["😀", "😀"]);
|
||||
|
||||
resolveTextChunkLimitMock.mockReturnValue(1.5);
|
||||
const mixed = makeClient();
|
||||
await sendMessageMatrix("room:!room:example", "😀AB", {
|
||||
client: mixed.client,
|
||||
cfg: {} as never,
|
||||
});
|
||||
expect(
|
||||
mixed.sendMessage.mock.calls.map((call) => requireRecord(call[1], "mixed content").body),
|
||||
).toEqual(["😀", "A", "B"]);
|
||||
|
||||
resolveTextChunkLimitMock.mockReturnValue(1);
|
||||
const integer = makeClient();
|
||||
await sendMessageMatrix("room:!room:example", "😀AB", {
|
||||
client: integer.client,
|
||||
cfg: {} as never,
|
||||
});
|
||||
expect(
|
||||
integer.sendMessage.mock.calls.map((call) => requireRecord(call[1], "integer content").body),
|
||||
).toEqual(["😀", "A", "B"]);
|
||||
});
|
||||
|
||||
it("persists the complete event plan before the first provider dispatch", async () => {
|
||||
const { client, sendMessage } = makeClient();
|
||||
const deliveryIdentity = resolveMatrixDurableDeliveryIdentity({
|
||||
|
||||
@@ -37,6 +37,23 @@ type MatrixPreparedChunkedText = MatrixPreparedSingleText & {
|
||||
|
||||
const getCore = () => getMatrixRuntime();
|
||||
|
||||
function normalizeMatrixEventLimit(limit: number): number {
|
||||
if (!Number.isFinite(limit) || limit <= 0) {
|
||||
return limit;
|
||||
}
|
||||
return Math.max(1, Math.floor(limit));
|
||||
}
|
||||
|
||||
function resolveMatrixChunkOverflow(chunk: string, limit: number): number {
|
||||
const body = markdownToMatrixBody(chunk);
|
||||
const renderedLength = Math.max(chunk.length, body.length);
|
||||
if (limit === 1 && Array.from(chunk).length === 1 && Array.from(body).length === 1) {
|
||||
// One astral code point occupies two UTF-16 units but cannot be split into a valid event.
|
||||
return 0;
|
||||
}
|
||||
return Math.max(0, renderedLength - limit);
|
||||
}
|
||||
|
||||
function protectMatrixUnderlineTags(markdown: string): MatrixSpoilerProtection {
|
||||
const codeRegions = findCodeRegions(markdown);
|
||||
const metadataRanges = findMatrixMarkdownMetadataRanges(markdown);
|
||||
@@ -170,11 +187,13 @@ export function prepareMatrixSingleText(
|
||||
accountId: opts.accountId,
|
||||
supportsBlockTables: MATRIX_FORMAT_PROFILE.constructs.table === "native",
|
||||
});
|
||||
const singleEventLimit = Math.min(
|
||||
getCore().channel.text.resolveTextChunkLimit(cfg, "matrix", opts.accountId),
|
||||
MATRIX_FORMAT_PROFILE.chunk.limit,
|
||||
);
|
||||
const convertedText = renderMatrixMarkdownTables(trimmedText, tableMode);
|
||||
const singleEventLimit = normalizeMatrixEventLimit(
|
||||
Math.min(
|
||||
getCore().channel.text.resolveTextChunkLimit(cfg, "matrix", opts.accountId),
|
||||
MATRIX_FORMAT_PROFILE.chunk.limit,
|
||||
),
|
||||
);
|
||||
const eventTextLength = Math.max(
|
||||
convertedText.length,
|
||||
markdownToMatrixBody(convertedText).length,
|
||||
@@ -241,10 +260,8 @@ export function chunkMatrixText(
|
||||
});
|
||||
const overflow = Math.max(
|
||||
0,
|
||||
...restored.map(
|
||||
(chunk) =>
|
||||
Math.max(chunk.length, markdownToMatrixBody(chunk).length) -
|
||||
preparedText.singleEventLimit,
|
||||
...restored.map((chunk) =>
|
||||
resolveMatrixChunkOverflow(chunk, preparedText.singleEventLimit),
|
||||
),
|
||||
);
|
||||
if (overflow === 0) {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
// Matrix tests cover outbound plugin behavior.
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import type { OpenClawConfig } from "../runtime-api.js";
|
||||
import { chunkTextForOutbound, type OpenClawConfig } from "../runtime-api.js";
|
||||
|
||||
const mocks = vi.hoisted(() => ({
|
||||
sendMessageMatrix: vi.fn(),
|
||||
@@ -63,6 +63,24 @@ describe("matrixOutbound cfg threading", () => {
|
||||
expect(chunker("hello world", 5)).toEqual(["hello", "world"]);
|
||||
});
|
||||
|
||||
it("makes progress for fractional BMP and astral limits", () => {
|
||||
const chunker = matrixOutbound.chunker;
|
||||
if (!chunker) {
|
||||
throw new Error("matrixOutbound.chunker missing");
|
||||
}
|
||||
|
||||
expect(chunker("ABCD", 0.5)).toEqual(["A", "B", "C", "D"]);
|
||||
expect(chunker("😀😀", 1.5)).toEqual(["😀", "😀"]);
|
||||
expect(chunkTextForOutbound("ABCD", 0.5)).toEqual(["A", "B", "C", "D"]);
|
||||
expect(chunkTextForOutbound("😀😀", 1.5)).toEqual(["😀", "😀"]);
|
||||
});
|
||||
|
||||
it("preserves Matrix compatibility behavior", () => {
|
||||
expect(chunkTextForOutbound("", 5)).toEqual([""]);
|
||||
expect(chunkTextForOutbound("", 0.5)).toEqual([""]);
|
||||
expect(chunkTextForOutbound("abcdef ", 5)).toEqual(["abcde", "f "]);
|
||||
});
|
||||
|
||||
it("passes resolved cfg to sendMessageMatrix for text sends", async () => {
|
||||
const cfg = {
|
||||
channels: {
|
||||
|
||||
10
packages/markdown-core/src/chunk-text.test.ts
Normal file
10
packages/markdown-core/src/chunk-text.test.ts
Normal file
@@ -0,0 +1,10 @@
|
||||
// Markdown Core tests cover plain-text chunking behavior.
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { chunkText } from "./chunk-text.js";
|
||||
|
||||
describe("chunkText", () => {
|
||||
it("normalizes positive fractional limits without emitting empty chunks", () => {
|
||||
expect(chunkText("abc", 0.5)).toEqual(["a", "b", "c"]);
|
||||
expect(chunkText("😀😀", 0.5)).toEqual(["😀", "😀"]);
|
||||
});
|
||||
});
|
||||
@@ -1,8 +1,14 @@
|
||||
// Markdown Core module implements chunk text behavior.
|
||||
import { resolveIntegerOption } from "@openclaw/normalization-core/number-coercion";
|
||||
import { avoidTrailingHighSurrogateBreak } from "@openclaw/normalization-core/utf16-slice";
|
||||
|
||||
export { avoidTrailingHighSurrogateBreak };
|
||||
|
||||
function normalizeChunkLimit(limit: number): number {
|
||||
// String slicing truncates fractional indexes, so positive limits need an integer progress step.
|
||||
return Number.isFinite(limit) && limit > 0 ? resolveIntegerOption(limit, 1, { min: 1 }) : limit;
|
||||
}
|
||||
|
||||
function resolveChunkEarlyReturn(text: string, limit: number): string[] | undefined {
|
||||
if (!text) {
|
||||
return [];
|
||||
@@ -89,14 +95,15 @@ export function chunkTextRanges(text: string, options: ChunkTextRangesOptions):
|
||||
if (!text) {
|
||||
return [];
|
||||
}
|
||||
if (options.limit <= 0 || text.length <= options.limit) {
|
||||
const normalizedLimit = normalizeChunkLimit(options.limit);
|
||||
if (normalizedLimit <= 0 || text.length <= normalizedLimit) {
|
||||
return [{ start: 0, end: text.length }];
|
||||
}
|
||||
|
||||
const ranges: TextChunkRange[] = [];
|
||||
let start = 0;
|
||||
while (start < text.length) {
|
||||
const maxEnd = Math.min(text.length, start + options.limit);
|
||||
const maxEnd = Math.min(text.length, start + normalizedLimit);
|
||||
const preferredEnd =
|
||||
options.mode === "preferred" && maxEnd < text.length
|
||||
? findPreferredRangeEnd(text, start, maxEnd)
|
||||
@@ -115,7 +122,8 @@ export function chunkTextRanges(text: string, options: ChunkTextRangesOptions):
|
||||
* Returns the original text as one chunk when the limit is non-positive.
|
||||
*/
|
||||
export function chunkText(text: string, limit: number): string[] {
|
||||
const early = resolveChunkEarlyReturn(text, limit);
|
||||
const normalizedLimit = normalizeChunkLimit(limit);
|
||||
const early = resolveChunkEarlyReturn(text, normalizedLimit);
|
||||
if (early) {
|
||||
return early;
|
||||
}
|
||||
@@ -123,11 +131,11 @@ export function chunkText(text: string, limit: number): string[] {
|
||||
const chunks: string[] = [];
|
||||
let cursor = 0;
|
||||
while (cursor < text.length) {
|
||||
if (text.length - cursor <= limit) {
|
||||
if (text.length - cursor <= normalizedLimit) {
|
||||
chunks.push(text.slice(cursor));
|
||||
break;
|
||||
}
|
||||
const windowEnd = Math.min(text.length, cursor + limit);
|
||||
const windowEnd = Math.min(text.length, cursor + normalizedLimit);
|
||||
const window = text.slice(cursor, windowEnd);
|
||||
const { lastNewline, lastWhitespace } = scanParenAwareBreakpoints(window);
|
||||
// Prefer block boundaries, then spaces, then a hard size cut when no
|
||||
|
||||
@@ -570,6 +570,15 @@ describe("chunkByNewline", () => {
|
||||
expect(chunks.every((chunk) => !/[\uD800-\uDBFF]$/u.test(chunk))).toBe(true);
|
||||
expect(chunks.every((chunk) => !/^[\uDC00-\uDFFF]/u.test(chunk))).toBe(true);
|
||||
});
|
||||
|
||||
it("normalizes fractional limits before an astral hard split", () => {
|
||||
const text = "😀😀";
|
||||
const chunks = chunkByNewline(text, 1.5);
|
||||
|
||||
expect(chunks).toEqual(["😀", "😀"]);
|
||||
expect(chunks).not.toContain("");
|
||||
expect(chunks.join("")).toBe(text);
|
||||
});
|
||||
});
|
||||
|
||||
describe("chunkTextWithMode", () => {
|
||||
@@ -665,6 +674,13 @@ describe("chunkMarkdownTextWithMode", () => {
|
||||
it("keeps an astral character whole when a positive hard limit starts on its pair", () => {
|
||||
expect(chunkMarkdownTextWithMode("A😀B", 1, "length")).toEqual(["A", "😀", "B"]);
|
||||
});
|
||||
|
||||
it.each(["length", "newline"] as const)(
|
||||
"keeps astral text with a fractional limit in %s mode",
|
||||
(mode) => {
|
||||
expect(chunkMarkdownTextWithMode("😀", 1.5, mode)).toEqual(["😀"]);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
describe("resolveChunkMode", () => {
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
// unintentionally breaking on newlines. Using [\s\S] keeps newlines inside
|
||||
// the chunk so messages are only split when they truly exceed the limit.
|
||||
|
||||
import { resolveIntegerOption } from "@openclaw/normalization-core/number-coercion";
|
||||
import {
|
||||
findFenceSpanAt,
|
||||
isSafeFenceBreak,
|
||||
@@ -32,6 +33,11 @@ export type ChunkMode = "length" | "newline";
|
||||
const DEFAULT_CHUNK_LIMIT = 4000;
|
||||
const DEFAULT_CHUNK_MODE: ChunkMode = "length";
|
||||
|
||||
function normalizeChunkLimit(limit: number): number {
|
||||
// String slicing truncates fractional indexes, so positive limits need an integer progress step.
|
||||
return Number.isFinite(limit) && limit > 0 ? resolveIntegerOption(limit, 1, { min: 1 }) : limit;
|
||||
}
|
||||
|
||||
type ProviderChunkConfig = {
|
||||
textChunkLimit?: number;
|
||||
streaming?: unknown;
|
||||
@@ -132,7 +138,8 @@ export function chunkByNewline(
|
||||
if (!text) {
|
||||
return [];
|
||||
}
|
||||
if (maxLineLength <= 0) {
|
||||
const lineLimit = normalizeChunkLimit(maxLineLength);
|
||||
if (lineLimit <= 0) {
|
||||
return text.trim() ? [text] : [];
|
||||
}
|
||||
const splitLongLines = opts?.splitLongLines !== false;
|
||||
@@ -148,26 +155,26 @@ export function chunkByNewline(
|
||||
continue;
|
||||
}
|
||||
|
||||
const maxPrefix = Math.max(0, maxLineLength - 1);
|
||||
const maxPrefix = Math.max(0, lineLimit - 1);
|
||||
const cappedBlankLines = pendingBlankLines > 0 ? Math.min(pendingBlankLines, maxPrefix) : 0;
|
||||
const prefix = cappedBlankLines > 0 ? "\n".repeat(cappedBlankLines) : "";
|
||||
pendingBlankLines = 0;
|
||||
|
||||
const lineValue = trimLines ? trimmed : line;
|
||||
if (!splitLongLines || lineValue.length + prefix.length <= maxLineLength) {
|
||||
if (!splitLongLines || lineValue.length + prefix.length <= lineLimit) {
|
||||
chunks.push(prefix + lineValue);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Back the head cut off to a code-point boundary so an over-long line never splits a surrogate
|
||||
// pair; the recursive chunkText below is already surrogate-safe, only this first cut was raw.
|
||||
const rawLimit = Math.max(1, maxLineLength - prefix.length);
|
||||
const rawLimit = Math.max(1, lineLimit - prefix.length);
|
||||
const firstLimit = avoidTrailingHighSurrogateBreak(lineValue, 0, rawLimit);
|
||||
const first = lineValue.slice(0, firstLimit);
|
||||
chunks.push(prefix + first);
|
||||
const remaining = lineValue.slice(firstLimit);
|
||||
if (remaining) {
|
||||
chunks.push(...chunkText(remaining, maxLineLength));
|
||||
chunks.push(...chunkText(remaining, lineLimit));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -293,21 +300,24 @@ export function chunkTextWithMode(text: string, limit: number, mode: ChunkMode):
|
||||
}
|
||||
|
||||
export function chunkMarkdownTextWithMode(text: string, limit: number, mode: ChunkMode): string[] {
|
||||
const normalizedLimit = normalizeChunkLimit(limit);
|
||||
if (mode === "newline") {
|
||||
// Paragraph chunking is fence-safe because we never split at arbitrary indices.
|
||||
// If a paragraph must be split by length, defer to the markdown-aware chunker.
|
||||
const paragraphChunks = chunkByParagraph(text, limit, { splitLongParagraphs: false });
|
||||
const paragraphChunks = chunkByParagraph(text, normalizedLimit, {
|
||||
splitLongParagraphs: false,
|
||||
});
|
||||
const out: string[] = [];
|
||||
for (const chunk of paragraphChunks.flatMap((paragraphChunk) =>
|
||||
paragraphChunk.length > limit
|
||||
paragraphChunk.length > normalizedLimit
|
||||
? splitPackedFenceParagraphChunk(paragraphChunk)
|
||||
: paragraphChunk,
|
||||
)) {
|
||||
out.push(...chunkMarkdownText(chunk, limit));
|
||||
out.push(...chunkMarkdownText(chunk, normalizedLimit));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
return chunkMarkdownText(text, limit);
|
||||
return chunkMarkdownText(text, normalizedLimit);
|
||||
}
|
||||
|
||||
function splitByNewline(
|
||||
@@ -381,7 +391,8 @@ export function chunkText(text: string, limit: number): string[] {
|
||||
}
|
||||
|
||||
export function chunkMarkdownText(text: string, limit: number): string[] {
|
||||
const early = resolveChunkEarlyReturn(text, limit);
|
||||
const normalizedLimit = normalizeChunkLimit(limit);
|
||||
const early = resolveChunkEarlyReturn(text, normalizedLimit);
|
||||
if (early) {
|
||||
return early;
|
||||
}
|
||||
@@ -393,7 +404,7 @@ export function chunkMarkdownText(text: string, limit: number): string[] {
|
||||
|
||||
while (start < text.length) {
|
||||
const reopenPrefix = reopenFence ? `${reopenFence.openLine}\n` : "";
|
||||
const contentLimit = Math.max(1, limit - reopenPrefix.length);
|
||||
const contentLimit = Math.max(1, normalizedLimit - reopenPrefix.length);
|
||||
if (text.length - start <= contentLimit) {
|
||||
const finalChunk = `${reopenPrefix}${text.slice(start)}`;
|
||||
if (finalChunk.length > 0) {
|
||||
|
||||
@@ -22,6 +22,13 @@ describe("chunkTextForOutbound", () => {
|
||||
expect(chunkTextForOutbound("abc", 0, { preserveWhitespace: false })).toEqual(["abc"]);
|
||||
});
|
||||
|
||||
it("normalizes positive fractional limits across outbound modes", () => {
|
||||
expect(chunkTextForOutbound("abc", 0.5)).toEqual(["a", "b", "c"]);
|
||||
expect(chunkTextForOutbound("abc", 0.5, { preserveWhitespace: true })).toEqual(["a", "b", "c"]);
|
||||
expect(chunkTextForOutbound("😀😀", 0.5)).toEqual(["😀", "😀"]);
|
||||
expect(chunkTextForOutbound("😀😀", 0.5, { preserveWhitespace: true })).toEqual(["😀", "😀"]);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{
|
||||
name: "returns empty for empty input",
|
||||
@@ -78,6 +85,24 @@ describe("chunkTextRanges", () => {
|
||||
expect(chunkTextRanges("abc", { limit: 0 })).toEqual([{ start: 0, end: 3 }]);
|
||||
});
|
||||
|
||||
it("normalizes positive fractional range limits", () => {
|
||||
expect(chunkTextRanges("abc", { limit: 0.5 })).toEqual([
|
||||
{ start: 0, end: 1 },
|
||||
{ start: 1, end: 2 },
|
||||
{ start: 2, end: 3 },
|
||||
]);
|
||||
});
|
||||
|
||||
it.each(["hard", "preferred"] as const)(
|
||||
"keeps astral characters whole with fractional limits in %s mode",
|
||||
(mode) => {
|
||||
expect(chunkTextRanges("😀😀", { limit: 0.5, mode })).toEqual([
|
||||
{ start: 0, end: 2 },
|
||||
{ start: 2, end: 4 },
|
||||
]);
|
||||
},
|
||||
);
|
||||
|
||||
it.each(["hard", "preferred"] as const)("keeps surrogate pairs intact in %s mode", (mode) => {
|
||||
expect(chunkTextRanges("a😀b", { limit: 2, mode })).toEqual([
|
||||
{ start: 0, end: 1 },
|
||||
|
||||
@@ -157,7 +157,7 @@ const RUNTIME_API_EXPORT_GUARDS: Record<string, readonly string[]> = {
|
||||
'export type { PluginRuntime, RuntimeLogger } from "openclaw/plugin-sdk/plugin-runtime";',
|
||||
'export type { RuntimeEnv } from "openclaw/plugin-sdk/runtime-env";',
|
||||
'export type { WizardPrompter } from "openclaw/plugin-sdk/setup";',
|
||||
'export function chunkTextForOutbound(text: string, limit: number): string[] { const chunks: string[] = []; let remaining = text; while (remaining.length > limit) { const window = remaining.slice(0, limit); const splitAt = Math.max(window.lastIndexOf("\\n"), window.lastIndexOf(" ")); const breakAt = splitAt > 0 ? splitAt : limit; chunks.push(remaining.slice(0, breakAt).trimEnd()); remaining = remaining.slice(breakAt).trimStart(); } if (remaining.length > 0 || text.length === 0) { chunks.push(remaining); } return chunks; }',
|
||||
'export function chunkTextForOutbound(text: string, limit: number): string[] { if (text.length === 0) { return [""]; } if (Number.isFinite(limit) && limit > 0 && !Number.isInteger(limit)) { return chunkTextForOutboundSdk(text, limit); } const chunks: string[] = []; let remaining = text; while (remaining.length > limit) { const window = remaining.slice(0, limit); const splitAt = Math.max(window.lastIndexOf("\\n"), window.lastIndexOf(" ")); const breakAt = splitAt > 0 ? splitAt : limit; chunks.push(remaining.slice(0, breakAt).trimEnd()); remaining = remaining.slice(breakAt).trimStart(); } if (remaining.length > 0) { chunks.push(remaining); } return chunks; }',
|
||||
],
|
||||
[bundledPluginFile({
|
||||
rootDir: ROOT_DIR,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
// Text chunking tests cover splitting text into bounded model-safe chunks.
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { chunkTextByBreakResolver } from "./text-chunking.js";
|
||||
import { chunkTextByBreakResolver, splitLongTextLine } from "./text-chunking.js";
|
||||
|
||||
describe("shared/text-chunking", () => {
|
||||
it("returns empty for blank input and the full text when under limit", () => {
|
||||
@@ -26,6 +26,19 @@ describe("shared/text-chunking", () => {
|
||||
]);
|
||||
expect(chunkTextByBreakResolver("abcdefghij", 4, () => 99)).toEqual(["abcd", "efgh", "ij"]);
|
||||
expect(chunkTextByBreakResolver("abcdefghij", 4, () => 0)).toEqual(["abcd", "efgh", "ij"]);
|
||||
expect(chunkTextByBreakResolver("abcdefghij", 4, () => 0.5)).toEqual(["abcd", "efgh", "ij"]);
|
||||
});
|
||||
|
||||
it("normalizes positive fractional limits before splitting", () => {
|
||||
expect(chunkTextByBreakResolver("abc", 0.5, (window) => window.lastIndexOf(" "))).toEqual([
|
||||
"a",
|
||||
"b",
|
||||
"c",
|
||||
]);
|
||||
expect(splitLongTextLine("abc", 0.5, { preserveWhitespace: true })).toEqual(["a", "b", "c"]);
|
||||
expect(chunkTextByBreakResolver("😀😀", 0.5, () => -1)).toEqual(["😀", "😀"]);
|
||||
expect(splitLongTextLine("😀😀", 0.5, { preserveWhitespace: true })).toEqual(["😀", "😀"]);
|
||||
expect(splitLongTextLine("😀😀", 0.5, { preserveWhitespace: false })).toEqual(["😀", "😀"]);
|
||||
});
|
||||
|
||||
it("skips empty chunks created by whitespace-only segments", () => {
|
||||
|
||||
@@ -1,9 +1,15 @@
|
||||
import { resolveIntegerOption } from "@openclaw/normalization-core/number-coercion";
|
||||
import { avoidTrailingHighSurrogateBreak } from "@openclaw/normalization-core/utf16-slice";
|
||||
|
||||
export { avoidTrailingHighSurrogateBreak };
|
||||
|
||||
const CJK_PUNCTUATION_BREAK_AFTER_RE = /[、。,.!?;:)]}〉》」』】〕〗〙]/u;
|
||||
|
||||
function normalizeChunkLimit(limit: number): number {
|
||||
// String slicing truncates fractional indexes, so positive limits need an integer progress step.
|
||||
return Number.isFinite(limit) && limit > 0 ? resolveIntegerOption(limit, 1, { min: 1 }) : limit;
|
||||
}
|
||||
|
||||
function clampToCodePointBoundary(text: string, index: number): number {
|
||||
const boundary = Math.min(Math.max(0, index), text.length);
|
||||
return avoidTrailingHighSurrogateBreak(text, 0, boundary);
|
||||
@@ -35,21 +41,22 @@ export function splitLongTextLine(
|
||||
limit: number,
|
||||
options: { preserveWhitespace: boolean },
|
||||
): string[] {
|
||||
if (limit <= 0 || line.length <= limit) {
|
||||
const normalizedLimit = normalizeChunkLimit(limit);
|
||||
if (normalizedLimit <= 0 || line.length <= normalizedLimit) {
|
||||
return [line];
|
||||
}
|
||||
const chunks: string[] = [];
|
||||
let remaining = line;
|
||||
while (remaining.length > limit) {
|
||||
let breakIndex = clampToCodePointBoundary(remaining, limit);
|
||||
while (remaining.length > normalizedLimit) {
|
||||
let breakIndex = clampToCodePointBoundary(remaining, normalizedLimit);
|
||||
if (!options.preserveWhitespace) {
|
||||
const window = remaining.slice(0, limit);
|
||||
const window = remaining.slice(0, normalizedLimit);
|
||||
breakIndex = findWhitespaceBreak(window);
|
||||
if (breakIndex <= 0) {
|
||||
breakIndex = findCjkPunctuationBreak(window);
|
||||
}
|
||||
if (breakIndex <= 0) {
|
||||
breakIndex = clampToCodePointBoundary(remaining, limit);
|
||||
breakIndex = clampToCodePointBoundary(remaining, normalizedLimit);
|
||||
}
|
||||
}
|
||||
chunks.push(remaining.slice(0, breakIndex));
|
||||
@@ -75,19 +82,20 @@ export function chunkTextByBreakResolver(
|
||||
if (!text) {
|
||||
return [];
|
||||
}
|
||||
if (limit <= 0 || text.length <= limit) {
|
||||
const normalizedLimit = normalizeChunkLimit(limit);
|
||||
if (normalizedLimit <= 0 || text.length <= normalizedLimit) {
|
||||
return [text];
|
||||
}
|
||||
const chunks: string[] = [];
|
||||
let remaining = text;
|
||||
while (remaining.length > limit) {
|
||||
const window = remaining.slice(0, limit);
|
||||
while (remaining.length > normalizedLimit) {
|
||||
const window = remaining.slice(0, normalizedLimit);
|
||||
const candidateBreak = resolveBreakIndex(window);
|
||||
// Invalid or zero-width soft breaks would stall the loop, so fall back to the hard limit.
|
||||
// Invalid, fractional, or zero-width soft breaks would stall the loop.
|
||||
const breakIdx =
|
||||
Number.isFinite(candidateBreak) && candidateBreak > 0 && candidateBreak <= limit
|
||||
Number.isInteger(candidateBreak) && candidateBreak > 0 && candidateBreak <= normalizedLimit
|
||||
? candidateBreak
|
||||
: limit;
|
||||
: normalizedLimit;
|
||||
const safeBreakIdx = avoidTrailingHighSurrogateBreak(remaining, 0, breakIdx);
|
||||
const rawChunk = remaining.slice(0, safeBreakIdx);
|
||||
const chunk = rawChunk.trimEnd();
|
||||
|
||||
Reference in New Issue
Block a user