From 4cbbb86d5e71aef0c953791ac0fd30caec08bf2c Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sun, 12 Jul 2026 23:46:16 -0700 Subject: [PATCH] refactor(zalouser): share offset-preserving text chunk ranges (#105842) * refactor(zalouser): share text chunk ranges * chore(plugin-sdk): refresh combined API baseline --- .../.generated/plugin-sdk-api-baseline.sha256 | 4 +- docs/plugins/sdk-migration.md | 2 +- docs/plugins/sdk-subpaths.md | 2 +- extensions/zalouser/src/send.test.ts | 36 ++++++++ extensions/zalouser/src/send.ts | 92 +------------------ packages/markdown-core/src/chunk-text.ts | 63 +++++++++++++ packages/markdown-core/src/index.ts | 1 + scripts/plugin-sdk-surface-report.mjs | 4 +- src/plugin-sdk/text-chunking.test.ts | 43 ++++++++- src/plugin-sdk/text-chunking.ts | 7 ++ 10 files changed, 160 insertions(+), 94 deletions(-) diff --git a/docs/.generated/plugin-sdk-api-baseline.sha256 b/docs/.generated/plugin-sdk-api-baseline.sha256 index 832a162b7f64..f344e06dd687 100644 --- a/docs/.generated/plugin-sdk-api-baseline.sha256 +++ b/docs/.generated/plugin-sdk-api-baseline.sha256 @@ -1,2 +1,2 @@ -bc84c5da613ba34f84e9265ed0580ae99d8793a6033519dee33e11863daaff62 plugin-sdk-api-baseline.json -a25e6c9ae393270029eb39de5e00a45f5a3ba13e526ff7532668494f3772159b plugin-sdk-api-baseline.jsonl +21f63180d1606d450af8d2509c2e12ae7bcb0e92b77429ee35b5d350d4f9a427 plugin-sdk-api-baseline.json +7ea611a65949cbd82c54a63f3fc1818d88bf6b6515fcb53ef59a911b2ca4a822 plugin-sdk-api-baseline.jsonl diff --git a/docs/plugins/sdk-migration.md b/docs/plugins/sdk-migration.md index 6bda8ff8f2d0..1eddc02694cc 100644 --- a/docs/plugins/sdk-migration.md +++ b/docs/plugins/sdk-migration.md @@ -500,7 +500,7 @@ SDK. | `plugin-sdk/media-generation-runtime` | Shared media-generation helpers | Shared failover helpers, candidate selection, and missing-model messaging for image/video/music generation | | `plugin-sdk/media-understanding` | Media-understanding helpers | Media understanding provider types plus provider-facing image/audio helper exports | | `plugin-sdk/text-runtime` | Deprecated broad text compatibility export | Use `string-coerce-runtime`, `text-chunking`, `text-utility-runtime`, and `logging-core` | - | `plugin-sdk/text-chunking` | Text chunking helpers | Outbound text chunking helper | + | `plugin-sdk/text-chunking` | Text chunking helpers | Outbound text and offset-preserving range chunking helpers | | `plugin-sdk/speech` | Speech helpers | Speech provider types plus provider-facing directive, registry, validation helpers, and OpenAI-compatible TTS builder | | `plugin-sdk/speech-core` | Shared speech core | Speech provider types, registry, directives, normalization | | `plugin-sdk/realtime-transcription` | Realtime transcription helpers | Provider types, registry helpers, and shared WebSocket session helper | diff --git a/docs/plugins/sdk-subpaths.md b/docs/plugins/sdk-subpaths.md index ba60b100f94d..14781a8bfb77 100644 --- a/docs/plugins/sdk-subpaths.md +++ b/docs/plugins/sdk-subpaths.md @@ -352,7 +352,7 @@ usage endpoint failed or returned no usable usage data. | `plugin-sdk/media-store` | Narrow media store helpers such as `saveMediaBuffer` and `saveMediaStream` | | `plugin-sdk/media-generation-runtime` | Shared media-generation failover helpers, candidate selection, and missing-model messaging | | `plugin-sdk/media-understanding` | Media understanding provider types plus provider-facing image/audio/structured-extraction helper exports | - | `plugin-sdk/text-chunking` | Outbound text and markdown chunking/render helpers, markdown table conversion, directive-tag stripping, and safe-text utilities | + | `plugin-sdk/text-chunking` | Outbound text and offset-preserving range chunking, markdown chunking/render helpers, markdown table conversion, directive-tag stripping, and safe-text utilities | | `plugin-sdk/speech` | Speech provider types plus provider-facing directive, registry, validation, OpenAI-compatible TTS builder, and speech helper exports | | `plugin-sdk/speech-core` | Shared speech provider types, registry, directive, normalization, and speech helper exports | | `plugin-sdk/realtime-transcription` | Realtime transcription provider types, registry helpers, and shared WebSocket session helper | diff --git a/extensions/zalouser/src/send.test.ts b/extensions/zalouser/src/send.test.ts index a7e4cfb5379a..511ce6349e61 100644 --- a/extensions/zalouser/src/send.test.ts +++ b/extensions/zalouser/src/send.test.ts @@ -296,6 +296,42 @@ describe("zalouser send helpers", () => { expectResultFields(result, { ok: true, messageId: "mid-2d-4" }); }); + it.each([ + { + name: "fenced code with hard limits", + text: "```ts\nconst alpha = 1;\nconst beta = 2;\n```", + textChunkLimit: 16, + textChunkMode: "length" as const, + expected: ["const alpha = 1;", "\nconst beta = 2;"], + }, + { + name: "tables with newline-preferred limits", + text: "| A | B |\n| --- | --- |\n| 1 | 2 |\n| 3 | 4 |", + textChunkLimit: 24, + textChunkMode: "newline" as const, + expected: ["| A | B |\n| --- | --- |\n", "| 1 | 2 |\n| 3 | 4 |"], + }, + { + name: "repeated whitespace around paragraph boundaries", + text: "alpha beta\n\ngamma delta", + textChunkLimit: 12, + textChunkMode: "newline" as const, + expected: ["alpha beta\n", "\ngamma delta"], + }, + ])("preserves exact rendered chunks for $name", async (fixture) => { + mockSendText.mockResolvedValue(sendResult("mid-parity", "thread-parity")); + + await sendMessageZalouser("thread-parity", fixture.text, { + textMode: "markdown", + textChunkLimit: fixture.textChunkLimit, + textChunkMode: fixture.textChunkMode, + }); + + const renderedChunks = mockSendText.mock.calls.map((call) => call[1]); + expect(renderedChunks).toEqual(fixture.expected); + expect(renderedChunks.join("")).toBe(parseZalouserTextStyles(fixture.text).text); + }); + it("respects an explicit text chunk limit when splitting formatted markdown", async () => { const text = `**${"a".repeat(1501)}**`; mockSendText diff --git a/extensions/zalouser/src/send.ts b/extensions/zalouser/src/send.ts index df49ba3eaa3f..a6facb0803e4 100644 --- a/extensions/zalouser/src/send.ts +++ b/extensions/zalouser/src/send.ts @@ -1,4 +1,5 @@ // Zalouser plugin module implements send behavior. +import { chunkTextRanges } from "openclaw/plugin-sdk/text-chunking"; import { createZalouserSendReceipt } from "./send-receipt.js"; import { parseZalouserTextStyles } from "./text-styles.js"; import type { ZaloEventMessage, ZaloSendOptions, ZaloSendResult } from "./types.js"; @@ -19,15 +20,12 @@ type ZalouserSendOptions = ZaloSendOptions & { type ZalouserSendResult = ZaloSendResult; const ZALO_TEXT_LIMIT = 2000; -const DEFAULT_TEXT_CHUNK_MODE = "length"; type StyledTextChunk = { text: string; styles?: ZaloSendOptions["textStyles"]; }; -type TextChunkMode = NonNullable; - export async function sendMessageZalouser( threadId: string, text: string, @@ -155,7 +153,10 @@ function splitStyledText( } const chunks: StyledTextChunk[] = []; - for (const range of splitTextRanges(text, limit, mode ?? DEFAULT_TEXT_CHUNK_MODE)) { + for (const range of chunkTextRanges(text, { + limit, + mode: mode === "newline" ? "preferred" : "hard", + })) { const { start, end } = range; chunks.push({ text: text.slice(start, end), @@ -201,86 +202,3 @@ function sliceTextStyles( return chunkStyles.length > 0 ? chunkStyles : undefined; } - -function splitTextRanges( - text: string, - limit: number, - mode: TextChunkMode, -): Array<{ start: number; end: number }> { - if (mode === "newline") { - return splitTextRangesByPreferredBreaks(text, limit); - } - - const ranges: Array<{ start: number; end: number }> = []; - for (let start = 0; start < text.length; start += limit) { - ranges.push({ - start, - end: Math.min(text.length, start + limit), - }); - } - return ranges; -} - -function splitTextRangesByPreferredBreaks( - text: string, - limit: number, -): Array<{ start: number; end: number }> { - const ranges: Array<{ start: number; end: number }> = []; - let start = 0; - - while (start < text.length) { - const maxEnd = Math.min(text.length, start + limit); - let end = maxEnd; - if (maxEnd < text.length) { - end = - findParagraphBreak(text, start, maxEnd) ?? - findLastBreak(text, "\n", start, maxEnd) ?? - findLastWhitespaceBreak(text, start, maxEnd) ?? - maxEnd; - } - - if (end <= start) { - end = maxEnd; - } - - ranges.push({ start, end }); - start = end; - } - - return ranges; -} - -function findParagraphBreak(text: string, start: number, end: number): number | undefined { - const slice = text.slice(start, end); - const matches = slice.matchAll(/\n[\t ]*\n+/g); - let lastMatch: RegExpMatchArray | undefined; - for (const match of matches) { - lastMatch = match; - } - if (!lastMatch || lastMatch.index === undefined) { - return undefined; - } - return start + lastMatch.index + lastMatch[0].length; -} - -function findLastBreak( - text: string, - marker: string, - start: number, - end: number, -): number | undefined { - const index = text.lastIndexOf(marker, end - 1); - if (index < start) { - return undefined; - } - return index + marker.length; -} - -function findLastWhitespaceBreak(text: string, start: number, end: number): number | undefined { - for (let index = end - 1; index > start; index -= 1) { - if (/\s/.test(text.charAt(index))) { - return index + 1; - } - } - return undefined; -} diff --git a/packages/markdown-core/src/chunk-text.ts b/packages/markdown-core/src/chunk-text.ts index 7887444ae592..e2accdf13de9 100644 --- a/packages/markdown-core/src/chunk-text.ts +++ b/packages/markdown-core/src/chunk-text.ts @@ -42,6 +42,69 @@ function scanParenAwareBreakpoints(text: string): { lastNewline: number; lastWhi return { lastNewline, lastWhitespace }; } +export type TextChunkRange = { + start: number; + end: number; +}; + +export type ChunkTextRangesOptions = { + limit: number; + mode?: "hard" | "preferred"; +}; + +function findPreferredRangeEnd(text: string, start: number, end: number): number | undefined { + const slice = text.slice(start, end); + let paragraphEnd: number | undefined; + for (const match of slice.matchAll(/\n[\t ]*\n+/g)) { + if (match.index !== undefined) { + paragraphEnd = start + match.index + match[0].length; + } + } + if (paragraphEnd !== undefined) { + return paragraphEnd; + } + + const newlineIndex = text.lastIndexOf("\n", end - 1); + if (newlineIndex >= start) { + return newlineIndex + 1; + } + + for (let index = end - 1; index > start; index -= 1) { + if (/\s/.test(text.charAt(index))) { + return index + 1; + } + } + return undefined; +} + +/** + * Splits text into contiguous UTF-16 ranges without dropping separator whitespace. + * Preferred mode selects paragraph, newline, then whitespace boundaries. + */ +export function chunkTextRanges(text: string, options: ChunkTextRangesOptions): TextChunkRange[] { + if (!text) { + return []; + } + if (options.limit <= 0 || text.length <= options.limit) { + return [{ start: 0, end: text.length }]; + } + + const ranges: TextChunkRange[] = []; + let start = 0; + while (start < text.length) { + const maxEnd = Math.min(text.length, start + options.limit); + const preferredEnd = + options.mode === "preferred" && maxEnd < text.length + ? findPreferredRangeEnd(text, start, maxEnd) + : undefined; + const candidateEnd = preferredEnd && preferredEnd > start ? preferredEnd : maxEnd; + const end = avoidTrailingHighSurrogateBreak(text, start, candidateEnd); + ranges.push({ start, end }); + start = end; + } + return ranges; +} + /** * Keeps UTF-16 chunk boundaries from separating a supplementary-plane character. * A one-unit positive limit still needs to emit an entire surrogate pair. diff --git a/packages/markdown-core/src/index.ts b/packages/markdown-core/src/index.ts index e5fbf9ac3533..7e25e7dc31b0 100644 --- a/packages/markdown-core/src/index.ts +++ b/packages/markdown-core/src/index.ts @@ -1,4 +1,5 @@ /** Public Markdown parsing, rendering, chunking, and table-conversion utilities. */ +export * from "./chunk-text.js"; export * from "./code-spans.js"; export * from "./fences.js"; export * from "./frontmatter.js"; diff --git a/scripts/plugin-sdk-surface-report.mjs b/scripts/plugin-sdk-surface-report.mjs index 2315ecb05186..23759606826d 100644 --- a/scripts/plugin-sdk-surface-report.mjs +++ b/scripts/plugin-sdk-surface-report.mjs @@ -203,12 +203,12 @@ export function readPluginSdkSurfaceBudgets(env = process.env) { ), publicExports: readPluginSdkSurfaceBudgetEnv( "OPENCLAW_PLUGIN_SDK_MAX_PUBLIC_EXPORTS", - 10641, + 10644, env, ), publicFunctionExports: readPluginSdkSurfaceBudgetEnv( "OPENCLAW_PLUGIN_SDK_MAX_PUBLIC_FUNCTION_EXPORTS", - 5357, + 5358, env, ), publicDeprecatedExports: readPluginSdkSurfaceBudgetEnv( diff --git a/src/plugin-sdk/text-chunking.test.ts b/src/plugin-sdk/text-chunking.test.ts index 1707777aced5..9b1c38035280 100644 --- a/src/plugin-sdk/text-chunking.test.ts +++ b/src/plugin-sdk/text-chunking.test.ts @@ -2,7 +2,7 @@ * Tests text and Markdown chunking helpers exported by the plugin SDK. */ import { describe, expect, it } from "vitest"; -import { chunkTextForOutbound } from "./text-chunking.js"; +import { chunkTextForOutbound, chunkTextRanges } from "./text-chunking.js"; describe("chunkTextForOutbound", () => { it.each([ @@ -28,3 +28,44 @@ describe("chunkTextForOutbound", () => { expect(chunkTextForOutbound(text, maxLen)).toEqual(expected); }); }); + +describe("chunkTextRanges", () => { + it("returns contiguous hard ranges without dropping whitespace", () => { + const text = "alpha beta\n\ngamma delta"; + const ranges = chunkTextRanges(text, { limit: 12, mode: "hard" }); + + expect(ranges).toEqual([ + { start: 0, end: 12 }, + { start: 12, end: 24 }, + ]); + expect(ranges.map(({ start, end }) => text.slice(start, end))).toEqual([ + "alpha beta\n", + "\ngamma delta", + ]); + }); + + it("prefers paragraph, newline, then whitespace boundaries", () => { + const text = "a\n\nb\nc xyz"; + const ranges = chunkTextRanges(text, { limit: 8, mode: "preferred" }); + + expect(ranges.map(({ start, end }) => text.slice(start, end))).toEqual(["a\n\n", "b\nc xyz"]); + }); + + it("falls back to hard ranges and handles empty or non-positive limits", () => { + expect(chunkTextRanges("abcdefgh", { limit: 3, mode: "preferred" })).toEqual([ + { start: 0, end: 3 }, + { start: 3, end: 6 }, + { start: 6, end: 8 }, + ]); + expect(chunkTextRanges("", { limit: 3 })).toEqual([]); + expect(chunkTextRanges("abc", { limit: 0 })).toEqual([{ start: 0, end: 3 }]); + }); + + it.each(["hard", "preferred"] as const)("keeps surrogate pairs intact in %s mode", (mode) => { + expect(chunkTextRanges("a😀b", { limit: 2, mode })).toEqual([ + { start: 0, end: 1 }, + { start: 1, end: 3 }, + { start: 3, end: 4 }, + ]); + }); +}); diff --git a/src/plugin-sdk/text-chunking.ts b/src/plugin-sdk/text-chunking.ts index 14d3ce42e45a..9888a2773754 100644 --- a/src/plugin-sdk/text-chunking.ts +++ b/src/plugin-sdk/text-chunking.ts @@ -1,6 +1,13 @@ // Text chunking helpers split long outbound text while preserving readable line boundaries. import { chunkTextByBreakResolver } from "../shared/text-chunking.js"; +/** Offset-preserving text ranges for transports with native style metadata. */ +export { + chunkTextRanges, + type ChunkTextRangesOptions, + type TextChunkRange, +} from "../../packages/markdown-core/src/chunk-text.js"; + /** * Splits outbound channel text into chunks no longer than the requested limit. * Newline boundaries win over spaces; text without usable separators falls back