fix: prevent fractional chunk limits from stalling text splitting (#117727)

* fix: normalize fractional text chunk limits

* fix: normalize markdown chunk limits

* fix: normalize direct newline chunk limits

* fix(matrix): reuse progress-safe text chunker

* test(matrix): align runtime API guard

* test(matrix): keep outbound shard topology stable

* fix(matrix): preserve facade chunk compatibility

* test(matrix): keep runtime export guard stable

* fix(matrix): normalize render-aware chunk limits

* test(matrix): type real-send assertions

* fix(matrix): preserve one-unit event limits

---------

Co-authored-by: Tak Hoffman <781889+Takhoffman@users.noreply.github.com>
Co-authored-by: Vincent Koc <vincentkoc@ieee.org>
This commit is contained in:
Simone
2026-08-02 11:53:31 +02:00
committed by GitHub
parent a75b134231
commit 3f9b4519a9
12 changed files with 221 additions and 39 deletions

View File

@@ -1,4 +1,6 @@
// Matrix API module exposes the plugin public contract.
import { chunkTextForOutbound as chunkTextForOutboundSdk } from "openclaw/plugin-sdk/text-chunking";
export {
type MatrixResolvedStringField,
type MatrixResolvedStringValues,
@@ -53,7 +55,15 @@ export type { PluginRuntime, RuntimeLogger } from "openclaw/plugin-sdk/plugin-ru
export type { RuntimeEnv } from "openclaw/plugin-sdk/runtime-env";
export type { WizardPrompter } from "openclaw/plugin-sdk/setup";
// This facade shipped distinct empty and whitespace behavior. Preserve that
// contract while delegating fractional limits to the progress-safe SDK owner.
export function chunkTextForOutbound(text: string, limit: number): string[] {
if (text.length === 0) {
return [""];
}
if (Number.isFinite(limit) && limit > 0 && !Number.isInteger(limit)) {
return chunkTextForOutboundSdk(text, limit);
}
const chunks: string[] = [];
let remaining = text;
while (remaining.length > limit) {
@@ -63,7 +73,7 @@ export function chunkTextForOutbound(text: string, limit: number): string[] {
chunks.push(remaining.slice(0, breakAt).trimEnd());
remaining = remaining.slice(breakAt).trimStart();
}
if (remaining.length > 0 || text.length === 0) {
if (remaining.length > 0) {
chunks.push(remaining);
}
return chunks;

View File

@@ -441,6 +441,52 @@ describe("sendMessageMatrix durable delivery", () => {
fs.rmSync(stateDir, { recursive: true, force: true });
});
it("dispatches fractional BMP and astral limits through the real send path", async () => {
chunkMarkdownTextWithModeMock.mockImplementation((text) => Array.from(text));
resolveTextChunkLimitMock.mockReturnValue(0.5);
const bmp = makeClient();
await sendMessageMatrix("room:!room:example", "ABCD", {
client: bmp.client,
cfg: {} as never,
});
expect(bmp.sendMessage).toHaveBeenCalledTimes(4);
expect(
bmp.sendMessage.mock.calls.map((call) => requireRecord(call[1], "BMP content").body),
).toEqual(["A", "B", "C", "D"]);
resolveTextChunkLimitMock.mockReturnValue(1.5);
const astral = makeClient();
await sendMessageMatrix("room:!room:example", "😀😀", {
client: astral.client,
cfg: {} as never,
});
expect(astral.sendMessage).toHaveBeenCalledTimes(2);
expect(
astral.sendMessage.mock.calls.map((call) => requireRecord(call[1], "astral content").body),
).toEqual(["😀", "😀"]);
resolveTextChunkLimitMock.mockReturnValue(1.5);
const mixed = makeClient();
await sendMessageMatrix("room:!room:example", "😀AB", {
client: mixed.client,
cfg: {} as never,
});
expect(
mixed.sendMessage.mock.calls.map((call) => requireRecord(call[1], "mixed content").body),
).toEqual(["😀", "A", "B"]);
resolveTextChunkLimitMock.mockReturnValue(1);
const integer = makeClient();
await sendMessageMatrix("room:!room:example", "😀AB", {
client: integer.client,
cfg: {} as never,
});
expect(
integer.sendMessage.mock.calls.map((call) => requireRecord(call[1], "integer content").body),
).toEqual(["😀", "A", "B"]);
});
it("persists the complete event plan before the first provider dispatch", async () => {
const { client, sendMessage } = makeClient();
const deliveryIdentity = resolveMatrixDurableDeliveryIdentity({

View File

@@ -37,6 +37,23 @@ type MatrixPreparedChunkedText = MatrixPreparedSingleText & {
const getCore = () => getMatrixRuntime();
function normalizeMatrixEventLimit(limit: number): number {
if (!Number.isFinite(limit) || limit <= 0) {
return limit;
}
return Math.max(1, Math.floor(limit));
}
function resolveMatrixChunkOverflow(chunk: string, limit: number): number {
const body = markdownToMatrixBody(chunk);
const renderedLength = Math.max(chunk.length, body.length);
if (limit === 1 && Array.from(chunk).length === 1 && Array.from(body).length === 1) {
// One astral code point occupies two UTF-16 units but cannot be split into a valid event.
return 0;
}
return Math.max(0, renderedLength - limit);
}
function protectMatrixUnderlineTags(markdown: string): MatrixSpoilerProtection {
const codeRegions = findCodeRegions(markdown);
const metadataRanges = findMatrixMarkdownMetadataRanges(markdown);
@@ -170,11 +187,13 @@ export function prepareMatrixSingleText(
accountId: opts.accountId,
supportsBlockTables: MATRIX_FORMAT_PROFILE.constructs.table === "native",
});
const singleEventLimit = Math.min(
getCore().channel.text.resolveTextChunkLimit(cfg, "matrix", opts.accountId),
MATRIX_FORMAT_PROFILE.chunk.limit,
);
const convertedText = renderMatrixMarkdownTables(trimmedText, tableMode);
const singleEventLimit = normalizeMatrixEventLimit(
Math.min(
getCore().channel.text.resolveTextChunkLimit(cfg, "matrix", opts.accountId),
MATRIX_FORMAT_PROFILE.chunk.limit,
),
);
const eventTextLength = Math.max(
convertedText.length,
markdownToMatrixBody(convertedText).length,
@@ -241,10 +260,8 @@ export function chunkMatrixText(
});
const overflow = Math.max(
0,
...restored.map(
(chunk) =>
Math.max(chunk.length, markdownToMatrixBody(chunk).length) -
preparedText.singleEventLimit,
...restored.map((chunk) =>
resolveMatrixChunkOverflow(chunk, preparedText.singleEventLimit),
),
);
if (overflow === 0) {

View File

@@ -1,6 +1,6 @@
// Matrix tests cover outbound plugin behavior.
import { beforeEach, describe, expect, it, vi } from "vitest";
import type { OpenClawConfig } from "../runtime-api.js";
import { chunkTextForOutbound, type OpenClawConfig } from "../runtime-api.js";
const mocks = vi.hoisted(() => ({
sendMessageMatrix: vi.fn(),
@@ -63,6 +63,24 @@ describe("matrixOutbound cfg threading", () => {
expect(chunker("hello world", 5)).toEqual(["hello", "world"]);
});
it("makes progress for fractional BMP and astral limits", () => {
const chunker = matrixOutbound.chunker;
if (!chunker) {
throw new Error("matrixOutbound.chunker missing");
}
expect(chunker("ABCD", 0.5)).toEqual(["A", "B", "C", "D"]);
expect(chunker("😀😀", 1.5)).toEqual(["😀", "😀"]);
expect(chunkTextForOutbound("ABCD", 0.5)).toEqual(["A", "B", "C", "D"]);
expect(chunkTextForOutbound("😀😀", 1.5)).toEqual(["😀", "😀"]);
});
it("preserves Matrix compatibility behavior", () => {
expect(chunkTextForOutbound("", 5)).toEqual([""]);
expect(chunkTextForOutbound("", 0.5)).toEqual([""]);
expect(chunkTextForOutbound("abcdef ", 5)).toEqual(["abcde", "f "]);
});
it("passes resolved cfg to sendMessageMatrix for text sends", async () => {
const cfg = {
channels: {

View File

@@ -0,0 +1,10 @@
// Markdown Core tests cover plain-text chunking behavior.
import { describe, expect, it } from "vitest";
import { chunkText } from "./chunk-text.js";
describe("chunkText", () => {
it("normalizes positive fractional limits without emitting empty chunks", () => {
expect(chunkText("abc", 0.5)).toEqual(["a", "b", "c"]);
expect(chunkText("😀😀", 0.5)).toEqual(["😀", "😀"]);
});
});

View File

@@ -1,8 +1,14 @@
// Markdown Core module implements chunk text behavior.
import { resolveIntegerOption } from "@openclaw/normalization-core/number-coercion";
import { avoidTrailingHighSurrogateBreak } from "@openclaw/normalization-core/utf16-slice";
export { avoidTrailingHighSurrogateBreak };
function normalizeChunkLimit(limit: number): number {
// String slicing truncates fractional indexes, so positive limits need an integer progress step.
return Number.isFinite(limit) && limit > 0 ? resolveIntegerOption(limit, 1, { min: 1 }) : limit;
}
function resolveChunkEarlyReturn(text: string, limit: number): string[] | undefined {
if (!text) {
return [];
@@ -89,14 +95,15 @@ export function chunkTextRanges(text: string, options: ChunkTextRangesOptions):
if (!text) {
return [];
}
if (options.limit <= 0 || text.length <= options.limit) {
const normalizedLimit = normalizeChunkLimit(options.limit);
if (normalizedLimit <= 0 || text.length <= normalizedLimit) {
return [{ start: 0, end: text.length }];
}
const ranges: TextChunkRange[] = [];
let start = 0;
while (start < text.length) {
const maxEnd = Math.min(text.length, start + options.limit);
const maxEnd = Math.min(text.length, start + normalizedLimit);
const preferredEnd =
options.mode === "preferred" && maxEnd < text.length
? findPreferredRangeEnd(text, start, maxEnd)
@@ -115,7 +122,8 @@ export function chunkTextRanges(text: string, options: ChunkTextRangesOptions):
* Returns the original text as one chunk when the limit is non-positive.
*/
export function chunkText(text: string, limit: number): string[] {
const early = resolveChunkEarlyReturn(text, limit);
const normalizedLimit = normalizeChunkLimit(limit);
const early = resolveChunkEarlyReturn(text, normalizedLimit);
if (early) {
return early;
}
@@ -123,11 +131,11 @@ export function chunkText(text: string, limit: number): string[] {
const chunks: string[] = [];
let cursor = 0;
while (cursor < text.length) {
if (text.length - cursor <= limit) {
if (text.length - cursor <= normalizedLimit) {
chunks.push(text.slice(cursor));
break;
}
const windowEnd = Math.min(text.length, cursor + limit);
const windowEnd = Math.min(text.length, cursor + normalizedLimit);
const window = text.slice(cursor, windowEnd);
const { lastNewline, lastWhitespace } = scanParenAwareBreakpoints(window);
// Prefer block boundaries, then spaces, then a hard size cut when no

View File

@@ -570,6 +570,15 @@ describe("chunkByNewline", () => {
expect(chunks.every((chunk) => !/[\uD800-\uDBFF]$/u.test(chunk))).toBe(true);
expect(chunks.every((chunk) => !/^[\uDC00-\uDFFF]/u.test(chunk))).toBe(true);
});
it("normalizes fractional limits before an astral hard split", () => {
const text = "😀😀";
const chunks = chunkByNewline(text, 1.5);
expect(chunks).toEqual(["😀", "😀"]);
expect(chunks).not.toContain("");
expect(chunks.join("")).toBe(text);
});
});
describe("chunkTextWithMode", () => {
@@ -665,6 +674,13 @@ describe("chunkMarkdownTextWithMode", () => {
it("keeps an astral character whole when a positive hard limit starts on its pair", () => {
expect(chunkMarkdownTextWithMode("A😀B", 1, "length")).toEqual(["A", "😀", "B"]);
});
it.each(["length", "newline"] as const)(
"keeps astral text with a fractional limit in %s mode",
(mode) => {
expect(chunkMarkdownTextWithMode("😀", 1.5, mode)).toEqual(["😀"]);
},
);
});
describe("resolveChunkMode", () => {

View File

@@ -2,6 +2,7 @@
// unintentionally breaking on newlines. Using [\s\S] keeps newlines inside
// the chunk so messages are only split when they truly exceed the limit.
import { resolveIntegerOption } from "@openclaw/normalization-core/number-coercion";
import {
findFenceSpanAt,
isSafeFenceBreak,
@@ -32,6 +33,11 @@ export type ChunkMode = "length" | "newline";
const DEFAULT_CHUNK_LIMIT = 4000;
const DEFAULT_CHUNK_MODE: ChunkMode = "length";
function normalizeChunkLimit(limit: number): number {
// String slicing truncates fractional indexes, so positive limits need an integer progress step.
return Number.isFinite(limit) && limit > 0 ? resolveIntegerOption(limit, 1, { min: 1 }) : limit;
}
type ProviderChunkConfig = {
textChunkLimit?: number;
streaming?: unknown;
@@ -132,7 +138,8 @@ export function chunkByNewline(
if (!text) {
return [];
}
if (maxLineLength <= 0) {
const lineLimit = normalizeChunkLimit(maxLineLength);
if (lineLimit <= 0) {
return text.trim() ? [text] : [];
}
const splitLongLines = opts?.splitLongLines !== false;
@@ -148,26 +155,26 @@ export function chunkByNewline(
continue;
}
const maxPrefix = Math.max(0, maxLineLength - 1);
const maxPrefix = Math.max(0, lineLimit - 1);
const cappedBlankLines = pendingBlankLines > 0 ? Math.min(pendingBlankLines, maxPrefix) : 0;
const prefix = cappedBlankLines > 0 ? "\n".repeat(cappedBlankLines) : "";
pendingBlankLines = 0;
const lineValue = trimLines ? trimmed : line;
if (!splitLongLines || lineValue.length + prefix.length <= maxLineLength) {
if (!splitLongLines || lineValue.length + prefix.length <= lineLimit) {
chunks.push(prefix + lineValue);
continue;
}
// Back the head cut off to a code-point boundary so an over-long line never splits a surrogate
// pair; the recursive chunkText below is already surrogate-safe, only this first cut was raw.
const rawLimit = Math.max(1, maxLineLength - prefix.length);
const rawLimit = Math.max(1, lineLimit - prefix.length);
const firstLimit = avoidTrailingHighSurrogateBreak(lineValue, 0, rawLimit);
const first = lineValue.slice(0, firstLimit);
chunks.push(prefix + first);
const remaining = lineValue.slice(firstLimit);
if (remaining) {
chunks.push(...chunkText(remaining, maxLineLength));
chunks.push(...chunkText(remaining, lineLimit));
}
}
@@ -293,21 +300,24 @@ export function chunkTextWithMode(text: string, limit: number, mode: ChunkMode):
}
export function chunkMarkdownTextWithMode(text: string, limit: number, mode: ChunkMode): string[] {
const normalizedLimit = normalizeChunkLimit(limit);
if (mode === "newline") {
// Paragraph chunking is fence-safe because we never split at arbitrary indices.
// If a paragraph must be split by length, defer to the markdown-aware chunker.
const paragraphChunks = chunkByParagraph(text, limit, { splitLongParagraphs: false });
const paragraphChunks = chunkByParagraph(text, normalizedLimit, {
splitLongParagraphs: false,
});
const out: string[] = [];
for (const chunk of paragraphChunks.flatMap((paragraphChunk) =>
paragraphChunk.length > limit
paragraphChunk.length > normalizedLimit
? splitPackedFenceParagraphChunk(paragraphChunk)
: paragraphChunk,
)) {
out.push(...chunkMarkdownText(chunk, limit));
out.push(...chunkMarkdownText(chunk, normalizedLimit));
}
return out;
}
return chunkMarkdownText(text, limit);
return chunkMarkdownText(text, normalizedLimit);
}
function splitByNewline(
@@ -381,7 +391,8 @@ export function chunkText(text: string, limit: number): string[] {
}
export function chunkMarkdownText(text: string, limit: number): string[] {
const early = resolveChunkEarlyReturn(text, limit);
const normalizedLimit = normalizeChunkLimit(limit);
const early = resolveChunkEarlyReturn(text, normalizedLimit);
if (early) {
return early;
}
@@ -393,7 +404,7 @@ export function chunkMarkdownText(text: string, limit: number): string[] {
while (start < text.length) {
const reopenPrefix = reopenFence ? `${reopenFence.openLine}\n` : "";
const contentLimit = Math.max(1, limit - reopenPrefix.length);
const contentLimit = Math.max(1, normalizedLimit - reopenPrefix.length);
if (text.length - start <= contentLimit) {
const finalChunk = `${reopenPrefix}${text.slice(start)}`;
if (finalChunk.length > 0) {

View File

@@ -22,6 +22,13 @@ describe("chunkTextForOutbound", () => {
expect(chunkTextForOutbound("abc", 0, { preserveWhitespace: false })).toEqual(["abc"]);
});
it("normalizes positive fractional limits across outbound modes", () => {
expect(chunkTextForOutbound("abc", 0.5)).toEqual(["a", "b", "c"]);
expect(chunkTextForOutbound("abc", 0.5, { preserveWhitespace: true })).toEqual(["a", "b", "c"]);
expect(chunkTextForOutbound("😀😀", 0.5)).toEqual(["😀", "😀"]);
expect(chunkTextForOutbound("😀😀", 0.5, { preserveWhitespace: true })).toEqual(["😀", "😀"]);
});
it.each([
{
name: "returns empty for empty input",
@@ -78,6 +85,24 @@ describe("chunkTextRanges", () => {
expect(chunkTextRanges("abc", { limit: 0 })).toEqual([{ start: 0, end: 3 }]);
});
it("normalizes positive fractional range limits", () => {
expect(chunkTextRanges("abc", { limit: 0.5 })).toEqual([
{ start: 0, end: 1 },
{ start: 1, end: 2 },
{ start: 2, end: 3 },
]);
});
it.each(["hard", "preferred"] as const)(
"keeps astral characters whole with fractional limits in %s mode",
(mode) => {
expect(chunkTextRanges("😀😀", { limit: 0.5, mode })).toEqual([
{ start: 0, end: 2 },
{ start: 2, end: 4 },
]);
},
);
it.each(["hard", "preferred"] as const)("keeps surrogate pairs intact in %s mode", (mode) => {
expect(chunkTextRanges("a😀b", { limit: 2, mode })).toEqual([
{ start: 0, end: 1 },

View File

@@ -157,7 +157,7 @@ const RUNTIME_API_EXPORT_GUARDS: Record<string, readonly string[]> = {
'export type { PluginRuntime, RuntimeLogger } from "openclaw/plugin-sdk/plugin-runtime";',
'export type { RuntimeEnv } from "openclaw/plugin-sdk/runtime-env";',
'export type { WizardPrompter } from "openclaw/plugin-sdk/setup";',
'export function chunkTextForOutbound(text: string, limit: number): string[] { const chunks: string[] = []; let remaining = text; while (remaining.length > limit) { const window = remaining.slice(0, limit); const splitAt = Math.max(window.lastIndexOf("\\n"), window.lastIndexOf(" ")); const breakAt = splitAt > 0 ? splitAt : limit; chunks.push(remaining.slice(0, breakAt).trimEnd()); remaining = remaining.slice(breakAt).trimStart(); } if (remaining.length > 0 || text.length === 0) { chunks.push(remaining); } return chunks; }',
'export function chunkTextForOutbound(text: string, limit: number): string[] { if (text.length === 0) { return [""]; } if (Number.isFinite(limit) && limit > 0 && !Number.isInteger(limit)) { return chunkTextForOutboundSdk(text, limit); } const chunks: string[] = []; let remaining = text; while (remaining.length > limit) { const window = remaining.slice(0, limit); const splitAt = Math.max(window.lastIndexOf("\\n"), window.lastIndexOf(" ")); const breakAt = splitAt > 0 ? splitAt : limit; chunks.push(remaining.slice(0, breakAt).trimEnd()); remaining = remaining.slice(breakAt).trimStart(); } if (remaining.length > 0) { chunks.push(remaining); } return chunks; }',
],
[bundledPluginFile({
rootDir: ROOT_DIR,

View File

@@ -1,6 +1,6 @@
// Text chunking tests cover splitting text into bounded model-safe chunks.
import { describe, expect, it } from "vitest";
import { chunkTextByBreakResolver } from "./text-chunking.js";
import { chunkTextByBreakResolver, splitLongTextLine } from "./text-chunking.js";
describe("shared/text-chunking", () => {
it("returns empty for blank input and the full text when under limit", () => {
@@ -26,6 +26,19 @@ describe("shared/text-chunking", () => {
]);
expect(chunkTextByBreakResolver("abcdefghij", 4, () => 99)).toEqual(["abcd", "efgh", "ij"]);
expect(chunkTextByBreakResolver("abcdefghij", 4, () => 0)).toEqual(["abcd", "efgh", "ij"]);
expect(chunkTextByBreakResolver("abcdefghij", 4, () => 0.5)).toEqual(["abcd", "efgh", "ij"]);
});
it("normalizes positive fractional limits before splitting", () => {
expect(chunkTextByBreakResolver("abc", 0.5, (window) => window.lastIndexOf(" "))).toEqual([
"a",
"b",
"c",
]);
expect(splitLongTextLine("abc", 0.5, { preserveWhitespace: true })).toEqual(["a", "b", "c"]);
expect(chunkTextByBreakResolver("😀😀", 0.5, () => -1)).toEqual(["😀", "😀"]);
expect(splitLongTextLine("😀😀", 0.5, { preserveWhitespace: true })).toEqual(["😀", "😀"]);
expect(splitLongTextLine("😀😀", 0.5, { preserveWhitespace: false })).toEqual(["😀", "😀"]);
});
it("skips empty chunks created by whitespace-only segments", () => {

View File

@@ -1,9 +1,15 @@
import { resolveIntegerOption } from "@openclaw/normalization-core/number-coercion";
import { avoidTrailingHighSurrogateBreak } from "@openclaw/normalization-core/utf16-slice";
export { avoidTrailingHighSurrogateBreak };
const CJK_PUNCTUATION_BREAK_AFTER_RE = /[]/u;
function normalizeChunkLimit(limit: number): number {
// String slicing truncates fractional indexes, so positive limits need an integer progress step.
return Number.isFinite(limit) && limit > 0 ? resolveIntegerOption(limit, 1, { min: 1 }) : limit;
}
function clampToCodePointBoundary(text: string, index: number): number {
const boundary = Math.min(Math.max(0, index), text.length);
return avoidTrailingHighSurrogateBreak(text, 0, boundary);
@@ -35,21 +41,22 @@ export function splitLongTextLine(
limit: number,
options: { preserveWhitespace: boolean },
): string[] {
if (limit <= 0 || line.length <= limit) {
const normalizedLimit = normalizeChunkLimit(limit);
if (normalizedLimit <= 0 || line.length <= normalizedLimit) {
return [line];
}
const chunks: string[] = [];
let remaining = line;
while (remaining.length > limit) {
let breakIndex = clampToCodePointBoundary(remaining, limit);
while (remaining.length > normalizedLimit) {
let breakIndex = clampToCodePointBoundary(remaining, normalizedLimit);
if (!options.preserveWhitespace) {
const window = remaining.slice(0, limit);
const window = remaining.slice(0, normalizedLimit);
breakIndex = findWhitespaceBreak(window);
if (breakIndex <= 0) {
breakIndex = findCjkPunctuationBreak(window);
}
if (breakIndex <= 0) {
breakIndex = clampToCodePointBoundary(remaining, limit);
breakIndex = clampToCodePointBoundary(remaining, normalizedLimit);
}
}
chunks.push(remaining.slice(0, breakIndex));
@@ -75,19 +82,20 @@ export function chunkTextByBreakResolver(
if (!text) {
return [];
}
if (limit <= 0 || text.length <= limit) {
const normalizedLimit = normalizeChunkLimit(limit);
if (normalizedLimit <= 0 || text.length <= normalizedLimit) {
return [text];
}
const chunks: string[] = [];
let remaining = text;
while (remaining.length > limit) {
const window = remaining.slice(0, limit);
while (remaining.length > normalizedLimit) {
const window = remaining.slice(0, normalizedLimit);
const candidateBreak = resolveBreakIndex(window);
// Invalid or zero-width soft breaks would stall the loop, so fall back to the hard limit.
// Invalid, fractional, or zero-width soft breaks would stall the loop.
const breakIdx =
Number.isFinite(candidateBreak) && candidateBreak > 0 && candidateBreak <= limit
Number.isInteger(candidateBreak) && candidateBreak > 0 && candidateBreak <= normalizedLimit
? candidateBreak
: limit;
: normalizedLimit;
const safeBreakIdx = avoidTrailingHighSurrogateBreak(remaining, 0, breakIdx);
const rawChunk = remaining.slice(0, safeBreakIdx);
const chunk = rawChunk.trimEnd();