From f7cc6ebe1e0b8d76bde694ee78fdd57037101fdb Mon Sep 17 00:00:00 2001 From: wings1029 Date: Thu, 9 Jul 2026 17:08:11 +0800 Subject: [PATCH] fix(agent-core,memory-core): keep compaction summary and memory snippet truncation UTF-16 safe (#102542) * fix(agent-core,memory-core): keep compaction summary and memory snippet truncation UTF-16 safe * fix: make compaction and memory truncation UTF-16 safe --------- Co-authored-by: Peter Steinberger --- .../src/short-term-promotion.test.ts | 66 +++++++++++++++++++ .../memory-core/src/short-term-promotion.ts | 5 +- packages/agent-core/package.json | 1 + .../src/harness/compaction/utils.test.ts | 14 ++++ .../src/harness/compaction/utils.ts | 6 +- pnpm-lock.yaml | 3 + 6 files changed, 91 insertions(+), 4 deletions(-) diff --git a/extensions/memory-core/src/short-term-promotion.test.ts b/extensions/memory-core/src/short-term-promotion.test.ts index 4c377d8e3940..c481781c21ad 100644 --- a/extensions/memory-core/src/short-term-promotion.test.ts +++ b/extensions/memory-core/src/short-term-promotion.test.ts @@ -3588,4 +3588,70 @@ describe("short-term promotion", () => { }); }); }); + describe("UTF-16 snippet bounds", () => { + it("stores a complete-code-point short-term recall snippet", async () => { + await withTempWorkspace(async (workspaceDir) => { + const prefix = "y".repeat(testing.SHORT_TERM_RECALL_MAX_SNIPPET_CHARS - 1); + await recordShortTermRecalls({ + workspaceDir, + query: "utf16 recall", + results: [ + { + path: "memory/2026-04-03.md", + source: "memory", + startLine: 1, + endLine: 1, + score: 0.9, + snippet: `${prefix}🚀tail`, + }, + ], + }); + + const entries = Object.values(await readRecallStoreEntries(workspaceDir)); + expect(entries).toHaveLength(1); + expect(readEntrySnippet(entries[0])).toBe(prefix); + }); + }); + + it("writes a complete-code-point promoted MEMORY.md snippet", async () => { + await withTempWorkspace(async (workspaceDir) => { + const prefix = "a".repeat(7); + const snippet = `${prefix}🚀tail`; + await writeDailyMemoryNote(workspaceDir, "2026-04-03", [snippet]); + await recordShortTermRecalls({ + workspaceDir, + query: "utf16 promotion", + results: [ + { + path: "memory/2026-04-03.md", + source: "memory", + startLine: 1, + endLine: 1, + score: 0.9, + snippet, + }, + ], + }); + const ranked = await rankShortTermPromotionCandidates({ + workspaceDir, + minScore: 0, + minRecallCount: 0, + minUniqueQueries: 0, + }); + + await applyShortTermPromotions({ + workspaceDir, + candidates: ranked, + minScore: 0, + minRecallCount: 0, + minUniqueQueries: 0, + maxPromotedSnippetTokens: 2, + }); + + const memoryText = await fs.readFile(path.join(workspaceDir, "MEMORY.md"), "utf-8"); + expect(memoryText).toContain(`- ${prefix}... [`); + expect(memoryText).not.toContain("🚀"); + }); + }); + }); }); diff --git a/extensions/memory-core/src/short-term-promotion.ts b/extensions/memory-core/src/short-term-promotion.ts index a60fc5436479..2a1cf5b3a48d 100644 --- a/extensions/memory-core/src/short-term-promotion.ts +++ b/extensions/memory-core/src/short-term-promotion.ts @@ -16,6 +16,7 @@ import { normalizeStringEntries, uniqueStrings, } from "openclaw/plugin-sdk/string-coerce-runtime"; +import { truncateUtf16Safe } from "openclaw/plugin-sdk/text-utility-runtime"; import { deriveConceptTags, MAX_CONCEPT_TAGS, @@ -359,7 +360,7 @@ function truncateShortTermSnippet(snippet: string): string { if (snippet.length <= SHORT_TERM_RECALL_MAX_SNIPPET_CHARS) { return snippet; } - return snippet.slice(0, SHORT_TERM_RECALL_MAX_SNIPPET_CHARS).trimEnd(); + return truncateUtf16Safe(snippet, SHORT_TERM_RECALL_MAX_SNIPPET_CHARS).trimEnd(); } function enforceShortTermRecallSnippetCap(store: ShortTermRecallStore): void { @@ -2351,7 +2352,7 @@ function truncatePromotedSnippet(snippet: string, maxTokens: number): string { if (limit === 0 || snippet.length <= limit) { return snippet; } - const hardLimit = snippet.slice(0, limit); + const hardLimit = truncateUtf16Safe(snippet, limit); const sentenceBoundary = Math.max( hardLimit.lastIndexOf(". "), hardLimit.lastIndexOf("! "), diff --git a/packages/agent-core/package.json b/packages/agent-core/package.json index cf3a337ee25c..849f92332bd2 100644 --- a/packages/agent-core/package.json +++ b/packages/agent-core/package.json @@ -96,6 +96,7 @@ }, "dependencies": { "@openclaw/ai": "workspace:*", + "@openclaw/normalization-core": "workspace:*", "typebox": "1.3.3" } } diff --git a/packages/agent-core/src/harness/compaction/utils.test.ts b/packages/agent-core/src/harness/compaction/utils.test.ts index c1ef2536d8f9..18ed3bfbed3d 100644 --- a/packages/agent-core/src/harness/compaction/utils.test.ts +++ b/packages/agent-core/src/harness/compaction/utils.test.ts @@ -33,4 +33,18 @@ describe("serializeConversation", () => { expect(serializeConversation(messages)).toBe(`[Tool result]: ${expected}`); }); + + it("keeps truncated tool results UTF-16 safe and reports the exact omitted count", () => { + const prefix = "a".repeat(1_999); + const messages = [ + { + role: "toolResult", + content: [{ type: "toolResult", content: `${prefix}🚀tail` }], + }, + ] as unknown as Message[]; + + expect(serializeConversation(messages)).toBe( + `[Tool result]: ${prefix}\n\n[... 6 more characters truncated]`, + ); + }); }); diff --git a/packages/agent-core/src/harness/compaction/utils.ts b/packages/agent-core/src/harness/compaction/utils.ts index 5a69d2f103c9..450d1e4cf9fa 100644 --- a/packages/agent-core/src/harness/compaction/utils.ts +++ b/packages/agent-core/src/harness/compaction/utils.ts @@ -1,4 +1,5 @@ // Agent Core helper module supports utils behavior. +import { truncateUtf16Safe } from "@openclaw/normalization-core/utf16-slice"; import type { Message } from "../../../../llm-core/src/index.js"; import type { AgentMessage } from "../../types.js"; @@ -105,8 +106,9 @@ function truncateForSummary(text: string, maxChars: number): string { if (text.length <= maxChars) { return text; } - const truncatedChars = text.length - maxChars; - return `${text.slice(0, maxChars)}\n\n[... ${truncatedChars} more characters truncated]`; + const sliced = truncateUtf16Safe(text, maxChars); + const truncatedChars = text.length - sliced.length; + return `${sliced}\n\n[... ${truncatedChars} more characters truncated]`; } /** Extract text that compaction both estimates and includes in summary prompts. */ diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ff98fa7cdd77..ee137ad25319 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1864,6 +1864,9 @@ importers: '@openclaw/ai': specifier: workspace:* version: link:../ai + '@openclaw/normalization-core': + specifier: workspace:* + version: link:../normalization-core typebox: specifier: 1.3.3 version: 1.3.3