refactor(core): share JSON utf8 byte counting helper

This commit is contained in:
Peter Steinberger
2026-03-02 05:19:30 +00:00
parent 4a1be98254
commit 264599cc1d
5 changed files with 26 additions and 24 deletions

View File

@@ -2,6 +2,7 @@ import { Type } from "@sinclair/typebox";
import { loadConfig } from "../../config/config.js";
import { callGateway } from "../../gateway/call.js";
import { capArrayByJsonBytes } from "../../gateway/session-utils.fs.js";
import { jsonUtf8Bytes } from "../../infra/json-utf8-bytes.js";
import { redactSensitiveText } from "../../logging/redact.js";
import { truncateUtf16Safe } from "../../utils.js";
import type { AnyAgentTool } from "./common.js";
@@ -140,14 +141,6 @@ function sanitizeHistoryMessage(message: unknown): {
return { message: entry, truncated, redacted };
}
function jsonUtf8Bytes(value: unknown): number {
try {
return Buffer.byteLength(JSON.stringify(value), "utf8");
} catch {
return Buffer.byteLength(String(value), "utf8");
}
}
function enforceSessionsHistoryHardCap(params: {
items: unknown[];
bytes: number;

View File

@@ -9,6 +9,7 @@ import { createReplyDispatcher } from "../../auto-reply/reply/reply-dispatcher.j
import type { MsgContext } from "../../auto-reply/templating.js";
import { createReplyPrefixOptions } from "../../channels/reply-prefix.js";
import { resolveSessionFilePath } from "../../config/sessions.js";
import { jsonUtf8Bytes } from "../../infra/json-utf8-bytes.js";
import { resolveSendPolicy } from "../../sessions/send-policy.js";
import {
stripInlineDirectiveTagsForDisplay,
@@ -198,14 +199,6 @@ function sanitizeChatHistoryMessages(messages: unknown[]): unknown[] {
return changed ? next : messages;
}
function jsonUtf8Bytes(value: unknown): number {
try {
return Buffer.byteLength(JSON.stringify(value), "utf8");
} catch {
return Buffer.byteLength(String(value), "utf8");
}
}
function buildOversizedHistoryPlaceholder(message?: unknown): Record<string, unknown> {
const role =
message &&

View File

@@ -10,6 +10,7 @@ import {
resolveSessionTranscriptPathInDir,
} from "../config/sessions.js";
import { resolveRequiredHomeDir } from "../infra/home-dir.js";
import { jsonUtf8Bytes } from "../infra/json-utf8-bytes.js";
import { hasInterSessionUserProvenance } from "../sessions/input-provenance.js";
import { stripInlineDirectiveTagsForDisplay } from "../utils/directive-tags.js";
import { extractToolCallNames, hasToolCall } from "../utils/transcript-tools.js";
@@ -265,14 +266,6 @@ export async function cleanupArchivedSessionTranscripts(opts: {
return { removed, scanned };
}
function jsonUtf8Bytes(value: unknown): number {
try {
return Buffer.byteLength(JSON.stringify(value), "utf8");
} catch {
return Buffer.byteLength(String(value), "utf8");
}
}
export function capArrayByJsonBytes<T>(
items: T[],
maxBytes: number,

View File

@@ -0,0 +1,16 @@
import { describe, expect, it } from "vitest";
import { jsonUtf8Bytes } from "./json-utf8-bytes.js";
describe("jsonUtf8Bytes", () => {
it("returns utf8 byte length for serializable values", () => {
expect(jsonUtf8Bytes({ a: "x", b: [1, 2, 3] })).toBe(
Buffer.byteLength(JSON.stringify({ a: "x", b: [1, 2, 3] }), "utf8"),
);
});
it("falls back to string conversion when JSON serialization throws", () => {
const circular: { self?: unknown } = {};
circular.self = circular;
expect(jsonUtf8Bytes(circular)).toBe(Buffer.byteLength("[object Object]", "utf8"));
});
});

View File

@@ -0,0 +1,7 @@
export function jsonUtf8Bytes(value: unknown): number {
try {
return Buffer.byteLength(JSON.stringify(value), "utf8");
} catch {
return Buffer.byteLength(String(value), "utf8");
}
}