perf: slim memory host imports

This commit is contained in:
Peter Steinberger
2026-04-25 12:47:50 +01:00
parent 6eec38ad5a
commit 4a76a66872
10 changed files with 57 additions and 23 deletions

View File

@@ -3,7 +3,7 @@ import {
normalizeBatchBaseUrl,
type BatchHttpClientConfig,
} from "./batch-utils.js";
import { hashText } from "./internal.js";
import { hashText } from "./hash.js";
import { withRemoteHttpResponse } from "./remote-http.js";
export async function uploadBatchJsonlFile(params: {

View File

@@ -2,7 +2,8 @@ import { estimateUtf8Bytes, splitTextToUtf8ByteLimit } from "./embedding-input-l
import { hasNonTextEmbeddingParts } from "./embedding-inputs.js";
import { resolveEmbeddingMaxInputTokens } from "./embedding-model-limits.js";
import type { EmbeddingProvider } from "./embeddings.js";
import { hashText, type MemoryChunk } from "./internal.js";
import { hashText } from "./hash.js";
import type { MemoryChunk } from "./internal.js";
export function enforceEmbeddingMaxInputTokens(
provider: EmbeddingProvider,

View File

@@ -0,0 +1,5 @@
import crypto from "node:crypto";
export function hashText(value: string): string {
return crypto.createHash("sha256").update(value).digest("hex");
}

View File

@@ -20,6 +20,9 @@ import {
type MemoryMultimodalSettings,
} from "./multimodal.js";
export { hashText } from "./hash.js";
import { hashText } from "./hash.js";
export type MemoryFileEntry = {
path: string;
absPath: string;
@@ -204,10 +207,6 @@ export async function listMemoryFiles(
return deduped;
}
export function hashText(value: string): string {
return crypto.createHash("sha256").update(value).digest("hex");
}
export async function buildFileEntry(
absPath: string,
workspaceDir: string,

View File

@@ -4,10 +4,7 @@ import { stripInboundMetadata } from "../../../../src/auto-reply/reply/strip-inb
import { isUsageCountedSessionTranscriptFileName } from "../../../../src/config/sessions/artifacts.js";
import { resolveSessionTranscriptsDirForAgent } from "../../../../src/config/sessions/paths.js";
import { redactSensitiveText } from "../../../../src/logging/redact.js";
import { createSubsystemLogger } from "../../../../src/logging/subsystem.js";
import { hashText } from "./internal.js";
const log = createSubsystemLogger("memory");
import { hashText } from "./hash.js";
export type SessionFileEntry = {
path: string;
@@ -62,6 +59,11 @@ export function sessionPathForFile(absPath: string): string {
return path.join("sessions", path.basename(absPath)).replace(/\\/g, "/");
}
async function logSessionFileReadFailure(absPath: string, err: unknown): Promise<void> {
const { createSubsystemLogger } = await import("../../../../src/logging/subsystem.js");
createSubsystemLogger("memory").debug(`Failed reading session file ${absPath}: ${String(err)}`);
}
function normalizeSessionText(value: string): string {
return value
.replace(/\s*\n+\s*/g, " ")
@@ -174,7 +176,7 @@ export async function buildSessionEntry(absPath: string): Promise<SessionFileEnt
...(generatedByDreamingNarrative ? { generatedByDreamingNarrative: true } : {}),
};
} catch (err) {
log.debug(`Failed reading session file ${absPath}: ${String(err)}`);
void logSessionFileReadFailure(absPath, err);
return null;
}
}

View File

@@ -3,7 +3,7 @@ import {
normalizeBatchBaseUrl,
type BatchHttpClientConfig,
} from "./batch-utils.js";
import { hashText } from "./internal.js";
import { hashText } from "./hash.js";
import { withRemoteHttpResponse } from "./remote-http.js";
export async function uploadBatchJsonlFile(params: {

View File

@@ -2,7 +2,8 @@ import { estimateUtf8Bytes, splitTextToUtf8ByteLimit } from "./embedding-input-l
import { hasNonTextEmbeddingParts } from "./embedding-inputs.js";
import { resolveEmbeddingMaxInputTokens } from "./embedding-model-limits.js";
import type { EmbeddingProvider } from "./embeddings.js";
import { hashText, type MemoryChunk } from "./internal.js";
import { hashText } from "./hash.js";
import type { MemoryChunk } from "./internal.js";
export function enforceEmbeddingMaxInputTokens(
provider: EmbeddingProvider,

View File

@@ -0,0 +1,5 @@
import crypto from "node:crypto";
export function hashText(value: string): string {
return crypto.createHash("sha256").update(value).digest("hex");
}

View File

@@ -20,6 +20,9 @@ import {
type MemoryMultimodalSettings,
} from "./multimodal.js";
export { hashText } from "./hash.js";
import { hashText } from "./hash.js";
export type MemoryFileEntry = {
path: string;
absPath: string;
@@ -204,10 +207,6 @@ export async function listMemoryFiles(
return deduped;
}
export function hashText(value: string): string {
return crypto.createHash("sha256").update(value).digest("hex");
}
export async function buildFileEntry(
absPath: string,
workspaceDir: string,

View File

@@ -1,3 +1,4 @@
import fsSync from "node:fs";
import fs from "node:fs/promises";
import path from "node:path";
import { stripInternalRuntimeContext } from "../../agents/internal-runtime-context.js";
@@ -10,14 +11,11 @@ import {
isUsageCountedSessionTranscriptFileName,
} from "../../config/sessions/artifacts.js";
import { resolveSessionTranscriptsDirForAgent } from "../../config/sessions/paths.js";
import { loadSessionStore } from "../../config/sessions/store-load.js";
import { isExecCompletionEvent } from "../../infra/heartbeat-events-filter.js";
import { redactSensitiveText } from "../../logging/redact.js";
import { createSubsystemLogger } from "../../logging/subsystem.js";
import { isCronRunSessionKey } from "../../sessions/session-key-utils.js";
import { hashText } from "./internal.js";
import { hashText } from "./hash.js";
const log = createSubsystemLogger("memory");
const DREAMING_NARRATIVE_RUN_PREFIX = "dreaming-narrative-";
// Keep the historical one-line-per-message export shape for normal turns, but
// wrap pathological long messages so downstream indexers never ingest a single
@@ -55,6 +53,11 @@ export type SessionTranscriptClassification = {
cronRunTranscriptPaths: ReadonlySet<string>;
};
type SessionTranscriptStoreEntry = {
sessionFile?: unknown;
sessionId?: unknown;
};
function isCheckpointTranscriptFileName(fileName: string): boolean {
return fileName.endsWith(".jsonl") && fileName.includes(".checkpoint.");
}
@@ -169,7 +172,7 @@ export function loadSessionTranscriptClassificationForSessionsDir(
sessionsDir: string,
): SessionTranscriptClassification {
const storePath = path.join(sessionsDir, "sessions.json");
const store = loadSessionStore(storePath);
const store = readSessionTranscriptClassificationStore(storePath);
const dreamingTranscriptPaths = new Set<string>();
const cronRunTranscriptPaths = new Set<string>();
for (const [sessionKey, entry] of Object.entries(store)) {
@@ -190,6 +193,20 @@ export function loadSessionTranscriptClassificationForSessionsDir(
};
}
function readSessionTranscriptClassificationStore(
storePath: string,
): Record<string, SessionTranscriptStoreEntry> {
try {
const parsed = JSON.parse(fsSync.readFileSync(storePath, "utf-8")) as unknown;
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
return {};
}
return parsed as Record<string, SessionTranscriptStoreEntry>;
} catch {
return {};
}
}
export function loadDreamingNarrativeTranscriptPathSetForAgent(
agentId: string,
): ReadonlySet<string> {
@@ -236,6 +253,11 @@ export function sessionPathForFile(absPath: string): string {
return path.join("sessions", path.basename(absPath)).replace(/\\/g, "/");
}
async function logSessionFileReadFailure(absPath: string, err: unknown): Promise<void> {
const { createSubsystemLogger } = await import("../../logging/subsystem.js");
createSubsystemLogger("memory").debug(`Failed reading session file ${absPath}: ${String(err)}`);
}
function normalizeSessionText(value: string): string {
return value
.replace(/\s*\n+\s*/g, " ")
@@ -528,7 +550,7 @@ export async function buildSessionEntry(
...(generatedByCronRun ? { generatedByCronRun: true } : {}),
};
} catch (err) {
log.debug(`Failed reading session file ${absPath}: ${String(err)}`);
void logSessionFileReadFailure(absPath, err);
return null;
}
}