mirror of
https://github.com/openclaw/openclaw.git
synced 2026-05-06 08:50:43 +00:00
perf: slim memory host imports
This commit is contained in:
@@ -3,7 +3,7 @@ import {
|
||||
normalizeBatchBaseUrl,
|
||||
type BatchHttpClientConfig,
|
||||
} from "./batch-utils.js";
|
||||
import { hashText } from "./internal.js";
|
||||
import { hashText } from "./hash.js";
|
||||
import { withRemoteHttpResponse } from "./remote-http.js";
|
||||
|
||||
export async function uploadBatchJsonlFile(params: {
|
||||
|
||||
@@ -2,7 +2,8 @@ import { estimateUtf8Bytes, splitTextToUtf8ByteLimit } from "./embedding-input-l
|
||||
import { hasNonTextEmbeddingParts } from "./embedding-inputs.js";
|
||||
import { resolveEmbeddingMaxInputTokens } from "./embedding-model-limits.js";
|
||||
import type { EmbeddingProvider } from "./embeddings.js";
|
||||
import { hashText, type MemoryChunk } from "./internal.js";
|
||||
import { hashText } from "./hash.js";
|
||||
import type { MemoryChunk } from "./internal.js";
|
||||
|
||||
export function enforceEmbeddingMaxInputTokens(
|
||||
provider: EmbeddingProvider,
|
||||
|
||||
5
packages/memory-host-sdk/src/host/hash.ts
Normal file
5
packages/memory-host-sdk/src/host/hash.ts
Normal file
@@ -0,0 +1,5 @@
|
||||
import crypto from "node:crypto";
|
||||
|
||||
export function hashText(value: string): string {
|
||||
return crypto.createHash("sha256").update(value).digest("hex");
|
||||
}
|
||||
@@ -20,6 +20,9 @@ import {
|
||||
type MemoryMultimodalSettings,
|
||||
} from "./multimodal.js";
|
||||
|
||||
export { hashText } from "./hash.js";
|
||||
import { hashText } from "./hash.js";
|
||||
|
||||
export type MemoryFileEntry = {
|
||||
path: string;
|
||||
absPath: string;
|
||||
@@ -204,10 +207,6 @@ export async function listMemoryFiles(
|
||||
return deduped;
|
||||
}
|
||||
|
||||
export function hashText(value: string): string {
|
||||
return crypto.createHash("sha256").update(value).digest("hex");
|
||||
}
|
||||
|
||||
export async function buildFileEntry(
|
||||
absPath: string,
|
||||
workspaceDir: string,
|
||||
|
||||
@@ -4,10 +4,7 @@ import { stripInboundMetadata } from "../../../../src/auto-reply/reply/strip-inb
|
||||
import { isUsageCountedSessionTranscriptFileName } from "../../../../src/config/sessions/artifacts.js";
|
||||
import { resolveSessionTranscriptsDirForAgent } from "../../../../src/config/sessions/paths.js";
|
||||
import { redactSensitiveText } from "../../../../src/logging/redact.js";
|
||||
import { createSubsystemLogger } from "../../../../src/logging/subsystem.js";
|
||||
import { hashText } from "./internal.js";
|
||||
|
||||
const log = createSubsystemLogger("memory");
|
||||
import { hashText } from "./hash.js";
|
||||
|
||||
export type SessionFileEntry = {
|
||||
path: string;
|
||||
@@ -62,6 +59,11 @@ export function sessionPathForFile(absPath: string): string {
|
||||
return path.join("sessions", path.basename(absPath)).replace(/\\/g, "/");
|
||||
}
|
||||
|
||||
async function logSessionFileReadFailure(absPath: string, err: unknown): Promise<void> {
|
||||
const { createSubsystemLogger } = await import("../../../../src/logging/subsystem.js");
|
||||
createSubsystemLogger("memory").debug(`Failed reading session file ${absPath}: ${String(err)}`);
|
||||
}
|
||||
|
||||
function normalizeSessionText(value: string): string {
|
||||
return value
|
||||
.replace(/\s*\n+\s*/g, " ")
|
||||
@@ -174,7 +176,7 @@ export async function buildSessionEntry(absPath: string): Promise<SessionFileEnt
|
||||
...(generatedByDreamingNarrative ? { generatedByDreamingNarrative: true } : {}),
|
||||
};
|
||||
} catch (err) {
|
||||
log.debug(`Failed reading session file ${absPath}: ${String(err)}`);
|
||||
void logSessionFileReadFailure(absPath, err);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@ import {
|
||||
normalizeBatchBaseUrl,
|
||||
type BatchHttpClientConfig,
|
||||
} from "./batch-utils.js";
|
||||
import { hashText } from "./internal.js";
|
||||
import { hashText } from "./hash.js";
|
||||
import { withRemoteHttpResponse } from "./remote-http.js";
|
||||
|
||||
export async function uploadBatchJsonlFile(params: {
|
||||
|
||||
@@ -2,7 +2,8 @@ import { estimateUtf8Bytes, splitTextToUtf8ByteLimit } from "./embedding-input-l
|
||||
import { hasNonTextEmbeddingParts } from "./embedding-inputs.js";
|
||||
import { resolveEmbeddingMaxInputTokens } from "./embedding-model-limits.js";
|
||||
import type { EmbeddingProvider } from "./embeddings.js";
|
||||
import { hashText, type MemoryChunk } from "./internal.js";
|
||||
import { hashText } from "./hash.js";
|
||||
import type { MemoryChunk } from "./internal.js";
|
||||
|
||||
export function enforceEmbeddingMaxInputTokens(
|
||||
provider: EmbeddingProvider,
|
||||
|
||||
5
src/memory-host-sdk/host/hash.ts
Normal file
5
src/memory-host-sdk/host/hash.ts
Normal file
@@ -0,0 +1,5 @@
|
||||
import crypto from "node:crypto";
|
||||
|
||||
export function hashText(value: string): string {
|
||||
return crypto.createHash("sha256").update(value).digest("hex");
|
||||
}
|
||||
@@ -20,6 +20,9 @@ import {
|
||||
type MemoryMultimodalSettings,
|
||||
} from "./multimodal.js";
|
||||
|
||||
export { hashText } from "./hash.js";
|
||||
import { hashText } from "./hash.js";
|
||||
|
||||
export type MemoryFileEntry = {
|
||||
path: string;
|
||||
absPath: string;
|
||||
@@ -204,10 +207,6 @@ export async function listMemoryFiles(
|
||||
return deduped;
|
||||
}
|
||||
|
||||
export function hashText(value: string): string {
|
||||
return crypto.createHash("sha256").update(value).digest("hex");
|
||||
}
|
||||
|
||||
export async function buildFileEntry(
|
||||
absPath: string,
|
||||
workspaceDir: string,
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import fsSync from "node:fs";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { stripInternalRuntimeContext } from "../../agents/internal-runtime-context.js";
|
||||
@@ -10,14 +11,11 @@ import {
|
||||
isUsageCountedSessionTranscriptFileName,
|
||||
} from "../../config/sessions/artifacts.js";
|
||||
import { resolveSessionTranscriptsDirForAgent } from "../../config/sessions/paths.js";
|
||||
import { loadSessionStore } from "../../config/sessions/store-load.js";
|
||||
import { isExecCompletionEvent } from "../../infra/heartbeat-events-filter.js";
|
||||
import { redactSensitiveText } from "../../logging/redact.js";
|
||||
import { createSubsystemLogger } from "../../logging/subsystem.js";
|
||||
import { isCronRunSessionKey } from "../../sessions/session-key-utils.js";
|
||||
import { hashText } from "./internal.js";
|
||||
import { hashText } from "./hash.js";
|
||||
|
||||
const log = createSubsystemLogger("memory");
|
||||
const DREAMING_NARRATIVE_RUN_PREFIX = "dreaming-narrative-";
|
||||
// Keep the historical one-line-per-message export shape for normal turns, but
|
||||
// wrap pathological long messages so downstream indexers never ingest a single
|
||||
@@ -55,6 +53,11 @@ export type SessionTranscriptClassification = {
|
||||
cronRunTranscriptPaths: ReadonlySet<string>;
|
||||
};
|
||||
|
||||
type SessionTranscriptStoreEntry = {
|
||||
sessionFile?: unknown;
|
||||
sessionId?: unknown;
|
||||
};
|
||||
|
||||
function isCheckpointTranscriptFileName(fileName: string): boolean {
|
||||
return fileName.endsWith(".jsonl") && fileName.includes(".checkpoint.");
|
||||
}
|
||||
@@ -169,7 +172,7 @@ export function loadSessionTranscriptClassificationForSessionsDir(
|
||||
sessionsDir: string,
|
||||
): SessionTranscriptClassification {
|
||||
const storePath = path.join(sessionsDir, "sessions.json");
|
||||
const store = loadSessionStore(storePath);
|
||||
const store = readSessionTranscriptClassificationStore(storePath);
|
||||
const dreamingTranscriptPaths = new Set<string>();
|
||||
const cronRunTranscriptPaths = new Set<string>();
|
||||
for (const [sessionKey, entry] of Object.entries(store)) {
|
||||
@@ -190,6 +193,20 @@ export function loadSessionTranscriptClassificationForSessionsDir(
|
||||
};
|
||||
}
|
||||
|
||||
function readSessionTranscriptClassificationStore(
|
||||
storePath: string,
|
||||
): Record<string, SessionTranscriptStoreEntry> {
|
||||
try {
|
||||
const parsed = JSON.parse(fsSync.readFileSync(storePath, "utf-8")) as unknown;
|
||||
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
|
||||
return {};
|
||||
}
|
||||
return parsed as Record<string, SessionTranscriptStoreEntry>;
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
export function loadDreamingNarrativeTranscriptPathSetForAgent(
|
||||
agentId: string,
|
||||
): ReadonlySet<string> {
|
||||
@@ -236,6 +253,11 @@ export function sessionPathForFile(absPath: string): string {
|
||||
return path.join("sessions", path.basename(absPath)).replace(/\\/g, "/");
|
||||
}
|
||||
|
||||
async function logSessionFileReadFailure(absPath: string, err: unknown): Promise<void> {
|
||||
const { createSubsystemLogger } = await import("../../logging/subsystem.js");
|
||||
createSubsystemLogger("memory").debug(`Failed reading session file ${absPath}: ${String(err)}`);
|
||||
}
|
||||
|
||||
function normalizeSessionText(value: string): string {
|
||||
return value
|
||||
.replace(/\s*\n+\s*/g, " ")
|
||||
@@ -528,7 +550,7 @@ export async function buildSessionEntry(
|
||||
...(generatedByCronRun ? { generatedByCronRun: true } : {}),
|
||||
};
|
||||
} catch (err) {
|
||||
log.debug(`Failed reading session file ${absPath}: ${String(err)}`);
|
||||
void logSessionFileReadFailure(absPath, err);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user