From dc3df62e67c7178efdfca063a815561356f02e5f Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Tue, 28 Apr 2026 05:48:59 +0100 Subject: [PATCH] refactor(memory-host): own package contract surface --- .../memory-host-sdk/src/engine-embeddings.ts | 74 +++- .../memory-host-sdk/src/engine-foundation.ts | 49 ++- packages/memory-host-sdk/src/engine-qmd.ts | 27 +- .../memory-host-sdk/src/engine-storage.ts | 49 ++- packages/memory-host-sdk/src/engine.ts | 8 +- .../src/host/backend-config.ts | 2 +- .../src/host/batch-error-utils.ts | 34 +- .../memory-host-sdk/src/host/batch-http.ts | 2 + .../src/host/embedding-defaults.ts | 2 + .../host/embedding-provider-adapter-utils.ts | 29 ++ .../src/host/embeddings-remote-client.ts | 8 +- .../src/host/embeddings-remote-fetch.ts | 2 + .../src/host/embeddings-remote-provider.ts | 4 +- .../memory-host-sdk/src/host/embeddings.ts | 86 +++- .../src/host/embeddings.types.ts | 56 +++ .../memory-host-sdk/src/host/multimodal.ts | 2 +- .../memory-host-sdk/src/host/node-llama.ts | 4 +- .../memory-host-sdk/src/host/post-json.ts | 2 + .../src/host/qmd-query-parser.ts | 2 +- .../memory-host-sdk/src/host/qmd-scope.ts | 4 +- .../src/host/query-expansion.ts | 2 +- .../src/host/read-file-shared.ts | 114 +++++ .../memory-host-sdk/src/host/read-file.ts | 6 +- .../memory-host-sdk/src/host/remote-http.ts | 25 +- .../memory-host-sdk/src/host/session-files.ts | 396 +++++++++++++++++- .../memory-host-sdk/src/host/sqlite-vec.ts | 2 +- packages/memory-host-sdk/src/host/sqlite.ts | 29 ++ .../memory-host-sdk/src/host/string-utils.ts | 19 + .../src/host/test-helpers/ssrf.ts | 2 +- packages/memory-host-sdk/src/host/types.ts | 24 +- packages/memory-host-sdk/src/multimodal.ts | 6 +- packages/memory-host-sdk/src/query.ts | 2 +- packages/memory-host-sdk/src/runtime-cli.ts | 12 +- packages/memory-host-sdk/src/runtime-core.ts | 42 +- packages/memory-host-sdk/src/runtime-files.ts | 11 +- packages/memory-host-sdk/src/runtime.ts | 7 +- packages/memory-host-sdk/src/secret.ts | 5 +- packages/memory-host-sdk/src/status.ts | 7 +- scripts/plugin-boundary-report.ts | 13 +- src/memory-host-sdk/engine-embeddings.ts | 74 +--- src/memory-host-sdk/engine-foundation.ts | 49 +-- src/memory-host-sdk/engine-qmd.ts | 27 +- src/memory-host-sdk/engine-storage.ts | 49 +-- src/memory-host-sdk/engine.ts | 8 +- src/memory-host-sdk/host/batch-error-utils.ts | 34 +- src/memory-host-sdk/host/embeddings.ts | 86 +--- src/memory-host-sdk/host/read-file-shared.ts | 115 +---- src/memory-host-sdk/multimodal.ts | 6 +- src/memory-host-sdk/query.ts | 2 +- src/memory-host-sdk/runtime-cli.ts | 12 +- src/memory-host-sdk/runtime-core.ts | 42 +- src/memory-host-sdk/runtime-files.ts | 11 +- src/memory-host-sdk/runtime.ts | 7 +- src/memory-host-sdk/secret.ts | 5 +- src/memory-host-sdk/status.ts | 7 +- ...tension-package-project-boundaries.test.ts | 9 +- 56 files changed, 1111 insertions(+), 602 deletions(-) create mode 100644 packages/memory-host-sdk/src/host/embedding-defaults.ts create mode 100644 packages/memory-host-sdk/src/host/embedding-provider-adapter-utils.ts create mode 100644 packages/memory-host-sdk/src/host/embeddings.types.ts create mode 100644 packages/memory-host-sdk/src/host/read-file-shared.ts create mode 100644 packages/memory-host-sdk/src/host/string-utils.ts diff --git a/packages/memory-host-sdk/src/engine-embeddings.ts b/packages/memory-host-sdk/src/engine-embeddings.ts index 9a2916d1926..0e003f3b725 100644 --- a/packages/memory-host-sdk/src/engine-embeddings.ts +++ b/packages/memory-host-sdk/src/engine-embeddings.ts @@ -1 +1,73 @@ -export * from "../../../src/memory-host-sdk/engine-embeddings.js"; +// Real workspace contract for memory embedding providers and batch helpers. + +export { + getMemoryEmbeddingProvider, + listRegisteredMemoryEmbeddingProviders, + listMemoryEmbeddingProviders, + listRegisteredMemoryEmbeddingProviderAdapters, +} from "../../../src/plugins/memory-embedding-provider-runtime.js"; +export type { + MemoryEmbeddingBatchChunk, + MemoryEmbeddingBatchOptions, + MemoryEmbeddingProvider, + MemoryEmbeddingProviderAdapter, + MemoryEmbeddingProviderCreateOptions, + MemoryEmbeddingProviderCreateResult, + MemoryEmbeddingProviderRuntime, +} from "../../../src/plugins/memory-embedding-providers.js"; +export { createLocalEmbeddingProvider, DEFAULT_LOCAL_MODEL } from "./host/embeddings.js"; +export { extractBatchErrorMessage, formatUnavailableBatchError } from "./host/batch-error-utils.js"; +export { postJsonWithRetry } from "./host/batch-http.js"; +export { applyEmbeddingBatchOutputLine } from "./host/batch-output.js"; +export { + EMBEDDING_BATCH_ENDPOINT, + type EmbeddingBatchStatus, + type ProviderBatchOutputLine, +} from "./host/batch-provider-common.js"; +export { + buildEmbeddingBatchGroupOptions, + runEmbeddingBatchGroups, + type EmbeddingBatchExecutionParams, +} from "./host/batch-runner.js"; +export { + resolveBatchCompletionFromStatus, + resolveCompletedBatchResult, + throwIfBatchTerminalFailure, + type BatchCompletionResult, +} from "./host/batch-status.js"; +export { uploadBatchJsonlFile } from "./host/batch-upload.js"; +export { + buildBatchHeaders, + normalizeBatchBaseUrl, + type BatchHttpClientConfig, +} from "./host/batch-utils.js"; +export { enforceEmbeddingMaxInputTokens } from "./host/embedding-chunk-limits.js"; +export { + isMissingEmbeddingApiKeyError, + mapBatchEmbeddingsByIndex, + sanitizeEmbeddingCacheHeaders, +} from "./host/embedding-provider-adapter-utils.js"; +export { sanitizeAndNormalizeEmbedding } from "./host/embedding-vectors.js"; +export { debugEmbeddingsLog } from "./host/embeddings-debug.js"; +export { normalizeEmbeddingModelWithPrefixes } from "./host/embeddings-model-normalize.js"; +export { + resolveRemoteEmbeddingBearerClient, + type RemoteEmbeddingProviderId, +} from "./host/embeddings-remote-client.js"; +export { + createRemoteEmbeddingProvider, + resolveRemoteEmbeddingClient, + type RemoteEmbeddingClient, +} from "./host/embeddings-remote-provider.js"; +export { fetchRemoteEmbeddingVectors } from "./host/embeddings-remote-fetch.js"; +export { + estimateStructuredEmbeddingInputBytes, + estimateUtf8Bytes, +} from "./host/embedding-input-limits.js"; +export { hasNonTextEmbeddingParts, type EmbeddingInput } from "./host/embedding-inputs.js"; +export { buildRemoteBaseUrlPolicy, withRemoteHttpResponse } from "./host/remote-http.js"; +export { + buildCaseInsensitiveExtensionGlob, + classifyMemoryMultimodalPath, + getMemoryMultimodalExtensions, +} from "./host/multimodal.js"; diff --git a/packages/memory-host-sdk/src/engine-foundation.ts b/packages/memory-host-sdk/src/engine-foundation.ts index c048f59890d..d05dd4c2625 100644 --- a/packages/memory-host-sdk/src/engine-foundation.ts +++ b/packages/memory-host-sdk/src/engine-foundation.ts @@ -1 +1,48 @@ -export * from "../../../src/memory-host-sdk/engine-foundation.js"; +// Real workspace contract for memory engine foundation concerns. + +export { + resolveAgentContextLimits, + resolveAgentDir, + resolveAgentWorkspaceDir, + resolveDefaultAgentId, + resolveSessionAgentId, +} from "../../../src/agents/agent-scope.js"; +export { + resolveMemorySearchConfig, + resolveMemorySearchSyncConfig, + type ResolvedMemorySearchConfig, + type ResolvedMemorySearchSyncConfig, +} from "../../../src/agents/memory-search.js"; +export { parseDurationMs } from "../../../src/cli/parse-duration.js"; +export { loadConfig } from "../../../src/config/config.js"; +export { resolveStateDir } from "../../../src/config/paths.js"; +export { resolveSessionTranscriptsDirForAgent } from "../../../src/config/sessions/paths.js"; +export { + hasConfiguredSecretInput, + normalizeResolvedSecretInputString, +} from "../../../src/config/types.secrets.js"; +export { writeFileWithinRoot } from "../../../src/infra/fs-safe.js"; +export { createSubsystemLogger } from "../../../src/logging/subsystem.js"; +export { detectMime } from "../../../src/media/mime.js"; +export { resolveGlobalSingleton } from "../../../src/shared/global-singleton.js"; +export { onSessionTranscriptUpdate } from "../../../src/sessions/transcript-events.js"; +export { splitShellArgs } from "../../../src/utils/shell-argv.js"; +export { runTasksWithConcurrency } from "../../../src/utils/run-with-concurrency.js"; +export { + shortenHomeInString, + shortenHomePath, + resolveUserPath, + truncateUtf16Safe, +} from "../../../src/utils.js"; +export type { OpenClawConfig } from "../../../src/config/config.js"; +export type { SessionSendPolicyConfig } from "../../../src/config/types.base.js"; +export type { SecretInput } from "../../../src/config/types.secrets.js"; +export type { + MemoryBackend, + MemoryCitationsMode, + MemoryQmdConfig, + MemoryQmdIndexPath, + MemoryQmdMcporterConfig, + MemoryQmdSearchMode, +} from "../../../src/config/types.memory.js"; +export type { MemorySearchConfig } from "../../../src/config/types.tools.js"; diff --git a/packages/memory-host-sdk/src/engine-qmd.ts b/packages/memory-host-sdk/src/engine-qmd.ts index 41f161481b2..3fe6db650d7 100644 --- a/packages/memory-host-sdk/src/engine-qmd.ts +++ b/packages/memory-host-sdk/src/engine-qmd.ts @@ -1 +1,26 @@ -export * from "../../../src/memory-host-sdk/engine-qmd.js"; +// Real workspace contract for QMD/session/query helpers used by the memory engine. + +export { extractKeywords, isQueryStopWordToken } from "./host/query-expansion.js"; +export { + buildSessionEntry, + listSessionFilesForAgent, + loadDreamingNarrativeTranscriptPathSetForAgent, + loadSessionTranscriptClassificationForAgent, + normalizeSessionTranscriptPathForComparison, + sessionPathForFile, + type BuildSessionEntryOptions, + type SessionFileEntry, + type SessionTranscriptClassification, +} from "./host/session-files.js"; +export { parseUsageCountedSessionIdFromFileName } from "../../../src/config/sessions/artifacts.js"; +export { parseQmdQueryJson, type QmdQueryResult } from "./host/qmd-query-parser.js"; +export { + deriveQmdScopeChannel, + deriveQmdScopeChatType, + isQmdScopeAllowed, +} from "./host/qmd-scope.js"; +export { + checkQmdBinaryAvailability, + resolveCliSpawnInvocation, + runCliCommand, +} from "./host/qmd-process.js"; diff --git a/packages/memory-host-sdk/src/engine-storage.ts b/packages/memory-host-sdk/src/engine-storage.ts index 7b6a5606c86..0159cff9605 100644 --- a/packages/memory-host-sdk/src/engine-storage.ts +++ b/packages/memory-host-sdk/src/engine-storage.ts @@ -1 +1,48 @@ -export * from "../../../src/memory-host-sdk/engine-storage.js"; +// Real workspace contract for memory engine storage/index helpers. + +export { + buildFileEntry, + buildMultimodalChunkForIndexing, + chunkMarkdown, + cosineSimilarity, + ensureDir, + hashText, + listMemoryFiles, + normalizeExtraMemoryPaths, + parseEmbedding, + remapChunkLines, + runWithConcurrency, + type MemoryChunk, + type MemoryFileEntry, +} from "./host/internal.js"; +export { readMemoryFile } from "./host/read-file.js"; +export { + buildMemoryReadResult, + buildMemoryReadResultFromSlice, + DEFAULT_MEMORY_READ_LINES, + DEFAULT_MEMORY_READ_MAX_CHARS, + type MemoryReadResult, +} from "./host/read-file-shared.js"; +export { resolveMemoryBackendConfig } from "./host/backend-config.js"; +export type { + ResolvedMemoryBackendConfig, + ResolvedQmdConfig, + ResolvedQmdMcporterConfig, +} from "./host/backend-config.js"; +export type { + MemoryEmbeddingProbeResult, + MemoryProviderStatus, + MemorySearchManager, + MemorySearchRuntimeDebug, + MemorySearchResult, + MemorySource, + MemorySyncProgressUpdate, +} from "./host/types.js"; +export { ensureMemoryIndexSchema } from "./host/memory-schema.js"; +export { loadSqliteVecExtension } from "./host/sqlite-vec.js"; +export { + closeMemorySqliteWalMaintenance, + configureMemorySqliteWalMaintenance, + requireNodeSqlite, +} from "./host/sqlite.js"; +export { isFileMissingError, statRegularFile } from "./host/fs-utils.js"; diff --git a/packages/memory-host-sdk/src/engine.ts b/packages/memory-host-sdk/src/engine.ts index 25269114848..a18fef9e8ba 100644 --- a/packages/memory-host-sdk/src/engine.ts +++ b/packages/memory-host-sdk/src/engine.ts @@ -1 +1,7 @@ -export * from "../../../src/memory-host-sdk/engine.js"; +// Aggregate workspace contract for the memory engine surface. +// Keep focused subpaths preferred for new code. + +export * from "./engine-foundation.js"; +export * from "./engine-storage.js"; +export * from "./engine-embeddings.js"; +export * from "./engine-qmd.js"; diff --git a/packages/memory-host-sdk/src/host/backend-config.ts b/packages/memory-host-sdk/src/host/backend-config.ts index 31f53e32f9a..905e45cefc8 100644 --- a/packages/memory-host-sdk/src/host/backend-config.ts +++ b/packages/memory-host-sdk/src/host/backend-config.ts @@ -14,9 +14,9 @@ import type { } from "../../../../src/config/types.memory.js"; import { CANONICAL_ROOT_MEMORY_FILENAME } from "../../../../src/memory/root-memory-files.js"; import { normalizeAgentId } from "../../../../src/routing/session-key.js"; -import { normalizeLowercaseStringOrEmpty } from "../../../../src/shared/string-coerce.js"; import { resolveUserPath } from "../../../../src/utils.js"; import { splitShellArgs } from "../../../../src/utils/shell-argv.js"; +import { normalizeLowercaseStringOrEmpty } from "./string-utils.js"; export type ResolvedMemoryBackendConfig = { backend: MemoryBackend; diff --git a/packages/memory-host-sdk/src/host/batch-error-utils.ts b/packages/memory-host-sdk/src/host/batch-error-utils.ts index d4094cb9ebe..02a1000ec9c 100644 --- a/packages/memory-host-sdk/src/host/batch-error-utils.ts +++ b/packages/memory-host-sdk/src/host/batch-error-utils.ts @@ -1 +1,33 @@ -export * from "../../../../src/memory-host-sdk/host/batch-error-utils.js"; +import { formatErrorMessage } from "../../../../src/infra/errors.js"; + +type BatchOutputErrorLike = { + error?: { message?: string }; + response?: { + body?: + | string + | { + error?: { message?: string }; + }; + }; +}; + +function getResponseErrorMessage(line: BatchOutputErrorLike | undefined): string | undefined { + const body = line?.response?.body; + if (typeof body === "string") { + return body || undefined; + } + if (!body || typeof body !== "object") { + return undefined; + } + return typeof body.error?.message === "string" ? body.error.message : undefined; +} + +export function extractBatchErrorMessage(lines: BatchOutputErrorLike[]): string | undefined { + const first = lines.find((line) => line.error?.message || getResponseErrorMessage(line)); + return first?.error?.message ?? getResponseErrorMessage(first); +} + +export function formatUnavailableBatchError(err: unknown): string | undefined { + const message = formatErrorMessage(err); + return message ? `error file unavailable: ${message}` : undefined; +} diff --git a/packages/memory-host-sdk/src/host/batch-http.ts b/packages/memory-host-sdk/src/host/batch-http.ts index 7e9e5fa0fbd..d890d039a97 100644 --- a/packages/memory-host-sdk/src/host/batch-http.ts +++ b/packages/memory-host-sdk/src/host/batch-http.ts @@ -6,6 +6,7 @@ export async function postJsonWithRetry(params: { url: string; headers: Record; ssrfPolicy?: SsrFPolicy; + fetchImpl?: typeof fetch; body: unknown; errorPrefix: string; }): Promise { @@ -15,6 +16,7 @@ export async function postJsonWithRetry(params: { url: params.url, headers: params.headers, ssrfPolicy: params.ssrfPolicy, + fetchImpl: params.fetchImpl, body: params.body, errorPrefix: params.errorPrefix, attachStatus: true, diff --git a/packages/memory-host-sdk/src/host/embedding-defaults.ts b/packages/memory-host-sdk/src/host/embedding-defaults.ts new file mode 100644 index 00000000000..fc503c9aca3 --- /dev/null +++ b/packages/memory-host-sdk/src/host/embedding-defaults.ts @@ -0,0 +1,2 @@ +export const DEFAULT_LOCAL_MODEL = + "hf:ggml-org/embeddinggemma-300m-qat-q8_0-GGUF/embeddinggemma-300m-qat-Q8_0.gguf"; diff --git a/packages/memory-host-sdk/src/host/embedding-provider-adapter-utils.ts b/packages/memory-host-sdk/src/host/embedding-provider-adapter-utils.ts new file mode 100644 index 00000000000..dc2e03e22b6 --- /dev/null +++ b/packages/memory-host-sdk/src/host/embedding-provider-adapter-utils.ts @@ -0,0 +1,29 @@ +import { normalizeLowercaseStringOrEmpty } from "./string-utils.js"; + +export function isMissingEmbeddingApiKeyError(err: unknown): boolean { + return err instanceof Error && err.message.includes("No API key found for provider"); +} + +export function sanitizeEmbeddingCacheHeaders( + headers: Record, + excludedHeaderNames: string[], +): Array<[string, string]> { + const excluded = new Set( + excludedHeaderNames.map((name) => normalizeLowercaseStringOrEmpty(name)), + ); + return Object.entries(headers) + .filter(([key]) => !excluded.has(normalizeLowercaseStringOrEmpty(key))) + .toSorted(([a], [b]) => a.localeCompare(b)) + .map(([key, value]) => [key, value]); +} + +export function mapBatchEmbeddingsByIndex( + byCustomId: Map, + count: number, +): number[][] { + const embeddings: number[][] = []; + for (let index = 0; index < count; index += 1) { + embeddings.push(byCustomId.get(String(index)) ?? []); + } + return embeddings; +} diff --git a/packages/memory-host-sdk/src/host/embeddings-remote-client.ts b/packages/memory-host-sdk/src/host/embeddings-remote-client.ts index f5fd0b79920..c32bcd31a4e 100644 --- a/packages/memory-host-sdk/src/host/embeddings-remote-client.ts +++ b/packages/memory-host-sdk/src/host/embeddings-remote-client.ts @@ -1,6 +1,7 @@ import { requireApiKey, resolveApiKeyForProvider } from "../../../../src/agents/model-auth.js"; import type { SsrFPolicy } from "../../../../src/infra/net/ssrf.js"; -import type { EmbeddingProviderOptions } from "./embeddings.js"; +import { normalizeOptionalString } from "../../../../src/shared/string-coerce.js"; +import type { EmbeddingProviderOptions } from "./embeddings.types.js"; import { buildRemoteBaseUrlPolicy } from "./remote-http.js"; import { resolveMemorySecretInputString } from "./secret-input.js"; @@ -16,7 +17,7 @@ export async function resolveRemoteEmbeddingBearerClient(params: { value: remote?.apiKey, path: "agents.*.memorySearch.remote.apiKey", }); - const remoteBaseUrl = remote?.baseUrl?.trim(); + const remoteBaseUrl = normalizeOptionalString(remote?.baseUrl); const providerConfig = params.options.config.models?.providers?.[params.provider]; const apiKey = remoteApiKey ? remoteApiKey @@ -28,7 +29,8 @@ export async function resolveRemoteEmbeddingBearerClient(params: { }), params.provider, ); - const baseUrl = remoteBaseUrl || providerConfig?.baseUrl?.trim() || params.defaultBaseUrl; + const baseUrl = + remoteBaseUrl || normalizeOptionalString(providerConfig?.baseUrl) || params.defaultBaseUrl; const headerOverrides = Object.assign({}, providerConfig?.headers, remote?.headers); const headers: Record = { "Content-Type": "application/json", diff --git a/packages/memory-host-sdk/src/host/embeddings-remote-fetch.ts b/packages/memory-host-sdk/src/host/embeddings-remote-fetch.ts index b17549548b4..cbe2081a69c 100644 --- a/packages/memory-host-sdk/src/host/embeddings-remote-fetch.ts +++ b/packages/memory-host-sdk/src/host/embeddings-remote-fetch.ts @@ -5,6 +5,7 @@ export async function fetchRemoteEmbeddingVectors(params: { url: string; headers: Record; ssrfPolicy?: SsrFPolicy; + fetchImpl?: typeof fetch; body: unknown; errorPrefix: string; }): Promise { @@ -12,6 +13,7 @@ export async function fetchRemoteEmbeddingVectors(params: { url: params.url, headers: params.headers, ssrfPolicy: params.ssrfPolicy, + fetchImpl: params.fetchImpl, body: params.body, errorPrefix: params.errorPrefix, parse: (payload) => { diff --git a/packages/memory-host-sdk/src/host/embeddings-remote-provider.ts b/packages/memory-host-sdk/src/host/embeddings-remote-provider.ts index 2ed7c188619..f58e41d3d29 100644 --- a/packages/memory-host-sdk/src/host/embeddings-remote-provider.ts +++ b/packages/memory-host-sdk/src/host/embeddings-remote-provider.ts @@ -4,12 +4,13 @@ import { type RemoteEmbeddingProviderId, } from "./embeddings-remote-client.js"; import { fetchRemoteEmbeddingVectors } from "./embeddings-remote-fetch.js"; -import type { EmbeddingProvider, EmbeddingProviderOptions } from "./embeddings.js"; +import type { EmbeddingProvider, EmbeddingProviderOptions } from "./embeddings.types.js"; export type RemoteEmbeddingClient = { baseUrl: string; headers: Record; ssrfPolicy?: SsrFPolicy; + fetchImpl?: typeof fetch; model: string; }; @@ -30,6 +31,7 @@ export function createRemoteEmbeddingProvider(params: { url, headers: client.headers, ssrfPolicy: client.ssrfPolicy, + fetchImpl: client.fetchImpl, body: { model: client.model, input }, errorPrefix: params.errorPrefix, }); diff --git a/packages/memory-host-sdk/src/host/embeddings.ts b/packages/memory-host-sdk/src/host/embeddings.ts index 89aaf665439..756d3aaf1d7 100644 --- a/packages/memory-host-sdk/src/host/embeddings.ts +++ b/packages/memory-host-sdk/src/host/embeddings.ts @@ -1 +1,85 @@ -export * from "../../../../src/memory-host-sdk/host/embeddings.js"; +import { DEFAULT_LOCAL_MODEL } from "./embedding-defaults.js"; +import { sanitizeAndNormalizeEmbedding } from "./embedding-vectors.js"; +import type { EmbeddingProvider, EmbeddingProviderOptions } from "./embeddings.types.js"; +import { + importNodeLlamaCpp, + type Llama, + type LlamaEmbeddingContext, + type LlamaModel, +} from "./node-llama.js"; +import { normalizeOptionalString } from "./string-utils.js"; + +export type { + EmbeddingProvider, + EmbeddingProviderFallback, + EmbeddingProviderId, + EmbeddingProviderOptions, + EmbeddingProviderRequest, + GeminiTaskType, +} from "./embeddings.types.js"; + +export { DEFAULT_LOCAL_MODEL } from "./embedding-defaults.js"; + +export async function createLocalEmbeddingProvider( + options: EmbeddingProviderOptions, +): Promise { + const modelPath = normalizeOptionalString(options.local?.modelPath) || DEFAULT_LOCAL_MODEL; + const modelCacheDir = normalizeOptionalString(options.local?.modelCacheDir); + const contextSize: number | "auto" = options.local?.contextSize ?? 4096; + + // Lazy-load node-llama-cpp to keep startup light unless local is enabled. + const { getLlama, resolveModelFile, LlamaLogLevel } = await importNodeLlamaCpp(); + + let llama: Llama | null = null; + let embeddingModel: LlamaModel | null = null; + let embeddingContext: LlamaEmbeddingContext | null = null; + let initPromise: Promise | null = null; + + const ensureContext = async (): Promise => { + if (embeddingContext) { + return embeddingContext; + } + if (initPromise) { + return initPromise; + } + initPromise = (async () => { + try { + if (!llama) { + llama = await getLlama({ logLevel: LlamaLogLevel.error }); + } + if (!embeddingModel) { + const resolved = await resolveModelFile(modelPath, modelCacheDir || undefined); + embeddingModel = await llama.loadModel({ modelPath: resolved }); + } + if (!embeddingContext) { + embeddingContext = await embeddingModel.createEmbeddingContext({ contextSize }); + } + return embeddingContext; + } catch (err) { + initPromise = null; + throw err; + } + })(); + return initPromise; + }; + + return { + id: "local", + model: modelPath, + embedQuery: async (text) => { + const ctx = await ensureContext(); + const embedding = await ctx.getEmbeddingFor(text); + return sanitizeAndNormalizeEmbedding(Array.from(embedding.vector)); + }, + embedBatch: async (texts) => { + const ctx = await ensureContext(); + const embeddings = await Promise.all( + texts.map(async (text) => { + const embedding = await ctx.getEmbeddingFor(text); + return sanitizeAndNormalizeEmbedding(Array.from(embedding.vector)); + }), + ); + return embeddings; + }, + }; +} diff --git a/packages/memory-host-sdk/src/host/embeddings.types.ts b/packages/memory-host-sdk/src/host/embeddings.types.ts new file mode 100644 index 00000000000..9d074ddf8a0 --- /dev/null +++ b/packages/memory-host-sdk/src/host/embeddings.types.ts @@ -0,0 +1,56 @@ +import type { OpenClawConfig, SecretInput } from "../engine-foundation.js"; +import type { EmbeddingInput } from "./embedding-inputs.js"; + +export type EmbeddingProvider = { + id: string; + model: string; + maxInputTokens?: number; + embedQuery: (text: string) => Promise; + embedBatch: (texts: string[]) => Promise; + embedBatchInputs?: (inputs: EmbeddingInput[]) => Promise; +}; + +export type EmbeddingProviderId = string; +export type EmbeddingProviderRequest = string; +export type EmbeddingProviderFallback = string; + +export type GeminiTaskType = + | "RETRIEVAL_QUERY" + | "RETRIEVAL_DOCUMENT" + | "SEMANTIC_SIMILARITY" + | "CLASSIFICATION" + | "CLUSTERING" + | "QUESTION_ANSWERING" + | "FACT_VERIFICATION"; + +export type EmbeddingProviderOptions = { + config: OpenClawConfig; + agentDir?: string; + provider?: EmbeddingProviderRequest; + remote?: { + baseUrl?: string; + apiKey?: SecretInput; + headers?: Record; + }; + model: string; + inputType?: string; + queryInputType?: string; + documentInputType?: string; + fallback?: EmbeddingProviderFallback; + local?: { + modelPath?: string; + modelCacheDir?: string; + /** + * Context size passed to node-llama-cpp `createEmbeddingContext`. + * Default: 4096, chosen to cover typical memory-search chunks (128–512 tokens) + * while keeping non-weight VRAM bounded. + * Set `"auto"` to let node-llama-cpp use the model's trained maximum — not + * recommended for 8B+ models (e.g. Qwen3-Embedding-8B: up to 40 960 tokens → ~32 GB VRAM). + */ + contextSize?: number | "auto"; + }; + /** Provider-specific output vector dimensions for supported embedding families. */ + outputDimensionality?: number; + /** Gemini: override the default task type sent with embedding requests. */ + taskType?: GeminiTaskType; +}; diff --git a/packages/memory-host-sdk/src/host/multimodal.ts b/packages/memory-host-sdk/src/host/multimodal.ts index 32167b48f81..d953e147356 100644 --- a/packages/memory-host-sdk/src/host/multimodal.ts +++ b/packages/memory-host-sdk/src/host/multimodal.ts @@ -1,4 +1,4 @@ -import { normalizeLowercaseStringOrEmpty } from "../../../../src/shared/string-coerce.js"; +import { normalizeLowercaseStringOrEmpty } from "./string-utils.js"; const MEMORY_MULTIMODAL_SPECS = { image: { diff --git a/packages/memory-host-sdk/src/host/node-llama.ts b/packages/memory-host-sdk/src/host/node-llama.ts index 7b54e3fed2f..8871b65da2e 100644 --- a/packages/memory-host-sdk/src/host/node-llama.ts +++ b/packages/memory-host-sdk/src/host/node-llama.ts @@ -7,7 +7,9 @@ export type LlamaEmbeddingContext = { }; export type LlamaModel = { - createEmbeddingContext: () => Promise; + createEmbeddingContext: (options?: { + contextSize?: number | "auto"; + }) => Promise; }; export type Llama = { diff --git a/packages/memory-host-sdk/src/host/post-json.ts b/packages/memory-host-sdk/src/host/post-json.ts index 12f8bfb3930..e53a833caf8 100644 --- a/packages/memory-host-sdk/src/host/post-json.ts +++ b/packages/memory-host-sdk/src/host/post-json.ts @@ -5,6 +5,7 @@ export async function postJson(params: { url: string; headers: Record; ssrfPolicy?: SsrFPolicy; + fetchImpl?: typeof fetch; body: unknown; errorPrefix: string; attachStatus?: boolean; @@ -13,6 +14,7 @@ export async function postJson(params: { return await withRemoteHttpResponse({ url: params.url, ssrfPolicy: params.ssrfPolicy, + fetchImpl: params.fetchImpl, init: { method: "POST", headers: params.headers, diff --git a/packages/memory-host-sdk/src/host/qmd-query-parser.ts b/packages/memory-host-sdk/src/host/qmd-query-parser.ts index ddf166bd8db..a4c9c550df7 100644 --- a/packages/memory-host-sdk/src/host/qmd-query-parser.ts +++ b/packages/memory-host-sdk/src/host/qmd-query-parser.ts @@ -1,6 +1,6 @@ import { formatErrorMessage } from "../../../../src/infra/errors.js"; import { createSubsystemLogger } from "../../../../src/logging/subsystem.js"; -import { normalizeLowercaseStringOrEmpty } from "../../../../src/shared/string-coerce.js"; +import { normalizeLowercaseStringOrEmpty } from "./string-utils.js"; const log = createSubsystemLogger("memory"); diff --git a/packages/memory-host-sdk/src/host/qmd-scope.ts b/packages/memory-host-sdk/src/host/qmd-scope.ts index 116d5f7a3b7..3edb490fb68 100644 --- a/packages/memory-host-sdk/src/host/qmd-scope.ts +++ b/packages/memory-host-sdk/src/host/qmd-scope.ts @@ -1,9 +1,9 @@ import { parseAgentSessionKey } from "../../../../src/sessions/session-key-utils.js"; +import type { ResolvedQmdConfig } from "./backend-config.js"; import { normalizeLowercaseStringOrEmpty, normalizeOptionalLowercaseString, -} from "../../../../src/shared/string-coerce.js"; -import type { ResolvedQmdConfig } from "./backend-config.js"; +} from "./string-utils.js"; type ParsedQmdSessionScope = { channel?: string; diff --git a/packages/memory-host-sdk/src/host/query-expansion.ts b/packages/memory-host-sdk/src/host/query-expansion.ts index 062dbb0165b..859612a1f6c 100644 --- a/packages/memory-host-sdk/src/host/query-expansion.ts +++ b/packages/memory-host-sdk/src/host/query-expansion.ts @@ -1,4 +1,4 @@ -import { normalizeLowercaseStringOrEmpty } from "../../../../src/shared/string-coerce.js"; +import { normalizeLowercaseStringOrEmpty } from "./string-utils.js"; /** * Query expansion for FTS-only search mode. diff --git a/packages/memory-host-sdk/src/host/read-file-shared.ts b/packages/memory-host-sdk/src/host/read-file-shared.ts new file mode 100644 index 00000000000..e9fd4906408 --- /dev/null +++ b/packages/memory-host-sdk/src/host/read-file-shared.ts @@ -0,0 +1,114 @@ +import type { MemoryReadResult } from "./types.js"; + +export const DEFAULT_MEMORY_READ_LINES = 120; +export const DEFAULT_MEMORY_READ_MAX_CHARS = 12_000; + +export type { MemoryReadResult } from "./types.js"; + +function buildContinuationNotice(params: { + nextFrom: number | undefined; + suggestReadFallback?: boolean; +}): string { + const base = + typeof params.nextFrom === "number" + ? `[More content available. Use from=${params.nextFrom} to continue.]` + : "[More content available. Requested excerpt exceeded the default maxChars budget.]"; + const fallback = params.suggestReadFallback + ? " If you need the full raw line, use read on the source file." + : ""; + return `\n\n${base.slice(0, -1)}${fallback}]`; +} + +function fitLinesToCharBudget(params: { lines: string[]; maxChars: number }): { + text: string; + includedLines: number; + hardTruncatedSingleLine: boolean; +} { + const { lines, maxChars } = params; + if (lines.length === 0) { + return { text: "", includedLines: 0, hardTruncatedSingleLine: false }; + } + + let includedLines = lines.length; + let text = lines.join("\n"); + while (includedLines > 1 && text.length > maxChars) { + includedLines -= 1; + text = lines.slice(0, includedLines).join("\n"); + } + + if (text.length <= maxChars) { + return { text, includedLines, hardTruncatedSingleLine: false }; + } + + return { + text: text.slice(0, maxChars), + includedLines: 1, + hardTruncatedSingleLine: true, + }; +} + +export function buildMemoryReadResultFromSlice(params: { + selectedLines: string[]; + relPath: string; + startLine: number; + moreSourceLinesRemain?: boolean; + maxChars?: number; + suggestReadFallback?: boolean; +}): MemoryReadResult { + const start = Math.max(1, params.startLine); + const fitted = fitLinesToCharBudget({ + lines: params.selectedLines, + maxChars: Math.max(1, params.maxChars ?? DEFAULT_MEMORY_READ_MAX_CHARS), + }); + const moreSourceLinesRemain = params.moreSourceLinesRemain ?? false; + const charCapTruncated = + fitted.hardTruncatedSingleLine || fitted.includedLines < params.selectedLines.length; + const nextFrom = + !fitted.hardTruncatedSingleLine && + (moreSourceLinesRemain || fitted.includedLines < params.selectedLines.length) + ? start + fitted.includedLines + : undefined; + const truncated = charCapTruncated || moreSourceLinesRemain; + const text = + truncated && fitted.text + ? `${fitted.text}${buildContinuationNotice({ + nextFrom, + suggestReadFallback: fitted.hardTruncatedSingleLine && params.suggestReadFallback, + })}` + : fitted.text; + return { + text, + path: params.relPath, + from: start, + lines: fitted.includedLines, + ...(truncated ? { truncated: true } : {}), + ...(typeof nextFrom === "number" ? { nextFrom } : {}), + }; +} + +export function buildMemoryReadResult(params: { + content: string; + relPath: string; + from?: number; + lines?: number; + defaultLines?: number; + maxChars?: number; + suggestReadFallback?: boolean; +}): MemoryReadResult { + const fileLines = params.content.split("\n"); + const start = Math.max(1, params.from ?? 1); + const requestedCount = Math.max( + 1, + params.lines ?? params.defaultLines ?? DEFAULT_MEMORY_READ_LINES, + ); + const selectedLines = fileLines.slice(start - 1, start - 1 + requestedCount); + const moreSourceLinesRemain = start - 1 + selectedLines.length < fileLines.length; + return buildMemoryReadResultFromSlice({ + selectedLines, + relPath: params.relPath, + startLine: start, + moreSourceLinesRemain, + maxChars: params.maxChars, + suggestReadFallback: params.suggestReadFallback, + }); +} diff --git a/packages/memory-host-sdk/src/host/read-file.ts b/packages/memory-host-sdk/src/host/read-file.ts index 7543b416e68..5686ccb6dda 100644 --- a/packages/memory-host-sdk/src/host/read-file.ts +++ b/packages/memory-host-sdk/src/host/read-file.ts @@ -6,13 +6,13 @@ import { } from "../../../../src/agents/agent-scope.js"; import { resolveMemorySearchConfig } from "../../../../src/agents/memory-search.js"; import type { OpenClawConfig } from "../../../../src/config/config.js"; +import { isFileMissingError, statRegularFile } from "./fs-utils.js"; +import { isMemoryPath, normalizeExtraMemoryPaths } from "./internal.js"; import { buildMemoryReadResult, DEFAULT_MEMORY_READ_LINES, type MemoryReadResult, -} from "../../../../src/memory-host-sdk/host/read-file-shared.js"; -import { isFileMissingError, statRegularFile } from "./fs-utils.js"; -import { isMemoryPath, normalizeExtraMemoryPaths } from "./internal.js"; +} from "./read-file-shared.js"; export async function readMemoryFile(params: { workspaceDir: string; diff --git a/packages/memory-host-sdk/src/host/remote-http.ts b/packages/memory-host-sdk/src/host/remote-http.ts index 6676a70db8e..bf6537c3b1a 100644 --- a/packages/memory-host-sdk/src/host/remote-http.ts +++ b/packages/memory-host-sdk/src/host/remote-http.ts @@ -1,29 +1,17 @@ import { fetchWithSsrFGuard, GUARDED_FETCH_MODE } from "../../../../src/infra/net/fetch-guard.js"; import { shouldUseEnvHttpProxyForUrl } from "../../../../src/infra/net/proxy-env.js"; -import type { SsrFPolicy } from "../../../../src/infra/net/ssrf.js"; +import { + ssrfPolicyFromHttpBaseUrlAllowedHostname, + type SsrFPolicy, +} from "../../../../src/infra/net/ssrf.js"; -export function buildRemoteBaseUrlPolicy(baseUrl: string): SsrFPolicy | undefined { - const trimmed = baseUrl.trim(); - if (!trimmed) { - return undefined; - } - try { - const parsed = new URL(trimmed); - if (parsed.protocol !== "http:" && parsed.protocol !== "https:") { - return undefined; - } - // Keep policy tied to the configured host so private operator endpoints - // continue to work, while cross-host redirects stay blocked. - return { allowedHostnames: [parsed.hostname] }; - } catch { - return undefined; - } -} +export const buildRemoteBaseUrlPolicy = ssrfPolicyFromHttpBaseUrlAllowedHostname; export async function withRemoteHttpResponse(params: { url: string; init?: RequestInit; ssrfPolicy?: SsrFPolicy; + fetchImpl?: typeof fetch; fetchWithSsrFGuardImpl?: typeof fetchWithSsrFGuard; shouldUseEnvHttpProxyForUrlImpl?: typeof shouldUseEnvHttpProxyForUrl; auditContext?: string; @@ -33,6 +21,7 @@ export async function withRemoteHttpResponse(params: { const shouldUseEnvProxy = params.shouldUseEnvHttpProxyForUrlImpl ?? shouldUseEnvHttpProxyForUrl; const { response, release } = await guardedFetch({ url: params.url, + fetchImpl: params.fetchImpl, init: params.init, policy: params.ssrfPolicy, auditContext: params.auditContext ?? "memory-remote", diff --git a/packages/memory-host-sdk/src/host/session-files.ts b/packages/memory-host-sdk/src/host/session-files.ts index 34e6c6c88de..02a483a4f67 100644 --- a/packages/memory-host-sdk/src/host/session-files.ts +++ b/packages/memory-host-sdk/src/host/session-files.ts @@ -1,16 +1,31 @@ +import fsSync from "node:fs"; import fs from "node:fs/promises"; import path from "node:path"; +import { stripInternalRuntimeContext } from "../../../../src/agents/internal-runtime-context.js"; +import { isHeartbeatUserMessage } from "../../../../src/auto-reply/heartbeat-filter.js"; +import { HEARTBEAT_PROMPT } from "../../../../src/auto-reply/heartbeat.js"; import { stripInboundMetadata } from "../../../../src/auto-reply/reply/strip-inbound-meta.js"; +import { HEARTBEAT_TOKEN, isSilentReplyPayloadText } from "../../../../src/auto-reply/tokens.js"; import { isCompactionCheckpointTranscriptFileName, isSessionArchiveArtifactName, isUsageCountedSessionTranscriptFileName, } from "../../../../src/config/sessions/artifacts.js"; import { resolveSessionTranscriptsDirForAgent } from "../../../../src/config/sessions/paths.js"; +import { isExecCompletionEvent } from "../../../../src/infra/heartbeat-events-filter.js"; import { redactSensitiveText } from "../../../../src/logging/redact.js"; import { hasInterSessionUserProvenance } from "../../../../src/sessions/input-provenance.js"; +import { isCronRunSessionKey } from "../../../../src/sessions/session-key-utils.js"; import { hashText } from "./hash.js"; +const DREAMING_NARRATIVE_RUN_PREFIX = "dreaming-narrative-"; +// Keep the historical one-line-per-message export shape for normal turns, but +// wrap pathological long messages so downstream indexers never ingest a single +// toxic line. Wrapped continuation lines still map back to the same JSONL line. +// This limit applies to content only; the role label adds up to 11 chars. +const SESSION_EXPORT_CONTENT_WRAP_CHARS = 800; +const DIRECT_CRON_PROMPT_RE = /^\[cron:[^\]]+\]\s*/; + export type SessionFileEntry = { path: string; absPath: string; @@ -20,10 +35,38 @@ export type SessionFileEntry = { content: string; /** Maps each content line (0-indexed) to its 1-indexed JSONL source line. */ lineMap: number[]; + /** Maps each content line (0-indexed) to epoch ms; 0 means unknown timestamp. */ + messageTimestampsMs: number[]; /** True when this transcript belongs to an internal dreaming narrative run. */ generatedByDreamingNarrative?: boolean; + /** True when this transcript belongs to an isolated cron run session. */ + generatedByCronRun?: boolean; }; +export type BuildSessionEntryOptions = { + /** Optional preclassification from a caller-managed dreaming transcript lookup. */ + generatedByDreamingNarrative?: boolean; + /** Optional preclassification from a caller-managed cron transcript lookup. */ + generatedByCronRun?: boolean; +}; + +export type SessionTranscriptClassification = { + dreamingNarrativeTranscriptPaths: ReadonlySet; + cronRunTranscriptPaths: ReadonlySet; +}; + +type SessionTranscriptStoreEntry = { + sessionFile?: unknown; + sessionId?: unknown; +}; + +function shouldSkipTranscriptFileForDreaming(absPath: string): boolean { + const fileName = path.basename(absPath); + return ( + isSessionArchiveArtifactName(fileName) || isCompactionCheckpointTranscriptFileName(fileName) + ); +} + function isDreamingNarrativeBootstrapRecord(record: unknown): boolean { if (!record || typeof record !== "object" || Array.isArray(record)) { return false; @@ -43,16 +86,155 @@ function isDreamingNarrativeBootstrapRecord(record: unknown): boolean { return false; } const runId = (candidate.data as { runId?: unknown }).runId; - return typeof runId === "string" && runId.startsWith("dreaming-narrative-"); + return typeof runId === "string" && runId.startsWith(DREAMING_NARRATIVE_RUN_PREFIX); } -function shouldSkipTranscriptFileForDreaming(absPath: string): boolean { - const fileName = path.basename(absPath); - return ( - isSessionArchiveArtifactName(fileName) || isCompactionCheckpointTranscriptFileName(fileName) +function hasDreamingNarrativeRunId(value: unknown): boolean { + return typeof value === "string" && value.startsWith(DREAMING_NARRATIVE_RUN_PREFIX); +} + +function isDreamingNarrativeGeneratedRecord(record: unknown): boolean { + if (isDreamingNarrativeBootstrapRecord(record)) { + return true; + } + if (!record || typeof record !== "object" || Array.isArray(record)) { + return false; + } + const candidate = record as { + runId?: unknown; + sessionKey?: unknown; + data?: unknown; + }; + if ( + hasDreamingNarrativeRunId(candidate.runId) || + hasDreamingNarrativeRunId(candidate.sessionKey) + ) { + return true; + } + if (!candidate.data || typeof candidate.data !== "object" || Array.isArray(candidate.data)) { + return false; + } + const nested = candidate.data as { + runId?: unknown; + sessionKey?: unknown; + }; + return hasDreamingNarrativeRunId(nested.runId) || hasDreamingNarrativeRunId(nested.sessionKey); +} + +function isDreamingNarrativeSessionStoreKey(sessionKey: string): boolean { + const trimmed = sessionKey.trim(); + if (!trimmed) { + return false; + } + const firstSeparator = trimmed.indexOf(":"); + if (firstSeparator < 0) { + return trimmed.startsWith(DREAMING_NARRATIVE_RUN_PREFIX); + } + const secondSeparator = trimmed.indexOf(":", firstSeparator + 1); + const sessionSegment = secondSeparator < 0 ? trimmed : trimmed.slice(secondSeparator + 1); + return sessionSegment.startsWith(DREAMING_NARRATIVE_RUN_PREFIX); +} + +function normalizeComparablePath(pathname: string): string { + const resolved = path.resolve(pathname); + return process.platform === "win32" ? resolved.toLowerCase() : resolved; +} + +export function normalizeSessionTranscriptPathForComparison(pathname: string): string { + return normalizeComparablePath(pathname); +} + +function resolveSessionStoreTranscriptPath( + sessionsDir: string, + entry: { sessionFile?: unknown; sessionId?: unknown } | undefined, +): string | null { + if (typeof entry?.sessionFile === "string" && entry.sessionFile.trim().length > 0) { + const sessionFile = entry.sessionFile.trim(); + const resolved = path.isAbsolute(sessionFile) + ? sessionFile + : path.resolve(sessionsDir, sessionFile); + return normalizeComparablePath(resolved); + } + if (typeof entry?.sessionId === "string" && entry.sessionId.trim().length > 0) { + return normalizeComparablePath(path.join(sessionsDir, `${entry.sessionId.trim()}.jsonl`)); + } + return null; +} + +export function loadDreamingNarrativeTranscriptPathSetForSessionsDir( + sessionsDir: string, +): ReadonlySet { + return loadSessionTranscriptClassificationForSessionsDir(sessionsDir) + .dreamingNarrativeTranscriptPaths; +} + +export function loadSessionTranscriptClassificationForSessionsDir( + sessionsDir: string, +): SessionTranscriptClassification { + const storePath = path.join(sessionsDir, "sessions.json"); + const store = readSessionTranscriptClassificationStore(storePath); + const dreamingTranscriptPaths = new Set(); + const cronRunTranscriptPaths = new Set(); + for (const [sessionKey, entry] of Object.entries(store)) { + const transcriptPath = resolveSessionStoreTranscriptPath(sessionsDir, entry); + if (!transcriptPath) { + continue; + } + if (isDreamingNarrativeSessionStoreKey(sessionKey)) { + dreamingTranscriptPaths.add(transcriptPath); + } + if (isCronRunSessionKey(sessionKey)) { + cronRunTranscriptPaths.add(transcriptPath); + } + } + return { + dreamingNarrativeTranscriptPaths: dreamingTranscriptPaths, + cronRunTranscriptPaths, + }; +} + +function readSessionTranscriptClassificationStore( + storePath: string, +): Record { + try { + const parsed = JSON.parse(fsSync.readFileSync(storePath, "utf-8")) as unknown; + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { + return {}; + } + return parsed as Record; + } catch { + return {}; + } +} + +export function loadDreamingNarrativeTranscriptPathSetForAgent( + agentId: string, +): ReadonlySet { + return loadSessionTranscriptClassificationForAgent(agentId).dreamingNarrativeTranscriptPaths; +} + +export function loadSessionTranscriptClassificationForAgent( + agentId: string, +): SessionTranscriptClassification { + return loadSessionTranscriptClassificationForSessionsDir( + resolveSessionTranscriptsDirForAgent(agentId), ); } +function classifySessionTranscriptFromSessionStore(absPath: string): { + generatedByDreamingNarrative: boolean; + generatedByCronRun: boolean; +} { + const sessionsDir = path.dirname(absPath); + const normalizedAbsPath = normalizeComparablePath(absPath); + const classification = loadSessionTranscriptClassificationForSessionsDir(sessionsDir); + return { + generatedByDreamingNarrative: + classification.dreamingNarrativeTranscriptPaths.has(normalizedAbsPath), + generatedByCronRun: classification.cronRunTranscriptPaths.has(normalizedAbsPath), + }; +} + export async function listSessionFilesForAgent(agentId: string): Promise { const dir = resolveSessionTranscriptsDirForAgent(agentId); try { @@ -103,11 +285,77 @@ function collectRawSessionText(content: unknown): string | null { return parts.length > 0 ? parts.join("\n") : null; } +function isHighSurrogate(code: number): boolean { + return code >= 0xd800 && code <= 0xdbff; +} + +function isLowSurrogate(code: number): boolean { + return code >= 0xdc00 && code <= 0xdfff; +} + +function splitLongSessionLine( + text: string, + maxChars: number = SESSION_EXPORT_CONTENT_WRAP_CHARS, +): string[] { + const normalized = text.trim(); + if (!normalized) { + return []; + } + if (normalized.length <= maxChars) { + return [normalized]; + } + + const segments: string[] = []; + let cursor = 0; + while (cursor < normalized.length) { + const remaining = normalized.length - cursor; + if (remaining <= maxChars) { + segments.push(normalized.slice(cursor).trim()); + break; + } + + const limit = cursor + maxChars; + let splitAt = limit; + for (let index = limit; index > cursor; index -= 1) { + if (normalized[index] === " ") { + splitAt = index; + break; + } + } + if ( + splitAt < normalized.length && + splitAt > cursor && + isHighSurrogate(normalized.charCodeAt(splitAt - 1)) && + isLowSurrogate(normalized.charCodeAt(splitAt)) + ) { + splitAt -= 1; + } + segments.push(normalized.slice(cursor, splitAt).trim()); + cursor = splitAt; + while (cursor < normalized.length && normalized[cursor] === " ") { + cursor += 1; + } + } + + return segments.filter(Boolean); +} + +function renderSessionExportLines(label: string, text: string): string[] { + return splitLongSessionLine(text).map((segment) => `${label}: ${segment}`); +} + /** - * Strip OpenClaw-injected inbound metadata envelopes from a raw text block - * on user-role messages before normalization. See the authoritative - * implementation in `src/memory-host-sdk/host/session-files.ts` for the - * full rationale; duplicated here to keep this parallel copy bug-free. + * Strip OpenClaw-injected inbound metadata envelopes from a raw text block. + * + * User-role messages arriving from external channels (Telegram, Discord, + * Slack, …) are stored with a multi-line prefix containing Conversation info, + * Sender info, and other AI-facing metadata blocks. These envelopes must be + * removed BEFORE normalization, because `stripInboundMetadata` relies on + * newline structure and fenced `json` code fences to locate sentinels; once + * `normalizeSessionText` collapses newlines into spaces, stripping is + * impossible. + * + * See: https://github.com/openclaw/openclaw/issues/63921 */ function stripInboundMetadataForUserRole(text: string, role: "user" | "assistant"): string { if (role !== "user") { @@ -116,6 +364,59 @@ function stripInboundMetadataForUserRole(text: string, role: "user" | "assistant return stripInboundMetadata(text); } +const GENERATED_SYSTEM_MESSAGE_RE = /^System(?: \(untrusted\))?: \[[^\]]+\]\s*/; + +function isGeneratedSystemWrapperMessage(text: string, role: "user" | "assistant"): boolean { + if (role !== "user") { + return false; + } + return GENERATED_SYSTEM_MESSAGE_RE.test(text); +} + +function isGeneratedCronPromptMessage(text: string, role: "user" | "assistant"): boolean { + if (role !== "user") { + return false; + } + return DIRECT_CRON_PROMPT_RE.test(text); +} + +function isGeneratedHeartbeatPromptMessage(text: string, role: "user" | "assistant"): boolean { + return role === "user" && isHeartbeatUserMessage({ role, content: text }, HEARTBEAT_PROMPT); +} + +function sanitizeSessionText(text: string, role: "user" | "assistant"): string | null { + const strippedInbound = stripInboundMetadataForUserRole(text, role); + const strippedInternal = stripInternalRuntimeContext(strippedInbound); + const normalized = normalizeSessionText(strippedInternal); + if (!normalized) { + return null; + } + if (isGeneratedSystemWrapperMessage(normalized, role)) { + return null; + } + if (isGeneratedCronPromptMessage(normalized, role)) { + return null; + } + if (isGeneratedHeartbeatPromptMessage(normalized, role)) { + return null; + } + if (isSilentReplyPayloadText(normalized)) { + return null; + } + // Assistant-side machinery acks: HEARTBEAT_OK is the canonical "all clear, + // nothing to do" reply to a heartbeat tick. Drop on the assistant side + // directly so we do not have to rely on cross-message coupling with the + // preceding user message (which a real user could spoof). + if (role === "assistant" && normalized === HEARTBEAT_TOKEN) { + return null; + } + const withoutSystemEnvelope = normalized.replace(GENERATED_SYSTEM_MESSAGE_RE, "").trim(); + if (isExecCompletionEvent(withoutSystemEnvelope)) { + return null; + } + return normalized; +} + export function extractSessionText( content: unknown, role: "user" | "assistant" = "assistant", @@ -124,12 +425,35 @@ export function extractSessionText( if (rawText === null) { return null; } - const stripped = stripInboundMetadataForUserRole(rawText, role); - const normalized = normalizeSessionText(stripped); - return normalized ? normalized : null; + return sanitizeSessionText(rawText, role); } -export async function buildSessionEntry(absPath: string): Promise { +function parseSessionTimestampMs( + record: { timestamp?: unknown }, + message: { timestamp?: unknown }, +): number { + const candidates = [message.timestamp, record.timestamp]; + for (const value of candidates) { + if (typeof value === "number" && Number.isFinite(value)) { + const ms = value > 0 && value < 1e11 ? value * 1000 : value; + if (Number.isFinite(ms) && ms > 0) { + return ms; + } + } + if (typeof value === "string") { + const parsed = Date.parse(value); + if (Number.isFinite(parsed) && parsed > 0) { + return parsed; + } + } + } + return 0; +} + +export async function buildSessionEntry( + absPath: string, + opts: BuildSessionEntryOptions = {}, +): Promise { try { const stat = await fs.stat(absPath); if (shouldSkipTranscriptFileForDreaming(absPath)) { @@ -141,14 +465,24 @@ export async function buildSessionEntry(absPath: string): Promise jsonlIdx + 1)); + messageTimestampsMs.push(...renderedLines.map(() => timestampMs)); } const content = collected.join("\n"); return { @@ -197,10 +551,12 @@ export async function buildSessionEntry(absPath: string): Promise string; diff --git a/packages/memory-host-sdk/src/host/sqlite.ts b/packages/memory-host-sdk/src/host/sqlite.ts index 4bc9b9ba02e..9a2ec1695a3 100644 --- a/packages/memory-host-sdk/src/host/sqlite.ts +++ b/packages/memory-host-sdk/src/host/sqlite.ts @@ -1,8 +1,15 @@ import { createRequire } from "node:module"; +import type { DatabaseSync } from "node:sqlite"; import { formatErrorMessage } from "../../../../src/infra/errors.js"; +import { + configureSqliteWalMaintenance, + type SqliteWalMaintenance, + type SqliteWalMaintenanceOptions, +} from "../../../../src/infra/sqlite-wal.js"; import { installProcessWarningFilter } from "../../../../src/infra/warning-filter.js"; const require = createRequire(import.meta.url); +const sqliteWalMaintenanceByDb = new WeakMap(); export function requireNodeSqlite(): typeof import("node:sqlite") { installProcessWarningFilter(); @@ -18,3 +25,25 @@ export function requireNodeSqlite(): typeof import("node:sqlite") { ); } } + +export function configureMemorySqliteWalMaintenance( + db: DatabaseSync, + options?: SqliteWalMaintenanceOptions, +): SqliteWalMaintenance { + const existing = sqliteWalMaintenanceByDb.get(db); + if (existing) { + return existing; + } + const maintenance = configureSqliteWalMaintenance(db, options); + sqliteWalMaintenanceByDb.set(db, maintenance); + return maintenance; +} + +export function closeMemorySqliteWalMaintenance(db: DatabaseSync): boolean { + const maintenance = sqliteWalMaintenanceByDb.get(db); + if (!maintenance) { + return true; + } + sqliteWalMaintenanceByDb.delete(db); + return maintenance.close(); +} diff --git a/packages/memory-host-sdk/src/host/string-utils.ts b/packages/memory-host-sdk/src/host/string-utils.ts new file mode 100644 index 00000000000..108f3eab3b1 --- /dev/null +++ b/packages/memory-host-sdk/src/host/string-utils.ts @@ -0,0 +1,19 @@ +export function normalizeNullableString(value: unknown): string | null { + if (typeof value !== "string") { + return null; + } + const trimmed = value.trim(); + return trimmed ? trimmed : null; +} + +export function normalizeOptionalString(value: unknown): string | undefined { + return normalizeNullableString(value) ?? undefined; +} + +export function normalizeOptionalLowercaseString(value: unknown): string | undefined { + return normalizeOptionalString(value)?.toLowerCase(); +} + +export function normalizeLowercaseStringOrEmpty(value: unknown): string { + return normalizeOptionalLowercaseString(value) ?? ""; +} diff --git a/packages/memory-host-sdk/src/host/test-helpers/ssrf.ts b/packages/memory-host-sdk/src/host/test-helpers/ssrf.ts index 0feb1a45381..8b2e160f439 100644 --- a/packages/memory-host-sdk/src/host/test-helpers/ssrf.ts +++ b/packages/memory-host-sdk/src/host/test-helpers/ssrf.ts @@ -1,6 +1,6 @@ import { vi } from "vitest"; import * as ssrf from "../../../../../src/infra/net/ssrf.js"; -import { normalizeLowercaseStringOrEmpty } from "../../../../../src/shared/string-coerce.js"; +import { normalizeLowercaseStringOrEmpty } from "../string-utils.js"; export function mockPublicPinnedHostname() { return vi.spyOn(ssrf, "resolvePinnedHostnameWithPolicy").mockImplementation(async (hostname) => { diff --git a/packages/memory-host-sdk/src/host/types.ts b/packages/memory-host-sdk/src/host/types.ts index ffe0fad3432..7c99da2d32f 100644 --- a/packages/memory-host-sdk/src/host/types.ts +++ b/packages/memory-host-sdk/src/host/types.ts @@ -27,6 +27,22 @@ export type MemorySyncProgressUpdate = { label?: string; }; +export type MemorySearchRuntimeDebug = { + backend: "builtin" | "qmd"; + configuredMode?: string; + effectiveMode?: string; + fallback?: string; +}; + +export type MemoryReadResult = { + text: string; + path: string; + truncated?: boolean; + from?: number; + lines?: number; + nextFrom?: number; +}; + export type MemoryProviderStatus = { backend: "builtin" | "qmd"; provider: string; @@ -71,14 +87,12 @@ export interface MemorySearchManager { maxResults?: number; minScore?: number; sessionKey?: string; + qmdSearchModeOverride?: "query" | "search" | "vsearch"; + onDebug?: (debug: MemorySearchRuntimeDebug) => void; sources?: MemorySource[]; }, ): Promise; - readFile(params: { - relPath: string; - from?: number; - lines?: number; - }): Promise<{ text: string; path: string }>; + readFile(params: { relPath: string; from?: number; lines?: number }): Promise; status(): MemoryProviderStatus; sync?(params?: { reason?: string; diff --git a/packages/memory-host-sdk/src/multimodal.ts b/packages/memory-host-sdk/src/multimodal.ts index af483ef2422..eb11867ac3a 100644 --- a/packages/memory-host-sdk/src/multimodal.ts +++ b/packages/memory-host-sdk/src/multimodal.ts @@ -1 +1,5 @@ -export * from "../../../src/memory-host-sdk/multimodal.js"; +export { + isMemoryMultimodalEnabled, + normalizeMemoryMultimodalSettings, + type MemoryMultimodalSettings, +} from "./host/multimodal.js"; diff --git a/packages/memory-host-sdk/src/query.ts b/packages/memory-host-sdk/src/query.ts index dd2605d656b..bb945afaa65 100644 --- a/packages/memory-host-sdk/src/query.ts +++ b/packages/memory-host-sdk/src/query.ts @@ -1 +1 @@ -export * from "../../../src/memory-host-sdk/query.js"; +export { extractKeywords, isQueryStopWordToken } from "./host/query-expansion.js"; diff --git a/packages/memory-host-sdk/src/runtime-cli.ts b/packages/memory-host-sdk/src/runtime-cli.ts index 3f2651422ee..9a1b858cd0d 100644 --- a/packages/memory-host-sdk/src/runtime-cli.ts +++ b/packages/memory-host-sdk/src/runtime-cli.ts @@ -1 +1,11 @@ -export * from "../../../src/memory-host-sdk/runtime-cli.js"; +// Focused runtime contract for memory CLI/UI helpers. + +export { formatErrorMessage, withManager } from "../../../src/cli/cli-utils.js"; +export { formatHelpExamples } from "../../../src/cli/help-format.js"; +export { resolveCommandSecretRefsViaGateway } from "../../../src/cli/command-secret-gateway.js"; +export { withProgress, withProgressTotals } from "../../../src/cli/progress.js"; +export { defaultRuntime } from "../../../src/runtime.js"; +export { formatDocsLink } from "../../../src/terminal/links.js"; +export { colorize, isRich, theme } from "../../../src/terminal/theme.js"; +export { isVerbose, setVerbose } from "../../../src/globals.js"; +export { shortenHomeInString, shortenHomePath } from "../../../src/utils.js"; diff --git a/packages/memory-host-sdk/src/runtime-core.ts b/packages/memory-host-sdk/src/runtime-core.ts index fa9bf04d6d8..9b14431b031 100644 --- a/packages/memory-host-sdk/src/runtime-core.ts +++ b/packages/memory-host-sdk/src/runtime-core.ts @@ -1 +1,41 @@ -export * from "../../../src/memory-host-sdk/runtime-core.js"; +// Focused runtime contract for memory plugin config/state/helpers. + +export type { AnyAgentTool } from "../../../src/agents/tools/common.js"; +export { resolveCronStyleNow } from "../../../src/agents/current-time.js"; +export { DEFAULT_PI_COMPACTION_RESERVE_TOKENS_FLOOR } from "../../../src/agents/pi-settings.js"; +export { resolveDefaultAgentId, resolveSessionAgentId } from "../../../src/agents/agent-scope.js"; +export { resolveMemorySearchConfig } from "../../../src/agents/memory-search.js"; +export { + asToolParamsRecord, + jsonResult, + readNumberParam, + readStringParam, +} from "../../../src/agents/tools/common.js"; +export { SILENT_REPLY_TOKEN } from "../../../src/auto-reply/tokens.js"; +export { parseNonNegativeByteSize } from "../../../src/config/byte-size.js"; +export { + getRuntimeConfig, + /** @deprecated Use getRuntimeConfig(), or pass the already loaded config through the call path. */ + loadConfig, +} from "../../../src/config/config.js"; +export { resolveStateDir } from "../../../src/config/paths.js"; +export { resolveSessionTranscriptsDirForAgent } from "../../../src/config/sessions/paths.js"; +export { emptyPluginConfigSchema } from "../../../src/plugins/config-schema.js"; +export { + buildMemoryPromptSection as buildActiveMemoryPromptSection, + listActiveMemoryPublicArtifacts, + getMemoryCapabilityRegistration, +} from "../../../src/plugins/memory-state.js"; +export { parseAgentSessionKey } from "../../../src/routing/session-key.js"; +export type { OpenClawConfig } from "../../../src/config/config.js"; +export type { MemoryCitationsMode } from "../../../src/config/types.memory.js"; +export type { + MemoryFlushPlan, + MemoryFlushPlanResolver, + MemoryPluginCapability, + MemoryPluginPublicArtifact, + MemoryPluginPublicArtifactsProvider, + MemoryPluginRuntime, + MemoryPromptSectionBuilder, +} from "../../../src/plugins/memory-state.js"; +export type { OpenClawPluginApi } from "../../../src/plugins/types.js"; diff --git a/packages/memory-host-sdk/src/runtime-files.ts b/packages/memory-host-sdk/src/runtime-files.ts index 5c8aa4b2ae0..77f0de0e1d2 100644 --- a/packages/memory-host-sdk/src/runtime-files.ts +++ b/packages/memory-host-sdk/src/runtime-files.ts @@ -1 +1,10 @@ -export * from "../../../src/memory-host-sdk/runtime-files.js"; +// Focused runtime contract for memory file/backend access. + +export { listMemoryFiles, normalizeExtraMemoryPaths } from "./host/internal.js"; +export { readAgentMemoryFile } from "./host/read-file.js"; +export { resolveMemoryBackendConfig } from "./host/backend-config.js"; +export type { + MemorySearchManager, + MemorySearchRuntimeDebug, + MemorySearchResult, +} from "./host/types.js"; diff --git a/packages/memory-host-sdk/src/runtime.ts b/packages/memory-host-sdk/src/runtime.ts index 0b3bef94c6c..6e152ea0dcb 100644 --- a/packages/memory-host-sdk/src/runtime.ts +++ b/packages/memory-host-sdk/src/runtime.ts @@ -1 +1,6 @@ -export * from "../../../src/memory-host-sdk/runtime.js"; +// Aggregate workspace contract for memory runtime/helper seams. +// Keep focused subpaths preferred for new code. + +export * from "./runtime-core.js"; +export * from "./runtime-cli.js"; +export * from "./runtime-files.js"; diff --git a/packages/memory-host-sdk/src/secret.ts b/packages/memory-host-sdk/src/secret.ts index c8ac074db0c..b2b6b94ab47 100644 --- a/packages/memory-host-sdk/src/secret.ts +++ b/packages/memory-host-sdk/src/secret.ts @@ -1 +1,4 @@ -export * from "../../../src/memory-host-sdk/secret.js"; +export { + hasConfiguredMemorySecretInput, + resolveMemorySecretInputString, +} from "./host/secret-input.js"; diff --git a/packages/memory-host-sdk/src/status.ts b/packages/memory-host-sdk/src/status.ts index 3e46beba8d6..dc718abd96b 100644 --- a/packages/memory-host-sdk/src/status.ts +++ b/packages/memory-host-sdk/src/status.ts @@ -1 +1,6 @@ -export * from "../../../src/memory-host-sdk/status.js"; +export { + resolveMemoryCacheSummary, + resolveMemoryFtsState, + resolveMemoryVectorState, + type Tone, +} from "./host/status-format.js"; diff --git a/scripts/plugin-boundary-report.ts b/scripts/plugin-boundary-report.ts index 9bd7cd54bda..55110b76910 100644 --- a/scripts/plugin-boundary-report.ts +++ b/scripts/plugin-boundary-report.ts @@ -106,7 +106,11 @@ type BoundaryReportSummary = { exportedSubpathCount: number; sourceBridgeFileCount: number; packageCoreReferenceFileCount: number; - implementation: "private-core-bridge" | "package-owned" | "mixed"; + implementation: + | "private-core-bridge" + | "private-package-core-integrated" + | "package-owned" + | "mixed"; }; }; @@ -365,10 +369,13 @@ function countByOwner(records: readonly CompatDebtRecord[]): Record 0) { + if (memoryHostSdk.privatePackage && memoryHostSdk.sourceBridgeFiles.length > 0) { return "private-core-bridge"; } - if (!memoryHostSdk.privatePackage && memoryHostSdk.packageCoreReferenceFiles.length === 0) { + if (memoryHostSdk.privatePackage && memoryHostSdk.packageCoreReferenceFiles.length > 0) { + return "private-package-core-integrated"; + } + if (memoryHostSdk.packageCoreReferenceFiles.length === 0) { return "package-owned"; } return "mixed"; diff --git a/src/memory-host-sdk/engine-embeddings.ts b/src/memory-host-sdk/engine-embeddings.ts index 39f140e419b..a5b7e2b91a9 100644 --- a/src/memory-host-sdk/engine-embeddings.ts +++ b/src/memory-host-sdk/engine-embeddings.ts @@ -1,73 +1 @@ -// Real workspace contract for memory embedding providers and batch helpers. - -export { - getMemoryEmbeddingProvider, - listRegisteredMemoryEmbeddingProviders, - listMemoryEmbeddingProviders, - listRegisteredMemoryEmbeddingProviderAdapters, -} from "../plugins/memory-embedding-provider-runtime.js"; -export type { - MemoryEmbeddingBatchChunk, - MemoryEmbeddingBatchOptions, - MemoryEmbeddingProvider, - MemoryEmbeddingProviderAdapter, - MemoryEmbeddingProviderCreateOptions, - MemoryEmbeddingProviderCreateResult, - MemoryEmbeddingProviderRuntime, -} from "../plugins/memory-embedding-providers.js"; -export { createLocalEmbeddingProvider, DEFAULT_LOCAL_MODEL } from "./host/embeddings.js"; -export { extractBatchErrorMessage, formatUnavailableBatchError } from "./host/batch-error-utils.js"; -export { postJsonWithRetry } from "./host/batch-http.js"; -export { applyEmbeddingBatchOutputLine } from "./host/batch-output.js"; -export { - EMBEDDING_BATCH_ENDPOINT, - type EmbeddingBatchStatus, - type ProviderBatchOutputLine, -} from "./host/batch-provider-common.js"; -export { - buildEmbeddingBatchGroupOptions, - runEmbeddingBatchGroups, - type EmbeddingBatchExecutionParams, -} from "./host/batch-runner.js"; -export { - resolveBatchCompletionFromStatus, - resolveCompletedBatchResult, - throwIfBatchTerminalFailure, - type BatchCompletionResult, -} from "./host/batch-status.js"; -export { uploadBatchJsonlFile } from "./host/batch-upload.js"; -export { - buildBatchHeaders, - normalizeBatchBaseUrl, - type BatchHttpClientConfig, -} from "./host/batch-utils.js"; -export { enforceEmbeddingMaxInputTokens } from "./host/embedding-chunk-limits.js"; -export { - isMissingEmbeddingApiKeyError, - mapBatchEmbeddingsByIndex, - sanitizeEmbeddingCacheHeaders, -} from "./host/embedding-provider-adapter-utils.js"; -export { sanitizeAndNormalizeEmbedding } from "./host/embedding-vectors.js"; -export { debugEmbeddingsLog } from "./host/embeddings-debug.js"; -export { normalizeEmbeddingModelWithPrefixes } from "./host/embeddings-model-normalize.js"; -export { - resolveRemoteEmbeddingBearerClient, - type RemoteEmbeddingProviderId, -} from "./host/embeddings-remote-client.js"; -export { - createRemoteEmbeddingProvider, - resolveRemoteEmbeddingClient, - type RemoteEmbeddingClient, -} from "./host/embeddings-remote-provider.js"; -export { fetchRemoteEmbeddingVectors } from "./host/embeddings-remote-fetch.js"; -export { - estimateStructuredEmbeddingInputBytes, - estimateUtf8Bytes, -} from "./host/embedding-input-limits.js"; -export { hasNonTextEmbeddingParts, type EmbeddingInput } from "./host/embedding-inputs.js"; -export { buildRemoteBaseUrlPolicy, withRemoteHttpResponse } from "./host/remote-http.js"; -export { - buildCaseInsensitiveExtensionGlob, - classifyMemoryMultimodalPath, - getMemoryMultimodalExtensions, -} from "./host/multimodal.js"; +export * from "../../packages/memory-host-sdk/src/engine-embeddings.js"; diff --git a/src/memory-host-sdk/engine-foundation.ts b/src/memory-host-sdk/engine-foundation.ts index 3276b797d19..84cd0cbdc5e 100644 --- a/src/memory-host-sdk/engine-foundation.ts +++ b/src/memory-host-sdk/engine-foundation.ts @@ -1,48 +1 @@ -// Real workspace contract for memory engine foundation concerns. - -export { - resolveAgentContextLimits, - resolveAgentDir, - resolveAgentWorkspaceDir, - resolveDefaultAgentId, - resolveSessionAgentId, -} from "../agents/agent-scope.js"; -export { - resolveMemorySearchConfig, - resolveMemorySearchSyncConfig, - type ResolvedMemorySearchConfig, - type ResolvedMemorySearchSyncConfig, -} from "../agents/memory-search.js"; -export { parseDurationMs } from "../cli/parse-duration.js"; -export { loadConfig } from "../config/config.js"; -export { resolveStateDir } from "../config/paths.js"; -export { resolveSessionTranscriptsDirForAgent } from "../config/sessions/paths.js"; -export { - hasConfiguredSecretInput, - normalizeResolvedSecretInputString, -} from "../config/types.secrets.js"; -export { writeFileWithinRoot } from "../infra/fs-safe.js"; -export { createSubsystemLogger } from "../logging/subsystem.js"; -export { detectMime } from "../media/mime.js"; -export { resolveGlobalSingleton } from "../shared/global-singleton.js"; -export { onSessionTranscriptUpdate } from "../sessions/transcript-events.js"; -export { splitShellArgs } from "../utils/shell-argv.js"; -export { runTasksWithConcurrency } from "../utils/run-with-concurrency.js"; -export { - shortenHomeInString, - shortenHomePath, - resolveUserPath, - truncateUtf16Safe, -} from "../utils.js"; -export type { OpenClawConfig } from "../config/config.js"; -export type { SessionSendPolicyConfig } from "../config/types.base.js"; -export type { SecretInput } from "../config/types.secrets.js"; -export type { - MemoryBackend, - MemoryCitationsMode, - MemoryQmdConfig, - MemoryQmdIndexPath, - MemoryQmdMcporterConfig, - MemoryQmdSearchMode, -} from "../config/types.memory.js"; -export type { MemorySearchConfig } from "../config/types.tools.js"; +export * from "../../packages/memory-host-sdk/src/engine-foundation.js"; diff --git a/src/memory-host-sdk/engine-qmd.ts b/src/memory-host-sdk/engine-qmd.ts index 119dd49ef21..21a0be44873 100644 --- a/src/memory-host-sdk/engine-qmd.ts +++ b/src/memory-host-sdk/engine-qmd.ts @@ -1,26 +1 @@ -// Real workspace contract for QMD/session/query helpers used by the memory engine. - -export { extractKeywords, isQueryStopWordToken } from "./host/query-expansion.js"; -export { - buildSessionEntry, - listSessionFilesForAgent, - loadDreamingNarrativeTranscriptPathSetForAgent, - loadSessionTranscriptClassificationForAgent, - normalizeSessionTranscriptPathForComparison, - sessionPathForFile, - type BuildSessionEntryOptions, - type SessionFileEntry, - type SessionTranscriptClassification, -} from "./host/session-files.js"; -export { parseUsageCountedSessionIdFromFileName } from "../config/sessions/artifacts.js"; -export { parseQmdQueryJson, type QmdQueryResult } from "./host/qmd-query-parser.js"; -export { - deriveQmdScopeChannel, - deriveQmdScopeChatType, - isQmdScopeAllowed, -} from "./host/qmd-scope.js"; -export { - checkQmdBinaryAvailability, - resolveCliSpawnInvocation, - runCliCommand, -} from "./host/qmd-process.js"; +export * from "../../packages/memory-host-sdk/src/engine-qmd.js"; diff --git a/src/memory-host-sdk/engine-storage.ts b/src/memory-host-sdk/engine-storage.ts index 0159cff9605..ee3f3a4e410 100644 --- a/src/memory-host-sdk/engine-storage.ts +++ b/src/memory-host-sdk/engine-storage.ts @@ -1,48 +1 @@ -// Real workspace contract for memory engine storage/index helpers. - -export { - buildFileEntry, - buildMultimodalChunkForIndexing, - chunkMarkdown, - cosineSimilarity, - ensureDir, - hashText, - listMemoryFiles, - normalizeExtraMemoryPaths, - parseEmbedding, - remapChunkLines, - runWithConcurrency, - type MemoryChunk, - type MemoryFileEntry, -} from "./host/internal.js"; -export { readMemoryFile } from "./host/read-file.js"; -export { - buildMemoryReadResult, - buildMemoryReadResultFromSlice, - DEFAULT_MEMORY_READ_LINES, - DEFAULT_MEMORY_READ_MAX_CHARS, - type MemoryReadResult, -} from "./host/read-file-shared.js"; -export { resolveMemoryBackendConfig } from "./host/backend-config.js"; -export type { - ResolvedMemoryBackendConfig, - ResolvedQmdConfig, - ResolvedQmdMcporterConfig, -} from "./host/backend-config.js"; -export type { - MemoryEmbeddingProbeResult, - MemoryProviderStatus, - MemorySearchManager, - MemorySearchRuntimeDebug, - MemorySearchResult, - MemorySource, - MemorySyncProgressUpdate, -} from "./host/types.js"; -export { ensureMemoryIndexSchema } from "./host/memory-schema.js"; -export { loadSqliteVecExtension } from "./host/sqlite-vec.js"; -export { - closeMemorySqliteWalMaintenance, - configureMemorySqliteWalMaintenance, - requireNodeSqlite, -} from "./host/sqlite.js"; -export { isFileMissingError, statRegularFile } from "./host/fs-utils.js"; +export * from "../../packages/memory-host-sdk/src/engine-storage.js"; diff --git a/src/memory-host-sdk/engine.ts b/src/memory-host-sdk/engine.ts index a18fef9e8ba..8b5f8578ce5 100644 --- a/src/memory-host-sdk/engine.ts +++ b/src/memory-host-sdk/engine.ts @@ -1,7 +1 @@ -// Aggregate workspace contract for the memory engine surface. -// Keep focused subpaths preferred for new code. - -export * from "./engine-foundation.js"; -export * from "./engine-storage.js"; -export * from "./engine-embeddings.js"; -export * from "./engine-qmd.js"; +export * from "../../packages/memory-host-sdk/src/engine.js"; diff --git a/src/memory-host-sdk/host/batch-error-utils.ts b/src/memory-host-sdk/host/batch-error-utils.ts index 4bff412b6f7..28cffda44b3 100644 --- a/src/memory-host-sdk/host/batch-error-utils.ts +++ b/src/memory-host-sdk/host/batch-error-utils.ts @@ -1,33 +1 @@ -import { formatErrorMessage } from "../../infra/errors.js"; - -type BatchOutputErrorLike = { - error?: { message?: string }; - response?: { - body?: - | string - | { - error?: { message?: string }; - }; - }; -}; - -function getResponseErrorMessage(line: BatchOutputErrorLike | undefined): string | undefined { - const body = line?.response?.body; - if (typeof body === "string") { - return body || undefined; - } - if (!body || typeof body !== "object") { - return undefined; - } - return typeof body.error?.message === "string" ? body.error.message : undefined; -} - -export function extractBatchErrorMessage(lines: BatchOutputErrorLike[]): string | undefined { - const first = lines.find((line) => line.error?.message || getResponseErrorMessage(line)); - return first?.error?.message ?? getResponseErrorMessage(first); -} - -export function formatUnavailableBatchError(err: unknown): string | undefined { - const message = formatErrorMessage(err); - return message ? `error file unavailable: ${message}` : undefined; -} +export * from "../../../packages/memory-host-sdk/src/host/batch-error-utils.js"; diff --git a/src/memory-host-sdk/host/embeddings.ts b/src/memory-host-sdk/host/embeddings.ts index 375127a8fc6..a56516b225c 100644 --- a/src/memory-host-sdk/host/embeddings.ts +++ b/src/memory-host-sdk/host/embeddings.ts @@ -1,85 +1 @@ -import { normalizeOptionalString } from "../../shared/string-coerce.js"; -import { DEFAULT_LOCAL_MODEL } from "./embedding-defaults.js"; -import { sanitizeAndNormalizeEmbedding } from "./embedding-vectors.js"; -import type { EmbeddingProvider, EmbeddingProviderOptions } from "./embeddings.types.js"; -import { - importNodeLlamaCpp, - type Llama, - type LlamaEmbeddingContext, - type LlamaModel, -} from "./node-llama.js"; - -export type { - EmbeddingProvider, - EmbeddingProviderFallback, - EmbeddingProviderId, - EmbeddingProviderOptions, - EmbeddingProviderRequest, - GeminiTaskType, -} from "./embeddings.types.js"; - -export { DEFAULT_LOCAL_MODEL } from "./embedding-defaults.js"; - -export async function createLocalEmbeddingProvider( - options: EmbeddingProviderOptions, -): Promise { - const modelPath = normalizeOptionalString(options.local?.modelPath) || DEFAULT_LOCAL_MODEL; - const modelCacheDir = normalizeOptionalString(options.local?.modelCacheDir); - const contextSize: number | "auto" = options.local?.contextSize ?? 4096; - - // Lazy-load node-llama-cpp to keep startup light unless local is enabled. - const { getLlama, resolveModelFile, LlamaLogLevel } = await importNodeLlamaCpp(); - - let llama: Llama | null = null; - let embeddingModel: LlamaModel | null = null; - let embeddingContext: LlamaEmbeddingContext | null = null; - let initPromise: Promise | null = null; - - const ensureContext = async (): Promise => { - if (embeddingContext) { - return embeddingContext; - } - if (initPromise) { - return initPromise; - } - initPromise = (async () => { - try { - if (!llama) { - llama = await getLlama({ logLevel: LlamaLogLevel.error }); - } - if (!embeddingModel) { - const resolved = await resolveModelFile(modelPath, modelCacheDir || undefined); - embeddingModel = await llama.loadModel({ modelPath: resolved }); - } - if (!embeddingContext) { - embeddingContext = await embeddingModel.createEmbeddingContext({ contextSize }); - } - return embeddingContext; - } catch (err) { - initPromise = null; - throw err; - } - })(); - return initPromise; - }; - - return { - id: "local", - model: modelPath, - embedQuery: async (text) => { - const ctx = await ensureContext(); - const embedding = await ctx.getEmbeddingFor(text); - return sanitizeAndNormalizeEmbedding(Array.from(embedding.vector)); - }, - embedBatch: async (texts) => { - const ctx = await ensureContext(); - const embeddings = await Promise.all( - texts.map(async (text) => { - const embedding = await ctx.getEmbeddingFor(text); - return sanitizeAndNormalizeEmbedding(Array.from(embedding.vector)); - }), - ); - return embeddings; - }, - }; -} +export * from "../../../packages/memory-host-sdk/src/host/embeddings.js"; diff --git a/src/memory-host-sdk/host/read-file-shared.ts b/src/memory-host-sdk/host/read-file-shared.ts index e9fd4906408..08d16a8e3d4 100644 --- a/src/memory-host-sdk/host/read-file-shared.ts +++ b/src/memory-host-sdk/host/read-file-shared.ts @@ -1,114 +1 @@ -import type { MemoryReadResult } from "./types.js"; - -export const DEFAULT_MEMORY_READ_LINES = 120; -export const DEFAULT_MEMORY_READ_MAX_CHARS = 12_000; - -export type { MemoryReadResult } from "./types.js"; - -function buildContinuationNotice(params: { - nextFrom: number | undefined; - suggestReadFallback?: boolean; -}): string { - const base = - typeof params.nextFrom === "number" - ? `[More content available. Use from=${params.nextFrom} to continue.]` - : "[More content available. Requested excerpt exceeded the default maxChars budget.]"; - const fallback = params.suggestReadFallback - ? " If you need the full raw line, use read on the source file." - : ""; - return `\n\n${base.slice(0, -1)}${fallback}]`; -} - -function fitLinesToCharBudget(params: { lines: string[]; maxChars: number }): { - text: string; - includedLines: number; - hardTruncatedSingleLine: boolean; -} { - const { lines, maxChars } = params; - if (lines.length === 0) { - return { text: "", includedLines: 0, hardTruncatedSingleLine: false }; - } - - let includedLines = lines.length; - let text = lines.join("\n"); - while (includedLines > 1 && text.length > maxChars) { - includedLines -= 1; - text = lines.slice(0, includedLines).join("\n"); - } - - if (text.length <= maxChars) { - return { text, includedLines, hardTruncatedSingleLine: false }; - } - - return { - text: text.slice(0, maxChars), - includedLines: 1, - hardTruncatedSingleLine: true, - }; -} - -export function buildMemoryReadResultFromSlice(params: { - selectedLines: string[]; - relPath: string; - startLine: number; - moreSourceLinesRemain?: boolean; - maxChars?: number; - suggestReadFallback?: boolean; -}): MemoryReadResult { - const start = Math.max(1, params.startLine); - const fitted = fitLinesToCharBudget({ - lines: params.selectedLines, - maxChars: Math.max(1, params.maxChars ?? DEFAULT_MEMORY_READ_MAX_CHARS), - }); - const moreSourceLinesRemain = params.moreSourceLinesRemain ?? false; - const charCapTruncated = - fitted.hardTruncatedSingleLine || fitted.includedLines < params.selectedLines.length; - const nextFrom = - !fitted.hardTruncatedSingleLine && - (moreSourceLinesRemain || fitted.includedLines < params.selectedLines.length) - ? start + fitted.includedLines - : undefined; - const truncated = charCapTruncated || moreSourceLinesRemain; - const text = - truncated && fitted.text - ? `${fitted.text}${buildContinuationNotice({ - nextFrom, - suggestReadFallback: fitted.hardTruncatedSingleLine && params.suggestReadFallback, - })}` - : fitted.text; - return { - text, - path: params.relPath, - from: start, - lines: fitted.includedLines, - ...(truncated ? { truncated: true } : {}), - ...(typeof nextFrom === "number" ? { nextFrom } : {}), - }; -} - -export function buildMemoryReadResult(params: { - content: string; - relPath: string; - from?: number; - lines?: number; - defaultLines?: number; - maxChars?: number; - suggestReadFallback?: boolean; -}): MemoryReadResult { - const fileLines = params.content.split("\n"); - const start = Math.max(1, params.from ?? 1); - const requestedCount = Math.max( - 1, - params.lines ?? params.defaultLines ?? DEFAULT_MEMORY_READ_LINES, - ); - const selectedLines = fileLines.slice(start - 1, start - 1 + requestedCount); - const moreSourceLinesRemain = start - 1 + selectedLines.length < fileLines.length; - return buildMemoryReadResultFromSlice({ - selectedLines, - relPath: params.relPath, - startLine: start, - moreSourceLinesRemain, - maxChars: params.maxChars, - suggestReadFallback: params.suggestReadFallback, - }); -} +export * from "../../../packages/memory-host-sdk/src/host/read-file-shared.js"; diff --git a/src/memory-host-sdk/multimodal.ts b/src/memory-host-sdk/multimodal.ts index eb11867ac3a..36b50cbbf4b 100644 --- a/src/memory-host-sdk/multimodal.ts +++ b/src/memory-host-sdk/multimodal.ts @@ -1,5 +1 @@ -export { - isMemoryMultimodalEnabled, - normalizeMemoryMultimodalSettings, - type MemoryMultimodalSettings, -} from "./host/multimodal.js"; +export * from "../../packages/memory-host-sdk/src/multimodal.js"; diff --git a/src/memory-host-sdk/query.ts b/src/memory-host-sdk/query.ts index bb945afaa65..2a2ef6bbed4 100644 --- a/src/memory-host-sdk/query.ts +++ b/src/memory-host-sdk/query.ts @@ -1 +1 @@ -export { extractKeywords, isQueryStopWordToken } from "./host/query-expansion.js"; +export * from "../../packages/memory-host-sdk/src/query.js"; diff --git a/src/memory-host-sdk/runtime-cli.ts b/src/memory-host-sdk/runtime-cli.ts index 63b918ad6b8..69ea0ceaad6 100644 --- a/src/memory-host-sdk/runtime-cli.ts +++ b/src/memory-host-sdk/runtime-cli.ts @@ -1,11 +1 @@ -// Focused runtime contract for memory CLI/UI helpers. - -export { formatErrorMessage, withManager } from "../cli/cli-utils.js"; -export { formatHelpExamples } from "../cli/help-format.js"; -export { resolveCommandSecretRefsViaGateway } from "../cli/command-secret-gateway.js"; -export { withProgress, withProgressTotals } from "../cli/progress.js"; -export { defaultRuntime } from "../runtime.js"; -export { formatDocsLink } from "../terminal/links.js"; -export { colorize, isRich, theme } from "../terminal/theme.js"; -export { isVerbose, setVerbose } from "../globals.js"; -export { shortenHomeInString, shortenHomePath } from "../utils.js"; +export * from "../../packages/memory-host-sdk/src/runtime-cli.js"; diff --git a/src/memory-host-sdk/runtime-core.ts b/src/memory-host-sdk/runtime-core.ts index 8f2765d96b8..f45bde01cb0 100644 --- a/src/memory-host-sdk/runtime-core.ts +++ b/src/memory-host-sdk/runtime-core.ts @@ -1,41 +1 @@ -// Focused runtime contract for memory plugin config/state/helpers. - -export type { AnyAgentTool } from "../agents/tools/common.js"; -export { resolveCronStyleNow } from "../agents/current-time.js"; -export { DEFAULT_PI_COMPACTION_RESERVE_TOKENS_FLOOR } from "../agents/pi-settings.js"; -export { resolveDefaultAgentId, resolveSessionAgentId } from "../agents/agent-scope.js"; -export { resolveMemorySearchConfig } from "../agents/memory-search.js"; -export { - asToolParamsRecord, - jsonResult, - readNumberParam, - readStringParam, -} from "../agents/tools/common.js"; -export { SILENT_REPLY_TOKEN } from "../auto-reply/tokens.js"; -export { parseNonNegativeByteSize } from "../config/byte-size.js"; -export { - getRuntimeConfig, - /** @deprecated Use getRuntimeConfig(), or pass the already loaded config through the call path. */ - loadConfig, -} from "../config/config.js"; -export { resolveStateDir } from "../config/paths.js"; -export { resolveSessionTranscriptsDirForAgent } from "../config/sessions/paths.js"; -export { emptyPluginConfigSchema } from "../plugins/config-schema.js"; -export { - buildMemoryPromptSection as buildActiveMemoryPromptSection, - listActiveMemoryPublicArtifacts, - getMemoryCapabilityRegistration, -} from "../plugins/memory-state.js"; -export { parseAgentSessionKey } from "../routing/session-key.js"; -export type { OpenClawConfig } from "../config/config.js"; -export type { MemoryCitationsMode } from "../config/types.memory.js"; -export type { - MemoryFlushPlan, - MemoryFlushPlanResolver, - MemoryPluginCapability, - MemoryPluginPublicArtifact, - MemoryPluginPublicArtifactsProvider, - MemoryPluginRuntime, - MemoryPromptSectionBuilder, -} from "../plugins/memory-state.js"; -export type { OpenClawPluginApi } from "../plugins/types.js"; +export * from "../../packages/memory-host-sdk/src/runtime-core.js"; diff --git a/src/memory-host-sdk/runtime-files.ts b/src/memory-host-sdk/runtime-files.ts index 77f0de0e1d2..f4daa70a22d 100644 --- a/src/memory-host-sdk/runtime-files.ts +++ b/src/memory-host-sdk/runtime-files.ts @@ -1,10 +1 @@ -// Focused runtime contract for memory file/backend access. - -export { listMemoryFiles, normalizeExtraMemoryPaths } from "./host/internal.js"; -export { readAgentMemoryFile } from "./host/read-file.js"; -export { resolveMemoryBackendConfig } from "./host/backend-config.js"; -export type { - MemorySearchManager, - MemorySearchRuntimeDebug, - MemorySearchResult, -} from "./host/types.js"; +export * from "../../packages/memory-host-sdk/src/runtime-files.js"; diff --git a/src/memory-host-sdk/runtime.ts b/src/memory-host-sdk/runtime.ts index 6e152ea0dcb..69f4648dc4a 100644 --- a/src/memory-host-sdk/runtime.ts +++ b/src/memory-host-sdk/runtime.ts @@ -1,6 +1 @@ -// Aggregate workspace contract for memory runtime/helper seams. -// Keep focused subpaths preferred for new code. - -export * from "./runtime-core.js"; -export * from "./runtime-cli.js"; -export * from "./runtime-files.js"; +export * from "../../packages/memory-host-sdk/src/runtime.js"; diff --git a/src/memory-host-sdk/secret.ts b/src/memory-host-sdk/secret.ts index b2b6b94ab47..f293730b357 100644 --- a/src/memory-host-sdk/secret.ts +++ b/src/memory-host-sdk/secret.ts @@ -1,4 +1 @@ -export { - hasConfiguredMemorySecretInput, - resolveMemorySecretInputString, -} from "./host/secret-input.js"; +export * from "../../packages/memory-host-sdk/src/secret.js"; diff --git a/src/memory-host-sdk/status.ts b/src/memory-host-sdk/status.ts index dc718abd96b..704b37737b4 100644 --- a/src/memory-host-sdk/status.ts +++ b/src/memory-host-sdk/status.ts @@ -1,6 +1 @@ -export { - resolveMemoryCacheSummary, - resolveMemoryFtsState, - resolveMemoryVectorState, - type Tone, -} from "./host/status-format.js"; +export * from "../../packages/memory-host-sdk/src/status.js"; diff --git a/src/plugins/contracts/extension-package-project-boundaries.test.ts b/src/plugins/contracts/extension-package-project-boundaries.test.ts index 2a227271a5d..096b9118470 100644 --- a/src/plugins/contracts/extension-package-project-boundaries.test.ts +++ b/src/plugins/contracts/extension-package-project-boundaries.test.ts @@ -129,9 +129,6 @@ describe("opt-in extension package boundaries", () => { expect(packageJson.exports?.["./acp-runtime"]?.types).toBe( "./dist/src/plugin-sdk/acp-runtime.d.ts", ); - expect(packageJson.exports?.["./browser-config"]?.types).toBe( - "./dist/src/plugin-sdk/browser-config.d.ts", - ); expect(packageJson.exports?.["./channel-secret-runtime"]?.types).toBe( "./dist/src/plugin-sdk/channel-secret-runtime.d.ts", ); @@ -193,7 +190,7 @@ describe("opt-in extension package boundaries", () => { ); }); - it("keeps memory-host-sdk as a private package bridge over the core-owned implementation", () => { + it("keeps memory-host-sdk as a private package-owned contract surface", () => { const packageJson = readJsonFile("packages/memory-host-sdk/package.json"); const packageExports = packageJson.exports as unknown as Record; @@ -210,9 +207,7 @@ describe("opt-in extension package boundaries", () => { throw new Error(`Missing memory-host-sdk export target for ${exportPath}`); } const source = readFileSync(resolve(REPO_ROOT, "packages/memory-host-sdk", target), "utf8"); - expect(source.trim(), target).toBe( - `export * from "../../../src/memory-host-sdk/${exportPath.slice(2)}.js";`, - ); + expect(source, target).not.toContain("src/memory-host-sdk/"); } }); });