/** * OpenClaw Memory (LanceDB) Plugin * * Long-term memory with vector search for AI conversations. * Uses LanceDB for storage and OpenAI for embeddings. * Provides seamless auto-recall and auto-capture via lifecycle hooks. */ import { Buffer } from "node:buffer"; import { randomUUID } from "node:crypto"; import type * as LanceDB from "@lancedb/lancedb"; import type { AgentToolResult } from "openclaw/plugin-sdk/agent-core"; import { optionalFiniteNumberSchema, optionalPositiveIntegerSchema, } from "openclaw/plugin-sdk/channel-actions"; import { BUNDLED_CHAT_CHANNEL_ENVELOPE_PREFIXES } from "openclaw/plugin-sdk/chat-channel-ids"; import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; import { expectDefined } from "openclaw/plugin-sdk/expect-runtime"; import { createLazyRuntimeModule } from "openclaw/plugin-sdk/lazy-runtime"; import type { MemoryEmbeddingProvider } from "openclaw/plugin-sdk/memory-core-host-engine-embeddings"; import { MESSAGE_TOOL_DELIVERY_HINTS } from "openclaw/plugin-sdk/message-tool-delivery-hints"; import { parseStrictPositiveInteger, resolveTimerTimeoutMs, } from "openclaw/plugin-sdk/number-runtime"; import { readFiniteNumberParam, readPositiveIntegerParam } from "openclaw/plugin-sdk/param-readers"; import { resolveLivePluginConfigObject } from "openclaw/plugin-sdk/plugin-config-runtime"; import { ensureGlobalUndiciEnvProxyDispatcher } from "openclaw/plugin-sdk/runtime-env"; import { asOptionalRecord as asRecord, normalizeLowercaseStringOrEmpty, } from "openclaw/plugin-sdk/string-coerce-runtime"; import { truncateUtf16Safe } from "openclaw/plugin-sdk/text-utility-runtime"; import { Type } from "typebox"; import { definePluginEntry, type OpenClawPluginApi } from "./api.js"; import { DEFAULT_CAPTURE_MAX_CHARS, DEFAULT_RECALL_MAX_CHARS, MEMORY_CATEGORIES, type MemoryConfig, type MemoryCategory, memoryConfigSchema, vectorDimsForModel, } from "./config.js"; import { loadLanceDbModule } from "./lancedb-runtime.js"; // ============================================================================ // Types // ============================================================================ type MemoryEntry = { id: string; text: string; vector: number[]; importance: number; category: MemoryCategory; createdAt: number; }; type MemoryListEntry = Omit; type MemoryListOptions = { orderByCreatedAt?: boolean; }; type MemorySearchResult = { entry: MemoryEntry; score: number; }; type AutoCaptureCursor = { nextIndex: number; lastMessageFingerprint?: string; }; type OpenAiEmbeddingClient = { post( path: string, options: { body: unknown; timeout?: number; maxRetries?: number }, ): Promise; }; const loadOpenAiModule = createLazyRuntimeModule(() => import("openai")); const loadMemoryEmbeddingProviderModule = createLazyRuntimeModule( () => import("openclaw/plugin-sdk/memory-core-host-engine-embeddings"), ); const loadMemoryHostCoreModule = createLazyRuntimeModule( () => import("openclaw/plugin-sdk/memory-host-core"), ); function extractUserTextContent(message: unknown): string[] { const msgObj = asRecord(message); if (!msgObj || msgObj.role !== "user") { return []; } const content = msgObj.content; if (typeof content === "string") { return [content]; } if (!Array.isArray(content)) { return []; } const texts: string[] = []; for (const block of content) { const blockObj = asRecord(block); if (blockObj?.type === "text" && typeof blockObj.text === "string") { texts.push(blockObj.text); } } return texts; } function extractLatestUserText(messages: unknown[]): string | undefined { for (let index = messages.length - 1; index >= 0; index--) { const text = extractUserTextContent(messages[index]).join("\n").trim(); if (text) { return text; } } return undefined; } export function normalizeRecallQuery( text: string, maxChars: number = DEFAULT_RECALL_MAX_CHARS, ): string { const normalized = text.replace(/\s+/g, " ").trim(); const limit = normalizeMaxChars(maxChars, DEFAULT_RECALL_MAX_CHARS); return normalized.length > limit ? truncateUtf16Safe(normalized, limit).trimEnd() : normalized; } function normalizeMaxChars(value: number | undefined, fallback: number): number { return typeof value === "number" && Number.isFinite(value) ? Math.max(0, Math.floor(value)) : fallback; } function messageFingerprint(message: unknown): string { const msgObj = asRecord(message); if (!msgObj) { return `${typeof message}:${String(message)}`; } try { return JSON.stringify({ role: msgObj.role, content: msgObj.content, }); } catch { return `${String(msgObj.role)}:${String(msgObj.content)}`; } } function resolveAutoCaptureStartIndex( messages: unknown[], cursor: AutoCaptureCursor | undefined, ): number { if (!cursor) { return 0; } if (cursor.lastMessageFingerprint && cursor.nextIndex > 0) { for (let index = messages.length - 1; index >= 0; index--) { if (messageFingerprint(messages[index]) === cursor.lastMessageFingerprint) { return index + 1; } } return 0; } if (cursor.nextIndex <= messages.length) { return cursor.nextIndex; } return 0; } // ============================================================================ // LanceDB Provider // ============================================================================ const TABLE_NAME = "memories"; const DEFAULT_AUTO_RECALL_TIMEOUT_MS = 15_000; const DEFAULT_TOOL_RECALL_TIMEOUT_MS = 15_000; const DEFAULT_TOOL_RECALL_COOLDOWN_MS = 60_000; const DEFAULT_TOOL_RECALL_OVERFETCH_EXTRA = 10; // Auto-recall over-fetches from the vector store, then filters envelope sludge // (contaminated memories that slipped past capture gating), then caps the // surviving results before prompt injection. The over-fetch limit must stay a // few multiples above the cap so a small number of contaminated top-K hits // still leave enough clean memories to surface; the cap mirrors prior // behavior of "at most 3 injected memories" so prompt budget impact stays // bounded. const DEFAULT_AUTO_RECALL_OVERFETCH_LIMIT = 10; const DEFAULT_AUTO_RECALL_RESULT_CAP = 3; const DUPLICATE_SEARCH_LIMIT = 5; function parsePositiveIntegerOption(value: string | undefined, flag: string): number | undefined { if (value === undefined) { return undefined; } const parsed = parseStrictPositiveInteger(value); if (parsed === undefined) { throw new Error(`${flag} must be a positive integer`); } return parsed; } class MemoryDB { private db: LanceDB.Connection | null = null; private table: LanceDB.Table | null = null; private initPromise: Promise | null = null; constructor( private readonly dbPath: string, private readonly vectorDim: number, private readonly storageOptions?: Record, ) {} private async ensureInitialized(): Promise { if (this.table) { return; } if (this.initPromise) { return this.initPromise; } this.initPromise = this.doInitialize().catch((error: unknown) => { this.initPromise = null; throw error; }); return this.initPromise; } private async doInitialize(): Promise { const lancedb = await loadLanceDbModule(); const connectionOptions: LanceDB.ConnectionOptions = this.storageOptions ? { storageOptions: this.storageOptions } : {}; this.db = await lancedb.connect(this.dbPath, connectionOptions); const tables = await this.db.tableNames(); if (tables.includes(TABLE_NAME)) { this.table = await this.db.openTable(TABLE_NAME); } else { this.table = await this.db.createTable(TABLE_NAME, [ { id: "__schema__", text: "", vector: Array.from({ length: this.vectorDim }).fill(0), importance: 0, category: "other", createdAt: 0, }, ]); await this.table.delete('id = "__schema__"'); } } async store(entry: Omit): Promise { await this.ensureInitialized(); const fullEntry: MemoryEntry = { ...entry, id: randomUUID(), createdAt: Date.now(), }; await this.table!.add([fullEntry]); return fullEntry; } async search(vector: number[], limit = 5, minScore = 0.5): Promise { await this.ensureInitialized(); const results = await this.table!.vectorSearch(vector).limit(limit).toArray(); // LanceDB uses L2 distance by default; convert to similarity score const mapped = results.map((row) => { const distance = row["_distance"] ?? 0; // Use inverse for a 0-1 range: sim = 1 / (1 + d) const score = 1 / (1 + distance); return { entry: { id: row.id as string, text: row.text as string, vector: row.vector as number[], importance: row.importance as number, category: row.category as MemoryEntry["category"], createdAt: row.createdAt as number, }, score, }; }); return mapped.filter((r) => r.score >= minScore); } async list(limit?: number, options: MemoryListOptions = {}): Promise { await this.ensureInitialized(); let query = this.table!.query().select(["id", "text", "importance", "category", "createdAt"]); // Push limit to LanceDB only when we don't need to sort in-memory. if (!options.orderByCreatedAt && limit !== undefined) { query = query.limit(limit); } const rows = await query.toArray(); const entries = rows.map((row) => ({ id: row.id as string, text: row.text as string, importance: row.importance as number, category: row.category as MemoryEntry["category"], createdAt: row.createdAt as number, })); if (options.orderByCreatedAt) { entries.sort((a, b) => b.createdAt - a.createdAt); } return limit === undefined ? entries : entries.slice(0, limit); } async delete(id: string): Promise { await this.ensureInitialized(); // Validate UUID format to prevent injection const uuidRegex = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; if (!uuidRegex.test(id)) { throw new Error(`Invalid memory ID format: ${id}`); } await this.table!.delete(`id = '${id}'`); return true; } async count(): Promise { await this.ensureInitialized(); return this.table!.countRows(); } async getTable(): Promise { await this.ensureInitialized(); return this.table!; } } // ============================================================================ // Embeddings // ============================================================================ type Embeddings = { embed(text: string, options?: { timeoutMs?: number }): Promise; }; class OpenAiCompatibleEmbeddings implements Embeddings { private clientPromise: Promise; constructor( apiKey: string, private model: string, baseUrl?: string, private dimensions?: number, ) { this.clientPromise = loadOpenAiModule().then( ({ default: OpenAI }) => new OpenAI({ apiKey, baseURL: baseUrl }) as OpenAiEmbeddingClient, ); } async embed(text: string, options?: { timeoutMs?: number }): Promise { const params: Record = { model: this.model, input: text, }; if (this.dimensions) { params.dimensions = this.dimensions; } ensureGlobalUndiciEnvProxyDispatcher(); // The OpenAI SDK's embeddings helper injects encoding_format=base64 when // omitted, then decodes the response. Several compatible providers either // reject encoding_format or always return float arrays, so use the generic // transport and normalize the response ourselves. const response = await ( await this.clientPromise ).post("/embeddings", { body: params, ...(options?.timeoutMs ? { timeout: options.timeoutMs, maxRetries: 0 } : {}), }); return normalizeEmbeddingVector(response.data?.[0]?.embedding); } } class ProviderAdapterEmbeddings implements Embeddings { private providerPromise: Promise | undefined; constructor( private api: OpenClawPluginApi, private embedding: MemoryConfig["embedding"], ) {} private getProvider(): Promise { // Auth profiles and local providers can be repaired while the Gateway stays up. // Cache successful setup, but retry after failed provider discovery/auth. this.providerPromise ??= this.createProvider().catch((err: unknown) => { this.providerPromise = undefined; throw err; }); return this.providerPromise; } private async createProvider(): Promise { const cfg = (this.api.runtime.config?.current?.() ?? this.api.config) as OpenClawConfig; const providerId = this.embedding.provider; const { getMemoryEmbeddingProvider } = await loadMemoryEmbeddingProviderModule(); const adapter = getMemoryEmbeddingProvider(providerId, cfg); if (!adapter) { throw new Error(`Unknown memory embedding provider: ${providerId}`); } const { resolveDefaultAgentId } = await loadMemoryHostCoreModule(); const defaultAgentId = resolveDefaultAgentId(cfg); const agentDir = this.api.runtime.agent.resolveAgentDir(cfg, defaultAgentId); const remote = this.embedding.apiKey || this.embedding.baseUrl ? { ...(this.embedding.apiKey ? { apiKey: this.embedding.apiKey } : {}), ...(this.embedding.baseUrl ? { baseUrl: this.embedding.baseUrl } : {}), } : undefined; const result = await adapter.create({ config: cfg, agentDir, provider: providerId, fallback: "none", model: this.embedding.model, ...(remote ? { remote } : {}), ...(typeof this.embedding.dimensions === "number" ? { outputDimensionality: this.embedding.dimensions } : {}), }); if (!result.provider) { throw new Error(`Memory embedding provider ${providerId} is unavailable.`); } return result.provider; } async embed(text: string, options?: { timeoutMs?: number }): Promise { const provider = await this.getProvider(); if (!options?.timeoutMs) { return await provider.embedQuery(text); } const controller = new AbortController(); let timer: ReturnType | undefined; try { timer = setTimeout( () => controller.abort(new Error("memory-lancedb embedding timed out")), resolveTimerTimeoutMs(options.timeoutMs, 1), ); timer.unref?.(); return await provider.embedQuery(text, { signal: controller.signal }); } finally { if (timer) { clearTimeout(timer); } } } } async function runWithTimeout(params: { timeoutMs: number; task: () => Promise; }): Promise<{ status: "ok"; value: T } | { status: "timeout" }> { let timeout: ReturnType | undefined; const TIMEOUT = Symbol("timeout"); const timeoutPromise = new Promise((resolve) => { timeout = setTimeout(() => resolve(TIMEOUT), resolveTimerTimeoutMs(params.timeoutMs, 1)); timeout.unref?.(); }); const taskPromise = params.task(); taskPromise.catch(() => undefined); try { const result = await Promise.race([taskPromise, timeoutPromise]); if (result === TIMEOUT) { return { status: "timeout" }; } return { status: "ok", value: result }; } finally { if (timeout) { clearTimeout(timeout); } } } function formatMemoryRecallError(error: unknown): string { return error instanceof Error ? error.message : String(error); } function buildMemoryRecallUnavailableResult(error: string): AgentToolResult<{ count: number; disabled: true; unavailable: true; error: string; }> { return { content: [{ type: "text", text: "Memory recall is unavailable right now." }], details: { count: 0, disabled: true, unavailable: true, error, }, }; } class MemoryRecallEmbeddingError extends Error { constructor(readonly originalError: unknown) { super(formatMemoryRecallError(originalError)); this.name = "MemoryRecallEmbeddingError"; } } export const testing = { runWithTimeout, } as const; function createEmbeddings(api: OpenClawPluginApi, cfg: MemoryConfig): Embeddings { const { provider, model, dimensions, apiKey, baseUrl } = cfg.embedding; if (provider === "openai" && apiKey) { return new OpenAiCompatibleEmbeddings(apiKey, model, baseUrl, dimensions); } return new ProviderAdapterEmbeddings(api, cfg.embedding); } type EmbeddingCreateResponse = { data?: Array<{ embedding?: unknown; }>; }; export function normalizeEmbeddingVector(value: unknown): number[] { if (Array.isArray(value)) { if (!value.every((item) => typeof item === "number" && Number.isFinite(item))) { throw new Error("Embedding response contains non-numeric values"); } return value; } if (typeof value === "string") { const bytes = Buffer.from(value, "base64"); if (bytes.byteLength % Float32Array.BYTES_PER_ELEMENT !== 0) { throw new Error("Base64 embedding response has invalid byte length"); } const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); const floats: number[] = []; for (let offset = 0; offset < bytes.byteLength; offset += Float32Array.BYTES_PER_ELEMENT) { floats.push(view.getFloat32(offset, true)); } return floats; } throw new Error("Embedding response is missing a vector"); } // ============================================================================ // Rule-based capture filter // ============================================================================ const MEMORY_TRIGGERS = [ /zapamatuj si|pamatuj|remember/i, /preferuji|radši|nechci|prefer/i, /rozhodli jsme|budeme používat/i, /\+\d{10,}/, /[\w.-]+@[\w.-]+\.\w+/, /můj\s+\w+\s+je|je\s+můj/i, /my\s+\w+\s+is|is\s+my/i, /i (like|prefer|hate|love|want|need)/i, /always|never|important/i, /记住|記住|记下|記下|我(喜欢|喜歡|偏好|讨厌|討厭|爱|愛|想要|需要)|我的.*是|以后都用这个|以後都用這個|决定|決定|总是|總是|从不|永远|永遠|重要/i, /覚えて|記憶して|忘れないで|私は.*(好き|嫌い|必要|欲しい)|好み|いつも|絶対|重要/i, /기억해|기억해줘|잊지 마|나는.*(좋아|싫어|원해|필요)|내.*(이야|입니다)|항상|절대|중요/i, ]; const CJK_TEXT = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]/u; const PROMPT_INJECTION_PATTERNS = [ /\b(ignore|disregard|forget|override)\b.{0,60}\b(all|any|previous|above|prior|earlier|system|developer)\b.{0,30}\binstructions?\b/i, /do not follow (the )?(system|developer)/i, /system prompt/i, /developer message/i, /<\s*(system|assistant|developer|tool|function|relevant-memories)\b/i, /\b(run|execute|call|invoke)\b.{0,40}\b(tool|command)\b/i, ]; const PROMPT_ESCAPE_MAP: Record = { "&": "&", "<": "<", ">": ">", '"': """, "'": "'", }; export function looksLikePromptInjection(text: string): boolean { const normalized = text.replace(/\s+/g, " ").trim(); if (!normalized) { return false; } return PROMPT_INJECTION_PATTERNS.some((pattern) => pattern.test(normalized)); } /** * Pattern matching [media attached: ...] and [media attached N/M: ...] annotations. * These are written by the Gateway's claim-check offload when a user sends an image. * When a message containing such an annotation is stored as a long-term memory and * later recalled, the verbatim text must NOT be re-interpreted as a live media * reference by detectImageReferences() because that makes old memories look like * fresh media attachments. */ const MEDIA_ATTACHED_PATTERN = /\[media attached(?:\s+\d+\/\d+)?:[^\]]*\]/gi; /** Same pattern without the `g` flag, safe for repeated `.test()` calls. */ const MEDIA_ATTACHED_PATTERN_TEST = /\[media attached(?:\s+\d+\/\d+)?:[^\]]*\]/i; export function escapeMemoryForPrompt(text: string): string { return stripMediaAttachedAnnotations(text).replace( /[&<>"']/g, (char) => PROMPT_ESCAPE_MAP[char] ?? char, ); } function stripMediaAttachedAnnotations(text: string): string { // Strip [media attached: ...] annotations before HTML-escaping so that // detectImageReferences() cannot re-parse them as live media references. const hadMedia = MEDIA_ATTACHED_PATTERN_TEST.test(text); let stripped = text.replace(MEDIA_ATTACHED_PATTERN, ""); // Collapse runs of spaces/tabs only when media was actually stripped; otherwise // intentional multi-space formatting (tabular data, indented code references, // etc.) is preserved. Newlines are deliberately excluded from the collapse so // multi-line memories keep their line structure after media removal. if (hadMedia) { stripped = stripped.replace(/[ \t]{2,}/g, " ").trim(); } return stripped; } function sanitizeRecallMemoryText(text: string): string | null { const stripped = stripMediaAttachedAnnotations(text); if (!stripped.trim()) { return null; } return looksLikeEnvelopeSludge(stripped) ? null : stripped; } async function findCleanDuplicateMemory( db: { search(vector: number[], limit?: number, minScore?: number): Promise; }, vector: number[], ): Promise { const existing = await db.search(vector, DUPLICATE_SEARCH_LIMIT, 0.95); return existing.find((result) => sanitizeRecallMemoryText(result.entry.text) !== null); } function cleanMemorySearchResults(results: MemorySearchResult[]): Array<{ result: MemorySearchResult; text: string; }> { return results.flatMap((result) => { const text = sanitizeRecallMemoryText(result.entry.text); return text ? [{ result, text }] : []; }); } // ============================================================================ // Envelope / transport metadata contamination detection // ============================================================================ /** * Explicit sentinel strings used by `sanitizeForMemoryCapture` to locate and * surgically strip individual blocks. Canonical source: * src/auto-reply/reply/strip-inbound-meta.ts. Duplicated here because * extensions must not import core internals. * * NOTE: `looksLikeEnvelopeSludge` deliberately uses the broader * `INBOUND_META_LABEL_RE` below instead of this list, because * `buildInboundUserContextPrefix` in core also injects label variants such as * `Location (untrusted metadata):`, `Structured object (untrusted metadata):`, * and arbitrary ` (untrusted metadata):` blocks (from * `UntrustedStructuredContext`). Detection must stay forward-compatible with * those without bloating this explicit list every time core adds a new label. */ const INBOUND_META_SENTINELS = [ "Conversation info (untrusted metadata):", "Sender (untrusted metadata):", "Thread starter (untrusted, for context):", "Reply target of current user message (untrusted, for context):", "Replied message (untrusted, for context):", "Forwarded message context (untrusted metadata):", "Conversation context (untrusted, chronological, selected for current message):", "Current local chat window (untrusted, chronological, before current message):", "Nearby reply target window (untrusted, chronological, around replied-to message):", "Chat history since last reply (untrusted, for context):", ] as const; const INBOUND_META_SENTINEL_LINE_RE = new RegExp( `^(?:${INBOUND_META_SENTINELS.map((sentinel) => sentinel.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"), ).join("|")})[^\\n]*$`, "m", ); const MESSAGE_TOOL_DELIVERY_HINT_RE = new RegExp( `^\\s*(?:${MESSAGE_TOOL_DELIVERY_HINTS.map((hint) => hint.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"), ).join("|")})\\s*$`, "m", ); const HISTORY_CONTEXT_MARKER = "[Chat messages since your last reply - for context]"; const CURRENT_MESSAGE_MARKER = "[Current message - respond to this]"; const HISTORY_CONTEXT_MARKERS = [ HISTORY_CONTEXT_MARKER, "[Chat messages since your last reply \u2014 CONTEXT ONLY]", "[Merged earlier messages \u2014 CONTEXT ONLY]", ] as const; const CURRENT_MESSAGE_MARKERS = [ CURRENT_MESSAGE_MARKER, "[CURRENT MESSAGE \u2014 reply to this]", "[CURRENT MESSAGE \u2014 reply using the context above]", ] as const; const ACTIVE_TURN_RECOVERY_RE = /active-turn-recovery/i; /** * Line-anchored pattern matching any inbound-meta block header injected by * `buildInboundUserContextPrefix`. Covers both `(untrusted metadata):` labels * (Conversation info, Sender, Forwarded, Location, Structured object, plus any * future `