From df918c4de530f3880b1169b33f70d79167c7b8cf Mon Sep 17 00:00:00 2001 From: Xin Sun Date: Wed, 15 Apr 2026 16:07:49 +0800 Subject: [PATCH] feat(memory-lancedb): add cloud storage support to memory-lancedb (#63502) * feat(memory-lancedb): add cloud storage support to memory-lancedb - Pass storageOptions to LanceDB connection # Conflicts: # extensions/memory-lancedb/index.ts # Conflicts: # extensions/memory-lancedb/config.ts * support env var * make storageOptions sensitive --- extensions/memory-lancedb/config.ts | 28 +++++- extensions/memory-lancedb/index.test.ts | 90 +++++++++++++++++++ extensions/memory-lancedb/index.ts | 11 ++- .../memory-lancedb/openclaw.plugin.json | 11 +++ 4 files changed, 136 insertions(+), 4 deletions(-) diff --git a/extensions/memory-lancedb/config.ts b/extensions/memory-lancedb/config.ts index 29ae376a54f..c834beaf1c3 100644 --- a/extensions/memory-lancedb/config.ts +++ b/extensions/memory-lancedb/config.ts @@ -15,6 +15,7 @@ export type MemoryConfig = { autoCapture?: boolean; autoRecall?: boolean; captureMaxChars?: number; + storageOptions?: Record; }; export const MEMORY_CATEGORIES = ["preference", "fact", "decision", "entity", "other"] as const; @@ -98,7 +99,7 @@ export const memoryConfigSchema = { const cfg = value as Record; assertAllowedKeys( cfg, - ["embedding", "dreaming", "dbPath", "autoCapture", "autoRecall", "captureMaxChars"], + ["embedding", "dreaming", "dbPath", "autoCapture", "autoRecall", "captureMaxChars", "storageOptions"], "memory config", ); @@ -128,6 +129,23 @@ export const memoryConfigSchema = { throw new Error("dreaming config must be an object"); })(); + // Parse storageOptions (object with string values) + let storageOptions: Record | undefined; + const storageOpts = cfg.storageOptions as Record | undefined; + if (storageOpts !== undefined && storageOpts !== null) { + if (!storageOpts || typeof storageOpts !== "object" || Array.isArray(storageOpts)) { + throw new Error("storageOptions must be an object"); + } + storageOptions = {}; + // Validate all values are strings + for (const [key, value] of Object.entries(storageOpts)) { + if (typeof value !== "string") { + throw new Error(`storageOptions.${key} must be a string`); + } + storageOptions[key] = resolveEnvVars(value); + } + } + return { embedding: { provider: "openai", @@ -142,6 +160,7 @@ export const memoryConfigSchema = { autoCapture: cfg.autoCapture === true, autoRecall: cfg.autoRecall !== false, captureMaxChars: captureMaxChars ?? DEFAULT_CAPTURE_MAX_CHARS, + ...(storageOptions ? { storageOptions } : {}), }; }, uiHints: { @@ -172,6 +191,7 @@ export const memoryConfigSchema = { label: "Database Path", placeholder: "~/.openclaw/memory/lancedb", advanced: true, + help: "Local filesystem path or cloud storage URI (s3://, gs://) for LanceDB database", }, autoCapture: { label: "Auto-Capture", @@ -187,5 +207,11 @@ export const memoryConfigSchema = { advanced: true, placeholder: String(DEFAULT_CAPTURE_MAX_CHARS), }, + storageOptions: { + label: "Storage Options", + sensitive: true, + advanced: true, + help: "Storage configuration options (access_key, secret_key, endpoint, etc.); supports ${ENV_VAR} values", + }, }, }; diff --git a/extensions/memory-lancedb/index.test.ts b/extensions/memory-lancedb/index.test.ts index c02c499386a..c0aa2cbf62d 100644 --- a/extensions/memory-lancedb/index.test.ts +++ b/extensions/memory-lancedb/index.test.ts @@ -31,6 +31,7 @@ type MemoryPluginTestConfig = { captureMaxChars?: number; autoCapture?: boolean; autoRecall?: boolean; + storageOptions?: Record; }; const TEST_RUNTIME_MANIFEST = { @@ -279,6 +280,95 @@ describe("memory plugin e2e", () => { } }); + test("config schema accepts storageOptions with string values", async () => { + const { default: memoryPlugin } = await import("./index.js"); + + const config = memoryPlugin.configSchema?.parse?.({ + embedding: { + apiKey: OPENAI_API_KEY, + model: "text-embedding-3-small", + }, + dbPath: getDbPath(), + storageOptions: { + region: "us-west-2", + access_key: "test-key", + secret_key: "test-secret", + }, + }) as MemoryPluginTestConfig | undefined; + + expect(config?.storageOptions).toEqual({ + region: "us-west-2", + access_key: "test-key", + secret_key: "test-secret", + }); + }); + + test("config schema resolves env vars in storageOptions", async () => { + const { default: memoryPlugin } = await import("./index.js"); + process.env.TEST_MEMORY_STORAGE_ACCESS_KEY = "env-access"; + process.env.TEST_MEMORY_STORAGE_SECRET_KEY = "env-secret"; + + try { + const config = memoryPlugin.configSchema?.parse?.({ + embedding: { + apiKey: OPENAI_API_KEY, + model: "text-embedding-3-small", + }, + dbPath: getDbPath(), + storageOptions: { + region: "us-west-2", + access_key: "${TEST_MEMORY_STORAGE_ACCESS_KEY}", + secret_key: "${TEST_MEMORY_STORAGE_SECRET_KEY}", + }, + }) as MemoryPluginTestConfig | undefined; + + expect(config?.storageOptions).toEqual({ + region: "us-west-2", + access_key: "env-access", + secret_key: "env-secret", + }); + } finally { + delete process.env.TEST_MEMORY_STORAGE_ACCESS_KEY; + delete process.env.TEST_MEMORY_STORAGE_SECRET_KEY; + } + }); + + test("config schema rejects missing env vars in storageOptions", async () => { + const { default: memoryPlugin } = await import("./index.js"); + delete process.env.TEST_MEMORY_STORAGE_MISSING; + + expect(() => { + memoryPlugin.configSchema?.parse?.({ + embedding: { + apiKey: OPENAI_API_KEY, + model: "text-embedding-3-small", + }, + dbPath: getDbPath(), + storageOptions: { + secret_key: "${TEST_MEMORY_STORAGE_MISSING}", + }, + }); + }).toThrow("Environment variable TEST_MEMORY_STORAGE_MISSING is not set"); + }); + + test("config schema rejects storageOptions with non-string values", async () => { + const { default: memoryPlugin } = await import("./index.js"); + + expect(() => { + memoryPlugin.configSchema?.parse?.({ + embedding: { + apiKey: OPENAI_API_KEY, + model: "text-embedding-3-small", + }, + dbPath: getDbPath(), + storageOptions: { + region: "us-west-2", + timeout: 30, // number, should fail + }, + }); + }).toThrow("storageOptions.timeout must be a string"); + }); + test("shouldCapture applies real capture rules", async () => { expect(shouldCapture("I prefer dark mode")).toBe(true); expect(shouldCapture("Remember that my name is John")).toBe(true); diff --git a/extensions/memory-lancedb/index.ts b/extensions/memory-lancedb/index.ts index e73835cddbb..82fac0fd7b1 100644 --- a/extensions/memory-lancedb/index.ts +++ b/extensions/memory-lancedb/index.ts @@ -56,6 +56,7 @@ class MemoryDB { constructor( private readonly dbPath: string, private readonly vectorDim: number, + private readonly storageOptions?: Record, ) {} private async ensureInitialized(): Promise { @@ -72,7 +73,10 @@ class MemoryDB { private async doInitialize(): Promise { const lancedb = await loadLanceDbModule(); - this.db = await lancedb.connect(this.dbPath); + const connectionOptions: LanceDB.ConnectionOptions = this.storageOptions + ? { storageOptions: this.storageOptions } + : {}; + this.db = await lancedb.connect(this.dbPath, connectionOptions); const tables = await this.db.tableNames(); if (tables.includes(TABLE_NAME)) { @@ -291,11 +295,12 @@ export default definePluginEntry({ register(api: OpenClawPluginApi) { const cfg = memoryConfigSchema.parse(api.pluginConfig); - const resolvedDbPath = api.resolvePath(cfg.dbPath!); + const dbPath = cfg.dbPath!; + const resolvedDbPath = dbPath.includes("://") ? dbPath : api.resolvePath(dbPath); const { model, dimensions, apiKey, baseUrl } = cfg.embedding; const vectorDim = dimensions ?? vectorDimsForModel(model); - const db = new MemoryDB(resolvedDbPath, vectorDim); + const db = new MemoryDB(resolvedDbPath, vectorDim, cfg.storageOptions); const embeddings = new Embeddings(apiKey, model, baseUrl, dimensions); api.logger.info(`memory-lancedb: plugin registered (db: ${resolvedDbPath}, lazy init)`); diff --git a/extensions/memory-lancedb/openclaw.plugin.json b/extensions/memory-lancedb/openclaw.plugin.json index b19e3f3a50f..7a4e38ccea9 100644 --- a/extensions/memory-lancedb/openclaw.plugin.json +++ b/extensions/memory-lancedb/openclaw.plugin.json @@ -47,6 +47,11 @@ "help": "Maximum message length eligible for auto-capture", "advanced": true, "placeholder": "500" + }, + "storageOptions": { + "label": "Storage Options", + "advanced": true, + "help": "Storage configuration options (access_key, secret_key, endpoint, etc.); supports ${ENV_VAR} values" } }, "configSchema": { @@ -88,6 +93,12 @@ "type": "number", "minimum": 100, "maximum": 10000 + }, + "storageOptions": { + "type": "object", + "additionalProperties": { + "type": "string" + } } }, "required": ["embedding"]