From 8e2383ea5fb0cb77ec86aa238dfc4c28935354ed Mon Sep 17 00:00:00 2001 From: Gustavo Madeira Santana Date: Sun, 8 Mar 2026 19:40:52 -0400 Subject: [PATCH] matrix-js: simplify storage paths --- docs/channels/matrix-js.md | 10 ++ .../src/matrix/client/create-client.ts | 9 +- .../src/matrix/client/storage.test.ts | 121 ++++++++++++++++++ .../matrix-js/src/matrix/client/storage.ts | 64 +++++++-- .../matrix-js/src/matrix/credentials.test.ts | 40 +++++- .../matrix-js/src/matrix/credentials.ts | 43 +++++++ .../src/matrix/sdk/idb-persistence.ts | 41 ++++-- 7 files changed, 305 insertions(+), 23 deletions(-) create mode 100644 extensions/matrix-js/src/matrix/client/storage.test.ts diff --git a/docs/channels/matrix-js.md b/docs/channels/matrix-js.md index eb20345b9b8..04a6a8801e3 100644 --- a/docs/channels/matrix-js.md +++ b/docs/channels/matrix-js.md @@ -73,6 +73,9 @@ Password-based setup (token is cached after login): } ``` +Matrix-js stores cached credentials in `~/.openclaw/credentials/matrix/`. +The default account uses `credentials.json`; named accounts use `credentials-.json`. + Environment variable equivalents (used when the config key is not set): - `MATRIX_HOMESERVER` @@ -218,6 +221,13 @@ Failed request attempts retry sooner than successful request creation by default Set `startupVerification: "off"` to disable automatic startup requests, or tune `startupVerificationCooldownHours` if you want a shorter or longer retry window. +Encrypted runtime state is stored per account and per access token in +`~/.openclaw/matrix/accounts//__//`. +That directory contains the sync store (`bot-storage.json`), crypto store (`crypto/`), +recovery key file (`recovery-key.json`), IndexedDB snapshot (`crypto-idb-snapshot.json`), +thread bindings (`thread-bindings.json`), and startup verification state (`startup-verification.json`) +when those features are in use. + ## Automatic verification notices Matrix-js now posts verification lifecycle notices directly into the Matrix room as `m.notice` messages. diff --git a/extensions/matrix-js/src/matrix/client/create-client.ts b/extensions/matrix-js/src/matrix/client/create-client.ts index 3a1d996233f..1782f4ba9cc 100644 --- a/extensions/matrix-js/src/matrix/client/create-client.ts +++ b/extensions/matrix-js/src/matrix/client/create-client.ts @@ -31,7 +31,14 @@ export async function createMatrixClient(params: { accountId: params.accountId, env, }); - maybeMigrateLegacyStorage({ storagePaths, env }); + maybeMigrateLegacyStorage({ + storagePaths, + homeserver: params.homeserver, + userId, + accessToken: params.accessToken, + accountId: params.accountId, + env, + }); fs.mkdirSync(storagePaths.rootDir, { recursive: true }); writeStorageMeta({ diff --git a/extensions/matrix-js/src/matrix/client/storage.test.ts b/extensions/matrix-js/src/matrix/client/storage.test.ts new file mode 100644 index 00000000000..682807a4569 --- /dev/null +++ b/extensions/matrix-js/src/matrix/client/storage.test.ts @@ -0,0 +1,121 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { setMatrixRuntime } from "../../runtime.js"; +import { maybeMigrateLegacyStorage, resolveMatrixStoragePaths } from "./storage.js"; + +describe("matrix client storage paths", () => { + const tempDirs: string[] = []; + + afterEach(() => { + for (const dir of tempDirs.splice(0)) { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); + + function setupStateDir(): string { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-matrix-storage-")); + tempDirs.push(dir); + setMatrixRuntime({ + state: { + resolveStateDir: () => dir, + }, + } as never); + return dir; + } + + it("uses the simplified matrix runtime root for account-scoped storage", () => { + const stateDir = setupStateDir(); + + const storagePaths = resolveMatrixStoragePaths({ + homeserver: "https://matrix.example.org", + userId: "@Bot:example.org", + accessToken: "secret-token", + accountId: "ops", + env: {}, + }); + + expect(storagePaths.rootDir).toBe( + path.join( + stateDir, + "matrix", + "accounts", + "ops", + "matrix.example.org__bot_example.org", + storagePaths.tokenHash, + ), + ); + expect(storagePaths.storagePath).toBe(path.join(storagePaths.rootDir, "bot-storage.json")); + expect(storagePaths.cryptoPath).toBe(path.join(storagePaths.rootDir, "crypto")); + expect(storagePaths.metaPath).toBe(path.join(storagePaths.rootDir, "storage-meta.json")); + expect(storagePaths.recoveryKeyPath).toBe(path.join(storagePaths.rootDir, "recovery-key.json")); + expect(storagePaths.idbSnapshotPath).toBe( + path.join(storagePaths.rootDir, "crypto-idb-snapshot.json"), + ); + }); + + it("migrates the nested legacy matrix-js account directory into the simplified root", () => { + const stateDir = setupStateDir(); + const storagePaths = resolveMatrixStoragePaths({ + homeserver: "https://matrix.example.org", + userId: "@bot:example.org", + accessToken: "secret-token", + accountId: "ops", + env: {}, + }); + const legacyRoot = path.join( + stateDir, + "credentials", + "matrix-js", + "accounts", + "ops", + "matrix.example.org__bot_example.org", + storagePaths.tokenHash, + ); + fs.mkdirSync(path.join(legacyRoot, "crypto"), { recursive: true }); + fs.writeFileSync(path.join(legacyRoot, "bot-storage.json"), "{}"); + fs.writeFileSync(path.join(legacyRoot, "recovery-key.json"), '{"key":"abc"}'); + fs.writeFileSync(path.join(legacyRoot, "crypto-idb-snapshot.json"), "[]"); + + maybeMigrateLegacyStorage({ + storagePaths, + homeserver: "https://matrix.example.org", + userId: "@bot:example.org", + accessToken: "secret-token", + accountId: "ops", + env: {}, + }); + + expect(fs.existsSync(legacyRoot)).toBe(false); + expect(fs.readFileSync(storagePaths.storagePath, "utf8")).toBe("{}"); + expect(fs.readFileSync(storagePaths.recoveryKeyPath, "utf8")).toBe('{"key":"abc"}'); + expect(fs.readFileSync(storagePaths.idbSnapshotPath, "utf8")).toBe("[]"); + expect(fs.existsSync(storagePaths.cryptoPath)).toBe(true); + }); + + it("falls back to migrating the older flat matrix-js storage layout", () => { + const stateDir = setupStateDir(); + const storagePaths = resolveMatrixStoragePaths({ + homeserver: "https://matrix.example.org", + userId: "@bot:example.org", + accessToken: "secret-token", + env: {}, + }); + const legacyRoot = path.join(stateDir, "credentials", "matrix-js"); + fs.mkdirSync(path.join(legacyRoot, "crypto"), { recursive: true }); + fs.writeFileSync(path.join(legacyRoot, "bot-storage.json"), '{"legacy":true}'); + + maybeMigrateLegacyStorage({ + storagePaths, + homeserver: "https://matrix.example.org", + userId: "@bot:example.org", + accessToken: "secret-token", + env: {}, + }); + + expect(fs.existsSync(path.join(legacyRoot, "bot-storage.json"))).toBe(false); + expect(fs.readFileSync(storagePaths.storagePath, "utf8")).toBe('{"legacy":true}'); + expect(fs.existsSync(storagePaths.cryptoPath)).toBe(true); + }); +}); diff --git a/extensions/matrix-js/src/matrix/client/storage.ts b/extensions/matrix-js/src/matrix/client/storage.ts index b09b6be2695..cd60989f9e7 100644 --- a/extensions/matrix-js/src/matrix/client/storage.ts +++ b/extensions/matrix-js/src/matrix/client/storage.ts @@ -44,6 +44,30 @@ function resolveLegacyStoragePaths(env: NodeJS.ProcessEnv = process.env): { }; } +function resolveLegacyMatrixJsAccountRoot(params: { + homeserver: string; + userId: string; + accessToken: string; + accountId?: string | null; + env?: NodeJS.ProcessEnv; +}): string { + const env = params.env ?? process.env; + const stateDir = getMatrixRuntime().state.resolveStateDir(env, os.homedir); + const accountKey = sanitizePathSegment(params.accountId ?? DEFAULT_ACCOUNT_KEY); + const userKey = sanitizePathSegment(params.userId); + const serverKey = resolveHomeserverKey(params.homeserver); + const tokenHash = hashAccessToken(params.accessToken); + return path.join( + stateDir, + "credentials", + "matrix-js", + "accounts", + accountKey, + `${serverKey}__${userKey}`, + tokenHash, + ); +} + export function resolveMatrixStoragePaths(params: { homeserver: string; userId: string; @@ -59,8 +83,7 @@ export function resolveMatrixStoragePaths(params: { const tokenHash = hashAccessToken(params.accessToken); const rootDir = path.join( stateDir, - "credentials", - "matrix-js", + "matrix", "accounts", accountKey, `${serverKey}__${userKey}`, @@ -80,20 +103,45 @@ export function resolveMatrixStoragePaths(params: { export function maybeMigrateLegacyStorage(params: { storagePaths: MatrixStoragePaths; + homeserver?: string; + userId?: string; + accessToken?: string; + accountId?: string | null; env?: NodeJS.ProcessEnv; }): void { + const hasNewStorage = + fs.existsSync(params.storagePaths.storagePath) || fs.existsSync(params.storagePaths.cryptoPath); + if (hasNewStorage) { + return; + } + + const legacyAccountRoot = + params.homeserver && params.userId && params.accessToken + ? resolveLegacyMatrixJsAccountRoot({ + homeserver: params.homeserver, + userId: params.userId, + accessToken: params.accessToken, + accountId: params.accountId, + env: params.env, + }) + : null; + + if (legacyAccountRoot && fs.existsSync(legacyAccountRoot)) { + fs.mkdirSync(path.dirname(params.storagePaths.rootDir), { recursive: true }); + try { + fs.renameSync(legacyAccountRoot, params.storagePaths.rootDir); + return; + } catch { + // Fall through to older one-off migration paths. + } + } + const legacy = resolveLegacyStoragePaths(params.env); const hasLegacyStorage = fs.existsSync(legacy.storagePath); const hasLegacyCrypto = fs.existsSync(legacy.cryptoPath); - const hasNewStorage = - fs.existsSync(params.storagePaths.storagePath) || fs.existsSync(params.storagePaths.cryptoPath); - if (!hasLegacyStorage && !hasLegacyCrypto) { return; } - if (hasNewStorage) { - return; - } fs.mkdirSync(params.storagePaths.rootDir, { recursive: true }); if (hasLegacyStorage) { diff --git a/extensions/matrix-js/src/matrix/credentials.test.ts b/extensions/matrix-js/src/matrix/credentials.test.ts index 840b755be09..08893616337 100644 --- a/extensions/matrix-js/src/matrix/credentials.test.ts +++ b/extensions/matrix-js/src/matrix/credentials.test.ts @@ -5,6 +5,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { setMatrixRuntime } from "../runtime.js"; import { loadMatrixCredentials, + clearMatrixCredentials, resolveMatrixCredentialsPath, saveMatrixCredentials, touchMatrixCredentials, @@ -31,7 +32,7 @@ describe("matrix credentials storage", () => { } it("writes credentials atomically with secure file permissions", async () => { - setupStateDir(); + const stateDir = setupStateDir(); await saveMatrixCredentials( { homeserver: "https://matrix.example.org", @@ -45,6 +46,7 @@ describe("matrix credentials storage", () => { const credPath = resolveMatrixCredentialsPath({}, "ops"); expect(fs.existsSync(credPath)).toBe(true); + expect(credPath).toBe(path.join(stateDir, "credentials", "matrix", "credentials-ops.json")); const mode = fs.statSync(credPath).mode & 0o777; expect(mode).toBe(0o600); }); @@ -77,4 +79,40 @@ describe("matrix credentials storage", () => { vi.useRealTimers(); } }); + + it("migrates legacy matrix-js credential files on read", async () => { + const stateDir = setupStateDir(); + const legacyPath = path.join(stateDir, "credentials", "matrix-js", "credentials.json"); + fs.mkdirSync(path.dirname(legacyPath), { recursive: true }); + fs.writeFileSync( + legacyPath, + JSON.stringify({ + homeserver: "https://matrix.example.org", + userId: "@bot:example.org", + accessToken: "legacy-token", + createdAt: "2026-03-01T10:00:00.000Z", + }), + ); + + const loaded = loadMatrixCredentials({}, "default"); + + expect(loaded?.accessToken).toBe("legacy-token"); + expect(fs.existsSync(legacyPath)).toBe(false); + expect(fs.existsSync(resolveMatrixCredentialsPath({}, "default"))).toBe(true); + }); + + it("clears both current and legacy credential paths", () => { + const stateDir = setupStateDir(); + const currentPath = path.join(stateDir, "credentials", "matrix", "credentials.json"); + const legacyPath = path.join(stateDir, "credentials", "matrix-js", "credentials.json"); + fs.mkdirSync(path.dirname(currentPath), { recursive: true }); + fs.mkdirSync(path.dirname(legacyPath), { recursive: true }); + fs.writeFileSync(currentPath, "{}"); + fs.writeFileSync(legacyPath, "{}"); + + clearMatrixCredentials({}, "default"); + + expect(fs.existsSync(currentPath)).toBe(false); + expect(fs.existsSync(legacyPath)).toBe(false); + }); }); diff --git a/extensions/matrix-js/src/matrix/credentials.ts b/extensions/matrix-js/src/matrix/credentials.ts index b7c848457f2..111c8bfd458 100644 --- a/extensions/matrix-js/src/matrix/credentials.ts +++ b/extensions/matrix-js/src/matrix/credentials.ts @@ -27,6 +27,14 @@ function credentialsFilename(accountId?: string | null): string { export function resolveMatrixCredentialsDir( env: NodeJS.ProcessEnv = process.env, stateDir?: string, +): string { + const resolvedStateDir = stateDir ?? getMatrixRuntime().state.resolveStateDir(env, os.homedir); + return path.join(resolvedStateDir, "credentials", "matrix"); +} + +function resolveLegacyMatrixJsCredentialsDir( + env: NodeJS.ProcessEnv = process.env, + stateDir?: string, ): string { const resolvedStateDir = stateDir ?? getMatrixRuntime().state.resolveStateDir(env, os.homedir); return path.join(resolvedStateDir, "credentials", "matrix-js"); @@ -40,10 +48,39 @@ export function resolveMatrixCredentialsPath( return path.join(dir, credentialsFilename(accountId)); } +function resolveLegacyMatrixJsCredentialsPath( + env: NodeJS.ProcessEnv = process.env, + accountId?: string | null, +): string { + const dir = resolveLegacyMatrixJsCredentialsDir(env); + return path.join(dir, credentialsFilename(accountId)); +} + +function maybeMigrateLegacyMatrixJsCredentials( + env: NodeJS.ProcessEnv = process.env, + accountId?: string | null, +): void { + const nextPath = resolveMatrixCredentialsPath(env, accountId); + if (fs.existsSync(nextPath)) { + return; + } + const legacyPath = resolveLegacyMatrixJsCredentialsPath(env, accountId); + if (!fs.existsSync(legacyPath)) { + return; + } + fs.mkdirSync(path.dirname(nextPath), { recursive: true }); + try { + fs.renameSync(legacyPath, nextPath); + } catch { + // Best-effort compatibility migration only. + } +} + export function loadMatrixCredentials( env: NodeJS.ProcessEnv = process.env, accountId?: string | null, ): MatrixStoredCredentials | null { + maybeMigrateLegacyMatrixJsCredentials(env, accountId); const credPath = resolveMatrixCredentialsPath(env, accountId); try { if (!fs.existsSync(credPath)) { @@ -69,6 +106,7 @@ export async function saveMatrixCredentials( env: NodeJS.ProcessEnv = process.env, accountId?: string | null, ): Promise { + maybeMigrateLegacyMatrixJsCredentials(env, accountId); const credPath = resolveMatrixCredentialsPath(env, accountId); const existing = loadMatrixCredentials(env, accountId); @@ -87,6 +125,7 @@ export async function touchMatrixCredentials( env: NodeJS.ProcessEnv = process.env, accountId?: string | null, ): Promise { + maybeMigrateLegacyMatrixJsCredentials(env, accountId); const existing = loadMatrixCredentials(env, accountId); if (!existing) { return; @@ -102,10 +141,14 @@ export function clearMatrixCredentials( accountId?: string | null, ): void { const credPath = resolveMatrixCredentialsPath(env, accountId); + const legacyPath = resolveLegacyMatrixJsCredentialsPath(env, accountId); try { if (fs.existsSync(credPath)) { fs.unlinkSync(credPath); } + if (fs.existsSync(legacyPath)) { + fs.unlinkSync(legacyPath); + } } catch { // ignore } diff --git a/extensions/matrix-js/src/matrix/sdk/idb-persistence.ts b/extensions/matrix-js/src/matrix/sdk/idb-persistence.ts index d6e355ca212..8bfd8042cee 100644 --- a/extensions/matrix-js/src/matrix/sdk/idb-persistence.ts +++ b/extensions/matrix-js/src/matrix/sdk/idb-persistence.ts @@ -120,6 +120,14 @@ async function restoreIndexedDatabases(snapshot: IdbDatabaseSnapshot[]): Promise } function resolveDefaultIdbSnapshotPath(): string { + const stateDir = + process.env.OPENCLAW_STATE_DIR || + process.env.MOLTBOT_STATE_DIR || + path.join(process.env.HOME || "/tmp", ".openclaw"); + return path.join(stateDir, "matrix", "crypto-idb-snapshot.json"); +} + +function resolveLegacyIdbSnapshotPath(): string { const stateDir = process.env.OPENCLAW_STATE_DIR || process.env.MOLTBOT_STATE_DIR || @@ -128,20 +136,27 @@ function resolveDefaultIdbSnapshotPath(): string { } export async function restoreIdbFromDisk(snapshotPath?: string): Promise { - const resolvedPath = snapshotPath ?? resolveDefaultIdbSnapshotPath(); - try { - const data = fs.readFileSync(resolvedPath, "utf8"); - const snapshot: IdbDatabaseSnapshot[] = JSON.parse(data); - if (!Array.isArray(snapshot) || snapshot.length === 0) return false; - await restoreIndexedDatabases(snapshot); - LogService.info( - "IdbPersistence", - `Restored ${snapshot.length} IndexedDB database(s) from ${resolvedPath}`, - ); - return true; - } catch { - return false; + const candidatePaths = snapshotPath + ? [snapshotPath] + : [resolveDefaultIdbSnapshotPath(), resolveLegacyIdbSnapshotPath()]; + for (const resolvedPath of candidatePaths) { + try { + const data = fs.readFileSync(resolvedPath, "utf8"); + const snapshot: IdbDatabaseSnapshot[] = JSON.parse(data); + if (!Array.isArray(snapshot) || snapshot.length === 0) { + continue; + } + await restoreIndexedDatabases(snapshot); + LogService.info( + "IdbPersistence", + `Restored ${snapshot.length} IndexedDB database(s) from ${resolvedPath}`, + ); + return true; + } catch { + continue; + } } + return false; } export async function persistIdbToDisk(params?: {