fix: migrate matrix sync store blobs

This commit is contained in:
Peter Steinberger
2026-05-16 06:36:47 +01:00
parent 9be633c160
commit e2c0fb7688
6 changed files with 45 additions and 26 deletions

View File

@@ -1,7 +1,10 @@
import fs from "node:fs";
import path from "node:path";
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
import { resetPluginStateStoreForTests } from "openclaw/plugin-sdk/plugin-state-runtime";
import {
resetPluginBlobStoreForTests,
resetPluginStateStoreForTests,
} from "openclaw/plugin-sdk/plugin-state-runtime";
import { withTempHome } from "openclaw/plugin-sdk/test-env";
import { afterEach, describe, expect, it } from "vitest";
import { detectLegacyMatrixState } from "./doctor-legacy-state-detection.js";
@@ -28,6 +31,7 @@ function writeLegacySyncStore(filePath: string) {
describe("matrix legacy state migration", () => {
afterEach(() => {
resetPluginStateStoreForTests();
resetPluginBlobStoreForTests();
});
it("migrates the flat legacy Matrix store into account-scoped storage", async () => {

View File

@@ -1,7 +1,7 @@
import fs from "node:fs";
import path from "node:path";
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
import { upsertPluginStateMigrationEntry } from "openclaw/plugin-sdk/migration-runtime";
import { upsertPluginBlobMigrationEntry } from "openclaw/plugin-sdk/migration-runtime";
import {
detectLegacyMatrixState,
type MatrixLegacyStateMigrationResult,
@@ -10,6 +10,7 @@ import {
MATRIX_SYNC_STORE_NAMESPACE,
parsePersistedMatrixSyncStore,
resolveMatrixSyncStoreKey,
serializePersistedMatrixSyncStoreBlob,
} from "./matrix/client/sqlite-sync-store.js";
const MATRIX_PLUGIN_ID = "matrix";
@@ -66,17 +67,19 @@ function importLegacySyncStore(params: {
params.warnings.push(`Skipped invalid Matrix legacy sync store: ${params.sourcePath}`);
return;
}
upsertPluginStateMigrationEntry({
const { metadata, blob } = serializePersistedMatrixSyncStoreBlob(parsed);
upsertPluginBlobMigrationEntry({
pluginId: MATRIX_PLUGIN_ID,
namespace: MATRIX_SYNC_STORE_NAMESPACE,
key: resolveMatrixSyncStoreKey(params.targetRootDir),
value: parsed,
metadata,
blob,
createdAt: fs.statSync(params.sourcePath).mtimeMs || Date.now(),
env: params.env,
});
fs.rmSync(params.sourcePath, { force: true });
params.changes.push(
`Imported Matrix legacy sync store into SQLite: ${params.sourcePath} -> matrix plugin state (${params.targetRootDir})`,
`Imported Matrix legacy sync store into SQLite: ${params.sourcePath} -> matrix plugin blob (${params.targetRootDir})`,
);
}

View File

@@ -95,7 +95,7 @@ async function applyPlan(stateDir: string, label: string) {
}
describe("Matrix legacy state migrations", () => {
it("imports sync store files into SQLite plugin state", async () => {
it("imports sync store files into SQLite plugin blobs", async () => {
const stateDir = makeStateDir();
const legacyRoot = makeLegacyAccountRoot(stateDir);
const storageFile = path.join(legacyRoot, "bot-storage.json");

View File

@@ -17,6 +17,7 @@ import {
MATRIX_SYNC_STORE_NAMESPACE,
parsePersistedMatrixSyncStore,
resolveMatrixSyncStoreKey,
serializePersistedMatrixSyncStoreBlob,
} from "./matrix/client/sqlite-sync-store.js";
import {
MATRIX_STORAGE_META_NAMESPACE,
@@ -214,11 +215,13 @@ function importSyncStoreFiles(root: string, env: NodeJS.ProcessEnv): ImportResul
warnings.push(`Skipped invalid Matrix sync store file: ${filePath}`);
continue;
}
upsertPluginStateMigrationEntry({
const { metadata, blob } = serializePersistedMatrixSyncStoreBlob(parsed);
upsertPluginBlobMigrationEntry({
pluginId: MATRIX_PLUGIN_ID,
namespace: MATRIX_SYNC_STORE_NAMESPACE,
key: resolveMatrixSyncStoreKey(path.dirname(filePath)),
value: parsed,
metadata,
blob,
createdAt: fs.statSync(filePath).mtimeMs || Date.now(),
env,
});
@@ -419,7 +422,6 @@ function pluginStatePlan(params: {
label: string;
sourcePath: string;
namespace:
| typeof MATRIX_SYNC_STORE_NAMESPACE
| typeof MATRIX_STORAGE_META_NAMESPACE
| typeof MATRIX_LEGACY_CRYPTO_MIGRATION_NAMESPACE
| "thread-bindings"
@@ -447,7 +449,7 @@ function pluginStatePlan(params: {
function pluginBlobPlan(params: {
label: string;
sourcePath: string;
namespace: typeof MATRIX_IDB_SNAPSHOT_NAMESPACE;
namespace: typeof MATRIX_IDB_SNAPSHOT_NAMESPACE | typeof MATRIX_SYNC_STORE_NAMESPACE;
importSource: (sourcePath: string, env: NodeJS.ProcessEnv) => ImportResult;
}): ChannelDoctorLegacyStateMigrationPlan {
return {
@@ -474,7 +476,7 @@ export function detectMatrixLegacyStateMigrations(params: {
const plans: ChannelDoctorLegacyStateMigrationPlan[] = [];
if (collectFiles(root, SYNC_STORE_FILENAME).length > 0) {
plans.push(
pluginStatePlan({
pluginBlobPlan({
label: "Matrix sync store",
sourcePath: root,
namespace: MATRIX_SYNC_STORE_NAMESPACE,

View File

@@ -52,7 +52,7 @@ export function formatMatrixLegacyStatePreview(
): string {
return [
"- Matrix plugin upgraded in place.",
`- Legacy sync store: ${detection.legacyStoragePath} -> SQLite plugin state (${detection.targetRootDir})`,
`- Legacy sync store: ${detection.legacyStoragePath} -> SQLite plugin blob (${detection.targetRootDir})`,
`- Legacy crypto store: ${detection.legacyCryptoPath} -> ${detection.targetCryptoPath}`,
...(detection.selectionNote ? [`- ${detection.selectionNote}`] : []),
'- Run "openclaw doctor --fix" to migrate this Matrix state now.',

View File

@@ -20,14 +20,14 @@ const STORE_VERSION = 1;
const PERSIST_DEBOUNCE_MS = 250;
export const MATRIX_SYNC_STORE_NAMESPACE = "sync-store";
type PersistedMatrixSyncStore = {
export type PersistedMatrixSyncStore = {
version: number;
savedSync: ISyncData | null;
clientOptions?: IStoredClientOpts;
cleanShutdown?: boolean;
};
type PersistedMatrixSyncStoreMetadata = {
export type PersistedMatrixSyncStoreMetadata = {
version: number;
cleanShutdown?: boolean;
hasSavedSync: boolean;
@@ -141,6 +141,26 @@ function toStoredJson<T>(value: T): T {
return JSON.parse(JSON.stringify(value)) as T;
}
export function serializePersistedMatrixSyncStoreBlob(payload: PersistedMatrixSyncStore): {
metadata: PersistedMatrixSyncStoreMetadata;
blob: Buffer;
} {
const storedPayload = toStoredJson(payload);
const blob = Buffer.from(JSON.stringify(storedPayload));
return {
metadata: {
version: STORE_VERSION,
cleanShutdown: storedPayload.cleanShutdown === true,
hasSavedSync: storedPayload.savedSync !== null,
...(storedPayload.savedSync?.nextBatch
? { nextBatch: storedPayload.savedSync.nextBatch }
: {}),
payloadBytes: blob.byteLength,
},
blob,
};
}
function syncDataToSyncResponse(syncData: ISyncData): ISyncResponse {
return {
next_batch: syncData.nextBatch,
@@ -300,20 +320,10 @@ export class SqliteBackedMatrixSyncStore extends MemoryStore {
cleanShutdown: this.cleanShutdown,
...(this.savedClientOptions ? { clientOptions: cloneJson(this.savedClientOptions) } : {}),
});
const blob = Buffer.from(JSON.stringify(payload));
const { metadata, blob } = serializePersistedMatrixSyncStoreBlob(payload);
try {
await this.persistLock(async () => {
this.syncStore.register(
resolveMatrixSyncStoreKey(this.rootDir),
{
version: STORE_VERSION,
cleanShutdown: payload.cleanShutdown === true,
hasSavedSync: payload.savedSync !== null,
...(payload.savedSync?.nextBatch ? { nextBatch: payload.savedSync.nextBatch } : {}),
payloadBytes: blob.byteLength,
},
blob,
);
this.syncStore.register(resolveMatrixSyncStoreKey(this.rootDir), metadata, blob);
claimCurrentTokenStorageState({
rootDir: this.rootDir,
});