From 6bc57ca73afd6cb4dfaabb37d079e30f910fdde0 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Tue, 16 Jun 2026 02:44:26 +0200 Subject: [PATCH] fix(migrate-hermes): snapshot live SQLite archive --- extensions/migrate-hermes/apply.ts | 64 ++++++++++++++++- .../migrate-hermes/files-and-skills.test.ts | 70 +++++++++++++++++++ 2 files changed, 132 insertions(+), 2 deletions(-) diff --git a/extensions/migrate-hermes/apply.ts b/extensions/migrate-hermes/apply.ts index 2f34e86fbf7..7da154bfa9a 100644 --- a/extensions/migrate-hermes/apply.ts +++ b/extensions/migrate-hermes/apply.ts @@ -1,6 +1,11 @@ // Migrate Hermes plugin module implements apply behavior. +import fs from "node:fs/promises"; import path from "node:path"; -import { markMigrationItemSkipped, summarizeMigrationItems } from "openclaw/plugin-sdk/migration"; +import { + markMigrationItemError, + markMigrationItemSkipped, + summarizeMigrationItems, +} from "openclaw/plugin-sdk/migration"; import { archiveMigrationItem, copyMigrationFileItem, @@ -13,6 +18,7 @@ import type { MigrationPlan, MigrationProviderContext, } from "openclaw/plugin-sdk/plugin-entry"; +import { resolvePreferredOpenClawTmpDir, withTempWorkspace } from "openclaw/plugin-sdk/temp-path"; import { applyAuthItem } from "./auth.js"; import { applyConfigItem, applyManualItem } from "./config.js"; import { appendItem } from "./helpers.js"; @@ -22,6 +28,60 @@ import { applySecretItem } from "./secrets.js"; import { resolveTargets } from "./targets.js"; const HERMES_REASON_BLOCKED_BY_APPLY_CONFLICT = "blocked by earlier apply conflict"; +const HERMES_STATE_DB_ARCHIVE_ITEM_ID = "archive:state.db"; +const HERMES_STATE_DB_SNAPSHOT_PREFIX = "openclaw-migrate-hermes-state-"; + +async function archiveHermesItem(item: MigrationItem, reportDir: string): Promise { + if (item.id !== HERMES_STATE_DB_ARCHIVE_ITEM_ID || !item.source) { + return await archiveMigrationItem(item, reportDir); + } + const sourcePath = item.source; + + let sourceStat: import("node:fs").Stats; + try { + sourceStat = await fs.lstat(sourcePath); + } catch { + return await archiveMigrationItem(item, reportDir); + } + if (!sourceStat.isFile()) { + return await archiveMigrationItem(item, reportDir); + } + + try { + // A raw state.db copy can omit committed rows that still live in state.db-wal. + // Snapshot the live database into one self-contained archive artifact. + return await withTempWorkspace( + { rootDir: resolvePreferredOpenClawTmpDir(), prefix: HERMES_STATE_DB_SNAPSHOT_PREFIX }, + async ({ dir: tempDir }) => { + const snapshotPath = path.join(tempDir, "state.db"); + const { DatabaseSync } = await import("node:sqlite"); + const source = new DatabaseSync(sourcePath, { readOnly: true }); + try { + source.exec("PRAGMA busy_timeout = 30000;"); + source.prepare("VACUUM INTO ?").run(snapshotPath); + } finally { + source.close(); + } + await fs.chmod(snapshotPath, 0o600); + const archived = await archiveMigrationItem({ ...item, source: snapshotPath }, reportDir); + return { ...archived, source: sourcePath }; + }, + ); + } catch (err) { + const snapshotReason = err instanceof Error ? err.message : String(err); + const rawArchive = await archiveMigrationItem(item, reportDir); + if (rawArchive.status === "migrated") { + return markMigrationItemError( + rawArchive, + `SQLite snapshot failed; raw state.db preserved for manual review: ${snapshotReason}`, + ); + } + return markMigrationItemError( + rawArchive, + `SQLite snapshot failed: ${snapshotReason}; raw archive failed: ${rawArchive.reason ?? rawArchive.status}`, + ); + } +} export async function applyHermesPlan(params: { ctx: MigrationProviderContext; @@ -55,7 +115,7 @@ export async function applyHermesPlan(params: { } else if (item.kind === "manual") { appliedItem = applyManualItem(item); } else if (item.action === "archive") { - appliedItem = await archiveMigrationItem(item, reportDir); + appliedItem = await archiveHermesItem(item, reportDir); } else if (item.kind === "auth") { appliedItem = await applyAuthItem(applyCtx, item, targets); } else if (item.kind === "secret") { diff --git a/extensions/migrate-hermes/files-and-skills.test.ts b/extensions/migrate-hermes/files-and-skills.test.ts index e0bffc9795e..47832102ae4 100644 --- a/extensions/migrate-hermes/files-and-skills.test.ts +++ b/extensions/migrate-hermes/files-and-skills.test.ts @@ -1,6 +1,7 @@ // Migrate Hermes tests cover files and skills plugin behavior. import fs from "node:fs/promises"; import path from "node:path"; +import { DatabaseSync } from "node:sqlite"; import { loadAuthProfileStoreWithoutExternalProfiles } from "openclaw/plugin-sdk/agent-runtime"; import { MIGRATION_REASON_TARGET_EXISTS } from "openclaw/plugin-sdk/migration"; import { afterEach, describe, expect, it } from "vitest"; @@ -203,6 +204,75 @@ describe("Hermes migration file and skill items", () => { await expectPathMissing(path.join(workspaceDir, "logs", "session.log")); }); + it("archives committed Hermes SQLite WAL state", async () => { + const root = await makeTempRoot(); + const source = path.join(root, "hermes"); + const workspaceDir = path.join(root, "workspace"); + const stateDir = path.join(root, "state"); + const reportDir = path.join(root, "report"); + const stateDbPath = path.join(source, "state.db"); + await fs.mkdir(source, { recursive: true }); + + const sourceDb = new DatabaseSync(stateDbPath); + try { + sourceDb.exec(` + PRAGMA journal_mode = WAL; + CREATE TABLE marker(value TEXT NOT NULL); + PRAGMA wal_checkpoint(TRUNCATE); + `); + sourceDb.prepare("INSERT INTO marker(value) VALUES (?)").run("committed-only-in-wal"); + expect((await fs.stat(`${stateDbPath}-wal`)).size).toBeGreaterThan(0); + + const provider = buildHermesMigrationProvider(); + const result = await provider.apply( + makeContext({ source, stateDir, workspaceDir, reportDir }), + ); + + const archivedState = itemById(result.items, "archive:state.db"); + const archivedStatePath = path.join(reportDir, "archive", "state.db"); + expect(archivedState?.status).toBe("migrated"); + expect(archivedState?.source).toBe(stateDbPath); + expect(archivedState?.target).toBe(archivedStatePath); + + const archivedDb = new DatabaseSync(archivedStatePath, { readOnly: true }); + try { + expect(archivedDb.prepare("SELECT value FROM marker").all()).toEqual([ + { value: "committed-only-in-wal" }, + ]); + expect(archivedDb.prepare("PRAGMA integrity_check").get()).toEqual({ + integrity_check: "ok", + }); + } finally { + archivedDb.close(); + } + } finally { + sourceDb.close(); + } + }); + + it("preserves raw Hermes state when SQLite snapshotting fails", async () => { + const root = await makeTempRoot(); + const source = path.join(root, "hermes"); + const workspaceDir = path.join(root, "workspace"); + const stateDir = path.join(root, "state"); + const reportDir = path.join(root, "report"); + const stateDbPath = path.join(source, "state.db"); + const archivedStatePath = path.join(reportDir, "archive", "state.db"); + await writeFile(stateDbPath, "legacy non-SQLite Hermes state\n"); + + const provider = buildHermesMigrationProvider(); + const result = await provider.apply(makeContext({ source, stateDir, workspaceDir, reportDir })); + + const archivedState = itemById(result.items, "archive:state.db"); + expect(archivedState?.status).toBe("error"); + expect(archivedState?.target).toBe(archivedStatePath); + expect(archivedState?.reason).toContain( + "SQLite snapshot failed; raw state.db preserved for manual review", + ); + expect(await fs.readFile(archivedStatePath, "utf8")).toBe("legacy non-SQLite Hermes state\n"); + expect(result.summary.errors).toBe(1); + }); + it("reports legacy Hermes OpenAI auth.json OAuth state as manual reauth work", async () => { const root = await makeTempRoot(); const source = path.join(root, "hermes");