diff --git a/docs/cli/doctor.md b/docs/cli/doctor.md index 60e904fefda4..b5c36ca09875 100644 --- a/docs/cli/doctor.md +++ b/docs/cli/doctor.md @@ -291,8 +291,10 @@ SQLite deletes reclaim pages inside the database first; they do not necessarily shrink the database file immediately. After deleting or archiving large transcripts, run `openclaw doctor --session-sqlite compact --session-sqlite-all-agents` to checkpoint WAL files, run `VACUUM`, and report before/after database and WAL -sizes. This is explicit doctor maintenance so normal Gateway writes do not pause -for background vacuum work. +sizes. Compaction requires a regular file with the current agent schema, the +selected agent's durable owner metadata, and no open handle in the doctor +process. This is explicit offline maintenance: stop the Gateway first so normal +writes cannot race the checkpoint or `VACUUM`. Each import writes a manifest under `~/.openclaw/session-sqlite-migration-runs/` before moving transcript artifacts @@ -308,11 +310,19 @@ manifest's archived artifacts, validates the affected targets, refreshes the sanitized `.failure.md` and `.failure.json` reports, and prepares a GitHub issue body that avoids transcript contents, raw environment, secrets, and unbounded config. When no failed migration manifest exists but a selected agent SQLite -database is corrupt or not a database, recovery preserves the DB, WAL, and SHM -files by renaming them with a `.corrupt-` suffix so the next startup -can create a fresh database. With `--github-issue --yes`, doctor uses the GitHub -CLI to create the issue in `openclaw/openclaw`; without confirmation it writes -the local support report and prints a prefilled issue URL. +database is corrupt, not a database, or has journal sidecars without a main +database, recovery copies the complete file set to a temporary inspection +directory. SQLite can roll back a valid hot journal in that disposable copy +before `quick_check`, `integrity_check`, and `foreign_key_check` run, while the +original forensic files remain untouched. Failed integrity checks or orphaned +sidecars preserve the DB, WAL, SHM, and rollback-journal files by renaming the +whole discovered set with one `.corrupt-` suffix. A caught rename +failure rolls already-moved files back before reporting failure, so a +recoverable file set is not silently split. Stop the Gateway before recovery; +copying or renaming an actively changing SQLite file set is unsafe and behaves +differently across operating systems. With `--github-issue --yes`, doctor uses +the GitHub CLI to create the issue in `openclaw/openclaw`; without confirmation +it writes the local support report and prints a prefilled issue URL. `restore` remains the lower-level undo operation. It uses manifest `sourcePath -> archivePath` records, moves archived artifacts back only when the diff --git a/scripts/check-kysely-guardrails.mjs b/scripts/check-kysely-guardrails.mjs index 589a715aabdd..e52a89b4a66b 100644 --- a/scripts/check-kysely-guardrails.mjs +++ b/scripts/check-kysely-guardrails.mjs @@ -57,6 +57,7 @@ const rawSqliteAllowPathGroups = { "src/commands/doctor-sqlite-compact.ts", "src/commands/doctor-session-sqlite.ts", "src/commands/doctor-session-sqlite-readers.ts", + "src/commands/doctor-session-sqlite-recover-report.ts", "src/commands/doctor-state-sqlite-compact.ts", "src/infra/state-migrations.ts", "src/infra/state-migrations.debug-proxy.ts", diff --git a/src/commands/doctor-session-sqlite-compact.ts b/src/commands/doctor-session-sqlite-compact.ts index 774bac79d57d..1e0c4671062d 100644 --- a/src/commands/doctor-session-sqlite-compact.ts +++ b/src/commands/doctor-session-sqlite-compact.ts @@ -1,6 +1,11 @@ /** Runs doctor-owned SQLite file compaction for migrated session stores. */ import fs from "node:fs"; import type { SessionStoreTarget } from "../config/sessions/targets.js"; +import { + assertOpenClawAgentDatabaseForMaintenance, + ensureOpenClawAgentDatabasePermissions, + isOpenClawAgentDatabaseOpen, +} from "../state/openclaw-agent-db.js"; import { resolveTargetSqlitePath } from "./doctor-session-sqlite-readers.js"; import type { DoctorSessionSqliteCompactReport } from "./doctor-session-sqlite-types.js"; import { compactDoctorSqliteFile } from "./doctor-sqlite-compact.js"; @@ -11,7 +16,8 @@ export function compactDoctorSessionSqliteTarget( ): DoctorSessionSqliteCompactReport { const sqlitePath = resolveTargetSqlitePath(target); const beforeFileSizes = readSqliteFileSizes(sqlitePath); - if (!fs.existsSync(sqlitePath)) { + const stat = readSessionDatabaseStat(sqlitePath); + if (!stat) { return { dbSizeAfterBytes: 0, dbSizeBeforeBytes: 0, @@ -24,8 +30,28 @@ export function compactDoctorSessionSqliteTarget( walSizeBeforeBytes: beforeFileSizes.walSizeBytes, }; } + if (!stat.isFile()) { + throw new Error(`OpenClaw agent database is not a regular file: ${sqlitePath}`); + } + if (isOpenClawAgentDatabaseOpen(sqlitePath)) { + throw new Error( + `OpenClaw agent database ${sqlitePath} is already open in this process. Stop OpenClaw and retry.`, + ); + } - const compact = compactDoctorSqliteFile({ sqlitePath }); + const compact = compactDoctorSqliteFile({ + afterMutation: () => + ensureOpenClawAgentDatabasePermissions(sqlitePath, { + agentId: target.agentId, + path: sqlitePath, + }), + sqlitePath, + validateBeforeMutation: (database) => + assertOpenClawAgentDatabaseForMaintenance(database, { + agentId: target.agentId, + pathname: sqlitePath, + }), + }); return { dbSizeAfterBytes: compact.after.dbSizeBytes, dbSizeBeforeBytes: compact.before.dbSizeBytes, @@ -39,6 +65,17 @@ export function compactDoctorSessionSqliteTarget( }; } +function readSessionDatabaseStat(sqlitePath: string): fs.Stats | undefined { + try { + return fs.lstatSync(sqlitePath); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + return undefined; + } + throw error; + } +} + function readSqliteFileSizes(sqlitePath: string): { dbSizeBytes: number; walSizeBytes: number } { return { dbSizeBytes: fileSize(sqlitePath), diff --git a/src/commands/doctor-session-sqlite-recover-report.ts b/src/commands/doctor-session-sqlite-recover-report.ts index 8037e0d3796a..efea18719018 100644 --- a/src/commands/doctor-session-sqlite-recover-report.ts +++ b/src/commands/doctor-session-sqlite-recover-report.ts @@ -1,6 +1,13 @@ /** Builds doctor reports for session SQLite migration recovery mode. */ import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import type { DatabaseSync } from "node:sqlite"; import type { SessionStoreTarget } from "../config/sessions/targets.js"; +import { requireNodeSqlite } from "../infra/node-sqlite.js"; +import { resolveSqliteDatabaseFilePaths } from "../infra/sqlite-files.js"; +import { assertSqliteIntegrity } from "../infra/sqlite-integrity.js"; +import { OPENCLAW_SQLITE_BUSY_TIMEOUT_MS } from "../state/openclaw-state-db.js"; import { createSessionSqliteMigrationFailureIssue, findLatestFailedSessionSqliteMigrationManifest, @@ -9,7 +16,7 @@ import { writeSessionSqliteMigrationFailureReports, type SessionSqliteMigrationTargetInput, } from "./doctor-session-sqlite-migration-run.js"; -import { readOnlySqliteDbStats, resolveTargetSqlitePath } from "./doctor-session-sqlite-readers.js"; +import { resolveTargetSqlitePath } from "./doctor-session-sqlite-readers.js"; import type { DoctorSessionSqliteOptions, DoctorSessionSqliteReport, @@ -85,29 +92,76 @@ function recoverCorruptSqliteTargets( ): DoctorSessionSqliteTargetReport[] { return targets.flatMap((target) => { const sqlitePath = resolveTargetSqlitePath(target); - if (!fs.existsSync(sqlitePath)) { + let recoveryFiles: ReturnType; + try { + recoveryFiles = inspectSqliteRecoveryFiles(sqlitePath); + } catch (error) { + return [createRecoverInspectionFailureTargetReport(target, sqlitePath, error)]; + } + if (recoveryFiles.existing.length === 0) { return []; } - const stats = readOnlySqliteDbStats(target); - if (stats.ok) { - if (stats.stats.integrityCheck && stats.stats.integrityCheck !== "ok") { - return [ - recoverCorruptSqliteTarget( - target, - sqlitePath, - new Error(`SQLite quick_check reported: ${stats.stats.integrityCheck}`), - ), - ]; - } + if (!recoveryFiles.existing.includes(sqlitePath)) { + return [ + recoverCorruptSqliteTarget( + target, + sqlitePath, + new Error(`SQLite sidecars exist without their main database: ${sqlitePath}`), + ), + ]; + } + const inspection = inspectSqliteForRecovery(sqlitePath, recoveryFiles.existing); + if (inspection.ok) { return []; } - if (!isSqliteCorruptionError(stats.error)) { - return [createRecoverInspectionFailureTargetReport(target, sqlitePath, stats.error)]; + if (!isSqliteCorruptionError(inspection.error)) { + return [createRecoverInspectionFailureTargetReport(target, sqlitePath, inspection.error)]; } - return [recoverCorruptSqliteTarget(target, sqlitePath, stats.error)]; + return [recoverCorruptSqliteTarget(target, sqlitePath, inspection.error)]; }); } +function inspectSqliteForRecovery( + sqlitePath: string, + sourcePaths: readonly string[], +): { ok: true } | { error: unknown; ok: false } { + let inspectionDir: string | undefined; + let database: DatabaseSync | undefined; + let inspectionError: unknown; + try { + const sqlite = requireNodeSqlite(); + inspectionDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-sqlite-recovery-")); + const inspectionPath = path.join(inspectionDir, path.basename(sqlitePath)); + for (const sourcePath of sourcePaths) { + const suffix = sourcePath.slice(sqlitePath.length); + const inspectionFilePath = `${inspectionPath}${suffix}`; + fs.copyFileSync(sourcePath, inspectionFilePath, fs.constants.COPYFILE_EXCL); + fs.chmodSync(inspectionFilePath, 0o600); + } + // Writable inspection of the disposable copy lets SQLite roll back a hot + // journal without changing the original forensic file set. + database = new sqlite.DatabaseSync(inspectionPath); + database.exec(`PRAGMA busy_timeout = ${OPENCLAW_SQLITE_BUSY_TIMEOUT_MS};`); + database.exec("PRAGMA trusted_schema = OFF;"); + assertSqliteIntegrity(database, inspectionPath); + } catch (error) { + inspectionError = error; + } + try { + database?.close(); + } catch (error) { + inspectionError ??= error; + } + try { + if (inspectionDir) { + fs.rmSync(inspectionDir, { force: true, recursive: true }); + } + } catch (error) { + inspectionError ??= error; + } + return inspectionError === undefined ? { ok: true } : { error: inspectionError, ok: false }; +} + function recoverCorruptSqliteTarget( target: SessionStoreTarget, sqlitePath: string, @@ -142,29 +196,106 @@ function moveCorruptSqliteFilesAside(sqlitePath: string): { movedFiles: string[]; skippedFiles: string[]; } { - const movedFiles: string[] = []; - const skippedFiles: string[] = []; - const suffix = `.corrupt-${Date.now()}`; - for (const candidate of [sqlitePath, `${sqlitePath}-wal`, `${sqlitePath}-shm`]) { - if (!fs.existsSync(candidate)) { - skippedFiles.push(candidate); - continue; + const recoveryFiles = inspectSqliteRecoveryFiles(sqlitePath); + const moves = planCorruptSqliteMoves(recoveryFiles.existing); + const completed: typeof moves = []; + try { + // Preserve every journal before removing the main pathname. Recovery is + // offline; rollback restores the set after a caught rename failure. + for (const move of moves.toSorted((left, right) => { + if (left.sourcePath === sqlitePath) { + return 1; + } + if (right.sourcePath === sqlitePath) { + return -1; + } + return left.sourcePath.localeCompare(right.sourcePath); + })) { + fs.renameSync(move.sourcePath, move.destinationPath); + completed.push(move); } - const destination = uniqueRecoveryPath(`${candidate}${suffix}`); - fs.renameSync(candidate, destination); - movedFiles.push(destination); + } catch (error) { + const rollbackErrors: unknown[] = []; + for (const move of completed.toReversed()) { + try { + if (pathExists(move.sourcePath)) { + throw new Error(`rollback source was recreated: ${move.sourcePath}`, { + cause: error, + }); + } + fs.renameSync(move.destinationPath, move.sourcePath); + } catch (rollbackError) { + rollbackErrors.push(rollbackError); + } + } + if (rollbackErrors.length > 0) { + const rollbackDetails = rollbackErrors + .map((rollbackError) => String(rollbackError)) + .join("; "); + throw new Error( + `Could not move corrupt SQLite file set aside or restore it: ${sqlitePath}; rollback failures: ${rollbackDetails}`, + { cause: error }, + ); + } + throw error; } - return { movedFiles, skippedFiles }; + return { + movedFiles: moves.map((move) => move.destinationPath), + skippedFiles: recoveryFiles.missing, + }; } -function uniqueRecoveryPath(basePath: string): string { - for (let attempt = 0; attempt < 100; attempt += 1) { - const candidate = attempt === 0 ? basePath : `${basePath}.${attempt}`; - if (!fs.existsSync(candidate)) { - return candidate; +function inspectSqliteRecoveryFiles(sqlitePath: string): { + existing: string[]; + missing: string[]; +} { + const existing: string[] = []; + const missing: string[] = []; + for (const candidate of resolveSqliteDatabaseFilePaths(sqlitePath)) { + try { + const stat = fs.lstatSync(candidate); + if (!stat.isFile()) { + throw new Error(`SQLite recovery path is not a regular file: ${candidate}`); + } + existing.push(candidate); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + missing.push(candidate); + continue; + } + throw error; } } - throw new Error(`Could not choose recovery path for ${basePath}`); + return { existing, missing }; +} + +function planCorruptSqliteMoves( + sourcePaths: readonly string[], +): Array<{ destinationPath: string; sourcePath: string }> { + const timestampSuffix = `.corrupt-${Date.now()}`; + for (let attempt = 0; attempt < 100; attempt += 1) { + const suffix = attempt === 0 ? timestampSuffix : `${timestampSuffix}.${attempt}`; + const moves = sourcePaths.map((sourcePath) => ({ + destinationPath: `${sourcePath}${suffix}`, + sourcePath, + })); + if (moves.every((move) => !pathExists(move.destinationPath))) { + return moves; + } + } + throw new Error(`Could not choose recovery paths for ${sourcePaths[0] ?? "SQLite files"}`); +} + +function pathExists(filePath: string): boolean { + try { + fs.lstatSync(filePath); + return true; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + return false; + } + throw error; + } } function isSqliteCorruptionError(error: unknown): boolean { @@ -173,7 +304,13 @@ function isSqliteCorruptionError(error: unknown): boolean { return true; } const message = String(error).toLowerCase(); - return message.includes("database disk image is malformed") || message.includes("not a database"); + return ( + message.includes("database disk image is malformed") || + message.includes("not a database") || + message.includes("sqlite quick_check failed") || + message.includes("sqlite integrity_check failed") || + message.includes("sqlite foreign_key_check failed") + ); } function resolveRecoverTargets( diff --git a/src/commands/doctor-session-sqlite.test.ts b/src/commands/doctor-session-sqlite.test.ts index 77d56ea8778f..5951ee83f01c 100644 --- a/src/commands/doctor-session-sqlite.test.ts +++ b/src/commands/doctor-session-sqlite.test.ts @@ -2,6 +2,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; +import type { DatabaseSync } from "node:sqlite"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { loadExactSqliteSessionEntry, @@ -9,9 +10,14 @@ import { readSqliteTranscriptStatsSync, upsertSqliteSessionEntry, } from "../config/sessions/session-accessor.sqlite.js"; -import { requireNodeSqlite } from "../infra/node-sqlite.js"; +import * as nodeSqlite from "../infra/node-sqlite.js"; import * as replaceFile from "../infra/replace-file.js"; -import { closeOpenClawAgentDatabasesForTest } from "../state/openclaw-agent-db.js"; +import { resolveSqliteDatabaseFilePaths } from "../infra/sqlite-files.js"; +import { + closeOpenClawAgentDatabasesForTest, + openOpenClawAgentDatabase, + OPENCLAW_AGENT_SCHEMA_VERSION, +} from "../state/openclaw-agent-db.js"; import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js"; import { assertSafeSessionSqliteMigrationMove, @@ -335,7 +341,7 @@ describe("runDoctorSessionSqlite", () => { }); const sqlitePath = importReport.targets[0]?.sqlitePath; expect(sqlitePath).toBeTruthy(); - const sqlite = requireNodeSqlite(); + const sqlite = nodeSqlite.requireNodeSqlite(); const db = new sqlite.DatabaseSync(sqlitePath ?? ""); try { db.exec("DELETE FROM transcript_events;"); @@ -361,6 +367,162 @@ describe("runDoctorSessionSqlite", () => { ); }); + it("refuses compaction while this process owns an open agent database handle", async () => { + const { sqlitePath, store } = await createImportedStoreForCompaction(); + openOpenClawAgentDatabase({ + agentId: "main", + env: store.env, + path: sqlitePath, + }); + + const report = await runDoctorSessionSqlite({ + env: store.env, + mode: "compact", + store: store.storePath, + }); + + expect(report.targets[0]?.issues).toEqual([ + expect.objectContaining({ + code: "sqlite_compact_failed", + message: expect.stringMatching(/already open in this process/iu), + }), + ]); + }); + + it.each([ + { + label: "wrong schema role", + mutate: (database: DatabaseSync) => { + database.prepare("UPDATE schema_meta SET role = 'global' WHERE meta_key = 'primary'").run(); + }, + message: /schema role global.*expected agent/iu, + }, + { + label: "wrong agent owner", + mutate: (database: DatabaseSync) => { + database + .prepare("UPDATE schema_meta SET agent_id = 'work' WHERE meta_key = 'primary'") + .run(); + }, + message: /belongs to agent work.*requested agent main/iu, + }, + { + label: "stale metadata version", + mutate: (database: DatabaseSync) => { + database + .prepare("UPDATE schema_meta SET schema_version = ? WHERE meta_key = 'primary'") + .run(OPENCLAW_AGENT_SCHEMA_VERSION - 1); + }, + message: /metadata schema version .* does not match/iu, + }, + { + label: "stale user version", + mutate: (database: DatabaseSync) => { + database.exec(`PRAGMA user_version = ${OPENCLAW_AGENT_SCHEMA_VERSION - 1};`); + }, + message: /run openclaw doctor --fix before compacting/iu, + }, + ])("rejects $label before compaction", async ({ mutate, message }) => { + const { sqlitePath, store } = await createImportedStoreForCompaction(); + const sqlite = nodeSqlite.requireNodeSqlite(); + const database = new sqlite.DatabaseSync(sqlitePath); + try { + mutate(database); + } finally { + database.close(); + } + + const report = await runDoctorSessionSqlite({ + env: store.env, + mode: "compact", + store: store.storePath, + }); + + expect(report.targets[0]?.issues).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + code: "sqlite_compact_failed", + message: expect.stringMatching(message), + }), + ]), + ); + }); + + it.skipIf(process.platform === "win32")( + "refuses a symlink at the agent database path", + async () => { + const { sqlitePath, store } = await createImportedStoreForCompaction(); + const realPath = `${sqlitePath}.real`; + fs.renameSync(sqlitePath, realPath); + fs.symlinkSync(realPath, sqlitePath); + + const report = await runDoctorSessionSqlite({ + env: store.env, + mode: "compact", + store: store.storePath, + }); + + expect(report.targets[0]?.issues).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + code: "sqlite_compact_failed", + message: expect.stringMatching(/not a regular file/iu), + }), + ]), + ); + }, + ); + + it.skipIf(process.platform === "win32")( + "reapplies owner-only permissions after compaction", + async () => { + const { sqlitePath, store } = await createImportedStoreForCompaction(); + fs.chmodSync(sqlitePath, 0o666); + + const report = await runDoctorSessionSqlite({ + env: store.env, + mode: "compact", + store: store.storePath, + }); + + expect(report.totals.issues).toBe(0); + expect(fs.statSync(sqlitePath).mode & 0o777).toBe(0o600); + }, + ); + + it("rejects stale secondary indexes before compacting and quarantines them in recovery", async () => { + const { sqlitePath, store } = await createImportedStoreForCompaction(); + createUnsafeIndexDrift(sqlitePath); + + const report = await runDoctorSessionSqlite({ + env: store.env, + mode: "compact", + store: store.storePath, + }); + + expect(report.targets[0]?.issues).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + code: "sqlite_compact_failed", + message: expect.stringMatching( + /integrity_check failed.*missing from index unsafe_session_index/iu, + ), + }), + ]), + ); + + const recovery = await runDoctorSessionSqlite({ + env: store.env, + mode: "recover", + store: store.storePath, + }); + expect(recovery.totals.issues).toBe(0); + expect(recovery.targets[0]?.corruptRecovery?.movedFiles).toEqual( + expect.arrayContaining([expect.stringMatching(/openclaw-agent\.sqlite\.corrupt-/u)]), + ); + expect(fs.existsSync(sqlitePath)).toBe(false); + }); + it("does not report SQLite markers as missing transcript files", async () => { const store = createLegacyStore(); fs.rmSync(store.transcriptPath); @@ -1639,6 +1801,8 @@ describe("runDoctorSessionSqlite", () => { fs.mkdirSync(path.dirname(sqlitePath), { recursive: true }); fs.writeFileSync(sqlitePath, "not a sqlite database\n", { mode: 0o600 }); fs.writeFileSync(`${sqlitePath}-wal`, "wal", { mode: 0o600 }); + fs.writeFileSync(`${sqlitePath}-shm`, "shm", { mode: 0o600 }); + fs.writeFileSync(`${sqlitePath}-journal`, "journal", { mode: 0o600 }); const report = await runDoctorSessionSqlite({ env: store.env, @@ -1647,15 +1811,128 @@ describe("runDoctorSessionSqlite", () => { }); expect(report.totals.issues).toBe(0); - expect(report.targets[0]?.corruptRecovery?.movedFiles.length).toBeGreaterThanOrEqual(2); - expect(fs.existsSync(sqlitePath)).toBe(false); + expect(report.targets[0]?.corruptRecovery?.movedFiles).toHaveLength(4); + expect(report.targets[0]?.corruptRecovery?.skippedFiles).toEqual([]); + for (const candidate of resolveSqliteDatabaseFilePaths(sqlitePath)) { + expect(fs.existsSync(candidate)).toBe(false); + expect( + report.targets[0]?.corruptRecovery?.movedFiles.some((filePath) => + filePath.startsWith(`${candidate}.corrupt-`), + ), + ).toBe(true); + } + }); + + it.skipIf(process.platform === "win32")( + "recovers owner-readable corrupt SQLite database files", + async () => { + const store = createLegacyStore(); + const sqlitePath = path.join( + store.stateDir, + "agents", + "main", + "agent", + "openclaw-agent.sqlite", + ); + fs.mkdirSync(path.dirname(sqlitePath), { recursive: true }); + fs.writeFileSync(sqlitePath, "not a sqlite database\n", { mode: 0o400 }); + + const report = await runDoctorSessionSqlite({ + env: store.env, + mode: "recover", + store: store.storePath, + }); + + expect(report.totals.issues).toBe(0); + expect(report.targets[0]?.corruptRecovery?.movedFiles).toEqual([ + expect.stringMatching(/openclaw-agent\.sqlite\.corrupt-/u), + ]); + expect(fs.existsSync(sqlitePath)).toBe(false); + }, + ); + + it("moves orphaned SQLite sidecars aside during recovery", async () => { + const store = createLegacyStore(); + const sqlitePath = path.join( + store.stateDir, + "agents", + "main", + "agent", + "openclaw-agent.sqlite", + ); + fs.mkdirSync(path.dirname(sqlitePath), { recursive: true }); + fs.writeFileSync(`${sqlitePath}-wal`, "wal", { mode: 0o600 }); + fs.writeFileSync(`${sqlitePath}-journal`, "journal", { mode: 0o600 }); + + const report = await runDoctorSessionSqlite({ + env: store.env, + mode: "recover", + store: store.storePath, + }); + + expect(report.totals.issues).toBe(0); + expect(report.targets[0]?.corruptRecovery?.movedFiles).toHaveLength(2); + expect(report.targets[0]?.corruptRecovery?.skippedFiles).toEqual([ + sqlitePath, + `${sqlitePath}-shm`, + ]); expect(fs.existsSync(`${sqlitePath}-wal`)).toBe(false); - expect(fs.existsSync(`${sqlitePath}-shm`)).toBe(false); + expect(fs.existsSync(`${sqlitePath}-journal`)).toBe(false); + }); + + it("rolls back every completed corrupt-file move when a later rename fails", async () => { + const store = createLegacyStore(); + const sqlitePath = path.join( + store.stateDir, + "agents", + "main", + "agent", + "openclaw-agent.sqlite", + ); + fs.mkdirSync(path.dirname(sqlitePath), { recursive: true }); + const expectedContents = new Map(); + for (const [candidate, contents] of [ + [sqlitePath, "not a sqlite database\n"], + [`${sqlitePath}-wal`, "wal"], + [`${sqlitePath}-shm`, "shm"], + [`${sqlitePath}-journal`, "journal"], + ] as const) { + fs.writeFileSync(candidate, contents, { mode: 0o600 }); + expectedContents.set(candidate, contents); + } + const renameSync = fs.renameSync.bind(fs); + let renameCalls = 0; + const renameSpy = vi.spyOn(fs, "renameSync").mockImplementation((source, destination) => { + renameCalls += 1; + if (renameCalls === 2) { + throw new Error("forced corrupt recovery rename failure"); + } + renameSync(source, destination); + }); + + let report: Awaited> | undefined; + try { + report = await runDoctorSessionSqlite({ + env: store.env, + mode: "recover", + store: store.storePath, + }); + } finally { + renameSpy.mockRestore(); + } + + expect(report?.totals.issues).toBe(1); + expect(report?.targets[0]?.corruptRecovery).toBeUndefined(); + expect(report?.targets[0]?.issues[0]).toMatchObject({ + code: "sqlite_corrupt_recovery_failed", + message: expect.stringContaining("forced corrupt recovery rename failure"), + }); + for (const [candidate, contents] of expectedContents) { + expect(fs.readFileSync(candidate, "utf8")).toBe(contents); + } expect( - report.targets[0]?.corruptRecovery?.movedFiles.every((filePath) => - filePath.includes(".corrupt-"), - ), - ).toBe(true); + fs.readdirSync(path.dirname(sqlitePath)).filter((entry) => entry.includes(".corrupt-")), + ).toEqual([]); }); it("does not move SQLite paths aside for non-corruption recovery inspection failures", async () => { @@ -1681,6 +1958,41 @@ describe("runDoctorSessionSqlite", () => { expect(fs.statSync(sqlitePath).isDirectory()).toBe(true); }); + it("reports SQLite loader failures without aborting recovery", async () => { + const store = createLegacyStore(); + const sqlitePath = path.join( + store.stateDir, + "agents", + "main", + "agent", + "openclaw-agent.sqlite", + ); + fs.mkdirSync(path.dirname(sqlitePath), { recursive: true }); + fs.writeFileSync(sqlitePath, "not a sqlite database\n", { mode: 0o600 }); + const requireSqlite = vi.spyOn(nodeSqlite, "requireNodeSqlite").mockImplementationOnce(() => { + throw new Error("node:sqlite unavailable"); + }); + + let report: Awaited> | undefined; + try { + report = await runDoctorSessionSqlite({ + env: store.env, + mode: "recover", + store: store.storePath, + }); + } finally { + requireSqlite.mockRestore(); + } + + expect(report?.totals.issues).toBe(1); + expect(report?.targets[0]?.issues[0]).toMatchObject({ + code: "sqlite_recovery_inspect_failed", + message: expect.stringContaining("node:sqlite unavailable"), + }); + expect(report?.targets[0]?.corruptRecovery).toBeUndefined(); + expect(fs.existsSync(sqlitePath)).toBe(true); + }); + it("does not truncate existing SQLite transcript rows when re-importing a duplicate fragment", async () => { const store = createLegacyStore({ transcriptLines: [ @@ -1866,6 +2178,60 @@ describe("runDoctorSessionSqlite", () => { }); }); +async function createImportedStoreForCompaction(): Promise<{ + sqlitePath: string; + store: TestStore; +}> { + const store = createLegacyStore(); + const report = await runDoctorSessionSqlite({ + env: store.env, + mode: "import", + store: store.storePath, + }); + const sqlitePath = report.targets[0]?.sqlitePath; + if (!sqlitePath) { + throw new Error("expected imported agent SQLite path"); + } + closeOpenClawAgentDatabasesForTest(); + return { sqlitePath, store }; +} + +function createUnsafeIndexDrift(sqlitePath: string): void { + const sqlite = nodeSqlite.requireNodeSqlite(); + const database = new sqlite.DatabaseSync(sqlitePath); + try { + database.exec(` + CREATE TABLE unsafe_session_index_records ( + id INTEGER PRIMARY KEY, + indexed_value TEXT NOT NULL, + alternate_value TEXT NOT NULL + ); + CREATE INDEX unsafe_session_index + ON unsafe_session_index_records(indexed_value); + INSERT INTO unsafe_session_index_records (indexed_value, alternate_value) + VALUES ('alpha', 'zeta'), ('beta', 'eta'), ('gamma', 'theta'); + `); + database.enableDefensive?.(false); + database.exec("PRAGMA writable_schema = ON;"); + database + .prepare( + "UPDATE sqlite_schema SET sql = 'CREATE INDEX unsafe_session_index ON unsafe_session_index_records(alternate_value)' WHERE name = 'unsafe_session_index'", + ) + .run(); + database.exec("PRAGMA writable_schema = OFF;"); + const schemaVersionRow = database.prepare("PRAGMA schema_version;").get() as + | Record + | undefined; + const schemaVersion = Number( + schemaVersionRow?.schema_version ?? + (schemaVersionRow ? Object.values(schemaVersionRow)[0] : undefined), + ); + database.exec(`PRAGMA schema_version = ${schemaVersion + 1};`); + } finally { + database.close(); + } +} + function createLegacyStore( params: { agentDirName?: string; diff --git a/src/commands/doctor-session-sqlite.ts b/src/commands/doctor-session-sqlite.ts index 174cd93ff210..736d36459211 100644 --- a/src/commands/doctor-session-sqlite.ts +++ b/src/commands/doctor-session-sqlite.ts @@ -23,6 +23,7 @@ import type { SessionEntry } from "../config/sessions/types.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import { resolveStoredSessionOwnerAgentId } from "../gateway/session-store-key.js"; import { normalizeAgentId } from "../routing/session-key.js"; +import { closeOpenClawAgentDatabaseByPath } from "../state/openclaw-agent-db.js"; import { compactDoctorSessionSqliteTarget } from "./doctor-session-sqlite-compact.js"; import { assertSafeSessionSqliteMigrationDirectory, @@ -295,7 +296,7 @@ async function inspectOrMigrateTarget(params: { if (validationPassed) { // Post-import compact retrofits auto_vacuum=INCREMENTAL onto pre-flip // databases and returns the pages the import churn freed. - compactSqliteDatabase(params.target, report); + compactSqliteDatabase(params.target, report, { closeImportedHandle: true }); } } report.unreferencedJsonlFiles = listUnreferencedJsonlFiles(params.target.storePath, [ @@ -983,8 +984,12 @@ function appendSqliteDbStats( function compactSqliteDatabase( target: SessionStoreTarget, report: DoctorSessionSqliteTargetReport, + options: { closeImportedHandle?: boolean } = {}, ): void { try { + if (options.closeImportedHandle) { + closeOpenClawAgentDatabaseByPath(resolveTargetSqlitePath(target)); + } report.compact = compactDoctorSessionSqliteTarget(target); } catch (err) { report.issues.push({ diff --git a/src/state/openclaw-agent-db.ts b/src/state/openclaw-agent-db.ts index b0cc5bccdcc1..2a7519c26e47 100644 --- a/src/state/openclaw-agent-db.ts +++ b/src/state/openclaw-agent-db.ts @@ -121,6 +121,7 @@ const cachedDatabases = new Map(); type ExistingSchemaMeta = { agentId: string | null; role: string | null; + schemaVersion: number | null; }; type MigratedSessionEntry = Record; @@ -509,7 +510,7 @@ function backfillOpenClawAgentSchema(db: DatabaseSync, previousVersion: number): } } -function ensureOpenClawAgentDatabasePermissions( +export function ensureOpenClawAgentDatabasePermissions( pathname: string, options: OpenClawAgentDatabaseOptions, ): void { @@ -540,14 +541,15 @@ function readExistingSchemaMeta(db: DatabaseSync): ExistingSchemaMeta | null { return null; } const row = db - .prepare("SELECT role, agent_id FROM schema_meta WHERE meta_key = 'primary'") - .get() as { agent_id?: unknown; role?: unknown } | undefined; + .prepare("SELECT role, schema_version, agent_id FROM schema_meta WHERE meta_key = 'primary'") + .get() as { agent_id?: unknown; role?: unknown; schema_version?: unknown } | undefined; if (!row) { return null; } return { agentId: typeof row.agent_id === "string" ? row.agent_id : null, role: typeof row.role === "string" ? row.role : null, + schemaVersion: typeof row.schema_version === "number" ? row.schema_version : null, }; } @@ -575,6 +577,41 @@ function assertExistingSchemaOwner( } } +/** Require the exact agent owner and schema before offline file maintenance. */ +export function assertOpenClawAgentDatabaseForMaintenance( + database: DatabaseSync, + options: { agentId: string; pathname: string }, +): void { + const agentId = normalizeAgentId(options.agentId); + const metadata = readExistingSchemaMeta(database); + if (!metadata) { + throw new Error( + `OpenClaw agent database ${options.pathname} has no schema ownership metadata.`, + ); + } + assertExistingSchemaOwner(metadata, agentId, options.pathname); + + const userVersion = readSqliteUserVersion(database); + if (userVersion > OPENCLAW_AGENT_SCHEMA_VERSION) { + throw createNewerSqliteSchemaVersionError( + "OpenClaw agent database", + options.pathname, + userVersion, + OPENCLAW_AGENT_SCHEMA_VERSION, + ); + } + if (userVersion !== OPENCLAW_AGENT_SCHEMA_VERSION) { + throw new Error( + `OpenClaw agent database ${options.pathname} uses schema version ${userVersion}; run openclaw doctor --fix before compacting it.`, + ); + } + if (metadata.schemaVersion !== OPENCLAW_AGENT_SCHEMA_VERSION) { + throw new Error( + `OpenClaw agent database ${options.pathname} metadata schema version ${metadata.schemaVersion ?? "invalid"} does not match ${OPENCLAW_AGENT_SCHEMA_VERSION}; run openclaw doctor --fix before compacting it.`, + ); + } +} + function ensureAgentSchema(db: DatabaseSync, agentId: string, pathname: string): void { // FK enforcement must be off before BEGIN: PRAGMA foreign_keys is a silent // no-op inside a transaction, and the v1 sessions rebuild would otherwise @@ -953,6 +990,12 @@ function closeCachedOpenClawAgentDatabase(database: OpenClawAgentDatabase): void } } +/** Return whether the exact cached agent database pathname is still open. */ +export function isOpenClawAgentDatabaseOpen(pathname: string): boolean { + const database = cachedDatabases.get(path.resolve(pathname)); + return database?.db.isOpen === true; +} + /** Close one cached agent database identified by its exact resolved pathname. */ export function closeOpenClawAgentDatabaseByPath(pathname: string): boolean { // Cache keys are lexical resolved paths. Do not realpath aliases here: a