fix(sqlite): preserve complete database file sets during recovery (#105370)

* fix(sqlite): harden agent database recovery

* chore(sqlite): allow recovery maintenance primitive
This commit is contained in:
Vincent Koc
2026-07-12 15:10:11 +02:00
committed by GitHub
parent eb9bd4e457
commit 720774aa67
7 changed files with 656 additions and 57 deletions

View File

@@ -291,8 +291,10 @@ SQLite deletes reclaim pages inside the database first; they do not necessarily
shrink the database file immediately. After deleting or archiving large
transcripts, run `openclaw doctor --session-sqlite compact --session-sqlite-all-agents`
to checkpoint WAL files, run `VACUUM`, and report before/after database and WAL
sizes. This is explicit doctor maintenance so normal Gateway writes do not pause
for background vacuum work.
sizes. Compaction requires a regular file with the current agent schema, the
selected agent's durable owner metadata, and no open handle in the doctor
process. This is explicit offline maintenance: stop the Gateway first so normal
writes cannot race the checkpoint or `VACUUM`.
Each import writes a manifest under
`~/.openclaw/session-sqlite-migration-runs/` before moving transcript artifacts
@@ -308,11 +310,19 @@ manifest's archived artifacts, validates the affected targets, refreshes the
sanitized `.failure.md` and `.failure.json` reports, and prepares a GitHub issue
body that avoids transcript contents, raw environment, secrets, and unbounded
config. When no failed migration manifest exists but a selected agent SQLite
database is corrupt or not a database, recovery preserves the DB, WAL, and SHM
files by renaming them with a `.corrupt-<timestamp>` suffix so the next startup
can create a fresh database. With `--github-issue --yes`, doctor uses the GitHub
CLI to create the issue in `openclaw/openclaw`; without confirmation it writes
the local support report and prints a prefilled issue URL.
database is corrupt, not a database, or has journal sidecars without a main
database, recovery copies the complete file set to a temporary inspection
directory. SQLite can roll back a valid hot journal in that disposable copy
before `quick_check`, `integrity_check`, and `foreign_key_check` run, while the
original forensic files remain untouched. Failed integrity checks or orphaned
sidecars preserve the DB, WAL, SHM, and rollback-journal files by renaming the
whole discovered set with one `.corrupt-<timestamp>` suffix. A caught rename
failure rolls already-moved files back before reporting failure, so a
recoverable file set is not silently split. Stop the Gateway before recovery;
copying or renaming an actively changing SQLite file set is unsafe and behaves
differently across operating systems. With `--github-issue --yes`, doctor uses
the GitHub CLI to create the issue in `openclaw/openclaw`; without confirmation
it writes the local support report and prints a prefilled issue URL.
`restore` remains the lower-level undo operation. It uses manifest
`sourcePath -> archivePath` records, moves archived artifacts back only when the

View File

@@ -57,6 +57,7 @@ const rawSqliteAllowPathGroups = {
"src/commands/doctor-sqlite-compact.ts",
"src/commands/doctor-session-sqlite.ts",
"src/commands/doctor-session-sqlite-readers.ts",
"src/commands/doctor-session-sqlite-recover-report.ts",
"src/commands/doctor-state-sqlite-compact.ts",
"src/infra/state-migrations.ts",
"src/infra/state-migrations.debug-proxy.ts",

View File

@@ -1,6 +1,11 @@
/** Runs doctor-owned SQLite file compaction for migrated session stores. */
import fs from "node:fs";
import type { SessionStoreTarget } from "../config/sessions/targets.js";
import {
assertOpenClawAgentDatabaseForMaintenance,
ensureOpenClawAgentDatabasePermissions,
isOpenClawAgentDatabaseOpen,
} from "../state/openclaw-agent-db.js";
import { resolveTargetSqlitePath } from "./doctor-session-sqlite-readers.js";
import type { DoctorSessionSqliteCompactReport } from "./doctor-session-sqlite-types.js";
import { compactDoctorSqliteFile } from "./doctor-sqlite-compact.js";
@@ -11,7 +16,8 @@ export function compactDoctorSessionSqliteTarget(
): DoctorSessionSqliteCompactReport {
const sqlitePath = resolveTargetSqlitePath(target);
const beforeFileSizes = readSqliteFileSizes(sqlitePath);
if (!fs.existsSync(sqlitePath)) {
const stat = readSessionDatabaseStat(sqlitePath);
if (!stat) {
return {
dbSizeAfterBytes: 0,
dbSizeBeforeBytes: 0,
@@ -24,8 +30,28 @@ export function compactDoctorSessionSqliteTarget(
walSizeBeforeBytes: beforeFileSizes.walSizeBytes,
};
}
if (!stat.isFile()) {
throw new Error(`OpenClaw agent database is not a regular file: ${sqlitePath}`);
}
if (isOpenClawAgentDatabaseOpen(sqlitePath)) {
throw new Error(
`OpenClaw agent database ${sqlitePath} is already open in this process. Stop OpenClaw and retry.`,
);
}
const compact = compactDoctorSqliteFile({ sqlitePath });
const compact = compactDoctorSqliteFile({
afterMutation: () =>
ensureOpenClawAgentDatabasePermissions(sqlitePath, {
agentId: target.agentId,
path: sqlitePath,
}),
sqlitePath,
validateBeforeMutation: (database) =>
assertOpenClawAgentDatabaseForMaintenance(database, {
agentId: target.agentId,
pathname: sqlitePath,
}),
});
return {
dbSizeAfterBytes: compact.after.dbSizeBytes,
dbSizeBeforeBytes: compact.before.dbSizeBytes,
@@ -39,6 +65,17 @@ export function compactDoctorSessionSqliteTarget(
};
}
function readSessionDatabaseStat(sqlitePath: string): fs.Stats | undefined {
try {
return fs.lstatSync(sqlitePath);
} catch (error) {
if ((error as NodeJS.ErrnoException).code === "ENOENT") {
return undefined;
}
throw error;
}
}
function readSqliteFileSizes(sqlitePath: string): { dbSizeBytes: number; walSizeBytes: number } {
return {
dbSizeBytes: fileSize(sqlitePath),

View File

@@ -1,6 +1,13 @@
/** Builds doctor reports for session SQLite migration recovery mode. */
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import type { DatabaseSync } from "node:sqlite";
import type { SessionStoreTarget } from "../config/sessions/targets.js";
import { requireNodeSqlite } from "../infra/node-sqlite.js";
import { resolveSqliteDatabaseFilePaths } from "../infra/sqlite-files.js";
import { assertSqliteIntegrity } from "../infra/sqlite-integrity.js";
import { OPENCLAW_SQLITE_BUSY_TIMEOUT_MS } from "../state/openclaw-state-db.js";
import {
createSessionSqliteMigrationFailureIssue,
findLatestFailedSessionSqliteMigrationManifest,
@@ -9,7 +16,7 @@ import {
writeSessionSqliteMigrationFailureReports,
type SessionSqliteMigrationTargetInput,
} from "./doctor-session-sqlite-migration-run.js";
import { readOnlySqliteDbStats, resolveTargetSqlitePath } from "./doctor-session-sqlite-readers.js";
import { resolveTargetSqlitePath } from "./doctor-session-sqlite-readers.js";
import type {
DoctorSessionSqliteOptions,
DoctorSessionSqliteReport,
@@ -85,29 +92,76 @@ function recoverCorruptSqliteTargets(
): DoctorSessionSqliteTargetReport[] {
return targets.flatMap((target) => {
const sqlitePath = resolveTargetSqlitePath(target);
if (!fs.existsSync(sqlitePath)) {
let recoveryFiles: ReturnType<typeof inspectSqliteRecoveryFiles>;
try {
recoveryFiles = inspectSqliteRecoveryFiles(sqlitePath);
} catch (error) {
return [createRecoverInspectionFailureTargetReport(target, sqlitePath, error)];
}
if (recoveryFiles.existing.length === 0) {
return [];
}
const stats = readOnlySqliteDbStats(target);
if (stats.ok) {
if (stats.stats.integrityCheck && stats.stats.integrityCheck !== "ok") {
return [
recoverCorruptSqliteTarget(
target,
sqlitePath,
new Error(`SQLite quick_check reported: ${stats.stats.integrityCheck}`),
),
];
}
if (!recoveryFiles.existing.includes(sqlitePath)) {
return [
recoverCorruptSqliteTarget(
target,
sqlitePath,
new Error(`SQLite sidecars exist without their main database: ${sqlitePath}`),
),
];
}
const inspection = inspectSqliteForRecovery(sqlitePath, recoveryFiles.existing);
if (inspection.ok) {
return [];
}
if (!isSqliteCorruptionError(stats.error)) {
return [createRecoverInspectionFailureTargetReport(target, sqlitePath, stats.error)];
if (!isSqliteCorruptionError(inspection.error)) {
return [createRecoverInspectionFailureTargetReport(target, sqlitePath, inspection.error)];
}
return [recoverCorruptSqliteTarget(target, sqlitePath, stats.error)];
return [recoverCorruptSqliteTarget(target, sqlitePath, inspection.error)];
});
}
function inspectSqliteForRecovery(
sqlitePath: string,
sourcePaths: readonly string[],
): { ok: true } | { error: unknown; ok: false } {
let inspectionDir: string | undefined;
let database: DatabaseSync | undefined;
let inspectionError: unknown;
try {
const sqlite = requireNodeSqlite();
inspectionDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-sqlite-recovery-"));
const inspectionPath = path.join(inspectionDir, path.basename(sqlitePath));
for (const sourcePath of sourcePaths) {
const suffix = sourcePath.slice(sqlitePath.length);
const inspectionFilePath = `${inspectionPath}${suffix}`;
fs.copyFileSync(sourcePath, inspectionFilePath, fs.constants.COPYFILE_EXCL);
fs.chmodSync(inspectionFilePath, 0o600);
}
// Writable inspection of the disposable copy lets SQLite roll back a hot
// journal without changing the original forensic file set.
database = new sqlite.DatabaseSync(inspectionPath);
database.exec(`PRAGMA busy_timeout = ${OPENCLAW_SQLITE_BUSY_TIMEOUT_MS};`);
database.exec("PRAGMA trusted_schema = OFF;");
assertSqliteIntegrity(database, inspectionPath);
} catch (error) {
inspectionError = error;
}
try {
database?.close();
} catch (error) {
inspectionError ??= error;
}
try {
if (inspectionDir) {
fs.rmSync(inspectionDir, { force: true, recursive: true });
}
} catch (error) {
inspectionError ??= error;
}
return inspectionError === undefined ? { ok: true } : { error: inspectionError, ok: false };
}
function recoverCorruptSqliteTarget(
target: SessionStoreTarget,
sqlitePath: string,
@@ -142,29 +196,106 @@ function moveCorruptSqliteFilesAside(sqlitePath: string): {
movedFiles: string[];
skippedFiles: string[];
} {
const movedFiles: string[] = [];
const skippedFiles: string[] = [];
const suffix = `.corrupt-${Date.now()}`;
for (const candidate of [sqlitePath, `${sqlitePath}-wal`, `${sqlitePath}-shm`]) {
if (!fs.existsSync(candidate)) {
skippedFiles.push(candidate);
continue;
const recoveryFiles = inspectSqliteRecoveryFiles(sqlitePath);
const moves = planCorruptSqliteMoves(recoveryFiles.existing);
const completed: typeof moves = [];
try {
// Preserve every journal before removing the main pathname. Recovery is
// offline; rollback restores the set after a caught rename failure.
for (const move of moves.toSorted((left, right) => {
if (left.sourcePath === sqlitePath) {
return 1;
}
if (right.sourcePath === sqlitePath) {
return -1;
}
return left.sourcePath.localeCompare(right.sourcePath);
})) {
fs.renameSync(move.sourcePath, move.destinationPath);
completed.push(move);
}
const destination = uniqueRecoveryPath(`${candidate}${suffix}`);
fs.renameSync(candidate, destination);
movedFiles.push(destination);
} catch (error) {
const rollbackErrors: unknown[] = [];
for (const move of completed.toReversed()) {
try {
if (pathExists(move.sourcePath)) {
throw new Error(`rollback source was recreated: ${move.sourcePath}`, {
cause: error,
});
}
fs.renameSync(move.destinationPath, move.sourcePath);
} catch (rollbackError) {
rollbackErrors.push(rollbackError);
}
}
if (rollbackErrors.length > 0) {
const rollbackDetails = rollbackErrors
.map((rollbackError) => String(rollbackError))
.join("; ");
throw new Error(
`Could not move corrupt SQLite file set aside or restore it: ${sqlitePath}; rollback failures: ${rollbackDetails}`,
{ cause: error },
);
}
throw error;
}
return { movedFiles, skippedFiles };
return {
movedFiles: moves.map((move) => move.destinationPath),
skippedFiles: recoveryFiles.missing,
};
}
function uniqueRecoveryPath(basePath: string): string {
for (let attempt = 0; attempt < 100; attempt += 1) {
const candidate = attempt === 0 ? basePath : `${basePath}.${attempt}`;
if (!fs.existsSync(candidate)) {
return candidate;
function inspectSqliteRecoveryFiles(sqlitePath: string): {
existing: string[];
missing: string[];
} {
const existing: string[] = [];
const missing: string[] = [];
for (const candidate of resolveSqliteDatabaseFilePaths(sqlitePath)) {
try {
const stat = fs.lstatSync(candidate);
if (!stat.isFile()) {
throw new Error(`SQLite recovery path is not a regular file: ${candidate}`);
}
existing.push(candidate);
} catch (error) {
if ((error as NodeJS.ErrnoException).code === "ENOENT") {
missing.push(candidate);
continue;
}
throw error;
}
}
throw new Error(`Could not choose recovery path for ${basePath}`);
return { existing, missing };
}
function planCorruptSqliteMoves(
sourcePaths: readonly string[],
): Array<{ destinationPath: string; sourcePath: string }> {
const timestampSuffix = `.corrupt-${Date.now()}`;
for (let attempt = 0; attempt < 100; attempt += 1) {
const suffix = attempt === 0 ? timestampSuffix : `${timestampSuffix}.${attempt}`;
const moves = sourcePaths.map((sourcePath) => ({
destinationPath: `${sourcePath}${suffix}`,
sourcePath,
}));
if (moves.every((move) => !pathExists(move.destinationPath))) {
return moves;
}
}
throw new Error(`Could not choose recovery paths for ${sourcePaths[0] ?? "SQLite files"}`);
}
function pathExists(filePath: string): boolean {
try {
fs.lstatSync(filePath);
return true;
} catch (error) {
if ((error as NodeJS.ErrnoException).code === "ENOENT") {
return false;
}
throw error;
}
}
function isSqliteCorruptionError(error: unknown): boolean {
@@ -173,7 +304,13 @@ function isSqliteCorruptionError(error: unknown): boolean {
return true;
}
const message = String(error).toLowerCase();
return message.includes("database disk image is malformed") || message.includes("not a database");
return (
message.includes("database disk image is malformed") ||
message.includes("not a database") ||
message.includes("sqlite quick_check failed") ||
message.includes("sqlite integrity_check failed") ||
message.includes("sqlite foreign_key_check failed")
);
}
function resolveRecoverTargets(

View File

@@ -2,6 +2,7 @@
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import type { DatabaseSync } from "node:sqlite";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import {
loadExactSqliteSessionEntry,
@@ -9,9 +10,14 @@ import {
readSqliteTranscriptStatsSync,
upsertSqliteSessionEntry,
} from "../config/sessions/session-accessor.sqlite.js";
import { requireNodeSqlite } from "../infra/node-sqlite.js";
import * as nodeSqlite from "../infra/node-sqlite.js";
import * as replaceFile from "../infra/replace-file.js";
import { closeOpenClawAgentDatabasesForTest } from "../state/openclaw-agent-db.js";
import { resolveSqliteDatabaseFilePaths } from "../infra/sqlite-files.js";
import {
closeOpenClawAgentDatabasesForTest,
openOpenClawAgentDatabase,
OPENCLAW_AGENT_SCHEMA_VERSION,
} from "../state/openclaw-agent-db.js";
import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js";
import {
assertSafeSessionSqliteMigrationMove,
@@ -335,7 +341,7 @@ describe("runDoctorSessionSqlite", () => {
});
const sqlitePath = importReport.targets[0]?.sqlitePath;
expect(sqlitePath).toBeTruthy();
const sqlite = requireNodeSqlite();
const sqlite = nodeSqlite.requireNodeSqlite();
const db = new sqlite.DatabaseSync(sqlitePath ?? "");
try {
db.exec("DELETE FROM transcript_events;");
@@ -361,6 +367,162 @@ describe("runDoctorSessionSqlite", () => {
);
});
it("refuses compaction while this process owns an open agent database handle", async () => {
const { sqlitePath, store } = await createImportedStoreForCompaction();
openOpenClawAgentDatabase({
agentId: "main",
env: store.env,
path: sqlitePath,
});
const report = await runDoctorSessionSqlite({
env: store.env,
mode: "compact",
store: store.storePath,
});
expect(report.targets[0]?.issues).toEqual([
expect.objectContaining({
code: "sqlite_compact_failed",
message: expect.stringMatching(/already open in this process/iu),
}),
]);
});
it.each([
{
label: "wrong schema role",
mutate: (database: DatabaseSync) => {
database.prepare("UPDATE schema_meta SET role = 'global' WHERE meta_key = 'primary'").run();
},
message: /schema role global.*expected agent/iu,
},
{
label: "wrong agent owner",
mutate: (database: DatabaseSync) => {
database
.prepare("UPDATE schema_meta SET agent_id = 'work' WHERE meta_key = 'primary'")
.run();
},
message: /belongs to agent work.*requested agent main/iu,
},
{
label: "stale metadata version",
mutate: (database: DatabaseSync) => {
database
.prepare("UPDATE schema_meta SET schema_version = ? WHERE meta_key = 'primary'")
.run(OPENCLAW_AGENT_SCHEMA_VERSION - 1);
},
message: /metadata schema version .* does not match/iu,
},
{
label: "stale user version",
mutate: (database: DatabaseSync) => {
database.exec(`PRAGMA user_version = ${OPENCLAW_AGENT_SCHEMA_VERSION - 1};`);
},
message: /run openclaw doctor --fix before compacting/iu,
},
])("rejects $label before compaction", async ({ mutate, message }) => {
const { sqlitePath, store } = await createImportedStoreForCompaction();
const sqlite = nodeSqlite.requireNodeSqlite();
const database = new sqlite.DatabaseSync(sqlitePath);
try {
mutate(database);
} finally {
database.close();
}
const report = await runDoctorSessionSqlite({
env: store.env,
mode: "compact",
store: store.storePath,
});
expect(report.targets[0]?.issues).toEqual(
expect.arrayContaining([
expect.objectContaining({
code: "sqlite_compact_failed",
message: expect.stringMatching(message),
}),
]),
);
});
it.skipIf(process.platform === "win32")(
"refuses a symlink at the agent database path",
async () => {
const { sqlitePath, store } = await createImportedStoreForCompaction();
const realPath = `${sqlitePath}.real`;
fs.renameSync(sqlitePath, realPath);
fs.symlinkSync(realPath, sqlitePath);
const report = await runDoctorSessionSqlite({
env: store.env,
mode: "compact",
store: store.storePath,
});
expect(report.targets[0]?.issues).toEqual(
expect.arrayContaining([
expect.objectContaining({
code: "sqlite_compact_failed",
message: expect.stringMatching(/not a regular file/iu),
}),
]),
);
},
);
it.skipIf(process.platform === "win32")(
"reapplies owner-only permissions after compaction",
async () => {
const { sqlitePath, store } = await createImportedStoreForCompaction();
fs.chmodSync(sqlitePath, 0o666);
const report = await runDoctorSessionSqlite({
env: store.env,
mode: "compact",
store: store.storePath,
});
expect(report.totals.issues).toBe(0);
expect(fs.statSync(sqlitePath).mode & 0o777).toBe(0o600);
},
);
it("rejects stale secondary indexes before compacting and quarantines them in recovery", async () => {
const { sqlitePath, store } = await createImportedStoreForCompaction();
createUnsafeIndexDrift(sqlitePath);
const report = await runDoctorSessionSqlite({
env: store.env,
mode: "compact",
store: store.storePath,
});
expect(report.targets[0]?.issues).toEqual(
expect.arrayContaining([
expect.objectContaining({
code: "sqlite_compact_failed",
message: expect.stringMatching(
/integrity_check failed.*missing from index unsafe_session_index/iu,
),
}),
]),
);
const recovery = await runDoctorSessionSqlite({
env: store.env,
mode: "recover",
store: store.storePath,
});
expect(recovery.totals.issues).toBe(0);
expect(recovery.targets[0]?.corruptRecovery?.movedFiles).toEqual(
expect.arrayContaining([expect.stringMatching(/openclaw-agent\.sqlite\.corrupt-/u)]),
);
expect(fs.existsSync(sqlitePath)).toBe(false);
});
it("does not report SQLite markers as missing transcript files", async () => {
const store = createLegacyStore();
fs.rmSync(store.transcriptPath);
@@ -1639,6 +1801,8 @@ describe("runDoctorSessionSqlite", () => {
fs.mkdirSync(path.dirname(sqlitePath), { recursive: true });
fs.writeFileSync(sqlitePath, "not a sqlite database\n", { mode: 0o600 });
fs.writeFileSync(`${sqlitePath}-wal`, "wal", { mode: 0o600 });
fs.writeFileSync(`${sqlitePath}-shm`, "shm", { mode: 0o600 });
fs.writeFileSync(`${sqlitePath}-journal`, "journal", { mode: 0o600 });
const report = await runDoctorSessionSqlite({
env: store.env,
@@ -1647,15 +1811,128 @@ describe("runDoctorSessionSqlite", () => {
});
expect(report.totals.issues).toBe(0);
expect(report.targets[0]?.corruptRecovery?.movedFiles.length).toBeGreaterThanOrEqual(2);
expect(fs.existsSync(sqlitePath)).toBe(false);
expect(report.targets[0]?.corruptRecovery?.movedFiles).toHaveLength(4);
expect(report.targets[0]?.corruptRecovery?.skippedFiles).toEqual([]);
for (const candidate of resolveSqliteDatabaseFilePaths(sqlitePath)) {
expect(fs.existsSync(candidate)).toBe(false);
expect(
report.targets[0]?.corruptRecovery?.movedFiles.some((filePath) =>
filePath.startsWith(`${candidate}.corrupt-`),
),
).toBe(true);
}
});
it.skipIf(process.platform === "win32")(
"recovers owner-readable corrupt SQLite database files",
async () => {
const store = createLegacyStore();
const sqlitePath = path.join(
store.stateDir,
"agents",
"main",
"agent",
"openclaw-agent.sqlite",
);
fs.mkdirSync(path.dirname(sqlitePath), { recursive: true });
fs.writeFileSync(sqlitePath, "not a sqlite database\n", { mode: 0o400 });
const report = await runDoctorSessionSqlite({
env: store.env,
mode: "recover",
store: store.storePath,
});
expect(report.totals.issues).toBe(0);
expect(report.targets[0]?.corruptRecovery?.movedFiles).toEqual([
expect.stringMatching(/openclaw-agent\.sqlite\.corrupt-/u),
]);
expect(fs.existsSync(sqlitePath)).toBe(false);
},
);
it("moves orphaned SQLite sidecars aside during recovery", async () => {
const store = createLegacyStore();
const sqlitePath = path.join(
store.stateDir,
"agents",
"main",
"agent",
"openclaw-agent.sqlite",
);
fs.mkdirSync(path.dirname(sqlitePath), { recursive: true });
fs.writeFileSync(`${sqlitePath}-wal`, "wal", { mode: 0o600 });
fs.writeFileSync(`${sqlitePath}-journal`, "journal", { mode: 0o600 });
const report = await runDoctorSessionSqlite({
env: store.env,
mode: "recover",
store: store.storePath,
});
expect(report.totals.issues).toBe(0);
expect(report.targets[0]?.corruptRecovery?.movedFiles).toHaveLength(2);
expect(report.targets[0]?.corruptRecovery?.skippedFiles).toEqual([
sqlitePath,
`${sqlitePath}-shm`,
]);
expect(fs.existsSync(`${sqlitePath}-wal`)).toBe(false);
expect(fs.existsSync(`${sqlitePath}-shm`)).toBe(false);
expect(fs.existsSync(`${sqlitePath}-journal`)).toBe(false);
});
it("rolls back every completed corrupt-file move when a later rename fails", async () => {
const store = createLegacyStore();
const sqlitePath = path.join(
store.stateDir,
"agents",
"main",
"agent",
"openclaw-agent.sqlite",
);
fs.mkdirSync(path.dirname(sqlitePath), { recursive: true });
const expectedContents = new Map<string, string>();
for (const [candidate, contents] of [
[sqlitePath, "not a sqlite database\n"],
[`${sqlitePath}-wal`, "wal"],
[`${sqlitePath}-shm`, "shm"],
[`${sqlitePath}-journal`, "journal"],
] as const) {
fs.writeFileSync(candidate, contents, { mode: 0o600 });
expectedContents.set(candidate, contents);
}
const renameSync = fs.renameSync.bind(fs);
let renameCalls = 0;
const renameSpy = vi.spyOn(fs, "renameSync").mockImplementation((source, destination) => {
renameCalls += 1;
if (renameCalls === 2) {
throw new Error("forced corrupt recovery rename failure");
}
renameSync(source, destination);
});
let report: Awaited<ReturnType<typeof runDoctorSessionSqlite>> | undefined;
try {
report = await runDoctorSessionSqlite({
env: store.env,
mode: "recover",
store: store.storePath,
});
} finally {
renameSpy.mockRestore();
}
expect(report?.totals.issues).toBe(1);
expect(report?.targets[0]?.corruptRecovery).toBeUndefined();
expect(report?.targets[0]?.issues[0]).toMatchObject({
code: "sqlite_corrupt_recovery_failed",
message: expect.stringContaining("forced corrupt recovery rename failure"),
});
for (const [candidate, contents] of expectedContents) {
expect(fs.readFileSync(candidate, "utf8")).toBe(contents);
}
expect(
report.targets[0]?.corruptRecovery?.movedFiles.every((filePath) =>
filePath.includes(".corrupt-"),
),
).toBe(true);
fs.readdirSync(path.dirname(sqlitePath)).filter((entry) => entry.includes(".corrupt-")),
).toEqual([]);
});
it("does not move SQLite paths aside for non-corruption recovery inspection failures", async () => {
@@ -1681,6 +1958,41 @@ describe("runDoctorSessionSqlite", () => {
expect(fs.statSync(sqlitePath).isDirectory()).toBe(true);
});
it("reports SQLite loader failures without aborting recovery", async () => {
const store = createLegacyStore();
const sqlitePath = path.join(
store.stateDir,
"agents",
"main",
"agent",
"openclaw-agent.sqlite",
);
fs.mkdirSync(path.dirname(sqlitePath), { recursive: true });
fs.writeFileSync(sqlitePath, "not a sqlite database\n", { mode: 0o600 });
const requireSqlite = vi.spyOn(nodeSqlite, "requireNodeSqlite").mockImplementationOnce(() => {
throw new Error("node:sqlite unavailable");
});
let report: Awaited<ReturnType<typeof runDoctorSessionSqlite>> | undefined;
try {
report = await runDoctorSessionSqlite({
env: store.env,
mode: "recover",
store: store.storePath,
});
} finally {
requireSqlite.mockRestore();
}
expect(report?.totals.issues).toBe(1);
expect(report?.targets[0]?.issues[0]).toMatchObject({
code: "sqlite_recovery_inspect_failed",
message: expect.stringContaining("node:sqlite unavailable"),
});
expect(report?.targets[0]?.corruptRecovery).toBeUndefined();
expect(fs.existsSync(sqlitePath)).toBe(true);
});
it("does not truncate existing SQLite transcript rows when re-importing a duplicate fragment", async () => {
const store = createLegacyStore({
transcriptLines: [
@@ -1866,6 +2178,60 @@ describe("runDoctorSessionSqlite", () => {
});
});
async function createImportedStoreForCompaction(): Promise<{
sqlitePath: string;
store: TestStore;
}> {
const store = createLegacyStore();
const report = await runDoctorSessionSqlite({
env: store.env,
mode: "import",
store: store.storePath,
});
const sqlitePath = report.targets[0]?.sqlitePath;
if (!sqlitePath) {
throw new Error("expected imported agent SQLite path");
}
closeOpenClawAgentDatabasesForTest();
return { sqlitePath, store };
}
function createUnsafeIndexDrift(sqlitePath: string): void {
const sqlite = nodeSqlite.requireNodeSqlite();
const database = new sqlite.DatabaseSync(sqlitePath);
try {
database.exec(`
CREATE TABLE unsafe_session_index_records (
id INTEGER PRIMARY KEY,
indexed_value TEXT NOT NULL,
alternate_value TEXT NOT NULL
);
CREATE INDEX unsafe_session_index
ON unsafe_session_index_records(indexed_value);
INSERT INTO unsafe_session_index_records (indexed_value, alternate_value)
VALUES ('alpha', 'zeta'), ('beta', 'eta'), ('gamma', 'theta');
`);
database.enableDefensive?.(false);
database.exec("PRAGMA writable_schema = ON;");
database
.prepare(
"UPDATE sqlite_schema SET sql = 'CREATE INDEX unsafe_session_index ON unsafe_session_index_records(alternate_value)' WHERE name = 'unsafe_session_index'",
)
.run();
database.exec("PRAGMA writable_schema = OFF;");
const schemaVersionRow = database.prepare("PRAGMA schema_version;").get() as
| Record<string, unknown>
| undefined;
const schemaVersion = Number(
schemaVersionRow?.schema_version ??
(schemaVersionRow ? Object.values(schemaVersionRow)[0] : undefined),
);
database.exec(`PRAGMA schema_version = ${schemaVersion + 1};`);
} finally {
database.close();
}
}
function createLegacyStore(
params: {
agentDirName?: string;

View File

@@ -23,6 +23,7 @@ import type { SessionEntry } from "../config/sessions/types.js";
import type { OpenClawConfig } from "../config/types.openclaw.js";
import { resolveStoredSessionOwnerAgentId } from "../gateway/session-store-key.js";
import { normalizeAgentId } from "../routing/session-key.js";
import { closeOpenClawAgentDatabaseByPath } from "../state/openclaw-agent-db.js";
import { compactDoctorSessionSqliteTarget } from "./doctor-session-sqlite-compact.js";
import {
assertSafeSessionSqliteMigrationDirectory,
@@ -295,7 +296,7 @@ async function inspectOrMigrateTarget(params: {
if (validationPassed) {
// Post-import compact retrofits auto_vacuum=INCREMENTAL onto pre-flip
// databases and returns the pages the import churn freed.
compactSqliteDatabase(params.target, report);
compactSqliteDatabase(params.target, report, { closeImportedHandle: true });
}
}
report.unreferencedJsonlFiles = listUnreferencedJsonlFiles(params.target.storePath, [
@@ -983,8 +984,12 @@ function appendSqliteDbStats(
function compactSqliteDatabase(
target: SessionStoreTarget,
report: DoctorSessionSqliteTargetReport,
options: { closeImportedHandle?: boolean } = {},
): void {
try {
if (options.closeImportedHandle) {
closeOpenClawAgentDatabaseByPath(resolveTargetSqlitePath(target));
}
report.compact = compactDoctorSessionSqliteTarget(target);
} catch (err) {
report.issues.push({

View File

@@ -121,6 +121,7 @@ const cachedDatabases = new Map<string, OpenClawAgentDatabase>();
type ExistingSchemaMeta = {
agentId: string | null;
role: string | null;
schemaVersion: number | null;
};
type MigratedSessionEntry = Record<string, unknown>;
@@ -509,7 +510,7 @@ function backfillOpenClawAgentSchema(db: DatabaseSync, previousVersion: number):
}
}
function ensureOpenClawAgentDatabasePermissions(
export function ensureOpenClawAgentDatabasePermissions(
pathname: string,
options: OpenClawAgentDatabaseOptions,
): void {
@@ -540,14 +541,15 @@ function readExistingSchemaMeta(db: DatabaseSync): ExistingSchemaMeta | null {
return null;
}
const row = db
.prepare("SELECT role, agent_id FROM schema_meta WHERE meta_key = 'primary'")
.get() as { agent_id?: unknown; role?: unknown } | undefined;
.prepare("SELECT role, schema_version, agent_id FROM schema_meta WHERE meta_key = 'primary'")
.get() as { agent_id?: unknown; role?: unknown; schema_version?: unknown } | undefined;
if (!row) {
return null;
}
return {
agentId: typeof row.agent_id === "string" ? row.agent_id : null,
role: typeof row.role === "string" ? row.role : null,
schemaVersion: typeof row.schema_version === "number" ? row.schema_version : null,
};
}
@@ -575,6 +577,41 @@ function assertExistingSchemaOwner(
}
}
/** Require the exact agent owner and schema before offline file maintenance. */
export function assertOpenClawAgentDatabaseForMaintenance(
database: DatabaseSync,
options: { agentId: string; pathname: string },
): void {
const agentId = normalizeAgentId(options.agentId);
const metadata = readExistingSchemaMeta(database);
if (!metadata) {
throw new Error(
`OpenClaw agent database ${options.pathname} has no schema ownership metadata.`,
);
}
assertExistingSchemaOwner(metadata, agentId, options.pathname);
const userVersion = readSqliteUserVersion(database);
if (userVersion > OPENCLAW_AGENT_SCHEMA_VERSION) {
throw createNewerSqliteSchemaVersionError(
"OpenClaw agent database",
options.pathname,
userVersion,
OPENCLAW_AGENT_SCHEMA_VERSION,
);
}
if (userVersion !== OPENCLAW_AGENT_SCHEMA_VERSION) {
throw new Error(
`OpenClaw agent database ${options.pathname} uses schema version ${userVersion}; run openclaw doctor --fix before compacting it.`,
);
}
if (metadata.schemaVersion !== OPENCLAW_AGENT_SCHEMA_VERSION) {
throw new Error(
`OpenClaw agent database ${options.pathname} metadata schema version ${metadata.schemaVersion ?? "invalid"} does not match ${OPENCLAW_AGENT_SCHEMA_VERSION}; run openclaw doctor --fix before compacting it.`,
);
}
}
function ensureAgentSchema(db: DatabaseSync, agentId: string, pathname: string): void {
// FK enforcement must be off before BEGIN: PRAGMA foreign_keys is a silent
// no-op inside a transaction, and the v1 sessions rebuild would otherwise
@@ -953,6 +990,12 @@ function closeCachedOpenClawAgentDatabase(database: OpenClawAgentDatabase): void
}
}
/** Return whether the exact cached agent database pathname is still open. */
export function isOpenClawAgentDatabaseOpen(pathname: string): boolean {
const database = cachedDatabases.get(path.resolve(pathname));
return database?.db.isOpen === true;
}
/** Close one cached agent database identified by its exact resolved pathname. */
export function closeOpenClawAgentDatabaseByPath(pathname: string): boolean {
// Cache keys are lexical resolved paths. Do not realpath aliases here: a