mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-18 23:51:34 +00:00
fix(sqlite): preserve complete database file sets during recovery (#105370)
* fix(sqlite): harden agent database recovery * chore(sqlite): allow recovery maintenance primitive
This commit is contained in:
@@ -291,8 +291,10 @@ SQLite deletes reclaim pages inside the database first; they do not necessarily
|
||||
shrink the database file immediately. After deleting or archiving large
|
||||
transcripts, run `openclaw doctor --session-sqlite compact --session-sqlite-all-agents`
|
||||
to checkpoint WAL files, run `VACUUM`, and report before/after database and WAL
|
||||
sizes. This is explicit doctor maintenance so normal Gateway writes do not pause
|
||||
for background vacuum work.
|
||||
sizes. Compaction requires a regular file with the current agent schema, the
|
||||
selected agent's durable owner metadata, and no open handle in the doctor
|
||||
process. This is explicit offline maintenance: stop the Gateway first so normal
|
||||
writes cannot race the checkpoint or `VACUUM`.
|
||||
|
||||
Each import writes a manifest under
|
||||
`~/.openclaw/session-sqlite-migration-runs/` before moving transcript artifacts
|
||||
@@ -308,11 +310,19 @@ manifest's archived artifacts, validates the affected targets, refreshes the
|
||||
sanitized `.failure.md` and `.failure.json` reports, and prepares a GitHub issue
|
||||
body that avoids transcript contents, raw environment, secrets, and unbounded
|
||||
config. When no failed migration manifest exists but a selected agent SQLite
|
||||
database is corrupt or not a database, recovery preserves the DB, WAL, and SHM
|
||||
files by renaming them with a `.corrupt-<timestamp>` suffix so the next startup
|
||||
can create a fresh database. With `--github-issue --yes`, doctor uses the GitHub
|
||||
CLI to create the issue in `openclaw/openclaw`; without confirmation it writes
|
||||
the local support report and prints a prefilled issue URL.
|
||||
database is corrupt, not a database, or has journal sidecars without a main
|
||||
database, recovery copies the complete file set to a temporary inspection
|
||||
directory. SQLite can roll back a valid hot journal in that disposable copy
|
||||
before `quick_check`, `integrity_check`, and `foreign_key_check` run, while the
|
||||
original forensic files remain untouched. Failed integrity checks or orphaned
|
||||
sidecars preserve the DB, WAL, SHM, and rollback-journal files by renaming the
|
||||
whole discovered set with one `.corrupt-<timestamp>` suffix. A caught rename
|
||||
failure rolls already-moved files back before reporting failure, so a
|
||||
recoverable file set is not silently split. Stop the Gateway before recovery;
|
||||
copying or renaming an actively changing SQLite file set is unsafe and behaves
|
||||
differently across operating systems. With `--github-issue --yes`, doctor uses
|
||||
the GitHub CLI to create the issue in `openclaw/openclaw`; without confirmation
|
||||
it writes the local support report and prints a prefilled issue URL.
|
||||
|
||||
`restore` remains the lower-level undo operation. It uses manifest
|
||||
`sourcePath -> archivePath` records, moves archived artifacts back only when the
|
||||
|
||||
@@ -57,6 +57,7 @@ const rawSqliteAllowPathGroups = {
|
||||
"src/commands/doctor-sqlite-compact.ts",
|
||||
"src/commands/doctor-session-sqlite.ts",
|
||||
"src/commands/doctor-session-sqlite-readers.ts",
|
||||
"src/commands/doctor-session-sqlite-recover-report.ts",
|
||||
"src/commands/doctor-state-sqlite-compact.ts",
|
||||
"src/infra/state-migrations.ts",
|
||||
"src/infra/state-migrations.debug-proxy.ts",
|
||||
|
||||
@@ -1,6 +1,11 @@
|
||||
/** Runs doctor-owned SQLite file compaction for migrated session stores. */
|
||||
import fs from "node:fs";
|
||||
import type { SessionStoreTarget } from "../config/sessions/targets.js";
|
||||
import {
|
||||
assertOpenClawAgentDatabaseForMaintenance,
|
||||
ensureOpenClawAgentDatabasePermissions,
|
||||
isOpenClawAgentDatabaseOpen,
|
||||
} from "../state/openclaw-agent-db.js";
|
||||
import { resolveTargetSqlitePath } from "./doctor-session-sqlite-readers.js";
|
||||
import type { DoctorSessionSqliteCompactReport } from "./doctor-session-sqlite-types.js";
|
||||
import { compactDoctorSqliteFile } from "./doctor-sqlite-compact.js";
|
||||
@@ -11,7 +16,8 @@ export function compactDoctorSessionSqliteTarget(
|
||||
): DoctorSessionSqliteCompactReport {
|
||||
const sqlitePath = resolveTargetSqlitePath(target);
|
||||
const beforeFileSizes = readSqliteFileSizes(sqlitePath);
|
||||
if (!fs.existsSync(sqlitePath)) {
|
||||
const stat = readSessionDatabaseStat(sqlitePath);
|
||||
if (!stat) {
|
||||
return {
|
||||
dbSizeAfterBytes: 0,
|
||||
dbSizeBeforeBytes: 0,
|
||||
@@ -24,8 +30,28 @@ export function compactDoctorSessionSqliteTarget(
|
||||
walSizeBeforeBytes: beforeFileSizes.walSizeBytes,
|
||||
};
|
||||
}
|
||||
if (!stat.isFile()) {
|
||||
throw new Error(`OpenClaw agent database is not a regular file: ${sqlitePath}`);
|
||||
}
|
||||
if (isOpenClawAgentDatabaseOpen(sqlitePath)) {
|
||||
throw new Error(
|
||||
`OpenClaw agent database ${sqlitePath} is already open in this process. Stop OpenClaw and retry.`,
|
||||
);
|
||||
}
|
||||
|
||||
const compact = compactDoctorSqliteFile({ sqlitePath });
|
||||
const compact = compactDoctorSqliteFile({
|
||||
afterMutation: () =>
|
||||
ensureOpenClawAgentDatabasePermissions(sqlitePath, {
|
||||
agentId: target.agentId,
|
||||
path: sqlitePath,
|
||||
}),
|
||||
sqlitePath,
|
||||
validateBeforeMutation: (database) =>
|
||||
assertOpenClawAgentDatabaseForMaintenance(database, {
|
||||
agentId: target.agentId,
|
||||
pathname: sqlitePath,
|
||||
}),
|
||||
});
|
||||
return {
|
||||
dbSizeAfterBytes: compact.after.dbSizeBytes,
|
||||
dbSizeBeforeBytes: compact.before.dbSizeBytes,
|
||||
@@ -39,6 +65,17 @@ export function compactDoctorSessionSqliteTarget(
|
||||
};
|
||||
}
|
||||
|
||||
function readSessionDatabaseStat(sqlitePath: string): fs.Stats | undefined {
|
||||
try {
|
||||
return fs.lstatSync(sqlitePath);
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code === "ENOENT") {
|
||||
return undefined;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
function readSqliteFileSizes(sqlitePath: string): { dbSizeBytes: number; walSizeBytes: number } {
|
||||
return {
|
||||
dbSizeBytes: fileSize(sqlitePath),
|
||||
|
||||
@@ -1,6 +1,13 @@
|
||||
/** Builds doctor reports for session SQLite migration recovery mode. */
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import type { DatabaseSync } from "node:sqlite";
|
||||
import type { SessionStoreTarget } from "../config/sessions/targets.js";
|
||||
import { requireNodeSqlite } from "../infra/node-sqlite.js";
|
||||
import { resolveSqliteDatabaseFilePaths } from "../infra/sqlite-files.js";
|
||||
import { assertSqliteIntegrity } from "../infra/sqlite-integrity.js";
|
||||
import { OPENCLAW_SQLITE_BUSY_TIMEOUT_MS } from "../state/openclaw-state-db.js";
|
||||
import {
|
||||
createSessionSqliteMigrationFailureIssue,
|
||||
findLatestFailedSessionSqliteMigrationManifest,
|
||||
@@ -9,7 +16,7 @@ import {
|
||||
writeSessionSqliteMigrationFailureReports,
|
||||
type SessionSqliteMigrationTargetInput,
|
||||
} from "./doctor-session-sqlite-migration-run.js";
|
||||
import { readOnlySqliteDbStats, resolveTargetSqlitePath } from "./doctor-session-sqlite-readers.js";
|
||||
import { resolveTargetSqlitePath } from "./doctor-session-sqlite-readers.js";
|
||||
import type {
|
||||
DoctorSessionSqliteOptions,
|
||||
DoctorSessionSqliteReport,
|
||||
@@ -85,29 +92,76 @@ function recoverCorruptSqliteTargets(
|
||||
): DoctorSessionSqliteTargetReport[] {
|
||||
return targets.flatMap((target) => {
|
||||
const sqlitePath = resolveTargetSqlitePath(target);
|
||||
if (!fs.existsSync(sqlitePath)) {
|
||||
let recoveryFiles: ReturnType<typeof inspectSqliteRecoveryFiles>;
|
||||
try {
|
||||
recoveryFiles = inspectSqliteRecoveryFiles(sqlitePath);
|
||||
} catch (error) {
|
||||
return [createRecoverInspectionFailureTargetReport(target, sqlitePath, error)];
|
||||
}
|
||||
if (recoveryFiles.existing.length === 0) {
|
||||
return [];
|
||||
}
|
||||
const stats = readOnlySqliteDbStats(target);
|
||||
if (stats.ok) {
|
||||
if (stats.stats.integrityCheck && stats.stats.integrityCheck !== "ok") {
|
||||
return [
|
||||
recoverCorruptSqliteTarget(
|
||||
target,
|
||||
sqlitePath,
|
||||
new Error(`SQLite quick_check reported: ${stats.stats.integrityCheck}`),
|
||||
),
|
||||
];
|
||||
}
|
||||
if (!recoveryFiles.existing.includes(sqlitePath)) {
|
||||
return [
|
||||
recoverCorruptSqliteTarget(
|
||||
target,
|
||||
sqlitePath,
|
||||
new Error(`SQLite sidecars exist without their main database: ${sqlitePath}`),
|
||||
),
|
||||
];
|
||||
}
|
||||
const inspection = inspectSqliteForRecovery(sqlitePath, recoveryFiles.existing);
|
||||
if (inspection.ok) {
|
||||
return [];
|
||||
}
|
||||
if (!isSqliteCorruptionError(stats.error)) {
|
||||
return [createRecoverInspectionFailureTargetReport(target, sqlitePath, stats.error)];
|
||||
if (!isSqliteCorruptionError(inspection.error)) {
|
||||
return [createRecoverInspectionFailureTargetReport(target, sqlitePath, inspection.error)];
|
||||
}
|
||||
return [recoverCorruptSqliteTarget(target, sqlitePath, stats.error)];
|
||||
return [recoverCorruptSqliteTarget(target, sqlitePath, inspection.error)];
|
||||
});
|
||||
}
|
||||
|
||||
function inspectSqliteForRecovery(
|
||||
sqlitePath: string,
|
||||
sourcePaths: readonly string[],
|
||||
): { ok: true } | { error: unknown; ok: false } {
|
||||
let inspectionDir: string | undefined;
|
||||
let database: DatabaseSync | undefined;
|
||||
let inspectionError: unknown;
|
||||
try {
|
||||
const sqlite = requireNodeSqlite();
|
||||
inspectionDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-sqlite-recovery-"));
|
||||
const inspectionPath = path.join(inspectionDir, path.basename(sqlitePath));
|
||||
for (const sourcePath of sourcePaths) {
|
||||
const suffix = sourcePath.slice(sqlitePath.length);
|
||||
const inspectionFilePath = `${inspectionPath}${suffix}`;
|
||||
fs.copyFileSync(sourcePath, inspectionFilePath, fs.constants.COPYFILE_EXCL);
|
||||
fs.chmodSync(inspectionFilePath, 0o600);
|
||||
}
|
||||
// Writable inspection of the disposable copy lets SQLite roll back a hot
|
||||
// journal without changing the original forensic file set.
|
||||
database = new sqlite.DatabaseSync(inspectionPath);
|
||||
database.exec(`PRAGMA busy_timeout = ${OPENCLAW_SQLITE_BUSY_TIMEOUT_MS};`);
|
||||
database.exec("PRAGMA trusted_schema = OFF;");
|
||||
assertSqliteIntegrity(database, inspectionPath);
|
||||
} catch (error) {
|
||||
inspectionError = error;
|
||||
}
|
||||
try {
|
||||
database?.close();
|
||||
} catch (error) {
|
||||
inspectionError ??= error;
|
||||
}
|
||||
try {
|
||||
if (inspectionDir) {
|
||||
fs.rmSync(inspectionDir, { force: true, recursive: true });
|
||||
}
|
||||
} catch (error) {
|
||||
inspectionError ??= error;
|
||||
}
|
||||
return inspectionError === undefined ? { ok: true } : { error: inspectionError, ok: false };
|
||||
}
|
||||
|
||||
function recoverCorruptSqliteTarget(
|
||||
target: SessionStoreTarget,
|
||||
sqlitePath: string,
|
||||
@@ -142,29 +196,106 @@ function moveCorruptSqliteFilesAside(sqlitePath: string): {
|
||||
movedFiles: string[];
|
||||
skippedFiles: string[];
|
||||
} {
|
||||
const movedFiles: string[] = [];
|
||||
const skippedFiles: string[] = [];
|
||||
const suffix = `.corrupt-${Date.now()}`;
|
||||
for (const candidate of [sqlitePath, `${sqlitePath}-wal`, `${sqlitePath}-shm`]) {
|
||||
if (!fs.existsSync(candidate)) {
|
||||
skippedFiles.push(candidate);
|
||||
continue;
|
||||
const recoveryFiles = inspectSqliteRecoveryFiles(sqlitePath);
|
||||
const moves = planCorruptSqliteMoves(recoveryFiles.existing);
|
||||
const completed: typeof moves = [];
|
||||
try {
|
||||
// Preserve every journal before removing the main pathname. Recovery is
|
||||
// offline; rollback restores the set after a caught rename failure.
|
||||
for (const move of moves.toSorted((left, right) => {
|
||||
if (left.sourcePath === sqlitePath) {
|
||||
return 1;
|
||||
}
|
||||
if (right.sourcePath === sqlitePath) {
|
||||
return -1;
|
||||
}
|
||||
return left.sourcePath.localeCompare(right.sourcePath);
|
||||
})) {
|
||||
fs.renameSync(move.sourcePath, move.destinationPath);
|
||||
completed.push(move);
|
||||
}
|
||||
const destination = uniqueRecoveryPath(`${candidate}${suffix}`);
|
||||
fs.renameSync(candidate, destination);
|
||||
movedFiles.push(destination);
|
||||
} catch (error) {
|
||||
const rollbackErrors: unknown[] = [];
|
||||
for (const move of completed.toReversed()) {
|
||||
try {
|
||||
if (pathExists(move.sourcePath)) {
|
||||
throw new Error(`rollback source was recreated: ${move.sourcePath}`, {
|
||||
cause: error,
|
||||
});
|
||||
}
|
||||
fs.renameSync(move.destinationPath, move.sourcePath);
|
||||
} catch (rollbackError) {
|
||||
rollbackErrors.push(rollbackError);
|
||||
}
|
||||
}
|
||||
if (rollbackErrors.length > 0) {
|
||||
const rollbackDetails = rollbackErrors
|
||||
.map((rollbackError) => String(rollbackError))
|
||||
.join("; ");
|
||||
throw new Error(
|
||||
`Could not move corrupt SQLite file set aside or restore it: ${sqlitePath}; rollback failures: ${rollbackDetails}`,
|
||||
{ cause: error },
|
||||
);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
return { movedFiles, skippedFiles };
|
||||
return {
|
||||
movedFiles: moves.map((move) => move.destinationPath),
|
||||
skippedFiles: recoveryFiles.missing,
|
||||
};
|
||||
}
|
||||
|
||||
function uniqueRecoveryPath(basePath: string): string {
|
||||
for (let attempt = 0; attempt < 100; attempt += 1) {
|
||||
const candidate = attempt === 0 ? basePath : `${basePath}.${attempt}`;
|
||||
if (!fs.existsSync(candidate)) {
|
||||
return candidate;
|
||||
function inspectSqliteRecoveryFiles(sqlitePath: string): {
|
||||
existing: string[];
|
||||
missing: string[];
|
||||
} {
|
||||
const existing: string[] = [];
|
||||
const missing: string[] = [];
|
||||
for (const candidate of resolveSqliteDatabaseFilePaths(sqlitePath)) {
|
||||
try {
|
||||
const stat = fs.lstatSync(candidate);
|
||||
if (!stat.isFile()) {
|
||||
throw new Error(`SQLite recovery path is not a regular file: ${candidate}`);
|
||||
}
|
||||
existing.push(candidate);
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code === "ENOENT") {
|
||||
missing.push(candidate);
|
||||
continue;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
throw new Error(`Could not choose recovery path for ${basePath}`);
|
||||
return { existing, missing };
|
||||
}
|
||||
|
||||
function planCorruptSqliteMoves(
|
||||
sourcePaths: readonly string[],
|
||||
): Array<{ destinationPath: string; sourcePath: string }> {
|
||||
const timestampSuffix = `.corrupt-${Date.now()}`;
|
||||
for (let attempt = 0; attempt < 100; attempt += 1) {
|
||||
const suffix = attempt === 0 ? timestampSuffix : `${timestampSuffix}.${attempt}`;
|
||||
const moves = sourcePaths.map((sourcePath) => ({
|
||||
destinationPath: `${sourcePath}${suffix}`,
|
||||
sourcePath,
|
||||
}));
|
||||
if (moves.every((move) => !pathExists(move.destinationPath))) {
|
||||
return moves;
|
||||
}
|
||||
}
|
||||
throw new Error(`Could not choose recovery paths for ${sourcePaths[0] ?? "SQLite files"}`);
|
||||
}
|
||||
|
||||
function pathExists(filePath: string): boolean {
|
||||
try {
|
||||
fs.lstatSync(filePath);
|
||||
return true;
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code === "ENOENT") {
|
||||
return false;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
function isSqliteCorruptionError(error: unknown): boolean {
|
||||
@@ -173,7 +304,13 @@ function isSqliteCorruptionError(error: unknown): boolean {
|
||||
return true;
|
||||
}
|
||||
const message = String(error).toLowerCase();
|
||||
return message.includes("database disk image is malformed") || message.includes("not a database");
|
||||
return (
|
||||
message.includes("database disk image is malformed") ||
|
||||
message.includes("not a database") ||
|
||||
message.includes("sqlite quick_check failed") ||
|
||||
message.includes("sqlite integrity_check failed") ||
|
||||
message.includes("sqlite foreign_key_check failed")
|
||||
);
|
||||
}
|
||||
|
||||
function resolveRecoverTargets(
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import type { DatabaseSync } from "node:sqlite";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
loadExactSqliteSessionEntry,
|
||||
@@ -9,9 +10,14 @@ import {
|
||||
readSqliteTranscriptStatsSync,
|
||||
upsertSqliteSessionEntry,
|
||||
} from "../config/sessions/session-accessor.sqlite.js";
|
||||
import { requireNodeSqlite } from "../infra/node-sqlite.js";
|
||||
import * as nodeSqlite from "../infra/node-sqlite.js";
|
||||
import * as replaceFile from "../infra/replace-file.js";
|
||||
import { closeOpenClawAgentDatabasesForTest } from "../state/openclaw-agent-db.js";
|
||||
import { resolveSqliteDatabaseFilePaths } from "../infra/sqlite-files.js";
|
||||
import {
|
||||
closeOpenClawAgentDatabasesForTest,
|
||||
openOpenClawAgentDatabase,
|
||||
OPENCLAW_AGENT_SCHEMA_VERSION,
|
||||
} from "../state/openclaw-agent-db.js";
|
||||
import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js";
|
||||
import {
|
||||
assertSafeSessionSqliteMigrationMove,
|
||||
@@ -335,7 +341,7 @@ describe("runDoctorSessionSqlite", () => {
|
||||
});
|
||||
const sqlitePath = importReport.targets[0]?.sqlitePath;
|
||||
expect(sqlitePath).toBeTruthy();
|
||||
const sqlite = requireNodeSqlite();
|
||||
const sqlite = nodeSqlite.requireNodeSqlite();
|
||||
const db = new sqlite.DatabaseSync(sqlitePath ?? "");
|
||||
try {
|
||||
db.exec("DELETE FROM transcript_events;");
|
||||
@@ -361,6 +367,162 @@ describe("runDoctorSessionSqlite", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("refuses compaction while this process owns an open agent database handle", async () => {
|
||||
const { sqlitePath, store } = await createImportedStoreForCompaction();
|
||||
openOpenClawAgentDatabase({
|
||||
agentId: "main",
|
||||
env: store.env,
|
||||
path: sqlitePath,
|
||||
});
|
||||
|
||||
const report = await runDoctorSessionSqlite({
|
||||
env: store.env,
|
||||
mode: "compact",
|
||||
store: store.storePath,
|
||||
});
|
||||
|
||||
expect(report.targets[0]?.issues).toEqual([
|
||||
expect.objectContaining({
|
||||
code: "sqlite_compact_failed",
|
||||
message: expect.stringMatching(/already open in this process/iu),
|
||||
}),
|
||||
]);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{
|
||||
label: "wrong schema role",
|
||||
mutate: (database: DatabaseSync) => {
|
||||
database.prepare("UPDATE schema_meta SET role = 'global' WHERE meta_key = 'primary'").run();
|
||||
},
|
||||
message: /schema role global.*expected agent/iu,
|
||||
},
|
||||
{
|
||||
label: "wrong agent owner",
|
||||
mutate: (database: DatabaseSync) => {
|
||||
database
|
||||
.prepare("UPDATE schema_meta SET agent_id = 'work' WHERE meta_key = 'primary'")
|
||||
.run();
|
||||
},
|
||||
message: /belongs to agent work.*requested agent main/iu,
|
||||
},
|
||||
{
|
||||
label: "stale metadata version",
|
||||
mutate: (database: DatabaseSync) => {
|
||||
database
|
||||
.prepare("UPDATE schema_meta SET schema_version = ? WHERE meta_key = 'primary'")
|
||||
.run(OPENCLAW_AGENT_SCHEMA_VERSION - 1);
|
||||
},
|
||||
message: /metadata schema version .* does not match/iu,
|
||||
},
|
||||
{
|
||||
label: "stale user version",
|
||||
mutate: (database: DatabaseSync) => {
|
||||
database.exec(`PRAGMA user_version = ${OPENCLAW_AGENT_SCHEMA_VERSION - 1};`);
|
||||
},
|
||||
message: /run openclaw doctor --fix before compacting/iu,
|
||||
},
|
||||
])("rejects $label before compaction", async ({ mutate, message }) => {
|
||||
const { sqlitePath, store } = await createImportedStoreForCompaction();
|
||||
const sqlite = nodeSqlite.requireNodeSqlite();
|
||||
const database = new sqlite.DatabaseSync(sqlitePath);
|
||||
try {
|
||||
mutate(database);
|
||||
} finally {
|
||||
database.close();
|
||||
}
|
||||
|
||||
const report = await runDoctorSessionSqlite({
|
||||
env: store.env,
|
||||
mode: "compact",
|
||||
store: store.storePath,
|
||||
});
|
||||
|
||||
expect(report.targets[0]?.issues).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
code: "sqlite_compact_failed",
|
||||
message: expect.stringMatching(message),
|
||||
}),
|
||||
]),
|
||||
);
|
||||
});
|
||||
|
||||
it.skipIf(process.platform === "win32")(
|
||||
"refuses a symlink at the agent database path",
|
||||
async () => {
|
||||
const { sqlitePath, store } = await createImportedStoreForCompaction();
|
||||
const realPath = `${sqlitePath}.real`;
|
||||
fs.renameSync(sqlitePath, realPath);
|
||||
fs.symlinkSync(realPath, sqlitePath);
|
||||
|
||||
const report = await runDoctorSessionSqlite({
|
||||
env: store.env,
|
||||
mode: "compact",
|
||||
store: store.storePath,
|
||||
});
|
||||
|
||||
expect(report.targets[0]?.issues).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
code: "sqlite_compact_failed",
|
||||
message: expect.stringMatching(/not a regular file/iu),
|
||||
}),
|
||||
]),
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
it.skipIf(process.platform === "win32")(
|
||||
"reapplies owner-only permissions after compaction",
|
||||
async () => {
|
||||
const { sqlitePath, store } = await createImportedStoreForCompaction();
|
||||
fs.chmodSync(sqlitePath, 0o666);
|
||||
|
||||
const report = await runDoctorSessionSqlite({
|
||||
env: store.env,
|
||||
mode: "compact",
|
||||
store: store.storePath,
|
||||
});
|
||||
|
||||
expect(report.totals.issues).toBe(0);
|
||||
expect(fs.statSync(sqlitePath).mode & 0o777).toBe(0o600);
|
||||
},
|
||||
);
|
||||
|
||||
it("rejects stale secondary indexes before compacting and quarantines them in recovery", async () => {
|
||||
const { sqlitePath, store } = await createImportedStoreForCompaction();
|
||||
createUnsafeIndexDrift(sqlitePath);
|
||||
|
||||
const report = await runDoctorSessionSqlite({
|
||||
env: store.env,
|
||||
mode: "compact",
|
||||
store: store.storePath,
|
||||
});
|
||||
|
||||
expect(report.targets[0]?.issues).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
code: "sqlite_compact_failed",
|
||||
message: expect.stringMatching(
|
||||
/integrity_check failed.*missing from index unsafe_session_index/iu,
|
||||
),
|
||||
}),
|
||||
]),
|
||||
);
|
||||
|
||||
const recovery = await runDoctorSessionSqlite({
|
||||
env: store.env,
|
||||
mode: "recover",
|
||||
store: store.storePath,
|
||||
});
|
||||
expect(recovery.totals.issues).toBe(0);
|
||||
expect(recovery.targets[0]?.corruptRecovery?.movedFiles).toEqual(
|
||||
expect.arrayContaining([expect.stringMatching(/openclaw-agent\.sqlite\.corrupt-/u)]),
|
||||
);
|
||||
expect(fs.existsSync(sqlitePath)).toBe(false);
|
||||
});
|
||||
|
||||
it("does not report SQLite markers as missing transcript files", async () => {
|
||||
const store = createLegacyStore();
|
||||
fs.rmSync(store.transcriptPath);
|
||||
@@ -1639,6 +1801,8 @@ describe("runDoctorSessionSqlite", () => {
|
||||
fs.mkdirSync(path.dirname(sqlitePath), { recursive: true });
|
||||
fs.writeFileSync(sqlitePath, "not a sqlite database\n", { mode: 0o600 });
|
||||
fs.writeFileSync(`${sqlitePath}-wal`, "wal", { mode: 0o600 });
|
||||
fs.writeFileSync(`${sqlitePath}-shm`, "shm", { mode: 0o600 });
|
||||
fs.writeFileSync(`${sqlitePath}-journal`, "journal", { mode: 0o600 });
|
||||
|
||||
const report = await runDoctorSessionSqlite({
|
||||
env: store.env,
|
||||
@@ -1647,15 +1811,128 @@ describe("runDoctorSessionSqlite", () => {
|
||||
});
|
||||
|
||||
expect(report.totals.issues).toBe(0);
|
||||
expect(report.targets[0]?.corruptRecovery?.movedFiles.length).toBeGreaterThanOrEqual(2);
|
||||
expect(fs.existsSync(sqlitePath)).toBe(false);
|
||||
expect(report.targets[0]?.corruptRecovery?.movedFiles).toHaveLength(4);
|
||||
expect(report.targets[0]?.corruptRecovery?.skippedFiles).toEqual([]);
|
||||
for (const candidate of resolveSqliteDatabaseFilePaths(sqlitePath)) {
|
||||
expect(fs.existsSync(candidate)).toBe(false);
|
||||
expect(
|
||||
report.targets[0]?.corruptRecovery?.movedFiles.some((filePath) =>
|
||||
filePath.startsWith(`${candidate}.corrupt-`),
|
||||
),
|
||||
).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
it.skipIf(process.platform === "win32")(
|
||||
"recovers owner-readable corrupt SQLite database files",
|
||||
async () => {
|
||||
const store = createLegacyStore();
|
||||
const sqlitePath = path.join(
|
||||
store.stateDir,
|
||||
"agents",
|
||||
"main",
|
||||
"agent",
|
||||
"openclaw-agent.sqlite",
|
||||
);
|
||||
fs.mkdirSync(path.dirname(sqlitePath), { recursive: true });
|
||||
fs.writeFileSync(sqlitePath, "not a sqlite database\n", { mode: 0o400 });
|
||||
|
||||
const report = await runDoctorSessionSqlite({
|
||||
env: store.env,
|
||||
mode: "recover",
|
||||
store: store.storePath,
|
||||
});
|
||||
|
||||
expect(report.totals.issues).toBe(0);
|
||||
expect(report.targets[0]?.corruptRecovery?.movedFiles).toEqual([
|
||||
expect.stringMatching(/openclaw-agent\.sqlite\.corrupt-/u),
|
||||
]);
|
||||
expect(fs.existsSync(sqlitePath)).toBe(false);
|
||||
},
|
||||
);
|
||||
|
||||
it("moves orphaned SQLite sidecars aside during recovery", async () => {
|
||||
const store = createLegacyStore();
|
||||
const sqlitePath = path.join(
|
||||
store.stateDir,
|
||||
"agents",
|
||||
"main",
|
||||
"agent",
|
||||
"openclaw-agent.sqlite",
|
||||
);
|
||||
fs.mkdirSync(path.dirname(sqlitePath), { recursive: true });
|
||||
fs.writeFileSync(`${sqlitePath}-wal`, "wal", { mode: 0o600 });
|
||||
fs.writeFileSync(`${sqlitePath}-journal`, "journal", { mode: 0o600 });
|
||||
|
||||
const report = await runDoctorSessionSqlite({
|
||||
env: store.env,
|
||||
mode: "recover",
|
||||
store: store.storePath,
|
||||
});
|
||||
|
||||
expect(report.totals.issues).toBe(0);
|
||||
expect(report.targets[0]?.corruptRecovery?.movedFiles).toHaveLength(2);
|
||||
expect(report.targets[0]?.corruptRecovery?.skippedFiles).toEqual([
|
||||
sqlitePath,
|
||||
`${sqlitePath}-shm`,
|
||||
]);
|
||||
expect(fs.existsSync(`${sqlitePath}-wal`)).toBe(false);
|
||||
expect(fs.existsSync(`${sqlitePath}-shm`)).toBe(false);
|
||||
expect(fs.existsSync(`${sqlitePath}-journal`)).toBe(false);
|
||||
});
|
||||
|
||||
it("rolls back every completed corrupt-file move when a later rename fails", async () => {
|
||||
const store = createLegacyStore();
|
||||
const sqlitePath = path.join(
|
||||
store.stateDir,
|
||||
"agents",
|
||||
"main",
|
||||
"agent",
|
||||
"openclaw-agent.sqlite",
|
||||
);
|
||||
fs.mkdirSync(path.dirname(sqlitePath), { recursive: true });
|
||||
const expectedContents = new Map<string, string>();
|
||||
for (const [candidate, contents] of [
|
||||
[sqlitePath, "not a sqlite database\n"],
|
||||
[`${sqlitePath}-wal`, "wal"],
|
||||
[`${sqlitePath}-shm`, "shm"],
|
||||
[`${sqlitePath}-journal`, "journal"],
|
||||
] as const) {
|
||||
fs.writeFileSync(candidate, contents, { mode: 0o600 });
|
||||
expectedContents.set(candidate, contents);
|
||||
}
|
||||
const renameSync = fs.renameSync.bind(fs);
|
||||
let renameCalls = 0;
|
||||
const renameSpy = vi.spyOn(fs, "renameSync").mockImplementation((source, destination) => {
|
||||
renameCalls += 1;
|
||||
if (renameCalls === 2) {
|
||||
throw new Error("forced corrupt recovery rename failure");
|
||||
}
|
||||
renameSync(source, destination);
|
||||
});
|
||||
|
||||
let report: Awaited<ReturnType<typeof runDoctorSessionSqlite>> | undefined;
|
||||
try {
|
||||
report = await runDoctorSessionSqlite({
|
||||
env: store.env,
|
||||
mode: "recover",
|
||||
store: store.storePath,
|
||||
});
|
||||
} finally {
|
||||
renameSpy.mockRestore();
|
||||
}
|
||||
|
||||
expect(report?.totals.issues).toBe(1);
|
||||
expect(report?.targets[0]?.corruptRecovery).toBeUndefined();
|
||||
expect(report?.targets[0]?.issues[0]).toMatchObject({
|
||||
code: "sqlite_corrupt_recovery_failed",
|
||||
message: expect.stringContaining("forced corrupt recovery rename failure"),
|
||||
});
|
||||
for (const [candidate, contents] of expectedContents) {
|
||||
expect(fs.readFileSync(candidate, "utf8")).toBe(contents);
|
||||
}
|
||||
expect(
|
||||
report.targets[0]?.corruptRecovery?.movedFiles.every((filePath) =>
|
||||
filePath.includes(".corrupt-"),
|
||||
),
|
||||
).toBe(true);
|
||||
fs.readdirSync(path.dirname(sqlitePath)).filter((entry) => entry.includes(".corrupt-")),
|
||||
).toEqual([]);
|
||||
});
|
||||
|
||||
it("does not move SQLite paths aside for non-corruption recovery inspection failures", async () => {
|
||||
@@ -1681,6 +1958,41 @@ describe("runDoctorSessionSqlite", () => {
|
||||
expect(fs.statSync(sqlitePath).isDirectory()).toBe(true);
|
||||
});
|
||||
|
||||
it("reports SQLite loader failures without aborting recovery", async () => {
|
||||
const store = createLegacyStore();
|
||||
const sqlitePath = path.join(
|
||||
store.stateDir,
|
||||
"agents",
|
||||
"main",
|
||||
"agent",
|
||||
"openclaw-agent.sqlite",
|
||||
);
|
||||
fs.mkdirSync(path.dirname(sqlitePath), { recursive: true });
|
||||
fs.writeFileSync(sqlitePath, "not a sqlite database\n", { mode: 0o600 });
|
||||
const requireSqlite = vi.spyOn(nodeSqlite, "requireNodeSqlite").mockImplementationOnce(() => {
|
||||
throw new Error("node:sqlite unavailable");
|
||||
});
|
||||
|
||||
let report: Awaited<ReturnType<typeof runDoctorSessionSqlite>> | undefined;
|
||||
try {
|
||||
report = await runDoctorSessionSqlite({
|
||||
env: store.env,
|
||||
mode: "recover",
|
||||
store: store.storePath,
|
||||
});
|
||||
} finally {
|
||||
requireSqlite.mockRestore();
|
||||
}
|
||||
|
||||
expect(report?.totals.issues).toBe(1);
|
||||
expect(report?.targets[0]?.issues[0]).toMatchObject({
|
||||
code: "sqlite_recovery_inspect_failed",
|
||||
message: expect.stringContaining("node:sqlite unavailable"),
|
||||
});
|
||||
expect(report?.targets[0]?.corruptRecovery).toBeUndefined();
|
||||
expect(fs.existsSync(sqlitePath)).toBe(true);
|
||||
});
|
||||
|
||||
it("does not truncate existing SQLite transcript rows when re-importing a duplicate fragment", async () => {
|
||||
const store = createLegacyStore({
|
||||
transcriptLines: [
|
||||
@@ -1866,6 +2178,60 @@ describe("runDoctorSessionSqlite", () => {
|
||||
});
|
||||
});
|
||||
|
||||
async function createImportedStoreForCompaction(): Promise<{
|
||||
sqlitePath: string;
|
||||
store: TestStore;
|
||||
}> {
|
||||
const store = createLegacyStore();
|
||||
const report = await runDoctorSessionSqlite({
|
||||
env: store.env,
|
||||
mode: "import",
|
||||
store: store.storePath,
|
||||
});
|
||||
const sqlitePath = report.targets[0]?.sqlitePath;
|
||||
if (!sqlitePath) {
|
||||
throw new Error("expected imported agent SQLite path");
|
||||
}
|
||||
closeOpenClawAgentDatabasesForTest();
|
||||
return { sqlitePath, store };
|
||||
}
|
||||
|
||||
function createUnsafeIndexDrift(sqlitePath: string): void {
|
||||
const sqlite = nodeSqlite.requireNodeSqlite();
|
||||
const database = new sqlite.DatabaseSync(sqlitePath);
|
||||
try {
|
||||
database.exec(`
|
||||
CREATE TABLE unsafe_session_index_records (
|
||||
id INTEGER PRIMARY KEY,
|
||||
indexed_value TEXT NOT NULL,
|
||||
alternate_value TEXT NOT NULL
|
||||
);
|
||||
CREATE INDEX unsafe_session_index
|
||||
ON unsafe_session_index_records(indexed_value);
|
||||
INSERT INTO unsafe_session_index_records (indexed_value, alternate_value)
|
||||
VALUES ('alpha', 'zeta'), ('beta', 'eta'), ('gamma', 'theta');
|
||||
`);
|
||||
database.enableDefensive?.(false);
|
||||
database.exec("PRAGMA writable_schema = ON;");
|
||||
database
|
||||
.prepare(
|
||||
"UPDATE sqlite_schema SET sql = 'CREATE INDEX unsafe_session_index ON unsafe_session_index_records(alternate_value)' WHERE name = 'unsafe_session_index'",
|
||||
)
|
||||
.run();
|
||||
database.exec("PRAGMA writable_schema = OFF;");
|
||||
const schemaVersionRow = database.prepare("PRAGMA schema_version;").get() as
|
||||
| Record<string, unknown>
|
||||
| undefined;
|
||||
const schemaVersion = Number(
|
||||
schemaVersionRow?.schema_version ??
|
||||
(schemaVersionRow ? Object.values(schemaVersionRow)[0] : undefined),
|
||||
);
|
||||
database.exec(`PRAGMA schema_version = ${schemaVersion + 1};`);
|
||||
} finally {
|
||||
database.close();
|
||||
}
|
||||
}
|
||||
|
||||
function createLegacyStore(
|
||||
params: {
|
||||
agentDirName?: string;
|
||||
|
||||
@@ -23,6 +23,7 @@ import type { SessionEntry } from "../config/sessions/types.js";
|
||||
import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
import { resolveStoredSessionOwnerAgentId } from "../gateway/session-store-key.js";
|
||||
import { normalizeAgentId } from "../routing/session-key.js";
|
||||
import { closeOpenClawAgentDatabaseByPath } from "../state/openclaw-agent-db.js";
|
||||
import { compactDoctorSessionSqliteTarget } from "./doctor-session-sqlite-compact.js";
|
||||
import {
|
||||
assertSafeSessionSqliteMigrationDirectory,
|
||||
@@ -295,7 +296,7 @@ async function inspectOrMigrateTarget(params: {
|
||||
if (validationPassed) {
|
||||
// Post-import compact retrofits auto_vacuum=INCREMENTAL onto pre-flip
|
||||
// databases and returns the pages the import churn freed.
|
||||
compactSqliteDatabase(params.target, report);
|
||||
compactSqliteDatabase(params.target, report, { closeImportedHandle: true });
|
||||
}
|
||||
}
|
||||
report.unreferencedJsonlFiles = listUnreferencedJsonlFiles(params.target.storePath, [
|
||||
@@ -983,8 +984,12 @@ function appendSqliteDbStats(
|
||||
function compactSqliteDatabase(
|
||||
target: SessionStoreTarget,
|
||||
report: DoctorSessionSqliteTargetReport,
|
||||
options: { closeImportedHandle?: boolean } = {},
|
||||
): void {
|
||||
try {
|
||||
if (options.closeImportedHandle) {
|
||||
closeOpenClawAgentDatabaseByPath(resolveTargetSqlitePath(target));
|
||||
}
|
||||
report.compact = compactDoctorSessionSqliteTarget(target);
|
||||
} catch (err) {
|
||||
report.issues.push({
|
||||
|
||||
@@ -121,6 +121,7 @@ const cachedDatabases = new Map<string, OpenClawAgentDatabase>();
|
||||
type ExistingSchemaMeta = {
|
||||
agentId: string | null;
|
||||
role: string | null;
|
||||
schemaVersion: number | null;
|
||||
};
|
||||
|
||||
type MigratedSessionEntry = Record<string, unknown>;
|
||||
@@ -509,7 +510,7 @@ function backfillOpenClawAgentSchema(db: DatabaseSync, previousVersion: number):
|
||||
}
|
||||
}
|
||||
|
||||
function ensureOpenClawAgentDatabasePermissions(
|
||||
export function ensureOpenClawAgentDatabasePermissions(
|
||||
pathname: string,
|
||||
options: OpenClawAgentDatabaseOptions,
|
||||
): void {
|
||||
@@ -540,14 +541,15 @@ function readExistingSchemaMeta(db: DatabaseSync): ExistingSchemaMeta | null {
|
||||
return null;
|
||||
}
|
||||
const row = db
|
||||
.prepare("SELECT role, agent_id FROM schema_meta WHERE meta_key = 'primary'")
|
||||
.get() as { agent_id?: unknown; role?: unknown } | undefined;
|
||||
.prepare("SELECT role, schema_version, agent_id FROM schema_meta WHERE meta_key = 'primary'")
|
||||
.get() as { agent_id?: unknown; role?: unknown; schema_version?: unknown } | undefined;
|
||||
if (!row) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
agentId: typeof row.agent_id === "string" ? row.agent_id : null,
|
||||
role: typeof row.role === "string" ? row.role : null,
|
||||
schemaVersion: typeof row.schema_version === "number" ? row.schema_version : null,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -575,6 +577,41 @@ function assertExistingSchemaOwner(
|
||||
}
|
||||
}
|
||||
|
||||
/** Require the exact agent owner and schema before offline file maintenance. */
|
||||
export function assertOpenClawAgentDatabaseForMaintenance(
|
||||
database: DatabaseSync,
|
||||
options: { agentId: string; pathname: string },
|
||||
): void {
|
||||
const agentId = normalizeAgentId(options.agentId);
|
||||
const metadata = readExistingSchemaMeta(database);
|
||||
if (!metadata) {
|
||||
throw new Error(
|
||||
`OpenClaw agent database ${options.pathname} has no schema ownership metadata.`,
|
||||
);
|
||||
}
|
||||
assertExistingSchemaOwner(metadata, agentId, options.pathname);
|
||||
|
||||
const userVersion = readSqliteUserVersion(database);
|
||||
if (userVersion > OPENCLAW_AGENT_SCHEMA_VERSION) {
|
||||
throw createNewerSqliteSchemaVersionError(
|
||||
"OpenClaw agent database",
|
||||
options.pathname,
|
||||
userVersion,
|
||||
OPENCLAW_AGENT_SCHEMA_VERSION,
|
||||
);
|
||||
}
|
||||
if (userVersion !== OPENCLAW_AGENT_SCHEMA_VERSION) {
|
||||
throw new Error(
|
||||
`OpenClaw agent database ${options.pathname} uses schema version ${userVersion}; run openclaw doctor --fix before compacting it.`,
|
||||
);
|
||||
}
|
||||
if (metadata.schemaVersion !== OPENCLAW_AGENT_SCHEMA_VERSION) {
|
||||
throw new Error(
|
||||
`OpenClaw agent database ${options.pathname} metadata schema version ${metadata.schemaVersion ?? "invalid"} does not match ${OPENCLAW_AGENT_SCHEMA_VERSION}; run openclaw doctor --fix before compacting it.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function ensureAgentSchema(db: DatabaseSync, agentId: string, pathname: string): void {
|
||||
// FK enforcement must be off before BEGIN: PRAGMA foreign_keys is a silent
|
||||
// no-op inside a transaction, and the v1 sessions rebuild would otherwise
|
||||
@@ -953,6 +990,12 @@ function closeCachedOpenClawAgentDatabase(database: OpenClawAgentDatabase): void
|
||||
}
|
||||
}
|
||||
|
||||
/** Return whether the exact cached agent database pathname is still open. */
|
||||
export function isOpenClawAgentDatabaseOpen(pathname: string): boolean {
|
||||
const database = cachedDatabases.get(path.resolve(pathname));
|
||||
return database?.db.isOpen === true;
|
||||
}
|
||||
|
||||
/** Close one cached agent database identified by its exact resolved pathname. */
|
||||
export function closeOpenClawAgentDatabaseByPath(pathname: string): boolean {
|
||||
// Cache keys are lexical resolved paths. Do not realpath aliases here: a
|
||||
|
||||
Reference in New Issue
Block a user