From 46aa7aed3a71d963fabcdd25bfb2e3f3df60092d Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 12 Jul 2026 12:13:37 +0800 Subject: [PATCH] fix(sqlite): validate state before writable setup (#104952) --- src/state/openclaw-state-db.test.ts | 231 ++++++++++++++++++++++++++++ src/state/openclaw-state-db.ts | 17 +- 2 files changed, 246 insertions(+), 2 deletions(-) diff --git a/src/state/openclaw-state-db.test.ts b/src/state/openclaw-state-db.test.ts index 1357a1fe65b9..5739becf8542 100644 --- a/src/state/openclaw-state-db.test.ts +++ b/src/state/openclaw-state-db.test.ts @@ -27,6 +27,7 @@ import { OPENCLAW_STATE_SCHEMA_VERSION, repairOpenClawStateDatabaseSchema, runOpenClawStateWriteTransaction, + withOpenClawStateStartupMigrationCheckpointDatabase, } from "./openclaw-state-db.js"; import { resolveOpenClawStateSqlitePath } from "./openclaw-state-db.paths.js"; import { @@ -214,6 +215,156 @@ function insertAuditMarker( ); } +function createUnsafeIndexDrift(databasePath: string): void { + const { DatabaseSync } = requireNodeSqlite(); + const database = new DatabaseSync(databasePath); + try { + database.exec(` + CREATE TABLE unsafe_index_records ( + id INTEGER PRIMARY KEY, + indexed_value TEXT NOT NULL, + alternate_value TEXT NOT NULL + ); + CREATE INDEX unsafe_index_records_value ON unsafe_index_records(indexed_value); + INSERT INTO unsafe_index_records (indexed_value, alternate_value) + VALUES ('alpha', 'zeta'), ('beta', 'eta'), ('gamma', 'theta'); + `); + database.enableDefensive?.(false); + database.exec("PRAGMA writable_schema = ON;"); + database + .prepare( + "UPDATE sqlite_schema SET sql = 'CREATE INDEX unsafe_index_records_value ON unsafe_index_records(alternate_value)' WHERE name = 'unsafe_index_records_value'", + ) + .run(); + const schemaVersion = readSqliteNumberPragma(database, "schema_version"); + database.exec(`PRAGMA writable_schema = OFF; PRAGMA schema_version = ${schemaVersion + 1};`); + } finally { + database.close(); + } +} + +function runHotRollbackJournalRecoveryProbe(params: { moduleUrl: string; rootDir: string }): { + integrity: string; + journalExistsAfterRecovery: boolean; + value: string; +} { + const probeSource = ` + import { spawn } from "node:child_process"; + import fs from "node:fs"; + import path from "node:path"; + import { DatabaseSync } from "node:sqlite"; + + const moduleUrl = ${JSON.stringify(params.moduleUrl)}; + const databasePath = path.join(${JSON.stringify(params.rootDir)}, "hot-journal.sqlite"); + const readyPath = path.join(${JSON.stringify(params.rootDir)}, "writer-ready"); + const { + closeOpenClawStateDatabaseForTest, + openOpenClawStateDatabase, + } = await import(moduleUrl); + + const initial = openOpenClawStateDatabase({ path: databasePath }); + initial.db.exec(\` + CREATE TABLE hot_journal_probe ( + id INTEGER PRIMARY KEY, + value TEXT NOT NULL + ); + INSERT INTO hot_journal_probe (id, value) VALUES (1, 'committed'); + \`); + closeOpenClawStateDatabaseForTest(); + + const rollbackMode = new DatabaseSync(databasePath); + rollbackMode.exec("PRAGMA journal_mode = DELETE;"); + rollbackMode.close(); + + const writerSource = \` + import fs from "node:fs"; + import { DatabaseSync } from "node:sqlite"; + + const database = new DatabaseSync(process.env.OPENCLAW_HOT_JOURNAL_DATABASE_PATH); + database.exec("PRAGMA journal_mode = DELETE; PRAGMA synchronous = FULL; BEGIN IMMEDIATE;"); + database + .prepare("UPDATE hot_journal_probe SET value = ? WHERE id = 1") + .run("uncommitted"); + fs.writeFileSync(process.env.OPENCLAW_HOT_JOURNAL_READY_PATH, "ready"); + setInterval(() => {}, 1_000); + \`; + const writer = spawn( + process.execPath, + ["--input-type=module", "-e", writerSource], + { + env: { + ...process.env, + OPENCLAW_HOT_JOURNAL_DATABASE_PATH: databasePath, + OPENCLAW_HOT_JOURNAL_READY_PATH: readyPath, + }, + stdio: ["ignore", "ignore", "pipe"], + }, + ); + let writerStderr = ""; + writer.stderr.on("data", (chunk) => { + writerStderr += chunk; + }); + const writerClosed = new Promise((resolve, reject) => { + writer.once("error", reject); + writer.once("close", (code, signal) => resolve({ code, signal })); + }); + + try { + const deadline = Date.now() + 15_000; + while (!fs.existsSync(readyPath)) { + if (writer.exitCode !== null || writer.signalCode !== null) { + throw new Error(\`writer exited before creating a hot journal: \${writerStderr}\`); + } + if (Date.now() >= deadline) { + throw new Error("timed out waiting for hot rollback journal writer"); + } + await new Promise((resolve) => setTimeout(resolve, 2)); + } + const journalPath = \`\${databasePath}-journal\`; + if (!fs.existsSync(journalPath) || fs.statSync(journalPath).size === 0) { + throw new Error("writer did not leave a rollback journal"); + } + writer.kill("SIGKILL"); + const outcome = await writerClosed; + if (outcome.signal !== "SIGKILL") { + throw new Error(\`writer was not killed: \${JSON.stringify(outcome)} \${writerStderr}\`); + } + + const reopened = openOpenClawStateDatabase({ path: databasePath }); + const row = reopened.db + .prepare("SELECT value FROM hot_journal_probe WHERE id = 1") + .get(); + const integrity = reopened.db.prepare("PRAGMA integrity_check").get(); + closeOpenClawStateDatabaseForTest(); + console.log(JSON.stringify({ + integrity: integrity?.integrity_check, + journalExistsAfterRecovery: fs.existsSync(journalPath), + value: row?.value, + })); + } finally { + if (writer.exitCode === null && writer.signalCode === null) { + writer.kill("SIGKILL"); + await writerClosed; + } + closeOpenClawStateDatabaseForTest(); + } + `; + const output = execFileSync( + process.execPath, + ["--import", "tsx", "--input-type=module", "-e", probeSource], + { encoding: "utf8", timeout: 30_000 }, + ); + const resultLine = output.trim().split("\n").at(-1); + if (!resultLine) { + throw new Error("hot rollback journal recovery probe produced no result"); + } + return JSON.parse(resultLine) as { + integrity: string; + journalExistsAfterRecovery: boolean; + value: string; + }; +} + function expectNoncanonicalAuditSchemaRejected(stateDir: string, databasePath: string): void { const options = { env: { OPENCLAW_STATE_DIR: stateDir } }; expect(detectOpenClawStateDatabaseSchemaMigrations(options)).toEqual([ @@ -1013,6 +1164,86 @@ describe("openclaw state database", () => { expect(listOpenFileDescriptorsForPath(databasePath)).toEqual([]); }); + it("rejects stale secondary indexes before writable initialization", () => { + const stateDir = createTempStateDir(); + const databasePath = createCanonicalAuditStateDatabase(stateDir); + const options = { env: { OPENCLAW_STATE_DIR: stateDir } }; + createUnsafeIndexDrift(databasePath); + const { DatabaseSync } = requireNodeSqlite(); + const before = new DatabaseSync(databasePath, { readOnly: true }); + let metadataBefore: unknown; + try { + expect(before.prepare("PRAGMA quick_check").get()).toEqual({ quick_check: "ok" }); + expect(before.prepare("PRAGMA integrity_check").all()).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + integrity_check: expect.stringMatching(/missing from index unsafe_index_records_value/), + }), + ]), + ); + metadataBefore = before + .prepare( + "SELECT schema_version, updated_at FROM schema_meta WHERE meta_key = 'primary' LIMIT 1", + ) + .get(); + } finally { + before.close(); + } + + expect(() => openOpenClawStateDatabase(options)).toThrow( + /integrity_check failed.*missing from index unsafe_index_records_value/iu, + ); + expect(repairOpenClawStateDatabaseSchema(options)).toEqual({ + changes: [], + warnings: [ + expect.stringMatching( + /integrity_check failed.*missing from index unsafe_index_records_value/iu, + ), + ], + }); + const checkpointCallback = vi.fn(); + expect(() => + withOpenClawStateStartupMigrationCheckpointDatabase(checkpointCallback, options), + ).toThrow(/integrity_check failed.*missing from index unsafe_index_records_value/iu); + expect(checkpointCallback).not.toHaveBeenCalled(); + + const after = new DatabaseSync(databasePath, { readOnly: true }); + try { + expect( + after + .prepare( + "SELECT schema_version, updated_at FROM schema_meta WHERE meta_key = 'primary' LIMIT 1", + ) + .get(), + ).toEqual(metadataBefore); + expect(after.prepare("PRAGMA integrity_check").all()).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + integrity_check: expect.stringMatching(/missing from index unsafe_index_records_value/), + }), + ]), + ); + } finally { + after.close(); + } + }); + + it.skipIf(process.platform === "win32")( + "recovers a hot rollback journal before checking integrity", + () => { + expect( + runHotRollbackJournalRecoveryProbe({ + moduleUrl: new URL("./openclaw-state-db.ts", import.meta.url).href, + rootDir: createTempStateDir(), + }), + ).toEqual({ + integrity: "ok", + journalExistsAfterRecovery: false, + value: "committed", + }); + }, + ); + it("adds gateway boot lifecycle startup markers to existing state databases", () => { const stateDir = createTempStateDir(); const database = openOpenClawStateDatabase({ diff --git a/src/state/openclaw-state-db.ts b/src/state/openclaw-state-db.ts index 80bc59e67080..5fc80bb621be 100644 --- a/src/state/openclaw-state-db.ts +++ b/src/state/openclaw-state-db.ts @@ -11,6 +11,7 @@ import { import { requireNodeSqlite } from "../infra/node-sqlite.js"; import { applyPrivateModeSync } from "../infra/private-mode.js"; import { resolveSqliteDatabaseFilePaths } from "../infra/sqlite-files.js"; +import { assertSqliteIntegrity } from "../infra/sqlite-integrity.js"; import { runSqliteImmediateTransactionSync } from "../infra/sqlite-transaction.js"; import { createNewerSqliteSchemaVersionError, @@ -726,8 +727,8 @@ export function repairOpenClawStateDatabaseSchema(options: OpenClawStateDatabase ensureOpenClawStatePermissions(pathname, env); const sqlite = requireNodeSqlite(); const db = new sqlite.DatabaseSync(pathname); - db.exec(`PRAGMA busy_timeout = ${OPENCLAW_SQLITE_BUSY_TIMEOUT_MS};`); try { + assertStateDatabaseIntegrityBeforeMutation(db, pathname); assertSupportedSchemaVersion(db, pathname); const changes = runSqliteImmediateTransactionSync( db, @@ -814,8 +815,8 @@ export function withOpenClawStateStartupMigrationCheckpointDatabase( ensureOpenClawStatePermissions(pathname, env); const sqlite = requireNodeSqlite(); const db = new sqlite.DatabaseSync(pathname); - db.exec(`PRAGMA busy_timeout = ${OPENCLAW_SQLITE_BUSY_TIMEOUT_MS};`); try { + assertStateDatabaseIntegrityBeforeMutation(db, pathname); ensureStartupMigrationCheckpointSchema(db, pathname); return callback(db); } finally { @@ -1501,6 +1502,17 @@ function resolveDatabasePath(options: OpenClawStateDatabaseOptions = {}): string return path.resolve(options.path ?? resolveOpenClawStateSqlitePath(options.env ?? process.env)); } +function assertStateDatabaseIntegrityBeforeMutation( + database: DatabaseSync, + pathname: string, +): void { + database.exec(`PRAGMA busy_timeout = ${OPENCLAW_SQLITE_BUSY_TIMEOUT_MS};`); + // A writable handle lets SQLite recover a hot journal or WAL after an + // interrupted writer. OpenClaw mutations start only after recovery and a + // full table/index consistency check both succeed. + assertSqliteIntegrity(database, pathname); +} + /** Open or return a cached shared state database after schema and migration checks. */ export function openOpenClawStateDatabase( options: OpenClawStateDatabaseOptions = {}, @@ -1524,6 +1536,7 @@ export function openOpenClawStateDatabase( const walMaintenance = (() => { let maintenance: SqliteWalMaintenance | undefined; try { + assertStateDatabaseIntegrityBeforeMutation(db, pathname); configureSqlitePreSchemaPragmas(db, { busyTimeoutMs: OPENCLAW_SQLITE_BUSY_TIMEOUT_MS, });