diff --git a/docs/.generated/sqlite-session-transcript-schema-baseline.sha256 b/docs/.generated/sqlite-session-transcript-schema-baseline.sha256 index 5d339e202540..6557f58ccdaa 100644 --- a/docs/.generated/sqlite-session-transcript-schema-baseline.sha256 +++ b/docs/.generated/sqlite-session-transcript-schema-baseline.sha256 @@ -1 +1 @@ -5afc3eb779d73bf3a38ab2924d3d3a2095ce6e928a791d47fef7c27747cfb0f7 sqlite-session-transcript-schema-baseline.sql +fc43dca6d2a716398d23a4bbc327c9fd6c5d92094d4cf92865a37e40d60b0747 sqlite-session-transcript-schema-baseline.sql diff --git a/docs/cli/backup.md b/docs/cli/backup.md index 4c7c27b292e2..b86e00bd51f7 100644 --- a/docs/cli/backup.md +++ b/docs/cli/backup.md @@ -25,7 +25,7 @@ openclaw backup verify ./2026-03-09T08-00-00.000+08-00-openclaw-backup.tar.gz - The archive embeds a `manifest.json` with the resolved source paths and archive layout. - Default output is a timestamped `.tar.gz` archive in the current working directory. Timestamped filenames use your machine's local timezone and include the UTC offset. If the current working directory is inside a backed-up source tree, OpenClaw falls back to your home directory for the default archive location. - Existing archive files are never overwritten. Output paths inside the source state/workspace trees are rejected to avoid self-inclusion. -- `openclaw backup verify ` checks that the archive contains exactly one root manifest, rejects traversal-style archive paths, and confirms every manifest-declared payload exists in the tarball. `openclaw backup create --verify` runs that validation immediately after writing the archive. +- `openclaw backup verify ` checks that the archive contains exactly one root manifest, rejects traversal-style archive paths and SQLite sidecars, confirms every manifest-declared payload exists, validates every SQLite snapshot's file shape, and runs full integrity and role checks on canonical OpenClaw databases. Dedicated plugin schemas remain opaque because they may require owner-defined SQLite capabilities. `openclaw backup create --verify` runs that validation immediately after writing the archive. - `openclaw backup create --only-config` backs up just the active JSON config file. ## What gets backed up @@ -43,7 +43,7 @@ Auth profiles and other per-agent runtime state live in SQLite under the state d OpenClaw canonicalizes paths before building the archive: if config, the credentials directory, or a workspace already live inside the state directory, they are not duplicated as separate top-level backup sources. Missing paths are skipped. -During archive creation, OpenClaw skips known live-mutation files with no restoration value: active agent session transcripts, cron run logs, rolling logs, delivery queues, socket/pid/temp files under the state directory, and related durable-queue temp files. The JSON result's `skippedVolatileCount` reports how many files were intentionally omitted. SQLite databases under the state directory are snapshotted safely (`VACUUM INTO`) rather than copied live, so open WAL/SHM files do not corrupt the backup. +During archive creation, OpenClaw skips known live-mutation files with no restoration value: active agent session transcripts, cron run logs, rolling logs, delivery queues, socket/pid/temp files under the state directory, and related durable-queue temp files. The JSON result's `skippedVolatileCount` reports how many files were intentionally omitted. SQLite databases under the state directory are compacted with `VACUUM INTO` so deleted-page remnants do not enter the archive, and live WAL/SHM files are not copied. A plugin-owned database that requires unavailable owner-defined SQLite capabilities fails closed rather than falling back to a raw page copy. SQLite files included through workspace backups are copied as workspace files and are not covered by the compaction guarantee. Installed plugin source and manifest files under the state directory's `extensions/` tree are included, but their nested `node_modules/` dependency trees are skipped as rebuildable install artifacts. After restoring an archive, use `openclaw plugins update ` or reinstall with `openclaw plugins install --force` if a restored plugin reports missing dependencies. diff --git a/docs/cli/doctor.md b/docs/cli/doctor.md index 856e759c2537..30030b91f1f2 100644 --- a/docs/cli/doctor.md +++ b/docs/cli/doctor.md @@ -17,20 +17,15 @@ Related: ## Postures -| Posture | Command | Behavior | -| ------- | ------------------------ | --------------------------------------------------------------------------- | -| Inspect | `openclaw doctor` | Human-oriented checks and guided prompts. | -| Repair | `openclaw doctor --fix` | Applies supported repairs, prompting unless non-interactive repair is safe. | -| Lint | `openclaw doctor --lint` | Read-only structured findings for CI, preflight, and review gates. | +Doctor has five postures: -Doctor has four postures: - -| Posture | Command | Behavior | -| ------------------------ | ----------------------------------------- | ------------------------------------------------------------------------------- | -| Inspect | `openclaw doctor` | Human-oriented checks and guided prompts. | -| Repair | `openclaw doctor --fix` | Applies supported repairs, using prompts unless non-interactive repair is safe. | -| Lint | `openclaw doctor --lint` | Read-only structured findings for CI, preflight, and review gates. | -| Session SQLite migration | `openclaw doctor --session-sqlite ` | Inspects, imports, validates, compacts, recovers, or restores session state. | +| Posture | Command | Behavior | +| ------------------------- | ----------------------------------------- | ------------------------------------------------------------------------------- | +| Inspect | `openclaw doctor` | Human-oriented checks and guided prompts. | +| Repair | `openclaw doctor --fix` | Applies supported repairs, using prompts unless non-interactive repair is safe. | +| Lint | `openclaw doctor --lint` | Read-only structured findings for CI, preflight, and review gates. | +| Shared SQLite maintenance | `openclaw doctor --state-sqlite compact` | Explicitly checkpoints, compacts, and verifies the canonical shared state DB. | +| Session SQLite migration | `openclaw doctor --session-sqlite ` | Inspects, imports, validates, compacts, recovers, or restores session state. | Prefer `--lint` when automation needs a stable result. Prefer `--fix` when a human operator wants doctor to edit config or state. @@ -49,6 +44,8 @@ openclaw doctor --fix --non-interactive openclaw doctor --generate-gateway-token openclaw doctor --post-upgrade openclaw doctor --post-upgrade --json +openclaw doctor --state-sqlite compact +openclaw doctor --state-sqlite compact --json openclaw doctor --session-sqlite inspect --session-sqlite-all-agents openclaw doctor --session-sqlite dry-run --session-sqlite-agent main --json openclaw doctor --session-sqlite import --session-sqlite-all-agents @@ -81,18 +78,19 @@ openclaw channels status --probe | `--deep` | Scan system services for extra gateway installs; report recent Gateway supervisor restart handoffs. | | `--lint` | Run modernized health checks in read-only mode and emit diagnostic findings. | | `--post-upgrade` | Run post-upgrade plugin compatibility probes; findings go to stdout; exit code 1 if any error-level finding is present. | +| `--state-sqlite ` | Run explicit shared state SQLite maintenance. The only mode is `compact`. | | `--session-sqlite ` | Run the targeted session SQLite migration mode: `inspect`, `dry-run`, `import`, `validate`, `compact`, `recover`, or `restore`. | | `--session-sqlite-store ` | With `--session-sqlite`: select one legacy `sessions.json` store path. | | `--session-sqlite-agent ` | With `--session-sqlite`: select one configured agent. | | `--session-sqlite-all-agents` | With `--session-sqlite`: select configured and discovered agent stores. | | `--github-issue` | With `--session-sqlite recover`: prepare a sanitized openclaw/openclaw issue report; doctor creates it with `gh` after `--yes` or interactive confirmation. | -| `--json` | With `--lint`: JSON findings. With `--post-upgrade`: machine-readable envelope `{ probesRun, findings }`. With `--session-sqlite`: the migration report as JSON. | +| `--json` | With `--lint`: JSON findings. With `--post-upgrade`: `{ probesRun, findings }`. With `--state-sqlite` or `--session-sqlite`: the maintenance report as JSON. | | `--severity-min ` | With `--lint`: drop findings below `info`, `warning`, or `error`. | | `--all` | With `--lint`: run all registered checks, including opt-in checks excluded from the default set. | | `--skip ` | With `--lint`: skip a check id. Repeatable. | | `--only ` | With `--lint`: run only the given check id(s). Repeatable. | -`--severity-min`, `--all`, `--only`, and `--skip` are only accepted together with `--lint`; `--json` is accepted with `--lint`, `--post-upgrade`, and `--session-sqlite`. +`--severity-min`, `--all`, `--only`, and `--skip` are only accepted together with `--lint`; `--json` is accepted with `--lint`, `--post-upgrade`, `--state-sqlite`, and `--session-sqlite`. ## Lint mode @@ -199,6 +197,46 @@ finish safely, startup exits and tells you to run the same image once with `openclaw doctor --fix` against the same mounted state/config before restarting the container normally. +## Shared state SQLite compaction + +`openclaw doctor --state-sqlite compact` is explicit offline maintenance for +the canonical shared state database at +`/state/openclaw.sqlite`. It does not accept an arbitrary database +path, is never invoked by normal Gateway operation, and is not part of +`openclaw doctor --fix`. + +Stop the Gateway and create a verified backup first: + +```bash +openclaw gateway stop +openclaw backup create --verify +openclaw doctor --state-sqlite compact --json +openclaw gateway start +``` + +The command: + +1. Requires a regular file at the canonical shared-state path. A missing + database is reported as `skipped` and exits successfully. +2. Validates the current supported schema version and + `schema_meta.role = "global"` before checkpointing or changing the file. +3. Requires a non-busy `wal_checkpoint(TRUNCATE)`. Stop any remaining OpenClaw + process and retry if the checkpoint is busy. +4. Sets `auto_vacuum` to `INCREMENTAL`, runs a full `VACUUM`, and checkpoints + again. +5. Runs both `quick_check` and `integrity_check`, then reapplies owner-only + permissions to the database and SQLite sidecar files. + +JSON output reports the database and WAL sizes, freelist pages, page size, and +`auto_vacuum` value before and after compaction, plus reclaimed bytes and both +verification results. SQLite reports `auto_vacuum` as `0` for none, `1` for +full, and `2` for incremental. + +Compaction fails without mutation when the schema is old, newer than the +running OpenClaw build, or belongs to an agent database. Run +`openclaw doctor --fix` first for an older shared-state schema. Restore a +compatible backup or upgrade OpenClaw for a newer schema. + ## Session SQLite migration OpenClaw imports legacy session rows and transcript history into each agent's diff --git a/docs/docs_map.md b/docs/docs_map.md index a154ec7315e6..abb1f893a7e2 100644 --- a/docs/docs_map.md +++ b/docs/docs_map.md @@ -1491,6 +1491,7 @@ Do not edit it by hand; run `pnpm docs:map:gen`. - H2: Structured health checks - H2: Check selection - H2: Post-upgrade mode + - H2: Shared state SQLite compaction - H2: Session SQLite migration - H3: Downgrading After Session SQLite Migration - H2: Notes diff --git a/docs/refactor/database-first.md b/docs/refactor/database-first.md index 958e5d54dfa4..757a83993d57 100644 --- a/docs/refactor/database-first.md +++ b/docs/refactor/database-first.md @@ -1346,8 +1346,8 @@ sessionId})`; create, branch, continue, list, and fork flows live in their Runtime tests no longer create invalid or empty `runs.json` fixtures to prove registry behavior; they seed/read SQLite rows directly. - Backup stages the state directory before archiving, copies non-database files, - snapshots `*.sqlite` databases with `VACUUM INTO`, omits live WAL/SHM - sidecars, records snapshot metadata in the archive manifest, and records + snapshots databases with `VACUUM INTO`, omits live WAL/SHM sidecars, records + snapshot metadata in the archive manifest, and records completed backup runs in SQLite with the archive manifest. `openclaw backup create` validates the written archive by default; `--no-verify` is the explicit fast path. @@ -1864,12 +1864,14 @@ SQLite-native: 1. Stop long-running write activity or enter a short backup barrier. 2. For every global and agent database, run a checkpoint. -3. Snapshot each database using SQLite backup semantics or `VACUUM INTO` into a - temporary backup directory. -4. Archive the compacted database snapshots, config file, credentials directory, - selected workspaces, and a manifest. -5. Verify the archive by opening every included SQLite snapshot and running - `PRAGMA integrity_check`. +3. Snapshot databases with `VACUUM INTO` into a temporary backup directory. + Plugin schemas that require owner-defined SQLite capabilities fail closed + until the owner provides a safe snapshot contract. +4. Archive the database snapshots, config file, credentials directory, selected + workspaces, and a manifest. +5. Verify every SQLite snapshot's file shape, then open canonical OpenClaw + databases and run `PRAGMA integrity_check` plus role validation. Dedicated + plugin schemas remain opaque unless their owner supplies a verifier. `openclaw backup create` does this by default; `--no-verify` is only for intentionally skipping the post-write archive pass. @@ -1985,10 +1987,12 @@ payload. lifetime and cancellation behavior are boring. 8. Backup integration. - - Teach backup to snapshot global and agent databases via SQLite backup or - `VACUUM INTO`. Done for discovered `*.sqlite` files under the state asset. - - Add backup verification for SQLite integrity and schema version. Done for - backup creation and default archive verification integrity checks. + - Teach backup to snapshot global, agent, and plugin databases with + `VACUUM INTO`. Done for discovered `*.sqlite` files under the state asset; + plugin schemas requiring unavailable owner capabilities fail closed. + - Add backup verification for canonical SQLite integrity and schema identity, + plus generic file-shape validation for dedicated plugin snapshots. Done for + backup creation and default archive verification. - Record backup run metadata in SQLite. Done via the shared `backup_runs` table with archive path, status, and manifest JSON. - Add restore from verified archive snapshots. Done: `openclaw backup diff --git a/scripts/check-kysely-guardrails.mjs b/scripts/check-kysely-guardrails.mjs index 924e02fc6187..47bc59b6f666 100644 --- a/scripts/check-kysely-guardrails.mjs +++ b/scripts/check-kysely-guardrails.mjs @@ -51,11 +51,12 @@ const rawSqliteAllowPathGroups = { "src/commands/doctor-db-bloat.ts", "src/commands/status.scan.shared.ts", ], - "doctor legacy state migration": [ + "doctor SQLite maintenance and legacy state migration": [ "src/commands/doctor/cron/migration-ledger.ts", + "src/commands/doctor-sqlite-compact.ts", "src/commands/doctor-session-sqlite.ts", - "src/commands/doctor-session-sqlite-compact.ts", "src/commands/doctor-session-sqlite-readers.ts", + "src/commands/doctor-state-sqlite-compact.ts", "src/infra/state-migrations.ts", "src/infra/state-migrations.debug-proxy.ts", ], diff --git a/src/cli/program/register.maintenance.test.ts b/src/cli/program/register.maintenance.test.ts index d35a98cfb4f3..f47a5f36ffbb 100644 --- a/src/cli/program/register.maintenance.test.ts +++ b/src/cli/program/register.maintenance.test.ts @@ -182,6 +182,63 @@ describe("registerMaintenanceCommands doctor action", () => { expect(runtime.exit).toHaveBeenCalledWith(0); }); + it("passes shared-state sqlite compact mode and JSON output to doctor command", async () => { + doctorCommand.mockResolvedValue(undefined); + + await runMaintenanceCli(["doctor", "--state-sqlite", "compact", "--json"]); + + expect(doctorCommand).toHaveBeenCalledTimes(1); + const [, options] = commandCall(doctorCommand); + expect(options.stateSqlite).toBe("compact"); + expect(options.json).toBe(true); + expect(runtime.exit).toHaveBeenCalledWith(0); + }); + + it("rejects simultaneous shared-state and session SQLite modes", async () => { + await runMaintenanceCli(["doctor", "--state-sqlite", "compact", "--session-sqlite", "compact"]); + + expect(doctorCommand).not.toHaveBeenCalled(); + expect(runtime.error).toHaveBeenCalledWith( + "doctor shared-state SQLite maintenance can only be combined with --json.", + ); + expect(runtime.exit).toHaveBeenCalledWith(2); + }); + + it("rejects shared-state SQLite maintenance combined with lint mode", async () => { + await runMaintenanceCli(["doctor", "--state-sqlite", "compact", "--lint"]); + + expect(doctorCommand).not.toHaveBeenCalled(); + expect(runDoctorLintCli).not.toHaveBeenCalled(); + expect(runtime.error).toHaveBeenCalledWith( + "doctor shared-state SQLite maintenance can only be combined with --json.", + ); + expect(runtime.exit).toHaveBeenCalledWith(2); + }); + + it.each([ + ["workspace suggestions", ["--no-workspace-suggestions"]], + ["yes mode", ["--yes"]], + ["repair mode", ["--repair"]], + ["fix mode", ["--fix"]], + ["force mode", ["--force"]], + ["non-interactive mode", ["--non-interactive"]], + ["gateway token generation", ["--generate-gateway-token"]], + ["exec secret resolution", ["--allow-exec"]], + ["deep scans", ["--deep"]], + ["post-upgrade mode", ["--post-upgrade"]], + ["session SQLite selectors", ["--session-sqlite-agent", "main"]], + ["lint selectors", ["--only", "core/example"]], + ])("rejects shared-state SQLite maintenance combined with %s", async (_label, args) => { + await runMaintenanceCli(["doctor", "--state-sqlite", "compact", ...args]); + + expect(doctorCommand).not.toHaveBeenCalled(); + expect(runDoctorLintCli).not.toHaveBeenCalled(); + expect(runtime.error).toHaveBeenCalledWith( + "doctor shared-state SQLite maintenance can only be combined with --json.", + ); + expect(runtime.exit).toHaveBeenCalledWith(2); + }); + it("rejects session sqlite selectors without session sqlite mode", async () => { await runMaintenanceCli(["doctor", "--session-sqlite-agent", "main"]); diff --git a/src/cli/program/register.maintenance.ts b/src/cli/program/register.maintenance.ts index d5bb3b9e1730..28a6502555ea 100644 --- a/src/cli/program/register.maintenance.ts +++ b/src/cli/program/register.maintenance.ts @@ -5,6 +5,30 @@ import { theme } from "../../../packages/terminal-core/src/theme.js"; import { resolveDoctorCrossStateDirImports } from "../../commands/doctor-invocation.js"; import { defaultRuntime } from "../../runtime.js"; import { runCommandWithRuntime } from "../cli-utils.js"; +import { hasExplicitOptions } from "../command-options.js"; + +const STATE_SQLITE_CONFLICTING_OPTION_NAMES = [ + "workspaceSuggestions", + "yes", + "repair", + "fix", + "force", + "nonInteractive", + "generateGatewayToken", + "allowExec", + "deep", + "lint", + "postUpgrade", + "sessionSqlite", + "sessionSqliteStore", + "sessionSqliteAgent", + "sessionSqliteAllAgents", + "githubIssue", + "severityMin", + "all", + "skip", + "only", +] as const; /** Register maintenance commands that inspect or mutate local OpenClaw state. */ export function registerMaintenanceCommands(program: Command) { @@ -39,6 +63,7 @@ export function registerMaintenanceCommands(program: Command) { "--session-sqlite ", "Run session SQLite migration mode (dry-run|import|validate|inspect|compact|restore|recover)", ) + .option("--state-sqlite ", "Run shared state SQLite maintenance mode (compact)") .option("--session-sqlite-store ", "With --session-sqlite: inspect one session store") .option("--session-sqlite-agent ", "With --session-sqlite: inspect one agent") .option( @@ -53,7 +78,7 @@ export function registerMaintenanceCommands(program: Command) { ) .option( "--json", - "With --lint, --post-upgrade, or --session-sqlite: emit machine-readable JSON output", + "With --lint, --post-upgrade, --state-sqlite, or --session-sqlite: emit machine-readable JSON output", false, ) .option( @@ -73,7 +98,17 @@ export function registerMaintenanceCommands(program: Command) { (v: string, prev: string[]) => [...prev, v], [], ) - .action(async (opts) => { + .action(async (opts, command) => { + if ( + typeof opts.stateSqlite === "string" && + hasExplicitOptions(command, STATE_SQLITE_CONFLICTING_OPTION_NAMES) + ) { + defaultRuntime.error( + "doctor shared-state SQLite maintenance can only be combined with --json.", + ); + defaultRuntime.exit(2); + return; + } if (opts.lint === true) { await runCommandWithRuntime( defaultRuntime, @@ -113,6 +148,7 @@ export function registerMaintenanceCommands(program: Command) { } await runCommandWithRuntime(defaultRuntime, async () => { const { doctorCommand } = await import("../../commands/doctor.js"); + const stateSqlite = parseDoctorStateSqliteMode(opts.stateSqlite); const sessionSqlite = parseDoctorSessionSqliteMode(opts.sessionSqlite); await doctorCommand(defaultRuntime, { workspaceSuggestions: opts.workspaceSuggestions, @@ -124,6 +160,7 @@ export function registerMaintenanceCommands(program: Command) { allowExec: Boolean(opts.allowExec), deep: Boolean(opts.deep), postUpgrade: Boolean(opts.postUpgrade), + ...(stateSqlite ? { stateSqlite } : {}), ...(sessionSqlite ? { sessionSqlite } : {}), ...(typeof opts.sessionSqliteStore === "string" ? { sessionSqliteStore: opts.sessionSqliteStore } @@ -220,6 +257,7 @@ export function registerMaintenanceCommands(program: Command) { function hasLintOnlyDoctorOptions(opts: { readonly json?: boolean; readonly postUpgrade?: boolean; + readonly stateSqlite?: unknown; readonly sessionSqlite?: unknown; readonly severityMin?: unknown; readonly all?: boolean; @@ -227,7 +265,10 @@ function hasLintOnlyDoctorOptions(opts: { readonly only?: unknown; }): boolean { return ( - (opts.json === true && opts.postUpgrade !== true && typeof opts.sessionSqlite !== "string") || + (opts.json === true && + opts.postUpgrade !== true && + typeof opts.stateSqlite !== "string" && + typeof opts.sessionSqlite !== "string") || typeof opts.severityMin === "string" || opts.all === true || (Array.isArray(opts.skip) && opts.skip.length > 0) || @@ -251,6 +292,18 @@ function hasSessionSqliteOnlyDoctorOptions(opts: { ); } +function parseDoctorStateSqliteMode(value: unknown): "compact" | undefined { + if (value === undefined) { + return undefined; + } + if (value === "compact") { + return value; + } + defaultRuntime.error("Invalid --state-sqlite mode. Use compact."); + defaultRuntime.exit(2); + throw new Error("unreachable"); +} + function parseDoctorSessionSqliteMode( value: unknown, ): "dry-run" | "import" | "validate" | "inspect" | "compact" | "restore" | "recover" | undefined { diff --git a/src/commands/backup-verify.test.ts b/src/commands/backup-verify.test.ts index a05f245b75db..fcf6aa239f1b 100644 --- a/src/commands/backup-verify.test.ts +++ b/src/commands/backup-verify.test.ts @@ -2,13 +2,17 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; +import type { DatabaseSync } from "node:sqlite"; import { gzipSync } from "node:zlib"; import * as tar from "tar"; import { afterEach, describe, expect, it, vi } from "vitest"; +import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js"; +import { requireNodeSqlite } from "../infra/node-sqlite.js"; import { buildBackupArchiveRoot } from "./backup-shared.js"; -import { backupVerifyCommand } from "./backup-verify.js"; +import { backupVerifyCommand, testApi } from "./backup-verify.js"; const TEST_ARCHIVE_ROOT = "2026-03-09T00-00-00.000Z-openclaw-backup"; +const tempDirs = useAutoCleanupTempDirTracker(afterEach); const createBackupVerifyRuntime = () => ({ log: vi.fn(), @@ -16,7 +20,11 @@ const createBackupVerifyRuntime = () => ({ exit: vi.fn(), }); -function createBackupManifest(assetArchivePath: string, archiveRoot = TEST_ARCHIVE_ROOT) { +function createBackupManifest( + assetArchivePath: string, + archiveRoot = TEST_ARCHIVE_ROOT, + stateDir = "/tmp/.openclaw", +) { return { schemaVersion: 1, createdAt: "2026-03-09T00:00:00.000Z", @@ -24,10 +32,13 @@ function createBackupManifest(assetArchivePath: string, archiveRoot = TEST_ARCHI runtimeVersion: "test", platform: process.platform, nodeVersion: process.version, + paths: { + stateDir, + }, assets: [ { kind: "state", - sourcePath: "/tmp/.openclaw", + sourcePath: stateDir, archivePath: assetArchivePath, }, ], @@ -105,7 +116,8 @@ async function withBrokenArchiveFixture( options: { tempPrefix: string; manifestAssetArchivePath: string; - payloads: Array<{ fileName: string; contents: string; archivePath?: string }>; + manifest?: ReturnType; + payloads: Array<{ fileName: string; contents: string | Uint8Array; archivePath?: string }>; buildTarEntries?: (paths: { manifestPath: string; payloadPaths: string[] }) => string[]; }, run: (archivePath: string) => Promise, @@ -130,7 +142,7 @@ async function withBrokenArchiveFixture( try { await fs.writeFile( manifestPath, - `${JSON.stringify(createBackupManifest(options.manifestAssetArchivePath), null, 2)}\n`, + `${JSON.stringify(options.manifest ?? createBackupManifest(options.manifestAssetArchivePath), null, 2)}\n`, "utf8", ); await tar.c( @@ -161,6 +173,23 @@ async function withBrokenArchiveFixture( } } +async function createSqlitePayload(setup: (database: DatabaseSync) => void): Promise { + const tempDir = tempDirs.make("openclaw-backup-verify-sqlite-db-"); + const databasePath = path.join(tempDir, "snapshot.sqlite"); + try { + const sqlite = requireNodeSqlite(); + const database = new sqlite.DatabaseSync(databasePath); + try { + setup(database); + } finally { + database.close(); + } + return await fs.readFile(databasePath); + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } +} + describe("backupVerifyCommand", () => { afterEach(async () => { vi.restoreAllMocks(); @@ -210,6 +239,526 @@ describe("backupVerifyCommand", () => { } }); + it("verifies SQLite integrity and the canonical shared-state role", async () => { + const stateAssetArchivePath = `${TEST_ARCHIVE_ROOT}/payload/posix/tmp/.openclaw`; + const sqliteArchivePath = `${stateAssetArchivePath}/state/openclaw.sqlite`; + const sqlitePayload = await createSqlitePayload((database) => { + database.exec(` + CREATE TABLE schema_meta ( + meta_key TEXT NOT NULL PRIMARY KEY, + role TEXT NOT NULL + ); + INSERT INTO schema_meta (meta_key, role) VALUES ('primary', 'global'); + `); + }); + + await withBrokenArchiveFixture( + { + tempPrefix: "openclaw-backup-valid-sqlite-", + manifestAssetArchivePath: stateAssetArchivePath, + payloads: [ + { + fileName: "openclaw.sqlite", + contents: sqlitePayload, + archivePath: sqliteArchivePath, + }, + ], + }, + async (archivePath) => { + const runtime = createBackupVerifyRuntime(); + await expect(backupVerifyCommand(runtime, { archive: archivePath })).resolves.toMatchObject( + { + ok: true, + }, + ); + }, + ); + }); + + it("does not interpret plugin-owned SQLite schemas without their owner runtime", async () => { + const stateAssetArchivePath = `${TEST_ARCHIVE_ROOT}/payload/posix/tmp/.openclaw`; + const sqliteArchivePath = `${stateAssetArchivePath}/plugins/dedicated/custom.sqlite`; + const sqlitePayload = await createSqlitePayload((database) => { + database.function("plugin_double", { deterministic: true }, (value) => Number(value) * 2); + database.exec(` + CREATE TABLE records (value INTEGER NOT NULL); + INSERT INTO records (value) VALUES (1), (2); + CREATE INDEX records_double ON records(plugin_double(value)); + `); + }); + + await withBrokenArchiveFixture( + { + tempPrefix: "openclaw-backup-plugin-owned-sqlite-", + manifestAssetArchivePath: stateAssetArchivePath, + payloads: [ + { + fileName: "custom.sqlite", + contents: sqlitePayload, + archivePath: sqliteArchivePath, + }, + ], + }, + async (archivePath) => { + const runtime = createBackupVerifyRuntime(); + await expect(backupVerifyCommand(runtime, { archive: archivePath })).resolves.toMatchObject( + { + ok: true, + }, + ); + }, + ); + }); + + it("rejects a structurally valid archive containing a malformed SQLite snapshot", async () => { + const stateAssetArchivePath = `${TEST_ARCHIVE_ROOT}/payload/posix/tmp/.openclaw`; + const sqliteArchivePath = `${stateAssetArchivePath}/state/openclaw.sqlite`; + const invalidSqlite = Buffer.from("not a sqlite database", "utf8"); + expect(invalidSqlite.byteLength).toBe(21); + + await withBrokenArchiveFixture( + { + tempPrefix: "openclaw-backup-invalid-sqlite-", + manifestAssetArchivePath: stateAssetArchivePath, + payloads: [ + { + fileName: "openclaw.sqlite", + contents: invalidSqlite, + archivePath: sqliteArchivePath, + }, + ], + }, + async (archivePath) => { + const verificationTempRoot = tempDirs.make("openclaw-backup-verify-cleanup-"); + const tmpdirSpy = vi.spyOn(os, "tmpdir").mockReturnValue(verificationTempRoot); + try { + const runtime = createBackupVerifyRuntime(); + await expect(backupVerifyCommand(runtime, { archive: archivePath })).rejects.toThrow( + /Backup SQLite snapshot failed verification.*openclaw\.sqlite/iu, + ); + await expect(fs.readdir(verificationTempRoot)).resolves.toEqual([]); + } finally { + tmpdirSpy.mockRestore(); + await fs.rm(verificationTempRoot, { recursive: true, force: true }); + } + }, + ); + }); + + it("rejects an empty SQLite snapshot instead of accepting a new empty database", async () => { + const stateAssetArchivePath = `${TEST_ARCHIVE_ROOT}/payload/posix/tmp/.openclaw`; + const sqliteArchivePath = `${stateAssetArchivePath}/plugins/dedicated/empty.sqlite`; + + await withBrokenArchiveFixture( + { + tempPrefix: "openclaw-backup-empty-sqlite-", + manifestAssetArchivePath: stateAssetArchivePath, + payloads: [ + { + fileName: "empty.sqlite", + contents: new Uint8Array(), + archivePath: sqliteArchivePath, + }, + ], + }, + async (archivePath) => { + const runtime = createBackupVerifyRuntime(); + await expect(backupVerifyCommand(runtime, { archive: archivePath })).rejects.toThrow( + /SQLite snapshot is empty.*empty\.sqlite/iu, + ); + }, + ); + }); + + it.each(["-wal", "-WAL"])( + "rejects SQLite sidecars that could change restored snapshot contents (%s)", + async (sidecarSuffix) => { + const stateAssetArchivePath = `${TEST_ARCHIVE_ROOT}/payload/posix/tmp/.openclaw`; + const sqliteArchivePath = `${stateAssetArchivePath}/state/openclaw.sqlite`; + const sqlitePayload = await createSqlitePayload((database) => { + database.exec(` + CREATE TABLE schema_meta ( + meta_key TEXT NOT NULL PRIMARY KEY, + role TEXT NOT NULL + ); + INSERT INTO schema_meta (meta_key, role) VALUES ('primary', 'global'); + `); + }); + + await withBrokenArchiveFixture( + { + tempPrefix: "openclaw-backup-sqlite-sidecar-", + manifestAssetArchivePath: stateAssetArchivePath, + payloads: [ + { + fileName: "openclaw.sqlite", + contents: sqlitePayload, + archivePath: sqliteArchivePath, + }, + { + fileName: "openclaw.sqlite-wal", + contents: "unverified transaction data", + archivePath: `${sqliteArchivePath}${sidecarSuffix}`, + }, + ], + }, + async (archivePath) => { + const runtime = createBackupVerifyRuntime(); + await expect(backupVerifyCommand(runtime, { archive: archivePath })).rejects.toThrow( + /contains a SQLite snapshot sidecar.*openclaw\.sqlite-wal/iu, + ); + }, + ); + }, + ); + + it("rejects case-mangled canonical SQLite paths", async () => { + const stateAssetArchivePath = `${TEST_ARCHIVE_ROOT}/payload/posix/tmp/.openclaw`; + const sqliteArchivePath = `${stateAssetArchivePath}/State/OpenClaw.SQLITE`; + const sqlitePayload = await createSqlitePayload((database) => { + database.exec(` + CREATE TABLE schema_meta ( + meta_key TEXT NOT NULL PRIMARY KEY, + role TEXT NOT NULL + ); + INSERT INTO schema_meta (meta_key, role) VALUES ('primary', 'global'); + `); + }); + + await withBrokenArchiveFixture( + { + tempPrefix: "openclaw-backup-sqlite-case-alias-", + manifestAssetArchivePath: stateAssetArchivePath, + payloads: [ + { + fileName: "openclaw.sqlite", + contents: sqlitePayload, + archivePath: sqliteArchivePath, + }, + ], + }, + async (archivePath) => { + const runtime = createBackupVerifyRuntime(); + await expect(backupVerifyCommand(runtime, { archive: archivePath })).rejects.toThrow( + /case-mangled canonical SQLite path.*State\/OpenClaw\.SQLITE/u, + ); + }, + ); + }); + + it("rejects case-mangled aliases of the state asset root", async () => { + const stateAssetArchivePath = `${TEST_ARCHIVE_ROOT}/payload/posix/tmp/.openclaw`; + const statePayloadArchivePath = `${stateAssetArchivePath}/payload.txt`; + const aliasSidecarArchivePath = `${TEST_ARCHIVE_ROOT}/PAYLOAD/posix/tmp/.openclaw/plugins/dedicated/custom.sqlite-wal`; + + await withBrokenArchiveFixture( + { + tempPrefix: "openclaw-backup-state-root-case-alias-", + manifestAssetArchivePath: stateAssetArchivePath, + payloads: [ + { + fileName: "payload.txt", + contents: "payload\n", + archivePath: statePayloadArchivePath, + }, + { + fileName: "custom.sqlite-wal", + contents: "unverified transaction data", + archivePath: aliasSidecarArchivePath, + }, + ], + }, + async (archivePath) => { + const runtime = createBackupVerifyRuntime(); + await expect(backupVerifyCommand(runtime, { archive: archivePath })).rejects.toThrow( + /case-mangled state asset path.*PAYLOAD.*custom\.sqlite-wal/iu, + ); + }, + ); + }); + + it("rejects a truncated SQLite snapshot with a valid database header", async () => { + const stateAssetArchivePath = `${TEST_ARCHIVE_ROOT}/payload/posix/tmp/.openclaw`; + const sqliteArchivePath = `${stateAssetArchivePath}/plugins/dedicated/corrupt.sqlite`; + const sqlitePayload = await createSqlitePayload((database) => { + database.exec("CREATE TABLE records (id INTEGER PRIMARY KEY, value TEXT NOT NULL);"); + const insert = database.prepare("INSERT INTO records (value) VALUES (?)"); + for (let index = 0; index < 100; index += 1) { + insert.run(`record-${index}-${"x".repeat(100)}`); + } + }); + const encodedPageSize = sqlitePayload.readUInt16BE(16); + const pageSize = encodedPageSize === 1 ? 65_536 : encodedPageSize; + const truncatedPayload = sqlitePayload.subarray( + 0, + sqlitePayload.byteLength - Math.floor(pageSize / 2), + ); + expect(truncatedPayload.subarray(0, 16).toString("utf8")).toBe("SQLite format 3\u0000"); + expect(truncatedPayload.byteLength % pageSize).not.toBe(0); + + await withBrokenArchiveFixture( + { + tempPrefix: "openclaw-backup-corrupt-sqlite-", + manifestAssetArchivePath: stateAssetArchivePath, + payloads: [ + { + fileName: "corrupt.sqlite", + contents: truncatedPayload, + archivePath: sqliteArchivePath, + }, + ], + }, + async (archivePath) => { + const runtime = createBackupVerifyRuntime(); + await expect(backupVerifyCommand(runtime, { archive: archivePath })).rejects.toThrow( + /Backup SQLite snapshot failed verification.*corrupt\.sqlite/iu, + ); + }, + ); + }); + + it("rejects a page-aligned truncated plugin SQLite snapshot", async () => { + const stateAssetArchivePath = `${TEST_ARCHIVE_ROOT}/payload/posix/tmp/.openclaw`; + const sqliteArchivePath = `${stateAssetArchivePath}/plugins/dedicated/corrupt.sqlite`; + const sqlitePayload = await createSqlitePayload((database) => { + database.exec("CREATE TABLE records (id INTEGER PRIMARY KEY, value TEXT NOT NULL);"); + const insert = database.prepare("INSERT INTO records (value) VALUES (?)"); + for (let index = 0; index < 100; index += 1) { + insert.run(`record-${index}-${"x".repeat(100)}`); + } + }); + const encodedPageSize = sqlitePayload.readUInt16BE(16); + const pageSize = encodedPageSize === 1 ? 65_536 : encodedPageSize; + const declaredPageCount = sqlitePayload.readUInt32BE(28); + expect(sqlitePayload.readUInt32BE(24)).toBe(sqlitePayload.readUInt32BE(92)); + expect(declaredPageCount).toBeGreaterThan(1); + expect(declaredPageCount).toBe(sqlitePayload.byteLength / pageSize); + const truncatedPayload = sqlitePayload.subarray(0, sqlitePayload.byteLength - pageSize); + expect(truncatedPayload.byteLength % pageSize).toBe(0); + + await withBrokenArchiveFixture( + { + tempPrefix: "openclaw-backup-page-truncated-sqlite-", + manifestAssetArchivePath: stateAssetArchivePath, + payloads: [ + { + fileName: "corrupt.sqlite", + contents: truncatedPayload, + archivePath: sqliteArchivePath, + }, + ], + }, + async (archivePath) => { + const runtime = createBackupVerifyRuntime(); + await expect(backupVerifyCommand(runtime, { archive: archivePath })).rejects.toThrow( + /Backup SQLite snapshot failed verification.*corrupt\.sqlite/iu, + ); + }, + ); + }); + + it("rejects a canonical SQLite snapshot with the wrong database role", async () => { + const stateAssetArchivePath = `${TEST_ARCHIVE_ROOT}/payload/posix/tmp/.openclaw`; + const sqliteArchivePath = `${stateAssetArchivePath}/state/openclaw.sqlite`; + const sqlitePayload = await createSqlitePayload((database) => { + database.exec(` + CREATE TABLE schema_meta ( + meta_key TEXT NOT NULL PRIMARY KEY, + role TEXT NOT NULL + ); + INSERT INTO schema_meta (meta_key, role) VALUES ('primary', 'agent'); + `); + }); + + await withBrokenArchiveFixture( + { + tempPrefix: "openclaw-backup-wrong-sqlite-role-", + manifestAssetArchivePath: stateAssetArchivePath, + payloads: [ + { + fileName: "openclaw.sqlite", + contents: sqlitePayload, + archivePath: sqliteArchivePath, + }, + ], + }, + async (archivePath) => { + const runtime = createBackupVerifyRuntime(); + await expect(backupVerifyCommand(runtime, { archive: archivePath })).rejects.toThrow( + /has role agent; expected global/iu, + ); + }, + ); + }); + + it("validates a canonical agent database whose agent id is node_modules", async () => { + const stateAssetArchivePath = `${TEST_ARCHIVE_ROOT}/payload/posix/tmp/.openclaw`; + const sqliteArchivePath = `${stateAssetArchivePath}/agents/node_modules/agent/openclaw-agent.sqlite`; + const sqlitePayload = await createSqlitePayload((database) => { + database.exec(` + CREATE TABLE schema_meta ( + meta_key TEXT NOT NULL PRIMARY KEY, + role TEXT NOT NULL + ); + INSERT INTO schema_meta (meta_key, role) VALUES ('primary', 'global'); + `); + }); + + await withBrokenArchiveFixture( + { + tempPrefix: "openclaw-backup-agent-node-modules-", + manifestAssetArchivePath: stateAssetArchivePath, + payloads: [ + { + fileName: "openclaw-agent.sqlite", + contents: sqlitePayload, + archivePath: sqliteArchivePath, + }, + ], + }, + async (archivePath) => { + const runtime = createBackupVerifyRuntime(); + await expect(backupVerifyCommand(runtime, { archive: archivePath })).rejects.toThrow( + /has role global; expected agent/iu, + ); + }, + ); + }); + + it("rejects a state asset root that does not encode its declared source path", async () => { + const declaredStateAssetRoot = `${TEST_ARCHIVE_ROOT}/payload`; + const sqliteArchivePath = `${declaredStateAssetRoot}/posix/tmp/.openclaw/state/openclaw.sqlite`; + const sqlitePayload = await createSqlitePayload((database) => { + database.exec(` + CREATE TABLE schema_meta ( + meta_key TEXT NOT NULL PRIMARY KEY, + role TEXT NOT NULL + ); + INSERT INTO schema_meta (meta_key, role) VALUES ('primary', 'agent'); + `); + }); + + await withBrokenArchiveFixture( + { + tempPrefix: "openclaw-backup-state-root-bypass-", + manifestAssetArchivePath: declaredStateAssetRoot, + manifest: createBackupManifest(declaredStateAssetRoot), + payloads: [ + { + fileName: "openclaw.sqlite", + contents: sqlitePayload, + archivePath: sqliteArchivePath, + }, + ], + }, + async (archivePath) => { + const runtime = createBackupVerifyRuntime(); + await expect(backupVerifyCommand(runtime, { archive: archivePath })).rejects.toThrow( + /state asset archivePath does not match its sourcePath/iu, + ); + }, + ); + }); + + it("rejects SQLite extraction before writing when temporary space is insufficient", () => { + expect(() => + testApi.assertSqliteExtractionBudget({ + entries: [ + { + raw: "backup/payload/state/openclaw.sqlite", + normalized: "backup/payload/state/openclaw.sqlite", + stateAssetRoot: "backup/payload", + type: "File", + size: 2 * 1024 * 1024, + }, + ], + tempRoot: "/tmp", + readDiskSpace: () => ({ + targetPath: "/tmp", + checkedPath: "/tmp", + availableBytes: 128 * 1024 * 1024, + totalBytes: 1024 * 1024 * 1024, + }), + }), + ).toThrow(/only 128 MiB is available/iu); + }); + + it("rejects SQLite extraction beyond the verification hard limit", () => { + expect(() => + testApi.assertSqliteExtractionBudget({ + entries: [ + { + raw: "backup/payload/state/openclaw.sqlite", + normalized: "backup/payload/state/openclaw.sqlite", + stateAssetRoot: "backup/payload", + type: "File", + size: 64 * 1024 * 1024 * 1024 + 1, + }, + ], + tempRoot: "/tmp", + readDiskSpace: () => null, + }), + ).toThrow(/verification limit is 64 GiB/iu); + }); + + it("ignores package-owned and transient SQLite-shaped state files", async () => { + const stateAssetArchivePath = `${TEST_ARCHIVE_ROOT}/payload/posix/tmp/.openclaw`; + const transientId = "11111111-2222-3333-4444-555555555555"; + const invalidSqlite = "not a sqlite database"; + + await withBrokenArchiveFixture( + { + tempPrefix: "openclaw-backup-excluded-sqlite-", + manifestAssetArchivePath: stateAssetArchivePath, + payloads: [ + { + fileName: "root-fixture.sqlite", + contents: invalidSqlite, + archivePath: `${stateAssetArchivePath}/node_modules/root-dep/fixture.sqlite`, + }, + { + fileName: "root-fixture.sqlite-wal", + contents: invalidSqlite, + archivePath: `${stateAssetArchivePath}/node_modules/root-dep/fixture.sqlite-wal`, + }, + { + fileName: "managed-fixture.sqlite", + contents: invalidSqlite, + archivePath: `${stateAssetArchivePath}/npm/projects/demo/node_modules/dep/fixture.sqlite`, + }, + { + fileName: "reindex-lock.sqlite", + contents: invalidSqlite, + archivePath: `${stateAssetArchivePath}/memory/main.sqlite.reindex-lock.sqlite`, + }, + { + fileName: "reindex-tmp", + contents: invalidSqlite, + archivePath: `${stateAssetArchivePath}/memory/main.sqlite.tmp-${transientId}`, + }, + { + fileName: "reindex-backup", + contents: invalidSqlite, + archivePath: `${stateAssetArchivePath}/memory/main.sqlite.backup-${transientId}`, + }, + { + fileName: "memory-reindex", + contents: invalidSqlite, + archivePath: `${stateAssetArchivePath}/agents/main/agent.sqlite.memory-reindex-${transientId}`, + }, + ], + }, + async (archivePath) => { + const runtime = createBackupVerifyRuntime(); + await expect(backupVerifyCommand(runtime, { archive: archivePath })).resolves.toMatchObject( + { + ok: true, + }, + ); + }, + ); + }); + it("fails when the archive does not contain a manifest", async () => { const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-backup-no-manifest-")); const archivePath = path.join(tempDir, "broken.tar.gz"); @@ -520,6 +1069,18 @@ describe("backupVerifyCommand", () => { ], error: /duplicate entry path/i, }, + { + tempPrefix: "openclaw-backup-portable-path-collision-", + payloads: [ + { fileName: "payload-a.txt", contents: "payload-a\n", archivePath: payloadArchivePath }, + { + fileName: "payload-b.txt", + contents: "payload-b\n", + archivePath: payloadArchivePath.toUpperCase(), + }, + ], + error: /portable path collision/i, + }, ]) { await withBrokenArchiveFixture( { diff --git a/src/commands/backup-verify.ts b/src/commands/backup-verify.ts index 3e8197cab201..11dac378d838 100644 --- a/src/commands/backup-verify.ts +++ b/src/commands/backup-verify.ts @@ -1,12 +1,25 @@ // Verifies backup archives by validating their manifest, payload entries, and hardlink targets. +import fs from "node:fs/promises"; +import os from "node:os"; import path from "node:path"; +import type { DatabaseSync } from "node:sqlite"; import { readStringValue } from "@openclaw/normalization-core/string-coerce"; import * as tar from "tar"; +import { loadSqliteVecExtension } from "../../packages/memory-host-sdk/src/engine-storage.js"; +import { formatDiskSpaceBytes, tryReadDiskSpace } from "../infra/disk-space.js"; +import { requireNodeSqlite } from "../infra/node-sqlite.js"; import { type RuntimeEnv, writeRuntimeJson } from "../runtime.js"; import { isRecord, resolveUserPath } from "../utils.js"; +import { buildBackupArchivePath } from "./backup-shared.js"; const WINDOWS_ABSOLUTE_ARCHIVE_PATH_RE = /^[A-Za-z]:[\\/]/; const MAX_MANIFEST_BYTES = 1024 * 1024; +const MAX_SQLITE_SNAPSHOT_EXTRACT_BYTES = 64 * 1024 * 1024 * 1024; +const SQLITE_SNAPSHOT_FREE_SPACE_RESERVE_BYTES = 256 * 1024 * 1024; +const SQLITE_SNAPSHOT_SIDECAR_SUFFIXES = ["-wal", "-shm", "-journal"] as const; +const SQLITE_BACKUP_EXCLUDED_SUFFIXES = [".reindex-lock.sqlite"] as const; +const SQLITE_BACKUP_REINDEX_TRANSIENT_PATTERN = + /\.sqlite\.(?:backup|memory-reindex|tmp)-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/iu; type BackupManifestAsset = { kind: string; @@ -57,9 +70,23 @@ type BackupVerifyResult = { type ArchiveEntry = { path: string; linkpath?: string; + size?: number; type?: string; }; +type NormalizedArchiveEntry = { + raw: string; + normalized: string; + size?: number; + type?: string; +}; + +type SqliteSnapshotEntry = NormalizedArchiveEntry & { + stateAssetRoot: string; +}; + +type ExpectedSqliteRole = "agent" | "global"; + function stripTrailingSlashes(value: string): string { return value.replace(/\/+$/u, ""); } @@ -183,6 +210,7 @@ async function listArchiveEntries(archivePath: string): Promise entries.push({ path: entry.path, ...(entry.linkpath ? { linkpath: entry.linkpath } : {}), + ...(Number.isSafeInteger(entry.size) && entry.size >= 0 ? { size: entry.size } : {}), ...(entry.type ? { type: entry.type } : {}), }); entry.resume(); @@ -345,7 +373,408 @@ function findDuplicateNormalizedEntryPath( return undefined; } -/** Verify a backup archive without extracting payload files to disk. */ +function resolvePortableArchivePathKey(value: string): string { + return value.normalize("NFC").toLowerCase(); +} + +function findPortableArchiveEntryPathCollision( + entries: Array<{ normalized: string }>, +): { first: string; second: string } | undefined { + const seen = new Map(); + for (const entry of entries) { + const key = resolvePortableArchivePathKey(entry.normalized); + const first = seen.get(key); + if (first && first !== entry.normalized) { + return { first, second: entry.normalized }; + } + seen.set(key, entry.normalized); + } + return undefined; +} + +function isRegularArchiveFile(entryType: string | undefined): boolean { + return entryType === "File" || entryType === "OldFile" || entryType === "ContiguousFile"; +} + +function resolveCanonicalStateAssetRoot(manifest: BackupManifest): string | undefined { + const stateAssets = manifest.assets.filter((asset) => asset.kind === "state"); + if (stateAssets.length === 0) { + return undefined; + } + if (stateAssets.length !== 1) { + throw new Error( + `Backup manifest must contain at most one state asset; found ${stateAssets.length}.`, + ); + } + + const stateAsset = stateAssets[0]; + if (!stateAsset) { + return undefined; + } + + const stateAssetRoot = normalizeArchivePath( + stateAsset.archivePath, + "Backup manifest state asset path", + ); + const expectedStateAssetRoot = buildBackupArchivePath( + normalizeArchiveRoot(manifest.archiveRoot), + stateAsset.sourcePath, + ); + if (stateAssetRoot !== expectedStateAssetRoot) { + throw new Error("Backup manifest state asset archivePath does not match its sourcePath."); + } + return stateAssetRoot; +} + +function isSqliteSnapshotRelativePath(relativePath: string): boolean { + const portablePath = resolvePortableArchivePathKey(relativePath); + if (!portablePath.endsWith(".sqlite")) { + return false; + } + if (resolveExpectedSqliteRoleFromRelativePath(relativePath)) { + return true; + } + return ( + !portablePath.split("/").includes("node_modules") && + !SQLITE_BACKUP_REINDEX_TRANSIENT_PATTERN.test(relativePath) && + !SQLITE_BACKUP_EXCLUDED_SUFFIXES.some((suffix) => portablePath.endsWith(suffix)) + ); +} + +function resolveSqliteSnapshotSidecarDatabasePath(relativePath: string): string | undefined { + const portablePath = resolvePortableArchivePathKey(relativePath); + for (const suffix of SQLITE_SNAPSHOT_SIDECAR_SUFFIXES) { + if (portablePath.endsWith(suffix)) { + const databasePath = relativePath.slice(0, -suffix.length); + return isSqliteSnapshotRelativePath(databasePath) ? databasePath : undefined; + } + } + return undefined; +} + +function assertCanonicalSqlitePathCasing(relativePath: string, archivePath: string): void { + const segments = relativePath.split("/"); + const portablePath = resolvePortableArchivePathKey(relativePath); + const isGlobalAlias = + portablePath === "state/openclaw.sqlite" && relativePath !== "state/openclaw.sqlite"; + const isAgentAlias = + segments.length === 4 && + segments[0]?.toLowerCase() === "agents" && + Boolean(segments[1]) && + segments[2]?.toLowerCase() === "agent" && + segments[3]?.toLowerCase() === "openclaw-agent.sqlite" && + (segments[0] !== "agents" || + segments[2] !== "agent" || + segments[3] !== "openclaw-agent.sqlite"); + if (isGlobalAlias || isAgentAlias) { + throw new Error(`Backup contains a case-mangled canonical SQLite path: ${archivePath}`); + } +} + +function listSqliteSnapshotEntries( + manifest: BackupManifest, + entries: NormalizedArchiveEntry[], +): SqliteSnapshotEntry[] { + const declaredStateAssetRoots = manifest.assets + .filter((asset) => asset.kind === "state") + .map((asset) => normalizeArchivePath(asset.archivePath, "Backup manifest state asset path")); + for (const root of declaredStateAssetRoots) { + const portableRoot = resolvePortableArchivePathKey(root); + for (const entry of entries) { + const isExactStateEntry = isArchivePathWithin(entry.normalized, root); + const isPortableStateEntry = isArchivePathWithin( + resolvePortableArchivePathKey(entry.normalized), + portableRoot, + ); + if (isPortableStateEntry && !isExactStateEntry) { + throw new Error(`Backup contains a case-mangled state asset path: ${entry.normalized}`); + } + } + } + + const hasSqliteCandidate = entries.some((entry) => + declaredStateAssetRoots.some((root) => { + if (!isArchivePathWithin(entry.normalized, root)) { + return false; + } + const relativePath = path.posix.relative(root, entry.normalized); + return ( + isSqliteSnapshotRelativePath(relativePath) || + resolveSqliteSnapshotSidecarDatabasePath(relativePath) !== undefined + ); + }), + ); + if (!hasSqliteCandidate) { + return []; + } + + const stateAssetRoot = resolveCanonicalStateAssetRoot(manifest); + if (!stateAssetRoot) { + return []; + } + + for (const entry of entries) { + if (!isArchivePathWithin(entry.normalized, stateAssetRoot)) { + continue; + } + const relativePath = path.posix.relative(stateAssetRoot, entry.normalized); + assertCanonicalSqlitePathCasing(relativePath, entry.normalized); + if (resolveSqliteSnapshotSidecarDatabasePath(relativePath)) { + throw new Error(`Backup contains a SQLite snapshot sidecar: ${entry.normalized}`); + } + } + + return entries.flatMap((entry) => { + if (!isArchivePathWithin(entry.normalized, stateAssetRoot)) { + return []; + } + const relativePath = path.posix.relative(stateAssetRoot, entry.normalized); + // Only state-owned database snapshots should be opened during verification. + // Package content, excluded reindex artifacts, and noncanonical symlinks are + // preserved or skipped by backup creation without becoming SQLite snapshots. + if (!isSqliteSnapshotRelativePath(relativePath)) { + return []; + } + const candidate = { ...entry, stateAssetRoot }; + if (!resolveExpectedSqliteRole(candidate) && !isRegularArchiveFile(entry.type)) { + return []; + } + return [candidate]; + }); +} + +function resolveExpectedSqliteRole(entry: SqliteSnapshotEntry): ExpectedSqliteRole | undefined { + const relativePath = path.posix.relative(entry.stateAssetRoot, entry.normalized); + return resolveExpectedSqliteRoleFromRelativePath(relativePath); +} + +function resolveExpectedSqliteRoleFromRelativePath( + relativePath: string, +): ExpectedSqliteRole | undefined { + if (relativePath === "state/openclaw.sqlite") { + return "global"; + } + const segments = relativePath.split("/"); + if ( + segments.length === 4 && + segments[0] === "agents" && + segments[1] && + segments[2] === "agent" && + segments[3] === "openclaw-agent.sqlite" + ) { + return "agent"; + } + return undefined; +} + +function resolveSqliteExtractionBytes(entries: SqliteSnapshotEntry[]): number { + let totalBytes = 0; + for (const entry of entries) { + if (!Number.isSafeInteger(entry.size) || (entry.size ?? -1) < 0) { + throw new Error(`SQLite snapshot has an invalid archive size: ${entry.normalized}`); + } + if (entry.size === 0) { + throw new Error(`SQLite snapshot is empty: ${entry.normalized}`); + } + totalBytes += entry.size ?? 0; + if (!Number.isSafeInteger(totalBytes)) { + throw new Error("SQLite snapshot extraction size exceeds the supported integer range."); + } + } + return totalBytes; +} + +function assertSqliteExtractionBudget(params: { + entries: SqliteSnapshotEntry[]; + tempRoot: string; + readDiskSpace?: typeof tryReadDiskSpace; +}): void { + const totalBytes = resolveSqliteExtractionBytes(params.entries); + if (totalBytes > MAX_SQLITE_SNAPSHOT_EXTRACT_BYTES) { + throw new Error( + `SQLite snapshots require ${formatDiskSpaceBytes(totalBytes)} of extraction space; the verification limit is ${formatDiskSpaceBytes(MAX_SQLITE_SNAPSHOT_EXTRACT_BYTES)}.`, + ); + } + + const diskSpace = (params.readDiskSpace ?? tryReadDiskSpace)(params.tempRoot); + if ( + diskSpace && + totalBytes + SQLITE_SNAPSHOT_FREE_SPACE_RESERVE_BYTES > diskSpace.availableBytes + ) { + throw new Error( + `SQLite snapshots require ${formatDiskSpaceBytes(totalBytes)} of extraction space, but only ${formatDiskSpaceBytes(diskSpace.availableBytes)} is available near ${params.tempRoot}; verification reserves ${formatDiskSpaceBytes(SQLITE_SNAPSHOT_FREE_SPACE_RESERVE_BYTES)} for the host.`, + ); + } +} + +function assertSqliteCheckOk(params: { + database: DatabaseSync; + archivePath: string; + pragma: "integrity_check" | "quick_check"; +}): void { + const rows = params.database.prepare(`PRAGMA ${params.pragma}`).all() as Array< + Record + >; + const results = rows.map((row) => row[params.pragma]); + if (results.length === 0 || results.some((result) => result !== "ok")) { + const details = results.map((result) => String(result)).join("; ") || "no result"; + throw new Error(`SQLite ${params.pragma} failed for ${params.archivePath}: ${details}`); + } +} + +function assertExpectedSqliteRole( + database: DatabaseSync, + archivePath: string, + expectedRole: ExpectedSqliteRole, +): void { + const schemaMetaTable = database + .prepare("SELECT type FROM sqlite_schema WHERE name = 'schema_meta'") + .get() as { type?: unknown } | undefined; + if (schemaMetaTable?.type !== "table") { + throw new Error(`SQLite snapshot ${archivePath} is missing the expected schema_meta table.`); + } + + const metadata = database + .prepare("SELECT role FROM schema_meta WHERE meta_key = 'primary'") + .get() as { role?: unknown } | undefined; + const actualRole = typeof metadata?.role === "string" ? metadata.role : "missing"; + if (actualRole !== expectedRole) { + throw new Error( + `SQLite snapshot ${archivePath} has role ${actualRole}; expected ${expectedRole}.`, + ); + } +} + +async function assertSqliteSnapshotFileShape( + extractedPath: string, + archivePath: string, + expectedSize: number, +): Promise { + const header = Buffer.alloc(100); + const handle = await fs.open(extractedPath, "r"); + try { + const { bytesRead } = await handle.read(header, 0, header.byteLength, 0); + if ( + bytesRead !== header.byteLength || + header.subarray(0, 16).toString("utf8") !== "SQLite format 3\u0000" + ) { + throw new Error(`SQLite snapshot ${archivePath} has an invalid database header.`); + } + } finally { + await handle.close(); + } + + const encodedPageSize = header.readUInt16BE(16); + const pageSize = encodedPageSize === 1 ? 65_536 : encodedPageSize; + const validPageSize = pageSize >= 512 && pageSize <= 65_536 && (pageSize & (pageSize - 1)) === 0; + if (!validPageSize || expectedSize % pageSize !== 0) { + throw new Error(`SQLite snapshot ${archivePath} has an invalid page layout.`); + } + + const changeCounter = header.readUInt32BE(24); + const declaredPageCount = header.readUInt32BE(28); + const versionValidFor = header.readUInt32BE(92); + const hasAuthoritativePageCount = declaredPageCount !== 0 && changeCounter === versionValidFor; + if (hasAuthoritativePageCount && declaredPageCount !== expectedSize / pageSize) { + throw new Error(`SQLite snapshot ${archivePath} has an invalid page layout.`); + } +} + +async function verifySqliteSnapshots(params: { + archivePath: string; + entries: NormalizedArchiveEntry[]; + manifest: BackupManifest; +}): Promise { + const sqliteEntries = listSqliteSnapshotEntries(params.manifest, params.entries); + if (sqliteEntries.length === 0) { + return; + } + for (const entry of sqliteEntries) { + if (!isRegularArchiveFile(entry.type)) { + throw new Error(`SQLite snapshot must be a regular archive file: ${entry.normalized}`); + } + } + + const tempRoot = os.tmpdir(); + assertSqliteExtractionBudget({ entries: sqliteEntries, tempRoot }); + const tempDir = await fs.mkdtemp(path.join(tempRoot, "openclaw-backup-verify-sqlite-")); + try { + const sqliteEntriesByRawPath = new Map(sqliteEntries.map((entry) => [entry.raw, entry])); + await tar.x({ + file: params.archivePath, + gzip: true, + cwd: tempDir, + strict: true, + preserveOwner: false, + noChmod: true, + filter: (entryPath, archiveEntry) => { + const expected = sqliteEntriesByRawPath.get(entryPath); + if (!expected) { + return false; + } + if (archiveEntry.size !== expected.size) { + throw new Error(`SQLite snapshot size changed during verification: ${entryPath}`); + } + return true; + }, + }); + + for (const entry of sqliteEntries) { + const extractedPath = path.join(tempDir, ...entry.normalized.split("/")); + const extractedStat = await fs.lstat(extractedPath); + if (!extractedStat.isFile()) { + throw new Error(`Extracted SQLite snapshot is not a regular file: ${entry.normalized}`); + } + if (extractedStat.size !== entry.size) { + throw new Error( + `Extracted SQLite snapshot size does not match archive: ${entry.normalized}`, + ); + } + + let database: DatabaseSync | undefined; + try { + await assertSqliteSnapshotFileShape(extractedPath, entry.normalized, extractedStat.size); + const expectedRole = resolveExpectedSqliteRole(entry); + if (!expectedRole) { + // Plugin-owned databases may require owner-specific functions, + // collations, or virtual-table modules. Core can validate their + // snapshot shape, but only canonical schemas are safe to interpret. + continue; + } + const sqlite = requireNodeSqlite(); + database = new sqlite.DatabaseSync(extractedPath, { + allowExtension: true, + readOnly: true, + }); + database.exec("PRAGMA query_only = ON; PRAGMA trusted_schema = OFF;"); + await loadSqliteVecExtension({ db: database }); + assertSqliteCheckOk({ + database, + archivePath: entry.normalized, + pragma: "quick_check", + }); + assertSqliteCheckOk({ + database, + archivePath: entry.normalized, + pragma: "integrity_check", + }); + assertExpectedSqliteRole(database, entry.normalized, expectedRole); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + throw new Error( + `Backup SQLite snapshot failed verification: ${entry.normalized}. ${message}`, + { cause: err }, + ); + } finally { + database?.close(); + } + } + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } +} + +/** Verify a backup archive, including snapshot shape and canonical SQLite integrity checks. */ export async function backupVerifyCommand( runtime: RuntimeEnv, opts: BackupVerifyOptions, @@ -359,6 +788,8 @@ export async function backupVerifyCommand( const entries = rawEntries.map((entry) => ({ raw: entry.path, normalized: normalizeArchivePath(entry.path, "Archive entry"), + ...(entry.size !== undefined ? { size: entry.size } : {}), + ...(entry.type ? { type: entry.type } : {}), })); const hardlinkTargets = rawEntries .filter((entry) => entry.type === "Link" && entry.linkpath) @@ -379,6 +810,12 @@ export async function backupVerifyCommand( if (duplicateEntryPath) { throw new Error(`Archive contains duplicate entry path: ${duplicateEntryPath}`); } + const portablePathCollision = findPortableArchiveEntryPathCollision(entries); + if (portablePathCollision) { + throw new Error( + `Archive contains a portable path collision: ${portablePathCollision.first} and ${portablePathCollision.second}`, + ); + } const manifestEntryPath = manifestMatches[0]?.raw; if (!manifestEntryPath) { throw new Error("Backup archive manifest entry could not be resolved."); @@ -392,6 +829,7 @@ export async function backupVerifyCommand( manifest.archiveRoot, normalizedEntrySet, ); + await verifySqliteSnapshots({ archivePath, entries, manifest }); const result: BackupVerifyResult = { ok: true, @@ -410,3 +848,7 @@ export async function backupVerifyCommand( } return result; } + +export const testApi = { + assertSqliteExtractionBudget, +}; diff --git a/src/commands/doctor-session-sqlite-compact.ts b/src/commands/doctor-session-sqlite-compact.ts index ba39eb38f265..774bac79d57d 100644 --- a/src/commands/doctor-session-sqlite-compact.ts +++ b/src/commands/doctor-session-sqlite-compact.ts @@ -1,18 +1,9 @@ /** Runs doctor-owned SQLite file compaction for migrated session stores. */ import fs from "node:fs"; -import type { DatabaseSync } from "node:sqlite"; import type { SessionStoreTarget } from "../config/sessions/targets.js"; -import { requireNodeSqlite } from "../infra/node-sqlite.js"; -import { OPENCLAW_SQLITE_BUSY_TIMEOUT_MS } from "../state/openclaw-state-db.js"; import { resolveTargetSqlitePath } from "./doctor-session-sqlite-readers.js"; import type { DoctorSessionSqliteCompactReport } from "./doctor-session-sqlite-types.js"; - -type SqliteFileCompactSnapshot = { - dbSizeBytes: number; - freelistPages: number; - pageSizeBytes: number; - walSizeBytes: number; -}; +import { compactDoctorSqliteFile } from "./doctor-sqlite-compact.js"; /** Reclaim free pages from one agent session SQLite database. */ export function compactDoctorSessionSqliteTarget( @@ -34,69 +25,20 @@ export function compactDoctorSessionSqliteTarget( }; } - const sqlite = requireNodeSqlite(); - const database = new sqlite.DatabaseSync(sqlitePath); - try { - database.exec(`PRAGMA busy_timeout = ${OPENCLAW_SQLITE_BUSY_TIMEOUT_MS};`); - checkpointTruncate(database); - const before = readCompactSnapshot(database, sqlitePath); - // Doctor's offline VACUUM is the one sanctioned window to retrofit - // incremental auto-vacuum onto databases created before the pragma - // existed; runtime maintenance then releases pages in bounded passes. - database.exec("PRAGMA auto_vacuum = INCREMENTAL;"); - database.exec("VACUUM;"); - checkpointTruncate(database); - const after = readCompactSnapshot(database, sqlitePath); - return { - dbSizeAfterBytes: after.dbSizeBytes, - dbSizeBeforeBytes: before.dbSizeBytes, - freelistAfterPages: after.freelistPages, - freelistBeforePages: before.freelistPages, - pageSizeBytes: before.pageSizeBytes || after.pageSizeBytes, - reclaimedBytes: Math.max(0, before.dbSizeBytes - after.dbSizeBytes), - skipped: false, - walSizeAfterBytes: after.walSizeBytes, - walSizeBeforeBytes: before.walSizeBytes, - }; - } finally { - database.close(); - } -} - -function checkpointTruncate(database: DatabaseSync): void { - database.exec("PRAGMA wal_checkpoint(TRUNCATE);"); -} - -function readCompactSnapshot( - database: DatabaseSync, - sqlitePath: string, -): SqliteFileCompactSnapshot { - const sizes = readSqliteFileSizes(sqlitePath); + const compact = compactDoctorSqliteFile({ sqlitePath }); return { - dbSizeBytes: sizes.dbSizeBytes, - freelistPages: readPragmaNumber(database, "freelist_count"), - pageSizeBytes: readPragmaNumber(database, "page_size"), - walSizeBytes: sizes.walSizeBytes, + dbSizeAfterBytes: compact.after.dbSizeBytes, + dbSizeBeforeBytes: compact.before.dbSizeBytes, + freelistAfterPages: compact.after.freelistPages, + freelistBeforePages: compact.before.freelistPages, + pageSizeBytes: compact.before.pageSizeBytes || compact.after.pageSizeBytes, + reclaimedBytes: compact.reclaimedBytes, + skipped: false, + walSizeAfterBytes: compact.after.walSizeBytes, + walSizeBeforeBytes: compact.before.walSizeBytes, }; } -function readPragmaNumber( - database: DatabaseSync, - pragmaName: "freelist_count" | "page_size", -): number { - const row = database.prepare(`PRAGMA ${pragmaName};`).get() as - | Record - | undefined; - const value = row?.[pragmaName] ?? (row ? Object.values(row)[0] : undefined); - if (typeof value === "number" && Number.isFinite(value)) { - return value; - } - if (typeof value === "bigint") { - return Number(value); - } - return 0; -} - function readSqliteFileSizes(sqlitePath: string): { dbSizeBytes: number; walSizeBytes: number } { return { dbSizeBytes: fileSize(sqlitePath), diff --git a/src/commands/doctor-sqlite-compact.ts b/src/commands/doctor-sqlite-compact.ts new file mode 100644 index 000000000000..0b78b0ee16f1 --- /dev/null +++ b/src/commands/doctor-sqlite-compact.ts @@ -0,0 +1,165 @@ +/** Shared doctor-only SQLite compaction mechanics. */ +import fs from "node:fs"; +import type { DatabaseSync } from "node:sqlite"; +import { requireNodeSqlite } from "../infra/node-sqlite.js"; +import { OPENCLAW_SQLITE_BUSY_TIMEOUT_MS } from "../state/openclaw-state-db.js"; + +export type DoctorSqliteCompactSnapshot = { + autoVacuum: number; + dbSizeBytes: number; + freelistPages: number; + pageSizeBytes: number; + walSizeBytes: number; +}; + +export type DoctorSqliteCompactResult = { + after: DoctorSqliteCompactSnapshot; + before: DoctorSqliteCompactSnapshot; + integrityCheck: "ok"; + quickCheck: "ok"; + reclaimedBytes: number; +}; + +type DoctorSqliteCompactOptions = { + afterMutation?: () => void; + sqlitePath: string; + validateBeforeMutation?: (database: DatabaseSync) => void; +}; + +/** + * Compact one SQLite file during an explicit offline doctor operation. + * + * Validation runs before the first checkpoint because checkpointing mutates + * the database files. A busy checkpoint is a hard failure, never partial + * success, so VACUUM cannot race an active reader or writer. + */ +export function compactDoctorSqliteFile( + options: DoctorSqliteCompactOptions, +): DoctorSqliteCompactResult { + const sqlite = requireNodeSqlite(); + const database = new sqlite.DatabaseSync(options.sqlitePath); + let mutationStarted = false; + let operationError: unknown; + let result: DoctorSqliteCompactResult | undefined; + try { + database.exec(`PRAGMA busy_timeout = ${OPENCLAW_SQLITE_BUSY_TIMEOUT_MS};`); + database.exec("PRAGMA trusted_schema = OFF;"); + options.validateBeforeMutation?.(database); + const before = readCompactSnapshot(database, options.sqlitePath); + mutationStarted = true; + checkpointTruncate(database, options.sqlitePath); + database.exec("PRAGMA auto_vacuum = INCREMENTAL;"); + database.exec("VACUUM;"); + checkpointTruncate(database, options.sqlitePath); + const quickCheck = runDatabaseCheck(database, "quick_check"); + const integrityCheck = runDatabaseCheck(database, "integrity_check"); + const after = readCompactSnapshot(database, options.sqlitePath); + const beforeBytes = before.dbSizeBytes + before.walSizeBytes; + const afterBytes = after.dbSizeBytes + after.walSizeBytes; + result = { + after, + before, + integrityCheck, + quickCheck, + reclaimedBytes: Math.max(0, beforeBytes - afterBytes), + }; + } catch (error) { + operationError = error; + } + try { + database.close(); + } catch (error) { + operationError ??= error; + } + if (mutationStarted) { + try { + options.afterMutation?.(); + } catch (error) { + operationError ??= error; + } + } + if (operationError !== undefined) { + throw operationError instanceof Error + ? operationError + : new Error("SQLite compaction failed with a non-Error value."); + } + if (!result) { + throw new Error(`SQLite compaction produced no result for ${options.sqlitePath}.`); + } + return result; +} + +function checkpointTruncate(database: DatabaseSync, sqlitePath: string): void { + const row = database.prepare("PRAGMA wal_checkpoint(TRUNCATE);").get() as + | Record + | undefined; + const busy = readFiniteNumber(row?.busy ?? (row ? Object.values(row)[0] : undefined)); + if (busy === undefined) { + throw new Error(`SQLite checkpoint returned an invalid result for ${sqlitePath}.`); + } + if (busy !== 0) { + throw new Error(`SQLite checkpoint remained busy for ${sqlitePath}. Stop OpenClaw and retry.`); + } +} + +function runDatabaseCheck(database: DatabaseSync, pragma: "integrity_check" | "quick_check"): "ok" { + const rows = database.prepare(`PRAGMA ${pragma};`).all() as Array>; + const results = rows.map((row) => readTextValue(row[pragma] ?? Object.values(row)[0])); + if (results.length === 1 && results[0] === "ok") { + return "ok"; + } + throw new Error(`SQLite ${pragma} failed: ${results.filter(Boolean).join("; ") || "no result"}`); +} + +function readTextValue(value: unknown): string { + return typeof value === "string" ? value : ""; +} + +function readCompactSnapshot( + database: DatabaseSync, + sqlitePath: string, +): DoctorSqliteCompactSnapshot { + return { + autoVacuum: readPragmaNumber(database, "auto_vacuum"), + dbSizeBytes: fileSize(sqlitePath), + freelistPages: readPragmaNumber(database, "freelist_count"), + pageSizeBytes: readPragmaNumber(database, "page_size"), + walSizeBytes: fileSize(`${sqlitePath}-wal`), + }; +} + +function readPragmaNumber( + database: DatabaseSync, + pragmaName: "auto_vacuum" | "freelist_count" | "page_size", +): number { + const row = database.prepare(`PRAGMA ${pragmaName};`).get() as + | Record + | undefined; + const value = readFiniteNumber(row?.[pragmaName] ?? (row ? Object.values(row)[0] : undefined)); + if (value === undefined) { + throw new Error(`SQLite PRAGMA ${pragmaName} returned an invalid result.`); + } + return value; +} + +function readFiniteNumber(value: unknown): number | undefined { + if (typeof value === "number" && Number.isFinite(value)) { + return value; + } + if (typeof value === "bigint") { + const numberValue = Number(value); + return Number.isFinite(numberValue) ? numberValue : undefined; + } + return undefined; +} + +function fileSize(filePath: string): number { + try { + return fs.statSync(filePath).size; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + return 0; + } + throw error; + } +} diff --git a/src/commands/doctor-state-sqlite-compact.test.ts b/src/commands/doctor-state-sqlite-compact.test.ts new file mode 100644 index 000000000000..9af046fa94ed --- /dev/null +++ b/src/commands/doctor-state-sqlite-compact.test.ts @@ -0,0 +1,220 @@ +import fs from "node:fs"; +import path from "node:path"; +import type { DatabaseSync } from "node:sqlite"; +import { afterEach, describe, expect, it } from "vitest"; +import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js"; +import { requireNodeSqlite } from "../infra/node-sqlite.js"; +import { + closeOpenClawStateDatabase, + openOpenClawStateDatabase, + OPENCLAW_STATE_SCHEMA_VERSION, +} from "../state/openclaw-state-db.js"; +import { resolveOpenClawStateSqlitePath } from "../state/openclaw-state-db.paths.js"; +import { + type DoctorStateSqliteCompactReport, + runDoctorStateSqliteCompact, +} from "./doctor-state-sqlite-compact.js"; + +const tempDirs = useAutoCleanupTempDirTracker((cleanup) => { + afterEach(() => { + closeOpenClawStateDatabase(); + cleanup(); + }); +}); +type CompletedStateSqliteCompactReport = Extract< + DoctorStateSqliteCompactReport, + { skipped: false } +>; + +function createStateEnv(): NodeJS.ProcessEnv { + const stateDir = tempDirs.make("openclaw-state-compact-"); + return { ...process.env, OPENCLAW_STATE_DIR: stateDir }; +} + +function seedStateDatabase(params: { + env: NodeJS.ProcessEnv; + role?: string; + schemaVersion?: number; + withBloat?: boolean; +}): string { + const sqlitePath = resolveOpenClawStateSqlitePath(params.env); + fs.mkdirSync(path.dirname(sqlitePath), { recursive: true }); + const sqlite = requireNodeSqlite(); + const database = new sqlite.DatabaseSync(sqlitePath); + const schemaVersion = params.schemaVersion ?? OPENCLAW_STATE_SCHEMA_VERSION; + try { + database.exec(` + PRAGMA auto_vacuum = NONE; + PRAGMA journal_mode = WAL; + CREATE TABLE schema_meta ( + meta_key TEXT PRIMARY KEY, + role TEXT NOT NULL, + schema_version INTEGER NOT NULL + ); + CREATE TABLE compact_payload ( + id INTEGER PRIMARY KEY, + payload TEXT NOT NULL + ); + PRAGMA user_version = ${schemaVersion}; + `); + database + .prepare("INSERT INTO schema_meta (meta_key, role, schema_version) VALUES (?, ?, ?)") + .run("primary", params.role ?? "global", schemaVersion); + if (params.withBloat) { + const insert = database.prepare("INSERT INTO compact_payload (payload) VALUES (?)"); + database.exec("BEGIN IMMEDIATE;"); + for (let index = 0; index < 512; index += 1) { + insert.run(`${index}:${"x".repeat(8_192)}`); + } + database.exec("COMMIT; DELETE FROM compact_payload; PRAGMA wal_checkpoint(TRUNCATE);"); + } + } finally { + database.close(); + } + if (process.platform !== "win32") { + fs.chmodSync(sqlitePath, 0o666); + } + return sqlitePath; +} + +function readPragma(database: DatabaseSync, name: string): number { + const row = database.prepare(`PRAGMA ${name};`).get() as Record; + return Number(row[name] ?? Object.values(row)[0]); +} + +function expectCompletedReport( + report: DoctorStateSqliteCompactReport, +): asserts report is CompletedStateSqliteCompactReport { + expect(report.skipped).toBe(false); + if (report.skipped) { + throw new Error("expected state SQLite compaction report"); + } +} + +function expectOwnerOnlySqlitePermissions(sqlitePath: string): void { + expect(fs.statSync(path.dirname(sqlitePath)).mode & 0o777).toBe(0o700); + for (const candidate of [sqlitePath, `${sqlitePath}-wal`, `${sqlitePath}-shm`]) { + if (fs.existsSync(candidate)) { + expect(fs.statSync(candidate).mode & 0o777).toBe(0o600); + } + } +} + +describe("runDoctorStateSqliteCompact", () => { + it("reports a missing canonical database as skipped", () => { + const env = createStateEnv(); + + expect(runDoctorStateSqliteCompact({ env })).toEqual({ + mode: "compact", + path: resolveOpenClawStateSqlitePath(env), + reason: "missing", + skipped: true, + }); + }); + + it("compacts the canonical database and reports verified before/after state", () => { + const env = createStateEnv(); + const sqlitePath = seedStateDatabase({ env, withBloat: true }); + + const report = runDoctorStateSqliteCompact({ env }); + + expectCompletedReport(report); + expect(report.path).toBe(sqlitePath); + expect(report.before.autoVacuum).toBe(0); + expect(report.after.autoVacuum).toBe(2); + expect(report.before.freelistPages).toBeGreaterThan(0); + expect(report.after.freelistPages).toBe(0); + expect(report.after.dbSizeBytes).toBeLessThan(report.before.dbSizeBytes); + expect(report.after.walSizeBytes).toBe(0); + expect(report.after.pageSizeBytes).toBeGreaterThan(0); + expect(report.reclaimedBytes).toBeGreaterThan(0); + expect(report.quickCheck).toBe("ok"); + expect(report.integrityCheck).toBe("ok"); + }); + + it.skipIf(process.platform === "win32")("reapplies owner-only SQLite permissions", () => { + const env = createStateEnv(); + const sqlitePath = seedStateDatabase({ env, withBloat: true }); + + runDoctorStateSqliteCompact({ env }); + + expectOwnerOnlySqlitePermissions(sqlitePath); + }); + + it("rejects non-global schema metadata before mutation", () => { + const env = createStateEnv(); + const sqlitePath = seedStateDatabase({ env, role: "agent", withBloat: true }); + + expect(() => runDoctorStateSqliteCompact({ env })).toThrow(/schema role agent.*global/); + + const sqlite = requireNodeSqlite(); + const database = new sqlite.DatabaseSync(sqlitePath, { readOnly: true }); + try { + expect(readPragma(database, "auto_vacuum")).toBe(0); + expect(readPragma(database, "freelist_count")).toBeGreaterThan(0); + } finally { + database.close(); + } + }); + + it.each([ + ["legacy", OPENCLAW_STATE_SCHEMA_VERSION - 1, /doctor --fix before compacting/], + ["future", OPENCLAW_STATE_SCHEMA_VERSION + 1, /uses newer schema version/], + ] as const)( + "rejects a %s shared-state schema before mutation", + (_label, schemaVersion, message) => { + const env = createStateEnv(); + const sqlitePath = seedStateDatabase({ env, schemaVersion }); + + expect(() => runDoctorStateSqliteCompact({ env })).toThrow(message); + + const sqlite = requireNodeSqlite(); + const database = new sqlite.DatabaseSync(sqlitePath, { readOnly: true }); + try { + expect(readPragma(database, "auto_vacuum")).toBe(0); + } finally { + database.close(); + } + }, + ); + + it.skipIf(process.platform === "win32")( + "refuses a symlink at the canonical database path", + () => { + const env = createStateEnv(); + const canonicalPath = resolveOpenClawStateSqlitePath(env); + const externalEnv = createStateEnv(); + const externalPath = seedStateDatabase({ env: externalEnv }); + fs.mkdirSync(path.dirname(canonicalPath), { recursive: true }); + fs.symlinkSync(externalPath, canonicalPath); + + expect(() => runDoctorStateSqliteCompact({ env })).toThrow(/not a regular file/); + }, + ); + + it("refuses compaction while this process owns an open shared-state handle", () => { + const env = createStateEnv(); + openOpenClawStateDatabase({ env }); + + expect(() => runDoctorStateSqliteCompact({ env })).toThrow(/already open in this process/); + }); + + it("treats a busy truncating checkpoint as failure", () => { + const env = createStateEnv(); + const sqlitePath = seedStateDatabase({ env }); + const sqlite = requireNodeSqlite(); + const reader = new sqlite.DatabaseSync(sqlitePath); + const writer = new sqlite.DatabaseSync(sqlitePath); + try { + reader.exec("BEGIN; SELECT COUNT(*) FROM compact_payload;"); + writer.exec("INSERT INTO compact_payload (payload) VALUES ('newer wal frame');"); + + expect(() => runDoctorStateSqliteCompact({ env })).toThrow(/checkpoint remained busy/); + expect(readPragma(writer, "auto_vacuum")).toBe(0); + } finally { + reader.exec("ROLLBACK;"); + reader.close(); + writer.close(); + } + }); +}); diff --git a/src/commands/doctor-state-sqlite-compact.ts b/src/commands/doctor-state-sqlite-compact.ts new file mode 100644 index 000000000000..a8b6b1cbb393 --- /dev/null +++ b/src/commands/doctor-state-sqlite-compact.ts @@ -0,0 +1,121 @@ +/** Explicit doctor maintenance for the canonical shared state SQLite database. */ +import fs from "node:fs"; +import type { DatabaseSync } from "node:sqlite"; +import { + createNewerSqliteSchemaVersionError, + readSqliteUserVersion, +} from "../infra/sqlite-user-version.js"; +import { + ensureOpenClawStatePermissions, + isOpenClawStateDatabaseOpen, + OPENCLAW_STATE_SCHEMA_VERSION, +} from "../state/openclaw-state-db.js"; +import { resolveOpenClawStateSqlitePath } from "../state/openclaw-state-db.paths.js"; +import { + compactDoctorSqliteFile, + type DoctorSqliteCompactSnapshot, +} from "./doctor-sqlite-compact.js"; + +export type DoctorStateSqliteCompactReport = + | { + mode: "compact"; + path: string; + reason: "missing"; + skipped: true; + } + | { + after: DoctorSqliteCompactSnapshot; + before: DoctorSqliteCompactSnapshot; + integrityCheck: "ok"; + mode: "compact"; + path: string; + quickCheck: "ok"; + reclaimedBytes: number; + skipped: false; + }; + +type DoctorStateSqliteCompactOptions = { + env?: NodeJS.ProcessEnv; +}; + +/** Compact only the canonical shared state database resolved for this invocation. */ +export function runDoctorStateSqliteCompact( + options: DoctorStateSqliteCompactOptions = {}, +): DoctorStateSqliteCompactReport { + const env = options.env ?? process.env; + const sqlitePath = resolveOpenClawStateSqlitePath(env); + const stat = readCanonicalStateDatabaseStat(sqlitePath); + if (!stat) { + return { + mode: "compact", + path: sqlitePath, + reason: "missing", + skipped: true, + }; + } + if (!stat.isFile()) { + throw new Error(`Canonical OpenClaw state database is not a regular file: ${sqlitePath}`); + } + if (isOpenClawStateDatabaseOpen()) { + throw new Error( + "The shared OpenClaw state database is already open in this process. Stop OpenClaw and retry.", + ); + } + + const compact = compactDoctorSqliteFile({ + afterMutation: () => ensureOpenClawStatePermissions(sqlitePath, env), + sqlitePath, + validateBeforeMutation: (database) => validateCanonicalStateDatabase(database, sqlitePath), + }); + return { + ...compact, + mode: "compact", + path: sqlitePath, + skipped: false, + }; +} + +function readCanonicalStateDatabaseStat(sqlitePath: string): fs.Stats | undefined { + try { + return fs.lstatSync(sqlitePath); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + return undefined; + } + throw error; + } +} + +function validateCanonicalStateDatabase(database: DatabaseSync, sqlitePath: string): void { + const userVersion = readSqliteUserVersion(database); + if (userVersion > OPENCLAW_STATE_SCHEMA_VERSION) { + throw createNewerSqliteSchemaVersionError( + "OpenClaw state database", + sqlitePath, + userVersion, + OPENCLAW_STATE_SCHEMA_VERSION, + ); + } + if (userVersion !== OPENCLAW_STATE_SCHEMA_VERSION) { + throw new Error( + `OpenClaw state database ${sqlitePath} uses schema version ${userVersion}; run openclaw doctor --fix before compacting it.`, + ); + } + + const metadata = database + .prepare("SELECT role, schema_version FROM schema_meta WHERE meta_key = 'primary' LIMIT 1") + .get() as { role?: unknown; schema_version?: unknown } | undefined; + if (metadata?.role !== "global") { + const role = typeof metadata?.role === "string" ? metadata.role : "missing"; + throw new Error( + `OpenClaw state database ${sqlitePath} has schema role ${role}; expected global.`, + ); + } + if (metadata.schema_version !== OPENCLAW_STATE_SCHEMA_VERSION) { + const schemaVersion = + typeof metadata.schema_version === "number" ? metadata.schema_version : "invalid"; + throw new Error( + `OpenClaw state database ${sqlitePath} metadata schema version ${schemaVersion} does not match ${OPENCLAW_STATE_SCHEMA_VERSION}; run openclaw doctor --fix before compacting it.`, + ); + } +} diff --git a/src/commands/doctor.test.ts b/src/commands/doctor.test.ts index 6051e37112b4..bd1ba70dcc56 100644 --- a/src/commands/doctor.test.ts +++ b/src/commands/doctor.test.ts @@ -4,6 +4,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; const mocks = vi.hoisted(() => ({ createSessionSqliteGithubIssue: vi.fn(), runPostUpgradeProbes: vi.fn(), + runDoctorStateSqliteCompact: vi.fn(), runDoctorSessionSqlite: vi.fn(), resolveInstalledPluginIndexStorePath: vi.fn(() => "/tmp/openclaw-installed-plugins.json"), })); @@ -16,6 +17,10 @@ vi.mock("./doctor-session-sqlite.js", () => ({ runDoctorSessionSqlite: mocks.runDoctorSessionSqlite, })); +vi.mock("./doctor-state-sqlite-compact.js", () => ({ + runDoctorStateSqliteCompact: mocks.runDoctorStateSqliteCompact, +})); + vi.mock("./doctor-session-sqlite-github-issue.js", () => ({ createSessionSqliteGithubIssue: mocks.createSessionSqliteGithubIssue, })); @@ -107,6 +112,53 @@ describe("doctorCommand", () => { expect(runtime.exit).toHaveBeenCalledWith(0); }); + it("writes shared-state sqlite compaction JSON through the runtime", async () => { + const report = { + after: { + autoVacuum: 2, + dbSizeBytes: 8_192, + freelistPages: 0, + pageSizeBytes: 4_096, + walSizeBytes: 0, + }, + before: { + autoVacuum: 0, + dbSizeBytes: 16_384, + freelistPages: 2, + pageSizeBytes: 4_096, + walSizeBytes: 4_096, + }, + integrityCheck: "ok", + mode: "compact", + path: "/tmp/openclaw/state/openclaw.sqlite", + quickCheck: "ok", + reclaimedBytes: 12_288, + skipped: false, + }; + mocks.runDoctorStateSqliteCompact.mockReturnValueOnce(report); + const runtime = { + log: vi.fn(), + error: vi.fn(), + writeStdout: vi.fn(), + writeJson: vi.fn(), + exit: vi.fn((code: number) => { + throw new Error(`exit:${code}`); + }), + }; + + await expect( + doctorCommand(runtime, { + json: true, + stateSqlite: "compact", + }), + ).rejects.toThrow("exit:0"); + + expect(mocks.runDoctorStateSqliteCompact).toHaveBeenCalledWith(); + expect(runtime.writeJson).toHaveBeenCalledWith(report, 2); + expect(runtime.log).not.toHaveBeenCalled(); + expect(runtime.exit).toHaveBeenCalledWith(0); + }); + it("creates a GitHub issue for approved session sqlite recovery reports", async () => { const supportIssue = { body: "sanitized body", diff --git a/src/commands/doctor.ts b/src/commands/doctor.ts index a987b4629719..42144e1ff2a7 100644 --- a/src/commands/doctor.ts +++ b/src/commands/doctor.ts @@ -6,6 +6,28 @@ import type { DoctorSessionSqliteReport } from "./doctor-session-sqlite.js"; /** Runs doctor or the post-upgrade probe submode using the provided runtime. */ export async function doctorCommand(runtime?: RuntimeEnv, options?: DoctorOptions): Promise { + if (options?.stateSqlite) { + const outputRuntime = runtime ?? defaultRuntime; + const { runDoctorStateSqliteCompact } = await import("./doctor-state-sqlite-compact.js"); + const report = runDoctorStateSqliteCompact(); + if (options.json) { + writeRuntimeJson(outputRuntime, report); + } else if (report.skipped) { + outputRuntime.log(`state-sqlite compact: skipped; database missing at ${report.path}`); + } else { + outputRuntime.log( + `state-sqlite compact: reclaimed=${report.reclaimedBytes} bytes, db=${report.before.dbSizeBytes}->${report.after.dbSizeBytes} bytes, wal=${report.before.walSizeBytes}->${report.after.walSizeBytes} bytes`, + ); + outputRuntime.log( + `- freelist=${report.before.freelistPages}->${report.after.freelistPages} pages, page-size=${report.after.pageSizeBytes} bytes, auto-vacuum=${report.before.autoVacuum}->${report.after.autoVacuum}`, + ); + outputRuntime.log( + `- quick-check=${report.quickCheck}, integrity-check=${report.integrityCheck}, path=${report.path}`, + ); + } + outputRuntime.exit(0); + return; + } if (options?.sessionSqlite) { const outputRuntime = runtime ?? defaultRuntime; const { runDoctorSessionSqlite } = await import("./doctor-session-sqlite.js"); diff --git a/src/commands/doctor.types.ts b/src/commands/doctor.types.ts index 1ed76c03a8c4..742b05f99c99 100644 --- a/src/commands/doctor.types.ts +++ b/src/commands/doctor.types.ts @@ -9,6 +9,7 @@ export type DoctorOptions = { generateGatewayToken?: boolean; allowExec?: boolean; postUpgrade?: boolean; + stateSqlite?: "compact"; sessionSqlite?: "dry-run" | "import" | "validate" | "inspect" | "compact" | "restore" | "recover"; sessionSqliteStore?: string; sessionSqliteAgent?: string; diff --git a/src/config/sessions/session-accessor.reply-init-concurrency.test.ts b/src/config/sessions/session-accessor.reply-init-concurrency.test.ts index f32d759f0788..ac1579446cc6 100644 --- a/src/config/sessions/session-accessor.reply-init-concurrency.test.ts +++ b/src/config/sessions/session-accessor.reply-init-concurrency.test.ts @@ -4,8 +4,17 @@ import os from "node:os"; import path from "node:path"; import process from "node:process"; import { pathToFileURL } from "node:url"; -import { describe, expect, it, vi } from "vitest"; -import { loadSessionEntry, updateSessionEntry, upsertSessionEntry } from "./session-accessor.js"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { useAutoCleanupTempDirTracker } from "../../../test/helpers/temp-dir.js"; +import { + appendTranscriptMessage, + loadSessionEntry, + loadTranscriptEvents, + updateSessionEntry, + upsertSessionEntry, + withTranscriptWriteLock, +} from "./session-accessor.js"; +import { replaceSqliteTranscriptEvents } from "./session-accessor.sqlite.js"; vi.mock("../config.js", async () => ({ ...(await vi.importActual("../config.js")), @@ -31,10 +40,19 @@ type ChildResult = revision: string; }; +type TranscriptRewriteChildResult = + | { ok: true } + | { + message: string; + name: string; + ok: false; + }; + const POLL_MS = 20; const WAIT_TIMEOUT_MS = 10_000; const SESSION_KEY = "agent:main:main"; const AGENT_ID = "main"; +const tempDirs = useAutoCleanupTempDirTracker(afterEach); async function waitForFile(filePath: string): Promise { const deadline = Date.now() + WAIT_TIMEOUT_MS; @@ -126,7 +144,117 @@ await writeJsonFile(resultPath, committed); `; } -async function waitForChild(child: ReturnType): Promise { +function createTranscriptRewriteChildScript(sessionAccessorUrl: string): string { + return ` +const fs = await import("node:fs/promises"); +const { withTranscriptWriteLock } = await import(${JSON.stringify(sessionAccessorUrl)}); + +const POLL_MS = ${POLL_MS}; +const WAIT_TIMEOUT_MS = ${WAIT_TIMEOUT_MS}; +const SESSION_KEY = ${JSON.stringify(SESSION_KEY)}; +const AGENT_ID = ${JSON.stringify(AGENT_ID)}; + +async function waitForFile(filePath) { + const deadline = Date.now() + WAIT_TIMEOUT_MS; + while (Date.now() < deadline) { + try { + await fs.access(filePath); + return; + } catch { + await new Promise((resolve) => { + setTimeout(resolve, POLL_MS); + }); + } + } + throw new Error(\`timeout waiting for \${filePath}\`); +} + +async function writeJsonFile(filePath, value) { + const tempPath = filePath + "." + process.pid + ".tmp"; + await fs.writeFile(tempPath, \`\${JSON.stringify(value, null, 2)}\\n\`, "utf8"); + await fs.rename(tempPath, filePath); +} + +const storePath = process.env.TRANSCRIPT_REWRITE_STORE_PATH; +const sessionId = process.env.TRANSCRIPT_REWRITE_SESSION_ID; +const readyPath = process.env.TRANSCRIPT_REWRITE_READY_PATH; +const proceedPath = process.env.TRANSCRIPT_REWRITE_PROCEED_PATH; +const resultPath = process.env.TRANSCRIPT_REWRITE_RESULT_PATH; +const rewriteMode = process.env.TRANSCRIPT_REWRITE_MODE ?? "read-then-replace"; +if (!storePath || !sessionId || !readyPath || !proceedPath || !resultPath) { + throw new Error("transcript rewrite child env is incomplete"); +} + +let result; +try { + await withTranscriptWriteLock( + { + agentId: AGENT_ID, + sessionId, + sessionKey: SESSION_KEY, + storePath, + }, + async (transcript) => { + if (rewriteMode === "replace-twice") { + const firstReplacement = [ + { type: "session", version: 3, id: sessionId }, + { + type: "message", + id: "first-replacement", + parentId: null, + message: { role: "assistant", content: "first replacement" }, + }, + ]; + await transcript.replaceEvents(firstReplacement); + await writeJsonFile(readyPath, { eventCount: firstReplacement.length }); + await waitForFile(proceedPath); + await transcript.replaceEvents([ + firstReplacement[0], + { + type: "message", + id: "first-replacement", + parentId: null, + message: { role: "assistant", content: "second replacement" }, + }, + ]); + return; + } + const events = await transcript.readEvents(); + await writeJsonFile(readyPath, { eventCount: events.length }); + await waitForFile(proceedPath); + const rewrittenEvents = events.map((event) => { + if ( + typeof event !== "object" || + event === null || + Array.isArray(event) || + event.id !== "rewrite-target" + ) { + return event; + } + return { + ...event, + message: { + ...event.message, + content: "rewritten content", + }, + }; + }); + await transcript.replaceEvents(rewrittenEvents); + }, + ); + result = { ok: true }; +} catch (error) { + result = { + ok: false, + name: error instanceof Error ? error.name : typeof error, + message: error instanceof Error ? error.message : String(error), + }; +} +await writeJsonFile(resultPath, result); +`; +} + +async function waitForChild(child: ReturnType, label: string): Promise { let childStdout = ""; let childStderr = ""; child.stdout?.setEncoding("utf8"); @@ -146,12 +274,12 @@ async function waitForChild(child: ReturnType): Promise { ); if (childExit.code !== 0) { throw new Error( - `reply initialization child failed code=${String(childExit.code)} signal=${String(childExit.signal)}\nstdout:\n${childStdout}\nstderr:\n${childStderr}`, + `${label} child failed code=${String(childExit.code)} signal=${String(childExit.signal)}\nstdout:\n${childStdout}\nstderr:\n${childStderr}`, ); } } -describe("reply session initialization concurrency", () => { +describe("session accessor cross-process concurrency", () => { it("commits after same-session activity from another process", async () => { const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-reply-init-")); const sessionAccessorUrl = pathToFileURL( @@ -211,7 +339,7 @@ describe("reply session initialization concurrency", () => { { skipMaintenance: true }, ); await fs.writeFile(proceedPath, "go\n", "utf8"); - await waitForChild(child); + await waitForChild(child, "reply initialization"); const result = await readJsonFile(resultPath); expect(result).toMatchObject({ @@ -231,4 +359,289 @@ describe("reply session initialization concurrency", () => { await fs.rm(tempDir, { recursive: true, force: true }); } }, 15_000); + + it("rejects a transcript rewrite after another process commits an append", async () => { + const tempDir = tempDirs.make("openclaw-transcript-rewrite-"); + const sessionAccessorUrl = pathToFileURL( + path.resolve("src/config/sessions/session-accessor.ts"), + ).href; + const storePath = path.join(tempDir, "sessions.json"); + const readyPath = path.join(tempDir, "rewrite-ready.json"); + const proceedPath = path.join(tempDir, "proceed"); + const resultPath = path.join(tempDir, "result.json"); + const sessionId = "cross-process-transcript"; + const scope = { + agentId: AGENT_ID, + sessionId, + sessionKey: SESSION_KEY, + storePath, + }; + let child: ReturnType | undefined; + + try { + await upsertSessionEntry(scope, { + sessionId, + updatedAt: Date.now(), + }); + await replaceSqliteTranscriptEvents(scope, [ + { type: "session", version: 3, id: sessionId }, + { + type: "message", + id: "rewrite-target", + parentId: null, + message: { role: "assistant", content: "original content" }, + }, + ]); + + child = spawn( + process.execPath, + [ + "--import", + "tsx", + "--input-type=module", + "--eval", + createTranscriptRewriteChildScript(sessionAccessorUrl), + ], + { + env: { + ...process.env, + TRANSCRIPT_REWRITE_PROCEED_PATH: proceedPath, + TRANSCRIPT_REWRITE_READY_PATH: readyPath, + TRANSCRIPT_REWRITE_RESULT_PATH: resultPath, + TRANSCRIPT_REWRITE_SESSION_ID: sessionId, + TRANSCRIPT_REWRITE_STORE_PATH: storePath, + }, + stdio: ["ignore", "pipe", "pipe"], + }, + ); + await waitForFile(readyPath); + expect(await readJsonFile<{ eventCount: number }>(readyPath)).toEqual({ eventCount: 2 }); + + await appendTranscriptMessage(scope, { + cwd: tempDir, + message: { + role: "user", + content: "committed concurrent append", + timestamp: Date.now(), + }, + }); + await fs.writeFile(proceedPath, "go\n", "utf8"); + await waitForChild(child, "transcript rewrite"); + + const result = await readJsonFile(resultPath); + expect(result).toMatchObject({ + ok: false, + name: "SqliteTranscriptMutationConflictError", + message: `SQLite transcript changed while preparing rewrite for ${sessionId}`, + }); + await expect(loadTranscriptEvents(scope)).resolves.toEqual([ + { type: "session", version: 3, id: sessionId }, + { + type: "message", + id: "rewrite-target", + parentId: null, + message: { role: "assistant", content: "original content" }, + }, + expect.objectContaining({ + type: "message", + message: expect.objectContaining({ + role: "user", + content: "committed concurrent append", + }), + }), + ]); + } finally { + if (child?.exitCode === null) { + child.kill(); + } + await fs.rm(tempDir, { recursive: true, force: true }); + } + }, 15_000); + + it("preserves locked replaceEvents without a prior readEvents call", async () => { + const tempDir = tempDirs.make("openclaw-transcript-replace-"); + const storePath = path.join(tempDir, "sessions.json"); + const sessionId = "replace-without-read"; + const scope = { + agentId: AGENT_ID, + sessionId, + sessionKey: SESSION_KEY, + storePath, + }; + const replacement = [ + { type: "session", version: 3, id: sessionId }, + { + type: "message", + id: "replacement", + parentId: null, + message: { role: "assistant", content: "replacement content" }, + }, + ]; + + try { + await upsertSessionEntry(scope, { sessionId, updatedAt: Date.now() }); + await withTranscriptWriteLock(scope, async (transcript) => { + await transcript.replaceEvents(replacement); + }); + + await expect(loadTranscriptEvents(scope)).resolves.toEqual(replacement); + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } + }); + + it("guards a second replace after replacing without a prior read", async () => { + const tempDir = tempDirs.make("openclaw-transcript-double-replace-"); + const sessionAccessorUrl = pathToFileURL( + path.resolve("src/config/sessions/session-accessor.ts"), + ).href; + const storePath = path.join(tempDir, "sessions.json"); + const readyPath = path.join(tempDir, "rewrite-ready.json"); + const proceedPath = path.join(tempDir, "rewrite-proceed"); + const resultPath = path.join(tempDir, "rewrite-result.json"); + const sessionId = "double-replace-without-read"; + const scope = { + agentId: AGENT_ID, + sessionId, + sessionKey: SESSION_KEY, + storePath, + }; + const firstReplacement = [ + { type: "session", version: 3, id: sessionId }, + { + type: "message", + id: "first-replacement", + parentId: null, + message: { role: "assistant", content: "first replacement" }, + }, + ]; + let child: ReturnType | undefined; + + try { + await upsertSessionEntry(scope, { sessionId, updatedAt: Date.now() }); + child = spawn( + process.execPath, + [ + "--import", + "tsx", + "--input-type=module", + "--eval", + createTranscriptRewriteChildScript(sessionAccessorUrl), + ], + { + env: { + ...process.env, + TRANSCRIPT_REWRITE_MODE: "replace-twice", + TRANSCRIPT_REWRITE_PROCEED_PATH: proceedPath, + TRANSCRIPT_REWRITE_READY_PATH: readyPath, + TRANSCRIPT_REWRITE_RESULT_PATH: resultPath, + TRANSCRIPT_REWRITE_SESSION_ID: sessionId, + TRANSCRIPT_REWRITE_STORE_PATH: storePath, + }, + stdio: ["ignore", "pipe", "pipe"], + }, + ); + await waitForFile(readyPath); + expect(await readJsonFile<{ eventCount: number }>(readyPath)).toEqual({ eventCount: 2 }); + await appendTranscriptMessage(scope, { + cwd: tempDir, + eventId: "concurrent-append", + message: { role: "user", content: "concurrent append" }, + parentId: "first-replacement", + }); + await fs.writeFile(proceedPath, "go\n", "utf8"); + await waitForChild(child, "double transcript rewrite"); + + expect(await readJsonFile(resultPath)).toMatchObject({ + ok: false, + name: "SqliteTranscriptMutationConflictError", + message: `SQLite transcript changed while preparing rewrite for ${sessionId}`, + }); + await expect(loadTranscriptEvents(scope)).resolves.toEqual([ + ...firstReplacement, + expect.objectContaining({ + type: "message", + id: "concurrent-append", + parentId: "first-replacement", + message: expect.objectContaining({ + role: "user", + content: "concurrent append", + }), + }), + ]); + } finally { + if (child?.exitCode === null) { + child.kill(); + } + await fs.rm(tempDir, { recursive: true, force: true }); + } + }, 15_000); + + it("refreshes a read snapshot after an append in the same locked callback", async () => { + const tempDir = tempDirs.make("openclaw-transcript-self-append-"); + const storePath = path.join(tempDir, "sessions.json"); + const sessionId = "rewrite-after-own-append"; + const scope = { + agentId: AGENT_ID, + sessionId, + sessionKey: SESSION_KEY, + storePath, + }; + + try { + await upsertSessionEntry(scope, { sessionId, updatedAt: Date.now() }); + await replaceSqliteTranscriptEvents(scope, [ + { type: "session", version: 3, id: sessionId }, + { + type: "message", + id: "rewrite-target", + parentId: null, + message: { role: "assistant", content: "original content" }, + }, + ]); + + await withTranscriptWriteLock(scope, async (transcript) => { + await transcript.readEvents(); + await transcript.appendMessage({ + cwd: tempDir, + eventId: "owned-append", + message: { role: "user", content: "owned append" }, + parentId: "rewrite-target", + }); + const currentEvents = await loadTranscriptEvents(scope); + const rewrittenEvents = currentEvents.map((event) => { + if ( + typeof event !== "object" || + event === null || + Array.isArray(event) || + (event as { id?: unknown }).id !== "rewrite-target" + ) { + return event; + } + return Object.assign({}, event, { + message: { role: "assistant", content: "rewritten content" }, + }); + }); + await transcript.replaceEvents(rewrittenEvents); + }); + + await expect(loadTranscriptEvents(scope)).resolves.toEqual([ + { type: "session", version: 3, id: sessionId }, + { + type: "message", + id: "rewrite-target", + parentId: null, + message: { role: "assistant", content: "rewritten content" }, + }, + expect.objectContaining({ + type: "message", + id: "owned-append", + parentId: "rewrite-target", + message: { role: "user", content: "owned append" }, + }), + ]); + } finally { + await fs.rm(tempDir, { recursive: true, force: true }); + } + }); }); diff --git a/src/config/sessions/session-accessor.sqlite.ts b/src/config/sessions/session-accessor.sqlite.ts index 55fe35252898..ca1cb43ba1c6 100644 --- a/src/config/sessions/session-accessor.sqlite.ts +++ b/src/config/sessions/session-accessor.sqlite.ts @@ -212,6 +212,13 @@ class SqliteSessionMutationConflictError extends Error { } } +class SqliteTranscriptMutationConflictError extends Error { + constructor(sessionId: string) { + super(`SQLite transcript changed while preparing rewrite for ${sessionId}`); + this.name = "SqliteTranscriptMutationConflictError"; + } +} + type ResolvedSqliteScope = { agentId: string; env?: NodeJS.ProcessEnv; @@ -322,6 +329,15 @@ export type SqliteTranscriptWriteLockContext = { replaceEvents: (events: readonly TranscriptEvent[]) => Promise; }; +type SqliteTranscriptSnapshotRow = { + eventJson: string; + seq: number; +}; + +type SqliteTranscriptSnapshotState = + | { kind: "current"; rows: SqliteTranscriptSnapshotRow[] } + | { kind: "stale" }; + const SQLITE_SESSION_WRITER_QUEUES = new Map(); /** Loads one session entry from the additive SQLite session store. */ @@ -1371,6 +1387,31 @@ function loadSqliteTranscriptEventsFromDatabase( return rows.map((row) => JSON.parse(row.event_json) as TranscriptEvent); } +function readSqliteTranscriptSnapshot( + database: OpenClawAgentDatabase, + sessionId: string, +): { + events: TranscriptEvent[]; + rows: SqliteTranscriptSnapshotRow[]; +} { + const db = getSessionKysely(database.db); + const rows = executeSqliteQuerySync( + database.db, + db + .selectFrom("transcript_events") + .select(["event_json", "seq"]) + .where("session_id", "=", sessionId) + .orderBy("seq", "asc"), + ).rows; + return { + events: rows.map((row) => JSON.parse(row.event_json) as TranscriptEvent), + rows: rows.map((row) => ({ + eventJson: row.event_json, + seq: normalizeSqliteNumber(row.seq), + })), + }; +} + function sqliteTranscriptJsonlByteSize() { return /* kysely-allow-raw: JSONL size includes event bytes plus newline separators. */ sql`COALESCE(SUM(LENGTH(CAST(event_json AS BLOB))), 0) + CASE WHEN COUNT(*) > 0 THEN COUNT(*) - 1 ELSE 0 END`.as("size_bytes"); @@ -1444,7 +1485,7 @@ export function loadLatestSqliteAssistantText( .select("te.event_json as event_json") .where("te.session_id", "=", resolved.sessionId) .where("ti.event_type", "=", "message") - .orderBy("te.seq", "desc"), + .orderBy("ti.seq", "desc"), ); for (const row of rows) { const latest = parseLatestAssistantMessageEvent(row.event_json, options); @@ -1477,7 +1518,7 @@ export function loadLatestSqliteAssistantMessage( .select("te.event_json as event_json") .where("te.session_id", "=", resolved.sessionId) .where("ti.event_type", "=", "message") - .orderBy("te.seq", "desc"), + .orderBy("ti.seq", "desc"), ); for (const row of rows) { const latest = parseLatestAssistantMessageEvent(row.event_json, options); @@ -1506,7 +1547,7 @@ export function loadLatestSqliteMessage( .select("te.event_json as event_json") .where("te.session_id", "=", resolved.sessionId) .where("ti.event_type", "=", "message") - .orderBy("te.seq", "desc") + .orderBy("ti.seq", "desc") .limit(1), ); return row ? parseLatestMessageEvent(row.event_json, options) : undefined; @@ -1953,18 +1994,57 @@ export async function withSqliteTranscriptWriteLock( const resolved = resolveSqliteTranscriptScope(scope); return await runExclusiveSqliteSessionWrite(resolved, async () => { const database = openOpenClawAgentDatabase(toDatabaseOptions(resolved)); + let transcriptSnapshot: SqliteTranscriptSnapshotState | undefined; return await run({ - readEvents: async () => loadSqliteTranscriptEventsFromDatabase(database, resolved.sessionId), + readEvents: async () => { + const snapshot = readSqliteTranscriptSnapshot(database, resolved.sessionId); + transcriptSnapshot = { kind: "current", rows: snapshot.rows }; + return snapshot.events; + }, replaceEvents: async (events) => { - runOpenClawAgentWriteTransaction((writeDatabase) => { + if (transcriptSnapshot?.kind === "stale") { + throw new SqliteTranscriptMutationConflictError(resolved.sessionId); + } + const expectedSnapshot = transcriptSnapshot?.rows; + const nextSnapshot = runOpenClawAgentWriteTransaction((writeDatabase) => { + if (expectedSnapshot !== undefined) { + // The writer queue is process-local. Revalidate after BEGIN IMMEDIATE + // so a committed cross-process append cannot be deleted by the rewrite. + assertSqliteTranscriptSnapshotUnchanged( + writeDatabase, + resolved.sessionId, + expectedSnapshot, + ); + } replaceSqliteTranscriptEventsInTransaction(writeDatabase, resolved, events); + return readSqliteTranscriptSnapshot(writeDatabase, resolved.sessionId).rows; }, toDatabaseOptions(resolved)); + transcriptSnapshot = { kind: "current", rows: nextSnapshot }; }, appendMessage: async (options) => { let result: TranscriptMessageAppendResult | undefined; + const snapshotState = transcriptSnapshot; + let nextSnapshotState = snapshotState; runOpenClawAgentWriteTransaction((writeDatabase) => { + const snapshotStillCurrent = + snapshotState?.kind === "current" + ? isSqliteTranscriptSnapshotUnchanged( + writeDatabase, + resolved.sessionId, + snapshotState.rows, + ) + : false; result = appendSqliteTranscriptMessageInTransaction(writeDatabase, resolved, options); + if (snapshotState?.kind === "current") { + nextSnapshotState = snapshotStillCurrent + ? { + kind: "current", + rows: readSqliteTranscriptSnapshot(writeDatabase, resolved.sessionId).rows, + } + : { kind: "stale" }; + } }, toDatabaseOptions(resolved)); + transcriptSnapshot = nextSnapshotState; return result as TranscriptMessageAppendResult | undefined; }, }); @@ -1986,6 +2066,31 @@ export async function withSqliteTranscriptWriteTransaction( ); } +function isSqliteTranscriptSnapshotUnchanged( + database: OpenClawAgentDatabase, + sessionId: string, + expected: readonly SqliteTranscriptSnapshotRow[], +): boolean { + const current = readSqliteTranscriptSnapshot(database, sessionId).rows; + return ( + current.length === expected.length && + current.every( + (row, index) => + row.seq === expected[index]?.seq && row.eventJson === expected[index]?.eventJson, + ) + ); +} + +function assertSqliteTranscriptSnapshotUnchanged( + database: OpenClawAgentDatabase, + sessionId: string, + expected: readonly SqliteTranscriptSnapshotRow[], +): void { + if (!isSqliteTranscriptSnapshotUnchanged(database, sessionId, expected)) { + throw new SqliteTranscriptMutationConflictError(sessionId); + } +} + function appendSqliteTranscriptMessageInTransaction( database: OpenClawAgentDatabase, resolved: ResolvedTranscriptScope, diff --git a/src/infra/backup-create.test.ts b/src/infra/backup-create.test.ts index e0ee964ab3a5..838b60ef4aa7 100644 --- a/src/infra/backup-create.test.ts +++ b/src/infra/backup-create.test.ts @@ -699,6 +699,76 @@ describe("createBackupArchive", () => { ); }); + it("snapshots and verifies a canonical agent database when the agent id is node_modules", async () => { + await withOpenClawTestState( + { + layout: "state-only", + prefix: "openclaw-backup-agent-node-modules-", + scenario: "minimal", + }, + async (state) => { + const outputDir = state.path("backups"); + const extractDir = state.path("extract"); + const dbPath = state.statePath("agents", "node_modules", "agent", "openclaw-agent.sqlite"); + await fs.mkdir(path.dirname(dbPath), { recursive: true }); + await fs.mkdir(outputDir, { recursive: true }); + await fs.mkdir(extractDir, { recursive: true }); + const sqlite = requireNodeSqlite(); + const db = new sqlite.DatabaseSync(dbPath); + try { + db.exec(` + PRAGMA journal_mode = WAL; + PRAGMA wal_autocheckpoint = 0; + CREATE TABLE schema_meta ( + meta_key TEXT NOT NULL PRIMARY KEY, + role TEXT NOT NULL + ); + INSERT INTO schema_meta (meta_key, role) VALUES ('primary', 'agent'); + CREATE TABLE markers (id INTEGER PRIMARY KEY, value TEXT NOT NULL); + PRAGMA wal_checkpoint(TRUNCATE); + INSERT INTO markers (value) VALUES ('committed-in-wal'); + `); + await expect(fs.access(`${dbPath}-wal`)).resolves.toBeUndefined(); + + const result = await createBackupArchive({ + output: outputDir, + includeWorkspace: false, + nowMs: Date.UTC(2026, 4, 9, 8, 31, 30), + }); + const entries = await listArchiveEntries(result.archivePath); + const archivedDbEntry = entries.find((entry) => + entry.endsWith("/state/agents/node_modules/agent/openclaw-agent.sqlite"), + ); + expect(archivedDbEntry).toBeDefined(); + expect( + entries.some((entry) => + entry.endsWith("/state/agents/node_modules/agent/openclaw-agent.sqlite-wal"), + ), + ).toBe(false); + + const runtime: RuntimeEnv = { log: vi.fn(), error: vi.fn(), exit: vi.fn() }; + await expect( + backupVerifyCommand(runtime, { archive: result.archivePath }), + ).resolves.toMatchObject({ ok: true }); + + await tar.x({ file: result.archivePath, gzip: true, cwd: extractDir }); + const archivedDb = new sqlite.DatabaseSync(path.join(extractDir, archivedDbEntry!), { + readOnly: true, + }); + try { + expect(archivedDb.prepare("SELECT value FROM markers").get()).toEqual({ + value: "committed-in-wal", + }); + } finally { + archivedDb.close(); + } + } finally { + db.close(); + } + }, + ); + }); + it("snapshots nested live SQLite databases with transaction continuity", async () => { await withOpenClawTestState( { @@ -792,6 +862,90 @@ describe("createBackupArchive", () => { ); }); + it("fails closed when a plugin SQLite schema cannot be compacted safely", async () => { + await withOpenClawTestState( + { + layout: "state-only", + prefix: "openclaw-backup-plugin-capability-", + scenario: "minimal", + }, + async (state) => { + const outputDir = state.path("backups"); + const dbPath = state.statePath("plugins", "dedicated", "custom.sqlite"); + await fs.mkdir(path.dirname(dbPath), { recursive: true }); + await fs.mkdir(outputDir, { recursive: true }); + const sqlite = requireNodeSqlite(); + const db = new sqlite.DatabaseSync(dbPath); + db.function("plugin_double", { deterministic: true }, (value) => Number(value) * 2); + db.exec(` + CREATE TABLE records (value INTEGER NOT NULL); + INSERT INTO records (value) VALUES (1), (2); + CREATE INDEX records_double ON records(plugin_double(value)); + `); + db.close(); + + await expect( + createBackupArchive({ + output: outputDir, + includeWorkspace: false, + nowMs: Date.UTC(2026, 4, 9, 8, 33, 0), + }), + ).rejects.toThrow(/cannot be compacted safely.*custom\.sqlite/iu); + }, + ); + }); + + it("scrubs deleted plugin SQLite bytes from archive snapshots", async () => { + await withOpenClawTestState( + { + layout: "state-only", + prefix: "openclaw-backup-plugin-deleted-bytes-", + scenario: "minimal", + }, + async (state) => { + const outputDir = state.path("backups"); + const extractDir = state.path("extract"); + const dbPath = state.statePath("plugins", "dedicated", "deleted.sqlite"); + const deletedValue = `deleted-plugin-secret-${"x".repeat(256)}`; + await fs.mkdir(path.dirname(dbPath), { recursive: true }); + await fs.mkdir(outputDir, { recursive: true }); + await fs.mkdir(extractDir, { recursive: true }); + const sqlite = requireNodeSqlite(); + const db = new sqlite.DatabaseSync(dbPath); + db.exec("PRAGMA secure_delete = OFF; CREATE TABLE records (value TEXT NOT NULL);"); + const insert = db.prepare("INSERT INTO records (value) VALUES (?)"); + insert.run("survivor"); + insert.run(deletedValue); + db.prepare("DELETE FROM records WHERE value = ?").run(deletedValue); + db.close(); + + expect((await fs.readFile(dbPath)).includes(deletedValue)).toBe(true); + const result = await createBackupArchive({ + output: outputDir, + includeWorkspace: false, + nowMs: Date.UTC(2026, 4, 9, 8, 34, 0), + }); + const entries = await listArchiveEntries(result.archivePath); + const archivedDbEntry = entries.find((entry) => + entry.endsWith("/state/plugins/dedicated/deleted.sqlite"), + ); + expect(archivedDbEntry).toBeDefined(); + + await tar.x({ file: result.archivePath, gzip: true, cwd: extractDir }); + const archivedPath = path.join(extractDir, archivedDbEntry!); + expect((await fs.readFile(archivedPath)).includes(deletedValue)).toBe(false); + const archivedDb = new sqlite.DatabaseSync(archivedPath, { readOnly: true }); + try { + expect(archivedDb.prepare("SELECT value FROM records").all()).toEqual([ + { value: "survivor" }, + ]); + } finally { + archivedDb.close(); + } + }, + ); + }); + it("fails instead of raw-copying malformed nested SQLite databases", async () => { await withOpenClawTestState( { @@ -979,6 +1133,10 @@ describe("createBackupArchive", () => { expect( entries.find((entry) => entry.path.endsWith("/state/plugins/dedicated/linked.sqlite")), ).toMatchObject({ type: "SymbolicLink" }); + const runtime: RuntimeEnv = { log: vi.fn(), error: vi.fn(), exit: vi.fn() }; + await expect( + backupVerifyCommand(runtime, { archive: result.archivePath }), + ).resolves.toMatchObject({ ok: true }); }, ); }); @@ -1019,6 +1177,11 @@ describe("createBackupArchive", () => { CREATE TABLE delivery_queue_entries ( id TEXT PRIMARY KEY ); + CREATE TABLE schema_meta ( + meta_key TEXT NOT NULL PRIMARY KEY, + role TEXT NOT NULL + ); + INSERT INTO schema_meta (meta_key, role) VALUES ('primary', 'global'); PRAGMA wal_checkpoint(TRUNCATE); INSERT INTO durable_state (id, value) VALUES (1, 'must-stay'); INSERT INTO delivery_queue_entries (id) VALUES ('must-drop'); diff --git a/src/infra/backup-create.ts b/src/infra/backup-create.ts index 8521c0c4590f..4238d7e5b325 100644 --- a/src/infra/backup-create.ts +++ b/src/infra/backup-create.ts @@ -538,11 +538,35 @@ const SQLITE_BACKUP_EXCLUDED_SUFFIXES = [".reindex-lock.sqlite"] as const; const SQLITE_BACKUP_REINDEX_TRANSIENT_PATTERN = /\.sqlite\.(?:backup|memory-reindex|tmp)-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/iu; +function isCanonicalAgentSqlitePathOrAncestor(sourcePath: string, stateDir: string): boolean { + const relativePath = path.relative(path.resolve(stateDir), path.resolve(sourcePath)); + const segments = relativePath.split(path.sep); + if (segments[0] !== "agents" || !segments[1]) { + return false; + } + if (segments.length === 2) { + return true; + } + if (segments[2] !== "agent") { + return false; + } + if (segments.length === 3) { + return true; + } + if (segments.length !== 4) { + return false; + } + return SQLITE_BACKUP_SOURCE_SUFFIXES.some( + (suffix) => segments[3] === `openclaw-agent.sqlite${suffix}`, + ); +} + function isStatePackageContentPath(sourcePath: string, stateDir: string): boolean { const resolvedStateDir = path.resolve(stateDir); const resolvedSourcePath = path.resolve(sourcePath); return ( isPathWithin(resolvedSourcePath, resolvedStateDir) && + !isCanonicalAgentSqlitePathOrAncestor(resolvedSourcePath, resolvedStateDir) && path.relative(resolvedStateDir, resolvedSourcePath).split(path.sep).includes("node_modules") ); } @@ -634,7 +658,11 @@ async function listStateSqlitePaths(params: { if (extensionsFilter(entryPath) && !isStatePackageContentPath(entryPath, params.stateDir)) { await visit(entryPath); } - } else if (entry.isFile() && extensionsFilter(entryPath)) { + } else if ( + entry.isFile() && + extensionsFilter(entryPath) && + !isStatePackageContentPath(entryPath, params.stateDir) + ) { const resolvedEntryPath = path.resolve(entryPath); if (resolveSqliteBackupDatabasePath(resolvedEntryPath)) { discoveredSourcePaths.add(resolvedEntryPath); @@ -726,11 +754,18 @@ async function createStateSqliteBackupPlan(params: { const sourcePath = path.join(params.tempDir, `openclaw-state-db-${snapshots.length}.sqlite`); try { source.exec("PRAGMA busy_timeout = 30000;"); - // VACUUM INTO removes deleted-page remnants before the snapshot enters - // the archive. Load sqlite-vec best-effort so memory indexes using vec0 - // can still be compacted without weakening that privacy property. - await loadSqliteVecExtension({ db: source }); - source.prepare("VACUUM INTO ?").run(sourcePath); + try { + // VACUUM INTO removes deleted-page remnants before the snapshot enters + // the archive. Load known bundled extensions, but fail closed when an + // owner schema needs capabilities core cannot safely reproduce. + await loadSqliteVecExtension({ db: source }); + source.prepare("VACUUM INTO ?").run(sourcePath); + } catch (err) { + throw new Error( + `SQLite database cannot be compacted safely for backup: ${archiveSourcePath}. Required SQLite capabilities are unavailable; raw page backup was refused because it can retain deleted data.`, + { cause: err }, + ); + } } finally { source.close(); } diff --git a/src/infra/sqlite-pragma.test-support.ts b/src/infra/sqlite-pragma.test-support.ts index 6a2d3914d757..05de4fba9a34 100644 --- a/src/infra/sqlite-pragma.test-support.ts +++ b/src/infra/sqlite-pragma.test-support.ts @@ -3,6 +3,7 @@ import type { DatabaseSync } from "node:sqlite"; // SQLite pragma test helpers normalize node:sqlite bigint/number outputs. export type SqliteNumberPragma = + | "auto_vacuum" | "busy_timeout" | "foreign_keys" | "synchronous" diff --git a/src/infra/sqlite-transaction.ts b/src/infra/sqlite-transaction.ts index dabf3a3015d2..eb1e16372c83 100644 --- a/src/infra/sqlite-transaction.ts +++ b/src/infra/sqlite-transaction.ts @@ -59,7 +59,7 @@ function sqlitePrimaryResultCode(error: unknown): number | undefined { return errcode === undefined ? undefined : errcode & SQLITE_PRIMARY_RESULT_CODE_MASK; } -function isTransactionLockError(error: unknown): boolean { +export function isSqliteLockError(error: unknown): boolean { const code = sqliteErrorCode(error); if (code !== undefined && SQLITE_LOCK_ERROR_CODES.has(code)) { return true; @@ -141,7 +141,7 @@ function execTimedTransactionStep(params: { return elapsedMs; } catch (error) { const elapsedMs = Date.now() - startedAt; - if (isTransactionLockError(error)) { + if (isSqliteLockError(error)) { const sqliteErrcode = sqliteExtendedResultCode(error); const sqlitePrimaryCode = sqlitePrimaryResultCode(error); transactionLogger(params.options).warn("SQLite transaction lock wait failed", { diff --git a/src/infra/sqlite-wal.test.ts b/src/infra/sqlite-wal.test.ts index fc582113ecf4..8ac3f7f8d58c 100644 --- a/src/infra/sqlite-wal.test.ts +++ b/src/infra/sqlite-wal.test.ts @@ -5,18 +5,27 @@ import os from "node:os"; import path from "node:path"; import type { DatabaseSync } from "node:sqlite"; import { afterEach, describe, expect, it, vi } from "vitest"; +import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js"; import { MAX_TIMER_TIMEOUT_MS } from "../shared/number-coercion.js"; +import { requireNodeSqlite } from "./node-sqlite.js"; import { DEFAULT_SQLITE_WAL_AUTOCHECKPOINT_PAGES, configureSqliteConnectionPragmas, + configureSqlitePreSchemaPragmas, configureSqliteWalMaintenance, } from "./sqlite-wal.js"; +const tempDirs = useAutoCleanupTempDirTracker(afterEach); + function createMockDb(): DatabaseSync { return { exec: vi.fn(), - prepare: vi.fn(() => ({ - get: vi.fn(() => ({ journal_mode: "delete" })), + prepare: vi.fn((sql: string) => ({ + get: vi.fn(() => + sql.includes("wal_checkpoint") + ? { busy: 0, log: 0, checkpointed: 0 } + : { journal_mode: "delete" }, + ), })), } as unknown as DatabaseSync; } @@ -457,16 +466,16 @@ describe("sqlite WAL maintenance", () => { expect(db["exec"]).toHaveBeenCalledTimes(2); vi.advanceTimersByTime(100); - expect(db["exec"]).toHaveBeenNthCalledWith(3, "PRAGMA wal_checkpoint(PASSIVE);"); - expect(db["exec"]).toHaveBeenNthCalledWith(4, "PRAGMA incremental_vacuum(512);"); - expect(db["exec"]).toHaveBeenCalledTimes(4); + expect(db["prepare"]).toHaveBeenCalledWith("PRAGMA wal_checkpoint(PASSIVE);"); + expect(db["exec"]).toHaveBeenNthCalledWith(3, "PRAGMA incremental_vacuum(512);"); + expect(db["exec"]).toHaveBeenCalledTimes(3); expect(maintenance.close()).toBe(true); - expect(db["exec"]).toHaveBeenLastCalledWith("PRAGMA wal_checkpoint(TRUNCATE);"); - expect(db["exec"]).toHaveBeenCalledTimes(5); + expect(db["prepare"]).toHaveBeenCalledWith("PRAGMA wal_checkpoint(TRUNCATE);"); + expect(db["exec"]).toHaveBeenCalledTimes(3); vi.advanceTimersByTime(200); - expect(db["exec"]).toHaveBeenCalledTimes(5); + expect(db["exec"]).toHaveBeenCalledTimes(3); }); it("clamps oversized checkpoint intervals before arming timers", () => { @@ -494,11 +503,75 @@ describe("sqlite WAL maintenance", () => { }); vi.advanceTimersByTime(100); - expect(db["exec"]).toHaveBeenNthCalledWith(3, "PRAGMA wal_checkpoint(FULL);"); - expect(db["exec"]).toHaveBeenNthCalledWith(4, "PRAGMA incremental_vacuum(512);"); + expect(db["prepare"]).toHaveBeenCalledWith("PRAGMA wal_checkpoint(FULL);"); + expect(db["exec"]).toHaveBeenNthCalledWith(3, "PRAGMA incremental_vacuum(512);"); expect(maintenance.close()).toBe(true); - expect(db["exec"]).toHaveBeenLastCalledWith("PRAGMA wal_checkpoint(FULL);"); + expect(db["prepare"]).toHaveBeenLastCalledWith("PRAGMA wal_checkpoint(FULL);"); + }); + + it("reports a busy checkpoint result as incomplete", () => { + const db = createMockDb(); + const onCheckpointError = vi.fn(); + vi.spyOn(process, "platform", "get").mockReturnValue("linux"); + vi.mocked(db["prepare"]).mockImplementation( + (sql) => + ({ + get: vi.fn(() => + sql.includes("wal_checkpoint") + ? { busy: 1, log: 4, checkpointed: 3 } + : { journal_mode: "delete" }, + ), + }) as unknown as ReturnType, + ); + + const maintenance = configureSqliteWalMaintenance(db, { + checkpointIntervalMs: 0, + databaseLabel: "test-db", + onCheckpointError, + }); + + expect(maintenance.checkpoint()).toBe(false); + expect(onCheckpointError).toHaveBeenCalledWith( + expect.objectContaining({ message: "test-db WAL checkpoint TRUNCATE remained busy" }), + ); + }); + + it("detects a checkpoint blocked by another connection's reader", () => { + const tempDir = tempDirs.make("openclaw-sqlite-checkpoint-busy-"); + const databasePath = path.join(tempDir, "state.sqlite"); + const { DatabaseSync } = requireNodeSqlite(); + const writer = new DatabaseSync(databasePath); + let reader: InstanceType | undefined; + let maintenance: ReturnType | undefined; + try { + writer.exec(` + PRAGMA journal_mode = WAL; + CREATE TABLE events (id INTEGER PRIMARY KEY, value TEXT NOT NULL); + INSERT INTO events (value) VALUES ('before-reader'); + PRAGMA wal_checkpoint(TRUNCATE); + `); + reader = new DatabaseSync(databasePath); + reader.exec("BEGIN;"); + reader.prepare("SELECT COUNT(*) FROM events").get(); + writer.prepare("INSERT INTO events (value) VALUES (?)").run("after-reader"); + + maintenance = configureSqliteWalMaintenance(writer, { checkpointIntervalMs: 0 }); + + expect(maintenance.checkpoint()).toBe(false); + reader.exec("ROLLBACK;"); + expect(maintenance.checkpoint()).toBe(true); + } finally { + if (reader?.isOpen) { + try { + reader.exec("ROLLBACK;"); + } catch {} + reader.close(); + } + maintenance?.close(); + writer.close(); + fs.rmSync(tempDir, { recursive: true, force: true }); + } }); it("reports checkpoint errors without throwing from background maintenance", () => { @@ -506,10 +579,13 @@ describe("sqlite WAL maintenance", () => { const error = new Error("busy"); const onCheckpointError = vi.fn(); vi.spyOn(process, "platform", "get").mockReturnValue("linux"); - vi.mocked(db["exec"]).mockImplementation((sql) => { + vi.mocked(db["prepare"]).mockImplementation((sql) => { if (sql.includes("wal_checkpoint")) { throw error; } + return { + get: vi.fn(() => ({ journal_mode: "delete" })), + } as unknown as ReturnType; }); const maintenance = configureSqliteWalMaintenance(db, { @@ -542,6 +618,53 @@ describe("sqlite WAL maintenance", () => { expect(db["exec"]).toHaveBeenNthCalledWith(5, "PRAGMA foreign_keys = ON;"); }); + it("retries the WAL transition when SQLite bypasses the busy handler", () => { + const db = createMockDb(); + vi.spyOn(process, "platform", "get").mockReturnValue("linux"); + let journalModeAttempts = 0; + vi.mocked(db["exec"]).mockImplementation((sql) => { + if (sql === "PRAGMA journal_mode = WAL;" && journalModeAttempts++ === 0) { + throw Object.assign(new Error("database is locked"), { + code: "ERR_SQLITE_ERROR", + errcode: 5, + }); + } + }); + + configureSqliteConnectionPragmas(db, { + busyTimeoutMs: 50, + checkpointIntervalMs: 0, + }); + + expect(journalModeAttempts).toBe(2); + expect( + vi.mocked(db["exec"]).mock.calls.filter(([sql]) => sql.startsWith("PRAGMA busy_timeout")), + ).toEqual([ + ["PRAGMA busy_timeout = 50;"], + ["PRAGMA busy_timeout = 0;"], + ["PRAGMA busy_timeout = 50;"], + ]); + }); + + it("configures lock retry before inspecting a fresh database header", () => { + const db = createMockDb(); + vi.mocked(db["prepare"]).mockImplementation( + (sql: string) => + ({ + get: vi.fn(() => (sql === "PRAGMA page_count" ? { page_count: 0 } : undefined)), + }) as unknown as ReturnType, + ); + + configureSqlitePreSchemaPragmas(db, { busyTimeoutMs: 5000 }); + + expect(db["exec"]).toHaveBeenNthCalledWith(1, "PRAGMA busy_timeout = 5000;"); + expect(db["prepare"]).toHaveBeenCalledWith("PRAGMA page_count"); + expect(db["exec"]).toHaveBeenNthCalledWith(2, "PRAGMA auto_vacuum = INCREMENTAL;"); + expect(vi.mocked(db["exec"]).mock.invocationCallOrder[0]).toBeLessThan( + vi.mocked(db["prepare"]).mock.invocationCallOrder[0], + ); + }); + it("sets busy timeout before rollback journaling on NFS-backed volumes", () => { const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-sqlite-nfs-")); try { diff --git a/src/infra/sqlite-wal.ts b/src/infra/sqlite-wal.ts index 7693ad745c25..a1463bdd31f7 100644 --- a/src/infra/sqlite-wal.ts +++ b/src/infra/sqlite-wal.ts @@ -4,6 +4,7 @@ import fs from "node:fs"; import path from "node:path"; import type { DatabaseSync } from "node:sqlite"; import { MAX_TIMER_TIMEOUT_MS } from "../shared/number-coercion.js"; +import { isSqliteLockError } from "./sqlite-transaction.js"; // WAL maintenance configures SQLite write-ahead logging and schedules bounded // checkpoints so state databases do not accumulate unbounded WAL files. @@ -23,6 +24,8 @@ const LINUX_CIFS_SUPER_MAGIC = 0xff534d42; const LINUX_SMB2_SUPER_MAGIC = 0xfe534d42; const PROC_MOUNTINFO_PATH = "/proc/self/mountinfo"; const NETWORK_FILESYSTEM_TYPES = new Set(["cifs", "smbfs", "smb2", "smb3"]); +const JOURNAL_MODE_RETRY_INTERVAL_MS = 10; +const JOURNAL_MODE_RETRY_SLEEP = new Int32Array(new SharedArrayBuffer(4)); type IntervalHandle = ReturnType & { unref?: () => void; @@ -40,6 +43,7 @@ export type SqliteWalMaintenance = { /** Options controlling WAL autocheckpoint and periodic checkpoint behavior. */ export type SqliteWalMaintenanceOptions = { autoCheckpointPages?: number; + busyTimeoutMs?: number; checkpointIntervalMs?: number; checkpointMode?: SqliteWalCheckpointMode; databaseLabel?: string; @@ -48,11 +52,39 @@ export type SqliteWalMaintenanceOptions = { }; export type SqliteConnectionPragmaOptions = SqliteWalMaintenanceOptions & { - busyTimeoutMs?: number; foreignKeys?: boolean; synchronous?: "NORMAL"; }; +function configureSqliteBusyTimeout(db: DatabaseSync, busyTimeoutMs: number): number { + const normalizedTimeoutMs = normalizeNonNegativeInteger(busyTimeoutMs, "busyTimeoutMs"); + db.exec(`PRAGMA busy_timeout = ${normalizedTimeoutMs};`); + return normalizedTimeoutMs; +} + +// auto_vacuum only takes effect when set before the first page is written. +// Existing databases require an offline VACUUM owned by doctor/maintenance. +export function enableIncrementalAutoVacuumForFreshDatabase(db: DatabaseSync): void { + const row = db.prepare("PRAGMA page_count").get() as { page_count?: unknown } | undefined; + if (row?.page_count === 0) { + db.exec("PRAGMA auto_vacuum = INCREMENTAL;"); + } +} + +/** + * Configure lock retry before inspecting or mutating a fresh database header. + * Concurrent first opens can otherwise fail before schema transactions begin. + */ +export function configureSqlitePreSchemaPragmas( + db: DatabaseSync, + options: Pick = {}, +): void { + if (options.busyTimeoutMs !== undefined) { + configureSqliteBusyTimeout(db, options.busyTimeoutMs); + } + enableIncrementalAutoVacuumForFreshDatabase(db); +} + function normalizeNonNegativeInteger(value: number, label: string): number { if (!Number.isInteger(value) || value < 0) { throw new Error(`${label} must be a non-negative integer`); @@ -264,6 +296,15 @@ function readJournalModeResult(row: unknown): string | null { return typeof value === "string" ? value.toLowerCase() : null; } +function readCheckpointBusyResult(row: unknown): boolean { + if (!row || typeof row !== "object") { + return false; + } + const record = row as Record; + const value = record.busy ?? Object.values(record)[0]; + return value === 1 || value === 1n; +} + function requireRollbackJournalMode(db: DatabaseSync, options: SqliteWalMaintenanceOptions): void { const row = db.prepare("PRAGMA journal_mode = DELETE;").get(); const journalMode = readJournalModeResult(row); @@ -277,6 +318,40 @@ function requireRollbackJournalMode(db: DatabaseSync, options: SqliteWalMaintena } } +function enableWalJournalMode(db: DatabaseSync, retryTimeoutMs: number): void { + const deadline = Date.now() + retryTimeoutMs; + let restoreBusyTimeout = false; + try { + while (true) { + try { + db.exec("PRAGMA journal_mode = WAL;"); + return; + } catch (error) { + const remainingMs = deadline - Date.now(); + if (!isSqliteLockError(error) || remainingMs <= 0) { + throw error; + } + if (!restoreBusyTimeout) { + // A busy handler can be bypassed to avoid deadlock. Disable it after + // the first BUSY so explicit retries cannot overrun this deadline. + configureSqliteBusyTimeout(db, 0); + restoreBusyTimeout = true; + } + Atomics.wait( + JOURNAL_MODE_RETRY_SLEEP, + 0, + 0, + Math.min(JOURNAL_MODE_RETRY_INTERVAL_MS, remainingMs), + ); + } + } + } finally { + if (restoreBusyTimeout) { + configureSqliteBusyTimeout(db, retryTimeoutMs); + } + } +} + function enableMacosCheckpointFullfsync(db: DatabaseSync): void { if (process.platform !== "darwin") { return; @@ -303,6 +378,8 @@ export function configureSqliteWalMaintenance( db: DatabaseSync, options: SqliteWalMaintenanceOptions = {}, ): SqliteWalMaintenance { + const busyTimeoutMs = + options.busyTimeoutMs === undefined ? 0 : configureSqliteBusyTimeout(db, options.busyTimeoutMs); const autoCheckpointPages = normalizeNonNegativeInteger( options.autoCheckpointPages ?? DEFAULT_SQLITE_WAL_AUTOCHECKPOINT_PAGES, "autoCheckpointPages", @@ -327,13 +404,19 @@ export function configureSqliteWalMaintenance( close: () => true, }; } - db.exec("PRAGMA journal_mode = WAL;"); + enableWalJournalMode(db, busyTimeoutMs); enableMacosCheckpointFullfsync(db); db.exec(`PRAGMA wal_autocheckpoint = ${autoCheckpointPages};`); const runCheckpoint = (mode: SqliteWalCheckpointMode): boolean => { try { - db.exec(`PRAGMA wal_checkpoint(${mode});`); + const row = db.prepare(`PRAGMA wal_checkpoint(${mode});`).get(); + if (readCheckpointBusyResult(row)) { + const label = options.databaseLabel ?? "sqlite database"; + const error = new Error(`${label} WAL checkpoint ${mode} remained busy`); + options.onCheckpointError?.(error); + return false; + } return true; } catch (error) { options.onCheckpointError?.(error); @@ -399,12 +482,7 @@ export function configureSqliteConnectionPragmas( db: DatabaseSync, options: SqliteConnectionPragmaOptions = {}, ): SqliteWalMaintenance { - const { busyTimeoutMs, foreignKeys, synchronous, ...walOptions } = options; - if (busyTimeoutMs !== undefined) { - db.exec( - `PRAGMA busy_timeout = ${normalizeNonNegativeInteger(busyTimeoutMs, "busyTimeoutMs")};`, - ); - } + const { foreignKeys, synchronous, ...walOptions } = options; const maintenance = configureSqliteWalMaintenance(db, walOptions); if (synchronous) { db.exec(`PRAGMA synchronous = ${synchronous};`); diff --git a/src/state/openclaw-agent-db.test.ts b/src/state/openclaw-agent-db.test.ts index 1b99c15354f1..9ffc094895c9 100644 --- a/src/state/openclaw-agent-db.test.ts +++ b/src/state/openclaw-agent-db.test.ts @@ -665,6 +665,19 @@ describe("openclaw agent database", () => { env, }), ).toThrow(/run openclaw doctor --fix/); + + fs.rmSync(stateDatabasePath); + const reopened = openOpenClawAgentDatabase({ + agentId: "worker-1", + env, + }); + expect(reopened.db.isOpen).toBe(true); + expect(listOpenClawRegisteredAgentDatabases({ env })).toEqual([ + expect.objectContaining({ + agentId: "worker-1", + path: reopened.path, + }), + ]); }); it("keys explicit relative paths by resolved database pathname", () => { @@ -815,6 +828,16 @@ describe("openclaw agent database", () => { ]); expect(disposeOpenClawAgentDatabaseByPath(first.path, { env })).toBe(false); expect(second.db.isOpen).toBe(true); + + const reopened = openOpenClawAgentDatabase({ + agentId: "worker-1", + env, + path: first.path, + }); + expect(listOpenClawRegisteredAgentDatabases({ env })).toEqual([ + expect.objectContaining({ agentId: "worker-1", path: reopened.path }), + expect.objectContaining({ agentId: "worker-2", path: second.path }), + ]); }); it("serializes concurrent ownership claims for one unowned database", async () => { @@ -999,6 +1022,7 @@ describe("openclaw agent database", () => { ); expect(readSqliteNumberPragma(database.db, "foreign_keys")).toBe(1); expect(readSqliteNumberPragma(database.db, "synchronous")).toBe(1); + expect(readSqliteNumberPragma(database.db, "auto_vacuum")).toBe(2); expect(readSqliteNumberPragma(database.db, "user_version")).toBe(OPENCLAW_AGENT_SCHEMA_VERSION); expect(readSqliteNumberPragma(database.db, "wal_autocheckpoint")).toBe(1000); const journalMode = database.db.prepare("PRAGMA journal_mode").get() as @@ -1007,6 +1031,58 @@ describe("openclaw agent database", () => { expect(journalMode?.journal_mode?.toLowerCase()).toBe("wal"); }); + it("replaces the main v5 session indexes during migration", () => { + const stateDir = createTempStateDir(); + const env = { OPENCLAW_STATE_DIR: stateDir }; + const database = openOpenClawAgentDatabase({ agentId: "worker-1", env }); + const databasePath = database.path; + closeOpenClawAgentDatabasesForTest(); + + const { DatabaseSync } = requireNodeSqlite(); + const legacy = new DatabaseSync(databasePath); + try { + legacy.exec(` + DROP INDEX idx_agent_session_entries_session_updated; + DROP INDEX idx_agent_transcript_event_sequence; + CREATE INDEX idx_agent_session_entries_session_id + ON session_entries(session_id); + PRAGMA user_version = 5; + `); + } finally { + legacy.close(); + } + + const migrated = openOpenClawAgentDatabase({ agentId: "worker-1", env }); + const indexNames = migrated.db + .prepare( + `SELECT name + FROM sqlite_master + WHERE type = 'index' + AND name IN ( + 'idx_agent_session_entries_session_id', + 'idx_agent_session_entries_session_updated', + 'idx_agent_transcript_event_sequence' + ) + ORDER BY name`, + ) + .all() + .map((row) => (row as { name: string }).name); + + expect(indexNames).toEqual([ + "idx_agent_session_entries_session_updated", + "idx_agent_transcript_event_sequence", + ]); + const transcriptIndex = migrated.db + .prepare("SELECT sql FROM sqlite_master WHERE type = 'index' AND name = ?") + .get("idx_agent_transcript_event_sequence") as { sql?: unknown } | undefined; + expect( + typeof transcriptIndex?.sql === "string" + ? transcriptIndex.sql.replace(/\s+/g, " ").trim() + : transcriptIndex?.sql, + ).toContain("ON transcript_event_identities(session_id, event_type, seq DESC)"); + expect(readSqliteNumberPragma(migrated.db, "user_version")).toBe(OPENCLAW_AGENT_SCHEMA_VERSION); + }); + it("records durable per-agent schema metadata", () => { const stateDir = createTempStateDir(); const database = openOpenClawAgentDatabase({ diff --git a/src/state/openclaw-agent-db.ts b/src/state/openclaw-agent-db.ts index 94ebc7525f21..940b8968153f 100644 --- a/src/state/openclaw-agent-db.ts +++ b/src/state/openclaw-agent-db.ts @@ -20,6 +20,7 @@ import { } from "../infra/sqlite-user-version.js"; import { configureSqliteConnectionPragmas, + configureSqlitePreSchemaPragmas, registerSqliteCacheExitClose, type SqliteWalMaintenance, } from "../infra/sqlite-wal.js"; @@ -46,10 +47,11 @@ export { resolveOpenClawAgentSqlitePath } from "./openclaw-agent-db.paths.js"; * per pathname, protected with private file modes, and registered in the shared * OpenClaw state database for discovery and maintenance. */ -// v5 = transcript mutation watermark. The v4 session/transcript flip and main's v2 memory-identity +// v6 = session/transcript hot-path indexes. v5 added transcript mutation watermarks. +// The v4 session/transcript flip and main's v2 memory-identity // change is folded in structure-gated (migrateMemoryIndexSourcesIdentity), so // v2 main DBs and pre-merge v4 flip DBs both converge on this schema. -export const OPENCLAW_AGENT_SCHEMA_VERSION = 5; +export const OPENCLAW_AGENT_SCHEMA_VERSION = 6; const OPENCLAW_AGENT_DB_DIR_MODE = 0o700; const OPENCLAW_AGENT_DB_FILE_MODE = 0o600; const OPENCLAW_AGENT_DB_SLOW_OPEN_MS = 1_000; @@ -81,7 +83,6 @@ type OpenClawAgentMetadataDatabase = Pick; const cachedDatabases = new Map(); -const registeredDatabasePaths = new Set(); type ExistingSchemaMeta = { agentId: string | null; @@ -182,6 +183,9 @@ function migrateOpenClawAgentSchema(db: DatabaseSync): void { if (userVersion >= OPENCLAW_AGENT_SCHEMA_VERSION) { return; } + if (userVersion < 6) { + db.exec("DROP INDEX IF EXISTS idx_agent_session_entries_session_id;"); + } if (userVersion < 3) { db.exec("DROP INDEX IF EXISTS idx_agent_transcript_events_session;"); } @@ -578,18 +582,6 @@ function ensureAgentSchema(db: DatabaseSync, agentId: string, pathname: string): } } -// auto_vacuum only takes effect when set before the first page is written; -// existing databases keep their mode until a doctor-owned full VACUUM. -// INCREMENTAL lets maintenance release freed pages in bounded steps so -// per-agent DBs shrink after retention deletes rows instead of pinning -// their high-water mark forever. -function enableIncrementalAutoVacuumForFreshDatabase(db: DatabaseSync): void { - const row = db.prepare("PRAGMA page_count").get() as { page_count?: unknown } | undefined; - if (row?.page_count === 0) { - db.exec("PRAGMA auto_vacuum = INCREMENTAL;"); - } -} - /** Initialize agent schema/ownership metadata on an independently managed connection. */ export function ensureOpenClawAgentDatabaseSchema( db: DatabaseSync, @@ -599,7 +591,9 @@ export function ensureOpenClawAgentDatabaseSchema( const databaseOptions = { ...options, agentId }; const pathname = resolveOpenClawAgentSqlitePath(databaseOptions); ensureOpenClawAgentDatabasePermissions(pathname, databaseOptions); - enableIncrementalAutoVacuumForFreshDatabase(db); + configureSqlitePreSchemaPragmas(db, { + busyTimeoutMs: OPENCLAW_SQLITE_BUSY_TIMEOUT_MS, + }); ensureAgentSchema(db, agentId, pathname); ensureOpenClawAgentDatabasePermissions(pathname, databaseOptions); if (options.register === true) { @@ -822,7 +816,9 @@ export function openOpenClawAgentDatabase( const walMaintenance = (() => { let maintenance: SqliteWalMaintenance | undefined; try { - enableIncrementalAutoVacuumForFreshDatabase(db); + configureSqlitePreSchemaPragmas(db, { + busyTimeoutMs: OPENCLAW_SQLITE_BUSY_TIMEOUT_MS, + }); maintenance = configureSqliteConnectionPragmas(db, { busyTimeoutMs: OPENCLAW_SQLITE_BUSY_TIMEOUT_MS, databaseLabel: `openclaw-agent:${agentId}`, @@ -840,14 +836,16 @@ export function openOpenClawAgentDatabase( })(); ensureOpenClawAgentDatabasePermissions(pathname, databaseOptions); const database = { agentId, db, path: pathname, walMaintenance }; + try { + registerAgentDatabase({ agentId, path: pathname, env: options.env }); + } catch (error) { + closeCachedOpenClawAgentDatabase(database); + throw error; + } cachedDatabases.set(pathname, database); // Safety net for processes that end without an orderly close: agent DBs have // no shutdown owner like the ACP/gateway state DB closes. Closing unregisters. unregisterExitClose ??= registerSqliteCacheExitClose(closeOpenClawAgentDatabases); - if (!registeredDatabasePaths.has(pathname)) { - registerAgentDatabase({ agentId, path: pathname, env: options.env }); - registeredDatabasePaths.add(pathname); - } logSlowAgentDatabaseOpen({ agentId, elapsedMs: Date.now() - openStartedAt, @@ -943,7 +941,6 @@ export function closeOpenClawAgentDatabases(): void { closeCachedOpenClawAgentDatabase(database); } cachedDatabases.clear(); - registeredDatabasePaths.clear(); } /** Test alias for closing cached agent database handles from teardown code. */ diff --git a/src/state/openclaw-agent-schema.generated.ts b/src/state/openclaw-agent-schema.generated.ts index 7b58064f30d6..0ffb3c13d3e1 100644 --- a/src/state/openclaw-agent-schema.generated.ts +++ b/src/state/openclaw-agent-schema.generated.ts @@ -118,8 +118,8 @@ CREATE TABLE IF NOT EXISTS session_entries ( CREATE INDEX IF NOT EXISTS idx_agent_session_entries_updated_at ON session_entries(updated_at DESC, session_key); -CREATE INDEX IF NOT EXISTS idx_agent_session_entries_session_id - ON session_entries(session_id); +CREATE INDEX IF NOT EXISTS idx_agent_session_entries_session_updated + ON session_entries(session_id, updated_at DESC, session_key); CREATE TABLE IF NOT EXISTS transcript_events ( session_id TEXT NOT NULL, @@ -164,6 +164,9 @@ CREATE INDEX IF NOT EXISTS idx_agent_transcript_event_parent ON transcript_event_identities(session_id, parent_id) WHERE parent_id IS NOT NULL; +CREATE INDEX IF NOT EXISTS idx_agent_transcript_event_sequence + ON transcript_event_identities(session_id, event_type, seq DESC); + CREATE TABLE IF NOT EXISTS cache_entries ( scope TEXT NOT NULL, key TEXT NOT NULL, diff --git a/src/state/openclaw-agent-schema.sql b/src/state/openclaw-agent-schema.sql index c38f7b1b4366..4428f1aca6b6 100644 --- a/src/state/openclaw-agent-schema.sql +++ b/src/state/openclaw-agent-schema.sql @@ -113,8 +113,8 @@ CREATE TABLE IF NOT EXISTS session_entries ( CREATE INDEX IF NOT EXISTS idx_agent_session_entries_updated_at ON session_entries(updated_at DESC, session_key); -CREATE INDEX IF NOT EXISTS idx_agent_session_entries_session_id - ON session_entries(session_id); +CREATE INDEX IF NOT EXISTS idx_agent_session_entries_session_updated + ON session_entries(session_id, updated_at DESC, session_key); CREATE TABLE IF NOT EXISTS transcript_events ( session_id TEXT NOT NULL, @@ -159,6 +159,9 @@ CREATE INDEX IF NOT EXISTS idx_agent_transcript_event_parent ON transcript_event_identities(session_id, parent_id) WHERE parent_id IS NOT NULL; +CREATE INDEX IF NOT EXISTS idx_agent_transcript_event_sequence + ON transcript_event_identities(session_id, event_type, seq DESC); + CREATE TABLE IF NOT EXISTS cache_entries ( scope TEXT NOT NULL, key TEXT NOT NULL, diff --git a/src/state/openclaw-state-db.test.ts b/src/state/openclaw-state-db.test.ts index e31f5d3863d4..1357a1fe65b9 100644 --- a/src/state/openclaw-state-db.test.ts +++ b/src/state/openclaw-state-db.test.ts @@ -226,6 +226,244 @@ function expectNoncanonicalAuditSchemaRejected(stateDir: string, databasePath: s }); } +function runConcurrentSchemaProbe(params: { + mode: "fresh" | "upgrade"; + moduleUrl: string; + rootDir: string; +}): string[] { + const workerSource = ` + import fs from "node:fs"; + import { DatabaseSync } from "node:sqlite"; + + const pageBarrierDir = process.env.OPENCLAW_SCHEMA_TEST_PAGE_BARRIER_DIR; + if (pageBarrierDir) { + const pageReadyPath = process.env.OPENCLAW_SCHEMA_TEST_PAGE_READY_PATH; + const workerCount = Number(process.env.OPENCLAW_SCHEMA_TEST_WORKER_COUNT); + const originalPrepare = DatabaseSync.prototype.prepare; + const sleepBuffer = new Int32Array(new SharedArrayBuffer(4)); + DatabaseSync.prototype.prepare = function (sql) { + const statement = originalPrepare.call(this, sql); + if (sql !== "PRAGMA page_count") { + return statement; + } + return new Proxy(statement, { + get(target, property) { + if (property === "get") { + return (...args) => { + const row = target.get(...args); + if (row?.page_count !== 0) { + throw new Error("fresh database acquired pages before the initialization barrier"); + } + fs.writeFileSync(pageReadyPath, "ready"); + const deadline = Date.now() + 15_000; + while ( + fs.readdirSync(pageBarrierDir).filter((name) => name.startsWith("page-ready-")) + .length < workerCount + ) { + if (Date.now() >= deadline) { + throw new Error("timed out waiting for fresh database page-count barrier"); + } + Atomics.wait(sleepBuffer, 0, 0, 2); + } + return row; + }; + } + const value = Reflect.get(target, property, target); + return typeof value === "function" ? value.bind(target) : value; + }, + }); + }; + } + + const { + closeOpenClawStateDatabaseForTest, + openOpenClawStateDatabase, + } = await import(process.env.OPENCLAW_SCHEMA_TEST_MODULE_URL); + const databasePath = process.env.OPENCLAW_SCHEMA_TEST_DATABASE_PATH; + const readyPath = process.env.OPENCLAW_SCHEMA_TEST_READY_PATH; + const startPath = process.env.OPENCLAW_SCHEMA_TEST_START_PATH; + fs.writeFileSync(readyPath, "ready"); + const deadline = Date.now() + 15_000; + while (!fs.existsSync(startPath)) { + if (Date.now() >= deadline) { + throw new Error("timed out waiting for concurrent schema upgrade start"); + } + await new Promise((resolve) => setTimeout(resolve, 2)); + } + try { + const database = openOpenClawStateDatabase({ path: databasePath }); + const integrity = database.db.prepare("PRAGMA integrity_check").get(); + if (integrity?.integrity_check !== "ok") { + throw new Error("state database integrity check failed"); + } + } finally { + closeOpenClawStateDatabaseForTest(); + } + `; + const orchestratorSource = ` + import { spawn } from "node:child_process"; + import fs from "node:fs"; + import path from "node:path"; + import { DatabaseSync } from "node:sqlite"; + + const moduleUrl = ${JSON.stringify(params.moduleUrl)}; + const rootDir = ${JSON.stringify(params.rootDir)}; + const mode = ${JSON.stringify(params.mode)}; + const workerSource = ${JSON.stringify(workerSource)}; + const workerCount = 8; + const roundCount = 3; + const databasePaths = []; + const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + + function waitForChild(child) { + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (chunk) => { + stdout += chunk; + }); + child.stderr.on("data", (chunk) => { + stderr += chunk; + }); + return new Promise((resolve, reject) => { + child.once("error", reject); + child.once("close", (code, signal) => resolve({ code, signal, stderr, stdout })); + }); + } + + for (let round = 0; round < roundCount; round += 1) { + const databasePath = path.join(rootDir, \`concurrent-\${mode}-\${round}.sqlite\`); + const barrierDir = path.join(rootDir, \`barrier-\${round}\`); + fs.mkdirSync(barrierDir, { recursive: true }); + + if (mode === "upgrade") { + const { + closeOpenClawStateDatabaseForTest, + openOpenClawStateDatabase, + } = await import(moduleUrl); + openOpenClawStateDatabase({ path: databasePath }); + closeOpenClawStateDatabaseForTest(); + + const legacy = new DatabaseSync(databasePath); + legacy + .prepare( + \`INSERT INTO task_runs ( + task_id, runtime, requester_session_key, owner_key, scope_kind, + child_session_key, agent_id, task, status, delivery_status, + notify_policy, created_at, last_event_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)\`, + ) + .run( + \`legacy-concurrent-\${round}\`, + "subagent", + "agent:main:main", + "agent:main:main", + "session", + \`agent:worker:subagent:concurrent-\${round}\`, + "main", + "Verify concurrent schema upgrade", + "running", + "pending", + "done_only", + 100, + 100, + ); + legacy.exec(\` + DROP TABLE worker_environment_credentials; + ALTER TABLE gateway_boot_lifecycle DROP COLUMN startup_reason; + ALTER TABLE task_runs DROP COLUMN requester_agent_id; + ALTER TABLE official_external_plugin_catalog_snapshots DROP COLUMN trust_mode; + ALTER TABLE official_external_plugin_catalog_snapshots DROP COLUMN trust_key_id; + ALTER TABLE official_external_plugin_catalog_snapshots DROP COLUMN trust_signature_count; + ALTER TABLE official_external_plugin_catalog_snapshots DROP COLUMN trust_threshold; + ALTER TABLE official_external_plugin_catalog_snapshots DROP COLUMN trust_verified_at; + ALTER TABLE worker_environments DROP COLUMN bootstrap_bundle_hash; + ALTER TABLE worker_environments DROP COLUMN bootstrap_openclaw_version; + ALTER TABLE worker_environments DROP COLUMN bootstrap_protocol_features_json; + ALTER TABLE worker_environments DROP COLUMN owner_epoch; + ALTER TABLE worker_environments DROP COLUMN teardown_terminal_state; + ALTER TABLE worker_environments DROP COLUMN ssh_host_key; + PRAGMA user_version = 1; + UPDATE schema_meta + SET schema_version = 1, + updated_at = 1 + WHERE meta_key = 'primary'; + \`); + legacy.close(); + } + + const startPath = path.join(barrierDir, "start"); + const workers = Array.from({ length: workerCount }, (_, index) => { + const readyPath = path.join(barrierDir, \`ready-\${index}\`); + const pageReadyPath = path.join(barrierDir, \`page-ready-\${index}\`); + return spawn( + process.execPath, + ["--import", "tsx", "--input-type=module", "-e", workerSource], + { + env: { + ...process.env, + OPENCLAW_SCHEMA_TEST_DATABASE_PATH: databasePath, + OPENCLAW_SCHEMA_TEST_MODULE_URL: moduleUrl, + OPENCLAW_SCHEMA_TEST_READY_PATH: readyPath, + OPENCLAW_SCHEMA_TEST_START_PATH: startPath, + ...(mode === "fresh" + ? { + OPENCLAW_SCHEMA_TEST_PAGE_BARRIER_DIR: barrierDir, + OPENCLAW_SCHEMA_TEST_PAGE_READY_PATH: pageReadyPath, + OPENCLAW_SCHEMA_TEST_WORKER_COUNT: String(workerCount), + } + : {}), + }, + stdio: ["ignore", "pipe", "pipe"], + }, + ); + }); + const outcomes = workers.map(waitForChild); + try { + const readyDeadline = Date.now() + 15_000; + while ( + !workers.every((_worker, index) => + fs.existsSync(path.join(barrierDir, \`ready-\${index}\`)), + ) + ) { + if (workers.some((worker) => worker.exitCode !== null || worker.signalCode !== null)) { + break; + } + if (Date.now() >= readyDeadline) { + throw new Error(\`round \${round} timed out waiting for workers\`); + } + await sleep(2); + } + fs.writeFileSync(startPath, "start"); + const results = await Promise.all(outcomes); + const failures = results.filter((result) => result.code !== 0); + if (failures.length > 0) { + throw new Error(\`round \${round} worker failures: \${JSON.stringify(failures)}\`); + } + } finally { + for (const worker of workers) { + if (worker.exitCode === null && worker.signalCode === null) { + worker.kill(); + } + } + await Promise.allSettled(outcomes); + } + databasePaths.push(databasePath); + } + + console.log(JSON.stringify(databasePaths)); + `; + const output = execFileSync( + process.execPath, + ["--import", "tsx", "--input-type=module", "-e", orchestratorSource], + { encoding: "utf8", timeout: 60_000 }, + ); + const resultLine = output.trim().split("\n").at(-1); + if (!resultLine) { + throw new Error(`concurrent schema ${params.mode} probe produced no result`); + } + return JSON.parse(resultLine) as string[]; +} + afterAll(() => { cleanupTempDirs(stateDbTempDirs); }); @@ -941,6 +1179,67 @@ describe("openclaw state database", () => { ); }); + it("serializes concurrent additive schema upgrades across processes", () => { + const rootDir = createTempStateDir(); + const moduleUrl = new URL("./openclaw-state-db.ts", import.meta.url).href; + const databasePaths = runConcurrentSchemaProbe({ mode: "upgrade", moduleUrl, rootDir }); + const expectedShape = createSqliteSchemaShapeFromSql( + new URL("./openclaw-state-schema.sql", import.meta.url), + ); + const { DatabaseSync } = requireNodeSqlite(); + + expect(databasePaths).toHaveLength(3); + for (const [round, databasePath] of databasePaths.entries()) { + const db = new DatabaseSync(databasePath, { readOnly: true }); + try { + expect(db.prepare("PRAGMA integrity_check").get()).toEqual({ integrity_check: "ok" }); + expect(db.prepare("PRAGMA foreign_key_check").all()).toEqual([]); + expect(readSqliteNumberPragma(db, "user_version")).toBe(OPENCLAW_STATE_SCHEMA_VERSION); + expect( + db.prepare("SELECT schema_version FROM schema_meta WHERE meta_key = 'primary'").get(), + ).toEqual({ schema_version: OPENCLAW_STATE_SCHEMA_VERSION }); + expect( + db + .prepare("SELECT agent_id, requester_agent_id FROM task_runs WHERE task_id = ?") + .get(`legacy-concurrent-${round}`), + ).toEqual({ + agent_id: "worker", + requester_agent_id: "main", + }); + expect(collectSqliteSchemaShape(db)).toEqual(expectedShape); + } finally { + db.close(); + } + } + }, 60_000); + + it("serializes concurrent fresh database initialization across processes", () => { + const rootDir = createTempStateDir(); + const moduleUrl = new URL("./openclaw-state-db.ts", import.meta.url).href; + const databasePaths = runConcurrentSchemaProbe({ mode: "fresh", moduleUrl, rootDir }); + const expectedShape = createSqliteSchemaShapeFromSql( + new URL("./openclaw-state-schema.sql", import.meta.url), + ); + const { DatabaseSync } = requireNodeSqlite(); + + expect(databasePaths).toHaveLength(3); + for (const databasePath of databasePaths) { + const db = new DatabaseSync(databasePath, { readOnly: true }); + try { + expect(db.prepare("PRAGMA integrity_check").get()).toEqual({ integrity_check: "ok" }); + expect(db.prepare("PRAGMA foreign_key_check").all()).toEqual([]); + expect(readSqliteNumberPragma(db, "auto_vacuum")).toBe(2); + expect(readSqliteNumberPragma(db, "user_version")).toBe(OPENCLAW_STATE_SCHEMA_VERSION); + expect( + db.prepare("SELECT schema_version FROM schema_meta WHERE meta_key = 'primary'").get(), + ).toEqual({ schema_version: OPENCLAW_STATE_SCHEMA_VERSION }); + expect(collectSqliteSchemaShape(db)).toEqual(expectedShape); + } finally { + db.close(); + } + } + }, 60_000); + it("migrates requester and executor attribution for existing cross-agent tasks", () => { const stateDir = createTempStateDir(); const database = openOpenClawStateDatabase({ @@ -1565,6 +1864,7 @@ describe("openclaw state database", () => { ); expect(readSqliteNumberPragma(database.db, "foreign_keys")).toBe(1); expect(readSqliteNumberPragma(database.db, "synchronous")).toBe(1); + expect(readSqliteNumberPragma(database.db, "auto_vacuum")).toBe(2); expect(readSqliteNumberPragma(database.db, "user_version")).toBe(OPENCLAW_STATE_SCHEMA_VERSION); expect(readSqliteNumberPragma(database.db, "wal_autocheckpoint")).toBe(1000); const journalMode = database.db.prepare("PRAGMA journal_mode").get() as diff --git a/src/state/openclaw-state-db.ts b/src/state/openclaw-state-db.ts index 1867ddfdc5b9..80bc59e67080 100644 --- a/src/state/openclaw-state-db.ts +++ b/src/state/openclaw-state-db.ts @@ -18,6 +18,7 @@ import { } from "../infra/sqlite-user-version.js"; import { configureSqliteConnectionPragmas, + configureSqlitePreSchemaPragmas, type SqliteWalMaintenance, } from "../infra/sqlite-wal.js"; import { createSubsystemLogger } from "../logging/subsystem.js"; @@ -763,35 +764,45 @@ export function repairOpenClawStateDatabaseSchema(options: OpenClawStateDatabase } function ensureStartupMigrationCheckpointSchema(db: DatabaseSync, pathname: string): void { - assertSupportedSchemaVersion(db, pathname); - db.exec(` - CREATE TABLE IF NOT EXISTS schema_meta ( - meta_key TEXT NOT NULL PRIMARY KEY, - role TEXT NOT NULL, - schema_version INTEGER NOT NULL, - agent_id TEXT, - app_version TEXT, - created_at INTEGER NOT NULL, - updated_at INTEGER NOT NULL - ); - CREATE TABLE IF NOT EXISTS state_leases ( - scope TEXT NOT NULL, - lease_key TEXT NOT NULL, - owner TEXT NOT NULL, - expires_at INTEGER, - heartbeat_at INTEGER, - payload_json TEXT, - created_at INTEGER NOT NULL, - updated_at INTEGER NOT NULL, - PRIMARY KEY (scope, lease_key) - ); - CREATE INDEX IF NOT EXISTS idx_state_leases_expiry - ON state_leases(expires_at, scope, lease_key) - WHERE expires_at IS NOT NULL; - CREATE INDEX IF NOT EXISTS idx_state_leases_owner - ON state_leases(owner, updated_at DESC); - `); - ensureColumn(db, "schema_meta", "app_version TEXT"); + runSqliteImmediateTransactionSync( + db, + () => { + assertSupportedSchemaVersion(db, pathname); + db.exec(` + CREATE TABLE IF NOT EXISTS schema_meta ( + meta_key TEXT NOT NULL PRIMARY KEY, + role TEXT NOT NULL, + schema_version INTEGER NOT NULL, + agent_id TEXT, + app_version TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL + ); + CREATE TABLE IF NOT EXISTS state_leases ( + scope TEXT NOT NULL, + lease_key TEXT NOT NULL, + owner TEXT NOT NULL, + expires_at INTEGER, + heartbeat_at INTEGER, + payload_json TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + PRIMARY KEY (scope, lease_key) + ); + CREATE INDEX IF NOT EXISTS idx_state_leases_expiry + ON state_leases(expires_at, scope, lease_key) + WHERE expires_at IS NOT NULL; + CREATE INDEX IF NOT EXISTS idx_state_leases_owner + ON state_leases(owner, updated_at DESC); + `); + ensureColumn(db, "schema_meta", "app_version TEXT"); + }, + { + busyTimeoutMs: OPENCLAW_SQLITE_BUSY_TIMEOUT_MS, + databaseLabel: pathname, + operationLabel: "state.schema.ensure-startup-checkpoint", + }, + ); } export function withOpenClawStateStartupMigrationCheckpointDatabase( @@ -1241,7 +1252,9 @@ function backfillDeliveryQueueEntriesFromEntryJson(db: DatabaseSync): void { } } -function ensureAdditiveStateColumns(db: DatabaseSync, pathname: string): void { +// The caller owns the state.schema.ensure transaction so every probe, DDL +// change, and backfill observes one authoritative schema across processes. +function ensureAdditiveStateColumns(db: DatabaseSync): void { ensureColumn(db, "device_pairing_pending", "refreshed_at_ms INTEGER"); ensureColumn(db, "device_pairing_paired", "approved_via TEXT"); ensureColumn(db, "device_pairing_paired", "operator_label TEXT"); @@ -1340,12 +1353,10 @@ function ensureAdditiveStateColumns(db: DatabaseSync, pathname: string): void { ensureColumn(db, "cron_jobs", "schedule_identity TEXT"); ensureColumn(db, "cron_jobs", "sort_order INTEGER NOT NULL DEFAULT 0"); backfillCronJobsFromJobJson(db); - runSqliteImmediateTransactionSync(db, () => { - const addedDeliveryThreadIdType = ensureColumn(db, "cron_jobs", "delivery_thread_id_type TEXT"); - if (addedDeliveryThreadIdType) { - migrateLegacyCronDeliveryThreadIds(db); - } - }); + const addedDeliveryThreadIdType = ensureColumn(db, "cron_jobs", "delivery_thread_id_type TEXT"); + if (addedDeliveryThreadIdType) { + migrateLegacyCronDeliveryThreadIds(db); + } ensureColumn(db, "sandbox_registry_entries", "session_key TEXT"); ensureColumn(db, "sandbox_registry_entries", "backend_id TEXT"); ensureColumn(db, "sandbox_registry_entries", "runtime_label TEXT"); @@ -1412,21 +1423,11 @@ function ensureAdditiveStateColumns(db: DatabaseSync, pathname: string): void { ensureColumn(db, "official_external_plugin_catalog_snapshots", "trust_signature_count INTEGER"); ensureColumn(db, "official_external_plugin_catalog_snapshots", "trust_threshold INTEGER"); ensureColumn(db, "official_external_plugin_catalog_snapshots", "trust_verified_at TEXT"); - runSqliteImmediateTransactionSync( - db, - () => { - const addedTaskRequesterAgentId = ensureColumn(db, "task_runs", "requester_agent_id TEXT"); - if (addedTaskRequesterAgentId) { - repairLegacyTaskAgentAttribution(db); - } - repairLegacyTaskDeliveryStatuses(db); - }, - { - busyTimeoutMs: OPENCLAW_SQLITE_BUSY_TIMEOUT_MS, - databaseLabel: pathname, - operationLabel: "state.schema.ensure-columns", - }, - ); + const addedTaskRequesterAgentId = ensureColumn(db, "task_runs", "requester_agent_id TEXT"); + if (addedTaskRequesterAgentId) { + repairLegacyTaskAgentAttribution(db); + } + repairLegacyTaskDeliveryStatuses(db); ensureColumn(db, "task_runs", "tool_use_count INTEGER"); ensureColumn(db, "task_runs", "last_tool_name TEXT"); ensureColumn(db, "subagent_runs", "task_name TEXT"); @@ -1448,14 +1449,14 @@ function ensureAdditiveStateColumns(db: DatabaseSync, pathname: string): void { } function ensureSchema(db: DatabaseSync, pathname: string): void { - assertSupportedSchemaVersion(db, pathname); - ensureAdditiveStateColumns(db, pathname); - assertCanonicalStateSchemaShape(db, pathname); const now = Date.now(); const kysely = getNodeSqliteKysely(db); runSqliteImmediateTransactionSync( db, () => { + assertSupportedSchemaVersion(db, pathname); + ensureAdditiveStateColumns(db); + assertCanonicalStateSchemaShape(db, pathname); db.exec(OPENCLAW_STATE_SCHEMA_SQL); // Retired node_pairing_* tables were created by earlier schema revisions but // never had a shipped writer (the node surface lives on device_pairing_paired @@ -1494,7 +1495,6 @@ function ensureSchema(db: DatabaseSync, pathname: string): void { operationLabel: "state.schema.ensure", }, ); - ensureAdditiveStateColumns(db, pathname); } function resolveDatabasePath(options: OpenClawStateDatabaseOptions = {}): string { @@ -1524,6 +1524,9 @@ export function openOpenClawStateDatabase( const walMaintenance = (() => { let maintenance: SqliteWalMaintenance | undefined; try { + configureSqlitePreSchemaPragmas(db, { + busyTimeoutMs: OPENCLAW_SQLITE_BUSY_TIMEOUT_MS, + }); maintenance = configureSqliteConnectionPragmas(db, { busyTimeoutMs: OPENCLAW_SQLITE_BUSY_TIMEOUT_MS, databaseLabel: "openclaw-state", diff --git a/src/state/sqlite-query-plan.test.ts b/src/state/sqlite-query-plan.test.ts index 1c433ccbbcb3..fffa42c89aaa 100644 --- a/src/state/sqlite-query-plan.test.ts +++ b/src/state/sqlite-query-plan.test.ts @@ -190,8 +190,37 @@ describe("sqlite hot query plans", () => { FROM cache_entries WHERE scope = ? AND expires_at IS NOT NULL ORDER BY expires_at ASC, key - LIMIT 50 + LIMIT 50 `, }); + expectPlanUsesIndex({ + db: database.db, + indexName: "idx_agent_session_entries_session_updated", + params: ["session-1"], + sql: ` + SELECT session_key + FROM session_entries + WHERE session_id = ? + ORDER BY updated_at DESC, session_key ASC + LIMIT 1 + `, + }); + const latestMessagePlan = explainQueryPlan( + database.db, + ` + SELECT te.event_json + FROM transcript_events AS te + JOIN transcript_event_identities AS ti + ON ti.session_id = te.session_id AND ti.seq = te.seq + WHERE te.session_id = ? AND ti.event_type = 'message' + ORDER BY ti.seq DESC + LIMIT 1 + `, + ["session-1"], + ); + expect(latestMessagePlan).toContain( + "USING COVERING INDEX idx_agent_transcript_event_sequence (session_id=? AND event_type=?)", + ); + expect(latestMessagePlan).not.toContain("USE TEMP B-TREE FOR ORDER BY"); }); });