diff --git a/CHANGELOG.md b/CHANGELOG.md index bb3f2e21af14..0f02e8969073 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ Docs: https://docs.openclaw.ai ### Changes +- **SQLite snapshots:** add `openclaw backup sqlite create|list|verify|restore` for compact, verified global and per-agent database artifacts with fresh-target-only restore. (#94805) Thanks @giodl73-repo. - **GPT-5.6 Ultra and runtime switching:** support Sol, Terra, and Luna across OpenClaw and Codex engines; keep model, runtime, and thinking selection atomic through `/model` and fallback; and add live matrix coverage for both harnesses. (#98021) Thanks @anyech. - **OpenAI GPT-5.6 defaults:** use `openai/gpt-5.6` (Sol alias) for fresh API-key setup and exact `openai/gpt-5.6-sol` for fresh Codex/OAuth setup, while preserving existing primaries, fallbacks, aliases, and explicit GPT-5.5 selections. (#103234) - **Meta provider:** add bundled `muse-spark-1.1` model support with Responses API streaming, tool calls, encrypted reasoning replay, onboarding, and standalone npm/ClawHub distribution. (#102873) Thanks @HamidShojanazeri. diff --git a/docs/cli/backup.md b/docs/cli/backup.md index b86e00bd51f7..75d5b2253963 100644 --- a/docs/cli/backup.md +++ b/docs/cli/backup.md @@ -1,7 +1,8 @@ --- -summary: "CLI reference for `openclaw backup` (create local backup archives)" +summary: "CLI reference for `openclaw backup` (archives and SQLite snapshots)" read_when: - You want a first-class backup archive for local OpenClaw state + - You need a compact, verified snapshot of one OpenClaw SQLite database - You want to preview which paths would be included before reset or uninstall title: "Backup" --- @@ -18,6 +19,12 @@ openclaw backup create --verify openclaw backup create --no-include-workspace openclaw backup create --only-config openclaw backup verify ./2026-03-09T08-00-00.000+08-00-openclaw-backup.tar.gz +openclaw backup sqlite create --global --repository ~/Backups/openclaw-sqlite +openclaw backup sqlite create --agent main --repository ~/Backups/openclaw-sqlite +openclaw backup sqlite list --repository ~/Backups/openclaw-sqlite +openclaw backup sqlite verify ~/Backups/openclaw-sqlite/ +openclaw backup sqlite verify ~/Backups/openclaw-sqlite/ --scratch ~/Private/openclaw-scratch +openclaw backup sqlite restore ~/Backups/openclaw-sqlite/ --target ./restored/openclaw.sqlite ``` ## Notes @@ -28,6 +35,45 @@ openclaw backup verify ./2026-03-09T08-00-00.000+08-00-openclaw-backup.tar.gz - `openclaw backup verify ` checks that the archive contains exactly one root manifest, rejects traversal-style archive paths and SQLite sidecars, confirms every manifest-declared payload exists, validates every SQLite snapshot's file shape, and runs full integrity and role checks on canonical OpenClaw databases. Dedicated plugin schemas remain opaque because they may require owner-defined SQLite capabilities. `openclaw backup create --verify` runs that validation immediately after writing the archive. - `openclaw backup create --only-config` backs up just the active JSON config file. +## SQLite snapshots + +Use `openclaw backup sqlite` when you need a portable artifact for one OpenClaw-owned SQLite database instead of a broad state archive. + +Snapshot creation accepts exactly one named source: + +| Command | Database | +| --------------------------------------------------------------- | ---------------------- | +| `openclaw backup sqlite create --global --repository ` | Shared OpenClaw state | +| `openclaw backup sqlite create --agent --repository ` | One per-agent database | + +The repository contains one directory per committed snapshot. Each snapshot directory contains exactly: + +- `manifest.json` +- `database.sqlite` + +Snapshot creation verifies the live database before reading it, uses SQLite `VACUUM INTO` to capture committed WAL state into a compact database, verifies the generated database again, and publishes the completed directory without overwriting existing paths. Global snapshots remove transient delivery queue rows and compact again so deleted queue payloads are not retained in free pages. + +Do not copy live `.sqlite`, `-wal`, `-shm`, or `-journal` files as a portability artifact. Copy only completed snapshot directories. + +SQLite snapshots can contain auth profiles, session state, plugin state, and other sensitive records. Protect repositories with the same permissions, encryption, retention policy, and destination restrictions as the live OpenClaw state directory. + +### Verify and restore + +```bash +openclaw backup sqlite verify +openclaw backup sqlite restore --target +``` + +Verification checks the strict manifest shape, artifact size and SHA-256, SQLite integrity, foreign keys, schema version, database role and owner, and OpenClaw-owned index definitions. + +Verification validates a private content-pinned copy so pathname races cannot swap the bytes SQLite inspects. By default, that temporary copy is created beside the snapshot repository and removed before the command returns. The staging root and its ancestor chain must prevent other users from replacing it. POSIX roots must be current-user-owned and not group/world writable; sticky ancestors such as `/tmp` are accepted for user-owned children. macOS ACL grants that expose or make staging replaceable are rejected. Windows roots and ancestors must be owned by the current user or a trusted OS principal, with ACLs that deny untrusted staging access. For a read-only mount or network share, pass `--scratch ` on storage with equivalent encryption and destination controls. + +Snapshot creation applies the same owner, ACL, ancestor, and path-identity checks to the repository before staging or publishing database bytes. + +Restore repeats verification and writes only to a fresh target. It refuses an existing target, `-wal`, `-shm`, or `-journal` sidecar and never performs an in-place replacement of a live OpenClaw database. The target parent has the same path-security requirements as verification scratch. Activating a restored database remains an explicit offline operator step. + +Snapshot repositories are local directories. Scheduling, upload, retention, incremental WAL bundles, failover, and restore-on-boot behavior are intentionally outside this command. + ## What gets backed up `openclaw backup create` plans sources from your local OpenClaw install: diff --git a/src/cli/program/core-command-descriptors.ts b/src/cli/program/core-command-descriptors.ts index e9832ff7ee66..caf10b2f10f5 100644 --- a/src/cli/program/core-command-descriptors.ts +++ b/src/cli/program/core-command-descriptors.ts @@ -34,7 +34,7 @@ const coreCliCommandCatalog = defineCommandDescriptorCatalog([ }, { name: "backup", - description: "Create and verify local backup archives for OpenClaw state", + description: "Create and verify backup archives and SQLite snapshots", hasSubcommands: true, }, { diff --git a/src/cli/program/preaction.test.ts b/src/cli/program/preaction.test.ts index 47fba188b87e..0c446bb1a0b7 100644 --- a/src/cli/program/preaction.test.ts +++ b/src/cli/program/preaction.test.ts @@ -719,6 +719,25 @@ describe("registerPreActionHooks", () => { expect(ensurePluginRegistryLoadedMock).not.toHaveBeenCalled(); }); + it("bypasses config guard for SQLite snapshot recovery commands", async () => { + await runPreAction({ + parseArgv: ["backup", "sqlite", "restore"], + processArgv: [ + "node", + "openclaw", + "backup", + "sqlite", + "restore", + "/tmp/snapshot", + "--target", + "/tmp/restore.sqlite", + ], + }); + + expect(ensureConfigReadyMock).not.toHaveBeenCalled(); + expect(ensurePluginRegistryLoadedMock).not.toHaveBeenCalled(); + }); + it("routes logs to stderr during plugin loading in --json mode and restores after", async () => { let stderrDuringPluginLoad = false; ensurePluginRegistryLoadedMock.mockImplementation(() => { diff --git a/src/cli/program/register.backup.test.ts b/src/cli/program/register.backup.test.ts index ddf5bae6f405..4eb95f7d96cf 100644 --- a/src/cli/program/register.backup.test.ts +++ b/src/cli/program/register.backup.test.ts @@ -5,6 +5,10 @@ import { registerBackupCommand } from "./register.backup.js"; const mocks = vi.hoisted(() => ({ backupCreateCommand: vi.fn(), + backupSqliteCreateCommand: vi.fn(), + backupSqliteListCommand: vi.fn(), + backupSqliteRestoreCommand: vi.fn(), + backupSqliteVerifyCommand: vi.fn(), backupVerifyCommand: vi.fn(), runtime: { log: vi.fn(), @@ -14,6 +18,10 @@ const mocks = vi.hoisted(() => ({ })); const backupCreateCommand = mocks.backupCreateCommand; +const backupSqliteCreateCommand = mocks.backupSqliteCreateCommand; +const backupSqliteListCommand = mocks.backupSqliteListCommand; +const backupSqliteRestoreCommand = mocks.backupSqliteRestoreCommand; +const backupSqliteVerifyCommand = mocks.backupSqliteVerifyCommand; const backupVerifyCommand = mocks.backupVerifyCommand; const runtime = mocks.runtime; @@ -25,6 +33,13 @@ vi.mock("../../commands/backup-verify.js", () => ({ backupVerifyCommand: mocks.backupVerifyCommand, })); +vi.mock("../../commands/backup-sqlite.js", () => ({ + backupSqliteCreateCommand: mocks.backupSqliteCreateCommand, + backupSqliteListCommand: mocks.backupSqliteListCommand, + backupSqliteRestoreCommand: mocks.backupSqliteRestoreCommand, + backupSqliteVerifyCommand: mocks.backupSqliteVerifyCommand, +})); + vi.mock("../../runtime.js", () => ({ defaultRuntime: mocks.runtime, })); @@ -39,6 +54,10 @@ describe("registerBackupCommand", () => { beforeEach(() => { vi.clearAllMocks(); backupCreateCommand.mockResolvedValue(undefined); + backupSqliteCreateCommand.mockResolvedValue(undefined); + backupSqliteListCommand.mockResolvedValue(undefined); + backupSqliteRestoreCommand.mockResolvedValue(undefined); + backupSqliteVerifyCommand.mockResolvedValue(undefined); backupVerifyCommand.mockResolvedValue(undefined); }); @@ -93,4 +112,91 @@ describe("registerBackupCommand", () => { expect(options.archive).toBe("/tmp/openclaw-backup.tar.gz"); expect(options.json).toBe(true); }); + + it("registers the SQLite snapshot command group", () => { + const program = new Command(); + + registerBackupCommand(program); + + const backup = program.commands.find((command) => command.name() === "backup"); + const sqlite = backup?.commands.find((command) => command.name() === "sqlite"); + expect(sqlite?.commands.map((command) => command.name()).toSorted()).toEqual([ + "create", + "list", + "restore", + "verify", + ]); + }); + + it("runs SQLite snapshot create for named OpenClaw databases", async () => { + await runCli([ + "backup", + "sqlite", + "create", + "--global", + "--repository", + "/tmp/snapshots", + "--json", + ]); + + expect(backupSqliteCreateCommand).toHaveBeenCalledWith(runtime, { + global: true, + agent: undefined, + repository: "/tmp/snapshots", + json: true, + }); + + await runCli([ + "backup", + "sqlite", + "create", + "--agent", + "main", + "--repository", + "/tmp/snapshots", + ]); + + expect(backupSqliteCreateCommand).toHaveBeenLastCalledWith(runtime, { + global: false, + agent: "main", + repository: "/tmp/snapshots", + json: false, + }); + }); + + it("runs SQLite snapshot list, verify, and restore", async () => { + await runCli(["backup", "sqlite", "list", "--repository", "/tmp/snapshots", "--json"]); + expect(backupSqliteListCommand).toHaveBeenCalledWith(runtime, { + repository: "/tmp/snapshots", + json: true, + }); + + await runCli([ + "backup", + "sqlite", + "verify", + "/tmp/snapshots/one", + "--scratch", + "/tmp/private-scratch", + "--json", + ]); + expect(backupSqliteVerifyCommand).toHaveBeenCalledWith(runtime, "/tmp/snapshots/one", { + scratch: "/tmp/private-scratch", + json: true, + }); + + await runCli([ + "backup", + "sqlite", + "restore", + "/tmp/snapshots/one", + "--target", + "/tmp/restored.sqlite", + "--json", + ]); + expect(backupSqliteRestoreCommand).toHaveBeenCalledWith(runtime, "/tmp/snapshots/one", { + target: "/tmp/restored.sqlite", + json: true, + }); + }); }); diff --git a/src/cli/program/register.backup.ts b/src/cli/program/register.backup.ts index 0fd8d8d1a0b6..31d64427bffb 100644 --- a/src/cli/program/register.backup.ts +++ b/src/cli/program/register.backup.ts @@ -2,6 +2,12 @@ import type { Command } from "commander"; import { formatDocsLink } from "../../../packages/terminal-core/src/links.js"; import { theme } from "../../../packages/terminal-core/src/theme.js"; +import { + backupSqliteCreateCommand, + backupSqliteListCommand, + backupSqliteRestoreCommand, + backupSqliteVerifyCommand, +} from "../../commands/backup-sqlite.js"; import { backupVerifyCommand } from "../../commands/backup-verify.js"; import { backupCreateCommand } from "../../commands/backup.js"; import { defaultRuntime } from "../../runtime.js"; @@ -12,7 +18,7 @@ import { formatHelpExamples } from "../help-format.js"; export function registerBackupCommand(program: Command) { const backup = program .command("backup") - .description("Create and verify local backup archives for OpenClaw state") + .description("Create and verify backup archives and SQLite snapshots") .addHelpText( "after", () => @@ -91,4 +97,90 @@ export function registerBackupCommand(program: Command) { }); }); }); + + registerBackupSqliteCommands(backup); +} + +function registerBackupSqliteCommands(backup: Command): void { + const sqlite = backup + .command("sqlite") + .description("Create, list, verify, and restore SQLite snapshots") + .action(() => { + sqlite.outputHelp(); + process.exitCode = 1; + }); + + sqlite + .command("create") + .description("Create a compact, verified snapshot of an OpenClaw SQLite database") + .option("--global", "Snapshot the shared OpenClaw state database", false) + .option("--agent ", "Snapshot one per-agent OpenClaw database") + .requiredOption("--repository ", "Snapshot repository directory") + .option("--json", "Output JSON", false) + .addHelpText( + "after", + () => + `\n${theme.heading("Examples:")}\n${formatHelpExamples([ + [ + "openclaw backup sqlite create --global --repository ~/Backups/openclaw-sqlite", + "Snapshot the shared state database.", + ], + [ + "openclaw backup sqlite create --agent main --repository ~/Backups/openclaw-sqlite", + "Snapshot the main agent database.", + ], + ])}`, + ) + .action(async (opts) => { + await runCommandWithRuntime(defaultRuntime, async () => { + await backupSqliteCreateCommand(defaultRuntime, { + global: Boolean(opts.global), + agent: opts.agent as string | undefined, + repository: opts.repository as string, + json: Boolean(opts.json), + }); + }); + }); + + sqlite + .command("list") + .description("List committed snapshots in a repository") + .requiredOption("--repository ", "Snapshot repository directory") + .option("--json", "Output JSON", false) + .action(async (opts) => { + await runCommandWithRuntime(defaultRuntime, async () => { + await backupSqliteListCommand(defaultRuntime, { + repository: opts.repository as string, + json: Boolean(opts.json), + }); + }); + }); + + sqlite + .command("verify ") + .description("Verify a snapshot manifest, artifact hash, SQLite integrity, and database owner") + .option("--scratch ", "Existing private directory for verification copies") + .option("--json", "Output JSON", false) + .action(async (snapshot, opts) => { + await runCommandWithRuntime(defaultRuntime, async () => { + await backupSqliteVerifyCommand(defaultRuntime, snapshot as string, { + scratch: opts.scratch as string | undefined, + json: Boolean(opts.json), + }); + }); + }); + + sqlite + .command("restore ") + .description("Restore a verified snapshot to a new SQLite database path") + .requiredOption("--target ", "Fresh target path; existing files and sidecars are refused") + .option("--json", "Output JSON", false) + .action(async (snapshot, opts) => { + await runCommandWithRuntime(defaultRuntime, async () => { + await backupSqliteRestoreCommand(defaultRuntime, snapshot as string, { + target: opts.target as string, + json: Boolean(opts.json), + }); + }); + }); } diff --git a/src/commands/backup-sqlite.test.ts b/src/commands/backup-sqlite.test.ts new file mode 100644 index 000000000000..0612a3715eab --- /dev/null +++ b/src/commands/backup-sqlite.test.ts @@ -0,0 +1,290 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js"; +import { requireNodeSqlite } from "../infra/node-sqlite.js"; +import type { RuntimeEnv } from "../runtime.js"; +import { createLocalSqliteSnapshotProvider } from "../snapshot/local-repository.js"; +import { OPENCLAW_AGENT_SCHEMA_VERSION } from "../state/openclaw-agent-db.js"; +import { resolveOpenClawAgentSqlitePath } from "../state/openclaw-agent-db.paths.js"; +import { OPENCLAW_AGENT_SCHEMA_SQL } from "../state/openclaw-agent-schema.generated.js"; +import { OPENCLAW_STATE_SCHEMA_VERSION } from "../state/openclaw-state-db.js"; +import { resolveOpenClawStateSqlitePath } from "../state/openclaw-state-db.paths.js"; +import { OPENCLAW_STATE_SCHEMA_SQL } from "../state/openclaw-state-schema.generated.js"; +import { + backupSqliteCreateCommand, + backupSqliteListCommand, + backupSqliteRestoreCommand, + backupSqliteVerifyCommand, +} from "./backup-sqlite.js"; + +const tempDirs = useAutoCleanupTempDirTracker(afterEach); +let previousStateDir: string | undefined; + +beforeEach(() => { + previousStateDir = process.env.OPENCLAW_STATE_DIR; +}); + +afterEach(() => { + if (previousStateDir === undefined) { + delete process.env.OPENCLAW_STATE_DIR; + } else { + process.env.OPENCLAW_STATE_DIR = previousStateDir; + } +}); + +function createGlobalDatabase(databasePath: string): void { + const sqlite = requireNodeSqlite(); + const database = new sqlite.DatabaseSync(databasePath); + try { + database.exec(` + PRAGMA journal_mode = WAL; + PRAGMA wal_autocheckpoint = 0; + ${OPENCLAW_STATE_SCHEMA_SQL} + PRAGMA user_version = ${OPENCLAW_STATE_SCHEMA_VERSION}; + CREATE TABLE durable_entries ( + id INTEGER PRIMARY KEY, + value TEXT NOT NULL + ); + `); + database + .prepare( + ` + INSERT INTO schema_meta ( + meta_key, + role, + schema_version, + agent_id, + app_version, + created_at, + updated_at + ) VALUES ('primary', 'global', ?, NULL, NULL, 1, 1) + `, + ) + .run(OPENCLAW_STATE_SCHEMA_VERSION); + database + .prepare( + ` + INSERT INTO delivery_queue_entries ( + queue_name, + id, + status, + entry_json, + enqueued_at, + updated_at + ) VALUES ('delivery', 'queued', 'pending', ?, 1, 1) + `, + ) + .run('{"payload":"do-not-restore"}'); + database.prepare("INSERT INTO durable_entries (value) VALUES (?)").run("checkpointed"); + database.exec("PRAGMA wal_checkpoint(TRUNCATE);"); + database.prepare("INSERT INTO durable_entries (value) VALUES (?)").run("committed-in-wal"); + } finally { + database.close(); + } +} + +function createAgentDatabase(databasePath: string, agentId: string): void { + const sqlite = requireNodeSqlite(); + const database = new sqlite.DatabaseSync(databasePath); + try { + database.exec(` + ${OPENCLAW_AGENT_SCHEMA_SQL} + PRAGMA user_version = ${OPENCLAW_AGENT_SCHEMA_VERSION}; + CREATE TABLE durable_entries ( + id INTEGER PRIMARY KEY, + value TEXT NOT NULL + ); + `); + database + .prepare( + ` + INSERT INTO schema_meta ( + meta_key, + role, + schema_version, + agent_id, + app_version, + created_at, + updated_at + ) VALUES ('primary', 'agent', ?, ?, NULL, 1, 1) + `, + ) + .run(OPENCLAW_AGENT_SCHEMA_VERSION, agentId); + database.prepare("INSERT INTO durable_entries (value) VALUES (?)").run("agent-state"); + } finally { + database.close(); + } +} + +describe("SQLite backup commands", () => { + it("creates, lists, verifies, and fresh-restores the global database", async () => { + const tempDir = tempDirs.make("openclaw-backup-sqlite-"); + const stateDir = path.join(tempDir, "state"); + const repositoryPath = path.join(tempDir, "snapshots"); + const scratchPath = path.join(tempDir, "scratch"); + const restorePath = path.join(tempDir, "restore", "openclaw.sqlite"); + process.env.OPENCLAW_STATE_DIR = stateDir; + const databasePath = resolveOpenClawStateSqlitePath(); + await fs.mkdir(path.dirname(databasePath), { recursive: true }); + await fs.mkdir(scratchPath, { mode: 0o700 }); + await fs.chmod(scratchPath, 0o700); + createGlobalDatabase(databasePath); + const runtime = createRuntimeCapture(); + + const created = await backupSqliteCreateCommand(runtime, { + global: true, + repository: repositoryPath, + json: true, + }); + expect(created.manifest.database).toMatchObject({ + role: "global", + basename: "openclaw.sqlite", + userVersion: OPENCLAW_STATE_SCHEMA_VERSION, + }); + expect(JSON.parse(runtime.logs.shift() ?? "{}")).toEqual(created); + + const listed = await backupSqliteListCommand(runtime, { + repository: repositoryPath, + json: true, + }); + expect(listed.snapshots).toHaveLength(1); + expect(listed.snapshots[0]?.manifest.snapshotId).toBe(created.manifest.snapshotId); + + const verified = await backupSqliteVerifyCommand(runtime, created.snapshotPath, { + scratch: scratchPath, + json: true, + }); + expect(verified.manifest).toEqual(created.manifest); + await expect(fs.readdir(scratchPath)).resolves.toEqual([]); + + const restored = await backupSqliteRestoreCommand(runtime, created.snapshotPath, { + target: restorePath, + json: true, + }); + expect(restored).toMatchObject({ + ok: true, + snapshotPath: created.snapshotPath, + targetPath: restorePath, + }); + expect(runtime.errors).toEqual([]); + + const sqlite = requireNodeSqlite(); + const restoredDatabase = new sqlite.DatabaseSync(restorePath, { readOnly: true }); + try { + expect( + restoredDatabase.prepare("SELECT value FROM durable_entries ORDER BY id").all(), + ).toEqual([{ value: "checkpointed" }, { value: "committed-in-wal" }]); + expect( + restoredDatabase.prepare("SELECT COUNT(*) AS count FROM delivery_queue_entries").get(), + ).toEqual({ count: 0 }); + } finally { + restoredDatabase.close(); + } + }); + + it("creates a snapshot for a normalized per-agent database", async () => { + const tempDir = tempDirs.make("openclaw-backup-sqlite-"); + const stateDir = path.join(tempDir, "state"); + const repositoryPath = path.join(tempDir, "snapshots"); + process.env.OPENCLAW_STATE_DIR = stateDir; + const databasePath = resolveOpenClawAgentSqlitePath({ agentId: "ops-team" }); + await fs.mkdir(path.dirname(databasePath), { recursive: true }); + createAgentDatabase(databasePath, "ops-team"); + const runtime = createRuntimeCapture(); + + const created = await backupSqliteCreateCommand(runtime, { + agent: "Ops Team", + repository: repositoryPath, + }); + + expect(created.manifest.database).toEqual({ + role: "agent", + agentId: "ops-team", + basename: "openclaw-agent.sqlite", + userVersion: OPENCLAW_AGENT_SCHEMA_VERSION, + }); + expect(runtime.logs).toEqual([expect.stringContaining("Database: agent:ops-team")]); + expect(runtime.errors).toEqual([]); + }); + + it("requires exactly one named OpenClaw database source", async () => { + const runtime = createRuntimeCapture(); + + await expect( + backupSqliteCreateCommand(runtime, { repository: "/tmp/snapshots" }), + ).rejects.toThrow("Choose a SQLite snapshot source"); + await expect( + backupSqliteCreateCommand(runtime, { + global: true, + agent: "main", + repository: "/tmp/snapshots", + }), + ).rejects.toThrow("Choose exactly one SQLite snapshot source"); + }); + + it("requires repository, snapshot, and restore target paths", async () => { + const runtime = createRuntimeCapture(); + + await expect(backupSqliteCreateCommand(runtime, { global: true })).rejects.toThrow( + "Missing required --repository value", + ); + await expect(backupSqliteVerifyCommand(runtime, " ", {})).rejects.toThrow( + "Missing required value", + ); + await expect(backupSqliteRestoreCommand(runtime, "/tmp/snapshot", {})).rejects.toThrow( + "Missing required --target value", + ); + }); + + it("rejects generic provider artifacts before verify or restore", async () => { + const tempDir = tempDirs.make("openclaw-backup-sqlite-"); + const databasePath = path.join(tempDir, "generic.sqlite"); + const repositoryPath = path.join(tempDir, "snapshots"); + const restorePath = path.join(tempDir, "restore", "generic.sqlite"); + const sqlite = requireNodeSqlite(); + const database = new sqlite.DatabaseSync(databasePath); + try { + database.exec("CREATE TABLE entries (id INTEGER PRIMARY KEY);"); + } finally { + database.close(); + } + const snapshot = await createLocalSqliteSnapshotProvider({ repositoryPath }).create({ + path: databasePath, + identity: { role: "generic", id: "generic-test" }, + }); + const runtime = createRuntimeCapture(); + + await expect(backupSqliteListCommand(runtime, { repository: repositoryPath })).rejects.toThrow( + /database role generic is not allowed/u, + ); + await expect(backupSqliteVerifyCommand(runtime, snapshot.ref.path, {})).rejects.toThrow( + /database role generic is not allowed/u, + ); + await expect( + backupSqliteRestoreCommand(runtime, snapshot.ref.path, { target: restorePath }), + ).rejects.toThrow(/database role generic is not allowed/u); + await expect(fs.access(restorePath)).rejects.toMatchObject({ code: "ENOENT" }); + }); +}); + +function createRuntimeCapture(): RuntimeEnv & { + logs: string[]; + errors: string[]; +} { + const logs: string[] = []; + const errors: string[] = []; + return { + logs, + errors, + log(value) { + logs.push(String(value)); + }, + error(value) { + errors.push(String(value)); + }, + exit(code) { + throw new Error(`exit ${code}`); + }, + }; +} diff --git a/src/commands/backup-sqlite.ts b/src/commands/backup-sqlite.ts new file mode 100644 index 000000000000..b69b68dfa8d5 --- /dev/null +++ b/src/commands/backup-sqlite.ts @@ -0,0 +1,269 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { normalizeAgentId } from "../routing/session-key.js"; +import { type RuntimeEnv, writeRuntimeJson } from "../runtime.js"; +import { createLocalSqliteSnapshotProvider } from "../snapshot/local-repository.js"; +import type { + SnapshotDatabaseManifest, + SnapshotManifest, + SnapshotRef, + SnapshotSummary, +} from "../snapshot/snapshot-provider.js"; +import { resolveOpenClawAgentSqlitePath } from "../state/openclaw-agent-db.paths.js"; +import { resolveOpenClawStateSqlitePath } from "../state/openclaw-state-db.paths.js"; +import { resolveUserPath, shortenHomePath } from "../utils.js"; + +export type BackupSqliteCreateOptions = { + global?: boolean; + agent?: string; + repository?: string; + json?: boolean; +}; + +export type BackupSqliteRepositoryOptions = { + repository?: string; + json?: boolean; +}; + +export type BackupSqliteJsonOptions = { + json?: boolean; +}; + +export type BackupSqliteVerifyOptions = BackupSqliteJsonOptions & { + scratch?: string; +}; + +export type BackupSqliteRestoreOptions = BackupSqliteJsonOptions & { + target?: string; +}; + +type BackupSqliteCreateResult = { + ok: true; + snapshotPath: string; + manifest: SnapshotManifest; +}; + +type BackupSqliteListResult = { + ok: true; + repositoryPath: string; + snapshots: SnapshotSummary[]; +}; + +type BackupSqliteVerifyResult = { + ok: true; + snapshotPath: string; + manifest: SnapshotManifest; +}; + +type BackupSqliteRestoreResult = BackupSqliteVerifyResult & { + targetPath: string; +}; + +type ResolvedSnapshotDatabase = { + path: string; + identity: { role: "global" } | { role: "agent"; agentId: string }; +}; + +const OPENCLAW_SNAPSHOT_READ_OPTIONS = { + allowedDatabaseRoles: ["global", "agent"], +} as const; + +export async function backupSqliteCreateCommand( + runtime: RuntimeEnv, + options: BackupSqliteCreateOptions, +): Promise { + const repositoryPath = resolveRequiredPath(options.repository, "--repository"); + const database = await resolveSnapshotDatabase(options); + const result = await createLocalSqliteSnapshotProvider({ repositoryPath }).create(database); + const report: BackupSqliteCreateResult = { + ok: true, + snapshotPath: result.ref.path, + manifest: result.manifest, + }; + writeCreateResult(runtime, options, report); + return report; +} + +export async function backupSqliteListCommand( + runtime: RuntimeEnv, + options: BackupSqliteRepositoryOptions, +): Promise { + const repositoryPath = resolveRequiredPath(options.repository, "--repository"); + const snapshots = await createLocalSqliteSnapshotProvider({ + repositoryPath, + ...OPENCLAW_SNAPSHOT_READ_OPTIONS, + }).list(); + const report: BackupSqliteListResult = { + ok: true, + repositoryPath, + snapshots, + }; + writeListResult(runtime, options, report); + return report; +} + +export async function backupSqliteVerifyCommand( + runtime: RuntimeEnv, + snapshot: string, + options: BackupSqliteVerifyOptions, +): Promise { + const resolved = resolveSnapshot(snapshot, options.scratch); + const verified = await resolved.provider.verify(resolved.ref); + const report: BackupSqliteVerifyResult = { + ok: true, + snapshotPath: resolved.ref.path, + manifest: verified.manifest, + }; + writeVerifyResult(runtime, options, report); + return report; +} + +export async function backupSqliteRestoreCommand( + runtime: RuntimeEnv, + snapshot: string, + options: BackupSqliteRestoreOptions, +): Promise { + const resolved = resolveSnapshot(snapshot); + const targetPath = resolveRequiredPath(options.target, "--target"); + const restored = await resolved.provider.restoreFresh(resolved.ref, targetPath); + const report: BackupSqliteRestoreResult = { + ok: true, + snapshotPath: resolved.ref.path, + targetPath, + manifest: restored.manifest, + }; + writeRestoreResult(runtime, options, report); + return report; +} + +async function resolveSnapshotDatabase( + options: BackupSqliteCreateOptions, +): Promise { + const rawAgentId = options.agent?.trim(); + if (options.global === true && rawAgentId) { + throw new Error("Choose exactly one SQLite snapshot source: --global or --agent ."); + } + if (options.global !== true && !rawAgentId) { + throw new Error("Choose a SQLite snapshot source: --global or --agent ."); + } + if (options.global === true) { + return { + path: await fs.realpath(resolveOpenClawStateSqlitePath()), + identity: { role: "global" }, + }; + } + const agentId = normalizeAgentId(rawAgentId); + return { + path: await fs.realpath(resolveOpenClawAgentSqlitePath({ agentId })), + identity: { role: "agent", agentId }, + }; +} + +function resolveSnapshot( + snapshot: string, + scratch?: string, +): { + provider: ReturnType; + ref: SnapshotRef; +} { + const snapshotPath = resolveRequiredPath(snapshot, ""); + const repositoryPath = path.dirname(snapshotPath); + const validationRootPath = scratch + ? resolveRequiredPath(scratch, "--scratch") + : path.dirname(repositoryPath); + return { + provider: createLocalSqliteSnapshotProvider({ + repositoryPath, + validationRootPath, + ...OPENCLAW_SNAPSHOT_READ_OPTIONS, + }), + ref: { path: snapshotPath }, + }; +} + +function resolveRequiredPath(value: string | undefined, label: string): string { + const trimmed = value?.trim(); + if (!trimmed) { + throw new Error(`Missing required ${label} value.`); + } + return path.resolve(resolveUserPath(trimmed)); +} + +function formatDatabaseIdentity(database: SnapshotDatabaseManifest): string { + if (database.role === "global") { + return "global"; + } + if (database.role === "agent") { + return `agent:${database.agentId}`; + } + return database.id; +} + +function writeCreateResult( + runtime: RuntimeEnv, + options: BackupSqliteJsonOptions, + report: BackupSqliteCreateResult, +): void { + if (options.json) { + writeRuntimeJson(runtime, report); + return; + } + runtime.log( + [ + `SQLite snapshot created: ${shortenHomePath(report.snapshotPath)}`, + `Database: ${formatDatabaseIdentity(report.manifest.database)}`, + `Size: ${report.manifest.artifact.sizeBytes} bytes`, + ].join("\n"), + ); +} + +function writeListResult( + runtime: RuntimeEnv, + options: BackupSqliteJsonOptions, + report: BackupSqliteListResult, +): void { + if (options.json) { + writeRuntimeJson(runtime, report); + return; + } + if (report.snapshots.length === 0) { + runtime.log(`No SQLite snapshots in ${shortenHomePath(report.repositoryPath)}.`); + return; + } + runtime.log( + report.snapshots + .map( + (snapshot) => + `${snapshot.manifest.createdAt} ${formatDatabaseIdentity(snapshot.manifest.database)} ${snapshot.manifest.artifact.sizeBytes} bytes ${shortenHomePath(snapshot.ref.path)}`, + ) + .join("\n"), + ); +} + +function writeVerifyResult( + runtime: RuntimeEnv, + options: BackupSqliteJsonOptions, + report: BackupSqliteVerifyResult, +): void { + if (options.json) { + writeRuntimeJson(runtime, report); + return; + } + runtime.log( + `SQLite snapshot verified: ${shortenHomePath(report.snapshotPath)} (${formatDatabaseIdentity(report.manifest.database)})`, + ); +} + +function writeRestoreResult( + runtime: RuntimeEnv, + options: BackupSqliteJsonOptions, + report: BackupSqliteRestoreResult, +): void { + if (options.json) { + writeRuntimeJson(runtime, report); + return; + } + runtime.log( + `SQLite snapshot restored: ${shortenHomePath(report.targetPath)} (${formatDatabaseIdentity(report.manifest.database)})`, + ); +} diff --git a/src/infra/sqlite-snapshot.test.ts b/src/infra/sqlite-snapshot.test.ts index c2faf5b94ccf..c508e7284e90 100644 --- a/src/infra/sqlite-snapshot.test.ts +++ b/src/infra/sqlite-snapshot.test.ts @@ -4,13 +4,18 @@ import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; import { requireNodeSqlite } from "./node-sqlite.js"; -import { createVerifiedSqliteSnapshot } from "./sqlite-snapshot.js"; +import { createPrivateSqliteDirectory, createVerifiedSqliteSnapshot } from "./sqlite-snapshot.js"; const tempDirs: string[] = []; async function createTempDir(): Promise { const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-sqlite-snapshot-")); tempDirs.push(tempDir); + if (process.platform === "win32") { + const privateTempDir = path.join(tempDir, "private"); + await createPrivateSqliteDirectory(privateTempDir); + return privateTempDir; + } return tempDir; } @@ -49,6 +54,24 @@ function createUnsafeIndexDrift(sqlitePath: string): void { } describe("createVerifiedSqliteSnapshot", () => { + it.runIf(process.platform === "win32")( + "creates private staging directories exclusively under races", + async () => { + const tempDir = await createTempDir(); + const directoryPath = path.join(tempDir, "private"); + const results = await Promise.allSettled([ + createPrivateSqliteDirectory(directoryPath), + createPrivateSqliteDirectory(directoryPath), + ]); + + expect(results.filter((result) => result.status === "fulfilled")).toHaveLength(1); + const rejected = results.find((result) => result.status === "rejected"); + expect(rejected).toBeDefined(); + expect((rejected as PromiseRejectedResult).reason).toMatchObject({ code: "EEXIST" }); + await expect(fs.lstat(directoryPath)).resolves.toMatchObject({}); + }, + ); + it("captures committed WAL state and removes deleted page contents", async () => { const tempDir = await createTempDir(); const sourcePath = path.join(tempDir, "source.sqlite"); diff --git a/src/infra/sqlite-snapshot.ts b/src/infra/sqlite-snapshot.ts index dafd7844e63b..736125eba604 100644 --- a/src/infra/sqlite-snapshot.ts +++ b/src/infra/sqlite-snapshot.ts @@ -6,12 +6,78 @@ import type { FileHandle } from "node:fs/promises"; import path from "node:path"; import type { DatabaseSync } from "node:sqlite"; import { loadSqliteVecExtension } from "../../packages/memory-host-sdk/src/engine-storage.js"; +import { runExec } from "../process/exec.js"; import { formatErrorMessage } from "./errors.js"; import { sameFileIdentity } from "./fs-safe-advanced.js"; import { requireNodeSqlite } from "./node-sqlite.js"; +import { resolveSystemBin } from "./resolve-system-bin.js"; import { assertSqliteIntegrity } from "./sqlite-integrity.js"; import { readSqliteUserVersion } from "./sqlite-user-version.js"; +const SQLITE_DIRECTORY_MODE = 0o700; +const WINDOWS_DIRECTORY_EXISTS_MARKER = "OPENCLAW_SQLITE_DIRECTORY_EXISTS"; +// Managed directory creation accepts existing paths. CreateDirectoryW applies the +// protected DACL atomically while preserving fail-if-exists semantics. +const WINDOWS_PRIVATE_DIRECTORY_NATIVE_SOURCE = ` +using System; +using System.Runtime.InteropServices; + +public static class OpenClawPrivateDirectory +{ + [StructLayout(LayoutKind.Sequential)] + private struct SecurityAttributes + { + public int Length; + public IntPtr SecurityDescriptor; + public int InheritHandle; + } + + [DllImport("advapi32.dll", CharSet = CharSet.Unicode, SetLastError = true)] + private static extern bool ConvertStringSecurityDescriptorToSecurityDescriptorW( + string securityDescriptor, + uint revision, + out IntPtr convertedSecurityDescriptor, + out uint convertedSecurityDescriptorSize); + + [DllImport("kernel32.dll", CharSet = CharSet.Unicode, SetLastError = true)] + private static extern bool CreateDirectoryW( + string path, + ref SecurityAttributes securityAttributes); + + [DllImport("kernel32.dll")] + private static extern IntPtr LocalFree(IntPtr memory); + + public static int Create(string path, string securityDescriptor) + { + IntPtr descriptor; + uint descriptorSize; + if (!ConvertStringSecurityDescriptorToSecurityDescriptorW( + securityDescriptor, + 1, + out descriptor, + out descriptorSize)) + { + return Marshal.GetLastWin32Error(); + } + + try + { + var attributes = new SecurityAttributes + { + Length = Marshal.SizeOf(typeof(SecurityAttributes)), + SecurityDescriptor = descriptor, + InheritHandle = 0, + }; + return CreateDirectoryW(path, ref attributes) ? 0 : Marshal.GetLastWin32Error(); + } + finally + { + LocalFree(descriptor); + } + } +} +`; + export type SqliteSnapshotValidator = (database: DatabaseSync, databaseLabel: string) => void; export type CreateVerifiedSqliteSnapshotOptions = { @@ -51,6 +117,71 @@ export type VerifiedSqliteSnapshot = { userVersion: number; }; +export async function createPrivateSqliteDirectory(directoryPath: string): Promise { + if (process.platform !== "win32") { + await fs.mkdir(directoryPath, { mode: SQLITE_DIRECTORY_MODE }); + return; + } + const encodedPath = Buffer.from(directoryPath, "utf8").toString("base64"); + const encodedNativeSource = Buffer.from(WINDOWS_PRIVATE_DIRECTORY_NATIVE_SOURCE, "utf8").toString( + "base64", + ); + const command = [ + "$ErrorActionPreference = 'Stop'", + `$path = [Text.Encoding]::UTF8.GetString([Convert]::FromBase64String('${encodedPath}'))`, + `$nativeSource = [Text.Encoding]::UTF8.GetString([Convert]::FromBase64String('${encodedNativeSource}'))`, + "Add-Type -TypeDefinition $nativeSource -Language CSharp", + "$current = [System.Security.Principal.WindowsIdentity]::GetCurrent().User", + "$security = New-Object System.Security.AccessControl.DirectorySecurity", + "$security.SetAccessRuleProtection($true, $false)", + "$security.SetOwner($current)", + "$inheritance = [System.Security.AccessControl.InheritanceFlags]::ContainerInherit -bor [System.Security.AccessControl.InheritanceFlags]::ObjectInherit", + "$propagation = [System.Security.AccessControl.PropagationFlags]::None", + "foreach ($sidValue in @($current.Value, 'S-1-5-18', 'S-1-5-32-544')) { $sid = New-Object System.Security.Principal.SecurityIdentifier($sidValue); $rule = New-Object System.Security.AccessControl.FileSystemAccessRule($sid, [System.Security.AccessControl.FileSystemRights]::FullControl, $inheritance, $propagation, [System.Security.AccessControl.AccessControlType]::Allow); [void]$security.AddAccessRule($rule) }", + "$sections = [System.Security.AccessControl.AccessControlSections]::Owner -bor [System.Security.AccessControl.AccessControlSections]::Access", + "$sddl = $security.GetSecurityDescriptorSddlForm($sections)", + "$errorCode = [OpenClawPrivateDirectory]::Create($path, $sddl)", + `if ($errorCode -eq 80 -or $errorCode -eq 183) { throw '${WINDOWS_DIRECTORY_EXISTS_MARKER}' }`, + "if ($errorCode -ne 0) { $exception = New-Object System.ComponentModel.Win32Exception($errorCode); throw $exception }", + ].join("; "); + const powershell = resolveSystemBin("powershell"); + if (!powershell) { + throw new Error("Unable to resolve PowerShell for private Windows SQLite staging."); + } + const encodedCommand = Buffer.from(command, "utf16le").toString("base64"); + try { + await runExec( + powershell, + ["-NoLogo", "-NoProfile", "-NonInteractive", "-EncodedCommand", encodedCommand], + { + timeoutMs: 10_000, + maxBuffer: 64 * 1024, + }, + ); + } catch (error) { + if (String(error).includes(WINDOWS_DIRECTORY_EXISTS_MARKER)) { + const existsError = new Error(`Private SQLite directory already exists: ${directoryPath}`); + (existsError as NodeJS.ErrnoException).code = "EEXIST"; + throw existsError; + } + throw new Error(`Unable to create private Windows SQLite directory: ${directoryPath}`, { + cause: error, + }); + } +} + +export async function createPrivateSqliteTempDirectory( + rootPath: string, + prefix: string, +): Promise { + if (process.platform !== "win32") { + return await fs.mkdtemp(path.join(rootPath, prefix)); + } + const directoryPath = path.join(rootPath, `${prefix}${randomUUID()}`); + await createPrivateSqliteDirectory(directoryPath); + return directoryPath; +} + async function assertRegularSourceFile(sourcePath: string): Promise { const stat = await fs.lstat(sourcePath); if (!stat.isFile()) { @@ -419,8 +550,9 @@ export async function publishVerifiedSqliteFile( ): Promise { await assertTargetAbsent(options.targetPath); const targetDirectory = path.dirname(options.targetPath); - const stagingDir = await fs.mkdtemp( - path.join(targetDirectory, `.sqlite-publish-${randomUUID()}-`), + const stagingDir = await createPrivateSqliteTempDirectory( + targetDirectory, + `.sqlite-publish-${randomUUID()}-`, ); const stagedPath = path.join(stagingDir, "database.sqlite"); let stagingIdentity: Stats | undefined; @@ -646,8 +778,9 @@ export async function createVerifiedSqliteSnapshot( await assertRegularSourceFile(options.sourcePath); await assertTargetAbsent(options.targetPath); - const stagingDir = await fs.mkdtemp( - path.join(path.dirname(options.targetPath), ".sqlite-snapshot-"), + const stagingDir = await createPrivateSqliteTempDirectory( + path.dirname(options.targetPath), + ".sqlite-snapshot-", ); await fs.chmod(stagingDir, 0o700); const stagedPath = path.join(stagingDir, "database.sqlite"); diff --git a/src/snapshot/local-repository.test.ts b/src/snapshot/local-repository.test.ts new file mode 100644 index 000000000000..fae8225170d6 --- /dev/null +++ b/src/snapshot/local-repository.test.ts @@ -0,0 +1,1660 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js"; +import { requireNodeSqlite } from "../infra/node-sqlite.js"; +import { createPrivateSqliteDirectory } from "../infra/sqlite-snapshot.js"; +import { runExec } from "../process/exec.js"; +import { OPENCLAW_AGENT_SCHEMA_VERSION } from "../state/openclaw-agent-db.js"; +import { OPENCLAW_AGENT_SCHEMA_SQL } from "../state/openclaw-agent-schema.generated.js"; +import { OPENCLAW_STATE_SCHEMA_VERSION } from "../state/openclaw-state-db.js"; +import { OPENCLAW_STATE_SCHEMA_SQL } from "../state/openclaw-state-schema.generated.js"; +import { createLocalSqliteSnapshotProvider } from "./local-repository.js"; +import { hashSnapshotArtifact, parseSnapshotManifest, readSnapshotManifest } from "./manifest.js"; +import { + SNAPSHOT_MANIFEST_FILENAME, + SNAPSHOT_SQLITE_FILENAME, + type SnapshotManifest, + type SnapshotResult, +} from "./snapshot-provider.js"; + +const tempDirs = useAutoCleanupTempDirTracker(afterEach); + +async function createTempDir(): Promise { + const tempDir = tempDirs.make("openclaw-snapshot-repository-"); + if (process.platform === "win32") { + const privateTempDir = path.join(tempDir, "private"); + await createPrivateSqliteDirectory(privateTempDir); + return privateTempDir; + } + return tempDir; +} + +function createGenericDatabase( + databasePath: string, + options: { userVersion?: number; values?: string[]; wal?: boolean } = {}, +): void { + const sqlite = requireNodeSqlite(); + const database = new sqlite.DatabaseSync(databasePath); + try { + database.exec(` + ${options.wal ? "PRAGMA journal_mode = WAL; PRAGMA wal_autocheckpoint = 0;" : ""} + PRAGMA user_version = ${options.userVersion ?? 7}; + CREATE TABLE entries ( + id INTEGER PRIMARY KEY, + value TEXT NOT NULL + ); + `); + const insert = database.prepare("INSERT INTO entries (value) VALUES (?)"); + for (const value of options.values ?? ["one"]) { + insert.run(value); + } + } finally { + database.close(); + } +} + +function createGlobalDatabase(databasePath: string): void { + const sqlite = requireNodeSqlite(); + const database = new sqlite.DatabaseSync(databasePath); + try { + database.exec(` + ${OPENCLAW_STATE_SCHEMA_SQL} + PRAGMA user_version = ${OPENCLAW_STATE_SCHEMA_VERSION}; + `); + database + .prepare( + ` + INSERT INTO schema_meta ( + meta_key, + role, + schema_version, + agent_id, + app_version, + created_at, + updated_at + ) VALUES ('primary', 'global', ?, NULL, NULL, 1, 1) + `, + ) + .run(OPENCLAW_STATE_SCHEMA_VERSION); + database + .prepare( + ` + INSERT INTO delivery_queue_entries ( + queue_name, + id, + status, + entry_json, + enqueued_at, + updated_at + ) VALUES ('delivery', 'queued', 'pending', ?, 1, 1) + `, + ) + .run('{"payload":"do-not-restore"}'); + } finally { + database.close(); + } +} + +function createAgentDatabase(databasePath: string, agentId: string): void { + const sqlite = requireNodeSqlite(); + const database = new sqlite.DatabaseSync(databasePath); + try { + database.exec(` + ${OPENCLAW_AGENT_SCHEMA_SQL} + PRAGMA user_version = ${OPENCLAW_AGENT_SCHEMA_VERSION}; + `); + database + .prepare( + ` + INSERT INTO schema_meta ( + meta_key, + role, + schema_version, + agent_id, + app_version, + created_at, + updated_at + ) VALUES ('primary', 'agent', ?, ?, NULL, 1, 1) + `, + ) + .run(OPENCLAW_AGENT_SCHEMA_VERSION, agentId); + } finally { + database.close(); + } +} + +function disableDefensiveModeForSchemaCorruption(database: object): void { + ( + database as { + enableDefensive?: (active: boolean) => void; + } + ).enableDefensive?.(false); +} + +function createUnsafeIndexDrift(databasePath: string): void { + const sqlite = requireNodeSqlite(); + const database = new sqlite.DatabaseSync(databasePath); + try { + disableDefensiveModeForSchemaCorruption(database); + database.exec(` + CREATE TABLE records ( + id INTEGER PRIMARY KEY, + indexed_value TEXT NOT NULL, + alternate_value TEXT NOT NULL + ); + CREATE INDEX records_value ON records(indexed_value); + INSERT INTO records (indexed_value, alternate_value) + VALUES ('alpha', 'zeta'), ('beta', 'eta'), ('gamma', 'theta'); + PRAGMA writable_schema = ON; + `); + database + .prepare( + "UPDATE sqlite_schema SET sql = 'CREATE INDEX records_value ON records(alternate_value)' WHERE name = 'records_value'", + ) + .run(); + const schemaVersion = Number( + Object.values(database.prepare("PRAGMA schema_version").get() as Record)[0], + ); + database.exec(`PRAGMA writable_schema = OFF; PRAGMA schema_version = ${schemaVersion + 1};`); + } finally { + database.close(); + } +} + +async function rewriteManifest( + result: SnapshotResult, + mutate: (manifest: SnapshotManifest) => SnapshotManifest, +): Promise { + const manifestPath = path.join(result.ref.path, SNAPSHOT_MANIFEST_FILENAME); + const manifest = await readSnapshotManifest(result.ref.path); + await fs.writeFile(manifestPath, `${JSON.stringify(mutate(manifest), null, 2)}\n`); +} + +async function refreshArtifactManifest(result: SnapshotResult): Promise { + const digest = await hashSnapshotArtifact(result.ref.path); + await rewriteManifest(result, (manifest) => ({ + ...manifest, + artifact: { + ...manifest.artifact, + sha256: digest.sha256, + sizeBytes: digest.sizeBytes, + }, + })); +} + +describe("local SQLite snapshot repository", () => { + it("creates, lists, verifies, and fresh-restores committed WAL state", async () => { + const tempDir = await createTempDir(); + const sourcePath = path.join(tempDir, "source.sqlite"); + const repositoryPath = path.join( + tempDir, + process.platform === "win32" ? "snapshots-\u00e9 #" : "snapshots ? #", + ); + const restorePath = path.join(tempDir, "restore", "source.sqlite"); + const sqlite = requireNodeSqlite(); + const source = new sqlite.DatabaseSync(sourcePath); + try { + source.exec(` + PRAGMA journal_mode = WAL; + PRAGMA wal_autocheckpoint = 0; + PRAGMA user_version = 42; + CREATE TABLE entries (id INTEGER PRIMARY KEY, value TEXT NOT NULL); + INSERT INTO entries (value) VALUES ('checkpointed'); + PRAGMA wal_checkpoint(TRUNCATE); + INSERT INTO entries (value) VALUES ('committed-in-wal'); + `); + const provider = createLocalSqliteSnapshotProvider({ + repositoryPath, + now: () => new Date("2026-07-12T14:00:00.000Z"), + }); + const snapshot = await provider.create({ + path: sourcePath, + identity: { role: "generic", id: "test-database" }, + }); + + expect(snapshot.manifest).toMatchObject({ + schemaVersion: 1, + createdAt: "2026-07-12T14:00:00.000Z", + database: { + role: "generic", + id: "test-database", + basename: "source.sqlite", + userVersion: 42, + }, + artifact: { + path: SNAPSHOT_SQLITE_FILENAME, + }, + }); + expect(snapshot.manifest.artifact.sha256).toMatch(/^[a-f0-9]{64}$/u); + await expect(provider.verify(snapshot.ref)).resolves.toEqual({ + ok: true, + manifest: snapshot.manifest, + }); + await expect(provider.list()).resolves.toEqual([snapshot]); + await expect(provider.restoreFresh(snapshot.ref, restorePath)).resolves.toEqual({ + ok: true, + manifest: snapshot.manifest, + }); + await expect(fs.readFile(restorePath)).resolves.toEqual( + await fs.readFile(path.join(snapshot.ref.path, SNAPSHOT_SQLITE_FILENAME)), + ); + expect((await fs.readdir(repositoryPath)).every((name) => !name.startsWith(".tmp-"))).toBe( + true, + ); + await expect(fs.readdir(path.dirname(restorePath))).resolves.toEqual(["source.sqlite"]); + } finally { + source.close(); + } + + const restored = new sqlite.DatabaseSync(restorePath, { readOnly: true }); + try { + expect(restored.prepare("SELECT value FROM entries ORDER BY id").all()).toEqual([ + { value: "checkpointed" }, + { value: "committed-in-wal" }, + ]); + expect(restored.prepare("PRAGMA user_version").get()).toEqual({ user_version: 42 }); + } finally { + restored.close(); + } + if (process.platform !== "win32") { + expect((await fs.stat(repositoryPath)).mode & 0o777).toBe(0o700); + expect((await fs.stat(restorePath)).mode & 0o777).toBe(0o600); + } + }); + + it("sorts snapshots newest first and ignores incomplete staging directories", async () => { + const tempDir = await createTempDir(); + const sourcePath = path.join(tempDir, "source.sqlite"); + const repositoryPath = path.join(tempDir, "snapshots"); + createGenericDatabase(sourcePath); + const dates = [new Date("2026-07-12T14:00:00.000Z"), new Date("2026-07-12T14:01:00.000Z")]; + const provider = createLocalSqliteSnapshotProvider({ + repositoryPath, + now: () => dates.shift() ?? new Date("invalid"), + }); + const first = await provider.create({ + path: sourcePath, + identity: { role: "generic", id: "test-database" }, + }); + const second = await provider.create({ + path: sourcePath, + identity: { role: "generic", id: "test-database" }, + }); + await fs.mkdir(path.join(repositoryPath, ".tmp-interrupted")); + await fs.mkdir(path.join(repositoryPath, "interrupted-final")); + await fs.writeFile(path.join(repositoryPath, "interrupted-final", ".pending"), ""); + await fs.mkdir(path.join(repositoryPath, "empty-final")); + + await expect(provider.list()).resolves.toEqual([second, first]); + }); + + it("uses caller-owned verification scratch and stages restore beside the target", async () => { + const tempDir = await createTempDir(); + const sourcePath = path.join(tempDir, "source.sqlite"); + const repositoryPath = path.join(tempDir, "snapshots"); + const validationRootPath = path.join(tempDir, "validation"); + const restoreParentPath = path.join(tempDir, "restore"); + const restorePath = path.join(restoreParentPath, "source.sqlite"); + createGenericDatabase(sourcePath); + await fs.mkdir(validationRootPath, { mode: 0o700 }); + await fs.chmod(validationRootPath, 0o700); + const provider = createLocalSqliteSnapshotProvider({ + repositoryPath, + validationRootPath, + }); + const snapshot = await provider.create({ + path: sourcePath, + identity: { role: "generic", id: "protected-scratch" }, + }); + const canonicalTempDir = await fs.realpath(tempDir); + const originalMkdtemp = fs.mkdtemp.bind(fs); + const prefixes: string[] = []; + const mkdtempSpy = vi.spyOn(fs, "mkdtemp").mockImplementation(async (prefix, options) => { + prefixes.push(prefix); + return await originalMkdtemp(prefix, options); + }); + + try { + await provider.verify(snapshot.ref); + await provider.restoreFresh(snapshot.ref, restorePath); + } finally { + mkdtempSpy.mockRestore(); + } + + if (process.platform === "win32") { + expect(prefixes).toEqual([]); + } else { + expect(prefixes.filter((prefix) => path.basename(prefix).startsWith(".tmp-"))).toEqual([ + path.join(canonicalTempDir, "validation", ".tmp-verify-"), + path.join(canonicalTempDir, "restore", ".tmp-restore-"), + path.join(canonicalTempDir, "restore", ".tmp-verify-"), + ]); + } + }); + + it("fails loudly when private verification scratch cannot be removed", async () => { + const tempDir = await createTempDir(); + const sourcePath = path.join(tempDir, "source.sqlite"); + const repositoryPath = path.join(tempDir, "snapshots"); + const validationRootPath = path.join(tempDir, "validation"); + createGenericDatabase(sourcePath); + await fs.mkdir(validationRootPath, { mode: 0o700 }); + await fs.chmod(validationRootPath, 0o700); + const provider = createLocalSqliteSnapshotProvider({ + repositoryPath, + validationRootPath, + }); + const snapshot = await provider.create({ + path: sourcePath, + identity: { role: "generic", id: "cleanup-failure" }, + }); + const originalUnlink = fs.unlink.bind(fs); + const unlinkSpy = vi.spyOn(fs, "unlink").mockImplementation(async (filePath) => { + if (path.basename(path.dirname(String(filePath))).startsWith(".tmp-verify-")) { + throw Object.assign(new Error("cleanup denied"), { code: "EACCES" }); + } + return await originalUnlink(filePath); + }); + + try { + await expect(provider.verify(snapshot.ref)).rejects.toThrow( + /Failed to clean private SQLite staging directory/u, + ); + } finally { + unlinkSpy.mockRestore(); + } + expect( + (await fs.readdir(validationRootPath)).some((entry) => entry.startsWith(".tmp-verify-")), + ).toBe(true); + }); + + it("removes SQLite sidecars left in private verification scratch", async () => { + const tempDir = await createTempDir(); + const sourcePath = path.join(tempDir, "source.sqlite"); + const repositoryPath = path.join(tempDir, "snapshots"); + const validationRootPath = path.join(tempDir, "validation"); + createGenericDatabase(sourcePath, { wal: true }); + await fs.mkdir(validationRootPath, { mode: 0o700 }); + await fs.chmod(validationRootPath, 0o700); + const provider = createLocalSqliteSnapshotProvider({ + repositoryPath, + validationRootPath, + }); + const snapshot = await provider.create({ + path: sourcePath, + identity: { role: "generic", id: "validation-sidecars" }, + }); + const originalReaddir = fs.readdir.bind(fs); + let injectedSidecars = false; + const readdirSpy = vi.spyOn(fs, "readdir").mockImplementation((async (...args: unknown[]) => { + const directoryPath = path.resolve(String(args[0])); + if (!injectedSidecars && path.basename(directoryPath).startsWith(".tmp-verify-")) { + injectedSidecars = true; + await Promise.all( + ["-wal", "-shm", "-journal"].map( + async (suffix) => + await fs.writeFile( + path.join(directoryPath, `${SNAPSHOT_SQLITE_FILENAME}${suffix}`), + "sqlite sidecar", + { mode: 0o600 }, + ), + ), + ); + } + return await (originalReaddir as (...readdirArgs: unknown[]) => Promise)(...args); + }) as typeof fs.readdir); + + try { + await expect(provider.verify(snapshot.ref)).resolves.toMatchObject({ ok: true }); + } finally { + readdirSpy.mockRestore(); + } + expect(injectedSidecars).toBe(true); + await expect(fs.readdir(validationRootPath)).resolves.toEqual([]); + }); + + it.runIf(process.platform !== "win32")( + "rejects snapshot repositories beneath a replaceable ancestor", + async () => { + const tempDir = await createTempDir(); + const sourcePath = path.join(tempDir, "source.sqlite"); + const sharedPath = path.join(tempDir, "shared"); + const repositoryPath = path.join(sharedPath, "snapshots"); + createGenericDatabase(sourcePath); + await fs.mkdir(sharedPath, { mode: 0o777 }); + await fs.chmod(sharedPath, 0o777); + const provider = createLocalSqliteSnapshotProvider({ repositoryPath }); + + await expect( + provider.create({ + path: sourcePath, + identity: { role: "generic", id: "replaceable-repository-ancestor" }, + }), + ).rejects.toThrow(/ancestor must not allow another user/u); + await expect(fs.readdir(repositoryPath)).resolves.toEqual([]); + }, + ); + + it.runIf(process.platform === "win32")( + "rejects snapshot repositories with inheritable Everyone access", + async () => { + const tempDir = await createTempDir(); + const sourcePath = path.join(tempDir, "source.sqlite"); + const sharedPath = path.join(tempDir, "shared"); + const repositoryPath = path.join(sharedPath, "snapshots"); + const icacls = path.join( + process.env.SystemRoot ?? process.env.WINDIR ?? "C:\\Windows", + "System32", + "icacls.exe", + ); + createGenericDatabase(sourcePath); + await fs.mkdir(sharedPath); + await runExec(icacls, [sharedPath, "/grant", "*S-1-1-0:(OI)(CI)(F)"]); + const provider = createLocalSqliteSnapshotProvider({ repositoryPath }); + + try { + await expect( + provider.create({ + path: sourcePath, + identity: { role: "generic", id: "windows-everyone-repository" }, + }), + ).rejects.toThrow(/Windows ACL permits untrusted SQLite staging access/u); + await expect(fs.readdir(repositoryPath)).resolves.toEqual([]); + } finally { + await runExec(icacls, [sharedPath, "/remove:g", "*S-1-1-0"]).catch(() => undefined); + } + }, + ); + + it.runIf(process.platform !== "win32")( + "rejects verification and restore staging roots writable by other users", + async () => { + const tempDir = await createTempDir(); + const sourcePath = path.join(tempDir, "source.sqlite"); + const repositoryPath = path.join(tempDir, "snapshots"); + const validationRootPath = path.join(tempDir, "validation"); + const restoreParentPath = path.join(tempDir, "restore"); + const restorePath = path.join(restoreParentPath, "source.sqlite"); + createGenericDatabase(sourcePath); + await fs.mkdir(validationRootPath, { mode: 0o777 }); + await fs.chmod(validationRootPath, 0o777); + await fs.mkdir(restoreParentPath, { mode: 0o777 }); + await fs.chmod(restoreParentPath, 0o777); + const provider = createLocalSqliteSnapshotProvider({ + repositoryPath, + validationRootPath, + }); + const snapshot = await provider.create({ + path: sourcePath, + identity: { role: "generic", id: "untrusted-staging-root" }, + }); + + await expect(provider.verify(snapshot.ref)).rejects.toThrow(/not writable by other users/u); + await expect(provider.restoreFresh(snapshot.ref, restorePath)).rejects.toThrow( + /not writable by other users/u, + ); + await expect(fs.access(restorePath)).rejects.toMatchObject({ code: "ENOENT" }); + }, + ); + + it.runIf(process.platform !== "win32")( + "rejects private staging roots beneath a replaceable ancestor", + async () => { + const tempDir = await createTempDir(); + const sourcePath = path.join(tempDir, "source.sqlite"); + const repositoryPath = path.join(tempDir, "snapshots"); + const sharedPath = path.join(tempDir, "shared"); + const validationRootPath = path.join(sharedPath, "validation"); + const restoreParentPath = path.join(sharedPath, "restore"); + const restorePath = path.join(restoreParentPath, "source.sqlite"); + createGenericDatabase(sourcePath); + await fs.mkdir(validationRootPath, { recursive: true, mode: 0o700 }); + await fs.chmod(validationRootPath, 0o700); + await fs.mkdir(restoreParentPath, { mode: 0o700 }); + await fs.chmod(restoreParentPath, 0o700); + await fs.chmod(sharedPath, 0o777); + const provider = createLocalSqliteSnapshotProvider({ + repositoryPath, + validationRootPath, + }); + const snapshot = await provider.create({ + path: sourcePath, + identity: { role: "generic", id: "replaceable-ancestor" }, + }); + + await expect(provider.verify(snapshot.ref)).rejects.toThrow( + /ancestor must not allow another user/u, + ); + await expect(provider.restoreFresh(snapshot.ref, restorePath)).rejects.toThrow( + /ancestor must not allow another user/u, + ); + await expect(fs.access(restorePath)).rejects.toMatchObject({ code: "ENOENT" }); + }, + ); + + it.runIf(process.platform !== "win32")( + "rejects sticky staging ancestors owned by another user", + async () => { + const tempDir = await createTempDir(); + const sourcePath = path.join(tempDir, "source.sqlite"); + const repositoryPath = path.join(tempDir, "snapshots"); + const sharedPath = path.join(tempDir, "shared"); + const validationRootPath = path.join(sharedPath, "validation"); + createGenericDatabase(sourcePath); + await fs.mkdir(validationRootPath, { recursive: true, mode: 0o700 }); + await fs.chmod(validationRootPath, 0o700); + await fs.chmod(sharedPath, 0o1777); + const provider = createLocalSqliteSnapshotProvider({ + repositoryPath, + validationRootPath, + }); + const snapshot = await provider.create({ + path: sourcePath, + identity: { role: "generic", id: "untrusted-sticky-owner" }, + }); + const canonicalSharedPath = await fs.realpath(sharedPath); + const originalLstat = fs.lstat.bind(fs); + const lstatSpy = vi.spyOn(fs, "lstat").mockImplementation(async (...args) => { + const stat = await originalLstat(...args); + if (path.resolve(String(args[0])) !== canonicalSharedPath) { + return stat; + } + return new Proxy(stat, { + get(target, property, receiver) { + if (property === "uid") { + return typeof target.uid === "bigint" ? target.uid + 1n : target.uid + 1; + } + const value = Reflect.get(target, property, receiver); + return typeof value === "function" ? value.bind(target) : value; + }, + }); + }); + + try { + await expect(provider.verify(snapshot.ref)).rejects.toThrow( + /ancestor must not allow another user/u, + ); + await fs.chmod(sharedPath, 0o755); + await expect(provider.verify(snapshot.ref)).rejects.toThrow( + /ancestor must not allow another user/u, + ); + await fs.chmod(sharedPath, 0o555); + await expect(provider.verify(snapshot.ref)).rejects.toThrow( + /ancestor must not allow another user/u, + ); + } finally { + await fs.chmod(sharedPath, 0o700); + lstatSpy.mockRestore(); + } + }, + ); + + it.runIf(process.platform !== "win32")( + "accepts protected symlinked ancestors through their canonical path", + async () => { + const tempDir = await createTempDir(); + const sourcePath = path.join(tempDir, "source.sqlite"); + const repositoryPath = path.join(tempDir, "snapshots"); + const realSharedPath = path.join(tempDir, "real-shared"); + const aliasSharedPath = path.join(tempDir, "alias-shared"); + const validationRootPath = path.join(aliasSharedPath, "validation"); + const restorePath = path.join(aliasSharedPath, "restore", "source.sqlite"); + createGenericDatabase(sourcePath, { values: ["canonical-staging"] }); + await fs.mkdir(path.join(realSharedPath, "validation"), { recursive: true, mode: 0o700 }); + await fs.chmod(path.join(realSharedPath, "validation"), 0o700); + await fs.symlink(realSharedPath, aliasSharedPath, "dir"); + const provider = createLocalSqliteSnapshotProvider({ + repositoryPath, + validationRootPath, + }); + const snapshot = await provider.create({ + path: sourcePath, + identity: { role: "generic", id: "canonical-staging" }, + }); + + await expect(provider.verify(snapshot.ref)).resolves.toMatchObject({ ok: true }); + await expect(provider.restoreFresh(snapshot.ref, restorePath)).resolves.toMatchObject({ + ok: true, + }); + const sqlite = requireNodeSqlite(); + const restored = new sqlite.DatabaseSync(restorePath, { readOnly: true }); + try { + expect(restored.prepare("SELECT value FROM entries").all()).toEqual([ + { value: "canonical-staging" }, + ]); + } finally { + restored.close(); + } + }, + ); + + it.runIf(process.platform === "darwin")( + "rejects snapshot repositories beneath a granting macOS ACL", + async () => { + const tempDir = await createTempDir(); + const sourcePath = path.join(tempDir, "source.sqlite"); + const sharedPath = path.join(tempDir, "shared"); + const repositoryPath = path.join(sharedPath, "snapshots"); + createGenericDatabase(sourcePath); + await fs.mkdir(sharedPath, { mode: 0o700 }); + await runExec("/bin/chmod", ["+a", "everyone allow add_file,delete_child", sharedPath]); + const provider = createLocalSqliteSnapshotProvider({ repositoryPath }); + + try { + await expect( + provider.create({ + path: sourcePath, + identity: { role: "generic", id: "macos-acl-repository" }, + }), + ).rejects.toThrow(/macOS ACL permits untrusted SQLite staging access/u); + await expect(fs.readdir(repositoryPath)).resolves.toEqual([]); + } finally { + await runExec("/bin/chmod", ["-N", sharedPath]).catch(() => undefined); + } + }, + ); + + it.runIf(process.platform === "darwin")( + "rejects granting macOS ACLs on private roots, ancestors, and staging directories", + async () => { + const tempDir = await createTempDir(); + const sourcePath = path.join(tempDir, "source.sqlite"); + const repositoryPath = path.join(tempDir, "snapshots"); + const sharedPath = path.join(tempDir, "shared"); + const validationRootPath = path.join(sharedPath, "validation"); + createGenericDatabase(sourcePath); + await fs.mkdir(validationRootPath, { recursive: true, mode: 0o700 }); + await fs.chmod(validationRootPath, 0o700); + const provider = createLocalSqliteSnapshotProvider({ + repositoryPath, + validationRootPath, + }); + const snapshot = await provider.create({ + path: sourcePath, + identity: { role: "generic", id: "macos-acl" }, + }); + + try { + await runExec("/bin/chmod", [ + "+a", + `${os.userInfo().username} allow read,write,delete`, + validationRootPath, + ]); + await expect(provider.verify(snapshot.ref)).resolves.toMatchObject({ ok: true }); + await runExec("/bin/chmod", ["-N", validationRootPath]); + + await runExec("/bin/chmod", ["+a", "everyone deny delete", validationRootPath]); + await expect(provider.verify(snapshot.ref)).resolves.toMatchObject({ ok: true }); + await runExec("/bin/chmod", ["-N", validationRootPath]); + + await runExec("/bin/chmod", ["+a", "everyone allow read,write,delete", validationRootPath]); + await expect(provider.verify(snapshot.ref)).rejects.toThrow( + /macOS ACL permits untrusted SQLite staging access/u, + ); + await runExec("/bin/chmod", ["-N", validationRootPath]); + + await runExec("/bin/chmod", ["+a", "everyone allow add_file,delete_child", sharedPath]); + await expect(provider.verify(snapshot.ref)).rejects.toThrow( + /macOS ACL permits untrusted SQLite staging access/u, + ); + await runExec("/bin/chmod", ["-N", sharedPath]); + + const originalMkdtemp = fs.mkdtemp.bind(fs); + const mkdtempSpy = vi.spyOn(fs, "mkdtemp").mockImplementation(async (prefix, options) => { + const directoryPath = await originalMkdtemp(prefix, options); + if (path.basename(directoryPath).startsWith(".tmp-verify-")) { + await runExec("/bin/chmod", ["+a", "everyone allow read,write,delete", directoryPath]); + } + return directoryPath; + }); + try { + await expect(provider.verify(snapshot.ref)).rejects.toThrow( + /macOS ACL permits untrusted SQLite staging access/u, + ); + } finally { + mkdtempSpy.mockRestore(); + } + } finally { + await Promise.all( + [validationRootPath, sharedPath].map( + async (pathname) => + await runExec("/bin/chmod", ["-N", pathname]).catch(() => undefined), + ), + ); + } + }, + ); + + it.runIf(process.platform !== "win32")( + "restores from a read-only snapshot repository without changing its permissions", + async () => { + const tempDir = await createTempDir(); + const sourcePath = path.join(tempDir, "source.sqlite"); + const repositoryPath = path.join(tempDir, "snapshots"); + const restorePath = path.join(tempDir, "restore", "source.sqlite"); + createGenericDatabase(sourcePath, { values: ["read-only-source"] }); + const provider = createLocalSqliteSnapshotProvider({ repositoryPath }); + const snapshot = await provider.create({ + path: sourcePath, + identity: { role: "generic", id: "read-only-repository" }, + }); + const snapshotEntries = [ + path.join(snapshot.ref.path, SNAPSHOT_MANIFEST_FILENAME), + path.join(snapshot.ref.path, SNAPSHOT_SQLITE_FILENAME), + ]; + for (const entryPath of snapshotEntries) { + await fs.chmod(entryPath, 0o400); + } + await fs.chmod(snapshot.ref.path, 0o500); + await fs.chmod(repositoryPath, 0o500); + + try { + await expect(provider.list()).resolves.toEqual([snapshot]); + await expect(provider.verify(snapshot.ref)).resolves.toMatchObject({ ok: true }); + await expect(provider.restoreFresh(snapshot.ref, restorePath)).resolves.toMatchObject({ + ok: true, + }); + expect((await fs.stat(repositoryPath)).mode & 0o777).toBe(0o500); + expect((await fs.stat(snapshot.ref.path)).mode & 0o777).toBe(0o500); + const sqlite = requireNodeSqlite(); + const restored = new sqlite.DatabaseSync(restorePath, { readOnly: true }); + try { + expect(restored.prepare("SELECT value FROM entries").all()).toEqual([ + { value: "read-only-source" }, + ]); + } finally { + restored.close(); + } + } finally { + await fs.chmod(repositoryPath, 0o700); + await fs.chmod(snapshot.ref.path, 0o700); + for (const entryPath of snapshotEntries) { + await fs.chmod(entryPath, 0o600); + } + } + }, + ); + + it("preserves both restore and cleanup failures", async () => { + const tempDir = await createTempDir(); + const sourcePath = path.join(tempDir, "source.sqlite"); + const repositoryPath = path.join(tempDir, "snapshots"); + const restorePath = path.join(tempDir, "restore", "source.sqlite"); + createGenericDatabase(sourcePath); + const provider = createLocalSqliteSnapshotProvider({ repositoryPath }); + const snapshot = await provider.create({ + path: sourcePath, + identity: { role: "generic", id: "combined-failure" }, + }); + const linkSpy = vi + .spyOn(fs, "link") + .mockRejectedValue(Object.assign(new Error("hard links unsupported"), { code: "ENOTSUP" })); + const originalUnlink = fs.unlink.bind(fs); + const unlinkSpy = vi.spyOn(fs, "unlink").mockImplementation(async (filePath) => { + if (path.basename(path.dirname(String(filePath))).startsWith(".tmp-restore-")) { + throw Object.assign(new Error("cleanup denied"), { code: "EACCES" }); + } + return await originalUnlink(filePath); + }); + + try { + const error = await provider + .restoreFresh(snapshot.ref, restorePath) + .catch((cause: unknown) => cause); + expect(error).toBeInstanceOf(AggregateError); + expect((error as AggregateError).errors.map(String)).toEqual([ + expect.stringMatching(/requires hard-link support/u), + expect.stringMatching(/cleanup denied/u), + ]); + } finally { + unlinkSpy.mockRestore(); + linkSpy.mockRestore(); + } + }); + + it("preserves a published restore when staging cleanup fails", async () => { + const tempDir = await createTempDir(); + const sourcePath = path.join(tempDir, "source.sqlite"); + const repositoryPath = path.join(tempDir, "snapshots"); + const restorePath = path.join(tempDir, "restore", "source.sqlite"); + createGenericDatabase(sourcePath, { values: ["published-before-cleanup"] }); + const provider = createLocalSqliteSnapshotProvider({ repositoryPath }); + const snapshot = await provider.create({ + path: sourcePath, + identity: { role: "generic", id: "restore-cleanup-failure" }, + }); + const originalUnlink = fs.unlink.bind(fs); + const unlinkSpy = vi.spyOn(fs, "unlink").mockImplementation(async (filePath) => { + if (path.basename(path.dirname(String(filePath))).startsWith(".tmp-restore-")) { + throw Object.assign(new Error("restore cleanup denied"), { code: "EACCES" }); + } + return await originalUnlink(filePath); + }); + + try { + await expect(provider.restoreFresh(snapshot.ref, restorePath)).rejects.toThrow( + /Failed to clean private SQLite staging directory/u, + ); + } finally { + unlinkSpy.mockRestore(); + } + const sqlite = requireNodeSqlite(); + const restored = new sqlite.DatabaseSync(restorePath, { readOnly: true }); + try { + expect(restored.prepare("SELECT value FROM entries").all()).toEqual([ + { value: "published-before-cleanup" }, + ]); + } finally { + restored.close(); + } + }); + + it("does not report best-effort directory sync as a failed restore", async () => { + const tempDir = await createTempDir(); + const sourcePath = path.join(tempDir, "source.sqlite"); + const repositoryPath = path.join(tempDir, "snapshots"); + const restoreParentPath = path.join(tempDir, "restore"); + const restorePath = path.join(restoreParentPath, "source.sqlite"); + createGenericDatabase(sourcePath); + const provider = createLocalSqliteSnapshotProvider({ repositoryPath }); + const snapshot = await provider.create({ + path: sourcePath, + identity: { role: "generic", id: "best-effort-directory-sync" }, + }); + const originalOpen = fs.open.bind(fs); + const openSpy = vi.spyOn(fs, "open").mockImplementation(async (filePath, flags, mode) => { + if (path.resolve(String(filePath)) === restoreParentPath && flags === "r") { + const entries = await fs.readdir(restoreParentPath); + if ( + entries.includes(path.basename(restorePath)) && + entries.every((entry) => !entry.startsWith(".tmp-restore-")) + ) { + throw Object.assign(new Error("directory sync unavailable"), { code: "EIO" }); + } + } + return await originalOpen(filePath, flags, mode); + }); + + try { + await expect(provider.restoreFresh(snapshot.ref, restorePath)).resolves.toMatchObject({ + ok: true, + }); + } finally { + openSpy.mockRestore(); + } + await expect(fs.access(restorePath)).resolves.toBeUndefined(); + }); + + it.runIf(process.platform !== "win32")( + "never replaces a snapshot directory raced into place", + async () => { + const tempDir = await createTempDir(); + const sourcePath = path.join(tempDir, "source.sqlite"); + const repositoryPath = path.join(tempDir, "snapshots"); + createGenericDatabase(sourcePath); + const provider = createLocalSqliteSnapshotProvider({ + repositoryPath, + now: () => new Date("2026-07-12T14:00:00.000Z"), + }); + const originalMkdir = fs.mkdir.bind(fs); + let racedPath: string | undefined; + const mkdirSpy = vi.spyOn(fs, "mkdir").mockImplementation(async (directoryPath, options) => { + const resolvedPath = path.resolve(String(directoryPath)); + if ( + path.basename(path.dirname(resolvedPath)) === path.basename(repositoryPath) && + !path.basename(resolvedPath).startsWith(".tmp-") + ) { + racedPath = resolvedPath; + await originalMkdir(resolvedPath, options); + await fs.writeFile(path.join(resolvedPath, "keep"), "racer"); + } + return await originalMkdir(directoryPath, options); + }); + + try { + await expect( + provider.create({ + path: sourcePath, + identity: { role: "generic", id: "directory-race" }, + }), + ).rejects.toThrow(/directory already exists/u); + } finally { + mkdirSpy.mockRestore(); + } + expect(racedPath).toBeDefined(); + await expect(fs.readFile(path.join(racedPath!, "keep"), "utf8")).resolves.toBe("racer"); + }, + ); + + it("rejects an artifact changed after entering the final directory", async () => { + const tempDir = await createTempDir(); + const sourcePath = path.join(tempDir, "source.sqlite"); + const repositoryPath = path.join(tempDir, "snapshots"); + createGenericDatabase(sourcePath); + const provider = createLocalSqliteSnapshotProvider({ repositoryPath }); + const originalLink = fs.link.bind(fs); + const linkSpy = vi.spyOn(fs, "link").mockImplementation(async (source, target) => { + await originalLink(source, target); + if ( + path.basename(String(target)) === SNAPSHOT_MANIFEST_FILENAME && + !path.basename(path.dirname(String(target))).startsWith(".tmp-") + ) { + await fs.appendFile( + path.join(path.dirname(String(target)), SNAPSHOT_SQLITE_FILENAME), + "changed-after-final-move", + ); + } + }); + + try { + await expect( + provider.create({ + path: sourcePath, + identity: { role: "generic", id: "final-directory-race" }, + }), + ).rejects.toThrow(/size mismatch/u); + await expect(provider.list()).resolves.toEqual([]); + } finally { + linkSpy.mockRestore(); + } + }); + + it("cleans a linked entry when post-link inspection fails", async () => { + const tempDir = await createTempDir(); + const sourcePath = path.join(tempDir, "source.sqlite"); + const repositoryPath = path.join(tempDir, "snapshots"); + createGenericDatabase(sourcePath); + const provider = createLocalSqliteSnapshotProvider({ repositoryPath }); + const originalLink = fs.link.bind(fs); + const originalLstat = fs.lstat.bind(fs); + let linkedArtifactPath: string | undefined; + let failedInspection = false; + const linkSpy = vi.spyOn(fs, "link").mockImplementation(async (source, target) => { + await originalLink(source, target); + if ( + path.basename(String(target)) === SNAPSHOT_SQLITE_FILENAME && + !path.basename(path.dirname(String(target))).startsWith(".tmp-") + ) { + linkedArtifactPath = path.resolve(String(target)); + } + }); + const lstatSpy = vi.spyOn(fs, "lstat").mockImplementation(async (filePath) => { + if ( + linkedArtifactPath && + !failedInspection && + path.resolve(String(filePath)) === linkedArtifactPath + ) { + failedInspection = true; + throw Object.assign(new Error("post-link inspection failed"), { code: "EIO" }); + } + return await originalLstat(filePath); + }); + + try { + await expect( + provider.create({ + path: sourcePath, + identity: { role: "generic", id: "post-link-inspection" }, + }), + ).rejects.toThrow(/post-link inspection failed/u); + await expect(provider.list()).resolves.toEqual([]); + await expect(fs.readdir(repositoryPath)).resolves.toEqual([]); + } finally { + lstatSpy.mockRestore(); + linkSpy.mockRestore(); + } + }); + + it("never overwrites a file raced into the final snapshot directory", async () => { + const tempDir = await createTempDir(); + const sourcePath = path.join(tempDir, "source.sqlite"); + const repositoryPath = path.join(tempDir, "snapshots"); + createGenericDatabase(sourcePath); + const provider = createLocalSqliteSnapshotProvider({ repositoryPath }); + const originalLink = fs.link.bind(fs); + let racedPath: string | undefined; + const linkSpy = vi.spyOn(fs, "link").mockImplementation(async (source, target) => { + const targetPath = path.resolve(String(target)); + if ( + path.basename(targetPath) === SNAPSHOT_SQLITE_FILENAME && + path.dirname(targetPath) !== repositoryPath && + !path.basename(path.dirname(targetPath)).startsWith(".tmp-") + ) { + racedPath = targetPath; + await fs.writeFile(targetPath, "racer", { flag: "wx" }); + } + await originalLink(source, target); + }); + + try { + await expect( + provider.create({ + path: sourcePath, + identity: { role: "generic", id: "entry-race" }, + }), + ).rejects.toThrow(/EEXIST/u); + } finally { + linkSpy.mockRestore(); + } + expect(racedPath).toBeDefined(); + await expect(fs.readFile(racedPath!, "utf8")).resolves.toBe("racer"); + }); + + it("sanitizes transient global delivery rows and enforces the global owner", async () => { + const tempDir = await createTempDir(); + const sourcePath = path.join(tempDir, "openclaw.sqlite"); + const repositoryPath = path.join(tempDir, "snapshots"); + createGlobalDatabase(sourcePath); + const provider = createLocalSqliteSnapshotProvider({ repositoryPath }); + const snapshot = await provider.create({ + path: sourcePath, + identity: { role: "global" }, + }); + const artifactPath = path.join(snapshot.ref.path, SNAPSHOT_SQLITE_FILENAME); + expect((await fs.readFile(artifactPath)).includes("do-not-restore")).toBe(false); + const sqlite = requireNodeSqlite(); + const artifact = new sqlite.DatabaseSync(artifactPath, { readOnly: true }); + try { + expect( + artifact.prepare("SELECT COUNT(*) AS count FROM delivery_queue_entries").get(), + ).toEqual({ count: 0 }); + } finally { + artifact.close(); + } + + const wrongRolePath = path.join(tempDir, "wrong-role.sqlite"); + createAgentDatabase(wrongRolePath, "main"); + const wrongRole = new sqlite.DatabaseSync(wrongRolePath); + wrongRole.exec(`PRAGMA user_version = ${OPENCLAW_STATE_SCHEMA_VERSION};`); + wrongRole.close(); + await expect( + provider.create({ path: wrongRolePath, identity: { role: "global" } }), + ).rejects.toThrow(/expected global/u); + }); + + it("enforces the exact agent owner and canonical agent id", async () => { + const tempDir = await createTempDir(); + const sourcePath = path.join(tempDir, "openclaw-agent.sqlite"); + const repositoryPath = path.join(tempDir, "snapshots"); + createAgentDatabase(sourcePath, "worker-1"); + const provider = createLocalSqliteSnapshotProvider({ repositoryPath }); + + await expect( + provider.create({ + path: sourcePath, + identity: { role: "agent", agentId: "worker-2" }, + }), + ).rejects.toThrow(/belongs to agent worker-1/u); + await expect( + provider.create({ + path: sourcePath, + identity: { role: "agent", agentId: "Worker-1" }, + }), + ).rejects.toThrow(/must be canonical/u); + await expect( + provider.create({ + path: sourcePath, + identity: { role: "agent", agentId: "worker-1" }, + }), + ).resolves.toMatchObject({ + manifest: { + database: { + role: "agent", + agentId: "worker-1", + userVersion: OPENCLAW_AGENT_SCHEMA_VERSION, + }, + }, + }); + }); + + it("rejects foreign-key violations and unsafe index definitions at creation", async () => { + const tempDir = await createTempDir(); + const repositoryPath = path.join(tempDir, "snapshots"); + const foreignKeyPath = path.join(tempDir, "foreign-key.sqlite"); + const sqlite = requireNodeSqlite(); + const foreignKeyDatabase = new sqlite.DatabaseSync(foreignKeyPath); + try { + foreignKeyDatabase.exec(` + PRAGMA foreign_keys = OFF; + CREATE TABLE parents (id INTEGER PRIMARY KEY); + CREATE TABLE children ( + id INTEGER PRIMARY KEY, + parent_id INTEGER REFERENCES parents(id) + ); + INSERT INTO children VALUES (1, 99); + `); + } finally { + foreignKeyDatabase.close(); + } + const unsafeIndexPath = path.join(tempDir, "unsafe-index.sqlite"); + createUnsafeIndexDrift(unsafeIndexPath); + const provider = createLocalSqliteSnapshotProvider({ repositoryPath }); + + await expect( + provider.create({ + path: foreignKeyPath, + identity: { role: "generic", id: "foreign-key" }, + }), + ).rejects.toThrow(/foreign_key_check failed/u); + await expect( + provider.create({ + path: unsafeIndexPath, + identity: { role: "generic", id: "unsafe-index" }, + }), + ).rejects.toThrow(/integrity_check failed|malformed database schema/iu); + await expect(provider.list()).resolves.toEqual([]); + }); + + it("detects artifact hash, user_version, and unsafe-index drift after creation", async () => { + const tempDir = await createTempDir(); + const sourcePath = path.join(tempDir, "source.sqlite"); + const repositoryPath = path.join(tempDir, "snapshots"); + createGenericDatabase(sourcePath); + const provider = createLocalSqliteSnapshotProvider({ repositoryPath }); + + const hashSnapshot = await provider.create({ + path: sourcePath, + identity: { role: "generic", id: "hash" }, + }); + await fs.appendFile(path.join(hashSnapshot.ref.path, SNAPSHOT_SQLITE_FILENAME), "tamper"); + await expect(provider.verify(hashSnapshot.ref)).rejects.toThrow(/size mismatch/u); + + const versionSnapshot = await provider.create({ + path: sourcePath, + identity: { role: "generic", id: "version" }, + }); + const sqlite = requireNodeSqlite(); + const versionDatabase = new sqlite.DatabaseSync( + path.join(versionSnapshot.ref.path, SNAPSHOT_SQLITE_FILENAME), + ); + versionDatabase.exec("PRAGMA user_version = 99;"); + versionDatabase.close(); + await refreshArtifactManifest(versionSnapshot); + await expect(provider.verify(versionSnapshot.ref)).rejects.toThrow(/user_version mismatch/u); + + const unsafeSnapshot = await provider.create({ + path: sourcePath, + identity: { role: "generic", id: "unsafe" }, + }); + const unsafePath = path.join(unsafeSnapshot.ref.path, SNAPSHOT_SQLITE_FILENAME); + const unsafeDatabase = new sqlite.DatabaseSync(unsafePath); + try { + disableDefensiveModeForSchemaCorruption(unsafeDatabase); + unsafeDatabase.exec(` + CREATE TABLE indexed_records ( + id INTEGER PRIMARY KEY, + indexed_value TEXT NOT NULL, + alternate_value TEXT NOT NULL + ); + CREATE INDEX indexed_records_value ON indexed_records(indexed_value); + INSERT INTO indexed_records (indexed_value, alternate_value) + VALUES ('alpha', 'zeta'), ('beta', 'eta'); + PRAGMA writable_schema = ON; + `); + unsafeDatabase + .prepare( + "UPDATE sqlite_schema SET sql = 'CREATE INDEX indexed_records_value ON indexed_records(alternate_value)' WHERE name = 'indexed_records_value'", + ) + .run(); + const schemaVersion = Number( + Object.values( + unsafeDatabase.prepare("PRAGMA schema_version").get() as Record, + )[0], + ); + unsafeDatabase.exec( + `PRAGMA writable_schema = OFF; PRAGMA schema_version = ${schemaVersion + 1};`, + ); + } finally { + unsafeDatabase.close(); + } + await refreshArtifactManifest(unsafeSnapshot); + await expect(provider.verify(unsafeSnapshot.ref)).rejects.toThrow( + /integrity_check failed|malformed database schema/iu, + ); + }); + + it("never overwrites an existing target or orphan SQLite sidecar", async () => { + const tempDir = await createTempDir(); + const sourcePath = path.join(tempDir, "source.sqlite"); + const repositoryPath = path.join(tempDir, "snapshots"); + const restorePath = path.join(tempDir, "restore", "source.sqlite"); + createGenericDatabase(sourcePath); + const provider = createLocalSqliteSnapshotProvider({ repositoryPath }); + const snapshot = await provider.create({ + path: sourcePath, + identity: { role: "generic", id: "restore" }, + }); + await fs.mkdir(path.dirname(restorePath), { recursive: true }); + await fs.writeFile(restorePath, "keep"); + + await expect(provider.restoreFresh(snapshot.ref, restorePath)).rejects.toThrow( + /restore path already exists/u, + ); + await expect(fs.readFile(restorePath, "utf8")).resolves.toBe("keep"); + + await fs.unlink(restorePath); + await fs.writeFile(`${restorePath}-wal`, "keep-wal"); + await expect(provider.restoreFresh(snapshot.ref, restorePath)).rejects.toThrow( + /restore path already exists/u, + ); + await expect(fs.readFile(`${restorePath}-wal`, "utf8")).resolves.toBe("keep-wal"); + await expect(fs.access(restorePath)).rejects.toMatchObject({ code: "ENOENT" }); + + if (process.platform !== "win32") { + await fs.unlink(`${restorePath}-wal`); + const externalPath = path.join(tempDir, "external.sqlite"); + await fs.writeFile(externalPath, "external"); + await fs.symlink(externalPath, restorePath); + await expect(provider.restoreFresh(snapshot.ref, restorePath)).rejects.toThrow( + /restore path already exists/u, + ); + await expect(fs.readFile(externalPath, "utf8")).resolves.toBe("external"); + expect((await fs.lstat(restorePath)).isSymbolicLink()).toBe(true); + } + }); + + it("fails closed when fresh restore cannot publish atomically", async () => { + const tempDir = await createTempDir(); + const sourcePath = path.join(tempDir, "source.sqlite"); + const repositoryPath = path.join(tempDir, "snapshots"); + const restorePath = path.join(tempDir, "restore", "source.sqlite"); + createGenericDatabase(sourcePath); + const provider = createLocalSqliteSnapshotProvider({ repositoryPath }); + const snapshot = await provider.create({ + path: sourcePath, + identity: { role: "generic", id: "atomic-restore" }, + }); + const linkSpy = vi + .spyOn(fs, "link") + .mockRejectedValue(Object.assign(new Error("hard links unsupported"), { code: "ENOTSUP" })); + + try { + await expect(provider.restoreFresh(snapshot.ref, restorePath)).rejects.toThrow( + /requires hard-link support/u, + ); + await expect(fs.access(restorePath)).rejects.toMatchObject({ code: "ENOENT" }); + } finally { + linkSpy.mockRestore(); + } + }); + + it("rejects restore targets inside the snapshot repository", async () => { + const tempDir = await createTempDir(); + const sourcePath = path.join(tempDir, "source.sqlite"); + const repositoryPath = path.join(tempDir, "snapshots"); + createGenericDatabase(sourcePath); + const provider = createLocalSqliteSnapshotProvider({ repositoryPath }); + const snapshot = await provider.create({ + path: sourcePath, + identity: { role: "generic", id: "repository-boundary" }, + }); + + await expect( + provider.restoreFresh(snapshot.ref, path.join(repositoryPath, "restored.sqlite")), + ).rejects.toThrow(/outside snapshot repository/u); + await expect( + provider.restoreFresh(snapshot.ref, path.join(snapshot.ref.path, "restored.sqlite")), + ).rejects.toThrow(/outside snapshot repository/u); + await expect(provider.verify(snapshot.ref)).resolves.toMatchObject({ ok: true }); + }); + + it.runIf(process.platform !== "win32")( + "rejects a restore parent redirected after directory creation", + async () => { + const tempDir = await createTempDir(); + const sourcePath = path.join(tempDir, "source.sqlite"); + const repositoryPath = path.join(tempDir, "snapshots"); + const restoreParentPath = path.join(tempDir, "redirected"); + const restorePath = path.join(restoreParentPath, "restored.sqlite"); + createGenericDatabase(sourcePath); + const provider = createLocalSqliteSnapshotProvider({ repositoryPath }); + const snapshot = await provider.create({ + path: sourcePath, + identity: { role: "generic", id: "restore-parent-race" }, + }); + const canonicalRestoreParentPath = path.join( + await fs.realpath(tempDir), + path.basename(restoreParentPath), + ); + const originalRealpath = fs.realpath.bind(fs); + let redirected = false; + const realpathSpy = vi.spyOn(fs, "realpath").mockImplementation(async (...args) => { + const pathname = path.resolve(String(args[0])); + if (!redirected && pathname === canonicalRestoreParentPath) { + redirected = true; + await fs.rmdir(canonicalRestoreParentPath); + await fs.symlink(snapshot.ref.path, canonicalRestoreParentPath, "dir"); + } + return await originalRealpath(...args); + }); + + try { + await expect(provider.restoreFresh(snapshot.ref, restorePath)).rejects.toThrow( + /restore target changed|outside snapshot repository/u, + ); + } finally { + realpathSpy.mockRestore(); + } + expect(redirected).toBe(true); + await expect( + fs.access(path.join(snapshot.ref.path, path.basename(restorePath))), + ).rejects.toMatchObject({ code: "ENOENT" }); + }, + ); + + it.runIf(process.platform !== "win32")( + "binds restore to the exact artifact bytes recorded by the manifest", + async () => { + const tempDir = await createTempDir(); + const sourcePath = path.join(tempDir, "source.sqlite"); + const repositoryPath = path.join(tempDir, "snapshots"); + const restorePath = path.join(tempDir, "restore", "source.sqlite"); + createGenericDatabase(sourcePath); + const provider = createLocalSqliteSnapshotProvider({ repositoryPath }); + const snapshot = await provider.create({ + path: sourcePath, + identity: { role: "generic", id: "verified-bytes" }, + }); + const artifactPath = path.join(snapshot.ref.path, SNAPSHOT_SQLITE_FILENAME); + const originalOpen = fs.open.bind(fs); + const openSpy = vi.spyOn(fs, "open").mockImplementation(async (filePath, flags, mode) => { + const handle = await originalOpen(filePath, flags, mode); + if ( + flags === "wx+" && + path.basename(String(filePath)) === SNAPSHOT_SQLITE_FILENAME && + path.basename(path.dirname(String(filePath))).startsWith(".tmp-restore-") + ) { + const sqlite = requireNodeSqlite(); + const database = new sqlite.DatabaseSync(artifactPath); + database.prepare("INSERT INTO entries (value) VALUES (?)").run("raced"); + database.close(); + } + return handle; + }); + + try { + await expect(provider.restoreFresh(snapshot.ref, restorePath)).rejects.toThrow( + /hash mismatch|size mismatch/u, + ); + await expect(fs.access(restorePath)).rejects.toMatchObject({ code: "ENOENT" }); + } finally { + openSpy.mockRestore(); + } + }, + ); + + it.runIf(process.platform !== "win32")( + "never publishes replacement bytes when the pinned staging pathname changes", + async () => { + const tempDir = await createTempDir(); + const sourcePath = path.join(tempDir, "source.sqlite"); + const repositoryPath = path.join(tempDir, "snapshots"); + const restorePath = path.join(tempDir, "restore", "source.sqlite"); + createGenericDatabase(sourcePath, { values: ["original"] }); + const provider = createLocalSqliteSnapshotProvider({ repositoryPath }); + const snapshot = await provider.create({ + path: sourcePath, + identity: { role: "generic", id: "pinned-staging" }, + }); + const originalOpen = fs.open.bind(fs); + const openSpy = vi.spyOn(fs, "open").mockImplementation(async (filePath, flags, mode) => { + if ( + flags === "wx+" && + path.basename(path.dirname(String(filePath))).startsWith(".sqlite-publish-") + ) { + const stagingEntry = (await fs.readdir(path.dirname(restorePath))).find((entry) => + entry.startsWith(".tmp-restore-"), + ); + if (!stagingEntry) { + throw new Error("restore staging directory was not created"); + } + const stagedPath = path.join( + path.dirname(restorePath), + stagingEntry, + SNAPSHOT_SQLITE_FILENAME, + ); + await fs.unlink(stagedPath); + createGenericDatabase(stagedPath, { values: ["replacement"] }); + } + return await originalOpen(filePath, flags, mode); + }); + + let restored = false; + try { + await provider.restoreFresh(snapshot.ref, restorePath); + restored = true; + } catch (error) { + expect(String(error)).toMatch(/file changed while reading/u); + } finally { + openSpy.mockRestore(); + } + if (!restored) { + await expect(fs.access(restorePath)).rejects.toMatchObject({ code: "ENOENT" }); + return; + } + const sqlite = requireNodeSqlite(); + const database = new sqlite.DatabaseSync(restorePath, { readOnly: true }); + try { + expect(database.prepare("SELECT value FROM entries").all()).toEqual([ + { value: "original" }, + ]); + } finally { + database.close(); + } + }, + ); + + it.runIf(process.platform !== "win32")( + "removes only its restored target when a sidecar races publication", + async () => { + const tempDir = await createTempDir(); + const sourcePath = path.join(tempDir, "source.sqlite"); + const repositoryPath = path.join(tempDir, "snapshots"); + const restorePath = path.join(tempDir, "restore", "source.sqlite"); + createGenericDatabase(sourcePath); + const provider = createLocalSqliteSnapshotProvider({ repositoryPath }); + const snapshot = await provider.create({ + path: sourcePath, + identity: { role: "generic", id: "restore-race" }, + }); + const canonicalRestorePath = path.join( + await fs.realpath(tempDir), + "restore", + "source.sqlite", + ); + const originalLink = fs.link.bind(fs); + const linkSpy = vi.spyOn(fs, "link").mockImplementation(async (source, target) => { + await originalLink(source, target); + if (path.resolve(String(target)) === canonicalRestorePath) { + await fs.writeFile(`${canonicalRestorePath}-wal`, "racer"); + } + }); + + try { + await expect(provider.restoreFresh(snapshot.ref, restorePath)).rejects.toThrow( + /unexpected sidecar/u, + ); + await expect(fs.access(restorePath)).rejects.toMatchObject({ code: "ENOENT" }); + await expect(fs.readFile(`${restorePath}-wal`, "utf8")).resolves.toBe("racer"); + } finally { + linkSpy.mockRestore(); + } + }, + ); + + it("rejects snapshots outside the configured repository and unexpected contents", async () => { + const tempDir = await createTempDir(); + const sourcePath = path.join(tempDir, "source.sqlite"); + const repositoryPath = path.join(tempDir, "snapshots"); + createGenericDatabase(sourcePath); + const provider = createLocalSqliteSnapshotProvider({ repositoryPath }); + const snapshot = await provider.create({ + path: sourcePath, + identity: { role: "generic", id: "boundary" }, + }); + + await expect(provider.verify({ path: tempDir })).rejects.toThrow(/immediate child/u); + await fs.writeFile(path.join(snapshot.ref.path, `${SNAPSHOT_SQLITE_FILENAME}-wal`), "orphan"); + await expect(provider.verify(snapshot.ref)).rejects.toThrow(/unexpected entry/u); + await expect(provider.list()).rejects.toThrow(/unexpected entry/u); + }); + + it("bounds manifest reads before parsing untrusted snapshot metadata", async () => { + const tempDir = await createTempDir(); + const sourcePath = path.join(tempDir, "source.sqlite"); + const repositoryPath = path.join(tempDir, "snapshots"); + createGenericDatabase(sourcePath); + const provider = createLocalSqliteSnapshotProvider({ repositoryPath }); + const snapshot = await provider.create({ + path: sourcePath, + identity: { role: "generic", id: "bounded-manifest" }, + }); + await fs.writeFile( + path.join(snapshot.ref.path, SNAPSHOT_MANIFEST_FILENAME), + Buffer.alloc(1024 * 1024 + 1, 0x20), + ); + + await expect(provider.verify(snapshot.ref)).rejects.toThrow(/1048576 bytes/u); + }); + + it.runIf(process.platform !== "win32")( + "rejects symlinked repositories, snapshot files, and hardlinked artifacts", + async () => { + const tempDir = await createTempDir(); + const sourcePath = path.join(tempDir, "source.sqlite"); + const realRepositoryPath = path.join(tempDir, "real-snapshots"); + const repositoryLink = path.join(tempDir, "snapshot-link"); + createGenericDatabase(sourcePath); + await fs.mkdir(realRepositoryPath); + await fs.symlink(realRepositoryPath, repositoryLink); + const linkedProvider = createLocalSqliteSnapshotProvider({ + repositoryPath: repositoryLink, + }); + await expect( + linkedProvider.create({ + path: sourcePath, + identity: { role: "generic", id: "symlink-repository" }, + }), + ).rejects.toThrow(/symlink|Invalid path/iu); + + const provider = createLocalSqliteSnapshotProvider({ + repositoryPath: realRepositoryPath, + }); + const snapshot = await provider.create({ + path: sourcePath, + identity: { role: "generic", id: "links" }, + }); + const artifactPath = path.join(snapshot.ref.path, SNAPSHOT_SQLITE_FILENAME); + const externalArtifact = path.join(tempDir, "external.sqlite"); + await fs.link(artifactPath, externalArtifact); + await expect(provider.verify(snapshot.ref)).rejects.toThrow(/hardlink/iu); + await fs.unlink(externalArtifact); + + const manifestPath = path.join(snapshot.ref.path, SNAPSHOT_MANIFEST_FILENAME); + const realManifest = path.join(tempDir, "manifest.json"); + await fs.rename(manifestPath, realManifest); + await fs.symlink(realManifest, manifestPath); + await expect(provider.verify(snapshot.ref)).rejects.toThrow(/regular file|symlink/iu); + + await fs.unlink(manifestPath); + await fs.rename(realManifest, manifestPath); + }, + ); + + it.runIf(process.platform !== "win32")( + "rejects repository restore targets reached through another filesystem spelling", + async () => { + const tempDir = await createTempDir(); + const sourcePath = path.join(tempDir, "source.sqlite"); + const repositoryPath = path.join(tempDir, "snapshots"); + createGenericDatabase(sourcePath); + const provider = createLocalSqliteSnapshotProvider({ repositoryPath }); + const snapshot = await provider.create({ + path: sourcePath, + identity: { role: "generic", id: "canonical-boundary" }, + }); + const aliasRoot = path.join(tempDir, "alias"); + await fs.symlink(tempDir, aliasRoot); + const aliasRepositoryPath = path.join(aliasRoot, "snapshots"); + const aliasProvider = createLocalSqliteSnapshotProvider({ + repositoryPath: aliasRepositoryPath, + }); + const aliasSnapshot = { + path: path.join(aliasRepositoryPath, path.basename(snapshot.ref.path)), + }; + + await expect( + aliasProvider.restoreFresh(aliasSnapshot, path.join(repositoryPath, "restored.sqlite")), + ).rejects.toThrow(/outside snapshot repository/u); + await expect(provider.verify(snapshot.ref)).resolves.toMatchObject({ ok: true }); + }, + ); +}); + +describe("snapshot manifest parser", () => { + const manifestPath = "/snapshots/snapshot/manifest.json"; + const snapshotId = "snapshot"; + const validManifest: SnapshotManifest = { + schemaVersion: 1, + snapshotId, + createdAt: "2026-07-12T14:00:00.000Z", + database: { + role: "agent", + agentId: "worker-1", + basename: "openclaw-agent.sqlite", + userVersion: OPENCLAW_AGENT_SCHEMA_VERSION, + }, + artifact: { + path: SNAPSHOT_SQLITE_FILENAME, + sha256: "a".repeat(64), + sizeBytes: 4096, + }, + }; + + it.each([ + ["unknown top-level field", { ...validManifest, extra: true }, /fields must be exactly/u], + ["wrong directory id", validManifest, /does not match directory/u, "other"], + [ + "noncanonical timestamp", + { ...validManifest, createdAt: "2026-07-12T14:00:00Z" }, + /not canonical/u, + ], + [ + "artifact path traversal", + { ...validManifest, artifact: { ...validManifest.artifact, path: "../database.sqlite" } }, + /artifact\.path must be database\.sqlite/u, + ], + [ + "prefixed digest", + { + ...validManifest, + artifact: { ...validManifest.artifact, sha256: `sha256:${"a".repeat(64)}` }, + }, + /sha256 is invalid/u, + ], + [ + "noncanonical agent id", + { ...validManifest, database: { ...validManifest.database, agentId: "Worker-1" } }, + /agentId is invalid/u, + ], + [ + "unsafe basename", + { ...validManifest, database: { ...validManifest.database, basename: "../db.sqlite" } }, + /basename is invalid/u, + ], + [ + "out-of-range user version", + { ...validManifest, database: { ...validManifest.database, userVersion: 2 ** 31 } }, + /userVersion is invalid/u, + ], + [ + "zero-byte artifact", + { ...validManifest, artifact: { ...validManifest.artifact, sizeBytes: 0 } }, + /sizeBytes is invalid/u, + ], + ])("rejects %s", (_name, value, error, expectedId = snapshotId) => { + expect(() => parseSnapshotManifest(value, manifestPath, expectedId)).toThrow(error); + }); +}); diff --git a/src/snapshot/local-repository.ts b/src/snapshot/local-repository.ts new file mode 100644 index 000000000000..5bd4b40d0f07 --- /dev/null +++ b/src/snapshot/local-repository.ts @@ -0,0 +1,1479 @@ +import { randomUUID } from "node:crypto"; +import fsSync, { type Stats } from "node:fs"; +import fs from "node:fs/promises"; +import type { FileHandle } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import type { DatabaseSync } from "node:sqlite"; +import { isDeepStrictEqual } from "node:util"; +import { z } from "zod"; +import { loadSqliteVecExtension } from "../../packages/memory-host-sdk/src/engine-storage.js"; +import { sameFileIdentity } from "../infra/fs-safe-advanced.js"; +import { + canonicalPathFromExistingAncestor, + ensureAbsoluteDirectory, + isPathInside, +} from "../infra/fs-safe.js"; +import { requireNodeSqlite } from "../infra/node-sqlite.js"; +import { applyPrivateModeSync } from "../infra/private-mode.js"; +import { resolveSystemBin } from "../infra/resolve-system-bin.js"; +import { assertSqliteIntegrity } from "../infra/sqlite-integrity.js"; +import { + createPrivateSqliteDirectory, + createPrivateSqliteTempDirectory, + createVerifiedSqliteSnapshot, + publishVerifiedSqliteFile, + syncDirectoryBestEffort, + type SqliteSnapshotValidator, +} from "../infra/sqlite-snapshot.js"; +import { readSqliteUserVersion } from "../infra/sqlite-user-version.js"; +import { runExec } from "../process/exec.js"; +import { isValidAgentId, normalizeAgentId } from "../routing/session-key.js"; +import { assertOpenClawAgentDatabaseForMaintenance } from "../state/openclaw-agent-db.js"; +import { assertOpenClawStateDatabaseForMaintenance } from "../state/openclaw-state-db.js"; +import { + containsAsciiControlCharacter, + copySnapshotArtifact, + hashSnapshotArtifact, + readSnapshotManifest, + type SnapshotArtifactDigest, + writeSnapshotManifest, +} from "./manifest.js"; +import { + SNAPSHOT_MANIFEST_FILENAME, + SNAPSHOT_SQLITE_FILENAME, + type SnapshotDatabaseIdentity, + type SnapshotDatabaseManifest, + type SnapshotDatabaseRef, + type SnapshotManifest, + type SnapshotRef, + type SnapshotResult, + type SnapshotSummary, + type SnapshotVerificationResult, + type SqliteSnapshotProvider, +} from "./snapshot-provider.js"; + +const SNAPSHOT_DIRECTORY_MODE = 0o700; +const SNAPSHOT_FILE_MODE = 0o600; +const SNAPSHOT_PENDING_FILENAME = ".pending"; +const SQLITE_SIDECAR_SUFFIXES = ["-wal", "-shm", "-journal"] as const; +const SNAPSHOT_ARTIFACT_ENTRIES = new Set([ + SNAPSHOT_MANIFEST_FILENAME, + SNAPSHOT_PENDING_FILENAME, + SNAPSHOT_SQLITE_FILENAME, +]); +const RESTORE_STAGING_ENTRIES = new Set([SNAPSHOT_SQLITE_FILENAME]); +const VALIDATION_STAGING_ENTRIES = new Set([ + SNAPSHOT_SQLITE_FILENAME, + ...SQLITE_SIDECAR_SUFFIXES.map((suffix) => `${SNAPSHOT_SQLITE_FILENAME}${suffix}`), +]); +const MACOS_REPLACEMENT_ACL_PERMISSIONS = new Set([ + "add_file", + "add_subdirectory", + "chown", + "delete", + "delete_child", + "writesecurity", +]); +const WINDOWS_STAGING_ACCESS_RIGHTS = new Set([ + "F", + "M", + "RX", + "R", + "W", + "D", + "DE", + "RC", + "WDAC", + "WO", + "AS", + "MA", + "GR", + "GW", + "GE", + "GA", + "RD", + "WD", + "AD", + "REA", + "WEA", + "X", + "DC", + "RA", + "WA", + "UNKNOWN", +]); +const WINDOWS_STAGING_REPLACEMENT_RIGHTS = new Set([ + "F", + "M", + "D", + "DE", + "WDAC", + "WO", + "MA", + "GA", + "DC", + "UNKNOWN", +]); +const WINDOWS_TRUSTED_OWNER_SIDS = new Set([ + "S-1-5-18", // LocalSystem + "S-1-5-32-544", // Builtin Administrators + "S-1-5-80-956008885-3418522649-1831038044-1853292631-2271478464", // TrustedInstaller +]); +const WINDOWS_TRUSTED_ACCESS_SIDS = new Set([ + ...WINDOWS_TRUSTED_OWNER_SIDS, + "S-1-3-0", // Creator Owner resolves to the trusted creator on inherited ACEs. +]); +// Windows descriptors can approach 64 KiB each; batched JSON and base64 need +// bounded aggregate headroom across every ancestor. +const WINDOWS_ACL_METADATA_MAX_BUFFER = 16 * 1024 * 1024; +const WINDOWS_SID_PATTERN = /^S-\d+-\d+(?:-\d+)+$/iu; +const WINDOWS_SID_SCHEMA = z + .string() + .regex(WINDOWS_SID_PATTERN) + .transform((value) => value.toUpperCase()); +const WINDOWS_PRINCIPAL_SCHEMA = z + .string() + .min(1) + .transform((value) => value.toUpperCase()); +const WINDOWS_ACCESS_ENTRY_SCHEMA = z + .object({ + principal: WINDOWS_PRINCIPAL_SCHEMA, + accessType: z.enum(["Allow", "Deny"]), + rightsMask: z.number().int().nonnegative().max(0xffffffff), + inheritanceFlags: z.string(), + propagationFlags: z.string(), + }) + .strict(); +const WINDOWS_PATH_SECURITY_SCHEMA = z + .object({ + currentUserSid: WINDOWS_SID_SCHEMA, + paths: z + .array( + z + .object({ + path: z.string().min(1), + ownerSid: WINDOWS_SID_SCHEMA, + entries: z.array(WINDOWS_ACCESS_ENTRY_SCHEMA).min(1), + }) + .strict(), + ) + .min(1), + }) + .strict(); +const WINDOWS_FILE_RIGHTS = [ + [0x000001, "RD"], + [0x000002, "WD"], + [0x000004, "AD"], + [0x000008, "REA"], + [0x000010, "WEA"], + [0x000020, "X"], + [0x000040, "DC"], + [0x000080, "RA"], + [0x000100, "WA"], + [0x010000, "D"], + [0x020000, "RC"], + [0x040000, "WDAC"], + [0x080000, "WO"], + [0x100000, "S"], + [0x02000000, "MA"], + [0x10000000, "GA"], + [0x20000000, "GE"], + [0x40000000, "GW"], + [0x80000000, "GR"], +] as const; +const WINDOWS_KNOWN_FILE_RIGHTS_MASK = WINDOWS_FILE_RIGHTS.reduce( + (mask, [right]) => mask | right, + 0, +); +const WINDOWS_READ_RIGHTS_MASK = + 0x000001 | 0x000008 | 0x000020 | 0x000080 | 0x020000 | 0x10000000 | 0x20000000 | 0x80000000; +const WINDOWS_WRITE_RIGHTS_MASK = + 0x000002 | + 0x000004 | + 0x000010 | + 0x000040 | + 0x000100 | + 0x010000 | + 0x040000 | + 0x080000 | + 0x10000000 | + 0x40000000; +let macosTrustedAclPrincipalsPromise: Promise> | undefined; + +type WindowsAclEntry = { + readonly principal: string; + readonly rights: string[]; + readonly rawRights: string; + readonly canRead: boolean; + readonly canWrite: boolean; +}; + +export type LocalSqliteSnapshotProviderOptions = { + readonly allowedDatabaseRoles?: readonly SnapshotDatabaseIdentity["role"][]; + readonly repositoryPath: string; + readonly validationRootPath?: string; + readonly now?: () => Date; +}; + +export function createLocalSqliteSnapshotProvider( + options: LocalSqliteSnapshotProviderOptions, +): SqliteSnapshotProvider { + return new LocalSqliteSnapshotProvider(options); +} + +class LocalSqliteSnapshotProvider implements SqliteSnapshotProvider { + readonly #allowedDatabaseRoles: readonly SnapshotDatabaseIdentity["role"][] | undefined; + readonly #repositoryPath: string; + readonly #validationRootPath: string; + readonly #now: () => Date; + + constructor(options: LocalSqliteSnapshotProviderOptions) { + this.#allowedDatabaseRoles = options.allowedDatabaseRoles; + this.#repositoryPath = path.resolve(options.repositoryPath); + this.#validationRootPath = path.resolve( + options.validationRootPath ?? path.dirname(this.#repositoryPath), + ); + this.#now = options.now ?? (() => new Date()); + } + + async create(database: SnapshotDatabaseRef): Promise { + await ensurePrivateDirectory(this.#repositoryPath, "SQLite snapshot repository"); + const repositoryIdentity = await fs.lstat(this.#repositoryPath); + const trustedRepositoryPath = await assertTrustedStagingRoot( + repositoryIdentity, + this.#repositoryPath, + ); + const sourcePath = path.resolve(database.path); + const identity = normalizeSnapshotIdentity(database.identity); + const now = this.#now(); + if (!Number.isFinite(now.getTime())) { + throw new Error("SQLite snapshot timestamp is invalid."); + } + const snapshotId = buildSnapshotId(now); + const snapshotRefPath = path.join(this.#repositoryPath, snapshotId); + const snapshotDir = path.join(trustedRepositoryPath, snapshotId); + const stagingDir = path.join(trustedRepositoryPath, `.tmp-${randomUUID()}`); + const artifactPath = path.join(stagingDir, SNAPSHOT_SQLITE_FILENAME); + await assertDirectoryIdentity(trustedRepositoryPath, repositoryIdentity); + await createPrivateSqliteDirectory(stagingDir); + + let stagingIdentity: Stats | undefined; + let publishedDirectory: FileHandle | undefined; + let publishedIdentity: Stats | undefined; + const publishedEntries = new Map(); + let snapshotDirectoryCreated = false; + try { + await assertDirectoryIdentity(trustedRepositoryPath, repositoryIdentity); + stagingIdentity = await fs.lstat(stagingDir); + applyPrivateModeSync(stagingDir, SNAPSHOT_DIRECTORY_MODE); + await assertPrivateStagingDirectory(stagingIdentity, stagingDir); + await assertDirectoryIdentity(trustedRepositoryPath, repositoryIdentity); + const result = await createVerifiedSqliteSnapshot({ + sourcePath, + targetPath: artifactPath, + transform: identity.role === "global" ? sanitizeGlobalStateSnapshot : undefined, + validate: buildDatabaseValidator(identity), + }); + applyPrivateModeSync(artifactPath, SNAPSHOT_FILE_MODE); + const artifact = await hashSnapshotArtifact(stagingDir); + const manifest: SnapshotManifest = { + schemaVersion: 1, + snapshotId, + createdAt: now.toISOString(), + database: buildDatabaseManifest(identity, sourcePath, result.userVersion), + artifact: { + path: SNAPSHOT_SQLITE_FILENAME, + sha256: artifact.sha256, + sizeBytes: artifact.sizeBytes, + }, + }; + await writeSnapshotManifest(stagingDir, manifest); + applyPrivateModeSync(path.join(stagingDir, SNAPSHOT_MANIFEST_FILENAME), SNAPSHOT_FILE_MODE); + await readSnapshotManifest(stagingDir, snapshotId); + await syncDirectoryBestEffort(stagingDir); + + await assertDirectoryIdentity(trustedRepositoryPath, repositoryIdentity); + try { + await createPrivateSqliteDirectory(snapshotDir); + snapshotDirectoryCreated = true; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "EEXIST") { + throw new Error(`SQLite snapshot directory already exists: ${snapshotDir}`, { + cause: error, + }); + } + throw error; + } + await assertDirectoryIdentity(trustedRepositoryPath, repositoryIdentity); + publishedIdentity = await fs.lstat(snapshotDir); + applyPrivateModeSync(snapshotDir, SNAPSHOT_DIRECTORY_MODE); + await assertPrivateStagingDirectory(publishedIdentity, snapshotDir); + publishedDirectory = await fs.open(snapshotDir, "r"); + await assertOpenDirectoryIdentity(publishedDirectory, snapshotDir, publishedIdentity); + const pendingPath = path.join(snapshotDir, SNAPSHOT_PENDING_FILENAME); + await fs.writeFile(pendingPath, "", { + flag: "wx", + mode: SNAPSHOT_FILE_MODE, + }); + publishedEntries.set(SNAPSHOT_PENDING_FILENAME, await fs.lstat(pendingPath)); + await assertOpenDirectoryIdentity(publishedDirectory, snapshotDir, publishedIdentity); + await publishSnapshotEntryNoOverwrite( + path.join(stagingDir, SNAPSHOT_SQLITE_FILENAME), + path.join(snapshotDir, SNAPSHOT_SQLITE_FILENAME), + SNAPSHOT_SQLITE_FILENAME, + publishedEntries, + ); + await assertOpenDirectoryIdentity(publishedDirectory, snapshotDir, publishedIdentity); + await publishSnapshotEntryNoOverwrite( + path.join(stagingDir, SNAPSHOT_MANIFEST_FILENAME), + path.join(snapshotDir, SNAPSHOT_MANIFEST_FILENAME), + SNAPSHOT_MANIFEST_FILENAME, + publishedEntries, + ); + await assertOpenDirectoryIdentity(publishedDirectory, snapshotDir, publishedIdentity); + await syncDirectoryBestEffort(snapshotDir); + await assertPendingSnapshotContents(snapshotDir); + const publishedManifest = await readSnapshotManifest(snapshotDir, snapshotId); + if (!isDeepStrictEqual(publishedManifest, manifest)) { + throw new Error(`SQLite snapshot manifest changed during publication: ${snapshotDir}`); + } + const publishedArtifact = await hashSnapshotArtifact(snapshotDir); + const publishedArtifactPath = path.join(snapshotDir, SNAPSHOT_SQLITE_FILENAME); + assertArtifactMatchesManifest(publishedArtifactPath, publishedArtifact, publishedManifest); + await verifySnapshotDatabaseFile( + publishedArtifactPath, + publishedArtifact.stat, + publishedManifest, + trustedRepositoryPath, + ); + const expectedPendingIdentity = publishedEntries.get(SNAPSHOT_PENDING_FILENAME); + const currentPendingIdentity = fsSync.lstatSync(pendingPath); + if ( + !expectedPendingIdentity || + !sameFileIdentity(expectedPendingIdentity, currentPendingIdentity) + ) { + throw new Error(`SQLite snapshot pending marker changed: ${pendingPath}`); + } + fsSync.unlinkSync(pendingPath); + publishedEntries.delete(SNAPSHOT_PENDING_FILENAME); + await syncDirectoryBestEffort(snapshotDir); + await publishedDirectory.close(); + publishedDirectory = undefined; + const committedManifest = await readSnapshotManifest(snapshotDir, snapshotId); + if (!isDeepStrictEqual(committedManifest, manifest)) { + throw new Error(`SQLite snapshot manifest changed after commit: ${snapshotDir}`); + } + const committedArtifact = await hashSnapshotArtifact(snapshotDir); + assertArtifactMatchesManifest( + path.join(snapshotDir, SNAPSHOT_SQLITE_FILENAME), + committedArtifact, + committedManifest, + ); + const currentIdentity = await fs.lstat(snapshotDir); + if (!sameFileIdentity(publishedIdentity, currentIdentity)) { + throw new Error(`SQLite snapshot directory changed during publication: ${snapshotDir}`); + } + await assertExactSnapshotContents(snapshotDir); + await assertDirectoryIdentity(trustedRepositoryPath, repositoryIdentity); + await syncDirectoryBestEffort(trustedRepositoryPath); + return { ref: { path: snapshotRefPath }, manifest }; + } catch (error) { + await publishedDirectory?.close().catch(() => undefined); + publishedDirectory = undefined; + if (snapshotDirectoryCreated) { + publishedIdentity ??= await fs.lstat(snapshotDir).catch(() => undefined); + } + if (publishedIdentity) { + const removed = await removePublishedSnapshotDirectoryIfOwned( + snapshotDir, + publishedIdentity, + publishedEntries, + ); + if (removed) { + await syncDirectoryBestEffort(trustedRepositoryPath); + } + } + throw error; + } finally { + const removed = stagingIdentity + ? await removePrivateDirectoryIfOwned( + stagingDir, + stagingIdentity, + SNAPSHOT_ARTIFACT_ENTRIES, + ).catch(() => false) + : await fs + .rmdir(stagingDir) + .then(() => true) + .catch(() => false); + if (removed) { + await syncDirectoryBestEffort(trustedRepositoryPath).catch(() => undefined); + } + } + } + + async verify(snapshot: SnapshotRef): Promise { + const snapshotDir = await this.#resolveSnapshotDirectory(snapshot); + const manifest = await readVerifiedSnapshotManifest(snapshotDir); + assertAllowedDatabaseRole(manifest, this.#allowedDatabaseRoles); + const artifact = await hashSnapshotArtifact(snapshotDir); + const artifactPath = path.join(snapshotDir, SNAPSHOT_SQLITE_FILENAME); + assertArtifactMatchesManifest(artifactPath, artifact, manifest); + await verifySnapshotDatabaseFile( + artifactPath, + artifact.stat, + manifest, + this.#validationRootPath, + ); + await assertExactSnapshotContents(snapshotDir); + return { ok: true, manifest }; + } + + async restoreFresh( + snapshot: SnapshotRef, + targetPath: string, + ): Promise { + const snapshotDir = await this.#resolveSnapshotDirectory(snapshot); + const manifest = await readVerifiedSnapshotManifest(snapshotDir); + assertAllowedDatabaseRole(manifest, this.#allowedDatabaseRoles); + const resolvedTargetPath = path.resolve(targetPath); + await assertFreshRestorePathsAbsent(resolvedTargetPath); + const canonicalRepositoryPath = await fs.realpath(this.#repositoryPath); + const canonicalRestoreParentPath = await canonicalPathFromExistingAncestor( + path.dirname(resolvedTargetPath), + ); + const canonicalTargetPath = path.join( + canonicalRestoreParentPath, + path.basename(resolvedTargetPath), + ); + if (isPathInside(canonicalRepositoryPath, canonicalTargetPath)) { + throw new Error( + `SQLite restore target must be outside snapshot repository ${this.#repositoryPath}: ${resolvedTargetPath}`, + ); + } + const restoreParentPath = path.dirname(canonicalTargetPath); + await ensureRestoreParentDirectory(restoreParentPath); + const trustedRestoreParentPath = await fs.realpath(restoreParentPath); + const trustedTargetPath = path.join( + trustedRestoreParentPath, + path.basename(resolvedTargetPath), + ); + const targetPathChanged = + !isPathInside(canonicalTargetPath, trustedTargetPath) || + !isPathInside(trustedTargetPath, canonicalTargetPath); + if (targetPathChanged) { + throw new Error( + `SQLite restore target changed while creating its parent: ${resolvedTargetPath}`, + ); + } + if (isPathInside(canonicalRepositoryPath, trustedTargetPath)) { + throw new Error( + `SQLite restore target must be outside snapshot repository ${this.#repositoryPath}: ${resolvedTargetPath}`, + ); + } + const restoreParentIdentity = await fs.lstat(trustedRestoreParentPath); + // Existing databases need a crash-recoverable main/WAL/SHM swap protocol. + // This path is deliberately fresh-only and refuses every preexisting sidecar. + await assertFreshRestorePathsAbsent(trustedTargetPath); + + return await withPrivateSqliteStagingDirectory({ + rootPath: trustedRestoreParentPath, + expectedRootIdentity: restoreParentIdentity, + prefix: ".tmp-restore-", + allowedEntries: RESTORE_STAGING_ENTRIES, + operation: async (stagingDir, stagingIdentity) => { + const stagedSourcePath = path.join(stagingDir, SNAPSHOT_SQLITE_FILENAME); + const stagedArtifact = await copySnapshotArtifact(snapshotDir, stagedSourcePath); + await assertDirectoryIdentity(stagingDir, stagingIdentity); + assertArtifactMatchesManifest(stagedSourcePath, stagedArtifact, manifest); + await assertExactSnapshotContents(snapshotDir); + await verifySnapshotDatabaseFile( + stagedSourcePath, + stagedArtifact.stat, + manifest, + trustedRestoreParentPath, + ); + await publishVerifiedSqliteFile({ + sourceIdentity: stagedArtifact.stat, + sourcePath: stagedSourcePath, + targetPath: trustedTargetPath, + expectedContent: manifest.artifact, + requireAtomicPublication: true, + beforePublish: async () => { + await assertDirectoryIdentity(trustedRestoreParentPath, restoreParentIdentity); + await assertFreshRestorePathsAbsent(trustedTargetPath); + }, + afterPublish: (guard) => { + guard.assertTargetMatchesExpectedContent(() => { + assertDirectoryIdentitySync(trustedRestoreParentPath, restoreParentIdentity); + assertNoSqliteSidecarsSync(trustedTargetPath); + }); + }, + }); + return { ok: true, manifest }; + }, + }); + } + + async list(): Promise { + const repositoryStat = await lstatIfExists(this.#repositoryPath); + if (!repositoryStat) { + return []; + } + assertDirectory(repositoryStat, this.#repositoryPath, "SQLite snapshot repository"); + + const entries = await fs.readdir(this.#repositoryPath, { withFileTypes: true }); + const snapshots: SnapshotSummary[] = []; + for (const entry of entries) { + if (entry.name.startsWith(".tmp-")) { + if (entry.isSymbolicLink() || !entry.isDirectory()) { + throw new Error( + `SQLite snapshot repository contains unsafe staging entry: ${path.join(this.#repositoryPath, entry.name)}`, + ); + } + continue; + } + if (entry.isSymbolicLink() || !entry.isDirectory()) { + throw new Error( + `SQLite snapshot repository contains unexpected entry: ${path.join(this.#repositoryPath, entry.name)}`, + ); + } + const snapshotPath = path.join(this.#repositoryPath, entry.name); + if (await isIncompleteSnapshotDirectory(snapshotPath)) { + continue; + } + await assertExactSnapshotContents(snapshotPath); + const manifest = await readSnapshotManifest(snapshotPath); + assertAllowedDatabaseRole(manifest, this.#allowedDatabaseRoles); + snapshots.push({ + ref: { path: snapshotPath }, + manifest, + }); + } + return snapshots.toSorted( + (left, right) => + right.manifest.createdAt.localeCompare(left.manifest.createdAt) || + right.manifest.snapshotId.localeCompare(left.manifest.snapshotId), + ); + } + + async #resolveSnapshotDirectory(snapshot: SnapshotRef): Promise { + const snapshotDir = path.resolve(snapshot.path); + if (path.dirname(snapshotDir) !== this.#repositoryPath) { + throw new Error( + `SQLite snapshot must be an immediate child of repository ${this.#repositoryPath}: ${snapshotDir}`, + ); + } + const repositoryStat = await fs.lstat(this.#repositoryPath); + assertDirectory(repositoryStat, this.#repositoryPath, "SQLite snapshot repository"); + const snapshotStat = await fs.lstat(snapshotDir); + assertDirectory(snapshotStat, snapshotDir, "SQLite snapshot"); + return snapshotDir; + } +} + +async function readVerifiedSnapshotManifest(snapshotDir: string): Promise { + await assertExactSnapshotContents(snapshotDir); + return await readSnapshotManifest(snapshotDir); +} + +function assertArtifactMatchesManifest( + artifactPath: string, + artifact: SnapshotArtifactDigest, + manifest: SnapshotManifest, +): void { + if (artifact.sizeBytes !== manifest.artifact.sizeBytes) { + throw new Error( + `Snapshot artifact size mismatch for ${artifactPath}: expected ${manifest.artifact.sizeBytes}, got ${artifact.sizeBytes}`, + ); + } + if (artifact.sha256 !== manifest.artifact.sha256) { + throw new Error( + `Snapshot artifact hash mismatch for ${artifactPath}: expected ${manifest.artifact.sha256}, got ${artifact.sha256}`, + ); + } +} + +function assertAllowedDatabaseRole( + manifest: SnapshotManifest, + allowedRoles: readonly SnapshotDatabaseIdentity["role"][] | undefined, +): void { + if (!allowedRoles || allowedRoles.includes(manifest.database.role)) { + return; + } + throw new Error( + `SQLite snapshot database role ${manifest.database.role} is not allowed for this operation.`, + ); +} + +async function verifySnapshotDatabaseFile( + artifactPath: string, + expectedIdentity: Stats, + manifest: SnapshotManifest, + validationRootPath: string, +): Promise { + const beforeOpen = await fs.lstat(artifactPath); + if ( + beforeOpen.isSymbolicLink() || + !beforeOpen.isFile() || + beforeOpen.nlink > 1 || + !sameFileIdentity(expectedIdentity, beforeOpen) + ) { + throw new Error(`Snapshot artifact changed before SQLite verification: ${artifactPath}`); + } + + const validationRootIdentity = await fs.lstat(validationRootPath); + assertDirectory(validationRootIdentity, validationRootPath, "SQLite validation root"); + await withPrivateSqliteStagingDirectory({ + rootPath: validationRootPath, + expectedRootIdentity: validationRootIdentity, + prefix: ".tmp-verify-", + allowedEntries: VALIDATION_STAGING_ENTRIES, + operation: async (validationDir) => { + const validationPath = path.join(validationDir, SNAPSHOT_SQLITE_FILENAME); + const validationArtifact = await copySnapshotArtifact( + path.dirname(artifactPath), + validationPath, + ); + assertArtifactMatchesManifest(validationPath, validationArtifact, manifest); + const sqlite = requireNodeSqlite(); + const database = new sqlite.DatabaseSync(validationPath, { + allowExtension: true, + readOnly: true, + }); + try { + database.exec("PRAGMA busy_timeout = 30000; PRAGMA trusted_schema = OFF;"); + await loadSqliteVecExtension({ db: database }); + assertSqliteIntegrity(database, artifactPath); + buildManifestDatabaseValidator(manifest.database)(database, artifactPath); + } finally { + database.close(); + } + const validatedArtifact = await hashSnapshotArtifact(validationDir); + if (!sameFileIdentity(validationArtifact.stat, validatedArtifact.stat)) { + throw new Error(`Snapshot validation copy changed: ${validationPath}`); + } + assertArtifactMatchesManifest(validationPath, validatedArtifact, manifest); + }, + }); + const afterOpen = await fs.lstat(artifactPath); + if ( + afterOpen.isSymbolicLink() || + !afterOpen.isFile() || + afterOpen.nlink > 1 || + !sameFileIdentity(expectedIdentity, afterOpen) + ) { + throw new Error(`Snapshot artifact changed during SQLite verification: ${artifactPath}`); + } + const verifiedArtifact = await hashSnapshotArtifact(path.dirname(artifactPath)); + if (!sameFileIdentity(expectedIdentity, verifiedArtifact.stat)) { + throw new Error(`Snapshot artifact changed after SQLite verification: ${artifactPath}`); + } + assertArtifactMatchesManifest(artifactPath, verifiedArtifact, manifest); +} + +function normalizeSnapshotIdentity(identity: SnapshotDatabaseIdentity): SnapshotDatabaseIdentity { + if (identity.role === "global") { + return identity; + } + if (identity.role === "agent") { + const agentId = normalizeAgentId(identity.agentId); + if (!isValidAgentId(identity.agentId) || agentId !== identity.agentId) { + throw new Error(`SQLite snapshot agent id must be canonical: ${identity.agentId}`); + } + return { role: "agent", agentId }; + } + const id = identity.id.trim(); + if (!id || id !== identity.id || id.length > 256 || containsAsciiControlCharacter(id)) { + throw new Error("SQLite snapshot generic database id is invalid."); + } + return { role: "generic", id }; +} + +function buildDatabaseManifest( + identity: SnapshotDatabaseIdentity, + sourcePath: string, + userVersion: number, +): SnapshotDatabaseManifest { + const basename = path.basename(sourcePath); + if (identity.role === "global") { + return { role: "global", basename, userVersion }; + } + if (identity.role === "agent") { + return { role: "agent", agentId: identity.agentId, basename, userVersion }; + } + return { role: "generic", id: identity.id, basename, userVersion }; +} + +function buildDatabaseValidator( + identity: SnapshotDatabaseIdentity | SnapshotDatabaseManifest, +): SqliteSnapshotValidator { + if (identity.role === "global") { + return (database, pathname) => + assertOpenClawStateDatabaseForMaintenance(database, { pathname }); + } + if (identity.role === "agent") { + return (database, pathname) => + assertOpenClawAgentDatabaseForMaintenance(database, { + agentId: identity.agentId, + pathname, + }); + } + return () => undefined; +} + +function buildManifestDatabaseValidator( + manifest: SnapshotDatabaseManifest, +): SqliteSnapshotValidator { + const validateOwner = buildDatabaseValidator(manifest); + return (database, pathname) => { + validateOwner(database, pathname); + const userVersion = readSqliteUserVersion(database); + if (userVersion !== manifest.userVersion) { + throw new Error( + `Snapshot database user_version mismatch for ${pathname}: expected ${manifest.userVersion}, got ${userVersion}`, + ); + } + }; +} + +function sanitizeGlobalStateSnapshot(database: DatabaseSync): void { + database.prepare("DELETE FROM delivery_queue_entries").run(); +} + +function buildSnapshotId(now: Date): string { + const timestamp = now.toISOString().replaceAll(/[:.]/g, "-"); + return `${timestamp}-${randomUUID()}`; +} + +async function ensurePrivateDirectory(directoryPath: string, scopeLabel: string): Promise { + if (process.platform === "win32") { + const parentResult = await ensureAbsoluteDirectory(path.dirname(directoryPath), { + mode: SNAPSHOT_DIRECTORY_MODE, + scopeLabel, + }); + if (!parentResult.ok) { + throw parentResult.error; + } + try { + await createPrivateSqliteDirectory(directoryPath); + return; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EEXIST") { + throw error; + } + } + } + const result = await ensureAbsoluteDirectory(directoryPath, { + mode: SNAPSHOT_DIRECTORY_MODE, + scopeLabel, + }); + if (!result.ok) { + throw result.error; + } + applyPrivateModeSync(result.path, SNAPSHOT_DIRECTORY_MODE); +} + +async function ensureRestoreParentDirectory(directoryPath: string): Promise { + const result = await ensureAbsoluteDirectory(directoryPath, { + mode: SNAPSHOT_DIRECTORY_MODE, + scopeLabel: "SQLite restore target", + }); + if (!result.ok) { + throw result.error; + } +} + +function assertDirectory(stat: Stats, pathname: string, label: string): void { + if (stat.isSymbolicLink() || !stat.isDirectory()) { + throw new Error(`${label} must be a real directory: ${pathname}`); + } +} + +async function assertDirectoryIdentity( + directoryPath: string, + expectedIdentity: Stats, +): Promise { + const currentIdentity = await fs.lstat(directoryPath); + assertDirectory(currentIdentity, directoryPath, "SQLite staging directory"); + if (!sameFileIdentity(currentIdentity, expectedIdentity)) { + throw new Error(`SQLite staging directory changed during operation: ${directoryPath}`); + } +} + +async function assertOpenDirectoryIdentity( + handle: FileHandle, + directoryPath: string, + expectedIdentity: Stats, +): Promise { + const openedIdentity = await handle.stat(); + const currentIdentity = await fs.lstat(directoryPath); + assertDirectory(openedIdentity, directoryPath, "SQLite snapshot directory"); + assertDirectory(currentIdentity, directoryPath, "SQLite snapshot directory"); + if ( + !sameFileIdentity(openedIdentity, expectedIdentity) || + !sameFileIdentity(currentIdentity, expectedIdentity) + ) { + throw new Error(`SQLite snapshot directory changed during publication: ${directoryPath}`); + } +} + +function assertDirectoryIdentitySync(directoryPath: string, expectedIdentity: Stats): void { + const currentIdentity = fsSync.lstatSync(directoryPath); + assertDirectory(currentIdentity, directoryPath, "SQLite staging directory"); + if (!sameFileIdentity(currentIdentity, expectedIdentity)) { + throw new Error(`SQLite staging directory changed during operation: ${directoryPath}`); + } +} + +function isSnapshotEntryLinkFallbackError(error: unknown): boolean { + const code = (error as NodeJS.ErrnoException).code; + return ( + code === "EPERM" || + code === "EXDEV" || + code === "ENOTSUP" || + code === "EOPNOTSUPP" || + code === "ENOSYS" + ); +} + +async function publishSnapshotEntryNoOverwrite( + sourcePath: string, + targetPath: string, + entryName: string, + publishedEntries: Map, +): Promise { + let linked = false; + let linkedSourceIdentity: Stats | undefined; + try { + linkedSourceIdentity = await fs.lstat(sourcePath); + await fs.link(sourcePath, targetPath); + publishedEntries.set(entryName, linkedSourceIdentity); + linked = true; + } catch (error) { + if (!isSnapshotEntryLinkFallbackError(error)) { + throw error; + } + const copiedIdentity = await copySnapshotEntryExclusive(sourcePath, targetPath); + publishedEntries.set(entryName, copiedIdentity); + } + const expectedTargetIdentity = publishedEntries.get(entryName); + const initialTargetIdentity = await fs.lstat(targetPath); + if (!expectedTargetIdentity || !sameFileIdentity(expectedTargetIdentity, initialTargetIdentity)) { + throw new Error(`SQLite snapshot entry changed during publication: ${targetPath}`); + } + if (linked) { + if (!linkedSourceIdentity || !sameFileIdentity(linkedSourceIdentity, initialTargetIdentity)) { + throw new Error(`SQLite snapshot entry changed during publication: ${targetPath}`); + } + const sourceIdentity = await fs.lstat(sourcePath); + if (!sameFileIdentity(sourceIdentity, initialTargetIdentity)) { + throw new Error(`SQLite snapshot entry changed during publication: ${targetPath}`); + } + } + await fs.unlink(sourcePath); + const finalTargetIdentity = await fs.lstat(targetPath); + if (!sameFileIdentity(initialTargetIdentity, finalTargetIdentity)) { + throw new Error(`SQLite snapshot entry changed after publication: ${targetPath}`); + } + publishedEntries.set(entryName, finalTargetIdentity); +} + +async function copySnapshotEntryExclusive(sourcePath: string, targetPath: string): Promise { + const source = await fs.open(sourcePath, "r"); + let target: FileHandle | undefined; + let targetIdentity: Stats | undefined; + try { + target = await fs.open(targetPath, "wx+", SNAPSHOT_FILE_MODE); + targetIdentity = await target.stat(); + const buffer = Buffer.allocUnsafe(1024 * 1024); + let offset = 0; + while (true) { + const { bytesRead } = await source.read(buffer, 0, buffer.length, offset); + if (bytesRead === 0) { + break; + } + let bytesWritten = 0; + while (bytesWritten < bytesRead) { + const result = await target.write( + buffer, + bytesWritten, + bytesRead - bytesWritten, + offset + bytesWritten, + ); + if (result.bytesWritten === 0) { + throw new Error(`SQLite snapshot entry copy made no progress: ${targetPath}`); + } + bytesWritten += result.bytesWritten; + } + offset += bytesRead; + } + await target.sync(); + const finalIdentity = await target.stat(); + const currentIdentity = await fs.lstat(targetPath); + if ( + !sameFileIdentity(targetIdentity, finalIdentity) || + !sameFileIdentity(targetIdentity, currentIdentity) + ) { + throw new Error(`SQLite snapshot entry changed during copy: ${targetPath}`); + } + return finalIdentity; + } catch (error) { + if (targetIdentity) { + const currentIdentity = await fs.lstat(targetPath).catch(() => undefined); + if (currentIdentity && sameFileIdentity(currentIdentity, targetIdentity)) { + await fs.unlink(targetPath).catch(() => undefined); + } + } + throw error; + } finally { + await target?.close().catch(() => undefined); + await source.close().catch(() => undefined); + } +} + +async function assertExactSnapshotContents(snapshotDir: string): Promise { + await assertSnapshotContents( + snapshotDir, + new Set([SNAPSHOT_MANIFEST_FILENAME, SNAPSHOT_SQLITE_FILENAME]), + ); +} + +async function assertPendingSnapshotContents(snapshotDir: string): Promise { + await assertSnapshotContents( + snapshotDir, + new Set([SNAPSHOT_MANIFEST_FILENAME, SNAPSHOT_PENDING_FILENAME, SNAPSHOT_SQLITE_FILENAME]), + ); +} + +async function assertSnapshotContents(snapshotDir: string, expected: Set): Promise { + const entries = await fs.readdir(snapshotDir, { withFileTypes: true }); + for (const entry of entries) { + if (!expected.delete(entry.name)) { + throw new Error( + `SQLite snapshot contains unexpected entry: ${path.join(snapshotDir, entry.name)}`, + ); + } + if (entry.isSymbolicLink() || !entry.isFile()) { + throw new Error( + `SQLite snapshot entry must be a regular file: ${path.join(snapshotDir, entry.name)}`, + ); + } + const stat = await fs.lstat(path.join(snapshotDir, entry.name)); + if (stat.nlink > 1) { + throw new Error( + `SQLite snapshot entry must not be hardlinked: ${path.join(snapshotDir, entry.name)}`, + ); + } + } + if (expected.size > 0) { + throw new Error(`SQLite snapshot is missing ${[...expected].join(", ")}: ${snapshotDir}`); + } +} + +async function isIncompleteSnapshotDirectory(snapshotDir: string): Promise { + const entries = await fs.readdir(snapshotDir, { withFileTypes: true }); + const names = new Set(entries.map((entry) => entry.name)); + if (names.has(SNAPSHOT_PENDING_FILENAME)) { + return true; + } + if (names.has(SNAPSHOT_MANIFEST_FILENAME)) { + return false; + } + return entries.length === 0; +} + +async function assertFreshRestorePathsAbsent(databasePath: string): Promise { + for (const candidate of [ + databasePath, + ...SQLITE_SIDECAR_SUFFIXES.map((suffix) => `${databasePath}${suffix}`), + ]) { + if (await lstatIfExists(candidate)) { + throw new Error(`Fresh SQLite restore path already exists: ${candidate}`); + } + } +} + +function assertNoSqliteSidecarsSync(databasePath: string): void { + for (const suffix of SQLITE_SIDECAR_SUFFIXES) { + const sidecarPath = `${databasePath}${suffix}`; + try { + fsSync.lstatSync(sidecarPath); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + continue; + } + throw error; + } + throw new Error(`Restored SQLite database has unexpected sidecar: ${sidecarPath}`); + } +} + +async function lstatIfExists(pathname: string): Promise { + try { + return await fs.lstat(pathname); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + return undefined; + } + throw error; + } +} + +async function removePrivateDirectoryIfOwned( + directoryPath: string, + expectedIdentity: Stats, + allowedEntries: ReadonlySet, +): Promise { + const currentIdentity = await lstatIfExists(directoryPath); + if (!currentIdentity) { + return false; + } + if ( + currentIdentity.isSymbolicLink() || + !currentIdentity.isDirectory() || + !sameFileIdentity(currentIdentity, expectedIdentity) + ) { + throw new Error(`Private SQLite staging directory changed before cleanup: ${directoryPath}`); + } + const entries = await fs.readdir(directoryPath, { withFileTypes: true }); + const verifiedPaths: string[] = []; + for (const entry of entries) { + const entryPath = path.join(directoryPath, entry.name); + if (!allowedEntries.has(entry.name) || entry.isSymbolicLink() || !entry.isFile()) { + throw new Error(`Private SQLite staging directory has unexpected entry: ${entryPath}`); + } + const stat = await fs.lstat(entryPath); + if (stat.nlink > 1) { + throw new Error(`Private SQLite staging file must not be hardlinked: ${entryPath}`); + } + verifiedPaths.push(entryPath); + } + await Promise.all(verifiedPaths.map(async (entryPath) => await fs.unlink(entryPath))); + await fs.rmdir(directoryPath); + return true; +} + +async function withPrivateSqliteStagingDirectory(options: { + rootPath: string; + expectedRootIdentity: Stats; + prefix: string; + allowedEntries: ReadonlySet; + operation: (directoryPath: string, directoryIdentity: Stats) => Promise; +}): Promise { + const trustedRootPath = await assertTrustedStagingRoot( + options.expectedRootIdentity, + options.rootPath, + ); + await assertDirectoryIdentity(trustedRootPath, options.expectedRootIdentity); + const directoryPath = await createPrivateSqliteTempDirectory(trustedRootPath, options.prefix); + const directoryIdentity = await fs.lstat(directoryPath); + + let outcome: { ok: true; value: T } | { ok: false; error: unknown }; + try { + applyPrivateModeSync(directoryPath, SNAPSHOT_DIRECTORY_MODE); + await assertPrivateStagingDirectory(directoryIdentity, directoryPath); + await assertDirectoryIdentity(trustedRootPath, options.expectedRootIdentity); + outcome = { + ok: true, + value: await options.operation(directoryPath, directoryIdentity), + }; + } catch (error) { + outcome = { ok: false, error }; + } + + let cleanupOutcome: { ok: true } | { ok: false; error: unknown }; + try { + const removed = await removePrivateDirectoryIfOwned( + directoryPath, + directoryIdentity, + options.allowedEntries, + ); + if (!removed) { + throw new Error(`Private SQLite staging directory disappeared: ${directoryPath}`); + } + cleanupOutcome = { ok: true }; + } catch (error) { + cleanupOutcome = { ok: false, error }; + } + + if (!cleanupOutcome.ok) { + if (!outcome.ok) { + throw new AggregateError( + [outcome.error, cleanupOutcome.error], + `SQLite staging operation and cleanup both failed: ${directoryPath}`, + ); + } + throw new Error(`Failed to clean private SQLite staging directory: ${directoryPath}`, { + cause: cleanupOutcome.error, + }); + } + await syncDirectoryBestEffort(trustedRootPath).catch(() => undefined); + if (!outcome.ok) { + throw outcome.error; + } + return outcome.value; +} + +async function assertTrustedStagingRoot( + expectedIdentity: Stats, + rootPath: string, +): Promise { + const resolvedRootPath = path.resolve(rootPath); + const trustedRootPath = await fs.realpath(resolvedRootPath); + const rootIdentity = await fs.lstat(trustedRootPath); + assertDirectory(rootIdentity, trustedRootPath, "Private SQLite staging root"); + if (!sameFileIdentity(rootIdentity, expectedIdentity)) { + throw new Error(`Private SQLite staging root changed during operation: ${resolvedRootPath}`); + } + if (process.platform === "win32") { + await assertTrustedWindowsStagingPath(trustedRootPath); + return trustedRootPath; + } + const uid = typeof process.getuid === "function" ? process.getuid() : undefined; + if (uid === undefined || rootIdentity.uid !== uid || (rootIdentity.mode & 0o022) !== 0) { + throw new Error( + `Private SQLite staging root must be owned by the current user and not writable by other users: ${resolvedRootPath}`, + ); + } + if (process.platform === "darwin") { + await assertTrustedMacosAcl(trustedRootPath, true); + } + await assertTrustedPosixStagingAncestors(trustedRootPath, rootIdentity, uid); + return trustedRootPath; +} + +async function assertPrivateStagingDirectory( + expectedIdentity: Stats, + directoryPath: string, +): Promise { + const currentIdentity = await fs.lstat(directoryPath); + assertDirectory(currentIdentity, directoryPath, "Private SQLite staging directory"); + if (!sameFileIdentity(currentIdentity, expectedIdentity)) { + throw new Error(`Private SQLite staging directory changed during operation: ${directoryPath}`); + } + if (process.platform === "win32") { + // The parent root was already checked for private and inherit-only ACEs. + // An untrusted principal cannot alter or replace children beneath that root. + return; + } + const uid = typeof process.getuid === "function" ? process.getuid() : undefined; + if (uid === undefined || currentIdentity.uid !== uid || (currentIdentity.mode & 0o077) !== 0) { + throw new Error(`Private SQLite staging directory permissions are unsafe: ${directoryPath}`); + } + if (process.platform === "darwin") { + await assertTrustedMacosAcl(directoryPath, true); + } +} + +async function assertTrustedPosixStagingAncestors( + rootPath: string, + rootIdentity: Stats, + uid: number, +): Promise { + // A private root is still replaceable when one of its ancestors is writable + // by another user. Sticky directories are safe only for user-owned children. + let childIdentity = rootIdentity; + let currentPath = path.dirname(rootPath); + while (currentPath !== rootPath) { + const currentIdentity = await fs.lstat(currentPath); + assertDirectory(currentIdentity, currentPath, "SQLite staging ancestor"); + const writableByOtherUsers = (currentIdentity.mode & 0o022) !== 0; + const ownerCanReplaceChild = currentIdentity.uid !== uid && currentIdentity.uid !== 0; + const stickyOwnerIsTrusted = currentIdentity.uid === uid || currentIdentity.uid === 0; + const stickyProtectsChild = + (currentIdentity.mode & 0o1000) !== 0 && stickyOwnerIsTrusted && childIdentity.uid === uid; + if (ownerCanReplaceChild || (writableByOtherUsers && !stickyProtectsChild)) { + throw new Error( + `SQLite staging ancestor must not allow another user to replace its child: ${currentPath}`, + ); + } + if (process.platform === "darwin") { + await assertTrustedMacosAcl(currentPath, false); + } + const parentPath = path.dirname(currentPath); + if (parentPath === currentPath) { + return; + } + childIdentity = currentIdentity; + currentPath = parentPath; + } +} + +type MacosAclEntry = { + effect: "allow" | "deny"; + permissions: ReadonlySet; + principal: string; +}; + +function parseMacosAclEntries(output: string, pathname: string): MacosAclEntry[] { + const lines = output.split(/\r?\n/u); + const header = lines.shift(); + if (!header) { + throw new Error(`Unable to inspect macOS ACL for SQLite staging: ${pathname}`); + } + const entries: MacosAclEntry[] = []; + for (const line of lines) { + if (!/^\s*\d+:\s/u.test(line)) { + continue; + } + const match = line.match(/^\s*\d+:\s+(.+?)\s+(?:inherited\s+)?(allow|deny)\s+([a-z_,]+)\s*$/u); + if (!match) { + throw new Error(`Unable to parse macOS ACL for SQLite staging: ${pathname}`); + } + const [, principal, effect, permissions] = match; + if (!principal || !permissions || (effect !== "allow" && effect !== "deny")) { + throw new Error(`Unable to parse macOS ACL for SQLite staging: ${pathname}`); + } + entries.push({ + principal: normalizeAclPrincipal(principal), + effect, + permissions: new Set(permissions.split(",")), + }); + } + if (/^[^\s]{10}\+/u.test(header) && entries.length === 0) { + throw new Error(`Unable to parse macOS ACL for SQLite staging: ${pathname}`); + } + return entries; +} + +function normalizeAclPrincipal(principal: string): string { + return principal.trim().toLowerCase(); +} + +async function resolveTrustedMacosAclPrincipals(): Promise> { + macosTrustedAclPrincipalsPromise ??= (async () => { + const dsmemberutil = resolveSystemBin("dsmemberutil"); + if (!dsmemberutil) { + throw new Error("Unable to resolve dsmemberutil for macOS ACL verification."); + } + const currentUsername = os.userInfo().username; + const usernames = new Set([currentUsername, "root"]); + const trusted = new Set(); + for (const username of usernames) { + const { stdout } = await runExec(dsmemberutil, ["getuuid", "-U", username], { + timeoutMs: 5_000, + maxBuffer: 64 * 1024, + }); + const uuid = stdout.trim(); + if (!/^[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}$/iu.test(uuid)) { + throw new Error(`Unable to resolve trusted macOS ACL principal for ${username}.`); + } + trusted.add(normalizeAclPrincipal(uuid)); + trusted.add(normalizeAclPrincipal(username)); + trusted.add(normalizeAclPrincipal(`user:${username}`)); + } + return trusted; + })(); + return await macosTrustedAclPrincipalsPromise; +} + +async function assertTrustedMacosAcl(pathname: string, requirePrivate: boolean): Promise { + const ls = resolveSystemBin("ls"); + if (!ls) { + throw new Error(`Unable to verify macOS ACL for SQLite staging: ${pathname}`); + } + let entries: MacosAclEntry[]; + try { + const [result, trustedPrincipals] = await Promise.all([ + runExec(ls, ["-lden", "--", pathname], { + timeoutMs: 5_000, + maxBuffer: 1024 * 1024, + }), + resolveTrustedMacosAclPrincipals(), + ]); + entries = parseMacosAclEntries(result.stdout, pathname).filter( + (entry) => !trustedPrincipals.has(entry.principal), + ); + } catch (error) { + throw new Error(`Unable to verify macOS ACL for SQLite staging: ${pathname}`, { + cause: error, + }); + } + const unsafeEntry = entries.find( + (entry) => + entry.effect === "allow" && + (requirePrivate || + [...entry.permissions].some((permission) => + MACOS_REPLACEMENT_ACL_PERMISSIONS.has(permission), + )), + ); + if (unsafeEntry) { + throw new Error(`macOS ACL permits untrusted SQLite staging access: ${pathname}`); + } +} + +async function assertTrustedWindowsStagingPath(rootPath: string): Promise { + const paths = [rootPath]; + let currentPath = path.dirname(rootPath); + while (currentPath !== rootPath) { + paths.push(currentPath); + const parentPath = path.dirname(currentPath); + if (parentPath === currentPath) { + break; + } + currentPath = parentPath; + } + let security: z.infer; + try { + security = await inspectWindowsPathSecurity(paths); + } catch { + throw new Error(`Unable to verify private Windows ACL for SQLite staging: ${rootPath}`); + } + if (security.paths.length !== paths.length) { + throw new Error(`Unable to verify private Windows ACL for SQLite staging: ${rootPath}`); + } + for (const [index, pathname] of paths.entries()) { + const pathSecurity = security.paths[index]; + if (!pathSecurity || path.resolve(pathSecurity.path) !== path.resolve(pathname)) { + throw new Error(`Unable to verify private Windows ACL for SQLite staging: ${pathname}`); + } + assertTrustedWindowsAcl(pathname, index === 0, security.currentUserSid, pathSecurity); + } +} + +function assertTrustedWindowsAcl( + pathname: string, + requirePrivate: boolean, + currentUserSid: string, + security: z.infer["paths"][number], +): void { + if (security.ownerSid !== currentUserSid && !WINDOWS_TRUSTED_OWNER_SIDS.has(security.ownerSid)) { + throw new Error(`Windows staging path is owned by an untrusted principal: ${pathname}`); + } + const allowedEntries = security.entries.filter((entry) => entry.accessType === "Allow"); + if (allowedEntries.length === 0) { + throw new Error(`Unable to verify private Windows ACL for SQLite staging: ${pathname}`); + } + const unsafeEntries = allowedEntries + .filter( + (entry) => + entry.principal !== currentUserSid && !WINDOWS_TRUSTED_ACCESS_SIDS.has(entry.principal), + ) + .map(windowsSecurityEntryToAclEntry) + .filter((entry) => windowsAclEntryPermitsUnsafeStagingAccess(entry, requirePrivate)); + if (unsafeEntries.length > 0) { + throw new Error(`Windows ACL permits untrusted SQLite staging access: ${pathname}`); + } +} + +function windowsSecurityEntryToAclEntry( + entry: z.infer, +): WindowsAclEntry { + const rights: string[] = WINDOWS_FILE_RIGHTS.filter( + ([right]) => (entry.rightsMask & right) !== 0, + ).map(([, name]) => name); + if ((entry.rightsMask & ~WINDOWS_KNOWN_FILE_RIGHTS_MASK) !== 0) { + rights.push("UNKNOWN"); + } + const inheritanceFlags = new Set(entry.inheritanceFlags.split(",").map((flag) => flag.trim())); + const propagationFlags = new Set(entry.propagationFlags.split(",").map((flag) => flag.trim())); + const rawFlags = [ + inheritanceFlags.has("ObjectInherit") ? "(OI)" : "", + inheritanceFlags.has("ContainerInherit") ? "(CI)" : "", + propagationFlags.has("NoPropagateInherit") ? "(NP)" : "", + propagationFlags.has("InheritOnly") ? "(IO)" : "", + ].join(""); + return { + principal: entry.principal, + rights, + rawRights: `${rawFlags}(${rights.join(",")})`, + canRead: (entry.rightsMask & WINDOWS_READ_RIGHTS_MASK) !== 0, + canWrite: (entry.rightsMask & WINDOWS_WRITE_RIGHTS_MASK) !== 0, + }; +} + +function windowsAclEntryPermitsUnsafeStagingAccess( + entry: WindowsAclEntry, + requirePrivate: boolean, +): boolean { + // Inherit-only ACEs on ordinary ancestors are covered when the protected + // root is inspected. Private roots must also reject rights inherited by files. + if (!requirePrivate && /\(IO\)/iu.test(entry.rawRights)) { + return false; + } + const rights = entry.rights.map((right) => right.toUpperCase()); + const unsafeRights = requirePrivate + ? WINDOWS_STAGING_ACCESS_RIGHTS + : WINDOWS_STAGING_REPLACEMENT_RIGHTS; + return ( + (requirePrivate && (entry.canWrite || entry.canRead)) || + rights.some((right) => unsafeRights.has(right)) + ); +} + +async function inspectWindowsPathSecurity( + pathnames: readonly string[], +): Promise> { + const encodedPaths = Buffer.from(JSON.stringify(pathnames), "utf8").toString("base64"); + const command = [ + "$ErrorActionPreference = 'Stop'", + `$paths = ConvertFrom-Json ([Text.Encoding]::UTF8.GetString([Convert]::FromBase64String('${encodedPaths}')))`, + "$pathSecurity = @($paths | ForEach-Object { $path = [string]$_; $acl = Get-Acl -LiteralPath $path; $entries = @($acl.Access | ForEach-Object { $identity = $_.IdentityReference; try { $principal = $identity.Translate([System.Security.Principal.SecurityIdentifier]).Value } catch { $principal = [string]$identity.Value }; $rightsMask = ([int64][int32]$_.FileSystemRights) -band 0xffffffffL; [pscustomobject]@{ principal = $principal; accessType = [string]$_.AccessControlType; rightsMask = $rightsMask; inheritanceFlags = [string]$_.InheritanceFlags; propagationFlags = [string]$_.PropagationFlags } }); [pscustomobject]@{ path = $path; ownerSid = $acl.GetOwner([System.Security.Principal.SecurityIdentifier]).Value; entries = $entries } })", + "$payload = [pscustomobject]@{ currentUserSid = [System.Security.Principal.WindowsIdentity]::GetCurrent().User.Value; paths = $pathSecurity }", + "$json = ConvertTo-Json -InputObject $payload -Compress -Depth 4", + "[Console]::Out.Write([Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes($json)))", + ].join("; "); + const stdout = await runEncodedWindowsPowerShell(command, WINDOWS_ACL_METADATA_MAX_BUFFER); + let parsed: unknown; + try { + parsed = JSON.parse(Buffer.from(stdout.trim(), "base64").toString("utf8")); + } catch (error) { + throw new Error("Unable to parse Windows ACL metadata.", { cause: error }); + } + const result = WINDOWS_PATH_SECURITY_SCHEMA.safeParse(parsed); + if (!result.success) { + throw new Error("Invalid Windows ACL metadata.", { cause: result.error }); + } + return result.data; +} + +async function runEncodedWindowsPowerShell(command: string, maxBuffer: number): Promise { + const powershell = resolveSystemBin("powershell"); + if (!powershell) { + throw new Error("Unable to resolve PowerShell for Windows SQLite path security."); + } + const encodedCommand = Buffer.from(command, "utf16le").toString("base64"); + const { stdout } = await runExec( + powershell, + ["-NoLogo", "-NoProfile", "-NonInteractive", "-EncodedCommand", encodedCommand], + { + timeoutMs: 10_000, + maxBuffer, + }, + ); + return stdout; +} + +async function removePublishedSnapshotDirectoryIfOwned( + directoryPath: string, + expectedIdentity: Stats, + publishedEntries: ReadonlyMap, +): Promise { + const currentIdentity = await lstatIfExists(directoryPath); + if ( + !currentIdentity || + currentIdentity.isSymbolicLink() || + !currentIdentity.isDirectory() || + !sameFileIdentity(currentIdentity, expectedIdentity) + ) { + return false; + } + const entries = await fs.readdir(directoryPath, { withFileTypes: true }); + for (const entry of entries) { + const expectedEntryIdentity = publishedEntries.get(entry.name); + if (!expectedEntryIdentity || entry.isSymbolicLink() || !entry.isFile()) { + continue; + } + const entryPath = path.join(directoryPath, entry.name); + const currentEntryIdentity = await fs.lstat(entryPath); + if (sameFileIdentity(currentEntryIdentity, expectedEntryIdentity)) { + await fs.unlink(entryPath); + } + } + if ((await fs.readdir(directoryPath)).length > 0) { + return false; + } + await fs.rmdir(directoryPath); + return true; +} diff --git a/src/snapshot/manifest.ts b/src/snapshot/manifest.ts new file mode 100644 index 000000000000..25d485fff89d --- /dev/null +++ b/src/snapshot/manifest.ts @@ -0,0 +1,316 @@ +import { createHash } from "node:crypto"; +import type { BigIntStats, Stats } from "node:fs"; +import fs from "node:fs/promises"; +import path from "node:path"; +import { sameFileIdentity } from "../infra/fs-safe-advanced.js"; +import { root } from "../infra/fs-safe.js"; +import { isValidAgentId, normalizeAgentId } from "../routing/session-key.js"; +import { + SNAPSHOT_MANIFEST_FILENAME, + SNAPSHOT_SQLITE_FILENAME, + type SnapshotDatabaseManifest, + type SnapshotManifest, +} from "./snapshot-provider.js"; + +const MAX_MANIFEST_BYTES = 1024 * 1024; +const MAX_SQLITE_USER_VERSION = 2_147_483_647; +const MIN_SQLITE_USER_VERSION = -2_147_483_648; +const SNAPSHOT_ID_PATTERN = /^[a-zA-Z0-9][a-zA-Z0-9._-]{0,254}$/; +const SHA256_PATTERN = /^[a-f0-9]{64}$/; + +export type SnapshotArtifactDigest = { + sha256: string; + sizeBytes: number; + stat: Stats; +}; + +type OpenFileHandle = Awaited>; + +export function containsAsciiControlCharacter(value: string): boolean { + for (let index = 0; index < value.length; index += 1) { + const code = value.charCodeAt(index); + if (code <= 0x1f || code === 0x7f) { + return true; + } + } + return false; +} + +export async function hashSnapshotArtifact(snapshotDir: string): Promise { + const snapshotRoot = await root(snapshotDir); + const opened = await snapshotRoot.open(SNAPSHOT_SQLITE_FILENAME, { + hardlinks: "reject", + symlinks: "reject", + }); + try { + return { ...(await hashFileHandle(opened.handle)), stat: opened.stat }; + } finally { + await opened.handle.close(); + } +} + +export async function copySnapshotArtifact( + snapshotDir: string, + targetPath: string, +): Promise { + const snapshotRoot = await root(snapshotDir); + const source = await snapshotRoot.open(SNAPSHOT_SQLITE_FILENAME, { + hardlinks: "reject", + symlinks: "reject", + }); + let target: OpenFileHandle | undefined; + let targetIdentity: Stats | undefined; + try { + target = await fs.open(targetPath, "wx+", 0o600); + targetIdentity = await target.stat(); + const digest = await hashFileHandle(source.handle, target); + await target.sync(); + const finalIdentity = await target.stat(); + const currentIdentity = await fs.lstat(targetPath); + if ( + !sameFileIdentity(targetIdentity, finalIdentity) || + !sameFileIdentity(targetIdentity, currentIdentity) + ) { + throw new Error(`Snapshot restore staging file changed during copy: ${targetPath}`); + } + return { ...digest, stat: finalIdentity }; + } catch (error) { + await target?.close().catch(() => undefined); + target = undefined; + if (targetIdentity) { + const currentIdentity = await fs.lstat(targetPath).catch(() => undefined); + if (currentIdentity && sameFileIdentity(targetIdentity, currentIdentity)) { + await fs.unlink(targetPath).catch(() => undefined); + } + } + throw error; + } finally { + await target?.close().catch(() => undefined); + await source.handle.close().catch(() => undefined); + } +} + +async function hashFileHandle( + source: OpenFileHandle, + target?: OpenFileHandle, +): Promise> { + const initialStat = await source.stat({ bigint: true }); + const hash = createHash("sha256"); + const buffer = Buffer.allocUnsafe(1024 * 1024); + let sizeBytes = 0; + while (true) { + const { bytesRead } = await source.read(buffer, 0, buffer.length, sizeBytes); + if (bytesRead === 0) { + break; + } + hash.update(buffer.subarray(0, bytesRead)); + let bytesWritten = 0; + if (target) { + while (bytesWritten < bytesRead) { + const result = await target.write( + buffer, + bytesWritten, + bytesRead - bytesWritten, + sizeBytes + bytesWritten, + ); + if (result.bytesWritten === 0) { + throw new Error("Snapshot restore staging copy made no progress."); + } + bytesWritten += result.bytesWritten; + } + } + sizeBytes += bytesRead; + } + const finalStat = await source.stat({ bigint: true }); + if (!sameMutationFingerprint(initialStat, finalStat)) { + throw new Error("Snapshot artifact changed while being read."); + } + return { sha256: hash.digest("hex"), sizeBytes }; +} + +function sameMutationFingerprint(left: BigIntStats, right: BigIntStats): boolean { + return ( + left.birthtimeNs === right.birthtimeNs && + left.ctimeNs === right.ctimeNs && + left.dev === right.dev && + left.ino === right.ino && + left.mtimeNs === right.mtimeNs && + left.size === right.size + ); +} + +export async function writeSnapshotManifest( + snapshotDir: string, + manifest: SnapshotManifest, +): Promise { + const manifestPath = path.join(snapshotDir, SNAPSHOT_MANIFEST_FILENAME); + const handle = await fs.open(manifestPath, "wx+", 0o600); + try { + await handle.writeFile(`${JSON.stringify(manifest, null, 2)}\n`, "utf8"); + await handle.sync(); + } finally { + await handle.close(); + } +} + +export async function readSnapshotManifest( + snapshotDir: string, + expectedSnapshotId = path.basename(snapshotDir), +): Promise { + const snapshotRoot = await root(snapshotDir); + const manifestPath = path.join(snapshotDir, SNAPSHOT_MANIFEST_FILENAME); + const result = await snapshotRoot.read(SNAPSHOT_MANIFEST_FILENAME, { + hardlinks: "reject", + maxBytes: MAX_MANIFEST_BYTES, + symlinks: "reject", + }); + let parsed: unknown; + try { + parsed = JSON.parse(result.buffer.toString("utf8")) as unknown; + } catch (error) { + throw new Error(`Snapshot manifest is not valid JSON: ${manifestPath}`, { cause: error }); + } + return parseSnapshotManifest(parsed, manifestPath, expectedSnapshotId); +} + +export function parseSnapshotManifest( + value: unknown, + manifestPath: string, + expectedSnapshotId: string, +): SnapshotManifest { + const record = requireRecord(value, "manifest", manifestPath); + requireExactKeys(record, ["schemaVersion", "snapshotId", "createdAt", "database", "artifact"]); + if (record.schemaVersion !== 1) { + throw new Error( + `Unsupported snapshot manifest schemaVersion ${String(record.schemaVersion)}: ${manifestPath}`, + ); + } + const snapshotId = requireSnapshotId(record.snapshotId, manifestPath); + if (snapshotId !== expectedSnapshotId) { + throw new Error( + `Snapshot manifest id ${snapshotId} does not match directory ${expectedSnapshotId}: ${manifestPath}`, + ); + } + const createdAt = requireCanonicalTimestamp(record.createdAt, manifestPath); + const database = parseSnapshotDatabase(record.database, manifestPath); + const artifactRecord = requireRecord(record.artifact, "artifact", manifestPath); + requireExactKeys(artifactRecord, ["path", "sha256", "sizeBytes"]); + if (artifactRecord.path !== SNAPSHOT_SQLITE_FILENAME) { + throw new Error( + `Snapshot manifest artifact.path must be ${SNAPSHOT_SQLITE_FILENAME}: ${manifestPath}`, + ); + } + if (typeof artifactRecord.sha256 !== "string" || !SHA256_PATTERN.test(artifactRecord.sha256)) { + throw new Error(`Snapshot manifest artifact.sha256 is invalid: ${manifestPath}`); + } + if (!Number.isSafeInteger(artifactRecord.sizeBytes) || Number(artifactRecord.sizeBytes) <= 0) { + throw new Error(`Snapshot manifest artifact.sizeBytes is invalid: ${manifestPath}`); + } + return { + schemaVersion: 1, + snapshotId, + createdAt, + database, + artifact: { + path: SNAPSHOT_SQLITE_FILENAME, + sha256: artifactRecord.sha256, + sizeBytes: Number(artifactRecord.sizeBytes), + }, + }; +} + +function parseSnapshotDatabase(value: unknown, manifestPath: string): SnapshotDatabaseManifest { + const database = requireRecord(value, "database", manifestPath); + const role = database.role; + const basename = requireSafeText(database.basename, "database.basename", manifestPath, 255); + if (path.basename(basename) !== basename || basename === "." || basename === "..") { + throw new Error(`Snapshot manifest database.basename is invalid: ${manifestPath}`); + } + const userVersion = requireSqliteUserVersion(database.userVersion, manifestPath); + if (role === "global") { + requireExactKeys(database, ["role", "basename", "userVersion"]); + return { role, basename, userVersion }; + } + if (role === "agent") { + requireExactKeys(database, ["role", "agentId", "basename", "userVersion"]); + const agentId = requireSafeText(database.agentId, "database.agentId", manifestPath, 64); + if (!isValidAgentId(agentId) || normalizeAgentId(agentId) !== agentId) { + throw new Error(`Snapshot manifest database.agentId is invalid: ${manifestPath}`); + } + return { role, agentId, basename, userVersion }; + } + if (role === "generic") { + requireExactKeys(database, ["role", "id", "basename", "userVersion"]); + const id = requireSafeText(database.id, "database.id", manifestPath, 256); + return { role, id, basename, userVersion }; + } + throw new Error(`Snapshot manifest database.role is invalid: ${manifestPath}`); +} + +function requireRecord( + value: unknown, + field: string, + manifestPath: string, +): Record { + if (!value || typeof value !== "object" || Array.isArray(value)) { + throw new Error(`Snapshot manifest ${field} must be an object: ${manifestPath}`); + } + return value as Record; +} + +function requireExactKeys(record: Record, expectedKeys: readonly string[]): void { + const actual = Object.keys(record).toSorted(); + const expected = [...expectedKeys].toSorted(); + if (actual.length !== expected.length || actual.some((key, index) => key !== expected[index])) { + throw new Error( + `Snapshot manifest fields must be exactly ${expectedKeys.join(", ")}; got ${actual.join(", ")}`, + ); + } +} + +function requireSnapshotId(value: unknown, manifestPath: string): string { + if (typeof value !== "string" || !SNAPSHOT_ID_PATTERN.test(value)) { + throw new Error(`Snapshot manifest snapshotId is invalid: ${manifestPath}`); + } + return value; +} + +function requireCanonicalTimestamp(value: unknown, manifestPath: string): string { + if (typeof value !== "string") { + throw new Error(`Snapshot manifest createdAt is invalid: ${manifestPath}`); + } + const parsed = new Date(value); + if (!Number.isFinite(parsed.getTime()) || parsed.toISOString() !== value) { + throw new Error(`Snapshot manifest createdAt is not canonical ISO 8601: ${manifestPath}`); + } + return value; +} + +function requireSafeText( + value: unknown, + field: string, + manifestPath: string, + maxLength: number, +): string { + if ( + typeof value !== "string" || + value.length === 0 || + value.length > maxLength || + value.trim() !== value || + containsAsciiControlCharacter(value) + ) { + throw new Error(`Snapshot manifest ${field} is invalid: ${manifestPath}`); + } + return value; +} + +function requireSqliteUserVersion(value: unknown, manifestPath: string): number { + if ( + !Number.isSafeInteger(value) || + Number(value) < MIN_SQLITE_USER_VERSION || + Number(value) > MAX_SQLITE_USER_VERSION + ) { + throw new Error(`Snapshot manifest database.userVersion is invalid: ${manifestPath}`); + } + return Number(value); +} diff --git a/src/snapshot/snapshot-provider.ts b/src/snapshot/snapshot-provider.ts new file mode 100644 index 000000000000..79fffaf4b2da --- /dev/null +++ b/src/snapshot/snapshot-provider.ts @@ -0,0 +1,66 @@ +export const SNAPSHOT_MANIFEST_FILENAME = "manifest.json"; +export const SNAPSHOT_SQLITE_FILENAME = "database.sqlite"; + +export type SnapshotDatabaseIdentity = + | { readonly role: "global" } + | { readonly role: "agent"; readonly agentId: string } + | { readonly role: "generic"; readonly id: string }; + +export type SnapshotDatabaseRef = { + readonly path: string; + readonly identity: SnapshotDatabaseIdentity; +}; + +export type SnapshotDatabaseManifest = + | { + readonly role: "global"; + readonly basename: string; + readonly userVersion: number; + } + | { + readonly role: "agent"; + readonly agentId: string; + readonly basename: string; + readonly userVersion: number; + } + | { + readonly role: "generic"; + readonly id: string; + readonly basename: string; + readonly userVersion: number; + }; + +export type SnapshotManifest = { + readonly schemaVersion: 1; + readonly snapshotId: string; + readonly createdAt: string; + readonly database: SnapshotDatabaseManifest; + readonly artifact: { + readonly path: typeof SNAPSHOT_SQLITE_FILENAME; + readonly sha256: string; + readonly sizeBytes: number; + }; +}; + +export type SnapshotRef = { + readonly path: string; +}; + +export type SnapshotResult = { + readonly ref: SnapshotRef; + readonly manifest: SnapshotManifest; +}; + +export type SnapshotVerificationResult = { + readonly ok: true; + readonly manifest: SnapshotManifest; +}; + +export type SnapshotSummary = SnapshotResult; + +export type SqliteSnapshotProvider = { + create(database: SnapshotDatabaseRef): Promise; + list(): Promise; + restoreFresh(snapshot: SnapshotRef, targetPath: string): Promise; + verify(snapshot: SnapshotRef): Promise; +};