From d115fb4cf9f901f292981b338dc65f86cd54fdfd Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 30 May 2026 04:54:37 +0200 Subject: [PATCH] refactor: move task state to shared sqlite Move task run, delivery, and flow registry persistence onto the shared OpenClaw state SQLite database. Summary: - Store task runs, delivery state, and flow runs in state/openclaw.sqlite via the generated Kysely schema. - Migrate shipped task sidecars into the shared state DB and archive old sidecars, including invalid-config/read-only CLI paths. - Keep startup migration lightweight for read-only status/tasks paths while still detecting known legacy state markers and custom session stores. Verification: - .agents/skills/autoreview/scripts/autoreview --mode local: clean after final fix - pnpm test src/tasks/task-registry.store.test.ts src/tasks/task-flow-registry.store.test.ts src/commands/doctor-state-migrations.test.ts -- --reporter=verbose - pnpm test src/commands/doctor-state-migrations.test.ts src/cli/program/config-guard.test.ts src/cli/route.test.ts src/cli/command-path-policy.test.ts -- --reporter=verbose - pnpm test src/cli/program/config-guard.test.ts src/cli/route.test.ts src/cli/command-startup-policy.test.ts src/cli/command-path-policy.test.ts src/cli/command-execution-startup.test.ts -- --reporter=verbose - pnpm test src/cli/program/config-guard.test.ts src/cli/argv.test.ts src/cli/route.test.ts src/commands/doctor-config-preflight.state-migration.test.ts -- --reporter=verbose - pnpm test src/tasks/task-flow-registry.store.test.ts -- --reporter=verbose - pnpm test test/scripts/lint-suppressions.test.ts -- --reporter=verbose - pnpm db:kysely:check - pnpm lint:kysely - git diff --check HEAD - pnpm test:startup:memory - PR CI green on 2f7d76f0d5b91e675accdba48fb7c8b0fc6a1325 --- src/cli/argv.test.ts | 6 +- src/cli/argv.ts | 2 +- src/cli/command-catalog.ts | 5 - src/cli/command-execution-startup.test.ts | 6 +- src/cli/command-path-policy.test.ts | 9 +- src/cli/command-startup-policy.test.ts | 4 +- src/cli/program/config-guard.test.ts | 161 ++++- src/cli/program/config-guard.ts | 134 +++- src/cli/route.test.ts | 13 +- ...r-config-preflight.state-migration.test.ts | 78 ++ src/commands/doctor-config-preflight.ts | 28 +- src/commands/doctor-state-migrations.test.ts | 314 ++++++++ src/commands/doctor-state-migrations.ts | 2 + src/commands/doctor.e2e-harness.ts | 5 + src/infra/state-migrations.ts | 641 ++++++++++++++++- src/state/openclaw-state-db.paths.ts | 24 +- src/state/openclaw-state-db.test.ts | 11 + src/tasks/task-flow-registry.paths.ts | 10 - src/tasks/task-flow-registry.store.sqlite.ts | 532 ++++---------- src/tasks/task-flow-registry.store.test.ts | 299 +++----- src/tasks/task-flow-registry.store.ts | 4 +- src/tasks/task-flow-registry.types.ts | 34 + src/tasks/task-registry.paths.test.ts | 25 - src/tasks/task-registry.paths.ts | 32 - src/tasks/task-registry.store.sqlite.ts | 678 ++++++++---------- src/tasks/task-registry.store.test.ts | 648 ++++++++++------- src/tasks/task-registry.store.ts | 4 +- src/tasks/task-registry.ts | 21 +- src/tasks/task-registry.types.ts | 60 ++ 29 files changed, 2450 insertions(+), 1340 deletions(-) create mode 100644 src/commands/doctor-config-preflight.state-migration.test.ts delete mode 100644 src/tasks/task-flow-registry.paths.ts delete mode 100644 src/tasks/task-registry.paths.test.ts delete mode 100644 src/tasks/task-registry.paths.ts diff --git a/src/cli/argv.test.ts b/src/cli/argv.test.ts index cc64b5c70eb..bf73a5dd258 100644 --- a/src/cli/argv.test.ts +++ b/src/cli/argv.test.ts @@ -582,10 +582,10 @@ describe("argv helpers", () => { }); it.each([ - { argv: ["node", "openclaw", "status"], expected: false }, + { argv: ["node", "openclaw", "status"], expected: true }, { argv: ["node", "openclaw", "health"], expected: false }, { argv: ["node", "openclaw", "sessions"], expected: false }, - { argv: ["node", "openclaw", "--profile", "work", "status"], expected: false }, + { argv: ["node", "openclaw", "--profile", "work", "status"], expected: true }, { argv: ["node", "openclaw", "--log-level=debug", "models", "list"], expected: false }, { argv: ["node", "openclaw", "config", "get", "update"], expected: false }, { argv: ["node", "openclaw", "config", "unset", "update"], expected: false }, @@ -600,7 +600,7 @@ describe("argv helpers", () => { }); it.each([ - { path: ["status"], expected: false }, + { path: ["status"], expected: true }, { path: ["update", "status"], expected: false }, { path: ["config", "get"], expected: false }, { path: ["models", "status"], expected: false }, diff --git a/src/cli/argv.ts b/src/cli/argv.ts index 2a80f546e13..7241d424bb3 100644 --- a/src/cli/argv.ts +++ b/src/cli/argv.ts @@ -452,7 +452,7 @@ export function shouldMigrateStateFromPath(path: string[]): boolean { return true; } const [primary, secondary] = path; - if (primary === "health" || primary === "status" || primary === "sessions") { + if (primary === "health" || primary === "sessions") { return false; } if (primary === "update" && secondary === "status") { diff --git a/src/cli/command-catalog.ts b/src/cli/command-catalog.ts index 225a2030337..acd0e185f46 100644 --- a/src/cli/command-catalog.ts +++ b/src/cli/command-catalog.ts @@ -124,7 +124,6 @@ export const cliCommandCatalog: readonly CliCommandCatalogEntry[] = [ policy: { loadPlugins: "never", pluginRegistry: { scope: "channels" }, - routeConfigGuard: "when-suppressed", ensureCliPath: false, networkProxy: "bypass", }, @@ -180,7 +179,6 @@ export const cliCommandCatalog: readonly CliCommandCatalogEntry[] = [ commandPath: ["commitments"], policy: { ensureCliPath: false, - routeConfigGuard: "when-suppressed", loadPlugins: "never", networkProxy: "bypass", }, @@ -225,7 +223,6 @@ export const cliCommandCatalog: readonly CliCommandCatalogEntry[] = [ exact: true, policy: { ensureCliPath: false, - routeConfigGuard: "when-suppressed", loadPlugins: "never", networkProxy: "bypass", }, @@ -236,7 +233,6 @@ export const cliCommandCatalog: readonly CliCommandCatalogEntry[] = [ exact: true, policy: { ensureCliPath: false, - routeConfigGuard: "when-suppressed", loadPlugins: "never", networkProxy: "bypass", }, @@ -246,7 +242,6 @@ export const cliCommandCatalog: readonly CliCommandCatalogEntry[] = [ commandPath: ["tasks"], policy: { ensureCliPath: false, - routeConfigGuard: "when-suppressed", loadPlugins: "never", networkProxy: "bypass", }, diff --git a/src/cli/command-execution-startup.test.ts b/src/cli/command-execution-startup.test.ts index ba627f9acd8..5656ce82f14 100644 --- a/src/cli/command-execution-startup.test.ts +++ b/src/cli/command-execution-startup.test.ts @@ -45,7 +45,7 @@ describe("command-execution-startup", () => { startupPolicy: { suppressDoctorStdout: true, hideBanner: false, - skipConfigGuard: true, + skipConfigGuard: false, loadPlugins: false, pluginRegistry: { scope: "channels" }, }, @@ -168,7 +168,7 @@ describe("command-execution-startup", () => { startupPolicy: { suppressDoctorStdout: true, hideBanner: false, - skipConfigGuard: true, + skipConfigGuard: false, loadPlugins: false, pluginRegistry: { scope: "channels" }, }, @@ -181,7 +181,7 @@ describe("command-execution-startup", () => { allowInvalid: undefined, loadPlugins: false, pluginRegistry: { scope: "channels" }, - skipConfigGuard: true, + skipConfigGuard: false, }); const messageRuntime = {} as never; diff --git a/src/cli/command-path-policy.test.ts b/src/cli/command-path-policy.test.ts index 48fc2d09a70..5ab93dc609b 100644 --- a/src/cli/command-path-policy.test.ts +++ b/src/cli/command-path-policy.test.ts @@ -56,7 +56,7 @@ describe("command-path-policy", () => { it("resolves status policy with shared startup semantics", () => { expectResolvedPolicy(["status"], { - routeConfigGuard: "when-suppressed", + routeConfigGuard: "never", loadPlugins: "never", pluginRegistry: { scope: "channels" }, ensureCliPath: false, @@ -221,6 +221,13 @@ describe("command-path-policy", () => { loadPlugins: "never", networkProxy: "bypass", }); + for (const commandPath of [["tasks"], ["tasks", "list"], ["tasks", "audit"]]) { + expectResolvedPolicy(commandPath, { + ensureCliPath: false, + loadPlugins: "never", + networkProxy: "bypass", + }); + } for (const commandPath of [ ["plugins", "install"], ["plugins", "inspect"], diff --git a/src/cli/command-startup-policy.test.ts b/src/cli/command-startup-policy.test.ts index c8e5b197c3a..f5949ce1bb2 100644 --- a/src/cli/command-startup-policy.test.ts +++ b/src/cli/command-startup-policy.test.ts @@ -24,7 +24,7 @@ describe("command-startup-policy", () => { commandPath: ["status"], suppressDoctorStdout: true, }), - ).toBe(true); + ).toBe(false); expect( shouldSkipRouteConfigGuardForCommandPath({ commandPath: ["gateway", "status"], @@ -239,7 +239,7 @@ describe("command-startup-policy", () => { ).toEqual({ suppressDoctorStdout: true, hideBanner: false, - skipConfigGuard: true, + skipConfigGuard: false, loadPlugins: false, pluginRegistry: { scope: "channels" }, }); diff --git a/src/cli/program/config-guard.test.ts b/src/cli/program/config-guard.test.ts index 931a31be083..1f34cabfa95 100644 --- a/src/cli/program/config-guard.test.ts +++ b/src/cli/program/config-guard.test.ts @@ -1,4 +1,7 @@ -import { beforeEach, describe, expect, it, vi } from "vitest"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { note } from "../../terminal/note.js"; import { formatCliCommand } from "../command-format.js"; import { ensureConfigReady, testApi } from "./config-guard.js"; @@ -68,6 +71,10 @@ async function withCapturedStdout(run: () => Promise): Promise { describe("ensureConfigReady", () => { const resetConfigGuardStateForTests = testApi.resetConfigGuardStateForTests; + const originalHome = process.env.HOME; + const originalOpenClawHome = process.env.OPENCLAW_HOME; + const originalOpenClawStateDir = process.env.OPENCLAW_STATE_DIR; + const tempRoots: string[] = []; async function runEnsureConfigReady(commandPath: string[], suppressDoctorStdout = false) { const runtime = makeRuntime(); @@ -90,9 +97,38 @@ describe("ensureConfigReady", () => { }); } + function useTempOpenClawHome(): string { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-config-guard-")); + tempRoots.push(root); + process.env.OPENCLAW_HOME = root; + delete process.env.OPENCLAW_STATE_DIR; + return root; + } + + function writeLegacyTaskSidecarMarker(root: string): void { + const markerPath = path.join(root, ".openclaw", "tasks", "runs.sqlite"); + fs.mkdirSync(path.dirname(markerPath), { recursive: true }); + fs.writeFileSync(markerPath, ""); + } + + function writeStateMarker(root: string, relativePath: string): void { + const markerPath = path.join(root, ".openclaw", relativePath); + fs.mkdirSync(path.dirname(markerPath), { recursive: true }); + fs.writeFileSync(markerPath, "{}"); + } + beforeEach(() => { vi.clearAllMocks(); resetConfigGuardStateForTests(); + if (originalHome === undefined) { + delete process.env.HOME; + } else { + process.env.HOME = originalHome; + } + for (const root of tempRoots.splice(0)) { + fs.rmSync(root, { recursive: true, force: true }); + } + useTempOpenClawHome(); readConfigFileSnapshotMock.mockResolvedValue(makeSnapshot()); loadAndMaybeMigrateDoctorConfigMock.mockImplementation(async () => ({ snapshot: makeSnapshot(), @@ -100,9 +136,30 @@ describe("ensureConfigReady", () => { })); }); + afterEach(() => { + if (originalHome === undefined) { + delete process.env.HOME; + } else { + process.env.HOME = originalHome; + } + if (originalOpenClawHome === undefined) { + delete process.env.OPENCLAW_HOME; + } else { + process.env.OPENCLAW_HOME = originalOpenClawHome; + } + if (originalOpenClawStateDir === undefined) { + delete process.env.OPENCLAW_STATE_DIR; + } else { + process.env.OPENCLAW_STATE_DIR = originalOpenClawStateDir; + } + for (const root of tempRoots.splice(0)) { + fs.rmSync(root, { recursive: true, force: true }); + } + }); + it.each([ { - name: "skips doctor flow for read-only fast path commands", + name: "skips doctor flow for status task reads without legacy state", commandPath: ["status"], expectedDoctorCalls: 0, }, @@ -112,7 +169,7 @@ describe("ensureConfigReady", () => { expectedDoctorCalls: 0, }, { - name: "runs doctor flow for commands that may mutate state", + name: "runs doctor flow for commands that may mutate state without legacy state", commandPath: ["message"], expectedDoctorCalls: 1, }, @@ -121,13 +178,82 @@ describe("ensureConfigReady", () => { expect(loadAndMaybeMigrateDoctorConfigMock).toHaveBeenCalledTimes(expectedDoctorCalls); if (expectedDoctorCalls > 0) { expect(loadAndMaybeMigrateDoctorConfigMock).toHaveBeenCalledWith({ - migrateState: false, + migrateState: true, migrateLegacyConfig: false, invalidConfigNote: false, }); } }); + it("runs doctor flow when lightweight startup detection finds legacy state", async () => { + const root = useTempOpenClawHome(); + writeLegacyTaskSidecarMarker(root); + + await runEnsureConfigReady(["status"]); + + expect(loadAndMaybeMigrateDoctorConfigMock).toHaveBeenCalledWith({ + migrateState: true, + migrateLegacyConfig: false, + invalidConfigNote: false, + }); + }); + + it("runs doctor flow for legacy sessions without task sidecars", async () => { + const root = useTempOpenClawHome(); + fs.mkdirSync(path.join(root, ".openclaw", "sessions"), { recursive: true }); + + await runEnsureConfigReady(["status"]); + + expect(loadAndMaybeMigrateDoctorConfigMock).toHaveBeenCalledOnce(); + }); + + it.each([ + ["Discord model picker preferences", "discord/model-picker-preferences.json"], + ["Feishu dedupe sidecar", "feishu/dedup/default.json"], + ["Telegram bot info cache", "telegram/bot-info-default.json"], + ["Telegram pairing allowFrom", "credentials/telegram-allowFrom.json"], + ["WhatsApp root auth", "credentials/creds.json"], + ])("runs doctor flow for bundled channel legacy state: %s", async (_label, relativePath) => { + const root = useTempOpenClawHome(); + writeStateMarker(root, relativePath); + + await runEnsureConfigReady(["status"]); + + expect(loadAndMaybeMigrateDoctorConfigMock).toHaveBeenCalledOnce(); + }); + + it("uses shared tilde expansion for OPENCLAW_HOME in the startup detector", async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-config-guard-home-")); + tempRoots.push(root); + process.env.HOME = root; + process.env.OPENCLAW_HOME = "~/svc"; + delete process.env.OPENCLAW_STATE_DIR; + writeLegacyTaskSidecarMarker(path.join(root, "svc")); + + await runEnsureConfigReady(["status"]); + + expect(loadAndMaybeMigrateDoctorConfigMock).toHaveBeenCalledOnce(); + }); + + it("runs doctor flow for read-only commands with configured custom session stores", async () => { + const root = useTempOpenClawHome(); + const customStore = path.join(root, "sessions", "sessions.json"); + const snapshot = { + ...makeSnapshot(), + config: { session: { store: customStore } }, + runtimeConfig: { session: { store: customStore } }, + }; + readConfigFileSnapshotMock.mockResolvedValue(snapshot); + loadAndMaybeMigrateDoctorConfigMock.mockResolvedValue({ + snapshot, + baseConfig: {}, + }); + + await runEnsureConfigReady(["status"]); + + expect(loadAndMaybeMigrateDoctorConfigMock).toHaveBeenCalledOnce(); + }); + it("pins a valid preflight snapshot for command code reuse", async () => { const snapshot = { ...makeSnapshot(), @@ -137,7 +263,7 @@ describe("ensureConfigReady", () => { }; readConfigFileSnapshotMock.mockResolvedValue(snapshot); - await runEnsureConfigReady(["status"]); + await runEnsureConfigReady(["health"]); expect(setRuntimeConfigSnapshotMock).toHaveBeenCalledWith( snapshot.runtimeConfig, @@ -155,9 +281,9 @@ describe("ensureConfigReady", () => { .mockResolvedValueOnce(recoveredSnapshot); try { - await expect(runEnsureConfigReady(["status"])).rejects.toThrow(transientError); - await expect(runEnsureConfigReady(["status"])).resolves.toBeDefined(); - await expect(runEnsureConfigReady(["status"])).resolves.toBeDefined(); + await expect(runEnsureConfigReady(["health"])).rejects.toThrow(transientError); + await expect(runEnsureConfigReady(["health"])).resolves.toBeDefined(); + await expect(runEnsureConfigReady(["health"])).resolves.toBeDefined(); } finally { if (originalVitest === undefined) { delete process.env.VITEST; @@ -182,7 +308,7 @@ describe("ensureConfigReady", () => { "", `Fix: ${formatCliCommand("openclaw doctor --fix")}`, `Inspect: ${formatCliCommand("openclaw config validate")}`, - "Status, health, logs, and doctor commands still run with invalid config.", + "Status, health, logs, tasks list/audit, and doctor commands still run with invalid config.", ]); expect(runtime.exit).toHaveBeenCalledWith(1); }); @@ -227,6 +353,18 @@ describe("ensureConfigReady", () => { const gatewayRuntime = await runEnsureConfigReady(["gateway", "health"]); expect(gatewayRuntime.exit).not.toHaveBeenCalled(); + const tasksListRuntime = await runEnsureConfigReady(["tasks", "list"]); + expect(tasksListRuntime.exit).not.toHaveBeenCalled(); + + const tasksParentRuntime = await runEnsureConfigReady(["tasks"]); + expect(tasksParentRuntime.exit).not.toHaveBeenCalled(); + + const tasksAuditRuntime = await runEnsureConfigReady(["tasks", "audit"]); + expect(tasksAuditRuntime.exit).not.toHaveBeenCalled(); + + const tasksRunRuntime = await runEnsureConfigReady(["tasks", "run"]); + expect(tasksRunRuntime.exit).toHaveBeenCalledWith(1); + const doctorRuntime = await runEnsureConfigReady(["doctor", "fix"]); expect(doctorRuntime.exit).not.toHaveBeenCalled(); expect(doctorRuntime.error).toHaveBeenCalledWith(expect.stringContaining("agentRuntime")); @@ -244,6 +382,7 @@ describe("ensureConfigReady", () => { }); it("runs doctor migration flow only once per module instance", async () => { + writeLegacyTaskSidecarMarker(useTempOpenClawHome()); const runtimeA = makeRuntime(); const runtimeB = makeRuntime(); @@ -253,11 +392,13 @@ describe("ensureConfigReady", () => { }); it("still runs doctor flow when stdout suppression is enabled", async () => { + writeLegacyTaskSidecarMarker(useTempOpenClawHome()); await runEnsureConfigReady(["message"], true); expect(loadAndMaybeMigrateDoctorConfigMock).toHaveBeenCalledTimes(1); }); it("prevents preflight note noise when suppression is enabled", async () => { + writeLegacyTaskSidecarMarker(useTempOpenClawHome()); loadAndMaybeMigrateDoctorConfigMock.mockImplementation(async () => { note("Doctor warnings", "Config warnings"); return { @@ -272,6 +413,7 @@ describe("ensureConfigReady", () => { }); it("allows preflight note noise when suppression is not enabled", async () => { + writeLegacyTaskSidecarMarker(useTempOpenClawHome()); loadAndMaybeMigrateDoctorConfigMock.mockImplementation(async () => { note("Doctor warnings", "Config warnings"); return { @@ -286,6 +428,7 @@ describe("ensureConfigReady", () => { }); it("does not suppress unrelated concurrent stdout writes while suppressing preflight notes", async () => { + writeLegacyTaskSidecarMarker(useTempOpenClawHome()); let releasePreflight: (() => void) | undefined; let preflightStarted: (() => void) | undefined; const preflightStartedPromise = new Promise((resolve) => { diff --git a/src/cli/program/config-guard.ts b/src/cli/program/config-guard.ts index abe7466079e..f0a28f181b2 100644 --- a/src/cli/program/config-guard.ts +++ b/src/cli/program/config-guard.ts @@ -1,4 +1,9 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; import { readConfigFileSnapshot, setRuntimeConfigSnapshot } from "../../config/config.js"; +import { resolveLegacyStateDirs, resolveOAuthDir, resolveStateDir } from "../../config/paths.js"; +import { resolveRequiredHomeDir } from "../../infra/home-dir.js"; import type { RuntimeEnv } from "../../runtime.js"; import { withSuppressedNotes } from "../../terminal/note.js"; import { shouldMigrateStateFromPath } from "../argv.js"; @@ -17,6 +22,7 @@ const ALLOWED_INVALID_GATEWAY_SUBCOMMANDS = new Set([ "stop", "restart", ]); +const ALLOWED_INVALID_TASK_SUBCOMMANDS = new Set(["list", "audit"]); let didRunDoctorConfigFlow = false; let configSnapshotPromise: Promise>> | null = null; @@ -26,6 +32,91 @@ function resetConfigGuardStateForTests() { configSnapshotPromise = null; } +function fileOrDirExists(pathname: string): boolean { + try { + return fs.existsSync(pathname); + } catch { + return false; + } +} + +function dirHasFile(dir: string, predicate: (name: string) => boolean): boolean { + try { + return fs + .readdirSync(dir, { withFileTypes: true }) + .some((entry) => entry.isFile() && predicate(entry.name)); + } catch { + return false; + } +} + +function isLegacyWhatsAppAuthFile(name: string): boolean { + if (name === "creds.json" || name === "creds.json.bak") { + return true; + } + return name.endsWith(".json") && /^(app-state-sync|session|sender-key|pre-key)-/.test(name); +} + +function hasBundledChannelLegacyStateMigrationInputs(stateDir: string, oauthDir: string): boolean { + if (fileOrDirExists(path.join(stateDir, "discord", "model-picker-preferences.json"))) { + return true; + } + if (dirHasFile(path.join(stateDir, "feishu", "dedup"), (name) => name.endsWith(".json"))) { + return true; + } + if ( + fileOrDirExists(path.join(oauthDir, "telegram-allowFrom.json")) || + dirHasFile( + path.join(stateDir, "telegram"), + (name) => name.startsWith("bot-info-") && name.endsWith(".json"), + ) + ) { + return true; + } + return dirHasFile(oauthDir, isLegacyWhatsAppAuthFile); +} + +function hasLegacyStateMigrationInputs(): boolean { + const stateDir = resolveStateDir(process.env, os.homedir); + const oauthDir = resolveOAuthDir(process.env, stateDir); + if ( + !process.env.OPENCLAW_STATE_DIR?.trim() && + resolveLegacyStateDirs(() => resolveRequiredHomeDir(process.env, os.homedir)).some( + fileOrDirExists, + ) + ) { + return true; + } + return ( + [ + path.join(stateDir, "agent"), + path.join(stateDir, "agents"), + path.join(stateDir, "flows", "registry.sqlite"), + path.join(stateDir, "plugin-state", "state.sqlite"), + path.join(stateDir, "sessions"), + path.join(stateDir, "tasks", "runs.sqlite"), + ].some(fileOrDirExists) || hasBundledChannelLegacyStateMigrationInputs(stateDir, oauthDir) + ); +} + +function isReadOnlyStateMigrationCommand(commandPath: string[]): boolean { + const commandName = commandPath[0]; + const subcommandName = commandPath[1]; + return ( + commandName === "status" || + (commandName === "tasks" && + (subcommandName === undefined || ALLOWED_INVALID_TASK_SUBCOMMANDS.has(subcommandName))) + ); +} + +function snapshotHasConfiguredSessionStore( + snapshot: Awaited>, +): boolean { + const cfg = snapshot.runtimeConfig ?? snapshot.config; + const store = cfg?.session?.store; + return typeof store === "string" && store.trim().length > 0; +} + async function getConfigSnapshot() { // Tests often mutate config fixtures; caching can make those flaky. if (process.env.VITEST === "true") { @@ -51,30 +142,51 @@ export async function ensureConfigReady(params: { }): Promise { const commandPath = params.commandPath ?? []; let preflightSnapshot: Awaited> | null = null; - if (!didRunDoctorConfigFlow && shouldMigrateStateFromPath(commandPath)) { + const shouldConsiderStateMigration = shouldMigrateStateFromPath(commandPath); + const isReadOnlyMigrationCommand = isReadOnlyStateMigrationCommand(commandPath); + const runStateMigrationPreflight = async () => { didRunDoctorConfigFlow = true; const runDoctorConfigPreflight = async () => (await import("../../commands/doctor-config-preflight.js")).runDoctorConfigPreflight({ - // Keep ordinary CLI startup on the lightweight validation path. - migrateState: false, + migrateState: true, migrateLegacyConfig: false, invalidConfigNote: false, }); - if (!params.suppressDoctorStdout) { - preflightSnapshot = (await runDoctorConfigPreflight()).snapshot; - } else { - preflightSnapshot = (await withSuppressedNotes(runDoctorConfigPreflight)).snapshot; - } + return !params.suppressDoctorStdout + ? (await runDoctorConfigPreflight()).snapshot + : (await withSuppressedNotes(runDoctorConfigPreflight)).snapshot; + }; + if ( + !didRunDoctorConfigFlow && + shouldConsiderStateMigration && + (!isReadOnlyMigrationCommand || hasLegacyStateMigrationInputs()) + ) { + preflightSnapshot = await runStateMigrationPreflight(); } - const snapshot = preflightSnapshot ?? (await getConfigSnapshot()); + let snapshot = preflightSnapshot ?? (await getConfigSnapshot()); + if ( + !preflightSnapshot && + !didRunDoctorConfigFlow && + shouldConsiderStateMigration && + isReadOnlyMigrationCommand && + snapshot.valid && + snapshotHasConfiguredSessionStore(snapshot) + ) { + preflightSnapshot = await runStateMigrationPreflight(); + snapshot = preflightSnapshot; + } const commandName = commandPath[0]; const subcommandName = commandPath[1]; const isBareGatewayForegroundRun = commandName === "gateway" && (subcommandName === undefined || subcommandName.trim() === ""); + const isReadOnlyTaskStateCommand = + commandName === "tasks" && + (subcommandName === undefined || ALLOWED_INVALID_TASK_SUBCOMMANDS.has(subcommandName)); const allowInvalid = commandName ? params.allowInvalid === true || ALLOWED_INVALID_COMMANDS.has(commandName) || + isReadOnlyTaskStateCommand || isBareGatewayForegroundRun || (commandName === "gateway" && subcommandName && @@ -134,7 +246,9 @@ export async function ensureConfigReady(params: { `${muted("Inspect:")} ${commandText(formatCliCommand("openclaw config validate"))}`, ); params.runtime.error( - muted("Status, health, logs, and doctor commands still run with invalid config."), + muted( + "Status, health, logs, tasks list/audit, and doctor commands still run with invalid config.", + ), ); if (!allowInvalid) { params.runtime.exit(1); diff --git a/src/cli/route.test.ts b/src/cli/route.test.ts index 102f8115e66..627783d654f 100644 --- a/src/cli/route.test.ts +++ b/src/cli/route.test.ts @@ -83,10 +83,19 @@ describe("tryRouteCli", () => { } }); - it("skips config guard for routed status --json commands", async () => { + it("keeps config guard for routed status --json commands", async () => { await expect(tryRouteCli(["node", "openclaw", "status", "--json"])).resolves.toBe(true); - expect(ensureConfigReadyMock).not.toHaveBeenCalled(); + expect(ensureConfigReadyMock).toHaveBeenCalledTimes(1); + expect(firstConfigReadyCall()?.commandPath).toEqual(["status"]); + expect(ensurePluginRegistryLoadedMock).not.toHaveBeenCalled(); + }); + + it("keeps config guard for the parent tasks JSON list alias", async () => { + await expect(tryRouteCli(["node", "openclaw", "tasks", "--json"])).resolves.toBe(true); + + expect(ensureConfigReadyMock).toHaveBeenCalledTimes(1); + expect(firstConfigReadyCall()?.commandPath).toEqual(["tasks"]); expect(ensurePluginRegistryLoadedMock).not.toHaveBeenCalled(); }); diff --git a/src/commands/doctor-config-preflight.state-migration.test.ts b/src/commands/doctor-config-preflight.state-migration.test.ts new file mode 100644 index 00000000000..19fae8811f7 --- /dev/null +++ b/src/commands/doctor-config-preflight.state-migration.test.ts @@ -0,0 +1,78 @@ +import { describe, expect, it, vi } from "vitest"; + +const autoMigrateLegacyStateDir = vi.hoisted(() => + vi.fn(async () => ({ migrated: false, skipped: false, changes: [], warnings: [] })), +); +const autoMigrateLegacyState = vi.hoisted(() => + vi.fn(async () => ({ migrated: true, skipped: false, changes: ["imported"], warnings: [] })), +); +const autoMigrateLegacyTaskStateSidecars = vi.hoisted(() => + vi.fn(async () => ({ migrated: true, skipped: false, changes: ["task-imported"], warnings: [] })), +); +const readConfigFileSnapshot = vi.hoisted(() => + vi.fn(async () => ({ + exists: true, + valid: true, + config: { gateway: { mode: "local", port: 19091 } } as Record, + sourceConfig: { gateway: { mode: "local", port: 19091 } } as Record, + legacyIssues: [] as Array<{ path: string; message: string }>, + warnings: [] as Array<{ path: string; message: string }>, + issues: [] as Array<{ path: string; message: string }>, + })), +); +const note = vi.hoisted(() => vi.fn()); + +vi.mock("./doctor-state-migrations.js", () => ({ + autoMigrateLegacyState, + autoMigrateLegacyStateDir, + autoMigrateLegacyTaskStateSidecars, +})); + +vi.mock("../config/io.js", () => ({ + readConfigFileSnapshot, + recoverConfigFromJsonRootSuffix: vi.fn(), + recoverConfigFromLastKnownGood: vi.fn(), +})); + +vi.mock("../terminal/note.js", () => ({ note })); + +const { runDoctorConfigPreflight } = await import("./doctor-config-preflight.js"); + +describe("runDoctorConfigPreflight state migration", () => { + it("runs full state migrations after reading the config snapshot", async () => { + await runDoctorConfigPreflight({ + migrateLegacyConfig: false, + invalidConfigNote: false, + }); + + expect(autoMigrateLegacyStateDir).toHaveBeenCalledOnce(); + expect(readConfigFileSnapshot).toHaveBeenCalledOnce(); + expect(autoMigrateLegacyState).toHaveBeenCalledWith({ + cfg: { gateway: { mode: "local", port: 19091 } }, + env: process.env, + }); + expect(note).toHaveBeenCalledWith("- imported", "Doctor changes"); + }); + + it("limits invalid-config preflight to task sidecar migration", async () => { + vi.clearAllMocks(); + readConfigFileSnapshot.mockResolvedValueOnce({ + exists: true, + valid: false, + config: {}, + sourceConfig: {}, + legacyIssues: [], + warnings: [], + issues: [{ path: "gateway", message: "invalid" }], + }); + + await runDoctorConfigPreflight({ + migrateLegacyConfig: false, + invalidConfigNote: false, + }); + + expect(autoMigrateLegacyState).not.toHaveBeenCalled(); + expect(autoMigrateLegacyTaskStateSidecars).toHaveBeenCalledWith({ env: process.env }); + expect(note).toHaveBeenCalledWith("- task-imported", "Doctor changes"); + }); +}); diff --git a/src/commands/doctor-config-preflight.ts b/src/commands/doctor-config-preflight.ts index 4ba5128c52c..79f606554b0 100644 --- a/src/commands/doctor-config-preflight.ts +++ b/src/commands/doctor-config-preflight.ts @@ -89,6 +89,15 @@ export function shouldSkipPluginValidationForDoctorConfigPreflight( return isTruthyEnvValue(env.OPENCLAW_UPDATE_IN_PROGRESS); } +function noteStateMigrationResult(result: { changes: string[]; warnings: string[] }): void { + if (result.changes.length > 0) { + note(result.changes.map((entry) => `- ${entry}`).join("\n"), "Doctor changes"); + } + if (result.warnings.length > 0) { + note(result.warnings.map((entry) => `- ${entry}`).join("\n"), "Doctor warnings"); + } +} + export async function runDoctorConfigPreflight( options: { migrateState?: boolean; @@ -100,12 +109,7 @@ export async function runDoctorConfigPreflight( if (options.migrateState !== false) { const { autoMigrateLegacyStateDir } = await import("./doctor-state-migrations.js"); const stateDirResult = await autoMigrateLegacyStateDir({ env: process.env }); - if (stateDirResult.changes.length > 0) { - note(stateDirResult.changes.map((entry) => `- ${entry}`).join("\n"), "Doctor changes"); - } - if (stateDirResult.warnings.length > 0) { - note(stateDirResult.warnings.map((entry) => `- ${entry}`).join("\n"), "Doctor warnings"); - } + noteStateMigrationResult(stateDirResult); } if (options.migrateLegacyConfig !== false) { @@ -150,8 +154,18 @@ export async function runDoctorConfigPreflight( note(formatConfigIssueLines(warnings, "-").join("\n"), "Config warnings"); } + const baseConfig = snapshot.sourceConfig ?? snapshot.config ?? {}; + if (options.migrateState !== false) { + const { autoMigrateLegacyState, autoMigrateLegacyTaskStateSidecars } = + await import("./doctor-state-migrations.js"); + const stateResult = snapshot.valid + ? await autoMigrateLegacyState({ cfg: baseConfig, env: process.env }) + : await autoMigrateLegacyTaskStateSidecars({ env: process.env }); + noteStateMigrationResult(stateResult); + } + return { snapshot, - baseConfig: snapshot.sourceConfig ?? snapshot.config ?? {}, + baseConfig, }; } diff --git a/src/commands/doctor-state-migrations.test.ts b/src/commands/doctor-state-migrations.test.ts index 369870c07d4..d1b84e13197 100644 --- a/src/commands/doctor-state-migrations.test.ts +++ b/src/commands/doctor-state-migrations.test.ts @@ -10,9 +10,12 @@ import { resetPluginStateStoreForTests, } from "../plugin-state/plugin-state-store.js"; import { seedPluginStateEntriesForTests } from "../plugin-state/plugin-state-store.test-helpers.js"; +import { loadTaskFlowRegistryStateFromSqlite } from "../tasks/task-flow-registry.store.sqlite.js"; +import { loadTaskRegistryStateFromSqlite } from "../tasks/task-registry.store.sqlite.js"; import { autoMigrateLegacyStateDir, autoMigrateLegacyState, + autoMigrateLegacyTaskStateSidecars, detectLegacyStateMigrations, resetAutoMigrateLegacyStateDirForTest, resetAutoMigrateLegacyStateForTest, @@ -288,6 +291,126 @@ function writeLegacyPluginStateSidecar(root: string): string { return sourcePath; } +function writeLegacyTaskStateSidecars(root: string): { + taskRunsPath: string; + flowRunsPath: string; +} { + const taskRunsPath = path.join(root, "tasks", "runs.sqlite"); + fs.mkdirSync(path.dirname(taskRunsPath), { recursive: true }); + const sqlite = requireNodeSqlite(); + const tasksDb = new sqlite.DatabaseSync(taskRunsPath); + try { + tasksDb.exec(` + CREATE TABLE task_runs ( + task_id TEXT PRIMARY KEY, + runtime TEXT NOT NULL, + source_id TEXT, + requester_session_key TEXT NOT NULL, + child_session_key TEXT, + parent_task_id TEXT, + agent_id TEXT, + run_id TEXT, + label TEXT, + task TEXT NOT NULL, + status TEXT NOT NULL, + delivery_status TEXT NOT NULL, + notify_policy TEXT NOT NULL, + created_at INTEGER NOT NULL, + started_at INTEGER, + ended_at INTEGER, + last_event_at INTEGER, + cleanup_after INTEGER, + error TEXT, + progress_summary TEXT, + terminal_summary TEXT, + terminal_outcome TEXT + ); + CREATE TABLE task_delivery_state ( + task_id TEXT PRIMARY KEY, + requester_origin_json TEXT, + last_notified_event_at INTEGER + ); + `); + tasksDb + .prepare( + ` + INSERT INTO task_runs ( + task_id, runtime, source_id, requester_session_key, child_session_key, run_id, task, + status, delivery_status, notify_policy, created_at, last_event_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `, + ) + .run( + "legacy-task", + "cron", + "nightly", + "", + "agent:main:cron:nightly", + "legacy-task-run", + "Legacy cron task", + "running", + "not_applicable", + "silent", + 100, + 110, + ); + tasksDb + .prepare( + ` + INSERT INTO task_delivery_state ( + task_id, requester_origin_json, last_notified_event_at + ) VALUES (?, ?, ?) + `, + ) + .run("legacy-task", '{"channel":"test","to":"target"}', 120); + } finally { + tasksDb.close(); + } + + const flowRunsPath = path.join(root, "flows", "registry.sqlite"); + fs.mkdirSync(path.dirname(flowRunsPath), { recursive: true }); + const flowsDb = new sqlite.DatabaseSync(flowRunsPath); + try { + flowsDb.exec(` + CREATE TABLE flow_runs ( + flow_id TEXT PRIMARY KEY, + owner_session_key TEXT NOT NULL, + requester_origin_json TEXT, + status TEXT NOT NULL, + notify_policy TEXT NOT NULL, + goal TEXT NOT NULL, + current_step TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + ended_at INTEGER + ); + `); + flowsDb + .prepare( + ` + INSERT INTO flow_runs ( + flow_id, owner_session_key, status, notify_policy, goal, current_step, created_at, + updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) + `, + ) + .run( + "legacy-flow", + "agent:main:legacy-flow", + "running", + "done_only", + "Legacy flow", + "spawn_task", + 200, + 210, + ); + } finally { + flowsDb.close(); + } + + return { taskRunsPath, flowRunsPath }; +} + async function detectAndRunMigrations(params: { root: string; cfg: OpenClawConfig; @@ -986,6 +1109,197 @@ describe("doctor legacy state migrations", () => { }); }); + it("imports shipped task registry and flow SQLite sidecars into shared state", async () => { + const root = await makeTempRoot(); + const { taskRunsPath, flowRunsPath } = writeLegacyTaskStateSidecars(root); + + const detected = await detectLegacyStateMigrations({ + cfg: {}, + env: { OPENCLAW_STATE_DIR: root } as NodeJS.ProcessEnv, + }); + + expect(detected.taskStateSidecars).toEqual({ + taskRunsPath, + flowRunsPath, + hasLegacy: true, + }); + expect(detected.preview).toContain( + `- Task registry sidecar: ${taskRunsPath} → shared SQLite state`, + ); + expect(detected.preview).toContain( + `- Task flow sidecar: ${flowRunsPath} → shared SQLite state`, + ); + + const result = await runLegacyStateMigrations({ detected }); + + expect(result.warnings).toStrictEqual([]); + expect(result.changes).toContain("Migrated 1 task registry sidecar row → shared SQLite state"); + expect(result.changes).toContain("Migrated 1 task delivery sidecar row → shared SQLite state"); + expect(result.changes).toContain("Migrated 1 task flow sidecar row → shared SQLite state"); + expect(fs.existsSync(taskRunsPath)).toBe(false); + expect(fs.existsSync(`${taskRunsPath}.migrated`)).toBe(true); + expect(fs.existsSync(flowRunsPath)).toBe(false); + expect(fs.existsSync(`${flowRunsPath}.migrated`)).toBe(true); + + await withStateDir(root, async () => { + const taskState = loadTaskRegistryStateFromSqlite(); + const task = taskState.tasks.get("legacy-task"); + expect(task).toMatchObject({ + taskId: "legacy-task", + ownerKey: "system:cron:nightly", + scopeKind: "system", + requesterSessionKey: "", + runId: "legacy-task-run", + }); + expect(taskState.deliveryStates.get("legacy-task")).toMatchObject({ + taskId: "legacy-task", + lastNotifiedEventAt: 120, + }); + + const flowState = loadTaskFlowRegistryStateFromSqlite(); + expect(flowState.flows.get("legacy-flow")).toMatchObject({ + flowId: "legacy-flow", + ownerKey: "agent:main:legacy-flow", + syncMode: "managed", + controllerId: "core/legacy-restored", + revision: 0, + }); + }); + }); + + it("skips orphan task delivery sidecar rows while importing valid task rows", async () => { + const root = await makeTempRoot(); + const { taskRunsPath } = writeLegacyTaskStateSidecars(root); + const sqlite = requireNodeSqlite(); + const db = new sqlite.DatabaseSync(taskRunsPath); + try { + db.prepare( + ` + INSERT INTO task_delivery_state ( + task_id, requester_origin_json, last_notified_event_at + ) VALUES (?, ?, ?) + `, + ).run("missing-task", '{"channel":"stale","to":"target"}', 130); + } finally { + db.close(); + } + + const result = await autoMigrateLegacyTaskStateSidecars({ + env: { OPENCLAW_STATE_DIR: root } as NodeJS.ProcessEnv, + }); + + expect(result.changes).toContain("Migrated 1 task registry sidecar row → shared SQLite state"); + expect(result.changes).toContain("Migrated 1 task delivery sidecar row → shared SQLite state"); + expect(result.warnings).toContain( + "Skipped 1 orphan task delivery sidecar row with no task run", + ); + expect(fs.existsSync(`${taskRunsPath}.migrated`)).toBe(true); + + await withStateDir(root, async () => { + const taskState = loadTaskRegistryStateFromSqlite(); + expect(taskState.tasks.has("legacy-task")).toBe(true); + expect(taskState.deliveryStates.has("legacy-task")).toBe(true); + expect(taskState.deliveryStates.has("missing-task")).toBe(false); + }); + }); + + it("auto-migrates task sidecars without config-dependent state moves", async () => { + const root = await makeTempRoot(); + const { taskRunsPath, flowRunsPath } = writeLegacyTaskStateSidecars(root); + + const result = await autoMigrateLegacyTaskStateSidecars({ + env: { OPENCLAW_STATE_DIR: root } as NodeJS.ProcessEnv, + }); + + expect(result.warnings).toStrictEqual([]); + expect(result.changes).toContain("Migrated 1 task registry sidecar row → shared SQLite state"); + expect(result.changes).toContain("Migrated 1 task flow sidecar row → shared SQLite state"); + expect(fs.existsSync(`${taskRunsPath}.migrated`)).toBe(true); + expect(fs.existsSync(`${flowRunsPath}.migrated`)).toBe(true); + + await withStateDir(root, async () => { + expect(loadTaskRegistryStateFromSqlite().tasks.has("legacy-task")).toBe(true); + expect(loadTaskFlowRegistryStateFromSqlite().flows.has("legacy-flow")).toBe(true); + }); + }); + + it("keeps task sidecars when shared state already has conflicting task rows", async () => { + const root = await makeTempRoot(); + const { taskRunsPath, flowRunsPath } = writeLegacyTaskStateSidecars(root); + + await withStateDir(root, async () => { + const sqlite = requireNodeSqlite(); + const sharedPath = path.join(root, "state", "openclaw.sqlite"); + fs.mkdirSync(path.dirname(sharedPath), { recursive: true }); + const db = new sqlite.DatabaseSync(sharedPath); + try { + db.exec(` + CREATE TABLE IF NOT EXISTS task_runs ( + task_id TEXT NOT NULL PRIMARY KEY, + runtime TEXT NOT NULL, + task_kind TEXT, + source_id TEXT, + requester_session_key TEXT, + owner_key TEXT NOT NULL, + scope_kind TEXT NOT NULL, + child_session_key TEXT, + parent_flow_id TEXT, + parent_task_id TEXT, + agent_id TEXT, + run_id TEXT, + label TEXT, + task TEXT NOT NULL, + status TEXT NOT NULL, + delivery_status TEXT NOT NULL, + notify_policy TEXT NOT NULL, + created_at INTEGER NOT NULL, + started_at INTEGER, + ended_at INTEGER, + last_event_at INTEGER, + cleanup_after INTEGER, + error TEXT, + progress_summary TEXT, + terminal_summary TEXT, + terminal_outcome TEXT + ); + `); + db.prepare(` + INSERT INTO task_runs ( + task_id, runtime, requester_session_key, owner_key, scope_kind, task, status, + delivery_status, notify_policy, created_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `).run( + "legacy-task", + "cron", + "", + "system:cron:nightly", + "system", + "Different task", + "running", + "not_applicable", + "silent", + 100, + ); + } finally { + db.close(); + } + }); + + const detected = await detectLegacyStateMigrations({ + cfg: {}, + env: { OPENCLAW_STATE_DIR: root } as NodeJS.ProcessEnv, + }); + const result = await runLegacyStateMigrations({ detected }); + + expect(result.warnings).toStrictEqual([ + "Left task registry sidecar in place because 1 row already existed in shared state: legacy-task", + ]); + expect(fs.existsSync(taskRunsPath)).toBe(true); + expect(fs.existsSync(`${taskRunsPath}.migrated`)).toBe(false); + expect(fs.existsSync(flowRunsPath)).toBe(false); + expect(fs.existsSync(`${flowRunsPath}.migrated`)).toBe(true); + }); + it("routes legacy state to the default agent entry", async () => { const root = await makeTempRoot(); const cfg: OpenClawConfig = { diff --git a/src/commands/doctor-state-migrations.ts b/src/commands/doctor-state-migrations.ts index 50c59a3a0ad..8ea8b23b771 100644 --- a/src/commands/doctor-state-migrations.ts +++ b/src/commands/doctor-state-migrations.ts @@ -1,11 +1,13 @@ export type { LegacyStateDetection } from "../infra/state-migrations.js"; export { autoMigrateLegacyStateDir, + autoMigrateLegacyTaskStateSidecars, autoMigrateLegacyAgentDir, autoMigrateLegacyState, detectLegacyStateMigrations, migrateLegacyAgentDir, resetAutoMigrateLegacyStateDirForTest, + resetAutoMigrateLegacyTaskStateSidecarsForTest, resetAutoMigrateLegacyAgentDirForTest, resetAutoMigrateLegacyStateForTest, runLegacyStateMigrations, diff --git a/src/commands/doctor.e2e-harness.ts b/src/commands/doctor.e2e-harness.ts index fc34e695723..422f1edfa83 100644 --- a/src/commands/doctor.e2e-harness.ts +++ b/src/commands/doctor.e2e-harness.ts @@ -210,6 +210,11 @@ function createLegacyStateMigrationDetectionResult(params?: { sourcePath: "/tmp/state/plugin-state/state.sqlite", hasLegacy: false, }, + taskStateSidecars: { + taskRunsPath: "/tmp/state/tasks/runs.sqlite", + flowRunsPath: "/tmp/state/flows/registry.sqlite", + hasLegacy: false, + }, channelPlans: { hasLegacy: false, plans: [], diff --git a/src/infra/state-migrations.ts b/src/infra/state-migrations.ts index cc25a47feeb..b535144e0df 100644 --- a/src/infra/state-migrations.ts +++ b/src/infra/state-migrations.ts @@ -1,6 +1,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; +import type { DatabaseSync, SQLInputValue } from "node:sqlite"; import { resolveDefaultAgentId } from "../agents/agent-scope.js"; import { listBundledChannelLegacySessionSurfaces, @@ -81,6 +82,11 @@ export type LegacyStateDetection = { sourcePath: string; hasLegacy: boolean; }; + taskStateSidecars: { + taskRunsPath: string; + flowRunsPath: string; + hasLegacy: boolean; + }; preview: string[]; }; @@ -91,6 +97,7 @@ type MigrationLogger = { let autoMigrateChecked = false; let autoMigrateStateDirChecked = false; +let autoMigrateTaskStateSidecarsChecked = false; let cachedLegacySessionSurfaces: LegacySessionSurface[] | null = null; type LegacySessionSurface = { @@ -111,8 +118,10 @@ type LegacyPluginStateSidecarRow = { }; type LegacyPluginStateImportDatabase = Pick; +type SqliteBindRow = Record; const PLUGIN_STATE_SQLITE_SIDECAR_SUFFIXES = ["", "-shm", "-wal"] as const; +const TASK_STATE_SQLITE_SIDECAR_SUFFIXES = ["", "-shm", "-wal"] as const; class LegacyPluginStateSidecarConflictError extends Error { constructor(readonly conflictedKeys: string[]) { @@ -120,6 +129,12 @@ class LegacyPluginStateSidecarConflictError extends Error { } } +class LegacyTaskStateSidecarConflictError extends Error { + constructor(readonly conflictedKeys: string[]) { + super("legacy task-state sidecar conflicts with shared state"); + } +} + function getLegacySessionSurfaces(): LegacySessionSurface[] { // Legacy migrations run on cold doctor/startup paths. Prefer the narrower // setup plugin surface here so session-key cleanup does not materialize full @@ -160,6 +175,14 @@ function resolveLegacyPluginStateSidecarPath(stateDir: string): string { return path.join(stateDir, "plugin-state", "state.sqlite"); } +function resolveLegacyTaskRunsSidecarPath(stateDir: string): string { + return path.join(stateDir, "tasks", "runs.sqlite"); +} + +function resolveLegacyFlowRunsSidecarPath(stateDir: string): string { + return path.join(stateDir, "flows", "registry.sqlite"); +} + function readLegacyPluginStateSidecarRows(sourcePath: string): LegacyPluginStateSidecarRow[] { const sqlite = requireNodeSqlite(); const db = new sqlite.DatabaseSync(sourcePath, { readOnly: true }); @@ -230,6 +253,550 @@ function archiveLegacyPluginStateSidecar(params: { ); } +function archiveLegacyTaskStateSidecar(params: { + sourcePath: string; + label: string; + changes: string[]; + warnings: string[]; +}): void { + const existingSources = TASK_STATE_SQLITE_SIDECAR_SUFFIXES.map( + (suffix) => `${params.sourcePath}${suffix}`, + ).filter(fileExists); + const existingArchives = existingSources + .map((sourcePath) => `${sourcePath}.migrated`) + .filter(fileExists); + if (existingArchives.length > 0) { + params.warnings.push( + `Left migrated ${params.label} sidecar in place because archive already exists: ${existingArchives[0]}`, + ); + return; + } + for (const sourcePath of existingSources) { + try { + fs.renameSync(sourcePath, `${sourcePath}.migrated`); + } catch (err) { + params.warnings.push( + `Failed archiving ${params.label} sidecar ${sourcePath}: ${String(err)}`, + ); + return; + } + } + params.changes.push( + `Archived ${params.label} sidecar legacy source → ${params.sourcePath}.migrated`, + ); +} + +function listSqliteColumns(db: DatabaseSync, table: string): Set { + const rows = db.prepare(`PRAGMA table_info(${table})`).all() as Array<{ name?: string }>; + return new Set(rows.flatMap((row) => (row.name ? [row.name] : []))); +} + +function pickLegacyColumn(columns: Set, name: string, fallbackSql = "NULL"): string { + return columns.has(name) ? name : `${fallbackSql} AS ${name}`; +} + +function legacyBindValue(value: unknown): SQLInputValue { + if ( + value == null || + typeof value === "string" || + typeof value === "number" || + typeof value === "bigint" || + value instanceof Uint8Array + ) { + return value ?? null; + } + return JSON.stringify(value); +} + +function legacyStringValue(value: unknown): string { + return typeof value === "string" ? value : ""; +} + +function legacyKeyValue(value: SQLInputValue): string { + if (typeof value === "string") { + return value; + } + if (typeof value === "number" || typeof value === "bigint") { + return `${value}`; + } + return ""; +} + +function normalizeLegacyTaskRow(row: Record): SqliteBindRow { + const runtime = legacyStringValue(row.runtime); + const sourceId = typeof row.source_id === "string" ? row.source_id : ""; + const taskId = legacyStringValue(row.task_id); + const ownerRaw = typeof row.owner_key === "string" ? row.owner_key.trim() : ""; + const requesterRaw = + typeof row.requester_session_key === "string" ? row.requester_session_key.trim() : ""; + const ownerKey = ownerRaw || requesterRaw || `system:${runtime}:${sourceId || taskId}`; + const scopeRaw = typeof row.scope_kind === "string" ? row.scope_kind : ""; + const scopeKind = scopeRaw === "system" || ownerKey.startsWith("system:") ? "system" : "session"; + return { + task_id: taskId, + runtime, + task_kind: legacyBindValue(row.task_kind), + source_id: legacyBindValue(row.source_id), + requester_session_key: scopeKind === "system" ? "" : requesterRaw || ownerKey, + owner_key: ownerKey, + scope_kind: scopeKind, + child_session_key: legacyBindValue(row.child_session_key), + parent_flow_id: legacyBindValue(row.parent_flow_id), + parent_task_id: legacyBindValue(row.parent_task_id), + agent_id: legacyBindValue(row.agent_id), + run_id: legacyBindValue(row.run_id), + label: legacyBindValue(row.label), + task: legacyBindValue(row.task ?? ""), + status: legacyBindValue(row.status ?? ""), + delivery_status: legacyBindValue(row.delivery_status ?? ""), + notify_policy: legacyBindValue(row.notify_policy ?? ""), + created_at: normalizeLegacySqliteInteger(row.created_at as number | bigint | null) ?? 0, + started_at: normalizeLegacySqliteInteger(row.started_at as number | bigint | null), + ended_at: normalizeLegacySqliteInteger(row.ended_at as number | bigint | null), + last_event_at: normalizeLegacySqliteInteger(row.last_event_at as number | bigint | null), + cleanup_after: normalizeLegacySqliteInteger(row.cleanup_after as number | bigint | null), + error: legacyBindValue(row.error), + progress_summary: legacyBindValue(row.progress_summary), + terminal_summary: legacyBindValue(row.terminal_summary), + terminal_outcome: legacyBindValue(row.terminal_outcome), + }; +} + +function normalizeLegacyFlowRow(row: Record): SqliteBindRow { + const syncMode = + row.sync_mode === "task_mirrored" || row.shape === "single_task" ? "task_mirrored" : "managed"; + const ownerKey = + typeof row.owner_key === "string" && row.owner_key.trim() + ? row.owner_key.trim() + : typeof row.owner_session_key === "string" + ? row.owner_session_key.trim() + : ""; + const controllerId = + syncMode === "managed" + ? typeof row.controller_id === "string" && row.controller_id.trim() + ? row.controller_id.trim() + : "core/legacy-restored" + : null; + return { + flow_id: legacyBindValue(row.flow_id ?? ""), + shape: legacyBindValue(row.shape), + sync_mode: syncMode, + owner_key: ownerKey, + requester_origin_json: legacyBindValue(row.requester_origin_json), + controller_id: controllerId, + revision: normalizeLegacySqliteInteger(row.revision as number | bigint | null) ?? 0, + status: legacyBindValue(row.status ?? ""), + notify_policy: legacyBindValue(row.notify_policy ?? ""), + goal: legacyBindValue(row.goal ?? ""), + current_step: legacyBindValue(row.current_step), + blocked_task_id: legacyBindValue(row.blocked_task_id), + blocked_summary: legacyBindValue(row.blocked_summary), + state_json: legacyBindValue(row.state_json), + wait_json: legacyBindValue(row.wait_json), + cancel_requested_at: normalizeLegacySqliteInteger( + row.cancel_requested_at as number | bigint | null, + ), + created_at: normalizeLegacySqliteInteger(row.created_at as number | bigint | null) ?? 0, + updated_at: normalizeLegacySqliteInteger(row.updated_at as number | bigint | null) ?? 0, + ended_at: normalizeLegacySqliteInteger(row.ended_at as number | bigint | null), + }; +} + +function legacyRowsMatch( + existing: Record, + incoming: Record, + columns: string[], +): boolean { + return columns.every( + (column) => + normalizeLegacySqliteInteger(existing[column] as number | bigint | null) === + normalizeLegacySqliteInteger(incoming[column] as number | bigint | null), + ); +} + +function readLegacyTaskRows(sourcePath: string): SqliteBindRow[] { + const sqlite = requireNodeSqlite(); + const db = new sqlite.DatabaseSync(sourcePath, { readOnly: true }); + try { + const columns = listSqliteColumns(db, "task_runs"); + if (columns.size === 0) { + return []; + } + const selectColumns = [ + "task_id", + "runtime", + pickLegacyColumn(columns, "task_kind"), + pickLegacyColumn(columns, "source_id"), + pickLegacyColumn(columns, "requester_session_key"), + pickLegacyColumn(columns, "owner_key"), + pickLegacyColumn(columns, "scope_kind"), + pickLegacyColumn(columns, "child_session_key"), + pickLegacyColumn(columns, "parent_flow_id"), + pickLegacyColumn(columns, "parent_task_id"), + pickLegacyColumn(columns, "agent_id"), + pickLegacyColumn(columns, "run_id"), + pickLegacyColumn(columns, "label"), + "task", + "status", + "delivery_status", + "notify_policy", + "created_at", + pickLegacyColumn(columns, "started_at"), + pickLegacyColumn(columns, "ended_at"), + pickLegacyColumn(columns, "last_event_at"), + pickLegacyColumn(columns, "cleanup_after"), + pickLegacyColumn(columns, "error"), + pickLegacyColumn(columns, "progress_summary"), + pickLegacyColumn(columns, "terminal_summary"), + pickLegacyColumn(columns, "terminal_outcome"), + ]; + return db + .prepare( + `SELECT ${selectColumns.join(", ")} FROM task_runs ORDER BY created_at ASC, task_id ASC`, + ) + .all() + .map((row) => normalizeLegacyTaskRow(row as Record)); + } finally { + db.close(); + } +} + +function readLegacyTaskDeliveryRows(sourcePath: string): SqliteBindRow[] { + const sqlite = requireNodeSqlite(); + const db = new sqlite.DatabaseSync(sourcePath, { readOnly: true }); + try { + const columns = listSqliteColumns(db, "task_delivery_state"); + if (columns.size === 0) { + return []; + } + return db + .prepare( + `SELECT task_id, requester_origin_json, last_notified_event_at FROM task_delivery_state ORDER BY task_id ASC`, + ) + .all() as SqliteBindRow[]; + } finally { + db.close(); + } +} + +function readLegacyFlowRows(sourcePath: string): SqliteBindRow[] { + const sqlite = requireNodeSqlite(); + const db = new sqlite.DatabaseSync(sourcePath, { readOnly: true }); + try { + const columns = listSqliteColumns(db, "flow_runs"); + if (columns.size === 0) { + return []; + } + const selectColumns = [ + "flow_id", + pickLegacyColumn(columns, "shape"), + pickLegacyColumn(columns, "sync_mode"), + pickLegacyColumn(columns, "owner_key"), + pickLegacyColumn(columns, "owner_session_key"), + pickLegacyColumn(columns, "requester_origin_json"), + pickLegacyColumn(columns, "controller_id"), + pickLegacyColumn(columns, "revision", "0"), + "status", + "notify_policy", + "goal", + pickLegacyColumn(columns, "current_step"), + pickLegacyColumn(columns, "blocked_task_id"), + pickLegacyColumn(columns, "blocked_summary"), + pickLegacyColumn(columns, "state_json"), + pickLegacyColumn(columns, "wait_json"), + pickLegacyColumn(columns, "cancel_requested_at"), + "created_at", + "updated_at", + pickLegacyColumn(columns, "ended_at"), + ]; + return db + .prepare( + `SELECT ${selectColumns.join(", ")} FROM flow_runs ORDER BY created_at ASC, flow_id ASC`, + ) + .all() + .map((row) => normalizeLegacyFlowRow(row as Record)); + } finally { + db.close(); + } +} + +function insertTaskRunRowSql(db: DatabaseSync, row: SqliteBindRow): void { + db.prepare( + ` + INSERT INTO task_runs ( + task_id, runtime, task_kind, source_id, requester_session_key, owner_key, scope_kind, + child_session_key, parent_flow_id, parent_task_id, agent_id, run_id, label, task, status, + delivery_status, notify_policy, created_at, started_at, ended_at, last_event_at, + cleanup_after, error, progress_summary, terminal_summary, terminal_outcome + ) VALUES ( + @task_id, @runtime, @task_kind, @source_id, @requester_session_key, @owner_key, + @scope_kind, @child_session_key, @parent_flow_id, @parent_task_id, @agent_id, @run_id, + @label, @task, @status, @delivery_status, @notify_policy, @created_at, @started_at, + @ended_at, @last_event_at, @cleanup_after, @error, @progress_summary, @terminal_summary, + @terminal_outcome + ) + `, + ).run(row); +} + +function insertTaskDeliveryRowSql(db: DatabaseSync, row: SqliteBindRow): void { + db.prepare( + ` + INSERT INTO task_delivery_state ( + task_id, requester_origin_json, last_notified_event_at + ) VALUES ( + @task_id, @requester_origin_json, @last_notified_event_at + ) + `, + ).run(row); +} + +function insertFlowRunRowSql(db: DatabaseSync, row: SqliteBindRow): void { + db.prepare( + ` + INSERT INTO flow_runs ( + flow_id, shape, sync_mode, owner_key, requester_origin_json, controller_id, revision, + status, notify_policy, goal, current_step, blocked_task_id, blocked_summary, state_json, + wait_json, cancel_requested_at, created_at, updated_at, ended_at + ) VALUES ( + @flow_id, @shape, @sync_mode, @owner_key, @requester_origin_json, @controller_id, + @revision, @status, @notify_policy, @goal, @current_step, @blocked_task_id, + @blocked_summary, @state_json, @wait_json, @cancel_requested_at, @created_at, + @updated_at, @ended_at + ) + `, + ).run(row); +} + +async function migrateLegacyTaskRunsSidecar(params: { + stateDir: string; +}): Promise<{ changes: string[]; warnings: string[] }> { + const sourcePath = resolveLegacyTaskRunsSidecarPath(params.stateDir); + if (!fileExists(sourcePath)) { + return { changes: [], warnings: [] }; + } + const changes: string[] = []; + const warnings: string[] = []; + let taskRows: SqliteBindRow[]; + let deliveryRows: SqliteBindRow[]; + try { + taskRows = readLegacyTaskRows(sourcePath); + deliveryRows = readLegacyTaskDeliveryRows(sourcePath); + } catch (err) { + return { + changes, + warnings: [`Failed reading task registry sidecar ${sourcePath}: ${String(err)}`], + }; + } + + try { + const conflicts: string[] = []; + let importedTasks = 0; + let importedDeliveryStates = 0; + let skippedOrphanDeliveryStates = 0; + runOpenClawStateWriteTransaction( + ({ db }) => { + const taskColumns = [ + "runtime", + "task_kind", + "source_id", + "requester_session_key", + "owner_key", + "scope_kind", + "child_session_key", + "parent_flow_id", + "parent_task_id", + "agent_id", + "run_id", + "label", + "task", + "status", + "delivery_status", + "notify_policy", + "created_at", + "started_at", + "ended_at", + "last_event_at", + "cleanup_after", + "error", + "progress_summary", + "terminal_summary", + "terminal_outcome", + ]; + for (const row of taskRows) { + const existing = db + .prepare(`SELECT ${taskColumns.join(", ")} FROM task_runs WHERE task_id = ?`) + .get(legacyKeyValue(row.task_id)); + if (existing) { + if (!legacyRowsMatch(existing as Record, row, taskColumns)) { + conflicts.push(legacyKeyValue(row.task_id)); + } + continue; + } + insertTaskRunRowSql(db, row); + importedTasks++; + } + const deliveryColumns = ["requester_origin_json", "last_notified_event_at"]; + for (const row of deliveryRows) { + const existing = db + .prepare( + `SELECT requester_origin_json, last_notified_event_at FROM task_delivery_state WHERE task_id = ?`, + ) + .get(legacyKeyValue(row.task_id)); + if (existing) { + if (!legacyRowsMatch(existing as Record, row, deliveryColumns)) { + conflicts.push(`${legacyKeyValue(row.task_id)}/delivery`); + } + continue; + } + const taskExists = db + .prepare("SELECT 1 FROM task_runs WHERE task_id = ?") + .get(legacyKeyValue(row.task_id)); + if (!taskExists) { + skippedOrphanDeliveryStates++; + continue; + } + insertTaskDeliveryRowSql(db, row); + importedDeliveryStates++; + } + if (conflicts.length > 0) { + throw new LegacyTaskStateSidecarConflictError(conflicts); + } + }, + { env: { ...process.env, OPENCLAW_STATE_DIR: params.stateDir } }, + ); + if (importedTasks > 0) { + changes.push( + `Migrated ${importedTasks} task registry sidecar ${importedTasks === 1 ? "row" : "rows"} → shared SQLite state`, + ); + } + if (importedDeliveryStates > 0) { + changes.push( + `Migrated ${importedDeliveryStates} task delivery sidecar ${importedDeliveryStates === 1 ? "row" : "rows"} → shared SQLite state`, + ); + } + if (skippedOrphanDeliveryStates > 0) { + warnings.push( + `Skipped ${skippedOrphanDeliveryStates} orphan task delivery sidecar ${skippedOrphanDeliveryStates === 1 ? "row" : "rows"} with no task run`, + ); + } + } catch (err) { + if (err instanceof LegacyTaskStateSidecarConflictError) { + return { + changes, + warnings: [ + `Left task registry sidecar in place because ${err.conflictedKeys.length} ${err.conflictedKeys.length === 1 ? "row" : "rows"} already existed in shared state: ${err.conflictedKeys[0]}`, + ], + }; + } + return { + changes, + warnings: [`Failed migrating task registry sidecar ${sourcePath}: ${String(err)}`], + }; + } + + archiveLegacyTaskStateSidecar({ sourcePath, label: "task registry", changes, warnings }); + return { changes, warnings }; +} + +async function migrateLegacyFlowRunsSidecar(params: { + stateDir: string; +}): Promise<{ changes: string[]; warnings: string[] }> { + const sourcePath = resolveLegacyFlowRunsSidecarPath(params.stateDir); + if (!fileExists(sourcePath)) { + return { changes: [], warnings: [] }; + } + const changes: string[] = []; + const warnings: string[] = []; + let rows: SqliteBindRow[]; + try { + rows = readLegacyFlowRows(sourcePath); + } catch (err) { + return { + changes, + warnings: [`Failed reading task flow sidecar ${sourcePath}: ${String(err)}`], + }; + } + + try { + const conflicts: string[] = []; + let imported = 0; + runOpenClawStateWriteTransaction( + ({ db }) => { + const columns = [ + "shape", + "sync_mode", + "owner_key", + "requester_origin_json", + "controller_id", + "revision", + "status", + "notify_policy", + "goal", + "current_step", + "blocked_task_id", + "blocked_summary", + "state_json", + "wait_json", + "cancel_requested_at", + "created_at", + "updated_at", + "ended_at", + ]; + for (const row of rows) { + const existing = db + .prepare(`SELECT ${columns.join(", ")} FROM flow_runs WHERE flow_id = ?`) + .get(legacyKeyValue(row.flow_id)); + if (existing) { + if (!legacyRowsMatch(existing as Record, row, columns)) { + conflicts.push(legacyKeyValue(row.flow_id)); + } + continue; + } + insertFlowRunRowSql(db, row); + imported++; + } + if (conflicts.length > 0) { + throw new LegacyTaskStateSidecarConflictError(conflicts); + } + }, + { env: { ...process.env, OPENCLAW_STATE_DIR: params.stateDir } }, + ); + if (imported > 0) { + changes.push( + `Migrated ${imported} task flow sidecar ${imported === 1 ? "row" : "rows"} → shared SQLite state`, + ); + } + } catch (err) { + if (err instanceof LegacyTaskStateSidecarConflictError) { + return { + changes, + warnings: [ + `Left task flow sidecar in place because ${err.conflictedKeys.length} ${err.conflictedKeys.length === 1 ? "row" : "rows"} already existed in shared state: ${err.conflictedKeys[0]}`, + ], + }; + } + return { + changes, + warnings: [`Failed migrating task flow sidecar ${sourcePath}: ${String(err)}`], + }; + } + + archiveLegacyTaskStateSidecar({ sourcePath, label: "task flow", changes, warnings }); + return { changes, warnings }; +} + +async function migrateLegacyTaskStateSidecars(params: { + stateDir: string; +}): Promise<{ changes: string[]; warnings: string[] }> { + const taskRuns = await migrateLegacyTaskRunsSidecar(params); + const flowRuns = await migrateLegacyFlowRunsSidecar(params); + return { + changes: [...taskRuns.changes, ...flowRuns.changes], + warnings: [...taskRuns.warnings, ...flowRuns.warnings], + }; +} + async function migrateLegacyPluginStateSidecar(params: { stateDir: string; }): Promise<{ changes: string[]; warnings: string[] }> { @@ -985,6 +1552,7 @@ function removeDirIfEmpty(dir: string) { export function resetAutoMigrateLegacyStateForTest() { autoMigrateChecked = false; + autoMigrateTaskStateSidecarsChecked = false; cachedLegacySessionSurfaces = null; } @@ -996,6 +1564,10 @@ export function resetAutoMigrateLegacyStateDirForTest() { autoMigrateStateDirChecked = false; } +export function resetAutoMigrateLegacyTaskStateSidecarsForTest() { + autoMigrateTaskStateSidecarsChecked = false; +} + type StateDirMigrationResult = { migrated: boolean; skipped: boolean; @@ -1227,6 +1799,42 @@ export async function autoMigrateLegacyStateDir(params: { return { migrated: changes.length > 0, skipped: false, changes, warnings }; } +export async function autoMigrateLegacyTaskStateSidecars(params: { + env?: NodeJS.ProcessEnv; + homedir?: () => string; + log?: MigrationLogger; +}): Promise<{ + migrated: boolean; + skipped: boolean; + changes: string[]; + warnings: string[]; +}> { + if (autoMigrateTaskStateSidecarsChecked) { + return { migrated: false, skipped: true, changes: [], warnings: [] }; + } + autoMigrateTaskStateSidecarsChecked = true; + + const stateDir = resolveStateDir(params.env ?? process.env, params.homedir); + const result = await migrateLegacyTaskStateSidecars({ stateDir }); + const logger = params.log ?? createSubsystemLogger("state-migrations"); + if (result.changes.length > 0) { + logger.info( + `Auto-migrated legacy task state:\n${result.changes.map((entry) => `- ${entry}`).join("\n")}`, + ); + } + if (result.warnings.length > 0) { + logger.warn( + `Legacy task state migration warnings:\n${result.warnings.map((entry) => `- ${entry}`).join("\n")}`, + ); + } + return { + migrated: result.changes.length > 0, + skipped: false, + changes: result.changes, + warnings: result.warnings, + }; +} + async function collectChannelLegacyStateMigrationPlans(params: { cfg: OpenClawConfig; env: NodeJS.ProcessEnv; @@ -1301,6 +1909,9 @@ export async function detectLegacyStateMigrations(params: { const hasLegacyAgentDir = existsDir(legacyAgentDir); const pluginStateSidecarPath = resolveLegacyPluginStateSidecarPath(stateDir); const hasPluginStateSidecar = fileExists(pluginStateSidecarPath); + const taskRunsSidecarPath = resolveLegacyTaskRunsSidecarPath(stateDir); + const flowRunsSidecarPath = resolveLegacyFlowRunsSidecarPath(stateDir); + const hasTaskStateSidecars = fileExists(taskRunsSidecarPath) || fileExists(flowRunsSidecarPath); const channelPlans = await collectChannelLegacyStateMigrationPlans({ cfg: params.cfg, env, @@ -1321,6 +1932,12 @@ export async function detectLegacyStateMigrations(params: { if (hasPluginStateSidecar) { preview.push(`- Plugin state sidecar: ${pluginStateSidecarPath} → shared SQLite state`); } + if (fileExists(taskRunsSidecarPath)) { + preview.push(`- Task registry sidecar: ${taskRunsSidecarPath} → shared SQLite state`); + } + if (fileExists(flowRunsSidecarPath)) { + preview.push(`- Task flow sidecar: ${flowRunsSidecarPath} → shared SQLite state`); + } if (channelPlans.length > 0) { preview.push(...channelPlans.map(buildLegacyMigrationPreview)); } @@ -1352,6 +1969,11 @@ export async function detectLegacyStateMigrations(params: { sourcePath: pluginStateSidecarPath, hasLegacy: hasPluginStateSidecar, }, + taskStateSidecars: { + taskRunsPath: taskRunsSidecarPath, + flowRunsPath: flowRunsSidecarPath, + hasLegacy: hasTaskStateSidecars, + }, preview, }; } @@ -1539,6 +2161,9 @@ export async function runLegacyStateMigrations(params: { const pluginStateSidecar = await migrateLegacyPluginStateSidecar({ stateDir: detected.stateDir, }); + const taskStateSidecars = await migrateLegacyTaskStateSidecars({ + stateDir: detected.stateDir, + }); const preSessionChannelPlans = await runLegacyMigrationPlans( detected.channelPlans.plans.filter((plan) => plan.kind === "plugin-state-import"), ); @@ -1550,6 +2175,7 @@ export async function runLegacyStateMigrations(params: { return { changes: [ ...pluginStateSidecar.changes, + ...taskStateSidecars.changes, ...preSessionChannelPlans.changes, ...sessions.changes, ...agentDir.changes, @@ -1557,6 +2183,7 @@ export async function runLegacyStateMigrations(params: { ], warnings: [ ...pluginStateSidecar.warnings, + ...taskStateSidecars.warnings, ...preSessionChannelPlans.warnings, ...sessions.warnings, ...agentDir.warnings, @@ -1793,6 +2420,9 @@ export async function autoMigrateLegacyState(params: { const pluginStateSidecar = await migrateLegacyPluginStateSidecar({ stateDir: detected.stateDir, }); + const taskStateSidecars = await migrateLegacyTaskStateSidecars({ + stateDir: detected.stateDir, + }); const preSessionChannelPlans = await runLegacyMigrationPlans( detected.channelPlans.plans.filter((plan) => plan.kind === "plugin-state-import"), ); @@ -1800,12 +2430,14 @@ export async function autoMigrateLegacyState(params: { ...stateDirResult.changes, ...orphanKeys.changes, ...pluginStateSidecar.changes, + ...taskStateSidecars.changes, ...preSessionChannelPlans.changes, ]; const warnings = [ ...stateDirResult.warnings, ...orphanKeys.warnings, ...pluginStateSidecar.warnings, + ...taskStateSidecars.warnings, ...preSessionChannelPlans.warnings, ]; logMigrationResults(changes, warnings); @@ -1814,6 +2446,7 @@ export async function autoMigrateLegacyState(params: { stateDirResult.migrated || orphanKeys.changes.length > 0 || pluginStateSidecar.changes.length > 0 || + taskStateSidecars.changes.length > 0 || preSessionChannelPlans.changes.length > 0, skipped: true, changes, @@ -1824,7 +2457,8 @@ export async function autoMigrateLegacyState(params: { !detected.sessions.hasLegacy && !detected.agentDir.hasLegacy && !detected.channelPlans.hasLegacy && - !detected.pluginStateSidecar.hasLegacy + !detected.pluginStateSidecar.hasLegacy && + !detected.taskStateSidecars.hasLegacy ) { const changes = [...stateDirResult.changes, ...orphanKeys.changes]; const warnings = [...stateDirResult.warnings, ...orphanKeys.warnings]; @@ -1841,6 +2475,9 @@ export async function autoMigrateLegacyState(params: { const pluginStateSidecar = await migrateLegacyPluginStateSidecar({ stateDir: detected.stateDir, }); + const taskStateSidecars = await migrateLegacyTaskStateSidecars({ + stateDir: detected.stateDir, + }); const preSessionChannelPlans = await runLegacyMigrationPlans( detected.channelPlans.plans.filter((plan) => plan.kind === "plugin-state-import"), ); @@ -1853,6 +2490,7 @@ export async function autoMigrateLegacyState(params: { ...stateDirResult.changes, ...orphanKeys.changes, ...pluginStateSidecar.changes, + ...taskStateSidecars.changes, ...preSessionChannelPlans.changes, ...sessions.changes, ...agentDir.changes, @@ -1862,6 +2500,7 @@ export async function autoMigrateLegacyState(params: { ...stateDirResult.warnings, ...orphanKeys.warnings, ...pluginStateSidecar.warnings, + ...taskStateSidecars.warnings, ...preSessionChannelPlans.warnings, ...sessions.warnings, ...agentDir.warnings, diff --git a/src/state/openclaw-state-db.paths.ts b/src/state/openclaw-state-db.paths.ts index f3b3ceda732..b0aa0a60c9f 100644 --- a/src/state/openclaw-state-db.paths.ts +++ b/src/state/openclaw-state-db.paths.ts @@ -1,8 +1,30 @@ +import os from "node:os"; import path from "node:path"; +import { isMainThread, threadId } from "node:worker_threads"; import { resolveStateDir } from "../config/paths.js"; +import { parseStrictNonNegativeInteger } from "../infra/parse-finite-number.js"; + +function resolveOpenClawStateRootDir(env: NodeJS.ProcessEnv): string { + if (env.OPENCLAW_STATE_DIR?.trim()) { + return resolveStateDir(env); + } + if (env.VITEST || env.NODE_ENV === "test") { + const workerId = parseStrictNonNegativeInteger( + env.VITEST_WORKER_ID ?? env.VITEST_POOL_ID ?? "", + ); + const shardSuffix = + workerId !== undefined + ? `${process.pid}-${workerId}` + : isMainThread + ? String(process.pid) + : `${process.pid}-${threadId}`; + return path.join(os.tmpdir(), "openclaw-test-state", shardSuffix); + } + return resolveStateDir(env); +} export function resolveOpenClawStateSqliteDir(env: NodeJS.ProcessEnv = process.env): string { - return path.join(resolveStateDir(env), "state"); + return path.join(resolveOpenClawStateRootDir(env), "state"); } export function resolveOpenClawStateSqlitePath(env: NodeJS.ProcessEnv = process.env): string { diff --git a/src/state/openclaw-state-db.test.ts b/src/state/openclaw-state-db.test.ts index 87baebc2048..14a3a194dd2 100644 --- a/src/state/openclaw-state-db.test.ts +++ b/src/state/openclaw-state-db.test.ts @@ -40,6 +40,17 @@ describe("openclaw state database", () => { ); }); + it("keeps test default state under a worker-sharded temp directory", () => { + expect( + resolveOpenClawStateSqlitePath({ + VITEST: "true", + VITEST_WORKER_ID: "7", + } as NodeJS.ProcessEnv), + ).toBe( + path.join(os.tmpdir(), "openclaw-test-state", `${process.pid}-7`, "state", "openclaw.sqlite"), + ); + }); + it("creates the shared state schema from the committed SQL shape", () => { const stateDir = createTempStateDir(); const database = openOpenClawStateDatabase({ diff --git a/src/tasks/task-flow-registry.paths.ts b/src/tasks/task-flow-registry.paths.ts deleted file mode 100644 index 7e770df97a4..00000000000 --- a/src/tasks/task-flow-registry.paths.ts +++ /dev/null @@ -1,10 +0,0 @@ -import path from "node:path"; -import { resolveTaskStateDir } from "./task-registry.paths.js"; - -export function resolveTaskFlowRegistryDir(env: NodeJS.ProcessEnv = process.env): string { - return path.join(resolveTaskStateDir(env), "flows"); -} - -export function resolveTaskFlowRegistrySqlitePath(env: NodeJS.ProcessEnv = process.env): string { - return path.join(resolveTaskFlowRegistryDir(env), "registry.sqlite"); -} diff --git a/src/tasks/task-flow-registry.store.sqlite.ts b/src/tasks/task-flow-registry.store.sqlite.ts index 570df6ef7a5..025f4c2a783 100644 --- a/src/tasks/task-flow-registry.store.sqlite.ts +++ b/src/tasks/task-flow-registry.store.sqlite.ts @@ -1,106 +1,84 @@ -import type { DatabaseSync, StatementSync } from "node:sqlite"; -import { requireNodeSqlite } from "../infra/node-sqlite.js"; -import { configureSqliteWalMaintenance, type SqliteWalMaintenance } from "../infra/sqlite-wal.js"; +import type { DatabaseSync } from "node:sqlite"; +import type { Insertable, Selectable } from "kysely"; +import { executeSqliteQuerySync, getNodeSqliteKysely } from "../infra/kysely-sync.js"; +import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js"; import { - resolveTaskFlowRegistryDir, - resolveTaskFlowRegistrySqlitePath, -} from "./task-flow-registry.paths.js"; + closeOpenClawStateDatabase, + openOpenClawStateDatabase, + runOpenClawStateWriteTransaction, +} from "../state/openclaw-state-db.js"; import type { TaskFlowRegistryStoreSnapshot } from "./task-flow-registry.store.types.js"; -import type { TaskFlowRecord, TaskFlowSyncMode, JsonValue } from "./task-flow-registry.types.js"; import { - ensureSqliteStorePermissions, - normalizeSqliteNumber, - parseDeliveryContextJson, - parseSqliteJsonValue, -} from "./task-registry.sqlite.shared.js"; + parseOptionalTaskFlowSyncMode, + parseTaskFlowStatus, + type JsonValue, + type TaskFlowRecord, + type TaskFlowSyncMode, +} from "./task-flow-registry.types.js"; +import { parseDeliveryContextJson } from "./task-registry.sqlite.shared.js"; +import { parseTaskNotifyPolicy } from "./task-registry.types.js"; -type FlowRegistryRow = { - flow_id: string; - sync_mode: TaskFlowSyncMode | null; - shape?: string | null; - owner_key: string; - requester_origin_json: string | null; - controller_id: string | null; - revision: number | bigint | null; - status: TaskFlowRecord["status"]; - notify_policy: TaskFlowRecord["notifyPolicy"]; - goal: string; - current_step: string | null; - blocked_task_id: string | null; - blocked_summary: string | null; - state_json: string | null; - wait_json: string | null; - cancel_requested_at: number | bigint | null; - created_at: number | bigint; - updated_at: number | bigint; - ended_at: number | bigint | null; -}; +type FlowRunsTable = OpenClawStateKyselyDatabase["flow_runs"]; +type FlowRegistryStoreDatabase = Pick; -type FlowRegistryStatements = { - selectAll: StatementSync; - upsertRow: StatementSync; - deleteRow: StatementSync; - clearRows: StatementSync; +type FlowRegistryRow = Selectable & { + sync_mode: string | null; + status: string; + notify_policy: string; }; type FlowRegistryDatabase = { db: DatabaseSync; path: string; - statements: FlowRegistryStatements; - walMaintenance: SqliteWalMaintenance; }; let cachedDatabase: FlowRegistryDatabase | null = null; -const FLOW_REGISTRY_DIR_MODE = 0o700; -const FLOW_REGISTRY_FILE_MODE = 0o600; -const FLOW_RUNS_COLUMNS = ` - flow_id TEXT PRIMARY KEY, - shape TEXT, - sync_mode TEXT NOT NULL DEFAULT 'managed', - owner_key TEXT NOT NULL, - requester_origin_json TEXT, - controller_id TEXT, - revision INTEGER NOT NULL DEFAULT 0, - status TEXT NOT NULL, - notify_policy TEXT NOT NULL, - goal TEXT NOT NULL, - current_step TEXT, - blocked_task_id TEXT, - blocked_summary TEXT, - state_json TEXT, - wait_json TEXT, - cancel_requested_at INTEGER, - created_at INTEGER NOT NULL, - updated_at INTEGER NOT NULL, - ended_at INTEGER -`; + +function normalizeNumber(value: number | bigint | null): number | undefined { + if (typeof value === "bigint") { + return Number(value); + } + return typeof value === "number" ? value : undefined; +} function serializeJson(value: unknown): string | null { return value === undefined ? null : JSON.stringify(value); } +function parseJsonValue(raw: string | null): JsonValue | undefined { + if (!raw?.trim()) { + return undefined; + } + try { + return JSON.parse(raw) as JsonValue; + } catch { + return undefined; + } +} + function rowToSyncMode(row: FlowRegistryRow): TaskFlowSyncMode { - if (row.sync_mode === "task_mirrored" || row.sync_mode === "managed") { - return row.sync_mode; + const syncMode = parseOptionalTaskFlowSyncMode(row.sync_mode); + if (syncMode) { + return syncMode; } return row.shape === "single_task" ? "task_mirrored" : "managed"; } function rowToFlowRecord(row: FlowRegistryRow): TaskFlowRecord { - const endedAt = normalizeSqliteNumber(row.ended_at); - const cancelRequestedAt = normalizeSqliteNumber(row.cancel_requested_at); + const endedAt = normalizeNumber(row.ended_at); + const cancelRequestedAt = normalizeNumber(row.cancel_requested_at); const requesterOrigin = parseDeliveryContextJson(row.requester_origin_json); - const stateJson = parseSqliteJsonValue(row.state_json); - const waitJson = parseSqliteJsonValue(row.wait_json); + const stateJson = parseJsonValue(row.state_json); + const waitJson = parseJsonValue(row.wait_json); return { flowId: row.flow_id, syncMode: rowToSyncMode(row), ownerKey: row.owner_key, ...(requesterOrigin ? { requesterOrigin } : {}), ...(row.controller_id ? { controllerId: row.controller_id } : {}), - revision: normalizeSqliteNumber(row.revision) ?? 0, - status: row.status, - notifyPolicy: row.notify_policy, + revision: normalizeNumber(row.revision) ?? 0, + status: parseTaskFlowStatus(row.status), + notifyPolicy: parseTaskNotifyPolicy(row.notify_policy), goal: row.goal, ...(row.current_step ? { currentStep: row.current_step } : {}), ...(row.blocked_task_id ? { blockedTaskId: row.blocked_task_id } : {}), @@ -108,16 +86,17 @@ function rowToFlowRecord(row: FlowRegistryRow): TaskFlowRecord { ...(stateJson !== undefined ? { stateJson } : {}), ...(waitJson !== undefined ? { waitJson } : {}), ...(cancelRequestedAt != null ? { cancelRequestedAt } : {}), - createdAt: normalizeSqliteNumber(row.created_at) ?? 0, - updatedAt: normalizeSqliteNumber(row.updated_at) ?? 0, + createdAt: normalizeNumber(row.created_at) ?? 0, + updatedAt: normalizeNumber(row.updated_at) ?? 0, ...(endedAt != null ? { endedAt } : {}), }; } -function bindFlowRecord(record: TaskFlowRecord) { +function bindFlowRecord(record: TaskFlowRecord): Insertable { return { flow_id: record.flowId, sync_mode: record.syncMode, + shape: null, owner_key: record.ownerKey, requester_origin_json: serializeJson(record.requesterOrigin), controller_id: record.controllerId ?? null, @@ -137,346 +116,149 @@ function bindFlowRecord(record: TaskFlowRecord) { }; } -function createStatements(db: DatabaseSync): FlowRegistryStatements { - return { - selectAll: db.prepare(` - SELECT - flow_id, - sync_mode, - shape, - owner_key, - requester_origin_json, - controller_id, - revision, - status, - notify_policy, - goal, - current_step, - blocked_task_id, - blocked_summary, - state_json, - wait_json, - cancel_requested_at, - created_at, - updated_at, - ended_at - FROM flow_runs - ORDER BY created_at ASC, flow_id ASC - `), - upsertRow: db.prepare(` - INSERT INTO flow_runs ( - flow_id, - sync_mode, - owner_key, - requester_origin_json, - controller_id, - revision, - status, - notify_policy, - goal, - current_step, - blocked_task_id, - blocked_summary, - state_json, - wait_json, - cancel_requested_at, - created_at, - updated_at, - ended_at - ) VALUES ( - @flow_id, - @sync_mode, - @owner_key, - @requester_origin_json, - @controller_id, - @revision, - @status, - @notify_policy, - @goal, - @current_step, - @blocked_task_id, - @blocked_summary, - @state_json, - @wait_json, - @cancel_requested_at, - @created_at, - @updated_at, - @ended_at - ) - ON CONFLICT(flow_id) DO UPDATE SET - sync_mode = excluded.sync_mode, - owner_key = excluded.owner_key, - requester_origin_json = excluded.requester_origin_json, - controller_id = excluded.controller_id, - revision = excluded.revision, - status = excluded.status, - notify_policy = excluded.notify_policy, - goal = excluded.goal, - current_step = excluded.current_step, - blocked_task_id = excluded.blocked_task_id, - blocked_summary = excluded.blocked_summary, - state_json = excluded.state_json, - wait_json = excluded.wait_json, - cancel_requested_at = excluded.cancel_requested_at, - created_at = excluded.created_at, - updated_at = excluded.updated_at, - ended_at = excluded.ended_at - `), - deleteRow: db.prepare(`DELETE FROM flow_runs WHERE flow_id = ?`), - clearRows: db.prepare(`DELETE FROM flow_runs`), - }; +function getFlowRegistryKysely(db: DatabaseSync) { + return getNodeSqliteKysely(db); } -function hasFlowRunsColumn(db: DatabaseSync, columnName: string): boolean { - const rows = db.prepare(`PRAGMA table_info(flow_runs)`).all() as Array<{ name?: string }>; - return rows.some((row) => row.name === columnName); -} - -function rebuildLegacyFlowRunsTable(db: DatabaseSync) { - // Older live registries can retain owner_session_key TEXT NOT NULL even after owner_key is - // added. Current inserts do not write owner_session_key, so SQLite rejects mirrored flow rows - // until the table is rebuilt into the canonical schema. - db.exec(` - DROP TABLE IF EXISTS flow_runs_canonical_migration; - CREATE TABLE flow_runs_canonical_migration ( - ${FLOW_RUNS_COLUMNS} - ); - INSERT INTO flow_runs_canonical_migration ( - flow_id, - sync_mode, - owner_key, - requester_origin_json, - controller_id, - revision, - status, - notify_policy, - goal, - current_step, - blocked_task_id, - blocked_summary, - state_json, - wait_json, - cancel_requested_at, - created_at, - updated_at, - ended_at +function pruneFlowsNotInSnapshot(params: { db: DatabaseSync; ids: readonly string[] }) { + const tempTableName = "openclaw_live_flow_ids"; + params.db.exec(`CREATE TEMP TABLE IF NOT EXISTS ${tempTableName} (id TEXT PRIMARY KEY)`); + params.db.exec(`DELETE FROM ${tempTableName}`); + const insert = params.db.prepare(`INSERT OR IGNORE INTO ${tempTableName} (id) VALUES (?)`); + for (const id of params.ids) { + insert.run(id); + } + params.db.exec(` + DELETE FROM flow_runs + WHERE NOT EXISTS ( + SELECT 1 FROM ${tempTableName} + WHERE ${tempTableName}.id = flow_runs.flow_id ) - SELECT - flow_id, - CASE - WHEN sync_mode = 'task_mirrored' THEN 'task_mirrored' - ELSE 'managed' - END, - COALESCE(NULLIF(trim(owner_key), ''), owner_session_key), - requester_origin_json, - CASE - WHEN sync_mode = 'task_mirrored' THEN NULL - ELSE COALESCE(NULLIF(trim(controller_id), ''), 'core/legacy-restored') - END, - COALESCE(revision, 0), - status, - notify_policy, - goal, - current_step, - blocked_task_id, - blocked_summary, - state_json, - wait_json, - cancel_requested_at, - created_at, - updated_at, - ended_at - FROM flow_runs; - DROP TABLE flow_runs; - ALTER TABLE flow_runs_canonical_migration RENAME TO flow_runs; `); + params.db.exec(`DELETE FROM ${tempTableName}`); } -function ensureSchema(db: DatabaseSync) { - db.exec(` - CREATE TABLE IF NOT EXISTS flow_runs ( - ${FLOW_RUNS_COLUMNS} - ); - `); - if (!hasFlowRunsColumn(db, "owner_key") && hasFlowRunsColumn(db, "owner_session_key")) { - db.exec(`ALTER TABLE flow_runs ADD COLUMN owner_key TEXT;`); - db.exec(` - UPDATE flow_runs - SET owner_key = owner_session_key - WHERE owner_key IS NULL - `); - } - if (!hasFlowRunsColumn(db, "shape")) { - db.exec(`ALTER TABLE flow_runs ADD COLUMN shape TEXT;`); - } - if (!hasFlowRunsColumn(db, "sync_mode")) { - db.exec(`ALTER TABLE flow_runs ADD COLUMN sync_mode TEXT;`); - if (hasFlowRunsColumn(db, "shape")) { - db.exec(` - UPDATE flow_runs - SET sync_mode = CASE - WHEN shape = 'single_task' THEN 'task_mirrored' - ELSE 'managed' - END - WHERE sync_mode IS NULL - `); - } else { - db.exec(` - UPDATE flow_runs - SET sync_mode = 'managed' - WHERE sync_mode IS NULL - `); - } - } - if (!hasFlowRunsColumn(db, "controller_id")) { - db.exec(`ALTER TABLE flow_runs ADD COLUMN controller_id TEXT;`); - } - db.exec(` - UPDATE flow_runs - SET controller_id = 'core/legacy-restored' - WHERE sync_mode = 'managed' - AND (controller_id IS NULL OR trim(controller_id) = '') - `); - if (!hasFlowRunsColumn(db, "revision")) { - db.exec(`ALTER TABLE flow_runs ADD COLUMN revision INTEGER;`); - db.exec(` - UPDATE flow_runs - SET revision = 0 - WHERE revision IS NULL - `); - } - if (!hasFlowRunsColumn(db, "blocked_task_id")) { - db.exec(`ALTER TABLE flow_runs ADD COLUMN blocked_task_id TEXT;`); - } - if (!hasFlowRunsColumn(db, "blocked_summary")) { - db.exec(`ALTER TABLE flow_runs ADD COLUMN blocked_summary TEXT;`); - } - if (!hasFlowRunsColumn(db, "state_json")) { - db.exec(`ALTER TABLE flow_runs ADD COLUMN state_json TEXT;`); - } - if (!hasFlowRunsColumn(db, "wait_json")) { - db.exec(`ALTER TABLE flow_runs ADD COLUMN wait_json TEXT;`); - } - if (!hasFlowRunsColumn(db, "cancel_requested_at")) { - db.exec(`ALTER TABLE flow_runs ADD COLUMN cancel_requested_at INTEGER;`); - } - if (hasFlowRunsColumn(db, "owner_session_key")) { - // Populate the canonical fields before rebuilding so existing rows survive the legacy-column - // drop, including pre-sync-mode single-task flows and older managed flows with no controller. - db.exec(` - UPDATE flow_runs - SET owner_key = owner_session_key - WHERE (owner_key IS NULL OR trim(owner_key) = '') - `); - db.exec(` - UPDATE flow_runs - SET sync_mode = CASE - WHEN shape = 'single_task' THEN 'task_mirrored' - ELSE 'managed' - END - WHERE sync_mode IS NULL OR trim(sync_mode) = '' - `); - db.exec(` - UPDATE flow_runs - SET revision = 0 - WHERE revision IS NULL - `); - db.exec(` - UPDATE flow_runs - SET controller_id = 'core/legacy-restored' - WHERE sync_mode = 'managed' - AND (controller_id IS NULL OR trim(controller_id) = '') - `); - rebuildLegacyFlowRunsTable(db); - } - db.exec(`CREATE INDEX IF NOT EXISTS idx_flow_runs_status ON flow_runs(status);`); - db.exec(`CREATE INDEX IF NOT EXISTS idx_flow_runs_owner_key ON flow_runs(owner_key);`); - db.exec(`CREATE INDEX IF NOT EXISTS idx_flow_runs_updated_at ON flow_runs(updated_at);`); +function selectFlowRows(db: DatabaseSync): FlowRegistryRow[] { + const query = getFlowRegistryKysely(db) + .selectFrom("flow_runs") + .select([ + "flow_id", + "sync_mode", + "shape", + "owner_key", + "requester_origin_json", + "controller_id", + "revision", + "status", + "notify_policy", + "goal", + "current_step", + "blocked_task_id", + "blocked_summary", + "state_json", + "wait_json", + "cancel_requested_at", + "created_at", + "updated_at", + "ended_at", + ]) + .orderBy("created_at", "asc") + .orderBy("flow_id", "asc"); + return executeSqliteQuerySync(db, query).rows; } -function ensureFlowRegistryPermissions(pathname: string) { - ensureSqliteStorePermissions({ - dir: resolveTaskFlowRegistryDir(process.env), - pathname, - dirMode: FLOW_REGISTRY_DIR_MODE, - fileMode: FLOW_REGISTRY_FILE_MODE, - }); +function upsertFlowRow(db: DatabaseSync, row: Insertable): void { + executeSqliteQuerySync( + db, + getFlowRegistryKysely(db) + .insertInto("flow_runs") + .values(row) + .onConflict((conflict) => + conflict.column("flow_id").doUpdateSet({ + sync_mode: (eb) => eb.ref("excluded.sync_mode"), + owner_key: (eb) => eb.ref("excluded.owner_key"), + requester_origin_json: (eb) => eb.ref("excluded.requester_origin_json"), + controller_id: (eb) => eb.ref("excluded.controller_id"), + revision: (eb) => eb.ref("excluded.revision"), + status: (eb) => eb.ref("excluded.status"), + notify_policy: (eb) => eb.ref("excluded.notify_policy"), + goal: (eb) => eb.ref("excluded.goal"), + current_step: (eb) => eb.ref("excluded.current_step"), + blocked_task_id: (eb) => eb.ref("excluded.blocked_task_id"), + blocked_summary: (eb) => eb.ref("excluded.blocked_summary"), + state_json: (eb) => eb.ref("excluded.state_json"), + wait_json: (eb) => eb.ref("excluded.wait_json"), + cancel_requested_at: (eb) => eb.ref("excluded.cancel_requested_at"), + created_at: (eb) => eb.ref("excluded.created_at"), + updated_at: (eb) => eb.ref("excluded.updated_at"), + ended_at: (eb) => eb.ref("excluded.ended_at"), + }), + ), + ); } function openFlowRegistryDatabase(): FlowRegistryDatabase { - const pathname = resolveTaskFlowRegistrySqlitePath(process.env); - if (cachedDatabase && cachedDatabase.path === pathname) { + const database = openOpenClawStateDatabase(); + const pathname = database.path; + if (cachedDatabase && cachedDatabase.path === pathname && cachedDatabase.db.isOpen) { return cachedDatabase; } - if (cachedDatabase) { - cachedDatabase.walMaintenance.close(); - cachedDatabase.db.close(); + if (cachedDatabase && !cachedDatabase.db.isOpen) { cachedDatabase = null; } - ensureFlowRegistryPermissions(pathname); - const { DatabaseSync } = requireNodeSqlite(); - const db = new DatabaseSync(pathname); - const walMaintenance = configureSqliteWalMaintenance(db); - db.exec(`PRAGMA synchronous = NORMAL;`); - db.exec(`PRAGMA busy_timeout = 5000;`); - ensureSchema(db); - ensureFlowRegistryPermissions(pathname); cachedDatabase = { - db, + db: database.db, path: pathname, - statements: createStatements(db), - walMaintenance, }; return cachedDatabase; } -function withWriteTransaction(write: (statements: FlowRegistryStatements) => void) { - const { db, path, statements } = openFlowRegistryDatabase(); - db.exec("BEGIN IMMEDIATE"); - try { - write(statements); - db.exec("COMMIT"); - ensureFlowRegistryPermissions(path); - } catch (error) { - db.exec("ROLLBACK"); - throw error; - } +function withWriteTransaction(write: (database: FlowRegistryDatabase) => void) { + const database = openFlowRegistryDatabase(); + runOpenClawStateWriteTransaction(() => { + write(database); + }); } export function loadTaskFlowRegistryStateFromSqlite(): TaskFlowRegistryStoreSnapshot { - const { statements } = openFlowRegistryDatabase(); - const rows = statements.selectAll.all() as FlowRegistryRow[]; + const { db } = openFlowRegistryDatabase(); + const rows = selectFlowRows(db); return { flows: new Map(rows.map((row) => [row.flow_id, rowToFlowRecord(row)])), }; } export function saveTaskFlowRegistryStateToSqlite(snapshot: TaskFlowRegistryStoreSnapshot) { - withWriteTransaction((statements) => { - statements.clearRows.run(); + withWriteTransaction(({ db }) => { + const kysely = getFlowRegistryKysely(db); + const flowIds = [...snapshot.flows.keys()]; + if (flowIds.length === 0) { + executeSqliteQuerySync(db, kysely.deleteFrom("flow_runs")); + return; + } + pruneFlowsNotInSnapshot({ db, ids: flowIds }); for (const flow of snapshot.flows.values()) { - statements.upsertRow.run(bindFlowRecord(flow)); + upsertFlowRow(db, bindFlowRecord(flow)); } }); } export function upsertTaskFlowRegistryRecordToSqlite(flow: TaskFlowRecord) { - const store = openFlowRegistryDatabase(); - store.statements.upsertRow.run(bindFlowRecord(flow)); - ensureFlowRegistryPermissions(store.path); + withWriteTransaction(({ db }) => { + upsertFlowRow(db, bindFlowRecord(flow)); + }); } export function deleteTaskFlowRegistryRecordFromSqlite(flowId: string) { - const store = openFlowRegistryDatabase(); - store.statements.deleteRow.run(flowId); - ensureFlowRegistryPermissions(store.path); + withWriteTransaction(({ db }) => { + executeSqliteQuerySync( + db, + getFlowRegistryKysely(db).deleteFrom("flow_runs").where("flow_id", "=", flowId), + ); + }); } -export function closeTaskFlowRegistrySqliteStore() { - if (!cachedDatabase) { - return; - } - cachedDatabase.walMaintenance.close(); - cachedDatabase.db.close(); +export function closeTaskFlowRegistryDatabase() { cachedDatabase = null; + closeOpenClawStateDatabase(); } diff --git a/src/tasks/task-flow-registry.store.test.ts b/src/tasks/task-flow-registry.store.test.ts index 54dc7f660a5..0ccb302534f 100644 --- a/src/tasks/task-flow-registry.store.test.ts +++ b/src/tasks/task-flow-registry.store.test.ts @@ -1,6 +1,10 @@ import { statSync } from "node:fs"; +import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { requireNodeSqlite } from "../infra/node-sqlite.js"; +import { executeSqliteQuerySync, getNodeSqliteKysely } from "../infra/kysely-sync.js"; +import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js"; +import { openOpenClawStateDatabase } from "../state/openclaw-state-db.js"; +import { resolveOpenClawStateSqlitePath } from "../state/openclaw-state-db.paths.js"; import { withOpenClawTestState } from "../test-utils/openclaw-test-state.js"; import { createTaskFlowForTask, @@ -10,12 +14,19 @@ import { resetTaskFlowRegistryForTests, setFlowWaiting, } from "./task-flow-registry.js"; -import { - resolveTaskFlowRegistryDir, - resolveTaskFlowRegistrySqlitePath, -} from "./task-flow-registry.paths.js"; import { configureTaskFlowRegistryRuntime } from "./task-flow-registry.store.js"; -import type { TaskFlowRecord } from "./task-flow-registry.types.js"; +import { + loadTaskFlowRegistryStateFromSqlite, + saveTaskFlowRegistryStateToSqlite, +} from "./task-flow-registry.store.sqlite.js"; +import { + parseOptionalTaskFlowSyncMode, + parseTaskFlowStatus, + type TaskFlowRecord, +} from "./task-flow-registry.types.js"; +import { parseTaskNotifyPolicy } from "./task-registry.types.js"; + +type TaskFlowRegistryTestDatabase = Pick; function createStoredFlow(): TaskFlowRecord { return { @@ -126,6 +137,74 @@ describe("task-flow-registry store runtime", () => { expect(restoredFlow.goal).toBe("Restored flow"); }); + it("rejects invalid persisted flow enum values", () => { + expect(parseOptionalTaskFlowSyncMode("managed")).toBe("managed"); + expect(parseOptionalTaskFlowSyncMode(null)).toBeUndefined(); + expect(parseTaskFlowStatus("waiting")).toBe("waiting"); + expect(parseTaskNotifyPolicy("state_changes")).toBe("state_changes"); + + expect(() => parseOptionalTaskFlowSyncMode("legacy")).toThrow( + "Invalid persisted task flow sync mode", + ); + expect(() => parseTaskFlowStatus("done")).toThrow("Invalid persisted task flow status"); + expect(() => parseTaskNotifyPolicy("verbose")).toThrow("Invalid persisted task notify policy"); + }); + + it("rejects corrupt persisted flow rows during sqlite restore", async () => { + await withFlowRegistryTempDir(async (root) => { + process.env.OPENCLAW_STATE_DIR = root; + resetTaskFlowRegistryForTests(); + + const created = createManagedTaskFlow({ + ownerKey: "agent:main:main", + controllerId: "tests/corrupt-flow", + goal: "Corrupt flow", + status: "running", + }); + + const database = openOpenClawStateDatabase(); + const db = getNodeSqliteKysely(database.db); + executeSqliteQuerySync( + database.db, + db.updateTable("flow_runs").set({ status: "done" }).where("flow_id", "=", created.flowId), + ); + + expect(() => loadTaskFlowRegistryStateFromSqlite()).toThrow( + "Invalid persisted task flow status", + ); + }); + }); + + it("drops invalid requester origins during sqlite restore", async () => { + await withFlowRegistryTempDir(async (root) => { + process.env.OPENCLAW_STATE_DIR = root; + resetTaskFlowRegistryForTests(); + + const created = createManagedTaskFlow({ + ownerKey: "agent:main:main", + controllerId: "tests/invalid-origin-flow", + goal: "Invalid origin flow", + requesterOrigin: { + channel: "test-channel", + to: "C1234567890", + }, + }); + + const database = openOpenClawStateDatabase(); + const db = getNodeSqliteKysely(database.db); + executeSqliteQuerySync( + database.db, + db + .updateTable("flow_runs") + .set({ requester_origin_json: '{"channel":42}' }) + .where("flow_id", "=", created.flowId), + ); + + const restored = loadTaskFlowRegistryStateFromSqlite(); + expect(restored.flows.get(created.flowId)?.requesterOrigin).toBeUndefined(); + }); + }); + it("restores persisted wait-state, revision, and cancel intent from sqlite", async () => { await withFlowRegistryTempDir(async (root) => { process.env.OPENCLAW_STATE_DIR = root; @@ -194,194 +273,31 @@ describe("task-flow-registry store runtime", () => { }); }); - it("migrates legacy owner_session_key schema before mirrored flow inserts", async () => { + it("prunes large sqlite snapshots without binding every flow id at once", async () => { await withFlowRegistryTempDir(async (root) => { process.env.OPENCLAW_STATE_DIR = root; resetTaskFlowRegistryForTests(); - const sqlitePath = resolveTaskFlowRegistrySqlitePath(process.env); - const { DatabaseSync } = requireNodeSqlite(); - const db = new DatabaseSync(sqlitePath); - // This mirrors the live pre-migration table shape that kept owner_session_key as NOT NULL, - // which made current owner_key-only mirrored inserts fail with SQLITE_CONSTRAINT_NOTNULL. - db.exec(` - DROP TABLE IF EXISTS flow_runs; - CREATE TABLE flow_runs ( - flow_id TEXT PRIMARY KEY, - owner_session_key TEXT NOT NULL, - requester_origin_json TEXT, - status TEXT NOT NULL, - notify_policy TEXT NOT NULL, - goal TEXT NOT NULL, - current_step TEXT, - created_at INTEGER NOT NULL, - updated_at INTEGER NOT NULL, - ended_at INTEGER - ); - INSERT INTO flow_runs ( - flow_id, - owner_session_key, - status, - notify_policy, - goal, - current_step, - created_at, - updated_at - ) VALUES ( - 'legacy-flow', - 'agent:main:legacy', - 'running', - 'done_only', - 'Legacy flow', - 'legacy_step', - 10, - 15 - ); - `); - db.close(); - resetTaskFlowRegistryForTests({ persist: false }); + const flows = new Map(); + for (let index = 0; index < 1_200; index++) { + const flow: TaskFlowRecord = { + ...createStoredFlow(), + flowId: `flow-large-${index}`, + controllerId: `tests/large-flow-${index}`, + createdAt: index, + updatedAt: index, + }; + flows.set(flow.flowId, flow); + } - const mirrored = createTaskFlowForTask({ - task: { - ownerKey: "agent:main:main", - taskId: "task-mirrored", - notifyPolicy: "silent", - status: "queued", - label: "Context engine turn maintenance", - task: "Deferred context-engine maintenance after turn.", - createdAt: 20, - }, - }); + saveTaskFlowRegistryStateToSqlite({ flows }); + const retainedFlows = new Map([...flows].slice(100)); + saveTaskFlowRegistryStateToSqlite({ flows: retainedFlows }); - expect(mirrored.syncMode).toBe("task_mirrored"); - expect(mirrored.ownerKey).toBe("agent:main:main"); - expect(mirrored.controllerId).toBeUndefined(); - - const legacy = getTaskFlowById("legacy-flow"); - expect(legacy?.ownerKey).toBe("agent:main:legacy"); - expect(legacy?.controllerId).toBe("core/legacy-restored"); - - const managed = createManagedTaskFlow({ - ownerKey: "agent:main:fresh", - controllerId: "tests/migrated-flow", - goal: "Writable after migration", - }); - expect(managed).toMatchObject({ - ownerKey: "agent:main:fresh", - syncMode: "managed", - controllerId: "tests/migrated-flow", - }); - - const migratedDb = new DatabaseSync(sqlitePath); - const columns = migratedDb.prepare(`PRAGMA table_info(flow_runs)`).all() as Array<{ - name?: string; - notnull?: number; - }>; - migratedDb.close(); - expect(columns.map((column) => column.name)).not.toContain("owner_session_key"); - expect(columns.find((column) => column.name === "owner_key")?.notnull).toBe(1); - }); - }); - - it("backfills blank hybrid owner_key values before rebuilding legacy flow_runs tables", async () => { - await withFlowRegistryTempDir(async (root) => { - process.env.OPENCLAW_STATE_DIR = root; - resetTaskFlowRegistryForTests(); - - const sqlitePath = resolveTaskFlowRegistrySqlitePath(process.env); - const { DatabaseSync } = requireNodeSqlite(); - const db = new DatabaseSync(sqlitePath); - db.exec(` - DROP TABLE IF EXISTS flow_runs; - CREATE TABLE flow_runs ( - flow_id TEXT PRIMARY KEY, - owner_session_key TEXT NOT NULL, - owner_key TEXT, - requester_origin_json TEXT, - status TEXT NOT NULL, - notify_policy TEXT NOT NULL, - goal TEXT NOT NULL, - current_step TEXT, - created_at INTEGER NOT NULL, - updated_at INTEGER NOT NULL, - ended_at INTEGER - ); - INSERT INTO flow_runs ( - flow_id, - owner_session_key, - owner_key, - status, - notify_policy, - goal, - current_step, - created_at, - updated_at - ) VALUES ( - 'hybrid-flow', - 'agent:main:legacy-hybrid', - ' ', - 'queued', - 'done_only', - 'Hybrid flow', - NULL, - 20, - 20 - ); - `); - db.close(); - resetTaskFlowRegistryForTests({ persist: false }); - - const hybrid = getTaskFlowById("hybrid-flow"); - expect(hybrid).toMatchObject({ - flowId: "hybrid-flow", - ownerKey: "agent:main:legacy-hybrid", - syncMode: "managed", - controllerId: "core/legacy-restored", - revision: 0, - status: "queued", - }); - - const migratedDb = new DatabaseSync(sqlitePath); - const columns = migratedDb.prepare(`PRAGMA table_info(flow_runs)`).all() as Array<{ - name?: string; - notnull?: number; - }>; - migratedDb.close(); - expect(columns.map((column) => column.name)).not.toContain("owner_session_key"); - expect(columns.find((column) => column.name === "owner_key")?.notnull).toBe(1); - }); - }); - - it("drops malformed requester origin json from sqlite flow state", async () => { - await withFlowRegistryTempDir(async (root) => { - process.env.OPENCLAW_STATE_DIR = root; - resetTaskFlowRegistryForTests(); - - const created = createManagedTaskFlow({ - ownerKey: "agent:main:main", - requesterOrigin: { - channel: "notifychat", - to: "notifychat:123", - }, - controllerId: "tests/malformed-origin", - goal: "Restore malformed origin", - status: "running", - }); - - const sqlitePath = resolveTaskFlowRegistrySqlitePath(process.env); - const { DatabaseSync } = requireNodeSqlite(); - const db = new DatabaseSync(sqlitePath); - db.prepare(`UPDATE flow_runs SET requester_origin_json = ? WHERE flow_id = ?`).run( - JSON.stringify(["notifychat", "123"]), - created.flowId, - ); - db.close(); - - resetTaskFlowRegistryForTests({ persist: false }); - - const restored = getTaskFlowById(created.flowId); - expect(restored?.flowId).toBe(created.flowId); - expect(restored?.requesterOrigin).toBeUndefined(); + const restored = loadTaskFlowRegistryStateFromSqlite(); + expect(restored.flows.size).toBe(1_100); + expect(restored.flows.has("flow-large-0")).toBe(false); + expect(restored.flows.has("flow-large-1199")).toBe(true); }); }); @@ -403,10 +319,11 @@ describe("task-flow-registry store runtime", () => { waitJson: { kind: "task", taskId: "task-secured" }, }); - const registryDir = resolveTaskFlowRegistryDir(process.env); - const sqlitePath = resolveTaskFlowRegistrySqlitePath(process.env); + const databasePath = resolveOpenClawStateSqlitePath(process.env); + const registryDir = path.dirname(databasePath); + expect(databasePath.endsWith(path.join("state", "openclaw.sqlite"))).toBe(true); expect(statSync(registryDir).mode & 0o777).toBe(0o700); - expect(statSync(sqlitePath).mode & 0o777).toBe(0o600); + expect(statSync(databasePath).mode & 0o777).toBe(0o600); }); }); }); diff --git a/src/tasks/task-flow-registry.store.ts b/src/tasks/task-flow-registry.store.ts index 605ef75d6da..f163a255fe6 100644 --- a/src/tasks/task-flow-registry.store.ts +++ b/src/tasks/task-flow-registry.store.ts @@ -1,5 +1,5 @@ import { - closeTaskFlowRegistrySqliteStore, + closeTaskFlowRegistryDatabase, deleteTaskFlowRegistryRecordFromSqlite, loadTaskFlowRegistryStateFromSqlite, saveTaskFlowRegistryStateToSqlite, @@ -44,7 +44,7 @@ const defaultFlowRegistryStore: TaskFlowRegistryStore = { saveSnapshot: saveTaskFlowRegistryStateToSqlite, upsertFlow: upsertTaskFlowRegistryRecordToSqlite, deleteFlow: deleteTaskFlowRegistryRecordFromSqlite, - close: closeTaskFlowRegistrySqliteStore, + close: closeTaskFlowRegistryDatabase, }; let configuredFlowRegistryStore: TaskFlowRegistryStore = defaultFlowRegistryStore; diff --git a/src/tasks/task-flow-registry.types.ts b/src/tasks/task-flow-registry.types.ts index b602e5b15a6..a0d2d458a01 100644 --- a/src/tasks/task-flow-registry.types.ts +++ b/src/tasks/task-flow-registry.types.ts @@ -21,6 +21,40 @@ export type TaskFlowStatus = | "cancelled" | "lost"; +const TASK_FLOW_SYNC_MODES = new Set(["task_mirrored", "managed"]); +const TASK_FLOW_STATUSES = new Set([ + "queued", + "running", + "waiting", + "blocked", + "succeeded", + "failed", + "cancelled", + "lost", +]); + +function parsePersistedFlowValue( + value: unknown, + values: ReadonlySet, + label: string, +): T { + if (typeof value === "string" && values.has(value as T)) { + return value as T; + } + throw new Error(`Invalid persisted task flow ${label}: ${JSON.stringify(value)}`); +} + +export function parseOptionalTaskFlowSyncMode(value: unknown): TaskFlowSyncMode | undefined { + if (value == null || value === "") { + return undefined; + } + return parsePersistedFlowValue(value, TASK_FLOW_SYNC_MODES, "sync mode"); +} + +export function parseTaskFlowStatus(value: unknown): TaskFlowStatus { + return parsePersistedFlowValue(value, TASK_FLOW_STATUSES, "status"); +} + export type TaskFlowRecord = { flowId: string; syncMode: TaskFlowSyncMode; diff --git a/src/tasks/task-registry.paths.test.ts b/src/tasks/task-registry.paths.test.ts deleted file mode 100644 index 0aa97f3b0a7..00000000000 --- a/src/tasks/task-registry.paths.test.ts +++ /dev/null @@ -1,25 +0,0 @@ -import os from "node:os"; -import path from "node:path"; -import { describe, expect, it } from "vitest"; -import { resolveTaskStateDir } from "./task-registry.paths.js"; - -describe("task registry paths", () => { - it("uses the Vitest worker id to shard test state dirs", () => { - expect( - resolveTaskStateDir({ - VITEST: "true", - VITEST_POOL_ID: "7", - } as NodeJS.ProcessEnv), - ).toBe(path.join(os.tmpdir(), "openclaw-test-state", `${process.pid}-7`)); - }); - - it("prefers explicit state dir overrides over Vitest sharding", () => { - expect( - resolveTaskStateDir({ - OPENCLAW_STATE_DIR: "/tmp/openclaw-custom-state", - VITEST: "true", - VITEST_POOL_ID: "7", - } as NodeJS.ProcessEnv), - ).toBe("/tmp/openclaw-custom-state"); - }); -}); diff --git a/src/tasks/task-registry.paths.ts b/src/tasks/task-registry.paths.ts deleted file mode 100644 index ee8564b1cea..00000000000 --- a/src/tasks/task-registry.paths.ts +++ /dev/null @@ -1,32 +0,0 @@ -import os from "node:os"; -import path from "node:path"; -import { isMainThread, threadId } from "node:worker_threads"; -import { resolveStateDir } from "../config/paths.js"; -import { parseStrictNonNegativeInteger } from "../infra/parse-finite-number.js"; - -export function resolveTaskStateDir(env: NodeJS.ProcessEnv = process.env): string { - const explicit = env.OPENCLAW_STATE_DIR?.trim(); - if (explicit) { - return resolveStateDir(env); - } - if (env.VITEST || env.NODE_ENV === "test") { - const workerIdRaw = env.VITEST_WORKER_ID ?? env.VITEST_POOL_ID ?? ""; - const workerId = parseStrictNonNegativeInteger(workerIdRaw); - const shardSuffix = - workerId !== undefined - ? `${process.pid}-${workerId}` - : isMainThread - ? String(process.pid) - : `${process.pid}-${threadId}`; - return path.join(os.tmpdir(), "openclaw-test-state", shardSuffix); - } - return resolveStateDir(env); -} - -export function resolveTaskRegistryDir(env: NodeJS.ProcessEnv = process.env): string { - return path.join(resolveTaskStateDir(env), "tasks"); -} - -export function resolveTaskRegistrySqlitePath(env: NodeJS.ProcessEnv = process.env): string { - return path.join(resolveTaskRegistryDir(env), "runs.sqlite"); -} diff --git a/src/tasks/task-registry.store.sqlite.ts b/src/tasks/task-registry.store.sqlite.ts index 2529f4c0c50..cdcf7ce9338 100644 --- a/src/tasks/task-registry.store.sqlite.ts +++ b/src/tasks/task-registry.store.sqlite.ts @@ -1,124 +1,78 @@ -import type { DatabaseSync, StatementSync } from "node:sqlite"; -import { requireNodeSqlite } from "../infra/node-sqlite.js"; -import { configureSqliteWalMaintenance, type SqliteWalMaintenance } from "../infra/sqlite-wal.js"; -import { resolveTaskRegistryDir, resolveTaskRegistrySqlitePath } from "./task-registry.paths.js"; +import type { DatabaseSync } from "node:sqlite"; +import type { Insertable, Selectable } from "kysely"; +import { executeSqliteQuerySync, getNodeSqliteKysely } from "../infra/kysely-sync.js"; +import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js"; import { - ensureSqliteStorePermissions, - normalizeSqliteNumber, - parseDeliveryContextJson, -} from "./task-registry.sqlite.shared.js"; + closeOpenClawStateDatabase, + openOpenClawStateDatabase, + runOpenClawStateWriteTransaction, +} from "../state/openclaw-state-db.js"; +import { parseDeliveryContextJson } from "./task-registry.sqlite.shared.js"; import type { TaskRegistryStoreSnapshot } from "./task-registry.store.types.js"; -import type { TaskDeliveryState, TaskRecord } from "./task-registry.types.js"; +import { + parseOptionalTaskTerminalOutcome, + parseTaskDeliveryStatus, + parseTaskNotifyPolicy, + parseTaskRuntime, + parseTaskScopeKind, + parseTaskStatus, + type TaskDeliveryState, + type TaskRecord, +} from "./task-registry.types.js"; -type TaskRegistryRow = { - task_id: string; - runtime: TaskRecord["runtime"]; - task_kind: string | null; - source_id: string | null; - requester_session_key: string | null; - owner_key: string; - scope_kind: TaskRecord["scopeKind"]; - child_session_key: string | null; - parent_flow_id: string | null; - parent_task_id: string | null; - agent_id: string | null; - run_id: string | null; - label: string | null; - task: string; - status: TaskRecord["status"]; - delivery_status: TaskRecord["deliveryStatus"]; - notify_policy: TaskRecord["notifyPolicy"]; - created_at: number | bigint; - started_at: number | bigint | null; - ended_at: number | bigint | null; - last_event_at: number | bigint | null; - cleanup_after: number | bigint | null; - error: string | null; - progress_summary: string | null; - terminal_summary: string | null; - terminal_outcome: TaskRecord["terminalOutcome"] | null; +type TaskRunsTable = OpenClawStateKyselyDatabase["task_runs"]; +type TaskDeliveryStateTable = OpenClawStateKyselyDatabase["task_delivery_state"]; +type TaskRegistryStoreDatabase = Pick< + OpenClawStateKyselyDatabase, + "task_delivery_state" | "task_runs" +>; + +type TaskRegistryRow = Selectable & { + runtime: string; + scope_kind: string; + status: string; + delivery_status: string; + notify_policy: string; + terminal_outcome: string | null; }; -type TaskDeliveryStateRow = { - task_id: string; - requester_origin_json: string | null; - last_notified_event_at: number | bigint | null; -}; - -type TableInfoRow = { - name: string; -}; - -type TaskRegistryStatements = { - selectAll: StatementSync; - selectByOwnerKey: StatementSync; - selectAllDeliveryStates: StatementSync; - upsertRow: StatementSync; - replaceDeliveryState: StatementSync; - deleteRow: StatementSync; - deleteDeliveryState: StatementSync; - clearRows: StatementSync; - clearDeliveryStates: StatementSync; -}; +type TaskDeliveryStateRow = Selectable; type TaskRegistryDatabase = { db: DatabaseSync; path: string; - statements: TaskRegistryStatements; - walMaintenance: SqliteWalMaintenance; }; let cachedDatabase: TaskRegistryDatabase | null = null; -const TASK_REGISTRY_DIR_MODE = 0o700; -const TASK_REGISTRY_FILE_MODE = 0o600; -const TASK_RUN_SELECT_COLUMNS = ` - task_id, - runtime, - task_kind, - source_id, - requester_session_key, - owner_key, - scope_kind, - child_session_key, - parent_flow_id, - parent_task_id, - agent_id, - run_id, - label, - task, - status, - delivery_status, - notify_policy, - created_at, - started_at, - ended_at, - last_event_at, - cleanup_after, - error, - progress_summary, - terminal_summary, - terminal_outcome -`; + +function normalizeNumber(value: number | bigint | null): number | undefined { + if (typeof value === "bigint") { + return Number(value); + } + return typeof value === "number" ? value : undefined; +} function serializeJson(value: unknown): string | null { return value == null ? null : JSON.stringify(value); } function rowToTaskRecord(row: TaskRegistryRow): TaskRecord { - const startedAt = normalizeSqliteNumber(row.started_at); - const endedAt = normalizeSqliteNumber(row.ended_at); - const lastEventAt = normalizeSqliteNumber(row.last_event_at); - const cleanupAfter = normalizeSqliteNumber(row.cleanup_after); + const startedAt = normalizeNumber(row.started_at); + const endedAt = normalizeNumber(row.ended_at); + const lastEventAt = normalizeNumber(row.last_event_at); + const cleanupAfter = normalizeNumber(row.cleanup_after); + const scopeKind = parseTaskScopeKind(row.scope_kind); + const terminalOutcome = parseOptionalTaskTerminalOutcome(row.terminal_outcome); const requesterSessionKey = - row.scope_kind === "system" ? "" : row.requester_session_key?.trim() || row.owner_key; + scopeKind === "system" ? "" : row.requester_session_key?.trim() || row.owner_key; return { taskId: row.task_id, - runtime: row.runtime, + runtime: parseTaskRuntime(row.runtime), ...(row.task_kind ? { taskKind: row.task_kind } : {}), ...(row.source_id ? { sourceId: row.source_id } : {}), requesterSessionKey, ownerKey: row.owner_key, - scopeKind: row.scope_kind, + scopeKind, ...(row.child_session_key ? { childSessionKey: row.child_session_key } : {}), ...(row.parent_flow_id ? { parentFlowId: row.parent_flow_id } : {}), ...(row.parent_task_id ? { parentTaskId: row.parent_task_id } : {}), @@ -126,10 +80,10 @@ function rowToTaskRecord(row: TaskRegistryRow): TaskRecord { ...(row.run_id ? { runId: row.run_id } : {}), ...(row.label ? { label: row.label } : {}), task: row.task, - status: row.status, - deliveryStatus: row.delivery_status, - notifyPolicy: row.notify_policy, - createdAt: normalizeSqliteNumber(row.created_at) ?? 0, + status: parseTaskStatus(row.status), + deliveryStatus: parseTaskDeliveryStatus(row.delivery_status), + notifyPolicy: parseTaskNotifyPolicy(row.notify_policy), + createdAt: normalizeNumber(row.created_at) ?? 0, ...(startedAt != null ? { startedAt } : {}), ...(endedAt != null ? { endedAt } : {}), ...(lastEventAt != null ? { lastEventAt } : {}), @@ -137,13 +91,13 @@ function rowToTaskRecord(row: TaskRegistryRow): TaskRecord { ...(row.error ? { error: row.error } : {}), ...(row.progress_summary ? { progressSummary: row.progress_summary } : {}), ...(row.terminal_summary ? { terminalSummary: row.terminal_summary } : {}), - ...(row.terminal_outcome ? { terminalOutcome: row.terminal_outcome } : {}), + ...(terminalOutcome ? { terminalOutcome } : {}), }; } function rowToTaskDeliveryState(row: TaskDeliveryStateRow): TaskDeliveryState { const requesterOrigin = parseDeliveryContextJson(row.requester_origin_json); - const lastNotifiedEventAt = normalizeSqliteNumber(row.last_notified_event_at); + const lastNotifiedEventAt = normalizeNumber(row.last_notified_event_at); return { taskId: row.task_id, ...(requesterOrigin ? { requesterOrigin } : {}), @@ -151,7 +105,7 @@ function rowToTaskDeliveryState(row: TaskDeliveryStateRow): TaskDeliveryState { }; } -function bindTaskRecordBase(record: TaskRecord) { +function bindTaskRecordBase(record: TaskRecord): Insertable { return { task_id: record.taskId, runtime: record.runtime, @@ -182,7 +136,7 @@ function bindTaskRecordBase(record: TaskRecord) { }; } -function bindTaskDeliveryState(state: TaskDeliveryState) { +function bindTaskDeliveryState(state: TaskDeliveryState): Insertable { return { task_id: state.taskId, requester_origin_json: serializeJson(state.requesterOrigin), @@ -190,287 +144,160 @@ function bindTaskDeliveryState(state: TaskDeliveryState) { }; } -function createStatements(db: DatabaseSync): TaskRegistryStatements { - return { - selectAll: db.prepare(` - SELECT - ${TASK_RUN_SELECT_COLUMNS} - FROM task_runs - ORDER BY created_at ASC, task_id ASC - `), - selectByOwnerKey: db.prepare(` - SELECT - ${TASK_RUN_SELECT_COLUMNS} - FROM task_runs - WHERE owner_key = ? - ORDER BY created_at ASC, task_id ASC - `), - selectAllDeliveryStates: db.prepare(` - SELECT - task_id, - requester_origin_json, - last_notified_event_at - FROM task_delivery_state - ORDER BY task_id ASC - `), - upsertRow: db.prepare(` - INSERT INTO task_runs ( - task_id, - runtime, - task_kind, - source_id, - requester_session_key, - owner_key, - scope_kind, - child_session_key, - parent_flow_id, - parent_task_id, - agent_id, - run_id, - label, - task, - status, - delivery_status, - notify_policy, - created_at, - started_at, - ended_at, - last_event_at, - cleanup_after, - error, - progress_summary, - terminal_summary, - terminal_outcome - ) VALUES ( - @task_id, - @runtime, - @task_kind, - @source_id, - @requester_session_key, - @owner_key, - @scope_kind, - @child_session_key, - @parent_flow_id, - @parent_task_id, - @agent_id, - @run_id, - @label, - @task, - @status, - @delivery_status, - @notify_policy, - @created_at, - @started_at, - @ended_at, - @last_event_at, - @cleanup_after, - @error, - @progress_summary, - @terminal_summary, - @terminal_outcome - ) - ON CONFLICT(task_id) DO UPDATE SET - runtime = excluded.runtime, - task_kind = excluded.task_kind, - source_id = excluded.source_id, - requester_session_key = excluded.requester_session_key, - owner_key = excluded.owner_key, - scope_kind = excluded.scope_kind, - child_session_key = excluded.child_session_key, - parent_flow_id = excluded.parent_flow_id, - parent_task_id = excluded.parent_task_id, - agent_id = excluded.agent_id, - run_id = excluded.run_id, - label = excluded.label, - task = excluded.task, - status = excluded.status, - delivery_status = excluded.delivery_status, - notify_policy = excluded.notify_policy, - created_at = excluded.created_at, - started_at = excluded.started_at, - ended_at = excluded.ended_at, - last_event_at = excluded.last_event_at, - cleanup_after = excluded.cleanup_after, - error = excluded.error, - progress_summary = excluded.progress_summary, - terminal_summary = excluded.terminal_summary, - terminal_outcome = excluded.terminal_outcome - `), - replaceDeliveryState: db.prepare(` - INSERT OR REPLACE INTO task_delivery_state ( - task_id, - requester_origin_json, - last_notified_event_at - ) VALUES ( - @task_id, - @requester_origin_json, - @last_notified_event_at - ) - `), - deleteRow: db.prepare(`DELETE FROM task_runs WHERE task_id = ?`), - deleteDeliveryState: db.prepare(`DELETE FROM task_delivery_state WHERE task_id = ?`), - clearRows: db.prepare(`DELETE FROM task_runs`), - clearDeliveryStates: db.prepare(`DELETE FROM task_delivery_state`), - }; +function getTaskRegistryKysely(db: DatabaseSync) { + return getNodeSqliteKysely(db); } -function hasTaskRunsColumn(db: DatabaseSync, columnName: string): boolean { - const rows = db.prepare(`PRAGMA table_info(task_runs)`).all() as TableInfoRow[]; - return rows.some((row) => row.name === columnName); +function pruneRowsNotInSnapshot(params: { + db: DatabaseSync; + tableName: "task_delivery_state" | "task_runs"; + columnName: "task_id"; + tempTableName: string; + ids: readonly string[]; +}) { + params.db.exec(`CREATE TEMP TABLE IF NOT EXISTS ${params.tempTableName} (id TEXT PRIMARY KEY)`); + params.db.exec(`DELETE FROM ${params.tempTableName}`); + const insert = params.db.prepare(`INSERT OR IGNORE INTO ${params.tempTableName} (id) VALUES (?)`); + for (const id of params.ids) { + insert.run(id); + } + params.db.exec(` + DELETE FROM ${params.tableName} + WHERE NOT EXISTS ( + SELECT 1 FROM ${params.tempTableName} + WHERE ${params.tempTableName}.id = ${params.tableName}.${params.columnName} + ) + `); + params.db.exec(`DELETE FROM ${params.tempTableName}`); } -function migrateLegacyOwnerColumns(db: DatabaseSync) { - if (!hasTaskRunsColumn(db, "owner_key")) { - db.exec(`ALTER TABLE task_runs ADD COLUMN owner_key TEXT;`); - } - if (!hasTaskRunsColumn(db, "requester_session_key")) { - db.exec(`ALTER TABLE task_runs ADD COLUMN requester_session_key TEXT;`); - } - if (!hasTaskRunsColumn(db, "scope_kind")) { - db.exec(`ALTER TABLE task_runs ADD COLUMN scope_kind TEXT NOT NULL DEFAULT 'session';`); - } - if (hasTaskRunsColumn(db, "requester_session_key")) { - db.exec(` - UPDATE task_runs - SET owner_key = requester_session_key - WHERE owner_key IS NULL - `); - } - db.exec(` - UPDATE task_runs - SET owner_key = CASE - WHEN trim(COALESCE(owner_key, '')) <> '' THEN trim(owner_key) - ELSE 'system:' || runtime || ':' || COALESCE(NULLIF(source_id, ''), task_id) - END - `); - db.exec(` - UPDATE task_runs - SET scope_kind = CASE - WHEN scope_kind = 'system' THEN 'system' - WHEN owner_key LIKE 'system:%' THEN 'system' - ELSE 'session' - END - `); - db.exec(` - UPDATE task_runs - SET requester_session_key = CASE - WHEN scope_kind = 'system' THEN '' - WHEN trim(COALESCE(requester_session_key, '')) <> '' THEN trim(requester_session_key) - ELSE owner_key - END - `); +function selectTaskRows(db: DatabaseSync): TaskRegistryRow[] { + const query = getTaskRegistryKysely(db) + .selectFrom("task_runs") + .select([ + "task_id", + "runtime", + "task_kind", + "source_id", + "requester_session_key", + "owner_key", + "scope_kind", + "child_session_key", + "parent_flow_id", + "parent_task_id", + "agent_id", + "run_id", + "label", + "task", + "status", + "delivery_status", + "notify_policy", + "created_at", + "started_at", + "ended_at", + "last_event_at", + "cleanup_after", + "error", + "progress_summary", + "terminal_summary", + "terminal_outcome", + ]) + .orderBy("created_at", "asc") + .orderBy("task_id", "asc"); + return executeSqliteQuerySync(db, query).rows; } -function ensureSchema(db: DatabaseSync) { - db.exec(` - CREATE TABLE IF NOT EXISTS task_runs ( - task_id TEXT PRIMARY KEY, - runtime TEXT NOT NULL, - task_kind TEXT, - source_id TEXT, - requester_session_key TEXT, - owner_key TEXT NOT NULL, - scope_kind TEXT NOT NULL, - child_session_key TEXT, - parent_flow_id TEXT, - parent_task_id TEXT, - agent_id TEXT, - run_id TEXT, - label TEXT, - task TEXT NOT NULL, - status TEXT NOT NULL, - delivery_status TEXT NOT NULL, - notify_policy TEXT NOT NULL, - created_at INTEGER NOT NULL, - started_at INTEGER, - ended_at INTEGER, - last_event_at INTEGER, - cleanup_after INTEGER, - error TEXT, - progress_summary TEXT, - terminal_summary TEXT, - terminal_outcome TEXT - ); - `); - migrateLegacyOwnerColumns(db); - if (!hasTaskRunsColumn(db, "task_kind")) { - db.exec(`ALTER TABLE task_runs ADD COLUMN task_kind TEXT;`); - } - if (!hasTaskRunsColumn(db, "parent_flow_id")) { - db.exec(`ALTER TABLE task_runs ADD COLUMN parent_flow_id TEXT;`); - } - db.exec(` - CREATE TABLE IF NOT EXISTS task_delivery_state ( - task_id TEXT PRIMARY KEY, - requester_origin_json TEXT, - last_notified_event_at INTEGER - ); - `); - db.exec(`CREATE INDEX IF NOT EXISTS idx_task_runs_run_id ON task_runs(run_id);`); - db.exec(`CREATE INDEX IF NOT EXISTS idx_task_runs_status ON task_runs(status);`); - db.exec(`CREATE INDEX IF NOT EXISTS idx_task_runs_runtime_status ON task_runs(runtime, status);`); - db.exec(`CREATE INDEX IF NOT EXISTS idx_task_runs_cleanup_after ON task_runs(cleanup_after);`); - db.exec(`CREATE INDEX IF NOT EXISTS idx_task_runs_last_event_at ON task_runs(last_event_at);`); - db.exec(`CREATE INDEX IF NOT EXISTS idx_task_runs_owner_key ON task_runs(owner_key);`); - db.exec(`CREATE INDEX IF NOT EXISTS idx_task_runs_parent_flow_id ON task_runs(parent_flow_id);`); - db.exec( - `CREATE INDEX IF NOT EXISTS idx_task_runs_child_session_key ON task_runs(child_session_key);`, +function selectTaskDeliveryStateRows(db: DatabaseSync): TaskDeliveryStateRow[] { + const query = getTaskRegistryKysely(db) + .selectFrom("task_delivery_state") + .select(["task_id", "requester_origin_json", "last_notified_event_at"]) + .orderBy("task_id", "asc"); + return executeSqliteQuerySync(db, query).rows; +} + +function upsertTaskRow(db: DatabaseSync, row: Insertable): void { + executeSqliteQuerySync( + db, + getTaskRegistryKysely(db) + .insertInto("task_runs") + .values(row) + .onConflict((conflict) => + conflict.column("task_id").doUpdateSet({ + runtime: (eb) => eb.ref("excluded.runtime"), + task_kind: (eb) => eb.ref("excluded.task_kind"), + source_id: (eb) => eb.ref("excluded.source_id"), + requester_session_key: (eb) => eb.ref("excluded.requester_session_key"), + owner_key: (eb) => eb.ref("excluded.owner_key"), + scope_kind: (eb) => eb.ref("excluded.scope_kind"), + child_session_key: (eb) => eb.ref("excluded.child_session_key"), + parent_flow_id: (eb) => eb.ref("excluded.parent_flow_id"), + parent_task_id: (eb) => eb.ref("excluded.parent_task_id"), + agent_id: (eb) => eb.ref("excluded.agent_id"), + run_id: (eb) => eb.ref("excluded.run_id"), + label: (eb) => eb.ref("excluded.label"), + task: (eb) => eb.ref("excluded.task"), + status: (eb) => eb.ref("excluded.status"), + delivery_status: (eb) => eb.ref("excluded.delivery_status"), + notify_policy: (eb) => eb.ref("excluded.notify_policy"), + created_at: (eb) => eb.ref("excluded.created_at"), + started_at: (eb) => eb.ref("excluded.started_at"), + ended_at: (eb) => eb.ref("excluded.ended_at"), + last_event_at: (eb) => eb.ref("excluded.last_event_at"), + cleanup_after: (eb) => eb.ref("excluded.cleanup_after"), + error: (eb) => eb.ref("excluded.error"), + progress_summary: (eb) => eb.ref("excluded.progress_summary"), + terminal_summary: (eb) => eb.ref("excluded.terminal_summary"), + terminal_outcome: (eb) => eb.ref("excluded.terminal_outcome"), + }), + ), ); } -function ensureTaskRegistryPermissions(pathname: string) { - ensureSqliteStorePermissions({ - dir: resolveTaskRegistryDir(process.env), - pathname, - dirMode: TASK_REGISTRY_DIR_MODE, - fileMode: TASK_REGISTRY_FILE_MODE, - }); +function replaceTaskDeliveryStateRow( + db: DatabaseSync, + row: Insertable, +): void { + executeSqliteQuerySync( + db, + getTaskRegistryKysely(db) + .insertInto("task_delivery_state") + .values(row) + .onConflict((conflict) => + conflict.column("task_id").doUpdateSet({ + requester_origin_json: (eb) => eb.ref("excluded.requester_origin_json"), + last_notified_event_at: (eb) => eb.ref("excluded.last_notified_event_at"), + }), + ), + ); } function openTaskRegistryDatabase(): TaskRegistryDatabase { - const pathname = resolveTaskRegistrySqlitePath(process.env); - if (cachedDatabase && cachedDatabase.path === pathname) { + const database = openOpenClawStateDatabase(); + const pathname = database.path; + if (cachedDatabase && cachedDatabase.path === pathname && cachedDatabase.db.isOpen) { return cachedDatabase; } - if (cachedDatabase) { - cachedDatabase.walMaintenance.close(); - cachedDatabase.db.close(); + if (cachedDatabase && !cachedDatabase.db.isOpen) { cachedDatabase = null; } - ensureTaskRegistryPermissions(pathname); - const { DatabaseSync } = requireNodeSqlite(); - const db = new DatabaseSync(pathname); - const walMaintenance = configureSqliteWalMaintenance(db); - db.exec(`PRAGMA synchronous = NORMAL;`); - db.exec(`PRAGMA busy_timeout = 5000;`); - ensureSchema(db); - ensureTaskRegistryPermissions(pathname); cachedDatabase = { - db, + db: database.db, path: pathname, - statements: createStatements(db), - walMaintenance, }; return cachedDatabase; } -function withWriteTransaction(write: (statements: TaskRegistryStatements) => void) { - const { db, path, statements } = openTaskRegistryDatabase(); - db.exec("BEGIN IMMEDIATE"); - try { - write(statements); - db.exec("COMMIT"); - ensureTaskRegistryPermissions(path); - } catch (error) { - db.exec("ROLLBACK"); - throw error; - } +function withWriteTransaction(write: (database: TaskRegistryDatabase) => void) { + const database = openTaskRegistryDatabase(); + runOpenClawStateWriteTransaction(() => { + write(database); + }); } export function loadTaskRegistryStateFromSqlite(): TaskRegistryStoreSnapshot { - const { statements } = openTaskRegistryDatabase(); - const taskRows = statements.selectAll.all() as TaskRegistryRow[]; - const deliveryRows = statements.selectAllDeliveryStates.all() as TaskDeliveryStateRow[]; + const { db } = openTaskRegistryDatabase(); + const taskRows = selectTaskRows(db); + const deliveryRows = selectTaskDeliveryStateRows(db); return { tasks: new Map(taskRows.map((row) => [row.task_id, rowToTaskRecord(row)])), deliveryStates: new Map(deliveryRows.map((row) => [row.task_id, rowToTaskDeliveryState(row)])), @@ -482,71 +309,144 @@ export function listTaskRegistryRecordsByOwnerKeyFromSqlite(ownerKey: string): T if (!key) { return []; } - const { statements } = openTaskRegistryDatabase(); - const rows = statements.selectByOwnerKey.all(key) as TaskRegistryRow[]; + const { db } = openTaskRegistryDatabase(); + const query = getTaskRegistryKysely(db) + .selectFrom("task_runs") + .select([ + "task_id", + "runtime", + "task_kind", + "source_id", + "requester_session_key", + "owner_key", + "scope_kind", + "child_session_key", + "parent_flow_id", + "parent_task_id", + "agent_id", + "run_id", + "label", + "task", + "status", + "delivery_status", + "notify_policy", + "created_at", + "started_at", + "ended_at", + "last_event_at", + "cleanup_after", + "error", + "progress_summary", + "terminal_summary", + "terminal_outcome", + ]) + .where("owner_key", "=", key) + .orderBy("created_at", "asc") + .orderBy("task_id", "asc"); + const rows = executeSqliteQuerySync(db, query).rows; return rows.map(rowToTaskRecord); } export function saveTaskRegistryStateToSqlite(snapshot: TaskRegistryStoreSnapshot) { - withWriteTransaction((statements) => { - statements.clearDeliveryStates.run(); - statements.clearRows.run(); + withWriteTransaction(({ db }) => { + const kysely = getTaskRegistryKysely(db); + const taskIds = [...snapshot.tasks.keys()]; + if (taskIds.length === 0) { + executeSqliteQuerySync(db, kysely.deleteFrom("task_delivery_state")); + executeSqliteQuerySync(db, kysely.deleteFrom("task_runs")); + return; + } + pruneRowsNotInSnapshot({ + db, + tableName: "task_runs", + columnName: "task_id", + tempTableName: "openclaw_live_task_run_ids", + ids: taskIds, + }); + const deliveryTaskIds = [...snapshot.deliveryStates.keys()]; + if (deliveryTaskIds.length === 0) { + executeSqliteQuerySync(db, kysely.deleteFrom("task_delivery_state")); + } else { + pruneRowsNotInSnapshot({ + db, + tableName: "task_delivery_state", + columnName: "task_id", + tempTableName: "openclaw_live_task_delivery_ids", + ids: deliveryTaskIds, + }); + } for (const task of snapshot.tasks.values()) { - statements.upsertRow.run(bindTaskRecordBase(task)); + upsertTaskRow(db, bindTaskRecordBase(task)); } for (const state of snapshot.deliveryStates.values()) { - statements.replaceDeliveryState.run(bindTaskDeliveryState(state)); + replaceTaskDeliveryStateRow(db, bindTaskDeliveryState(state)); } }); } export function upsertTaskRegistryRecordToSqlite(task: TaskRecord) { - const store = openTaskRegistryDatabase(); - store.statements.upsertRow.run(bindTaskRecordBase(task)); + withWriteTransaction(({ db }) => { + upsertTaskRow(db, bindTaskRecordBase(task)); + }); } export function upsertTaskWithDeliveryStateToSqlite(params: { task: TaskRecord; deliveryState?: TaskDeliveryState; }) { - withWriteTransaction((statements) => { - statements.upsertRow.run(bindTaskRecordBase(params.task)); + withWriteTransaction(({ db }) => { + upsertTaskRow(db, bindTaskRecordBase(params.task)); if (params.deliveryState) { - statements.replaceDeliveryState.run(bindTaskDeliveryState(params.deliveryState)); + replaceTaskDeliveryStateRow(db, bindTaskDeliveryState(params.deliveryState)); } else { - statements.deleteDeliveryState.run(params.task.taskId); + executeSqliteQuerySync( + db, + getTaskRegistryKysely(db) + .deleteFrom("task_delivery_state") + .where("task_id", "=", params.task.taskId), + ); } }); } export function deleteTaskRegistryRecordFromSqlite(taskId: string) { - const store = openTaskRegistryDatabase(); - store.statements.deleteRow.run(taskId); - store.statements.deleteDeliveryState.run(taskId); + withWriteTransaction(({ db }) => { + const kysely = getTaskRegistryKysely(db); + executeSqliteQuerySync( + db, + kysely.deleteFrom("task_delivery_state").where("task_id", "=", taskId), + ); + executeSqliteQuerySync(db, kysely.deleteFrom("task_runs").where("task_id", "=", taskId)); + }); } export function deleteTaskAndDeliveryStateFromSqlite(taskId: string) { - withWriteTransaction((statements) => { - statements.deleteRow.run(taskId); - statements.deleteDeliveryState.run(taskId); + withWriteTransaction(({ db }) => { + const kysely = getTaskRegistryKysely(db); + executeSqliteQuerySync( + db, + kysely.deleteFrom("task_delivery_state").where("task_id", "=", taskId), + ); + executeSqliteQuerySync(db, kysely.deleteFrom("task_runs").where("task_id", "=", taskId)); }); } export function upsertTaskDeliveryStateToSqlite(state: TaskDeliveryState) { - const store = openTaskRegistryDatabase(); - store.statements.replaceDeliveryState.run(bindTaskDeliveryState(state)); + withWriteTransaction(({ db }) => { + replaceTaskDeliveryStateRow(db, bindTaskDeliveryState(state)); + }); } export function deleteTaskDeliveryStateFromSqlite(taskId: string) { - const store = openTaskRegistryDatabase(); - store.statements.deleteDeliveryState.run(taskId); + withWriteTransaction(({ db }) => { + executeSqliteQuerySync( + db, + getTaskRegistryKysely(db).deleteFrom("task_delivery_state").where("task_id", "=", taskId), + ); + }); } -export function closeTaskRegistrySqliteStore() { - if (!cachedDatabase) { - return; - } - cachedDatabase.walMaintenance.close(); - cachedDatabase.db.close(); +export function closeTaskRegistryDatabase() { cachedDatabase = null; + closeOpenClawStateDatabase(); } diff --git a/src/tasks/task-registry.store.test.ts b/src/tasks/task-registry.store.test.ts index cd04274bab4..f7a5450c67d 100644 --- a/src/tasks/task-registry.store.test.ts +++ b/src/tasks/task-registry.store.test.ts @@ -1,7 +1,13 @@ -import { mkdirSync, statSync } from "node:fs"; +import { statSync } from "node:fs"; import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; -import { requireNodeSqlite } from "../infra/node-sqlite.js"; +import { executeSqliteQuerySync, getNodeSqliteKysely } from "../infra/kysely-sync.js"; +import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js"; +import { + closeOpenClawStateDatabase, + openOpenClawStateDatabase, +} from "../state/openclaw-state-db.js"; +import { resolveOpenClawStateSqlitePath } from "../state/openclaw-state-db.paths.js"; import { withOpenClawTestState } from "../test-utils/openclaw-test-state.js"; import { createManagedTaskFlow, resetTaskFlowRegistryForTests } from "./task-flow-registry.js"; import { @@ -14,14 +20,29 @@ import { maybeDeliverTaskStateChangeUpdate, resetTaskRegistryForTests, } from "./task-registry.js"; -import { resolveTaskRegistryDir, resolveTaskRegistrySqlitePath } from "./task-registry.paths.js"; import { configureTaskRegistryRuntime, type TaskRegistryObserverEvent, } from "./task-registry.store.js"; -import type { TaskRecord } from "./task-registry.types.js"; +import { + loadTaskRegistryStateFromSqlite, + saveTaskRegistryStateToSqlite, +} from "./task-registry.store.sqlite.js"; +import type { TaskDeliveryState, TaskRecord } from "./task-registry.types.js"; +import { + parseOptionalTaskTerminalOutcome, + parseTaskDeliveryStatus, + parseTaskNotifyPolicy, + parseTaskRuntime, + parseTaskScopeKind, + parseTaskStatus, +} from "./task-registry.types.js"; const ORIGINAL_STATE_DIR = process.env.OPENCLAW_STATE_DIR; +type TaskRegistryTestDatabase = Pick< + OpenClawStateKyselyDatabase, + "task_delivery_state" | "task_runs" +>; function requireFirstUpsertParams(upsertTaskWithDeliveryState: ReturnType): { task?: { taskId?: string }; @@ -82,9 +103,10 @@ describe("task-registry store runtime", () => { }, }); - const restored = findTaskByRunId("run-restored"); - expect(restored?.taskId).toBe("task-restored"); - expect(restored?.task).toBe("Restored task"); + expect(findTaskByRunId("run-restored")).toMatchObject({ + taskId: "task-restored", + task: "Restored task", + }); expect(loadSnapshot).toHaveBeenCalledTimes(1); createTaskRecord({ @@ -128,6 +150,90 @@ describe("task-registry store runtime", () => { expect(loadSnapshot).toHaveBeenCalledTimes(1); }); + it("rejects invalid persisted task enum values", () => { + expect(parseTaskRuntime("cron")).toBe("cron"); + expect(parseTaskScopeKind("system")).toBe("system"); + expect(parseTaskStatus("running")).toBe("running"); + expect(parseTaskDeliveryStatus("pending")).toBe("pending"); + expect(parseTaskNotifyPolicy("done_only")).toBe("done_only"); + expect(parseOptionalTaskTerminalOutcome("blocked")).toBe("blocked"); + expect(parseOptionalTaskTerminalOutcome(null)).toBeUndefined(); + + expect(() => parseTaskRuntime("timer")).toThrow("Invalid persisted task runtime"); + expect(() => parseTaskScopeKind("workspace")).toThrow("Invalid persisted task scope kind"); + expect(() => parseTaskStatus("done")).toThrow("Invalid persisted task status"); + expect(() => parseTaskDeliveryStatus("ok")).toThrow("Invalid persisted task delivery status"); + expect(() => parseTaskNotifyPolicy("verbose")).toThrow("Invalid persisted task notify policy"); + expect(() => parseOptionalTaskTerminalOutcome("failed")).toThrow( + "Invalid persisted task terminal outcome", + ); + }); + + it("rejects corrupt persisted task rows during sqlite restore", async () => { + await withOpenClawTestState( + { layout: "state-only", prefix: "openclaw-task-store-corrupt-" }, + async () => { + resetTaskRegistryForTests(); + const created = createTaskRecord({ + runtime: "cron", + ownerKey: "agent:main:main", + scopeKind: "session", + sourceId: "job-corrupt", + runId: "run-corrupt-task-status", + task: "Corrupt task row", + status: "running", + deliveryStatus: "not_applicable", + notifyPolicy: "silent", + }); + + const database = openOpenClawStateDatabase(); + const db = getNodeSqliteKysely(database.db); + executeSqliteQuerySync( + database.db, + db.updateTable("task_runs").set({ status: "done" }).where("task_id", "=", created.taskId), + ); + + expect(() => loadTaskRegistryStateFromSqlite()).toThrow("Invalid persisted task status"); + }, + ); + }); + + it("drops invalid requester origins during sqlite restore", async () => { + await withOpenClawTestState( + { layout: "state-only", prefix: "openclaw-task-store-invalid-origin-" }, + async () => { + resetTaskRegistryForTests(); + const created = createTaskRecord({ + runtime: "acp", + ownerKey: "agent:main:main", + scopeKind: "session", + childSessionKey: "agent:main:acp:origin", + runId: "run-invalid-origin", + task: "Invalid origin task", + status: "running", + deliveryStatus: "pending", + requesterOrigin: { + channel: "test-channel", + to: "C1234567890", + }, + }); + + const database = openOpenClawStateDatabase(); + const db = getNodeSqliteKysely(database.db); + executeSqliteQuerySync( + database.db, + db + .updateTable("task_delivery_state") + .set({ requester_origin_json: '["bad-origin"]' }) + .where("task_id", "=", created.taskId), + ); + + const restored = loadTaskRegistryStateFromSqlite(); + expect(restored.deliveryStates.get(created.taskId)?.requesterOrigin).toBeUndefined(); + }, + ); + }); + it("emits incremental observer events for restore, mutation, and delete", () => { const events: TaskRegistryObserverEvent[] = []; configureTaskRegistryRuntime({ @@ -145,10 +251,11 @@ describe("task-registry store runtime", () => { }, }); - const restored = findTaskByRunId("run-restored"); - expect(restored?.runId).toBe("run-restored"); - expect(restored?.taskId).toBe("task-restored"); - expect(restored?.task).toBe("Restored task"); + expect(findTaskByRunId("run-restored")).toMatchObject({ + runId: "run-restored", + taskId: "task-restored", + task: "Restored task", + }); const created = createTaskRecord({ runtime: "acp", ownerKey: "agent:main:main", @@ -162,26 +269,18 @@ describe("task-registry store runtime", () => { expect(deleteTaskRecordById(created.taskId)).toBe(true); expect(events.map((event) => event.kind)).toEqual(["restored", "upserted", "deleted"]); - const restoredEvent = events[0]; - expect(restoredEvent?.kind).toBe("restored"); - if (restoredEvent?.kind !== "restored") { - throw new Error("Expected restored observer event"); - } - expect(restoredEvent.tasks.map((task) => task.taskId)).toEqual(["task-restored"]); - - const upsertedEvent = events[1]; - expect(upsertedEvent?.kind).toBe("upserted"); - if (upsertedEvent?.kind !== "upserted") { - throw new Error("Expected upserted observer event"); - } - expect(upsertedEvent.task.taskId).toBe(created.taskId); - - const deletedEvent = events[2]; - expect(deletedEvent?.kind).toBe("deleted"); - if (deletedEvent?.kind !== "deleted") { - throw new Error("Expected deleted observer event"); - } - expect(deletedEvent.taskId).toBe(created.taskId); + expect(events[0]).toMatchObject({ + kind: "restored", + tasks: [expect.objectContaining({ taskId: "task-restored" })], + }); + expect(events[1]).toMatchObject({ + kind: "upserted", + task: expect.objectContaining({ taskId: created.taskId }), + }); + expect(events[2]).toMatchObject({ + kind: "deleted", + taskId: created.taskId, + }); }); it("uses atomic task-plus-delivery store methods when available", async () => { @@ -219,7 +318,11 @@ describe("task-registry store runtime", () => { expect(deleteTaskRecordById(created.taskId)).toBe(true); expect(upsertTaskWithDeliveryState).toHaveBeenCalled(); - expect(requireFirstUpsertParams(upsertTaskWithDeliveryState).task?.taskId).toBe(created.taskId); + expect(requireFirstUpsertParams(upsertTaskWithDeliveryState)).toMatchObject({ + task: expect.objectContaining({ + taskId: created.taskId, + }), + }); expect( upsertTaskWithDeliveryState.mock.calls.some((call) => { const params = call[0] as { deliveryState?: { lastNotifiedEventAt?: number } }; @@ -229,6 +332,92 @@ describe("task-registry store runtime", () => { expect(deleteTaskWithDeliveryState).toHaveBeenCalledWith(created.taskId); }); + it("persists create requester origin with separate task and delivery store methods", () => { + const upsertTask = vi.fn(); + const upsertDeliveryState = vi.fn(); + configureTaskRegistryRuntime({ + store: { + loadSnapshot: () => ({ + tasks: new Map(), + deliveryStates: new Map(), + }), + saveSnapshot: vi.fn(), + upsertTask, + upsertDeliveryState, + }, + }); + + const created = createTaskRecord({ + runtime: "acp", + ownerKey: "agent:main:main", + scopeKind: "session", + childSessionKey: "agent:codex:acp:new", + runId: "run-separate-store-origin", + task: "Separate store task", + status: "running", + deliveryStatus: "pending", + requesterOrigin: { + channel: "test-channel", + to: "C1234567890", + }, + }); + + expect(upsertTask).toHaveBeenCalledWith(expect.objectContaining({ taskId: created.taskId })); + expect(upsertDeliveryState).toHaveBeenCalledWith({ + taskId: created.taskId, + requesterOrigin: { + channel: "test-channel", + to: "C1234567890", + }, + }); + const taskCallOrder = upsertTask.mock.invocationCallOrder[0]; + const deliveryCallOrder = upsertDeliveryState.mock.invocationCallOrder[0]; + if (taskCallOrder == null || deliveryCallOrder == null) { + throw new Error("Expected separate store upsert calls"); + } + expect(taskCallOrder).toBeLessThan(deliveryCallOrder); + }); + + it("falls back to full snapshots when custom stores cannot upsert delivery state", () => { + const saveSnapshot = vi.fn(); + const upsertTask = vi.fn(); + configureTaskRegistryRuntime({ + store: { + loadSnapshot: () => ({ + tasks: new Map(), + deliveryStates: new Map(), + }), + saveSnapshot, + upsertTask, + }, + }); + + const created = createTaskRecord({ + runtime: "acp", + ownerKey: "agent:main:main", + scopeKind: "session", + childSessionKey: "agent:codex:acp:snapshot-fallback", + runId: "run-snapshot-fallback-origin", + task: "Snapshot fallback task", + status: "running", + deliveryStatus: "pending", + requesterOrigin: { + channel: "test-channel", + to: "C1234567890", + }, + }); + + expect(upsertTask).not.toHaveBeenCalled(); + expect(saveSnapshot).toHaveBeenCalledOnce(); + const snapshot = saveSnapshot.mock.calls[0]?.[0] as { + deliveryStates: ReadonlyMap; + }; + expect(snapshot.deliveryStates.get(created.taskId)?.requesterOrigin).toEqual({ + channel: "test-channel", + to: "C1234567890", + }); + }); + it("restores persisted tasks from the default sqlite store", () => { const created = createTaskRecord({ runtime: "cron", @@ -244,10 +433,48 @@ describe("task-registry store runtime", () => { resetTaskRegistryForTests({ persist: false }); - const restored = findTaskByRunId("run-sqlite"); - expect(restored?.taskId).toBe(created.taskId); - expect(restored?.sourceId).toBe("job-123"); - expect(restored?.task).toBe("Run nightly cron"); + expect(findTaskByRunId("run-sqlite")).toMatchObject({ + taskId: created.taskId, + sourceId: "job-123", + task: "Run nightly cron", + }); + }); + + it("persists requester origin atomically when creating sqlite tasks", async () => { + await withOpenClawTestState( + { layout: "state-only", prefix: "openclaw-task-create-origin-" }, + async () => { + const created = createTaskRecord({ + runtime: "acp", + requesterSessionKey: "agent:main:workspace:channel:C1234567890", + ownerKey: "agent:main:main", + scopeKind: "session", + childSessionKey: "agent:main:workspace:channel:C1234567890", + runId: "run-create-origin", + task: "Reply to channel task", + status: "running", + deliveryStatus: "pending", + notifyPolicy: "done_only", + requesterOrigin: { + channel: "test-channel", + to: "C1234567890", + }, + }); + + resetTaskRegistryForTests({ persist: false }); + + expect(findTaskByRunId("run-create-origin")).toMatchObject({ + taskId: created.taskId, + }); + const deliveryState = getTaskRegistrySnapshot().deliveryStates.find( + (state) => state.taskId === created.taskId, + ); + expect(deliveryState?.requesterOrigin).toEqual({ + channel: "test-channel", + to: "C1234567890", + }); + }, + ); }); it("persists parentFlowId with task rows", () => { @@ -270,9 +497,10 @@ describe("task-registry store runtime", () => { resetTaskRegistryForTests({ persist: false }); - const restored = findTaskByRunId("run-flow-linked"); - expect(restored?.taskId).toBe(created.taskId); - expect(restored?.parentFlowId).toBe(flow.flowId); + expect(findTaskByRunId("run-flow-linked")).toMatchObject({ + taskId: created.taskId, + parentFlowId: flow.flowId, + }); }); it("preserves requesterSessionKey when it differs from ownerKey across sqlite restore", () => { @@ -291,54 +519,12 @@ describe("task-registry store runtime", () => { resetTaskRegistryForTests({ persist: false }); - const restored = findTaskByRunId("run-requester-session-restore"); - expect(restored?.taskId).toBe(created.taskId); - expect(restored?.requesterSessionKey).toBe("agent:main:workspace:channel:C1234567890"); - expect(restored?.ownerKey).toBe("agent:main:main"); - expect(restored?.childSessionKey).toBe("agent:main:workspace:channel:C1234567890"); - }); - - it("drops malformed requester origin json from sqlite delivery state", async () => { - await withOpenClawTestState( - { layout: "state-only", prefix: "openclaw-task-store-origin-shape-" }, - async () => { - const created = createTaskRecord({ - runtime: "acp", - ownerKey: "agent:main:main", - scopeKind: "session", - requesterOrigin: { - channel: "notifychat", - to: "notifychat:123", - }, - childSessionKey: "agent:main:acp:origin-shape", - runId: "run-origin-shape", - task: "Restore malformed origin", - status: "running", - deliveryStatus: "pending", - notifyPolicy: "state_changes", - }); - - const sqlitePath = resolveTaskRegistrySqlitePath(process.env); - const { DatabaseSync } = requireNodeSqlite(); - const db = new DatabaseSync(sqlitePath); - db.prepare( - `INSERT OR REPLACE INTO task_delivery_state ( - task_id, - requester_origin_json, - last_notified_event_at - ) VALUES (?, ?, ?)`, - ).run(created.taskId, JSON.stringify(["notifychat", "123"]), 321); - db.close(); - - resetTaskRegistryForTests({ persist: false }); - - const deliveryState = getTaskRegistrySnapshot().deliveryStates.find( - (state) => state.taskId === created.taskId, - ); - expect(deliveryState?.lastNotifiedEventAt).toBe(321); - expect(deliveryState?.requesterOrigin).toBeUndefined(); - }, - ); + expect(findTaskByRunId("run-requester-session-restore")).toMatchObject({ + taskId: created.taskId, + requesterSessionKey: "agent:main:workspace:channel:C1234567890", + ownerKey: "agent:main:main", + childSessionKey: "agent:main:workspace:channel:C1234567890", + }); }); it("preserves taskKind across sqlite restore", () => { @@ -357,10 +543,112 @@ describe("task-registry store runtime", () => { resetTaskRegistryForTests({ persist: false }); - const restored = findTaskByRunId("run-task-kind-restore"); - expect(restored?.taskId).toBe(created.taskId); - expect(restored?.taskKind).toBe("video_generation"); - expect(restored?.runId).toBe("run-task-kind-restore"); + expect(findTaskByRunId("run-task-kind-restore")).toMatchObject({ + taskId: created.taskId, + taskKind: "video_generation", + runId: "run-task-kind-restore", + }); + }); + + it("prunes stale sqlite delivery state while retaining current rows", async () => { + await withOpenClawTestState( + { layout: "state-only", prefix: "openclaw-task-delivery-prune-" }, + async () => { + const taskA = createStoredTask(); + const taskB: TaskRecord = { + ...createStoredTask(), + taskId: "task-retained-delivery-b", + runId: "run-retained-delivery-b", + }; + const deliveryA: TaskDeliveryState = { + taskId: taskA.taskId, + lastNotifiedEventAt: 100, + }; + const deliveryB: TaskDeliveryState = { + taskId: taskB.taskId, + lastNotifiedEventAt: 200, + }; + + saveTaskRegistryStateToSqlite({ + tasks: new Map([ + [taskA.taskId, taskA], + [taskB.taskId, taskB], + ]), + deliveryStates: new Map([ + [deliveryA.taskId, deliveryA], + [deliveryB.taskId, deliveryB], + ]), + }); + + saveTaskRegistryStateToSqlite({ + tasks: new Map([ + [taskA.taskId, taskA], + [taskB.taskId, taskB], + ]), + deliveryStates: new Map([[deliveryB.taskId, deliveryB]]), + }); + + const restored = loadTaskRegistryStateFromSqlite(); + expect(restored.deliveryStates.has(taskA.taskId)).toBe(false); + expect(restored.deliveryStates.get(taskB.taskId)).toEqual(deliveryB); + }, + ); + }); + + it("prunes large sqlite snapshots without binding every task id at once", async () => { + await withOpenClawTestState( + { layout: "state-only", prefix: "openclaw-task-large-prune-" }, + async () => { + const tasks = new Map(); + const deliveryStates = new Map(); + for (let index = 0; index < 1_200; index++) { + const task: TaskRecord = { + ...createStoredTask(), + taskId: `task-large-${index}`, + runId: `run-large-${index}`, + createdAt: index, + lastEventAt: index, + }; + tasks.set(task.taskId, task); + deliveryStates.set(task.taskId, { + taskId: task.taskId, + lastNotifiedEventAt: index, + }); + } + + saveTaskRegistryStateToSqlite({ tasks, deliveryStates }); + const retainedTasks = new Map([...tasks].slice(100)); + const retainedDeliveryStates = new Map([...deliveryStates].slice(100)); + saveTaskRegistryStateToSqlite({ + tasks: retainedTasks, + deliveryStates: retainedDeliveryStates, + }); + + const restored = loadTaskRegistryStateFromSqlite(); + expect(restored.tasks.size).toBe(1_100); + expect(restored.deliveryStates.size).toBe(1_100); + expect(restored.tasks.has("task-large-0")).toBe(false); + expect(restored.tasks.has("task-large-1199")).toBe(true); + }, + ); + }); + + it("reopens after the shared state database is closed", async () => { + await withOpenClawTestState( + { layout: "state-only", prefix: "openclaw-task-store-" }, + async () => { + const task = createStoredTask(); + saveTaskRegistryStateToSqlite({ + tasks: new Map([[task.taskId, task]]), + deliveryStates: new Map(), + }); + + closeOpenClawStateDatabase(); + + const restored = loadTaskRegistryStateFromSqlite(); + expect(restored.tasks.get(task.taskId)).toEqual(task); + }, + ); }); it("hardens the sqlite task store directory and file modes", async () => { @@ -382,181 +670,11 @@ describe("task-registry store runtime", () => { notifyPolicy: "silent", }); - const registryDir = resolveTaskRegistryDir(process.env); - const sqlitePath = resolveTaskRegistrySqlitePath(process.env); + const databasePath = resolveOpenClawStateSqlitePath(process.env); + const registryDir = path.dirname(databasePath); + expect(databasePath.endsWith(path.join("state", "openclaw.sqlite"))).toBe(true); expect(statSync(registryDir).mode & 0o777).toBe(0o700); - expect(statSync(sqlitePath).mode & 0o777).toBe(0o600); - }, - ); - }); - - it("migrates legacy ownerless cron rows to system scope", async () => { - await withOpenClawTestState( - { layout: "state-only", prefix: "openclaw-task-store-legacy-" }, - async () => { - const sqlitePath = resolveTaskRegistrySqlitePath(process.env); - mkdirSync(path.dirname(sqlitePath), { recursive: true }); - const { DatabaseSync } = requireNodeSqlite(); - const db = new DatabaseSync(sqlitePath); - db.exec(` - CREATE TABLE task_runs ( - task_id TEXT PRIMARY KEY, - runtime TEXT NOT NULL, - source_id TEXT, - requester_session_key TEXT NOT NULL, - child_session_key TEXT, - parent_task_id TEXT, - agent_id TEXT, - run_id TEXT, - label TEXT, - task TEXT NOT NULL, - status TEXT NOT NULL, - delivery_status TEXT NOT NULL, - notify_policy TEXT NOT NULL, - created_at INTEGER NOT NULL, - started_at INTEGER, - ended_at INTEGER, - last_event_at INTEGER, - cleanup_after INTEGER, - error TEXT, - progress_summary TEXT, - terminal_summary TEXT, - terminal_outcome TEXT - ); - `); - db.exec(` - CREATE TABLE task_delivery_state ( - task_id TEXT PRIMARY KEY, - requester_origin_json TEXT, - last_notified_event_at INTEGER - ); - `); - db.prepare(` - INSERT INTO task_runs ( - task_id, - runtime, - source_id, - requester_session_key, - child_session_key, - run_id, - task, - status, - delivery_status, - notify_policy, - created_at, - last_event_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - `).run( - "legacy-cron-task", - "cron", - "nightly-digest", - "", - "agent:main:cron:nightly-digest", - "legacy-cron-run", - "Nightly digest", - "running", - "not_applicable", - "silent", - 100, - 100, - ); - db.close(); - - resetTaskRegistryForTests({ persist: false }); - - const restored = findTaskByRunId("legacy-cron-run"); - expect(restored?.taskId).toBe("legacy-cron-task"); - expect(restored?.ownerKey).toBe("system:cron:nightly-digest"); - expect(restored?.scopeKind).toBe("system"); - expect(restored?.deliveryStatus).toBe("not_applicable"); - expect(restored?.notifyPolicy).toBe("silent"); - }, - ); - }); - - it("keeps legacy requester_session_key rows writable after restore", async () => { - await withOpenClawTestState( - { layout: "state-only", prefix: "openclaw-task-store-legacy-write-" }, - async () => { - const sqlitePath = resolveTaskRegistrySqlitePath(process.env); - mkdirSync(path.dirname(sqlitePath), { recursive: true }); - const { DatabaseSync } = requireNodeSqlite(); - const db = new DatabaseSync(sqlitePath); - db.exec(` - CREATE TABLE task_runs ( - task_id TEXT PRIMARY KEY, - runtime TEXT NOT NULL, - source_id TEXT, - requester_session_key TEXT NOT NULL, - child_session_key TEXT, - parent_task_id TEXT, - agent_id TEXT, - run_id TEXT, - label TEXT, - task TEXT NOT NULL, - status TEXT NOT NULL, - delivery_status TEXT NOT NULL, - notify_policy TEXT NOT NULL, - created_at INTEGER NOT NULL, - started_at INTEGER, - ended_at INTEGER, - last_event_at INTEGER, - cleanup_after INTEGER, - error TEXT, - progress_summary TEXT, - terminal_summary TEXT, - terminal_outcome TEXT - ); - `); - db.exec(` - CREATE TABLE task_delivery_state ( - task_id TEXT PRIMARY KEY, - requester_origin_json TEXT, - last_notified_event_at INTEGER - ); - `); - db.prepare(` - INSERT INTO task_runs ( - task_id, - runtime, - requester_session_key, - run_id, - task, - status, - delivery_status, - notify_policy, - created_at, - last_event_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - `).run( - "legacy-session-task", - "acp", - "agent:main:main", - "legacy-session-run", - "Legacy session task", - "running", - "pending", - "done_only", - 100, - 100, - ); - db.close(); - - resetTaskRegistryForTests({ persist: false }); - - const lost = markTaskLostById({ - taskId: "legacy-session-task", - endedAt: 200, - lastEventAt: 200, - error: "session missing", - }); - expect(lost?.taskId).toBe("legacy-session-task"); - expect(lost?.status).toBe("lost"); - expect(lost?.error).toBe("session missing"); - const restored = findTaskByRunId("legacy-session-run"); - expect(restored?.taskId).toBe("legacy-session-task"); - expect(restored?.status).toBe("lost"); - expect(restored?.error).toBe("session missing"); + expect(statSync(databasePath).mode & 0o777).toBe(0o600); }, ); }); diff --git a/src/tasks/task-registry.store.ts b/src/tasks/task-registry.store.ts index 853a425c6d1..b1f068c4000 100644 --- a/src/tasks/task-registry.store.ts +++ b/src/tasks/task-registry.store.ts @@ -1,5 +1,5 @@ import { - closeTaskRegistrySqliteStore, + closeTaskRegistryDatabase, deleteTaskAndDeliveryStateFromSqlite, deleteTaskDeliveryStateFromSqlite, deleteTaskRegistryRecordFromSqlite, @@ -62,7 +62,7 @@ const defaultTaskRegistryStore: TaskRegistryStore = { deleteTask: deleteTaskRegistryRecordFromSqlite, upsertDeliveryState: upsertTaskDeliveryStateToSqlite, deleteDeliveryState: deleteTaskDeliveryStateFromSqlite, - close: closeTaskRegistrySqliteStore, + close: closeTaskRegistryDatabase, }; let configuredTaskRegistryStore: TaskRegistryStore = defaultTaskRegistryStore; diff --git a/src/tasks/task-registry.ts b/src/tasks/task-registry.ts index 432beabb88f..74da8778880 100644 --- a/src/tasks/task-registry.ts +++ b/src/tasks/task-registry.ts @@ -280,7 +280,17 @@ function persistTaskUpsert(task: TaskRecord) { return; } if (store.upsertTask) { + if (deliveryState && !store.upsertDeliveryState) { + store.saveSnapshot({ + tasks, + deliveryStates: taskDeliveryStates, + }); + return; + } store.upsertTask(task); + if (deliveryState && store.upsertDeliveryState) { + store.upsertDeliveryState(deliveryState); + } return; } store.saveSnapshot({ @@ -1662,10 +1672,13 @@ export function createTaskRecord(params: { record.cleanupAfter = resolveTaskCleanupAfter(record); } tasks.set(taskId, record); - upsertTaskDeliveryState({ - taskId, - requesterOrigin: normalizeDeliveryContext(params.requesterOrigin), - }); + const requesterOrigin = normalizeDeliveryContext(params.requesterOrigin); + if (requesterOrigin) { + taskDeliveryStates.set(taskId, { + taskId, + requesterOrigin, + }); + } addRunIdIndex(taskId, record.runId); addOwnerKeyIndex(taskId, record); addParentFlowIdIndex(taskId, record); diff --git a/src/tasks/task-registry.types.ts b/src/tasks/task-registry.types.ts index 3df1a5a8f44..20bcdb7d01b 100644 --- a/src/tasks/task-registry.types.ts +++ b/src/tasks/task-registry.types.ts @@ -27,6 +27,66 @@ export type TaskScopeKind = "session" | "system"; export type TaskStatusCounts = Record; export type TaskRuntimeCounts = Record; +const TASK_RUNTIMES = new Set(["subagent", "acp", "cli", "cron"]); +const TASK_STATUSES = new Set([ + "queued", + "running", + "succeeded", + "failed", + "timed_out", + "cancelled", + "lost", +]); +const TASK_DELIVERY_STATUSES = new Set([ + "pending", + "delivered", + "session_queued", + "failed", + "parent_missing", + "not_applicable", +]); +const TASK_NOTIFY_POLICIES = new Set(["done_only", "state_changes", "silent"]); +const TASK_TERMINAL_OUTCOMES = new Set(["succeeded", "blocked"]); +const TASK_SCOPE_KINDS = new Set(["session", "system"]); + +function parsePersistedTaskValue( + value: unknown, + values: ReadonlySet, + label: string, +): T { + if (typeof value === "string" && values.has(value as T)) { + return value as T; + } + throw new Error(`Invalid persisted task ${label}: ${JSON.stringify(value)}`); +} + +export function parseTaskRuntime(value: unknown): TaskRuntime { + return parsePersistedTaskValue(value, TASK_RUNTIMES, "runtime"); +} + +export function parseTaskStatus(value: unknown): TaskStatus { + return parsePersistedTaskValue(value, TASK_STATUSES, "status"); +} + +export function parseTaskDeliveryStatus(value: unknown): TaskDeliveryStatus { + return parsePersistedTaskValue(value, TASK_DELIVERY_STATUSES, "delivery status"); +} + +export function parseTaskNotifyPolicy(value: unknown): TaskNotifyPolicy { + return parsePersistedTaskValue(value, TASK_NOTIFY_POLICIES, "notify policy"); +} + +export function parseTaskScopeKind(value: unknown): TaskScopeKind { + return parsePersistedTaskValue(value, TASK_SCOPE_KINDS, "scope kind"); +} + +export function parseOptionalTaskTerminalOutcome(value: unknown): TaskTerminalOutcome | undefined { + if (value == null || value === "") { + return undefined; + } + return parsePersistedTaskValue(value, TASK_TERMINAL_OUTCOMES, "terminal outcome"); +} + export type TaskRegistrySummary = { total: number; active: number;