diff --git a/src/cli/cron-cli/register.cron-simple.ts b/src/cli/cron-cli/register.cron-simple.ts index 6d4ee8ac0b12..f3b0c84cd6d2 100644 --- a/src/cli/cron-cli/register.cron-simple.ts +++ b/src/cli/cron-cli/register.cron-simple.ts @@ -64,7 +64,7 @@ async function waitForCronRunCompletion(params: { timeoutMs: number; pollIntervalMs: number; }): Promise { - // Poll the run log rather than cron.run because completion state is written asynchronously. + // Poll the task ledger rather than cron.run because completion state is written asynchronously. const startedAt = Date.now(); for (;;) { const page = (await callGatewayFromCli("cron.runs", params.opts, { diff --git a/src/commands/status.summary.ts b/src/commands/status.summary.ts index f85ed47e0ac4..f388532fd3a0 100644 --- a/src/commands/status.summary.ts +++ b/src/commands/status.summary.ts @@ -13,7 +13,6 @@ import { resolveStorePath } from "../config/sessions/paths.js"; import { listSessionEntries } from "../config/sessions/session-accessor.js"; import { resolveSessionTotalTokens, type SessionEntry } from "../config/sessions/types.js"; import type { OpenClawConfig } from "../config/types.js"; -import { resolveCronJobsStorePath } from "../cron/store.js"; import { listGatewayAgentsBasic } from "../gateway/agent-list.js"; import { resolveHeartbeatSummaryForAgent } from "../infra/heartbeat-summary.js"; import { peekSystemEvents } from "../infra/system-events.js"; @@ -338,10 +337,7 @@ export async function getStatusSummary( const mainSessionKey = resolveMainSessionKey(cfg); const queuedSystemEvents = peekSystemEvents(mainSessionKey); const taskMaintenanceModule = await loadTaskRegistryMaintenanceModule(); - // Configure maintenance store before reading task summaries so cron-backed tasks are in scope. - taskMaintenanceModule.configureTaskRegistryMaintenance({ - cronStorePath: resolveCronJobsStorePath(cfg.cron?.store), - }); + taskMaintenanceModule.configureTaskRegistryMaintenance(); const inspectableTasks = taskMaintenanceModule.reconcileInspectableTasks(); const rawTasks = taskMaintenanceModule.getInspectableTaskRegistrySummary(inspectableTasks); const taskAuditFindings = taskMaintenanceModule.getInspectableTaskAuditFindings(inspectableTasks); diff --git a/src/commands/tasks.ts b/src/commands/tasks.ts index 62e5b8b3b45f..2446e0a9077d 100644 --- a/src/commands/tasks.ts +++ b/src/commands/tasks.ts @@ -118,10 +118,7 @@ async function tryCancelGatewayOwnedTaskViaGateway( } function configureTaskMaintenanceFromConfig(): void { - const cfg = getRuntimeConfig(); - configureTaskRegistryMaintenance({ - cronStorePath: resolveCronJobsStorePath(cfg.cron?.store), - }); + configureTaskRegistryMaintenance(); } type SessionRegistryMaintenanceStoreSummary = { diff --git a/src/cron/service/failure-alerts.ts b/src/cron/service/failure-alerts.ts index 03d524498050..a40adf63b696 100644 --- a/src/cron/service/failure-alerts.ts +++ b/src/cron/service/failure-alerts.ts @@ -168,6 +168,8 @@ export function maybeEmitFailureAlert( error?: string; provider?: string; consecutiveCount: number; + delivery?: "emit" | "record-only"; + occurredAtMs?: number; }, ) { if (!params.alertConfig || params.consecutiveCount < params.alertConfig.after) { @@ -177,7 +179,7 @@ export function maybeEmitFailureAlert( if (isBestEffort) { return; } - const now = state.deps.nowMs(); + const now = params.occurredAtMs ?? state.deps.nowMs(); const lastAlert = params.job.state.lastFailureAlertAtMs; // Cooldown is stored on job state so process restarts and service reloads do // not spam operators with repeated alerts for the same failing job. @@ -186,16 +188,18 @@ export function maybeEmitFailureAlert( if (inCooldown) { return; } - emitFailureAlert(state, { - job: params.job, - error: params.error, - consecutiveErrors: params.consecutiveCount, - channel: params.alertConfig.channel, - to: params.alertConfig.to, - mode: params.alertConfig.mode, - accountId: params.alertConfig.accountId, - status: params.status, - provider: params.provider, - }); + if (params.delivery !== "record-only") { + emitFailureAlert(state, { + job: params.job, + error: params.error, + consecutiveErrors: params.consecutiveCount, + channel: params.alertConfig.channel, + to: params.alertConfig.to, + mode: params.alertConfig.mode, + accountId: params.alertConfig.accountId, + status: params.status, + provider: params.provider, + }); + } params.job.state.lastFailureAlertAtMs = now; } diff --git a/src/cron/service/ops.test.ts b/src/cron/service/ops.test.ts index 82d7a69560bd..e6918fb3619e 100644 --- a/src/cron/service/ops.test.ts +++ b/src/cron/service/ops.test.ts @@ -3,8 +3,12 @@ import fs from "node:fs/promises"; import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; import { runOpenClawStateWriteTransaction } from "../../state/openclaw-state-db.js"; -import * as detachedTaskRuntime from "../../tasks/detached-task-runtime.js"; -import { findTaskByRunId, resetTaskRegistryForTests } from "../../tasks/task-registry.js"; +import * as taskExecutor from "../../tasks/task-executor.js"; +import { + findTaskByRunId, + listTaskRecordsUnsorted, + resetTaskRegistryForTests, +} from "../../tasks/task-registry.js"; import { formatTaskStatusDetail } from "../../tasks/task-status.js"; import { withEnvAsync } from "../../test-utils/env.js"; import * as cronSchedule from "../schedule.js"; @@ -14,6 +18,7 @@ import { loadCronJobsStoreWithConfigJobs, loadCronStore } from "../store.js"; import type { CronJob } from "../types.js"; import { add, list, remove, run, start, stop, update } from "./ops.js"; import { createCronServiceState, type CronEvent } from "./state.js"; +import { tryCreateCronTaskRun, tryFinishCronTaskRun } from "./task-runs.js"; import { runMissedJobs } from "./timer.js"; const { logger, makeStorePath } = setupCronServiceSuite({ @@ -195,7 +200,7 @@ function expectTaskRun(params: { sourceId: string; progressSummary?: string; }) { - const task = findTaskByRunId(params.runId); + const task = findCronTaskByBaseRunId(params.runId); expect(task?.runtime).toBe(params.runtime); expect(task?.status).toBe(params.status); expect(task?.sourceId).toBe(params.sourceId); @@ -204,6 +209,13 @@ function expectTaskRun(params: { } } +function findCronTaskByBaseRunId(baseRunId: string) { + return ( + findTaskByRunId(baseRunId) ?? + listTaskRecordsUnsorted().find((task) => task.runId?.startsWith(`${baseRunId}:`)) + ); +} + function createMissedIsolatedJob(now: number): CronJob { return { id: "startup-timeout", @@ -388,6 +400,136 @@ describe("cron service ops seam coverage", () => { stop(state); }); + it("preserves a finalized canonical task run when startup finds its stale marker", async () => { + const { storePath } = await makeStorePath(); + const now = Date.parse("2026-03-23T12:00:00.000Z"); + const reservedAt = now - 30 * 60_000; + const startedAt = reservedAt + 250; + const endedAt = startedAt + 4_000; + + await withStateDirForStorePath(storePath, async () => { + const job = createInterruptedMainJob(now); + job.trigger = { script: "json({ fire: true })", once: true }; + job.state.triggerState = { cursor: "old" }; + await writeCronStoreSnapshot({ storePath, jobs: [job] }); + const events: CronEvent[] = []; + const state = createCronServiceState({ + storePath, + cronEnabled: true, + log: logger, + nowMs: () => now, + enqueueSystemEvent: vi.fn(), + requestHeartbeat: vi.fn(), + runIsolatedAgentJob: vi.fn(async () => ({ status: "ok" as const })), + onEvent: (event) => events.push(structuredClone(event)), + }); + const taskRunId = tryCreateCronTaskRun({ + state, + job, + startedAt, + runIdStartedAt: reservedAt, + }); + if (!taskRunId) { + throw new Error("expected reserved cron task run"); + } + + tryFinishCronTaskRun(state, { + taskRunId, + job, + triggerEval: { fired: true, stateChanged: true, state: { cursor: "new" } }, + event: { + jobId: job.id, + action: "finished", + job, + status: "ok", + summary: "completed before crash", + delivered: true, + deliveryStatus: "delivered", + failureNotificationDelivery: { status: "not-requested" }, + runAtMs: startedAt, + durationMs: endedAt - startedAt, + triggerFired: true, + }, + }); + + await start(state); + + expect(findTaskByRunId(taskRunId)).toMatchObject({ + status: "succeeded", + startedAt, + terminalSummary: "completed before crash", + endedAt, + detail: { kind: "cron-run", status: "ok", triggerFired: true }, + }); + const persisted = await loadCronStore(storePath); + expect(persisted.jobs[0]).toMatchObject({ + enabled: false, + state: { + lastRunAtMs: startedAt, + lastRunStatus: "ok", + lastStatus: "ok", + lastDurationMs: endedAt - startedAt, + lastDelivered: true, + lastDeliveryStatus: "delivered", + lastTriggerEvalAtMs: endedAt, + lastTriggerFireAtMs: endedAt, + triggerState: { cursor: "new" }, + }, + }); + expect(persisted.jobs[0]?.state.runningAtMs).toBeUndefined(); + expect(persisted.jobs[0]?.state.lastError).toBeUndefined(); + expect(persisted.jobs[0]?.state.nextRunAtMs).toBeUndefined(); + expect(events.filter((event) => event.action === "finished")).toEqual([]); + stop(state); + }); + }); + + it("restores finalized failure-alert cooldown without redelivery", async () => { + const { storePath } = await makeStorePath(); + const now = Date.parse("2026-03-23T12:00:00.000Z"); + const startedAt = now - 30 * 60_000; + const endedAt = startedAt + 4_000; + + await withStateDirForStorePath(storePath, async () => { + const job = createInterruptedMainJob(now); + await writeCronStoreSnapshot({ storePath, jobs: [job] }); + const sendCronFailureAlert = vi.fn(async () => undefined); + const state = createCronServiceState({ + storePath, + cronEnabled: true, + cronConfig: { failureAlert: { enabled: true, after: 1, cooldownMs: 60_000 } }, + log: logger, + nowMs: () => now, + enqueueSystemEvent: vi.fn(), + requestHeartbeat: vi.fn(), + sendCronFailureAlert, + runIsolatedAgentJob: vi.fn(async () => ({ status: "ok" as const })), + }); + + tryFinishCronTaskRun(state, { + job, + event: { + jobId: job.id, + action: "finished", + job, + status: "error", + error: "provider unavailable", + runAtMs: startedAt, + durationMs: endedAt - startedAt, + nextRunAtMs: now + 30 * 60_000, + }, + }); + + await start(state); + + const persisted = await loadCronStore(storePath); + expect(persisted.jobs[0]?.state.lastFailureAlertAtMs).toBe(endedAt); + expect(persisted.jobs[0]?.state.consecutiveErrors).toBe(1); + expect(sendCronFailureAlert).not.toHaveBeenCalled(); + stop(state); + }); + }); + it("start persists load-time updatedAtMs repairs to the state sidecar only", async () => { const { storePath } = await makeStorePath(); const now = Date.parse("2026-04-09T08:00:00.000Z"); @@ -453,7 +595,7 @@ describe("cron service ops seam coverage", () => { }); expectTaskRun({ - runId: `cron:isolated-timeout:${now}`, + runId: `cron:isolated-timeout:${now}:${manualRunId}`, runtime: "cron", status: "succeeded", sourceId: "isolated-timeout", @@ -483,6 +625,51 @@ describe("cron service ops seam coverage", () => { status: "timed_out", sourceId: "isolated-timeout", }); + expect(findCronTaskByBaseRunId(`cron:isolated-timeout:${now}`)?.detail).toMatchObject({ + kind: "cron-run", + status: "error", + runAtMs: now, + durationMs: 0, + }); + }); + }); + + it("records failed manual runs with cron outcome detail", async () => { + const { storePath } = await makeStorePath(); + const now = Date.parse("2026-03-23T12:00:00.000Z"); + + await withStateDirForStorePath(storePath, async () => { + await writeDueIsolatedJobSnapshot(storePath, now); + const state = createCronServiceState({ + storePath, + cronEnabled: true, + log: logger, + nowMs: () => now, + enqueueSystemEvent: vi.fn(), + requestHeartbeat: vi.fn(), + runIsolatedAgentJob: vi.fn(async () => ({ + status: "error" as const, + error: "provider failed", + provider: "openai", + model: "gpt-test", + })), + }); + + await run(state, "isolated-timeout"); + + const task = findCronTaskByBaseRunId(`cron:isolated-timeout:${now}`); + expect(task).toMatchObject({ + status: "failed", + error: "provider failed", + detail: { + kind: "cron-run", + status: "error", + provider: "openai", + model: "gpt-test", + runAtMs: now, + durationMs: 0, + }, + }); }); }); @@ -496,7 +683,7 @@ describe("cron service ops seam coverage", () => { }); const createTaskRecordSpy = vi - .spyOn(detachedTaskRuntime, "createRunningTaskRun") + .spyOn(taskExecutor, "createRunningTaskRun") .mockImplementation(() => { throw new Error("disk full"); }); @@ -519,7 +706,7 @@ describe("cron service ops seam coverage", () => { await writeDueIsolatedJobSnapshot(storePath, now); const updateTaskRecordSpy = vi - .spyOn(detachedTaskRuntime, "completeTaskRunByRunId") + .spyOn(taskExecutor, "finalizeTaskRunByRunId") .mockImplementation(() => { throw new Error("disk full"); }); @@ -653,7 +840,7 @@ describe("cron service ops seam coverage", () => { expect(state.deps.runIsolatedAgentJob).toHaveBeenCalledTimes(1); }); - const task = findTaskByRunId(`cron:isolated-timeout:${now}`); + const task = findCronTaskByBaseRunId(`cron:isolated-timeout:${now}`); if (!task) { throw new Error("expected active manual cron task ledger record"); } diff --git a/src/cron/service/ops.ts b/src/cron/service/ops.ts index 5ae40cf0d488..26558d82f7fe 100644 --- a/src/cron/service/ops.ts +++ b/src/cron/service/ops.ts @@ -9,11 +9,6 @@ import { runWithGatewayIndependentRootWorkContinuation } from "../../process/gat import { CommandLane } from "../../process/lanes.js"; import { DEFAULT_AGENT_ID } from "../../routing/session-key.js"; import { resolveOpenClawStateSqlitePath } from "../../state/openclaw-state-db.paths.js"; -import { - completeTaskRunByRunId, - createRunningTaskRun, - failTaskRunByRunId, -} from "../../tasks/detached-task-runtime.js"; import { clearCronJobActive, isCronActiveJobMarkerCurrent, @@ -22,10 +17,10 @@ import { } from "../active-jobs.js"; import { resolveCronDeliveryPlan, resolveFailureDestination } from "../delivery-plan.js"; import { createCronRunDiagnosticsFromError } from "../run-diagnostics.js"; -import { createCronExecutionId } from "../run-id.js"; +import type { CronRunLogEntry } from "../run-log-types.js"; import { normalizeCronRunLogJobId } from "../run-log.js"; import { cronSchedulingInputsEqual } from "../schedule-identity.js"; -import type { CronJob, CronJobCreate, CronJobPatch, CronPayload } from "../types.js"; +import type { CronJob, CronJobCreate, CronJobPatch, CronPayload, CronRunStatus } from "../types.js"; import { normalizeCronRunErrorText } from "./execution-errors.js"; import { failureNotificationDeliveryFromJobState } from "./failure-alerts.js"; import { @@ -68,12 +63,19 @@ import { type CronRollbackSnapshot, warnIfDisabled, } from "./store.js"; -import { CRON_TASK_RUNNING_PROGRESS_SUMMARY } from "./task-ledger.js"; +import { + tryCreateCronTaskRun, + tryFindCronTaskRunIdForRecovery, + tryFindFinalizedCronTaskRun, + tryFinishCronTaskRun, + tryFinishCronTaskRunWithoutHistory, +} from "./task-runs.js"; import { applyJobResult, applyTriggerNoFireResult, applyTriggerRunResult, armTimer, + type CronTriggerEvalOutcome, executeJobCoreWithTimeout, type IsolatedAgentSetupTimeoutSignal, maybeNotifyIsolatedAgentSetupTimeout, @@ -86,6 +88,7 @@ const STARTUP_INTERRUPTED_ERROR = "cron: job interrupted by gateway restart"; type InterruptedStartupRun = { jobId: string; + taskRunId?: string; runAtMs: number; durationMs: number; }; @@ -146,6 +149,7 @@ function resolveInterruptedStartupFailureNotificationStatus(params: { function markInterruptedStartupRun(params: { state: CronServiceState; job: CronJob; + taskRunId?: string; runningAtMs: number; nowMs: number; }): InterruptedStartupRun { @@ -188,11 +192,61 @@ function markInterruptedStartupRun(params: { return { jobId: job.id, + ...(params.taskRunId ? { taskRunId: params.taskRunId } : {}), runAtMs: runningAtMs, durationMs: job.state.lastDurationMs, }; } +function restoreFinalizedStartupRun(params: { + state: CronServiceState; + job: CronJob; + runningAtMs: number; + entry: CronRunLogEntry & { status: CronRunStatus }; + triggerEval?: CronTriggerEvalOutcome; +}): boolean { + const { state, job, runningAtMs, entry } = params; + const startedAt = entry.runAtMs ?? runningAtMs; + const shouldDelete = applyJobResult( + state, + job, + { + status: entry.status, + error: entry.error, + deliveryError: entry.deliveryError, + diagnostics: entry.diagnostics, + delivered: entry.delivered, + provider: entry.provider, + startedAt, + endedAt: entry.ts, + }, + { replayFailureAlertAtMs: entry.ts }, + ); + + // The finalized row captured post-run state before the stale cron store write. + job.state.lastDurationMs = entry.durationMs ?? Math.max(0, entry.ts - startedAt); + job.state.lastErrorReason = entry.errorReason; + job.state.lastDelivered = entry.delivered; + job.state.lastDeliveryStatus = entry.deliveryStatus; + job.state.lastDeliveryError = entry.deliveryError; + job.state.lastFailureNotificationDelivered = entry.failureNotificationDelivery?.delivered; + job.state.lastFailureNotificationDeliveryStatus = entry.failureNotificationDelivery?.status; + job.state.lastFailureNotificationDeliveryError = entry.failureNotificationDelivery?.error; + job.state.nextRunAtMs = entry.nextRunAtMs; + if (params.triggerEval) { + applyTriggerRunResult(job, { + status: entry.status, + endedAt: entry.ts, + triggerEval: params.triggerEval, + }); + } + state.deps.log.info( + { jobId: job.id, runningAtMs, status: entry.status }, + "cron: restored finalized task-ledger run on startup", + ); + return shouldDelete; +} + function mergeManualRunSnapshotAfterReload(params: { state: CronServiceState; jobId: string; @@ -245,7 +299,8 @@ export async function start(state: CronServiceState) { const interruptedJobIds = new Set(); const interruptedRuns: InterruptedStartupRun[] = []; - let markedAnyInterruptedRun = false; + const completedJobIdsToDelete = new Set(); + let repairedAnyStartupRun = false; await locked(state, async () => { await ensureLoaded(state, { skipRecompute: true }); if (state.stopped) { @@ -255,20 +310,43 @@ export async function start(state: CronServiceState) { for (const job of jobs) { job.state ??= {}; if (typeof job.state.runningAtMs === "number") { + const runningAtMs = job.state.runningAtMs; + const taskRunId = tryFindCronTaskRunIdForRecovery(state, job.id, runningAtMs); + const finalized = tryFindFinalizedCronTaskRun(state, job.id, runningAtMs); + if (finalized) { + interruptedJobIds.add(job.id); + if ( + restoreFinalizedStartupRun({ + state, + job, + runningAtMs, + entry: finalized.entry, + ...(finalized.triggerEval ? { triggerEval: finalized.triggerEval } : {}), + }) + ) { + completedJobIdsToDelete.add(job.id); + } + repairedAnyStartupRun = true; + continue; + } const nowMs = state.deps.nowMs(); const interrupted = markInterruptedStartupRun({ state, job, - runningAtMs: job.state.runningAtMs, + taskRunId, + runningAtMs, nowMs, }); interruptedJobIds.add(job.id); interruptedRuns.push(interrupted); - markedAnyInterruptedRun = true; + repairedAnyStartupRun = true; } } - if (markedAnyInterruptedRun || jobs.length > 0) { - await persist(state, markedAnyInterruptedRun ? undefined : { stateOnly: true }); + if (completedJobIdsToDelete.size > 0 && state.store) { + state.store.jobs = jobs.filter((job) => !completedJobIdsToDelete.has(job.id)); + } + if (repairedAnyStartupRun || jobs.length > 0) { + await persist(state, repairedAnyStartupRun ? undefined : { stateOnly: true }); } }); @@ -294,20 +372,27 @@ export async function start(state: CronServiceState) { } for (const interrupted of interruptedRuns) { const job = state.store?.jobs.find((entry) => entry.id === interrupted.jobId); - emit(state, { - jobId: interrupted.jobId, - action: "finished", - job, - status: "error", - error: STARTUP_INTERRUPTED_ERROR, - delivered: false, - deliveryStatus: "unknown", - deliveryError: STARTUP_INTERRUPTED_ERROR, - failureNotificationDelivery: job ? failureNotificationDeliveryFromJobState(job) : undefined, - runAtMs: interrupted.runAtMs, - durationMs: interrupted.durationMs, - nextRunAtMs: job?.state.nextRunAtMs, - }); + emitCronRunFinished( + state, + { + jobId: interrupted.jobId, + action: "finished", + job, + status: "error", + error: STARTUP_INTERRUPTED_ERROR, + delivered: false, + deliveryStatus: "unknown", + deliveryError: STARTUP_INTERRUPTED_ERROR, + failureNotificationDelivery: job + ? failureNotificationDeliveryFromJobState(job) + : undefined, + runAtMs: interrupted.runAtMs, + durationMs: interrupted.durationMs, + nextRunAtMs: job?.state.nextRunAtMs, + }, + undefined, + interrupted.taskRunId, + ); } armTimer(state); state.deps.log.info( @@ -821,11 +906,19 @@ type ManualRunOptions = { type ManualRunTerminalTracker = { emitted: boolean }; -function emitManualRunFinished( +function emitCronRunFinished( state: CronServiceState, evt: CronEvent & { action: "finished" }, tracker?: ManualRunTerminalTracker, + taskRunId?: string, + triggerEval?: CronTriggerEvalOutcome, ): void { + tryFinishCronTaskRun(state, { + taskRunId, + job: evt.job, + event: evt, + ...(triggerEval ? { triggerEval } : {}), + }); emit(state, evt); if (tracker) { tracker.emitted = true; @@ -875,11 +968,12 @@ async function skipInvalidPersistedManualRun(params: { { preserveSchedule: params.mode === "force" }, ); - emitManualRunFinished( + emitCronRunFinished( params.state, { jobId: params.job.id, action: "finished", + job: params.job, status: "skipped", error: errorText, diagnostics, @@ -899,91 +993,6 @@ async function skipInvalidPersistedManualRun(params: { armTimer(params.state); } -function tryCreateManualTaskRun(params: { - state: CronServiceState; - job: CronJob; - startedAt: number; -}): string | undefined { - const runId = createCronExecutionId(params.job.id, params.startedAt); - try { - const task = createRunningTaskRun({ - runtime: "cron", - sourceId: params.job.id, - ownerKey: "", - scopeKind: "system", - childSessionKey: params.job.sessionKey, - agentId: params.job.agentId, - runId, - label: params.job.name, - task: params.job.name || params.job.id, - deliveryStatus: "not_applicable", - notifyPolicy: "silent", - startedAt: params.startedAt, - lastEventAt: params.startedAt, - progressSummary: CRON_TASK_RUNNING_PROGRESS_SUMMARY, - }); - if (!task) { - params.state.deps.log.warn( - { jobId: params.job.id }, - "cron: task ledger record was not persisted", - ); - return undefined; - } - return runId; - } catch (error) { - params.state.deps.log.warn( - { jobId: params.job.id, error }, - "cron: failed to create task ledger record", - ); - return undefined; - } -} - -function tryFinishManualTaskRun( - state: CronServiceState, - params: { - taskRunId?: string; - coreResult: Awaited>; - endedAt: number; - }, -): void { - if (!params.taskRunId) { - return; - } - try { - if (params.coreResult.status === "ok" || params.coreResult.status === "skipped") { - completeTaskRunByRunId({ - runId: params.taskRunId, - runtime: "cron", - endedAt: params.endedAt, - lastEventAt: params.endedAt, - terminalSummary: params.coreResult.summary ?? undefined, - }); - return; - } - failTaskRunByRunId({ - runId: params.taskRunId, - runtime: "cron", - status: - normalizeCronRunErrorText(params.coreResult.error) === "cron: job execution timed out" - ? "timed_out" - : "failed", - endedAt: params.endedAt, - lastEventAt: params.endedAt, - error: - params.coreResult.status === "error" - ? normalizeCronRunErrorText(params.coreResult.error) - : undefined, - terminalSummary: params.coreResult.summary ?? undefined, - }); - } catch (error) { - state.deps.log.warn( - { runId: params.taskRunId, jobStatus: params.coreResult.status, error }, - "cron: failed to update task ledger record", - ); - } -} - async function inspectManualRunPreflight( state: CronServiceState, id: string, @@ -1088,10 +1097,11 @@ async function prepareManualRun( return { ok: true, ran: false, reason: "stopped" as const }; } emit(state, { jobId: job.id, action: "started", job, runAtMs: preflight.now }); - const taskRunId = tryCreateManualTaskRun({ + const taskRunId = tryCreateCronTaskRun({ state, job, startedAt: preflight.now, + publicRunId: opts?.runId, }); const activeJobMarker = markManualCronJobActive(state, job); // Execute against a snapshot so later reload/merge can preserve delivery @@ -1141,16 +1151,16 @@ async function finishPreparedManualRun( coreResult = { status: "error", error: normalizeCronRunErrorText(err) }; } const endedAt = state.deps.nowMs(); + const triggerSkipped = coreResult.status === "ok" && coreResult.triggerEval?.fired === false; const emitMissingQueuedTerminal = () => { const tracker = prepared.terminalTracker; if (!tracker || tracker.emitted) { return; } const job = state.store?.jobs.find((entry) => entry.id === jobId); - const triggerSkipped = coreResult.status === "ok" && coreResult.triggerEval?.fired === false; // enqueueRun acknowledges a concrete run id, so every accepted request // needs one terminal event even if the job or service owner changes mid-run. - emitManualRunFinished( + emitCronRunFinished( state, { jobId, @@ -1176,13 +1186,22 @@ async function finishPreparedManualRun( usage: coreResult.usage, }, tracker, + taskRunId, ); }; - tryFinishManualTaskRun(state, { - taskRunId, - coreResult, - endedAt, - }); + if (!triggerSkipped) { + // Terminal state must land even if the store merge below throws; the later + // emitCronRunFinished re-finalizes the same row to attach history detail + // (same-status terminal updates apply, so this does not race precedence). + tryFinishCronTaskRunWithoutHistory(state, { + taskRunId, + status: coreResult.status, + error: coreResult.error, + endedAt, + summary: coreResult.summary, + childSessionKey: coreResult.sessionKey, + }); + } if (!isCronActiveJobMarkerCurrent(prepared.activeJobMarker)) { emitMissingQueuedTerminal(); return; @@ -1232,7 +1251,7 @@ async function finishPreparedManualRun( triggerEval: coreResult.triggerEval, }); - emitManualRunFinished( + emitCronRunFinished( state, { jobId: job.id, @@ -1259,6 +1278,8 @@ async function finishPreparedManualRun( usage: coreResult.usage, }, prepared.terminalTracker, + taskRunId, + coreResult.triggerEval, ); } @@ -1303,6 +1324,16 @@ async function finishPreparedManualRun( }); } if (finalized) { + if (triggerSkipped) { + tryFinishCronTaskRunWithoutHistory(state, { + taskRunId, + status: coreResult.status, + error: coreResult.error, + endedAt, + summary: coreResult.summary, + childSessionKey: coreResult.sessionKey, + }); + } armTimer(state); } emitMissingQueuedTerminal(); @@ -1344,7 +1375,7 @@ export async function enqueueRun(state: CronServiceState, id: string, mode?: "du if (result.reason !== "invalid-spec") { const finishedAt = state.deps.nowMs(); const job = state.store?.jobs.find((entry) => entry.id === id); - emitManualRunFinished( + emitCronRunFinished( state, { jobId: id, @@ -1387,7 +1418,7 @@ export async function enqueueRun(state: CronServiceState, id: string, mode?: "du } const finishedAt = state.deps.nowMs(); const job = state.store?.jobs.find((entry) => entry.id === id); - emitManualRunFinished( + emitCronRunFinished( state, { jobId: id, diff --git a/src/cron/service/task-runs.test.ts b/src/cron/service/task-runs.test.ts new file mode 100644 index 000000000000..f38d0d22b3f4 --- /dev/null +++ b/src/cron/service/task-runs.test.ts @@ -0,0 +1,700 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + getDetachedTaskLifecycleRuntime, + resetDetachedTaskLifecycleRuntimeForTests, + setDetachedTaskLifecycleRuntime, +} from "../../tasks/detached-task-runtime.js"; +import * as taskExecutor from "../../tasks/task-executor.js"; +import { finalizeTaskRunByRunId } from "../../tasks/task-executor.js"; +import * as taskRegistry from "../../tasks/task-registry.js"; +import { markTaskLostById, resetTaskRegistryForTests } from "../../tasks/task-registry.js"; +import { listTaskRegistryRecordsByRuntimeSourceIdFromSqlite } from "../../tasks/task-registry.store.sqlite.js"; +import { withOpenClawTestState } from "../../test-utils/openclaw-test-state.js"; +import { cronStoreKey } from "../store/key.js"; +import { readCronTaskRunHistoryPage } from "../task-run-history.js"; +import type { CronJob } from "../types.js"; +import { timeoutErrorMessage } from "./execution-errors.js"; +import { createCronServiceState } from "./state.js"; +import { + tryCreateCronTaskRun, + tryFindCronTaskRunIdForRecovery, + tryFindFinalizedCronTaskRun, + tryFinishCronTaskRun, + tryFinishCronTaskRunWithoutHistory, +} from "./task-runs.js"; + +afterEach(() => { + vi.restoreAllMocks(); + resetDetachedTaskLifecycleRuntimeForTests(); + resetTaskRegistryForTests({ persist: false }); +}); + +describe("cron task run terminal records", () => { + it("persists canonical history directly when a detached runtime is registered", async () => { + await withOpenClawTestState( + { layout: "state-only", prefix: "openclaw-cron-core-ledger-runtime-" }, + async () => { + resetTaskRegistryForTests(); + const customCreate = vi.fn(() => null); + const customFinalize = vi.fn(() => []); + setDetachedTaskLifecycleRuntime({ + ...getDetachedTaskLifecycleRuntime(), + createRunningTaskRun: customCreate, + finalizeTaskRunByRunId: customFinalize, + }); + const job: CronJob = { + id: "core-ledger-runtime", + name: "core ledger runtime", + enabled: true, + createdAtMs: 100, + updatedAtMs: 100, + schedule: { kind: "every", everyMs: 60_000, anchorMs: 100 }, + sessionTarget: "main", + wakeMode: "next-heartbeat", + payload: { kind: "systemEvent", text: "work" }, + state: {}, + }; + const state = createCronServiceState({ + storePath: "/tmp/jobs.json", + cronEnabled: true, + log: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + nowMs: () => 1_100, + enqueueSystemEvent: vi.fn(), + requestHeartbeat: vi.fn(), + runIsolatedAgentJob: vi.fn(async () => ({ status: "ok" as const })), + }); + + tryFinishCronTaskRun(state, { + job, + event: { + jobId: job.id, + action: "finished", + job, + status: "ok", + runAtMs: 1_000, + durationMs: 100, + }, + }); + + expect(customCreate).not.toHaveBeenCalled(); + expect(customFinalize).not.toHaveBeenCalled(); + expect( + readCronTaskRunHistoryPage({ + storeKey: cronStoreKey(state.deps.storePath), + jobId: job.id, + }).entries, + ).toEqual([expect.objectContaining({ status: "ok", runAtMs: 1_000 })]); + }, + ); + }); + + it("keeps task-registry lookup failures inside the best-effort boundary", () => { + const warn = vi.fn(); + const startedAt = 500; + const job: CronJob = { + id: "lookup-failure", + name: "lookup failure", + enabled: true, + createdAtMs: 100, + updatedAtMs: 100, + schedule: { kind: "every", everyMs: 60_000, anchorMs: 100 }, + sessionTarget: "main", + wakeMode: "next-heartbeat", + payload: { kind: "systemEvent", text: "work" }, + state: {}, + }; + const state = createCronServiceState({ + storePath: "/tmp/jobs.json", + cronEnabled: true, + log: { debug: vi.fn(), info: vi.fn(), warn, error: vi.fn() }, + nowMs: () => startedAt + 100, + enqueueSystemEvent: vi.fn(), + requestHeartbeat: vi.fn(), + runIsolatedAgentJob: vi.fn(async () => ({ status: "ok" as const })), + }); + vi.spyOn(taskRegistry, "findTaskByRunId").mockImplementation(() => { + throw new Error("task store unavailable"); + }); + vi.spyOn(taskRegistry, "listTaskRecordsUnsorted").mockImplementation(() => { + throw new Error("task store unavailable"); + }); + + expect(tryFindFinalizedCronTaskRun(state, job.id, startedAt)).toBeUndefined(); + expect(() => + tryFinishCronTaskRun(state, { + job, + event: { + jobId: job.id, + action: "finished", + job, + status: "ok", + runAtMs: startedAt, + durationMs: 100, + }, + }), + ).not.toThrow(); + expect(warn).toHaveBeenCalledWith( + expect.objectContaining({ jobId: job.id }), + "cron: failed to read finalized task ledger record", + ); + expect(warn).toHaveBeenCalledWith( + expect.objectContaining({ + runId: expect.stringMatching(new RegExp(`^cron:${job.id}:${startedAt}:`)), + }), + "cron: failed to update task ledger record", + ); + }); + + it("creates an immediately terminal task row for a skipped-only event", async () => { + await withOpenClawTestState( + { layout: "state-only", prefix: "openclaw-cron-skipped-task-" }, + async () => { + resetTaskRegistryForTests(); + const startedAt = 1_000; + const job: CronJob = { + id: "skipped-job", + name: "skipped job", + agentId: "finn", + enabled: true, + createdAtMs: 100, + updatedAtMs: 100, + schedule: { kind: "every", everyMs: 60_000, anchorMs: 100 }, + sessionTarget: "isolated", + wakeMode: "next-heartbeat", + payload: { kind: "agentTurn", message: "work" }, + state: { nextRunAtMs: 60_000 }, + }; + const state = createCronServiceState({ + storePath: "/tmp/jobs.json", + cronEnabled: true, + log: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + nowMs: () => startedAt, + enqueueSystemEvent: vi.fn(), + requestHeartbeat: vi.fn(), + runIsolatedAgentJob: vi.fn(async () => ({ status: "ok" as const })), + }); + + tryFinishCronTaskRun(state, { + job, + event: { + jobId: job.id, + action: "finished", + job, + status: "skipped", + error: "trigger condition not met", + runId: "manual:skipped-job:1", + runAtMs: startedAt, + durationMs: 0, + nextRunAtMs: 60_000, + }, + }); + + const rows = listTaskRegistryRecordsByRuntimeSourceIdFromSqlite({ + runtime: "cron", + sourceId: job.id, + }); + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ + runtime: "cron", + sourceId: job.id, + agentId: "finn", + status: "succeeded", + startedAt, + endedAt: startedAt, + error: "trigger condition not met", + detail: { + kind: "cron-run", + status: "skipped", + runId: "manual:skipped-job:1", + nextRunAtMs: 60_000, + }, + }); + expect( + readCronTaskRunHistoryPage({ + storeKey: cronStoreKey(state.deps.storePath), + jobId: job.id, + }).entries, + ).toEqual([ + expect.objectContaining({ + jobId: job.id, + status: "skipped", + runId: "manual:skipped-job:1", + }), + ]); + }, + ); + }); + + it("keeps same-millisecond cron executions as distinct task rows", async () => { + await withOpenClawTestState( + { layout: "state-only", prefix: "openclaw-cron-distinct-task-runs-" }, + async () => { + resetTaskRegistryForTests(); + const startedAt = 1_500; + const job: CronJob = { + id: "same-millisecond-job", + name: "same millisecond job", + enabled: true, + createdAtMs: 100, + updatedAtMs: 100, + schedule: { kind: "every", everyMs: 60_000, anchorMs: 100 }, + sessionTarget: "isolated", + wakeMode: "next-heartbeat", + payload: { kind: "agentTurn", message: "work" }, + state: { nextRunAtMs: 60_000 }, + }; + const state = createCronServiceState({ + storePath: "/tmp/jobs.json", + cronEnabled: true, + log: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + nowMs: () => startedAt + 100, + enqueueSystemEvent: vi.fn(), + requestHeartbeat: vi.fn(), + runIsolatedAgentJob: vi.fn(async () => ({ status: "ok" as const })), + }); + const publicRunIds = [ + "manual:same-millisecond-job:1500:1", + "manual:same-millisecond-job:1500:2", + ]; + const taskRunIds = publicRunIds.map((publicRunId) => + tryCreateCronTaskRun({ state, job, startedAt, publicRunId }), + ); + expect(new Set(taskRunIds).size).toBe(2); + + for (const [index, taskRunId] of taskRunIds.entries()) { + if (!taskRunId) { + throw new Error("expected unique cron task run id"); + } + tryFinishCronTaskRun(state, { + taskRunId, + job, + event: { + jobId: job.id, + action: "finished", + job, + status: "ok", + runId: publicRunIds[index], + runAtMs: startedAt, + durationMs: 100, + }, + }); + } + + const rows = listTaskRegistryRecordsByRuntimeSourceIdFromSqlite({ + runtime: "cron", + sourceId: job.id, + }); + expect(rows).toHaveLength(2); + expect(new Set(rows.map((row) => row.runId)).size).toBe(2); + expect( + readCronTaskRunHistoryPage({ + storeKey: cronStoreKey(state.deps.storePath), + jobId: job.id, + }).entries.map((entry) => entry.runId), + ).toEqual(expect.arrayContaining(publicRunIds)); + }, + ); + }); + + it("keeps operator cancellation while attaching terminal run history", async () => { + await withOpenClawTestState( + { layout: "state-only", prefix: "openclaw-cron-cancelled-task-" }, + async () => { + resetTaskRegistryForTests(); + const startedAt = 2_000; + const job: CronJob = { + id: "cancelled-job", + name: "cancelled job", + enabled: true, + createdAtMs: 100, + updatedAtMs: 100, + schedule: { kind: "every", everyMs: 60_000, anchorMs: 100 }, + sessionTarget: "isolated", + wakeMode: "next-heartbeat", + payload: { kind: "agentTurn", message: "work" }, + state: { nextRunAtMs: 60_000 }, + }; + const state = createCronServiceState({ + storePath: "/tmp/jobs.json", + cronEnabled: true, + log: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + nowMs: () => startedAt + 100, + enqueueSystemEvent: vi.fn(), + requestHeartbeat: vi.fn(), + runIsolatedAgentJob: vi.fn(async () => ({ status: "ok" as const })), + }); + const taskRunId = tryCreateCronTaskRun({ state, job, startedAt }); + if (!taskRunId) { + throw new Error("expected cron task run id"); + } + finalizeTaskRunByRunId({ + runId: taskRunId, + runtime: "cron", + status: "cancelled", + endedAt: startedAt + 50, + error: "cancelled by operator", + terminalSummary: "Cancelled by operator.", + }); + + tryFinishCronTaskRun(state, { + taskRunId, + job, + event: { + jobId: job.id, + action: "finished", + job, + status: "ok", + runAtMs: startedAt, + durationMs: 100, + }, + }); + + const [row] = listTaskRegistryRecordsByRuntimeSourceIdFromSqlite({ + runtime: "cron", + sourceId: job.id, + }); + expect(row).toMatchObject({ + status: "cancelled", + endedAt: startedAt + 100, + detail: { kind: "cron-run", status: "ok", durationMs: 100 }, + }); + expect(row?.error).toBeUndefined(); + expect(row?.terminalSummary).toBeUndefined(); + expect( + readCronTaskRunHistoryPage({ + storeKey: cronStoreKey(state.deps.storePath), + jobId: job.id, + }).entries, + ).toEqual([ + expect.objectContaining({ + jobId: job.id, + status: "ok", + error: undefined, + }), + ]); + }, + ); + }); + + it("retries the original outcome after an empty finalization result", async () => { + await withOpenClawTestState( + { layout: "state-only", prefix: "openclaw-cron-task-retry-" }, + async () => { + resetTaskRegistryForTests(); + const startedAt = 3_000; + const job: CronJob = { + id: "retry-job", + name: "retry job", + enabled: true, + createdAtMs: 100, + updatedAtMs: 100, + schedule: { kind: "every", everyMs: 60_000, anchorMs: 100 }, + sessionTarget: "isolated", + wakeMode: "next-heartbeat", + payload: { kind: "agentTurn", message: "work" }, + state: { nextRunAtMs: 60_000 }, + }; + const state = createCronServiceState({ + storePath: "/tmp/jobs.json", + cronEnabled: true, + log: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + nowMs: () => startedAt + 100, + enqueueSystemEvent: vi.fn(), + requestHeartbeat: vi.fn(), + runIsolatedAgentJob: vi.fn(async () => ({ status: "ok" as const })), + }); + const taskRunId = tryCreateCronTaskRun({ state, job, startedAt }); + if (!taskRunId) { + throw new Error("expected cron task run id"); + } + const finalize = taskExecutor.finalizeTaskRunByRunId; + vi.spyOn(taskExecutor, "finalizeTaskRunByRunId") + .mockReturnValueOnce([]) + .mockImplementation((params) => finalize(params)); + + tryFinishCronTaskRun(state, { + taskRunId, + job, + event: { + jobId: job.id, + action: "finished", + job, + status: "ok", + summary: "done", + sessionKey: "agent:main:cron:retry-job:run:actual", + runAtMs: startedAt, + durationMs: 100, + }, + }); + + const rows = listTaskRegistryRecordsByRuntimeSourceIdFromSqlite({ + runtime: "cron", + sourceId: job.id, + }); + expect(rows).toHaveLength(1); + const [row] = rows; + expect(row).toMatchObject({ + status: "succeeded", + childSessionKey: "agent:main:cron:retry-job:run:actual", + terminalSummary: "done", + detail: { kind: "cron-run", status: "ok" }, + }); + }, + ); + }); + + it("overwrites a lost canonical row with restart terminal history", async () => { + await withOpenClawTestState( + { layout: "state-only", prefix: "openclaw-cron-task-lost-recovery-" }, + async () => { + resetTaskRegistryForTests(); + const startedAt = 4_000; + const job: CronJob = { + id: "lost-job", + name: "lost job", + enabled: true, + createdAtMs: 100, + updatedAtMs: 100, + schedule: { kind: "every", everyMs: 60_000, anchorMs: 100 }, + sessionTarget: "isolated", + wakeMode: "next-heartbeat", + payload: { kind: "agentTurn", message: "work" }, + state: { nextRunAtMs: 60_000 }, + }; + const state = createCronServiceState({ + storePath: "/tmp/jobs.json", + cronEnabled: true, + log: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + nowMs: () => startedAt + 100, + enqueueSystemEvent: vi.fn(), + requestHeartbeat: vi.fn(), + runIsolatedAgentJob: vi.fn(async () => ({ status: "ok" as const })), + }); + const taskRunId = tryCreateCronTaskRun({ state, job, startedAt }); + if (!taskRunId) { + throw new Error("expected cron task run id"); + } + const [running] = listTaskRegistryRecordsByRuntimeSourceIdFromSqlite({ + runtime: "cron", + sourceId: job.id, + }); + if (!running) { + throw new Error("expected running cron task"); + } + taskRegistry.markTaskTerminalById({ + taskId: running.taskId, + status: "failed", + endedAt: startedAt + 40, + terminalSummary: "Task session disappeared.", + }); + markTaskLostById({ + taskId: running.taskId, + endedAt: startedAt + 50, + error: "backing session missing", + }); + const renamedJob = { ...job, name: "renamed lost job", sessionTarget: "main" as const }; + + tryFinishCronTaskRun(state, { + taskRunId, + job: renamedJob, + event: { + jobId: job.id, + action: "finished", + job: renamedJob, + status: "error", + error: "gateway restarted while job was running", + runAtMs: startedAt, + durationMs: 100, + }, + }); + + const rows = listTaskRegistryRecordsByRuntimeSourceIdFromSqlite({ + runtime: "cron", + sourceId: job.id, + }); + expect(rows).toHaveLength(1); + const [row] = rows; + expect(row).toMatchObject({ + taskId: running.taskId, + status: "failed", + endedAt: startedAt + 100, + error: "gateway restarted while job was running", + detail: { kind: "cron-run", status: "error", durationMs: 100 }, + }); + expect(row).not.toHaveProperty("childSessionKey"); + expect(row?.terminalSummary).toBeUndefined(); + }, + ); + }); + + it("overwrites a provisional timeout with restart terminal history", async () => { + await withOpenClawTestState( + { layout: "state-only", prefix: "openclaw-cron-task-timeout-recovery-" }, + async () => { + resetTaskRegistryForTests(); + const startedAt = 5_000; + const job: CronJob = { + id: "provisional-timeout-job", + name: "provisional timeout job", + enabled: true, + createdAtMs: 100, + updatedAtMs: 100, + schedule: { kind: "every", everyMs: 60_000, anchorMs: 100 }, + sessionTarget: "isolated", + wakeMode: "next-heartbeat", + payload: { kind: "agentTurn", message: "work" }, + state: { nextRunAtMs: 60_000 }, + }; + const state = createCronServiceState({ + storePath: "/tmp/jobs.json", + cronEnabled: true, + log: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + nowMs: () => startedAt + 200, + enqueueSystemEvent: vi.fn(), + requestHeartbeat: vi.fn(), + runIsolatedAgentJob: vi.fn(async () => ({ status: "ok" as const })), + }); + const taskRunId = tryCreateCronTaskRun({ state, job, startedAt }); + if (!taskRunId) { + throw new Error("expected cron task run id"); + } + tryFinishCronTaskRunWithoutHistory(state, { + taskRunId, + status: "error", + error: timeoutErrorMessage(), + endedAt: startedAt + 100, + }); + + tryFinishCronTaskRun(state, { + taskRunId, + job, + event: { + jobId: job.id, + action: "finished", + job, + status: "error", + error: "gateway restarted while job was running", + runAtMs: startedAt, + durationMs: 200, + }, + }); + + const [row] = listTaskRegistryRecordsByRuntimeSourceIdFromSqlite({ + runtime: "cron", + sourceId: job.id, + }); + expect(row).toMatchObject({ + status: "failed", + endedAt: startedAt + 200, + error: "gateway restarted while job was running", + detail: { kind: "cron-run", status: "error", durationMs: 200 }, + }); + }, + ); + }); + + it("recovers pre-discriminator task rows written by older releases", async () => { + await withOpenClawTestState( + { layout: "state-only", prefix: "openclaw-cron-task-legacy-runid-" }, + async () => { + resetTaskRegistryForTests(); + const startedAt = 7_000; + const state = createCronServiceState({ + storePath: "/tmp/jobs.json", + cronEnabled: true, + log: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + nowMs: () => startedAt + 100, + enqueueSystemEvent: vi.fn(), + requestHeartbeat: vi.fn(), + runIsolatedAgentJob: vi.fn(async () => ({ status: "ok" as const })), + }); + const legacyRunId = `cron:legacy-job:${startedAt}`; + // Older releases persisted the reservation id without a uniqueness suffix. + taskExecutor.createRunningTaskRun({ + runtime: "cron", + sourceId: "legacy-job", + ownerKey: "", + scopeKind: "system", + runId: legacyRunId, + task: "legacy job", + deliveryStatus: "not_applicable", + notifyPolicy: "silent", + startedAt, + }); + expect(tryFindCronTaskRunIdForRecovery(state, "legacy-job", startedAt)).toBe(legacyRunId); + finalizeTaskRunByRunId({ + runId: legacyRunId, + runtime: "cron", + status: "timed_out", + endedAt: startedAt + 50, + error: timeoutErrorMessage(), + }); + + tryFinishCronTaskRun(state, { + taskRunId: legacyRunId, + event: { + jobId: "legacy-job", + action: "finished", + status: "error", + error: "gateway restarted while job was running", + runAtMs: startedAt, + durationMs: 100, + }, + }); + + const [row] = listTaskRegistryRecordsByRuntimeSourceIdFromSqlite({ + runtime: "cron", + sourceId: "legacy-job", + }); + expect(row).toMatchObject({ + runId: legacyRunId, + status: "failed", + error: "gateway restarted while job was running", + detail: { kind: "cron-run", status: "error", durationMs: 100 }, + }); + }, + ); + }); + + it("keeps suffixed recovery identities scoped to the current cron store", async () => { + await withOpenClawTestState( + { layout: "state-only", prefix: "openclaw-cron-task-store-recovery-" }, + async (fixture) => { + resetTaskRegistryForTests(); + const startedAt = 8_000; + const job: CronJob = { + id: "shared-job", + name: "shared job", + enabled: true, + createdAtMs: 100, + updatedAtMs: 100, + schedule: { kind: "every", everyMs: 60_000, anchorMs: 100 }, + sessionTarget: "main", + wakeMode: "next-heartbeat", + payload: { kind: "systemEvent", text: "work" }, + state: {}, + }; + const createState = (storePath: string) => + createCronServiceState({ + storePath, + cronEnabled: true, + log: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + nowMs: () => startedAt, + enqueueSystemEvent: vi.fn(), + requestHeartbeat: vi.fn(), + runIsolatedAgentJob: vi.fn(async () => ({ status: "ok" as const })), + }); + const stateA = createState(fixture.path("cron-a", "jobs.json")); + const stateB = createState(fixture.path("cron-b", "jobs.json")); + const runA = tryCreateCronTaskRun({ state: stateA, job, startedAt }); + const runB = tryCreateCronTaskRun({ state: stateB, job, startedAt }); + expect(runA).toBeTruthy(); + expect(runB).toBeTruthy(); + expect(runA).not.toBe(runB); + + expect(tryFindCronTaskRunIdForRecovery(stateA, job.id, startedAt)).toBe(runA); + expect(tryFindCronTaskRunIdForRecovery(stateB, job.id, startedAt)).toBe(runB); + }, + ); + }); +}); diff --git a/src/cron/service/task-runs.ts b/src/cron/service/task-runs.ts index 3662438795b6..4db5635942ff 100644 --- a/src/cron/service/task-runs.ts +++ b/src/cron/service/task-runs.ts @@ -1,20 +1,33 @@ /** Detached task-ledger integration for cron runs. */ +import { randomUUID } from "node:crypto"; import { normalizeOptionalLowercaseString } from "@openclaw/normalization-core/string-coerce"; import { DEFAULT_AGENT_ID, normalizeAgentId, resolveAgentIdFromSessionKey, } from "../../routing/session-key.js"; +import { createRunningTaskRun, finalizeTaskRunByRunId } from "../../tasks/task-executor.js"; import { - completeTaskRunByRunId, - createRunningTaskRun, - failTaskRunByRunId, -} from "../../tasks/detached-task-runtime.js"; + findTaskByRunId, + listTaskRecordsUnsorted, + markTaskTerminalById, +} from "../../tasks/task-registry.js"; +import type { JsonValue, TaskRecord, TaskStatus } from "../../tasks/task-registry.types.js"; import { resolveCronAgentSessionKey } from "../isolated-agent/session-key.js"; import { createCronExecutionId } from "../run-id.js"; +import type { CronRunLogEntry } from "../run-log-types.js"; +import { cronStoreKey } from "../store/key.js"; +import { + cronRunLogEntryFromEvent, + cronRunLogEntryToTaskDetail, + cronRunStatusToTaskStatus, + cronTaskRecordStoreKey, + cronTaskRecordToRunLogEntry, + cronTaskRecordToTriggerEval, +} from "../task-run-detail.js"; import type { CronJob, CronRunStatus } from "../types.js"; import { normalizeCronRunErrorText, timeoutErrorMessage } from "./execution-errors.js"; -import type { CronServiceState } from "./state.js"; +import type { CronEvent, CronServiceState } from "./state.js"; import { CRON_TASK_RUNNING_PROGRESS_SUMMARY } from "./task-ledger.js"; /** Converts cron ids into bounded session-key path segments with a fallback for empty input. */ @@ -69,80 +82,312 @@ export function tryCreateCronTaskRun(params: { state: CronServiceState; job: CronJob; startedAt: number; + runIdStartedAt?: number; + publicRunId?: string; +}): string | undefined { + const runId = createCronTaskRunId( + params.job.id, + params.runIdStartedAt ?? params.startedAt, + params.publicRunId, + ); + return tryCreateCronTaskRunRecord({ + state: params.state, + job: params.job, + jobId: params.job.id, + startedAt: params.startedAt, + runId, + }); +} + +function createCronTaskRunId(jobId: string, reservationAt: number, publicRunId?: string): string { + const discriminator = publicRunId?.trim() || randomUUID(); + return `${createCronExecutionId(jobId, reservationAt)}:${discriminator}`; +} + +function findLatestCronTaskRunForRecovery( + jobId: string, + reservationAt: number, + storeKey: string, +): TaskRecord | undefined { + const reservationRunId = createCronExecutionId(jobId, reservationAt); + const prefix = `${reservationRunId}:`; + return listTaskRecordsUnsorted() + .filter((task) => { + if (task.runtime !== "cron" || task.sourceId !== jobId) { + return false; + } + const taskStoreKey = cronTaskRecordStoreKey(task); + if (taskStoreKey === undefined) { + // Exact match covers detail-less pre-discriminator rows from older releases. + return task.runId === reservationRunId; + } + return ( + taskStoreKey === storeKey && + (task.runId === reservationRunId || task.runId?.startsWith(prefix)) + ); + }) + .toSorted( + (left, right) => + Number(left.endedAt !== undefined) - Number(right.endedAt !== undefined) || + (right.endedAt ?? right.lastEventAt ?? right.createdAt) - + (left.endedAt ?? left.lastEventAt ?? left.createdAt) || + right.createdAt - left.createdAt || + right.taskId.localeCompare(left.taskId), + )[0]; +} + +/** Finds the unique task identity owned by one persisted cron reservation. */ +export function tryFindCronTaskRunIdForRecovery( + state: CronServiceState, + jobId: string, + startedAt: number, +): string | undefined { + try { + return findLatestCronTaskRunForRecovery(jobId, startedAt, cronStoreKey(state.deps.storePath)) + ?.runId; + } catch (error) { + state.deps.log.warn({ jobId, error }, "cron: failed to read task ledger recovery record"); + return undefined; + } +} + +/** Finds a completed canonical cron row for startup crash recovery. */ +export function tryFindFinalizedCronTaskRun( + state: CronServiceState, + jobId: string, + startedAt: number, +): + | { + entry: CronRunLogEntry & { status: CronRunStatus }; + triggerEval?: { fired: true; stateChanged: boolean; state?: JsonValue }; + } + | undefined { + try { + const task = findLatestCronTaskRunForRecovery( + jobId, + startedAt, + cronStoreKey(state.deps.storePath), + ); + if (task?.runtime !== "cron" || task.sourceId !== jobId || task.endedAt === undefined) { + return undefined; + } + const entry = cronTaskRecordToRunLogEntry(task); + if (!entry?.status) { + return undefined; + } + const triggerEval = cronTaskRecordToTriggerEval(task); + return { + entry: { ...entry, status: entry.status }, + ...(triggerEval ? { triggerEval } : {}), + }; + } catch (error) { + state.deps.log.warn({ jobId, error }, "cron: failed to read finalized task ledger record"); + return undefined; + } +} + +function tryCreateCronTaskRunRecord(params: { + state: CronServiceState; + job?: CronJob; + jobId: string; + startedAt: number; + runId: string; + childSessionKey?: string; }): string | undefined { - const runId = createCronExecutionId(params.job.id, params.startedAt); try { const task = createRunningTaskRun({ runtime: "cron", - sourceId: params.job.id, + sourceId: params.jobId, ownerKey: "", scopeKind: "system", - childSessionKey: resolveCronTaskChildSessionKey(params), - agentId: params.job.agentId, - runId, - label: params.job.name, - task: params.job.name || params.job.id, + childSessionKey: + params.childSessionKey ?? + (params.job + ? resolveCronTaskChildSessionKey({ + state: params.state, + job: params.job, + startedAt: params.startedAt, + }) + : undefined), + agentId: + params.job?.agentId ?? + resolveAgentIdFromSessionKey(params.childSessionKey) ?? + params.state.deps.defaultAgentId ?? + DEFAULT_AGENT_ID, + runId: params.runId, + label: params.job?.name, + task: params.job?.name || params.jobId, deliveryStatus: "not_applicable", notifyPolicy: "silent", startedAt: params.startedAt, lastEventAt: params.startedAt, progressSummary: CRON_TASK_RUNNING_PROGRESS_SUMMARY, + detail: { storeKey: cronStoreKey(params.state.deps.storePath) }, }); if (!task) { params.state.deps.log.warn( - { jobId: params.job.id }, + { jobId: params.jobId }, "cron: task ledger record was not persisted", ); return undefined; } - return runId; + return params.runId; } catch (error) { params.state.deps.log.warn( - { jobId: params.job.id, error }, + { jobId: params.jobId, error }, "cron: failed to create task ledger record", ); return undefined; } } -/** Completes or fails the detached task ledger row for a cron run when one exists. */ -export function tryFinishCronTaskRun( +/** Finalizes executions that intentionally do not produce a run-history row. */ +export function tryFinishCronTaskRunWithoutHistory( state: CronServiceState, result: { taskRunId?: string; - status: CronRunStatus; + status: "ok" | "error" | "skipped"; error?: unknown; endedAt: number; summary?: string; + childSessionKey?: string; }, ): void { if (!result.taskRunId) { return; } + const error = result.status === "error" ? normalizeCronRunErrorText(result.error) : undefined; try { - if (result.status === "ok" || result.status === "skipped") { - completeTaskRunByRunId({ - runId: result.taskRunId, - runtime: "cron", - endedAt: result.endedAt, - lastEventAt: result.endedAt, - terminalSummary: result.summary ?? undefined, - }); - return; - } - failTaskRunByRunId({ + finalizeTaskRunByRunId({ runId: result.taskRunId, runtime: "cron", status: - normalizeCronRunErrorText(result.error) === timeoutErrorMessage() ? "timed_out" : "failed", + result.status === "ok" || result.status === "skipped" + ? "succeeded" + : error === timeoutErrorMessage() + ? "timed_out" + : "failed", endedAt: result.endedAt, lastEventAt: result.endedAt, - error: result.status === "error" ? normalizeCronRunErrorText(result.error) : undefined, - terminalSummary: result.summary ?? undefined, + error, + terminalSummary: result.summary, + childSessionKey: result.childSessionKey, }); - } catch (error) { + } catch (cause) { state.deps.log.warn( - { runId: result.taskRunId, jobStatus: result.status, error }, + { runId: result.taskRunId, jobStatus: result.status, error: cause }, + "cron: failed to update task ledger record", + ); + } +} + +/** Finalizes the authoritative task row, creating one for terminal-only cron events. */ +export function tryFinishCronTaskRun( + state: CronServiceState, + result: { + taskRunId?: string; + job?: CronJob; + event: CronEvent & { action: "finished" }; + triggerEval?: { fired: boolean; stateChanged: boolean; state?: unknown }; + }, +): void { + const entry = cronRunLogEntryFromEvent(result.event, state.deps.nowMs()); + const startedAt = entry.runAtMs ?? entry.ts; + const candidateRunId = + result.taskRunId ?? createCronTaskRunId(entry.jobId, startedAt, entry.runId); + try { + const existingCandidate = findTaskByRunId(candidateRunId); + const taskRunId = + existingCandidate?.runtime === "cron" + ? candidateRunId + : tryCreateCronTaskRunRecord({ + state, + job: result.job ?? result.event.job, + jobId: entry.jobId, + startedAt, + runId: candidateRunId, + childSessionKey: entry.sessionKey, + }); + if (!taskRunId) { + return; + } + const storeKey = cronStoreKey(state.deps.storePath); + const legacyRecoveryRunId = createCronExecutionId(entry.jobId, startedAt); + const detail = cronRunLogEntryToTaskDetail(entry, { + storeKey, + ...(result.triggerEval ? { triggerEval: result.triggerEval } : {}), + }); + const finalize = ( + runId: string, + status: Extract< + TaskStatus, + "succeeded" | "failed" | "timed_out" | "cancelled" + > = cronRunStatusToTaskStatus(entry), + ) => + finalizeTaskRunByRunId({ + runId, + runtime: "cron", + status, + endedAt: entry.ts, + lastEventAt: entry.ts, + error: entry.error, + clearError: entry.error === undefined, + terminalSummary: entry.summary ?? null, + preserveTerminalSummary: true, + childSessionKey: entry.sessionKey ?? null, + detail, + }); + let updated = finalize(taskRunId); + if (updated.length === 0) { + const existing = findTaskByRunId(taskRunId); + if (existing?.runtime === "cron" && existing.status === "cancelled") { + // Operator cancellation owns task status, but its finished event still owns history detail. + updated = finalize(taskRunId, "cancelled"); + } else if ( + existing?.runtime === "cron" && + (existing.status === "lost" || + (cronTaskRecordStoreKey(existing) === storeKey && + cronTaskRecordToRunLogEntry(existing) === null) || + (existing.detail === undefined && existing.runId === legacyRecoveryRunId)) + ) { + // Pre-persist markers and exact legacy identities contain no history detail. + // Startup recovery replaces them with the durable interrupted outcome. + const recovered = markTaskTerminalById({ + taskId: existing.taskId, + status: cronRunStatusToTaskStatus(entry), + childSessionKey: entry.sessionKey ?? null, + endedAt: entry.ts, + lastEventAt: entry.ts, + error: entry.error, + terminalSummary: entry.summary ?? null, + preserveTerminalSummary: true, + detail, + }); + updated = recovered ? [recovered] : []; + } else if (existing?.runtime === "cron") { + // Keep the existing run/session scope when its first terminal write failed. + updated = finalize(taskRunId); + } else { + // A terminal event still owns one durable row if its active mirror vanished. + const recreatedRunId = tryCreateCronTaskRunRecord({ + state, + job: result.job ?? result.event.job, + jobId: entry.jobId, + startedAt, + runId: taskRunId, + childSessionKey: entry.sessionKey, + }); + if (recreatedRunId) { + updated = finalize(recreatedRunId); + } + } + } + if (updated.length === 0) { + state.deps.log.warn({ runId: taskRunId }, "cron: task ledger record was not finalized"); + } + } catch (error) { + state.deps.log.warn( + { runId: candidateRunId, jobStatus: entry.status, error }, "cron: failed to update task ledger record", ); } diff --git a/src/cron/service/timer.regression.test.ts b/src/cron/service/timer.regression.test.ts index a5db51bc0c8a..cd030b627beb 100644 --- a/src/cron/service/timer.regression.test.ts +++ b/src/cron/service/timer.regression.test.ts @@ -85,6 +85,14 @@ function firstMockArg(mock: unknown): unknown { return call[0]; } +function findCronTaskByBaseRunId(baseRunId: string) { + return listTaskRecords().find( + (entry) => + entry.runtime === "cron" && + (entry.runId === baseRunId || entry.runId?.startsWith(`${baseRunId}:`)), + ); +} + describe("cron service timer regressions", () => { it("caps timer delay to 60s for far-future schedules", async () => { const timeoutSpy = vi.spyOn(globalThis, "setTimeout"); @@ -843,9 +851,7 @@ describe("cron service timer regressions", () => { await runnerStarted.promise; const runId = `cron:no-timeout-cancel:${scheduledAt}`; - const task = listTaskRecords().find( - (entry) => entry.runtime === "cron" && entry.runId === runId, - ); + const task = findCronTaskByBaseRunId(runId); if (!task) { throw new Error("Expected timeout-disabled cron task row"); } @@ -1032,8 +1038,9 @@ describe("cron service timer regressions", () => { void timerPromise.then(() => { timerSettled = true; }); + const taskRunId = findCronTaskByBaseRunId(runId)?.runId; const cancelled = cancelActiveCronTaskRun({ - runId, + runId: taskRunId ?? runId, reason: "Cancelled by operator.", }); @@ -1106,9 +1113,7 @@ describe("cron service timer regressions", () => { await cleanupStarted.promise; const runId = `cron:late-cancel-after-timeout:${scheduledAt}`; - const task = listTaskRecords().find( - (entry) => entry.runtime === "cron" && entry.runId === runId, - ); + const task = findCronTaskByBaseRunId(runId); if (!task) { throw new Error("Expected timed-out cron task row"); } @@ -1689,9 +1694,7 @@ describe("cron service timer regressions", () => { } expect(runHeartbeatOnce).toHaveBeenCalledTimes(1); - const task = listTaskRecords().find( - (entry) => entry.runtime === "cron" && entry.runId === runId, - ); + const task = findCronTaskByBaseRunId(runId); if (!task) { throw new Error("Expected main-session cron task row"); } diff --git a/src/cron/service/timer.test.ts b/src/cron/service/timer.test.ts index dd0ae596a097..355baae17a1d 100644 --- a/src/cron/service/timer.test.ts +++ b/src/cron/service/timer.test.ts @@ -5,10 +5,16 @@ import { upsertSessionEntry } from "../../config/sessions/session-accessor.js"; import { setupCronServiceSuite, writeCronStoreSnapshot } from "../../cron/service.test-harness.js"; import { createCronServiceState } from "../../cron/service/state.js"; import { executeJobCore, onTimer } from "../../cron/service/timer.js"; +import * as cronStoreModule from "../../cron/store.js"; import { loadCronStore } from "../../cron/store.js"; +import { cronStoreKey } from "../../cron/store/key.js"; import type { CronJob } from "../../cron/types.js"; -import * as detachedTaskRuntime from "../../tasks/detached-task-runtime.js"; -import { findTaskByRunId, resetTaskRegistryForTests } from "../../tasks/task-registry.js"; +import * as taskExecutor from "../../tasks/task-executor.js"; +import { + findTaskByRunId, + listTaskRecordsUnsorted, + resetTaskRegistryForTests, +} from "../../tasks/task-registry.js"; import { formatTaskStatusDetail } from "../../tasks/task-status.js"; const { logger, makeStorePath } = setupCronServiceSuite({ @@ -63,6 +69,13 @@ function createDueCommandJob(params: { now: number }): CronJob { }; } +function findCronTaskByBaseRunId(baseRunId: string) { + return ( + findTaskByRunId(baseRunId) ?? + listTaskRecordsUnsorted().find((task) => task.runId?.startsWith(`${baseRunId}:`)) + ); +} + afterEach(() => { resetTaskRegistryForTests(); }); @@ -170,7 +183,7 @@ describe("cron service timer seam coverage", () => { expect(job.state.lastStatus).toBe("ok"); expect(job.state.runningAtMs).toBeUndefined(); expect(job.state.nextRunAtMs).toBe(now + 60_000); - const task = findTaskByRunId(`cron:main-heartbeat-job:${now}`); + const task = findCronTaskByBaseRunId(`cron:main-heartbeat-job:${now}`); if (!task) { throw new Error("expected cron task ledger record"); } @@ -179,7 +192,7 @@ describe("cron service timer seam coverage", () => { expect(task.ownerKey).toBe(""); expect(task.scopeKind).toBe("system"); expect(task.childSessionKey).toBe(cronRunSessionKey); - expect(task.runId).toBe(`cron:main-heartbeat-job:${now}`); + expect(task.runId).toMatch(new RegExp(`^cron:main-heartbeat-job:${now}:`)); expect(task.label).toBe("main heartbeat job"); expect(task.task).toBe("main heartbeat job"); expect(task.status).toBe("succeeded"); @@ -199,6 +212,102 @@ describe("cron service timer seam coverage", () => { timeoutSpy.mockRestore(); }); + it("uses the persisted reservation timestamp for the canonical timer task", async () => { + const { storePath } = await makeStorePath(); + const now = Date.parse("2026-03-23T12:00:00.000Z"); + let clock = now; + let persistedReservation: number | undefined; + let liveReservation: number | undefined; + let liveError: string | undefined; + let emittedStartedAt: number | undefined; + const job = createDueIsolatedAgentJob({ now }); + job.state.lastError = "previous failure"; + await writeCronStoreSnapshot({ + storePath, + jobs: [job], + }); + const state = createCronServiceState({ + storePath, + cronEnabled: true, + log: logger, + nowMs: () => clock++, + enqueueSystemEvent: vi.fn(), + requestHeartbeat: vi.fn(), + runIsolatedAgentJob: vi.fn(async () => { + persistedReservation = (await loadCronStore(storePath)).jobs[0]?.state.runningAtMs; + liveReservation = state.store?.jobs[0]?.state.runningAtMs; + liveError = state.store?.jobs[0]?.state.lastError; + return { status: "ok" as const }; + }), + onEvent: (event) => { + if (event.action === "started") { + emittedStartedAt = event.runAtMs; + } + }, + }); + + await onTimer(state); + + expect(persistedReservation).toEqual(expect.any(Number)); + expect(liveReservation).toBe(persistedReservation); + expect(liveError).toBeUndefined(); + expect(emittedStartedAt).toEqual(expect.any(Number)); + expect(emittedStartedAt).toBeGreaterThan(persistedReservation ?? 0); + expect( + findCronTaskByBaseRunId(`cron:isolated-agent-job:${persistedReservation}`), + ).toMatchObject({ + startedAt: emittedStartedAt, + status: "succeeded", + }); + }); + + it("finalizes quiet trigger tasks only after cron state persists", async () => { + const { storePath } = await makeStorePath(); + const now = Date.parse("2026-03-23T12:00:00.000Z"); + const job = { + ...createDueIsolatedAgentJob({ now }), + trigger: { script: "json({ fire: false })" }, + }; + await writeCronStoreSnapshot({ storePath, jobs: [job] }); + const order: string[] = []; + const save = cronStoreModule.saveCronJobsStore; + const finalize = taskExecutor.finalizeTaskRunByRunId; + const saveSpy = vi + .spyOn(cronStoreModule, "saveCronJobsStore") + .mockImplementation(async (...args) => { + order.push("persist"); + return await save(...args); + }); + const finalizeSpy = vi + .spyOn(taskExecutor, "finalizeTaskRunByRunId") + .mockImplementation((params) => { + order.push("finalize"); + return finalize(params); + }); + const state = createCronServiceState({ + storePath, + cronEnabled: true, + cronConfig: { triggers: { enabled: true } }, + log: logger, + nowMs: () => now, + enqueueSystemEvent: vi.fn(), + requestHeartbeat: vi.fn(), + evaluateCronTrigger: vi.fn(async () => ({ kind: "evaluated" as const, fire: false })), + runIsolatedAgentJob: vi.fn(async () => ({ status: "ok" as const })), + }); + + try { + await onTimer(state); + expect(order).toEqual(["persist", "persist", "finalize"]); + const task = findCronTaskByBaseRunId(`cron:${job.id}:${now}`); + expect(task).toMatchObject({ status: "succeeded" }); + expect(task?.detail).toEqual({ storeKey: cronStoreKey(storePath) }); + } finally { + saveSpy.mockRestore(); + finalizeSpy.mockRestore(); + } + }); + it("runs command cron jobs without isolated agent setup", async () => { const { storePath } = await makeStorePath(); const now = Date.parse("2026-03-23T12:00:00.000Z"); @@ -237,7 +346,12 @@ describe("cron service timer seam coverage", () => { const runIsolatedAgentJob = vi.fn(async () => ({ status: "ok" as const, summary: "done", + sessionId: "session-run-1", sessionKey: "agent:finn:cron:isolated-agent-job:run:run-1", + delivery: { intended: { channel: "telegram", to: "42" } }, + model: "gpt-test", + provider: "openai", + usage: { input_tokens: 5, output_tokens: 3, total_tokens: 8 }, })); await writeCronStoreSnapshot({ @@ -263,13 +377,24 @@ describe("cron service timer seam coverage", () => { message: "run isolated cron", }), ); - const task = findTaskByRunId(`cron:isolated-agent-job:${now}`); + const task = findCronTaskByBaseRunId(`cron:isolated-agent-job:${now}`); if (!task) { throw new Error("expected isolated cron task ledger record"); } - expect(task.childSessionKey).toBe("agent:finn:cron:isolated-agent-job"); + expect(task.childSessionKey).toBe("agent:finn:cron:isolated-agent-job:run:run-1"); expect(task.status).toBe("succeeded"); expect(task.terminalSummary).toBe("done"); + expect(task.detail).toMatchObject({ + kind: "cron-run", + status: "ok", + sessionId: "session-run-1", + durationMs: 0, + nextRunAtMs: now + 60_000, + delivery: { intended: { channel: "telegram", to: "42" } }, + model: "gpt-test", + provider: "openai", + usage: { input_tokens: 5, output_tokens: 3, total_tokens: 8 }, + }); }); it("records current-bound cron task runs against the backing cron session", async () => { @@ -304,11 +429,11 @@ describe("cron service timer seam coverage", () => { await onTimer(state); - const task = findTaskByRunId(`cron:isolated-agent-job:${now}`); + const task = findCronTaskByBaseRunId(`cron:isolated-agent-job:${now}`); if (!task) { throw new Error("expected current-bound cron task ledger record"); } - expect(task.childSessionKey).toBe("agent:finn:cron:isolated-agent-job"); + expect(task.childSessionKey).toBe("agent:finn:cron:isolated-agent-job:run:run-1"); expect(task.status).toBe("succeeded"); }); @@ -345,7 +470,7 @@ describe("cron service timer seam coverage", () => { expect(runIsolatedAgentJob).toHaveBeenCalledTimes(1); }); - const task = findTaskByRunId(`cron:isolated-agent-job:${now}`); + const task = findCronTaskByBaseRunId(`cron:isolated-agent-job:${now}`); if (!task) { throw new Error("expected active cron task ledger record"); } @@ -370,7 +495,7 @@ describe("cron service timer seam coverage", () => { }); const createTaskRecordSpy = vi - .spyOn(detachedTaskRuntime, "createRunningTaskRun") + .spyOn(taskExecutor, "createRunningTaskRun") .mockImplementation(() => { throw ledgerError; }); diff --git a/src/cron/service/timer.ts b/src/cron/service/timer.ts index 318c1027795c..f7f58ca229cd 100644 --- a/src/cron/service/timer.ts +++ b/src/cron/service/timer.ts @@ -90,6 +90,7 @@ import { resolveMainSessionCronRunSessionKey, tryCreateCronTaskRun, tryFinishCronTaskRun, + tryFinishCronTaskRunWithoutHistory, } from "./task-runs.js"; import { resolveCronJobTimeoutMs } from "./timeout-policy.js"; @@ -117,6 +118,7 @@ type TimedCronRunOutcome = CronRunOutcome & delivered?: boolean; deliveryAttempted?: boolean; deliveryError?: string; + delivery?: CronDeliveryTrace; isolatedAgentSetupTimeout?: IsolatedAgentSetupTimeoutSignal; activeJobMarker?: CronActiveJobMarker; startedAt: number; @@ -173,7 +175,7 @@ type ExecuteJobCoreOptions = { * Carries the already-resolved run attribution from watchdog-visible execution * state into a timer-built error outcome. The wall-clock/cancel paths return * their own outcome (the inner run result loses the Promise.race), so without - * this the persisted cron_run_logs row drops provider/model/session for a + * this the persisted cron run record drops provider/model/session for a * post-runner timeout or cancel even though they were already known. Stays * empty before the runner starts, so pre-execution setup timeouts read blank. */ @@ -712,6 +714,8 @@ export function applyJobResult( opts?: { // Preserve recurring "every" anchors for manual force runs. preserveSchedule?: boolean; + // Startup replay restores alert cooldown bookkeeping without redelivery. + replayFailureAlertAtMs?: number; }, ): boolean { const prevLastRunAtMs = job.state.lastRunAtMs; @@ -783,6 +787,9 @@ export function applyJobResult( error: result.error, provider: result.provider, consecutiveCount: job.state.consecutiveErrors, + ...(opts?.replayFailureAlertAtMs !== undefined + ? { delivery: "record-only" as const, occurredAtMs: opts.replayFailureAlertAtMs } + : {}), }); } else if (result.status === "skipped") { job.state.consecutiveErrors = 0; @@ -795,6 +802,9 @@ export function applyJobResult( error: result.error, provider: result.provider, consecutiveCount: job.state.consecutiveSkipped, + ...(opts?.replayFailureAlertAtMs !== undefined + ? { delivery: "record-only" as const, occurredAtMs: opts.replayFailureAlertAtMs } + : {}), }); } else { job.state.lastFailureAlertAtMs = undefined; @@ -1105,15 +1115,16 @@ function applyOutcomeToStoredJob( state: CronServiceState, result: TimedCronRunOutcome, ): CronJob | undefined { - tryFinishCronTaskRun(state, result); const store = state.store; if (!store) { + tryFinishCronTaskRunWithoutHistory(state, result); return undefined; } const jobs = store.jobs; const job = jobs.find((entry) => entry.id === result.jobId); if (!job) { if (result.status === "ok" && result.triggerEval?.fired === false) { + tryFinishCronTaskRunWithoutHistory(state, result); return undefined; } if (result.status === "ok") { @@ -1140,6 +1151,7 @@ function applyOutcomeToStoredJob( { jobId: result.jobId }, "cron: applyOutcomeToStoredJob — job not found after forceReload, result discarded", ); + tryFinishCronTaskRunWithoutHistory(state, result); return undefined; } @@ -1177,6 +1189,17 @@ function applyOutcomeToStoredJob( return undefined; } +function finishPersistedQuietCronTaskRuns( + state: CronServiceState, + outcomes: readonly TimedCronRunOutcome[], +): void { + for (const outcome of outcomes) { + if (outcome.status === "ok" && outcome.triggerEval && !outcome.triggerEval.fired) { + tryFinishCronTaskRunWithoutHistory(state, outcome); + } + } +} + function clearActiveMarkersForOutcomes(outcomes: readonly TimedCronRunOutcome[]): void { for (const outcome of outcomes) { clearCronJobActive(outcome.jobId, outcome.activeJobMarker); @@ -1197,7 +1220,7 @@ function finishRetiredCronTaskRuns( const current = new Set(currentOutcomes); for (const outcome of outcomes) { if (!current.has(outcome)) { - tryFinishCronTaskRun(state, outcome); + tryFinishCronTaskRunWithoutHistory(state, outcome); } } } @@ -1405,23 +1428,35 @@ async function onAdmittedTimer(state: CronServiceState) { }): Promise => { const { id, job } = params; const startedAt = state.deps.nowMs(); - job.state.runningAtMs = startedAt; + const executionJob = structuredClone(job); job.state.lastError = undefined; - const activeJobMarker = markCronJobActive(job.id, { - preserveAcrossGenerationAdvance: job.sessionTarget === "main", + executionJob.state.runningAtMs = startedAt; + executionJob.state.lastError = undefined; + const activeJobMarker = markCronJobActive(executionJob.id, { + preserveAcrossGenerationAdvance: executionJob.sessionTarget === "main", + }); + emit(state, { + jobId: executionJob.id, + action: "started", + job: executionJob, + runAtMs: startedAt, + }); + const jobTimeoutMs = resolveCronJobTimeoutMs(executionJob); + const taskRunId = tryCreateCronTaskRun({ + state, + job: executionJob, + startedAt, + runIdStartedAt: params.reservedAtMs, }); - emit(state, { jobId: job.id, action: "started", job, runAtMs: startedAt }); - const jobTimeoutMs = resolveCronJobTimeoutMs(job); - const taskRunId = tryCreateCronTaskRun({ state, job, startedAt }); try { - const result = await executeJobCoreWithTimeout(state, job, { + const result = await executeJobCoreWithTimeout(state, executionJob, { runId: taskRunId, activeJobMarker, }); return { jobId: id, - job, + job: executionJob, taskRunId, activeJobMarker, ...result, @@ -1431,12 +1466,12 @@ async function onAdmittedTimer(state: CronServiceState) { } catch (err) { const errorText = normalizeCronRunErrorText(err); state.deps.log.warn( - { jobId: id, jobName: job.name, timeoutMs: jobTimeoutMs ?? null }, + { jobId: id, jobName: executionJob.name, timeoutMs: jobTimeoutMs ?? null }, `cron: job failed: ${errorText}`, ); return { jobId: id, - job, + job: executionJob, taskRunId, activeJobMarker, status: "error", @@ -1488,6 +1523,7 @@ async function onAdmittedTimer(state: CronServiceState) { // daily cron schedules to jump 48 h instead of 24 h (#17852). recomputeNextRunsForMaintenance(state); await persistOrRestore(state, rollbackSnapshot); + finishPersistedQuietCronTaskRuns(state, finalizedResults); for (const removedJob of removedJobs) { emit(state, { jobId: removedJob.id, action: "removed", job: removedJob }); } @@ -1909,28 +1945,31 @@ async function runStartupCatchupCandidate( candidate: StartupCatchupCandidate, ): Promise { const startedAt = state.deps.nowMs(); + const executionJob = structuredClone(candidate.job); + executionJob.state.runningAtMs = startedAt; const taskRunId = tryCreateCronTaskRun({ state, - job: candidate.job, + job: executionJob, startedAt, + runIdStartedAt: candidate.reservedAtMs, }); - const activeJobMarker = markCronJobActive(candidate.job.id, { - preserveAcrossGenerationAdvance: candidate.job.sessionTarget === "main", + const activeJobMarker = markCronJobActive(executionJob.id, { + preserveAcrossGenerationAdvance: executionJob.sessionTarget === "main", }); emit(state, { - jobId: candidate.job.id, + jobId: executionJob.id, action: "started", - job: candidate.job, + job: executionJob, runAtMs: startedAt, }); try { - const result = await executeJobCoreWithTimeout(state, candidate.job, { + const result = await executeJobCoreWithTimeout(state, executionJob, { runId: taskRunId, activeJobMarker, }); return { jobId: candidate.jobId, - job: candidate.job, + job: executionJob, taskRunId, activeJobMarker, status: result.status, @@ -1954,7 +1993,7 @@ async function runStartupCatchupCandidate( } catch (err) { return { jobId: candidate.jobId, - job: candidate.job, + job: executionJob, taskRunId, activeJobMarker, status: "error", @@ -2050,6 +2089,7 @@ async function applyStartupCatchupOutcomes( // startup overflow deferrals survive until their staggered catch-up tick. recomputeNextRunsForMaintenance(state, { repairFutureCronNextRunAtMs: false }); await persistOrRestore(state, rollbackSnapshot); + finishPersistedQuietCronTaskRuns(state, finalizedOutcomes); for (const removedJob of removedJobs) { emit(state, { jobId: removedJob.id, action: "removed", job: removedJob }); } @@ -2424,16 +2464,10 @@ async function executeDetachedCronJob( function emitJobFinished( state: CronServiceState, job: CronJob, - result: { - status: CronRunStatus; - delivered?: boolean; - delivery?: CronDeliveryTrace; - triggerEval?: CronTriggerEvalOutcome; - } & CronRunOutcome & - CronRunTelemetry, + result: TimedCronRunOutcome, runAtMs: number, ) { - emit(state, { + const event = { jobId: job.id, action: "finished", job, @@ -2455,7 +2489,14 @@ function emitJobFinished( model: result.model, provider: result.provider, usage: result.usage, + } as const; + tryFinishCronTaskRun(state, { + taskRunId: result.taskRunId, + job, + event, + ...(result.triggerEval ? { triggerEval: result.triggerEval } : {}), }); + emit(state, event); } /** Clears the currently armed cron timer. */ diff --git a/src/cron/task-run-detail.ts b/src/cron/task-run-detail.ts new file mode 100644 index 000000000000..1ef7820d5785 --- /dev/null +++ b/src/cron/task-run-detail.ts @@ -0,0 +1,181 @@ +/** Cron-owned codec between task-ledger detail and the stable run-history wire shape. */ +import { resolveFailoverReasonFromError } from "../agents/failover-error.js"; +import type { JsonValue, TaskRecord, TaskStatus } from "../tasks/task-registry.types.js"; +import type { CronRunLogEntry } from "./run-log-types.js"; +import { parseCronRunLogEntryObject } from "./run-log/entry-codec.js"; +import { timeoutErrorMessage } from "./service/execution-errors.js"; +import type { CronEvent } from "./service/state.js"; +import type { CronRunStatus } from "./types.js"; + +const CRON_TASK_DETAIL_KIND = "cron-run"; + +type CronFinishedEvent = CronEvent & { action: "finished" }; + +function toJsonValue(value: unknown): JsonValue | undefined { + const serialized = JSON.stringify(value); + return serialized === undefined ? undefined : (JSON.parse(serialized) as JsonValue); +} + +function isJsonObject(value: JsonValue | undefined): value is { [key: string]: JsonValue } { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +function isCronRunStatus(value: unknown): value is CronRunStatus { + return value === "ok" || value === "error" || value === "skipped"; +} + +/** Uses execution timing for one timestamp shared by ledger and legacy dual-write paths. */ +export function resolveCronRunEndedAt(event: CronFinishedEvent, fallbackTs: number): number { + if ( + typeof event.runAtMs === "number" && + Number.isFinite(event.runAtMs) && + typeof event.durationMs === "number" && + Number.isFinite(event.durationMs) + ) { + return event.runAtMs + event.durationMs; + } + return fallbackTs; +} + +/** Builds the legacy run-history record from one finished service event. */ +export function cronRunLogEntryFromEvent( + event: CronFinishedEvent, + fallbackTs: number, +): CronRunLogEntry { + const errorReason = resolveFailoverReasonFromError(event.error, event.provider) ?? undefined; + return { + ts: resolveCronRunEndedAt(event, fallbackTs), + jobId: event.jobId, + action: "finished", + status: event.status, + error: event.error, + errorReason, + summary: event.summary, + diagnostics: event.diagnostics, + delivered: event.delivered, + deliveryStatus: event.deliveryStatus, + deliveryError: event.deliveryError, + failureNotificationDelivery: event.failureNotificationDelivery, + delivery: event.delivery, + sessionId: event.sessionId, + sessionKey: event.sessionKey, + runId: event.runId, + runAtMs: event.runAtMs, + durationMs: event.durationMs, + nextRunAtMs: event.nextRunAtMs, + triggerFired: event.triggerFired, + model: event.model, + provider: event.provider, + usage: event.usage, + }; +} + +/** Encodes cron-only outcome fields; generic lifecycle fields stay on TaskRecord. */ +export function cronRunLogEntryToTaskDetail( + entry: CronRunLogEntry, + options: { + storeKey: string; + triggerEval?: { fired: boolean; stateChanged: boolean; state?: unknown }; + }, +): JsonValue { + const detail = toJsonValue({ + kind: CRON_TASK_DETAIL_KIND, + status: entry.status, + storeKey: options.storeKey, + errorReason: entry.errorReason, + diagnostics: entry.diagnostics, + delivered: entry.delivered, + deliveryStatus: entry.deliveryStatus, + deliveryError: entry.deliveryError, + failureNotificationDelivery: entry.failureNotificationDelivery, + delivery: entry.delivery, + sessionId: entry.sessionId, + // TaskRecord.runId remains the internal cancellation identity. + runId: entry.runId, + runAtMs: entry.runAtMs, + durationMs: entry.durationMs, + nextRunAtMs: entry.nextRunAtMs, + triggerFired: entry.triggerFired, + triggerStateChanged: + options.triggerEval?.fired === true ? options.triggerEval.stateChanged : undefined, + triggerState: + options.triggerEval?.fired === true && options.triggerEval.stateChanged + ? options.triggerEval.state + : undefined, + model: entry.model, + provider: entry.provider, + usage: entry.usage, + }); + return detail ?? { kind: CRON_TASK_DETAIL_KIND }; +} + +/** Returns the cron store partition recorded on a task row. */ +export function cronTaskRecordStoreKey(task: TaskRecord): string | undefined { + return isJsonObject(task.detail) && typeof task.detail.storeKey === "string" + ? task.detail.storeKey + : undefined; +} + +/** Reads internal trigger recovery data without adding it to run-history responses. */ +export function cronTaskRecordToTriggerEval( + task: TaskRecord, +): { fired: true; stateChanged: boolean; state?: JsonValue } | undefined { + if (!isJsonObject(task.detail) || task.detail.triggerFired !== true) { + return undefined; + } + return { + fired: true, + stateChanged: task.detail.triggerStateChanged === true, + ...(task.detail.triggerStateChanged === true && "triggerState" in task.detail + ? { state: task.detail.triggerState } + : {}), + }; +} + +/** Maps the cron outcome vocabulary onto generic task terminal states. */ +export function cronRunStatusToTaskStatus( + entry: CronRunLogEntry, +): Extract { + if (entry.status === "ok" || entry.status === "skipped") { + return "succeeded"; + } + return entry.error === timeoutErrorMessage() ? "timed_out" : "failed"; +} + +/** Reconstructs the unchanged CronRunLogEntry wire shape from a cron task row. */ +export function cronTaskRecordToRunLogEntry(task: TaskRecord): CronRunLogEntry | null { + if (task.runtime !== "cron" || !task.sourceId || !isJsonObject(task.detail)) { + return null; + } + if (task.detail.kind !== CRON_TASK_DETAIL_KIND || !isCronRunStatus(task.detail.status)) { + return null; + } + const wireDetail = { ...task.detail }; + delete wireDetail.storeKey; + const entry = parseCronRunLogEntryObject( + { + ...wireDetail, + ts: task.endedAt ?? task.lastEventAt ?? task.createdAt, + jobId: task.sourceId, + action: "finished", + status: task.detail.status, + error: task.error, + summary: task.terminalSummary, + sessionKey: task.childSessionKey, + runId: typeof task.detail.runId === "string" ? task.detail.runId : undefined, + }, + { jobId: task.sourceId }, + ); + if (!entry) { + return null; + } + // The legacy SQLite reader materializes these indexed columns even when absent. + return { + ...entry, + delivered: entry.delivered, + deliveryStatus: entry.deliveryStatus, + deliveryError: entry.deliveryError, + sessionId: entry.sessionId, + sessionKey: entry.sessionKey, + }; +} diff --git a/src/cron/task-run-history.test.ts b/src/cron/task-run-history.test.ts new file mode 100644 index 000000000000..b4e8a8c40d13 --- /dev/null +++ b/src/cron/task-run-history.test.ts @@ -0,0 +1,466 @@ +import { describe, expect, it, vi } from "vitest"; +import { resetTaskRegistryForTests } from "../tasks/task-registry.js"; +import { saveTaskRegistryStateToSqlite } from "../tasks/task-registry.store.sqlite.js"; +import type { TaskRecord } from "../tasks/task-registry.types.js"; +import { withOpenClawTestState } from "../test-utils/openclaw-test-state.js"; +import type { CronRunLogEntry } from "./run-log-types.js"; +import { appendCronRunLog, readCronRunLogEntriesPage } from "./run-log.js"; +import { CronService } from "./service.js"; +import { createNoopLogger } from "./service.test-harness.js"; +import { cronStoreKey } from "./store/key.js"; +import { + cronRunLogEntryFromEvent, + cronRunLogEntryToTaskDetail, + cronRunStatusToTaskStatus, + cronTaskRecordToRunLogEntry, +} from "./task-run-detail.js"; +import { readCronTaskRunHistoryPage } from "./task-run-history.js"; + +const JOB_ID = "history-job"; + +function taskFromEntry(entry: CronRunLogEntry, index: number, storeKey: string): TaskRecord { + return { + taskId: `task-${index}`, + runtime: "cron", + sourceId: entry.jobId, + requesterSessionKey: "", + ownerKey: "", + scopeKind: "system", + ...(entry.sessionKey ? { childSessionKey: entry.sessionKey } : {}), + agentId: "main", + runId: `cron:${entry.jobId}:${entry.runAtMs ?? entry.ts}`, + task: JOB_ID, + status: cronRunStatusToTaskStatus(entry), + deliveryStatus: "not_applicable", + notifyPolicy: "silent", + createdAt: entry.runAtMs ?? entry.ts, + startedAt: entry.runAtMs, + endedAt: entry.ts, + lastEventAt: entry.ts, + error: entry.error, + terminalSummary: entry.summary, + detail: cronRunLogEntryToTaskDetail(entry, { storeKey }), + }; +} + +function futureCronDetailTask(storeKey: string): TaskRecord { + return { + ...taskFromEntry( + { + ts: 400, + jobId: JOB_ID, + action: "finished", + status: "ok", + runAtMs: 390, + durationMs: 10, + }, + 4, + storeKey, + ), + taskId: "future-detail", + detail: { kind: "future-cron-detail", status: "ok" }, + }; +} + +describe("cron task run history", () => { + it("matches legacy history for executions produced by the cron service", async () => { + await withOpenClawTestState( + { layout: "state-only", prefix: "openclaw-cron-task-service-history-" }, + async (state) => { + resetTaskRegistryForTests(); + const storePath = state.path("cron", "jobs.json"); + const legacyWrites: Promise[] = []; + let now = Date.parse("2026-07-12T12:00:00.000Z"); + const cron = new CronService({ + storePath, + cronEnabled: true, + cronConfig: { triggers: { enabled: true } }, + log: createNoopLogger(), + nowMs: () => now, + enqueueSystemEvent: vi.fn(), + requestHeartbeat: vi.fn(), + evaluateCronTrigger: vi.fn(async () => ({ + kind: "evaluated" as const, + fire: true, + })), + runIsolatedAgentJob: vi.fn(async ({ job }) => { + if (job.name === "error") { + return { status: "error" as const, error: "provider overloaded" }; + } + if (job.name === "timeout") { + return { status: "error" as const, error: "cron: job execution timed out" }; + } + if (job.name === "skipped") { + return { status: "skipped" as const, error: "trigger condition not met" }; + } + return { + status: "ok" as const, + summary: "delivered", + delivered: true, + deliveryAttempted: true, + delivery: { + intended: { channel: "telegram", to: "42" }, + resolved: { channel: "telegram", to: "42", ok: true }, + messageToolSentTo: [{ channel: "telegram", to: "42" }], + delivered: true, + }, + sessionId: "session-ok", + sessionKey: "agent:main:cron:history:run:ok", + model: "gpt-test", + provider: "openai", + usage: { input_tokens: 10, output_tokens: 5, total_tokens: 15 }, + }; + }), + onEvent: (event) => { + if (event.action === "finished") { + legacyWrites.push( + appendCronRunLog({ + storePath, + entry: cronRunLogEntryFromEvent(event, Date.now()), + }), + ); + } + }, + }); + try { + await cron.start(); + for (const name of ["ok", "error", "timeout", "skipped"]) { + const job = await cron.add({ + name, + enabled: true, + schedule: { kind: "every", everyMs: 60_000 }, + ...(name === "ok" ? { trigger: { script: "true" } } : {}), + sessionTarget: "isolated", + wakeMode: "next-heartbeat", + payload: { kind: "agentTurn", message: name }, + delivery: + name === "ok" + ? { mode: "announce", channel: "telegram", to: "42" } + : { mode: "none" }, + }); + if (name === "ok") { + now = job.state.nextRunAtMs ?? now; + } + expect(await cron.run(job.id, name === "ok" ? "due" : "force")).toEqual({ + ok: true, + ran: true, + }); + now += 10_000; + } + await Promise.all(legacyWrites); + + const ledger = readCronTaskRunHistoryPage({ + storeKey: cronStoreKey(storePath), + limit: 50, + sortDir: "asc", + }); + const legacy = await readCronRunLogEntriesPage({ + storePath, + limit: 50, + sortDir: "asc", + }); + expect(ledger).toEqual(legacy); + expect(ledger.entries.map((entry) => entry.status)).toEqual([ + "ok", + "error", + "error", + "skipped", + ]); + expect(ledger.entries[0]).toMatchObject({ + deliveryStatus: "delivered", + triggerFired: true, + nextRunAtMs: expect.any(Number), + model: "gpt-test", + provider: "openai", + usage: { input_tokens: 10, output_tokens: 5, total_tokens: 15 }, + }); + } finally { + cron.stop(); + resetTaskRegistryForTests({ persist: false }); + } + }, + ); + }); + + it("matches legacy run-log records across outcomes and telemetry", async () => { + await withOpenClawTestState( + { layout: "state-only", prefix: "openclaw-cron-task-history-" }, + async (state) => { + const storePath = state.path("jobs.json"); + const storeKey = cronStoreKey(storePath); + const entries: CronRunLogEntry[] = [ + { + ts: 1_100, + jobId: JOB_ID, + action: "finished", + status: "ok", + summary: "delivered\n needle", + diagnostics: { + summary: "healthy", + entries: [ + { + ts: 1_050, + source: "agent-run", + severity: "info", + message: "diagnostic needle", + }, + ], + }, + delivered: true, + deliveryStatus: "delivered", + failureNotificationDelivery: { status: "not-requested" }, + delivery: { + intended: { channel: "telegram", to: "123" }, + resolved: { channel: "telegram", to: "123", ok: true }, + messageToolSentTo: [{ channel: "telegram", to: "123" }], + delivered: true, + }, + sessionId: "session-ok", + sessionKey: "agent:main:cron:history:run:ok", + runId: "manual:history:ok", + runAtMs: 1_000, + durationMs: 100, + nextRunAtMs: 2_000, + triggerFired: true, + model: "gpt-test", + provider: "openai", + usage: { + input_tokens: 10, + output_tokens: 5, + total_tokens: 15, + cache_read_tokens: 2, + cache_write_tokens: 1, + }, + }, + { + ts: 2_250, + jobId: JOB_ID, + action: "finished", + status: "error", + error: "provider overloaded", + deliveryStatus: "not-delivered", + deliveryError: "send failed", + runId: "manual:history:error", + runAtMs: 2_000, + durationMs: 250, + nextRunAtMs: 3_000, + provider: "openai", + }, + { + ts: 3_500, + jobId: JOB_ID, + action: "finished", + status: "error", + error: "cron: job execution timed out", + runId: "manual:history:timeout", + runAtMs: 3_000, + durationMs: 500, + nextRunAtMs: 4_000, + }, + { + ts: 4_000, + jobId: JOB_ID, + action: "finished", + status: "skipped", + error: "trigger condition not met", + summary: "", + runId: "manual:history:skipped", + runAtMs: 4_000, + durationMs: 0, + nextRunAtMs: 5_000, + }, + ]; + saveTaskRegistryStateToSqlite({ + tasks: new Map( + entries.map((entry, index) => [`task-${index}`, taskFromEntry(entry, index, storeKey)]), + ), + deliveryStates: new Map(), + }); + for (const entry of entries) { + await appendCronRunLog({ storePath, entry }); + } + + const ledger = readCronTaskRunHistoryPage({ storeKey, jobId: JOB_ID, limit: 50 }); + const legacy = await readCronRunLogEntriesPage({ + storePath, + jobId: JOB_ID, + limit: 50, + }); + expect(ledger).toEqual(legacy); + expect(ledger.entries.map((entry) => entry.status)).toEqual([ + "skipped", + "error", + "error", + "ok", + ]); + }, + ); + }); + + it("preserves paging and text-query filtering", async () => { + await withOpenClawTestState( + { layout: "state-only", prefix: "openclaw-cron-task-history-page-" }, + async (state) => { + const storeKey = cronStoreKey(state.path("jobs.json")); + const entries: CronRunLogEntry[] = [ + { + ts: 100, + jobId: JOB_ID, + action: "finished", + status: "ok", + summary: "first", + runAtMs: 90, + durationMs: 10, + }, + { + ts: 200, + jobId: JOB_ID, + action: "finished", + status: "error", + error: "needle failure", + runAtMs: 180, + durationMs: 20, + }, + { + ts: 300, + jobId: JOB_ID, + action: "finished", + status: "skipped", + summary: "third", + runAtMs: 300, + durationMs: 0, + }, + ]; + saveTaskRegistryStateToSqlite({ + tasks: new Map([ + ...entries.map( + (entry, index) => [`task-${index}`, taskFromEntry(entry, index, storeKey)] as const, + ), + ["future-detail", futureCronDetailTask(storeKey)] as const, + [ + "other-store", + { + ...taskFromEntry( + { + ts: 250, + jobId: JOB_ID, + action: "finished", + status: "ok", + summary: "foreign partition", + }, + 5, + "/other/cron/store", + ), + taskId: "other-store", + }, + ] as const, + [ + "missing-store-key", + { + ...taskFromEntry(entries[0], 6, storeKey), + taskId: "missing-store-key", + detail: { kind: "cron-run", status: "ok" }, + }, + ] as const, + ]), + deliveryStates: new Map(), + }); + + expect( + readCronTaskRunHistoryPage({ storeKey, jobId: JOB_ID, limit: 1, offset: 1 }), + ).toMatchObject({ + entries: [expect.objectContaining({ ts: 200 })], + total: 3, + offset: 1, + limit: 1, + hasMore: true, + nextOffset: 2, + }); + expect( + readCronTaskRunHistoryPage({ + storeKey, + jobId: JOB_ID, + query: "needle", + status: "error", + limit: 50, + }).entries, + ).toEqual([expect.objectContaining({ ts: 200, error: "needle failure" })]); + }, + ); + }); + + it("keeps same-job histories and totals scoped to one cron store", async () => { + await withOpenClawTestState( + { layout: "state-only", prefix: "openclaw-cron-task-history-store-scope-" }, + async (state) => { + const storeA = cronStoreKey(state.path("cron-a", "jobs.json")); + const storeB = cronStoreKey(state.path("cron-b", "jobs.json")); + const entryA: CronRunLogEntry = { + ts: 100, + jobId: JOB_ID, + action: "finished", + status: "ok", + summary: "store a", + }; + const entryB: CronRunLogEntry = { + ts: 200, + jobId: JOB_ID, + action: "finished", + status: "error", + error: "store b", + }; + saveTaskRegistryStateToSqlite({ + tasks: new Map([ + ["store-a", { ...taskFromEntry(entryA, 1, storeA), taskId: "store-a" }], + ["store-b", { ...taskFromEntry(entryB, 2, storeB), taskId: "store-b" }], + ]), + deliveryStates: new Map(), + }); + + expect(readCronTaskRunHistoryPage({ storeKey: storeA, jobId: JOB_ID })).toMatchObject({ + entries: [expect.objectContaining({ summary: "store a" })], + total: 1, + hasMore: false, + }); + expect(readCronTaskRunHistoryPage({ storeKey: storeB, jobId: JOB_ID })).toMatchObject({ + entries: [expect.objectContaining({ error: "store b" })], + total: 1, + hasMore: false, + }); + }, + ); + }); + + it("keeps the internal store key out of the legacy wire record", () => { + const storeKey = "/internal/cron/store"; + const task = taskFromEntry( + { ts: 100, jobId: JOB_ID, action: "finished", status: "ok" }, + 1, + storeKey, + ); + const entry = cronTaskRecordToRunLogEntry(task); + expect(entry).not.toBeNull(); + expect(Object.hasOwn(entry ?? {}, "storeKey")).toBe(false); + }); + + it("locks the serialized detail shape: kind first, status second", () => { + // External tooling may prefix-match serialized detail; keep the codec's + // field order stable so those prefixes stay meaningful. + for (const status of ["ok", "error", "skipped"] as const) { + const detail = cronRunLogEntryToTaskDetail( + { + ts: 100, + jobId: JOB_ID, + action: "finished", + status, + }, + { storeKey: "/tmp/cron-history" }, + ); + const serialized = JSON.stringify(detail); + expect( + serialized.startsWith(`{"kind":"cron-run","status":"${status}"`), + `detail for status "${status}" must keep the stable prefix: ${serialized}`, + ).toBe(true); + } + }); +}); diff --git a/src/cron/task-run-history.ts b/src/cron/task-run-history.ts new file mode 100644 index 000000000000..9ae52e6ef567 --- /dev/null +++ b/src/cron/task-run-history.ts @@ -0,0 +1,180 @@ +/** Cron run-history reads backed by authoritative task-ledger rows. */ +import { + normalizeLowercaseStringOrEmpty, + normalizeOptionalString, +} from "@openclaw/normalization-core/string-coerce"; +import { uniqueValues } from "@openclaw/normalization-core/string-normalization"; +import { listTaskRegistryRecordsByRuntimeSourceIdFromSqlite } from "../tasks/task-registry.store.sqlite.js"; +import type { TaskRecord } from "../tasks/task-registry.types.js"; +import type { CronRunLogEntry } from "./run-log-types.js"; +import { cronTaskRecordStoreKey, cronTaskRecordToRunLogEntry } from "./task-run-detail.js"; +import type { CronDeliveryStatus, CronRunStatus } from "./types.js"; + +type CronRunHistorySortDir = "asc" | "desc"; +type CronRunHistoryStatusFilter = "all" | CronRunStatus; + +export type ReadCronTaskRunHistoryPageOptions = { + storeKey: string; + limit?: number; + offset?: number; + jobId?: string; + /** Narrows the page to these job ids (caller-scope filtering). */ + jobIds?: readonly string[]; + runId?: string; + status?: CronRunHistoryStatusFilter; + statuses?: CronRunStatus[]; + deliveryStatus?: CronDeliveryStatus; + deliveryStatuses?: CronDeliveryStatus[]; + query?: string; + sortDir?: CronRunHistorySortDir; + jobNameById?: Record; +}; + +export type CronTaskRunHistoryPage = { + entries: CronRunLogEntry[]; + total: number; + offset: number; + limit: number; + hasMore: boolean; + nextOffset: number | null; +}; + +const INVALID_CRON_TASK_RUN_JOB_ID_MESSAGE = "invalid cron task run job id"; + +export function normalizeCronTaskRunJobId(jobId: string): string { + const trimmed = jobId.trim(); + if (!trimmed || trimmed.includes("/") || trimmed.includes("\\") || trimmed.includes("\0")) { + throw new Error(INVALID_CRON_TASK_RUN_JOB_ID_MESSAGE); + } + return trimmed; +} + +export function isInvalidCronTaskRunJobIdError(error: unknown): boolean { + return error instanceof Error && error.message === INVALID_CRON_TASK_RUN_JOB_ID_MESSAGE; +} + +function normalizeStatuses(options: ReadCronTaskRunHistoryPageOptions): CronRunStatus[] | null { + if (options.statuses?.length) { + const statuses = options.statuses.filter(isCronRunStatus); + if (statuses.length > 0) { + return uniqueValues(statuses); + } + } + return isCronRunStatus(options.status) ? [options.status] : null; +} + +function isCronRunStatus(value: unknown): value is CronRunStatus { + return value === "ok" || value === "error" || value === "skipped"; +} + +function isCronDeliveryStatus(value: unknown): value is CronDeliveryStatus { + return ( + value === "delivered" || + value === "not-delivered" || + value === "unknown" || + value === "not-requested" + ); +} + +function normalizeDeliveryStatuses( + options: ReadCronTaskRunHistoryPageOptions, +): CronDeliveryStatus[] | null { + if (options.deliveryStatuses?.length) { + const statuses = options.deliveryStatuses.filter(isCronDeliveryStatus); + if (statuses.length > 0) { + return uniqueValues(statuses); + } + } + return isCronDeliveryStatus(options.deliveryStatus) ? [options.deliveryStatus] : null; +} + +function queryText(entry: CronRunLogEntry, jobNameById?: Record): string { + return [ + entry.summary ?? "", + entry.error ?? "", + entry.errorReason ?? "", + entry.diagnostics?.summary ?? "", + ...(entry.diagnostics?.entries ?? []).map((diagnostic) => diagnostic.message), + entry.jobId, + jobNameById?.[entry.jobId] ?? "", + entry.delivery?.intended?.channel ?? "", + entry.delivery?.resolved?.channel ?? "", + ...(entry.delivery?.messageToolSentTo ?? []).map((target) => target.channel), + ].join(" "); +} + +function compareHistoryRows( + left: { entry: CronRunLogEntry; task: TaskRecord }, + right: { entry: CronRunLogEntry; task: TaskRecord }, + direction: CronRunHistorySortDir, +): number { + const multiplier = direction === "asc" ? 1 : -1; + return ( + multiplier * (left.entry.ts - right.entry.ts) || + multiplier * (left.task.createdAt - right.task.createdAt) || + multiplier * left.task.taskId.localeCompare(right.task.taskId) + ); +} + +function attachJobNames(entries: CronRunLogEntry[], jobNameById?: Record): void { + for (const entry of entries) { + const jobName = jobNameById?.[entry.jobId]; + if (jobName) { + (entry as CronRunLogEntry & { jobName?: string }).jobName = jobName; + } + } +} + +/** Reads and filters cron task rows with the legacy run-history paging contract. */ +export function readCronTaskRunHistoryPage( + options: ReadCronTaskRunHistoryPageOptions, +): CronTaskRunHistoryPage { + const jobId = options.jobId ? normalizeCronTaskRunJobId(options.jobId) : undefined; + const limit = Math.max(1, Math.min(200, Math.floor(options.limit ?? 50))); + const offset = Math.max(0, Math.floor(options.offset ?? 0)); + const statuses = normalizeStatuses(options); + const deliveryStatuses = normalizeDeliveryStatuses(options); + const runId = normalizeOptionalString(options.runId); + const jobIds = options.jobIds ? new Set(options.jobIds) : undefined; + const query = normalizeLowercaseStringOrEmpty(options.query); + const sortDir: CronRunHistorySortDir = options.sortDir === "asc" ? "asc" : "desc"; + const rows = listTaskRegistryRecordsByRuntimeSourceIdFromSqlite({ + runtime: "cron", + sourceId: jobId, + }) + .filter((task) => cronTaskRecordStoreKey(task) === options.storeKey) + .map((task) => ({ task, entry: cronTaskRecordToRunLogEntry(task) })) + .filter((row): row is { task: TaskRecord; entry: CronRunLogEntry } => row.entry !== null) + .filter(({ entry }) => { + if (jobIds && !jobIds.has(entry.jobId)) { + return false; + } + if (runId && entry.runId !== runId) { + return false; + } + if (statuses && (!entry.status || !statuses.includes(entry.status))) { + return false; + } + if (deliveryStatuses && !deliveryStatuses.includes(entry.deliveryStatus ?? "not-requested")) { + return false; + } + return ( + !query || + normalizeLowercaseStringOrEmpty(queryText(entry, options.jobNameById)).includes(query) + ); + }) + .toSorted((left, right) => compareHistoryRows(left, right, sortDir)); + const total = rows.length; + const boundedOffset = Math.min(total, offset); + const entries = rows.slice(boundedOffset, boundedOffset + limit).map(({ entry }) => entry); + attachJobNames(entries, options.jobNameById); + const nextOffset = boundedOffset + entries.length; + return { + entries, + total, + offset: boundedOffset, + limit, + hasMore: nextOffset < total, + nextOffset: nextOffset < total ? nextOffset : null, + }; +} diff --git a/src/gateway/server-cron.ts b/src/gateway/server-cron.ts index d24bf50f9c39..42ba93558bb3 100644 --- a/src/gateway/server-cron.ts +++ b/src/gateway/server-cron.ts @@ -31,6 +31,7 @@ import { resolveCronSessionTargetSessionKey, } from "../cron/session-target.js"; import { resolveCronJobsStorePath } from "../cron/store.js"; +import { cronRunLogEntryFromEvent } from "../cron/task-run-detail.js"; import { createCronTriggerEvaluator } from "../cron/trigger-script.js"; import type { CronJob, CronPayload } from "../cron/types.js"; import { formatErrorMessage } from "../infra/errors.js"; @@ -784,6 +785,7 @@ export function buildGatewayCronService(params: { } if (evt.action === "finished") { const job = evt.job ?? cron.getJob(evt.jobId); + const runLogEntry = cronRunLogEntryFromEvent({ ...evt, action: "finished" }, Date.now()); dispatchGatewayCronFinishedNotifications({ evt, job, @@ -797,30 +799,7 @@ export function buildGatewayCronService(params: { void runWithGatewayIndependentRootWorkAdmission(async () => { await appendCronRunLog({ storePath, - entry: { - ts: Date.now(), - jobId: evt.jobId, - action: "finished", - status: evt.status, - error: evt.error, - summary: evt.summary, - diagnostics: evt.diagnostics, - delivered: evt.delivered, - deliveryStatus: evt.deliveryStatus, - deliveryError: evt.deliveryError, - failureNotificationDelivery: evt.failureNotificationDelivery, - delivery: evt.delivery, - sessionId: evt.sessionId, - sessionKey: evt.sessionKey, - runId: evt.runId, - runAtMs: evt.runAtMs, - durationMs: evt.durationMs, - nextRunAtMs: evt.nextRunAtMs, - triggerFired: evt.triggerFired, - model: evt.model, - provider: evt.provider, - usage: evt.usage, - }, + entry: runLogEntry, opts: { keepLines: runLogPrune.keepLines }, }); }).catch((err: unknown) => { diff --git a/src/gateway/server-methods/cron.ts b/src/gateway/server-methods/cron.ts index e1340021077f..7ba7f728d1b3 100644 --- a/src/gateway/server-methods/cron.ts +++ b/src/gateway/server-methods/cron.ts @@ -23,11 +23,6 @@ import { import { resolveCronDeliveryPreviews } from "../../cron/delivery-preview.js"; import { assertCronDeliveryInputNonBlankFields } from "../../cron/delivery-target-validation.js"; import { normalizeCronJobCreate, normalizeCronJobPatch } from "../../cron/normalize.js"; -import { - isInvalidCronRunLogJobIdError, - readCronRunLogEntriesPage, - readCronRunLogEntriesPageAll, -} from "../../cron/run-log.js"; import { applyJobPatch } from "../../cron/service/jobs.js"; import type { CronListPageOptions, @@ -37,6 +32,11 @@ import { isInvalidCronSessionTargetIdError, resolveCronSessionTargetSessionKey, } from "../../cron/session-target.js"; +import { cronStoreKey } from "../../cron/store/key.js"; +import { + isInvalidCronTaskRunJobIdError, + readCronTaskRunHistoryPage, +} from "../../cron/task-run-history.js"; import type { CronJob, CronJobCreate, CronJobPatch } from "../../cron/types.js"; import { validateScheduleTimestamp } from "../../cron/validate-timestamp.js"; import { formatErrorMessage } from "../../infra/errors.js"; @@ -940,8 +940,8 @@ export const cronHandlers: GatewayRequestHandlers = { .filter((job) => typeof job.id === "string" && typeof job.name === "string") .map((job) => [job.id, job.name]), ); - const page = await readCronRunLogEntriesPageAll({ - storePath: context.cronStorePath, + const page = readCronTaskRunHistoryPage({ + storeKey: cronStoreKey(context.cronStorePath), ...cronRunLogPageFilters(p), ...(p.agentId ? { jobIds: jobs.map((job) => job.id) } : {}), jobNameById, @@ -972,15 +972,15 @@ export const cronHandlers: GatewayRequestHandlers = { matchedJob && typeof matchedJob.name === "string" ? { [jobId as string]: matchedJob.name } : undefined; - const page = await readCronRunLogEntriesPage({ - storePath: context.cronStorePath, + const page = readCronTaskRunHistoryPage({ + storeKey: cronStoreKey(context.cronStorePath), jobId: jobId as string, ...cronRunLogPageFilters(p), jobNameById, }); respond(true, page, undefined); } catch (err) { - if (!isInvalidCronRunLogJobIdError(err)) { + if (!isInvalidCronTaskRunJobIdError(err)) { throw err; } respond( diff --git a/src/gateway/server-startup-early.ts b/src/gateway/server-startup-early.ts index 101ff24d1989..7a5735e72cc5 100644 --- a/src/gateway/server-startup-early.ts +++ b/src/gateway/server-startup-early.ts @@ -2,7 +2,6 @@ // Starts discovery, remote skills, task maintenance, and delayed maintenance setup. import type { GatewayTailscaleMode } from "../config/types.gateway.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; -import { resolveCronJobsStorePath } from "../cron/store.js"; import type { PluginRegistry } from "../plugins/registry-types.js"; type Awaitable = T | Promise; @@ -133,9 +132,8 @@ export async function startGatewayEarlyRuntime(params: { setSkillsRemoteRegistry(params.nodeRegistry); void primeRemoteSkillsCache(); // Task registry maintenance is authoritative in the Gateway process so - // restart-blocker counts reflect the same cron store as runtime execution. + // restart-blocker counts reflect the same live cron runtime. taskRegistryMaintenance.configureTaskRegistryMaintenance({ - cronStorePath: resolveCronJobsStorePath(params.cfgAtStart.cron?.store), runtimeAuthoritative: true, }); taskRegistryMaintenance.startTaskRegistryMaintenance(); diff --git a/src/infra/state-migrations.cron-run-logs.test.ts b/src/infra/state-migrations.cron-run-logs.test.ts new file mode 100644 index 000000000000..4b457c993797 --- /dev/null +++ b/src/infra/state-migrations.cron-run-logs.test.ts @@ -0,0 +1,186 @@ +import { DatabaseSync } from "node:sqlite"; +import { describe, expect, it } from "vitest"; +import type { CronRunLogEntry } from "../cron/run-log-types.js"; +import { readCronRunLogEntriesPage } from "../cron/run-log.js"; +import { insertCronRunLogEntry } from "../cron/run-log/sqlite-store.js"; +import { cronStoreKey } from "../cron/store/key.js"; +import { cronRunLogEntryToTaskDetail, cronRunStatusToTaskStatus } from "../cron/task-run-detail.js"; +import { readCronTaskRunHistoryPage } from "../cron/task-run-history.js"; +import { resetTaskRegistryForTests } from "../tasks/task-registry.js"; +import { withOpenClawTestState } from "../test-utils/openclaw-test-state.js"; +import { + closeOpenClawStateDatabaseForTest, + openOpenClawStateDatabase, +} from "../state/openclaw-state-db.js"; +import { + CRON_RUN_LOG_TASK_IMPORT_MIGRATION_ID, + migrateLegacyCronRunLogsToTaskRuns, +} from "./state-migrations.cron-run-logs.js"; + +describe("cron run-log task import", () => { + it("imports legacy cron history into task runs once at state database open", async () => { + await withOpenClawTestState( + { layout: "state-only", prefix: "openclaw-cron-run-log-import-" }, + async (state) => { + const storePath = state.path("cron", "jobs.json"); + const storeKey = cronStoreKey(storePath); + const jobId = "legacy-history-job"; + const entries: CronRunLogEntry[] = [ + { + ts: 1_100, + jobId, + action: "finished", + status: "ok", + summary: "legacy one", + sessionKey: "agent:main:cron:legacy:run:1", + runId: "manual:legacy:1", + runAtMs: 1_000, + durationMs: 100, + }, + { + ts: 2_100, + jobId, + action: "finished", + status: "error", + error: "legacy failure", + runAtMs: 2_000, + durationMs: 100, + }, + { + ts: 2_100, + jobId, + action: "finished", + status: "ok", + summary: "same millisecond legacy run", + runAtMs: 2_001, + durationMs: 99, + }, + { + ts: 3_100, + jobId, + action: "finished", + status: "error", + error: "different public run id", + runId: "manual:legacy:same-ts", + runAtMs: 3_000, + durationMs: 100, + }, + { + ts: 3_100, + jobId, + action: "finished", + status: "skipped", + runId: "manual:mirrored:3", + runAtMs: 3_001, + durationMs: 99, + }, + { + ts: 4_100, + jobId, + action: "finished", + status: "ok", + summary: "mirrored without public run id", + runAtMs: 4_000, + durationMs: 100, + }, + ]; + + const initial = openOpenClawStateDatabase(); + const databasePath = initial.path; + closeOpenClawStateDatabaseForTest(); + const fixture = new DatabaseSync(databasePath); + try { + fixture + .prepare("DELETE FROM migration_runs WHERE id = ?") + .run(CRON_RUN_LOG_TASK_IMPORT_MIGRATION_ID); + for (const entry of entries) { + insertCronRunLogEntry(fixture, storeKey, entry); + } + const insertMirrored = fixture.prepare( + `INSERT INTO task_runs ( + task_id, runtime, source_id, requester_session_key, owner_key, scope_kind, + child_session_key, run_id, task, status, delivery_status, notify_policy, created_at, + started_at, ended_at, last_event_at, error, terminal_summary, terminal_outcome, + detail_json + ) VALUES (?, 'cron', ?, '', '', 'system', ?, ?, ?, ?, 'not_applicable', 'silent', + ?, ?, ?, ?, ?, ?, ?, ?)`, + ); + for (const [index, mirrored] of entries.slice(4).entries()) { + const mirroredStatus = cronRunStatusToTaskStatus(mirrored); + insertMirrored.run( + `already-mirrored-${index}`, + jobId, + mirrored.sessionKey ?? null, + `cron:legacy-history-job:${mirrored.runAtMs}:mirrored`, + jobId, + mirroredStatus, + mirrored.runAtMs ?? mirrored.ts, + mirrored.runAtMs ?? null, + mirrored.ts, + mirrored.ts, + mirrored.error ?? null, + mirrored.summary ?? null, + mirroredStatus === "succeeded" ? "succeeded" : null, + JSON.stringify(cronRunLogEntryToTaskDetail(mirrored, { storeKey })), + ); + } + fixture + .prepare( + `INSERT INTO cron_run_logs + (store_key, job_id, seq, ts, entry_json, created_at) + VALUES (?, ?, 7, 5100, '{', 5100)`, + ) + .run(storeKey, jobId); + } finally { + fixture.close(); + } + + const reopened = openOpenClawStateDatabase(); + const report = reopened.db + .prepare("SELECT report_json FROM migration_runs WHERE id = ?") + .get(CRON_RUN_LOG_TASK_IMPORT_MIGRATION_ID) as { report_json: string }; + expect(JSON.parse(report.report_json)).toEqual({ + imported: 4, + alreadyMirrored: 2, + malformed: 1, + skipped: false, + }); + const ledgerEntries = readCronTaskRunHistoryPage({ + storeKey, + jobId, + limit: 50, + sortDir: "asc", + }).entries; + const legacyEntries = ( + await readCronRunLogEntriesPage({ storePath, jobId, limit: 50, sortDir: "asc" }) + ).entries; + expect(ledgerEntries).toEqual(legacyEntries); + const imported = reopened.db + .prepare( + "SELECT task_id, cleanup_after FROM task_runs WHERE task_id LIKE 'cron-runlog-import:%' ORDER BY task_id", + ) + .all() as Array<{ task_id: string; cleanup_after: number | null }>; + expect(imported).toHaveLength(4); + expect(imported.map((row) => row.task_id)).toEqual([ + expect.stringMatching(/^cron-runlog-import:[a-f0-9]{16}:legacy-history-job:1100:1$/u), + expect.stringMatching(/^cron-runlog-import:[a-f0-9]{16}:legacy-history-job:2100:2$/u), + expect.stringMatching(/^cron-runlog-import:[a-f0-9]{16}:legacy-history-job:2100:3$/u), + expect.stringMatching(/^cron-runlog-import:[a-f0-9]{16}:legacy-history-job:3100:4$/u), + ]); + expect(imported.every((row) => row.cleanup_after === null)).toBe(true); + + closeOpenClawStateDatabaseForTest(); + const secondOpen = openOpenClawStateDatabase(); + expect(secondOpen.db.prepare("SELECT COUNT(*) AS count FROM task_runs").get()).toEqual({ + count: 6, + }); + expect( + secondOpen.db + .prepare("SELECT report_json FROM migration_runs WHERE id = ?") + .get(CRON_RUN_LOG_TASK_IMPORT_MIGRATION_ID), + ).toEqual({ report_json: report.report_json }); + resetTaskRegistryForTests({ persist: false }); + }, + ); + }); +}); diff --git a/src/infra/state-migrations.cron-run-logs.ts b/src/infra/state-migrations.cron-run-logs.ts new file mode 100644 index 000000000000..d2632c36becd --- /dev/null +++ b/src/infra/state-migrations.cron-run-logs.ts @@ -0,0 +1,170 @@ +/** One-shot import of legacy cron run history into the authoritative task ledger. */ +import type { DatabaseSync } from "node:sqlite"; +import type { Selectable } from "kysely"; +import { parseStoredRunLogEntry } from "../cron/run-log/sqlite-store.js"; +import { cronRunLogEntryToTaskDetail, cronRunStatusToTaskStatus } from "../cron/task-run-detail.js"; +import type { DB as OpenClawStateDatabase } from "../state/openclaw-state-db.generated.js"; +import { sha256HexPrefix } from "./crypto-digest.js"; + +export const CRON_RUN_LOG_TASK_IMPORT_MIGRATION_ID = "state:cron-run-logs-to-task-runs:v1"; + +type CronRunLogRow = Selectable; +type MirroredTask = { + source_id: string | null; + ended_at: number | null; + detail_json: string | null; +}; +type MirroredIdentity = { endedAt: number | null; runId?: string }; + +export type CronRunLogTaskImportResult = { + imported: number; + alreadyMirrored: number; + malformed: number; + skipped: boolean; +}; + +function parseDetail(raw: string | null): Record | undefined { + if (!raw) { + return undefined; + } + try { + const parsed: unknown = JSON.parse(raw); + return parsed !== null && typeof parsed === "object" && !Array.isArray(parsed) + ? (parsed as Record) + : undefined; + } catch { + return undefined; + } +} + +function mirroredKey(sourceId: string, storeKey: string): string { + return `${sourceId}\0${storeKey}`; +} + +function collectMirroredTasks(db: DatabaseSync): Map { + const rows = db + .prepare( + `SELECT source_id, ended_at, detail_json + FROM task_runs + WHERE runtime = 'cron' AND source_id IS NOT NULL AND detail_json IS NOT NULL`, + ) + .all() as MirroredTask[]; + const bySource = new Map(); + for (const row of rows) { + const detail = parseDetail(row.detail_json); + const storeKey = typeof detail?.storeKey === "string" ? detail.storeKey : undefined; + if (!row.source_id || !storeKey) { + continue; + } + const key = mirroredKey(row.source_id, storeKey); + const identities = bySource.get(key) ?? []; + identities.push({ + endedAt: row.ended_at, + ...(typeof detail.runId === "string" ? { runId: detail.runId } : {}), + }); + bySource.set(key, identities); + } + return bySource; +} + +function consumeMirroredIdentity( + identities: MirroredIdentity[], + runId: string | undefined, + endedAt: number, +): boolean { + let index = runId ? identities.findIndex((identity) => identity.runId === runId) : -1; + if (index < 0) { + index = identities.findIndex( + (identity) => identity.runId === undefined && identity.endedAt === endedAt, + ); + } + if (index < 0 && !runId) { + index = identities.findIndex((identity) => identity.endedAt === endedAt); + } + if (index < 0) { + return false; + } + identities.splice(index, 1); + return true; +} + +/** Runs inside the state schema transaction; completed receipt makes later opens no-ops. */ +export function migrateLegacyCronRunLogsToTaskRuns(db: DatabaseSync): CronRunLogTaskImportResult { + const receipt = db + .prepare("SELECT status FROM migration_runs WHERE id = ?") + .get(CRON_RUN_LOG_TASK_IMPORT_MIGRATION_ID) as { status?: unknown } | undefined; + if (receipt?.status === "completed") { + return { imported: 0, alreadyMirrored: 0, malformed: 0, skipped: true }; + } + + const rows = db + .prepare("SELECT * FROM cron_run_logs ORDER BY store_key, job_id, seq") + .all() as CronRunLogRow[]; + const mirrored = collectMirroredTasks(db); + // Leave cleanup unset; follow-up source-aware retention owns imported cron history. + const insert = db.prepare(` + INSERT INTO task_runs ( + task_id, runtime, task_kind, source_id, requester_session_key, owner_key, scope_kind, + child_session_key, parent_flow_id, parent_task_id, agent_id, requester_agent_id, run_id, + label, task, status, delivery_status, notify_policy, created_at, started_at, ended_at, + last_event_at, cleanup_after, error, progress_summary, terminal_summary, terminal_outcome, + detail_json + ) VALUES ( + @task_id, 'cron', NULL, @source_id, '', '', 'system', @child_session_key, NULL, NULL, + NULL, NULL, @run_id, NULL, @task, @status, 'not_applicable', 'silent', @created_at, + @started_at, @ended_at, @ended_at, NULL, @error, NULL, @terminal_summary, + @terminal_outcome, @detail_json + ) + `); + let imported = 0; + let alreadyMirroredCount = 0; + let malformed = 0; + for (const row of rows) { + const entry = parseStoredRunLogEntry(row); + if (!entry) { + malformed++; + continue; + } + const key = mirroredKey(row.job_id, row.store_key); + const identities = mirrored.get(key) ?? []; + if (consumeMirroredIdentity(identities, entry.runId, entry.ts)) { + alreadyMirroredCount++; + continue; + } + const taskId = `cron-runlog-import:${sha256HexPrefix(row.store_key, 16)}:${row.job_id}:${entry.ts}:${String(row.seq)}`; + const status = cronRunStatusToTaskStatus(entry); + insert.run({ + task_id: taskId, + source_id: row.job_id, + child_session_key: entry.sessionKey ?? null, + run_id: taskId, + task: row.job_id, + status, + created_at: entry.runAtMs ?? entry.ts, + started_at: entry.runAtMs ?? null, + ended_at: entry.ts, + error: entry.error ?? null, + terminal_summary: entry.summary ?? null, + terminal_outcome: status === "succeeded" ? "succeeded" : null, + detail_json: JSON.stringify(cronRunLogEntryToTaskDetail(entry, { storeKey: row.store_key })), + }); + imported++; + } + + const result = { + imported, + alreadyMirrored: alreadyMirroredCount, + malformed, + skipped: false, + }; + const now = Date.now(); + db.prepare( + `INSERT INTO migration_runs (id, started_at, finished_at, status, report_json) + VALUES (?, ?, ?, 'completed', ?) + ON CONFLICT(id) DO UPDATE SET + finished_at = excluded.finished_at, + status = excluded.status, + report_json = excluded.report_json`, + ).run(CRON_RUN_LOG_TASK_IMPORT_MIGRATION_ID, now, now, JSON.stringify(result)); + return result; +} diff --git a/src/infra/state-migrations.storage.ts b/src/infra/state-migrations.storage.ts index c9a8c279afd4..ea0119c76f77 100644 --- a/src/infra/state-migrations.storage.ts +++ b/src/infra/state-migrations.storage.ts @@ -612,6 +612,7 @@ function normalizeLegacyTaskRow(row: Record): SqliteBindRow { progress_summary: legacyBindValue(row.progress_summary), terminal_summary: legacyBindValue(row.terminal_summary), terminal_outcome: legacyBindValue(row.terminal_outcome), + detail_json: legacyBindValue(row.detail_json), }; } @@ -703,6 +704,7 @@ function readLegacyTaskRows(sourcePath: string): SqliteBindRow[] { pickLegacyColumn(columns, "progress_summary"), pickLegacyColumn(columns, "terminal_summary"), pickLegacyColumn(columns, "terminal_outcome"), + pickLegacyColumn(columns, "detail_json"), ]; return db .prepare( @@ -781,13 +783,14 @@ function insertTaskRunRowSql(db: DatabaseSync, row: SqliteBindRow): void { task_id, runtime, task_kind, source_id, requester_session_key, owner_key, scope_kind, child_session_key, parent_flow_id, parent_task_id, agent_id, requester_agent_id, run_id, label, task, status, delivery_status, notify_policy, created_at, started_at, ended_at, - last_event_at, cleanup_after, error, progress_summary, terminal_summary, terminal_outcome + last_event_at, cleanup_after, error, progress_summary, terminal_summary, terminal_outcome, + detail_json ) VALUES ( @task_id, @runtime, @task_kind, @source_id, @requester_session_key, @owner_key, @scope_kind, @child_session_key, @parent_flow_id, @parent_task_id, @agent_id, @requester_agent_id, @run_id, @label, @task, @status, @delivery_status, @notify_policy, @created_at, @started_at, @ended_at, @last_event_at, @cleanup_after, @error, - @progress_summary, @terminal_summary, @terminal_outcome + @progress_summary, @terminal_summary, @terminal_outcome, @detail_json ) `, ).run(row); @@ -882,6 +885,7 @@ async function migrateLegacyTaskRunsSidecar(params: { "progress_summary", "terminal_summary", "terminal_outcome", + "detail_json", ]; for (const row of taskRows) { const taskId = legacyKeyValue(expectDefined(row.task_id, "task migration row key")); diff --git a/src/infra/state-migrations.ts b/src/infra/state-migrations.ts index 2d6f41b79aa3..bb54a5e6401a 100644 --- a/src/infra/state-migrations.ts +++ b/src/infra/state-migrations.ts @@ -18,3 +18,7 @@ export { resetAutoMigrateLegacyStateDirForTest, resetAutoMigrateLegacyTaskStateSidecarsForTest, } from "./state-migrations.state-dir.js"; +export { + CRON_RUN_LOG_TASK_IMPORT_MIGRATION_ID, + migrateLegacyCronRunLogsToTaskRuns, +} from "./state-migrations.cron-run-logs.js"; diff --git a/src/state/openclaw-state-db.generated.d.ts b/src/state/openclaw-state-db.generated.d.ts index 4c0265415d5d..1eb814a0e026 100644 --- a/src/state/openclaw-state-db.generated.d.ts +++ b/src/state/openclaw-state-db.generated.d.ts @@ -1004,6 +1004,7 @@ export interface TaskRuns { cleanup_after: number | null; created_at: number; delivery_status: string; + detail_json: string | null; ended_at: number | null; error: string | null; label: string | null; diff --git a/src/state/openclaw-state-db.test.ts b/src/state/openclaw-state-db.test.ts index 3417bd38e130..5526839f9c36 100644 --- a/src/state/openclaw-state-db.test.ts +++ b/src/state/openclaw-state-db.test.ts @@ -2272,6 +2272,28 @@ describe("openclaw state database", () => { closeOpenClawStateDatabaseForTest(); }); + it("adds task detail storage to an existing state database", () => { + const stateDir = createTempStateDir(); + const database = openOpenClawStateDatabase({ + env: { OPENCLAW_STATE_DIR: stateDir }, + }); + const databasePath = database.path; + closeOpenClawStateDatabaseForTest(); + + const { DatabaseSync } = requireNodeSqlite(); + const legacyDb = new DatabaseSync(databasePath); + legacyDb.exec("ALTER TABLE task_runs DROP COLUMN detail_json"); + legacyDb.close(); + + const reopened = openOpenClawStateDatabase({ + env: { OPENCLAW_STATE_DIR: stateDir }, + }); + const columns = reopened.db.prepare("PRAGMA table_info(task_runs)").all() as Array<{ + name?: string; + }>; + expect(columns.some((column) => column.name === "detail_json")).toBe(true); + }); + it("rolls back the requester attribution column when its backfill fails", () => { const stateDir = createTempStateDir(); const database = openOpenClawStateDatabase({ diff --git a/src/state/openclaw-state-db.ts b/src/state/openclaw-state-db.ts index fde39ed0a527..a92cc3371d21 100644 --- a/src/state/openclaw-state-db.ts +++ b/src/state/openclaw-state-db.ts @@ -30,6 +30,7 @@ import { configureSqlitePreSchemaPragmas, type SqliteWalMaintenance, } from "../infra/sqlite-wal.js"; +import { migrateLegacyCronRunLogsToTaskRuns } from "../infra/state-migrations.cron-run-logs.js"; import { createSubsystemLogger } from "../logging/subsystem.js"; import type { DB as OpenClawStateKyselyDatabase } from "./openclaw-state-db.generated.js"; import { @@ -1525,6 +1526,7 @@ function ensureAdditiveStateColumns(db: DatabaseSync): void { repairLegacyTaskDeliveryStatuses(db); ensureColumn(db, "task_runs", "tool_use_count INTEGER"); ensureColumn(db, "task_runs", "last_tool_name TEXT"); + ensureColumn(db, "task_runs", "detail_json TEXT"); ensureColumn(db, "subagent_runs", "task_name TEXT"); ensureColumn(db, "worker_environments", "bootstrap_bundle_hash TEXT"); ensureColumn(db, "worker_environments", "bootstrap_openclaw_version TEXT"); @@ -1553,6 +1555,7 @@ function ensureSchema(db: DatabaseSync, pathname: string): void { ensureAdditiveStateColumns(db); assertCanonicalStateSchemaShape(db, pathname); db.exec(OPENCLAW_STATE_SCHEMA_SQL); + migrateLegacyCronRunLogsToTaskRuns(db); repairCanonicalSqliteUniqueIndexes(db, pathname, OPENCLAW_STATE_CANONICAL_UNIQUE_INDEXES); // Retired node_pairing_* tables were created by earlier schema revisions but // never had a shipped writer (the node surface lives on device_pairing_paired diff --git a/src/state/openclaw-state-schema.generated.ts b/src/state/openclaw-state-schema.generated.ts index 8d70918e1f69..c36aac6e19a1 100644 --- a/src/state/openclaw-state-schema.generated.ts +++ b/src/state/openclaw-state-schema.generated.ts @@ -1312,7 +1312,8 @@ CREATE TABLE IF NOT EXISTS task_runs ( error TEXT, progress_summary TEXT, terminal_summary TEXT, - terminal_outcome TEXT + terminal_outcome TEXT, + detail_json TEXT ); CREATE INDEX IF NOT EXISTS idx_task_runs_run_id ON task_runs(run_id); @@ -1323,6 +1324,10 @@ CREATE INDEX IF NOT EXISTS idx_task_runs_last_event_at ON task_runs(last_event_a CREATE INDEX IF NOT EXISTS idx_task_runs_owner_key ON task_runs(owner_key); CREATE INDEX IF NOT EXISTS idx_task_runs_parent_flow_id ON task_runs(parent_flow_id); CREATE INDEX IF NOT EXISTS idx_task_runs_child_session_key ON task_runs(child_session_key); +CREATE INDEX IF NOT EXISTS idx_task_runs_runtime_source_ended + ON task_runs(runtime, source_id, ended_at, created_at, task_id); +CREATE INDEX IF NOT EXISTS idx_task_runs_runtime_ended + ON task_runs(runtime, ended_at, created_at, task_id); CREATE TABLE IF NOT EXISTS subagent_runs ( run_id TEXT NOT NULL PRIMARY KEY, diff --git a/src/state/openclaw-state-schema.sql b/src/state/openclaw-state-schema.sql index ca0fa73854cd..bf222fd64284 100644 --- a/src/state/openclaw-state-schema.sql +++ b/src/state/openclaw-state-schema.sql @@ -1307,7 +1307,8 @@ CREATE TABLE IF NOT EXISTS task_runs ( error TEXT, progress_summary TEXT, terminal_summary TEXT, - terminal_outcome TEXT + terminal_outcome TEXT, + detail_json TEXT ); CREATE INDEX IF NOT EXISTS idx_task_runs_run_id ON task_runs(run_id); @@ -1318,6 +1319,10 @@ CREATE INDEX IF NOT EXISTS idx_task_runs_last_event_at ON task_runs(last_event_a CREATE INDEX IF NOT EXISTS idx_task_runs_owner_key ON task_runs(owner_key); CREATE INDEX IF NOT EXISTS idx_task_runs_parent_flow_id ON task_runs(parent_flow_id); CREATE INDEX IF NOT EXISTS idx_task_runs_child_session_key ON task_runs(child_session_key); +CREATE INDEX IF NOT EXISTS idx_task_runs_runtime_source_ended + ON task_runs(runtime, source_id, ended_at, created_at, task_id); +CREATE INDEX IF NOT EXISTS idx_task_runs_runtime_ended + ON task_runs(runtime, ended_at, created_at, task_id); CREATE TABLE IF NOT EXISTS subagent_runs ( run_id TEXT NOT NULL PRIMARY KEY, diff --git a/src/tasks/detached-task-runtime-contract.ts b/src/tasks/detached-task-runtime-contract.ts index 045134f09306..18637feeb1cd 100644 --- a/src/tasks/detached-task-runtime-contract.ts +++ b/src/tasks/detached-task-runtime-contract.ts @@ -1,6 +1,7 @@ // Defines the detached task runtime contract and spawn options. import type { OpenClawConfig } from "../config/types.openclaw.js"; import type { + JsonValue, TaskDeliveryState, TaskDeliveryStatus, TaskNotifyPolicy, @@ -34,6 +35,7 @@ export type DetachedTaskCreateParams = { preferMetadata?: boolean; notifyPolicy?: TaskNotifyPolicy; deliveryStatus?: TaskDeliveryStatus; + detail?: JsonValue; }; export type DetachedRunningTaskCreateParams = DetachedTaskCreateParams & { @@ -65,11 +67,14 @@ type DetachedTaskCompleteParams = { runId: string; runtime?: TaskRuntime; sessionKey?: string; + childSessionKey?: string | null; endedAt: number; lastEventAt?: number; progressSummary?: string | null; terminalSummary?: string | null; + preserveTerminalSummary?: boolean; terminalOutcome?: TaskTerminalOutcome | null; + detail?: JsonValue; suppressDelivery?: boolean; }; @@ -77,12 +82,15 @@ type DetachedTaskFailParams = { runId: string; runtime?: TaskRuntime; sessionKey?: string; + childSessionKey?: string | null; status?: Extract; endedAt: number; lastEventAt?: number; error?: string; progressSummary?: string | null; terminalSummary?: string | null; + preserveTerminalSummary?: boolean; + detail?: JsonValue; suppressDelivery?: boolean; }; @@ -90,13 +98,17 @@ export type DetachedTaskFinalizeParams = { runId: string; runtime?: TaskRuntime; sessionKey?: string; + childSessionKey?: string | null; status: Extract; endedAt: number; lastEventAt?: number; error?: string; + clearError?: boolean; progressSummary?: string | null; terminalSummary?: string | null; + preserveTerminalSummary?: boolean; terminalOutcome?: TaskTerminalOutcome | null; + detail?: JsonValue; suppressDelivery?: boolean; }; diff --git a/src/tasks/task-executor.ts b/src/tasks/task-executor.ts index 19e5c3fcb526..11cca513bafc 100644 --- a/src/tasks/task-executor.ts +++ b/src/tasks/task-executor.ts @@ -36,6 +36,7 @@ import { } from "./task-flow-runtime-internal.js"; import { summarizeTaskRecords } from "./task-registry.summary.js"; import type { + JsonValue, TaskDeliveryState, TaskDeliveryStatus, TaskNotifyPolicy, @@ -175,11 +176,14 @@ export function completeTaskRunByRunId(params: { runId: string; runtime?: TaskRuntime; sessionKey?: string; + childSessionKey?: string | null; endedAt: number; lastEventAt?: number; progressSummary?: string | null; terminalSummary?: string | null; + preserveTerminalSummary?: boolean; terminalOutcome?: TaskTerminalOutcome | null; + detail?: JsonValue; suppressDelivery?: boolean; }) { return finalizeTaskRunByRunId({ @@ -196,12 +200,15 @@ export function failTaskRunByRunId(params: { runId: string; runtime?: TaskRuntime; sessionKey?: string; + childSessionKey?: string | null; status?: Extract; endedAt: number; lastEventAt?: number; error?: string; progressSummary?: string | null; terminalSummary?: string | null; + preserveTerminalSummary?: boolean; + detail?: JsonValue; suppressDelivery?: boolean; }) { return finalizeTaskRunByRunId({ diff --git a/src/tasks/task-flow-registry.types.ts b/src/tasks/task-flow-registry.types.ts index e92472e68d7f..b622fa4242ad 100644 --- a/src/tasks/task-flow-registry.types.ts +++ b/src/tasks/task-flow-registry.types.ts @@ -1,15 +1,8 @@ // Defines managed task-flow registry records and parser helpers. import type { DeliveryContext } from "../utils/delivery-context.types.js"; -import type { TaskNotifyPolicy } from "./task-registry.types.js"; +import type { JsonValue, TaskNotifyPolicy } from "./task-registry.types.js"; -/** JSON value shape persisted with task-flow state and wait metadata. */ -export type JsonValue = - | null - | boolean - | number - | string - | JsonValue[] - | { [key: string]: JsonValue }; +export type { JsonValue } from "./task-registry.types.js"; export type TaskFlowSyncMode = "task_mirrored" | "managed"; diff --git a/src/tasks/task-registry.maintenance.issue-60299.test.ts b/src/tasks/task-registry.maintenance.issue-60299.test.ts index 6b9f77eea4ef..be9b9ef7aaf0 100644 --- a/src/tasks/task-registry.maintenance.issue-60299.test.ts +++ b/src/tasks/task-registry.maintenance.issue-60299.test.ts @@ -2,8 +2,6 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import type { AcpSessionStoreEntry } from "../acp/runtime/session-meta.js"; import type { SessionEntry } from "../config/sessions.js"; -import type { CronRunLogEntry } from "../cron/run-log.js"; -import type { CronStoreFile } from "../cron/types.js"; import type { ParsedAgentSessionKey } from "../routing/session-key.js"; import { resetDetachedTaskLifecycleRuntimeForTests, @@ -63,8 +61,7 @@ function createTaskRegistryMaintenanceHarness(params: { activeCronJobIds?: string[]; activeRunIds?: string[]; activeAcpSessionKeys?: string[]; - cronStore?: CronStoreFile; - cronRunLogEntries?: Record; + durableCronTaskRows?: Record; runtimeAuthoritative?: boolean; }) { const sessionStore = params.sessionStore ?? {}; @@ -72,7 +69,7 @@ function createTaskRegistryMaintenanceHarness(params: { const activeCronJobIds = new Set(params.activeCronJobIds ?? []); const activeRunIds = new Set(params.activeRunIds ?? []); const activeAcpSessionKeys = new Set(params.activeAcpSessionKeys ?? []); - const cronRunLogEntries = params.cronRunLogEntries ?? {}; + const durableCronTaskRows = params.durableCronTaskRows ?? {}; const currentTasks = new Map(params.tasks.map((task) => [task.taskId, { ...task }])); const runtime: TaskRegistryMaintenanceRuntime = { @@ -159,8 +156,13 @@ function createTaskRegistryMaintenanceHarness(params: { endedAt: patch.endedAt, lastEventAt: patch.lastEventAt ?? patch.endedAt, ...(patch.terminalSummary !== undefined - ? { terminalSummary: patch.terminalSummary ?? undefined } + ? { + terminalSummary: patch.preserveTerminalSummary + ? (patch.terminalSummary ?? undefined) + : patch.terminalSummary?.replace(/\s+/g, " ").trim() || undefined, + } : {}), + ...(patch.detail !== undefined ? { detail: patch.detail } : {}), } satisfies TaskRecord; if (Object.hasOwn(patch, "error")) { if (patch.error === undefined) { @@ -184,9 +186,8 @@ function createTaskRegistryMaintenanceHarness(params: { return next; }, isRuntimeAuthoritative: () => params.runtimeAuthoritative ?? true, - resolveCronJobsStorePath: () => "/tmp/openclaw-test-cron/jobs.json", - loadCronJobsStoreSync: () => params.cronStore ?? { version: 1, jobs: [] }, - readCronRunLogEntriesSync: ({ jobId }) => (jobId ? (cronRunLogEntries[jobId] ?? []) : []), + listTaskRegistryRecordsByRuntimeSourceIdFromSqlite: ({ sourceId }) => + sourceId ? (durableCronTaskRows[sourceId] ?? []) : Object.values(durableCronTaskRows).flat(), }; setTaskRegistryMaintenanceRuntimeForTests(runtime); @@ -287,6 +288,16 @@ describe("task-registry maintenance issue #60299", () => { const { currentTasks } = createTaskRegistryMaintenanceHarness({ tasks: [task], activeCronJobIds: ["cron-job-2"], + durableCronTaskRows: { + "cron-job-2": [ + { + ...task, + status: "succeeded", + endedAt: Date.now(), + detail: { kind: "cron-run", status: "ok" }, + }, + ], + }, }); expectMaintenanceCounts(await runTaskRegistryMaintenance(), { reconciled: 0 }); @@ -473,7 +484,7 @@ describe("task-registry maintenance issue #60299", () => { expectTaskStatus(currentTasks, task.taskId, "running"); }); - it("recovers finished cron tasks from durable run logs before marking them lost", async () => { + it("recovers finished cron tasks from durable ledger detail before marking them lost", async () => { const startedAt = Date.now() - 60 * 60_000; const task = makeStaleTask({ runtime: "cron", @@ -485,16 +496,15 @@ describe("task-registry maintenance issue #60299", () => { const { currentTasks } = createTaskRegistryMaintenanceHarness({ tasks: [task], - cronRunLogEntries: { + durableCronTaskRows: { "cron-job-run-log-ok": [ { - ts: startedAt + 1250, - jobId: "cron-job-run-log-ok", - action: "finished", - status: "ok", - summary: "done", - runAtMs: startedAt, - durationMs: 1250, + ...task, + status: "succeeded", + endedAt: startedAt + 1250, + lastEventAt: startedAt + 1250, + terminalSummary: "done", + detail: { kind: "cron-run", status: "ok", durationMs: 1250 }, }, ], }, @@ -513,9 +523,47 @@ describe("task-registry maintenance issue #60299", () => { expect(storedTask.status).toBe("succeeded"); expect(storedTask.endedAt).toBe(startedAt + 1250); expect(storedTask.terminalSummary).toBe("done"); + expect(storedTask.detail).toEqual({ kind: "cron-run", status: "ok", durationMs: 1250 }); }); - it("does not recover cron tasks from malformed run id timestamps", async () => { + it("recovers cancelled cron tasks with exact durable summaries", async () => { + const startedAt = Date.now() - 60 * 60_000; + const task = makeStaleTask({ + runtime: "cron", + sourceId: "cron-job-cancelled", + runId: `cron:cron-job-cancelled:${startedAt}`, + startedAt, + lastEventAt: startedAt, + }); + const terminalSummary = "cancelled\n summary"; + + const { currentTasks } = createTaskRegistryMaintenanceHarness({ + tasks: [task], + durableCronTaskRows: { + "cron-job-cancelled": [ + { + ...task, + status: "cancelled", + error: "cancelled by operator", + endedAt: startedAt + 500, + lastEventAt: startedAt + 500, + terminalSummary, + detail: { kind: "cron-run", status: "error", durationMs: 500 }, + }, + ], + }, + }); + + expectMaintenanceCounts(await runTaskRegistryMaintenance(), { reconciled: 0, recovered: 1 }); + expect(currentTasks.get(task.taskId)).toMatchObject({ + status: "cancelled", + endedAt: startedAt + 500, + terminalSummary, + detail: { kind: "cron-run", status: "error", durationMs: 500 }, + }); + }); + + it("does not recover cron tasks from an unrelated ledger row", async () => { const task = makeStaleTask({ runtime: "cron", sourceId: "cron-job-run-log-ok", @@ -524,16 +572,17 @@ describe("task-registry maintenance issue #60299", () => { const { currentTasks } = createTaskRegistryMaintenanceHarness({ tasks: [task], - cronRunLogEntries: { + durableCronTaskRows: { "cron-job-run-log-ok": [ { - ts: 1250, - jobId: "cron-job-run-log-ok", - action: "finished", - status: "ok", - summary: "done", - runAtMs: 1000, - durationMs: 250, + ...task, + taskId: "different-task", + runId: "cron:cron-job-run-log-ok:1000", + status: "succeeded", + endedAt: 1250, + lastEventAt: 1250, + terminalSummary: "done", + detail: { kind: "cron-run", status: "ok" }, }, ], }, @@ -544,7 +593,7 @@ describe("task-registry maintenance issue #60299", () => { expectTaskStatus(currentTasks, task.taskId, "lost"); }); - it("recovers terminal lost cron tasks from durable run logs", async () => { + it("recovers terminal lost cron tasks from the durable ledger", async () => { const startedAt = Date.now() - GRACE_EXPIRED_MS; const task = makeStaleTask({ runtime: "cron", @@ -560,16 +609,16 @@ describe("task-registry maintenance issue #60299", () => { const { currentTasks } = createTaskRegistryMaintenanceHarness({ tasks: [task], - cronRunLogEntries: { + durableCronTaskRows: { "cron-job-terminal-lost-ok": [ { - ts: startedAt + 1250, - jobId: "cron-job-terminal-lost-ok", - action: "finished", - status: "ok", - summary: "done", - runAtMs: startedAt, - durationMs: 1250, + ...task, + status: "succeeded", + error: undefined, + endedAt: startedAt + 1250, + lastEventAt: startedAt + 1250, + terminalSummary: "done", + detail: { kind: "cron-run", status: "ok" }, }, ], }, @@ -611,16 +660,15 @@ describe("task-registry maintenance issue #60299", () => { const { currentTasks } = createTaskRegistryMaintenanceHarness({ tasks: [task], - cronRunLogEntries: { + durableCronTaskRows: { "cron-job-terminal-lost-no-error": [ { - ts: startedAt + 1250, - jobId: "cron-job-terminal-lost-no-error", - action: "finished", - status: "ok", - summary: "done", - runAtMs: startedAt, - durationMs: 1250, + ...task, + status: "succeeded", + endedAt: startedAt + 1250, + lastEventAt: startedAt + 1250, + terminalSummary: "done", + detail: { kind: "cron-run", status: "ok" }, }, ], }, @@ -649,16 +697,16 @@ describe("task-registry maintenance issue #60299", () => { const { currentTasks } = createTaskRegistryMaintenanceHarness({ tasks: [task], - cronRunLogEntries: { + durableCronTaskRows: { "cron-job-terminal-lost-other-error": [ { - ts: startedAt + 1250, - jobId: "cron-job-terminal-lost-other-error", - action: "finished", - status: "ok", - summary: "done", - runAtMs: startedAt, - durationMs: 1250, + ...task, + status: "succeeded", + error: undefined, + endedAt: startedAt + 1250, + lastEventAt: startedAt + 1250, + terminalSummary: "done", + detail: { kind: "cron-run", status: "ok" }, }, ], }, @@ -672,7 +720,7 @@ describe("task-registry maintenance issue #60299", () => { }); }); - it("recovers interrupted cron tasks from durable cron job state when run logs are absent", async () => { + it("does not recover cron tasks from cron job state without a terminal ledger row", async () => { const startedAt = Date.now() - GRACE_EXPIRED_MS; const task = makeStaleTask({ runtime: "cron", @@ -682,38 +730,11 @@ describe("task-registry maintenance issue #60299", () => { lastEventAt: startedAt, }); - const { currentTasks } = createTaskRegistryMaintenanceHarness({ - tasks: [task], - cronStore: { - version: 1, - jobs: [ - { - id: "cron-job-state-error", - name: "state error", - enabled: true, - createdAtMs: startedAt - 60_000, - updatedAtMs: startedAt, - schedule: { kind: "every", everyMs: 60_000, anchorMs: startedAt - 60_000 }, - sessionTarget: "isolated", - wakeMode: "next-heartbeat", - payload: { kind: "agentTurn", message: "work" }, - state: { - lastRunAtMs: startedAt, - lastRunStatus: "error", - lastError: "cron: job interrupted by gateway restart", - lastDurationMs: 5000, - }, - }, - ], - }, - }); + const { currentTasks } = createTaskRegistryMaintenanceHarness({ tasks: [task] }); - expectMaintenanceCounts(previewTaskRegistryMaintenance(), { reconciled: 0, recovered: 1 }); - expectMaintenanceCounts(await runTaskRegistryMaintenance(), { reconciled: 0, recovered: 1 }); - const storedTask = requireTaskRecord(currentTasks, task.taskId); - expect(storedTask.status).toBe("failed"); - expect(storedTask.endedAt).toBe(startedAt + 5000); - expect(storedTask.error).toBe("cron: job interrupted by gateway restart"); + expectMaintenanceCounts(previewTaskRegistryMaintenance(), { reconciled: 1, recovered: 0 }); + expectMaintenanceCounts(await runTaskRegistryMaintenance(), { reconciled: 1, recovered: 0 }); + expectTaskStatus(currentTasks, task.taskId, "lost"); }); it("marks chat-backed cli tasks lost after the owning run context disappears", async () => { diff --git a/src/tasks/task-registry.maintenance.ts b/src/tasks/task-registry.maintenance.ts index ea5ffb7c77d4..2cb54aec4b95 100644 --- a/src/tasks/task-registry.maintenance.ts +++ b/src/tasks/task-registry.maintenance.ts @@ -19,13 +19,8 @@ import { } from "../config/sessions/session-accessor.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import { isCronJobActive } from "../cron/active-jobs.js"; -import { readCronRunLogEntriesSync } from "../cron/run-log.js"; -import type { CronRunLogEntry } from "../cron/run-log.js"; -import { loadCronJobsStoreSync, resolveCronJobsStorePath } from "../cron/store.js"; -import type { CronJob, CronStoreFile } from "../cron/types.js"; import { getAgentRunContext } from "../infra/agent-events.js"; import { getSessionBindingService } from "../infra/outbound/session-binding-service.js"; -import { parseStrictNonNegativeInteger } from "../infra/parse-finite-number.js"; import { createSubsystemLogger } from "../logging/subsystem.js"; import { isPluginStateDatabaseOpen, @@ -64,6 +59,7 @@ import { summarizeTaskAuditFindings, } from "./task-registry.audit.js"; import type { TaskAuditFinding, TaskAuditSummary } from "./task-registry.audit.js"; +import { listTaskRegistryRecordsByRuntimeSourceIdFromSqlite } from "./task-registry.store.sqlite.js"; import { summarizeTaskRecords } from "./task-registry.summary.js"; import type { TaskRecord, TaskRegistrySummary, TaskStatus } from "./task-registry.types.js"; import type { ActiveTaskRestartBlocker } from "./task-restart-blocker.js"; @@ -88,7 +84,6 @@ const SWEEP_YIELD_BATCH_SIZE = 25; let sweeper: NodeJS.Timeout | null = null; let deferredSweep: NodeJS.Timeout | null = null; let sweepInProgress = false; -let configuredCronStorePath: string | undefined; let configuredRuntimeAuthoritative = false; type TaskRegistryMaintenanceRuntime = { @@ -119,9 +114,7 @@ type TaskRegistryMaintenanceRuntime = { resolveTaskForLookupToken: typeof resolveTaskForLookupToken; setTaskCleanupAfterById: typeof setTaskCleanupAfterById; isRuntimeAuthoritative: () => boolean; - resolveCronJobsStorePath: typeof resolveCronJobsStorePath; - loadCronJobsStoreSync: typeof loadCronJobsStoreSync; - readCronRunLogEntriesSync: typeof readCronRunLogEntriesSync; + listTaskRegistryRecordsByRuntimeSourceIdFromSqlite: typeof listTaskRegistryRecordsByRuntimeSourceIdFromSqlite; }; const defaultTaskRegistryMaintenanceRuntime: TaskRegistryMaintenanceRuntime = { @@ -159,9 +152,7 @@ const defaultTaskRegistryMaintenanceRuntime: TaskRegistryMaintenanceRuntime = { resolveTaskForLookupToken, setTaskCleanupAfterById, isRuntimeAuthoritative: () => configuredRuntimeAuthoritative, - resolveCronJobsStorePath: () => configuredCronStorePath ?? resolveCronJobsStorePath(), - loadCronJobsStoreSync, - readCronRunLogEntriesSync, + listTaskRegistryRecordsByRuntimeSourceIdFromSqlite, }; let taskRegistryMaintenanceRuntime: TaskRegistryMaintenanceRuntime = @@ -197,23 +188,17 @@ export type TaskRegistryMaintenanceDiagnostics = { staleRunningTasks: TaskRegistryMaintenanceTaskDiagnostic[]; }; -type CronExecutionId = { - jobId: string; - startedAt: number; -}; - type CronTerminalRecovery = { - status: Extract; + status: Extract; endedAt: number; lastEventAt: number; error?: string; terminalSummary?: string; + detail?: TaskRecord["detail"]; }; type CronRecoveryContext = { - storePath: string; - store?: CronStoreFile | null; - runLogsByJobId: Map; + taskRowsByJobId: Map; }; type SessionEntryLookup = { @@ -227,8 +212,7 @@ type BackingSessionLookupContext = { function createCronRecoveryContext(): CronRecoveryContext { return { - storePath: taskRegistryMaintenanceRuntime.resolveCronJobsStorePath(), - runLogsByJobId: new Map(), + taskRowsByJobId: new Map(), }; } @@ -319,30 +303,6 @@ function hasLostGraceExpired(task: TaskRecord, now: number): boolean { return now - referenceAt >= graceMs; } -function parseCronExecutionId(task: TaskRecord): CronExecutionId | undefined { - const runId = task.runId?.trim(); - if (!runId?.startsWith("cron:")) { - return undefined; - } - const separator = runId.lastIndexOf(":"); - if (separator <= "cron:".length) { - return undefined; - } - const startedAt = parseStrictNonNegativeInteger(runId.slice(separator + 1)); - if (startedAt === undefined) { - return undefined; - } - const jobId = runId.slice("cron:".length, separator).trim(); - if (!jobId || (task.sourceId?.trim() && task.sourceId.trim() !== jobId)) { - return undefined; - } - return { jobId, startedAt }; -} - -function isTimeoutCronError(error: string | undefined): boolean { - return error === "cron: job execution timed out"; -} - function isRecoverableLostCronTask(task: TaskRecord): boolean { if (task.status !== "lost") { return false; @@ -351,97 +311,31 @@ function isRecoverableLostCronTask(task: TaskRecord): boolean { return Boolean(error?.includes("backing session missing")); } -function mapCronTerminalStatus(status: unknown, error?: string): CronTerminalRecovery["status"] { - if (status === "ok" || status === "skipped") { - return "succeeded"; - } - return isTimeoutCronError(error) ? "timed_out" : "failed"; +function isCronTerminalTaskStatus(status: TaskStatus): status is CronTerminalRecovery["status"] { + return ( + status === "succeeded" || + status === "failed" || + status === "timed_out" || + status === "cancelled" + ); } -function getCronRunLogEntries(context: CronRecoveryContext, jobId: string): CronRunLogEntry[] { - const cached = context.runLogsByJobId.get(jobId); +function getCronTaskRows(context: CronRecoveryContext, jobId: string): TaskRecord[] { + const cached = context.taskRowsByJobId.get(jobId); if (cached) { return cached; } - let entries: CronRunLogEntry[]; + let rows: TaskRecord[]; try { - entries = taskRegistryMaintenanceRuntime.readCronRunLogEntriesSync({ - storePath: context.storePath, - jobId, - limit: 5000, + rows = taskRegistryMaintenanceRuntime.listTaskRegistryRecordsByRuntimeSourceIdFromSqlite({ + runtime: "cron", + sourceId: jobId, }); } catch { - entries = []; + rows = []; } - context.runLogsByJobId.set(jobId, entries); - return entries; -} - -function getCronStore(context: CronRecoveryContext): CronStoreFile | null { - if (context.store !== undefined) { - return context.store; - } - try { - context.store = taskRegistryMaintenanceRuntime.loadCronJobsStoreSync(context.storePath); - } catch { - context.store = null; - } - return context.store; -} - -function resolveCronRunLogRecovery( - execution: CronExecutionId, - context: CronRecoveryContext, -): CronTerminalRecovery | undefined { - const entries = getCronRunLogEntries(context, execution.jobId); - const entry = entries.findLast( - (candidate) => - candidate.jobId === execution.jobId && - candidate.action === "finished" && - candidate.runAtMs === execution.startedAt && - (candidate.status === "ok" || candidate.status === "skipped" || candidate.status === "error"), - ); - if (!entry) { - return undefined; - } - const durationMs = - typeof entry.durationMs === "number" && Number.isFinite(entry.durationMs) - ? Math.max(0, entry.durationMs) - : undefined; - const endedAt = durationMs === undefined ? entry.ts : execution.startedAt + durationMs; - return { - status: mapCronTerminalStatus(entry.status, entry.error), - endedAt, - lastEventAt: endedAt, - ...(entry.error !== undefined ? { error: entry.error } : {}), - ...(entry.summary !== undefined ? { terminalSummary: entry.summary } : {}), - }; -} - -function resolveCronJobStateRecovery( - execution: CronExecutionId, - context: CronRecoveryContext, -): CronTerminalRecovery | undefined { - const store = getCronStore(context); - const job: CronJob | undefined = store?.jobs.find((entry) => entry.id === execution.jobId); - if (!job || job.state.lastRunAtMs !== execution.startedAt) { - return undefined; - } - const status = job.state.lastRunStatus ?? job.state.lastStatus; - if (status !== "ok" && status !== "skipped" && status !== "error") { - return undefined; - } - const durationMs = - typeof job.state.lastDurationMs === "number" && Number.isFinite(job.state.lastDurationMs) - ? Math.max(0, job.state.lastDurationMs) - : 0; - const endedAt = execution.startedAt + durationMs; - return { - status: mapCronTerminalStatus(status, job.state.lastError), - endedAt, - lastEventAt: endedAt, - ...(job.state.lastError !== undefined ? { error: job.state.lastError } : {}), - }; + context.taskRowsByJobId.set(jobId, rows); + return rows; } function resolveDurableCronTaskRecovery( @@ -451,13 +345,33 @@ function resolveDurableCronTaskRecovery( if (task.runtime !== "cron" || (!isActiveTask(task) && !isRecoverableLostCronTask(task))) { return undefined; } - const execution = parseCronExecutionId(task); - if (!execution) { + const jobId = task.sourceId?.trim(); + if (!jobId) { return undefined; } - return ( - resolveCronRunLogRecovery(execution, context) ?? resolveCronJobStateRecovery(execution, context) + if ( + taskRegistryMaintenanceRuntime.isRuntimeAuthoritative() && + taskRegistryMaintenanceRuntime.isCronJobActive(jobId) + ) { + return undefined; + } + const row = getCronTaskRows(context, jobId).find( + (candidate) => + candidate.taskId === task.taskId || + (Boolean(task.runId?.trim()) && candidate.runId === task.runId), ); + if (!row || !isCronTerminalTaskStatus(row.status)) { + return undefined; + } + const endedAt = row.endedAt ?? row.lastEventAt ?? row.createdAt; + return { + status: row.status, + endedAt, + lastEventAt: row.lastEventAt ?? endedAt, + ...(row.error !== undefined ? { error: row.error } : {}), + ...(row.terminalSummary !== undefined ? { terminalSummary: row.terminalSummary } : {}), + ...(row.detail !== undefined ? { detail: row.detail } : {}), + }; } function hasActiveCliRun(task: TaskRecord): boolean { @@ -804,8 +718,9 @@ function markTaskRecovered(task: TaskRecord, recovery: CronTerminalRecovery): Ta lastEventAt: recovery.lastEventAt, error: recovery.error, ...(recovery.terminalSummary !== undefined - ? { terminalSummary: recovery.terminalSummary } + ? { terminalSummary: recovery.terminalSummary, preserveTerminalSummary: true } : {}), + ...(recovery.detail !== undefined ? { detail: recovery.detail } : {}), }) ?? projectTaskRecovered(task, recovery); void taskRegistryMaintenanceRuntime.maybeDeliverTaskTerminalUpdate(updated.taskId); return updated; @@ -821,6 +736,7 @@ function projectTaskRecovered(task: TaskRecord, recovery: CronTerminalRecovery): ...(recovery.terminalSummary !== undefined ? { terminalSummary: recovery.terminalSummary } : {}), + ...(recovery.detail !== undefined ? { detail: recovery.detail } : {}), }; if (recovery.error === undefined) { delete projected.error; @@ -1232,16 +1148,13 @@ export function setTaskRegistryMaintenanceRuntimeForTests( export function resetTaskRegistryMaintenanceRuntimeForTests(): void { taskRegistryMaintenanceRuntime = defaultTaskRegistryMaintenanceRuntime; - configuredCronStorePath = undefined; configuredRuntimeAuthoritative = false; } -export function configureTaskRegistryMaintenance(options: { - cronStorePath?: string; +export function configureTaskRegistryMaintenance(options?: { runtimeAuthoritative?: boolean; }): void { - configuredCronStorePath = options.cronStorePath?.trim() || undefined; - if (options.runtimeAuthoritative !== undefined) { + if (options?.runtimeAuthoritative !== undefined) { configuredRuntimeAuthoritative = options.runtimeAuthoritative; } } diff --git a/src/tasks/task-registry.store.sqlite.ts b/src/tasks/task-registry.store.sqlite.ts index d3c0c110255b..460e4b6ea666 100644 --- a/src/tasks/task-registry.store.sqlite.ts +++ b/src/tasks/task-registry.store.sqlite.ts @@ -21,7 +21,9 @@ import { parseTaskScopeKind, parseTaskStatus, type TaskDeliveryState, + type JsonValue, type TaskRecord, + type TaskRuntime, } from "./task-registry.types.js"; type TaskRunsTable = OpenClawStateKyselyDatabase["task_runs"]; @@ -78,12 +80,24 @@ const TASK_RUN_SELECT_COLUMNS = [ "progress_summary", "terminal_summary", "terminal_outcome", + "detail_json", ] as const; let cachedDatabase: TaskRegistryDatabase | null = null; function serializeJson(value: unknown): string | null { - return value == null ? null : JSON.stringify(value); + return value === undefined ? null : (JSON.stringify(value) ?? null); +} + +function parseJsonValue(raw: string | null): JsonValue | undefined { + if (!raw?.trim()) { + return undefined; + } + try { + return JSON.parse(raw) as JsonValue; + } catch { + return undefined; + } } function rowToTaskRecord(row: TaskRegistryRow): TaskRecord { @@ -94,6 +108,7 @@ function rowToTaskRecord(row: TaskRegistryRow): TaskRecord { const toolUseCount = normalizeSqliteNumber(row.tool_use_count); const scopeKind = parseTaskScopeKind(row.scope_kind); const terminalOutcome = parseOptionalTaskTerminalOutcome(row.terminal_outcome); + const detail = parseJsonValue(row.detail_json); // System tasks intentionally have no requester session; ownerKey is the lookup anchor. const requesterSessionKey = scopeKind === "system" ? "" : row.requester_session_key?.trim() || row.owner_key; @@ -125,8 +140,9 @@ function rowToTaskRecord(row: TaskRegistryRow): TaskRecord { ...(row.last_tool_name ? { lastToolName: row.last_tool_name } : {}), ...(row.error ? { error: row.error } : {}), ...(row.progress_summary ? { progressSummary: row.progress_summary } : {}), - ...(row.terminal_summary ? { terminalSummary: row.terminal_summary } : {}), + ...(row.terminal_summary !== null ? { terminalSummary: row.terminal_summary } : {}), ...(terminalOutcome ? { terminalOutcome } : {}), + ...(detail !== undefined ? { detail } : {}), }; } @@ -171,6 +187,7 @@ function bindTaskRecordBase(record: TaskRecord): Insertable { progress_summary: record.progressSummary ?? null, terminal_summary: record.terminalSummary ?? null, terminal_outcome: record.terminalOutcome ?? null, + detail_json: serializeJson(record.detail), }; } @@ -232,6 +249,22 @@ function selectTaskRowsByOwnerKey(db: DatabaseSync, ownerKey: string): TaskRegis .all(ownerKey) as TaskRegistryRow[]; } +function selectTaskRowsByRuntimeSourceId( + db: DatabaseSync, + runtime: TaskRuntime, + sourceId?: string, +): TaskRegistryRow[] { + let query = getTaskRegistryKysely(db) + .selectFrom("task_runs") + .select(TASK_RUN_SELECT_COLUMNS) + .where("runtime", "=", runtime); + if (sourceId !== undefined) { + query = query.where("source_id", "=", sourceId); + } + return executeSqliteQuerySync(db, query.orderBy("created_at", "asc").orderBy("task_id", "asc")) + .rows; +} + function selectTaskDeliveryStateRows(db: DatabaseSync): TaskDeliveryStateRow[] { const query = getTaskRegistryKysely(db) .selectFrom("task_delivery_state") @@ -276,6 +309,7 @@ function upsertTaskRow(db: DatabaseSync, row: Insertable): void { progress_summary: (eb) => eb.ref("excluded.progress_summary"), terminal_summary: (eb) => eb.ref("excluded.terminal_summary"), terminal_outcome: (eb) => eb.ref("excluded.terminal_outcome"), + detail_json: (eb) => eb.ref("excluded.detail_json"), }), ), ); @@ -356,6 +390,19 @@ export function listTaskRegistryRecordsByOwnerKeyFromSqlite(ownerKey: string): T return selectTaskRowsByOwnerKey(db, key).map(rowToTaskRecord); } +/** Reads task rows for one runtime/source without restoring the process registry snapshot. */ +export function listTaskRegistryRecordsByRuntimeSourceIdFromSqlite(params: { + runtime: TaskRuntime; + sourceId?: string; +}): TaskRecord[] { + const sourceId = params.sourceId?.trim(); + if (params.sourceId !== undefined && !sourceId) { + return []; + } + const { db } = openTaskRegistryDatabase(); + return selectTaskRowsByRuntimeSourceId(db, params.runtime, sourceId).map(rowToTaskRecord); +} + export function saveTaskRegistryStateToSqlite(snapshot: TaskRegistryStoreSnapshot) { withWriteTransaction(({ db }) => { const kysely = getTaskRegistryKysely(db); diff --git a/src/tasks/task-registry.store.test.ts b/src/tasks/task-registry.store.test.ts index 09004db64183..3d7714dd1e03 100644 --- a/src/tasks/task-registry.store.test.ts +++ b/src/tasks/task-registry.store.test.ts @@ -420,6 +420,51 @@ describe("task-registry store runtime", () => { ); }); + it("round-trips runtime-owned task detail through sqlite", async () => { + await withOpenClawTestState( + { layout: "state-only", prefix: "openclaw-task-store-detail-" }, + async () => { + const task: TaskRecord = { + ...createStoredTask(), + runtime: "cron", + sourceId: "cron-detail-job", + detail: { + kind: "cron-run", + status: "ok", + usage: { input_tokens: 3, cached: false }, + }, + }; + saveTaskRegistryStateToSqlite({ + tasks: new Map([[task.taskId, task]]), + deliveryStates: new Map(), + }); + + expect(loadTaskRegistryStateFromSqlite().tasks.get(task.taskId)?.detail).toEqual( + task.detail, + ); + }, + ); + }); + + it("preserves explicit null task detail through sqlite", async () => { + await withOpenClawTestState( + { layout: "state-only", prefix: "openclaw-task-store-null-detail-" }, + async () => { + const task: TaskRecord = { + ...createStoredTask(), + detail: null, + }; + saveTaskRegistryStateToSqlite({ + tasks: new Map([[task.taskId, task]]), + deliveryStates: new Map(), + }); + + const restored = loadTaskRegistryStateFromSqlite().tasks.get(task.taskId); + expect(restored).toHaveProperty("detail", null); + }, + ); + }); + it("loads task and delivery rows from one sqlite read snapshot", async () => { await withOpenClawTestState( { layout: "state-only", prefix: "openclaw-task-store-read-snapshot-" }, diff --git a/src/tasks/task-registry.test.ts b/src/tasks/task-registry.test.ts index 0bd749f73ebd..207d8a64fd8a 100644 --- a/src/tasks/task-registry.test.ts +++ b/src/tasks/task-registry.test.ts @@ -239,9 +239,7 @@ function configureTaskRegistryMaintenanceRuntimeForTest(params: { return next; }, isRuntimeAuthoritative: () => true, - resolveCronJobsStorePath: () => "/tmp/openclaw-test-cron/jobs.json", - loadCronJobsStoreSync: () => ({ version: 1, jobs: [] }), - readCronRunLogEntriesSync: () => [], + listTaskRegistryRecordsByRuntimeSourceIdFromSqlite: () => [], }); } @@ -624,6 +622,35 @@ describe("task-registry", () => { }); }); + it("clears a provisional child session when the terminal outcome has none", async () => { + await withTaskRegistryTempDir(async () => { + resetTaskRegistryMemoryForTest(); + createTaskRecord({ + runtime: "cron", + ownerKey: "", + scopeKind: "system", + childSessionKey: "agent:main:cron:provisional", + runId: "cron:provisional:100", + task: "Provisional cron run", + status: "running", + deliveryStatus: "not_applicable", + startedAt: 100, + }); + + finalizeTaskRunByRunId({ + runId: "cron:provisional:100", + runtime: "cron", + childSessionKey: null, + status: "failed", + endedAt: 200, + error: "setup failed", + }); + reloadTaskRegistryFromStore(); + + expect(requireTaskByRunId("cron:provisional:100").childSessionKey).toBeUndefined(); + }); + }); + it("reuses an ACP run task when a derived flow id is linked before a duplicate create", async () => { await withTaskRegistryTempDir(async () => { resetTaskRegistryMemoryForTest({ persist: false }); @@ -3546,9 +3573,7 @@ describe("task-registry", () => { resolveTaskForLookupToken: () => undefined, setTaskCleanupAfterById: () => null, isRuntimeAuthoritative: () => true, - resolveCronJobsStorePath: () => "/tmp/openclaw-test-cron/jobs.json", - loadCronJobsStoreSync: () => ({ version: 1, jobs: [] }), - readCronRunLogEntriesSync: () => [], + listTaskRegistryRecordsByRuntimeSourceIdFromSqlite: () => [], }); try { diff --git a/src/tasks/task-registry.ts b/src/tasks/task-registry.ts index ec6a6e1e0706..c7c3ab12f909 100644 --- a/src/tasks/task-registry.ts +++ b/src/tasks/task-registry.ts @@ -55,6 +55,7 @@ import { type TaskRegistryObserverEvent, } from "./task-registry.store.js"; import type { + JsonValue, TaskDeliveryState, TaskDeliveryStatus, TaskEventKind, @@ -240,7 +241,10 @@ function ensureTaskCancellationReady(task: TaskRecord): void { } function cloneTaskRecord(record: TaskRecord): TaskRecord { - return { ...record }; + return { + ...record, + ...(record.detail !== undefined ? { detail: structuredClone(record.detail) } : {}), + }; } function normalizeTaskTimestamps(task: TaskRecord): TaskRecord { @@ -961,6 +965,7 @@ function mergeExistingTaskForCreate( preferMetadata?: boolean; deliveryStatus?: TaskDeliveryStatus; notifyPolicy?: TaskNotifyPolicy; + detail?: JsonValue; }, ): TaskRecord | null { ensureLinkedTaskFlowRegistryReady(existing); @@ -1024,6 +1029,9 @@ function mergeExistingTaskForCreate( if (notifyPolicy !== existing.notifyPolicy && existing.notifyPolicy === "silent") { patch.notifyPolicy = notifyPolicy; } + if (params.detail !== undefined) { + patch.detail = params.detail; + } if (Object.keys(patch).length === 0) { return cloneTaskRecord(existing); } @@ -1306,10 +1314,17 @@ function updateTask(taskId: string, patch: Partial): TaskRecord | nu if (!current) { return null; } - const next = normalizeTaskTimestamps({ ...current, ...patch }); + const next = normalizeTaskTimestamps({ + ...current, + ...patch, + ...(patch.detail !== undefined ? { detail: structuredClone(patch.detail) } : {}), + }); if (Object.hasOwn(patch, "error") && patch.error === undefined) { delete next.error; } + if (Object.hasOwn(patch, "childSessionKey") && patch.childSessionKey === undefined) { + delete next.childSessionKey; + } if (isTerminalTaskStatus(next.status) && typeof next.cleanupAfter !== "number") { next.cleanupAfter = resolveTaskCleanupAfter({ ...next, @@ -1747,19 +1762,29 @@ export function setTaskCleanupAfterById(params: { export function markTaskTerminalById(params: { taskId: string; status: Extract; + childSessionKey?: string | null; endedAt: number; lastEventAt?: number; error?: string; terminalSummary?: string | null; + preserveTerminalSummary?: boolean; terminalOutcome?: TaskTerminalOutcome | null; + detail?: JsonValue; }): TaskRecord | null { ensureTaskRegistryReady(); const patch: Partial = { status: params.status, + ...(params.childSessionKey !== undefined + ? { childSessionKey: params.childSessionKey?.trim() || undefined } + : {}), endedAt: params.endedAt, lastEventAt: params.lastEventAt ?? params.endedAt, ...(params.terminalSummary !== undefined - ? { terminalSummary: normalizeTaskSummary(params.terminalSummary) } + ? { + terminalSummary: params.preserveTerminalSummary + ? (params.terminalSummary ?? undefined) + : normalizeTaskSummary(params.terminalSummary), + } : {}), ...(params.terminalOutcome !== undefined ? { @@ -1769,6 +1794,7 @@ export function markTaskTerminalById(params: { }), } : {}), + ...(params.detail !== undefined ? { detail: structuredClone(params.detail) } : {}), }; if (Object.hasOwn(params, "error")) { patch.error = params.error; @@ -1937,6 +1963,7 @@ export function createTaskRecord(params: { progressSummary?: string | null; terminalSummary?: string | null; terminalOutcome?: TaskTerminalOutcome | null; + detail?: JsonValue; }): TaskRecord | null { ensureTaskRegistryReady(); const requesterSessionKey = resolveTaskRequesterSessionKey(params); @@ -2026,6 +2053,7 @@ export function createTaskRecord(params: { status, terminalOutcome: params.terminalOutcome, }), + ...(params.detail !== undefined ? { detail: structuredClone(params.detail) } : {}), }); if (isTerminalTaskStatus(record.status) && typeof record.cleanupAfter !== "number") { record.cleanupAfter = resolveTaskCleanupAfter(record); @@ -2063,14 +2091,18 @@ function updateTaskStateByRunId(params: { runId: string; runtime?: TaskRuntime; sessionKey?: string; + childSessionKey?: string | null; status?: TaskStatus; startedAt?: number; endedAt?: number; lastEventAt?: number; error?: string; + clearError?: boolean; progressSummary?: string | null; terminalSummary?: string | null; + preserveTerminalSummary?: boolean; terminalOutcome?: TaskTerminalOutcome | null; + detail?: JsonValue; eventSummary?: string | null; suppressDelivery?: boolean; }) { @@ -2111,7 +2143,12 @@ function updateTaskStateByRunId(params: { if (params.lastEventAt != null) { patch.lastEventAt = params.lastEventAt; } - if ( + if (params.childSessionKey !== undefined) { + patch.childSessionKey = params.childSessionKey?.trim() || undefined; + } + if (params.clearError) { + patch.error = undefined; + } else if ( current.status === "cancelled" && nextStatus !== "cancelled" && params.error === undefined @@ -2124,7 +2161,9 @@ function updateTaskStateByRunId(params: { patch.progressSummary = normalizeTaskSummary(params.progressSummary); } if (params.terminalSummary !== undefined) { - patch.terminalSummary = normalizeTaskSummary(params.terminalSummary); + patch.terminalSummary = params.preserveTerminalSummary + ? (params.terminalSummary ?? undefined) + : normalizeTaskSummary(params.terminalSummary); } if (params.terminalOutcome !== undefined) { patch.terminalOutcome = resolveTaskTerminalOutcome({ @@ -2132,6 +2171,9 @@ function updateTaskStateByRunId(params: { terminalOutcome: params.terminalOutcome, }); } + if (params.detail !== undefined) { + patch.detail = params.detail; + } if (params.suppressDelivery) { // Teardown suppression must survive redundant lifecycle finalizers that // arrive after queues are cleared, or they can repopulate the stopped session. @@ -2234,28 +2276,36 @@ export function finalizeTaskRunByRunId(params: { runId: string; runtime?: TaskRuntime; sessionKey?: string; + childSessionKey?: string | null; status: Extract; startedAt?: number; endedAt: number; lastEventAt?: number; error?: string; + clearError?: boolean; progressSummary?: string | null; terminalSummary?: string | null; + preserveTerminalSummary?: boolean; terminalOutcome?: TaskTerminalOutcome | null; + detail?: JsonValue; suppressDelivery?: boolean; }) { return updateTaskStateByRunId({ runId: params.runId, runtime: params.runtime, sessionKey: params.sessionKey, + childSessionKey: params.childSessionKey, status: params.status, startedAt: params.startedAt, endedAt: params.endedAt, lastEventAt: params.lastEventAt, error: params.error, + clearError: params.clearError, progressSummary: params.progressSummary, terminalSummary: params.terminalSummary, + preserveTerminalSummary: params.preserveTerminalSummary, terminalOutcome: params.terminalOutcome, + detail: params.detail, suppressDelivery: params.suppressDelivery, }); } diff --git a/src/tasks/task-registry.types.ts b/src/tasks/task-registry.types.ts index a70819972bda..f50330003bfe 100644 --- a/src/tasks/task-registry.types.ts +++ b/src/tasks/task-registry.types.ts @@ -1,6 +1,15 @@ // Defines task registry records, statuses, delivery state, and parser helpers. import type { DeliveryContext } from "../utils/delivery-context.types.js"; +/** JSON value shape persisted with runtime-owned task detail. */ +export type JsonValue = + | null + | boolean + | number + | string + | JsonValue[] + | { [key: string]: JsonValue }; + /** Runtime family that owns a task run lifecycle. */ export type TaskRuntime = "subagent" | "acp" | "cli" | "cron"; @@ -147,4 +156,5 @@ export type TaskRecord = { progressSummary?: string; terminalSummary?: string; terminalOutcome?: TaskTerminalOutcome; + detail?: JsonValue; };