mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-14 00:56:07 +00:00
feat(cron): serve cron run history from the task ledger
Cron executions now write full-fidelity task_runs rows (detail_json column, store-key scoped) and cron.runs/CLI/task-maintenance read from the ledger; legacy cron_run_logs dual-write stays as a revert safety net. Existing history auto-imports at first state-DB open (verified against a production DB copy: 2750/2750 rows, byte-identical parity, idempotent). Startup crash recovery restores finished runs from finalized ledger rows instead of reporting synthetic interruptions. Part 1 of 2 for #106041.
This commit is contained in:
Peter Steinberger
@@ -64,7 +64,7 @@ async function waitForCronRunCompletion(params: {
|
||||
timeoutMs: number;
|
||||
pollIntervalMs: number;
|
||||
}): Promise<CronRunLogEntryResult> {
|
||||
// Poll the run log rather than cron.run because completion state is written asynchronously.
|
||||
// Poll the task ledger rather than cron.run because completion state is written asynchronously.
|
||||
const startedAt = Date.now();
|
||||
for (;;) {
|
||||
const page = (await callGatewayFromCli("cron.runs", params.opts, {
|
||||
|
||||
@@ -13,7 +13,6 @@ import { resolveStorePath } from "../config/sessions/paths.js";
|
||||
import { listSessionEntries } from "../config/sessions/session-accessor.js";
|
||||
import { resolveSessionTotalTokens, type SessionEntry } from "../config/sessions/types.js";
|
||||
import type { OpenClawConfig } from "../config/types.js";
|
||||
import { resolveCronJobsStorePath } from "../cron/store.js";
|
||||
import { listGatewayAgentsBasic } from "../gateway/agent-list.js";
|
||||
import { resolveHeartbeatSummaryForAgent } from "../infra/heartbeat-summary.js";
|
||||
import { peekSystemEvents } from "../infra/system-events.js";
|
||||
@@ -338,10 +337,7 @@ export async function getStatusSummary(
|
||||
const mainSessionKey = resolveMainSessionKey(cfg);
|
||||
const queuedSystemEvents = peekSystemEvents(mainSessionKey);
|
||||
const taskMaintenanceModule = await loadTaskRegistryMaintenanceModule();
|
||||
// Configure maintenance store before reading task summaries so cron-backed tasks are in scope.
|
||||
taskMaintenanceModule.configureTaskRegistryMaintenance({
|
||||
cronStorePath: resolveCronJobsStorePath(cfg.cron?.store),
|
||||
});
|
||||
taskMaintenanceModule.configureTaskRegistryMaintenance();
|
||||
const inspectableTasks = taskMaintenanceModule.reconcileInspectableTasks();
|
||||
const rawTasks = taskMaintenanceModule.getInspectableTaskRegistrySummary(inspectableTasks);
|
||||
const taskAuditFindings = taskMaintenanceModule.getInspectableTaskAuditFindings(inspectableTasks);
|
||||
|
||||
@@ -118,10 +118,7 @@ async function tryCancelGatewayOwnedTaskViaGateway(
|
||||
}
|
||||
|
||||
function configureTaskMaintenanceFromConfig(): void {
|
||||
const cfg = getRuntimeConfig();
|
||||
configureTaskRegistryMaintenance({
|
||||
cronStorePath: resolveCronJobsStorePath(cfg.cron?.store),
|
||||
});
|
||||
configureTaskRegistryMaintenance();
|
||||
}
|
||||
|
||||
type SessionRegistryMaintenanceStoreSummary = {
|
||||
|
||||
@@ -168,6 +168,8 @@ export function maybeEmitFailureAlert(
|
||||
error?: string;
|
||||
provider?: string;
|
||||
consecutiveCount: number;
|
||||
delivery?: "emit" | "record-only";
|
||||
occurredAtMs?: number;
|
||||
},
|
||||
) {
|
||||
if (!params.alertConfig || params.consecutiveCount < params.alertConfig.after) {
|
||||
@@ -177,7 +179,7 @@ export function maybeEmitFailureAlert(
|
||||
if (isBestEffort) {
|
||||
return;
|
||||
}
|
||||
const now = state.deps.nowMs();
|
||||
const now = params.occurredAtMs ?? state.deps.nowMs();
|
||||
const lastAlert = params.job.state.lastFailureAlertAtMs;
|
||||
// Cooldown is stored on job state so process restarts and service reloads do
|
||||
// not spam operators with repeated alerts for the same failing job.
|
||||
@@ -186,16 +188,18 @@ export function maybeEmitFailureAlert(
|
||||
if (inCooldown) {
|
||||
return;
|
||||
}
|
||||
emitFailureAlert(state, {
|
||||
job: params.job,
|
||||
error: params.error,
|
||||
consecutiveErrors: params.consecutiveCount,
|
||||
channel: params.alertConfig.channel,
|
||||
to: params.alertConfig.to,
|
||||
mode: params.alertConfig.mode,
|
||||
accountId: params.alertConfig.accountId,
|
||||
status: params.status,
|
||||
provider: params.provider,
|
||||
});
|
||||
if (params.delivery !== "record-only") {
|
||||
emitFailureAlert(state, {
|
||||
job: params.job,
|
||||
error: params.error,
|
||||
consecutiveErrors: params.consecutiveCount,
|
||||
channel: params.alertConfig.channel,
|
||||
to: params.alertConfig.to,
|
||||
mode: params.alertConfig.mode,
|
||||
accountId: params.alertConfig.accountId,
|
||||
status: params.status,
|
||||
provider: params.provider,
|
||||
});
|
||||
}
|
||||
params.job.state.lastFailureAlertAtMs = now;
|
||||
}
|
||||
|
||||
@@ -3,8 +3,12 @@ import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { runOpenClawStateWriteTransaction } from "../../state/openclaw-state-db.js";
|
||||
import * as detachedTaskRuntime from "../../tasks/detached-task-runtime.js";
|
||||
import { findTaskByRunId, resetTaskRegistryForTests } from "../../tasks/task-registry.js";
|
||||
import * as taskExecutor from "../../tasks/task-executor.js";
|
||||
import {
|
||||
findTaskByRunId,
|
||||
listTaskRecordsUnsorted,
|
||||
resetTaskRegistryForTests,
|
||||
} from "../../tasks/task-registry.js";
|
||||
import { formatTaskStatusDetail } from "../../tasks/task-status.js";
|
||||
import { withEnvAsync } from "../../test-utils/env.js";
|
||||
import * as cronSchedule from "../schedule.js";
|
||||
@@ -14,6 +18,7 @@ import { loadCronJobsStoreWithConfigJobs, loadCronStore } from "../store.js";
|
||||
import type { CronJob } from "../types.js";
|
||||
import { add, list, remove, run, start, stop, update } from "./ops.js";
|
||||
import { createCronServiceState, type CronEvent } from "./state.js";
|
||||
import { tryCreateCronTaskRun, tryFinishCronTaskRun } from "./task-runs.js";
|
||||
import { runMissedJobs } from "./timer.js";
|
||||
|
||||
const { logger, makeStorePath } = setupCronServiceSuite({
|
||||
@@ -195,7 +200,7 @@ function expectTaskRun(params: {
|
||||
sourceId: string;
|
||||
progressSummary?: string;
|
||||
}) {
|
||||
const task = findTaskByRunId(params.runId);
|
||||
const task = findCronTaskByBaseRunId(params.runId);
|
||||
expect(task?.runtime).toBe(params.runtime);
|
||||
expect(task?.status).toBe(params.status);
|
||||
expect(task?.sourceId).toBe(params.sourceId);
|
||||
@@ -204,6 +209,13 @@ function expectTaskRun(params: {
|
||||
}
|
||||
}
|
||||
|
||||
function findCronTaskByBaseRunId(baseRunId: string) {
|
||||
return (
|
||||
findTaskByRunId(baseRunId) ??
|
||||
listTaskRecordsUnsorted().find((task) => task.runId?.startsWith(`${baseRunId}:`))
|
||||
);
|
||||
}
|
||||
|
||||
function createMissedIsolatedJob(now: number): CronJob {
|
||||
return {
|
||||
id: "startup-timeout",
|
||||
@@ -388,6 +400,136 @@ describe("cron service ops seam coverage", () => {
|
||||
stop(state);
|
||||
});
|
||||
|
||||
it("preserves a finalized canonical task run when startup finds its stale marker", async () => {
|
||||
const { storePath } = await makeStorePath();
|
||||
const now = Date.parse("2026-03-23T12:00:00.000Z");
|
||||
const reservedAt = now - 30 * 60_000;
|
||||
const startedAt = reservedAt + 250;
|
||||
const endedAt = startedAt + 4_000;
|
||||
|
||||
await withStateDirForStorePath(storePath, async () => {
|
||||
const job = createInterruptedMainJob(now);
|
||||
job.trigger = { script: "json({ fire: true })", once: true };
|
||||
job.state.triggerState = { cursor: "old" };
|
||||
await writeCronStoreSnapshot({ storePath, jobs: [job] });
|
||||
const events: CronEvent[] = [];
|
||||
const state = createCronServiceState({
|
||||
storePath,
|
||||
cronEnabled: true,
|
||||
log: logger,
|
||||
nowMs: () => now,
|
||||
enqueueSystemEvent: vi.fn(),
|
||||
requestHeartbeat: vi.fn(),
|
||||
runIsolatedAgentJob: vi.fn(async () => ({ status: "ok" as const })),
|
||||
onEvent: (event) => events.push(structuredClone(event)),
|
||||
});
|
||||
const taskRunId = tryCreateCronTaskRun({
|
||||
state,
|
||||
job,
|
||||
startedAt,
|
||||
runIdStartedAt: reservedAt,
|
||||
});
|
||||
if (!taskRunId) {
|
||||
throw new Error("expected reserved cron task run");
|
||||
}
|
||||
|
||||
tryFinishCronTaskRun(state, {
|
||||
taskRunId,
|
||||
job,
|
||||
triggerEval: { fired: true, stateChanged: true, state: { cursor: "new" } },
|
||||
event: {
|
||||
jobId: job.id,
|
||||
action: "finished",
|
||||
job,
|
||||
status: "ok",
|
||||
summary: "completed before crash",
|
||||
delivered: true,
|
||||
deliveryStatus: "delivered",
|
||||
failureNotificationDelivery: { status: "not-requested" },
|
||||
runAtMs: startedAt,
|
||||
durationMs: endedAt - startedAt,
|
||||
triggerFired: true,
|
||||
},
|
||||
});
|
||||
|
||||
await start(state);
|
||||
|
||||
expect(findTaskByRunId(taskRunId)).toMatchObject({
|
||||
status: "succeeded",
|
||||
startedAt,
|
||||
terminalSummary: "completed before crash",
|
||||
endedAt,
|
||||
detail: { kind: "cron-run", status: "ok", triggerFired: true },
|
||||
});
|
||||
const persisted = await loadCronStore(storePath);
|
||||
expect(persisted.jobs[0]).toMatchObject({
|
||||
enabled: false,
|
||||
state: {
|
||||
lastRunAtMs: startedAt,
|
||||
lastRunStatus: "ok",
|
||||
lastStatus: "ok",
|
||||
lastDurationMs: endedAt - startedAt,
|
||||
lastDelivered: true,
|
||||
lastDeliveryStatus: "delivered",
|
||||
lastTriggerEvalAtMs: endedAt,
|
||||
lastTriggerFireAtMs: endedAt,
|
||||
triggerState: { cursor: "new" },
|
||||
},
|
||||
});
|
||||
expect(persisted.jobs[0]?.state.runningAtMs).toBeUndefined();
|
||||
expect(persisted.jobs[0]?.state.lastError).toBeUndefined();
|
||||
expect(persisted.jobs[0]?.state.nextRunAtMs).toBeUndefined();
|
||||
expect(events.filter((event) => event.action === "finished")).toEqual([]);
|
||||
stop(state);
|
||||
});
|
||||
});
|
||||
|
||||
it("restores finalized failure-alert cooldown without redelivery", async () => {
|
||||
const { storePath } = await makeStorePath();
|
||||
const now = Date.parse("2026-03-23T12:00:00.000Z");
|
||||
const startedAt = now - 30 * 60_000;
|
||||
const endedAt = startedAt + 4_000;
|
||||
|
||||
await withStateDirForStorePath(storePath, async () => {
|
||||
const job = createInterruptedMainJob(now);
|
||||
await writeCronStoreSnapshot({ storePath, jobs: [job] });
|
||||
const sendCronFailureAlert = vi.fn(async () => undefined);
|
||||
const state = createCronServiceState({
|
||||
storePath,
|
||||
cronEnabled: true,
|
||||
cronConfig: { failureAlert: { enabled: true, after: 1, cooldownMs: 60_000 } },
|
||||
log: logger,
|
||||
nowMs: () => now,
|
||||
enqueueSystemEvent: vi.fn(),
|
||||
requestHeartbeat: vi.fn(),
|
||||
sendCronFailureAlert,
|
||||
runIsolatedAgentJob: vi.fn(async () => ({ status: "ok" as const })),
|
||||
});
|
||||
|
||||
tryFinishCronTaskRun(state, {
|
||||
job,
|
||||
event: {
|
||||
jobId: job.id,
|
||||
action: "finished",
|
||||
job,
|
||||
status: "error",
|
||||
error: "provider unavailable",
|
||||
runAtMs: startedAt,
|
||||
durationMs: endedAt - startedAt,
|
||||
nextRunAtMs: now + 30 * 60_000,
|
||||
},
|
||||
});
|
||||
|
||||
await start(state);
|
||||
|
||||
const persisted = await loadCronStore(storePath);
|
||||
expect(persisted.jobs[0]?.state.lastFailureAlertAtMs).toBe(endedAt);
|
||||
expect(persisted.jobs[0]?.state.consecutiveErrors).toBe(1);
|
||||
expect(sendCronFailureAlert).not.toHaveBeenCalled();
|
||||
stop(state);
|
||||
});
|
||||
});
|
||||
|
||||
it("start persists load-time updatedAtMs repairs to the state sidecar only", async () => {
|
||||
const { storePath } = await makeStorePath();
|
||||
const now = Date.parse("2026-04-09T08:00:00.000Z");
|
||||
@@ -453,7 +595,7 @@ describe("cron service ops seam coverage", () => {
|
||||
});
|
||||
|
||||
expectTaskRun({
|
||||
runId: `cron:isolated-timeout:${now}`,
|
||||
runId: `cron:isolated-timeout:${now}:${manualRunId}`,
|
||||
runtime: "cron",
|
||||
status: "succeeded",
|
||||
sourceId: "isolated-timeout",
|
||||
@@ -483,6 +625,51 @@ describe("cron service ops seam coverage", () => {
|
||||
status: "timed_out",
|
||||
sourceId: "isolated-timeout",
|
||||
});
|
||||
expect(findCronTaskByBaseRunId(`cron:isolated-timeout:${now}`)?.detail).toMatchObject({
|
||||
kind: "cron-run",
|
||||
status: "error",
|
||||
runAtMs: now,
|
||||
durationMs: 0,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it("records failed manual runs with cron outcome detail", async () => {
|
||||
const { storePath } = await makeStorePath();
|
||||
const now = Date.parse("2026-03-23T12:00:00.000Z");
|
||||
|
||||
await withStateDirForStorePath(storePath, async () => {
|
||||
await writeDueIsolatedJobSnapshot(storePath, now);
|
||||
const state = createCronServiceState({
|
||||
storePath,
|
||||
cronEnabled: true,
|
||||
log: logger,
|
||||
nowMs: () => now,
|
||||
enqueueSystemEvent: vi.fn(),
|
||||
requestHeartbeat: vi.fn(),
|
||||
runIsolatedAgentJob: vi.fn(async () => ({
|
||||
status: "error" as const,
|
||||
error: "provider failed",
|
||||
provider: "openai",
|
||||
model: "gpt-test",
|
||||
})),
|
||||
});
|
||||
|
||||
await run(state, "isolated-timeout");
|
||||
|
||||
const task = findCronTaskByBaseRunId(`cron:isolated-timeout:${now}`);
|
||||
expect(task).toMatchObject({
|
||||
status: "failed",
|
||||
error: "provider failed",
|
||||
detail: {
|
||||
kind: "cron-run",
|
||||
status: "error",
|
||||
provider: "openai",
|
||||
model: "gpt-test",
|
||||
runAtMs: now,
|
||||
durationMs: 0,
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -496,7 +683,7 @@ describe("cron service ops seam coverage", () => {
|
||||
});
|
||||
|
||||
const createTaskRecordSpy = vi
|
||||
.spyOn(detachedTaskRuntime, "createRunningTaskRun")
|
||||
.spyOn(taskExecutor, "createRunningTaskRun")
|
||||
.mockImplementation(() => {
|
||||
throw new Error("disk full");
|
||||
});
|
||||
@@ -519,7 +706,7 @@ describe("cron service ops seam coverage", () => {
|
||||
await writeDueIsolatedJobSnapshot(storePath, now);
|
||||
|
||||
const updateTaskRecordSpy = vi
|
||||
.spyOn(detachedTaskRuntime, "completeTaskRunByRunId")
|
||||
.spyOn(taskExecutor, "finalizeTaskRunByRunId")
|
||||
.mockImplementation(() => {
|
||||
throw new Error("disk full");
|
||||
});
|
||||
@@ -653,7 +840,7 @@ describe("cron service ops seam coverage", () => {
|
||||
expect(state.deps.runIsolatedAgentJob).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
const task = findTaskByRunId(`cron:isolated-timeout:${now}`);
|
||||
const task = findCronTaskByBaseRunId(`cron:isolated-timeout:${now}`);
|
||||
if (!task) {
|
||||
throw new Error("expected active manual cron task ledger record");
|
||||
}
|
||||
|
||||
@@ -9,11 +9,6 @@ import { runWithGatewayIndependentRootWorkContinuation } from "../../process/gat
|
||||
import { CommandLane } from "../../process/lanes.js";
|
||||
import { DEFAULT_AGENT_ID } from "../../routing/session-key.js";
|
||||
import { resolveOpenClawStateSqlitePath } from "../../state/openclaw-state-db.paths.js";
|
||||
import {
|
||||
completeTaskRunByRunId,
|
||||
createRunningTaskRun,
|
||||
failTaskRunByRunId,
|
||||
} from "../../tasks/detached-task-runtime.js";
|
||||
import {
|
||||
clearCronJobActive,
|
||||
isCronActiveJobMarkerCurrent,
|
||||
@@ -22,10 +17,10 @@ import {
|
||||
} from "../active-jobs.js";
|
||||
import { resolveCronDeliveryPlan, resolveFailureDestination } from "../delivery-plan.js";
|
||||
import { createCronRunDiagnosticsFromError } from "../run-diagnostics.js";
|
||||
import { createCronExecutionId } from "../run-id.js";
|
||||
import type { CronRunLogEntry } from "../run-log-types.js";
|
||||
import { normalizeCronRunLogJobId } from "../run-log.js";
|
||||
import { cronSchedulingInputsEqual } from "../schedule-identity.js";
|
||||
import type { CronJob, CronJobCreate, CronJobPatch, CronPayload } from "../types.js";
|
||||
import type { CronJob, CronJobCreate, CronJobPatch, CronPayload, CronRunStatus } from "../types.js";
|
||||
import { normalizeCronRunErrorText } from "./execution-errors.js";
|
||||
import { failureNotificationDeliveryFromJobState } from "./failure-alerts.js";
|
||||
import {
|
||||
@@ -68,12 +63,19 @@ import {
|
||||
type CronRollbackSnapshot,
|
||||
warnIfDisabled,
|
||||
} from "./store.js";
|
||||
import { CRON_TASK_RUNNING_PROGRESS_SUMMARY } from "./task-ledger.js";
|
||||
import {
|
||||
tryCreateCronTaskRun,
|
||||
tryFindCronTaskRunIdForRecovery,
|
||||
tryFindFinalizedCronTaskRun,
|
||||
tryFinishCronTaskRun,
|
||||
tryFinishCronTaskRunWithoutHistory,
|
||||
} from "./task-runs.js";
|
||||
import {
|
||||
applyJobResult,
|
||||
applyTriggerNoFireResult,
|
||||
applyTriggerRunResult,
|
||||
armTimer,
|
||||
type CronTriggerEvalOutcome,
|
||||
executeJobCoreWithTimeout,
|
||||
type IsolatedAgentSetupTimeoutSignal,
|
||||
maybeNotifyIsolatedAgentSetupTimeout,
|
||||
@@ -86,6 +88,7 @@ const STARTUP_INTERRUPTED_ERROR = "cron: job interrupted by gateway restart";
|
||||
|
||||
type InterruptedStartupRun = {
|
||||
jobId: string;
|
||||
taskRunId?: string;
|
||||
runAtMs: number;
|
||||
durationMs: number;
|
||||
};
|
||||
@@ -146,6 +149,7 @@ function resolveInterruptedStartupFailureNotificationStatus(params: {
|
||||
function markInterruptedStartupRun(params: {
|
||||
state: CronServiceState;
|
||||
job: CronJob;
|
||||
taskRunId?: string;
|
||||
runningAtMs: number;
|
||||
nowMs: number;
|
||||
}): InterruptedStartupRun {
|
||||
@@ -188,11 +192,61 @@ function markInterruptedStartupRun(params: {
|
||||
|
||||
return {
|
||||
jobId: job.id,
|
||||
...(params.taskRunId ? { taskRunId: params.taskRunId } : {}),
|
||||
runAtMs: runningAtMs,
|
||||
durationMs: job.state.lastDurationMs,
|
||||
};
|
||||
}
|
||||
|
||||
function restoreFinalizedStartupRun(params: {
|
||||
state: CronServiceState;
|
||||
job: CronJob;
|
||||
runningAtMs: number;
|
||||
entry: CronRunLogEntry & { status: CronRunStatus };
|
||||
triggerEval?: CronTriggerEvalOutcome;
|
||||
}): boolean {
|
||||
const { state, job, runningAtMs, entry } = params;
|
||||
const startedAt = entry.runAtMs ?? runningAtMs;
|
||||
const shouldDelete = applyJobResult(
|
||||
state,
|
||||
job,
|
||||
{
|
||||
status: entry.status,
|
||||
error: entry.error,
|
||||
deliveryError: entry.deliveryError,
|
||||
diagnostics: entry.diagnostics,
|
||||
delivered: entry.delivered,
|
||||
provider: entry.provider,
|
||||
startedAt,
|
||||
endedAt: entry.ts,
|
||||
},
|
||||
{ replayFailureAlertAtMs: entry.ts },
|
||||
);
|
||||
|
||||
// The finalized row captured post-run state before the stale cron store write.
|
||||
job.state.lastDurationMs = entry.durationMs ?? Math.max(0, entry.ts - startedAt);
|
||||
job.state.lastErrorReason = entry.errorReason;
|
||||
job.state.lastDelivered = entry.delivered;
|
||||
job.state.lastDeliveryStatus = entry.deliveryStatus;
|
||||
job.state.lastDeliveryError = entry.deliveryError;
|
||||
job.state.lastFailureNotificationDelivered = entry.failureNotificationDelivery?.delivered;
|
||||
job.state.lastFailureNotificationDeliveryStatus = entry.failureNotificationDelivery?.status;
|
||||
job.state.lastFailureNotificationDeliveryError = entry.failureNotificationDelivery?.error;
|
||||
job.state.nextRunAtMs = entry.nextRunAtMs;
|
||||
if (params.triggerEval) {
|
||||
applyTriggerRunResult(job, {
|
||||
status: entry.status,
|
||||
endedAt: entry.ts,
|
||||
triggerEval: params.triggerEval,
|
||||
});
|
||||
}
|
||||
state.deps.log.info(
|
||||
{ jobId: job.id, runningAtMs, status: entry.status },
|
||||
"cron: restored finalized task-ledger run on startup",
|
||||
);
|
||||
return shouldDelete;
|
||||
}
|
||||
|
||||
function mergeManualRunSnapshotAfterReload(params: {
|
||||
state: CronServiceState;
|
||||
jobId: string;
|
||||
@@ -245,7 +299,8 @@ export async function start(state: CronServiceState) {
|
||||
|
||||
const interruptedJobIds = new Set<string>();
|
||||
const interruptedRuns: InterruptedStartupRun[] = [];
|
||||
let markedAnyInterruptedRun = false;
|
||||
const completedJobIdsToDelete = new Set<string>();
|
||||
let repairedAnyStartupRun = false;
|
||||
await locked(state, async () => {
|
||||
await ensureLoaded(state, { skipRecompute: true });
|
||||
if (state.stopped) {
|
||||
@@ -255,20 +310,43 @@ export async function start(state: CronServiceState) {
|
||||
for (const job of jobs) {
|
||||
job.state ??= {};
|
||||
if (typeof job.state.runningAtMs === "number") {
|
||||
const runningAtMs = job.state.runningAtMs;
|
||||
const taskRunId = tryFindCronTaskRunIdForRecovery(state, job.id, runningAtMs);
|
||||
const finalized = tryFindFinalizedCronTaskRun(state, job.id, runningAtMs);
|
||||
if (finalized) {
|
||||
interruptedJobIds.add(job.id);
|
||||
if (
|
||||
restoreFinalizedStartupRun({
|
||||
state,
|
||||
job,
|
||||
runningAtMs,
|
||||
entry: finalized.entry,
|
||||
...(finalized.triggerEval ? { triggerEval: finalized.triggerEval } : {}),
|
||||
})
|
||||
) {
|
||||
completedJobIdsToDelete.add(job.id);
|
||||
}
|
||||
repairedAnyStartupRun = true;
|
||||
continue;
|
||||
}
|
||||
const nowMs = state.deps.nowMs();
|
||||
const interrupted = markInterruptedStartupRun({
|
||||
state,
|
||||
job,
|
||||
runningAtMs: job.state.runningAtMs,
|
||||
taskRunId,
|
||||
runningAtMs,
|
||||
nowMs,
|
||||
});
|
||||
interruptedJobIds.add(job.id);
|
||||
interruptedRuns.push(interrupted);
|
||||
markedAnyInterruptedRun = true;
|
||||
repairedAnyStartupRun = true;
|
||||
}
|
||||
}
|
||||
if (markedAnyInterruptedRun || jobs.length > 0) {
|
||||
await persist(state, markedAnyInterruptedRun ? undefined : { stateOnly: true });
|
||||
if (completedJobIdsToDelete.size > 0 && state.store) {
|
||||
state.store.jobs = jobs.filter((job) => !completedJobIdsToDelete.has(job.id));
|
||||
}
|
||||
if (repairedAnyStartupRun || jobs.length > 0) {
|
||||
await persist(state, repairedAnyStartupRun ? undefined : { stateOnly: true });
|
||||
}
|
||||
});
|
||||
|
||||
@@ -294,20 +372,27 @@ export async function start(state: CronServiceState) {
|
||||
}
|
||||
for (const interrupted of interruptedRuns) {
|
||||
const job = state.store?.jobs.find((entry) => entry.id === interrupted.jobId);
|
||||
emit(state, {
|
||||
jobId: interrupted.jobId,
|
||||
action: "finished",
|
||||
job,
|
||||
status: "error",
|
||||
error: STARTUP_INTERRUPTED_ERROR,
|
||||
delivered: false,
|
||||
deliveryStatus: "unknown",
|
||||
deliveryError: STARTUP_INTERRUPTED_ERROR,
|
||||
failureNotificationDelivery: job ? failureNotificationDeliveryFromJobState(job) : undefined,
|
||||
runAtMs: interrupted.runAtMs,
|
||||
durationMs: interrupted.durationMs,
|
||||
nextRunAtMs: job?.state.nextRunAtMs,
|
||||
});
|
||||
emitCronRunFinished(
|
||||
state,
|
||||
{
|
||||
jobId: interrupted.jobId,
|
||||
action: "finished",
|
||||
job,
|
||||
status: "error",
|
||||
error: STARTUP_INTERRUPTED_ERROR,
|
||||
delivered: false,
|
||||
deliveryStatus: "unknown",
|
||||
deliveryError: STARTUP_INTERRUPTED_ERROR,
|
||||
failureNotificationDelivery: job
|
||||
? failureNotificationDeliveryFromJobState(job)
|
||||
: undefined,
|
||||
runAtMs: interrupted.runAtMs,
|
||||
durationMs: interrupted.durationMs,
|
||||
nextRunAtMs: job?.state.nextRunAtMs,
|
||||
},
|
||||
undefined,
|
||||
interrupted.taskRunId,
|
||||
);
|
||||
}
|
||||
armTimer(state);
|
||||
state.deps.log.info(
|
||||
@@ -821,11 +906,19 @@ type ManualRunOptions = {
|
||||
|
||||
type ManualRunTerminalTracker = { emitted: boolean };
|
||||
|
||||
function emitManualRunFinished(
|
||||
function emitCronRunFinished(
|
||||
state: CronServiceState,
|
||||
evt: CronEvent & { action: "finished" },
|
||||
tracker?: ManualRunTerminalTracker,
|
||||
taskRunId?: string,
|
||||
triggerEval?: CronTriggerEvalOutcome,
|
||||
): void {
|
||||
tryFinishCronTaskRun(state, {
|
||||
taskRunId,
|
||||
job: evt.job,
|
||||
event: evt,
|
||||
...(triggerEval ? { triggerEval } : {}),
|
||||
});
|
||||
emit(state, evt);
|
||||
if (tracker) {
|
||||
tracker.emitted = true;
|
||||
@@ -875,11 +968,12 @@ async function skipInvalidPersistedManualRun(params: {
|
||||
{ preserveSchedule: params.mode === "force" },
|
||||
);
|
||||
|
||||
emitManualRunFinished(
|
||||
emitCronRunFinished(
|
||||
params.state,
|
||||
{
|
||||
jobId: params.job.id,
|
||||
action: "finished",
|
||||
job: params.job,
|
||||
status: "skipped",
|
||||
error: errorText,
|
||||
diagnostics,
|
||||
@@ -899,91 +993,6 @@ async function skipInvalidPersistedManualRun(params: {
|
||||
armTimer(params.state);
|
||||
}
|
||||
|
||||
function tryCreateManualTaskRun(params: {
|
||||
state: CronServiceState;
|
||||
job: CronJob;
|
||||
startedAt: number;
|
||||
}): string | undefined {
|
||||
const runId = createCronExecutionId(params.job.id, params.startedAt);
|
||||
try {
|
||||
const task = createRunningTaskRun({
|
||||
runtime: "cron",
|
||||
sourceId: params.job.id,
|
||||
ownerKey: "",
|
||||
scopeKind: "system",
|
||||
childSessionKey: params.job.sessionKey,
|
||||
agentId: params.job.agentId,
|
||||
runId,
|
||||
label: params.job.name,
|
||||
task: params.job.name || params.job.id,
|
||||
deliveryStatus: "not_applicable",
|
||||
notifyPolicy: "silent",
|
||||
startedAt: params.startedAt,
|
||||
lastEventAt: params.startedAt,
|
||||
progressSummary: CRON_TASK_RUNNING_PROGRESS_SUMMARY,
|
||||
});
|
||||
if (!task) {
|
||||
params.state.deps.log.warn(
|
||||
{ jobId: params.job.id },
|
||||
"cron: task ledger record was not persisted",
|
||||
);
|
||||
return undefined;
|
||||
}
|
||||
return runId;
|
||||
} catch (error) {
|
||||
params.state.deps.log.warn(
|
||||
{ jobId: params.job.id, error },
|
||||
"cron: failed to create task ledger record",
|
||||
);
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function tryFinishManualTaskRun(
|
||||
state: CronServiceState,
|
||||
params: {
|
||||
taskRunId?: string;
|
||||
coreResult: Awaited<ReturnType<typeof executeJobCoreWithTimeout>>;
|
||||
endedAt: number;
|
||||
},
|
||||
): void {
|
||||
if (!params.taskRunId) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
if (params.coreResult.status === "ok" || params.coreResult.status === "skipped") {
|
||||
completeTaskRunByRunId({
|
||||
runId: params.taskRunId,
|
||||
runtime: "cron",
|
||||
endedAt: params.endedAt,
|
||||
lastEventAt: params.endedAt,
|
||||
terminalSummary: params.coreResult.summary ?? undefined,
|
||||
});
|
||||
return;
|
||||
}
|
||||
failTaskRunByRunId({
|
||||
runId: params.taskRunId,
|
||||
runtime: "cron",
|
||||
status:
|
||||
normalizeCronRunErrorText(params.coreResult.error) === "cron: job execution timed out"
|
||||
? "timed_out"
|
||||
: "failed",
|
||||
endedAt: params.endedAt,
|
||||
lastEventAt: params.endedAt,
|
||||
error:
|
||||
params.coreResult.status === "error"
|
||||
? normalizeCronRunErrorText(params.coreResult.error)
|
||||
: undefined,
|
||||
terminalSummary: params.coreResult.summary ?? undefined,
|
||||
});
|
||||
} catch (error) {
|
||||
state.deps.log.warn(
|
||||
{ runId: params.taskRunId, jobStatus: params.coreResult.status, error },
|
||||
"cron: failed to update task ledger record",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function inspectManualRunPreflight(
|
||||
state: CronServiceState,
|
||||
id: string,
|
||||
@@ -1088,10 +1097,11 @@ async function prepareManualRun(
|
||||
return { ok: true, ran: false, reason: "stopped" as const };
|
||||
}
|
||||
emit(state, { jobId: job.id, action: "started", job, runAtMs: preflight.now });
|
||||
const taskRunId = tryCreateManualTaskRun({
|
||||
const taskRunId = tryCreateCronTaskRun({
|
||||
state,
|
||||
job,
|
||||
startedAt: preflight.now,
|
||||
publicRunId: opts?.runId,
|
||||
});
|
||||
const activeJobMarker = markManualCronJobActive(state, job);
|
||||
// Execute against a snapshot so later reload/merge can preserve delivery
|
||||
@@ -1141,16 +1151,16 @@ async function finishPreparedManualRun(
|
||||
coreResult = { status: "error", error: normalizeCronRunErrorText(err) };
|
||||
}
|
||||
const endedAt = state.deps.nowMs();
|
||||
const triggerSkipped = coreResult.status === "ok" && coreResult.triggerEval?.fired === false;
|
||||
const emitMissingQueuedTerminal = () => {
|
||||
const tracker = prepared.terminalTracker;
|
||||
if (!tracker || tracker.emitted) {
|
||||
return;
|
||||
}
|
||||
const job = state.store?.jobs.find((entry) => entry.id === jobId);
|
||||
const triggerSkipped = coreResult.status === "ok" && coreResult.triggerEval?.fired === false;
|
||||
// enqueueRun acknowledges a concrete run id, so every accepted request
|
||||
// needs one terminal event even if the job or service owner changes mid-run.
|
||||
emitManualRunFinished(
|
||||
emitCronRunFinished(
|
||||
state,
|
||||
{
|
||||
jobId,
|
||||
@@ -1176,13 +1186,22 @@ async function finishPreparedManualRun(
|
||||
usage: coreResult.usage,
|
||||
},
|
||||
tracker,
|
||||
taskRunId,
|
||||
);
|
||||
};
|
||||
tryFinishManualTaskRun(state, {
|
||||
taskRunId,
|
||||
coreResult,
|
||||
endedAt,
|
||||
});
|
||||
if (!triggerSkipped) {
|
||||
// Terminal state must land even if the store merge below throws; the later
|
||||
// emitCronRunFinished re-finalizes the same row to attach history detail
|
||||
// (same-status terminal updates apply, so this does not race precedence).
|
||||
tryFinishCronTaskRunWithoutHistory(state, {
|
||||
taskRunId,
|
||||
status: coreResult.status,
|
||||
error: coreResult.error,
|
||||
endedAt,
|
||||
summary: coreResult.summary,
|
||||
childSessionKey: coreResult.sessionKey,
|
||||
});
|
||||
}
|
||||
if (!isCronActiveJobMarkerCurrent(prepared.activeJobMarker)) {
|
||||
emitMissingQueuedTerminal();
|
||||
return;
|
||||
@@ -1232,7 +1251,7 @@ async function finishPreparedManualRun(
|
||||
triggerEval: coreResult.triggerEval,
|
||||
});
|
||||
|
||||
emitManualRunFinished(
|
||||
emitCronRunFinished(
|
||||
state,
|
||||
{
|
||||
jobId: job.id,
|
||||
@@ -1259,6 +1278,8 @@ async function finishPreparedManualRun(
|
||||
usage: coreResult.usage,
|
||||
},
|
||||
prepared.terminalTracker,
|
||||
taskRunId,
|
||||
coreResult.triggerEval,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1303,6 +1324,16 @@ async function finishPreparedManualRun(
|
||||
});
|
||||
}
|
||||
if (finalized) {
|
||||
if (triggerSkipped) {
|
||||
tryFinishCronTaskRunWithoutHistory(state, {
|
||||
taskRunId,
|
||||
status: coreResult.status,
|
||||
error: coreResult.error,
|
||||
endedAt,
|
||||
summary: coreResult.summary,
|
||||
childSessionKey: coreResult.sessionKey,
|
||||
});
|
||||
}
|
||||
armTimer(state);
|
||||
}
|
||||
emitMissingQueuedTerminal();
|
||||
@@ -1344,7 +1375,7 @@ export async function enqueueRun(state: CronServiceState, id: string, mode?: "du
|
||||
if (result.reason !== "invalid-spec") {
|
||||
const finishedAt = state.deps.nowMs();
|
||||
const job = state.store?.jobs.find((entry) => entry.id === id);
|
||||
emitManualRunFinished(
|
||||
emitCronRunFinished(
|
||||
state,
|
||||
{
|
||||
jobId: id,
|
||||
@@ -1387,7 +1418,7 @@ export async function enqueueRun(state: CronServiceState, id: string, mode?: "du
|
||||
}
|
||||
const finishedAt = state.deps.nowMs();
|
||||
const job = state.store?.jobs.find((entry) => entry.id === id);
|
||||
emitManualRunFinished(
|
||||
emitCronRunFinished(
|
||||
state,
|
||||
{
|
||||
jobId: id,
|
||||
|
||||
700
src/cron/service/task-runs.test.ts
Normal file
700
src/cron/service/task-runs.test.ts
Normal file
@@ -0,0 +1,700 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
getDetachedTaskLifecycleRuntime,
|
||||
resetDetachedTaskLifecycleRuntimeForTests,
|
||||
setDetachedTaskLifecycleRuntime,
|
||||
} from "../../tasks/detached-task-runtime.js";
|
||||
import * as taskExecutor from "../../tasks/task-executor.js";
|
||||
import { finalizeTaskRunByRunId } from "../../tasks/task-executor.js";
|
||||
import * as taskRegistry from "../../tasks/task-registry.js";
|
||||
import { markTaskLostById, resetTaskRegistryForTests } from "../../tasks/task-registry.js";
|
||||
import { listTaskRegistryRecordsByRuntimeSourceIdFromSqlite } from "../../tasks/task-registry.store.sqlite.js";
|
||||
import { withOpenClawTestState } from "../../test-utils/openclaw-test-state.js";
|
||||
import { cronStoreKey } from "../store/key.js";
|
||||
import { readCronTaskRunHistoryPage } from "../task-run-history.js";
|
||||
import type { CronJob } from "../types.js";
|
||||
import { timeoutErrorMessage } from "./execution-errors.js";
|
||||
import { createCronServiceState } from "./state.js";
|
||||
import {
|
||||
tryCreateCronTaskRun,
|
||||
tryFindCronTaskRunIdForRecovery,
|
||||
tryFindFinalizedCronTaskRun,
|
||||
tryFinishCronTaskRun,
|
||||
tryFinishCronTaskRunWithoutHistory,
|
||||
} from "./task-runs.js";
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
resetDetachedTaskLifecycleRuntimeForTests();
|
||||
resetTaskRegistryForTests({ persist: false });
|
||||
});
|
||||
|
||||
describe("cron task run terminal records", () => {
|
||||
it("persists canonical history directly when a detached runtime is registered", async () => {
|
||||
await withOpenClawTestState(
|
||||
{ layout: "state-only", prefix: "openclaw-cron-core-ledger-runtime-" },
|
||||
async () => {
|
||||
resetTaskRegistryForTests();
|
||||
const customCreate = vi.fn(() => null);
|
||||
const customFinalize = vi.fn(() => []);
|
||||
setDetachedTaskLifecycleRuntime({
|
||||
...getDetachedTaskLifecycleRuntime(),
|
||||
createRunningTaskRun: customCreate,
|
||||
finalizeTaskRunByRunId: customFinalize,
|
||||
});
|
||||
const job: CronJob = {
|
||||
id: "core-ledger-runtime",
|
||||
name: "core ledger runtime",
|
||||
enabled: true,
|
||||
createdAtMs: 100,
|
||||
updatedAtMs: 100,
|
||||
schedule: { kind: "every", everyMs: 60_000, anchorMs: 100 },
|
||||
sessionTarget: "main",
|
||||
wakeMode: "next-heartbeat",
|
||||
payload: { kind: "systemEvent", text: "work" },
|
||||
state: {},
|
||||
};
|
||||
const state = createCronServiceState({
|
||||
storePath: "/tmp/jobs.json",
|
||||
cronEnabled: true,
|
||||
log: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() },
|
||||
nowMs: () => 1_100,
|
||||
enqueueSystemEvent: vi.fn(),
|
||||
requestHeartbeat: vi.fn(),
|
||||
runIsolatedAgentJob: vi.fn(async () => ({ status: "ok" as const })),
|
||||
});
|
||||
|
||||
tryFinishCronTaskRun(state, {
|
||||
job,
|
||||
event: {
|
||||
jobId: job.id,
|
||||
action: "finished",
|
||||
job,
|
||||
status: "ok",
|
||||
runAtMs: 1_000,
|
||||
durationMs: 100,
|
||||
},
|
||||
});
|
||||
|
||||
expect(customCreate).not.toHaveBeenCalled();
|
||||
expect(customFinalize).not.toHaveBeenCalled();
|
||||
expect(
|
||||
readCronTaskRunHistoryPage({
|
||||
storeKey: cronStoreKey(state.deps.storePath),
|
||||
jobId: job.id,
|
||||
}).entries,
|
||||
).toEqual([expect.objectContaining({ status: "ok", runAtMs: 1_000 })]);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps task-registry lookup failures inside the best-effort boundary", () => {
|
||||
const warn = vi.fn();
|
||||
const startedAt = 500;
|
||||
const job: CronJob = {
|
||||
id: "lookup-failure",
|
||||
name: "lookup failure",
|
||||
enabled: true,
|
||||
createdAtMs: 100,
|
||||
updatedAtMs: 100,
|
||||
schedule: { kind: "every", everyMs: 60_000, anchorMs: 100 },
|
||||
sessionTarget: "main",
|
||||
wakeMode: "next-heartbeat",
|
||||
payload: { kind: "systemEvent", text: "work" },
|
||||
state: {},
|
||||
};
|
||||
const state = createCronServiceState({
|
||||
storePath: "/tmp/jobs.json",
|
||||
cronEnabled: true,
|
||||
log: { debug: vi.fn(), info: vi.fn(), warn, error: vi.fn() },
|
||||
nowMs: () => startedAt + 100,
|
||||
enqueueSystemEvent: vi.fn(),
|
||||
requestHeartbeat: vi.fn(),
|
||||
runIsolatedAgentJob: vi.fn(async () => ({ status: "ok" as const })),
|
||||
});
|
||||
vi.spyOn(taskRegistry, "findTaskByRunId").mockImplementation(() => {
|
||||
throw new Error("task store unavailable");
|
||||
});
|
||||
vi.spyOn(taskRegistry, "listTaskRecordsUnsorted").mockImplementation(() => {
|
||||
throw new Error("task store unavailable");
|
||||
});
|
||||
|
||||
expect(tryFindFinalizedCronTaskRun(state, job.id, startedAt)).toBeUndefined();
|
||||
expect(() =>
|
||||
tryFinishCronTaskRun(state, {
|
||||
job,
|
||||
event: {
|
||||
jobId: job.id,
|
||||
action: "finished",
|
||||
job,
|
||||
status: "ok",
|
||||
runAtMs: startedAt,
|
||||
durationMs: 100,
|
||||
},
|
||||
}),
|
||||
).not.toThrow();
|
||||
expect(warn).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ jobId: job.id }),
|
||||
"cron: failed to read finalized task ledger record",
|
||||
);
|
||||
expect(warn).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
runId: expect.stringMatching(new RegExp(`^cron:${job.id}:${startedAt}:`)),
|
||||
}),
|
||||
"cron: failed to update task ledger record",
|
||||
);
|
||||
});
|
||||
|
||||
it("creates an immediately terminal task row for a skipped-only event", async () => {
|
||||
await withOpenClawTestState(
|
||||
{ layout: "state-only", prefix: "openclaw-cron-skipped-task-" },
|
||||
async () => {
|
||||
resetTaskRegistryForTests();
|
||||
const startedAt = 1_000;
|
||||
const job: CronJob = {
|
||||
id: "skipped-job",
|
||||
name: "skipped job",
|
||||
agentId: "finn",
|
||||
enabled: true,
|
||||
createdAtMs: 100,
|
||||
updatedAtMs: 100,
|
||||
schedule: { kind: "every", everyMs: 60_000, anchorMs: 100 },
|
||||
sessionTarget: "isolated",
|
||||
wakeMode: "next-heartbeat",
|
||||
payload: { kind: "agentTurn", message: "work" },
|
||||
state: { nextRunAtMs: 60_000 },
|
||||
};
|
||||
const state = createCronServiceState({
|
||||
storePath: "/tmp/jobs.json",
|
||||
cronEnabled: true,
|
||||
log: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() },
|
||||
nowMs: () => startedAt,
|
||||
enqueueSystemEvent: vi.fn(),
|
||||
requestHeartbeat: vi.fn(),
|
||||
runIsolatedAgentJob: vi.fn(async () => ({ status: "ok" as const })),
|
||||
});
|
||||
|
||||
tryFinishCronTaskRun(state, {
|
||||
job,
|
||||
event: {
|
||||
jobId: job.id,
|
||||
action: "finished",
|
||||
job,
|
||||
status: "skipped",
|
||||
error: "trigger condition not met",
|
||||
runId: "manual:skipped-job:1",
|
||||
runAtMs: startedAt,
|
||||
durationMs: 0,
|
||||
nextRunAtMs: 60_000,
|
||||
},
|
||||
});
|
||||
|
||||
const rows = listTaskRegistryRecordsByRuntimeSourceIdFromSqlite({
|
||||
runtime: "cron",
|
||||
sourceId: job.id,
|
||||
});
|
||||
expect(rows).toHaveLength(1);
|
||||
expect(rows[0]).toMatchObject({
|
||||
runtime: "cron",
|
||||
sourceId: job.id,
|
||||
agentId: "finn",
|
||||
status: "succeeded",
|
||||
startedAt,
|
||||
endedAt: startedAt,
|
||||
error: "trigger condition not met",
|
||||
detail: {
|
||||
kind: "cron-run",
|
||||
status: "skipped",
|
||||
runId: "manual:skipped-job:1",
|
||||
nextRunAtMs: 60_000,
|
||||
},
|
||||
});
|
||||
expect(
|
||||
readCronTaskRunHistoryPage({
|
||||
storeKey: cronStoreKey(state.deps.storePath),
|
||||
jobId: job.id,
|
||||
}).entries,
|
||||
).toEqual([
|
||||
expect.objectContaining({
|
||||
jobId: job.id,
|
||||
status: "skipped",
|
||||
runId: "manual:skipped-job:1",
|
||||
}),
|
||||
]);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps same-millisecond cron executions as distinct task rows", async () => {
|
||||
await withOpenClawTestState(
|
||||
{ layout: "state-only", prefix: "openclaw-cron-distinct-task-runs-" },
|
||||
async () => {
|
||||
resetTaskRegistryForTests();
|
||||
const startedAt = 1_500;
|
||||
const job: CronJob = {
|
||||
id: "same-millisecond-job",
|
||||
name: "same millisecond job",
|
||||
enabled: true,
|
||||
createdAtMs: 100,
|
||||
updatedAtMs: 100,
|
||||
schedule: { kind: "every", everyMs: 60_000, anchorMs: 100 },
|
||||
sessionTarget: "isolated",
|
||||
wakeMode: "next-heartbeat",
|
||||
payload: { kind: "agentTurn", message: "work" },
|
||||
state: { nextRunAtMs: 60_000 },
|
||||
};
|
||||
const state = createCronServiceState({
|
||||
storePath: "/tmp/jobs.json",
|
||||
cronEnabled: true,
|
||||
log: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() },
|
||||
nowMs: () => startedAt + 100,
|
||||
enqueueSystemEvent: vi.fn(),
|
||||
requestHeartbeat: vi.fn(),
|
||||
runIsolatedAgentJob: vi.fn(async () => ({ status: "ok" as const })),
|
||||
});
|
||||
const publicRunIds = [
|
||||
"manual:same-millisecond-job:1500:1",
|
||||
"manual:same-millisecond-job:1500:2",
|
||||
];
|
||||
const taskRunIds = publicRunIds.map((publicRunId) =>
|
||||
tryCreateCronTaskRun({ state, job, startedAt, publicRunId }),
|
||||
);
|
||||
expect(new Set(taskRunIds).size).toBe(2);
|
||||
|
||||
for (const [index, taskRunId] of taskRunIds.entries()) {
|
||||
if (!taskRunId) {
|
||||
throw new Error("expected unique cron task run id");
|
||||
}
|
||||
tryFinishCronTaskRun(state, {
|
||||
taskRunId,
|
||||
job,
|
||||
event: {
|
||||
jobId: job.id,
|
||||
action: "finished",
|
||||
job,
|
||||
status: "ok",
|
||||
runId: publicRunIds[index],
|
||||
runAtMs: startedAt,
|
||||
durationMs: 100,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
const rows = listTaskRegistryRecordsByRuntimeSourceIdFromSqlite({
|
||||
runtime: "cron",
|
||||
sourceId: job.id,
|
||||
});
|
||||
expect(rows).toHaveLength(2);
|
||||
expect(new Set(rows.map((row) => row.runId)).size).toBe(2);
|
||||
expect(
|
||||
readCronTaskRunHistoryPage({
|
||||
storeKey: cronStoreKey(state.deps.storePath),
|
||||
jobId: job.id,
|
||||
}).entries.map((entry) => entry.runId),
|
||||
).toEqual(expect.arrayContaining(publicRunIds));
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps operator cancellation while attaching terminal run history", async () => {
|
||||
await withOpenClawTestState(
|
||||
{ layout: "state-only", prefix: "openclaw-cron-cancelled-task-" },
|
||||
async () => {
|
||||
resetTaskRegistryForTests();
|
||||
const startedAt = 2_000;
|
||||
const job: CronJob = {
|
||||
id: "cancelled-job",
|
||||
name: "cancelled job",
|
||||
enabled: true,
|
||||
createdAtMs: 100,
|
||||
updatedAtMs: 100,
|
||||
schedule: { kind: "every", everyMs: 60_000, anchorMs: 100 },
|
||||
sessionTarget: "isolated",
|
||||
wakeMode: "next-heartbeat",
|
||||
payload: { kind: "agentTurn", message: "work" },
|
||||
state: { nextRunAtMs: 60_000 },
|
||||
};
|
||||
const state = createCronServiceState({
|
||||
storePath: "/tmp/jobs.json",
|
||||
cronEnabled: true,
|
||||
log: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() },
|
||||
nowMs: () => startedAt + 100,
|
||||
enqueueSystemEvent: vi.fn(),
|
||||
requestHeartbeat: vi.fn(),
|
||||
runIsolatedAgentJob: vi.fn(async () => ({ status: "ok" as const })),
|
||||
});
|
||||
const taskRunId = tryCreateCronTaskRun({ state, job, startedAt });
|
||||
if (!taskRunId) {
|
||||
throw new Error("expected cron task run id");
|
||||
}
|
||||
finalizeTaskRunByRunId({
|
||||
runId: taskRunId,
|
||||
runtime: "cron",
|
||||
status: "cancelled",
|
||||
endedAt: startedAt + 50,
|
||||
error: "cancelled by operator",
|
||||
terminalSummary: "Cancelled by operator.",
|
||||
});
|
||||
|
||||
tryFinishCronTaskRun(state, {
|
||||
taskRunId,
|
||||
job,
|
||||
event: {
|
||||
jobId: job.id,
|
||||
action: "finished",
|
||||
job,
|
||||
status: "ok",
|
||||
runAtMs: startedAt,
|
||||
durationMs: 100,
|
||||
},
|
||||
});
|
||||
|
||||
const [row] = listTaskRegistryRecordsByRuntimeSourceIdFromSqlite({
|
||||
runtime: "cron",
|
||||
sourceId: job.id,
|
||||
});
|
||||
expect(row).toMatchObject({
|
||||
status: "cancelled",
|
||||
endedAt: startedAt + 100,
|
||||
detail: { kind: "cron-run", status: "ok", durationMs: 100 },
|
||||
});
|
||||
expect(row?.error).toBeUndefined();
|
||||
expect(row?.terminalSummary).toBeUndefined();
|
||||
expect(
|
||||
readCronTaskRunHistoryPage({
|
||||
storeKey: cronStoreKey(state.deps.storePath),
|
||||
jobId: job.id,
|
||||
}).entries,
|
||||
).toEqual([
|
||||
expect.objectContaining({
|
||||
jobId: job.id,
|
||||
status: "ok",
|
||||
error: undefined,
|
||||
}),
|
||||
]);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it("retries the original outcome after an empty finalization result", async () => {
|
||||
await withOpenClawTestState(
|
||||
{ layout: "state-only", prefix: "openclaw-cron-task-retry-" },
|
||||
async () => {
|
||||
resetTaskRegistryForTests();
|
||||
const startedAt = 3_000;
|
||||
const job: CronJob = {
|
||||
id: "retry-job",
|
||||
name: "retry job",
|
||||
enabled: true,
|
||||
createdAtMs: 100,
|
||||
updatedAtMs: 100,
|
||||
schedule: { kind: "every", everyMs: 60_000, anchorMs: 100 },
|
||||
sessionTarget: "isolated",
|
||||
wakeMode: "next-heartbeat",
|
||||
payload: { kind: "agentTurn", message: "work" },
|
||||
state: { nextRunAtMs: 60_000 },
|
||||
};
|
||||
const state = createCronServiceState({
|
||||
storePath: "/tmp/jobs.json",
|
||||
cronEnabled: true,
|
||||
log: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() },
|
||||
nowMs: () => startedAt + 100,
|
||||
enqueueSystemEvent: vi.fn(),
|
||||
requestHeartbeat: vi.fn(),
|
||||
runIsolatedAgentJob: vi.fn(async () => ({ status: "ok" as const })),
|
||||
});
|
||||
const taskRunId = tryCreateCronTaskRun({ state, job, startedAt });
|
||||
if (!taskRunId) {
|
||||
throw new Error("expected cron task run id");
|
||||
}
|
||||
const finalize = taskExecutor.finalizeTaskRunByRunId;
|
||||
vi.spyOn(taskExecutor, "finalizeTaskRunByRunId")
|
||||
.mockReturnValueOnce([])
|
||||
.mockImplementation((params) => finalize(params));
|
||||
|
||||
tryFinishCronTaskRun(state, {
|
||||
taskRunId,
|
||||
job,
|
||||
event: {
|
||||
jobId: job.id,
|
||||
action: "finished",
|
||||
job,
|
||||
status: "ok",
|
||||
summary: "done",
|
||||
sessionKey: "agent:main:cron:retry-job:run:actual",
|
||||
runAtMs: startedAt,
|
||||
durationMs: 100,
|
||||
},
|
||||
});
|
||||
|
||||
const rows = listTaskRegistryRecordsByRuntimeSourceIdFromSqlite({
|
||||
runtime: "cron",
|
||||
sourceId: job.id,
|
||||
});
|
||||
expect(rows).toHaveLength(1);
|
||||
const [row] = rows;
|
||||
expect(row).toMatchObject({
|
||||
status: "succeeded",
|
||||
childSessionKey: "agent:main:cron:retry-job:run:actual",
|
||||
terminalSummary: "done",
|
||||
detail: { kind: "cron-run", status: "ok" },
|
||||
});
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it("overwrites a lost canonical row with restart terminal history", async () => {
|
||||
await withOpenClawTestState(
|
||||
{ layout: "state-only", prefix: "openclaw-cron-task-lost-recovery-" },
|
||||
async () => {
|
||||
resetTaskRegistryForTests();
|
||||
const startedAt = 4_000;
|
||||
const job: CronJob = {
|
||||
id: "lost-job",
|
||||
name: "lost job",
|
||||
enabled: true,
|
||||
createdAtMs: 100,
|
||||
updatedAtMs: 100,
|
||||
schedule: { kind: "every", everyMs: 60_000, anchorMs: 100 },
|
||||
sessionTarget: "isolated",
|
||||
wakeMode: "next-heartbeat",
|
||||
payload: { kind: "agentTurn", message: "work" },
|
||||
state: { nextRunAtMs: 60_000 },
|
||||
};
|
||||
const state = createCronServiceState({
|
||||
storePath: "/tmp/jobs.json",
|
||||
cronEnabled: true,
|
||||
log: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() },
|
||||
nowMs: () => startedAt + 100,
|
||||
enqueueSystemEvent: vi.fn(),
|
||||
requestHeartbeat: vi.fn(),
|
||||
runIsolatedAgentJob: vi.fn(async () => ({ status: "ok" as const })),
|
||||
});
|
||||
const taskRunId = tryCreateCronTaskRun({ state, job, startedAt });
|
||||
if (!taskRunId) {
|
||||
throw new Error("expected cron task run id");
|
||||
}
|
||||
const [running] = listTaskRegistryRecordsByRuntimeSourceIdFromSqlite({
|
||||
runtime: "cron",
|
||||
sourceId: job.id,
|
||||
});
|
||||
if (!running) {
|
||||
throw new Error("expected running cron task");
|
||||
}
|
||||
taskRegistry.markTaskTerminalById({
|
||||
taskId: running.taskId,
|
||||
status: "failed",
|
||||
endedAt: startedAt + 40,
|
||||
terminalSummary: "Task session disappeared.",
|
||||
});
|
||||
markTaskLostById({
|
||||
taskId: running.taskId,
|
||||
endedAt: startedAt + 50,
|
||||
error: "backing session missing",
|
||||
});
|
||||
const renamedJob = { ...job, name: "renamed lost job", sessionTarget: "main" as const };
|
||||
|
||||
tryFinishCronTaskRun(state, {
|
||||
taskRunId,
|
||||
job: renamedJob,
|
||||
event: {
|
||||
jobId: job.id,
|
||||
action: "finished",
|
||||
job: renamedJob,
|
||||
status: "error",
|
||||
error: "gateway restarted while job was running",
|
||||
runAtMs: startedAt,
|
||||
durationMs: 100,
|
||||
},
|
||||
});
|
||||
|
||||
const rows = listTaskRegistryRecordsByRuntimeSourceIdFromSqlite({
|
||||
runtime: "cron",
|
||||
sourceId: job.id,
|
||||
});
|
||||
expect(rows).toHaveLength(1);
|
||||
const [row] = rows;
|
||||
expect(row).toMatchObject({
|
||||
taskId: running.taskId,
|
||||
status: "failed",
|
||||
endedAt: startedAt + 100,
|
||||
error: "gateway restarted while job was running",
|
||||
detail: { kind: "cron-run", status: "error", durationMs: 100 },
|
||||
});
|
||||
expect(row).not.toHaveProperty("childSessionKey");
|
||||
expect(row?.terminalSummary).toBeUndefined();
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it("overwrites a provisional timeout with restart terminal history", async () => {
|
||||
await withOpenClawTestState(
|
||||
{ layout: "state-only", prefix: "openclaw-cron-task-timeout-recovery-" },
|
||||
async () => {
|
||||
resetTaskRegistryForTests();
|
||||
const startedAt = 5_000;
|
||||
const job: CronJob = {
|
||||
id: "provisional-timeout-job",
|
||||
name: "provisional timeout job",
|
||||
enabled: true,
|
||||
createdAtMs: 100,
|
||||
updatedAtMs: 100,
|
||||
schedule: { kind: "every", everyMs: 60_000, anchorMs: 100 },
|
||||
sessionTarget: "isolated",
|
||||
wakeMode: "next-heartbeat",
|
||||
payload: { kind: "agentTurn", message: "work" },
|
||||
state: { nextRunAtMs: 60_000 },
|
||||
};
|
||||
const state = createCronServiceState({
|
||||
storePath: "/tmp/jobs.json",
|
||||
cronEnabled: true,
|
||||
log: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() },
|
||||
nowMs: () => startedAt + 200,
|
||||
enqueueSystemEvent: vi.fn(),
|
||||
requestHeartbeat: vi.fn(),
|
||||
runIsolatedAgentJob: vi.fn(async () => ({ status: "ok" as const })),
|
||||
});
|
||||
const taskRunId = tryCreateCronTaskRun({ state, job, startedAt });
|
||||
if (!taskRunId) {
|
||||
throw new Error("expected cron task run id");
|
||||
}
|
||||
tryFinishCronTaskRunWithoutHistory(state, {
|
||||
taskRunId,
|
||||
status: "error",
|
||||
error: timeoutErrorMessage(),
|
||||
endedAt: startedAt + 100,
|
||||
});
|
||||
|
||||
tryFinishCronTaskRun(state, {
|
||||
taskRunId,
|
||||
job,
|
||||
event: {
|
||||
jobId: job.id,
|
||||
action: "finished",
|
||||
job,
|
||||
status: "error",
|
||||
error: "gateway restarted while job was running",
|
||||
runAtMs: startedAt,
|
||||
durationMs: 200,
|
||||
},
|
||||
});
|
||||
|
||||
const [row] = listTaskRegistryRecordsByRuntimeSourceIdFromSqlite({
|
||||
runtime: "cron",
|
||||
sourceId: job.id,
|
||||
});
|
||||
expect(row).toMatchObject({
|
||||
status: "failed",
|
||||
endedAt: startedAt + 200,
|
||||
error: "gateway restarted while job was running",
|
||||
detail: { kind: "cron-run", status: "error", durationMs: 200 },
|
||||
});
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it("recovers pre-discriminator task rows written by older releases", async () => {
|
||||
await withOpenClawTestState(
|
||||
{ layout: "state-only", prefix: "openclaw-cron-task-legacy-runid-" },
|
||||
async () => {
|
||||
resetTaskRegistryForTests();
|
||||
const startedAt = 7_000;
|
||||
const state = createCronServiceState({
|
||||
storePath: "/tmp/jobs.json",
|
||||
cronEnabled: true,
|
||||
log: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() },
|
||||
nowMs: () => startedAt + 100,
|
||||
enqueueSystemEvent: vi.fn(),
|
||||
requestHeartbeat: vi.fn(),
|
||||
runIsolatedAgentJob: vi.fn(async () => ({ status: "ok" as const })),
|
||||
});
|
||||
const legacyRunId = `cron:legacy-job:${startedAt}`;
|
||||
// Older releases persisted the reservation id without a uniqueness suffix.
|
||||
taskExecutor.createRunningTaskRun({
|
||||
runtime: "cron",
|
||||
sourceId: "legacy-job",
|
||||
ownerKey: "",
|
||||
scopeKind: "system",
|
||||
runId: legacyRunId,
|
||||
task: "legacy job",
|
||||
deliveryStatus: "not_applicable",
|
||||
notifyPolicy: "silent",
|
||||
startedAt,
|
||||
});
|
||||
expect(tryFindCronTaskRunIdForRecovery(state, "legacy-job", startedAt)).toBe(legacyRunId);
|
||||
finalizeTaskRunByRunId({
|
||||
runId: legacyRunId,
|
||||
runtime: "cron",
|
||||
status: "timed_out",
|
||||
endedAt: startedAt + 50,
|
||||
error: timeoutErrorMessage(),
|
||||
});
|
||||
|
||||
tryFinishCronTaskRun(state, {
|
||||
taskRunId: legacyRunId,
|
||||
event: {
|
||||
jobId: "legacy-job",
|
||||
action: "finished",
|
||||
status: "error",
|
||||
error: "gateway restarted while job was running",
|
||||
runAtMs: startedAt,
|
||||
durationMs: 100,
|
||||
},
|
||||
});
|
||||
|
||||
const [row] = listTaskRegistryRecordsByRuntimeSourceIdFromSqlite({
|
||||
runtime: "cron",
|
||||
sourceId: "legacy-job",
|
||||
});
|
||||
expect(row).toMatchObject({
|
||||
runId: legacyRunId,
|
||||
status: "failed",
|
||||
error: "gateway restarted while job was running",
|
||||
detail: { kind: "cron-run", status: "error", durationMs: 100 },
|
||||
});
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps suffixed recovery identities scoped to the current cron store", async () => {
|
||||
await withOpenClawTestState(
|
||||
{ layout: "state-only", prefix: "openclaw-cron-task-store-recovery-" },
|
||||
async (fixture) => {
|
||||
resetTaskRegistryForTests();
|
||||
const startedAt = 8_000;
|
||||
const job: CronJob = {
|
||||
id: "shared-job",
|
||||
name: "shared job",
|
||||
enabled: true,
|
||||
createdAtMs: 100,
|
||||
updatedAtMs: 100,
|
||||
schedule: { kind: "every", everyMs: 60_000, anchorMs: 100 },
|
||||
sessionTarget: "main",
|
||||
wakeMode: "next-heartbeat",
|
||||
payload: { kind: "systemEvent", text: "work" },
|
||||
state: {},
|
||||
};
|
||||
const createState = (storePath: string) =>
|
||||
createCronServiceState({
|
||||
storePath,
|
||||
cronEnabled: true,
|
||||
log: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() },
|
||||
nowMs: () => startedAt,
|
||||
enqueueSystemEvent: vi.fn(),
|
||||
requestHeartbeat: vi.fn(),
|
||||
runIsolatedAgentJob: vi.fn(async () => ({ status: "ok" as const })),
|
||||
});
|
||||
const stateA = createState(fixture.path("cron-a", "jobs.json"));
|
||||
const stateB = createState(fixture.path("cron-b", "jobs.json"));
|
||||
const runA = tryCreateCronTaskRun({ state: stateA, job, startedAt });
|
||||
const runB = tryCreateCronTaskRun({ state: stateB, job, startedAt });
|
||||
expect(runA).toBeTruthy();
|
||||
expect(runB).toBeTruthy();
|
||||
expect(runA).not.toBe(runB);
|
||||
|
||||
expect(tryFindCronTaskRunIdForRecovery(stateA, job.id, startedAt)).toBe(runA);
|
||||
expect(tryFindCronTaskRunIdForRecovery(stateB, job.id, startedAt)).toBe(runB);
|
||||
},
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -1,20 +1,33 @@
|
||||
/** Detached task-ledger integration for cron runs. */
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { normalizeOptionalLowercaseString } from "@openclaw/normalization-core/string-coerce";
|
||||
import {
|
||||
DEFAULT_AGENT_ID,
|
||||
normalizeAgentId,
|
||||
resolveAgentIdFromSessionKey,
|
||||
} from "../../routing/session-key.js";
|
||||
import { createRunningTaskRun, finalizeTaskRunByRunId } from "../../tasks/task-executor.js";
|
||||
import {
|
||||
completeTaskRunByRunId,
|
||||
createRunningTaskRun,
|
||||
failTaskRunByRunId,
|
||||
} from "../../tasks/detached-task-runtime.js";
|
||||
findTaskByRunId,
|
||||
listTaskRecordsUnsorted,
|
||||
markTaskTerminalById,
|
||||
} from "../../tasks/task-registry.js";
|
||||
import type { JsonValue, TaskRecord, TaskStatus } from "../../tasks/task-registry.types.js";
|
||||
import { resolveCronAgentSessionKey } from "../isolated-agent/session-key.js";
|
||||
import { createCronExecutionId } from "../run-id.js";
|
||||
import type { CronRunLogEntry } from "../run-log-types.js";
|
||||
import { cronStoreKey } from "../store/key.js";
|
||||
import {
|
||||
cronRunLogEntryFromEvent,
|
||||
cronRunLogEntryToTaskDetail,
|
||||
cronRunStatusToTaskStatus,
|
||||
cronTaskRecordStoreKey,
|
||||
cronTaskRecordToRunLogEntry,
|
||||
cronTaskRecordToTriggerEval,
|
||||
} from "../task-run-detail.js";
|
||||
import type { CronJob, CronRunStatus } from "../types.js";
|
||||
import { normalizeCronRunErrorText, timeoutErrorMessage } from "./execution-errors.js";
|
||||
import type { CronServiceState } from "./state.js";
|
||||
import type { CronEvent, CronServiceState } from "./state.js";
|
||||
import { CRON_TASK_RUNNING_PROGRESS_SUMMARY } from "./task-ledger.js";
|
||||
|
||||
/** Converts cron ids into bounded session-key path segments with a fallback for empty input. */
|
||||
@@ -69,80 +82,312 @@ export function tryCreateCronTaskRun(params: {
|
||||
state: CronServiceState;
|
||||
job: CronJob;
|
||||
startedAt: number;
|
||||
runIdStartedAt?: number;
|
||||
publicRunId?: string;
|
||||
}): string | undefined {
|
||||
const runId = createCronTaskRunId(
|
||||
params.job.id,
|
||||
params.runIdStartedAt ?? params.startedAt,
|
||||
params.publicRunId,
|
||||
);
|
||||
return tryCreateCronTaskRunRecord({
|
||||
state: params.state,
|
||||
job: params.job,
|
||||
jobId: params.job.id,
|
||||
startedAt: params.startedAt,
|
||||
runId,
|
||||
});
|
||||
}
|
||||
|
||||
function createCronTaskRunId(jobId: string, reservationAt: number, publicRunId?: string): string {
|
||||
const discriminator = publicRunId?.trim() || randomUUID();
|
||||
return `${createCronExecutionId(jobId, reservationAt)}:${discriminator}`;
|
||||
}
|
||||
|
||||
function findLatestCronTaskRunForRecovery(
|
||||
jobId: string,
|
||||
reservationAt: number,
|
||||
storeKey: string,
|
||||
): TaskRecord | undefined {
|
||||
const reservationRunId = createCronExecutionId(jobId, reservationAt);
|
||||
const prefix = `${reservationRunId}:`;
|
||||
return listTaskRecordsUnsorted()
|
||||
.filter((task) => {
|
||||
if (task.runtime !== "cron" || task.sourceId !== jobId) {
|
||||
return false;
|
||||
}
|
||||
const taskStoreKey = cronTaskRecordStoreKey(task);
|
||||
if (taskStoreKey === undefined) {
|
||||
// Exact match covers detail-less pre-discriminator rows from older releases.
|
||||
return task.runId === reservationRunId;
|
||||
}
|
||||
return (
|
||||
taskStoreKey === storeKey &&
|
||||
(task.runId === reservationRunId || task.runId?.startsWith(prefix))
|
||||
);
|
||||
})
|
||||
.toSorted(
|
||||
(left, right) =>
|
||||
Number(left.endedAt !== undefined) - Number(right.endedAt !== undefined) ||
|
||||
(right.endedAt ?? right.lastEventAt ?? right.createdAt) -
|
||||
(left.endedAt ?? left.lastEventAt ?? left.createdAt) ||
|
||||
right.createdAt - left.createdAt ||
|
||||
right.taskId.localeCompare(left.taskId),
|
||||
)[0];
|
||||
}
|
||||
|
||||
/** Finds the unique task identity owned by one persisted cron reservation. */
|
||||
export function tryFindCronTaskRunIdForRecovery(
|
||||
state: CronServiceState,
|
||||
jobId: string,
|
||||
startedAt: number,
|
||||
): string | undefined {
|
||||
try {
|
||||
return findLatestCronTaskRunForRecovery(jobId, startedAt, cronStoreKey(state.deps.storePath))
|
||||
?.runId;
|
||||
} catch (error) {
|
||||
state.deps.log.warn({ jobId, error }, "cron: failed to read task ledger recovery record");
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/** Finds a completed canonical cron row for startup crash recovery. */
|
||||
export function tryFindFinalizedCronTaskRun(
|
||||
state: CronServiceState,
|
||||
jobId: string,
|
||||
startedAt: number,
|
||||
):
|
||||
| {
|
||||
entry: CronRunLogEntry & { status: CronRunStatus };
|
||||
triggerEval?: { fired: true; stateChanged: boolean; state?: JsonValue };
|
||||
}
|
||||
| undefined {
|
||||
try {
|
||||
const task = findLatestCronTaskRunForRecovery(
|
||||
jobId,
|
||||
startedAt,
|
||||
cronStoreKey(state.deps.storePath),
|
||||
);
|
||||
if (task?.runtime !== "cron" || task.sourceId !== jobId || task.endedAt === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
const entry = cronTaskRecordToRunLogEntry(task);
|
||||
if (!entry?.status) {
|
||||
return undefined;
|
||||
}
|
||||
const triggerEval = cronTaskRecordToTriggerEval(task);
|
||||
return {
|
||||
entry: { ...entry, status: entry.status },
|
||||
...(triggerEval ? { triggerEval } : {}),
|
||||
};
|
||||
} catch (error) {
|
||||
state.deps.log.warn({ jobId, error }, "cron: failed to read finalized task ledger record");
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function tryCreateCronTaskRunRecord(params: {
|
||||
state: CronServiceState;
|
||||
job?: CronJob;
|
||||
jobId: string;
|
||||
startedAt: number;
|
||||
runId: string;
|
||||
childSessionKey?: string;
|
||||
}): string | undefined {
|
||||
const runId = createCronExecutionId(params.job.id, params.startedAt);
|
||||
try {
|
||||
const task = createRunningTaskRun({
|
||||
runtime: "cron",
|
||||
sourceId: params.job.id,
|
||||
sourceId: params.jobId,
|
||||
ownerKey: "",
|
||||
scopeKind: "system",
|
||||
childSessionKey: resolveCronTaskChildSessionKey(params),
|
||||
agentId: params.job.agentId,
|
||||
runId,
|
||||
label: params.job.name,
|
||||
task: params.job.name || params.job.id,
|
||||
childSessionKey:
|
||||
params.childSessionKey ??
|
||||
(params.job
|
||||
? resolveCronTaskChildSessionKey({
|
||||
state: params.state,
|
||||
job: params.job,
|
||||
startedAt: params.startedAt,
|
||||
})
|
||||
: undefined),
|
||||
agentId:
|
||||
params.job?.agentId ??
|
||||
resolveAgentIdFromSessionKey(params.childSessionKey) ??
|
||||
params.state.deps.defaultAgentId ??
|
||||
DEFAULT_AGENT_ID,
|
||||
runId: params.runId,
|
||||
label: params.job?.name,
|
||||
task: params.job?.name || params.jobId,
|
||||
deliveryStatus: "not_applicable",
|
||||
notifyPolicy: "silent",
|
||||
startedAt: params.startedAt,
|
||||
lastEventAt: params.startedAt,
|
||||
progressSummary: CRON_TASK_RUNNING_PROGRESS_SUMMARY,
|
||||
detail: { storeKey: cronStoreKey(params.state.deps.storePath) },
|
||||
});
|
||||
if (!task) {
|
||||
params.state.deps.log.warn(
|
||||
{ jobId: params.job.id },
|
||||
{ jobId: params.jobId },
|
||||
"cron: task ledger record was not persisted",
|
||||
);
|
||||
return undefined;
|
||||
}
|
||||
return runId;
|
||||
return params.runId;
|
||||
} catch (error) {
|
||||
params.state.deps.log.warn(
|
||||
{ jobId: params.job.id, error },
|
||||
{ jobId: params.jobId, error },
|
||||
"cron: failed to create task ledger record",
|
||||
);
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/** Completes or fails the detached task ledger row for a cron run when one exists. */
|
||||
export function tryFinishCronTaskRun(
|
||||
/** Finalizes executions that intentionally do not produce a run-history row. */
|
||||
export function tryFinishCronTaskRunWithoutHistory(
|
||||
state: CronServiceState,
|
||||
result: {
|
||||
taskRunId?: string;
|
||||
status: CronRunStatus;
|
||||
status: "ok" | "error" | "skipped";
|
||||
error?: unknown;
|
||||
endedAt: number;
|
||||
summary?: string;
|
||||
childSessionKey?: string;
|
||||
},
|
||||
): void {
|
||||
if (!result.taskRunId) {
|
||||
return;
|
||||
}
|
||||
const error = result.status === "error" ? normalizeCronRunErrorText(result.error) : undefined;
|
||||
try {
|
||||
if (result.status === "ok" || result.status === "skipped") {
|
||||
completeTaskRunByRunId({
|
||||
runId: result.taskRunId,
|
||||
runtime: "cron",
|
||||
endedAt: result.endedAt,
|
||||
lastEventAt: result.endedAt,
|
||||
terminalSummary: result.summary ?? undefined,
|
||||
});
|
||||
return;
|
||||
}
|
||||
failTaskRunByRunId({
|
||||
finalizeTaskRunByRunId({
|
||||
runId: result.taskRunId,
|
||||
runtime: "cron",
|
||||
status:
|
||||
normalizeCronRunErrorText(result.error) === timeoutErrorMessage() ? "timed_out" : "failed",
|
||||
result.status === "ok" || result.status === "skipped"
|
||||
? "succeeded"
|
||||
: error === timeoutErrorMessage()
|
||||
? "timed_out"
|
||||
: "failed",
|
||||
endedAt: result.endedAt,
|
||||
lastEventAt: result.endedAt,
|
||||
error: result.status === "error" ? normalizeCronRunErrorText(result.error) : undefined,
|
||||
terminalSummary: result.summary ?? undefined,
|
||||
error,
|
||||
terminalSummary: result.summary,
|
||||
childSessionKey: result.childSessionKey,
|
||||
});
|
||||
} catch (error) {
|
||||
} catch (cause) {
|
||||
state.deps.log.warn(
|
||||
{ runId: result.taskRunId, jobStatus: result.status, error },
|
||||
{ runId: result.taskRunId, jobStatus: result.status, error: cause },
|
||||
"cron: failed to update task ledger record",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/** Finalizes the authoritative task row, creating one for terminal-only cron events. */
|
||||
export function tryFinishCronTaskRun(
|
||||
state: CronServiceState,
|
||||
result: {
|
||||
taskRunId?: string;
|
||||
job?: CronJob;
|
||||
event: CronEvent & { action: "finished" };
|
||||
triggerEval?: { fired: boolean; stateChanged: boolean; state?: unknown };
|
||||
},
|
||||
): void {
|
||||
const entry = cronRunLogEntryFromEvent(result.event, state.deps.nowMs());
|
||||
const startedAt = entry.runAtMs ?? entry.ts;
|
||||
const candidateRunId =
|
||||
result.taskRunId ?? createCronTaskRunId(entry.jobId, startedAt, entry.runId);
|
||||
try {
|
||||
const existingCandidate = findTaskByRunId(candidateRunId);
|
||||
const taskRunId =
|
||||
existingCandidate?.runtime === "cron"
|
||||
? candidateRunId
|
||||
: tryCreateCronTaskRunRecord({
|
||||
state,
|
||||
job: result.job ?? result.event.job,
|
||||
jobId: entry.jobId,
|
||||
startedAt,
|
||||
runId: candidateRunId,
|
||||
childSessionKey: entry.sessionKey,
|
||||
});
|
||||
if (!taskRunId) {
|
||||
return;
|
||||
}
|
||||
const storeKey = cronStoreKey(state.deps.storePath);
|
||||
const legacyRecoveryRunId = createCronExecutionId(entry.jobId, startedAt);
|
||||
const detail = cronRunLogEntryToTaskDetail(entry, {
|
||||
storeKey,
|
||||
...(result.triggerEval ? { triggerEval: result.triggerEval } : {}),
|
||||
});
|
||||
const finalize = (
|
||||
runId: string,
|
||||
status: Extract<
|
||||
TaskStatus,
|
||||
"succeeded" | "failed" | "timed_out" | "cancelled"
|
||||
> = cronRunStatusToTaskStatus(entry),
|
||||
) =>
|
||||
finalizeTaskRunByRunId({
|
||||
runId,
|
||||
runtime: "cron",
|
||||
status,
|
||||
endedAt: entry.ts,
|
||||
lastEventAt: entry.ts,
|
||||
error: entry.error,
|
||||
clearError: entry.error === undefined,
|
||||
terminalSummary: entry.summary ?? null,
|
||||
preserveTerminalSummary: true,
|
||||
childSessionKey: entry.sessionKey ?? null,
|
||||
detail,
|
||||
});
|
||||
let updated = finalize(taskRunId);
|
||||
if (updated.length === 0) {
|
||||
const existing = findTaskByRunId(taskRunId);
|
||||
if (existing?.runtime === "cron" && existing.status === "cancelled") {
|
||||
// Operator cancellation owns task status, but its finished event still owns history detail.
|
||||
updated = finalize(taskRunId, "cancelled");
|
||||
} else if (
|
||||
existing?.runtime === "cron" &&
|
||||
(existing.status === "lost" ||
|
||||
(cronTaskRecordStoreKey(existing) === storeKey &&
|
||||
cronTaskRecordToRunLogEntry(existing) === null) ||
|
||||
(existing.detail === undefined && existing.runId === legacyRecoveryRunId))
|
||||
) {
|
||||
// Pre-persist markers and exact legacy identities contain no history detail.
|
||||
// Startup recovery replaces them with the durable interrupted outcome.
|
||||
const recovered = markTaskTerminalById({
|
||||
taskId: existing.taskId,
|
||||
status: cronRunStatusToTaskStatus(entry),
|
||||
childSessionKey: entry.sessionKey ?? null,
|
||||
endedAt: entry.ts,
|
||||
lastEventAt: entry.ts,
|
||||
error: entry.error,
|
||||
terminalSummary: entry.summary ?? null,
|
||||
preserveTerminalSummary: true,
|
||||
detail,
|
||||
});
|
||||
updated = recovered ? [recovered] : [];
|
||||
} else if (existing?.runtime === "cron") {
|
||||
// Keep the existing run/session scope when its first terminal write failed.
|
||||
updated = finalize(taskRunId);
|
||||
} else {
|
||||
// A terminal event still owns one durable row if its active mirror vanished.
|
||||
const recreatedRunId = tryCreateCronTaskRunRecord({
|
||||
state,
|
||||
job: result.job ?? result.event.job,
|
||||
jobId: entry.jobId,
|
||||
startedAt,
|
||||
runId: taskRunId,
|
||||
childSessionKey: entry.sessionKey,
|
||||
});
|
||||
if (recreatedRunId) {
|
||||
updated = finalize(recreatedRunId);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (updated.length === 0) {
|
||||
state.deps.log.warn({ runId: taskRunId }, "cron: task ledger record was not finalized");
|
||||
}
|
||||
} catch (error) {
|
||||
state.deps.log.warn(
|
||||
{ runId: candidateRunId, jobStatus: entry.status, error },
|
||||
"cron: failed to update task ledger record",
|
||||
);
|
||||
}
|
||||
|
||||
@@ -85,6 +85,14 @@ function firstMockArg(mock: unknown): unknown {
|
||||
return call[0];
|
||||
}
|
||||
|
||||
function findCronTaskByBaseRunId(baseRunId: string) {
|
||||
return listTaskRecords().find(
|
||||
(entry) =>
|
||||
entry.runtime === "cron" &&
|
||||
(entry.runId === baseRunId || entry.runId?.startsWith(`${baseRunId}:`)),
|
||||
);
|
||||
}
|
||||
|
||||
describe("cron service timer regressions", () => {
|
||||
it("caps timer delay to 60s for far-future schedules", async () => {
|
||||
const timeoutSpy = vi.spyOn(globalThis, "setTimeout");
|
||||
@@ -843,9 +851,7 @@ describe("cron service timer regressions", () => {
|
||||
await runnerStarted.promise;
|
||||
|
||||
const runId = `cron:no-timeout-cancel:${scheduledAt}`;
|
||||
const task = listTaskRecords().find(
|
||||
(entry) => entry.runtime === "cron" && entry.runId === runId,
|
||||
);
|
||||
const task = findCronTaskByBaseRunId(runId);
|
||||
if (!task) {
|
||||
throw new Error("Expected timeout-disabled cron task row");
|
||||
}
|
||||
@@ -1032,8 +1038,9 @@ describe("cron service timer regressions", () => {
|
||||
void timerPromise.then(() => {
|
||||
timerSettled = true;
|
||||
});
|
||||
const taskRunId = findCronTaskByBaseRunId(runId)?.runId;
|
||||
const cancelled = cancelActiveCronTaskRun({
|
||||
runId,
|
||||
runId: taskRunId ?? runId,
|
||||
reason: "Cancelled by operator.",
|
||||
});
|
||||
|
||||
@@ -1106,9 +1113,7 @@ describe("cron service timer regressions", () => {
|
||||
await cleanupStarted.promise;
|
||||
|
||||
const runId = `cron:late-cancel-after-timeout:${scheduledAt}`;
|
||||
const task = listTaskRecords().find(
|
||||
(entry) => entry.runtime === "cron" && entry.runId === runId,
|
||||
);
|
||||
const task = findCronTaskByBaseRunId(runId);
|
||||
if (!task) {
|
||||
throw new Error("Expected timed-out cron task row");
|
||||
}
|
||||
@@ -1689,9 +1694,7 @@ describe("cron service timer regressions", () => {
|
||||
}
|
||||
expect(runHeartbeatOnce).toHaveBeenCalledTimes(1);
|
||||
|
||||
const task = listTaskRecords().find(
|
||||
(entry) => entry.runtime === "cron" && entry.runId === runId,
|
||||
);
|
||||
const task = findCronTaskByBaseRunId(runId);
|
||||
if (!task) {
|
||||
throw new Error("Expected main-session cron task row");
|
||||
}
|
||||
|
||||
@@ -5,10 +5,16 @@ import { upsertSessionEntry } from "../../config/sessions/session-accessor.js";
|
||||
import { setupCronServiceSuite, writeCronStoreSnapshot } from "../../cron/service.test-harness.js";
|
||||
import { createCronServiceState } from "../../cron/service/state.js";
|
||||
import { executeJobCore, onTimer } from "../../cron/service/timer.js";
|
||||
import * as cronStoreModule from "../../cron/store.js";
|
||||
import { loadCronStore } from "../../cron/store.js";
|
||||
import { cronStoreKey } from "../../cron/store/key.js";
|
||||
import type { CronJob } from "../../cron/types.js";
|
||||
import * as detachedTaskRuntime from "../../tasks/detached-task-runtime.js";
|
||||
import { findTaskByRunId, resetTaskRegistryForTests } from "../../tasks/task-registry.js";
|
||||
import * as taskExecutor from "../../tasks/task-executor.js";
|
||||
import {
|
||||
findTaskByRunId,
|
||||
listTaskRecordsUnsorted,
|
||||
resetTaskRegistryForTests,
|
||||
} from "../../tasks/task-registry.js";
|
||||
import { formatTaskStatusDetail } from "../../tasks/task-status.js";
|
||||
|
||||
const { logger, makeStorePath } = setupCronServiceSuite({
|
||||
@@ -63,6 +69,13 @@ function createDueCommandJob(params: { now: number }): CronJob {
|
||||
};
|
||||
}
|
||||
|
||||
function findCronTaskByBaseRunId(baseRunId: string) {
|
||||
return (
|
||||
findTaskByRunId(baseRunId) ??
|
||||
listTaskRecordsUnsorted().find((task) => task.runId?.startsWith(`${baseRunId}:`))
|
||||
);
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
resetTaskRegistryForTests();
|
||||
});
|
||||
@@ -170,7 +183,7 @@ describe("cron service timer seam coverage", () => {
|
||||
expect(job.state.lastStatus).toBe("ok");
|
||||
expect(job.state.runningAtMs).toBeUndefined();
|
||||
expect(job.state.nextRunAtMs).toBe(now + 60_000);
|
||||
const task = findTaskByRunId(`cron:main-heartbeat-job:${now}`);
|
||||
const task = findCronTaskByBaseRunId(`cron:main-heartbeat-job:${now}`);
|
||||
if (!task) {
|
||||
throw new Error("expected cron task ledger record");
|
||||
}
|
||||
@@ -179,7 +192,7 @@ describe("cron service timer seam coverage", () => {
|
||||
expect(task.ownerKey).toBe("");
|
||||
expect(task.scopeKind).toBe("system");
|
||||
expect(task.childSessionKey).toBe(cronRunSessionKey);
|
||||
expect(task.runId).toBe(`cron:main-heartbeat-job:${now}`);
|
||||
expect(task.runId).toMatch(new RegExp(`^cron:main-heartbeat-job:${now}:`));
|
||||
expect(task.label).toBe("main heartbeat job");
|
||||
expect(task.task).toBe("main heartbeat job");
|
||||
expect(task.status).toBe("succeeded");
|
||||
@@ -199,6 +212,102 @@ describe("cron service timer seam coverage", () => {
|
||||
timeoutSpy.mockRestore();
|
||||
});
|
||||
|
||||
it("uses the persisted reservation timestamp for the canonical timer task", async () => {
|
||||
const { storePath } = await makeStorePath();
|
||||
const now = Date.parse("2026-03-23T12:00:00.000Z");
|
||||
let clock = now;
|
||||
let persistedReservation: number | undefined;
|
||||
let liveReservation: number | undefined;
|
||||
let liveError: string | undefined;
|
||||
let emittedStartedAt: number | undefined;
|
||||
const job = createDueIsolatedAgentJob({ now });
|
||||
job.state.lastError = "previous failure";
|
||||
await writeCronStoreSnapshot({
|
||||
storePath,
|
||||
jobs: [job],
|
||||
});
|
||||
const state = createCronServiceState({
|
||||
storePath,
|
||||
cronEnabled: true,
|
||||
log: logger,
|
||||
nowMs: () => clock++,
|
||||
enqueueSystemEvent: vi.fn(),
|
||||
requestHeartbeat: vi.fn(),
|
||||
runIsolatedAgentJob: vi.fn(async () => {
|
||||
persistedReservation = (await loadCronStore(storePath)).jobs[0]?.state.runningAtMs;
|
||||
liveReservation = state.store?.jobs[0]?.state.runningAtMs;
|
||||
liveError = state.store?.jobs[0]?.state.lastError;
|
||||
return { status: "ok" as const };
|
||||
}),
|
||||
onEvent: (event) => {
|
||||
if (event.action === "started") {
|
||||
emittedStartedAt = event.runAtMs;
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
await onTimer(state);
|
||||
|
||||
expect(persistedReservation).toEqual(expect.any(Number));
|
||||
expect(liveReservation).toBe(persistedReservation);
|
||||
expect(liveError).toBeUndefined();
|
||||
expect(emittedStartedAt).toEqual(expect.any(Number));
|
||||
expect(emittedStartedAt).toBeGreaterThan(persistedReservation ?? 0);
|
||||
expect(
|
||||
findCronTaskByBaseRunId(`cron:isolated-agent-job:${persistedReservation}`),
|
||||
).toMatchObject({
|
||||
startedAt: emittedStartedAt,
|
||||
status: "succeeded",
|
||||
});
|
||||
});
|
||||
|
||||
it("finalizes quiet trigger tasks only after cron state persists", async () => {
|
||||
const { storePath } = await makeStorePath();
|
||||
const now = Date.parse("2026-03-23T12:00:00.000Z");
|
||||
const job = {
|
||||
...createDueIsolatedAgentJob({ now }),
|
||||
trigger: { script: "json({ fire: false })" },
|
||||
};
|
||||
await writeCronStoreSnapshot({ storePath, jobs: [job] });
|
||||
const order: string[] = [];
|
||||
const save = cronStoreModule.saveCronJobsStore;
|
||||
const finalize = taskExecutor.finalizeTaskRunByRunId;
|
||||
const saveSpy = vi
|
||||
.spyOn(cronStoreModule, "saveCronJobsStore")
|
||||
.mockImplementation(async (...args) => {
|
||||
order.push("persist");
|
||||
return await save(...args);
|
||||
});
|
||||
const finalizeSpy = vi
|
||||
.spyOn(taskExecutor, "finalizeTaskRunByRunId")
|
||||
.mockImplementation((params) => {
|
||||
order.push("finalize");
|
||||
return finalize(params);
|
||||
});
|
||||
const state = createCronServiceState({
|
||||
storePath,
|
||||
cronEnabled: true,
|
||||
cronConfig: { triggers: { enabled: true } },
|
||||
log: logger,
|
||||
nowMs: () => now,
|
||||
enqueueSystemEvent: vi.fn(),
|
||||
requestHeartbeat: vi.fn(),
|
||||
evaluateCronTrigger: vi.fn(async () => ({ kind: "evaluated" as const, fire: false })),
|
||||
runIsolatedAgentJob: vi.fn(async () => ({ status: "ok" as const })),
|
||||
});
|
||||
|
||||
try {
|
||||
await onTimer(state);
|
||||
expect(order).toEqual(["persist", "persist", "finalize"]);
|
||||
const task = findCronTaskByBaseRunId(`cron:${job.id}:${now}`);
|
||||
expect(task).toMatchObject({ status: "succeeded" });
|
||||
expect(task?.detail).toEqual({ storeKey: cronStoreKey(storePath) });
|
||||
} finally {
|
||||
saveSpy.mockRestore();
|
||||
finalizeSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it("runs command cron jobs without isolated agent setup", async () => {
|
||||
const { storePath } = await makeStorePath();
|
||||
const now = Date.parse("2026-03-23T12:00:00.000Z");
|
||||
@@ -237,7 +346,12 @@ describe("cron service timer seam coverage", () => {
|
||||
const runIsolatedAgentJob = vi.fn(async () => ({
|
||||
status: "ok" as const,
|
||||
summary: "done",
|
||||
sessionId: "session-run-1",
|
||||
sessionKey: "agent:finn:cron:isolated-agent-job:run:run-1",
|
||||
delivery: { intended: { channel: "telegram", to: "42" } },
|
||||
model: "gpt-test",
|
||||
provider: "openai",
|
||||
usage: { input_tokens: 5, output_tokens: 3, total_tokens: 8 },
|
||||
}));
|
||||
|
||||
await writeCronStoreSnapshot({
|
||||
@@ -263,13 +377,24 @@ describe("cron service timer seam coverage", () => {
|
||||
message: "run isolated cron",
|
||||
}),
|
||||
);
|
||||
const task = findTaskByRunId(`cron:isolated-agent-job:${now}`);
|
||||
const task = findCronTaskByBaseRunId(`cron:isolated-agent-job:${now}`);
|
||||
if (!task) {
|
||||
throw new Error("expected isolated cron task ledger record");
|
||||
}
|
||||
expect(task.childSessionKey).toBe("agent:finn:cron:isolated-agent-job");
|
||||
expect(task.childSessionKey).toBe("agent:finn:cron:isolated-agent-job:run:run-1");
|
||||
expect(task.status).toBe("succeeded");
|
||||
expect(task.terminalSummary).toBe("done");
|
||||
expect(task.detail).toMatchObject({
|
||||
kind: "cron-run",
|
||||
status: "ok",
|
||||
sessionId: "session-run-1",
|
||||
durationMs: 0,
|
||||
nextRunAtMs: now + 60_000,
|
||||
delivery: { intended: { channel: "telegram", to: "42" } },
|
||||
model: "gpt-test",
|
||||
provider: "openai",
|
||||
usage: { input_tokens: 5, output_tokens: 3, total_tokens: 8 },
|
||||
});
|
||||
});
|
||||
|
||||
it("records current-bound cron task runs against the backing cron session", async () => {
|
||||
@@ -304,11 +429,11 @@ describe("cron service timer seam coverage", () => {
|
||||
|
||||
await onTimer(state);
|
||||
|
||||
const task = findTaskByRunId(`cron:isolated-agent-job:${now}`);
|
||||
const task = findCronTaskByBaseRunId(`cron:isolated-agent-job:${now}`);
|
||||
if (!task) {
|
||||
throw new Error("expected current-bound cron task ledger record");
|
||||
}
|
||||
expect(task.childSessionKey).toBe("agent:finn:cron:isolated-agent-job");
|
||||
expect(task.childSessionKey).toBe("agent:finn:cron:isolated-agent-job:run:run-1");
|
||||
expect(task.status).toBe("succeeded");
|
||||
});
|
||||
|
||||
@@ -345,7 +470,7 @@ describe("cron service timer seam coverage", () => {
|
||||
expect(runIsolatedAgentJob).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
const task = findTaskByRunId(`cron:isolated-agent-job:${now}`);
|
||||
const task = findCronTaskByBaseRunId(`cron:isolated-agent-job:${now}`);
|
||||
if (!task) {
|
||||
throw new Error("expected active cron task ledger record");
|
||||
}
|
||||
@@ -370,7 +495,7 @@ describe("cron service timer seam coverage", () => {
|
||||
});
|
||||
|
||||
const createTaskRecordSpy = vi
|
||||
.spyOn(detachedTaskRuntime, "createRunningTaskRun")
|
||||
.spyOn(taskExecutor, "createRunningTaskRun")
|
||||
.mockImplementation(() => {
|
||||
throw ledgerError;
|
||||
});
|
||||
|
||||
@@ -90,6 +90,7 @@ import {
|
||||
resolveMainSessionCronRunSessionKey,
|
||||
tryCreateCronTaskRun,
|
||||
tryFinishCronTaskRun,
|
||||
tryFinishCronTaskRunWithoutHistory,
|
||||
} from "./task-runs.js";
|
||||
import { resolveCronJobTimeoutMs } from "./timeout-policy.js";
|
||||
|
||||
@@ -117,6 +118,7 @@ type TimedCronRunOutcome = CronRunOutcome &
|
||||
delivered?: boolean;
|
||||
deliveryAttempted?: boolean;
|
||||
deliveryError?: string;
|
||||
delivery?: CronDeliveryTrace;
|
||||
isolatedAgentSetupTimeout?: IsolatedAgentSetupTimeoutSignal;
|
||||
activeJobMarker?: CronActiveJobMarker;
|
||||
startedAt: number;
|
||||
@@ -173,7 +175,7 @@ type ExecuteJobCoreOptions = {
|
||||
* Carries the already-resolved run attribution from watchdog-visible execution
|
||||
* state into a timer-built error outcome. The wall-clock/cancel paths return
|
||||
* their own outcome (the inner run result loses the Promise.race), so without
|
||||
* this the persisted cron_run_logs row drops provider/model/session for a
|
||||
* this the persisted cron run record drops provider/model/session for a
|
||||
* post-runner timeout or cancel even though they were already known. Stays
|
||||
* empty before the runner starts, so pre-execution setup timeouts read blank.
|
||||
*/
|
||||
@@ -712,6 +714,8 @@ export function applyJobResult(
|
||||
opts?: {
|
||||
// Preserve recurring "every" anchors for manual force runs.
|
||||
preserveSchedule?: boolean;
|
||||
// Startup replay restores alert cooldown bookkeeping without redelivery.
|
||||
replayFailureAlertAtMs?: number;
|
||||
},
|
||||
): boolean {
|
||||
const prevLastRunAtMs = job.state.lastRunAtMs;
|
||||
@@ -783,6 +787,9 @@ export function applyJobResult(
|
||||
error: result.error,
|
||||
provider: result.provider,
|
||||
consecutiveCount: job.state.consecutiveErrors,
|
||||
...(opts?.replayFailureAlertAtMs !== undefined
|
||||
? { delivery: "record-only" as const, occurredAtMs: opts.replayFailureAlertAtMs }
|
||||
: {}),
|
||||
});
|
||||
} else if (result.status === "skipped") {
|
||||
job.state.consecutiveErrors = 0;
|
||||
@@ -795,6 +802,9 @@ export function applyJobResult(
|
||||
error: result.error,
|
||||
provider: result.provider,
|
||||
consecutiveCount: job.state.consecutiveSkipped,
|
||||
...(opts?.replayFailureAlertAtMs !== undefined
|
||||
? { delivery: "record-only" as const, occurredAtMs: opts.replayFailureAlertAtMs }
|
||||
: {}),
|
||||
});
|
||||
} else {
|
||||
job.state.lastFailureAlertAtMs = undefined;
|
||||
@@ -1105,15 +1115,16 @@ function applyOutcomeToStoredJob(
|
||||
state: CronServiceState,
|
||||
result: TimedCronRunOutcome,
|
||||
): CronJob | undefined {
|
||||
tryFinishCronTaskRun(state, result);
|
||||
const store = state.store;
|
||||
if (!store) {
|
||||
tryFinishCronTaskRunWithoutHistory(state, result);
|
||||
return undefined;
|
||||
}
|
||||
const jobs = store.jobs;
|
||||
const job = jobs.find((entry) => entry.id === result.jobId);
|
||||
if (!job) {
|
||||
if (result.status === "ok" && result.triggerEval?.fired === false) {
|
||||
tryFinishCronTaskRunWithoutHistory(state, result);
|
||||
return undefined;
|
||||
}
|
||||
if (result.status === "ok") {
|
||||
@@ -1140,6 +1151,7 @@ function applyOutcomeToStoredJob(
|
||||
{ jobId: result.jobId },
|
||||
"cron: applyOutcomeToStoredJob — job not found after forceReload, result discarded",
|
||||
);
|
||||
tryFinishCronTaskRunWithoutHistory(state, result);
|
||||
return undefined;
|
||||
}
|
||||
|
||||
@@ -1177,6 +1189,17 @@ function applyOutcomeToStoredJob(
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function finishPersistedQuietCronTaskRuns(
|
||||
state: CronServiceState,
|
||||
outcomes: readonly TimedCronRunOutcome[],
|
||||
): void {
|
||||
for (const outcome of outcomes) {
|
||||
if (outcome.status === "ok" && outcome.triggerEval && !outcome.triggerEval.fired) {
|
||||
tryFinishCronTaskRunWithoutHistory(state, outcome);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function clearActiveMarkersForOutcomes(outcomes: readonly TimedCronRunOutcome[]): void {
|
||||
for (const outcome of outcomes) {
|
||||
clearCronJobActive(outcome.jobId, outcome.activeJobMarker);
|
||||
@@ -1197,7 +1220,7 @@ function finishRetiredCronTaskRuns(
|
||||
const current = new Set(currentOutcomes);
|
||||
for (const outcome of outcomes) {
|
||||
if (!current.has(outcome)) {
|
||||
tryFinishCronTaskRun(state, outcome);
|
||||
tryFinishCronTaskRunWithoutHistory(state, outcome);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1405,23 +1428,35 @@ async function onAdmittedTimer(state: CronServiceState) {
|
||||
}): Promise<TimedCronRunOutcome> => {
|
||||
const { id, job } = params;
|
||||
const startedAt = state.deps.nowMs();
|
||||
job.state.runningAtMs = startedAt;
|
||||
const executionJob = structuredClone(job);
|
||||
job.state.lastError = undefined;
|
||||
const activeJobMarker = markCronJobActive(job.id, {
|
||||
preserveAcrossGenerationAdvance: job.sessionTarget === "main",
|
||||
executionJob.state.runningAtMs = startedAt;
|
||||
executionJob.state.lastError = undefined;
|
||||
const activeJobMarker = markCronJobActive(executionJob.id, {
|
||||
preserveAcrossGenerationAdvance: executionJob.sessionTarget === "main",
|
||||
});
|
||||
emit(state, {
|
||||
jobId: executionJob.id,
|
||||
action: "started",
|
||||
job: executionJob,
|
||||
runAtMs: startedAt,
|
||||
});
|
||||
const jobTimeoutMs = resolveCronJobTimeoutMs(executionJob);
|
||||
const taskRunId = tryCreateCronTaskRun({
|
||||
state,
|
||||
job: executionJob,
|
||||
startedAt,
|
||||
runIdStartedAt: params.reservedAtMs,
|
||||
});
|
||||
emit(state, { jobId: job.id, action: "started", job, runAtMs: startedAt });
|
||||
const jobTimeoutMs = resolveCronJobTimeoutMs(job);
|
||||
const taskRunId = tryCreateCronTaskRun({ state, job, startedAt });
|
||||
|
||||
try {
|
||||
const result = await executeJobCoreWithTimeout(state, job, {
|
||||
const result = await executeJobCoreWithTimeout(state, executionJob, {
|
||||
runId: taskRunId,
|
||||
activeJobMarker,
|
||||
});
|
||||
return {
|
||||
jobId: id,
|
||||
job,
|
||||
job: executionJob,
|
||||
taskRunId,
|
||||
activeJobMarker,
|
||||
...result,
|
||||
@@ -1431,12 +1466,12 @@ async function onAdmittedTimer(state: CronServiceState) {
|
||||
} catch (err) {
|
||||
const errorText = normalizeCronRunErrorText(err);
|
||||
state.deps.log.warn(
|
||||
{ jobId: id, jobName: job.name, timeoutMs: jobTimeoutMs ?? null },
|
||||
{ jobId: id, jobName: executionJob.name, timeoutMs: jobTimeoutMs ?? null },
|
||||
`cron: job failed: ${errorText}`,
|
||||
);
|
||||
return {
|
||||
jobId: id,
|
||||
job,
|
||||
job: executionJob,
|
||||
taskRunId,
|
||||
activeJobMarker,
|
||||
status: "error",
|
||||
@@ -1488,6 +1523,7 @@ async function onAdmittedTimer(state: CronServiceState) {
|
||||
// daily cron schedules to jump 48 h instead of 24 h (#17852).
|
||||
recomputeNextRunsForMaintenance(state);
|
||||
await persistOrRestore(state, rollbackSnapshot);
|
||||
finishPersistedQuietCronTaskRuns(state, finalizedResults);
|
||||
for (const removedJob of removedJobs) {
|
||||
emit(state, { jobId: removedJob.id, action: "removed", job: removedJob });
|
||||
}
|
||||
@@ -1909,28 +1945,31 @@ async function runStartupCatchupCandidate(
|
||||
candidate: StartupCatchupCandidate,
|
||||
): Promise<TimedCronRunOutcome> {
|
||||
const startedAt = state.deps.nowMs();
|
||||
const executionJob = structuredClone(candidate.job);
|
||||
executionJob.state.runningAtMs = startedAt;
|
||||
const taskRunId = tryCreateCronTaskRun({
|
||||
state,
|
||||
job: candidate.job,
|
||||
job: executionJob,
|
||||
startedAt,
|
||||
runIdStartedAt: candidate.reservedAtMs,
|
||||
});
|
||||
const activeJobMarker = markCronJobActive(candidate.job.id, {
|
||||
preserveAcrossGenerationAdvance: candidate.job.sessionTarget === "main",
|
||||
const activeJobMarker = markCronJobActive(executionJob.id, {
|
||||
preserveAcrossGenerationAdvance: executionJob.sessionTarget === "main",
|
||||
});
|
||||
emit(state, {
|
||||
jobId: candidate.job.id,
|
||||
jobId: executionJob.id,
|
||||
action: "started",
|
||||
job: candidate.job,
|
||||
job: executionJob,
|
||||
runAtMs: startedAt,
|
||||
});
|
||||
try {
|
||||
const result = await executeJobCoreWithTimeout(state, candidate.job, {
|
||||
const result = await executeJobCoreWithTimeout(state, executionJob, {
|
||||
runId: taskRunId,
|
||||
activeJobMarker,
|
||||
});
|
||||
return {
|
||||
jobId: candidate.jobId,
|
||||
job: candidate.job,
|
||||
job: executionJob,
|
||||
taskRunId,
|
||||
activeJobMarker,
|
||||
status: result.status,
|
||||
@@ -1954,7 +1993,7 @@ async function runStartupCatchupCandidate(
|
||||
} catch (err) {
|
||||
return {
|
||||
jobId: candidate.jobId,
|
||||
job: candidate.job,
|
||||
job: executionJob,
|
||||
taskRunId,
|
||||
activeJobMarker,
|
||||
status: "error",
|
||||
@@ -2050,6 +2089,7 @@ async function applyStartupCatchupOutcomes(
|
||||
// startup overflow deferrals survive until their staggered catch-up tick.
|
||||
recomputeNextRunsForMaintenance(state, { repairFutureCronNextRunAtMs: false });
|
||||
await persistOrRestore(state, rollbackSnapshot);
|
||||
finishPersistedQuietCronTaskRuns(state, finalizedOutcomes);
|
||||
for (const removedJob of removedJobs) {
|
||||
emit(state, { jobId: removedJob.id, action: "removed", job: removedJob });
|
||||
}
|
||||
@@ -2424,16 +2464,10 @@ async function executeDetachedCronJob(
|
||||
function emitJobFinished(
|
||||
state: CronServiceState,
|
||||
job: CronJob,
|
||||
result: {
|
||||
status: CronRunStatus;
|
||||
delivered?: boolean;
|
||||
delivery?: CronDeliveryTrace;
|
||||
triggerEval?: CronTriggerEvalOutcome;
|
||||
} & CronRunOutcome &
|
||||
CronRunTelemetry,
|
||||
result: TimedCronRunOutcome,
|
||||
runAtMs: number,
|
||||
) {
|
||||
emit(state, {
|
||||
const event = {
|
||||
jobId: job.id,
|
||||
action: "finished",
|
||||
job,
|
||||
@@ -2455,7 +2489,14 @@ function emitJobFinished(
|
||||
model: result.model,
|
||||
provider: result.provider,
|
||||
usage: result.usage,
|
||||
} as const;
|
||||
tryFinishCronTaskRun(state, {
|
||||
taskRunId: result.taskRunId,
|
||||
job,
|
||||
event,
|
||||
...(result.triggerEval ? { triggerEval: result.triggerEval } : {}),
|
||||
});
|
||||
emit(state, event);
|
||||
}
|
||||
|
||||
/** Clears the currently armed cron timer. */
|
||||
|
||||
181
src/cron/task-run-detail.ts
Normal file
181
src/cron/task-run-detail.ts
Normal file
@@ -0,0 +1,181 @@
|
||||
/** Cron-owned codec between task-ledger detail and the stable run-history wire shape. */
|
||||
import { resolveFailoverReasonFromError } from "../agents/failover-error.js";
|
||||
import type { JsonValue, TaskRecord, TaskStatus } from "../tasks/task-registry.types.js";
|
||||
import type { CronRunLogEntry } from "./run-log-types.js";
|
||||
import { parseCronRunLogEntryObject } from "./run-log/entry-codec.js";
|
||||
import { timeoutErrorMessage } from "./service/execution-errors.js";
|
||||
import type { CronEvent } from "./service/state.js";
|
||||
import type { CronRunStatus } from "./types.js";
|
||||
|
||||
const CRON_TASK_DETAIL_KIND = "cron-run";
|
||||
|
||||
type CronFinishedEvent = CronEvent & { action: "finished" };
|
||||
|
||||
function toJsonValue(value: unknown): JsonValue | undefined {
|
||||
const serialized = JSON.stringify(value);
|
||||
return serialized === undefined ? undefined : (JSON.parse(serialized) as JsonValue);
|
||||
}
|
||||
|
||||
function isJsonObject(value: JsonValue | undefined): value is { [key: string]: JsonValue } {
|
||||
return value !== null && typeof value === "object" && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function isCronRunStatus(value: unknown): value is CronRunStatus {
|
||||
return value === "ok" || value === "error" || value === "skipped";
|
||||
}
|
||||
|
||||
/** Uses execution timing for one timestamp shared by ledger and legacy dual-write paths. */
|
||||
export function resolveCronRunEndedAt(event: CronFinishedEvent, fallbackTs: number): number {
|
||||
if (
|
||||
typeof event.runAtMs === "number" &&
|
||||
Number.isFinite(event.runAtMs) &&
|
||||
typeof event.durationMs === "number" &&
|
||||
Number.isFinite(event.durationMs)
|
||||
) {
|
||||
return event.runAtMs + event.durationMs;
|
||||
}
|
||||
return fallbackTs;
|
||||
}
|
||||
|
||||
/** Builds the legacy run-history record from one finished service event. */
|
||||
export function cronRunLogEntryFromEvent(
|
||||
event: CronFinishedEvent,
|
||||
fallbackTs: number,
|
||||
): CronRunLogEntry {
|
||||
const errorReason = resolveFailoverReasonFromError(event.error, event.provider) ?? undefined;
|
||||
return {
|
||||
ts: resolveCronRunEndedAt(event, fallbackTs),
|
||||
jobId: event.jobId,
|
||||
action: "finished",
|
||||
status: event.status,
|
||||
error: event.error,
|
||||
errorReason,
|
||||
summary: event.summary,
|
||||
diagnostics: event.diagnostics,
|
||||
delivered: event.delivered,
|
||||
deliveryStatus: event.deliveryStatus,
|
||||
deliveryError: event.deliveryError,
|
||||
failureNotificationDelivery: event.failureNotificationDelivery,
|
||||
delivery: event.delivery,
|
||||
sessionId: event.sessionId,
|
||||
sessionKey: event.sessionKey,
|
||||
runId: event.runId,
|
||||
runAtMs: event.runAtMs,
|
||||
durationMs: event.durationMs,
|
||||
nextRunAtMs: event.nextRunAtMs,
|
||||
triggerFired: event.triggerFired,
|
||||
model: event.model,
|
||||
provider: event.provider,
|
||||
usage: event.usage,
|
||||
};
|
||||
}
|
||||
|
||||
/** Encodes cron-only outcome fields; generic lifecycle fields stay on TaskRecord. */
|
||||
export function cronRunLogEntryToTaskDetail(
|
||||
entry: CronRunLogEntry,
|
||||
options: {
|
||||
storeKey: string;
|
||||
triggerEval?: { fired: boolean; stateChanged: boolean; state?: unknown };
|
||||
},
|
||||
): JsonValue {
|
||||
const detail = toJsonValue({
|
||||
kind: CRON_TASK_DETAIL_KIND,
|
||||
status: entry.status,
|
||||
storeKey: options.storeKey,
|
||||
errorReason: entry.errorReason,
|
||||
diagnostics: entry.diagnostics,
|
||||
delivered: entry.delivered,
|
||||
deliveryStatus: entry.deliveryStatus,
|
||||
deliveryError: entry.deliveryError,
|
||||
failureNotificationDelivery: entry.failureNotificationDelivery,
|
||||
delivery: entry.delivery,
|
||||
sessionId: entry.sessionId,
|
||||
// TaskRecord.runId remains the internal cancellation identity.
|
||||
runId: entry.runId,
|
||||
runAtMs: entry.runAtMs,
|
||||
durationMs: entry.durationMs,
|
||||
nextRunAtMs: entry.nextRunAtMs,
|
||||
triggerFired: entry.triggerFired,
|
||||
triggerStateChanged:
|
||||
options.triggerEval?.fired === true ? options.triggerEval.stateChanged : undefined,
|
||||
triggerState:
|
||||
options.triggerEval?.fired === true && options.triggerEval.stateChanged
|
||||
? options.triggerEval.state
|
||||
: undefined,
|
||||
model: entry.model,
|
||||
provider: entry.provider,
|
||||
usage: entry.usage,
|
||||
});
|
||||
return detail ?? { kind: CRON_TASK_DETAIL_KIND };
|
||||
}
|
||||
|
||||
/** Returns the cron store partition recorded on a task row. */
|
||||
export function cronTaskRecordStoreKey(task: TaskRecord): string | undefined {
|
||||
return isJsonObject(task.detail) && typeof task.detail.storeKey === "string"
|
||||
? task.detail.storeKey
|
||||
: undefined;
|
||||
}
|
||||
|
||||
/** Reads internal trigger recovery data without adding it to run-history responses. */
|
||||
export function cronTaskRecordToTriggerEval(
|
||||
task: TaskRecord,
|
||||
): { fired: true; stateChanged: boolean; state?: JsonValue } | undefined {
|
||||
if (!isJsonObject(task.detail) || task.detail.triggerFired !== true) {
|
||||
return undefined;
|
||||
}
|
||||
return {
|
||||
fired: true,
|
||||
stateChanged: task.detail.triggerStateChanged === true,
|
||||
...(task.detail.triggerStateChanged === true && "triggerState" in task.detail
|
||||
? { state: task.detail.triggerState }
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
|
||||
/** Maps the cron outcome vocabulary onto generic task terminal states. */
|
||||
export function cronRunStatusToTaskStatus(
|
||||
entry: CronRunLogEntry,
|
||||
): Extract<TaskStatus, "succeeded" | "failed" | "timed_out"> {
|
||||
if (entry.status === "ok" || entry.status === "skipped") {
|
||||
return "succeeded";
|
||||
}
|
||||
return entry.error === timeoutErrorMessage() ? "timed_out" : "failed";
|
||||
}
|
||||
|
||||
/** Reconstructs the unchanged CronRunLogEntry wire shape from a cron task row. */
|
||||
export function cronTaskRecordToRunLogEntry(task: TaskRecord): CronRunLogEntry | null {
|
||||
if (task.runtime !== "cron" || !task.sourceId || !isJsonObject(task.detail)) {
|
||||
return null;
|
||||
}
|
||||
if (task.detail.kind !== CRON_TASK_DETAIL_KIND || !isCronRunStatus(task.detail.status)) {
|
||||
return null;
|
||||
}
|
||||
const wireDetail = { ...task.detail };
|
||||
delete wireDetail.storeKey;
|
||||
const entry = parseCronRunLogEntryObject(
|
||||
{
|
||||
...wireDetail,
|
||||
ts: task.endedAt ?? task.lastEventAt ?? task.createdAt,
|
||||
jobId: task.sourceId,
|
||||
action: "finished",
|
||||
status: task.detail.status,
|
||||
error: task.error,
|
||||
summary: task.terminalSummary,
|
||||
sessionKey: task.childSessionKey,
|
||||
runId: typeof task.detail.runId === "string" ? task.detail.runId : undefined,
|
||||
},
|
||||
{ jobId: task.sourceId },
|
||||
);
|
||||
if (!entry) {
|
||||
return null;
|
||||
}
|
||||
// The legacy SQLite reader materializes these indexed columns even when absent.
|
||||
return {
|
||||
...entry,
|
||||
delivered: entry.delivered,
|
||||
deliveryStatus: entry.deliveryStatus,
|
||||
deliveryError: entry.deliveryError,
|
||||
sessionId: entry.sessionId,
|
||||
sessionKey: entry.sessionKey,
|
||||
};
|
||||
}
|
||||
466
src/cron/task-run-history.test.ts
Normal file
466
src/cron/task-run-history.test.ts
Normal file
@@ -0,0 +1,466 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { resetTaskRegistryForTests } from "../tasks/task-registry.js";
|
||||
import { saveTaskRegistryStateToSqlite } from "../tasks/task-registry.store.sqlite.js";
|
||||
import type { TaskRecord } from "../tasks/task-registry.types.js";
|
||||
import { withOpenClawTestState } from "../test-utils/openclaw-test-state.js";
|
||||
import type { CronRunLogEntry } from "./run-log-types.js";
|
||||
import { appendCronRunLog, readCronRunLogEntriesPage } from "./run-log.js";
|
||||
import { CronService } from "./service.js";
|
||||
import { createNoopLogger } from "./service.test-harness.js";
|
||||
import { cronStoreKey } from "./store/key.js";
|
||||
import {
|
||||
cronRunLogEntryFromEvent,
|
||||
cronRunLogEntryToTaskDetail,
|
||||
cronRunStatusToTaskStatus,
|
||||
cronTaskRecordToRunLogEntry,
|
||||
} from "./task-run-detail.js";
|
||||
import { readCronTaskRunHistoryPage } from "./task-run-history.js";
|
||||
|
||||
const JOB_ID = "history-job";
|
||||
|
||||
function taskFromEntry(entry: CronRunLogEntry, index: number, storeKey: string): TaskRecord {
|
||||
return {
|
||||
taskId: `task-${index}`,
|
||||
runtime: "cron",
|
||||
sourceId: entry.jobId,
|
||||
requesterSessionKey: "",
|
||||
ownerKey: "",
|
||||
scopeKind: "system",
|
||||
...(entry.sessionKey ? { childSessionKey: entry.sessionKey } : {}),
|
||||
agentId: "main",
|
||||
runId: `cron:${entry.jobId}:${entry.runAtMs ?? entry.ts}`,
|
||||
task: JOB_ID,
|
||||
status: cronRunStatusToTaskStatus(entry),
|
||||
deliveryStatus: "not_applicable",
|
||||
notifyPolicy: "silent",
|
||||
createdAt: entry.runAtMs ?? entry.ts,
|
||||
startedAt: entry.runAtMs,
|
||||
endedAt: entry.ts,
|
||||
lastEventAt: entry.ts,
|
||||
error: entry.error,
|
||||
terminalSummary: entry.summary,
|
||||
detail: cronRunLogEntryToTaskDetail(entry, { storeKey }),
|
||||
};
|
||||
}
|
||||
|
||||
function futureCronDetailTask(storeKey: string): TaskRecord {
|
||||
return {
|
||||
...taskFromEntry(
|
||||
{
|
||||
ts: 400,
|
||||
jobId: JOB_ID,
|
||||
action: "finished",
|
||||
status: "ok",
|
||||
runAtMs: 390,
|
||||
durationMs: 10,
|
||||
},
|
||||
4,
|
||||
storeKey,
|
||||
),
|
||||
taskId: "future-detail",
|
||||
detail: { kind: "future-cron-detail", status: "ok" },
|
||||
};
|
||||
}
|
||||
|
||||
describe("cron task run history", () => {
|
||||
it("matches legacy history for executions produced by the cron service", async () => {
|
||||
await withOpenClawTestState(
|
||||
{ layout: "state-only", prefix: "openclaw-cron-task-service-history-" },
|
||||
async (state) => {
|
||||
resetTaskRegistryForTests();
|
||||
const storePath = state.path("cron", "jobs.json");
|
||||
const legacyWrites: Promise<void>[] = [];
|
||||
let now = Date.parse("2026-07-12T12:00:00.000Z");
|
||||
const cron = new CronService({
|
||||
storePath,
|
||||
cronEnabled: true,
|
||||
cronConfig: { triggers: { enabled: true } },
|
||||
log: createNoopLogger(),
|
||||
nowMs: () => now,
|
||||
enqueueSystemEvent: vi.fn(),
|
||||
requestHeartbeat: vi.fn(),
|
||||
evaluateCronTrigger: vi.fn(async () => ({
|
||||
kind: "evaluated" as const,
|
||||
fire: true,
|
||||
})),
|
||||
runIsolatedAgentJob: vi.fn(async ({ job }) => {
|
||||
if (job.name === "error") {
|
||||
return { status: "error" as const, error: "provider overloaded" };
|
||||
}
|
||||
if (job.name === "timeout") {
|
||||
return { status: "error" as const, error: "cron: job execution timed out" };
|
||||
}
|
||||
if (job.name === "skipped") {
|
||||
return { status: "skipped" as const, error: "trigger condition not met" };
|
||||
}
|
||||
return {
|
||||
status: "ok" as const,
|
||||
summary: "delivered",
|
||||
delivered: true,
|
||||
deliveryAttempted: true,
|
||||
delivery: {
|
||||
intended: { channel: "telegram", to: "42" },
|
||||
resolved: { channel: "telegram", to: "42", ok: true },
|
||||
messageToolSentTo: [{ channel: "telegram", to: "42" }],
|
||||
delivered: true,
|
||||
},
|
||||
sessionId: "session-ok",
|
||||
sessionKey: "agent:main:cron:history:run:ok",
|
||||
model: "gpt-test",
|
||||
provider: "openai",
|
||||
usage: { input_tokens: 10, output_tokens: 5, total_tokens: 15 },
|
||||
};
|
||||
}),
|
||||
onEvent: (event) => {
|
||||
if (event.action === "finished") {
|
||||
legacyWrites.push(
|
||||
appendCronRunLog({
|
||||
storePath,
|
||||
entry: cronRunLogEntryFromEvent(event, Date.now()),
|
||||
}),
|
||||
);
|
||||
}
|
||||
},
|
||||
});
|
||||
try {
|
||||
await cron.start();
|
||||
for (const name of ["ok", "error", "timeout", "skipped"]) {
|
||||
const job = await cron.add({
|
||||
name,
|
||||
enabled: true,
|
||||
schedule: { kind: "every", everyMs: 60_000 },
|
||||
...(name === "ok" ? { trigger: { script: "true" } } : {}),
|
||||
sessionTarget: "isolated",
|
||||
wakeMode: "next-heartbeat",
|
||||
payload: { kind: "agentTurn", message: name },
|
||||
delivery:
|
||||
name === "ok"
|
||||
? { mode: "announce", channel: "telegram", to: "42" }
|
||||
: { mode: "none" },
|
||||
});
|
||||
if (name === "ok") {
|
||||
now = job.state.nextRunAtMs ?? now;
|
||||
}
|
||||
expect(await cron.run(job.id, name === "ok" ? "due" : "force")).toEqual({
|
||||
ok: true,
|
||||
ran: true,
|
||||
});
|
||||
now += 10_000;
|
||||
}
|
||||
await Promise.all(legacyWrites);
|
||||
|
||||
const ledger = readCronTaskRunHistoryPage({
|
||||
storeKey: cronStoreKey(storePath),
|
||||
limit: 50,
|
||||
sortDir: "asc",
|
||||
});
|
||||
const legacy = await readCronRunLogEntriesPage({
|
||||
storePath,
|
||||
limit: 50,
|
||||
sortDir: "asc",
|
||||
});
|
||||
expect(ledger).toEqual(legacy);
|
||||
expect(ledger.entries.map((entry) => entry.status)).toEqual([
|
||||
"ok",
|
||||
"error",
|
||||
"error",
|
||||
"skipped",
|
||||
]);
|
||||
expect(ledger.entries[0]).toMatchObject({
|
||||
deliveryStatus: "delivered",
|
||||
triggerFired: true,
|
||||
nextRunAtMs: expect.any(Number),
|
||||
model: "gpt-test",
|
||||
provider: "openai",
|
||||
usage: { input_tokens: 10, output_tokens: 5, total_tokens: 15 },
|
||||
});
|
||||
} finally {
|
||||
cron.stop();
|
||||
resetTaskRegistryForTests({ persist: false });
|
||||
}
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it("matches legacy run-log records across outcomes and telemetry", async () => {
|
||||
await withOpenClawTestState(
|
||||
{ layout: "state-only", prefix: "openclaw-cron-task-history-" },
|
||||
async (state) => {
|
||||
const storePath = state.path("jobs.json");
|
||||
const storeKey = cronStoreKey(storePath);
|
||||
const entries: CronRunLogEntry[] = [
|
||||
{
|
||||
ts: 1_100,
|
||||
jobId: JOB_ID,
|
||||
action: "finished",
|
||||
status: "ok",
|
||||
summary: "delivered\n needle",
|
||||
diagnostics: {
|
||||
summary: "healthy",
|
||||
entries: [
|
||||
{
|
||||
ts: 1_050,
|
||||
source: "agent-run",
|
||||
severity: "info",
|
||||
message: "diagnostic needle",
|
||||
},
|
||||
],
|
||||
},
|
||||
delivered: true,
|
||||
deliveryStatus: "delivered",
|
||||
failureNotificationDelivery: { status: "not-requested" },
|
||||
delivery: {
|
||||
intended: { channel: "telegram", to: "123" },
|
||||
resolved: { channel: "telegram", to: "123", ok: true },
|
||||
messageToolSentTo: [{ channel: "telegram", to: "123" }],
|
||||
delivered: true,
|
||||
},
|
||||
sessionId: "session-ok",
|
||||
sessionKey: "agent:main:cron:history:run:ok",
|
||||
runId: "manual:history:ok",
|
||||
runAtMs: 1_000,
|
||||
durationMs: 100,
|
||||
nextRunAtMs: 2_000,
|
||||
triggerFired: true,
|
||||
model: "gpt-test",
|
||||
provider: "openai",
|
||||
usage: {
|
||||
input_tokens: 10,
|
||||
output_tokens: 5,
|
||||
total_tokens: 15,
|
||||
cache_read_tokens: 2,
|
||||
cache_write_tokens: 1,
|
||||
},
|
||||
},
|
||||
{
|
||||
ts: 2_250,
|
||||
jobId: JOB_ID,
|
||||
action: "finished",
|
||||
status: "error",
|
||||
error: "provider overloaded",
|
||||
deliveryStatus: "not-delivered",
|
||||
deliveryError: "send failed",
|
||||
runId: "manual:history:error",
|
||||
runAtMs: 2_000,
|
||||
durationMs: 250,
|
||||
nextRunAtMs: 3_000,
|
||||
provider: "openai",
|
||||
},
|
||||
{
|
||||
ts: 3_500,
|
||||
jobId: JOB_ID,
|
||||
action: "finished",
|
||||
status: "error",
|
||||
error: "cron: job execution timed out",
|
||||
runId: "manual:history:timeout",
|
||||
runAtMs: 3_000,
|
||||
durationMs: 500,
|
||||
nextRunAtMs: 4_000,
|
||||
},
|
||||
{
|
||||
ts: 4_000,
|
||||
jobId: JOB_ID,
|
||||
action: "finished",
|
||||
status: "skipped",
|
||||
error: "trigger condition not met",
|
||||
summary: "",
|
||||
runId: "manual:history:skipped",
|
||||
runAtMs: 4_000,
|
||||
durationMs: 0,
|
||||
nextRunAtMs: 5_000,
|
||||
},
|
||||
];
|
||||
saveTaskRegistryStateToSqlite({
|
||||
tasks: new Map(
|
||||
entries.map((entry, index) => [`task-${index}`, taskFromEntry(entry, index, storeKey)]),
|
||||
),
|
||||
deliveryStates: new Map(),
|
||||
});
|
||||
for (const entry of entries) {
|
||||
await appendCronRunLog({ storePath, entry });
|
||||
}
|
||||
|
||||
const ledger = readCronTaskRunHistoryPage({ storeKey, jobId: JOB_ID, limit: 50 });
|
||||
const legacy = await readCronRunLogEntriesPage({
|
||||
storePath,
|
||||
jobId: JOB_ID,
|
||||
limit: 50,
|
||||
});
|
||||
expect(ledger).toEqual(legacy);
|
||||
expect(ledger.entries.map((entry) => entry.status)).toEqual([
|
||||
"skipped",
|
||||
"error",
|
||||
"error",
|
||||
"ok",
|
||||
]);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it("preserves paging and text-query filtering", async () => {
|
||||
await withOpenClawTestState(
|
||||
{ layout: "state-only", prefix: "openclaw-cron-task-history-page-" },
|
||||
async (state) => {
|
||||
const storeKey = cronStoreKey(state.path("jobs.json"));
|
||||
const entries: CronRunLogEntry[] = [
|
||||
{
|
||||
ts: 100,
|
||||
jobId: JOB_ID,
|
||||
action: "finished",
|
||||
status: "ok",
|
||||
summary: "first",
|
||||
runAtMs: 90,
|
||||
durationMs: 10,
|
||||
},
|
||||
{
|
||||
ts: 200,
|
||||
jobId: JOB_ID,
|
||||
action: "finished",
|
||||
status: "error",
|
||||
error: "needle failure",
|
||||
runAtMs: 180,
|
||||
durationMs: 20,
|
||||
},
|
||||
{
|
||||
ts: 300,
|
||||
jobId: JOB_ID,
|
||||
action: "finished",
|
||||
status: "skipped",
|
||||
summary: "third",
|
||||
runAtMs: 300,
|
||||
durationMs: 0,
|
||||
},
|
||||
];
|
||||
saveTaskRegistryStateToSqlite({
|
||||
tasks: new Map([
|
||||
...entries.map(
|
||||
(entry, index) => [`task-${index}`, taskFromEntry(entry, index, storeKey)] as const,
|
||||
),
|
||||
["future-detail", futureCronDetailTask(storeKey)] as const,
|
||||
[
|
||||
"other-store",
|
||||
{
|
||||
...taskFromEntry(
|
||||
{
|
||||
ts: 250,
|
||||
jobId: JOB_ID,
|
||||
action: "finished",
|
||||
status: "ok",
|
||||
summary: "foreign partition",
|
||||
},
|
||||
5,
|
||||
"/other/cron/store",
|
||||
),
|
||||
taskId: "other-store",
|
||||
},
|
||||
] as const,
|
||||
[
|
||||
"missing-store-key",
|
||||
{
|
||||
...taskFromEntry(entries[0], 6, storeKey),
|
||||
taskId: "missing-store-key",
|
||||
detail: { kind: "cron-run", status: "ok" },
|
||||
},
|
||||
] as const,
|
||||
]),
|
||||
deliveryStates: new Map(),
|
||||
});
|
||||
|
||||
expect(
|
||||
readCronTaskRunHistoryPage({ storeKey, jobId: JOB_ID, limit: 1, offset: 1 }),
|
||||
).toMatchObject({
|
||||
entries: [expect.objectContaining({ ts: 200 })],
|
||||
total: 3,
|
||||
offset: 1,
|
||||
limit: 1,
|
||||
hasMore: true,
|
||||
nextOffset: 2,
|
||||
});
|
||||
expect(
|
||||
readCronTaskRunHistoryPage({
|
||||
storeKey,
|
||||
jobId: JOB_ID,
|
||||
query: "needle",
|
||||
status: "error",
|
||||
limit: 50,
|
||||
}).entries,
|
||||
).toEqual([expect.objectContaining({ ts: 200, error: "needle failure" })]);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps same-job histories and totals scoped to one cron store", async () => {
|
||||
await withOpenClawTestState(
|
||||
{ layout: "state-only", prefix: "openclaw-cron-task-history-store-scope-" },
|
||||
async (state) => {
|
||||
const storeA = cronStoreKey(state.path("cron-a", "jobs.json"));
|
||||
const storeB = cronStoreKey(state.path("cron-b", "jobs.json"));
|
||||
const entryA: CronRunLogEntry = {
|
||||
ts: 100,
|
||||
jobId: JOB_ID,
|
||||
action: "finished",
|
||||
status: "ok",
|
||||
summary: "store a",
|
||||
};
|
||||
const entryB: CronRunLogEntry = {
|
||||
ts: 200,
|
||||
jobId: JOB_ID,
|
||||
action: "finished",
|
||||
status: "error",
|
||||
error: "store b",
|
||||
};
|
||||
saveTaskRegistryStateToSqlite({
|
||||
tasks: new Map([
|
||||
["store-a", { ...taskFromEntry(entryA, 1, storeA), taskId: "store-a" }],
|
||||
["store-b", { ...taskFromEntry(entryB, 2, storeB), taskId: "store-b" }],
|
||||
]),
|
||||
deliveryStates: new Map(),
|
||||
});
|
||||
|
||||
expect(readCronTaskRunHistoryPage({ storeKey: storeA, jobId: JOB_ID })).toMatchObject({
|
||||
entries: [expect.objectContaining({ summary: "store a" })],
|
||||
total: 1,
|
||||
hasMore: false,
|
||||
});
|
||||
expect(readCronTaskRunHistoryPage({ storeKey: storeB, jobId: JOB_ID })).toMatchObject({
|
||||
entries: [expect.objectContaining({ error: "store b" })],
|
||||
total: 1,
|
||||
hasMore: false,
|
||||
});
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps the internal store key out of the legacy wire record", () => {
|
||||
const storeKey = "/internal/cron/store";
|
||||
const task = taskFromEntry(
|
||||
{ ts: 100, jobId: JOB_ID, action: "finished", status: "ok" },
|
||||
1,
|
||||
storeKey,
|
||||
);
|
||||
const entry = cronTaskRecordToRunLogEntry(task);
|
||||
expect(entry).not.toBeNull();
|
||||
expect(Object.hasOwn(entry ?? {}, "storeKey")).toBe(false);
|
||||
});
|
||||
|
||||
it("locks the serialized detail shape: kind first, status second", () => {
|
||||
// External tooling may prefix-match serialized detail; keep the codec's
|
||||
// field order stable so those prefixes stay meaningful.
|
||||
for (const status of ["ok", "error", "skipped"] as const) {
|
||||
const detail = cronRunLogEntryToTaskDetail(
|
||||
{
|
||||
ts: 100,
|
||||
jobId: JOB_ID,
|
||||
action: "finished",
|
||||
status,
|
||||
},
|
||||
{ storeKey: "/tmp/cron-history" },
|
||||
);
|
||||
const serialized = JSON.stringify(detail);
|
||||
expect(
|
||||
serialized.startsWith(`{"kind":"cron-run","status":"${status}"`),
|
||||
`detail for status "${status}" must keep the stable prefix: ${serialized}`,
|
||||
).toBe(true);
|
||||
}
|
||||
});
|
||||
});
|
||||
180
src/cron/task-run-history.ts
Normal file
180
src/cron/task-run-history.ts
Normal file
@@ -0,0 +1,180 @@
|
||||
/** Cron run-history reads backed by authoritative task-ledger rows. */
|
||||
import {
|
||||
normalizeLowercaseStringOrEmpty,
|
||||
normalizeOptionalString,
|
||||
} from "@openclaw/normalization-core/string-coerce";
|
||||
import { uniqueValues } from "@openclaw/normalization-core/string-normalization";
|
||||
import { listTaskRegistryRecordsByRuntimeSourceIdFromSqlite } from "../tasks/task-registry.store.sqlite.js";
|
||||
import type { TaskRecord } from "../tasks/task-registry.types.js";
|
||||
import type { CronRunLogEntry } from "./run-log-types.js";
|
||||
import { cronTaskRecordStoreKey, cronTaskRecordToRunLogEntry } from "./task-run-detail.js";
|
||||
import type { CronDeliveryStatus, CronRunStatus } from "./types.js";
|
||||
|
||||
type CronRunHistorySortDir = "asc" | "desc";
|
||||
type CronRunHistoryStatusFilter = "all" | CronRunStatus;
|
||||
|
||||
export type ReadCronTaskRunHistoryPageOptions = {
|
||||
storeKey: string;
|
||||
limit?: number;
|
||||
offset?: number;
|
||||
jobId?: string;
|
||||
/** Narrows the page to these job ids (caller-scope filtering). */
|
||||
jobIds?: readonly string[];
|
||||
runId?: string;
|
||||
status?: CronRunHistoryStatusFilter;
|
||||
statuses?: CronRunStatus[];
|
||||
deliveryStatus?: CronDeliveryStatus;
|
||||
deliveryStatuses?: CronDeliveryStatus[];
|
||||
query?: string;
|
||||
sortDir?: CronRunHistorySortDir;
|
||||
jobNameById?: Record<string, string>;
|
||||
};
|
||||
|
||||
export type CronTaskRunHistoryPage = {
|
||||
entries: CronRunLogEntry[];
|
||||
total: number;
|
||||
offset: number;
|
||||
limit: number;
|
||||
hasMore: boolean;
|
||||
nextOffset: number | null;
|
||||
};
|
||||
|
||||
const INVALID_CRON_TASK_RUN_JOB_ID_MESSAGE = "invalid cron task run job id";
|
||||
|
||||
export function normalizeCronTaskRunJobId(jobId: string): string {
|
||||
const trimmed = jobId.trim();
|
||||
if (!trimmed || trimmed.includes("/") || trimmed.includes("\\") || trimmed.includes("\0")) {
|
||||
throw new Error(INVALID_CRON_TASK_RUN_JOB_ID_MESSAGE);
|
||||
}
|
||||
return trimmed;
|
||||
}
|
||||
|
||||
export function isInvalidCronTaskRunJobIdError(error: unknown): boolean {
|
||||
return error instanceof Error && error.message === INVALID_CRON_TASK_RUN_JOB_ID_MESSAGE;
|
||||
}
|
||||
|
||||
function normalizeStatuses(options: ReadCronTaskRunHistoryPageOptions): CronRunStatus[] | null {
|
||||
if (options.statuses?.length) {
|
||||
const statuses = options.statuses.filter(isCronRunStatus);
|
||||
if (statuses.length > 0) {
|
||||
return uniqueValues(statuses);
|
||||
}
|
||||
}
|
||||
return isCronRunStatus(options.status) ? [options.status] : null;
|
||||
}
|
||||
|
||||
function isCronRunStatus(value: unknown): value is CronRunStatus {
|
||||
return value === "ok" || value === "error" || value === "skipped";
|
||||
}
|
||||
|
||||
function isCronDeliveryStatus(value: unknown): value is CronDeliveryStatus {
|
||||
return (
|
||||
value === "delivered" ||
|
||||
value === "not-delivered" ||
|
||||
value === "unknown" ||
|
||||
value === "not-requested"
|
||||
);
|
||||
}
|
||||
|
||||
function normalizeDeliveryStatuses(
|
||||
options: ReadCronTaskRunHistoryPageOptions,
|
||||
): CronDeliveryStatus[] | null {
|
||||
if (options.deliveryStatuses?.length) {
|
||||
const statuses = options.deliveryStatuses.filter(isCronDeliveryStatus);
|
||||
if (statuses.length > 0) {
|
||||
return uniqueValues(statuses);
|
||||
}
|
||||
}
|
||||
return isCronDeliveryStatus(options.deliveryStatus) ? [options.deliveryStatus] : null;
|
||||
}
|
||||
|
||||
function queryText(entry: CronRunLogEntry, jobNameById?: Record<string, string>): string {
|
||||
return [
|
||||
entry.summary ?? "",
|
||||
entry.error ?? "",
|
||||
entry.errorReason ?? "",
|
||||
entry.diagnostics?.summary ?? "",
|
||||
...(entry.diagnostics?.entries ?? []).map((diagnostic) => diagnostic.message),
|
||||
entry.jobId,
|
||||
jobNameById?.[entry.jobId] ?? "",
|
||||
entry.delivery?.intended?.channel ?? "",
|
||||
entry.delivery?.resolved?.channel ?? "",
|
||||
...(entry.delivery?.messageToolSentTo ?? []).map((target) => target.channel),
|
||||
].join(" ");
|
||||
}
|
||||
|
||||
function compareHistoryRows(
|
||||
left: { entry: CronRunLogEntry; task: TaskRecord },
|
||||
right: { entry: CronRunLogEntry; task: TaskRecord },
|
||||
direction: CronRunHistorySortDir,
|
||||
): number {
|
||||
const multiplier = direction === "asc" ? 1 : -1;
|
||||
return (
|
||||
multiplier * (left.entry.ts - right.entry.ts) ||
|
||||
multiplier * (left.task.createdAt - right.task.createdAt) ||
|
||||
multiplier * left.task.taskId.localeCompare(right.task.taskId)
|
||||
);
|
||||
}
|
||||
|
||||
function attachJobNames(entries: CronRunLogEntry[], jobNameById?: Record<string, string>): void {
|
||||
for (const entry of entries) {
|
||||
const jobName = jobNameById?.[entry.jobId];
|
||||
if (jobName) {
|
||||
(entry as CronRunLogEntry & { jobName?: string }).jobName = jobName;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Reads and filters cron task rows with the legacy run-history paging contract. */
|
||||
export function readCronTaskRunHistoryPage(
|
||||
options: ReadCronTaskRunHistoryPageOptions,
|
||||
): CronTaskRunHistoryPage {
|
||||
const jobId = options.jobId ? normalizeCronTaskRunJobId(options.jobId) : undefined;
|
||||
const limit = Math.max(1, Math.min(200, Math.floor(options.limit ?? 50)));
|
||||
const offset = Math.max(0, Math.floor(options.offset ?? 0));
|
||||
const statuses = normalizeStatuses(options);
|
||||
const deliveryStatuses = normalizeDeliveryStatuses(options);
|
||||
const runId = normalizeOptionalString(options.runId);
|
||||
const jobIds = options.jobIds ? new Set(options.jobIds) : undefined;
|
||||
const query = normalizeLowercaseStringOrEmpty(options.query);
|
||||
const sortDir: CronRunHistorySortDir = options.sortDir === "asc" ? "asc" : "desc";
|
||||
const rows = listTaskRegistryRecordsByRuntimeSourceIdFromSqlite({
|
||||
runtime: "cron",
|
||||
sourceId: jobId,
|
||||
})
|
||||
.filter((task) => cronTaskRecordStoreKey(task) === options.storeKey)
|
||||
.map((task) => ({ task, entry: cronTaskRecordToRunLogEntry(task) }))
|
||||
.filter((row): row is { task: TaskRecord; entry: CronRunLogEntry } => row.entry !== null)
|
||||
.filter(({ entry }) => {
|
||||
if (jobIds && !jobIds.has(entry.jobId)) {
|
||||
return false;
|
||||
}
|
||||
if (runId && entry.runId !== runId) {
|
||||
return false;
|
||||
}
|
||||
if (statuses && (!entry.status || !statuses.includes(entry.status))) {
|
||||
return false;
|
||||
}
|
||||
if (deliveryStatuses && !deliveryStatuses.includes(entry.deliveryStatus ?? "not-requested")) {
|
||||
return false;
|
||||
}
|
||||
return (
|
||||
!query ||
|
||||
normalizeLowercaseStringOrEmpty(queryText(entry, options.jobNameById)).includes(query)
|
||||
);
|
||||
})
|
||||
.toSorted((left, right) => compareHistoryRows(left, right, sortDir));
|
||||
const total = rows.length;
|
||||
const boundedOffset = Math.min(total, offset);
|
||||
const entries = rows.slice(boundedOffset, boundedOffset + limit).map(({ entry }) => entry);
|
||||
attachJobNames(entries, options.jobNameById);
|
||||
const nextOffset = boundedOffset + entries.length;
|
||||
return {
|
||||
entries,
|
||||
total,
|
||||
offset: boundedOffset,
|
||||
limit,
|
||||
hasMore: nextOffset < total,
|
||||
nextOffset: nextOffset < total ? nextOffset : null,
|
||||
};
|
||||
}
|
||||
@@ -31,6 +31,7 @@ import {
|
||||
resolveCronSessionTargetSessionKey,
|
||||
} from "../cron/session-target.js";
|
||||
import { resolveCronJobsStorePath } from "../cron/store.js";
|
||||
import { cronRunLogEntryFromEvent } from "../cron/task-run-detail.js";
|
||||
import { createCronTriggerEvaluator } from "../cron/trigger-script.js";
|
||||
import type { CronJob, CronPayload } from "../cron/types.js";
|
||||
import { formatErrorMessage } from "../infra/errors.js";
|
||||
@@ -784,6 +785,7 @@ export function buildGatewayCronService(params: {
|
||||
}
|
||||
if (evt.action === "finished") {
|
||||
const job = evt.job ?? cron.getJob(evt.jobId);
|
||||
const runLogEntry = cronRunLogEntryFromEvent({ ...evt, action: "finished" }, Date.now());
|
||||
dispatchGatewayCronFinishedNotifications({
|
||||
evt,
|
||||
job,
|
||||
@@ -797,30 +799,7 @@ export function buildGatewayCronService(params: {
|
||||
void runWithGatewayIndependentRootWorkAdmission(async () => {
|
||||
await appendCronRunLog({
|
||||
storePath,
|
||||
entry: {
|
||||
ts: Date.now(),
|
||||
jobId: evt.jobId,
|
||||
action: "finished",
|
||||
status: evt.status,
|
||||
error: evt.error,
|
||||
summary: evt.summary,
|
||||
diagnostics: evt.diagnostics,
|
||||
delivered: evt.delivered,
|
||||
deliveryStatus: evt.deliveryStatus,
|
||||
deliveryError: evt.deliveryError,
|
||||
failureNotificationDelivery: evt.failureNotificationDelivery,
|
||||
delivery: evt.delivery,
|
||||
sessionId: evt.sessionId,
|
||||
sessionKey: evt.sessionKey,
|
||||
runId: evt.runId,
|
||||
runAtMs: evt.runAtMs,
|
||||
durationMs: evt.durationMs,
|
||||
nextRunAtMs: evt.nextRunAtMs,
|
||||
triggerFired: evt.triggerFired,
|
||||
model: evt.model,
|
||||
provider: evt.provider,
|
||||
usage: evt.usage,
|
||||
},
|
||||
entry: runLogEntry,
|
||||
opts: { keepLines: runLogPrune.keepLines },
|
||||
});
|
||||
}).catch((err: unknown) => {
|
||||
|
||||
@@ -23,11 +23,6 @@ import {
|
||||
import { resolveCronDeliveryPreviews } from "../../cron/delivery-preview.js";
|
||||
import { assertCronDeliveryInputNonBlankFields } from "../../cron/delivery-target-validation.js";
|
||||
import { normalizeCronJobCreate, normalizeCronJobPatch } from "../../cron/normalize.js";
|
||||
import {
|
||||
isInvalidCronRunLogJobIdError,
|
||||
readCronRunLogEntriesPage,
|
||||
readCronRunLogEntriesPageAll,
|
||||
} from "../../cron/run-log.js";
|
||||
import { applyJobPatch } from "../../cron/service/jobs.js";
|
||||
import type {
|
||||
CronListPageOptions,
|
||||
@@ -37,6 +32,11 @@ import {
|
||||
isInvalidCronSessionTargetIdError,
|
||||
resolveCronSessionTargetSessionKey,
|
||||
} from "../../cron/session-target.js";
|
||||
import { cronStoreKey } from "../../cron/store/key.js";
|
||||
import {
|
||||
isInvalidCronTaskRunJobIdError,
|
||||
readCronTaskRunHistoryPage,
|
||||
} from "../../cron/task-run-history.js";
|
||||
import type { CronJob, CronJobCreate, CronJobPatch } from "../../cron/types.js";
|
||||
import { validateScheduleTimestamp } from "../../cron/validate-timestamp.js";
|
||||
import { formatErrorMessage } from "../../infra/errors.js";
|
||||
@@ -940,8 +940,8 @@ export const cronHandlers: GatewayRequestHandlers = {
|
||||
.filter((job) => typeof job.id === "string" && typeof job.name === "string")
|
||||
.map((job) => [job.id, job.name]),
|
||||
);
|
||||
const page = await readCronRunLogEntriesPageAll({
|
||||
storePath: context.cronStorePath,
|
||||
const page = readCronTaskRunHistoryPage({
|
||||
storeKey: cronStoreKey(context.cronStorePath),
|
||||
...cronRunLogPageFilters(p),
|
||||
...(p.agentId ? { jobIds: jobs.map((job) => job.id) } : {}),
|
||||
jobNameById,
|
||||
@@ -972,15 +972,15 @@ export const cronHandlers: GatewayRequestHandlers = {
|
||||
matchedJob && typeof matchedJob.name === "string"
|
||||
? { [jobId as string]: matchedJob.name }
|
||||
: undefined;
|
||||
const page = await readCronRunLogEntriesPage({
|
||||
storePath: context.cronStorePath,
|
||||
const page = readCronTaskRunHistoryPage({
|
||||
storeKey: cronStoreKey(context.cronStorePath),
|
||||
jobId: jobId as string,
|
||||
...cronRunLogPageFilters(p),
|
||||
jobNameById,
|
||||
});
|
||||
respond(true, page, undefined);
|
||||
} catch (err) {
|
||||
if (!isInvalidCronRunLogJobIdError(err)) {
|
||||
if (!isInvalidCronTaskRunJobIdError(err)) {
|
||||
throw err;
|
||||
}
|
||||
respond(
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
// Starts discovery, remote skills, task maintenance, and delayed maintenance setup.
|
||||
import type { GatewayTailscaleMode } from "../config/types.gateway.js";
|
||||
import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
import { resolveCronJobsStorePath } from "../cron/store.js";
|
||||
import type { PluginRegistry } from "../plugins/registry-types.js";
|
||||
|
||||
type Awaitable<T> = T | Promise<T>;
|
||||
@@ -133,9 +132,8 @@ export async function startGatewayEarlyRuntime(params: {
|
||||
setSkillsRemoteRegistry(params.nodeRegistry);
|
||||
void primeRemoteSkillsCache();
|
||||
// Task registry maintenance is authoritative in the Gateway process so
|
||||
// restart-blocker counts reflect the same cron store as runtime execution.
|
||||
// restart-blocker counts reflect the same live cron runtime.
|
||||
taskRegistryMaintenance.configureTaskRegistryMaintenance({
|
||||
cronStorePath: resolveCronJobsStorePath(params.cfgAtStart.cron?.store),
|
||||
runtimeAuthoritative: true,
|
||||
});
|
||||
taskRegistryMaintenance.startTaskRegistryMaintenance();
|
||||
|
||||
186
src/infra/state-migrations.cron-run-logs.test.ts
Normal file
186
src/infra/state-migrations.cron-run-logs.test.ts
Normal file
@@ -0,0 +1,186 @@
|
||||
import { DatabaseSync } from "node:sqlite";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { CronRunLogEntry } from "../cron/run-log-types.js";
|
||||
import { readCronRunLogEntriesPage } from "../cron/run-log.js";
|
||||
import { insertCronRunLogEntry } from "../cron/run-log/sqlite-store.js";
|
||||
import { cronStoreKey } from "../cron/store/key.js";
|
||||
import { cronRunLogEntryToTaskDetail, cronRunStatusToTaskStatus } from "../cron/task-run-detail.js";
|
||||
import { readCronTaskRunHistoryPage } from "../cron/task-run-history.js";
|
||||
import { resetTaskRegistryForTests } from "../tasks/task-registry.js";
|
||||
import { withOpenClawTestState } from "../test-utils/openclaw-test-state.js";
|
||||
import {
|
||||
closeOpenClawStateDatabaseForTest,
|
||||
openOpenClawStateDatabase,
|
||||
} from "../state/openclaw-state-db.js";
|
||||
import {
|
||||
CRON_RUN_LOG_TASK_IMPORT_MIGRATION_ID,
|
||||
migrateLegacyCronRunLogsToTaskRuns,
|
||||
} from "./state-migrations.cron-run-logs.js";
|
||||
|
||||
describe("cron run-log task import", () => {
|
||||
it("imports legacy cron history into task runs once at state database open", async () => {
|
||||
await withOpenClawTestState(
|
||||
{ layout: "state-only", prefix: "openclaw-cron-run-log-import-" },
|
||||
async (state) => {
|
||||
const storePath = state.path("cron", "jobs.json");
|
||||
const storeKey = cronStoreKey(storePath);
|
||||
const jobId = "legacy-history-job";
|
||||
const entries: CronRunLogEntry[] = [
|
||||
{
|
||||
ts: 1_100,
|
||||
jobId,
|
||||
action: "finished",
|
||||
status: "ok",
|
||||
summary: "legacy one",
|
||||
sessionKey: "agent:main:cron:legacy:run:1",
|
||||
runId: "manual:legacy:1",
|
||||
runAtMs: 1_000,
|
||||
durationMs: 100,
|
||||
},
|
||||
{
|
||||
ts: 2_100,
|
||||
jobId,
|
||||
action: "finished",
|
||||
status: "error",
|
||||
error: "legacy failure",
|
||||
runAtMs: 2_000,
|
||||
durationMs: 100,
|
||||
},
|
||||
{
|
||||
ts: 2_100,
|
||||
jobId,
|
||||
action: "finished",
|
||||
status: "ok",
|
||||
summary: "same millisecond legacy run",
|
||||
runAtMs: 2_001,
|
||||
durationMs: 99,
|
||||
},
|
||||
{
|
||||
ts: 3_100,
|
||||
jobId,
|
||||
action: "finished",
|
||||
status: "error",
|
||||
error: "different public run id",
|
||||
runId: "manual:legacy:same-ts",
|
||||
runAtMs: 3_000,
|
||||
durationMs: 100,
|
||||
},
|
||||
{
|
||||
ts: 3_100,
|
||||
jobId,
|
||||
action: "finished",
|
||||
status: "skipped",
|
||||
runId: "manual:mirrored:3",
|
||||
runAtMs: 3_001,
|
||||
durationMs: 99,
|
||||
},
|
||||
{
|
||||
ts: 4_100,
|
||||
jobId,
|
||||
action: "finished",
|
||||
status: "ok",
|
||||
summary: "mirrored without public run id",
|
||||
runAtMs: 4_000,
|
||||
durationMs: 100,
|
||||
},
|
||||
];
|
||||
|
||||
const initial = openOpenClawStateDatabase();
|
||||
const databasePath = initial.path;
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
const fixture = new DatabaseSync(databasePath);
|
||||
try {
|
||||
fixture
|
||||
.prepare("DELETE FROM migration_runs WHERE id = ?")
|
||||
.run(CRON_RUN_LOG_TASK_IMPORT_MIGRATION_ID);
|
||||
for (const entry of entries) {
|
||||
insertCronRunLogEntry(fixture, storeKey, entry);
|
||||
}
|
||||
const insertMirrored = fixture.prepare(
|
||||
`INSERT INTO task_runs (
|
||||
task_id, runtime, source_id, requester_session_key, owner_key, scope_kind,
|
||||
child_session_key, run_id, task, status, delivery_status, notify_policy, created_at,
|
||||
started_at, ended_at, last_event_at, error, terminal_summary, terminal_outcome,
|
||||
detail_json
|
||||
) VALUES (?, 'cron', ?, '', '', 'system', ?, ?, ?, ?, 'not_applicable', 'silent',
|
||||
?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
);
|
||||
for (const [index, mirrored] of entries.slice(4).entries()) {
|
||||
const mirroredStatus = cronRunStatusToTaskStatus(mirrored);
|
||||
insertMirrored.run(
|
||||
`already-mirrored-${index}`,
|
||||
jobId,
|
||||
mirrored.sessionKey ?? null,
|
||||
`cron:legacy-history-job:${mirrored.runAtMs}:mirrored`,
|
||||
jobId,
|
||||
mirroredStatus,
|
||||
mirrored.runAtMs ?? mirrored.ts,
|
||||
mirrored.runAtMs ?? null,
|
||||
mirrored.ts,
|
||||
mirrored.ts,
|
||||
mirrored.error ?? null,
|
||||
mirrored.summary ?? null,
|
||||
mirroredStatus === "succeeded" ? "succeeded" : null,
|
||||
JSON.stringify(cronRunLogEntryToTaskDetail(mirrored, { storeKey })),
|
||||
);
|
||||
}
|
||||
fixture
|
||||
.prepare(
|
||||
`INSERT INTO cron_run_logs
|
||||
(store_key, job_id, seq, ts, entry_json, created_at)
|
||||
VALUES (?, ?, 7, 5100, '{', 5100)`,
|
||||
)
|
||||
.run(storeKey, jobId);
|
||||
} finally {
|
||||
fixture.close();
|
||||
}
|
||||
|
||||
const reopened = openOpenClawStateDatabase();
|
||||
const report = reopened.db
|
||||
.prepare("SELECT report_json FROM migration_runs WHERE id = ?")
|
||||
.get(CRON_RUN_LOG_TASK_IMPORT_MIGRATION_ID) as { report_json: string };
|
||||
expect(JSON.parse(report.report_json)).toEqual({
|
||||
imported: 4,
|
||||
alreadyMirrored: 2,
|
||||
malformed: 1,
|
||||
skipped: false,
|
||||
});
|
||||
const ledgerEntries = readCronTaskRunHistoryPage({
|
||||
storeKey,
|
||||
jobId,
|
||||
limit: 50,
|
||||
sortDir: "asc",
|
||||
}).entries;
|
||||
const legacyEntries = (
|
||||
await readCronRunLogEntriesPage({ storePath, jobId, limit: 50, sortDir: "asc" })
|
||||
).entries;
|
||||
expect(ledgerEntries).toEqual(legacyEntries);
|
||||
const imported = reopened.db
|
||||
.prepare(
|
||||
"SELECT task_id, cleanup_after FROM task_runs WHERE task_id LIKE 'cron-runlog-import:%' ORDER BY task_id",
|
||||
)
|
||||
.all() as Array<{ task_id: string; cleanup_after: number | null }>;
|
||||
expect(imported).toHaveLength(4);
|
||||
expect(imported.map((row) => row.task_id)).toEqual([
|
||||
expect.stringMatching(/^cron-runlog-import:[a-f0-9]{16}:legacy-history-job:1100:1$/u),
|
||||
expect.stringMatching(/^cron-runlog-import:[a-f0-9]{16}:legacy-history-job:2100:2$/u),
|
||||
expect.stringMatching(/^cron-runlog-import:[a-f0-9]{16}:legacy-history-job:2100:3$/u),
|
||||
expect.stringMatching(/^cron-runlog-import:[a-f0-9]{16}:legacy-history-job:3100:4$/u),
|
||||
]);
|
||||
expect(imported.every((row) => row.cleanup_after === null)).toBe(true);
|
||||
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
const secondOpen = openOpenClawStateDatabase();
|
||||
expect(secondOpen.db.prepare("SELECT COUNT(*) AS count FROM task_runs").get()).toEqual({
|
||||
count: 6,
|
||||
});
|
||||
expect(
|
||||
secondOpen.db
|
||||
.prepare("SELECT report_json FROM migration_runs WHERE id = ?")
|
||||
.get(CRON_RUN_LOG_TASK_IMPORT_MIGRATION_ID),
|
||||
).toEqual({ report_json: report.report_json });
|
||||
resetTaskRegistryForTests({ persist: false });
|
||||
},
|
||||
);
|
||||
});
|
||||
});
|
||||
170
src/infra/state-migrations.cron-run-logs.ts
Normal file
170
src/infra/state-migrations.cron-run-logs.ts
Normal file
@@ -0,0 +1,170 @@
|
||||
/** One-shot import of legacy cron run history into the authoritative task ledger. */
|
||||
import type { DatabaseSync } from "node:sqlite";
|
||||
import type { Selectable } from "kysely";
|
||||
import { parseStoredRunLogEntry } from "../cron/run-log/sqlite-store.js";
|
||||
import { cronRunLogEntryToTaskDetail, cronRunStatusToTaskStatus } from "../cron/task-run-detail.js";
|
||||
import type { DB as OpenClawStateDatabase } from "../state/openclaw-state-db.generated.js";
|
||||
import { sha256HexPrefix } from "./crypto-digest.js";
|
||||
|
||||
export const CRON_RUN_LOG_TASK_IMPORT_MIGRATION_ID = "state:cron-run-logs-to-task-runs:v1";
|
||||
|
||||
type CronRunLogRow = Selectable<OpenClawStateDatabase["cron_run_logs"]>;
|
||||
type MirroredTask = {
|
||||
source_id: string | null;
|
||||
ended_at: number | null;
|
||||
detail_json: string | null;
|
||||
};
|
||||
type MirroredIdentity = { endedAt: number | null; runId?: string };
|
||||
|
||||
export type CronRunLogTaskImportResult = {
|
||||
imported: number;
|
||||
alreadyMirrored: number;
|
||||
malformed: number;
|
||||
skipped: boolean;
|
||||
};
|
||||
|
||||
function parseDetail(raw: string | null): Record<string, unknown> | undefined {
|
||||
if (!raw) {
|
||||
return undefined;
|
||||
}
|
||||
try {
|
||||
const parsed: unknown = JSON.parse(raw);
|
||||
return parsed !== null && typeof parsed === "object" && !Array.isArray(parsed)
|
||||
? (parsed as Record<string, unknown>)
|
||||
: undefined;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function mirroredKey(sourceId: string, storeKey: string): string {
|
||||
return `${sourceId}\0${storeKey}`;
|
||||
}
|
||||
|
||||
function collectMirroredTasks(db: DatabaseSync): Map<string, MirroredIdentity[]> {
|
||||
const rows = db
|
||||
.prepare(
|
||||
`SELECT source_id, ended_at, detail_json
|
||||
FROM task_runs
|
||||
WHERE runtime = 'cron' AND source_id IS NOT NULL AND detail_json IS NOT NULL`,
|
||||
)
|
||||
.all() as MirroredTask[];
|
||||
const bySource = new Map<string, MirroredIdentity[]>();
|
||||
for (const row of rows) {
|
||||
const detail = parseDetail(row.detail_json);
|
||||
const storeKey = typeof detail?.storeKey === "string" ? detail.storeKey : undefined;
|
||||
if (!row.source_id || !storeKey) {
|
||||
continue;
|
||||
}
|
||||
const key = mirroredKey(row.source_id, storeKey);
|
||||
const identities = bySource.get(key) ?? [];
|
||||
identities.push({
|
||||
endedAt: row.ended_at,
|
||||
...(typeof detail.runId === "string" ? { runId: detail.runId } : {}),
|
||||
});
|
||||
bySource.set(key, identities);
|
||||
}
|
||||
return bySource;
|
||||
}
|
||||
|
||||
function consumeMirroredIdentity(
|
||||
identities: MirroredIdentity[],
|
||||
runId: string | undefined,
|
||||
endedAt: number,
|
||||
): boolean {
|
||||
let index = runId ? identities.findIndex((identity) => identity.runId === runId) : -1;
|
||||
if (index < 0) {
|
||||
index = identities.findIndex(
|
||||
(identity) => identity.runId === undefined && identity.endedAt === endedAt,
|
||||
);
|
||||
}
|
||||
if (index < 0 && !runId) {
|
||||
index = identities.findIndex((identity) => identity.endedAt === endedAt);
|
||||
}
|
||||
if (index < 0) {
|
||||
return false;
|
||||
}
|
||||
identities.splice(index, 1);
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Runs inside the state schema transaction; completed receipt makes later opens no-ops. */
|
||||
export function migrateLegacyCronRunLogsToTaskRuns(db: DatabaseSync): CronRunLogTaskImportResult {
|
||||
const receipt = db
|
||||
.prepare("SELECT status FROM migration_runs WHERE id = ?")
|
||||
.get(CRON_RUN_LOG_TASK_IMPORT_MIGRATION_ID) as { status?: unknown } | undefined;
|
||||
if (receipt?.status === "completed") {
|
||||
return { imported: 0, alreadyMirrored: 0, malformed: 0, skipped: true };
|
||||
}
|
||||
|
||||
const rows = db
|
||||
.prepare("SELECT * FROM cron_run_logs ORDER BY store_key, job_id, seq")
|
||||
.all() as CronRunLogRow[];
|
||||
const mirrored = collectMirroredTasks(db);
|
||||
// Leave cleanup unset; follow-up source-aware retention owns imported cron history.
|
||||
const insert = db.prepare(`
|
||||
INSERT INTO task_runs (
|
||||
task_id, runtime, task_kind, source_id, requester_session_key, owner_key, scope_kind,
|
||||
child_session_key, parent_flow_id, parent_task_id, agent_id, requester_agent_id, run_id,
|
||||
label, task, status, delivery_status, notify_policy, created_at, started_at, ended_at,
|
||||
last_event_at, cleanup_after, error, progress_summary, terminal_summary, terminal_outcome,
|
||||
detail_json
|
||||
) VALUES (
|
||||
@task_id, 'cron', NULL, @source_id, '', '', 'system', @child_session_key, NULL, NULL,
|
||||
NULL, NULL, @run_id, NULL, @task, @status, 'not_applicable', 'silent', @created_at,
|
||||
@started_at, @ended_at, @ended_at, NULL, @error, NULL, @terminal_summary,
|
||||
@terminal_outcome, @detail_json
|
||||
)
|
||||
`);
|
||||
let imported = 0;
|
||||
let alreadyMirroredCount = 0;
|
||||
let malformed = 0;
|
||||
for (const row of rows) {
|
||||
const entry = parseStoredRunLogEntry(row);
|
||||
if (!entry) {
|
||||
malformed++;
|
||||
continue;
|
||||
}
|
||||
const key = mirroredKey(row.job_id, row.store_key);
|
||||
const identities = mirrored.get(key) ?? [];
|
||||
if (consumeMirroredIdentity(identities, entry.runId, entry.ts)) {
|
||||
alreadyMirroredCount++;
|
||||
continue;
|
||||
}
|
||||
const taskId = `cron-runlog-import:${sha256HexPrefix(row.store_key, 16)}:${row.job_id}:${entry.ts}:${String(row.seq)}`;
|
||||
const status = cronRunStatusToTaskStatus(entry);
|
||||
insert.run({
|
||||
task_id: taskId,
|
||||
source_id: row.job_id,
|
||||
child_session_key: entry.sessionKey ?? null,
|
||||
run_id: taskId,
|
||||
task: row.job_id,
|
||||
status,
|
||||
created_at: entry.runAtMs ?? entry.ts,
|
||||
started_at: entry.runAtMs ?? null,
|
||||
ended_at: entry.ts,
|
||||
error: entry.error ?? null,
|
||||
terminal_summary: entry.summary ?? null,
|
||||
terminal_outcome: status === "succeeded" ? "succeeded" : null,
|
||||
detail_json: JSON.stringify(cronRunLogEntryToTaskDetail(entry, { storeKey: row.store_key })),
|
||||
});
|
||||
imported++;
|
||||
}
|
||||
|
||||
const result = {
|
||||
imported,
|
||||
alreadyMirrored: alreadyMirroredCount,
|
||||
malformed,
|
||||
skipped: false,
|
||||
};
|
||||
const now = Date.now();
|
||||
db.prepare(
|
||||
`INSERT INTO migration_runs (id, started_at, finished_at, status, report_json)
|
||||
VALUES (?, ?, ?, 'completed', ?)
|
||||
ON CONFLICT(id) DO UPDATE SET
|
||||
finished_at = excluded.finished_at,
|
||||
status = excluded.status,
|
||||
report_json = excluded.report_json`,
|
||||
).run(CRON_RUN_LOG_TASK_IMPORT_MIGRATION_ID, now, now, JSON.stringify(result));
|
||||
return result;
|
||||
}
|
||||
@@ -612,6 +612,7 @@ function normalizeLegacyTaskRow(row: Record<string, unknown>): SqliteBindRow {
|
||||
progress_summary: legacyBindValue(row.progress_summary),
|
||||
terminal_summary: legacyBindValue(row.terminal_summary),
|
||||
terminal_outcome: legacyBindValue(row.terminal_outcome),
|
||||
detail_json: legacyBindValue(row.detail_json),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -703,6 +704,7 @@ function readLegacyTaskRows(sourcePath: string): SqliteBindRow[] {
|
||||
pickLegacyColumn(columns, "progress_summary"),
|
||||
pickLegacyColumn(columns, "terminal_summary"),
|
||||
pickLegacyColumn(columns, "terminal_outcome"),
|
||||
pickLegacyColumn(columns, "detail_json"),
|
||||
];
|
||||
return db
|
||||
.prepare(
|
||||
@@ -781,13 +783,14 @@ function insertTaskRunRowSql(db: DatabaseSync, row: SqliteBindRow): void {
|
||||
task_id, runtime, task_kind, source_id, requester_session_key, owner_key, scope_kind,
|
||||
child_session_key, parent_flow_id, parent_task_id, agent_id, requester_agent_id, run_id,
|
||||
label, task, status, delivery_status, notify_policy, created_at, started_at, ended_at,
|
||||
last_event_at, cleanup_after, error, progress_summary, terminal_summary, terminal_outcome
|
||||
last_event_at, cleanup_after, error, progress_summary, terminal_summary, terminal_outcome,
|
||||
detail_json
|
||||
) VALUES (
|
||||
@task_id, @runtime, @task_kind, @source_id, @requester_session_key, @owner_key,
|
||||
@scope_kind, @child_session_key, @parent_flow_id, @parent_task_id, @agent_id,
|
||||
@requester_agent_id, @run_id, @label, @task, @status, @delivery_status, @notify_policy,
|
||||
@created_at, @started_at, @ended_at, @last_event_at, @cleanup_after, @error,
|
||||
@progress_summary, @terminal_summary, @terminal_outcome
|
||||
@progress_summary, @terminal_summary, @terminal_outcome, @detail_json
|
||||
)
|
||||
`,
|
||||
).run(row);
|
||||
@@ -882,6 +885,7 @@ async function migrateLegacyTaskRunsSidecar(params: {
|
||||
"progress_summary",
|
||||
"terminal_summary",
|
||||
"terminal_outcome",
|
||||
"detail_json",
|
||||
];
|
||||
for (const row of taskRows) {
|
||||
const taskId = legacyKeyValue(expectDefined(row.task_id, "task migration row key"));
|
||||
|
||||
@@ -18,3 +18,7 @@ export {
|
||||
resetAutoMigrateLegacyStateDirForTest,
|
||||
resetAutoMigrateLegacyTaskStateSidecarsForTest,
|
||||
} from "./state-migrations.state-dir.js";
|
||||
export {
|
||||
CRON_RUN_LOG_TASK_IMPORT_MIGRATION_ID,
|
||||
migrateLegacyCronRunLogsToTaskRuns,
|
||||
} from "./state-migrations.cron-run-logs.js";
|
||||
|
||||
1
src/state/openclaw-state-db.generated.d.ts
vendored
1
src/state/openclaw-state-db.generated.d.ts
vendored
@@ -1004,6 +1004,7 @@ export interface TaskRuns {
|
||||
cleanup_after: number | null;
|
||||
created_at: number;
|
||||
delivery_status: string;
|
||||
detail_json: string | null;
|
||||
ended_at: number | null;
|
||||
error: string | null;
|
||||
label: string | null;
|
||||
|
||||
@@ -2272,6 +2272,28 @@ describe("openclaw state database", () => {
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
});
|
||||
|
||||
it("adds task detail storage to an existing state database", () => {
|
||||
const stateDir = createTempStateDir();
|
||||
const database = openOpenClawStateDatabase({
|
||||
env: { OPENCLAW_STATE_DIR: stateDir },
|
||||
});
|
||||
const databasePath = database.path;
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
|
||||
const { DatabaseSync } = requireNodeSqlite();
|
||||
const legacyDb = new DatabaseSync(databasePath);
|
||||
legacyDb.exec("ALTER TABLE task_runs DROP COLUMN detail_json");
|
||||
legacyDb.close();
|
||||
|
||||
const reopened = openOpenClawStateDatabase({
|
||||
env: { OPENCLAW_STATE_DIR: stateDir },
|
||||
});
|
||||
const columns = reopened.db.prepare("PRAGMA table_info(task_runs)").all() as Array<{
|
||||
name?: string;
|
||||
}>;
|
||||
expect(columns.some((column) => column.name === "detail_json")).toBe(true);
|
||||
});
|
||||
|
||||
it("rolls back the requester attribution column when its backfill fails", () => {
|
||||
const stateDir = createTempStateDir();
|
||||
const database = openOpenClawStateDatabase({
|
||||
|
||||
@@ -30,6 +30,7 @@ import {
|
||||
configureSqlitePreSchemaPragmas,
|
||||
type SqliteWalMaintenance,
|
||||
} from "../infra/sqlite-wal.js";
|
||||
import { migrateLegacyCronRunLogsToTaskRuns } from "../infra/state-migrations.cron-run-logs.js";
|
||||
import { createSubsystemLogger } from "../logging/subsystem.js";
|
||||
import type { DB as OpenClawStateKyselyDatabase } from "./openclaw-state-db.generated.js";
|
||||
import {
|
||||
@@ -1525,6 +1526,7 @@ function ensureAdditiveStateColumns(db: DatabaseSync): void {
|
||||
repairLegacyTaskDeliveryStatuses(db);
|
||||
ensureColumn(db, "task_runs", "tool_use_count INTEGER");
|
||||
ensureColumn(db, "task_runs", "last_tool_name TEXT");
|
||||
ensureColumn(db, "task_runs", "detail_json TEXT");
|
||||
ensureColumn(db, "subagent_runs", "task_name TEXT");
|
||||
ensureColumn(db, "worker_environments", "bootstrap_bundle_hash TEXT");
|
||||
ensureColumn(db, "worker_environments", "bootstrap_openclaw_version TEXT");
|
||||
@@ -1553,6 +1555,7 @@ function ensureSchema(db: DatabaseSync, pathname: string): void {
|
||||
ensureAdditiveStateColumns(db);
|
||||
assertCanonicalStateSchemaShape(db, pathname);
|
||||
db.exec(OPENCLAW_STATE_SCHEMA_SQL);
|
||||
migrateLegacyCronRunLogsToTaskRuns(db);
|
||||
repairCanonicalSqliteUniqueIndexes(db, pathname, OPENCLAW_STATE_CANONICAL_UNIQUE_INDEXES);
|
||||
// Retired node_pairing_* tables were created by earlier schema revisions but
|
||||
// never had a shipped writer (the node surface lives on device_pairing_paired
|
||||
|
||||
@@ -1312,7 +1312,8 @@ CREATE TABLE IF NOT EXISTS task_runs (
|
||||
error TEXT,
|
||||
progress_summary TEXT,
|
||||
terminal_summary TEXT,
|
||||
terminal_outcome TEXT
|
||||
terminal_outcome TEXT,
|
||||
detail_json TEXT
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_task_runs_run_id ON task_runs(run_id);
|
||||
@@ -1323,6 +1324,10 @@ CREATE INDEX IF NOT EXISTS idx_task_runs_last_event_at ON task_runs(last_event_a
|
||||
CREATE INDEX IF NOT EXISTS idx_task_runs_owner_key ON task_runs(owner_key);
|
||||
CREATE INDEX IF NOT EXISTS idx_task_runs_parent_flow_id ON task_runs(parent_flow_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_task_runs_child_session_key ON task_runs(child_session_key);
|
||||
CREATE INDEX IF NOT EXISTS idx_task_runs_runtime_source_ended
|
||||
ON task_runs(runtime, source_id, ended_at, created_at, task_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_task_runs_runtime_ended
|
||||
ON task_runs(runtime, ended_at, created_at, task_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS subagent_runs (
|
||||
run_id TEXT NOT NULL PRIMARY KEY,
|
||||
|
||||
@@ -1307,7 +1307,8 @@ CREATE TABLE IF NOT EXISTS task_runs (
|
||||
error TEXT,
|
||||
progress_summary TEXT,
|
||||
terminal_summary TEXT,
|
||||
terminal_outcome TEXT
|
||||
terminal_outcome TEXT,
|
||||
detail_json TEXT
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_task_runs_run_id ON task_runs(run_id);
|
||||
@@ -1318,6 +1319,10 @@ CREATE INDEX IF NOT EXISTS idx_task_runs_last_event_at ON task_runs(last_event_a
|
||||
CREATE INDEX IF NOT EXISTS idx_task_runs_owner_key ON task_runs(owner_key);
|
||||
CREATE INDEX IF NOT EXISTS idx_task_runs_parent_flow_id ON task_runs(parent_flow_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_task_runs_child_session_key ON task_runs(child_session_key);
|
||||
CREATE INDEX IF NOT EXISTS idx_task_runs_runtime_source_ended
|
||||
ON task_runs(runtime, source_id, ended_at, created_at, task_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_task_runs_runtime_ended
|
||||
ON task_runs(runtime, ended_at, created_at, task_id);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS subagent_runs (
|
||||
run_id TEXT NOT NULL PRIMARY KEY,
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
// Defines the detached task runtime contract and spawn options.
|
||||
import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
import type {
|
||||
JsonValue,
|
||||
TaskDeliveryState,
|
||||
TaskDeliveryStatus,
|
||||
TaskNotifyPolicy,
|
||||
@@ -34,6 +35,7 @@ export type DetachedTaskCreateParams = {
|
||||
preferMetadata?: boolean;
|
||||
notifyPolicy?: TaskNotifyPolicy;
|
||||
deliveryStatus?: TaskDeliveryStatus;
|
||||
detail?: JsonValue;
|
||||
};
|
||||
|
||||
export type DetachedRunningTaskCreateParams = DetachedTaskCreateParams & {
|
||||
@@ -65,11 +67,14 @@ type DetachedTaskCompleteParams = {
|
||||
runId: string;
|
||||
runtime?: TaskRuntime;
|
||||
sessionKey?: string;
|
||||
childSessionKey?: string | null;
|
||||
endedAt: number;
|
||||
lastEventAt?: number;
|
||||
progressSummary?: string | null;
|
||||
terminalSummary?: string | null;
|
||||
preserveTerminalSummary?: boolean;
|
||||
terminalOutcome?: TaskTerminalOutcome | null;
|
||||
detail?: JsonValue;
|
||||
suppressDelivery?: boolean;
|
||||
};
|
||||
|
||||
@@ -77,12 +82,15 @@ type DetachedTaskFailParams = {
|
||||
runId: string;
|
||||
runtime?: TaskRuntime;
|
||||
sessionKey?: string;
|
||||
childSessionKey?: string | null;
|
||||
status?: Extract<TaskStatus, "failed" | "timed_out" | "cancelled">;
|
||||
endedAt: number;
|
||||
lastEventAt?: number;
|
||||
error?: string;
|
||||
progressSummary?: string | null;
|
||||
terminalSummary?: string | null;
|
||||
preserveTerminalSummary?: boolean;
|
||||
detail?: JsonValue;
|
||||
suppressDelivery?: boolean;
|
||||
};
|
||||
|
||||
@@ -90,13 +98,17 @@ export type DetachedTaskFinalizeParams = {
|
||||
runId: string;
|
||||
runtime?: TaskRuntime;
|
||||
sessionKey?: string;
|
||||
childSessionKey?: string | null;
|
||||
status: Extract<TaskStatus, "succeeded" | "failed" | "timed_out" | "cancelled">;
|
||||
endedAt: number;
|
||||
lastEventAt?: number;
|
||||
error?: string;
|
||||
clearError?: boolean;
|
||||
progressSummary?: string | null;
|
||||
terminalSummary?: string | null;
|
||||
preserveTerminalSummary?: boolean;
|
||||
terminalOutcome?: TaskTerminalOutcome | null;
|
||||
detail?: JsonValue;
|
||||
suppressDelivery?: boolean;
|
||||
};
|
||||
|
||||
|
||||
@@ -36,6 +36,7 @@ import {
|
||||
} from "./task-flow-runtime-internal.js";
|
||||
import { summarizeTaskRecords } from "./task-registry.summary.js";
|
||||
import type {
|
||||
JsonValue,
|
||||
TaskDeliveryState,
|
||||
TaskDeliveryStatus,
|
||||
TaskNotifyPolicy,
|
||||
@@ -175,11 +176,14 @@ export function completeTaskRunByRunId(params: {
|
||||
runId: string;
|
||||
runtime?: TaskRuntime;
|
||||
sessionKey?: string;
|
||||
childSessionKey?: string | null;
|
||||
endedAt: number;
|
||||
lastEventAt?: number;
|
||||
progressSummary?: string | null;
|
||||
terminalSummary?: string | null;
|
||||
preserveTerminalSummary?: boolean;
|
||||
terminalOutcome?: TaskTerminalOutcome | null;
|
||||
detail?: JsonValue;
|
||||
suppressDelivery?: boolean;
|
||||
}) {
|
||||
return finalizeTaskRunByRunId({
|
||||
@@ -196,12 +200,15 @@ export function failTaskRunByRunId(params: {
|
||||
runId: string;
|
||||
runtime?: TaskRuntime;
|
||||
sessionKey?: string;
|
||||
childSessionKey?: string | null;
|
||||
status?: Extract<TaskStatus, "failed" | "timed_out" | "cancelled">;
|
||||
endedAt: number;
|
||||
lastEventAt?: number;
|
||||
error?: string;
|
||||
progressSummary?: string | null;
|
||||
terminalSummary?: string | null;
|
||||
preserveTerminalSummary?: boolean;
|
||||
detail?: JsonValue;
|
||||
suppressDelivery?: boolean;
|
||||
}) {
|
||||
return finalizeTaskRunByRunId({
|
||||
|
||||
@@ -1,15 +1,8 @@
|
||||
// Defines managed task-flow registry records and parser helpers.
|
||||
import type { DeliveryContext } from "../utils/delivery-context.types.js";
|
||||
import type { TaskNotifyPolicy } from "./task-registry.types.js";
|
||||
import type { JsonValue, TaskNotifyPolicy } from "./task-registry.types.js";
|
||||
|
||||
/** JSON value shape persisted with task-flow state and wait metadata. */
|
||||
export type JsonValue =
|
||||
| null
|
||||
| boolean
|
||||
| number
|
||||
| string
|
||||
| JsonValue[]
|
||||
| { [key: string]: JsonValue };
|
||||
export type { JsonValue } from "./task-registry.types.js";
|
||||
|
||||
export type TaskFlowSyncMode = "task_mirrored" | "managed";
|
||||
|
||||
|
||||
@@ -2,8 +2,6 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import type { AcpSessionStoreEntry } from "../acp/runtime/session-meta.js";
|
||||
import type { SessionEntry } from "../config/sessions.js";
|
||||
import type { CronRunLogEntry } from "../cron/run-log.js";
|
||||
import type { CronStoreFile } from "../cron/types.js";
|
||||
import type { ParsedAgentSessionKey } from "../routing/session-key.js";
|
||||
import {
|
||||
resetDetachedTaskLifecycleRuntimeForTests,
|
||||
@@ -63,8 +61,7 @@ function createTaskRegistryMaintenanceHarness(params: {
|
||||
activeCronJobIds?: string[];
|
||||
activeRunIds?: string[];
|
||||
activeAcpSessionKeys?: string[];
|
||||
cronStore?: CronStoreFile;
|
||||
cronRunLogEntries?: Record<string, CronRunLogEntry[]>;
|
||||
durableCronTaskRows?: Record<string, TaskRecord[]>;
|
||||
runtimeAuthoritative?: boolean;
|
||||
}) {
|
||||
const sessionStore = params.sessionStore ?? {};
|
||||
@@ -72,7 +69,7 @@ function createTaskRegistryMaintenanceHarness(params: {
|
||||
const activeCronJobIds = new Set(params.activeCronJobIds ?? []);
|
||||
const activeRunIds = new Set(params.activeRunIds ?? []);
|
||||
const activeAcpSessionKeys = new Set(params.activeAcpSessionKeys ?? []);
|
||||
const cronRunLogEntries = params.cronRunLogEntries ?? {};
|
||||
const durableCronTaskRows = params.durableCronTaskRows ?? {};
|
||||
const currentTasks = new Map(params.tasks.map((task) => [task.taskId, { ...task }]));
|
||||
|
||||
const runtime: TaskRegistryMaintenanceRuntime = {
|
||||
@@ -159,8 +156,13 @@ function createTaskRegistryMaintenanceHarness(params: {
|
||||
endedAt: patch.endedAt,
|
||||
lastEventAt: patch.lastEventAt ?? patch.endedAt,
|
||||
...(patch.terminalSummary !== undefined
|
||||
? { terminalSummary: patch.terminalSummary ?? undefined }
|
||||
? {
|
||||
terminalSummary: patch.preserveTerminalSummary
|
||||
? (patch.terminalSummary ?? undefined)
|
||||
: patch.terminalSummary?.replace(/\s+/g, " ").trim() || undefined,
|
||||
}
|
||||
: {}),
|
||||
...(patch.detail !== undefined ? { detail: patch.detail } : {}),
|
||||
} satisfies TaskRecord;
|
||||
if (Object.hasOwn(patch, "error")) {
|
||||
if (patch.error === undefined) {
|
||||
@@ -184,9 +186,8 @@ function createTaskRegistryMaintenanceHarness(params: {
|
||||
return next;
|
||||
},
|
||||
isRuntimeAuthoritative: () => params.runtimeAuthoritative ?? true,
|
||||
resolveCronJobsStorePath: () => "/tmp/openclaw-test-cron/jobs.json",
|
||||
loadCronJobsStoreSync: () => params.cronStore ?? { version: 1, jobs: [] },
|
||||
readCronRunLogEntriesSync: ({ jobId }) => (jobId ? (cronRunLogEntries[jobId] ?? []) : []),
|
||||
listTaskRegistryRecordsByRuntimeSourceIdFromSqlite: ({ sourceId }) =>
|
||||
sourceId ? (durableCronTaskRows[sourceId] ?? []) : Object.values(durableCronTaskRows).flat(),
|
||||
};
|
||||
|
||||
setTaskRegistryMaintenanceRuntimeForTests(runtime);
|
||||
@@ -287,6 +288,16 @@ describe("task-registry maintenance issue #60299", () => {
|
||||
const { currentTasks } = createTaskRegistryMaintenanceHarness({
|
||||
tasks: [task],
|
||||
activeCronJobIds: ["cron-job-2"],
|
||||
durableCronTaskRows: {
|
||||
"cron-job-2": [
|
||||
{
|
||||
...task,
|
||||
status: "succeeded",
|
||||
endedAt: Date.now(),
|
||||
detail: { kind: "cron-run", status: "ok" },
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
expectMaintenanceCounts(await runTaskRegistryMaintenance(), { reconciled: 0 });
|
||||
@@ -473,7 +484,7 @@ describe("task-registry maintenance issue #60299", () => {
|
||||
expectTaskStatus(currentTasks, task.taskId, "running");
|
||||
});
|
||||
|
||||
it("recovers finished cron tasks from durable run logs before marking them lost", async () => {
|
||||
it("recovers finished cron tasks from durable ledger detail before marking them lost", async () => {
|
||||
const startedAt = Date.now() - 60 * 60_000;
|
||||
const task = makeStaleTask({
|
||||
runtime: "cron",
|
||||
@@ -485,16 +496,15 @@ describe("task-registry maintenance issue #60299", () => {
|
||||
|
||||
const { currentTasks } = createTaskRegistryMaintenanceHarness({
|
||||
tasks: [task],
|
||||
cronRunLogEntries: {
|
||||
durableCronTaskRows: {
|
||||
"cron-job-run-log-ok": [
|
||||
{
|
||||
ts: startedAt + 1250,
|
||||
jobId: "cron-job-run-log-ok",
|
||||
action: "finished",
|
||||
status: "ok",
|
||||
summary: "done",
|
||||
runAtMs: startedAt,
|
||||
durationMs: 1250,
|
||||
...task,
|
||||
status: "succeeded",
|
||||
endedAt: startedAt + 1250,
|
||||
lastEventAt: startedAt + 1250,
|
||||
terminalSummary: "done",
|
||||
detail: { kind: "cron-run", status: "ok", durationMs: 1250 },
|
||||
},
|
||||
],
|
||||
},
|
||||
@@ -513,9 +523,47 @@ describe("task-registry maintenance issue #60299", () => {
|
||||
expect(storedTask.status).toBe("succeeded");
|
||||
expect(storedTask.endedAt).toBe(startedAt + 1250);
|
||||
expect(storedTask.terminalSummary).toBe("done");
|
||||
expect(storedTask.detail).toEqual({ kind: "cron-run", status: "ok", durationMs: 1250 });
|
||||
});
|
||||
|
||||
it("does not recover cron tasks from malformed run id timestamps", async () => {
|
||||
it("recovers cancelled cron tasks with exact durable summaries", async () => {
|
||||
const startedAt = Date.now() - 60 * 60_000;
|
||||
const task = makeStaleTask({
|
||||
runtime: "cron",
|
||||
sourceId: "cron-job-cancelled",
|
||||
runId: `cron:cron-job-cancelled:${startedAt}`,
|
||||
startedAt,
|
||||
lastEventAt: startedAt,
|
||||
});
|
||||
const terminalSummary = "cancelled\n summary";
|
||||
|
||||
const { currentTasks } = createTaskRegistryMaintenanceHarness({
|
||||
tasks: [task],
|
||||
durableCronTaskRows: {
|
||||
"cron-job-cancelled": [
|
||||
{
|
||||
...task,
|
||||
status: "cancelled",
|
||||
error: "cancelled by operator",
|
||||
endedAt: startedAt + 500,
|
||||
lastEventAt: startedAt + 500,
|
||||
terminalSummary,
|
||||
detail: { kind: "cron-run", status: "error", durationMs: 500 },
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
expectMaintenanceCounts(await runTaskRegistryMaintenance(), { reconciled: 0, recovered: 1 });
|
||||
expect(currentTasks.get(task.taskId)).toMatchObject({
|
||||
status: "cancelled",
|
||||
endedAt: startedAt + 500,
|
||||
terminalSummary,
|
||||
detail: { kind: "cron-run", status: "error", durationMs: 500 },
|
||||
});
|
||||
});
|
||||
|
||||
it("does not recover cron tasks from an unrelated ledger row", async () => {
|
||||
const task = makeStaleTask({
|
||||
runtime: "cron",
|
||||
sourceId: "cron-job-run-log-ok",
|
||||
@@ -524,16 +572,17 @@ describe("task-registry maintenance issue #60299", () => {
|
||||
|
||||
const { currentTasks } = createTaskRegistryMaintenanceHarness({
|
||||
tasks: [task],
|
||||
cronRunLogEntries: {
|
||||
durableCronTaskRows: {
|
||||
"cron-job-run-log-ok": [
|
||||
{
|
||||
ts: 1250,
|
||||
jobId: "cron-job-run-log-ok",
|
||||
action: "finished",
|
||||
status: "ok",
|
||||
summary: "done",
|
||||
runAtMs: 1000,
|
||||
durationMs: 250,
|
||||
...task,
|
||||
taskId: "different-task",
|
||||
runId: "cron:cron-job-run-log-ok:1000",
|
||||
status: "succeeded",
|
||||
endedAt: 1250,
|
||||
lastEventAt: 1250,
|
||||
terminalSummary: "done",
|
||||
detail: { kind: "cron-run", status: "ok" },
|
||||
},
|
||||
],
|
||||
},
|
||||
@@ -544,7 +593,7 @@ describe("task-registry maintenance issue #60299", () => {
|
||||
expectTaskStatus(currentTasks, task.taskId, "lost");
|
||||
});
|
||||
|
||||
it("recovers terminal lost cron tasks from durable run logs", async () => {
|
||||
it("recovers terminal lost cron tasks from the durable ledger", async () => {
|
||||
const startedAt = Date.now() - GRACE_EXPIRED_MS;
|
||||
const task = makeStaleTask({
|
||||
runtime: "cron",
|
||||
@@ -560,16 +609,16 @@ describe("task-registry maintenance issue #60299", () => {
|
||||
|
||||
const { currentTasks } = createTaskRegistryMaintenanceHarness({
|
||||
tasks: [task],
|
||||
cronRunLogEntries: {
|
||||
durableCronTaskRows: {
|
||||
"cron-job-terminal-lost-ok": [
|
||||
{
|
||||
ts: startedAt + 1250,
|
||||
jobId: "cron-job-terminal-lost-ok",
|
||||
action: "finished",
|
||||
status: "ok",
|
||||
summary: "done",
|
||||
runAtMs: startedAt,
|
||||
durationMs: 1250,
|
||||
...task,
|
||||
status: "succeeded",
|
||||
error: undefined,
|
||||
endedAt: startedAt + 1250,
|
||||
lastEventAt: startedAt + 1250,
|
||||
terminalSummary: "done",
|
||||
detail: { kind: "cron-run", status: "ok" },
|
||||
},
|
||||
],
|
||||
},
|
||||
@@ -611,16 +660,15 @@ describe("task-registry maintenance issue #60299", () => {
|
||||
|
||||
const { currentTasks } = createTaskRegistryMaintenanceHarness({
|
||||
tasks: [task],
|
||||
cronRunLogEntries: {
|
||||
durableCronTaskRows: {
|
||||
"cron-job-terminal-lost-no-error": [
|
||||
{
|
||||
ts: startedAt + 1250,
|
||||
jobId: "cron-job-terminal-lost-no-error",
|
||||
action: "finished",
|
||||
status: "ok",
|
||||
summary: "done",
|
||||
runAtMs: startedAt,
|
||||
durationMs: 1250,
|
||||
...task,
|
||||
status: "succeeded",
|
||||
endedAt: startedAt + 1250,
|
||||
lastEventAt: startedAt + 1250,
|
||||
terminalSummary: "done",
|
||||
detail: { kind: "cron-run", status: "ok" },
|
||||
},
|
||||
],
|
||||
},
|
||||
@@ -649,16 +697,16 @@ describe("task-registry maintenance issue #60299", () => {
|
||||
|
||||
const { currentTasks } = createTaskRegistryMaintenanceHarness({
|
||||
tasks: [task],
|
||||
cronRunLogEntries: {
|
||||
durableCronTaskRows: {
|
||||
"cron-job-terminal-lost-other-error": [
|
||||
{
|
||||
ts: startedAt + 1250,
|
||||
jobId: "cron-job-terminal-lost-other-error",
|
||||
action: "finished",
|
||||
status: "ok",
|
||||
summary: "done",
|
||||
runAtMs: startedAt,
|
||||
durationMs: 1250,
|
||||
...task,
|
||||
status: "succeeded",
|
||||
error: undefined,
|
||||
endedAt: startedAt + 1250,
|
||||
lastEventAt: startedAt + 1250,
|
||||
terminalSummary: "done",
|
||||
detail: { kind: "cron-run", status: "ok" },
|
||||
},
|
||||
],
|
||||
},
|
||||
@@ -672,7 +720,7 @@ describe("task-registry maintenance issue #60299", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("recovers interrupted cron tasks from durable cron job state when run logs are absent", async () => {
|
||||
it("does not recover cron tasks from cron job state without a terminal ledger row", async () => {
|
||||
const startedAt = Date.now() - GRACE_EXPIRED_MS;
|
||||
const task = makeStaleTask({
|
||||
runtime: "cron",
|
||||
@@ -682,38 +730,11 @@ describe("task-registry maintenance issue #60299", () => {
|
||||
lastEventAt: startedAt,
|
||||
});
|
||||
|
||||
const { currentTasks } = createTaskRegistryMaintenanceHarness({
|
||||
tasks: [task],
|
||||
cronStore: {
|
||||
version: 1,
|
||||
jobs: [
|
||||
{
|
||||
id: "cron-job-state-error",
|
||||
name: "state error",
|
||||
enabled: true,
|
||||
createdAtMs: startedAt - 60_000,
|
||||
updatedAtMs: startedAt,
|
||||
schedule: { kind: "every", everyMs: 60_000, anchorMs: startedAt - 60_000 },
|
||||
sessionTarget: "isolated",
|
||||
wakeMode: "next-heartbeat",
|
||||
payload: { kind: "agentTurn", message: "work" },
|
||||
state: {
|
||||
lastRunAtMs: startedAt,
|
||||
lastRunStatus: "error",
|
||||
lastError: "cron: job interrupted by gateway restart",
|
||||
lastDurationMs: 5000,
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
const { currentTasks } = createTaskRegistryMaintenanceHarness({ tasks: [task] });
|
||||
|
||||
expectMaintenanceCounts(previewTaskRegistryMaintenance(), { reconciled: 0, recovered: 1 });
|
||||
expectMaintenanceCounts(await runTaskRegistryMaintenance(), { reconciled: 0, recovered: 1 });
|
||||
const storedTask = requireTaskRecord(currentTasks, task.taskId);
|
||||
expect(storedTask.status).toBe("failed");
|
||||
expect(storedTask.endedAt).toBe(startedAt + 5000);
|
||||
expect(storedTask.error).toBe("cron: job interrupted by gateway restart");
|
||||
expectMaintenanceCounts(previewTaskRegistryMaintenance(), { reconciled: 1, recovered: 0 });
|
||||
expectMaintenanceCounts(await runTaskRegistryMaintenance(), { reconciled: 1, recovered: 0 });
|
||||
expectTaskStatus(currentTasks, task.taskId, "lost");
|
||||
});
|
||||
|
||||
it("marks chat-backed cli tasks lost after the owning run context disappears", async () => {
|
||||
|
||||
@@ -19,13 +19,8 @@ import {
|
||||
} from "../config/sessions/session-accessor.js";
|
||||
import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
import { isCronJobActive } from "../cron/active-jobs.js";
|
||||
import { readCronRunLogEntriesSync } from "../cron/run-log.js";
|
||||
import type { CronRunLogEntry } from "../cron/run-log.js";
|
||||
import { loadCronJobsStoreSync, resolveCronJobsStorePath } from "../cron/store.js";
|
||||
import type { CronJob, CronStoreFile } from "../cron/types.js";
|
||||
import { getAgentRunContext } from "../infra/agent-events.js";
|
||||
import { getSessionBindingService } from "../infra/outbound/session-binding-service.js";
|
||||
import { parseStrictNonNegativeInteger } from "../infra/parse-finite-number.js";
|
||||
import { createSubsystemLogger } from "../logging/subsystem.js";
|
||||
import {
|
||||
isPluginStateDatabaseOpen,
|
||||
@@ -64,6 +59,7 @@ import {
|
||||
summarizeTaskAuditFindings,
|
||||
} from "./task-registry.audit.js";
|
||||
import type { TaskAuditFinding, TaskAuditSummary } from "./task-registry.audit.js";
|
||||
import { listTaskRegistryRecordsByRuntimeSourceIdFromSqlite } from "./task-registry.store.sqlite.js";
|
||||
import { summarizeTaskRecords } from "./task-registry.summary.js";
|
||||
import type { TaskRecord, TaskRegistrySummary, TaskStatus } from "./task-registry.types.js";
|
||||
import type { ActiveTaskRestartBlocker } from "./task-restart-blocker.js";
|
||||
@@ -88,7 +84,6 @@ const SWEEP_YIELD_BATCH_SIZE = 25;
|
||||
let sweeper: NodeJS.Timeout | null = null;
|
||||
let deferredSweep: NodeJS.Timeout | null = null;
|
||||
let sweepInProgress = false;
|
||||
let configuredCronStorePath: string | undefined;
|
||||
let configuredRuntimeAuthoritative = false;
|
||||
|
||||
type TaskRegistryMaintenanceRuntime = {
|
||||
@@ -119,9 +114,7 @@ type TaskRegistryMaintenanceRuntime = {
|
||||
resolveTaskForLookupToken: typeof resolveTaskForLookupToken;
|
||||
setTaskCleanupAfterById: typeof setTaskCleanupAfterById;
|
||||
isRuntimeAuthoritative: () => boolean;
|
||||
resolveCronJobsStorePath: typeof resolveCronJobsStorePath;
|
||||
loadCronJobsStoreSync: typeof loadCronJobsStoreSync;
|
||||
readCronRunLogEntriesSync: typeof readCronRunLogEntriesSync;
|
||||
listTaskRegistryRecordsByRuntimeSourceIdFromSqlite: typeof listTaskRegistryRecordsByRuntimeSourceIdFromSqlite;
|
||||
};
|
||||
|
||||
const defaultTaskRegistryMaintenanceRuntime: TaskRegistryMaintenanceRuntime = {
|
||||
@@ -159,9 +152,7 @@ const defaultTaskRegistryMaintenanceRuntime: TaskRegistryMaintenanceRuntime = {
|
||||
resolveTaskForLookupToken,
|
||||
setTaskCleanupAfterById,
|
||||
isRuntimeAuthoritative: () => configuredRuntimeAuthoritative,
|
||||
resolveCronJobsStorePath: () => configuredCronStorePath ?? resolveCronJobsStorePath(),
|
||||
loadCronJobsStoreSync,
|
||||
readCronRunLogEntriesSync,
|
||||
listTaskRegistryRecordsByRuntimeSourceIdFromSqlite,
|
||||
};
|
||||
|
||||
let taskRegistryMaintenanceRuntime: TaskRegistryMaintenanceRuntime =
|
||||
@@ -197,23 +188,17 @@ export type TaskRegistryMaintenanceDiagnostics = {
|
||||
staleRunningTasks: TaskRegistryMaintenanceTaskDiagnostic[];
|
||||
};
|
||||
|
||||
type CronExecutionId = {
|
||||
jobId: string;
|
||||
startedAt: number;
|
||||
};
|
||||
|
||||
type CronTerminalRecovery = {
|
||||
status: Extract<TaskStatus, "succeeded" | "failed" | "timed_out">;
|
||||
status: Extract<TaskStatus, "succeeded" | "failed" | "timed_out" | "cancelled">;
|
||||
endedAt: number;
|
||||
lastEventAt: number;
|
||||
error?: string;
|
||||
terminalSummary?: string;
|
||||
detail?: TaskRecord["detail"];
|
||||
};
|
||||
|
||||
type CronRecoveryContext = {
|
||||
storePath: string;
|
||||
store?: CronStoreFile | null;
|
||||
runLogsByJobId: Map<string, CronRunLogEntry[]>;
|
||||
taskRowsByJobId: Map<string, TaskRecord[]>;
|
||||
};
|
||||
|
||||
type SessionEntryLookup = {
|
||||
@@ -227,8 +212,7 @@ type BackingSessionLookupContext = {
|
||||
|
||||
function createCronRecoveryContext(): CronRecoveryContext {
|
||||
return {
|
||||
storePath: taskRegistryMaintenanceRuntime.resolveCronJobsStorePath(),
|
||||
runLogsByJobId: new Map<string, CronRunLogEntry[]>(),
|
||||
taskRowsByJobId: new Map<string, TaskRecord[]>(),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -319,30 +303,6 @@ function hasLostGraceExpired(task: TaskRecord, now: number): boolean {
|
||||
return now - referenceAt >= graceMs;
|
||||
}
|
||||
|
||||
function parseCronExecutionId(task: TaskRecord): CronExecutionId | undefined {
|
||||
const runId = task.runId?.trim();
|
||||
if (!runId?.startsWith("cron:")) {
|
||||
return undefined;
|
||||
}
|
||||
const separator = runId.lastIndexOf(":");
|
||||
if (separator <= "cron:".length) {
|
||||
return undefined;
|
||||
}
|
||||
const startedAt = parseStrictNonNegativeInteger(runId.slice(separator + 1));
|
||||
if (startedAt === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
const jobId = runId.slice("cron:".length, separator).trim();
|
||||
if (!jobId || (task.sourceId?.trim() && task.sourceId.trim() !== jobId)) {
|
||||
return undefined;
|
||||
}
|
||||
return { jobId, startedAt };
|
||||
}
|
||||
|
||||
function isTimeoutCronError(error: string | undefined): boolean {
|
||||
return error === "cron: job execution timed out";
|
||||
}
|
||||
|
||||
function isRecoverableLostCronTask(task: TaskRecord): boolean {
|
||||
if (task.status !== "lost") {
|
||||
return false;
|
||||
@@ -351,97 +311,31 @@ function isRecoverableLostCronTask(task: TaskRecord): boolean {
|
||||
return Boolean(error?.includes("backing session missing"));
|
||||
}
|
||||
|
||||
function mapCronTerminalStatus(status: unknown, error?: string): CronTerminalRecovery["status"] {
|
||||
if (status === "ok" || status === "skipped") {
|
||||
return "succeeded";
|
||||
}
|
||||
return isTimeoutCronError(error) ? "timed_out" : "failed";
|
||||
function isCronTerminalTaskStatus(status: TaskStatus): status is CronTerminalRecovery["status"] {
|
||||
return (
|
||||
status === "succeeded" ||
|
||||
status === "failed" ||
|
||||
status === "timed_out" ||
|
||||
status === "cancelled"
|
||||
);
|
||||
}
|
||||
|
||||
function getCronRunLogEntries(context: CronRecoveryContext, jobId: string): CronRunLogEntry[] {
|
||||
const cached = context.runLogsByJobId.get(jobId);
|
||||
function getCronTaskRows(context: CronRecoveryContext, jobId: string): TaskRecord[] {
|
||||
const cached = context.taskRowsByJobId.get(jobId);
|
||||
if (cached) {
|
||||
return cached;
|
||||
}
|
||||
let entries: CronRunLogEntry[];
|
||||
let rows: TaskRecord[];
|
||||
try {
|
||||
entries = taskRegistryMaintenanceRuntime.readCronRunLogEntriesSync({
|
||||
storePath: context.storePath,
|
||||
jobId,
|
||||
limit: 5000,
|
||||
rows = taskRegistryMaintenanceRuntime.listTaskRegistryRecordsByRuntimeSourceIdFromSqlite({
|
||||
runtime: "cron",
|
||||
sourceId: jobId,
|
||||
});
|
||||
} catch {
|
||||
entries = [];
|
||||
rows = [];
|
||||
}
|
||||
context.runLogsByJobId.set(jobId, entries);
|
||||
return entries;
|
||||
}
|
||||
|
||||
function getCronStore(context: CronRecoveryContext): CronStoreFile | null {
|
||||
if (context.store !== undefined) {
|
||||
return context.store;
|
||||
}
|
||||
try {
|
||||
context.store = taskRegistryMaintenanceRuntime.loadCronJobsStoreSync(context.storePath);
|
||||
} catch {
|
||||
context.store = null;
|
||||
}
|
||||
return context.store;
|
||||
}
|
||||
|
||||
function resolveCronRunLogRecovery(
|
||||
execution: CronExecutionId,
|
||||
context: CronRecoveryContext,
|
||||
): CronTerminalRecovery | undefined {
|
||||
const entries = getCronRunLogEntries(context, execution.jobId);
|
||||
const entry = entries.findLast(
|
||||
(candidate) =>
|
||||
candidate.jobId === execution.jobId &&
|
||||
candidate.action === "finished" &&
|
||||
candidate.runAtMs === execution.startedAt &&
|
||||
(candidate.status === "ok" || candidate.status === "skipped" || candidate.status === "error"),
|
||||
);
|
||||
if (!entry) {
|
||||
return undefined;
|
||||
}
|
||||
const durationMs =
|
||||
typeof entry.durationMs === "number" && Number.isFinite(entry.durationMs)
|
||||
? Math.max(0, entry.durationMs)
|
||||
: undefined;
|
||||
const endedAt = durationMs === undefined ? entry.ts : execution.startedAt + durationMs;
|
||||
return {
|
||||
status: mapCronTerminalStatus(entry.status, entry.error),
|
||||
endedAt,
|
||||
lastEventAt: endedAt,
|
||||
...(entry.error !== undefined ? { error: entry.error } : {}),
|
||||
...(entry.summary !== undefined ? { terminalSummary: entry.summary } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
function resolveCronJobStateRecovery(
|
||||
execution: CronExecutionId,
|
||||
context: CronRecoveryContext,
|
||||
): CronTerminalRecovery | undefined {
|
||||
const store = getCronStore(context);
|
||||
const job: CronJob | undefined = store?.jobs.find((entry) => entry.id === execution.jobId);
|
||||
if (!job || job.state.lastRunAtMs !== execution.startedAt) {
|
||||
return undefined;
|
||||
}
|
||||
const status = job.state.lastRunStatus ?? job.state.lastStatus;
|
||||
if (status !== "ok" && status !== "skipped" && status !== "error") {
|
||||
return undefined;
|
||||
}
|
||||
const durationMs =
|
||||
typeof job.state.lastDurationMs === "number" && Number.isFinite(job.state.lastDurationMs)
|
||||
? Math.max(0, job.state.lastDurationMs)
|
||||
: 0;
|
||||
const endedAt = execution.startedAt + durationMs;
|
||||
return {
|
||||
status: mapCronTerminalStatus(status, job.state.lastError),
|
||||
endedAt,
|
||||
lastEventAt: endedAt,
|
||||
...(job.state.lastError !== undefined ? { error: job.state.lastError } : {}),
|
||||
};
|
||||
context.taskRowsByJobId.set(jobId, rows);
|
||||
return rows;
|
||||
}
|
||||
|
||||
function resolveDurableCronTaskRecovery(
|
||||
@@ -451,13 +345,33 @@ function resolveDurableCronTaskRecovery(
|
||||
if (task.runtime !== "cron" || (!isActiveTask(task) && !isRecoverableLostCronTask(task))) {
|
||||
return undefined;
|
||||
}
|
||||
const execution = parseCronExecutionId(task);
|
||||
if (!execution) {
|
||||
const jobId = task.sourceId?.trim();
|
||||
if (!jobId) {
|
||||
return undefined;
|
||||
}
|
||||
return (
|
||||
resolveCronRunLogRecovery(execution, context) ?? resolveCronJobStateRecovery(execution, context)
|
||||
if (
|
||||
taskRegistryMaintenanceRuntime.isRuntimeAuthoritative() &&
|
||||
taskRegistryMaintenanceRuntime.isCronJobActive(jobId)
|
||||
) {
|
||||
return undefined;
|
||||
}
|
||||
const row = getCronTaskRows(context, jobId).find(
|
||||
(candidate) =>
|
||||
candidate.taskId === task.taskId ||
|
||||
(Boolean(task.runId?.trim()) && candidate.runId === task.runId),
|
||||
);
|
||||
if (!row || !isCronTerminalTaskStatus(row.status)) {
|
||||
return undefined;
|
||||
}
|
||||
const endedAt = row.endedAt ?? row.lastEventAt ?? row.createdAt;
|
||||
return {
|
||||
status: row.status,
|
||||
endedAt,
|
||||
lastEventAt: row.lastEventAt ?? endedAt,
|
||||
...(row.error !== undefined ? { error: row.error } : {}),
|
||||
...(row.terminalSummary !== undefined ? { terminalSummary: row.terminalSummary } : {}),
|
||||
...(row.detail !== undefined ? { detail: row.detail } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
function hasActiveCliRun(task: TaskRecord): boolean {
|
||||
@@ -804,8 +718,9 @@ function markTaskRecovered(task: TaskRecord, recovery: CronTerminalRecovery): Ta
|
||||
lastEventAt: recovery.lastEventAt,
|
||||
error: recovery.error,
|
||||
...(recovery.terminalSummary !== undefined
|
||||
? { terminalSummary: recovery.terminalSummary }
|
||||
? { terminalSummary: recovery.terminalSummary, preserveTerminalSummary: true }
|
||||
: {}),
|
||||
...(recovery.detail !== undefined ? { detail: recovery.detail } : {}),
|
||||
}) ?? projectTaskRecovered(task, recovery);
|
||||
void taskRegistryMaintenanceRuntime.maybeDeliverTaskTerminalUpdate(updated.taskId);
|
||||
return updated;
|
||||
@@ -821,6 +736,7 @@ function projectTaskRecovered(task: TaskRecord, recovery: CronTerminalRecovery):
|
||||
...(recovery.terminalSummary !== undefined
|
||||
? { terminalSummary: recovery.terminalSummary }
|
||||
: {}),
|
||||
...(recovery.detail !== undefined ? { detail: recovery.detail } : {}),
|
||||
};
|
||||
if (recovery.error === undefined) {
|
||||
delete projected.error;
|
||||
@@ -1232,16 +1148,13 @@ export function setTaskRegistryMaintenanceRuntimeForTests(
|
||||
|
||||
export function resetTaskRegistryMaintenanceRuntimeForTests(): void {
|
||||
taskRegistryMaintenanceRuntime = defaultTaskRegistryMaintenanceRuntime;
|
||||
configuredCronStorePath = undefined;
|
||||
configuredRuntimeAuthoritative = false;
|
||||
}
|
||||
|
||||
export function configureTaskRegistryMaintenance(options: {
|
||||
cronStorePath?: string;
|
||||
export function configureTaskRegistryMaintenance(options?: {
|
||||
runtimeAuthoritative?: boolean;
|
||||
}): void {
|
||||
configuredCronStorePath = options.cronStorePath?.trim() || undefined;
|
||||
if (options.runtimeAuthoritative !== undefined) {
|
||||
if (options?.runtimeAuthoritative !== undefined) {
|
||||
configuredRuntimeAuthoritative = options.runtimeAuthoritative;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -21,7 +21,9 @@ import {
|
||||
parseTaskScopeKind,
|
||||
parseTaskStatus,
|
||||
type TaskDeliveryState,
|
||||
type JsonValue,
|
||||
type TaskRecord,
|
||||
type TaskRuntime,
|
||||
} from "./task-registry.types.js";
|
||||
|
||||
type TaskRunsTable = OpenClawStateKyselyDatabase["task_runs"];
|
||||
@@ -78,12 +80,24 @@ const TASK_RUN_SELECT_COLUMNS = [
|
||||
"progress_summary",
|
||||
"terminal_summary",
|
||||
"terminal_outcome",
|
||||
"detail_json",
|
||||
] as const;
|
||||
|
||||
let cachedDatabase: TaskRegistryDatabase | null = null;
|
||||
|
||||
function serializeJson(value: unknown): string | null {
|
||||
return value == null ? null : JSON.stringify(value);
|
||||
return value === undefined ? null : (JSON.stringify(value) ?? null);
|
||||
}
|
||||
|
||||
function parseJsonValue(raw: string | null): JsonValue | undefined {
|
||||
if (!raw?.trim()) {
|
||||
return undefined;
|
||||
}
|
||||
try {
|
||||
return JSON.parse(raw) as JsonValue;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function rowToTaskRecord(row: TaskRegistryRow): TaskRecord {
|
||||
@@ -94,6 +108,7 @@ function rowToTaskRecord(row: TaskRegistryRow): TaskRecord {
|
||||
const toolUseCount = normalizeSqliteNumber(row.tool_use_count);
|
||||
const scopeKind = parseTaskScopeKind(row.scope_kind);
|
||||
const terminalOutcome = parseOptionalTaskTerminalOutcome(row.terminal_outcome);
|
||||
const detail = parseJsonValue(row.detail_json);
|
||||
// System tasks intentionally have no requester session; ownerKey is the lookup anchor.
|
||||
const requesterSessionKey =
|
||||
scopeKind === "system" ? "" : row.requester_session_key?.trim() || row.owner_key;
|
||||
@@ -125,8 +140,9 @@ function rowToTaskRecord(row: TaskRegistryRow): TaskRecord {
|
||||
...(row.last_tool_name ? { lastToolName: row.last_tool_name } : {}),
|
||||
...(row.error ? { error: row.error } : {}),
|
||||
...(row.progress_summary ? { progressSummary: row.progress_summary } : {}),
|
||||
...(row.terminal_summary ? { terminalSummary: row.terminal_summary } : {}),
|
||||
...(row.terminal_summary !== null ? { terminalSummary: row.terminal_summary } : {}),
|
||||
...(terminalOutcome ? { terminalOutcome } : {}),
|
||||
...(detail !== undefined ? { detail } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -171,6 +187,7 @@ function bindTaskRecordBase(record: TaskRecord): Insertable<TaskRunsTable> {
|
||||
progress_summary: record.progressSummary ?? null,
|
||||
terminal_summary: record.terminalSummary ?? null,
|
||||
terminal_outcome: record.terminalOutcome ?? null,
|
||||
detail_json: serializeJson(record.detail),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -232,6 +249,22 @@ function selectTaskRowsByOwnerKey(db: DatabaseSync, ownerKey: string): TaskRegis
|
||||
.all(ownerKey) as TaskRegistryRow[];
|
||||
}
|
||||
|
||||
function selectTaskRowsByRuntimeSourceId(
|
||||
db: DatabaseSync,
|
||||
runtime: TaskRuntime,
|
||||
sourceId?: string,
|
||||
): TaskRegistryRow[] {
|
||||
let query = getTaskRegistryKysely(db)
|
||||
.selectFrom("task_runs")
|
||||
.select(TASK_RUN_SELECT_COLUMNS)
|
||||
.where("runtime", "=", runtime);
|
||||
if (sourceId !== undefined) {
|
||||
query = query.where("source_id", "=", sourceId);
|
||||
}
|
||||
return executeSqliteQuerySync(db, query.orderBy("created_at", "asc").orderBy("task_id", "asc"))
|
||||
.rows;
|
||||
}
|
||||
|
||||
function selectTaskDeliveryStateRows(db: DatabaseSync): TaskDeliveryStateRow[] {
|
||||
const query = getTaskRegistryKysely(db)
|
||||
.selectFrom("task_delivery_state")
|
||||
@@ -276,6 +309,7 @@ function upsertTaskRow(db: DatabaseSync, row: Insertable<TaskRunsTable>): void {
|
||||
progress_summary: (eb) => eb.ref("excluded.progress_summary"),
|
||||
terminal_summary: (eb) => eb.ref("excluded.terminal_summary"),
|
||||
terminal_outcome: (eb) => eb.ref("excluded.terminal_outcome"),
|
||||
detail_json: (eb) => eb.ref("excluded.detail_json"),
|
||||
}),
|
||||
),
|
||||
);
|
||||
@@ -356,6 +390,19 @@ export function listTaskRegistryRecordsByOwnerKeyFromSqlite(ownerKey: string): T
|
||||
return selectTaskRowsByOwnerKey(db, key).map(rowToTaskRecord);
|
||||
}
|
||||
|
||||
/** Reads task rows for one runtime/source without restoring the process registry snapshot. */
|
||||
export function listTaskRegistryRecordsByRuntimeSourceIdFromSqlite(params: {
|
||||
runtime: TaskRuntime;
|
||||
sourceId?: string;
|
||||
}): TaskRecord[] {
|
||||
const sourceId = params.sourceId?.trim();
|
||||
if (params.sourceId !== undefined && !sourceId) {
|
||||
return [];
|
||||
}
|
||||
const { db } = openTaskRegistryDatabase();
|
||||
return selectTaskRowsByRuntimeSourceId(db, params.runtime, sourceId).map(rowToTaskRecord);
|
||||
}
|
||||
|
||||
export function saveTaskRegistryStateToSqlite(snapshot: TaskRegistryStoreSnapshot) {
|
||||
withWriteTransaction(({ db }) => {
|
||||
const kysely = getTaskRegistryKysely(db);
|
||||
|
||||
@@ -420,6 +420,51 @@ describe("task-registry store runtime", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("round-trips runtime-owned task detail through sqlite", async () => {
|
||||
await withOpenClawTestState(
|
||||
{ layout: "state-only", prefix: "openclaw-task-store-detail-" },
|
||||
async () => {
|
||||
const task: TaskRecord = {
|
||||
...createStoredTask(),
|
||||
runtime: "cron",
|
||||
sourceId: "cron-detail-job",
|
||||
detail: {
|
||||
kind: "cron-run",
|
||||
status: "ok",
|
||||
usage: { input_tokens: 3, cached: false },
|
||||
},
|
||||
};
|
||||
saveTaskRegistryStateToSqlite({
|
||||
tasks: new Map([[task.taskId, task]]),
|
||||
deliveryStates: new Map(),
|
||||
});
|
||||
|
||||
expect(loadTaskRegistryStateFromSqlite().tasks.get(task.taskId)?.detail).toEqual(
|
||||
task.detail,
|
||||
);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it("preserves explicit null task detail through sqlite", async () => {
|
||||
await withOpenClawTestState(
|
||||
{ layout: "state-only", prefix: "openclaw-task-store-null-detail-" },
|
||||
async () => {
|
||||
const task: TaskRecord = {
|
||||
...createStoredTask(),
|
||||
detail: null,
|
||||
};
|
||||
saveTaskRegistryStateToSqlite({
|
||||
tasks: new Map([[task.taskId, task]]),
|
||||
deliveryStates: new Map(),
|
||||
});
|
||||
|
||||
const restored = loadTaskRegistryStateFromSqlite().tasks.get(task.taskId);
|
||||
expect(restored).toHaveProperty("detail", null);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it("loads task and delivery rows from one sqlite read snapshot", async () => {
|
||||
await withOpenClawTestState(
|
||||
{ layout: "state-only", prefix: "openclaw-task-store-read-snapshot-" },
|
||||
|
||||
@@ -239,9 +239,7 @@ function configureTaskRegistryMaintenanceRuntimeForTest(params: {
|
||||
return next;
|
||||
},
|
||||
isRuntimeAuthoritative: () => true,
|
||||
resolveCronJobsStorePath: () => "/tmp/openclaw-test-cron/jobs.json",
|
||||
loadCronJobsStoreSync: () => ({ version: 1, jobs: [] }),
|
||||
readCronRunLogEntriesSync: () => [],
|
||||
listTaskRegistryRecordsByRuntimeSourceIdFromSqlite: () => [],
|
||||
});
|
||||
}
|
||||
|
||||
@@ -624,6 +622,35 @@ describe("task-registry", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("clears a provisional child session when the terminal outcome has none", async () => {
|
||||
await withTaskRegistryTempDir(async () => {
|
||||
resetTaskRegistryMemoryForTest();
|
||||
createTaskRecord({
|
||||
runtime: "cron",
|
||||
ownerKey: "",
|
||||
scopeKind: "system",
|
||||
childSessionKey: "agent:main:cron:provisional",
|
||||
runId: "cron:provisional:100",
|
||||
task: "Provisional cron run",
|
||||
status: "running",
|
||||
deliveryStatus: "not_applicable",
|
||||
startedAt: 100,
|
||||
});
|
||||
|
||||
finalizeTaskRunByRunId({
|
||||
runId: "cron:provisional:100",
|
||||
runtime: "cron",
|
||||
childSessionKey: null,
|
||||
status: "failed",
|
||||
endedAt: 200,
|
||||
error: "setup failed",
|
||||
});
|
||||
reloadTaskRegistryFromStore();
|
||||
|
||||
expect(requireTaskByRunId("cron:provisional:100").childSessionKey).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
it("reuses an ACP run task when a derived flow id is linked before a duplicate create", async () => {
|
||||
await withTaskRegistryTempDir(async () => {
|
||||
resetTaskRegistryMemoryForTest({ persist: false });
|
||||
@@ -3546,9 +3573,7 @@ describe("task-registry", () => {
|
||||
resolveTaskForLookupToken: () => undefined,
|
||||
setTaskCleanupAfterById: () => null,
|
||||
isRuntimeAuthoritative: () => true,
|
||||
resolveCronJobsStorePath: () => "/tmp/openclaw-test-cron/jobs.json",
|
||||
loadCronJobsStoreSync: () => ({ version: 1, jobs: [] }),
|
||||
readCronRunLogEntriesSync: () => [],
|
||||
listTaskRegistryRecordsByRuntimeSourceIdFromSqlite: () => [],
|
||||
});
|
||||
|
||||
try {
|
||||
|
||||
@@ -55,6 +55,7 @@ import {
|
||||
type TaskRegistryObserverEvent,
|
||||
} from "./task-registry.store.js";
|
||||
import type {
|
||||
JsonValue,
|
||||
TaskDeliveryState,
|
||||
TaskDeliveryStatus,
|
||||
TaskEventKind,
|
||||
@@ -240,7 +241,10 @@ function ensureTaskCancellationReady(task: TaskRecord): void {
|
||||
}
|
||||
|
||||
function cloneTaskRecord(record: TaskRecord): TaskRecord {
|
||||
return { ...record };
|
||||
return {
|
||||
...record,
|
||||
...(record.detail !== undefined ? { detail: structuredClone(record.detail) } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeTaskTimestamps(task: TaskRecord): TaskRecord {
|
||||
@@ -961,6 +965,7 @@ function mergeExistingTaskForCreate(
|
||||
preferMetadata?: boolean;
|
||||
deliveryStatus?: TaskDeliveryStatus;
|
||||
notifyPolicy?: TaskNotifyPolicy;
|
||||
detail?: JsonValue;
|
||||
},
|
||||
): TaskRecord | null {
|
||||
ensureLinkedTaskFlowRegistryReady(existing);
|
||||
@@ -1024,6 +1029,9 @@ function mergeExistingTaskForCreate(
|
||||
if (notifyPolicy !== existing.notifyPolicy && existing.notifyPolicy === "silent") {
|
||||
patch.notifyPolicy = notifyPolicy;
|
||||
}
|
||||
if (params.detail !== undefined) {
|
||||
patch.detail = params.detail;
|
||||
}
|
||||
if (Object.keys(patch).length === 0) {
|
||||
return cloneTaskRecord(existing);
|
||||
}
|
||||
@@ -1306,10 +1314,17 @@ function updateTask(taskId: string, patch: Partial<TaskRecord>): TaskRecord | nu
|
||||
if (!current) {
|
||||
return null;
|
||||
}
|
||||
const next = normalizeTaskTimestamps({ ...current, ...patch });
|
||||
const next = normalizeTaskTimestamps({
|
||||
...current,
|
||||
...patch,
|
||||
...(patch.detail !== undefined ? { detail: structuredClone(patch.detail) } : {}),
|
||||
});
|
||||
if (Object.hasOwn(patch, "error") && patch.error === undefined) {
|
||||
delete next.error;
|
||||
}
|
||||
if (Object.hasOwn(patch, "childSessionKey") && patch.childSessionKey === undefined) {
|
||||
delete next.childSessionKey;
|
||||
}
|
||||
if (isTerminalTaskStatus(next.status) && typeof next.cleanupAfter !== "number") {
|
||||
next.cleanupAfter = resolveTaskCleanupAfter({
|
||||
...next,
|
||||
@@ -1747,19 +1762,29 @@ export function setTaskCleanupAfterById(params: {
|
||||
export function markTaskTerminalById(params: {
|
||||
taskId: string;
|
||||
status: Extract<TaskStatus, "succeeded" | "failed" | "timed_out" | "cancelled">;
|
||||
childSessionKey?: string | null;
|
||||
endedAt: number;
|
||||
lastEventAt?: number;
|
||||
error?: string;
|
||||
terminalSummary?: string | null;
|
||||
preserveTerminalSummary?: boolean;
|
||||
terminalOutcome?: TaskTerminalOutcome | null;
|
||||
detail?: JsonValue;
|
||||
}): TaskRecord | null {
|
||||
ensureTaskRegistryReady();
|
||||
const patch: Partial<TaskRecord> = {
|
||||
status: params.status,
|
||||
...(params.childSessionKey !== undefined
|
||||
? { childSessionKey: params.childSessionKey?.trim() || undefined }
|
||||
: {}),
|
||||
endedAt: params.endedAt,
|
||||
lastEventAt: params.lastEventAt ?? params.endedAt,
|
||||
...(params.terminalSummary !== undefined
|
||||
? { terminalSummary: normalizeTaskSummary(params.terminalSummary) }
|
||||
? {
|
||||
terminalSummary: params.preserveTerminalSummary
|
||||
? (params.terminalSummary ?? undefined)
|
||||
: normalizeTaskSummary(params.terminalSummary),
|
||||
}
|
||||
: {}),
|
||||
...(params.terminalOutcome !== undefined
|
||||
? {
|
||||
@@ -1769,6 +1794,7 @@ export function markTaskTerminalById(params: {
|
||||
}),
|
||||
}
|
||||
: {}),
|
||||
...(params.detail !== undefined ? { detail: structuredClone(params.detail) } : {}),
|
||||
};
|
||||
if (Object.hasOwn(params, "error")) {
|
||||
patch.error = params.error;
|
||||
@@ -1937,6 +1963,7 @@ export function createTaskRecord(params: {
|
||||
progressSummary?: string | null;
|
||||
terminalSummary?: string | null;
|
||||
terminalOutcome?: TaskTerminalOutcome | null;
|
||||
detail?: JsonValue;
|
||||
}): TaskRecord | null {
|
||||
ensureTaskRegistryReady();
|
||||
const requesterSessionKey = resolveTaskRequesterSessionKey(params);
|
||||
@@ -2026,6 +2053,7 @@ export function createTaskRecord(params: {
|
||||
status,
|
||||
terminalOutcome: params.terminalOutcome,
|
||||
}),
|
||||
...(params.detail !== undefined ? { detail: structuredClone(params.detail) } : {}),
|
||||
});
|
||||
if (isTerminalTaskStatus(record.status) && typeof record.cleanupAfter !== "number") {
|
||||
record.cleanupAfter = resolveTaskCleanupAfter(record);
|
||||
@@ -2063,14 +2091,18 @@ function updateTaskStateByRunId(params: {
|
||||
runId: string;
|
||||
runtime?: TaskRuntime;
|
||||
sessionKey?: string;
|
||||
childSessionKey?: string | null;
|
||||
status?: TaskStatus;
|
||||
startedAt?: number;
|
||||
endedAt?: number;
|
||||
lastEventAt?: number;
|
||||
error?: string;
|
||||
clearError?: boolean;
|
||||
progressSummary?: string | null;
|
||||
terminalSummary?: string | null;
|
||||
preserveTerminalSummary?: boolean;
|
||||
terminalOutcome?: TaskTerminalOutcome | null;
|
||||
detail?: JsonValue;
|
||||
eventSummary?: string | null;
|
||||
suppressDelivery?: boolean;
|
||||
}) {
|
||||
@@ -2111,7 +2143,12 @@ function updateTaskStateByRunId(params: {
|
||||
if (params.lastEventAt != null) {
|
||||
patch.lastEventAt = params.lastEventAt;
|
||||
}
|
||||
if (
|
||||
if (params.childSessionKey !== undefined) {
|
||||
patch.childSessionKey = params.childSessionKey?.trim() || undefined;
|
||||
}
|
||||
if (params.clearError) {
|
||||
patch.error = undefined;
|
||||
} else if (
|
||||
current.status === "cancelled" &&
|
||||
nextStatus !== "cancelled" &&
|
||||
params.error === undefined
|
||||
@@ -2124,7 +2161,9 @@ function updateTaskStateByRunId(params: {
|
||||
patch.progressSummary = normalizeTaskSummary(params.progressSummary);
|
||||
}
|
||||
if (params.terminalSummary !== undefined) {
|
||||
patch.terminalSummary = normalizeTaskSummary(params.terminalSummary);
|
||||
patch.terminalSummary = params.preserveTerminalSummary
|
||||
? (params.terminalSummary ?? undefined)
|
||||
: normalizeTaskSummary(params.terminalSummary);
|
||||
}
|
||||
if (params.terminalOutcome !== undefined) {
|
||||
patch.terminalOutcome = resolveTaskTerminalOutcome({
|
||||
@@ -2132,6 +2171,9 @@ function updateTaskStateByRunId(params: {
|
||||
terminalOutcome: params.terminalOutcome,
|
||||
});
|
||||
}
|
||||
if (params.detail !== undefined) {
|
||||
patch.detail = params.detail;
|
||||
}
|
||||
if (params.suppressDelivery) {
|
||||
// Teardown suppression must survive redundant lifecycle finalizers that
|
||||
// arrive after queues are cleared, or they can repopulate the stopped session.
|
||||
@@ -2234,28 +2276,36 @@ export function finalizeTaskRunByRunId(params: {
|
||||
runId: string;
|
||||
runtime?: TaskRuntime;
|
||||
sessionKey?: string;
|
||||
childSessionKey?: string | null;
|
||||
status: Extract<TaskStatus, "succeeded" | "failed" | "timed_out" | "cancelled">;
|
||||
startedAt?: number;
|
||||
endedAt: number;
|
||||
lastEventAt?: number;
|
||||
error?: string;
|
||||
clearError?: boolean;
|
||||
progressSummary?: string | null;
|
||||
terminalSummary?: string | null;
|
||||
preserveTerminalSummary?: boolean;
|
||||
terminalOutcome?: TaskTerminalOutcome | null;
|
||||
detail?: JsonValue;
|
||||
suppressDelivery?: boolean;
|
||||
}) {
|
||||
return updateTaskStateByRunId({
|
||||
runId: params.runId,
|
||||
runtime: params.runtime,
|
||||
sessionKey: params.sessionKey,
|
||||
childSessionKey: params.childSessionKey,
|
||||
status: params.status,
|
||||
startedAt: params.startedAt,
|
||||
endedAt: params.endedAt,
|
||||
lastEventAt: params.lastEventAt,
|
||||
error: params.error,
|
||||
clearError: params.clearError,
|
||||
progressSummary: params.progressSummary,
|
||||
terminalSummary: params.terminalSummary,
|
||||
preserveTerminalSummary: params.preserveTerminalSummary,
|
||||
terminalOutcome: params.terminalOutcome,
|
||||
detail: params.detail,
|
||||
suppressDelivery: params.suppressDelivery,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1,6 +1,15 @@
|
||||
// Defines task registry records, statuses, delivery state, and parser helpers.
|
||||
import type { DeliveryContext } from "../utils/delivery-context.types.js";
|
||||
|
||||
/** JSON value shape persisted with runtime-owned task detail. */
|
||||
export type JsonValue =
|
||||
| null
|
||||
| boolean
|
||||
| number
|
||||
| string
|
||||
| JsonValue[]
|
||||
| { [key: string]: JsonValue };
|
||||
|
||||
/** Runtime family that owns a task run lifecycle. */
|
||||
export type TaskRuntime = "subagent" | "acp" | "cli" | "cron";
|
||||
|
||||
@@ -147,4 +156,5 @@ export type TaskRecord = {
|
||||
progressSummary?: string;
|
||||
terminalSummary?: string;
|
||||
terminalOutcome?: TaskTerminalOutcome;
|
||||
detail?: JsonValue;
|
||||
};
|
||||
|
||||
Reference in New Issue
Block a user