diff --git a/src/commands/doctor/cron/index.test.ts b/src/commands/doctor/cron/index.test.ts index 0f41b062d451..a667d5aa7b45 100644 --- a/src/commands/doctor/cron/index.test.ts +++ b/src/commands/doctor/cron/index.test.ts @@ -543,6 +543,88 @@ describe("maybeRepairLegacyCronStore", () => { }); }); + describe("chronic failure advisory", () => { + it("warns about repeatedly failing jobs without touching the store", async () => { + const storePath = await makeTempStorePath(); + await writeCurrentCronStore(storePath, [ + createCurrentCronJob({ + id: "failing-job", + state: { lastRunStatus: "error", consecutiveErrors: 5, lastError: "boom" }, + }), + ]); + const prompter = makePrompter(true); + + await maybeRepairLegacyCronStore({ + cfg: createCronConfig(storePath), + options: {}, + prompter, + }); + + expectNoteContaining("1 cron job has failed 3+ runs in a row", "Cron"); + expectNoteContaining("re-fires it on error backoff", "Cron"); + expectNoteContaining("resets on the next successful run", "Cron"); + expectNoteContaining("interrupted by a gateway restart", "Cron"); + expectNoteContaining("openclaw cron show ", "Cron"); + + // Observer-only: no repair prompt and the failure counters stay untouched. + expect(prompter.confirm).not.toHaveBeenCalled(); + const jobs = await readPersistedJobs(storePath); + const state = requireRecord(requirePersistedJob(jobs, 0).state, "cron state"); + expect(state.consecutiveErrors).toBe(5); + }); + + it("pluralizes and only counts enabled jobs at or above the threshold", async () => { + const storePath = await makeTempStorePath(); + await writeCurrentCronStore(storePath, [ + createCurrentCronJob({ + id: "failing-a", + state: { lastRunStatus: "error", consecutiveErrors: 3 }, + }), + createCurrentCronJob({ + id: "failing-b", + state: { lastRunStatus: "error", consecutiveErrors: 12 }, + }), + createCurrentCronJob({ + id: "recovering", + state: { lastRunStatus: "error", consecutiveErrors: 2 }, + }), + // Exhausted one-shot jobs get disabled with their error state retained; + // they no longer re-fire, so the advisory must not count them. + createCurrentCronJob({ + id: "disabled-exhausted", + enabled: false, + state: { lastRunStatus: "error", consecutiveErrors: 9 }, + }), + ]); + + await maybeRepairLegacyCronStore({ + cfg: createCronConfig(storePath), + options: {}, + prompter: makePrompter(true), + }); + + expectNoteContaining("2 cron jobs have failed 3+ runs in a row", "Cron"); + }); + + it("stays silent when failure streaks are below the threshold", async () => { + const storePath = await makeTempStorePath(); + await writeCurrentCronStore(storePath, [ + createCurrentCronJob({ + id: "single-failure", + state: { lastRunStatus: "error", consecutiveErrors: 2 }, + }), + ]); + + await maybeRepairLegacyCronStore({ + cfg: createCronConfig(storePath), + options: {}, + prompter: makePrompter(true), + }); + + expectNoNoteContaining("runs in a row", "Cron"); + }); + }); + it("repairs legacy cron store fields and migrates notify fallback to webhook delivery", async () => { const storePath = await makeTempStorePath(); await writeCronStore(storePath, [createLegacyCronJob()]); diff --git a/src/commands/doctor/cron/index.ts b/src/commands/doctor/cron/index.ts index 43597b642967..cbc9eb5ac2e4 100644 --- a/src/commands/doctor/cron/index.ts +++ b/src/commands/doctor/cron/index.ts @@ -85,6 +85,38 @@ function countInFlightCronJobs(jobs: Array>): number { }).length; } +// Fixed advisory threshold: three failures in a row is a clear chronic signal on +// its own. It coincides with the scheduler's default transient-retry budget, but +// `cron.retry.maxAttempts` is per-job configurable and doctor deliberately does +// not mirror retry config or exhaustion semantics (`consecutiveErrors > maxAttempts`). +const CHRONIC_FAILURE_MIN_CONSECUTIVE_ERRORS = 3; + +// Count enabled jobs stuck in repeated run failures. `state.consecutiveErrors` +// resets to 0 on the next successful run and also increments for runs interrupted +// by a gateway restart (startup marks in-flight runs failed, `src/cron/service/ops.ts`), +// so a streak can mean task failures, interrupted runs, or a mix — the note says so. +// Failure alerts are opt-in, so by default nothing else surfaces the streak. +// Disabled jobs no longer re-fire (e.g. the scheduler disables exhausted +// one-shot jobs with their error state retained), so they are excluded. +function countChronicallyFailingCronJobs(jobs: Array>): number { + return jobs.filter((job) => { + // Missing `enabled` counts as enabled, matching `isJobEnabled` + // (`src/cron/service/jobs.ts`); only an explicit `false` is excluded. + if (job.enabled === false) { + return false; + } + const state = job.state; + if (typeof state !== "object" || state === null) { + return false; + } + const consecutiveErrors = (state as { consecutiveErrors?: unknown }).consecutiveErrors; + return ( + typeof consecutiveErrors === "number" && + consecutiveErrors >= CHRONIC_FAILURE_MIN_CONSECUTIVE_ERRORS + ); + }).length; +} + type LegacyCronRepairState = { storePath: string; quarantinePath: string; @@ -575,6 +607,18 @@ export async function maybeRepairLegacyCronStore(params: { ); } + const chronicFailureCount = countChronicallyFailingCronJobs(rawJobs); + if (chronicFailureCount > 0) { + note( + [ + `${pluralize(chronicFailureCount, "cron job")} ${chronicFailureCount === 1 ? "has" : "have"} failed ${CHRONIC_FAILURE_MIN_CONSECUTIVE_ERRORS}+ runs in a row (\`state.consecutiveErrors\`), so the scheduler only re-fires ${chronicFailureCount === 1 ? "it" : "them"} on error backoff.`, + `- The count resets on the next successful run and also counts runs interrupted by a gateway restart, so a lasting streak means repeated task failures, repeatedly interrupted runs, or a mix. Failure alerts are opt-in, so this may be the only notice.`, + `- Review with ${formatCliCommand("openclaw cron list")} or ${formatCliCommand("openclaw cron show ")}.`, + ].join("\n"), + "Cron", + ); + } + const normalized = normalizeStoredCronJobs(rawJobs); const notifyCount = rawJobs.filter((job) => job.notify === true).length; const dreamingStaleCount = countStaleDreamingJobs(rawJobs);