mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-18 18:51:39 +00:00
fix(cron): honor configured retry.backoffMs for recurring error backoff floor (#93051)
Merged via squash.
Prepared head SHA: c8026d0aef
Co-authored-by: Alix-007 <267018309+Alix-007@users.noreply.github.com>
Co-authored-by: steipete <58493+steipete@users.noreply.github.com>
Reviewed-by: @steipete
This commit is contained in:
80
src/cron/service/timer.backoff-config-readback.test.ts
Normal file
80
src/cron/service/timer.backoff-config-readback.test.ts
Normal file
@@ -0,0 +1,80 @@
|
||||
// Verifies the configured retry.backoffMs floor for a recurring job survives a
|
||||
// real cron service run and is persisted to the SQLite-backed store, not just
|
||||
// computed in memory by applyJobResult.
|
||||
import path from "node:path";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { resetTaskRegistryForTests } from "../../tasks/task-registry.js";
|
||||
import { withEnvAsync } from "../../test-utils/env.js";
|
||||
import { setupCronServiceSuite, writeCronStoreSnapshot } from "../service.test-harness.js";
|
||||
import { loadCronStore } from "../store.js";
|
||||
import type { CronJob } from "../types.js";
|
||||
import { run } from "./ops.js";
|
||||
import { createCronServiceState } from "./state.js";
|
||||
|
||||
const { logger, makeStorePath } = setupCronServiceSuite({
|
||||
prefix: "cron-backoff-config-readback",
|
||||
});
|
||||
|
||||
function createDueRecurringJob(now: number): CronJob {
|
||||
return {
|
||||
id: "recurring-backoff-readback",
|
||||
name: "recurring backoff readback",
|
||||
enabled: true,
|
||||
createdAtMs: now - 60_000,
|
||||
updatedAtMs: now - 60_000,
|
||||
schedule: { kind: "every", everyMs: 1_000, anchorMs: now - 60_000 },
|
||||
sessionTarget: "isolated",
|
||||
wakeMode: "next-heartbeat",
|
||||
payload: { kind: "agentTurn", message: "ping" },
|
||||
sessionKey: "agent:main:main",
|
||||
state: { nextRunAtMs: now - 1 },
|
||||
};
|
||||
}
|
||||
|
||||
describe("recurring error backoff floor persistence", () => {
|
||||
it("persists the configured retry.backoffMs floor across a real run and SQLite readback", async () => {
|
||||
const now = Date.parse("2026-03-02T12:00:00.000Z");
|
||||
const { storePath } = await makeStorePath();
|
||||
const stateRoot = path.dirname(path.dirname(storePath));
|
||||
const job = createDueRecurringJob(now);
|
||||
|
||||
let persistedJob: CronJob | undefined;
|
||||
resetTaskRegistryForTests();
|
||||
try {
|
||||
await withEnvAsync({ OPENCLAW_STATE_DIR: stateRoot }, async () => {
|
||||
await writeCronStoreSnapshot({ storePath, jobs: [job] });
|
||||
|
||||
const state = createCronServiceState({
|
||||
storePath,
|
||||
cronEnabled: true,
|
||||
log: logger,
|
||||
nowMs: () => now,
|
||||
enqueueSystemEvent: vi.fn(),
|
||||
requestHeartbeat: vi.fn(),
|
||||
// Permanent (non-retryable) error -> recurring safety-net backoff
|
||||
// floor, the branch that must honor the configured backoffMs.
|
||||
runIsolatedAgentJob: vi.fn(async () => {
|
||||
throw new Error("permanent: bad request");
|
||||
}),
|
||||
cronConfig: { retry: { backoffMs: [300_000] } },
|
||||
});
|
||||
|
||||
// mode "due" (not "force") keeps preserveSchedule false, so the error
|
||||
// path computes the safety-net backoff floor rather than preserving the
|
||||
// recurring anchor.
|
||||
await expect(run(state, job.id, "due")).resolves.toEqual({ ok: true, ran: true });
|
||||
|
||||
const persisted = (await loadCronStore(storePath)) as { jobs: CronJob[] };
|
||||
persistedJob = persisted.jobs.find((entry) => entry.id === job.id);
|
||||
});
|
||||
} finally {
|
||||
resetTaskRegistryForTests();
|
||||
}
|
||||
|
||||
// The floor read back from the SQLite-backed store must be endedAt(=now) +
|
||||
// the configured backoffMs[0], not the hardcoded 30000 default.
|
||||
expect(persistedJob?.state.nextRunAtMs).toBe(now + 300_000);
|
||||
expect(persistedJob?.state.lastStatus).toBe("error");
|
||||
expect(persistedJob?.state.consecutiveErrors).toBe(1);
|
||||
});
|
||||
});
|
||||
@@ -893,7 +893,10 @@ export function applyJobResult(
|
||||
}
|
||||
}
|
||||
// Apply exponential backoff for errored jobs to prevent retry storms.
|
||||
const backoff = errorBackoffMs(job.state.consecutiveErrors ?? 1);
|
||||
const backoff = errorBackoffMs(
|
||||
job.state.consecutiveErrors ?? 1,
|
||||
state.deps.cronConfig?.retry?.backoffMs ?? DEFAULT_ERROR_BACKOFF_SCHEDULE_MS,
|
||||
);
|
||||
normalNext = computeNormalNext();
|
||||
const backoffNext = result.endedAt + backoff;
|
||||
// Use whichever is later: the natural next run or the backoff delay.
|
||||
|
||||
Reference in New Issue
Block a user