From c06739d773da4d1ed3b88bfc69a957e0056e9f1f Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 2 May 2026 14:31:11 +0100 Subject: [PATCH] fix(heartbeat): type wake scheduling intent Co-authored-by: Jordan Baker --- CHANGELOG.md | 1 + docs/plugins/sdk-runtime.md | 7 +- src/agents/acp-spawn-parent-stream.test.ts | 10 +- src/agents/acp-spawn-parent-stream.ts | 6 +- src/agents/bash-tools.exec-runtime.test.ts | 20 +- src/agents/bash-tools.exec-runtime.ts | 20 +- src/agents/cli-runner.reliability.test.ts | 6 +- src/agents/cli-runner.test-support.ts | 10 +- src/agents/cli-runner/execute.ts | 12 +- src/cli/deps.ts | 2 +- src/cron/service.armtimer-tight-loop.test.ts | 2 +- src/cron/service.delivery-plan.test.ts | 6 +- src/cron/service.every-jobs-fire.test.ts | 4 +- src/cron/service.failure-alert.test.ts | 2 +- src/cron/service.get-job.test.ts | 2 +- ...ce.heartbeat-ok-summary-suppressed.test.ts | 16 +- ...ervice.issue-16156-list-skips-cron.test.ts | 2 +- .../service.issue-35195-backup-timing.test.ts | 4 +- ...ce.issue-66019-unresolved-next-run.test.ts | 2 +- .../service.issue-regressions.test-helpers.ts | 8 +- src/cron/service.issue-regressions.test.ts | 2 +- ...n-job-passes-heartbeat-target-last.test.ts | 22 +- .../service.persists-delivered-status.test.ts | 2 +- .../service.prevents-duplicate-timers.test.ts | 8 +- src/cron/service.read-ops-nonblocking.test.ts | 12 +- .../service.rearm-timer-when-running.test.ts | 2 +- src/cron/service.restart-catchup.test.ts | 48 +-- ...runs-one-shot-main-job-disables-it.test.ts | 44 +- .../service.session-reaper-in-finally.test.ts | 6 +- ...s-main-jobs-empty-systemevent-text.test.ts | 10 +- ...ervice.store-load-invalid-main-job.test.ts | 6 +- src/cron/service.test-harness.ts | 20 +- .../jobs.schedule-error-isolation.test.ts | 2 +- src/cron/service/jobs.ts | 4 +- src/cron/service/ops.regression.test.ts | 14 +- src/cron/service/ops.test.ts | 10 +- src/cron/service/state.test.ts | 8 +- src/cron/service/state.ts | 6 +- .../store.load-missing-session-target.test.ts | 2 +- src/cron/service/store.test.ts | 2 +- src/cron/service/timer.regression.test.ts | 62 +-- src/cron/service/timer.test.ts | 18 +- src/cron/service/timer.ts | 22 +- src/gateway/server-cron.test.ts | 26 +- src/gateway/server-cron.ts | 10 +- src/gateway/server-node-events.runtime.ts | 2 +- src/gateway/server-node-events.test.ts | 50 ++- src/gateway/server-node-events.ts | 18 +- src/gateway/server-restart-sentinel.test.ts | 28 +- src/gateway/server-restart-sentinel.ts | 18 +- src/gateway/server/hooks.agent-trust.test.ts | 8 +- src/gateway/server/hooks.ts | 12 +- src/infra/heartbeat-cooldown.test.ts | 400 ++++++++++++++++++ src/infra/heartbeat-cooldown.ts | 164 +++++++ src/infra/heartbeat-reason.test.ts | 50 +-- src/infra/heartbeat-reason.ts | 56 +-- .../heartbeat-runner.commitments.test.ts | 4 +- .../heartbeat-runner.ghost-reminder.test.ts | 4 + ...beat-runner.isolated-key-stability.test.ts | 2 +- ...tbeat-runner.returns-default-unset.test.ts | 9 + src/infra/heartbeat-runner.scheduler.test.ts | 289 ++++++++++++- src/infra/heartbeat-runner.ts | 169 ++++++-- src/infra/heartbeat-wake.test.ts | 105 +++-- src/infra/heartbeat-wake.ts | 92 ++-- .../test-helpers/plugin-runtime-mock.ts | 1 + src/plugins/runtime/index.test.ts | 42 +- src/plugins/runtime/runtime-system.ts | 14 +- src/plugins/runtime/types-core.ts | 17 +- src/tasks/task-registry.ts | 10 +- .../cron/service-regression-fixtures.ts | 2 +- test/scripts/audit-seams.test.ts | 9 +- 71 files changed, 1601 insertions(+), 484 deletions(-) create mode 100644 src/infra/heartbeat-cooldown.test.ts create mode 100644 src/infra/heartbeat-cooldown.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 43c65924a95..1457dec865f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -314,6 +314,7 @@ Docs: https://docs.openclaw.ai - Agents/tools: reuse the auth profile store already loaded for the active run when deciding media and generation tool availability, avoiding repeated provider-auth runtime discovery during reply startup. Thanks @shakkernerd. - Agents/tools: keep image, video, and music generation tool registration on manifest/auth control-plane checks instead of loading runtime provider registries during reply startup, reducing live-path tool-prep blocking while leaving provider runtime resolution for execution and list actions. Thanks @shakkernerd. - Discord: document canonical mention formatting in agent prompt hints and channel docs so outbound replies use `<@USER_ID>`, `<#CHANNEL_ID>`, and `<@&ROLE_ID>` instead of legacy nickname mentions. (#75173) +- Heartbeat scheduler: gate exec-event/notification/spawn/retry wakes through a centralized cooldown so backgrounded `process.start` exit notifications can no longer self-feed runaway heartbeat runs (configured `every: "30m"` was firing every ~10s in production, pegging the gateway event loop with `eventLoopDelayMaxMs >6s` spikes that stalled control-UI asset serving and TUI handshakes). Documented wake-now paths (`manual`, `wake`, task completion, blocked-task follow-up, `/hooks/wake mode=now`, and cron `--wake now`) remain immediate; retryable busy skips no longer poison the cooldown for the next retry; per-agent flood guard caps any unexpected feedback loop at 5 runs/60s. (#64016, refs #17797 and #75436) Thanks @hexsprite. - fix: block workspace CLOUDSDK_PYTHON override and always set trusted interpreter for gcloud. (#74492) Thanks @pgondhi987. - Providers/Z.AI: move the bundled GLM catalog and auth env metadata into the plugin manifest, so `models list --all --provider zai` shows the full known catalog without duplicated runtime seed data. Thanks @shakkernerd. - Providers/Qianfan and Providers/Stepfun: declare setup auth metadata (`api-key` method, `QIANFAN_API_KEY`, `STEPFUN_API_KEY`) in the plugin manifest so onboarding and `models setup` surface the expected env var without falling back to legacy `providerAuthEnvVars` runtime seed data. Thanks @shakkernerd. diff --git a/docs/plugins/sdk-runtime.md b/docs/plugins/sdk-runtime.md index 1d36afe5401..e0dee0e8824 100644 --- a/docs/plugins/sdk-runtime.md +++ b/docs/plugins/sdk-runtime.md @@ -360,7 +360,12 @@ Provider and channel execution paths must use the active runtime config snapshot ```typescript await api.runtime.system.enqueueSystemEvent(event); - api.runtime.system.requestHeartbeatNow(); + api.runtime.system.requestHeartbeat({ + source: "other", + intent: "event", + reason: "plugin-event", + }); + api.runtime.system.requestHeartbeatNow({ reason: "plugin-event" }); // Deprecated compatibility alias. const output = await api.runtime.system.runCommandWithTimeout(cmd, args, opts); const hint = api.runtime.system.formatNativeDependencyHint(pkg); ``` diff --git a/src/agents/acp-spawn-parent-stream.test.ts b/src/agents/acp-spawn-parent-stream.test.ts index 53b06c364e5..aa6ae95f0ad 100644 --- a/src/agents/acp-spawn-parent-stream.test.ts +++ b/src/agents/acp-spawn-parent-stream.test.ts @@ -2,7 +2,7 @@ import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vite import { mergeMockedModule } from "../test-utils/vitest-module-mocks.js"; const enqueueSystemEventMock = vi.fn(); -const requestHeartbeatNowMock = vi.fn(); +const requestHeartbeatMock = vi.fn(); const readAcpSessionEntryMock = vi.fn(); const resolveSessionFilePathMock = vi.fn(); const resolveSessionFilePathOptionsMock = vi.fn(); @@ -17,7 +17,7 @@ vi.mock("../infra/heartbeat-wake.js", async () => { "../infra/heartbeat-wake.js", ), () => ({ - requestHeartbeatNow: (...args: unknown[]) => requestHeartbeatNowMock(...args), + requestHeartbeat: (...args: unknown[]) => requestHeartbeatMock(...args), }), ); }); @@ -63,7 +63,7 @@ describe("startAcpSpawnParentStreamRelay", () => { beforeEach(() => { enqueueSystemEventMock.mockClear(); - requestHeartbeatNowMock.mockClear(); + requestHeartbeatMock.mockClear(); readAcpSessionEntryMock.mockReset(); resolveSessionFilePathMock.mockReset(); resolveSessionFilePathOptionsMock.mockReset(); @@ -129,7 +129,7 @@ describe("startAcpSpawnParentStreamRelay", () => { trusted: false, }), ); - expect(requestHeartbeatNowMock).toHaveBeenCalledWith( + expect(requestHeartbeatMock).toHaveBeenCalledWith( expect.objectContaining({ reason: "acp:spawn:stream", sessionKey: "agent:main:main", @@ -255,7 +255,7 @@ describe("startAcpSpawnParentStreamRelay", () => { }); expect(collectedTexts()).toEqual([]); - expect(requestHeartbeatNowMock).not.toHaveBeenCalled(); + expect(requestHeartbeatMock).not.toHaveBeenCalled(); relay.dispose(); }); diff --git a/src/agents/acp-spawn-parent-stream.ts b/src/agents/acp-spawn-parent-stream.ts index badd5d156e7..6c380caa3a3 100644 --- a/src/agents/acp-spawn-parent-stream.ts +++ b/src/agents/acp-spawn-parent-stream.ts @@ -3,7 +3,7 @@ import path from "node:path"; import { readAcpSessionEntry } from "../acp/runtime/session-meta.js"; import { resolveSessionFilePath, resolveSessionFilePathOptions } from "../config/sessions/paths.js"; import { onAgentEvent } from "../infra/agent-events.js"; -import { requestHeartbeatNow } from "../infra/heartbeat-wake.js"; +import { requestHeartbeat } from "../infra/heartbeat-wake.js"; import { enqueueSystemEvent } from "../infra/system-events.js"; import { scopedHeartbeatWakeOptions } from "../routing/session-key.js"; import { normalizeAssistantPhase } from "../shared/chat-message-content.js"; @@ -181,8 +181,10 @@ export function startAcpSpawnParentStreamRelay(params: { if (!shouldSurfaceUpdates) { return; } - requestHeartbeatNow( + requestHeartbeat( scopedHeartbeatWakeOptions(parentSessionKey, { + source: "acp-spawn", + intent: "event", reason: "acp:spawn:stream", }), ); diff --git a/src/agents/bash-tools.exec-runtime.test.ts b/src/agents/bash-tools.exec-runtime.test.ts index 8f61bfc0e8d..4b55ebd161a 100644 --- a/src/agents/bash-tools.exec-runtime.test.ts +++ b/src/agents/bash-tools.exec-runtime.test.ts @@ -1,13 +1,13 @@ import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; -const requestHeartbeatNowMock = vi.hoisted(() => vi.fn()); +const requestHeartbeatMock = vi.hoisted(() => vi.fn()); const enqueueSystemEventMock = vi.hoisted(() => vi.fn()); const supervisorMock = vi.hoisted(() => ({ spawn: vi.fn(), })); vi.mock("../infra/heartbeat-wake.js", () => ({ - requestHeartbeatNow: requestHeartbeatNowMock, + requestHeartbeat: requestHeartbeatMock, })); vi.mock("../infra/system-events.js", () => ({ @@ -43,7 +43,7 @@ beforeAll(async () => { }); beforeEach(() => { - requestHeartbeatNowMock.mockClear(); + requestHeartbeatMock.mockClear(); enqueueSystemEventMock.mockClear(); supervisorMock.spawn.mockReset(); }); @@ -392,7 +392,7 @@ describe("exec notifyOnExit suppression", () => { expect(outcome.status).toBe("failed"); expect(enqueueSystemEventMock).not.toHaveBeenCalled(); - expect(requestHeartbeatNowMock).not.toHaveBeenCalled(); + expect(requestHeartbeatMock).not.toHaveBeenCalled(); }); it("notifies for manual-cancelled background execs with output", async () => { @@ -402,7 +402,7 @@ describe("exec notifyOnExit suppression", () => { expect.stringContaining("partial output"), expect.objectContaining({ sessionKey: "agent:main:main" }), ); - expect(requestHeartbeatNowMock).toHaveBeenCalled(); + expect(requestHeartbeatMock).toHaveBeenCalled(); }); it("still notifies for no-output background exec timeouts", async () => { @@ -412,13 +412,13 @@ describe("exec notifyOnExit suppression", () => { expect.stringContaining("Exec failed"), expect.objectContaining({ sessionKey: "agent:main:main" }), ); - expect(requestHeartbeatNowMock).toHaveBeenCalled(); + expect(requestHeartbeatMock).toHaveBeenCalled(); }); }); describe("emitExecSystemEvent", () => { beforeEach(() => { - requestHeartbeatNowMock.mockClear(); + requestHeartbeatMock.mockClear(); enqueueSystemEventMock.mockClear(); }); @@ -442,7 +442,7 @@ describe("emitExecSystemEvent", () => { threadId: 47, }, }); - expect(requestHeartbeatNowMock).toHaveBeenCalledWith( + expect(requestHeartbeatMock).toHaveBeenCalledWith( expect.objectContaining({ coalesceMs: 0, reason: "exec-event", @@ -461,7 +461,7 @@ describe("emitExecSystemEvent", () => { sessionKey: "global", contextKey: "exec:run-global", }); - expect(requestHeartbeatNowMock).toHaveBeenCalledWith( + expect(requestHeartbeatMock).toHaveBeenCalledWith( expect.objectContaining({ coalesceMs: 0, reason: "exec-event", @@ -476,7 +476,7 @@ describe("emitExecSystemEvent", () => { }); expect(enqueueSystemEventMock).not.toHaveBeenCalled(); - expect(requestHeartbeatNowMock).not.toHaveBeenCalled(); + expect(requestHeartbeatMock).not.toHaveBeenCalled(); }); }); diff --git a/src/agents/bash-tools.exec-runtime.ts b/src/agents/bash-tools.exec-runtime.ts index 2ea8c9788e5..059f4c498a0 100644 --- a/src/agents/bash-tools.exec-runtime.ts +++ b/src/agents/bash-tools.exec-runtime.ts @@ -8,7 +8,7 @@ import { type ExecApprovalDecision, type ExecTarget, } from "../infra/exec-approvals.js"; -import { requestHeartbeatNow } from "../infra/heartbeat-wake.js"; +import { requestHeartbeat } from "../infra/heartbeat-wake.js"; import { isDangerousHostInheritedEnvVarName } from "../infra/host-env-security.js"; import { findPathKey, mergePathPrepend } from "../infra/path-prepend.js"; import { enqueueSystemEvent } from "../infra/system-events.js"; @@ -344,8 +344,13 @@ function maybeNotifyOnExit(session: ProcessSession, status: "completed" | "faile deliveryContext: session.notifyDeliveryContext, trusted: false, }); - requestHeartbeatNow( - scopedHeartbeatWakeOptions(sessionKey, { reason: "exec-event", coalesceMs: 0 }), + requestHeartbeat( + scopedHeartbeatWakeOptions(sessionKey, { + source: "exec-event", + intent: "event", + reason: "exec-event", + coalesceMs: 0, + }), ); } @@ -422,8 +427,13 @@ export function emitExecSystemEvent( contextKey: opts.contextKey, deliveryContext: opts.deliveryContext, }); - requestHeartbeatNow( - scopedHeartbeatWakeOptions(sessionKey, { reason: "exec-event", coalesceMs: 0 }), + requestHeartbeat( + scopedHeartbeatWakeOptions(sessionKey, { + source: "exec-event", + intent: "event", + reason: "exec-event", + coalesceMs: 0, + }), ); } diff --git a/src/agents/cli-runner.reliability.test.ts b/src/agents/cli-runner.reliability.test.ts index ce3a74e73fb..eacb7e5014c 100644 --- a/src/agents/cli-runner.reliability.test.ts +++ b/src/agents/cli-runner.reliability.test.ts @@ -14,7 +14,7 @@ import { runPreparedCliAgent } from "./cli-runner.js"; import { createManagedRun, enqueueSystemEventMock, - requestHeartbeatNowMock, + requestHeartbeatMock, supervisorSpawnMock, } from "./cli-runner.test-support.js"; import { executePreparedCliRun } from "./cli-runner/execute.js"; @@ -235,7 +235,9 @@ describe("runCliAgent reliability", () => { expect(String(notice)).toContain("produced no output"); expect(String(notice)).toContain("interactive input or an approval prompt"); expect(opts).toMatchObject({ sessionKey: "agent:main:main" }); - expect(requestHeartbeatNowMock).toHaveBeenCalledWith({ + expect(requestHeartbeatMock).toHaveBeenCalledWith({ + source: "cli-watchdog", + intent: "event", reason: "cli:watchdog:stall", sessionKey: "agent:main:main", }); diff --git a/src/agents/cli-runner.test-support.ts b/src/agents/cli-runner.test-support.ts index baf5db15bb5..aa9b4af2390 100644 --- a/src/agents/cli-runner.test-support.ts +++ b/src/agents/cli-runner.test-support.ts @@ -1,6 +1,6 @@ import type { Mock } from "vitest"; import { beforeEach, vi } from "vitest"; -import type { requestHeartbeatNow } from "../infra/heartbeat-wake.js"; +import type { requestHeartbeat } from "../infra/heartbeat-wake.js"; import type { enqueueSystemEvent } from "../infra/system-events.js"; import type { getProcessSupervisor } from "../process/supervisor/index.js"; import { setCliRunnerExecuteTestDeps } from "./cli-runner/execute.js"; @@ -11,7 +11,7 @@ import type { WorkspaceBootstrapFile } from "./workspace.js"; type ProcessSupervisor = ReturnType; type SupervisorSpawnFn = ProcessSupervisor["spawn"]; type EnqueueSystemEventFn = typeof enqueueSystemEvent; -type RequestHeartbeatNowFn = typeof requestHeartbeatNow; +type RequestHeartbeatFn = typeof requestHeartbeat; type UnknownMock = Mock<(...args: unknown[]) => unknown>; type BootstrapContext = { bootstrapFiles: WorkspaceBootstrapFile[]; @@ -21,7 +21,7 @@ type ResolveBootstrapContextForRunMock = Mock<() => Promise>; export const supervisorSpawnMock: UnknownMock = vi.fn(); export const enqueueSystemEventMock: UnknownMock = vi.fn(); -export const requestHeartbeatNowMock: UnknownMock = vi.fn(); +export const requestHeartbeatMock: UnknownMock = vi.fn(); const hoisted = vi.hoisted( (): { @@ -49,8 +49,8 @@ setCliRunnerExecuteTestDeps({ text: Parameters[0], options: Parameters[1], ) => enqueueSystemEventMock(text, options) as ReturnType, - requestHeartbeatNow: (options?: Parameters[0]) => - requestHeartbeatNowMock(options) as ReturnType, + requestHeartbeat: (options?: Parameters[0]) => + requestHeartbeatMock(options) as ReturnType, }); setCliRunnerPrepareTestDeps({ diff --git a/src/agents/cli-runner/execute.ts b/src/agents/cli-runner/execute.ts index 360f8268005..43659f5d54e 100644 --- a/src/agents/cli-runner/execute.ts +++ b/src/agents/cli-runner/execute.ts @@ -2,7 +2,7 @@ import crypto from "node:crypto"; import { shouldLogVerbose } from "../../globals.js"; import { emitAgentEvent } from "../../infra/agent-events.js"; import { isTruthyEnvValue } from "../../infra/env.js"; -import { requestHeartbeatNow as requestHeartbeatNowImpl } from "../../infra/heartbeat-wake.js"; +import { requestHeartbeat as requestHeartbeatImpl } from "../../infra/heartbeat-wake.js"; import { sanitizeHostExecEnv } from "../../infra/host-env-security.js"; import { enqueueSystemEvent as enqueueSystemEventImpl } from "../../infra/system-events.js"; import { getProcessSupervisor as getProcessSupervisorImpl } from "../../process/supervisor/index.js"; @@ -42,7 +42,7 @@ import type { PreparedCliRunContext } from "./types.js"; const executeDeps = { getProcessSupervisor: getProcessSupervisorImpl, enqueueSystemEvent: enqueueSystemEventImpl, - requestHeartbeatNow: requestHeartbeatNowImpl, + requestHeartbeat: requestHeartbeatImpl, }; export function setCliRunnerExecuteTestDeps(overrides: Partial): void { @@ -545,8 +545,12 @@ export async function executePreparedCliRun( "For Claude Code, prefer --permission-mode bypassPermissions --print.", ].join(" "); executeDeps.enqueueSystemEvent(stallNotice, { sessionKey: params.sessionKey }); - executeDeps.requestHeartbeatNow( - scopedHeartbeatWakeOptions(params.sessionKey, { reason: "cli:watchdog:stall" }), + executeDeps.requestHeartbeat( + scopedHeartbeatWakeOptions(params.sessionKey, { + source: "cli-watchdog", + intent: "event", + reason: "cli:watchdog:stall", + }), ); } throw new FailoverError(timeoutReason, { diff --git a/src/cli/deps.ts b/src/cli/deps.ts index 8f0ece675b9..f595f6da075 100644 --- a/src/cli/deps.ts +++ b/src/cli/deps.ts @@ -33,7 +33,7 @@ const NON_CHANNEL_DEP_KEYS = new Set([ "migrateOrphanedSessionKeys", "nowMs", "onEvent", - "requestHeartbeatNow", + "requestHeartbeat", "resolveSessionStorePath", "runHeartbeatOnce", "runIsolatedAgentJob", diff --git a/src/cron/service.armtimer-tight-loop.test.ts b/src/cron/service.armtimer-tight-loop.test.ts index c8cca51af98..87e74b84387 100644 --- a/src/cron/service.armtimer-tight-loop.test.ts +++ b/src/cron/service.armtimer-tight-loop.test.ts @@ -57,7 +57,7 @@ describe("CronService - armTimer tight loop prevention", () => { log: noopLogger, nowMs: () => params.now, enqueueSystemEvent: vi.fn(), - requestHeartbeatNow: vi.fn(), + requestHeartbeat: vi.fn(), runIsolatedAgentJob: params.runIsolatedAgentJob ?? vi.fn().mockResolvedValue({ status: "ok" }), }); diff --git a/src/cron/service.delivery-plan.test.ts b/src/cron/service.delivery-plan.test.ts index f118a08f22b..d12363d1b27 100644 --- a/src/cron/service.delivery-plan.test.ts +++ b/src/cron/service.delivery-plan.test.ts @@ -25,7 +25,7 @@ async function withCronService( run: (context: { cron: CronService; enqueueSystemEvent: ReturnType; - requestHeartbeatNow: ReturnType; + requestHeartbeat: ReturnType; }) => Promise, ) { await withCronServiceForTest( @@ -108,7 +108,7 @@ describe("CronService delivery plan consistency", () => { delivered: true, })), }, - async ({ cron, enqueueSystemEvent, requestHeartbeatNow }) => { + async ({ cron, enqueueSystemEvent, requestHeartbeat }) => { const job = await addIsolatedAgentTurnJob(cron, { name: "announce-delivered", wakeMode: "now", @@ -118,7 +118,7 @@ describe("CronService delivery plan consistency", () => { const result = await cron.run(job.id, "force"); expect(result).toEqual({ ok: true, ran: true }); expect(enqueueSystemEvent).not.toHaveBeenCalled(); - expect(requestHeartbeatNow).not.toHaveBeenCalled(); + expect(requestHeartbeat).not.toHaveBeenCalled(); }, ); }); diff --git a/src/cron/service.every-jobs-fire.test.ts b/src/cron/service.every-jobs-fire.test.ts index 0a6f878755a..2cbdffbe096 100644 --- a/src/cron/service.every-jobs-fire.test.ts +++ b/src/cron/service.every-jobs-fire.test.ts @@ -116,7 +116,7 @@ describe("CronService interval/cron jobs fire on time", () => { it("keeps legacy every jobs due while minute cron jobs recompute schedules", async () => { const store = await makeStorePath(); const enqueueSystemEvent = vi.fn(); - const requestHeartbeatNow = vi.fn(); + const requestHeartbeat = vi.fn(); const nowMs = Date.parse("2025-12-13T00:00:00.000Z"); await writeCronStoreSnapshot({ @@ -154,7 +154,7 @@ describe("CronService interval/cron jobs fire on time", () => { cronEnabled: true, log: noopLogger, enqueueSystemEvent, - requestHeartbeatNow, + requestHeartbeat, runIsolatedAgentJob: vi.fn(async () => ({ status: "ok" as const })), }); diff --git a/src/cron/service.failure-alert.test.ts b/src/cron/service.failure-alert.test.ts index cb2f7f4654c..c6b09c7f8f7 100644 --- a/src/cron/service.failure-alert.test.ts +++ b/src/cron/service.failure-alert.test.ts @@ -35,7 +35,7 @@ function createFailureAlertCron(params: { cronConfig: params.cronConfig, log: noopLogger, enqueueSystemEvent: vi.fn(), - requestHeartbeatNow: vi.fn(), + requestHeartbeat: vi.fn(), runIsolatedAgentJob: params.runIsolatedAgentJob, sendCronFailureAlert: params.sendCronFailureAlert, }); diff --git a/src/cron/service.get-job.test.ts b/src/cron/service.get-job.test.ts index c03a1380ab2..93fb0d3a561 100644 --- a/src/cron/service.get-job.test.ts +++ b/src/cron/service.get-job.test.ts @@ -16,7 +16,7 @@ function createCronService(storePath: string) { cronEnabled: true, log: logger, enqueueSystemEvent: vi.fn(), - requestHeartbeatNow: vi.fn(), + requestHeartbeat: vi.fn(), runIsolatedAgentJob: vi.fn(async () => ({ status: "ok" as const })), }); } diff --git a/src/cron/service.heartbeat-ok-summary-suppressed.test.ts b/src/cron/service.heartbeat-ok-summary-suppressed.test.ts index d2a620e1439..aae72e9fd20 100644 --- a/src/cron/service.heartbeat-ok-summary-suppressed.test.ts +++ b/src/cron/service.heartbeat-ok-summary-suppressed.test.ts @@ -32,14 +32,14 @@ function createCronServiceForSummary(params: { storePath: string; summary: string; enqueueSystemEvent: CronServiceParams["enqueueSystemEvent"]; - requestHeartbeatNow: CronServiceParams["requestHeartbeatNow"]; + requestHeartbeat: CronServiceParams["requestHeartbeat"]; }) { return new CronService({ storePath: params.storePath, cronEnabled: true, log: logger, enqueueSystemEvent: params.enqueueSystemEvent, - requestHeartbeatNow: params.requestHeartbeatNow, + requestHeartbeat: params.requestHeartbeat, runHeartbeatOnce: vi.fn(), runIsolatedAgentJob: vi.fn(async () => ({ status: "ok" as const, @@ -71,19 +71,19 @@ describe("cron isolated job HEARTBEAT_OK summary suppression (#32013)", () => { await writeCronStoreSnapshot({ storePath, jobs: [job] }); const enqueueSystemEvent = vi.fn(); - const requestHeartbeatNow = vi.fn(); + const requestHeartbeat = vi.fn(); const cron = createCronServiceForSummary({ storePath, summary: "HEARTBEAT_OK", enqueueSystemEvent, - requestHeartbeatNow, + requestHeartbeat, }); await runScheduledCron(cron); // HEARTBEAT_OK should NOT leak into the main session as a system event. expect(enqueueSystemEvent).not.toHaveBeenCalled(); - expect(requestHeartbeatNow).not.toHaveBeenCalled(); + expect(requestHeartbeat).not.toHaveBeenCalled(); }); it("does not revive legacy main-session relay for real cron summaries", async () => { @@ -99,17 +99,17 @@ describe("cron isolated job HEARTBEAT_OK summary suppression (#32013)", () => { await writeCronStoreSnapshot({ storePath, jobs: [job] }); const enqueueSystemEvent = vi.fn(); - const requestHeartbeatNow = vi.fn(); + const requestHeartbeat = vi.fn(); const cron = createCronServiceForSummary({ storePath, summary: "Weather update: sunny, 72°F", enqueueSystemEvent, - requestHeartbeatNow, + requestHeartbeat, }); await runScheduledCron(cron); expect(enqueueSystemEvent).not.toHaveBeenCalled(); - expect(requestHeartbeatNow).not.toHaveBeenCalled(); + expect(requestHeartbeat).not.toHaveBeenCalled(); }); }); diff --git a/src/cron/service.issue-16156-list-skips-cron.test.ts b/src/cron/service.issue-16156-list-skips-cron.test.ts index c0cda6d20bd..614ac940515 100644 --- a/src/cron/service.issue-16156-list-skips-cron.test.ts +++ b/src/cron/service.issue-16156-list-skips-cron.test.ts @@ -23,7 +23,7 @@ function createCronFromStorePath(storePath: string) { cronEnabled: true, log: noopLogger, enqueueSystemEvent: vi.fn(), - requestHeartbeatNow: vi.fn(), + requestHeartbeat: vi.fn(), runIsolatedAgentJob: vi.fn(async () => ({ status: "ok" as const })), }); } diff --git a/src/cron/service.issue-35195-backup-timing.test.ts b/src/cron/service.issue-35195-backup-timing.test.ts index e1831bff748..a91706908a6 100644 --- a/src/cron/service.issue-35195-backup-timing.test.ts +++ b/src/cron/service.issue-35195-backup-timing.test.ts @@ -34,7 +34,7 @@ describe("cron backup timing for edit", () => { cronEnabled: true, log: noopLogger, enqueueSystemEvent: vi.fn(), - requestHeartbeatNow: vi.fn(), + requestHeartbeat: vi.fn(), runIsolatedAgentJob: vi.fn(async () => ({ status: "ok" as const })), }); @@ -66,7 +66,7 @@ describe("cron backup timing for edit", () => { cronEnabled: true, log: noopLogger, enqueueSystemEvent: vi.fn(), - requestHeartbeatNow: vi.fn(), + requestHeartbeat: vi.fn(), runIsolatedAgentJob: vi.fn(async () => ({ status: "ok" as const })), }); diff --git a/src/cron/service.issue-66019-unresolved-next-run.test.ts b/src/cron/service.issue-66019-unresolved-next-run.test.ts index 6927e47e373..52d958d66df 100644 --- a/src/cron/service.issue-66019-unresolved-next-run.test.ts +++ b/src/cron/service.issue-66019-unresolved-next-run.test.ts @@ -34,7 +34,7 @@ function createIssue66019State(params: { log: noopLogger, nowMs: params.nowMs, enqueueSystemEvent: vi.fn(), - requestHeartbeatNow: vi.fn(), + requestHeartbeat: vi.fn(), runIsolatedAgentJob: params.runIsolatedAgentJob, }); } diff --git a/src/cron/service.issue-regressions.test-helpers.ts b/src/cron/service.issue-regressions.test-helpers.ts index 7c2ca29d1be..3e4cdb85fad 100644 --- a/src/cron/service.issue-regressions.test-helpers.ts +++ b/src/cron/service.issue-regressions.test-helpers.ts @@ -35,14 +35,14 @@ export async function startCronForStore(params: { storePath: string; cronEnabled?: boolean; enqueueSystemEvent?: CronServiceOptions["enqueueSystemEvent"]; - requestHeartbeatNow?: CronServiceOptions["requestHeartbeatNow"]; + requestHeartbeat?: CronServiceOptions["requestHeartbeat"]; runIsolatedAgentJob?: CronServiceOptions["runIsolatedAgentJob"]; onEvent?: CronServiceOptions["onEvent"]; }) { const enqueueSystemEvent = params.enqueueSystemEvent ?? (vi.fn() as unknown as CronServiceOptions["enqueueSystemEvent"]); - const requestHeartbeatNow = - params.requestHeartbeatNow ?? (vi.fn() as unknown as CronServiceOptions["requestHeartbeatNow"]); + const requestHeartbeat = + params.requestHeartbeat ?? (vi.fn() as unknown as CronServiceOptions["requestHeartbeat"]); const runIsolatedAgentJob = params.runIsolatedAgentJob ?? createDefaultIsolatedRunner(); const cron = new CronService({ @@ -50,7 +50,7 @@ export async function startCronForStore(params: { storePath: params.storePath, log: noopLogger, enqueueSystemEvent, - requestHeartbeatNow, + requestHeartbeat, runIsolatedAgentJob, ...(params.onEvent ? { onEvent: params.onEvent } : {}), }); diff --git a/src/cron/service.issue-regressions.test.ts b/src/cron/service.issue-regressions.test.ts index 49595cb593b..cacaa2cfd13 100644 --- a/src/cron/service.issue-regressions.test.ts +++ b/src/cron/service.issue-regressions.test.ts @@ -82,7 +82,7 @@ describe("Cron issue regressions", () => { storePath: store.storePath, log: noopLogger, enqueueSystemEvent: vi.fn(), - requestHeartbeatNow: vi.fn(), + requestHeartbeat: vi.fn(), runIsolatedAgentJob: vi.fn().mockResolvedValue({ status: "ok", summary: "ok" }), }); await cron.start(); diff --git a/src/cron/service.main-job-passes-heartbeat-target-last.test.ts b/src/cron/service.main-job-passes-heartbeat-target-last.test.ts index 460d44f3787..13f78c4f01d 100644 --- a/src/cron/service.main-job-passes-heartbeat-target-last.test.ts +++ b/src/cron/service.main-job-passes-heartbeat-target-last.test.ts @@ -33,17 +33,17 @@ describe("cron main job passes heartbeat target=last", () => { function createCronWithSpies(params: { storePath: string; runHeartbeatOnce: RunHeartbeatOnce }) { const enqueueSystemEvent = vi.fn(); - const requestHeartbeatNow = vi.fn(); + const requestHeartbeat = vi.fn(); const cron = new CronService({ storePath: params.storePath, cronEnabled: true, log: logger, enqueueSystemEvent, - requestHeartbeatNow, + requestHeartbeat, runHeartbeatOnce: params.runHeartbeatOnce, runIsolatedAgentJob: vi.fn(async () => ({ status: "ok" as const })), }); - return { cron, requestHeartbeatNow }; + return { cron, requestHeartbeat }; } async function runSingleTick(cron: CronService) { @@ -89,7 +89,7 @@ describe("cron main job passes heartbeat target=last", () => { expect(callArgs?.heartbeat?.target).toBe("last"); }); - it("should preserve heartbeat.target=last when wakeMode=now falls back to requestHeartbeatNow", async () => { + it("should preserve heartbeat.target=last when wakeMode=now falls back to requestHeartbeat", async () => { const { storePath } = await makeStorePath(); const now = Date.now(); @@ -106,7 +106,7 @@ describe("cron main job passes heartbeat target=last", () => { reason: "cron-in-progress", })); - const { cron, requestHeartbeatNow } = createCronWithSpies({ + const { cron, requestHeartbeat } = createCronWithSpies({ storePath, runHeartbeatOnce, }); @@ -114,8 +114,10 @@ describe("cron main job passes heartbeat target=last", () => { await runSingleTick(cron); expect(runHeartbeatOnce).toHaveBeenCalled(); - expect(requestHeartbeatNow).toHaveBeenCalledWith( + expect(requestHeartbeat).toHaveBeenCalledWith( expect.objectContaining({ + source: "cron", + intent: "immediate", reason: "cron:test-main-delivery-busy", heartbeat: { target: "last" }, }), @@ -139,16 +141,18 @@ describe("cron main job passes heartbeat target=last", () => { durationMs: 50, })); - const { cron, requestHeartbeatNow } = createCronWithSpies({ + const { cron, requestHeartbeat } = createCronWithSpies({ storePath, runHeartbeatOnce, }); await runSingleTick(cron); - expect(requestHeartbeatNow).toHaveBeenCalled(); - expect(requestHeartbeatNow).toHaveBeenCalledWith( + expect(requestHeartbeat).toHaveBeenCalled(); + expect(requestHeartbeat).toHaveBeenCalledWith( expect.objectContaining({ + source: "cron", + intent: "event", reason: "cron:test-next-heartbeat", heartbeat: { target: "last" }, }), diff --git a/src/cron/service.persists-delivered-status.test.ts b/src/cron/service.persists-delivered-status.test.ts index 5f5746b660a..c0a6fec0d6e 100644 --- a/src/cron/service.persists-delivered-status.test.ts +++ b/src/cron/service.persists-delivered-status.test.ts @@ -56,7 +56,7 @@ function createIsolatedCronWithFinishedBarrier(params: { cronEnabled: true, log: noopLogger, enqueueSystemEvent: vi.fn(), - requestHeartbeatNow: vi.fn(), + requestHeartbeat: vi.fn(), runIsolatedAgentJob: vi.fn(async () => ({ status: "ok" as const, summary: "done", diff --git a/src/cron/service.prevents-duplicate-timers.test.ts b/src/cron/service.prevents-duplicate-timers.test.ts index c5b2f348403..ba7f71ee897 100644 --- a/src/cron/service.prevents-duplicate-timers.test.ts +++ b/src/cron/service.prevents-duplicate-timers.test.ts @@ -17,7 +17,7 @@ describe("CronService", () => { it("avoids duplicate runs when two services share a store", async () => { const store = await makeStorePath(); const enqueueSystemEvent = vi.fn(); - const requestHeartbeatNow = vi.fn(); + const requestHeartbeat = vi.fn(); const runIsolatedAgentJob = vi.fn(async () => ({ status: "ok" as const })); const cronA = new CronService({ @@ -25,7 +25,7 @@ describe("CronService", () => { cronEnabled: true, log: noopLogger, enqueueSystemEvent, - requestHeartbeatNow, + requestHeartbeat, runIsolatedAgentJob, }); @@ -45,7 +45,7 @@ describe("CronService", () => { cronEnabled: true, log: noopLogger, enqueueSystemEvent, - requestHeartbeatNow, + requestHeartbeat, runIsolatedAgentJob, }); @@ -57,7 +57,7 @@ describe("CronService", () => { await cronB.status(); expect(enqueueSystemEvent).toHaveBeenCalledTimes(1); - expect(requestHeartbeatNow).toHaveBeenCalledTimes(1); + expect(requestHeartbeat).toHaveBeenCalledTimes(1); cronA.stop(); cronB.stop(); diff --git a/src/cron/service.read-ops-nonblocking.test.ts b/src/cron/service.read-ops-nonblocking.test.ts index f6d55ec30de..311dffb995c 100644 --- a/src/cron/service.read-ops-nonblocking.test.ts +++ b/src/cron/service.read-ops-nonblocking.test.ts @@ -82,7 +82,7 @@ describe("CronService read ops while job is running", () => { vi.setSystemTime(new Date("2025-12-13T00:00:00.000Z")); const store = await makeStorePath(); const enqueueSystemEvent = vi.fn(); - const requestHeartbeatNow = vi.fn(); + const requestHeartbeat = vi.fn(); let resolveFinished: (() => void) | undefined; const finished = new Promise((resolve) => { resolveFinished = resolve; @@ -95,7 +95,7 @@ describe("CronService read ops while job is running", () => { cronEnabled: true, log: noopLogger, enqueueSystemEvent, - requestHeartbeatNow, + requestHeartbeat, runIsolatedAgentJob: isolatedRun.runIsolatedAgentJob, onEvent: (evt) => { if (evt.action === "finished" && evt.status === "ok") { @@ -164,7 +164,7 @@ describe("CronService read ops while job is running", () => { it("keeps list and status responsive during manual cron.run execution", async () => { const store = await makeStorePath(); const enqueueSystemEvent = vi.fn(); - const requestHeartbeatNow = vi.fn(); + const requestHeartbeat = vi.fn(); const isolatedRun = createDeferredIsolatedRun(); const cron = new CronService({ @@ -172,7 +172,7 @@ describe("CronService read ops while job is running", () => { cronEnabled: true, log: noopLogger, enqueueSystemEvent, - requestHeartbeatNow, + requestHeartbeat, runIsolatedAgentJob: isolatedRun.runIsolatedAgentJob, }); @@ -217,7 +217,7 @@ describe("CronService read ops while job is running", () => { it("keeps list and status responsive after startup defers catch-up runs", async () => { const store = await makeStorePath(); const enqueueSystemEvent = vi.fn(); - const requestHeartbeatNow = vi.fn(); + const requestHeartbeat = vi.fn(); const nowMs = Date.parse("2025-12-13T00:00:00.000Z"); await writeCronStoreSnapshot({ @@ -247,7 +247,7 @@ describe("CronService read ops while job is running", () => { log: noopLogger, nowMs: () => nowMs, enqueueSystemEvent, - requestHeartbeatNow, + requestHeartbeat, runIsolatedAgentJob: isolatedRun.runIsolatedAgentJob, startupDeferredMissedAgentJobDelayMs: 120_000, }); diff --git a/src/cron/service.rearm-timer-when-running.test.ts b/src/cron/service.rearm-timer-when-running.test.ts index aac531d85f5..be9be3d3704 100644 --- a/src/cron/service.rearm-timer-when-running.test.ts +++ b/src/cron/service.rearm-timer-when-running.test.ts @@ -125,7 +125,7 @@ describe("CronService - timer re-arm when running (#12025)", () => { log: noopLogger, nowMs: () => now, enqueueSystemEvent: vi.fn(), - requestHeartbeatNow: vi.fn(), + requestHeartbeat: vi.fn(), runIsolatedAgentJob: vi.fn(async () => await deferredRun.promise), }); diff --git a/src/cron/service.restart-catchup.test.ts b/src/cron/service.restart-catchup.test.ts index b16260176c9..3cedc26d1ef 100644 --- a/src/cron/service.restart-catchup.test.ts +++ b/src/cron/service.restart-catchup.test.ts @@ -21,7 +21,7 @@ describe("CronService restart catch-up", () => { function createRestartCronService(params: { storePath: string; enqueueSystemEvent: ReturnType; - requestHeartbeatNow: ReturnType; + requestHeartbeat: ReturnType; onEvent?: ReturnType; nowMs?: () => number; runIsolatedAgentJob?: ReturnType; @@ -33,7 +33,7 @@ describe("CronService restart catch-up", () => { log: noopLogger, ...(params.nowMs ? { nowMs: params.nowMs } : {}), enqueueSystemEvent: params.enqueueSystemEvent as never, - requestHeartbeatNow: params.requestHeartbeatNow as never, + requestHeartbeat: params.requestHeartbeat as never, runIsolatedAgentJob: (params.runIsolatedAgentJob as never) ?? (vi.fn(async () => ({ status: "ok" as const })) as never), @@ -64,13 +64,13 @@ describe("CronService restart catch-up", () => { run: (params: { cron: CronService; enqueueSystemEvent: ReturnType; - requestHeartbeatNow: ReturnType; + requestHeartbeat: ReturnType; onEvent: ReturnType; }) => Promise, ) { const store = await makeStorePath(); const enqueueSystemEvent = vi.fn(); - const requestHeartbeatNow = vi.fn(); + const requestHeartbeat = vi.fn(); const onEvent = vi.fn(); await writeStoreJobs(store.storePath, jobs); @@ -78,13 +78,13 @@ describe("CronService restart catch-up", () => { const cron = createRestartCronService({ storePath: store.storePath, enqueueSystemEvent, - requestHeartbeatNow, + requestHeartbeat, onEvent, }); try { await cron.start(); - await run({ cron, enqueueSystemEvent, requestHeartbeatNow, onEvent }); + await run({ cron, enqueueSystemEvent, requestHeartbeat, onEvent }); } finally { cron.stop(); await store.cleanup(); @@ -114,12 +114,12 @@ describe("CronService restart catch-up", () => { }, }, ], - async ({ cron, enqueueSystemEvent, requestHeartbeatNow }) => { + async ({ cron, enqueueSystemEvent, requestHeartbeat }) => { expect(enqueueSystemEvent).toHaveBeenCalledWith( "digest now", expect.objectContaining({ agentId: undefined }), ); - expect(requestHeartbeatNow).toHaveBeenCalled(); + expect(requestHeartbeat).toHaveBeenCalled(); const listedJobs = await cron.list({ includeDisabled: true }); const updated = listedJobs.find((job) => job.id === "restart-overdue-job"); @@ -135,7 +135,7 @@ describe("CronService restart catch-up", () => { const startNow = Date.parse("2025-12-13T17:00:00.000Z"); const runIsolatedAgentJob = vi.fn(async () => ({ status: "ok" as const })); const enqueueSystemEvent = vi.fn(); - const requestHeartbeatNow = vi.fn(); + const requestHeartbeat = vi.fn(); await writeStoreJobs(store.storePath, [ { @@ -155,7 +155,7 @@ describe("CronService restart catch-up", () => { const cron = createRestartCronService({ storePath: store.storePath, enqueueSystemEvent, - requestHeartbeatNow, + requestHeartbeat, runIsolatedAgentJob, nowMs: () => startNow, startupDeferredMissedAgentJobDelayMs: 120_000, @@ -166,7 +166,7 @@ describe("CronService restart catch-up", () => { expect(runIsolatedAgentJob).not.toHaveBeenCalled(); expect(enqueueSystemEvent).not.toHaveBeenCalled(); - expect(requestHeartbeatNow).not.toHaveBeenCalled(); + expect(requestHeartbeat).not.toHaveBeenCalled(); const listedJobs = await cron.list({ includeDisabled: true }); const updated = listedJobs.find((job) => job.id === "startup-isolated-agent"); @@ -201,14 +201,14 @@ describe("CronService restart catch-up", () => { }, }, ], - async ({ cron, enqueueSystemEvent, requestHeartbeatNow, onEvent }) => { + async ({ cron, enqueueSystemEvent, requestHeartbeat, onEvent }) => { expect(noopLogger.warn).toHaveBeenCalledWith( expect.objectContaining({ jobId: "restart-stale-running" }), "cron: marking interrupted running job failed on startup", ); expect(enqueueSystemEvent).not.toHaveBeenCalled(); - expect(requestHeartbeatNow).not.toHaveBeenCalled(); + expect(requestHeartbeat).not.toHaveBeenCalled(); const listedJobs = await cron.list({ includeDisabled: true }); const updated = listedJobs.find((job) => job.id === "restart-stale-running"); @@ -253,12 +253,12 @@ describe("CronService restart catch-up", () => { }, }, ], - async ({ cron, enqueueSystemEvent, requestHeartbeatNow }) => { + async ({ cron, enqueueSystemEvent, requestHeartbeat }) => { expect(enqueueSystemEvent).toHaveBeenCalledWith( "catch missed slot", expect.objectContaining({ agentId: undefined }), ); - expect(requestHeartbeatNow).toHaveBeenCalled(); + expect(requestHeartbeat).toHaveBeenCalled(); const listedJobs = await cron.list({ includeDisabled: true }); const updated = listedJobs.find((job) => job.id === "restart-missed-slot"); @@ -289,9 +289,9 @@ describe("CronService restart catch-up", () => { }, }, ], - async ({ cron, enqueueSystemEvent, requestHeartbeatNow, onEvent }) => { + async ({ cron, enqueueSystemEvent, requestHeartbeat, onEvent }) => { expect(enqueueSystemEvent).not.toHaveBeenCalled(); - expect(requestHeartbeatNow).not.toHaveBeenCalled(); + expect(requestHeartbeat).not.toHaveBeenCalled(); const listedJobs = await cron.list({ includeDisabled: true }); const updated = listedJobs.find((job) => job.id === "restart-stale-one-shot"); @@ -336,9 +336,9 @@ describe("CronService restart catch-up", () => { }, }, ], - async ({ enqueueSystemEvent, requestHeartbeatNow }) => { + async ({ enqueueSystemEvent, requestHeartbeat }) => { expect(enqueueSystemEvent).not.toHaveBeenCalled(); - expect(requestHeartbeatNow).not.toHaveBeenCalled(); + expect(requestHeartbeat).not.toHaveBeenCalled(); }, ); }); @@ -366,9 +366,9 @@ describe("CronService restart catch-up", () => { }, }, ], - async ({ enqueueSystemEvent, requestHeartbeatNow }) => { + async ({ enqueueSystemEvent, requestHeartbeat }) => { expect(enqueueSystemEvent).not.toHaveBeenCalled(); - expect(requestHeartbeatNow).not.toHaveBeenCalled(); + expect(requestHeartbeat).not.toHaveBeenCalled(); }, ); }); @@ -397,12 +397,12 @@ describe("CronService restart catch-up", () => { }, }, ], - async ({ enqueueSystemEvent, requestHeartbeatNow }) => { + async ({ enqueueSystemEvent, requestHeartbeat }) => { expect(enqueueSystemEvent).toHaveBeenCalledWith( "replay after backoff elapsed", expect.objectContaining({ agentId: undefined }), ); - expect(requestHeartbeatNow).toHaveBeenCalled(); + expect(requestHeartbeat).toHaveBeenCalled(); }, ); }); @@ -424,7 +424,7 @@ describe("CronService restart catch-up", () => { log: noopLogger, nowMs: () => now, enqueueSystemEvent: vi.fn(), - requestHeartbeatNow: vi.fn(), + requestHeartbeat: vi.fn(), runIsolatedAgentJob: vi.fn(async () => { now += 6_000; return { status: "ok" as const, summary: "ok" }; diff --git a/src/cron/service.runs-one-shot-main-job-disables-it.test.ts b/src/cron/service.runs-one-shot-main-job-disables-it.test.ts index 6547c0d0603..eb29c51022d 100644 --- a/src/cron/service.runs-one-shot-main-job-disables-it.test.ts +++ b/src/cron/service.runs-one-shot-main-job-disables-it.test.ts @@ -232,7 +232,7 @@ async function createCronHarness(options: CronHarnessOptions = {}) { ensureDir(fixturesRoot); const store = await makeStorePath(); const enqueueSystemEvent = vi.fn(); - const requestHeartbeatNow = vi.fn(); + const requestHeartbeat = vi.fn(); const events = options.withEvents === false ? undefined : createCronEventHarness(); const cron = new CronService({ @@ -247,7 +247,7 @@ async function createCronHarness(options: CronHarnessOptions = {}) { ? { wakeNowHeartbeatBusyRetryDelayMs: options.wakeNowHeartbeatBusyRetryDelayMs } : {}), enqueueSystemEvent, - requestHeartbeatNow, + requestHeartbeat, ...(options.runHeartbeatOnce ? { runHeartbeatOnce: options.runHeartbeatOnce } : {}), runIsolatedAgentJob: options.runIsolatedAgentJob ?? @@ -257,7 +257,7 @@ async function createCronHarness(options: CronHarnessOptions = {}) { ...(events ? { onEvent: events.onEvent } : {}), }); await cron.start(); - return { store, cron, enqueueSystemEvent, requestHeartbeatNow, events }; + return { store, cron, enqueueSystemEvent, requestHeartbeat, events }; } async function createMainOneShotHarness() { @@ -390,7 +390,7 @@ function createStartedCronService( cronEnabled: true, log: noopLogger, enqueueSystemEvent: vi.fn(), - requestHeartbeatNow: vi.fn(), + requestHeartbeat: vi.fn(), runIsolatedAgentJob: runIsolatedAgentJob ?? vi.fn(async () => ({ status: "ok" as const })), }); } @@ -410,7 +410,7 @@ async function expectNoMainSummaryForIsolatedRun(params: { runIsolatedAgentJob: CronServiceDeps["runIsolatedAgentJob"]; name: string; }) { - const { store, cron, enqueueSystemEvent, requestHeartbeatNow, events } = + const { store, cron, enqueueSystemEvent, requestHeartbeat, events } = await createIsolatedAnnounceHarness(params.runIsolatedAgentJob); await runIsolatedAnnounceScenario({ cron, @@ -418,13 +418,13 @@ async function expectNoMainSummaryForIsolatedRun(params: { name: params.name, }); expect(enqueueSystemEvent).not.toHaveBeenCalled(); - expect(requestHeartbeatNow).not.toHaveBeenCalled(); + expect(requestHeartbeat).not.toHaveBeenCalled(); await stopCronAndCleanup(cron, store); } describe("CronService", () => { it("runs a one-shot main job and disables it after success when requested", async () => { - const { store, cron, enqueueSystemEvent, requestHeartbeatNow, events, atMs, job } = + const { store, cron, enqueueSystemEvent, requestHeartbeat, events, atMs, job } = await createMainOneShotJobHarness({ name: "one-shot hello", deleteAfterRun: false, @@ -440,14 +440,14 @@ describe("CronService", () => { const updated = jobs.find((j) => j.id === job.id); expect(updated?.enabled).toBe(false); expectMainSystemEventPosted(enqueueSystemEvent, "hello"); - expect(requestHeartbeatNow).toHaveBeenCalled(); + expect(requestHeartbeat).toHaveBeenCalled(); await cron.list({ includeDisabled: true }); await stopCronAndCleanup(cron, store); }); it("runs a one-shot job and deletes it after success by default", async () => { - const { store, cron, enqueueSystemEvent, requestHeartbeatNow, events, job } = + const { store, cron, enqueueSystemEvent, requestHeartbeat, events, job } = await createMainOneShotJobHarness({ name: "one-shot delete", }); @@ -459,7 +459,7 @@ describe("CronService", () => { const jobs = await cron.list({ includeDisabled: true }); expect(jobs.find((j) => j.id === job.id)).toBeUndefined(); expectMainSystemEventPosted(enqueueSystemEvent, "hello"); - expect(requestHeartbeatNow).toHaveBeenCalled(); + expect(requestHeartbeat).toHaveBeenCalled(); await stopCronAndCleanup(cron, store); }); @@ -480,7 +480,7 @@ describe("CronService", () => { }); }); - const { store, cron, enqueueSystemEvent, requestHeartbeatNow } = + const { store, cron, enqueueSystemEvent, requestHeartbeat } = await createWakeModeNowMainHarness({ runHeartbeatOnce, nowMs, @@ -491,7 +491,7 @@ describe("CronService", () => { await heartbeatStarted.promise; expect(runHeartbeatOnce).toHaveBeenCalledTimes(1); - expect(requestHeartbeatNow).not.toHaveBeenCalled(); + expect(requestHeartbeat).not.toHaveBeenCalled(); expectMainSystemEventPosted(enqueueSystemEvent, "hello"); expect(job.state.runningAtMs).toBeTypeOf("number"); @@ -536,7 +536,7 @@ describe("CronService", () => { return now; }; - const { store, cron, requestHeartbeatNow } = await createWakeModeNowMainHarness({ + const { store, cron, requestHeartbeat } = await createWakeModeNowMainHarness({ runHeartbeatOnce, nowMs, // Perf: avoid advancing fake timers by 2+ minutes for the busy-heartbeat fallback. @@ -553,7 +553,7 @@ describe("CronService", () => { await cron.run(job.id, "force"); expect(runHeartbeatOnce).toHaveBeenCalled(); - expect(requestHeartbeatNow).toHaveBeenCalledWith( + expect(requestHeartbeat).toHaveBeenCalledWith( expect.objectContaining({ reason: `cron:${job.id}`, sessionKey, @@ -572,7 +572,7 @@ describe("CronService", () => { reason: HEARTBEAT_SKIP_CRON_IN_PROGRESS, })); - const { store, cron, requestHeartbeatNow } = await createWakeModeNowMainHarness({ + const { store, cron, requestHeartbeat } = await createWakeModeNowMainHarness({ runHeartbeatOnce, }); @@ -585,7 +585,7 @@ describe("CronService", () => { await cron.run(job.id, "force"); expect(runHeartbeatOnce).toHaveBeenCalledTimes(1); - expect(requestHeartbeatNow).toHaveBeenCalledWith( + expect(requestHeartbeat).toHaveBeenCalledWith( expect.objectContaining({ reason: `cron:${job.id}`, sessionKey, @@ -600,12 +600,12 @@ describe("CronService", () => { it("runs an isolated job without posting a fallback summary to main", async () => { const runIsolatedAgentJob = vi.fn(async () => ({ status: "ok" as const, summary: "done" })); - const { store, cron, enqueueSystemEvent, requestHeartbeatNow, events } = + const { store, cron, enqueueSystemEvent, requestHeartbeat, events } = await createIsolatedAnnounceHarness(runIsolatedAgentJob); await runIsolatedAnnounceScenario({ cron, events, name: "weekly" }); expect(runIsolatedAgentJob).toHaveBeenCalledTimes(1); expect(enqueueSystemEvent).not.toHaveBeenCalled(); - expect(requestHeartbeatNow).not.toHaveBeenCalled(); + expect(requestHeartbeat).not.toHaveBeenCalled(); await stopCronAndCleanup(cron, store); }); @@ -642,7 +642,7 @@ describe("CronService", () => { summary: "last output", error: "boom", })); - const { store, cron, enqueueSystemEvent, requestHeartbeatNow, events } = + const { store, cron, enqueueSystemEvent, requestHeartbeat, events } = await createIsolatedAnnounceHarness(runIsolatedAgentJob); await runIsolatedAnnounceJobAndWait({ cron, @@ -652,7 +652,7 @@ describe("CronService", () => { }); expect(enqueueSystemEvent).not.toHaveBeenCalled(); - expect(requestHeartbeatNow).not.toHaveBeenCalled(); + expect(requestHeartbeat).not.toHaveBeenCalled(); await stopCronAndCleanup(cron, store); }); @@ -663,7 +663,7 @@ describe("CronService", () => { error: "Channel is required when multiple channels are configured: telegram, discord", errorKind: "delivery-target" as const, })); - const { store, cron, enqueueSystemEvent, requestHeartbeatNow, events } = + const { store, cron, enqueueSystemEvent, requestHeartbeat, events } = await createIsolatedAnnounceHarness(runIsolatedAgentJob); await runIsolatedAnnounceJobAndWait({ cron, @@ -673,7 +673,7 @@ describe("CronService", () => { }); expect(enqueueSystemEvent).not.toHaveBeenCalled(); - expect(requestHeartbeatNow).not.toHaveBeenCalled(); + expect(requestHeartbeat).not.toHaveBeenCalled(); await stopCronAndCleanup(cron, store); }); diff --git a/src/cron/service.session-reaper-in-finally.test.ts b/src/cron/service.session-reaper-in-finally.test.ts index f76b51219f5..8cec12738df 100644 --- a/src/cron/service.session-reaper-in-finally.test.ts +++ b/src/cron/service.session-reaper-in-finally.test.ts @@ -70,7 +70,7 @@ describe("CronService - session reaper runs in finally block (#31946)", () => { log: noopLogger, nowMs: () => now, enqueueSystemEvent: vi.fn(), - requestHeartbeatNow: vi.fn(), + requestHeartbeat: vi.fn(), // This will throw, simulating a failure during job execution. runIsolatedAgentJob: vi.fn().mockRejectedValue(new Error("gateway down")), sessionStorePath, @@ -109,7 +109,7 @@ describe("CronService - session reaper runs in finally block (#31946)", () => { log: noopLogger, nowMs: () => now, enqueueSystemEvent: vi.fn(), - requestHeartbeatNow: vi.fn(), + requestHeartbeat: vi.fn(), runIsolatedAgentJob: vi.fn().mockResolvedValue({ status: "ok", summary: "done" }), resolveSessionStorePath: (agentId) => { const p = path.join(path.dirname(store.storePath), `${agentId}-sessions`, "sessions.json"); @@ -156,7 +156,7 @@ describe("CronService - session reaper runs in finally block (#31946)", () => { log: noopLogger, nowMs: () => now, enqueueSystemEvent: vi.fn(), - requestHeartbeatNow: vi.fn(), + requestHeartbeat: vi.fn(), runIsolatedAgentJob: vi.fn(), sessionStorePath, }); diff --git a/src/cron/service.skips-main-jobs-empty-systemevent-text.test.ts b/src/cron/service.skips-main-jobs-empty-systemevent-text.test.ts index 4818677b135..3cac77f6db3 100644 --- a/src/cron/service.skips-main-jobs-empty-systemevent-text.test.ts +++ b/src/cron/service.skips-main-jobs-empty-systemevent-text.test.ts @@ -31,7 +31,7 @@ async function withCronService( run: (params: { cron: CronService; enqueueSystemEvent: ReturnType; - requestHeartbeatNow: ReturnType; + requestHeartbeat: ReturnType; }) => Promise, ) { await withCronServiceForTest( @@ -60,7 +60,7 @@ describe("CronService", () => { }); it("skips main jobs with empty systemEvent text", async () => { - await withCronService(true, async ({ cron, enqueueSystemEvent, requestHeartbeatNow }) => { + await withCronService(true, async ({ cron, enqueueSystemEvent, requestHeartbeat }) => { const atMs = Date.parse("2025-12-13T00:00:01.000Z"); await cron.add({ name: "empty systemEvent test", @@ -75,7 +75,7 @@ describe("CronService", () => { await vi.runOnlyPendingTimersAsync(); expect(enqueueSystemEvent).not.toHaveBeenCalled(); - expect(requestHeartbeatNow).not.toHaveBeenCalled(); + expect(requestHeartbeat).not.toHaveBeenCalled(); const job = await waitForFirstJob(cron, (current) => current?.state.lastStatus === "skipped"); expect(job?.state.lastStatus).toBe("skipped"); @@ -84,7 +84,7 @@ describe("CronService", () => { }); it("does not schedule timers when cron is disabled", async () => { - await withCronService(false, async ({ cron, enqueueSystemEvent, requestHeartbeatNow }) => { + await withCronService(false, async ({ cron, enqueueSystemEvent, requestHeartbeat }) => { const atMs = Date.parse("2025-12-13T00:00:01.000Z"); await cron.add({ name: "disabled cron job", @@ -103,7 +103,7 @@ describe("CronService", () => { await vi.runOnlyPendingTimersAsync(); expect(enqueueSystemEvent).not.toHaveBeenCalled(); - expect(requestHeartbeatNow).not.toHaveBeenCalled(); + expect(requestHeartbeat).not.toHaveBeenCalled(); expect(noopLogger.warn).toHaveBeenCalled(); }); }); diff --git a/src/cron/service.store-load-invalid-main-job.test.ts b/src/cron/service.store-load-invalid-main-job.test.ts index 906d735c97b..6b080affb73 100644 --- a/src/cron/service.store-load-invalid-main-job.test.ts +++ b/src/cron/service.store-load-invalid-main-job.test.ts @@ -36,7 +36,7 @@ describe("CronService store load", () => { const { dir, storePath } = await makeStorePath(); tempDir = dir; const enqueueSystemEvent = vi.fn(); - const requestHeartbeatNow = vi.fn(); + const requestHeartbeat = vi.fn(); const job = { id: "job-1", @@ -58,7 +58,7 @@ describe("CronService store load", () => { cronEnabled: true, log: noopLogger, enqueueSystemEvent, - requestHeartbeatNow, + requestHeartbeat, runIsolatedAgentJob: vi.fn(async () => ({ status: "ok" as const })), }); @@ -67,7 +67,7 @@ describe("CronService store load", () => { await cron.run("job-1", "due"); expect(enqueueSystemEvent).not.toHaveBeenCalled(); - expect(requestHeartbeatNow).not.toHaveBeenCalled(); + expect(requestHeartbeat).not.toHaveBeenCalled(); const jobs = await cron.list({ includeDisabled: true }); expect(jobs[0]?.state.lastStatus).toBe("skipped"); diff --git a/src/cron/service.test-harness.ts b/src/cron/service.test-harness.ts index 8888f0d2ab9..2e9706343e8 100644 --- a/src/cron/service.test-harness.ts +++ b/src/cron/service.test-harness.ts @@ -127,22 +127,22 @@ export function createStartedCronServiceWithFinishedBarrier(params: { }): { cron: CronService; enqueueSystemEvent: MockFn; - requestHeartbeatNow: MockFn; + requestHeartbeat: MockFn; finished: ReturnType; } { const enqueueSystemEvent = vi.fn(); - const requestHeartbeatNow = vi.fn(); + const requestHeartbeat = vi.fn(); const finished = createFinishedBarrier(); const cron = new CronService({ storePath: params.storePath, cronEnabled: true, log: params.logger, enqueueSystemEvent, - requestHeartbeatNow, + requestHeartbeat, runIsolatedAgentJob: vi.fn(async () => ({ status: "ok" as const })), onEvent: finished.onEvent, }); - return { cron, enqueueSystemEvent, requestHeartbeatNow, finished }; + return { cron, enqueueSystemEvent, requestHeartbeat, finished }; } export async function withCronServiceForTest( @@ -155,18 +155,18 @@ export async function withCronServiceForTest( run: (context: { cron: CronService; enqueueSystemEvent: ReturnType; - requestHeartbeatNow: ReturnType; + requestHeartbeat: ReturnType; }) => Promise, ): Promise { const store = await params.makeStorePath(); const enqueueSystemEvent = vi.fn(); - const requestHeartbeatNow = vi.fn(); + const requestHeartbeat = vi.fn(); const cron = new CronService({ cronEnabled: params.cronEnabled, storePath: store.storePath, log: params.logger, enqueueSystemEvent, - requestHeartbeatNow, + requestHeartbeat, runIsolatedAgentJob: params.runIsolatedAgentJob ?? (vi.fn(async () => ({ status: "ok" as const, summary: "done" })) as never), @@ -174,7 +174,7 @@ export async function withCronServiceForTest( await cron.start(); try { - await run({ cron, enqueueSystemEvent, requestHeartbeatNow }); + await run({ cron, enqueueSystemEvent, requestHeartbeat }); } finally { cron.stop(); await store.cleanup(); @@ -193,7 +193,7 @@ export function createRunningCronServiceState(params: { log: params.log, nowMs: params.nowMs, enqueueSystemEvent: vi.fn(), - requestHeartbeatNow: vi.fn(), + requestHeartbeat: vi.fn(), runIsolatedAgentJob: vi.fn().mockResolvedValue({ status: "ok", summary: "ok" }), }); state.running = true; @@ -251,7 +251,7 @@ export function createMockCronStateForJobs(params: { cronEnabled: true, nowMs: () => nowMs, enqueueSystemEvent: () => {}, - requestHeartbeatNow: () => {}, + requestHeartbeat: () => {}, runIsolatedAgentJob: async () => ({ status: "ok" }), log: { debug: () => {}, diff --git a/src/cron/service/jobs.schedule-error-isolation.test.ts b/src/cron/service/jobs.schedule-error-isolation.test.ts index 84cd8e0a1e9..f6e43ae0b17 100644 --- a/src/cron/service/jobs.schedule-error-isolation.test.ts +++ b/src/cron/service/jobs.schedule-error-isolation.test.ts @@ -16,7 +16,7 @@ function createMockState(jobs: CronJob[]): CronServiceState { error: vi.fn(), }, enqueueSystemEvent: vi.fn(), - requestHeartbeatNow: vi.fn(), + requestHeartbeat: vi.fn(), runHeartbeatOnce: vi.fn(), runIsolatedAgentJob: vi.fn(), onEvent: vi.fn(), diff --git a/src/cron/service/jobs.ts b/src/cron/service/jobs.ts index 329c4cd0759..a3e387a6f46 100644 --- a/src/cron/service/jobs.ts +++ b/src/cron/service/jobs.ts @@ -343,7 +343,9 @@ export function recordScheduleComputeError(params: { sessionKey: job.sessionKey, contextKey: `cron:${job.id}:auto-disabled`, }); - state.deps.requestHeartbeatNow({ + state.deps.requestHeartbeat({ + source: "cron", + intent: "event", reason: `cron:${job.id}:auto-disabled`, agentId: job.agentId, sessionKey: job.sessionKey, diff --git a/src/cron/service/ops.regression.test.ts b/src/cron/service/ops.regression.test.ts index 1afff372e22..f3d8e84dc46 100644 --- a/src/cron/service/ops.regression.test.ts +++ b/src/cron/service/ops.regression.test.ts @@ -35,7 +35,7 @@ describe("cron service ops regressions", () => { storePath: store.storePath, log: noopLogger, enqueueSystemEvent: vi.fn(), - requestHeartbeatNow: vi.fn(), + requestHeartbeat: vi.fn(), runIsolatedAgentJob: vi.fn(), }); state.store = { @@ -90,7 +90,7 @@ describe("cron service ops regressions", () => { storePath: store.storePath, log: noopLogger, enqueueSystemEvent: vi.fn(), - requestHeartbeatNow: vi.fn(), + requestHeartbeat: vi.fn(), runIsolatedAgentJob, onEvent: (evt: CronEvent) => { if (evt.jobId !== job.id) { @@ -152,7 +152,7 @@ describe("cron service ops regressions", () => { log: noopLogger, nowMs: () => now, enqueueSystemEvent: vi.fn(), - requestHeartbeatNow: vi.fn(), + requestHeartbeat: vi.fn(), runIsolatedAgentJob, onEvent: (evt: CronEvent) => { if (evt.jobId === job.id && evt.action === "finished") { @@ -214,7 +214,7 @@ describe("cron service ops regressions", () => { storePath: store.storePath, log: noopLogger, enqueueSystemEvent: vi.fn(), - requestHeartbeatNow: vi.fn(), + requestHeartbeat: vi.fn(), runIsolatedAgentJob: vi.fn().mockResolvedValue({ status: "ok", summary: "ok" }), }); @@ -249,7 +249,7 @@ describe("cron service ops regressions", () => { storePath: store.storePath, log: noopLogger, enqueueSystemEvent: vi.fn(), - requestHeartbeatNow: vi.fn(), + requestHeartbeat: vi.fn(), runIsolatedAgentJob: abortAwareRunner.runIsolatedAgentJob, }); @@ -301,7 +301,7 @@ describe("cron service ops regressions", () => { log: noopLogger, nowMs: () => now, enqueueSystemEvent, - requestHeartbeatNow: vi.fn(), + requestHeartbeat: vi.fn(), runIsolatedAgentJob: vi.fn().mockResolvedValue({ status: "ok", summary: "ok" }), }); @@ -365,7 +365,7 @@ describe("cron service ops regressions", () => { log: noopLogger, nowMs: () => now, enqueueSystemEvent: vi.fn(), - requestHeartbeatNow: vi.fn(), + requestHeartbeat: vi.fn(), runIsolatedAgentJob, onEvent: (evt) => { if (evt.action === "finished" && evt.jobId === second.id && evt.status === "ok") { diff --git a/src/cron/service/ops.test.ts b/src/cron/service/ops.test.ts index 5d7b21eaeb9..d308f4deac5 100644 --- a/src/cron/service/ops.test.ts +++ b/src/cron/service/ops.test.ts @@ -35,7 +35,7 @@ function createTimedOutIsolatedCronState(params: { storePath: string; now: numbe log: logger, nowMs: () => params.now, enqueueSystemEvent: vi.fn(), - requestHeartbeatNow: vi.fn(), + requestHeartbeat: vi.fn(), runIsolatedAgentJob: vi.fn(async () => { throw new Error("cron: job execution timed out"); }), @@ -49,7 +49,7 @@ function createOkIsolatedCronState(params: { storePath: string; now: number; sum log: logger, nowMs: () => params.now, enqueueSystemEvent: vi.fn(), - requestHeartbeatNow: vi.fn(), + requestHeartbeat: vi.fn(), runIsolatedAgentJob: vi.fn(async () => ({ status: "ok" as const, ...(params.summary === undefined ? {} : { summary: params.summary }), @@ -133,7 +133,7 @@ describe("cron service ops seam coverage", () => { const { storePath } = await makeStorePath(); const now = Date.parse("2026-03-23T12:00:00.000Z"); const enqueueSystemEvent = vi.fn(); - const requestHeartbeatNow = vi.fn(); + const requestHeartbeat = vi.fn(); const timeoutSpy = vi.spyOn(globalThis, "setTimeout"); await writeCronStoreSnapshot({ @@ -147,7 +147,7 @@ describe("cron service ops seam coverage", () => { log: logger, nowMs: () => now, enqueueSystemEvent, - requestHeartbeatNow, + requestHeartbeat, runIsolatedAgentJob: vi.fn(async () => ({ status: "ok" as const })), }); @@ -158,7 +158,7 @@ describe("cron service ops seam coverage", () => { "cron: marking interrupted running job failed on startup", ); expect(enqueueSystemEvent).not.toHaveBeenCalled(); - expect(requestHeartbeatNow).not.toHaveBeenCalled(); + expect(requestHeartbeat).not.toHaveBeenCalled(); expect(state.timer).not.toBeNull(); const persisted = (await loadCronStore(storePath)) as { diff --git a/src/cron/service/state.test.ts b/src/cron/service/state.test.ts index f36e2a2b0b3..fdc7c790c08 100644 --- a/src/cron/service/state.test.ts +++ b/src/cron/service/state.test.ts @@ -5,7 +5,7 @@ describe("cron service state seam coverage", () => { it("threads heartbeat and session-store dependencies into internal state", () => { const nowMs = vi.fn(() => 123_456); const enqueueSystemEvent = vi.fn(); - const requestHeartbeatNow = vi.fn(); + const requestHeartbeat = vi.fn(); const runHeartbeatOnce = vi.fn(); const resolveSessionStorePath = vi.fn((agentId?: string) => `/tmp/${agentId ?? "main"}.json`); @@ -23,7 +23,7 @@ describe("cron service state seam coverage", () => { sessionStorePath: "/tmp/sessions.json", resolveSessionStorePath, enqueueSystemEvent, - requestHeartbeatNow, + requestHeartbeat, runHeartbeatOnce, runIsolatedAgentJob: vi.fn(async () => ({ status: "ok" as const })), }); @@ -41,7 +41,7 @@ describe("cron service state seam coverage", () => { expect(state.deps.sessionStorePath).toBe("/tmp/sessions.json"); expect(state.deps.resolveSessionStorePath).toBe(resolveSessionStorePath); expect(state.deps.enqueueSystemEvent).toBe(enqueueSystemEvent); - expect(state.deps.requestHeartbeatNow).toBe(requestHeartbeatNow); + expect(state.deps.requestHeartbeat).toBe(requestHeartbeat); expect(state.deps.runHeartbeatOnce).toBe(runHeartbeatOnce); expect(state.deps.nowMs()).toBe(123_456); }); @@ -59,7 +59,7 @@ describe("cron service state seam coverage", () => { storePath: "/tmp/cron/jobs.json", cronEnabled: false, enqueueSystemEvent: vi.fn(), - requestHeartbeatNow: vi.fn(), + requestHeartbeat: vi.fn(), runIsolatedAgentJob: vi.fn(async () => ({ status: "ok" as const })), }); diff --git a/src/cron/service/state.ts b/src/cron/service/state.ts index 5b7d9387037..e81bea18aec 100644 --- a/src/cron/service/state.ts +++ b/src/cron/service/state.ts @@ -74,8 +74,10 @@ export type CronServiceDeps = { text: string, opts?: { agentId?: string; sessionKey?: string; contextKey?: string; trusted?: boolean }, ) => void; - requestHeartbeatNow: (opts?: HeartbeatWakeRequest) => void; + requestHeartbeat: (opts: HeartbeatWakeRequest) => void; runHeartbeatOnce?: (opts?: { + source?: HeartbeatWakeRequest["source"]; + intent?: HeartbeatWakeRequest["intent"]; reason?: string; agentId?: string; sessionKey?: string; @@ -85,7 +87,7 @@ export type CronServiceDeps = { /** * WakeMode=now: max time to wait for runHeartbeatOnce to stop returning * { status:"skipped", reason:"requests-in-flight" } before falling back to - * requestHeartbeatNow. + * requestHeartbeat. */ wakeNowHeartbeatBusyMaxWaitMs?: number; /** WakeMode=now: delay between runHeartbeatOnce retries while busy. */ diff --git a/src/cron/service/store.load-missing-session-target.test.ts b/src/cron/service/store.load-missing-session-target.test.ts index 643bc00dae9..bdcaade72c9 100644 --- a/src/cron/service/store.load-missing-session-target.test.ts +++ b/src/cron/service/store.load-missing-session-target.test.ts @@ -24,7 +24,7 @@ function createStoreTestState(storePath: string) { log: logger, nowMs: () => STORE_TEST_NOW, enqueueSystemEvent: vi.fn(), - requestHeartbeatNow: vi.fn(), + requestHeartbeat: vi.fn(), runIsolatedAgentJob: vi.fn(async () => ({ status: "ok" as const })), }); } diff --git a/src/cron/service/store.test.ts b/src/cron/service/store.test.ts index a28a9da17f4..213f4fe23d4 100644 --- a/src/cron/service/store.test.ts +++ b/src/cron/service/store.test.ts @@ -37,7 +37,7 @@ function createStoreTestState(storePath: string) { log: logger, nowMs: () => STORE_TEST_NOW, enqueueSystemEvent: vi.fn(), - requestHeartbeatNow: vi.fn(), + requestHeartbeat: vi.fn(), runIsolatedAgentJob: vi.fn(async () => ({ status: "ok" as const })), }); } diff --git a/src/cron/service/timer.regression.test.ts b/src/cron/service/timer.regression.test.ts index 9aeb0339f96..35197a0f83a 100644 --- a/src/cron/service/timer.regression.test.ts +++ b/src/cron/service/timer.regression.test.ts @@ -39,7 +39,7 @@ describe("cron service timer regressions", () => { storePath: store.storePath, log: noopLogger, enqueueSystemEvent: vi.fn(), - requestHeartbeatNow: vi.fn(), + requestHeartbeat: vi.fn(), runIsolatedAgentJob: createDefaultIsolatedRunner(), }); @@ -124,7 +124,7 @@ describe("cron service timer regressions", () => { log: noopLogger, nowMs: () => now, enqueueSystemEvent: vi.fn(), - requestHeartbeatNow: vi.fn(), + requestHeartbeat: vi.fn(), runIsolatedAgentJob, }); @@ -196,7 +196,7 @@ describe("cron service timer regressions", () => { log: noopLogger, nowMs: () => now, enqueueSystemEvent: vi.fn(), - requestHeartbeatNow: vi.fn(), + requestHeartbeat: vi.fn(), runIsolatedAgentJob, }); @@ -239,7 +239,7 @@ describe("cron service timer regressions", () => { log: noopLogger, nowMs: () => now, enqueueSystemEvent: vi.fn(), - requestHeartbeatNow: vi.fn(), + requestHeartbeat: vi.fn(), runIsolatedAgentJob, cronConfig: { retry: { maxAttempts: 2, backoffMs: [1000, 2000] }, @@ -285,7 +285,7 @@ describe("cron service timer regressions", () => { log: noopLogger, nowMs: () => now, enqueueSystemEvent: vi.fn(), - requestHeartbeatNow: vi.fn(), + requestHeartbeat: vi.fn(), runIsolatedAgentJob, cronConfig: { retry: { maxAttempts: 1, backoffMs: [1000], retryOn: ["overloaded"] }, @@ -334,7 +334,7 @@ describe("cron service timer regressions", () => { log: noopLogger, nowMs: () => now, enqueueSystemEvent: vi.fn(), - requestHeartbeatNow: vi.fn(), + requestHeartbeat: vi.fn(), runIsolatedAgentJob, cronConfig: { retry: { maxAttempts: 1, backoffMs: [1000], retryOn: ["rate_limit"] }, @@ -380,7 +380,7 @@ describe("cron service timer regressions", () => { log: noopLogger, nowMs: () => now, enqueueSystemEvent: vi.fn(), - requestHeartbeatNow: vi.fn(), + requestHeartbeat: vi.fn(), runIsolatedAgentJob: vi.fn().mockResolvedValue({ status: "error", error: "invalid API key", @@ -418,7 +418,7 @@ describe("cron service timer regressions", () => { log: noopLogger, nowMs: () => now, enqueueSystemEvent: vi.fn(), - requestHeartbeatNow: vi.fn(), + requestHeartbeat: vi.fn(), runIsolatedAgentJob: vi.fn(async () => { now += 7; fireCount += 1; @@ -457,7 +457,7 @@ describe("cron service timer regressions", () => { log: noopLogger, nowMs: () => now, enqueueSystemEvent: vi.fn(), - requestHeartbeatNow: vi.fn(), + requestHeartbeat: vi.fn(), runIsolatedAgentJob: vi.fn(async () => { now += 100; return { status: "ok" as const, summary: "done" }; @@ -493,7 +493,7 @@ describe("cron service timer regressions", () => { log: noopLogger, nowMs: () => now, enqueueSystemEvent: vi.fn(), - requestHeartbeatNow: vi.fn(), + requestHeartbeat: vi.fn(), runIsolatedAgentJob: vi.fn(async () => { const result = await deferredRun.promise; now += 5; @@ -548,7 +548,7 @@ describe("cron service timer regressions", () => { log: noopLogger, nowMs: () => now, enqueueSystemEvent: vi.fn(), - requestHeartbeatNow: vi.fn(), + requestHeartbeat: vi.fn(), runIsolatedAgentJob, }); @@ -593,7 +593,7 @@ describe("cron service timer regressions", () => { log: noopLogger, nowMs: () => now, enqueueSystemEvent: vi.fn(), - requestHeartbeatNow: vi.fn(), + requestHeartbeat: vi.fn(), runIsolatedAgentJob: vi.fn(async (params) => { const result = await abortAwareRunner.runIsolatedAgentJob(params); now += 5; @@ -640,7 +640,7 @@ describe("cron service timer regressions", () => { log: noopLogger, nowMs: () => now, enqueueSystemEvent: vi.fn(), - requestHeartbeatNow: vi.fn(), + requestHeartbeat: vi.fn(), runIsolatedAgentJob: vi.fn(async ({ abortSignal, onExecutionStarted }) => { observedAbortSignal = abortSignal; runnerEntered.resolve(); @@ -708,7 +708,7 @@ describe("cron service timer regressions", () => { log: noopLogger, nowMs: () => now, enqueueSystemEvent, - requestHeartbeatNow: vi.fn(), + requestHeartbeat: vi.fn(), runIsolatedAgentJob: vi.fn(async (params) => { const result = await abortAwareRunner.runIsolatedAgentJob(params); now += 100; @@ -755,7 +755,7 @@ describe("cron service timer regressions", () => { log: noopLogger, nowMs: () => now, enqueueSystemEvent: vi.fn(), - requestHeartbeatNow: vi.fn(), + requestHeartbeat: vi.fn(), runIsolatedAgentJob: vi.fn(async (params) => { const result = await abortAwareRunner.runIsolatedAgentJob(params); now += 5; @@ -786,7 +786,7 @@ describe("cron service timer regressions", () => { }), ); const enqueueSystemEvent = vi.fn(); - const requestHeartbeatNow = vi.fn(); + const requestHeartbeat = vi.fn(); const mainJob: CronJob = { id: "main-abort", name: "main abort", @@ -805,7 +805,7 @@ describe("cron service timer regressions", () => { log: noopLogger, nowMs: () => Date.now(), enqueueSystemEvent, - requestHeartbeatNow, + requestHeartbeat, runHeartbeatOnce, wakeNowHeartbeatBusyMaxWaitMs: 30, wakeNowHeartbeatBusyRetryDelayMs: 5, @@ -824,7 +824,7 @@ describe("cron service timer regressions", () => { expect(result.error).toContain("timed out"); expect(enqueueSystemEvent).toHaveBeenCalledTimes(1); expect(runHeartbeatOnce).toHaveBeenCalled(); - expect(requestHeartbeatNow).not.toHaveBeenCalled(); + expect(requestHeartbeat).not.toHaveBeenCalled(); }); it("retries recurring wake-now main jobs until temporary lane pressure clears (#75964)", async () => { @@ -838,7 +838,7 @@ describe("cron service timer regressions", () => { .mockResolvedValueOnce({ status: "skipped", reason: HEARTBEAT_SKIP_LANES_BUSY }) .mockResolvedValueOnce({ status: "ran", durationMs: 12 }); const enqueueSystemEvent = vi.fn(); - const requestHeartbeatNow = vi.fn(); + const requestHeartbeat = vi.fn(); const job: CronJob = { id: "busy-recurring-main", name: "busy recurring main", @@ -857,7 +857,7 @@ describe("cron service timer regressions", () => { log: noopLogger, nowMs, enqueueSystemEvent, - requestHeartbeatNow, + requestHeartbeat, runHeartbeatOnce, wakeNowHeartbeatBusyMaxWaitMs: 120_000, wakeNowHeartbeatBusyRetryDelayMs: 1, @@ -871,7 +871,7 @@ describe("cron service timer regressions", () => { expect(enqueueSystemEvent).toHaveBeenCalledTimes(1); expect(runHeartbeatOnce).toHaveBeenCalledTimes(2); - expect(requestHeartbeatNow).not.toHaveBeenCalled(); + expect(requestHeartbeat).not.toHaveBeenCalled(); expect(job.state.lastStatus).toBe("ok"); expect(job.state.runningAtMs).toBeUndefined(); }); @@ -923,7 +923,7 @@ describe("cron service timer regressions", () => { log: noopLogger, nowMs: () => now, enqueueSystemEvent: vi.fn(), - requestHeartbeatNow: vi.fn(), + requestHeartbeat: vi.fn(), onEvent: (evt) => { events.push(evt); }, @@ -978,7 +978,7 @@ describe("cron service timer regressions", () => { log: noopLogger, nowMs: () => now, enqueueSystemEvent: vi.fn(), - requestHeartbeatNow: vi.fn(), + requestHeartbeat: vi.fn(), runIsolatedAgentJob: vi.fn(async (params: { job: { id: string } }) => { activeRuns += 1; peakActiveRuns = Math.max(peakActiveRuns, activeRuns); @@ -1044,7 +1044,7 @@ describe("cron service timer regressions", () => { log, nowMs: () => dueAt, enqueueSystemEvent: vi.fn(), - requestHeartbeatNow: vi.fn(), + requestHeartbeat: vi.fn(), onEvent: (evt) => { events.push(evt); }, @@ -1102,7 +1102,7 @@ describe("cron service timer regressions", () => { log, nowMs: () => dueAt, enqueueSystemEvent: vi.fn(), - requestHeartbeatNow: vi.fn(), + requestHeartbeat: vi.fn(), onEvent: (evt) => { events.push(evt); }, @@ -1154,7 +1154,7 @@ describe("cron service timer regressions", () => { log: noopLogger, nowMs: () => now, enqueueSystemEvent: vi.fn(), - requestHeartbeatNow: vi.fn(), + requestHeartbeat: vi.fn(), runIsolatedAgentJob: vi.fn( async ({ abortSignal, @@ -1236,7 +1236,7 @@ describe("cron service timer regressions", () => { log: noopLogger, nowMs: () => now, enqueueSystemEvent: vi.fn(), - requestHeartbeatNow: vi.fn(), + requestHeartbeat: vi.fn(), cleanupTimedOutAgentRun, runIsolatedAgentJob: vi.fn( async ({ @@ -1299,7 +1299,7 @@ describe("cron service timer regressions", () => { log: noopLogger, nowMs: () => endedAt, enqueueSystemEvent: vi.fn(), - requestHeartbeatNow: vi.fn(), + requestHeartbeat: vi.fn(), runIsolatedAgentJob: createDefaultIsolatedRunner(), }); const job = createIsolatedRegressionJob({ @@ -1337,7 +1337,7 @@ describe("cron service timer regressions", () => { log: noopLogger, nowMs: () => endedAt, enqueueSystemEvent: vi.fn(), - requestHeartbeatNow: vi.fn(), + requestHeartbeat: vi.fn(), runIsolatedAgentJob: createDefaultIsolatedRunner(), }); const job = createIsolatedRegressionJob({ @@ -1376,7 +1376,7 @@ describe("cron service timer regressions", () => { log: noopLogger, nowMs: () => endedAt, enqueueSystemEvent: vi.fn(), - requestHeartbeatNow: vi.fn(), + requestHeartbeat: vi.fn(), runIsolatedAgentJob: createDefaultIsolatedRunner(), }); const job = createIsolatedRegressionJob({ @@ -1417,7 +1417,7 @@ describe("cron service timer regressions", () => { log: noopLogger, nowMs: () => endedAt, enqueueSystemEvent: vi.fn(), - requestHeartbeatNow: vi.fn(), + requestHeartbeat: vi.fn(), runIsolatedAgentJob: createDefaultIsolatedRunner(), }); const job = createIsolatedRegressionJob({ diff --git a/src/cron/service/timer.test.ts b/src/cron/service/timer.test.ts index f96fb5cdb76..72a3ddfa731 100644 --- a/src/cron/service/timer.test.ts +++ b/src/cron/service/timer.test.ts @@ -37,7 +37,7 @@ describe("cron service timer seam coverage", () => { const { storePath } = await makeStorePath(); const now = Date.parse("2026-03-23T12:00:00.000Z"); const enqueueSystemEvent = vi.fn(); - const requestHeartbeatNow = vi.fn(); + const requestHeartbeat = vi.fn(); const timeoutSpy = vi.spyOn(globalThis, "setTimeout"); await writeCronStoreSnapshot({ @@ -51,7 +51,7 @@ describe("cron service timer seam coverage", () => { log: logger, nowMs: () => now, enqueueSystemEvent, - requestHeartbeatNow, + requestHeartbeat, runIsolatedAgentJob: vi.fn(async () => ({ status: "ok" as const })), }); @@ -62,7 +62,9 @@ describe("cron service timer seam coverage", () => { sessionKey: "agent:main:main", contextKey: "cron:main-heartbeat-job", }); - expect(requestHeartbeatNow).toHaveBeenCalledWith({ + expect(requestHeartbeat).toHaveBeenCalledWith({ + source: "cron", + intent: "event", reason: "cron:main-heartbeat-job", agentId: undefined, sessionKey: "agent:main:main", @@ -94,7 +96,7 @@ describe("cron service timer seam coverage", () => { const { storePath } = await makeStorePath(); const now = Date.parse("2026-03-23T12:00:00.000Z"); const enqueueSystemEvent = vi.fn(); - const requestHeartbeatNow = vi.fn(); + const requestHeartbeat = vi.fn(); await writeCronStoreSnapshot({ storePath, @@ -113,7 +115,7 @@ describe("cron service timer seam coverage", () => { log: logger, nowMs: () => now, enqueueSystemEvent, - requestHeartbeatNow, + requestHeartbeat, runIsolatedAgentJob: vi.fn(async () => ({ status: "ok" as const })), }); @@ -137,7 +139,7 @@ describe("cron service timer seam coverage", () => { const now = Date.parse("2026-03-23T06:00:00.000Z"); const staleNextRunAtMs = now; const enqueueSystemEvent = vi.fn(); - const requestHeartbeatNow = vi.fn(); + const requestHeartbeat = vi.fn(); await saveCronStore(storePath, { version: 1, @@ -169,14 +171,14 @@ describe("cron service timer seam coverage", () => { log: logger, nowMs: () => now, enqueueSystemEvent, - requestHeartbeatNow, + requestHeartbeat, runIsolatedAgentJob: vi.fn(async () => ({ status: "ok" as const })), }); await onTimer(state); expect(enqueueSystemEvent).not.toHaveBeenCalled(); - expect(requestHeartbeatNow).not.toHaveBeenCalled(); + expect(requestHeartbeat).not.toHaveBeenCalled(); const persisted = await loadCronStore(storePath); const job = persisted.jobs[0]; diff --git a/src/cron/service/timer.ts b/src/cron/service/timer.ts index 48ca49f36ea..7eba1c74963 100644 --- a/src/cron/service/timer.ts +++ b/src/cron/service/timer.ts @@ -456,7 +456,11 @@ function emitFailureAlert( state.deps.enqueueSystemEvent(text, { agentId: params.job.agentId }); if (params.job.wakeMode === "now") { - state.deps.requestHeartbeatNow({ reason: `cron:${params.job.id}:failure-alert` }); + state.deps.requestHeartbeat({ + source: "cron", + intent: "immediate", + reason: `cron:${params.job.id}:failure-alert`, + }); } } @@ -1384,6 +1388,8 @@ async function executeMainSessionCronJob( return { status: "error", error: timeoutErrorMessage() }; } heartbeatResult = await state.deps.runHeartbeatOnce({ + source: "cron", + intent: "immediate", reason, agentId: job.agentId, sessionKey: targetMainSessionKey, @@ -1397,7 +1403,9 @@ async function executeMainSessionCronJob( } if (heartbeatResult.reason === HEARTBEAT_SKIP_CRON_IN_PROGRESS) { // The active cron marker blocks direct wake-now until this job returns. - state.deps.requestHeartbeatNow({ + state.deps.requestHeartbeat({ + source: "cron", + intent: "immediate", reason, agentId: job.agentId, sessionKey: targetMainSessionKey, @@ -1412,7 +1420,9 @@ async function executeMainSessionCronJob( if (abortSignal?.aborted) { return { status: "error", error: timeoutErrorMessage() }; } - state.deps.requestHeartbeatNow({ + state.deps.requestHeartbeat({ + source: "cron", + intent: "immediate", reason, agentId: job.agentId, sessionKey: targetMainSessionKey, @@ -1435,7 +1445,9 @@ async function executeMainSessionCronJob( if (abortSignal?.aborted) { return { status: "error", error: timeoutErrorMessage() }; } - state.deps.requestHeartbeatNow({ + state.deps.requestHeartbeat({ + source: "cron", + intent: job.wakeMode === "now" ? "immediate" : "event", reason: `cron:${job.id}`, agentId: job.agentId, sessionKey: targetMainSessionKey, @@ -1585,7 +1597,7 @@ export function wake( } state.deps.enqueueSystemEvent(text); if (opts.mode === "now") { - state.deps.requestHeartbeatNow({ reason: "wake" }); + state.deps.requestHeartbeat({ source: "manual", intent: "immediate", reason: "wake" }); } return { ok: true } as const; } diff --git a/src/gateway/server-cron.test.ts b/src/gateway/server-cron.test.ts index ceee265b3fd..b04c322d24e 100644 --- a/src/gateway/server-cron.test.ts +++ b/src/gateway/server-cron.test.ts @@ -8,7 +8,7 @@ import { mergeMockedModule } from "../test-utils/vitest-module-mocks.js"; const { enqueueSystemEventMock, - requestHeartbeatNowMock, + requestHeartbeatMock, runHeartbeatOnceMock, loadConfigMock, fetchWithSsrFGuardMock, @@ -18,7 +18,7 @@ const { runCronChangedMock, } = vi.hoisted(() => ({ enqueueSystemEventMock: vi.fn(), - requestHeartbeatNowMock: vi.fn(), + requestHeartbeatMock: vi.fn(), runHeartbeatOnceMock: vi.fn< (...args: unknown[]) => Promise<{ status: "ran"; durationMs: number }> >(async () => ({ status: "ran", durationMs: 1 })), @@ -37,8 +37,8 @@ function enqueueSystemEvent(...args: unknown[]) { return enqueueSystemEventMock(...args); } -function requestHeartbeatNow(...args: unknown[]) { - return requestHeartbeatNowMock(...args); +function requestHeartbeat(...args: unknown[]) { + return requestHeartbeatMock(...args); } function runHeartbeatOnce(...args: unknown[]) { @@ -55,7 +55,7 @@ vi.mock("../infra/heartbeat-wake.js", async () => { "../infra/heartbeat-wake.js", ), () => ({ - requestHeartbeatNow, + requestHeartbeat, }), ); }); @@ -113,7 +113,7 @@ function createCronConfig(name: string): OpenClawConfig { describe("buildGatewayCronService", () => { beforeEach(() => { enqueueSystemEventMock.mockClear(); - requestHeartbeatNowMock.mockClear(); + requestHeartbeatMock.mockClear(); runHeartbeatOnceMock.mockClear(); loadConfigMock.mockClear(); fetchWithSsrFGuardMock.mockClear(); @@ -264,7 +264,7 @@ describe("buildGatewayCronService", () => { sessionKey: "agent:main:discord:channel:ops", }), ); - expect(requestHeartbeatNowMock).toHaveBeenCalledWith( + expect(requestHeartbeatMock).toHaveBeenCalledWith( expect.objectContaining({ sessionKey: "agent:main:discord:channel:ops", }), @@ -288,10 +288,12 @@ describe("buildGatewayCronService", () => { state.cron as unknown as { state?: { deps?: { - requestHeartbeatNow?: (opts?: { + requestHeartbeat?: (opts?: { agentId?: string; sessionKey?: string | null; reason?: string; + source?: string; + intent?: string; heartbeat?: { target?: string }; }) => void; }; @@ -299,13 +301,17 @@ describe("buildGatewayCronService", () => { } ).state?.deps; - cronDeps?.requestHeartbeatNow?.({ + cronDeps?.requestHeartbeat?.({ + source: "cron", + intent: "event", reason: "cron:test", sessionKey: "discord:channel:ops", heartbeat: { target: "last" }, }); - expect(requestHeartbeatNowMock).toHaveBeenCalledWith({ + expect(requestHeartbeatMock).toHaveBeenCalledWith({ + source: "cron", + intent: "event", reason: "cron:test", agentId: "main", sessionKey: "agent:main:discord:channel:ops", diff --git a/src/gateway/server-cron.ts b/src/gateway/server-cron.ts index d0d6c12a538..3d13d406970 100644 --- a/src/gateway/server-cron.ts +++ b/src/gateway/server-cron.ts @@ -22,7 +22,7 @@ import { resolveCronStorePath } from "../cron/store.js"; import type { CronJob } from "../cron/types.js"; import { formatErrorMessage } from "../infra/errors.js"; import { runHeartbeatOnce } from "../infra/heartbeat-runner.js"; -import { requestHeartbeatNow } from "../infra/heartbeat-wake.js"; +import { requestHeartbeat } from "../infra/heartbeat-wake.js"; import { enqueueSystemEvent } from "../infra/system-events.js"; import { getChildLogger } from "../logging.js"; import { getGlobalHookRunner } from "../plugins/hook-runner-global.js"; @@ -247,9 +247,11 @@ export function buildGatewayCronService(params: { trusted: opts?.trusted, }); }, - requestHeartbeatNow: (opts) => { + requestHeartbeat: (opts) => { const { agentId, sessionKey } = resolveCronWakeTarget(opts); - requestHeartbeatNow({ + requestHeartbeat({ + source: opts?.source ?? "cron", + intent: opts?.intent ?? "event", reason: opts?.reason, agentId, sessionKey, @@ -279,6 +281,8 @@ export function buildGatewayCronService(params: { : undefined; return await runHeartbeatOnce({ cfg: runtimeConfig, + source: opts?.source ?? "cron", + intent: opts?.intent ?? "event", reason: opts?.reason, agentId, sessionKey, diff --git a/src/gateway/server-node-events.runtime.ts b/src/gateway/server-node-events.runtime.ts index fa501faccb0..a602e0ac489 100644 --- a/src/gateway/server-node-events.runtime.ts +++ b/src/gateway/server-node-events.runtime.ts @@ -6,7 +6,7 @@ export { agentCommandFromIngress } from "../commands/agent.js"; export { getRuntimeConfig } from "../config/io.js"; export { updateSessionStore } from "../config/sessions.js"; export { loadOrCreateDeviceIdentity } from "../infra/device-identity.js"; -export { requestHeartbeatNow } from "../infra/heartbeat-wake.js"; +export { requestHeartbeat } from "../infra/heartbeat-wake.js"; export { deliverOutboundPayloads } from "../infra/outbound/deliver.js"; export { buildOutboundSessionContext } from "../infra/outbound/session-context.js"; export { resolveOutboundTarget } from "../infra/outbound/targets.js"; diff --git a/src/gateway/server-node-events.test.ts b/src/gateway/server-node-events.test.ts index d00d7114d82..6d2fe5b6105 100644 --- a/src/gateway/server-node-events.test.ts +++ b/src/gateway/server-node-events.test.ts @@ -91,7 +91,7 @@ const runtimeMocks = vi.hoisted(() => ({ normalizeRpcAttachmentsToChatAttachments: vi.fn((attachments?: unknown[]) => attachments ?? []), parseMessageWithAttachments: parseMessageWithAttachmentsMock, registerApnsRegistration: registerApnsRegistrationMock, - requestHeartbeatNow: vi.fn(), + requestHeartbeat: vi.fn(), resolveChatAttachmentMaxBytes: vi.fn(() => 20 * 1024 * 1024), resolveGatewayModelSupportsImages: vi.fn( async ({ @@ -151,7 +151,7 @@ import { } from "./server-node-events.js"; const enqueueSystemEventMock = runtimeMocks.enqueueSystemEvent; -const requestHeartbeatNowMock = runtimeMocks.requestHeartbeatNow; +const requestHeartbeatMock = runtimeMocks.requestHeartbeat; const loadConfigMock = runtimeMocks.getRuntimeConfig; const agentCommandMock = runtimeMocks.agentCommandFromIngress; const updateSessionStoreMock = runtimeMocks.updateSessionStore; @@ -187,7 +187,7 @@ describe("node exec events", () => { resetNodeEventDeduplicationForTests(); enqueueSystemEventMock.mockClear(); enqueueSystemEventMock.mockReturnValue(true); - requestHeartbeatNowMock.mockClear(); + requestHeartbeatMock.mockClear(); registerApnsRegistrationVi.mockClear(); loadOrCreateDeviceIdentityMock.mockClear(); normalizeChannelIdVi.mockClear(); @@ -214,7 +214,7 @@ describe("node exec events", () => { "Exec started (node=node-1 id=run-1): ls -la", { sessionKey: "agent:main:main", contextKey: "exec:run-1", trusted: false }, ); - expect(requestHeartbeatNowMock).toHaveBeenCalledWith({ + expect(requestHeartbeatMock).toHaveBeenCalledWith({ reason: "exec-event", sessionKey: "agent:main:main", }); @@ -236,7 +236,7 @@ describe("node exec events", () => { "Exec finished (node=node-2 id=run-2, code 0)\ndone", { sessionKey: "node-node-2", contextKey: "exec:run-2", trusted: false }, ); - expect(requestHeartbeatNowMock).toHaveBeenCalledWith({ reason: "exec-event" }); + expect(requestHeartbeatMock).toHaveBeenCalledWith({ reason: "exec-event" }); }); it("dedupes duplicate exec.finished events for the same runId on the same session", async () => { @@ -259,7 +259,7 @@ describe("node exec events", () => { }); expect(enqueueSystemEventMock).toHaveBeenCalledTimes(1); - expect(requestHeartbeatNowMock).toHaveBeenCalledTimes(1); + expect(requestHeartbeatMock).toHaveBeenCalledTimes(1); expect(enqueueSystemEventMock).toHaveBeenCalledWith( "Exec finished (node=node-2 id=run-dup-finished, code 0)\ndone", { @@ -291,7 +291,7 @@ describe("node exec events", () => { "Exec finished (node=node-2 id=run-2, code 0)\ndone", { sessionKey: "agent:main:node-node-2", contextKey: "exec:run-2", trusted: false }, ); - expect(requestHeartbeatNowMock).toHaveBeenCalledWith({ + expect(requestHeartbeatMock).toHaveBeenCalledWith({ reason: "exec-event", sessionKey: "agent:main:node-node-2", }); @@ -310,7 +310,7 @@ describe("node exec events", () => { }); expect(enqueueSystemEventMock).not.toHaveBeenCalled(); - expect(requestHeartbeatNowMock).not.toHaveBeenCalled(); + expect(requestHeartbeatMock).not.toHaveBeenCalled(); }); it("truncates long exec.finished output in system events", async () => { @@ -330,7 +330,7 @@ describe("node exec events", () => { expect(text.startsWith("Exec finished (node=node-2 id=run-long, code 0)\n")).toBe(true); expect(text.endsWith("…")).toBe(true); expect(text.length).toBeLessThan(280); - expect(requestHeartbeatNowMock).toHaveBeenCalledWith({ reason: "exec-event" }); + expect(requestHeartbeatMock).toHaveBeenCalledWith({ reason: "exec-event" }); }); it("enqueues exec.denied events with reason", async () => { @@ -349,7 +349,7 @@ describe("node exec events", () => { "Exec denied (node=node-3 id=run-3, allowlist-miss): rm -rf /", { sessionKey: "agent:demo:main", contextKey: "exec:run-3", trusted: false }, ); - expect(requestHeartbeatNowMock).toHaveBeenCalledWith({ + expect(requestHeartbeatMock).toHaveBeenCalledWith({ reason: "exec-event", sessionKey: "agent:demo:main", }); @@ -374,7 +374,7 @@ describe("node exec events", () => { }); expect(enqueueSystemEventMock).not.toHaveBeenCalled(); - expect(requestHeartbeatNowMock).not.toHaveBeenCalled(); + expect(requestHeartbeatMock).not.toHaveBeenCalled(); }); it("suppresses exec.finished when notifyOnExit is false", async () => { @@ -397,7 +397,7 @@ describe("node exec events", () => { }); expect(enqueueSystemEventMock).not.toHaveBeenCalled(); - expect(requestHeartbeatNowMock).not.toHaveBeenCalled(); + expect(requestHeartbeatMock).not.toHaveBeenCalled(); }); it("suppresses exec.denied when notifyOnExit is false", async () => { @@ -420,7 +420,7 @@ describe("node exec events", () => { }); expect(enqueueSystemEventMock).not.toHaveBeenCalled(); - expect(requestHeartbeatNowMock).not.toHaveBeenCalled(); + expect(requestHeartbeatMock).not.toHaveBeenCalled(); }); it("sanitizes remote exec event content before enqueue", async () => { @@ -687,7 +687,7 @@ describe("voice transcript events", () => { describe("notifications changed events", () => { beforeEach(() => { enqueueSystemEventMock.mockClear(); - requestHeartbeatNowMock.mockClear(); + requestHeartbeatMock.mockClear(); loadSessionEntryMock.mockClear(); normalizeChannelIdVi.mockClear(); normalizeChannelIdVi.mockImplementation((channel?: string | null) => channel ?? null); @@ -712,7 +712,9 @@ describe("notifications changed events", () => { "Notification posted (node=node-n1 key=notif-1 package=com.example.chat): Message - Ping from Alex", { sessionKey: "node-node-n1", contextKey: "notification:notif-1", trusted: false }, ); - expect(requestHeartbeatNowMock).toHaveBeenCalledWith({ + expect(requestHeartbeatMock).toHaveBeenCalledWith({ + source: "notifications-event", + intent: "event", reason: "notifications-event", sessionKey: "node-node-n1", }); @@ -733,7 +735,9 @@ describe("notifications changed events", () => { "Notification removed (node=node-n2 key=notif-2 package=com.example.mail)", { sessionKey: "node-node-n2", contextKey: "notification:notif-2", trusted: false }, ); - expect(requestHeartbeatNowMock).toHaveBeenCalledWith({ + expect(requestHeartbeatMock).toHaveBeenCalledWith({ + source: "notifications-event", + intent: "event", reason: "notifications-event", sessionKey: "node-node-n2", }); @@ -750,7 +754,9 @@ describe("notifications changed events", () => { }), }); - expect(requestHeartbeatNowMock).toHaveBeenCalledWith({ + expect(requestHeartbeatMock).toHaveBeenCalledWith({ + source: "notifications-event", + intent: "event", reason: "notifications-event", sessionKey: "agent:main:main", }); @@ -779,7 +785,9 @@ describe("notifications changed events", () => { trusted: false, }, ); - expect(requestHeartbeatNowMock).toHaveBeenCalledWith({ + expect(requestHeartbeatMock).toHaveBeenCalledWith({ + source: "notifications-event", + intent: "event", reason: "notifications-event", sessionKey: "agent:main:node-node-n5", }); @@ -795,7 +803,7 @@ describe("notifications changed events", () => { }); expect(enqueueSystemEventMock).not.toHaveBeenCalled(); - expect(requestHeartbeatNowMock).not.toHaveBeenCalled(); + expect(requestHeartbeatMock).not.toHaveBeenCalled(); }); it("sanitizes notification text before enqueueing an untrusted system event", async () => { @@ -838,7 +846,7 @@ describe("notifications changed events", () => { }); expect(enqueueSystemEventMock).toHaveBeenCalledTimes(2); - expect(requestHeartbeatNowMock).toHaveBeenCalledTimes(1); + expect(requestHeartbeatMock).toHaveBeenCalledTimes(1); }); it("suppresses exec notifyOnExit events when payload opts out", async () => { @@ -855,7 +863,7 @@ describe("notifications changed events", () => { }); expect(enqueueSystemEventMock).not.toHaveBeenCalled(); - expect(requestHeartbeatNowMock).not.toHaveBeenCalled(); + expect(requestHeartbeatMock).not.toHaveBeenCalled(); }); }); diff --git a/src/gateway/server-node-events.ts b/src/gateway/server-node-events.ts index 79efd355635..499feffa02d 100644 --- a/src/gateway/server-node-events.ts +++ b/src/gateway/server-node-events.ts @@ -31,7 +31,7 @@ import { normalizeRpcAttachmentsToChatAttachments, parseMessageWithAttachments, registerApnsRegistration, - requestHeartbeatNow, + requestHeartbeat, resolveChatAttachmentMaxBytes, resolveGatewayModelSupportsImages, resolveOutboundTarget, @@ -646,7 +646,12 @@ export const handleNodeEvent = async ( trusted: false, }); if (queued) { - requestHeartbeatNow({ reason: "notifications-event", sessionKey }); + requestHeartbeat({ + source: "notifications-event", + intent: "event", + reason: "notifications-event", + sessionKey, + }); } return undefined; } @@ -749,8 +754,13 @@ export const handleNodeEvent = async ( // Scope wakes only for canonical agent sessions. Synthetic node-* fallback // keys should keep legacy unscoped behavior so enabled non-main heartbeat // agents still run when no explicit agent session is provided. - requestHeartbeatNow( - scopedHeartbeatWakeOptions(sessionKey, { reason: "exec-event", coalesceMs: 0 }), + requestHeartbeat( + scopedHeartbeatWakeOptions(sessionKey, { + source: "exec-event", + intent: "event", + reason: "exec-event", + coalesceMs: 0, + }), ); } return undefined; diff --git a/src/gateway/server-restart-sentinel.test.ts b/src/gateway/server-restart-sentinel.test.ts index 5a75cb6f6d5..1e596003dc2 100644 --- a/src/gateway/server-restart-sentinel.test.ts +++ b/src/gateway/server-restart-sentinel.test.ts @@ -73,7 +73,7 @@ const mocks = vi.hoisted(() => { ackDelivery: vi.fn(async () => {}), failDelivery: vi.fn(async () => {}), enqueueSystemEvent: vi.fn(), - requestHeartbeatNow: vi.fn(), + requestHeartbeat: vi.fn(), enqueueSessionDelivery: vi.fn(async (payload: Record) => { state.queuedSessionDelivery = payload; return "session-delivery-1"; @@ -204,7 +204,7 @@ vi.mock("../infra/heartbeat-wake.js", async () => { ); return { ...actual, - requestHeartbeatNow: mocks.requestHeartbeatNow, + requestHeartbeat: mocks.requestHeartbeat, }; }); @@ -274,7 +274,7 @@ describe("scheduleRestartSentinelWake", () => { mocks.ackDelivery.mockClear(); mocks.failDelivery.mockClear(); mocks.enqueueSystemEvent.mockClear(); - mocks.requestHeartbeatNow.mockClear(); + mocks.requestHeartbeat.mockClear(); mocks.enqueueSessionDelivery.mockClear(); mocks.drainPendingSessionDeliveries.mockClear(); mocks.recoverPendingSessionDeliveries.mockClear(); @@ -319,7 +319,9 @@ describe("scheduleRestartSentinelWake", () => { sessionKey: "agent:main:main", }), ); - expect(mocks.requestHeartbeatNow).toHaveBeenCalledWith({ + expect(mocks.requestHeartbeat).toHaveBeenCalledWith({ + source: "restart-sentinel", + intent: "immediate", reason: "wake", sessionKey: "agent:main:main", }); @@ -356,7 +358,7 @@ describe("scheduleRestartSentinelWake", () => { expect(mocks.ackDelivery).toHaveBeenCalledWith("queue-1"); expect(mocks.failDelivery).not.toHaveBeenCalled(); expect(mocks.enqueueSystemEvent).toHaveBeenCalledTimes(1); - expect(mocks.requestHeartbeatNow).toHaveBeenCalledTimes(1); + expect(mocks.requestHeartbeat).toHaveBeenCalledTimes(1); expect(mocks.logWarn).toHaveBeenCalledWith( expect.stringContaining("retrying in 1000ms"), expect.objectContaining({ @@ -759,11 +761,15 @@ describe("scheduleRestartSentinelWake", () => { }), }), ); - expect(mocks.requestHeartbeatNow).toHaveBeenNthCalledWith(1, { + expect(mocks.requestHeartbeat).toHaveBeenNthCalledWith(1, { + source: "restart-sentinel", + intent: "immediate", reason: "wake", sessionKey: "agent:main:main", }); - expect(mocks.requestHeartbeatNow).toHaveBeenNthCalledWith(2, { + expect(mocks.requestHeartbeat).toHaveBeenNthCalledWith(2, { + source: "restart-sentinel", + intent: "immediate", reason: "wake", sessionKey: "agent:main:main", }); @@ -899,7 +905,7 @@ describe("scheduleRestartSentinelWake", () => { sessionKey: "agent:main:main", }), ); - expect(mocks.requestHeartbeatNow).toHaveBeenCalledTimes(2); + expect(mocks.requestHeartbeat).toHaveBeenCalledTimes(2); expect(mocks.logWarn).not.toHaveBeenCalled(); }); @@ -974,7 +980,7 @@ describe("scheduleRestartSentinelWake", () => { expect(mocks.enqueueSystemEvent).toHaveBeenCalledWith("restart message", { sessionKey: "agent:main:main", }); - expect(mocks.requestHeartbeatNow).not.toHaveBeenCalled(); + expect(mocks.requestHeartbeat).not.toHaveBeenCalled(); expect(mocks.deliverOutboundPayloads).not.toHaveBeenCalled(); }); @@ -1042,7 +1048,7 @@ describe("scheduleRestartSentinelWake", () => { channel: "qa-channel", to: "channel:qa-room", }); - mocks.requestHeartbeatNow.mockImplementation(() => { + mocks.requestHeartbeat.mockImplementation(() => { mocks.deliveryContextFromSession.mockReturnValue({ channel: "qa-channel", to: "heartbeat", @@ -1055,7 +1061,7 @@ describe("scheduleRestartSentinelWake", () => { await scheduleRestartSentinelWake({ deps: {} as never }); - expect(mocks.requestHeartbeatNow).toHaveBeenCalledTimes(1); + expect(mocks.requestHeartbeat).toHaveBeenCalledTimes(1); expect(mocks.resolveOutboundTarget).toHaveBeenCalledWith( expect.objectContaining({ channel: "qa-channel", diff --git a/src/gateway/server-restart-sentinel.ts b/src/gateway/server-restart-sentinel.ts index 68555be2686..08a3ae8f2fc 100644 --- a/src/gateway/server-restart-sentinel.ts +++ b/src/gateway/server-restart-sentinel.ts @@ -8,7 +8,7 @@ import type { CliDeps } from "../cli/deps.types.js"; import { resolveMainSessionKeyFromConfig } from "../config/sessions.js"; import { parseSessionThreadInfo } from "../config/sessions/thread-info.js"; import { formatErrorMessage } from "../infra/errors.js"; -import { requestHeartbeatNow } from "../infra/heartbeat-wake.js"; +import { requestHeartbeat } from "../infra/heartbeat-wake.js"; import { deliverOutboundPayloads } from "../infra/outbound/deliver.js"; import { ackDelivery, enqueueDelivery, failDelivery } from "../infra/outbound/delivery-queue.js"; import { buildOutboundSessionContext } from "../infra/outbound/session-context.js"; @@ -80,7 +80,7 @@ function enqueueRestartSentinelWake( sessionKey, ...(deliveryContext ? { deliveryContext } : {}), }); - requestHeartbeatNow({ reason: "wake", sessionKey }); + requestHeartbeat({ source: "restart-sentinel", intent: "immediate", reason: "wake", sessionKey }); } async function waitForOutboundRetry(delayMs: number) { @@ -235,7 +235,12 @@ async function deliverQueuedSessionDelivery(params: { } : {}), }); - requestHeartbeatNow({ reason: "wake", sessionKey: canonicalKey }); + requestHeartbeat({ + source: "restart-sentinel", + intent: "immediate", + reason: "wake", + sessionKey: canonicalKey, + }); return; } @@ -250,7 +255,12 @@ async function deliverQueuedSessionDelivery(params: { } : {}), }); - requestHeartbeatNow({ reason: "wake", sessionKey: canonicalKey }); + requestHeartbeat({ + source: "restart-sentinel", + intent: "immediate", + reason: "wake", + sessionKey: canonicalKey, + }); return; } diff --git a/src/gateway/server/hooks.agent-trust.test.ts b/src/gateway/server/hooks.agent-trust.test.ts index e67888be200..dd1ffec295a 100644 --- a/src/gateway/server/hooks.agent-trust.test.ts +++ b/src/gateway/server/hooks.agent-trust.test.ts @@ -1,7 +1,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; const enqueueSystemEventMock = vi.fn(); -const requestHeartbeatNowMock = vi.fn(); +const requestHeartbeatMock = vi.fn(); const runCronIsolatedAgentTurnMock = vi.fn(); const resolveMainSessionKeyMock = vi.fn(() => "main-session"); const loadConfigMock = vi.fn(() => ({})); @@ -12,7 +12,7 @@ vi.mock("../../infra/system-events.js", () => ({ enqueueSystemEvent: enqueueSystemEventMock, })); vi.mock("../../infra/heartbeat-wake.js", () => ({ - requestHeartbeatNow: requestHeartbeatNowMock, + requestHeartbeat: requestHeartbeatMock, })); vi.mock("../../cron/isolated-agent.js", () => ({ runCronIsolatedAgentTurn: runCronIsolatedAgentTurnMock, @@ -101,7 +101,7 @@ describe("dispatchAgentHook trust handling", () => { await vi.waitFor(() => expect(runCronIsolatedAgentTurnMock).toHaveBeenCalledTimes(1)); expect(enqueueSystemEventMock).not.toHaveBeenCalled(); - expect(requestHeartbeatNowMock).not.toHaveBeenCalled(); + expect(requestHeartbeatMock).not.toHaveBeenCalled(); expect(logHooksInfoMock).toHaveBeenCalledWith( "hook agent run completed without announcement", expect.objectContaining({ @@ -191,7 +191,7 @@ describe("dispatchAgentHook trust handling", () => { await vi.waitFor(() => expect(runCronIsolatedAgentTurnMock).toHaveBeenCalledTimes(1)); expect(enqueueSystemEventMock).not.toHaveBeenCalled(); - expect(requestHeartbeatNowMock).not.toHaveBeenCalled(); + expect(requestHeartbeatMock).not.toHaveBeenCalled(); }); it("marks error events as untrusted and sanitizes hook names", async () => { diff --git a/src/gateway/server/hooks.ts b/src/gateway/server/hooks.ts index 5dffbd1337f..396c8aa968c 100644 --- a/src/gateway/server/hooks.ts +++ b/src/gateway/server/hooks.ts @@ -10,7 +10,7 @@ import { import type { OpenClawConfig } from "../../config/types.openclaw.js"; import type { RunCronAgentTurnResult } from "../../cron/isolated-agent/run.types.js"; import type { CronJob } from "../../cron/types.js"; -import { requestHeartbeatNow } from "../../infra/heartbeat-wake.js"; +import { requestHeartbeat } from "../../infra/heartbeat-wake.js"; import { enqueueSystemEvent } from "../../infra/system-events.js"; import type { createSubsystemLogger } from "../../logging/subsystem.js"; import { normalizeOptionalString } from "../../shared/string-coerce.js"; @@ -51,7 +51,7 @@ export function createGatewayHooksRequestHandler(params: { const sessionKey = resolveMainSessionKeyFromConfig(); enqueueSystemEvent(value.text, { sessionKey, trusted: false }); if (value.mode === "now") { - requestHeartbeatNow({ reason: "hook:wake" }); + requestHeartbeat({ source: "hook", intent: "immediate", reason: "hook:wake" }); } }; @@ -122,7 +122,7 @@ export function createGatewayHooksRequestHandler(params: { trusted: false, }); if (value.wakeMode === "now") { - requestHeartbeatNow({ reason: `hook:${jobId}` }); + requestHeartbeat({ source: "hook", intent: "immediate", reason: `hook:${jobId}` }); } } else if (result.status === "ok" && !value.deliver) { logHooks.info("hook agent run completed without announcement", { @@ -142,7 +142,11 @@ export function createGatewayHooksRequestHandler(params: { trusted: false, }); if (value.wakeMode === "now") { - requestHeartbeatNow({ reason: `hook:${jobId}:error` }); + requestHeartbeat({ + source: "hook", + intent: "immediate", + reason: `hook:${jobId}:error`, + }); } } })(); diff --git a/src/infra/heartbeat-cooldown.test.ts b/src/infra/heartbeat-cooldown.test.ts new file mode 100644 index 00000000000..aac55d2980f --- /dev/null +++ b/src/infra/heartbeat-cooldown.test.ts @@ -0,0 +1,400 @@ +import { describe, expect, it } from "vitest"; +import { + DEFAULT_FLOOD_THRESHOLD, + DEFAULT_MIN_WAKE_SPACING_MS, + recordRunStart, + shouldDeferWake, +} from "./heartbeat-cooldown.js"; + +describe("shouldDeferWake", () => { + type Input = Parameters[0]; + function decide(input: Omit & { intent?: Input["intent"] }) { + return shouldDeferWake({ intent: "event", ...input }); + } + + // After-a-run baseline: agent has already run once, so the cooldown gate is + // active for non-manual non-interval wakes. + const afterRun = { + nextDueMs: 100_000, + now: 50_000, + lastRunStartedAtMs: 49_000, + }; + + // Bootstrap baseline: agent has never run. nextDueMs is the first phase tick. + const beforeFirstRun = { + nextDueMs: 100_000, + now: 50_000, + lastRunStartedAtMs: undefined, + }; + + describe("manual wakes", () => { + it("never defers manual wakes even within nextDueMs", () => { + expect(decide({ ...afterRun, intent: "manual", reason: "manual" })).toEqual({ + defer: false, + }); + }); + + it("never defers manual wakes even within min-spacing window", () => { + expect( + decide({ + intent: "manual", + now: 200_000, + nextDueMs: 100_000, + lastRunStartedAtMs: 199_900, + reason: "manual", + }), + ).toEqual({ defer: false }); + }); + + it("never defers manual wakes even during a flood", () => { + const now = 1_000_000; + const recentRunStarts = [ + now - 50_000, + now - 40_000, + now - 30_000, + now - 20_000, + now - 10_000, + ]; + expect( + decide({ + intent: "manual", + now, + nextDueMs: 0, + lastRunStartedAtMs: now - 10_000, + recentRunStarts, + reason: "manual", + }), + ).toEqual({ defer: false }); + }); + }); + + describe("immediate wake intent (wake-now contracts)", () => { + it("does not defer 'wake' even within nextDueMs (system event --mode now contract)", () => { + expect(decide({ ...afterRun, intent: "immediate", reason: "wake" })).toEqual({ + defer: false, + }); + }); + + it("does not defer 'background-task' even within nextDueMs (task completion contract)", () => { + expect(decide({ ...afterRun, intent: "immediate", reason: "background-task" })).toEqual({ + defer: false, + }); + }); + + it("does not defer 'background-task-blocked' even within nextDueMs", () => { + expect( + decide({ ...afterRun, intent: "immediate", reason: "background-task-blocked" }), + ).toEqual({ defer: false }); + }); + + it("does not defer explicit hook wake-now calls even within nextDueMs", () => { + expect(decide({ ...afterRun, intent: "immediate", reason: "hook:wake" })).toEqual({ + defer: false, + }); + }); + + it("does not defer explicit cron wake-now calls even within nextDueMs", () => { + expect(decide({ ...afterRun, intent: "immediate", reason: "cron:morning-brief" })).toEqual({ + defer: false, + }); + }); + + it("does not defer 'wake' within min-spacing window", () => { + expect( + decide({ + intent: "immediate", + now: 200_000, + nextDueMs: 100_000, + lastRunStartedAtMs: 199_990, + reason: "wake", + }), + ).toEqual({ defer: false }); + }); + + it("defers acp spawn stream wakes when they use event intent", () => { + expect(decide({ ...afterRun, source: "acp-spawn", reason: "acp:spawn:stream" })).toEqual({ + defer: true, + reason: "not-due", + }); + }); + + it("flood guard still applies to 'wake' as a backstop against unexpected loops", () => { + const now = 1_000_000; + const recentRunStarts = [ + now - 50_000, + now - 40_000, + now - 30_000, + now - 20_000, + now - 10_000, + ]; + expect( + decide({ + intent: "immediate", + now, + nextDueMs: 0, + lastRunStartedAtMs: now - 10_000, + recentRunStarts, + reason: "wake", + }), + ).toEqual({ defer: true, reason: "flood" }); + }); + + it("flood guard still applies to 'background-task' as a backstop", () => { + const now = 1_000_000; + const recentRunStarts = [ + now - 50_000, + now - 40_000, + now - 30_000, + now - 20_000, + now - 10_000, + ]; + expect( + decide({ + intent: "immediate", + now, + nextDueMs: 0, + lastRunStartedAtMs: now - 10_000, + recentRunStarts, + reason: "background-task", + }), + ).toEqual({ defer: true, reason: "flood" }); + }); + + it("flood guard still applies to explicit wake-now bypass calls", () => { + const now = 1_000_000; + const recentRunStarts = [ + now - 50_000, + now - 40_000, + now - 30_000, + now - 20_000, + now - 10_000, + ]; + expect( + decide({ + intent: "immediate", + now, + nextDueMs: 0, + lastRunStartedAtMs: now - 10_000, + recentRunStarts, + reason: "hook:wake", + }), + ).toEqual({ defer: true, reason: "flood" }); + }); + }); + + describe("scheduled intent", () => { + it("defers with 'not-due' when now < nextDueMs (interval cooldown)", () => { + expect(decide({ ...afterRun, intent: "scheduled", reason: "interval" })).toEqual({ + defer: true, + reason: "not-due", + }); + }); + + it("defers interval wake before first run if nextDueMs is in future", () => { + expect(decide({ ...beforeFirstRun, intent: "scheduled", reason: "interval" })).toEqual({ + defer: true, + reason: "not-due", + }); + }); + + it("does not defer interval wake when now >= nextDueMs", () => { + expect( + decide({ + intent: "scheduled", + now: 100_001, + nextDueMs: 100_000, + lastRunStartedAtMs: 70_000, + reason: "interval", + }), + ).toEqual({ defer: false }); + }); + }); + + describe("event-driven wakes after a prior run (regression for #75436)", () => { + it("defers exec-event wakes when now < nextDueMs", () => { + expect(decide({ ...afterRun, source: "exec-event", reason: "exec-event" })).toEqual({ + defer: true, + reason: "not-due", + }); + }); + + it("defers cron wakes when now < nextDueMs", () => { + expect(decide({ ...afterRun, source: "cron", reason: "cron:morning-brief" })).toEqual({ + defer: true, + reason: "not-due", + }); + }); + + it("defers hook wakes when now < nextDueMs", () => { + expect(decide({ ...afterRun, source: "hook", reason: "hook:wake" })).toEqual({ + defer: true, + reason: "not-due", + }); + }); + + it("defers acp spawn stream wakes when now < nextDueMs", () => { + expect(decide({ ...afterRun, source: "acp-spawn", reason: "acp:spawn:stream" })).toEqual({ + defer: true, + reason: "not-due", + }); + }); + + it("defers unknown wake reasons when now < nextDueMs", () => { + expect(decide({ ...afterRun, source: "other", reason: "something-new" })).toEqual({ + defer: true, + reason: "not-due", + }); + }); + }); + + describe("event-driven wakes before any prior run (bootstrap)", () => { + it("does NOT defer the first exec-event wake (lets idle agent respond)", () => { + expect(decide({ ...beforeFirstRun, source: "exec-event", reason: "exec-event" })).toEqual({ + defer: false, + }); + }); + + it("does NOT defer the first cron wake", () => { + expect(decide({ ...beforeFirstRun, source: "cron", reason: "cron:job-x" })).toEqual({ + defer: false, + }); + }); + + it("does NOT defer the first hook wake", () => { + expect(decide({ ...beforeFirstRun, source: "hook", reason: "hook:wake" })).toEqual({ + defer: false, + }); + }); + }); + + describe("min-spacing floor", () => { + it("defers with 'min-spacing' when last run started within floor (post-cooldown race)", () => { + // nextDueMs has just been crossed, but a run started ~10s ago — second + // wake landed before the schedule advanced. + expect( + decide({ + source: "exec-event", + now: 200_000, + nextDueMs: 199_999, + lastRunStartedAtMs: 200_000 - DEFAULT_MIN_WAKE_SPACING_MS + 100, + reason: "exec-event", + }), + ).toEqual({ defer: true, reason: "min-spacing" }); + }); + + it("does not defer when last run is older than min-spacing", () => { + expect( + decide({ + source: "exec-event", + now: 200_000, + nextDueMs: 199_999, + lastRunStartedAtMs: 200_000 - DEFAULT_MIN_WAKE_SPACING_MS - 1, + reason: "exec-event", + }), + ).toEqual({ defer: false }); + }); + + it("respects override of minSpacingMs", () => { + expect( + decide({ + source: "exec-event", + now: 200_000, + nextDueMs: 199_999, + lastRunStartedAtMs: 199_500, // 500ms ago + minSpacingMs: 1_000, + reason: "exec-event", + }), + ).toEqual({ defer: true, reason: "min-spacing" }); + }); + + it("does not gate manual wakes on min-spacing", () => { + expect( + decide({ + intent: "manual", + now: 200_000, + nextDueMs: 100_000, + lastRunStartedAtMs: 199_999, + reason: "manual", + }), + ).toEqual({ defer: false }); + }); + }); + + describe("flood guard", () => { + it("defers with 'flood' when threshold runs land within window", () => { + const now = 1_000_000; + const recentRunStarts = [ + now - 50_000, + now - 40_000, + now - 30_000, + now - 20_000, + now - 10_000, + ]; + expect( + decide({ + source: "exec-event", + now, + nextDueMs: 0, + lastRunStartedAtMs: now - DEFAULT_MIN_WAKE_SPACING_MS - 1, + recentRunStarts, + reason: "exec-event", + }), + ).toEqual({ defer: true, reason: "flood" }); + }); + + it("does not flood-defer when recent runs are spread outside window", () => { + const now = 1_000_000; + const recentRunStarts = [ + now - 300_000, + now - 240_000, + now - 180_000, + now - 120_000, + now - 65_000, // just outside default 60s window + ]; + expect( + decide({ + source: "exec-event", + now, + nextDueMs: 0, + lastRunStartedAtMs: now - DEFAULT_MIN_WAKE_SPACING_MS - 1, + recentRunStarts, + reason: "exec-event", + }), + ).toEqual({ defer: false }); + }); + + it("does not flood-defer below threshold", () => { + const now = 1_000_000; + const recentRunStarts = [now - 30_000, now - 20_000, now - 10_000]; + expect( + decide({ + source: "exec-event", + now, + nextDueMs: 0, + lastRunStartedAtMs: now - DEFAULT_MIN_WAKE_SPACING_MS - 1, + recentRunStarts, + reason: "exec-event", + }), + ).toEqual({ defer: false }); + }); + }); +}); + +describe("recordRunStart", () => { + it("trims buffer to threshold + 1 entries", () => { + const buffer: number[] = []; + for (let i = 1; i <= DEFAULT_FLOOD_THRESHOLD + 5; i++) { + recordRunStart(buffer, i); + } + expect(buffer.length).toBe(DEFAULT_FLOOD_THRESHOLD + 1); + expect(buffer[buffer.length - 1]).toBe(DEFAULT_FLOOD_THRESHOLD + 5); + }); + + it("preserves insertion order", () => { + const buffer: number[] = []; + recordRunStart(buffer, 100); + recordRunStart(buffer, 200); + recordRunStart(buffer, 300); + expect(buffer).toEqual([100, 200, 300]); + }); +}); diff --git a/src/infra/heartbeat-cooldown.ts b/src/infra/heartbeat-cooldown.ts new file mode 100644 index 00000000000..52238460306 --- /dev/null +++ b/src/infra/heartbeat-cooldown.ts @@ -0,0 +1,164 @@ +// Centralized cooldown decision for heartbeat wakes. +// +// Background: a heartbeat run can be triggered by many wake sources — the +// scheduler's interval tick, a manual user request, a backgrounded `process.start` +// exit, a cron tick, an ACP spawn stream event, etc. Different sources used to +// take different code paths through the dispatcher, and historically the +// `nextDueMs` cooldown gate was only enforced on the `interval` branch. That let +// event-driven wakes (especially `exec-event`) fire heartbeat runs back-to-back +// when a heartbeat agent's tools triggered more wakes (#17797 → #75436). +// +// This module owns the single decision: "given this wake, should we run now or +// defer it?" Both the targeted and broadcast dispatch branches must call +// `shouldDeferWake` so the gate can never be forgotten on one path. + +import type { HeartbeatWakeIntent, HeartbeatWakeSource } from "./heartbeat-wake.js"; + +// Default minimum spacing between heartbeat runs for the same agent, regardless +// of configured `every`. Even when `nextDueMs` is enforced, two wakes arriving +// within milliseconds can race the schedule update; this floor prevents that. +export const DEFAULT_MIN_WAKE_SPACING_MS = 30_000; + +// Flood guard: if more than this many wakes for the same agent fall within the +// flood window, the dispatcher logs a warning and forces the wake to defer to +// the next scheduled tick. Tuned so a normal heartbeat that legitimately uses +// `manual` retry doesn't trip it but a feedback loop does. +export const DEFAULT_FLOOD_WINDOW_MS = 60_000; +export const DEFAULT_FLOOD_THRESHOLD = 5; + +export type DeferDecision = + | { defer: false } + | { defer: true; reason: "not-due" | "min-spacing" | "flood" }; + +export type ShouldDeferInput = { + /** Scheduler behavior requested by the wake producer. */ + intent: HeartbeatWakeIntent; + /** Wake producer, used for diagnostics and future source-specific telemetry. */ + source?: HeartbeatWakeSource; + /** Raw wake reason string for logs/model context. It does not drive scheduling policy. */ + reason: string | undefined; + /** Current monotonic-ish wall clock. Pass `Date.now()`. */ + now: number; + /** When this agent's next interval-tick run is due. */ + nextDueMs: number; + /** When this agent last *started* a run, if known. */ + lastRunStartedAtMs?: number; + /** Recent wake timestamps for flood detection. */ + recentRunStarts?: readonly number[]; + /** Override the minimum spacing floor. */ + minSpacingMs?: number; + /** Override the flood-window length. */ + floodWindowMs?: number; + /** Override the flood-window threshold. */ + floodThreshold?: number; +}; + +/** + * Decide whether an incoming wake should be deferred. + * + * The decision matrix: + * + * | Wake intent | First wake (no prior run) | Subsequent wakes | + * |---------------|----------------------------|-----------------------------------------| + * | manual | Run | Run (never deferred) | + * | immediate | Run | Run (never deferred, except flood) | + * | scheduled | Defer if now < nextDueMs | Defer if now < nextDueMs | + * | event | Run (bootstrap responsive) | Defer if now < nextDueMs OR within floor | + * + * Immediate is for documented wake-now delivery paths such as `openclaw system + * event --mode now`, task completion follow-ups, cron `--wake now`, and + * `/hooks/wake mode=now`. Event is for external/system notifications such as + * background exec exits, node notification changes, hook/cron next-heartbeat + * handoffs, ACP spawn stream updates, and retry wakes. + * + * Additional gates layered on top of the reason matrix: + * + * 1. **Minimum spacing floor** (`min-spacing`): even if `nextDueMs` has been + * passed, defer if a run started within the last `minSpacingMs`. Catches + * the race where a second wake arrives between `runOnce` returning and + * `advanceAgentSchedule` updating `nextDueMs`. + * 2. **Flood guard** (`flood`): if `recentRunStarts` shows ≥ `floodThreshold` + * runs within `floodWindowMs`, defer regardless of reason (except + * `manual`-class immediate intent). Caller should also emit a single + * warning log when this fires. + */ +export function shouldDeferWake(input: ShouldDeferInput): DeferDecision { + if (input.intent === "manual") { + return { defer: false }; + } + + if (input.intent === "immediate") { + // Even immediate wakes get rate-limited if a real flood is happening — but + // manual operator intent is fully exempt above. System-event, task, hook, + // and cron wake-now paths come from external systems we trust but cannot + // prove are loop-free, so the flood guard remains a backstop. + const floodDefer = checkFloodGuard(input); + return floodDefer ?? { defer: false }; + } + + // Flood guard applies to every non-immediate wake regardless of run history. + // It is the last line of defense against feedback loops. + const floodDefer = checkFloodGuard(input); + if (floodDefer) { + return floodDefer; + } + + if (input.intent === "scheduled") { + return input.now < input.nextDueMs ? { defer: true, reason: "not-due" } : { defer: false }; + } + + // Event-driven wakes. First wake (no prior run) bypasses cooldown gates so + // an idle agent can respond to an external event without waiting for the + // first scheduled phase tick. + if (input.lastRunStartedAtMs === undefined) { + return { defer: false }; + } + + if (input.now < input.nextDueMs) { + return { defer: true, reason: "not-due" }; + } + + const minSpacing = input.minSpacingMs ?? DEFAULT_MIN_WAKE_SPACING_MS; + if (minSpacing > 0 && input.now - input.lastRunStartedAtMs < minSpacing) { + return { defer: true, reason: "min-spacing" }; + } + + return { defer: false }; +} + +function checkFloodGuard(input: ShouldDeferInput): DeferDecision | null { + const floodWindow = input.floodWindowMs ?? DEFAULT_FLOOD_WINDOW_MS; + const floodThreshold = input.floodThreshold ?? DEFAULT_FLOOD_THRESHOLD; + if (!input.recentRunStarts || input.recentRunStarts.length < floodThreshold || floodWindow <= 0) { + return null; + } + const windowStart = input.now - floodWindow; + let inWindow = 0; + for (let i = input.recentRunStarts.length - 1; i >= 0; i--) { + const ts = input.recentRunStarts[i]; + if (ts === undefined || ts < windowStart) { + break; + } + inWindow += 1; + } + return inWindow >= floodThreshold ? { defer: true, reason: "flood" } : null; +} + +/** + * Append a run-start timestamp to a bounded recent-runs buffer. Caller passes + * the previous buffer; this returns a new (mutated) buffer with the entry + * appended and trimmed to `floodThreshold + 1` entries (only the newest matter + * for flood detection). + */ +export function recordRunStart( + buffer: number[], + ts: number, + floodThreshold: number = DEFAULT_FLOOD_THRESHOLD, +): number[] { + buffer.push(ts); + const max = floodThreshold + 1; + while (buffer.length > max) { + buffer.shift(); + } + return buffer; +} diff --git a/src/infra/heartbeat-reason.test.ts b/src/infra/heartbeat-reason.test.ts index ab0fe94ec06..05f5eb4405d 100644 --- a/src/infra/heartbeat-reason.test.ts +++ b/src/infra/heartbeat-reason.test.ts @@ -1,10 +1,5 @@ import { describe, expect, it } from "vitest"; -import { - isHeartbeatActionWakeReason, - isHeartbeatEventDrivenReason, - normalizeHeartbeatWakeReason, - resolveHeartbeatReasonKind, -} from "./heartbeat-reason.js"; +import { normalizeHeartbeatWakeReason } from "./heartbeat-reason.js"; describe("heartbeat-reason", () => { it.each([ @@ -14,47 +9,4 @@ describe("heartbeat-reason", () => { ])("normalizes wake reasons for %j", ({ value, expected }) => { expect(normalizeHeartbeatWakeReason(value)).toBe(expected); }); - - it.each([ - { value: "retry", expected: "retry" }, - { value: "interval", expected: "interval" }, - { value: "manual", expected: "manual" }, - { value: "exec-event", expected: "exec-event" }, - { value: "wake", expected: "wake" }, - { value: "acp:spawn:stream", expected: "wake" }, - { value: "acp:spawn:", expected: "wake" }, - { value: "cron:job-1", expected: "cron" }, - { value: "hook:wake", expected: "hook" }, - { value: " hook:wake ", expected: "hook" }, - { value: "requested", expected: "other" }, - { value: "slow", expected: "other" }, - { value: "", expected: "other" }, - { value: undefined, expected: "other" }, - ])("classifies reason kinds for %j", ({ value, expected }) => { - expect(resolveHeartbeatReasonKind(value)).toBe(expected); - }); - - it.each([ - { value: "exec-event", expected: true }, - { value: "cron:job-1", expected: true }, - { value: "wake", expected: true }, - { value: "acp:spawn:stream", expected: true }, - { value: "hook:gmail:sync", expected: true }, - { value: "interval", expected: false }, - { value: "manual", expected: false }, - { value: "other", expected: false }, - ])("matches event-driven behavior for %j", ({ value, expected }) => { - expect(isHeartbeatEventDrivenReason(value)).toBe(expected); - }); - - it.each([ - { value: "manual", expected: true }, - { value: "exec-event", expected: true }, - { value: "hook:wake", expected: true }, - { value: "interval", expected: false }, - { value: "cron:job-1", expected: false }, - { value: "retry", expected: false }, - ])("matches action-priority wake behavior for %j", ({ value, expected }) => { - expect(isHeartbeatActionWakeReason(value)).toBe(expected); - }); }); diff --git a/src/infra/heartbeat-reason.ts b/src/infra/heartbeat-reason.ts index ca5cde1336a..742f2ad01fa 100644 --- a/src/infra/heartbeat-reason.ts +++ b/src/infra/heartbeat-reason.ts @@ -1,59 +1,5 @@ import { normalizeOptionalString } from "../shared/string-coerce.js"; -type HeartbeatReasonKind = - | "retry" - | "interval" - | "manual" - | "exec-event" - | "wake" - | "cron" - | "hook" - | "other"; - -function trimReason(reason?: string): string { - return normalizeOptionalString(reason) ?? ""; -} - export function normalizeHeartbeatWakeReason(reason?: string): string { - const trimmed = trimReason(reason); - return trimmed.length > 0 ? trimmed : "requested"; -} - -export function resolveHeartbeatReasonKind(reason?: string): HeartbeatReasonKind { - const trimmed = trimReason(reason); - if (trimmed === "retry") { - return "retry"; - } - if (trimmed === "interval") { - return "interval"; - } - if (trimmed === "manual") { - return "manual"; - } - if (trimmed === "exec-event") { - return "exec-event"; - } - if (trimmed === "wake") { - return "wake"; - } - if (trimmed.startsWith("acp:spawn:")) { - return "wake"; - } - if (trimmed.startsWith("cron:")) { - return "cron"; - } - if (trimmed.startsWith("hook:")) { - return "hook"; - } - return "other"; -} - -export function isHeartbeatEventDrivenReason(reason?: string): boolean { - const kind = resolveHeartbeatReasonKind(reason); - return kind === "exec-event" || kind === "cron" || kind === "wake" || kind === "hook"; -} - -export function isHeartbeatActionWakeReason(reason?: string): boolean { - const kind = resolveHeartbeatReasonKind(reason); - return kind === "manual" || kind === "exec-event" || kind === "hook"; + return normalizeOptionalString(reason) ?? "requested"; } diff --git a/src/infra/heartbeat-runner.commitments.test.ts b/src/infra/heartbeat-runner.commitments.test.ts index 5a1c61dc333..fb9aab8e5fa 100644 --- a/src/infra/heartbeat-runner.commitments.test.ts +++ b/src/infra/heartbeat-runner.commitments.test.ts @@ -12,7 +12,7 @@ import { } from "./heartbeat-runner.js"; import { installHeartbeatRunnerTestRuntime } from "./heartbeat-runner.test-harness.js"; import { seedSessionStore, withTempHeartbeatSandbox } from "./heartbeat-runner.test-utils.js"; -import { requestHeartbeatNow, resetHeartbeatWakeStateForTests } from "./heartbeat-wake.js"; +import { requestHeartbeat, resetHeartbeatWakeStateForTests } from "./heartbeat-wake.js"; installHeartbeatRunnerTestRuntime(); @@ -348,7 +348,7 @@ describe("runHeartbeatOnce commitments", () => { stableSchedulerSeed: "commitment-target-none", }); - requestHeartbeatNow({ reason: "manual", coalesceMs: 0 }); + requestHeartbeat({ source: "manual", intent: "manual", reason: "manual", coalesceMs: 0 }); await vi.advanceTimersByTimeAsync(1); runner.stop(); diff --git a/src/infra/heartbeat-runner.ghost-reminder.test.ts b/src/infra/heartbeat-runner.ghost-reminder.test.ts index 735abe96971..7351a2b7703 100644 --- a/src/infra/heartbeat-runner.ghost-reminder.test.ts +++ b/src/infra/heartbeat-runner.ghost-reminder.test.ts @@ -511,6 +511,8 @@ describe("Ghost reminder bug (issue #13317)", () => { const result = await runHeartbeatOnce({ cfg, agentId: "main", + source: "hook", + intent: "immediate", reason: "wake", deps: { getReplyFromConfig: replySpy, @@ -551,6 +553,8 @@ describe("Ghost reminder bug (issue #13317)", () => { const result = await runHeartbeatOnce({ cfg, agentId: "main", + source: "hook", + intent: "immediate", reason: "wake", deps: { getReplyFromConfig: replySpy, diff --git a/src/infra/heartbeat-runner.isolated-key-stability.test.ts b/src/infra/heartbeat-runner.isolated-key-stability.test.ts index d6690acd61d..b585a0d5b9d 100644 --- a/src/infra/heartbeat-runner.isolated-key-stability.test.ts +++ b/src/infra/heartbeat-runner.isolated-key-stability.test.ts @@ -24,7 +24,7 @@ describe("runHeartbeatOnce – isolated session key stability (#59493)", () => { /** * Simulates the wake-request feedback loop: * 1. Normal heartbeat tick produces sessionKey "agent:main:main:heartbeat" - * 2. An exec/subagent event during that tick calls requestHeartbeatNow() + * 2. An exec/subagent event during that tick calls requestHeartbeat() * with the already-suffixed key "agent:main:main:heartbeat" * 3. The wake handler passes that key back into runHeartbeatOnce(sessionKey: ...) * diff --git a/src/infra/heartbeat-runner.returns-default-unset.test.ts b/src/infra/heartbeat-runner.returns-default-unset.test.ts index d614783d810..5a790370f53 100644 --- a/src/infra/heartbeat-runner.returns-default-unset.test.ts +++ b/src/infra/heartbeat-runner.returns-default-unset.test.ts @@ -1389,6 +1389,11 @@ describe("runHeartbeatOnce", () => { .mockResolvedValue({ messageId: "m1", toJid: "jid" }); const res = await runHeartbeatOnce({ cfg, + ...(params.reason === "wake" + ? { source: "hook" as const, intent: "immediate" as const } + : params.reason === "interval" + ? { source: "interval" as const, intent: "scheduled" as const } + : {}), reason: params.reason, deps: createHeartbeatDeps(sendWhatsApp, { getReplyFromConfig: replySpy }), }); @@ -1736,6 +1741,8 @@ tasks: try { const res = await runHeartbeatOnce({ cfg, + source: "interval", + intent: "scheduled", reason: "interval", deps: createHeartbeatDeps(sendWhatsApp, { getReplyFromConfig: replySpy }), }); @@ -1791,6 +1798,8 @@ tasks: try { const res = await runHeartbeatOnce({ cfg, + source: "exec-event", + intent: "event", reason: "exec-event", deps: createHeartbeatDeps(sendWhatsApp, { getReplyFromConfig: replySpy }), }); diff --git a/src/infra/heartbeat-runner.scheduler.test.ts b/src/infra/heartbeat-runner.scheduler.test.ts index cc1ec00f58d..113382b2d37 100644 --- a/src/infra/heartbeat-runner.scheduler.test.ts +++ b/src/infra/heartbeat-runner.scheduler.test.ts @@ -6,7 +6,7 @@ import { HEARTBEAT_SKIP_CRON_IN_PROGRESS, HEARTBEAT_SKIP_REQUESTS_IN_FLIGHT, type RetryableHeartbeatBusySkipReason, - requestHeartbeatNow, + requestHeartbeat, resetHeartbeatWakeStateForTests, } from "./heartbeat-wake.js"; @@ -61,10 +61,47 @@ describe("startHeartbeatRunner", () => { }); } + function wake( + reason: string, + opts: Partial[0]> = {}, + ): Parameters[0] { + const source = + opts.source ?? + (reason === "interval" + ? "interval" + : reason === "manual" + ? "manual" + : reason === "retry" + ? "retry" + : reason === "exec-event" + ? "exec-event" + : reason === "background-task" + ? "background-task" + : reason === "background-task-blocked" + ? "background-task-blocked" + : reason.startsWith("cron:") + ? "cron" + : reason.startsWith("hook:") + ? "hook" + : "other"); + const intent = + opts.intent ?? + (reason === "interval" + ? "scheduled" + : reason === "manual" + ? "manual" + : reason === "wake" || + reason === "background-task" || + reason === "background-task-blocked" + ? "immediate" + : "event"); + return { source, intent, reason, ...opts }; + } + async function expectWakeDispatch(params: { cfg: OpenClawConfig; runSpy: RunOnce; - wake: Parameters[0]; + wake: Parameters[0]; expectedCall: Record; }) { const runner = startHeartbeatRunner({ @@ -73,7 +110,7 @@ describe("startHeartbeatRunner", () => { stableSchedulerSeed: TEST_SCHEDULER_SEED, }); - requestHeartbeatNow(params.wake); + requestHeartbeat(params.wake); await vi.advanceTimersByTimeAsync(1); expect(params.runSpy).toHaveBeenCalledTimes(1); @@ -313,7 +350,7 @@ describe("startHeartbeatRunner", () => { // Simulate 4 more retries at short intervals (wake layer retries). for (let i = 0; i < 4; i++) { - requestHeartbeatNow({ reason: "retry", coalesceMs: 0 }); + requestHeartbeat(wake("retry", { coalesceMs: 0 })); await vi.advanceTimersByTimeAsync(1_000); } expect(callTimes.some((time) => time >= firstDueMs + intervalMs)).toBe(false); @@ -338,6 +375,8 @@ describe("startHeartbeatRunner", () => { } as OpenClawConfig, runSpy, wake: { + source: "cron", + intent: "event", reason: "cron:job-123", agentId: "ops", sessionKey: "agent:ops:discord:channel:alerts", @@ -360,6 +399,8 @@ describe("startHeartbeatRunner", () => { cfg: heartbeatConfig([{ id: "main" }, { id: "ops" }]), runSpy, wake: { + source: "cron", + intent: "event", reason: "cron:job-123", agentId: "ops", sessionKey: "agent:ops:discord:channel:alerts", @@ -394,6 +435,8 @@ describe("startHeartbeatRunner", () => { } as OpenClawConfig, runSpy, wake: { + source: "cron", + intent: "event", reason: "cron:job-123", agentId: "ops", sessionKey: "agent:ops:discord:channel:alerts", @@ -446,6 +489,8 @@ describe("startHeartbeatRunner", () => { } as OpenClawConfig, runSpy, wake: { + source: "exec-event", + intent: "event", reason: "exec-event", sessionKey: "agent:main:main", coalesceMs: 0, @@ -460,4 +505,240 @@ describe("startHeartbeatRunner", () => { runner.stop(); }); + + // Regression for runaway heartbeat loop: backgrounded `process.start` exits + // call `requestHeartbeat({reason: "exec-event"})` from + // `bash-tools.exec-runtime.ts:347` (`maybeNotifyOnExit`). If a heartbeat run + // uses backgrounded tools (response-tracker sync, conversation monitors, + // etc.), each background process exit triggers another heartbeat run because + // the dispatcher (`heartbeat-runner.ts:1805`) only enforces `nextDueMs` when + // `reason === "interval"`, and the targeted branch has no cooldown gate at + // all. Observed in production: heartbeat configured `every: 30m` fires every + // ~10s, pegging the gateway event loop with eventLoopDelayMaxMs >6s spikes. + it("does not bypass interval cooldown for repeated exec-event wakes within nextDueMs", async () => { + useFakeHeartbeatTime(); + const runSpy = vi.fn().mockResolvedValue({ status: "ran", durationMs: 1 }); + + const runner = startHeartbeatRunner({ + cfg: heartbeatConfig(), + runOnce: runSpy, + stableSchedulerSeed: TEST_SCHEDULER_SEED, + }); + + // First exec-event wake: agent just woke from a backgrounded tool exit. + // This one legitimately fires the run. + requestHeartbeat({ + source: "exec-event", + intent: "event", + reason: "exec-event", + sessionKey: "agent:main:main", + coalesceMs: 0, + }); + await vi.advanceTimersByTimeAsync(1); + expect(runSpy).toHaveBeenCalledTimes(1); + + // Simulate the runaway: 4 more exec-event wakes from backgrounded process + // exits, fired well within the configured 30m interval. These should be + // debounced by the cooldown — the agent just ran, nothing has changed. + for (let i = 0; i < 4; i++) { + await vi.advanceTimersByTimeAsync(10_000); // 10s between background exits + requestHeartbeat({ + source: "exec-event", + intent: "event", + reason: "exec-event", + sessionKey: "agent:main:main", + coalesceMs: 0, + }); + await vi.advanceTimersByTimeAsync(1); + } + + // Total elapsed: ~40s. Configured `every` is 30m. Subsequent exec-events + // should NOT trigger fresh runs within the cooldown window. + expect(runSpy).toHaveBeenCalledTimes(1); + + runner.stop(); + }); + + it("preserves immediate delivery for repeated bare wake reasons", async () => { + // 'wake' is the immediate-path reason from `openclaw system event --mode now` + // and must NOT be deferred. Verify the runner allows multiple back-to-back + // wake requests through (subject only to the flood guard backstop). + useFakeHeartbeatTime(); + const runSpy = vi.fn().mockResolvedValue({ status: "ran", durationMs: 1 }); + const runner = startHeartbeatRunner({ + cfg: heartbeatConfig(), + runOnce: runSpy, + stableSchedulerSeed: TEST_SCHEDULER_SEED, + }); + + // Three 'wake' requests with 200ms between them — none should be deferred. + for (let i = 0; i < 3; i++) { + requestHeartbeat({ + source: "manual", + intent: "immediate", + reason: "wake", + sessionKey: "agent:main:main", + coalesceMs: 0, + }); + await vi.advanceTimersByTimeAsync(1); + await vi.advanceTimersByTimeAsync(200); + } + + expect(runSpy).toHaveBeenCalledTimes(3); + runner.stop(); + }); + + it("preserves immediate delivery for repeated background-task wakes", async () => { + // Task-registry terminal updates wake the heartbeat with reason + // 'background-task'. Documented as immediate so users don't wait for the + // next scheduled tick to see task completion notifications. + useFakeHeartbeatTime(); + const runSpy = vi.fn().mockResolvedValue({ status: "ran", durationMs: 1 }); + const runner = startHeartbeatRunner({ + cfg: heartbeatConfig(), + runOnce: runSpy, + stableSchedulerSeed: TEST_SCHEDULER_SEED, + }); + + for (let i = 0; i < 3; i++) { + requestHeartbeat({ + source: "background-task", + intent: "immediate", + reason: "background-task", + sessionKey: "agent:main:main", + coalesceMs: 0, + }); + await vi.advanceTimersByTimeAsync(1); + await vi.advanceTimersByTimeAsync(200); + } + + expect(runSpy).toHaveBeenCalledTimes(3); + runner.stop(); + }); + + it("preserves immediate delivery for blocked background-task follow-ups", async () => { + useFakeHeartbeatTime(); + const runSpy = vi.fn().mockResolvedValue({ status: "ran", durationMs: 1 }); + const runner = startHeartbeatRunner({ + cfg: heartbeatConfig(), + runOnce: runSpy, + stableSchedulerSeed: TEST_SCHEDULER_SEED, + }); + + requestHeartbeat({ + source: "exec-event", + intent: "event", + reason: "exec-event", + sessionKey: "agent:main:main", + coalesceMs: 0, + }); + await vi.advanceTimersByTimeAsync(1); + expect(runSpy).toHaveBeenCalledTimes(1); + + requestHeartbeat({ + source: "background-task-blocked", + intent: "immediate", + reason: "background-task-blocked", + sessionKey: "agent:main:main", + coalesceMs: 0, + }); + await vi.advanceTimersByTimeAsync(1); + + expect(runSpy).toHaveBeenCalledTimes(2); + expect(runSpy.mock.calls[1]?.[0]).toEqual( + expect.objectContaining({ + reason: "background-task-blocked", + sessionKey: "agent:main:main", + }), + ); + runner.stop(); + }); + + it.each([ + { reason: "hook:wake", label: "hook wake-now" }, + { reason: "hook:job-123", label: "hook agent wake-now announcement" }, + { reason: "cron:job-123", label: "cron wake-now" }, + ])("preserves immediate delivery for $label after a recent run", async ({ reason }) => { + useFakeHeartbeatTime(); + const runSpy = vi.fn().mockResolvedValue({ status: "ran", durationMs: 1 }); + const runner = startHeartbeatRunner({ + cfg: heartbeatConfig(), + runOnce: runSpy, + stableSchedulerSeed: TEST_SCHEDULER_SEED, + }); + + requestHeartbeat({ + source: "exec-event", + intent: "event", + reason: "exec-event", + sessionKey: "agent:main:main", + coalesceMs: 0, + }); + await vi.advanceTimersByTimeAsync(1); + expect(runSpy).toHaveBeenCalledTimes(1); + + requestHeartbeat({ + source: reason.startsWith("cron:") ? "cron" : "hook", + intent: "immediate", + reason, + sessionKey: "agent:main:main", + coalesceMs: 0, + }); + await vi.advanceTimersByTimeAsync(1); + + expect(runSpy).toHaveBeenCalledTimes(2); + expect(runSpy.mock.calls[1]?.[0]).toEqual( + expect.objectContaining({ reason, sessionKey: "agent:main:main" }), + ); + runner.stop(); + }); + + it("retryable busy skip does not poison the cooldown for the next retry", async () => { + // Reproduces P2 finding from #75439 review: if a targeted exec-event wake + // hits requests-in-flight on its first attempt, the wake layer retries the + // same reason. The cooldown must NOT have been advanced by the busy attempt + // — otherwise the retry would falsely defer with `not-due`/`min-spacing`. + useFakeHeartbeatTime(); + let attempt = 0; + const runSpy = vi.fn().mockImplementation(async () => { + attempt += 1; + if (attempt === 1) { + return { status: "skipped", reason: HEARTBEAT_SKIP_REQUESTS_IN_FLIGHT } as const; + } + return { status: "ran", durationMs: 1 } as const; + }); + + const runner = startHeartbeatRunner({ + cfg: heartbeatConfig(), + runOnce: runSpy, + stableSchedulerSeed: TEST_SCHEDULER_SEED, + }); + + requestHeartbeat({ + source: "exec-event", + intent: "event", + reason: "exec-event", + sessionKey: "agent:main:main", + coalesceMs: 0, + }); + await vi.advanceTimersByTimeAsync(1); + expect(runSpy).toHaveBeenCalledTimes(1); + + // Wake layer retries via DEFAULT_RETRY_MS (1s). Advance past it. + await vi.advanceTimersByTimeAsync(1500); + + // The retry must NOT be deferred to `not-due` or `min-spacing`. Since the + // first attempt was a retryable busy skip, the cooldown bookkeeping was + // never recorded — so the retry should reach runOnce normally. + expect(runSpy).toHaveBeenCalledTimes(2); + expect(runSpy.mock.calls[1]?.[0]).toEqual( + expect.objectContaining({ reason: "exec-event", sessionKey: "agent:main:main" }), + ); + await expect(runSpy.mock.results[1]?.value).resolves.toEqual({ + status: "ran", + durationMs: 1, + }); + + runner.stop(); + }); }); diff --git a/src/infra/heartbeat-runner.ts b/src/infra/heartbeat-runner.ts index 3a1c97b3e4f..b23d704f194 100644 --- a/src/infra/heartbeat-runner.ts +++ b/src/infra/heartbeat-runner.ts @@ -84,6 +84,7 @@ import { MAX_SAFE_TIMEOUT_DELAY_MS, resolveSafeTimeoutDelayMs } from "../utils/t import { loadOrCreateDeviceIdentity } from "./device-identity.js"; import { formatErrorMessage, hasErrnoCode } from "./errors.js"; import { isWithinActiveHours } from "./heartbeat-active-hours.js"; +import { recordRunStart, shouldDeferWake, type DeferDecision } from "./heartbeat-cooldown.js"; import { buildCronEventPrompt, buildExecEventPrompt, @@ -92,7 +93,6 @@ import { isRelayableExecCompletionEvent, } from "./heartbeat-events-filter.js"; import { emitHeartbeatEvent, resolveIndicatorType } from "./heartbeat-events.js"; -import { resolveHeartbeatReasonKind } from "./heartbeat-reason.js"; import { computeNextHeartbeatPhaseDueMs, resolveHeartbeatPhaseMs, @@ -113,9 +113,11 @@ import { HEARTBEAT_SKIP_REQUESTS_IN_FLIGHT, type HeartbeatRunResult, type HeartbeatWakeHandler, + type HeartbeatWakeIntent, type HeartbeatWakeRequest, + type HeartbeatWakeSource, isRetryableHeartbeatBusySkipReason, - requestHeartbeatNow, + requestHeartbeat, setHeartbeatsEnabled, setHeartbeatWakeHandler, } from "./heartbeat-wake.js"; @@ -213,6 +215,12 @@ type HeartbeatAgentState = { intervalMs: number; phaseMs: number; nextDueMs: number; + /** Wall-clock start time of the most recent run for this agent. */ + lastRunStartedAtMs?: number; + /** Bounded ring buffer of recent run-start timestamps for flood detection. */ + recentRunStarts: number[]; + /** Set true after a flood-defer is logged to avoid log spam. Reset when a run actually fires. */ + floodLoggedSinceLastRun: boolean; }; export type HeartbeatRunner = { @@ -602,10 +610,10 @@ function normalizeHeartbeatToolNotification( return { shouldSkip: false, text: finalText, hasMedia: false }; } -type HeartbeatReasonFlags = { - isExecEventReason: boolean; - isCronEventReason: boolean; - isWakeReason: boolean; +type HeartbeatWakePayloadFlags = { + isExecEventWake: boolean; + isCronWake: boolean; + isWakePayload: boolean; }; type HeartbeatSkipReason = "empty-heartbeat-file"; @@ -661,7 +669,7 @@ Commitments: ${JSON.stringify(items, null, 2)}`; } -type HeartbeatPreflight = HeartbeatReasonFlags & { +type HeartbeatPreflight = HeartbeatWakePayloadFlags & { session: ReturnType; pendingEventEntries: ReturnType; turnSourceDeliveryContext: ReturnType; @@ -673,12 +681,33 @@ type HeartbeatPreflight = HeartbeatReasonFlags & { heartbeatFileContent?: string; }; -function resolveHeartbeatReasonFlags(reason?: string): HeartbeatReasonFlags { - const reasonKind = resolveHeartbeatReasonKind(reason); +function inferHeartbeatWakeSourceFromReason(reason?: string): HeartbeatWakeSource | undefined { + const trimmed = (reason ?? "").trim(); + if (trimmed === "exec-event") { + return "exec-event"; + } + if (trimmed.startsWith("cron:")) { + return "cron"; + } + if (trimmed === "wake" || trimmed.startsWith("hook:")) { + return "hook"; + } + if (trimmed.startsWith("acp:spawn:")) { + return "acp-spawn"; + } + return undefined; +} + +function resolveHeartbeatWakePayloadFlags(params: { + source?: HeartbeatWakeSource; + reason?: string; +}): HeartbeatWakePayloadFlags { + const source = params.source ?? inferHeartbeatWakeSourceFromReason(params.reason); + const reason = (params.reason ?? "").trim(); return { - isExecEventReason: reasonKind === "exec-event", - isCronEventReason: reasonKind === "cron", - isWakeReason: reasonKind === "wake" || reasonKind === "hook", + isExecEventWake: source === "exec-event", + isCronWake: source === "cron", + isWakePayload: source === "hook" || source === "acp-spawn" || reason === "wake", }; } @@ -688,9 +717,13 @@ async function resolveHeartbeatPreflight(params: { heartbeat?: HeartbeatConfig; forcedSessionKey?: string; reason?: string; + source?: HeartbeatWakeSource; nowMs?: number; }): Promise { - const reasonFlags = resolveHeartbeatReasonFlags(params.reason); + const wakeFlags = resolveHeartbeatWakePayloadFlags({ + source: params.source, + reason: params.reason, + }); const session = resolveHeartbeatSession( params.cfg, params.agentId, @@ -715,7 +748,7 @@ async function resolveHeartbeatPreflight(params: { // Wake-triggered runs should only inspect pending events when preflight peeks // the same queue that the run itself will execute/drain. const shouldInspectWakePendingEvents = (() => { - if (!reasonFlags.isWakeReason) { + if (!wakeFlags.isWakePayload) { return false; } if (params.heartbeat?.isolatedSession !== true) { @@ -730,17 +763,17 @@ async function resolveHeartbeatPreflight(params: { return isolatedSessionKey === session.sessionKey; })(); const shouldInspectPendingEvents = - reasonFlags.isExecEventReason || - reasonFlags.isCronEventReason || + wakeFlags.isExecEventWake || + wakeFlags.isCronWake || shouldInspectWakePendingEvents || hasTaggedCronEvents; const shouldBypassFileGates = - reasonFlags.isExecEventReason || - reasonFlags.isCronEventReason || - reasonFlags.isWakeReason || + wakeFlags.isExecEventWake || + wakeFlags.isCronWake || + wakeFlags.isWakePayload || hasTaggedCronEvents; const basePreflight = { - ...reasonFlags, + ...wakeFlags, session, pendingEventEntries, turnSourceDeliveryContext, @@ -870,7 +903,7 @@ function resolveHeartbeatRunPrompt(params: { const cronEvents = pendingEventEntries .filter( (event) => - (params.preflight.isCronEventReason || event.contextKey?.startsWith("cron:")) && + (params.preflight.isCronWake || event.contextKey?.startsWith("cron:")) && isCronSystemEvent(event.text), ) .map((event) => event.text); @@ -962,7 +995,7 @@ function selectSystemEventsConsumedByHeartbeat(params: { if (params.hasCronEvents) { return preflight.pendingEventEntries.filter( (event) => - (preflight.isCronEventReason || event.contextKey?.startsWith("cron:")) && + (preflight.isCronWake || event.contextKey?.startsWith("cron:")) && isCronSystemEvent(event.text), ); } @@ -974,6 +1007,8 @@ export async function runHeartbeatOnce(opts: { agentId?: string; sessionKey?: string; heartbeat?: HeartbeatConfig; + source?: HeartbeatWakeSource; + intent?: HeartbeatWakeIntent; reason?: string; deps?: HeartbeatDeps; }): Promise { @@ -1030,6 +1065,7 @@ export async function runHeartbeatOnce(opts: { agentId, heartbeat, forcedSessionKey: opts.sessionKey, + source: opts.source, reason: opts.reason, nowMs: startedAt, }); @@ -1157,7 +1193,7 @@ export async function runHeartbeatOnce(opts: { // Wake-triggered events should stay queued when the run short-circuits: // no reply turn ran, so there is nothing that actually consumed that wake payload. const shouldConsumeInspectedEvents = - !preflight.isWakeReason && preflight.shouldInspectPendingEvents; + !preflight.isWakePayload && preflight.shouldInspectPendingEvents; if (shouldConsumeInspectedEvents && inspectedSystemEventsToConsume.length > 0) { consumeSelectedSystemEventEntries(sessionKey, inspectedSystemEventsToConsume); } @@ -1756,6 +1792,45 @@ export function startHeartbeatRunner(opts: { now + agent.intervalMs; }; + // Centralized cooldown gate. Both targeted and broadcast dispatch branches + // call this before invoking `runOnce`. Manual wakes are never deferred. + // Everything else respects `nextDueMs`, the min-spacing floor, and the flood + // guard — see `heartbeat-cooldown.ts` for rationale and #75436. + const evaluateWakeDeferral = ( + agent: HeartbeatAgentState, + now: number, + reason?: string, + intent: HeartbeatWakeIntent = "event", + ): DeferDecision => { + const decision = shouldDeferWake({ + intent, + reason, + now, + nextDueMs: agent.nextDueMs, + lastRunStartedAtMs: agent.lastRunStartedAtMs, + recentRunStarts: agent.recentRunStarts, + }); + if (decision.defer && decision.reason === "flood") { + if (!agent.floodLoggedSinceLastRun) { + log.warn("heartbeat: flood guard tripped, deferring wake", { + agentId: agent.agentId, + reason: reason ?? "(none)", + recentRunCount: agent.recentRunStarts.length, + }); + agent.floodLoggedSinceLastRun = true; + } + } + return decision; + }; + + // Called immediately before `runOnce` actually executes. Updates the + // bookkeeping that the cooldown gate consults on the next wake. + const recordRunBookkeeping = (agent: HeartbeatAgentState, now: number) => { + agent.lastRunStartedAtMs = now; + recordRunStart(agent.recentRunStarts, now); + agent.floodLoggedSinceLastRun = false; + }; + const scheduleNext = () => { if (state.stopped) { return; @@ -1788,7 +1863,12 @@ export function startHeartbeatRunner(opts: { const delay = resolveSafeTimeoutDelayMs(rawDelay, { minMs: 0 }); state.timer = setTimeout(() => { state.timer = null; - requestHeartbeatNow({ reason: "interval", coalesceMs: 0 }); + requestHeartbeat({ + source: "interval", + intent: "scheduled", + reason: "interval", + coalesceMs: 0, + }); }, delay); state.timer.unref?.(); }; @@ -1821,6 +1901,9 @@ export function startHeartbeatRunner(opts: { intervalMs, phaseMs, nextDueMs, + lastRunStartedAtMs: prevState?.lastRunStartedAtMs, + recentRunStarts: prevState?.recentRunStarts ?? [], + floodLoggedSinceLastRun: prevState?.floodLoggedSinceLastRun ?? false, }); } @@ -1866,6 +1949,7 @@ export function startHeartbeatRunner(opts: { } const reason = params?.reason; + const intent = params.intent; const requestedAgentId = params?.agentId ? normalizeAgentId(params.agentId) : undefined; const requestedSessionKey = normalizeOptionalString(params?.sessionKey); const requestedHeartbeat = params?.heartbeat; @@ -1886,19 +1970,32 @@ export function startHeartbeatRunner(opts: { if (!targetAgent) { return { status: "skipped", reason: "disabled" }; } + const deferral = evaluateWakeDeferral(targetAgent, now, reason, intent); + if (deferral.defer) { + return { status: "skipped", reason: deferral.reason }; + } try { const res = await runOnce({ cfg: state.cfg, agentId: targetAgent.agentId, heartbeat: resolveRequestedHeartbeat(targetAgent.heartbeat), + source: params.source, + intent, reason, sessionKey: requestedSessionKey, deps: { runtime: state.runtime }, }); if (res.status === "skipped" && isRetryableHeartbeatBusySkipReason(res.reason)) { + // Retryable busy — do NOT record run bookkeeping. The wake layer + // retries the same reason shortly; if we recorded `lastRunStartedAtMs` + // here, the retry would falsely defer with `not-due`/`min-spacing` + // because the cooldown would treat this skipped attempt as a real run. retryableBusySkip = true; return res; } + // Non-retryable outcome (ran, disabled, failed-but-not-busy). Record + // bookkeeping so subsequent wakes within the cooldown window defer. + recordRunBookkeeping(targetAgent, now); if (res.status !== "skipped" || res.reason !== "disabled") { advanceAgentSchedule(targetAgent, now, reason); } @@ -1908,13 +2005,18 @@ export function startHeartbeatRunner(opts: { log.error(`heartbeat runner: targeted runOnce threw unexpectedly: ${errMsg}`, { error: errMsg, }); + // Throw counts as a non-retryable terminal attempt for cooldown + // purposes — record bookkeeping so the wake layer doesn't tight-loop + // on the same reason. + recordRunBookkeeping(targetAgent, now); advanceAgentSchedule(targetAgent, now, reason); return { status: "failed", reason: errMsg }; } } for (const agent of state.agents.values()) { - if (isInterval && now < agent.nextDueMs) { + const deferral = evaluateWakeDeferral(agent, now, reason, intent); + if (deferral.defer) { continue; } @@ -1924,23 +2026,30 @@ export function startHeartbeatRunner(opts: { cfg: state.cfg, agentId: agent.agentId, heartbeat: agent.heartbeat, + source: params.source, + intent, reason, deps: { runtime: state.runtime }, }); } catch (err) { const errMsg = formatErrorMessage(err); log.error(`heartbeat runner: runOnce threw unexpectedly: ${errMsg}`, { error: errMsg }); + // Throw counts as a non-retryable terminal attempt — see comment in + // targeted branch above. + recordRunBookkeeping(agent, now); advanceAgentSchedule(agent, now, reason); continue; } if (res.status === "skipped" && isRetryableHeartbeatBusySkipReason(res.reason)) { - // Do not advance the schedule — the main lane is busy and the wake - // layer will retry shortly (DEFAULT_RETRY_MS = 1 s). Calling - // scheduleNext() here would register a 0 ms timer that races with - // the wake layer's 1 s retry and wins, bypassing the cooldown. + // Do not advance the schedule or record run bookkeeping — the main + // lane is busy and the wake layer will retry the same reason shortly + // (DEFAULT_RETRY_MS = 1 s). Recording here would convert the retry + // into a false `not-due`/`min-spacing` defer. retryableBusySkip = true; return res; } + // Non-retryable outcome — record bookkeeping for cooldown gates. + recordRunBookkeeping(agent, now); if (res.status !== "skipped" || res.reason !== "disabled") { advanceAgentSchedule(agent, now, reason); } @@ -2014,6 +2123,8 @@ export function startHeartbeatRunner(opts: { agentId: params.agentId, sessionKey: params.sessionKey, heartbeat: params.heartbeat, + source: params.source, + intent: params.intent, }); const disposeWakeHandler = setHeartbeatWakeHandler(wakeHandler); updateConfig(state.cfg); diff --git a/src/infra/heartbeat-wake.test.ts b/src/infra/heartbeat-wake.test.ts index 8419d286372..b07f2735dac 100644 --- a/src/infra/heartbeat-wake.test.ts +++ b/src/infra/heartbeat-wake.test.ts @@ -5,12 +5,35 @@ import { HEARTBEAT_SKIP_REQUESTS_IN_FLIGHT, hasHeartbeatWakeHandler, hasPendingHeartbeatWake, - requestHeartbeatNow, + requestHeartbeat, resetHeartbeatWakeStateForTests, setHeartbeatWakeHandler, } from "./heartbeat-wake.js"; describe("heartbeat-wake", () => { + type WakeRequest = Parameters[0]; + function wake(reason: string, opts: Partial = {}): WakeRequest { + const source = + opts.source ?? + (reason === "interval" + ? "interval" + : reason === "manual" + ? "manual" + : reason === "retry" + ? "retry" + : reason === "exec-event" + ? "exec-event" + : reason.startsWith("cron:") + ? "cron" + : reason.startsWith("hook:") + ? "hook" + : "other"); + const intent = + opts.intent ?? + (reason === "interval" ? "scheduled" : reason === "manual" ? "manual" : "event"); + return { source, intent, reason, ...opts }; + } + function setRetryOnceHeartbeatHandler() { const handler = vi .fn() @@ -28,7 +51,7 @@ describe("heartbeat-wake", () => { setHeartbeatWakeHandler( params.handler as unknown as Parameters[0], ); - requestHeartbeatNow({ reason: params.initialReason, coalesceMs: 0 }); + requestHeartbeat(wake(params.initialReason, { coalesceMs: 0 })); await vi.advanceTimersByTimeAsync(1); expect(params.handler).toHaveBeenCalledTimes(1); @@ -38,7 +61,7 @@ describe("heartbeat-wake", () => { await vi.advanceTimersByTimeAsync(500); expect(params.handler).toHaveBeenCalledTimes(2); - expect(params.handler.mock.calls[1]?.[0]).toEqual({ reason: params.expectedRetryReason }); + expect(params.handler.mock.calls[1]?.[0]).toEqual(wake(params.expectedRetryReason)); } beforeEach(() => { @@ -56,9 +79,9 @@ describe("heartbeat-wake", () => { const handler = vi.fn().mockResolvedValue({ status: "skipped", reason: "disabled" }); setHeartbeatWakeHandler(handler); - requestHeartbeatNow({ reason: "interval", coalesceMs: 200 }); - requestHeartbeatNow({ reason: "exec-event", coalesceMs: 200 }); - requestHeartbeatNow({ reason: "retry", coalesceMs: 200 }); + requestHeartbeat(wake("interval", { coalesceMs: 200 })); + requestHeartbeat(wake("exec-event", { coalesceMs: 200 })); + requestHeartbeat(wake("retry", { coalesceMs: 200 })); expect(hasPendingHeartbeatWake()).toBe(true); @@ -67,7 +90,7 @@ describe("heartbeat-wake", () => { await vi.advanceTimersByTimeAsync(1); expect(handler).toHaveBeenCalledTimes(1); - expect(handler).toHaveBeenCalledWith({ reason: "exec-event" }); + expect(handler).toHaveBeenCalledWith(wake("exec-event")); expect(hasPendingHeartbeatWake()).toBe(false); }); @@ -104,18 +127,18 @@ describe("heartbeat-wake", () => { vi.useFakeTimers(); const handler = setRetryOnceHeartbeatHandler(); - requestHeartbeatNow({ reason: "interval", coalesceMs: 0 }); + requestHeartbeat(wake("interval", { coalesceMs: 0 })); await vi.advanceTimersByTimeAsync(1); expect(handler).toHaveBeenCalledTimes(1); // Retry is now waiting for 1000ms. This should not preempt cooldown. - requestHeartbeatNow({ reason: "hook:wake", coalesceMs: 0 }); + requestHeartbeat(wake("hook:wake", { coalesceMs: 0 })); await vi.advanceTimersByTimeAsync(998); expect(handler).toHaveBeenCalledTimes(1); await vi.advanceTimersByTimeAsync(1); expect(handler).toHaveBeenCalledTimes(2); - expect(handler.mock.calls[1]?.[0]).toEqual({ reason: "hook:wake" }); + expect(handler.mock.calls[1]?.[0]).toEqual(wake("hook:wake")); }); it("retries thrown handler errors after the default retry delay", async () => { @@ -147,7 +170,7 @@ describe("heartbeat-wake", () => { expect(hasHeartbeatWakeHandler()).toBe(true); // handlerB should still work - requestHeartbeatNow({ reason: "interval", coalesceMs: 0 }); + requestHeartbeat(wake("interval", { coalesceMs: 0 })); await vi.advanceTimersByTimeAsync(1); expect(handlerB).toHaveBeenCalledTimes(1); expect(handlerA).not.toHaveBeenCalled(); @@ -163,15 +186,15 @@ describe("heartbeat-wake", () => { setHeartbeatWakeHandler(handler); // Schedule for 5 seconds from now - requestHeartbeatNow({ reason: "slow", coalesceMs: 5000 }); + requestHeartbeat(wake("slow", { coalesceMs: 5000 })); // Schedule for 100ms from now — should preempt the 5s timer - requestHeartbeatNow({ reason: "fast", coalesceMs: 100 }); + requestHeartbeat(wake("fast", { coalesceMs: 100 })); await vi.advanceTimersByTimeAsync(100); expect(handler).toHaveBeenCalledTimes(1); // The reason should be "fast" since it was set last - expect(handler).toHaveBeenCalledWith({ reason: "fast" }); + expect(handler).toHaveBeenCalledWith(wake("fast")); }); it("keeps existing timer when later schedule is requested", async () => { @@ -180,10 +203,10 @@ describe("heartbeat-wake", () => { setHeartbeatWakeHandler(handler); // Schedule for 100ms from now - requestHeartbeatNow({ reason: "fast", coalesceMs: 100 }); + requestHeartbeat(wake("fast", { coalesceMs: 100 })); // Schedule for 5 seconds from now — should NOT preempt - requestHeartbeatNow({ reason: "slow", coalesceMs: 5000 }); + requestHeartbeat(wake("slow", { coalesceMs: 5000 })); await vi.advanceTimersByTimeAsync(100); expect(handler).toHaveBeenCalledTimes(1); @@ -194,12 +217,12 @@ describe("heartbeat-wake", () => { const handler = vi.fn().mockResolvedValue({ status: "ran", durationMs: 1 }); setHeartbeatWakeHandler(handler); - requestHeartbeatNow({ reason: "exec-event", coalesceMs: 100 }); - requestHeartbeatNow({ reason: "retry", coalesceMs: 100 }); + requestHeartbeat(wake("exec-event", { coalesceMs: 100 })); + requestHeartbeat(wake("retry", { coalesceMs: 100 })); await vi.advanceTimersByTimeAsync(100); expect(handler).toHaveBeenCalledTimes(1); - expect(handler).toHaveBeenCalledWith({ reason: "exec-event" }); + expect(handler).toHaveBeenCalledWith(wake("exec-event")); }); it("resets running/scheduled flags when new handler is registered", async () => { @@ -217,7 +240,7 @@ describe("heartbeat-wake", () => { setHeartbeatWakeHandler(handlerA); // Trigger the handler — it starts running but never finishes - requestHeartbeatNow({ reason: "interval", coalesceMs: 0 }); + requestHeartbeat(wake("interval", { coalesceMs: 0 })); await vi.advanceTimersByTimeAsync(1); expect(handlerA).toHaveBeenCalledTimes(1); @@ -227,7 +250,7 @@ describe("heartbeat-wake", () => { setHeartbeatWakeHandler(handlerB); // handlerB should be able to fire (running was reset) - requestHeartbeatNow({ reason: "interval", coalesceMs: 0 }); + requestHeartbeat(wake("interval", { coalesceMs: 0 })); await vi.advanceTimersByTimeAsync(1); expect(handlerB).toHaveBeenCalledTimes(1); @@ -243,7 +266,7 @@ describe("heartbeat-wake", () => { .mockResolvedValue({ status: "skipped", reason: HEARTBEAT_SKIP_REQUESTS_IN_FLIGHT }); setHeartbeatWakeHandler(handlerA); - requestHeartbeatNow({ reason: "interval", coalesceMs: 0 }); + requestHeartbeat(wake("interval", { coalesceMs: 0 })); await vi.advanceTimersByTimeAsync(1); expect(handlerA).toHaveBeenCalledTimes(1); @@ -251,16 +274,16 @@ describe("heartbeat-wake", () => { const handlerB = vi.fn().mockResolvedValue({ status: "ran", durationMs: 1 }); setHeartbeatWakeHandler(handlerB); - requestHeartbeatNow({ reason: "manual", coalesceMs: 0 }); + requestHeartbeat(wake("manual", { coalesceMs: 0 })); await vi.advanceTimersByTimeAsync(1); expect(handlerB).toHaveBeenCalledTimes(1); - expect(handlerB).toHaveBeenCalledWith({ reason: "manual" }); + expect(handlerB).toHaveBeenCalledWith(wake("manual")); }); it("drains pending wake once a handler is registered", async () => { vi.useFakeTimers(); - requestHeartbeatNow({ reason: "manual", coalesceMs: 0 }); + requestHeartbeat(wake("manual", { coalesceMs: 0 })); await vi.advanceTimersByTimeAsync(1); expect(hasPendingHeartbeatWake()).toBe(true); @@ -272,7 +295,7 @@ describe("heartbeat-wake", () => { await vi.advanceTimersByTimeAsync(1); expect(handler).toHaveBeenCalledTimes(1); - expect(handler).toHaveBeenCalledWith({ reason: "manual" }); + expect(handler).toHaveBeenCalledWith(wake("manual")); expect(hasPendingHeartbeatWake()).toBe(false); }); @@ -280,7 +303,9 @@ describe("heartbeat-wake", () => { vi.useFakeTimers(); const handler = setRetryOnceHeartbeatHandler(); - requestHeartbeatNow({ + requestHeartbeat({ + source: "cron", + intent: "immediate", reason: "cron:job-1", agentId: "ops", sessionKey: "agent:ops:guildchat:channel:alerts", @@ -291,6 +316,8 @@ describe("heartbeat-wake", () => { await vi.advanceTimersByTimeAsync(1); expect(handler).toHaveBeenCalledTimes(1); expect(handler.mock.calls[0]?.[0]).toEqual({ + source: "cron", + intent: "immediate", reason: "cron:job-1", agentId: "ops", sessionKey: "agent:ops:guildchat:channel:alerts", @@ -300,6 +327,8 @@ describe("heartbeat-wake", () => { await vi.advanceTimersByTimeAsync(1000); expect(handler).toHaveBeenCalledTimes(2); expect(handler.mock.calls[1]?.[0]).toEqual({ + source: "cron", + intent: "immediate", reason: "cron:job-1", agentId: "ops", sessionKey: "agent:ops:guildchat:channel:alerts", @@ -312,14 +341,18 @@ describe("heartbeat-wake", () => { const handler = vi.fn().mockResolvedValue({ status: "ran", durationMs: 1 }); setHeartbeatWakeHandler(handler); - requestHeartbeatNow({ + requestHeartbeat({ + source: "manual", + intent: "manual", reason: "manual", agentId: "ops", sessionKey: "agent:ops:guildchat:channel:alerts", heartbeat: { target: "last" }, coalesceMs: 100, }); - requestHeartbeatNow({ + requestHeartbeat({ + source: "manual", + intent: "manual", reason: "manual", agentId: "ops", sessionKey: "agent:ops:guildchat:channel:alerts", @@ -330,6 +363,8 @@ describe("heartbeat-wake", () => { expect(handler).toHaveBeenCalledTimes(1); expect(handler).toHaveBeenCalledWith({ + source: "manual", + intent: "manual", reason: "manual", agentId: "ops", sessionKey: "agent:ops:guildchat:channel:alerts", @@ -342,13 +377,17 @@ describe("heartbeat-wake", () => { const handler = vi.fn().mockResolvedValue({ status: "ran", durationMs: 1 }); setHeartbeatWakeHandler(handler); - requestHeartbeatNow({ + requestHeartbeat({ + source: "cron", + intent: "event", reason: "cron:job-a", agentId: "ops", sessionKey: "agent:ops:guildchat:channel:alerts", coalesceMs: 100, }); - requestHeartbeatNow({ + requestHeartbeat({ + source: "cron", + intent: "event", reason: "cron:job-b", agentId: "main", sessionKey: "agent:main:forum:group:-1001", @@ -361,11 +400,15 @@ describe("heartbeat-wake", () => { expect(handler.mock.calls.map((call) => call[0])).toEqual( expect.arrayContaining([ { + source: "cron", + intent: "event", reason: "cron:job-a", agentId: "ops", sessionKey: "agent:ops:guildchat:channel:alerts", }, { + source: "cron", + intent: "event", reason: "cron:job-b", agentId: "main", sessionKey: "agent:main:forum:group:-1001", diff --git a/src/infra/heartbeat-wake.ts b/src/infra/heartbeat-wake.ts index 629259bc7fd..572421a2609 100644 --- a/src/infra/heartbeat-wake.ts +++ b/src/infra/heartbeat-wake.ts @@ -1,9 +1,5 @@ import { normalizeOptionalString } from "../shared/string-coerce.js"; -import { - isHeartbeatActionWakeReason, - normalizeHeartbeatWakeReason, - resolveHeartbeatReasonKind, -} from "./heartbeat-reason.js"; +import { normalizeHeartbeatWakeReason } from "./heartbeat-reason.js"; export type HeartbeatRunResult = | { status: "ran"; durationMs: number } @@ -28,7 +24,26 @@ export function isRetryableHeartbeatBusySkipReason(reason: string): boolean { return RETRYABLE_BUSY_SKIP_REASONS.has(reason); } +export type HeartbeatWakeIntent = "scheduled" | "event" | "immediate" | "manual"; + +export type HeartbeatWakeSource = + | "interval" + | "manual" + | "exec-event" + | "notifications-event" + | "cron" + | "hook" + | "background-task" + | "background-task-blocked" + | "acp-spawn" + | "cli-watchdog" + | "restart-sentinel" + | "retry" + | "other"; + export type HeartbeatWakeRequest = { + source: HeartbeatWakeSource; + intent: HeartbeatWakeIntent; reason?: string; agentId?: string; sessionKey?: string; @@ -49,6 +64,8 @@ export function areHeartbeatsEnabled(): boolean { type WakeTimerKind = "normal" | "retry"; type PendingWakeReason = { + source: HeartbeatWakeSource; + intent: HeartbeatWakeIntent; reason: string; priority: number; requestedAt: number; @@ -75,17 +92,24 @@ const REASON_PRIORITY = { ACTION: 3, } as const; -function resolveReasonPriority(reason: string): number { - const kind = resolveHeartbeatReasonKind(reason); - if (kind === "retry") { +function resolveWakePriority(params: { + source: HeartbeatWakeSource; + intent: HeartbeatWakeIntent; + reason: string; +}): number { + if (params.intent === "manual" || params.intent === "immediate") { + return REASON_PRIORITY.ACTION; + } + if (params.source === "retry" || params.reason === "retry") { return REASON_PRIORITY.RETRY; } - if (kind === "interval") { + if ( + params.intent === "scheduled" || + params.source === "interval" || + params.reason === "interval" + ) { return REASON_PRIORITY.INTERVAL; } - if (isHeartbeatActionWakeReason(reason)) { - return REASON_PRIORITY.ACTION; - } return REASON_PRIORITY.DEFAULT; } @@ -104,28 +128,36 @@ function getWakeTargetKey(params: { agentId?: string; sessionKey?: string }) { return `${agentId ?? ""}::${sessionKey ?? ""}`; } -function queuePendingWakeReason(params?: { +function queuePendingWakeReason(params: { + source: HeartbeatWakeSource; + intent: HeartbeatWakeIntent; reason?: string; requestedAt?: number; agentId?: string; sessionKey?: string; heartbeat?: { target?: string }; }) { - const requestedAt = params?.requestedAt ?? Date.now(); - const normalizedReason = normalizeWakeReason(params?.reason); - const normalizedAgentId = normalizeWakeTarget(params?.agentId); - const normalizedSessionKey = normalizeWakeTarget(params?.sessionKey); + const requestedAt = params.requestedAt ?? Date.now(); + const normalizedReason = normalizeWakeReason(params.reason); + const normalizedAgentId = normalizeWakeTarget(params.agentId); + const normalizedSessionKey = normalizeWakeTarget(params.sessionKey); const wakeTargetKey = getWakeTargetKey({ agentId: normalizedAgentId, sessionKey: normalizedSessionKey, }); const next: PendingWakeReason = { + source: params.source, + intent: params.intent, reason: normalizedReason, - priority: resolveReasonPriority(normalizedReason), + priority: resolveWakePriority({ + source: params.source, + intent: params.intent, + reason: normalizedReason, + }), requestedAt, agentId: normalizedAgentId, sessionKey: normalizedSessionKey, - heartbeat: params?.heartbeat, + heartbeat: params.heartbeat, }; const previous = pendingWakes.get(wakeTargetKey); if (!previous) { @@ -187,6 +219,8 @@ function schedule(coalesceMs: number, kind: WakeTimerKind = "normal") { try { for (const pendingWake of pendingBatch) { const wakeOpts = { + source: pendingWake.source, + intent: pendingWake.intent, reason: pendingWake.reason ?? undefined, ...(pendingWake.agentId ? { agentId: pendingWake.agentId } : {}), ...(pendingWake.sessionKey ? { sessionKey: pendingWake.sessionKey } : {}), @@ -196,6 +230,8 @@ function schedule(coalesceMs: number, kind: WakeTimerKind = "normal") { if (res.status === "skipped" && isRetryableHeartbeatBusySkipReason(res.reason)) { // The target runtime is busy; retry this wake target soon. queuePendingWakeReason({ + source: pendingWake.source, + intent: pendingWake.intent, reason: pendingWake.reason ?? "retry", agentId: pendingWake.agentId, sessionKey: pendingWake.sessionKey, @@ -208,6 +244,8 @@ function schedule(coalesceMs: number, kind: WakeTimerKind = "normal") { // Error is already logged by the heartbeat runner; schedule a retry. for (const pendingWake of pendingBatch) { queuePendingWakeReason({ + source: pendingWake.source, + intent: pendingWake.intent, reason: pendingWake.reason ?? "retry", agentId: pendingWake.agentId, sessionKey: pendingWake.sessionKey, @@ -267,7 +305,9 @@ export function setHeartbeatWakeHandler(next: HeartbeatWakeHandler | null): () = }; } -export function requestHeartbeatNow(opts?: { +export function requestHeartbeat(opts: { + source: HeartbeatWakeSource; + intent: HeartbeatWakeIntent; reason?: string; coalesceMs?: number; agentId?: string; @@ -275,12 +315,14 @@ export function requestHeartbeatNow(opts?: { heartbeat?: { target?: string }; }) { queuePendingWakeReason({ - reason: opts?.reason, - agentId: opts?.agentId, - sessionKey: opts?.sessionKey, - heartbeat: opts?.heartbeat, + source: opts.source, + intent: opts.intent, + reason: opts.reason, + agentId: opts.agentId, + sessionKey: opts.sessionKey, + heartbeat: opts.heartbeat, }); - schedule(opts?.coalesceMs ?? DEFAULT_COALESCE_MS, "normal"); + schedule(opts.coalesceMs ?? DEFAULT_COALESCE_MS, "normal"); } export function hasHeartbeatWakeHandler() { diff --git a/src/plugin-sdk/test-helpers/plugin-runtime-mock.ts b/src/plugin-sdk/test-helpers/plugin-runtime-mock.ts index 85b1e89c240..23560b1ac07 100644 --- a/src/plugin-sdk/test-helpers/plugin-runtime-mock.ts +++ b/src/plugin-sdk/test-helpers/plugin-runtime-mock.ts @@ -370,6 +370,7 @@ export function createPluginRuntimeMock(overrides: DeepPartial = }, system: { enqueueSystemEvent: vi.fn() as unknown as PluginRuntime["system"]["enqueueSystemEvent"], + requestHeartbeat: vi.fn() as unknown as PluginRuntime["system"]["requestHeartbeat"], requestHeartbeatNow: vi.fn() as unknown as PluginRuntime["system"]["requestHeartbeatNow"], runHeartbeatOnce: vi.fn(async () => ({ status: "ran" as const, diff --git a/src/plugins/runtime/index.test.ts b/src/plugins/runtime/index.test.ts index fc751fc1323..7165e001ce8 100644 --- a/src/plugins/runtime/index.test.ts +++ b/src/plugins/runtime/index.test.ts @@ -6,7 +6,11 @@ import { type OpenClawConfig, } from "../../config/config.js"; import { onAgentEvent } from "../../infra/agent-events.js"; -import { requestHeartbeatNow } from "../../infra/heartbeat-wake.js"; +import { + requestHeartbeat, + resetHeartbeatWakeStateForTests, + setHeartbeatWakeHandler, +} from "../../infra/heartbeat-wake.js"; import * as execModule from "../../process/exec.js"; import { onSessionTranscriptUpdate } from "../../sessions/transcript-events.js"; import { VERSION } from "../../version.js"; @@ -161,10 +165,16 @@ describe("plugin runtime command execution", () => { expected: onSessionTranscriptUpdate, }, { - name: "exposes runtime.system.requestHeartbeatNow", + name: "exposes runtime.system.requestHeartbeat", readValue: (runtime: ReturnType) => - runtime.system.requestHeartbeatNow, - expected: requestHeartbeatNow, + runtime.system.requestHeartbeat, + expected: requestHeartbeat, + }, + { + name: "exposes deprecated runtime.system.requestHeartbeatNow", + readValue: (runtime: ReturnType) => + typeof runtime.system.requestHeartbeatNow, + expected: "function", }, { name: "exposes runtime.version from the shared VERSION constant", @@ -175,6 +185,30 @@ describe("plugin runtime command execution", () => { expectRuntimeValue(readValue, expected); }); + it("maps deprecated runtime.system.requestHeartbeatNow to a structured event wake", async () => { + vi.useFakeTimers(); + resetHeartbeatWakeStateForTests(); + const handler = vi.fn(async () => ({ status: "skipped" as const, reason: "disabled" })); + setHeartbeatWakeHandler(handler); + try { + createPluginRuntime().system.requestHeartbeatNow({ + reason: "legacy-plugin", + coalesceMs: 0, + }); + await vi.advanceTimersByTimeAsync(1); + expect(handler).toHaveBeenCalledWith( + expect.objectContaining({ + source: "other", + intent: "event", + reason: "legacy-plugin", + }), + ); + } finally { + resetHeartbeatWakeStateForTests(); + vi.useRealTimers(); + } + }); + it("resolves thinking policy with configured model compat from runtime config", () => { setRuntimeConfigSnapshot({ models: { diff --git a/src/plugins/runtime/runtime-system.ts b/src/plugins/runtime/runtime-system.ts index 538ee2bdcd3..52034a77177 100644 --- a/src/plugins/runtime/runtime-system.ts +++ b/src/plugins/runtime/runtime-system.ts @@ -1,4 +1,4 @@ -import { requestHeartbeatNow } from "../../infra/heartbeat-wake.js"; +import { requestHeartbeat } from "../../infra/heartbeat-wake.js"; import { enqueueSystemEvent } from "../../infra/system-events.js"; import { runCommandWithTimeout } from "../../process/exec.js"; import { createLazyRuntimeMethod, createLazyRuntimeModule } from "../../shared/lazy-runtime.js"; @@ -15,8 +15,20 @@ const runHeartbeatOnceInternal = createLazyRuntimeMethod( ); export function createRuntimeSystem(): PluginRuntime["system"] { + const requestHeartbeatNow: PluginRuntime["system"]["requestHeartbeatNow"] = (opts) => + requestHeartbeat({ + source: opts?.source ?? "other", + intent: opts?.intent ?? "event", + reason: opts?.reason, + coalesceMs: opts?.coalesceMs, + agentId: opts?.agentId, + sessionKey: opts?.sessionKey, + heartbeat: opts?.heartbeat, + }); + return { enqueueSystemEvent, + requestHeartbeat, requestHeartbeatNow, runHeartbeatOnce: (opts?: RunHeartbeatOnceOptions) => { // Destructure to forward only the plugin-safe subset; prevent cfg/deps injection at runtime. diff --git a/src/plugins/runtime/types-core.ts b/src/plugins/runtime/types-core.ts index bd088ca44ab..c318ef78c6b 100644 --- a/src/plugins/runtime/types-core.ts +++ b/src/plugins/runtime/types-core.ts @@ -10,6 +10,16 @@ import type { PluginRuntimeTaskFlows, PluginRuntimeTaskRuns } from "./runtime-ta export type { HeartbeatRunResult }; +export type RuntimeRequestHeartbeatOptions = Parameters< + typeof import("../../infra/heartbeat-wake.js").requestHeartbeat +>[0]; + +export type RuntimeRequestHeartbeatNowOptions = Omit< + RuntimeRequestHeartbeatOptions, + "source" | "intent" +> & + Partial>; + type RuntimeWriteConfigOptions = { envSnapshotForRestore?: Record; expectedConfigPath?: string; @@ -154,7 +164,12 @@ export type PluginRuntimeCore = { }; system: { enqueueSystemEvent: typeof import("../../infra/system-events.js").enqueueSystemEvent; - requestHeartbeatNow: typeof import("../../infra/heartbeat-wake.js").requestHeartbeatNow; + requestHeartbeat: typeof import("../../infra/heartbeat-wake.js").requestHeartbeat; + /** + * @deprecated Use `requestHeartbeat({ source, intent, reason })` so wake producers declare + * scheduler intent explicitly. + */ + requestHeartbeatNow: (opts?: RuntimeRequestHeartbeatNowOptions) => void; /** * Run a single heartbeat cycle immediately (bypassing the coalesce timer). * Accepts an optional `heartbeat` config override so callers can force diff --git a/src/tasks/task-registry.ts b/src/tasks/task-registry.ts index fed110ff396..87503b934fc 100644 --- a/src/tasks/task-registry.ts +++ b/src/tasks/task-registry.ts @@ -3,7 +3,7 @@ import { createRequire } from "node:module"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import { onAgentEvent } from "../infra/agent-events.js"; import { formatErrorMessage } from "../infra/errors.js"; -import { requestHeartbeatNow } from "../infra/heartbeat-wake.js"; +import { requestHeartbeat } from "../infra/heartbeat-wake.js"; import { enqueueSystemEvent } from "../infra/system-events.js"; import { createSubsystemLogger } from "../logging/subsystem.js"; import { parseAgentSessionKey } from "../routing/session-key.js"; @@ -1077,7 +1077,9 @@ function queueTaskSystemEvent(task: TaskRecord, text: string) { deliveryContext: owner.requesterOrigin, trusted: false, }); - requestHeartbeatNow({ + requestHeartbeat({ + source: "background-task", + intent: "immediate", reason: "background-task", sessionKey: ownerKey, }); @@ -1100,7 +1102,9 @@ function queueBlockedTaskFollowup(task: TaskRecord) { deliveryContext: owner.requesterOrigin, trusted: false, }); - requestHeartbeatNow({ + requestHeartbeat({ + source: "background-task-blocked", + intent: "immediate", reason: "background-task-blocked", sessionKey: ownerKey, }); diff --git a/test/helpers/cron/service-regression-fixtures.ts b/test/helpers/cron/service-regression-fixtures.ts index 78ece8ebb93..02817ebfbc9 100644 --- a/test/helpers/cron/service-regression-fixtures.ts +++ b/test/helpers/cron/service-regression-fixtures.ts @@ -85,7 +85,7 @@ export function createRunningCronServiceState(params: { log: params.log, nowMs: params.nowMs, enqueueSystemEvent: vi.fn(), - requestHeartbeatNow: vi.fn(), + requestHeartbeat: vi.fn(), runIsolatedAgentJob: vi.fn().mockResolvedValue({ status: "ok", summary: "ok" }), }); state.running = true; diff --git a/test/scripts/audit-seams.test.ts b/test/scripts/audit-seams.test.ts index 5649ef8e39b..753343a70c2 100644 --- a/test/scripts/audit-seams.test.ts +++ b/test/scripts/audit-seams.test.ts @@ -96,12 +96,17 @@ describe("audit-seams subagent seam classification", () => { it("detects parent-stream seams for ACP spawn relays", () => { const source = ` import { onAgentEvent } from "../infra/agent-events.js"; - import { requestHeartbeatNow } from "../infra/heartbeat-wake.js"; + import { requestHeartbeat } from "../infra/heartbeat-wake.js"; import { enqueueSystemEvent } from "../infra/system-events.js"; export function startAcpSpawnParentStreamRelay() { onAgentEvent("agent-output", () => {}); - requestHeartbeatNow({ sessionKey: "agent:main" }); + requestHeartbeat({ + source: "acp-spawn", + intent: "event", + reason: "acp:spawn:stream", + sessionKey: "agent:main", + }); enqueueSystemEvent("progress", { sessionKey: "agent:main", contextKey: "stream" }); return { streamTo: "parent" }; }