vultr/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-05-06 13:10:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(heartbeat): type wake scheduling intent

Browse Source 
Co-authored-by: Jordan Baker <jbb@scryent.com>
This commit is contained in:
Peter Steinberger
2026-05-02 14:31:11 +01:00
parent 0b09cfb8cd
commit c06739d773
71 changed files with 1601 additions and 484 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@@ -314,6 +314,7 @@ Docs: https://docs.openclaw.ai
- Agents/tools: reuse the auth profile store already loaded for the active run when deciding media and generation tool availability, avoiding repeated provider-auth runtime discovery during reply startup. Thanks @shakkernerd.
- Agents/tools: keep image, video, and music generation tool registration on manifest/auth control-plane checks instead of loading runtime provider registries during reply startup, reducing live-path tool-prep blocking while leaving provider runtime resolution for execution and list actions. Thanks @shakkernerd.
- Discord: document canonical mention formatting in agent prompt hints and channel docs so outbound replies use `<@USER_ID>`, `<#CHANNEL_ID>`, and `<@&ROLE_ID>` instead of legacy nickname mentions. (#75173)
- Heartbeat scheduler: gate exec-event/notification/spawn/retry wakes through a centralized cooldown so backgrounded `process.start` exit notifications can no longer self-feed runaway heartbeat runs (configured `every: "30m"` was firing every ~10s in production, pegging the gateway event loop with `eventLoopDelayMaxMs >6s` spikes that stalled control-UI asset serving and TUI handshakes). Documented wake-now paths (`manual`, `wake`, task completion, blocked-task follow-up, `/hooks/wake mode=now`, and cron `--wake now`) remain immediate; retryable busy skips no longer poison the cooldown for the next retry; per-agent flood guard caps any unexpected feedback loop at 5 runs/60s. (#64016, refs #17797 and #75436) Thanks @hexsprite.
- fix: block workspace CLOUDSDK_PYTHON override and always set trusted interpreter for gcloud. (#74492) Thanks @pgondhi987.
- Providers/Z.AI: move the bundled GLM catalog and auth env metadata into the plugin manifest, so `models list --all --provider zai` shows the full known catalog without duplicated runtime seed data. Thanks @shakkernerd.
- Providers/Qianfan and Providers/Stepfun: declare setup auth metadata (`api-key` method, `QIANFAN_API_KEY`, `STEPFUN_API_KEY`) in the plugin manifest so onboarding and `models setup` surface the expected env var without falling back to legacy `providerAuthEnvVars` runtime seed data. Thanks @shakkernerd.

7
docs/plugins/sdk-runtime.md
View File

@@ -360,7 +360,12 @@ Provider and channel execution paths must use the active runtime config snapshot
```typescript
await api.runtime.system.enqueueSystemEvent(event);
api.runtime.system.requestHeartbeatNow();
api.runtime.system.requestHeartbeat({
source: "other",
intent: "event",
reason: "plugin-event",
});
api.runtime.system.requestHeartbeatNow({ reason: "plugin-event" }); // Deprecated compatibility alias.
const output = await api.runtime.system.runCommandWithTimeout(cmd, args, opts);
const hint = api.runtime.system.formatNativeDependencyHint(pkg);
```

10
src/agents/acp-spawn-parent-stream.test.ts
View File

@@ -2,7 +2,7 @@ import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vite
import { mergeMockedModule } from "../test-utils/vitest-module-mocks.js";
const enqueueSystemEventMock = vi.fn();
const requestHeartbeatNowMock = vi.fn();
const requestHeartbeatMock = vi.fn();
const readAcpSessionEntryMock = vi.fn();
const resolveSessionFilePathMock = vi.fn();
const resolveSessionFilePathOptionsMock = vi.fn();
@@ -17,7 +17,7 @@ vi.mock("../infra/heartbeat-wake.js", async () => {
"../infra/heartbeat-wake.js",
),
() => ({
requestHeartbeatNow: (...args: unknown[]) => requestHeartbeatNowMock(...args),
requestHeartbeat: (...args: unknown[]) => requestHeartbeatMock(...args),
}),
);
});
@@ -63,7 +63,7 @@ describe("startAcpSpawnParentStreamRelay", () => {
beforeEach(() => {
enqueueSystemEventMock.mockClear();
requestHeartbeatNowMock.mockClear();
requestHeartbeatMock.mockClear();
readAcpSessionEntryMock.mockReset();
resolveSessionFilePathMock.mockReset();
resolveSessionFilePathOptionsMock.mockReset();
@@ -129,7 +129,7 @@ describe("startAcpSpawnParentStreamRelay", () => {
trusted: false,
}),
);
expect(requestHeartbeatNowMock).toHaveBeenCalledWith(
expect(requestHeartbeatMock).toHaveBeenCalledWith(
expect.objectContaining({
reason: "acp:spawn:stream",
sessionKey: "agent:main:main",
@@ -255,7 +255,7 @@ describe("startAcpSpawnParentStreamRelay", () => {
});
expect(collectedTexts()).toEqual([]);
expect(requestHeartbeatNowMock).not.toHaveBeenCalled();
expect(requestHeartbeatMock).not.toHaveBeenCalled();
relay.dispose();
});

6
src/agents/acp-spawn-parent-stream.ts
View File

@@ -3,7 +3,7 @@ import path from "node:path";
import { readAcpSessionEntry } from "../acp/runtime/session-meta.js";
import { resolveSessionFilePath, resolveSessionFilePathOptions } from "../config/sessions/paths.js";
import { onAgentEvent } from "../infra/agent-events.js";
import { requestHeartbeatNow } from "../infra/heartbeat-wake.js";
import { requestHeartbeat } from "../infra/heartbeat-wake.js";
import { enqueueSystemEvent } from "../infra/system-events.js";
import { scopedHeartbeatWakeOptions } from "../routing/session-key.js";
import { normalizeAssistantPhase } from "../shared/chat-message-content.js";
@@ -181,8 +181,10 @@ export function startAcpSpawnParentStreamRelay(params: {
if (!shouldSurfaceUpdates) {
return;
}
requestHeartbeatNow(
requestHeartbeat(
scopedHeartbeatWakeOptions(parentSessionKey, {
source: "acp-spawn",
intent: "event",
reason: "acp:spawn:stream",
}),
);

20
src/agents/bash-tools.exec-runtime.test.ts
View File

@@ -1,13 +1,13 @@
import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
const requestHeartbeatNowMock = vi.hoisted(() => vi.fn());
const requestHeartbeatMock = vi.hoisted(() => vi.fn());
const enqueueSystemEventMock = vi.hoisted(() => vi.fn());
const supervisorMock = vi.hoisted(() => ({
spawn: vi.fn(),
}));
vi.mock("../infra/heartbeat-wake.js", () => ({
requestHeartbeatNow: requestHeartbeatNowMock,
requestHeartbeat: requestHeartbeatMock,
}));
vi.mock("../infra/system-events.js", () => ({
@@ -43,7 +43,7 @@ beforeAll(async () => {
});
beforeEach(() => {
requestHeartbeatNowMock.mockClear();
requestHeartbeatMock.mockClear();
enqueueSystemEventMock.mockClear();
supervisorMock.spawn.mockReset();
});
@@ -392,7 +392,7 @@ describe("exec notifyOnExit suppression", () => {
expect(outcome.status).toBe("failed");
expect(enqueueSystemEventMock).not.toHaveBeenCalled();
expect(requestHeartbeatNowMock).not.toHaveBeenCalled();
expect(requestHeartbeatMock).not.toHaveBeenCalled();
});
it("notifies for manual-cancelled background execs with output", async () => {
@@ -402,7 +402,7 @@ describe("exec notifyOnExit suppression", () => {
expect.stringContaining("partial output"),
expect.objectContaining({ sessionKey: "agent:main:main" }),
);
expect(requestHeartbeatNowMock).toHaveBeenCalled();
expect(requestHeartbeatMock).toHaveBeenCalled();
});
it("still notifies for no-output background exec timeouts", async () => {
@@ -412,13 +412,13 @@ describe("exec notifyOnExit suppression", () => {
expect.stringContaining("Exec failed"),
expect.objectContaining({ sessionKey: "agent:main:main" }),
);
expect(requestHeartbeatNowMock).toHaveBeenCalled();
expect(requestHeartbeatMock).toHaveBeenCalled();
});
});
describe("emitExecSystemEvent", () => {
beforeEach(() => {
requestHeartbeatNowMock.mockClear();
requestHeartbeatMock.mockClear();
enqueueSystemEventMock.mockClear();
});
@@ -442,7 +442,7 @@ describe("emitExecSystemEvent", () => {
threadId: 47,
},
});
expect(requestHeartbeatNowMock).toHaveBeenCalledWith(
expect(requestHeartbeatMock).toHaveBeenCalledWith(
expect.objectContaining({
coalesceMs: 0,
reason: "exec-event",
@@ -461,7 +461,7 @@ describe("emitExecSystemEvent", () => {
sessionKey: "global",
contextKey: "exec:run-global",
});
expect(requestHeartbeatNowMock).toHaveBeenCalledWith(
expect(requestHeartbeatMock).toHaveBeenCalledWith(
expect.objectContaining({
coalesceMs: 0,
reason: "exec-event",
@@ -476,7 +476,7 @@ describe("emitExecSystemEvent", () => {
});
expect(enqueueSystemEventMock).not.toHaveBeenCalled();
expect(requestHeartbeatNowMock).not.toHaveBeenCalled();
expect(requestHeartbeatMock).not.toHaveBeenCalled();
});
});

20
src/agents/bash-tools.exec-runtime.ts
View File

@@ -8,7 +8,7 @@ import {
type ExecApprovalDecision,
type ExecTarget,
} from "../infra/exec-approvals.js";
import { requestHeartbeatNow } from "../infra/heartbeat-wake.js";
import { requestHeartbeat } from "../infra/heartbeat-wake.js";
import { isDangerousHostInheritedEnvVarName } from "../infra/host-env-security.js";
import { findPathKey, mergePathPrepend } from "../infra/path-prepend.js";
import { enqueueSystemEvent } from "../infra/system-events.js";
@@ -344,8 +344,13 @@ function maybeNotifyOnExit(session: ProcessSession, status: "completed" | "faile
deliveryContext: session.notifyDeliveryContext,
trusted: false,
});
requestHeartbeatNow(
scopedHeartbeatWakeOptions(sessionKey, { reason: "exec-event", coalesceMs: 0 }),
requestHeartbeat(
scopedHeartbeatWakeOptions(sessionKey, {
source: "exec-event",
intent: "event",
reason: "exec-event",
coalesceMs: 0,
}),
);
}
@@ -422,8 +427,13 @@ export function emitExecSystemEvent(
contextKey: opts.contextKey,
deliveryContext: opts.deliveryContext,
});
requestHeartbeatNow(
scopedHeartbeatWakeOptions(sessionKey, { reason: "exec-event", coalesceMs: 0 }),
requestHeartbeat(
scopedHeartbeatWakeOptions(sessionKey, {
source: "exec-event",
intent: "event",
reason: "exec-event",
coalesceMs: 0,
}),
);
}

6
src/agents/cli-runner.reliability.test.ts
View File

@@ -14,7 +14,7 @@ import { runPreparedCliAgent } from "./cli-runner.js";
import {
createManagedRun,
enqueueSystemEventMock,
requestHeartbeatNowMock,
requestHeartbeatMock,
supervisorSpawnMock,
} from "./cli-runner.test-support.js";
import { executePreparedCliRun } from "./cli-runner/execute.js";
@@ -235,7 +235,9 @@ describe("runCliAgent reliability", () => {
expect(String(notice)).toContain("produced no output");
expect(String(notice)).toContain("interactive input or an approval prompt");
expect(opts).toMatchObject({ sessionKey: "agent:main:main" });
expect(requestHeartbeatNowMock).toHaveBeenCalledWith({
expect(requestHeartbeatMock).toHaveBeenCalledWith({
source: "cli-watchdog",
intent: "event",
reason: "cli:watchdog:stall",
sessionKey: "agent:main:main",
});

10
src/agents/cli-runner.test-support.ts
View File

@@ -1,6 +1,6 @@
import type { Mock } from "vitest";
import { beforeEach, vi } from "vitest";
import type { requestHeartbeatNow } from "../infra/heartbeat-wake.js";
import type { requestHeartbeat } from "../infra/heartbeat-wake.js";
import type { enqueueSystemEvent } from "../infra/system-events.js";
import type { getProcessSupervisor } from "../process/supervisor/index.js";
import { setCliRunnerExecuteTestDeps } from "./cli-runner/execute.js";
@@ -11,7 +11,7 @@ import type { WorkspaceBootstrapFile } from "./workspace.js";
type ProcessSupervisor = ReturnType<typeof getProcessSupervisor>;
type SupervisorSpawnFn = ProcessSupervisor["spawn"];
type EnqueueSystemEventFn = typeof enqueueSystemEvent;
type RequestHeartbeatNowFn = typeof requestHeartbeatNow;
type RequestHeartbeatFn = typeof requestHeartbeat;
type UnknownMock = Mock<(...args: unknown[]) => unknown>;
type BootstrapContext = {
bootstrapFiles: WorkspaceBootstrapFile[];
@@ -21,7 +21,7 @@ type ResolveBootstrapContextForRunMock = Mock<() => Promise<BootstrapContext>>;
export const supervisorSpawnMock: UnknownMock = vi.fn();
export const enqueueSystemEventMock: UnknownMock = vi.fn();
export const requestHeartbeatNowMock: UnknownMock = vi.fn();
export const requestHeartbeatMock: UnknownMock = vi.fn();
const hoisted = vi.hoisted(
(): {
@@ -49,8 +49,8 @@ setCliRunnerExecuteTestDeps({
text: Parameters<EnqueueSystemEventFn>[0],
options: Parameters<EnqueueSystemEventFn>[1],
) => enqueueSystemEventMock(text, options) as ReturnType<EnqueueSystemEventFn>,
requestHeartbeatNow: (options?: Parameters<RequestHeartbeatNowFn>[0]) =>
requestHeartbeatNowMock(options) as ReturnType<RequestHeartbeatNowFn>,
requestHeartbeat: (options?: Parameters<RequestHeartbeatFn>[0]) =>
requestHeartbeatMock(options) as ReturnType<RequestHeartbeatFn>,
});
setCliRunnerPrepareTestDeps({

12
src/agents/cli-runner/execute.ts
View File

@@ -2,7 +2,7 @@ import crypto from "node:crypto";
import { shouldLogVerbose } from "../../globals.js";
import { emitAgentEvent } from "../../infra/agent-events.js";
import { isTruthyEnvValue } from "../../infra/env.js";
import { requestHeartbeatNow as requestHeartbeatNowImpl } from "../../infra/heartbeat-wake.js";
import { requestHeartbeat as requestHeartbeatImpl } from "../../infra/heartbeat-wake.js";
import { sanitizeHostExecEnv } from "../../infra/host-env-security.js";
import { enqueueSystemEvent as enqueueSystemEventImpl } from "../../infra/system-events.js";
import { getProcessSupervisor as getProcessSupervisorImpl } from "../../process/supervisor/index.js";
@@ -42,7 +42,7 @@ import type { PreparedCliRunContext } from "./types.js";
const executeDeps = {
getProcessSupervisor: getProcessSupervisorImpl,
enqueueSystemEvent: enqueueSystemEventImpl,
requestHeartbeatNow: requestHeartbeatNowImpl,
requestHeartbeat: requestHeartbeatImpl,
};
export function setCliRunnerExecuteTestDeps(overrides: Partial<typeof executeDeps>): void {
@@ -545,8 +545,12 @@ export async function executePreparedCliRun(
"For Claude Code, prefer --permission-mode bypassPermissions --print.",
].join(" ");
executeDeps.enqueueSystemEvent(stallNotice, { sessionKey: params.sessionKey });
executeDeps.requestHeartbeatNow(
scopedHeartbeatWakeOptions(params.sessionKey, { reason: "cli:watchdog:stall" }),
executeDeps.requestHeartbeat(
scopedHeartbeatWakeOptions(params.sessionKey, {
source: "cli-watchdog",
intent: "event",
reason: "cli:watchdog:stall",
}),
);
}
throw new FailoverError(timeoutReason, {

2
src/cli/deps.ts
View File

@@ -33,7 +33,7 @@ const NON_CHANNEL_DEP_KEYS = new Set([
"migrateOrphanedSessionKeys",
"nowMs",
"onEvent",
"requestHeartbeatNow",
"requestHeartbeat",
"resolveSessionStorePath",
"runHeartbeatOnce",
"runIsolatedAgentJob",

2
src/cron/service.armtimer-tight-loop.test.ts
View File

@@ -57,7 +57,7 @@ describe("CronService - armTimer tight loop prevention", () => {
log: noopLogger,
nowMs: () => params.now,
enqueueSystemEvent: vi.fn(),
requestHeartbeatNow: vi.fn(),
requestHeartbeat: vi.fn(),
runIsolatedAgentJob:
params.runIsolatedAgentJob ?? vi.fn().mockResolvedValue({ status: "ok" }),
});

6
src/cron/service.delivery-plan.test.ts
View File

@@ -25,7 +25,7 @@ async function withCronService(
run: (context: {
cron: CronService;
enqueueSystemEvent: ReturnType<typeof vi.fn>;
requestHeartbeatNow: ReturnType<typeof vi.fn>;
requestHeartbeat: ReturnType<typeof vi.fn>;
}) => Promise<void>,
) {
await withCronServiceForTest(
@@ -108,7 +108,7 @@ describe("CronService delivery plan consistency", () => {
delivered: true,
})),
},
async ({ cron, enqueueSystemEvent, requestHeartbeatNow }) => {
async ({ cron, enqueueSystemEvent, requestHeartbeat }) => {
const job = await addIsolatedAgentTurnJob(cron, {
name: "announce-delivered",
wakeMode: "now",
@@ -118,7 +118,7 @@ describe("CronService delivery plan consistency", () => {
const result = await cron.run(job.id, "force");
expect(result).toEqual({ ok: true, ran: true });
expect(enqueueSystemEvent).not.toHaveBeenCalled();
expect(requestHeartbeatNow).not.toHaveBeenCalled();
expect(requestHeartbeat).not.toHaveBeenCalled();
},
);
});

4
src/cron/service.every-jobs-fire.test.ts
View File

@@ -116,7 +116,7 @@ describe("CronService interval/cron jobs fire on time", () => {
it("keeps legacy every jobs due while minute cron jobs recompute schedules", async () => {
const store = await makeStorePath();
const enqueueSystemEvent = vi.fn();
const requestHeartbeatNow = vi.fn();
const requestHeartbeat = vi.fn();
const nowMs = Date.parse("2025-12-13T00:00:00.000Z");
await writeCronStoreSnapshot({
@@ -154,7 +154,7 @@ describe("CronService interval/cron jobs fire on time", () => {
cronEnabled: true,
log: noopLogger,
enqueueSystemEvent,
requestHeartbeatNow,
requestHeartbeat,
runIsolatedAgentJob: vi.fn(async () => ({ status: "ok" as const })),
});

2
src/cron/service.failure-alert.test.ts
View File

@@ -35,7 +35,7 @@ function createFailureAlertCron(params: {
cronConfig: params.cronConfig,
log: noopLogger,
enqueueSystemEvent: vi.fn(),
requestHeartbeatNow: vi.fn(),
requestHeartbeat: vi.fn(),
runIsolatedAgentJob: params.runIsolatedAgentJob,
sendCronFailureAlert: params.sendCronFailureAlert,
});

2
src/cron/service.get-job.test.ts
View File

@@ -16,7 +16,7 @@ function createCronService(storePath: string) {
cronEnabled: true,
log: logger,
enqueueSystemEvent: vi.fn(),
requestHeartbeatNow: vi.fn(),
requestHeartbeat: vi.fn(),
runIsolatedAgentJob: vi.fn(async () => ({ status: "ok" as const })),
});
}

16
src/cron/service.heartbeat-ok-summary-suppressed.test.ts
View File

@@ -32,14 +32,14 @@ function createCronServiceForSummary(params: {
storePath: string;
summary: string;
enqueueSystemEvent: CronServiceParams["enqueueSystemEvent"];
requestHeartbeatNow: CronServiceParams["requestHeartbeatNow"];
requestHeartbeat: CronServiceParams["requestHeartbeat"];
}) {
return new CronService({
storePath: params.storePath,
cronEnabled: true,
log: logger,
enqueueSystemEvent: params.enqueueSystemEvent,
requestHeartbeatNow: params.requestHeartbeatNow,
requestHeartbeat: params.requestHeartbeat,
runHeartbeatOnce: vi.fn(),
runIsolatedAgentJob: vi.fn(async () => ({
status: "ok" as const,
@@ -71,19 +71,19 @@ describe("cron isolated job HEARTBEAT_OK summary suppression (#32013)", () => {
await writeCronStoreSnapshot({ storePath, jobs: [job] });
const enqueueSystemEvent = vi.fn();
const requestHeartbeatNow = vi.fn();
const requestHeartbeat = vi.fn();
const cron = createCronServiceForSummary({
storePath,
summary: "HEARTBEAT_OK",
enqueueSystemEvent,
requestHeartbeatNow,
requestHeartbeat,
});
await runScheduledCron(cron);
// HEARTBEAT_OK should NOT leak into the main session as a system event.
expect(enqueueSystemEvent).not.toHaveBeenCalled();
expect(requestHeartbeatNow).not.toHaveBeenCalled();
expect(requestHeartbeat).not.toHaveBeenCalled();
});
it("does not revive legacy main-session relay for real cron summaries", async () => {
@@ -99,17 +99,17 @@ describe("cron isolated job HEARTBEAT_OK summary suppression (#32013)", () => {
await writeCronStoreSnapshot({ storePath, jobs: [job] });
const enqueueSystemEvent = vi.fn();
const requestHeartbeatNow = vi.fn();
const requestHeartbeat = vi.fn();
const cron = createCronServiceForSummary({
storePath,
summary: "Weather update: sunny, 72°F",
enqueueSystemEvent,
requestHeartbeatNow,
requestHeartbeat,
});
await runScheduledCron(cron);
expect(enqueueSystemEvent).not.toHaveBeenCalled();
expect(requestHeartbeatNow).not.toHaveBeenCalled();
expect(requestHeartbeat).not.toHaveBeenCalled();
});
});

2
src/cron/service.issue-16156-list-skips-cron.test.ts
View File

@@ -23,7 +23,7 @@ function createCronFromStorePath(storePath: string) {
cronEnabled: true,
log: noopLogger,
enqueueSystemEvent: vi.fn(),
requestHeartbeatNow: vi.fn(),
requestHeartbeat: vi.fn(),
runIsolatedAgentJob: vi.fn(async () => ({ status: "ok" as const })),
});
}

4
src/cron/service.issue-35195-backup-timing.test.ts
View File

@@ -34,7 +34,7 @@ describe("cron backup timing for edit", () => {
cronEnabled: true,
log: noopLogger,
enqueueSystemEvent: vi.fn(),
requestHeartbeatNow: vi.fn(),
requestHeartbeat: vi.fn(),
runIsolatedAgentJob: vi.fn(async () => ({ status: "ok" as const })),
});
@@ -66,7 +66,7 @@ describe("cron backup timing for edit", () => {
cronEnabled: true,
log: noopLogger,
enqueueSystemEvent: vi.fn(),
requestHeartbeatNow: vi.fn(),
requestHeartbeat: vi.fn(),
runIsolatedAgentJob: vi.fn(async () => ({ status: "ok" as const })),
});

2
src/cron/service.issue-66019-unresolved-next-run.test.ts
View File

@@ -34,7 +34,7 @@ function createIssue66019State(params: {
log: noopLogger,
nowMs: params.nowMs,
enqueueSystemEvent: vi.fn(),
requestHeartbeatNow: vi.fn(),
requestHeartbeat: vi.fn(),
runIsolatedAgentJob: params.runIsolatedAgentJob,
});
}

8
src/cron/service.issue-regressions.test-helpers.ts
View File

@@ -35,14 +35,14 @@ export async function startCronForStore(params: {
storePath: string;
cronEnabled?: boolean;
enqueueSystemEvent?: CronServiceOptions["enqueueSystemEvent"];
requestHeartbeatNow?: CronServiceOptions["requestHeartbeatNow"];
requestHeartbeat?: CronServiceOptions["requestHeartbeat"];
runIsolatedAgentJob?: CronServiceOptions["runIsolatedAgentJob"];
onEvent?: CronServiceOptions["onEvent"];
}) {
const enqueueSystemEvent =
params.enqueueSystemEvent ?? (vi.fn() as unknown as CronServiceOptions["enqueueSystemEvent"]);
const requestHeartbeatNow =
params.requestHeartbeatNow ?? (vi.fn() as unknown as CronServiceOptions["requestHeartbeatNow"]);
const requestHeartbeat =
params.requestHeartbeat ?? (vi.fn() as unknown as CronServiceOptions["requestHeartbeat"]);
const runIsolatedAgentJob = params.runIsolatedAgentJob ?? createDefaultIsolatedRunner();
const cron = new CronService({
@@ -50,7 +50,7 @@ export async function startCronForStore(params: {
storePath: params.storePath,
log: noopLogger,
enqueueSystemEvent,
requestHeartbeatNow,
requestHeartbeat,
runIsolatedAgentJob,
...(params.onEvent ? { onEvent: params.onEvent } : {}),
});

2
src/cron/service.issue-regressions.test.ts
View File

@@ -82,7 +82,7 @@ describe("Cron issue regressions", () => {
storePath: store.storePath,
log: noopLogger,
enqueueSystemEvent: vi.fn(),
requestHeartbeatNow: vi.fn(),
requestHeartbeat: vi.fn(),
runIsolatedAgentJob: vi.fn().mockResolvedValue({ status: "ok", summary: "ok" }),
});
await cron.start();

22
src/cron/service.main-job-passes-heartbeat-target-last.test.ts
View File

@@ -33,17 +33,17 @@ describe("cron main job passes heartbeat target=last", () => {
function createCronWithSpies(params: { storePath: string; runHeartbeatOnce: RunHeartbeatOnce }) {
const enqueueSystemEvent = vi.fn();
const requestHeartbeatNow = vi.fn();
const requestHeartbeat = vi.fn();
const cron = new CronService({
storePath: params.storePath,
cronEnabled: true,
log: logger,
enqueueSystemEvent,
requestHeartbeatNow,
requestHeartbeat,
runHeartbeatOnce: params.runHeartbeatOnce,
runIsolatedAgentJob: vi.fn(async () => ({ status: "ok" as const })),
});
return { cron, requestHeartbeatNow };
return { cron, requestHeartbeat };
}
async function runSingleTick(cron: CronService) {
@@ -89,7 +89,7 @@ describe("cron main job passes heartbeat target=last", () => {
expect(callArgs?.heartbeat?.target).toBe("last");
});
it("should preserve heartbeat.target=last when wakeMode=now falls back to requestHeartbeatNow", async () => {
it("should preserve heartbeat.target=last when wakeMode=now falls back to requestHeartbeat", async () => {
const { storePath } = await makeStorePath();
const now = Date.now();
@@ -106,7 +106,7 @@ describe("cron main job passes heartbeat target=last", () => {
reason: "cron-in-progress",
}));
const { cron, requestHeartbeatNow } = createCronWithSpies({
const { cron, requestHeartbeat } = createCronWithSpies({
storePath,
runHeartbeatOnce,
});
@@ -114,8 +114,10 @@ describe("cron main job passes heartbeat target=last", () => {
await runSingleTick(cron);
expect(runHeartbeatOnce).toHaveBeenCalled();
expect(requestHeartbeatNow).toHaveBeenCalledWith(
expect(requestHeartbeat).toHaveBeenCalledWith(
expect.objectContaining({
source: "cron",
intent: "immediate",
reason: "cron:test-main-delivery-busy",
heartbeat: { target: "last" },
}),
@@ -139,16 +141,18 @@ describe("cron main job passes heartbeat target=last", () => {
durationMs: 50,
}));
const { cron, requestHeartbeatNow } = createCronWithSpies({
const { cron, requestHeartbeat } = createCronWithSpies({
storePath,
runHeartbeatOnce,
});
await runSingleTick(cron);
expect(requestHeartbeatNow).toHaveBeenCalled();
expect(requestHeartbeatNow).toHaveBeenCalledWith(
expect(requestHeartbeat).toHaveBeenCalled();
expect(requestHeartbeat).toHaveBeenCalledWith(
expect.objectContaining({
source: "cron",
intent: "event",
reason: "cron:test-next-heartbeat",
heartbeat: { target: "last" },
}),

2
src/cron/service.persists-delivered-status.test.ts
View File

@@ -56,7 +56,7 @@ function createIsolatedCronWithFinishedBarrier(params: {
cronEnabled: true,
log: noopLogger,
enqueueSystemEvent: vi.fn(),
requestHeartbeatNow: vi.fn(),
requestHeartbeat: vi.fn(),
runIsolatedAgentJob: vi.fn(async () => ({
status: "ok" as const,
summary: "done",

8
src/cron/service.prevents-duplicate-timers.test.ts
View File

@@ -17,7 +17,7 @@ describe("CronService", () => {
it("avoids duplicate runs when two services share a store", async () => {
const store = await makeStorePath();
const enqueueSystemEvent = vi.fn();
const requestHeartbeatNow = vi.fn();
const requestHeartbeat = vi.fn();
const runIsolatedAgentJob = vi.fn(async () => ({ status: "ok" as const }));
const cronA = new CronService({
@@ -25,7 +25,7 @@ describe("CronService", () => {
cronEnabled: true,
log: noopLogger,
enqueueSystemEvent,
requestHeartbeatNow,
requestHeartbeat,
runIsolatedAgentJob,
});
@@ -45,7 +45,7 @@ describe("CronService", () => {
cronEnabled: true,
log: noopLogger,
enqueueSystemEvent,
requestHeartbeatNow,
requestHeartbeat,
runIsolatedAgentJob,
});
@@ -57,7 +57,7 @@ describe("CronService", () => {
await cronB.status();
expect(enqueueSystemEvent).toHaveBeenCalledTimes(1);
expect(requestHeartbeatNow).toHaveBeenCalledTimes(1);
expect(requestHeartbeat).toHaveBeenCalledTimes(1);
cronA.stop();
cronB.stop();

12
src/cron/service.read-ops-nonblocking.test.ts
View File

@@ -82,7 +82,7 @@ describe("CronService read ops while job is running", () => {
vi.setSystemTime(new Date("2025-12-13T00:00:00.000Z"));
const store = await makeStorePath();
const enqueueSystemEvent = vi.fn();
const requestHeartbeatNow = vi.fn();
const requestHeartbeat = vi.fn();
let resolveFinished: (() => void) | undefined;
const finished = new Promise<void>((resolve) => {
resolveFinished = resolve;
@@ -95,7 +95,7 @@ describe("CronService read ops while job is running", () => {
cronEnabled: true,
log: noopLogger,
enqueueSystemEvent,
requestHeartbeatNow,
requestHeartbeat,
runIsolatedAgentJob: isolatedRun.runIsolatedAgentJob,
onEvent: (evt) => {
if (evt.action === "finished" && evt.status === "ok") {
@@ -164,7 +164,7 @@ describe("CronService read ops while job is running", () => {
it("keeps list and status responsive during manual cron.run execution", async () => {
const store = await makeStorePath();
const enqueueSystemEvent = vi.fn();
const requestHeartbeatNow = vi.fn();
const requestHeartbeat = vi.fn();
const isolatedRun = createDeferredIsolatedRun();
const cron = new CronService({
@@ -172,7 +172,7 @@ describe("CronService read ops while job is running", () => {
cronEnabled: true,
log: noopLogger,
enqueueSystemEvent,
requestHeartbeatNow,
requestHeartbeat,
runIsolatedAgentJob: isolatedRun.runIsolatedAgentJob,
});
@@ -217,7 +217,7 @@ describe("CronService read ops while job is running", () => {
it("keeps list and status responsive after startup defers catch-up runs", async () => {
const store = await makeStorePath();
const enqueueSystemEvent = vi.fn();
const requestHeartbeatNow = vi.fn();
const requestHeartbeat = vi.fn();
const nowMs = Date.parse("2025-12-13T00:00:00.000Z");
await writeCronStoreSnapshot({
@@ -247,7 +247,7 @@ describe("CronService read ops while job is running", () => {
log: noopLogger,
nowMs: () => nowMs,
enqueueSystemEvent,
requestHeartbeatNow,
requestHeartbeat,
runIsolatedAgentJob: isolatedRun.runIsolatedAgentJob,
startupDeferredMissedAgentJobDelayMs: 120_000,
});

2
src/cron/service.rearm-timer-when-running.test.ts
View File

@@ -125,7 +125,7 @@ describe("CronService - timer re-arm when running (#12025)", () => {
log: noopLogger,
nowMs: () => now,
enqueueSystemEvent: vi.fn(),
requestHeartbeatNow: vi.fn(),
requestHeartbeat: vi.fn(),
runIsolatedAgentJob: vi.fn(async () => await deferredRun.promise),
});

48
src/cron/service.restart-catchup.test.ts
View File

@@ -21,7 +21,7 @@ describe("CronService restart catch-up", () => {
function createRestartCronService(params: {
storePath: string;
enqueueSystemEvent: ReturnType<typeof vi.fn>;
requestHeartbeatNow: ReturnType<typeof vi.fn>;
requestHeartbeat: ReturnType<typeof vi.fn>;
onEvent?: ReturnType<typeof vi.fn>;
nowMs?: () => number;
runIsolatedAgentJob?: ReturnType<typeof vi.fn>;
@@ -33,7 +33,7 @@ describe("CronService restart catch-up", () => {
log: noopLogger,
...(params.nowMs ? { nowMs: params.nowMs } : {}),
enqueueSystemEvent: params.enqueueSystemEvent as never,
requestHeartbeatNow: params.requestHeartbeatNow as never,
requestHeartbeat: params.requestHeartbeat as never,
runIsolatedAgentJob:
(params.runIsolatedAgentJob as never) ??
(vi.fn(async () => ({ status: "ok" as const })) as never),
@@ -64,13 +64,13 @@ describe("CronService restart catch-up", () => {
run: (params: {
cron: CronService;
enqueueSystemEvent: ReturnType<typeof vi.fn>;
requestHeartbeatNow: ReturnType<typeof vi.fn>;
requestHeartbeat: ReturnType<typeof vi.fn>;
onEvent: ReturnType<typeof vi.fn>;
}) => Promise<void>,
) {
const store = await makeStorePath();
const enqueueSystemEvent = vi.fn();
const requestHeartbeatNow = vi.fn();
const requestHeartbeat = vi.fn();
const onEvent = vi.fn();
await writeStoreJobs(store.storePath, jobs);
@@ -78,13 +78,13 @@ describe("CronService restart catch-up", () => {
const cron = createRestartCronService({
storePath: store.storePath,
enqueueSystemEvent,
requestHeartbeatNow,
requestHeartbeat,
onEvent,
});
try {
await cron.start();
await run({ cron, enqueueSystemEvent, requestHeartbeatNow, onEvent });
await run({ cron, enqueueSystemEvent, requestHeartbeat, onEvent });
} finally {
cron.stop();
await store.cleanup();
@@ -114,12 +114,12 @@ describe("CronService restart catch-up", () => {
},
},
],
async ({ cron, enqueueSystemEvent, requestHeartbeatNow }) => {
async ({ cron, enqueueSystemEvent, requestHeartbeat }) => {
expect(enqueueSystemEvent).toHaveBeenCalledWith(
"digest now",
expect.objectContaining({ agentId: undefined }),
);
expect(requestHeartbeatNow).toHaveBeenCalled();
expect(requestHeartbeat).toHaveBeenCalled();
const listedJobs = await cron.list({ includeDisabled: true });
const updated = listedJobs.find((job) => job.id === "restart-overdue-job");
@@ -135,7 +135,7 @@ describe("CronService restart catch-up", () => {
const startNow = Date.parse("2025-12-13T17:00:00.000Z");
const runIsolatedAgentJob = vi.fn(async () => ({ status: "ok" as const }));
const enqueueSystemEvent = vi.fn();
const requestHeartbeatNow = vi.fn();
const requestHeartbeat = vi.fn();
await writeStoreJobs(store.storePath, [
{
@@ -155,7 +155,7 @@ describe("CronService restart catch-up", () => {
const cron = createRestartCronService({
storePath: store.storePath,
enqueueSystemEvent,
requestHeartbeatNow,
requestHeartbeat,
runIsolatedAgentJob,
nowMs: () => startNow,
startupDeferredMissedAgentJobDelayMs: 120_000,
@@ -166,7 +166,7 @@ describe("CronService restart catch-up", () => {
expect(runIsolatedAgentJob).not.toHaveBeenCalled();
expect(enqueueSystemEvent).not.toHaveBeenCalled();
expect(requestHeartbeatNow).not.toHaveBeenCalled();
expect(requestHeartbeat).not.toHaveBeenCalled();
const listedJobs = await cron.list({ includeDisabled: true });
const updated = listedJobs.find((job) => job.id === "startup-isolated-agent");
@@ -201,14 +201,14 @@ describe("CronService restart catch-up", () => {
},
},
],
async ({ cron, enqueueSystemEvent, requestHeartbeatNow, onEvent }) => {
async ({ cron, enqueueSystemEvent, requestHeartbeat, onEvent }) => {
expect(noopLogger.warn).toHaveBeenCalledWith(
expect.objectContaining({ jobId: "restart-stale-running" }),
"cron: marking interrupted running job failed on startup",
);
expect(enqueueSystemEvent).not.toHaveBeenCalled();
expect(requestHeartbeatNow).not.toHaveBeenCalled();
expect(requestHeartbeat).not.toHaveBeenCalled();
const listedJobs = await cron.list({ includeDisabled: true });
const updated = listedJobs.find((job) => job.id === "restart-stale-running");
@@ -253,12 +253,12 @@ describe("CronService restart catch-up", () => {
},
},
],
async ({ cron, enqueueSystemEvent, requestHeartbeatNow }) => {
async ({ cron, enqueueSystemEvent, requestHeartbeat }) => {
expect(enqueueSystemEvent).toHaveBeenCalledWith(
"catch missed slot",
expect.objectContaining({ agentId: undefined }),
);
expect(requestHeartbeatNow).toHaveBeenCalled();
expect(requestHeartbeat).toHaveBeenCalled();
const listedJobs = await cron.list({ includeDisabled: true });
const updated = listedJobs.find((job) => job.id === "restart-missed-slot");
@@ -289,9 +289,9 @@ describe("CronService restart catch-up", () => {
},
},
],
async ({ cron, enqueueSystemEvent, requestHeartbeatNow, onEvent }) => {
async ({ cron, enqueueSystemEvent, requestHeartbeat, onEvent }) => {
expect(enqueueSystemEvent).not.toHaveBeenCalled();
expect(requestHeartbeatNow).not.toHaveBeenCalled();
expect(requestHeartbeat).not.toHaveBeenCalled();
const listedJobs = await cron.list({ includeDisabled: true });
const updated = listedJobs.find((job) => job.id === "restart-stale-one-shot");
@@ -336,9 +336,9 @@ describe("CronService restart catch-up", () => {
},
},
],
async ({ enqueueSystemEvent, requestHeartbeatNow }) => {
async ({ enqueueSystemEvent, requestHeartbeat }) => {
expect(enqueueSystemEvent).not.toHaveBeenCalled();
expect(requestHeartbeatNow).not.toHaveBeenCalled();
expect(requestHeartbeat).not.toHaveBeenCalled();
},
);
});
@@ -366,9 +366,9 @@ describe("CronService restart catch-up", () => {
},
},
],
async ({ enqueueSystemEvent, requestHeartbeatNow }) => {
async ({ enqueueSystemEvent, requestHeartbeat }) => {
expect(enqueueSystemEvent).not.toHaveBeenCalled();
expect(requestHeartbeatNow).not.toHaveBeenCalled();
expect(requestHeartbeat).not.toHaveBeenCalled();
},
);
});
@@ -397,12 +397,12 @@ describe("CronService restart catch-up", () => {
},
},
],
async ({ enqueueSystemEvent, requestHeartbeatNow }) => {
async ({ enqueueSystemEvent, requestHeartbeat }) => {
expect(enqueueSystemEvent).toHaveBeenCalledWith(
"replay after backoff elapsed",
expect.objectContaining({ agentId: undefined }),
);
expect(requestHeartbeatNow).toHaveBeenCalled();
expect(requestHeartbeat).toHaveBeenCalled();
},
);
});
@@ -424,7 +424,7 @@ describe("CronService restart catch-up", () => {
log: noopLogger,
nowMs: () => now,
enqueueSystemEvent: vi.fn(),
requestHeartbeatNow: vi.fn(),
requestHeartbeat: vi.fn(),
runIsolatedAgentJob: vi.fn(async () => {
now += 6_000;
return { status: "ok" as const, summary: "ok" };

44
src/cron/service.runs-one-shot-main-job-disables-it.test.ts
View File

@@ -232,7 +232,7 @@ async function createCronHarness(options: CronHarnessOptions = {}) {
ensureDir(fixturesRoot);
const store = await makeStorePath();
const enqueueSystemEvent = vi.fn();
const requestHeartbeatNow = vi.fn();
const requestHeartbeat = vi.fn();
const events = options.withEvents === false ? undefined : createCronEventHarness();
const cron = new CronService({
@@ -247,7 +247,7 @@ async function createCronHarness(options: CronHarnessOptions = {}) {
? { wakeNowHeartbeatBusyRetryDelayMs: options.wakeNowHeartbeatBusyRetryDelayMs }
: {}),
enqueueSystemEvent,
requestHeartbeatNow,
requestHeartbeat,
...(options.runHeartbeatOnce ? { runHeartbeatOnce: options.runHeartbeatOnce } : {}),
runIsolatedAgentJob:
options.runIsolatedAgentJob ??
@@ -257,7 +257,7 @@ async function createCronHarness(options: CronHarnessOptions = {}) {
...(events ? { onEvent: events.onEvent } : {}),
});
await cron.start();
return { store, cron, enqueueSystemEvent, requestHeartbeatNow, events };
return { store, cron, enqueueSystemEvent, requestHeartbeat, events };
}
async function createMainOneShotHarness() {
@@ -390,7 +390,7 @@ function createStartedCronService(
cronEnabled: true,
log: noopLogger,
enqueueSystemEvent: vi.fn(),
requestHeartbeatNow: vi.fn(),
requestHeartbeat: vi.fn(),
runIsolatedAgentJob: runIsolatedAgentJob ?? vi.fn(async () => ({ status: "ok" as const })),
});
}
@@ -410,7 +410,7 @@ async function expectNoMainSummaryForIsolatedRun(params: {
runIsolatedAgentJob: CronServiceDeps["runIsolatedAgentJob"];
name: string;
}) {
const { store, cron, enqueueSystemEvent, requestHeartbeatNow, events } =
const { store, cron, enqueueSystemEvent, requestHeartbeat, events } =
await createIsolatedAnnounceHarness(params.runIsolatedAgentJob);
await runIsolatedAnnounceScenario({
cron,
@@ -418,13 +418,13 @@ async function expectNoMainSummaryForIsolatedRun(params: {
name: params.name,
});
expect(enqueueSystemEvent).not.toHaveBeenCalled();
expect(requestHeartbeatNow).not.toHaveBeenCalled();
expect(requestHeartbeat).not.toHaveBeenCalled();
await stopCronAndCleanup(cron, store);
}
describe("CronService", () => {
it("runs a one-shot main job and disables it after success when requested", async () => {
const { store, cron, enqueueSystemEvent, requestHeartbeatNow, events, atMs, job } =
const { store, cron, enqueueSystemEvent, requestHeartbeat, events, atMs, job } =
await createMainOneShotJobHarness({
name: "one-shot hello",
deleteAfterRun: false,
@@ -440,14 +440,14 @@ describe("CronService", () => {
const updated = jobs.find((j) => j.id === job.id);
expect(updated?.enabled).toBe(false);
expectMainSystemEventPosted(enqueueSystemEvent, "hello");
expect(requestHeartbeatNow).toHaveBeenCalled();
expect(requestHeartbeat).toHaveBeenCalled();
await cron.list({ includeDisabled: true });
await stopCronAndCleanup(cron, store);
});
it("runs a one-shot job and deletes it after success by default", async () => {
const { store, cron, enqueueSystemEvent, requestHeartbeatNow, events, job } =
const { store, cron, enqueueSystemEvent, requestHeartbeat, events, job } =
await createMainOneShotJobHarness({
name: "one-shot delete",
});
@@ -459,7 +459,7 @@ describe("CronService", () => {
const jobs = await cron.list({ includeDisabled: true });
expect(jobs.find((j) => j.id === job.id)).toBeUndefined();
expectMainSystemEventPosted(enqueueSystemEvent, "hello");
expect(requestHeartbeatNow).toHaveBeenCalled();
expect(requestHeartbeat).toHaveBeenCalled();
await stopCronAndCleanup(cron, store);
});
@@ -480,7 +480,7 @@ describe("CronService", () => {
});
});
const { store, cron, enqueueSystemEvent, requestHeartbeatNow } =
const { store, cron, enqueueSystemEvent, requestHeartbeat } =
await createWakeModeNowMainHarness({
runHeartbeatOnce,
nowMs,
@@ -491,7 +491,7 @@ describe("CronService", () => {
await heartbeatStarted.promise;
expect(runHeartbeatOnce).toHaveBeenCalledTimes(1);
expect(requestHeartbeatNow).not.toHaveBeenCalled();
expect(requestHeartbeat).not.toHaveBeenCalled();
expectMainSystemEventPosted(enqueueSystemEvent, "hello");
expect(job.state.runningAtMs).toBeTypeOf("number");
@@ -536,7 +536,7 @@ describe("CronService", () => {
return now;
};
const { store, cron, requestHeartbeatNow } = await createWakeModeNowMainHarness({
const { store, cron, requestHeartbeat } = await createWakeModeNowMainHarness({
runHeartbeatOnce,
nowMs,
// Perf: avoid advancing fake timers by 2+ minutes for the busy-heartbeat fallback.
@@ -553,7 +553,7 @@ describe("CronService", () => {
await cron.run(job.id, "force");
expect(runHeartbeatOnce).toHaveBeenCalled();
expect(requestHeartbeatNow).toHaveBeenCalledWith(
expect(requestHeartbeat).toHaveBeenCalledWith(
expect.objectContaining({
reason: `cron:${job.id}`,
sessionKey,
@@ -572,7 +572,7 @@ describe("CronService", () => {
reason: HEARTBEAT_SKIP_CRON_IN_PROGRESS,
}));
const { store, cron, requestHeartbeatNow } = await createWakeModeNowMainHarness({
const { store, cron, requestHeartbeat } = await createWakeModeNowMainHarness({
runHeartbeatOnce,
});
@@ -585,7 +585,7 @@ describe("CronService", () => {
await cron.run(job.id, "force");
expect(runHeartbeatOnce).toHaveBeenCalledTimes(1);
expect(requestHeartbeatNow).toHaveBeenCalledWith(
expect(requestHeartbeat).toHaveBeenCalledWith(
expect.objectContaining({
reason: `cron:${job.id}`,
sessionKey,
@@ -600,12 +600,12 @@ describe("CronService", () => {
it("runs an isolated job without posting a fallback summary to main", async () => {
const runIsolatedAgentJob = vi.fn(async () => ({ status: "ok" as const, summary: "done" }));
const { store, cron, enqueueSystemEvent, requestHeartbeatNow, events } =
const { store, cron, enqueueSystemEvent, requestHeartbeat, events } =
await createIsolatedAnnounceHarness(runIsolatedAgentJob);
await runIsolatedAnnounceScenario({ cron, events, name: "weekly" });
expect(runIsolatedAgentJob).toHaveBeenCalledTimes(1);
expect(enqueueSystemEvent).not.toHaveBeenCalled();
expect(requestHeartbeatNow).not.toHaveBeenCalled();
expect(requestHeartbeat).not.toHaveBeenCalled();
await stopCronAndCleanup(cron, store);
});
@@ -642,7 +642,7 @@ describe("CronService", () => {
summary: "last output",
error: "boom",
}));
const { store, cron, enqueueSystemEvent, requestHeartbeatNow, events } =
const { store, cron, enqueueSystemEvent, requestHeartbeat, events } =
await createIsolatedAnnounceHarness(runIsolatedAgentJob);
await runIsolatedAnnounceJobAndWait({
cron,
@@ -652,7 +652,7 @@ describe("CronService", () => {
});
expect(enqueueSystemEvent).not.toHaveBeenCalled();
expect(requestHeartbeatNow).not.toHaveBeenCalled();
expect(requestHeartbeat).not.toHaveBeenCalled();
await stopCronAndCleanup(cron, store);
});
@@ -663,7 +663,7 @@ describe("CronService", () => {
error: "Channel is required when multiple channels are configured: telegram, discord",
errorKind: "delivery-target" as const,
}));
const { store, cron, enqueueSystemEvent, requestHeartbeatNow, events } =
const { store, cron, enqueueSystemEvent, requestHeartbeat, events } =
await createIsolatedAnnounceHarness(runIsolatedAgentJob);
await runIsolatedAnnounceJobAndWait({
cron,
@@ -673,7 +673,7 @@ describe("CronService", () => {
});
expect(enqueueSystemEvent).not.toHaveBeenCalled();
expect(requestHeartbeatNow).not.toHaveBeenCalled();
expect(requestHeartbeat).not.toHaveBeenCalled();
await stopCronAndCleanup(cron, store);
});

6
src/cron/service.session-reaper-in-finally.test.ts
View File

@@ -70,7 +70,7 @@ describe("CronService - session reaper runs in finally block (#31946)", () => {
log: noopLogger,
nowMs: () => now,
enqueueSystemEvent: vi.fn(),
requestHeartbeatNow: vi.fn(),
requestHeartbeat: vi.fn(),
// This will throw, simulating a failure during job execution.
runIsolatedAgentJob: vi.fn().mockRejectedValue(new Error("gateway down")),
sessionStorePath,
@@ -109,7 +109,7 @@ describe("CronService - session reaper runs in finally block (#31946)", () => {
log: noopLogger,
nowMs: () => now,
enqueueSystemEvent: vi.fn(),
requestHeartbeatNow: vi.fn(),
requestHeartbeat: vi.fn(),
runIsolatedAgentJob: vi.fn().mockResolvedValue({ status: "ok", summary: "done" }),
resolveSessionStorePath: (agentId) => {
const p = path.join(path.dirname(store.storePath), `${agentId}-sessions`, "sessions.json");
@@ -156,7 +156,7 @@ describe("CronService - session reaper runs in finally block (#31946)", () => {
log: noopLogger,
nowMs: () => now,
enqueueSystemEvent: vi.fn(),
requestHeartbeatNow: vi.fn(),
requestHeartbeat: vi.fn(),
runIsolatedAgentJob: vi.fn(),
sessionStorePath,
});

10
src/cron/service.skips-main-jobs-empty-systemevent-text.test.ts
View File

@@ -31,7 +31,7 @@ async function withCronService(
run: (params: {
cron: CronService;
enqueueSystemEvent: ReturnType<typeof vi.fn>;
requestHeartbeatNow: ReturnType<typeof vi.fn>;
requestHeartbeat: ReturnType<typeof vi.fn>;
}) => Promise<void>,
) {
await withCronServiceForTest(
@@ -60,7 +60,7 @@ describe("CronService", () => {
});
it("skips main jobs with empty systemEvent text", async () => {
await withCronService(true, async ({ cron, enqueueSystemEvent, requestHeartbeatNow }) => {
await withCronService(true, async ({ cron, enqueueSystemEvent, requestHeartbeat }) => {
const atMs = Date.parse("2025-12-13T00:00:01.000Z");
await cron.add({
name: "empty systemEvent test",
@@ -75,7 +75,7 @@ describe("CronService", () => {
await vi.runOnlyPendingTimersAsync();
expect(enqueueSystemEvent).not.toHaveBeenCalled();
expect(requestHeartbeatNow).not.toHaveBeenCalled();
expect(requestHeartbeat).not.toHaveBeenCalled();
const job = await waitForFirstJob(cron, (current) => current?.state.lastStatus === "skipped");
expect(job?.state.lastStatus).toBe("skipped");
@@ -84,7 +84,7 @@ describe("CronService", () => {
});
it("does not schedule timers when cron is disabled", async () => {
await withCronService(false, async ({ cron, enqueueSystemEvent, requestHeartbeatNow }) => {
await withCronService(false, async ({ cron, enqueueSystemEvent, requestHeartbeat }) => {
const atMs = Date.parse("2025-12-13T00:00:01.000Z");
await cron.add({
name: "disabled cron job",
@@ -103,7 +103,7 @@ describe("CronService", () => {
await vi.runOnlyPendingTimersAsync();
expect(enqueueSystemEvent).not.toHaveBeenCalled();
expect(requestHeartbeatNow).not.toHaveBeenCalled();
expect(requestHeartbeat).not.toHaveBeenCalled();
expect(noopLogger.warn).toHaveBeenCalled();
});
});

6
src/cron/service.store-load-invalid-main-job.test.ts
View File

@@ -36,7 +36,7 @@ describe("CronService store load", () => {
const { dir, storePath } = await makeStorePath();
tempDir = dir;
const enqueueSystemEvent = vi.fn();
const requestHeartbeatNow = vi.fn();
const requestHeartbeat = vi.fn();
const job = {
id: "job-1",
@@ -58,7 +58,7 @@ describe("CronService store load", () => {
cronEnabled: true,
log: noopLogger,
enqueueSystemEvent,
requestHeartbeatNow,
requestHeartbeat,
runIsolatedAgentJob: vi.fn(async () => ({ status: "ok" as const })),
});
@@ -67,7 +67,7 @@ describe("CronService store load", () => {
await cron.run("job-1", "due");
expect(enqueueSystemEvent).not.toHaveBeenCalled();
expect(requestHeartbeatNow).not.toHaveBeenCalled();
expect(requestHeartbeat).not.toHaveBeenCalled();
const jobs = await cron.list({ includeDisabled: true });
expect(jobs[0]?.state.lastStatus).toBe("skipped");

20
src/cron/service.test-harness.ts
View File

@@ -127,22 +127,22 @@ export function createStartedCronServiceWithFinishedBarrier(params: {
}): {
cron: CronService;
enqueueSystemEvent: MockFn;
requestHeartbeatNow: MockFn;
requestHeartbeat: MockFn;
finished: ReturnType<typeof createFinishedBarrier>;
} {
const enqueueSystemEvent = vi.fn();
const requestHeartbeatNow = vi.fn();
const requestHeartbeat = vi.fn();
const finished = createFinishedBarrier();
const cron = new CronService({
storePath: params.storePath,
cronEnabled: true,
log: params.logger,
enqueueSystemEvent,
requestHeartbeatNow,
requestHeartbeat,
runIsolatedAgentJob: vi.fn(async () => ({ status: "ok" as const })),
onEvent: finished.onEvent,
});
return { cron, enqueueSystemEvent, requestHeartbeatNow, finished };
return { cron, enqueueSystemEvent, requestHeartbeat, finished };
}
export async function withCronServiceForTest(
@@ -155,18 +155,18 @@ export async function withCronServiceForTest(
run: (context: {
cron: CronService;
enqueueSystemEvent: ReturnType<typeof vi.fn>;
requestHeartbeatNow: ReturnType<typeof vi.fn>;
requestHeartbeat: ReturnType<typeof vi.fn>;
}) => Promise<void>,
): Promise<void> {
const store = await params.makeStorePath();
const enqueueSystemEvent = vi.fn();
const requestHeartbeatNow = vi.fn();
const requestHeartbeat = vi.fn();
const cron = new CronService({
cronEnabled: params.cronEnabled,
storePath: store.storePath,
log: params.logger,
enqueueSystemEvent,
requestHeartbeatNow,
requestHeartbeat,
runIsolatedAgentJob:
params.runIsolatedAgentJob ??
(vi.fn(async () => ({ status: "ok" as const, summary: "done" })) as never),
@@ -174,7 +174,7 @@ export async function withCronServiceForTest(
await cron.start();
try {
await run({ cron, enqueueSystemEvent, requestHeartbeatNow });
await run({ cron, enqueueSystemEvent, requestHeartbeat });
} finally {
cron.stop();
await store.cleanup();
@@ -193,7 +193,7 @@ export function createRunningCronServiceState(params: {
log: params.log,
nowMs: params.nowMs,
enqueueSystemEvent: vi.fn(),
requestHeartbeatNow: vi.fn(),
requestHeartbeat: vi.fn(),
runIsolatedAgentJob: vi.fn().mockResolvedValue({ status: "ok", summary: "ok" }),
});
state.running = true;
@@ -251,7 +251,7 @@ export function createMockCronStateForJobs(params: {
cronEnabled: true,
nowMs: () => nowMs,
enqueueSystemEvent: () => {},
requestHeartbeatNow: () => {},
requestHeartbeat: () => {},
runIsolatedAgentJob: async () => ({ status: "ok" }),
log: {
debug: () => {},

2
src/cron/service/jobs.schedule-error-isolation.test.ts
View File

@@ -16,7 +16,7 @@ function createMockState(jobs: CronJob[]): CronServiceState {
error: vi.fn(),
},
enqueueSystemEvent: vi.fn(),
requestHeartbeatNow: vi.fn(),
requestHeartbeat: vi.fn(),
runHeartbeatOnce: vi.fn(),
runIsolatedAgentJob: vi.fn(),
onEvent: vi.fn(),

4
src/cron/service/jobs.ts
View File

@@ -343,7 +343,9 @@ export function recordScheduleComputeError(params: {
sessionKey: job.sessionKey,
contextKey: `cron:${job.id}:auto-disabled`,
});
state.deps.requestHeartbeatNow({
state.deps.requestHeartbeat({
source: "cron",
intent: "event",
reason: `cron:${job.id}:auto-disabled`,
agentId: job.agentId,
sessionKey: job.sessionKey,

14
src/cron/service/ops.regression.test.ts
View File

@@ -35,7 +35,7 @@ describe("cron service ops regressions", () => {
storePath: store.storePath,
log: noopLogger,
enqueueSystemEvent: vi.fn(),
requestHeartbeatNow: vi.fn(),
requestHeartbeat: vi.fn(),
runIsolatedAgentJob: vi.fn(),
});
state.store = {
@@ -90,7 +90,7 @@ describe("cron service ops regressions", () => {
storePath: store.storePath,
log: noopLogger,
enqueueSystemEvent: vi.fn(),
requestHeartbeatNow: vi.fn(),
requestHeartbeat: vi.fn(),
runIsolatedAgentJob,
onEvent: (evt: CronEvent) => {
if (evt.jobId !== job.id) {
@@ -152,7 +152,7 @@ describe("cron service ops regressions", () => {
log: noopLogger,
nowMs: () => now,
enqueueSystemEvent: vi.fn(),
requestHeartbeatNow: vi.fn(),
requestHeartbeat: vi.fn(),
runIsolatedAgentJob,
onEvent: (evt: CronEvent) => {
if (evt.jobId === job.id && evt.action === "finished") {
@@ -214,7 +214,7 @@ describe("cron service ops regressions", () => {
storePath: store.storePath,
log: noopLogger,
enqueueSystemEvent: vi.fn(),
requestHeartbeatNow: vi.fn(),
requestHeartbeat: vi.fn(),
runIsolatedAgentJob: vi.fn().mockResolvedValue({ status: "ok", summary: "ok" }),
});
@@ -249,7 +249,7 @@ describe("cron service ops regressions", () => {
storePath: store.storePath,
log: noopLogger,
enqueueSystemEvent: vi.fn(),
requestHeartbeatNow: vi.fn(),
requestHeartbeat: vi.fn(),
runIsolatedAgentJob: abortAwareRunner.runIsolatedAgentJob,
});
@@ -301,7 +301,7 @@ describe("cron service ops regressions", () => {
log: noopLogger,
nowMs: () => now,
enqueueSystemEvent,
requestHeartbeatNow: vi.fn(),
requestHeartbeat: vi.fn(),
runIsolatedAgentJob: vi.fn().mockResolvedValue({ status: "ok", summary: "ok" }),
});
@@ -365,7 +365,7 @@ describe("cron service ops regressions", () => {
log: noopLogger,
nowMs: () => now,
enqueueSystemEvent: vi.fn(),
requestHeartbeatNow: vi.fn(),
requestHeartbeat: vi.fn(),
runIsolatedAgentJob,
onEvent: (evt) => {
if (evt.action === "finished" && evt.jobId === second.id && evt.status === "ok") {

10
src/cron/service/ops.test.ts
View File

@@ -35,7 +35,7 @@ function createTimedOutIsolatedCronState(params: { storePath: string; now: numbe
log: logger,
nowMs: () => params.now,
enqueueSystemEvent: vi.fn(),
requestHeartbeatNow: vi.fn(),
requestHeartbeat: vi.fn(),
runIsolatedAgentJob: vi.fn(async () => {
throw new Error("cron: job execution timed out");
}),
@@ -49,7 +49,7 @@ function createOkIsolatedCronState(params: { storePath: string; now: number; sum
log: logger,
nowMs: () => params.now,
enqueueSystemEvent: vi.fn(),
requestHeartbeatNow: vi.fn(),
requestHeartbeat: vi.fn(),
runIsolatedAgentJob: vi.fn(async () => ({
status: "ok" as const,
...(params.summary === undefined ? {} : { summary: params.summary }),
@@ -133,7 +133,7 @@ describe("cron service ops seam coverage", () => {
const { storePath } = await makeStorePath();
const now = Date.parse("2026-03-23T12:00:00.000Z");
const enqueueSystemEvent = vi.fn();
const requestHeartbeatNow = vi.fn();
const requestHeartbeat = vi.fn();
const timeoutSpy = vi.spyOn(globalThis, "setTimeout");
await writeCronStoreSnapshot({
@@ -147,7 +147,7 @@ describe("cron service ops seam coverage", () => {
log: logger,
nowMs: () => now,
enqueueSystemEvent,
requestHeartbeatNow,
requestHeartbeat,
runIsolatedAgentJob: vi.fn(async () => ({ status: "ok" as const })),
});
@@ -158,7 +158,7 @@ describe("cron service ops seam coverage", () => {
"cron: marking interrupted running job failed on startup",
);
expect(enqueueSystemEvent).not.toHaveBeenCalled();
expect(requestHeartbeatNow).not.toHaveBeenCalled();
expect(requestHeartbeat).not.toHaveBeenCalled();
expect(state.timer).not.toBeNull();
const persisted = (await loadCronStore(storePath)) as {

8
src/cron/service/state.test.ts
View File

@@ -5,7 +5,7 @@ describe("cron service state seam coverage", () => {
it("threads heartbeat and session-store dependencies into internal state", () => {
const nowMs = vi.fn(() => 123_456);
const enqueueSystemEvent = vi.fn();
const requestHeartbeatNow = vi.fn();
const requestHeartbeat = vi.fn();
const runHeartbeatOnce = vi.fn();
const resolveSessionStorePath = vi.fn((agentId?: string) => `/tmp/${agentId ?? "main"}.json`);
@@ -23,7 +23,7 @@ describe("cron service state seam coverage", () => {
sessionStorePath: "/tmp/sessions.json",
resolveSessionStorePath,
enqueueSystemEvent,
requestHeartbeatNow,
requestHeartbeat,
runHeartbeatOnce,
runIsolatedAgentJob: vi.fn(async () => ({ status: "ok" as const })),
});
@@ -41,7 +41,7 @@ describe("cron service state seam coverage", () => {
expect(state.deps.sessionStorePath).toBe("/tmp/sessions.json");
expect(state.deps.resolveSessionStorePath).toBe(resolveSessionStorePath);
expect(state.deps.enqueueSystemEvent).toBe(enqueueSystemEvent);
expect(state.deps.requestHeartbeatNow).toBe(requestHeartbeatNow);
expect(state.deps.requestHeartbeat).toBe(requestHeartbeat);
expect(state.deps.runHeartbeatOnce).toBe(runHeartbeatOnce);
expect(state.deps.nowMs()).toBe(123_456);
});
@@ -59,7 +59,7 @@ describe("cron service state seam coverage", () => {
storePath: "/tmp/cron/jobs.json",
cronEnabled: false,
enqueueSystemEvent: vi.fn(),
requestHeartbeatNow: vi.fn(),
requestHeartbeat: vi.fn(),
runIsolatedAgentJob: vi.fn(async () => ({ status: "ok" as const })),
});

6
src/cron/service/state.ts
View File

@@ -74,8 +74,10 @@ export type CronServiceDeps = {
text: string,
opts?: { agentId?: string; sessionKey?: string; contextKey?: string; trusted?: boolean },
) => void;
requestHeartbeatNow: (opts?: HeartbeatWakeRequest) => void;
requestHeartbeat: (opts: HeartbeatWakeRequest) => void;
runHeartbeatOnce?: (opts?: {
source?: HeartbeatWakeRequest["source"];
intent?: HeartbeatWakeRequest["intent"];
reason?: string;
agentId?: string;
sessionKey?: string;
@@ -85,7 +87,7 @@ export type CronServiceDeps = {
/**
* WakeMode=now: max time to wait for runHeartbeatOnce to stop returning
* { status:"skipped", reason:"requests-in-flight" } before falling back to
* requestHeartbeatNow.
* requestHeartbeat.
*/
wakeNowHeartbeatBusyMaxWaitMs?: number;
/** WakeMode=now: delay between runHeartbeatOnce retries while busy. */

2
src/cron/service/store.load-missing-session-target.test.ts
View File

@@ -24,7 +24,7 @@ function createStoreTestState(storePath: string) {
log: logger,
nowMs: () => STORE_TEST_NOW,
enqueueSystemEvent: vi.fn(),
requestHeartbeatNow: vi.fn(),
requestHeartbeat: vi.fn(),
runIsolatedAgentJob: vi.fn(async () => ({ status: "ok" as const })),
});
}

2
src/cron/service/store.test.ts
View File

@@ -37,7 +37,7 @@ function createStoreTestState(storePath: string) {
log: logger,
nowMs: () => STORE_TEST_NOW,
enqueueSystemEvent: vi.fn(),
requestHeartbeatNow: vi.fn(),
requestHeartbeat: vi.fn(),
runIsolatedAgentJob: vi.fn(async () => ({ status: "ok" as const })),
});
}

62
src/cron/service/timer.regression.test.ts
View File

@@ -39,7 +39,7 @@ describe("cron service timer regressions", () => {
storePath: store.storePath,
log: noopLogger,
enqueueSystemEvent: vi.fn(),
requestHeartbeatNow: vi.fn(),
requestHeartbeat: vi.fn(),
runIsolatedAgentJob: createDefaultIsolatedRunner(),
});
@@ -124,7 +124,7 @@ describe("cron service timer regressions", () => {
log: noopLogger,
nowMs: () => now,
enqueueSystemEvent: vi.fn(),
requestHeartbeatNow: vi.fn(),
requestHeartbeat: vi.fn(),
runIsolatedAgentJob,
});
@@ -196,7 +196,7 @@ describe("cron service timer regressions", () => {
log: noopLogger,
nowMs: () => now,
enqueueSystemEvent: vi.fn(),
requestHeartbeatNow: vi.fn(),
requestHeartbeat: vi.fn(),
runIsolatedAgentJob,
});
@@ -239,7 +239,7 @@ describe("cron service timer regressions", () => {
log: noopLogger,
nowMs: () => now,
enqueueSystemEvent: vi.fn(),
requestHeartbeatNow: vi.fn(),
requestHeartbeat: vi.fn(),
runIsolatedAgentJob,
cronConfig: {
retry: { maxAttempts: 2, backoffMs: [1000, 2000] },
@@ -285,7 +285,7 @@ describe("cron service timer regressions", () => {
log: noopLogger,
nowMs: () => now,
enqueueSystemEvent: vi.fn(),
requestHeartbeatNow: vi.fn(),
requestHeartbeat: vi.fn(),
runIsolatedAgentJob,
cronConfig: {
retry: { maxAttempts: 1, backoffMs: [1000], retryOn: ["overloaded"] },
@@ -334,7 +334,7 @@ describe("cron service timer regressions", () => {
log: noopLogger,
nowMs: () => now,
enqueueSystemEvent: vi.fn(),
requestHeartbeatNow: vi.fn(),
requestHeartbeat: vi.fn(),
runIsolatedAgentJob,
cronConfig: {
retry: { maxAttempts: 1, backoffMs: [1000], retryOn: ["rate_limit"] },
@@ -380,7 +380,7 @@ describe("cron service timer regressions", () => {
log: noopLogger,
nowMs: () => now,
enqueueSystemEvent: vi.fn(),
requestHeartbeatNow: vi.fn(),
requestHeartbeat: vi.fn(),
runIsolatedAgentJob: vi.fn().mockResolvedValue({
status: "error",
error: "invalid API key",
@@ -418,7 +418,7 @@ describe("cron service timer regressions", () => {
log: noopLogger,
nowMs: () => now,
enqueueSystemEvent: vi.fn(),
requestHeartbeatNow: vi.fn(),
requestHeartbeat: vi.fn(),
runIsolatedAgentJob: vi.fn(async () => {
now += 7;
fireCount += 1;
@@ -457,7 +457,7 @@ describe("cron service timer regressions", () => {
log: noopLogger,
nowMs: () => now,
enqueueSystemEvent: vi.fn(),
requestHeartbeatNow: vi.fn(),
requestHeartbeat: vi.fn(),
runIsolatedAgentJob: vi.fn(async () => {
now += 100;
return { status: "ok" as const, summary: "done" };
@@ -493,7 +493,7 @@ describe("cron service timer regressions", () => {
log: noopLogger,
nowMs: () => now,
enqueueSystemEvent: vi.fn(),
requestHeartbeatNow: vi.fn(),
requestHeartbeat: vi.fn(),
runIsolatedAgentJob: vi.fn(async () => {
const result = await deferredRun.promise;
now += 5;
@@ -548,7 +548,7 @@ describe("cron service timer regressions", () => {
log: noopLogger,
nowMs: () => now,
enqueueSystemEvent: vi.fn(),
requestHeartbeatNow: vi.fn(),
requestHeartbeat: vi.fn(),
runIsolatedAgentJob,
});
@@ -593,7 +593,7 @@ describe("cron service timer regressions", () => {
log: noopLogger,
nowMs: () => now,
enqueueSystemEvent: vi.fn(),
requestHeartbeatNow: vi.fn(),
requestHeartbeat: vi.fn(),
runIsolatedAgentJob: vi.fn(async (params) => {
const result = await abortAwareRunner.runIsolatedAgentJob(params);
now += 5;
@@ -640,7 +640,7 @@ describe("cron service timer regressions", () => {
log: noopLogger,
nowMs: () => now,
enqueueSystemEvent: vi.fn(),
requestHeartbeatNow: vi.fn(),
requestHeartbeat: vi.fn(),
runIsolatedAgentJob: vi.fn(async ({ abortSignal, onExecutionStarted }) => {
observedAbortSignal = abortSignal;
runnerEntered.resolve();
@@ -708,7 +708,7 @@ describe("cron service timer regressions", () => {
log: noopLogger,
nowMs: () => now,
enqueueSystemEvent,
requestHeartbeatNow: vi.fn(),
requestHeartbeat: vi.fn(),
runIsolatedAgentJob: vi.fn(async (params) => {
const result = await abortAwareRunner.runIsolatedAgentJob(params);
now += 100;
@@ -755,7 +755,7 @@ describe("cron service timer regressions", () => {
log: noopLogger,
nowMs: () => now,
enqueueSystemEvent: vi.fn(),
requestHeartbeatNow: vi.fn(),
requestHeartbeat: vi.fn(),
runIsolatedAgentJob: vi.fn(async (params) => {
const result = await abortAwareRunner.runIsolatedAgentJob(params);
now += 5;
@@ -786,7 +786,7 @@ describe("cron service timer regressions", () => {
}),
);
const enqueueSystemEvent = vi.fn();
const requestHeartbeatNow = vi.fn();
const requestHeartbeat = vi.fn();
const mainJob: CronJob = {
id: "main-abort",
name: "main abort",
@@ -805,7 +805,7 @@ describe("cron service timer regressions", () => {
log: noopLogger,
nowMs: () => Date.now(),
enqueueSystemEvent,
requestHeartbeatNow,
requestHeartbeat,
runHeartbeatOnce,
wakeNowHeartbeatBusyMaxWaitMs: 30,
wakeNowHeartbeatBusyRetryDelayMs: 5,
@@ -824,7 +824,7 @@ describe("cron service timer regressions", () => {
expect(result.error).toContain("timed out");
expect(enqueueSystemEvent).toHaveBeenCalledTimes(1);
expect(runHeartbeatOnce).toHaveBeenCalled();
expect(requestHeartbeatNow).not.toHaveBeenCalled();
expect(requestHeartbeat).not.toHaveBeenCalled();
});
it("retries recurring wake-now main jobs until temporary lane pressure clears (#75964)", async () => {
@@ -838,7 +838,7 @@ describe("cron service timer regressions", () => {
.mockResolvedValueOnce({ status: "skipped", reason: HEARTBEAT_SKIP_LANES_BUSY })
.mockResolvedValueOnce({ status: "ran", durationMs: 12 });
const enqueueSystemEvent = vi.fn();
const requestHeartbeatNow = vi.fn();
const requestHeartbeat = vi.fn();
const job: CronJob = {
id: "busy-recurring-main",
name: "busy recurring main",
@@ -857,7 +857,7 @@ describe("cron service timer regressions", () => {
log: noopLogger,
nowMs,
enqueueSystemEvent,
requestHeartbeatNow,
requestHeartbeat,
runHeartbeatOnce,
wakeNowHeartbeatBusyMaxWaitMs: 120_000,
wakeNowHeartbeatBusyRetryDelayMs: 1,
@@ -871,7 +871,7 @@ describe("cron service timer regressions", () => {
expect(enqueueSystemEvent).toHaveBeenCalledTimes(1);
expect(runHeartbeatOnce).toHaveBeenCalledTimes(2);
expect(requestHeartbeatNow).not.toHaveBeenCalled();
expect(requestHeartbeat).not.toHaveBeenCalled();
expect(job.state.lastStatus).toBe("ok");
expect(job.state.runningAtMs).toBeUndefined();
});
@@ -923,7 +923,7 @@ describe("cron service timer regressions", () => {
log: noopLogger,
nowMs: () => now,
enqueueSystemEvent: vi.fn(),
requestHeartbeatNow: vi.fn(),
requestHeartbeat: vi.fn(),
onEvent: (evt) => {
events.push(evt);
},
@@ -978,7 +978,7 @@ describe("cron service timer regressions", () => {
log: noopLogger,
nowMs: () => now,
enqueueSystemEvent: vi.fn(),
requestHeartbeatNow: vi.fn(),
requestHeartbeat: vi.fn(),
runIsolatedAgentJob: vi.fn(async (params: { job: { id: string } }) => {
activeRuns += 1;
peakActiveRuns = Math.max(peakActiveRuns, activeRuns);
@@ -1044,7 +1044,7 @@ describe("cron service timer regressions", () => {
log,
nowMs: () => dueAt,
enqueueSystemEvent: vi.fn(),
requestHeartbeatNow: vi.fn(),
requestHeartbeat: vi.fn(),
onEvent: (evt) => {
events.push(evt);
},
@@ -1102,7 +1102,7 @@ describe("cron service timer regressions", () => {
log,
nowMs: () => dueAt,
enqueueSystemEvent: vi.fn(),
requestHeartbeatNow: vi.fn(),
requestHeartbeat: vi.fn(),
onEvent: (evt) => {
events.push(evt);
},
@@ -1154,7 +1154,7 @@ describe("cron service timer regressions", () => {
log: noopLogger,
nowMs: () => now,
enqueueSystemEvent: vi.fn(),
requestHeartbeatNow: vi.fn(),
requestHeartbeat: vi.fn(),
runIsolatedAgentJob: vi.fn(
async ({
abortSignal,
@@ -1236,7 +1236,7 @@ describe("cron service timer regressions", () => {
log: noopLogger,
nowMs: () => now,
enqueueSystemEvent: vi.fn(),
requestHeartbeatNow: vi.fn(),
requestHeartbeat: vi.fn(),
cleanupTimedOutAgentRun,
runIsolatedAgentJob: vi.fn(
async ({
@@ -1299,7 +1299,7 @@ describe("cron service timer regressions", () => {
log: noopLogger,
nowMs: () => endedAt,
enqueueSystemEvent: vi.fn(),
requestHeartbeatNow: vi.fn(),
requestHeartbeat: vi.fn(),
runIsolatedAgentJob: createDefaultIsolatedRunner(),
});
const job = createIsolatedRegressionJob({
@@ -1337,7 +1337,7 @@ describe("cron service timer regressions", () => {
log: noopLogger,
nowMs: () => endedAt,
enqueueSystemEvent: vi.fn(),
requestHeartbeatNow: vi.fn(),
requestHeartbeat: vi.fn(),
runIsolatedAgentJob: createDefaultIsolatedRunner(),
});
const job = createIsolatedRegressionJob({
@@ -1376,7 +1376,7 @@ describe("cron service timer regressions", () => {
log: noopLogger,
nowMs: () => endedAt,
enqueueSystemEvent: vi.fn(),
requestHeartbeatNow: vi.fn(),
requestHeartbeat: vi.fn(),
runIsolatedAgentJob: createDefaultIsolatedRunner(),
});
const job = createIsolatedRegressionJob({
@@ -1417,7 +1417,7 @@ describe("cron service timer regressions", () => {
log: noopLogger,
nowMs: () => endedAt,
enqueueSystemEvent: vi.fn(),
requestHeartbeatNow: vi.fn(),
requestHeartbeat: vi.fn(),
runIsolatedAgentJob: createDefaultIsolatedRunner(),
});
const job = createIsolatedRegressionJob({

18
src/cron/service/timer.test.ts
View File

@@ -37,7 +37,7 @@ describe("cron service timer seam coverage", () => {
const { storePath } = await makeStorePath();
const now = Date.parse("2026-03-23T12:00:00.000Z");
const enqueueSystemEvent = vi.fn();
const requestHeartbeatNow = vi.fn();
const requestHeartbeat = vi.fn();
const timeoutSpy = vi.spyOn(globalThis, "setTimeout");
await writeCronStoreSnapshot({
@@ -51,7 +51,7 @@ describe("cron service timer seam coverage", () => {
log: logger,
nowMs: () => now,
enqueueSystemEvent,
requestHeartbeatNow,
requestHeartbeat,
runIsolatedAgentJob: vi.fn(async () => ({ status: "ok" as const })),
});
@@ -62,7 +62,9 @@ describe("cron service timer seam coverage", () => {
sessionKey: "agent:main:main",
contextKey: "cron:main-heartbeat-job",
});
expect(requestHeartbeatNow).toHaveBeenCalledWith({
expect(requestHeartbeat).toHaveBeenCalledWith({
source: "cron",
intent: "event",
reason: "cron:main-heartbeat-job",
agentId: undefined,
sessionKey: "agent:main:main",
@@ -94,7 +96,7 @@ describe("cron service timer seam coverage", () => {
const { storePath } = await makeStorePath();
const now = Date.parse("2026-03-23T12:00:00.000Z");
const enqueueSystemEvent = vi.fn();
const requestHeartbeatNow = vi.fn();
const requestHeartbeat = vi.fn();
await writeCronStoreSnapshot({
storePath,
@@ -113,7 +115,7 @@ describe("cron service timer seam coverage", () => {
log: logger,
nowMs: () => now,
enqueueSystemEvent,
requestHeartbeatNow,
requestHeartbeat,
runIsolatedAgentJob: vi.fn(async () => ({ status: "ok" as const })),
});
@@ -137,7 +139,7 @@ describe("cron service timer seam coverage", () => {
const now = Date.parse("2026-03-23T06:00:00.000Z");
const staleNextRunAtMs = now;
const enqueueSystemEvent = vi.fn();
const requestHeartbeatNow = vi.fn();
const requestHeartbeat = vi.fn();
await saveCronStore(storePath, {
version: 1,
@@ -169,14 +171,14 @@ describe("cron service timer seam coverage", () => {
log: logger,
nowMs: () => now,
enqueueSystemEvent,
requestHeartbeatNow,
requestHeartbeat,
runIsolatedAgentJob: vi.fn(async () => ({ status: "ok" as const })),
});
await onTimer(state);
expect(enqueueSystemEvent).not.toHaveBeenCalled();
expect(requestHeartbeatNow).not.toHaveBeenCalled();
expect(requestHeartbeat).not.toHaveBeenCalled();
const persisted = await loadCronStore(storePath);
const job = persisted.jobs[0];

22
src/cron/service/timer.ts
View File

@@ -456,7 +456,11 @@ function emitFailureAlert(
state.deps.enqueueSystemEvent(text, { agentId: params.job.agentId });
if (params.job.wakeMode === "now") {
state.deps.requestHeartbeatNow({ reason: `cron:${params.job.id}:failure-alert` });
state.deps.requestHeartbeat({
source: "cron",
intent: "immediate",
reason: `cron:${params.job.id}:failure-alert`,
});
}
}
@@ -1384,6 +1388,8 @@ async function executeMainSessionCronJob(
return { status: "error", error: timeoutErrorMessage() };
}
heartbeatResult = await state.deps.runHeartbeatOnce({
source: "cron",
intent: "immediate",
reason,
agentId: job.agentId,
sessionKey: targetMainSessionKey,
@@ -1397,7 +1403,9 @@ async function executeMainSessionCronJob(
}
if (heartbeatResult.reason === HEARTBEAT_SKIP_CRON_IN_PROGRESS) {
// The active cron marker blocks direct wake-now until this job returns.
state.deps.requestHeartbeatNow({
state.deps.requestHeartbeat({
source: "cron",
intent: "immediate",
reason,
agentId: job.agentId,
sessionKey: targetMainSessionKey,
@@ -1412,7 +1420,9 @@ async function executeMainSessionCronJob(
if (abortSignal?.aborted) {
return { status: "error", error: timeoutErrorMessage() };
}
state.deps.requestHeartbeatNow({
state.deps.requestHeartbeat({
source: "cron",
intent: "immediate",
reason,
agentId: job.agentId,
sessionKey: targetMainSessionKey,
@@ -1435,7 +1445,9 @@ async function executeMainSessionCronJob(
if (abortSignal?.aborted) {
return { status: "error", error: timeoutErrorMessage() };
}
state.deps.requestHeartbeatNow({
state.deps.requestHeartbeat({
source: "cron",
intent: job.wakeMode === "now" ? "immediate" : "event",
reason: `cron:${job.id}`,
agentId: job.agentId,
sessionKey: targetMainSessionKey,
@@ -1585,7 +1597,7 @@ export function wake(
}
state.deps.enqueueSystemEvent(text);
if (opts.mode === "now") {
state.deps.requestHeartbeatNow({ reason: "wake" });
state.deps.requestHeartbeat({ source: "manual", intent: "immediate", reason: "wake" });
}
return { ok: true } as const;
}

26
src/gateway/server-cron.test.ts
View File

@@ -8,7 +8,7 @@ import { mergeMockedModule } from "../test-utils/vitest-module-mocks.js";
const {
enqueueSystemEventMock,
requestHeartbeatNowMock,
requestHeartbeatMock,
runHeartbeatOnceMock,
loadConfigMock,
fetchWithSsrFGuardMock,
@@ -18,7 +18,7 @@ const {
runCronChangedMock,
} = vi.hoisted(() => ({
enqueueSystemEventMock: vi.fn(),
requestHeartbeatNowMock: vi.fn(),
requestHeartbeatMock: vi.fn(),
runHeartbeatOnceMock: vi.fn<
(...args: unknown[]) => Promise<{ status: "ran"; durationMs: number }>
>(async () => ({ status: "ran", durationMs: 1 })),
@@ -37,8 +37,8 @@ function enqueueSystemEvent(...args: unknown[]) {
return enqueueSystemEventMock(...args);
}
function requestHeartbeatNow(...args: unknown[]) {
return requestHeartbeatNowMock(...args);
function requestHeartbeat(...args: unknown[]) {
return requestHeartbeatMock(...args);
}
function runHeartbeatOnce(...args: unknown[]) {
@@ -55,7 +55,7 @@ vi.mock("../infra/heartbeat-wake.js", async () => {
"../infra/heartbeat-wake.js",
),
() => ({
requestHeartbeatNow,
requestHeartbeat,
}),
);
});
@@ -113,7 +113,7 @@ function createCronConfig(name: string): OpenClawConfig {
describe("buildGatewayCronService", () => {
beforeEach(() => {
enqueueSystemEventMock.mockClear();
requestHeartbeatNowMock.mockClear();
requestHeartbeatMock.mockClear();
runHeartbeatOnceMock.mockClear();
loadConfigMock.mockClear();
fetchWithSsrFGuardMock.mockClear();
@@ -264,7 +264,7 @@ describe("buildGatewayCronService", () => {
sessionKey: "agent:main:discord:channel:ops",
}),
);
expect(requestHeartbeatNowMock).toHaveBeenCalledWith(
expect(requestHeartbeatMock).toHaveBeenCalledWith(
expect.objectContaining({
sessionKey: "agent:main:discord:channel:ops",
}),
@@ -288,10 +288,12 @@ describe("buildGatewayCronService", () => {
state.cron as unknown as {
state?: {
deps?: {
requestHeartbeatNow?: (opts?: {
requestHeartbeat?: (opts?: {
agentId?: string;
sessionKey?: string | null;
reason?: string;
source?: string;
intent?: string;
heartbeat?: { target?: string };
}) => void;
};
@@ -299,13 +301,17 @@ describe("buildGatewayCronService", () => {
}
).state?.deps;
cronDeps?.requestHeartbeatNow?.({
cronDeps?.requestHeartbeat?.({
source: "cron",
intent: "event",
reason: "cron:test",
sessionKey: "discord:channel:ops",
heartbeat: { target: "last" },
});
expect(requestHeartbeatNowMock).toHaveBeenCalledWith({
expect(requestHeartbeatMock).toHaveBeenCalledWith({
source: "cron",
intent: "event",
reason: "cron:test",
agentId: "main",
sessionKey: "agent:main:discord:channel:ops",

10
src/gateway/server-cron.ts
View File

@@ -22,7 +22,7 @@ import { resolveCronStorePath } from "../cron/store.js";
import type { CronJob } from "../cron/types.js";
import { formatErrorMessage } from "../infra/errors.js";
import { runHeartbeatOnce } from "../infra/heartbeat-runner.js";
import { requestHeartbeatNow } from "../infra/heartbeat-wake.js";
import { requestHeartbeat } from "../infra/heartbeat-wake.js";
import { enqueueSystemEvent } from "../infra/system-events.js";
import { getChildLogger } from "../logging.js";
import { getGlobalHookRunner } from "../plugins/hook-runner-global.js";
@@ -247,9 +247,11 @@ export function buildGatewayCronService(params: {
trusted: opts?.trusted,
});
},
requestHeartbeatNow: (opts) => {
requestHeartbeat: (opts) => {
const { agentId, sessionKey } = resolveCronWakeTarget(opts);
requestHeartbeatNow({
requestHeartbeat({
source: opts?.source ?? "cron",
intent: opts?.intent ?? "event",
reason: opts?.reason,
agentId,
sessionKey,
@@ -279,6 +281,8 @@ export function buildGatewayCronService(params: {
: undefined;
return await runHeartbeatOnce({
cfg: runtimeConfig,
source: opts?.source ?? "cron",
intent: opts?.intent ?? "event",
reason: opts?.reason,
agentId,
sessionKey,

2
src/gateway/server-node-events.runtime.ts
View File

@@ -6,7 +6,7 @@ export { agentCommandFromIngress } from "../commands/agent.js";
export { getRuntimeConfig } from "../config/io.js";
export { updateSessionStore } from "../config/sessions.js";
export { loadOrCreateDeviceIdentity } from "../infra/device-identity.js";
export { requestHeartbeatNow } from "../infra/heartbeat-wake.js";
export { requestHeartbeat } from "../infra/heartbeat-wake.js";
export { deliverOutboundPayloads } from "../infra/outbound/deliver.js";
export { buildOutboundSessionContext } from "../infra/outbound/session-context.js";
export { resolveOutboundTarget } from "../infra/outbound/targets.js";

50
src/gateway/server-node-events.test.ts
View File

@@ -91,7 +91,7 @@ const runtimeMocks = vi.hoisted(() => ({
normalizeRpcAttachmentsToChatAttachments: vi.fn((attachments?: unknown[]) => attachments ?? []),
parseMessageWithAttachments: parseMessageWithAttachmentsMock,
registerApnsRegistration: registerApnsRegistrationMock,
requestHeartbeatNow: vi.fn(),
requestHeartbeat: vi.fn(),
resolveChatAttachmentMaxBytes: vi.fn(() => 20 * 1024 * 1024),
resolveGatewayModelSupportsImages: vi.fn(
async ({
@@ -151,7 +151,7 @@ import {
} from "./server-node-events.js";
const enqueueSystemEventMock = runtimeMocks.enqueueSystemEvent;
const requestHeartbeatNowMock = runtimeMocks.requestHeartbeatNow;
const requestHeartbeatMock = runtimeMocks.requestHeartbeat;
const loadConfigMock = runtimeMocks.getRuntimeConfig;
const agentCommandMock = runtimeMocks.agentCommandFromIngress;
const updateSessionStoreMock = runtimeMocks.updateSessionStore;
@@ -187,7 +187,7 @@ describe("node exec events", () => {
resetNodeEventDeduplicationForTests();
enqueueSystemEventMock.mockClear();
enqueueSystemEventMock.mockReturnValue(true);
requestHeartbeatNowMock.mockClear();
requestHeartbeatMock.mockClear();
registerApnsRegistrationVi.mockClear();
loadOrCreateDeviceIdentityMock.mockClear();
normalizeChannelIdVi.mockClear();
@@ -214,7 +214,7 @@ describe("node exec events", () => {
"Exec started (node=node-1 id=run-1): ls -la",
{ sessionKey: "agent:main:main", contextKey: "exec:run-1", trusted: false },
);
expect(requestHeartbeatNowMock).toHaveBeenCalledWith({
expect(requestHeartbeatMock).toHaveBeenCalledWith({
reason: "exec-event",
sessionKey: "agent:main:main",
});
@@ -236,7 +236,7 @@ describe("node exec events", () => {
"Exec finished (node=node-2 id=run-2, code 0)\ndone",
{ sessionKey: "node-node-2", contextKey: "exec:run-2", trusted: false },
);
expect(requestHeartbeatNowMock).toHaveBeenCalledWith({ reason: "exec-event" });
expect(requestHeartbeatMock).toHaveBeenCalledWith({ reason: "exec-event" });
});
it("dedupes duplicate exec.finished events for the same runId on the same session", async () => {
@@ -259,7 +259,7 @@ describe("node exec events", () => {
});
expect(enqueueSystemEventMock).toHaveBeenCalledTimes(1);
expect(requestHeartbeatNowMock).toHaveBeenCalledTimes(1);
expect(requestHeartbeatMock).toHaveBeenCalledTimes(1);
expect(enqueueSystemEventMock).toHaveBeenCalledWith(
"Exec finished (node=node-2 id=run-dup-finished, code 0)\ndone",
{
@@ -291,7 +291,7 @@ describe("node exec events", () => {
"Exec finished (node=node-2 id=run-2, code 0)\ndone",
{ sessionKey: "agent:main:node-node-2", contextKey: "exec:run-2", trusted: false },
);
expect(requestHeartbeatNowMock).toHaveBeenCalledWith({
expect(requestHeartbeatMock).toHaveBeenCalledWith({
reason: "exec-event",
sessionKey: "agent:main:node-node-2",
});
@@ -310,7 +310,7 @@ describe("node exec events", () => {
});
expect(enqueueSystemEventMock).not.toHaveBeenCalled();
expect(requestHeartbeatNowMock).not.toHaveBeenCalled();
expect(requestHeartbeatMock).not.toHaveBeenCalled();
});
it("truncates long exec.finished output in system events", async () => {
@@ -330,7 +330,7 @@ describe("node exec events", () => {
expect(text.startsWith("Exec finished (node=node-2 id=run-long, code 0)\n")).toBe(true);
expect(text.endsWith("…")).toBe(true);
expect(text.length).toBeLessThan(280);
expect(requestHeartbeatNowMock).toHaveBeenCalledWith({ reason: "exec-event" });
expect(requestHeartbeatMock).toHaveBeenCalledWith({ reason: "exec-event" });
});
it("enqueues exec.denied events with reason", async () => {
@@ -349,7 +349,7 @@ describe("node exec events", () => {
"Exec denied (node=node-3 id=run-3, allowlist-miss): rm -rf /",
{ sessionKey: "agent:demo:main", contextKey: "exec:run-3", trusted: false },
);
expect(requestHeartbeatNowMock).toHaveBeenCalledWith({
expect(requestHeartbeatMock).toHaveBeenCalledWith({
reason: "exec-event",
sessionKey: "agent:demo:main",
});
@@ -374,7 +374,7 @@ describe("node exec events", () => {
});
expect(enqueueSystemEventMock).not.toHaveBeenCalled();
expect(requestHeartbeatNowMock).not.toHaveBeenCalled();
expect(requestHeartbeatMock).not.toHaveBeenCalled();
});
it("suppresses exec.finished when notifyOnExit is false", async () => {
@@ -397,7 +397,7 @@ describe("node exec events", () => {
});
expect(enqueueSystemEventMock).not.toHaveBeenCalled();
expect(requestHeartbeatNowMock).not.toHaveBeenCalled();
expect(requestHeartbeatMock).not.toHaveBeenCalled();
});
it("suppresses exec.denied when notifyOnExit is false", async () => {
@@ -420,7 +420,7 @@ describe("node exec events", () => {
});
expect(enqueueSystemEventMock).not.toHaveBeenCalled();
expect(requestHeartbeatNowMock).not.toHaveBeenCalled();
expect(requestHeartbeatMock).not.toHaveBeenCalled();
});
it("sanitizes remote exec event content before enqueue", async () => {
@@ -687,7 +687,7 @@ describe("voice transcript events", () => {
describe("notifications changed events", () => {
beforeEach(() => {
enqueueSystemEventMock.mockClear();
requestHeartbeatNowMock.mockClear();
requestHeartbeatMock.mockClear();
loadSessionEntryMock.mockClear();
normalizeChannelIdVi.mockClear();
normalizeChannelIdVi.mockImplementation((channel?: string | null) => channel ?? null);
@@ -712,7 +712,9 @@ describe("notifications changed events", () => {
"Notification posted (node=node-n1 key=notif-1 package=com.example.chat): Message - Ping from Alex",
{ sessionKey: "node-node-n1", contextKey: "notification:notif-1", trusted: false },
);
expect(requestHeartbeatNowMock).toHaveBeenCalledWith({
expect(requestHeartbeatMock).toHaveBeenCalledWith({
source: "notifications-event",
intent: "event",
reason: "notifications-event",
sessionKey: "node-node-n1",
});
@@ -733,7 +735,9 @@ describe("notifications changed events", () => {
"Notification removed (node=node-n2 key=notif-2 package=com.example.mail)",
{ sessionKey: "node-node-n2", contextKey: "notification:notif-2", trusted: false },
);
expect(requestHeartbeatNowMock).toHaveBeenCalledWith({
expect(requestHeartbeatMock).toHaveBeenCalledWith({
source: "notifications-event",
intent: "event",
reason: "notifications-event",
sessionKey: "node-node-n2",
});
@@ -750,7 +754,9 @@ describe("notifications changed events", () => {
}),
});
expect(requestHeartbeatNowMock).toHaveBeenCalledWith({
expect(requestHeartbeatMock).toHaveBeenCalledWith({
source: "notifications-event",
intent: "event",
reason: "notifications-event",
sessionKey: "agent:main:main",
});
@@ -779,7 +785,9 @@ describe("notifications changed events", () => {
trusted: false,
},
);
expect(requestHeartbeatNowMock).toHaveBeenCalledWith({
expect(requestHeartbeatMock).toHaveBeenCalledWith({
source: "notifications-event",
intent: "event",
reason: "notifications-event",
sessionKey: "agent:main:node-node-n5",
});
@@ -795,7 +803,7 @@ describe("notifications changed events", () => {
});
expect(enqueueSystemEventMock).not.toHaveBeenCalled();
expect(requestHeartbeatNowMock).not.toHaveBeenCalled();
expect(requestHeartbeatMock).not.toHaveBeenCalled();
});
it("sanitizes notification text before enqueueing an untrusted system event", async () => {
@@ -838,7 +846,7 @@ describe("notifications changed events", () => {
});
expect(enqueueSystemEventMock).toHaveBeenCalledTimes(2);
expect(requestHeartbeatNowMock).toHaveBeenCalledTimes(1);
expect(requestHeartbeatMock).toHaveBeenCalledTimes(1);
});
it("suppresses exec notifyOnExit events when payload opts out", async () => {
@@ -855,7 +863,7 @@ describe("notifications changed events", () => {
});
expect(enqueueSystemEventMock).not.toHaveBeenCalled();
expect(requestHeartbeatNowMock).not.toHaveBeenCalled();
expect(requestHeartbeatMock).not.toHaveBeenCalled();
});
});

18
src/gateway/server-node-events.ts
View File

@@ -31,7 +31,7 @@ import {
normalizeRpcAttachmentsToChatAttachments,
parseMessageWithAttachments,
registerApnsRegistration,
requestHeartbeatNow,
requestHeartbeat,
resolveChatAttachmentMaxBytes,
resolveGatewayModelSupportsImages,
resolveOutboundTarget,
@@ -646,7 +646,12 @@ export const handleNodeEvent = async (
trusted: false,
});
if (queued) {
requestHeartbeatNow({ reason: "notifications-event", sessionKey });
requestHeartbeat({
source: "notifications-event",
intent: "event",
reason: "notifications-event",
sessionKey,
});
}
return undefined;
}
@@ -749,8 +754,13 @@ export const handleNodeEvent = async (
// Scope wakes only for canonical agent sessions. Synthetic node-* fallback
// keys should keep legacy unscoped behavior so enabled non-main heartbeat
// agents still run when no explicit agent session is provided.
requestHeartbeatNow(
scopedHeartbeatWakeOptions(sessionKey, { reason: "exec-event", coalesceMs: 0 }),
requestHeartbeat(
scopedHeartbeatWakeOptions(sessionKey, {
source: "exec-event",
intent: "event",
reason: "exec-event",
coalesceMs: 0,
}),
);
}
return undefined;

28
src/gateway/server-restart-sentinel.test.ts
View File

@@ -73,7 +73,7 @@ const mocks = vi.hoisted(() => {
ackDelivery: vi.fn(async () => {}),
failDelivery: vi.fn(async () => {}),
enqueueSystemEvent: vi.fn(),
requestHeartbeatNow: vi.fn(),
requestHeartbeat: vi.fn(),
enqueueSessionDelivery: vi.fn(async (payload: Record<string, unknown>) => {
state.queuedSessionDelivery = payload;
return "session-delivery-1";
@@ -204,7 +204,7 @@ vi.mock("../infra/heartbeat-wake.js", async () => {
);
return {
...actual,
requestHeartbeatNow: mocks.requestHeartbeatNow,
requestHeartbeat: mocks.requestHeartbeat,
};
});
@@ -274,7 +274,7 @@ describe("scheduleRestartSentinelWake", () => {
mocks.ackDelivery.mockClear();
mocks.failDelivery.mockClear();
mocks.enqueueSystemEvent.mockClear();
mocks.requestHeartbeatNow.mockClear();
mocks.requestHeartbeat.mockClear();
mocks.enqueueSessionDelivery.mockClear();
mocks.drainPendingSessionDeliveries.mockClear();
mocks.recoverPendingSessionDeliveries.mockClear();
@@ -319,7 +319,9 @@ describe("scheduleRestartSentinelWake", () => {
sessionKey: "agent:main:main",
}),
);
expect(mocks.requestHeartbeatNow).toHaveBeenCalledWith({
expect(mocks.requestHeartbeat).toHaveBeenCalledWith({
source: "restart-sentinel",
intent: "immediate",
reason: "wake",
sessionKey: "agent:main:main",
});
@@ -356,7 +358,7 @@ describe("scheduleRestartSentinelWake", () => {
expect(mocks.ackDelivery).toHaveBeenCalledWith("queue-1");
expect(mocks.failDelivery).not.toHaveBeenCalled();
expect(mocks.enqueueSystemEvent).toHaveBeenCalledTimes(1);
expect(mocks.requestHeartbeatNow).toHaveBeenCalledTimes(1);
expect(mocks.requestHeartbeat).toHaveBeenCalledTimes(1);
expect(mocks.logWarn).toHaveBeenCalledWith(
expect.stringContaining("retrying in 1000ms"),
expect.objectContaining({
@@ -759,11 +761,15 @@ describe("scheduleRestartSentinelWake", () => {
}),
}),
);
expect(mocks.requestHeartbeatNow).toHaveBeenNthCalledWith(1, {
expect(mocks.requestHeartbeat).toHaveBeenNthCalledWith(1, {
source: "restart-sentinel",
intent: "immediate",
reason: "wake",
sessionKey: "agent:main:main",
});
expect(mocks.requestHeartbeatNow).toHaveBeenNthCalledWith(2, {
expect(mocks.requestHeartbeat).toHaveBeenNthCalledWith(2, {
source: "restart-sentinel",
intent: "immediate",
reason: "wake",
sessionKey: "agent:main:main",
});
@@ -899,7 +905,7 @@ describe("scheduleRestartSentinelWake", () => {
sessionKey: "agent:main:main",
}),
);
expect(mocks.requestHeartbeatNow).toHaveBeenCalledTimes(2);
expect(mocks.requestHeartbeat).toHaveBeenCalledTimes(2);
expect(mocks.logWarn).not.toHaveBeenCalled();
});
@@ -974,7 +980,7 @@ describe("scheduleRestartSentinelWake", () => {
expect(mocks.enqueueSystemEvent).toHaveBeenCalledWith("restart message", {
sessionKey: "agent:main:main",
});
expect(mocks.requestHeartbeatNow).not.toHaveBeenCalled();
expect(mocks.requestHeartbeat).not.toHaveBeenCalled();
expect(mocks.deliverOutboundPayloads).not.toHaveBeenCalled();
});
@@ -1042,7 +1048,7 @@ describe("scheduleRestartSentinelWake", () => {
channel: "qa-channel",
to: "channel:qa-room",
});
mocks.requestHeartbeatNow.mockImplementation(() => {
mocks.requestHeartbeat.mockImplementation(() => {
mocks.deliveryContextFromSession.mockReturnValue({
channel: "qa-channel",
to: "heartbeat",
@@ -1055,7 +1061,7 @@ describe("scheduleRestartSentinelWake", () => {
await scheduleRestartSentinelWake({ deps: {} as never });
expect(mocks.requestHeartbeatNow).toHaveBeenCalledTimes(1);
expect(mocks.requestHeartbeat).toHaveBeenCalledTimes(1);
expect(mocks.resolveOutboundTarget).toHaveBeenCalledWith(
expect.objectContaining({
channel: "qa-channel",

18
src/gateway/server-restart-sentinel.ts
View File

@@ -8,7 +8,7 @@ import type { CliDeps } from "../cli/deps.types.js";
import { resolveMainSessionKeyFromConfig } from "../config/sessions.js";
import { parseSessionThreadInfo } from "../config/sessions/thread-info.js";
import { formatErrorMessage } from "../infra/errors.js";
import { requestHeartbeatNow } from "../infra/heartbeat-wake.js";
import { requestHeartbeat } from "../infra/heartbeat-wake.js";
import { deliverOutboundPayloads } from "../infra/outbound/deliver.js";
import { ackDelivery, enqueueDelivery, failDelivery } from "../infra/outbound/delivery-queue.js";
import { buildOutboundSessionContext } from "../infra/outbound/session-context.js";
@@ -80,7 +80,7 @@ function enqueueRestartSentinelWake(
sessionKey,
...(deliveryContext ? { deliveryContext } : {}),
});
requestHeartbeatNow({ reason: "wake", sessionKey });
requestHeartbeat({ source: "restart-sentinel", intent: "immediate", reason: "wake", sessionKey });
}
async function waitForOutboundRetry(delayMs: number) {
@@ -235,7 +235,12 @@ async function deliverQueuedSessionDelivery(params: {
}
: {}),
});
requestHeartbeatNow({ reason: "wake", sessionKey: canonicalKey });
requestHeartbeat({
source: "restart-sentinel",
intent: "immediate",
reason: "wake",
sessionKey: canonicalKey,
});
return;
}
@@ -250,7 +255,12 @@ async function deliverQueuedSessionDelivery(params: {
}
: {}),
});
requestHeartbeatNow({ reason: "wake", sessionKey: canonicalKey });
requestHeartbeat({
source: "restart-sentinel",
intent: "immediate",
reason: "wake",
sessionKey: canonicalKey,
});
return;
}

8
src/gateway/server/hooks.agent-trust.test.ts
View File

@@ -1,7 +1,7 @@
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
const enqueueSystemEventMock = vi.fn();
const requestHeartbeatNowMock = vi.fn();
const requestHeartbeatMock = vi.fn();
const runCronIsolatedAgentTurnMock = vi.fn();
const resolveMainSessionKeyMock = vi.fn(() => "main-session");
const loadConfigMock = vi.fn(() => ({}));
@@ -12,7 +12,7 @@ vi.mock("../../infra/system-events.js", () => ({
enqueueSystemEvent: enqueueSystemEventMock,
}));
vi.mock("../../infra/heartbeat-wake.js", () => ({
requestHeartbeatNow: requestHeartbeatNowMock,
requestHeartbeat: requestHeartbeatMock,
}));
vi.mock("../../cron/isolated-agent.js", () => ({
runCronIsolatedAgentTurn: runCronIsolatedAgentTurnMock,
@@ -101,7 +101,7 @@ describe("dispatchAgentHook trust handling", () => {
await vi.waitFor(() => expect(runCronIsolatedAgentTurnMock).toHaveBeenCalledTimes(1));
expect(enqueueSystemEventMock).not.toHaveBeenCalled();
expect(requestHeartbeatNowMock).not.toHaveBeenCalled();
expect(requestHeartbeatMock).not.toHaveBeenCalled();
expect(logHooksInfoMock).toHaveBeenCalledWith(
"hook agent run completed without announcement",
expect.objectContaining({
@@ -191,7 +191,7 @@ describe("dispatchAgentHook trust handling", () => {
await vi.waitFor(() => expect(runCronIsolatedAgentTurnMock).toHaveBeenCalledTimes(1));
expect(enqueueSystemEventMock).not.toHaveBeenCalled();
expect(requestHeartbeatNowMock).not.toHaveBeenCalled();
expect(requestHeartbeatMock).not.toHaveBeenCalled();
});
it("marks error events as untrusted and sanitizes hook names", async () => {

12
src/gateway/server/hooks.ts
View File

@@ -10,7 +10,7 @@ import {
import type { OpenClawConfig } from "../../config/types.openclaw.js";
import type { RunCronAgentTurnResult } from "../../cron/isolated-agent/run.types.js";
import type { CronJob } from "../../cron/types.js";
import { requestHeartbeatNow } from "../../infra/heartbeat-wake.js";
import { requestHeartbeat } from "../../infra/heartbeat-wake.js";
import { enqueueSystemEvent } from "../../infra/system-events.js";
import type { createSubsystemLogger } from "../../logging/subsystem.js";
import { normalizeOptionalString } from "../../shared/string-coerce.js";
@@ -51,7 +51,7 @@ export function createGatewayHooksRequestHandler(params: {
const sessionKey = resolveMainSessionKeyFromConfig();
enqueueSystemEvent(value.text, { sessionKey, trusted: false });
if (value.mode === "now") {
requestHeartbeatNow({ reason: "hook:wake" });
requestHeartbeat({ source: "hook", intent: "immediate", reason: "hook:wake" });
}
};
@@ -122,7 +122,7 @@ export function createGatewayHooksRequestHandler(params: {
trusted: false,
});
if (value.wakeMode === "now") {
requestHeartbeatNow({ reason: `hook:${jobId}` });
requestHeartbeat({ source: "hook", intent: "immediate", reason: `hook:${jobId}` });
}
} else if (result.status === "ok" && !value.deliver) {
logHooks.info("hook agent run completed without announcement", {
@@ -142,7 +142,11 @@ export function createGatewayHooksRequestHandler(params: {
trusted: false,
});
if (value.wakeMode === "now") {
requestHeartbeatNow({ reason: `hook:${jobId}:error` });
requestHeartbeat({
source: "hook",
intent: "immediate",
reason: `hook:${jobId}:error`,
});
}
}
})();

400
src/infra/heartbeat-cooldown.test.ts Normal file
View File

@@ -0,0 +1,400 @@
import { describe, expect, it } from "vitest";
import {
DEFAULT_FLOOD_THRESHOLD,
DEFAULT_MIN_WAKE_SPACING_MS,
recordRunStart,
shouldDeferWake,
} from "./heartbeat-cooldown.js";
describe("shouldDeferWake", () => {
type Input = Parameters<typeof shouldDeferWake>[0];
function decide(input: Omit<Input, "intent"> & { intent?: Input["intent"] }) {
return shouldDeferWake({ intent: "event", ...input });
}
// After-a-run baseline: agent has already run once, so the cooldown gate is
// active for non-manual non-interval wakes.
const afterRun = {
nextDueMs: 100_000,
now: 50_000,
lastRunStartedAtMs: 49_000,
};
// Bootstrap baseline: agent has never run. nextDueMs is the first phase tick.
const beforeFirstRun = {
nextDueMs: 100_000,
now: 50_000,
lastRunStartedAtMs: undefined,
};
describe("manual wakes", () => {
it("never defers manual wakes even within nextDueMs", () => {
expect(decide({ ...afterRun, intent: "manual", reason: "manual" })).toEqual({
defer: false,
});
});
it("never defers manual wakes even within min-spacing window", () => {
expect(
decide({
intent: "manual",
now: 200_000,
nextDueMs: 100_000,
lastRunStartedAtMs: 199_900,
reason: "manual",
}),
).toEqual({ defer: false });
});
it("never defers manual wakes even during a flood", () => {
const now = 1_000_000;
const recentRunStarts = [
now - 50_000,
now - 40_000,
now - 30_000,
now - 20_000,
now - 10_000,
];
expect(
decide({
intent: "manual",
now,
nextDueMs: 0,
lastRunStartedAtMs: now - 10_000,
recentRunStarts,
reason: "manual",
}),
).toEqual({ defer: false });
});
});
describe("immediate wake intent (wake-now contracts)", () => {
it("does not defer 'wake' even within nextDueMs (system event --mode now contract)", () => {
expect(decide({ ...afterRun, intent: "immediate", reason: "wake" })).toEqual({
defer: false,
});
});
it("does not defer 'background-task' even within nextDueMs (task completion contract)", () => {
expect(decide({ ...afterRun, intent: "immediate", reason: "background-task" })).toEqual({
defer: false,
});
});
it("does not defer 'background-task-blocked' even within nextDueMs", () => {
expect(
decide({ ...afterRun, intent: "immediate", reason: "background-task-blocked" }),
).toEqual({ defer: false });
});
it("does not defer explicit hook wake-now calls even within nextDueMs", () => {
expect(decide({ ...afterRun, intent: "immediate", reason: "hook:wake" })).toEqual({
defer: false,
});
});
it("does not defer explicit cron wake-now calls even within nextDueMs", () => {
expect(decide({ ...afterRun, intent: "immediate", reason: "cron:morning-brief" })).toEqual({
defer: false,
});
});
it("does not defer 'wake' within min-spacing window", () => {
expect(
decide({
intent: "immediate",
now: 200_000,
nextDueMs: 100_000,
lastRunStartedAtMs: 199_990,
reason: "wake",
}),
).toEqual({ defer: false });
});
it("defers acp spawn stream wakes when they use event intent", () => {
expect(decide({ ...afterRun, source: "acp-spawn", reason: "acp:spawn:stream" })).toEqual({
defer: true,
reason: "not-due",
});
});
it("flood guard still applies to 'wake' as a backstop against unexpected loops", () => {
const now = 1_000_000;
const recentRunStarts = [
now - 50_000,
now - 40_000,
now - 30_000,
now - 20_000,
now - 10_000,
];
expect(
decide({
intent: "immediate",
now,
nextDueMs: 0,
lastRunStartedAtMs: now - 10_000,
recentRunStarts,
reason: "wake",
}),
).toEqual({ defer: true, reason: "flood" });
});
it("flood guard still applies to 'background-task' as a backstop", () => {
const now = 1_000_000;
const recentRunStarts = [
now - 50_000,
now - 40_000,
now - 30_000,
now - 20_000,
now - 10_000,
];
expect(
decide({
intent: "immediate",
now,
nextDueMs: 0,
lastRunStartedAtMs: now - 10_000,
recentRunStarts,
reason: "background-task",
}),
).toEqual({ defer: true, reason: "flood" });
});
it("flood guard still applies to explicit wake-now bypass calls", () => {
const now = 1_000_000;
const recentRunStarts = [
now - 50_000,
now - 40_000,
now - 30_000,
now - 20_000,
now - 10_000,
];
expect(
decide({
intent: "immediate",
now,
nextDueMs: 0,
lastRunStartedAtMs: now - 10_000,
recentRunStarts,
reason: "hook:wake",
}),
).toEqual({ defer: true, reason: "flood" });
});
});
describe("scheduled intent", () => {
it("defers with 'not-due' when now < nextDueMs (interval cooldown)", () => {
expect(decide({ ...afterRun, intent: "scheduled", reason: "interval" })).toEqual({
defer: true,
reason: "not-due",
});
});
it("defers interval wake before first run if nextDueMs is in future", () => {
expect(decide({ ...beforeFirstRun, intent: "scheduled", reason: "interval" })).toEqual({
defer: true,
reason: "not-due",
});
});
it("does not defer interval wake when now >= nextDueMs", () => {
expect(
decide({
intent: "scheduled",
now: 100_001,
nextDueMs: 100_000,
lastRunStartedAtMs: 70_000,
reason: "interval",
}),
).toEqual({ defer: false });
});
});
describe("event-driven wakes after a prior run (regression for #75436)", () => {
it("defers exec-event wakes when now < nextDueMs", () => {
expect(decide({ ...afterRun, source: "exec-event", reason: "exec-event" })).toEqual({
defer: true,
reason: "not-due",
});
});
it("defers cron wakes when now < nextDueMs", () => {
expect(decide({ ...afterRun, source: "cron", reason: "cron:morning-brief" })).toEqual({
defer: true,
reason: "not-due",
});
});
it("defers hook wakes when now < nextDueMs", () => {
expect(decide({ ...afterRun, source: "hook", reason: "hook:wake" })).toEqual({
defer: true,
reason: "not-due",
});
});
it("defers acp spawn stream wakes when now < nextDueMs", () => {
expect(decide({ ...afterRun, source: "acp-spawn", reason: "acp:spawn:stream" })).toEqual({
defer: true,
reason: "not-due",
});
});
it("defers unknown wake reasons when now < nextDueMs", () => {
expect(decide({ ...afterRun, source: "other", reason: "something-new" })).toEqual({
defer: true,
reason: "not-due",
});
});
});
describe("event-driven wakes before any prior run (bootstrap)", () => {
it("does NOT defer the first exec-event wake (lets idle agent respond)", () => {
expect(decide({ ...beforeFirstRun, source: "exec-event", reason: "exec-event" })).toEqual({
defer: false,
});
});
it("does NOT defer the first cron wake", () => {
expect(decide({ ...beforeFirstRun, source: "cron", reason: "cron:job-x" })).toEqual({
defer: false,
});
});
it("does NOT defer the first hook wake", () => {
expect(decide({ ...beforeFirstRun, source: "hook", reason: "hook:wake" })).toEqual({
defer: false,
});
});
});
describe("min-spacing floor", () => {
it("defers with 'min-spacing' when last run started within floor (post-cooldown race)", () => {
// nextDueMs has just been crossed, but a run started ~10s ago — second
// wake landed before the schedule advanced.
expect(
decide({
source: "exec-event",
now: 200_000,
nextDueMs: 199_999,
lastRunStartedAtMs: 200_000 - DEFAULT_MIN_WAKE_SPACING_MS + 100,
reason: "exec-event",
}),
).toEqual({ defer: true, reason: "min-spacing" });
});
it("does not defer when last run is older than min-spacing", () => {
expect(
decide({
source: "exec-event",
now: 200_000,
nextDueMs: 199_999,
lastRunStartedAtMs: 200_000 - DEFAULT_MIN_WAKE_SPACING_MS - 1,
reason: "exec-event",
}),
).toEqual({ defer: false });
});
it("respects override of minSpacingMs", () => {
expect(
decide({
source: "exec-event",
now: 200_000,
nextDueMs: 199_999,
lastRunStartedAtMs: 199_500, // 500ms ago
minSpacingMs: 1_000,
reason: "exec-event",
}),
).toEqual({ defer: true, reason: "min-spacing" });
});
it("does not gate manual wakes on min-spacing", () => {
expect(
decide({
intent: "manual",
now: 200_000,
nextDueMs: 100_000,
lastRunStartedAtMs: 199_999,
reason: "manual",
}),
).toEqual({ defer: false });
});
});
describe("flood guard", () => {
it("defers with 'flood' when threshold runs land within window", () => {
const now = 1_000_000;
const recentRunStarts = [
now - 50_000,
now - 40_000,
now - 30_000,
now - 20_000,
now - 10_000,
];
expect(
decide({
source: "exec-event",
now,
nextDueMs: 0,
lastRunStartedAtMs: now - DEFAULT_MIN_WAKE_SPACING_MS - 1,
recentRunStarts,
reason: "exec-event",
}),
).toEqual({ defer: true, reason: "flood" });
});
it("does not flood-defer when recent runs are spread outside window", () => {
const now = 1_000_000;
const recentRunStarts = [
now - 300_000,
now - 240_000,
now - 180_000,
now - 120_000,
now - 65_000, // just outside default 60s window
];
expect(
decide({
source: "exec-event",
now,
nextDueMs: 0,
lastRunStartedAtMs: now - DEFAULT_MIN_WAKE_SPACING_MS - 1,
recentRunStarts,
reason: "exec-event",
}),
).toEqual({ defer: false });
});
it("does not flood-defer below threshold", () => {
const now = 1_000_000;
const recentRunStarts = [now - 30_000, now - 20_000, now - 10_000];
expect(
decide({
source: "exec-event",
now,
nextDueMs: 0,
lastRunStartedAtMs: now - DEFAULT_MIN_WAKE_SPACING_MS - 1,
recentRunStarts,
reason: "exec-event",
}),
).toEqual({ defer: false });
});
});
});
describe("recordRunStart", () => {
it("trims buffer to threshold + 1 entries", () => {
const buffer: number[] = [];
for (let i = 1; i <= DEFAULT_FLOOD_THRESHOLD + 5; i++) {
recordRunStart(buffer, i);
}
expect(buffer.length).toBe(DEFAULT_FLOOD_THRESHOLD + 1);
expect(buffer[buffer.length - 1]).toBe(DEFAULT_FLOOD_THRESHOLD + 5);
});
it("preserves insertion order", () => {
const buffer: number[] = [];
recordRunStart(buffer, 100);
recordRunStart(buffer, 200);
recordRunStart(buffer, 300);
expect(buffer).toEqual([100, 200, 300]);
});
});

164
src/infra/heartbeat-cooldown.ts Normal file
View File

@@ -0,0 +1,164 @@
// Centralized cooldown decision for heartbeat wakes.
//
// Background: a heartbeat run can be triggered by many wake sources — the
// scheduler's interval tick, a manual user request, a backgrounded `process.start`
// exit, a cron tick, an ACP spawn stream event, etc. Different sources used to
// take different code paths through the dispatcher, and historically the
// `nextDueMs` cooldown gate was only enforced on the `interval` branch. That let
// event-driven wakes (especially `exec-event`) fire heartbeat runs back-to-back
// when a heartbeat agent's tools triggered more wakes (#17797 → #75436).
//
// This module owns the single decision: "given this wake, should we run now or
// defer it?" Both the targeted and broadcast dispatch branches must call
// `shouldDeferWake` so the gate can never be forgotten on one path.
import type { HeartbeatWakeIntent, HeartbeatWakeSource } from "./heartbeat-wake.js";
// Default minimum spacing between heartbeat runs for the same agent, regardless
// of configured `every`. Even when `nextDueMs` is enforced, two wakes arriving
// within milliseconds can race the schedule update; this floor prevents that.
export const DEFAULT_MIN_WAKE_SPACING_MS = 30_000;
// Flood guard: if more than this many wakes for the same agent fall within the
// flood window, the dispatcher logs a warning and forces the wake to defer to
// the next scheduled tick. Tuned so a normal heartbeat that legitimately uses
// `manual` retry doesn't trip it but a feedback loop does.
export const DEFAULT_FLOOD_WINDOW_MS = 60_000;
export const DEFAULT_FLOOD_THRESHOLD = 5;
export type DeferDecision =
| { defer: false }
| { defer: true; reason: "not-due" | "min-spacing" | "flood" };
export type ShouldDeferInput = {
/** Scheduler behavior requested by the wake producer. */
intent: HeartbeatWakeIntent;
/** Wake producer, used for diagnostics and future source-specific telemetry. */
source?: HeartbeatWakeSource;
/** Raw wake reason string for logs/model context. It does not drive scheduling policy. */
reason: string | undefined;
/** Current monotonic-ish wall clock. Pass `Date.now()`. */
now: number;
/** When this agent's next interval-tick run is due. */
nextDueMs: number;
/** When this agent last *started* a run, if known. */
lastRunStartedAtMs?: number;
/** Recent wake timestamps for flood detection. */
recentRunStarts?: readonly number[];
/** Override the minimum spacing floor. */
minSpacingMs?: number;
/** Override the flood-window length. */
floodWindowMs?: number;
/** Override the flood-window threshold. */
floodThreshold?: number;
};
/**
* Decide whether an incoming wake should be deferred.
*
* The decision matrix:
*
* | Wake intent | First wake (no prior run) | Subsequent wakes |
* |---------------|----------------------------|-----------------------------------------|
* | manual | Run | Run (never deferred) |
* | immediate | Run | Run (never deferred, except flood) |
* | scheduled | Defer if now < nextDueMs | Defer if now < nextDueMs |
* | event | Run (bootstrap responsive) | Defer if now < nextDueMs OR within floor |
*
* Immediate is for documented wake-now delivery paths such as `openclaw system
* event --mode now`, task completion follow-ups, cron `--wake now`, and
* `/hooks/wake mode=now`. Event is for external/system notifications such as
* background exec exits, node notification changes, hook/cron next-heartbeat
* handoffs, ACP spawn stream updates, and retry wakes.
*
* Additional gates layered on top of the reason matrix:
*
* 1. **Minimum spacing floor** (`min-spacing`): even if `nextDueMs` has been
* passed, defer if a run started within the last `minSpacingMs`. Catches
* the race where a second wake arrives between `runOnce` returning and
* `advanceAgentSchedule` updating `nextDueMs`.
* 2. **Flood guard** (`flood`): if `recentRunStarts` shows ≥ `floodThreshold`
* runs within `floodWindowMs`, defer regardless of reason (except
* `manual`-class immediate intent). Caller should also emit a single
* warning log when this fires.
*/
export function shouldDeferWake(input: ShouldDeferInput): DeferDecision {
if (input.intent === "manual") {
return { defer: false };
}
if (input.intent === "immediate") {
// Even immediate wakes get rate-limited if a real flood is happening — but
// manual operator intent is fully exempt above. System-event, task, hook,
// and cron wake-now paths come from external systems we trust but cannot
// prove are loop-free, so the flood guard remains a backstop.
const floodDefer = checkFloodGuard(input);
return floodDefer ?? { defer: false };
}
// Flood guard applies to every non-immediate wake regardless of run history.
// It is the last line of defense against feedback loops.
const floodDefer = checkFloodGuard(input);
if (floodDefer) {
return floodDefer;
}
if (input.intent === "scheduled") {
return input.now < input.nextDueMs ? { defer: true, reason: "not-due" } : { defer: false };
}
// Event-driven wakes. First wake (no prior run) bypasses cooldown gates so
// an idle agent can respond to an external event without waiting for the
// first scheduled phase tick.
if (input.lastRunStartedAtMs === undefined) {
return { defer: false };
}
if (input.now < input.nextDueMs) {
return { defer: true, reason: "not-due" };
}
const minSpacing = input.minSpacingMs ?? DEFAULT_MIN_WAKE_SPACING_MS;
if (minSpacing > 0 && input.now - input.lastRunStartedAtMs < minSpacing) {
return { defer: true, reason: "min-spacing" };
}
return { defer: false };
}
function checkFloodGuard(input: ShouldDeferInput): DeferDecision | null {
const floodWindow = input.floodWindowMs ?? DEFAULT_FLOOD_WINDOW_MS;
const floodThreshold = input.floodThreshold ?? DEFAULT_FLOOD_THRESHOLD;
if (!input.recentRunStarts || input.recentRunStarts.length < floodThreshold || floodWindow <= 0) {
return null;
}
const windowStart = input.now - floodWindow;
let inWindow = 0;
for (let i = input.recentRunStarts.length - 1; i >= 0; i--) {
const ts = input.recentRunStarts[i];
if (ts === undefined || ts < windowStart) {
break;
}
inWindow += 1;
}
return inWindow >= floodThreshold ? { defer: true, reason: "flood" } : null;
}
/**
* Append a run-start timestamp to a bounded recent-runs buffer. Caller passes
* the previous buffer; this returns a new (mutated) buffer with the entry
* appended and trimmed to `floodThreshold + 1` entries (only the newest matter
* for flood detection).
*/
export function recordRunStart(
buffer: number[],
ts: number,
floodThreshold: number = DEFAULT_FLOOD_THRESHOLD,
): number[] {
buffer.push(ts);
const max = floodThreshold + 1;
while (buffer.length > max) {
buffer.shift();
}
return buffer;
}

50
src/infra/heartbeat-reason.test.ts
View File

@@ -1,10 +1,5 @@
import { describe, expect, it } from "vitest";
import {
isHeartbeatActionWakeReason,
isHeartbeatEventDrivenReason,
normalizeHeartbeatWakeReason,
resolveHeartbeatReasonKind,
} from "./heartbeat-reason.js";
import { normalizeHeartbeatWakeReason } from "./heartbeat-reason.js";
describe("heartbeat-reason", () => {
it.each([
@@ -14,47 +9,4 @@ describe("heartbeat-reason", () => {
])("normalizes wake reasons for %j", ({ value, expected }) => {
expect(normalizeHeartbeatWakeReason(value)).toBe(expected);
});
it.each([
{ value: "retry", expected: "retry" },
{ value: "interval", expected: "interval" },
{ value: "manual", expected: "manual" },
{ value: "exec-event", expected: "exec-event" },
{ value: "wake", expected: "wake" },
{ value: "acp:spawn:stream", expected: "wake" },
{ value: "acp:spawn:", expected: "wake" },
{ value: "cron:job-1", expected: "cron" },
{ value: "hook:wake", expected: "hook" },
{ value: " hook:wake ", expected: "hook" },
{ value: "requested", expected: "other" },
{ value: "slow", expected: "other" },
{ value: "", expected: "other" },
{ value: undefined, expected: "other" },
])("classifies reason kinds for %j", ({ value, expected }) => {
expect(resolveHeartbeatReasonKind(value)).toBe(expected);
});
it.each([
{ value: "exec-event", expected: true },
{ value: "cron:job-1", expected: true },
{ value: "wake", expected: true },
{ value: "acp:spawn:stream", expected: true },
{ value: "hook:gmail:sync", expected: true },
{ value: "interval", expected: false },
{ value: "manual", expected: false },
{ value: "other", expected: false },
])("matches event-driven behavior for %j", ({ value, expected }) => {
expect(isHeartbeatEventDrivenReason(value)).toBe(expected);
});
it.each([
{ value: "manual", expected: true },
{ value: "exec-event", expected: true },
{ value: "hook:wake", expected: true },
{ value: "interval", expected: false },
{ value: "cron:job-1", expected: false },
{ value: "retry", expected: false },
])("matches action-priority wake behavior for %j", ({ value, expected }) => {
expect(isHeartbeatActionWakeReason(value)).toBe(expected);
});
});

56
src/infra/heartbeat-reason.ts
View File

@@ -1,59 +1,5 @@
import { normalizeOptionalString } from "../shared/string-coerce.js";
type HeartbeatReasonKind =
| "retry"
| "interval"
| "manual"
| "exec-event"
| "wake"
| "cron"
| "hook"
| "other";
function trimReason(reason?: string): string {
return normalizeOptionalString(reason) ?? "";
}
export function normalizeHeartbeatWakeReason(reason?: string): string {
const trimmed = trimReason(reason);
return trimmed.length > 0 ? trimmed : "requested";
}
export function resolveHeartbeatReasonKind(reason?: string): HeartbeatReasonKind {
const trimmed = trimReason(reason);
if (trimmed === "retry") {
return "retry";
}
if (trimmed === "interval") {
return "interval";
}
if (trimmed === "manual") {
return "manual";
}
if (trimmed === "exec-event") {
return "exec-event";
}
if (trimmed === "wake") {
return "wake";
}
if (trimmed.startsWith("acp:spawn:")) {
return "wake";
}
if (trimmed.startsWith("cron:")) {
return "cron";
}
if (trimmed.startsWith("hook:")) {
return "hook";
}
return "other";
}
export function isHeartbeatEventDrivenReason(reason?: string): boolean {
const kind = resolveHeartbeatReasonKind(reason);
return kind === "exec-event" || kind === "cron" || kind === "wake" || kind === "hook";
}
export function isHeartbeatActionWakeReason(reason?: string): boolean {
const kind = resolveHeartbeatReasonKind(reason);
return kind === "manual" || kind === "exec-event" || kind === "hook";
return normalizeOptionalString(reason) ?? "requested";
}

4
src/infra/heartbeat-runner.commitments.test.ts
View File

@@ -12,7 +12,7 @@ import {
} from "./heartbeat-runner.js";
import { installHeartbeatRunnerTestRuntime } from "./heartbeat-runner.test-harness.js";
import { seedSessionStore, withTempHeartbeatSandbox } from "./heartbeat-runner.test-utils.js";
import { requestHeartbeatNow, resetHeartbeatWakeStateForTests } from "./heartbeat-wake.js";
import { requestHeartbeat, resetHeartbeatWakeStateForTests } from "./heartbeat-wake.js";
installHeartbeatRunnerTestRuntime();
@@ -348,7 +348,7 @@ describe("runHeartbeatOnce commitments", () => {
stableSchedulerSeed: "commitment-target-none",
});
requestHeartbeatNow({ reason: "manual", coalesceMs: 0 });
requestHeartbeat({ source: "manual", intent: "manual", reason: "manual", coalesceMs: 0 });
await vi.advanceTimersByTimeAsync(1);
runner.stop();

4
src/infra/heartbeat-runner.ghost-reminder.test.ts
View File

@@ -511,6 +511,8 @@ describe("Ghost reminder bug (issue #13317)", () => {
const result = await runHeartbeatOnce({
cfg,
agentId: "main",
source: "hook",
intent: "immediate",
reason: "wake",
deps: {
getReplyFromConfig: replySpy,
@@ -551,6 +553,8 @@ describe("Ghost reminder bug (issue #13317)", () => {
const result = await runHeartbeatOnce({
cfg,
agentId: "main",
source: "hook",
intent: "immediate",
reason: "wake",
deps: {
getReplyFromConfig: replySpy,

2
src/infra/heartbeat-runner.isolated-key-stability.test.ts
View File

@@ -24,7 +24,7 @@ describe("runHeartbeatOnce – isolated session key stability (#59493)", () => {
/**
* Simulates the wake-request feedback loop:
* 1. Normal heartbeat tick produces sessionKey "agent:main:main:heartbeat"
* 2. An exec/subagent event during that tick calls requestHeartbeatNow()
* 2. An exec/subagent event during that tick calls requestHeartbeat()
* with the already-suffixed key "agent:main:main:heartbeat"
* 3. The wake handler passes that key back into runHeartbeatOnce(sessionKey: ...)
*

9
src/infra/heartbeat-runner.returns-default-unset.test.ts
View File

@@ -1389,6 +1389,11 @@ describe("runHeartbeatOnce", () => {
.mockResolvedValue({ messageId: "m1", toJid: "jid" });
const res = await runHeartbeatOnce({
cfg,
...(params.reason === "wake"
? { source: "hook" as const, intent: "immediate" as const }
: params.reason === "interval"
? { source: "interval" as const, intent: "scheduled" as const }
: {}),
reason: params.reason,
deps: createHeartbeatDeps(sendWhatsApp, { getReplyFromConfig: replySpy }),
});
@@ -1736,6 +1741,8 @@ tasks:
try {
const res = await runHeartbeatOnce({
cfg,
source: "interval",
intent: "scheduled",
reason: "interval",
deps: createHeartbeatDeps(sendWhatsApp, { getReplyFromConfig: replySpy }),
});
@@ -1791,6 +1798,8 @@ tasks:
try {
const res = await runHeartbeatOnce({
cfg,
source: "exec-event",
intent: "event",
reason: "exec-event",
deps: createHeartbeatDeps(sendWhatsApp, { getReplyFromConfig: replySpy }),
});

289
src/infra/heartbeat-runner.scheduler.test.ts
View File

@@ -6,7 +6,7 @@ import {
HEARTBEAT_SKIP_CRON_IN_PROGRESS,
HEARTBEAT_SKIP_REQUESTS_IN_FLIGHT,
type RetryableHeartbeatBusySkipReason,
requestHeartbeatNow,
requestHeartbeat,
resetHeartbeatWakeStateForTests,
} from "./heartbeat-wake.js";
@@ -61,10 +61,47 @@ describe("startHeartbeatRunner", () => {
});
}
function wake(
reason: string,
opts: Partial<Parameters<typeof requestHeartbeat>[0]> = {},
): Parameters<typeof requestHeartbeat>[0] {
const source =
opts.source ??
(reason === "interval"
? "interval"
: reason === "manual"
? "manual"
: reason === "retry"
? "retry"
: reason === "exec-event"
? "exec-event"
: reason === "background-task"
? "background-task"
: reason === "background-task-blocked"
? "background-task-blocked"
: reason.startsWith("cron:")
? "cron"
: reason.startsWith("hook:")
? "hook"
: "other");
const intent =
opts.intent ??
(reason === "interval"
? "scheduled"
: reason === "manual"
? "manual"
: reason === "wake" ||
reason === "background-task" ||
reason === "background-task-blocked"
? "immediate"
: "event");
return { source, intent, reason, ...opts };
}
async function expectWakeDispatch(params: {
cfg: OpenClawConfig;
runSpy: RunOnce;
wake: Parameters<typeof requestHeartbeatNow>[0];
wake: Parameters<typeof requestHeartbeat>[0];
expectedCall: Record<string, unknown>;
}) {
const runner = startHeartbeatRunner({
@@ -73,7 +110,7 @@ describe("startHeartbeatRunner", () => {
stableSchedulerSeed: TEST_SCHEDULER_SEED,
});
requestHeartbeatNow(params.wake);
requestHeartbeat(params.wake);
await vi.advanceTimersByTimeAsync(1);
expect(params.runSpy).toHaveBeenCalledTimes(1);
@@ -313,7 +350,7 @@ describe("startHeartbeatRunner", () => {
// Simulate 4 more retries at short intervals (wake layer retries).
for (let i = 0; i < 4; i++) {
requestHeartbeatNow({ reason: "retry", coalesceMs: 0 });
requestHeartbeat(wake("retry", { coalesceMs: 0 }));
await vi.advanceTimersByTimeAsync(1_000);
}
expect(callTimes.some((time) => time >= firstDueMs + intervalMs)).toBe(false);
@@ -338,6 +375,8 @@ describe("startHeartbeatRunner", () => {
} as OpenClawConfig,
runSpy,
wake: {
source: "cron",
intent: "event",
reason: "cron:job-123",
agentId: "ops",
sessionKey: "agent:ops:discord:channel:alerts",
@@ -360,6 +399,8 @@ describe("startHeartbeatRunner", () => {
cfg: heartbeatConfig([{ id: "main" }, { id: "ops" }]),
runSpy,
wake: {
source: "cron",
intent: "event",
reason: "cron:job-123",
agentId: "ops",
sessionKey: "agent:ops:discord:channel:alerts",
@@ -394,6 +435,8 @@ describe("startHeartbeatRunner", () => {
} as OpenClawConfig,
runSpy,
wake: {
source: "cron",
intent: "event",
reason: "cron:job-123",
agentId: "ops",
sessionKey: "agent:ops:discord:channel:alerts",
@@ -446,6 +489,8 @@ describe("startHeartbeatRunner", () => {
} as OpenClawConfig,
runSpy,
wake: {
source: "exec-event",
intent: "event",
reason: "exec-event",
sessionKey: "agent:main:main",
coalesceMs: 0,
@@ -460,4 +505,240 @@ describe("startHeartbeatRunner", () => {
runner.stop();
});
// Regression for runaway heartbeat loop: backgrounded `process.start` exits
// call `requestHeartbeat({reason: "exec-event"})` from
// `bash-tools.exec-runtime.ts:347` (`maybeNotifyOnExit`). If a heartbeat run
// uses backgrounded tools (response-tracker sync, conversation monitors,
// etc.), each background process exit triggers another heartbeat run because
// the dispatcher (`heartbeat-runner.ts:1805`) only enforces `nextDueMs` when
// `reason === "interval"`, and the targeted branch has no cooldown gate at
// all. Observed in production: heartbeat configured `every: 30m` fires every
// ~10s, pegging the gateway event loop with eventLoopDelayMaxMs >6s spikes.
it("does not bypass interval cooldown for repeated exec-event wakes within nextDueMs", async () => {
useFakeHeartbeatTime();
const runSpy = vi.fn().mockResolvedValue({ status: "ran", durationMs: 1 });
const runner = startHeartbeatRunner({
cfg: heartbeatConfig(),
runOnce: runSpy,
stableSchedulerSeed: TEST_SCHEDULER_SEED,
});
// First exec-event wake: agent just woke from a backgrounded tool exit.
// This one legitimately fires the run.
requestHeartbeat({
source: "exec-event",
intent: "event",
reason: "exec-event",
sessionKey: "agent:main:main",
coalesceMs: 0,
});
await vi.advanceTimersByTimeAsync(1);
expect(runSpy).toHaveBeenCalledTimes(1);
// Simulate the runaway: 4 more exec-event wakes from backgrounded process
// exits, fired well within the configured 30m interval. These should be
// debounced by the cooldown — the agent just ran, nothing has changed.
for (let i = 0; i < 4; i++) {
await vi.advanceTimersByTimeAsync(10_000); // 10s between background exits
requestHeartbeat({
source: "exec-event",
intent: "event",
reason: "exec-event",
sessionKey: "agent:main:main",
coalesceMs: 0,
});
await vi.advanceTimersByTimeAsync(1);
}
// Total elapsed: ~40s. Configured `every` is 30m. Subsequent exec-events
// should NOT trigger fresh runs within the cooldown window.
expect(runSpy).toHaveBeenCalledTimes(1);
runner.stop();
});
it("preserves immediate delivery for repeated bare wake reasons", async () => {
// 'wake' is the immediate-path reason from `openclaw system event --mode now`
// and must NOT be deferred. Verify the runner allows multiple back-to-back
// wake requests through (subject only to the flood guard backstop).
useFakeHeartbeatTime();
const runSpy = vi.fn().mockResolvedValue({ status: "ran", durationMs: 1 });
const runner = startHeartbeatRunner({
cfg: heartbeatConfig(),
runOnce: runSpy,
stableSchedulerSeed: TEST_SCHEDULER_SEED,
});
// Three 'wake' requests with 200ms between them — none should be deferred.
for (let i = 0; i < 3; i++) {
requestHeartbeat({
source: "manual",
intent: "immediate",
reason: "wake",
sessionKey: "agent:main:main",
coalesceMs: 0,
});
await vi.advanceTimersByTimeAsync(1);
await vi.advanceTimersByTimeAsync(200);
}
expect(runSpy).toHaveBeenCalledTimes(3);
runner.stop();
});
it("preserves immediate delivery for repeated background-task wakes", async () => {
// Task-registry terminal updates wake the heartbeat with reason
// 'background-task'. Documented as immediate so users don't wait for the
// next scheduled tick to see task completion notifications.
useFakeHeartbeatTime();
const runSpy = vi.fn().mockResolvedValue({ status: "ran", durationMs: 1 });
const runner = startHeartbeatRunner({
cfg: heartbeatConfig(),
runOnce: runSpy,
stableSchedulerSeed: TEST_SCHEDULER_SEED,
});
for (let i = 0; i < 3; i++) {
requestHeartbeat({
source: "background-task",
intent: "immediate",
reason: "background-task",
sessionKey: "agent:main:main",
coalesceMs: 0,
});
await vi.advanceTimersByTimeAsync(1);
await vi.advanceTimersByTimeAsync(200);
}
expect(runSpy).toHaveBeenCalledTimes(3);
runner.stop();
});
it("preserves immediate delivery for blocked background-task follow-ups", async () => {
useFakeHeartbeatTime();
const runSpy = vi.fn().mockResolvedValue({ status: "ran", durationMs: 1 });
const runner = startHeartbeatRunner({
cfg: heartbeatConfig(),
runOnce: runSpy,
stableSchedulerSeed: TEST_SCHEDULER_SEED,
});
requestHeartbeat({
source: "exec-event",
intent: "event",
reason: "exec-event",
sessionKey: "agent:main:main",
coalesceMs: 0,
});
await vi.advanceTimersByTimeAsync(1);
expect(runSpy).toHaveBeenCalledTimes(1);
requestHeartbeat({
source: "background-task-blocked",
intent: "immediate",
reason: "background-task-blocked",
sessionKey: "agent:main:main",
coalesceMs: 0,
});
await vi.advanceTimersByTimeAsync(1);
expect(runSpy).toHaveBeenCalledTimes(2);
expect(runSpy.mock.calls[1]?.[0]).toEqual(
expect.objectContaining({
reason: "background-task-blocked",
sessionKey: "agent:main:main",
}),
);
runner.stop();
});
it.each([
{ reason: "hook:wake", label: "hook wake-now" },
{ reason: "hook:job-123", label: "hook agent wake-now announcement" },
{ reason: "cron:job-123", label: "cron wake-now" },
])("preserves immediate delivery for $label after a recent run", async ({ reason }) => {
useFakeHeartbeatTime();
const runSpy = vi.fn().mockResolvedValue({ status: "ran", durationMs: 1 });
const runner = startHeartbeatRunner({
cfg: heartbeatConfig(),
runOnce: runSpy,
stableSchedulerSeed: TEST_SCHEDULER_SEED,
});
requestHeartbeat({
source: "exec-event",
intent: "event",
reason: "exec-event",
sessionKey: "agent:main:main",
coalesceMs: 0,
});
await vi.advanceTimersByTimeAsync(1);
expect(runSpy).toHaveBeenCalledTimes(1);
requestHeartbeat({
source: reason.startsWith("cron:") ? "cron" : "hook",
intent: "immediate",
reason,
sessionKey: "agent:main:main",
coalesceMs: 0,
});
await vi.advanceTimersByTimeAsync(1);
expect(runSpy).toHaveBeenCalledTimes(2);
expect(runSpy.mock.calls[1]?.[0]).toEqual(
expect.objectContaining({ reason, sessionKey: "agent:main:main" }),
);
runner.stop();
});
it("retryable busy skip does not poison the cooldown for the next retry", async () => {
// Reproduces P2 finding from #75439 review: if a targeted exec-event wake
// hits requests-in-flight on its first attempt, the wake layer retries the
// same reason. The cooldown must NOT have been advanced by the busy attempt
// — otherwise the retry would falsely defer with `not-due`/`min-spacing`.
useFakeHeartbeatTime();
let attempt = 0;
const runSpy = vi.fn().mockImplementation(async () => {
attempt += 1;
if (attempt === 1) {
return { status: "skipped", reason: HEARTBEAT_SKIP_REQUESTS_IN_FLIGHT } as const;
}
return { status: "ran", durationMs: 1 } as const;
});
const runner = startHeartbeatRunner({
cfg: heartbeatConfig(),
runOnce: runSpy,
stableSchedulerSeed: TEST_SCHEDULER_SEED,
});
requestHeartbeat({
source: "exec-event",
intent: "event",
reason: "exec-event",
sessionKey: "agent:main:main",
coalesceMs: 0,
});
await vi.advanceTimersByTimeAsync(1);
expect(runSpy).toHaveBeenCalledTimes(1);
// Wake layer retries via DEFAULT_RETRY_MS (1s). Advance past it.
await vi.advanceTimersByTimeAsync(1500);
// The retry must NOT be deferred to `not-due` or `min-spacing`. Since the
// first attempt was a retryable busy skip, the cooldown bookkeeping was
// never recorded — so the retry should reach runOnce normally.
expect(runSpy).toHaveBeenCalledTimes(2);
expect(runSpy.mock.calls[1]?.[0]).toEqual(
expect.objectContaining({ reason: "exec-event", sessionKey: "agent:main:main" }),
);
await expect(runSpy.mock.results[1]?.value).resolves.toEqual({
status: "ran",
durationMs: 1,
});
runner.stop();
});
});

169
src/infra/heartbeat-runner.ts
View File

@@ -84,6 +84,7 @@ import { MAX_SAFE_TIMEOUT_DELAY_MS, resolveSafeTimeoutDelayMs } from "../utils/t
import { loadOrCreateDeviceIdentity } from "./device-identity.js";
import { formatErrorMessage, hasErrnoCode } from "./errors.js";
import { isWithinActiveHours } from "./heartbeat-active-hours.js";
import { recordRunStart, shouldDeferWake, type DeferDecision } from "./heartbeat-cooldown.js";
import {
buildCronEventPrompt,
buildExecEventPrompt,
@@ -92,7 +93,6 @@ import {
isRelayableExecCompletionEvent,
} from "./heartbeat-events-filter.js";
import { emitHeartbeatEvent, resolveIndicatorType } from "./heartbeat-events.js";
import { resolveHeartbeatReasonKind } from "./heartbeat-reason.js";
import {
computeNextHeartbeatPhaseDueMs,
resolveHeartbeatPhaseMs,
@@ -113,9 +113,11 @@ import {
HEARTBEAT_SKIP_REQUESTS_IN_FLIGHT,
type HeartbeatRunResult,
type HeartbeatWakeHandler,
type HeartbeatWakeIntent,
type HeartbeatWakeRequest,
type HeartbeatWakeSource,
isRetryableHeartbeatBusySkipReason,
requestHeartbeatNow,
requestHeartbeat,
setHeartbeatsEnabled,
setHeartbeatWakeHandler,
} from "./heartbeat-wake.js";
@@ -213,6 +215,12 @@ type HeartbeatAgentState = {
intervalMs: number;
phaseMs: number;
nextDueMs: number;
/** Wall-clock start time of the most recent run for this agent. */
lastRunStartedAtMs?: number;
/** Bounded ring buffer of recent run-start timestamps for flood detection. */
recentRunStarts: number[];
/** Set true after a flood-defer is logged to avoid log spam. Reset when a run actually fires. */
floodLoggedSinceLastRun: boolean;
};
export type HeartbeatRunner = {
@@ -602,10 +610,10 @@ function normalizeHeartbeatToolNotification(
return { shouldSkip: false, text: finalText, hasMedia: false };
}
type HeartbeatReasonFlags = {
isExecEventReason: boolean;
isCronEventReason: boolean;
isWakeReason: boolean;
type HeartbeatWakePayloadFlags = {
isExecEventWake: boolean;
isCronWake: boolean;
isWakePayload: boolean;
};
type HeartbeatSkipReason = "empty-heartbeat-file";
@@ -661,7 +669,7 @@ Commitments:
${JSON.stringify(items, null, 2)}`;
}
type HeartbeatPreflight = HeartbeatReasonFlags & {
type HeartbeatPreflight = HeartbeatWakePayloadFlags & {
session: ReturnType<typeof resolveHeartbeatSession>;
pendingEventEntries: ReturnType<typeof peekSystemEventEntries>;
turnSourceDeliveryContext: ReturnType<typeof resolveSystemEventDeliveryContext>;
@@ -673,12 +681,33 @@ type HeartbeatPreflight = HeartbeatReasonFlags & {
heartbeatFileContent?: string;
};
function resolveHeartbeatReasonFlags(reason?: string): HeartbeatReasonFlags {
const reasonKind = resolveHeartbeatReasonKind(reason);
function inferHeartbeatWakeSourceFromReason(reason?: string): HeartbeatWakeSource | undefined {
const trimmed = (reason ?? "").trim();
if (trimmed === "exec-event") {
return "exec-event";
}
if (trimmed.startsWith("cron:")) {
return "cron";
}
if (trimmed === "wake" || trimmed.startsWith("hook:")) {
return "hook";
}
if (trimmed.startsWith("acp:spawn:")) {
return "acp-spawn";
}
return undefined;
}
function resolveHeartbeatWakePayloadFlags(params: {
source?: HeartbeatWakeSource;
reason?: string;
}): HeartbeatWakePayloadFlags {
const source = params.source ?? inferHeartbeatWakeSourceFromReason(params.reason);
const reason = (params.reason ?? "").trim();
return {
isExecEventReason: reasonKind === "exec-event",
isCronEventReason: reasonKind === "cron",
isWakeReason: reasonKind === "wake" || reasonKind === "hook",
isExecEventWake: source === "exec-event",
isCronWake: source === "cron",
isWakePayload: source === "hook" || source === "acp-spawn" || reason === "wake",
};
}
@@ -688,9 +717,13 @@ async function resolveHeartbeatPreflight(params: {
heartbeat?: HeartbeatConfig;
forcedSessionKey?: string;
reason?: string;
source?: HeartbeatWakeSource;
nowMs?: number;
}): Promise<HeartbeatPreflight> {
const reasonFlags = resolveHeartbeatReasonFlags(params.reason);
const wakeFlags = resolveHeartbeatWakePayloadFlags({
source: params.source,
reason: params.reason,
});
const session = resolveHeartbeatSession(
params.cfg,
params.agentId,
@@ -715,7 +748,7 @@ async function resolveHeartbeatPreflight(params: {
// Wake-triggered runs should only inspect pending events when preflight peeks
// the same queue that the run itself will execute/drain.
const shouldInspectWakePendingEvents = (() => {
if (!reasonFlags.isWakeReason) {
if (!wakeFlags.isWakePayload) {
return false;
}
if (params.heartbeat?.isolatedSession !== true) {
@@ -730,17 +763,17 @@ async function resolveHeartbeatPreflight(params: {
return isolatedSessionKey === session.sessionKey;
})();
const shouldInspectPendingEvents =
reasonFlags.isExecEventReason ||
reasonFlags.isCronEventReason ||
wakeFlags.isExecEventWake ||
wakeFlags.isCronWake ||
shouldInspectWakePendingEvents ||
hasTaggedCronEvents;
const shouldBypassFileGates =
reasonFlags.isExecEventReason ||
reasonFlags.isCronEventReason ||
reasonFlags.isWakeReason ||
wakeFlags.isExecEventWake ||
wakeFlags.isCronWake ||
wakeFlags.isWakePayload ||
hasTaggedCronEvents;
const basePreflight = {
...reasonFlags,
...wakeFlags,
session,
pendingEventEntries,
turnSourceDeliveryContext,
@@ -870,7 +903,7 @@ function resolveHeartbeatRunPrompt(params: {
const cronEvents = pendingEventEntries
.filter(
(event) =>
(params.preflight.isCronEventReason || event.contextKey?.startsWith("cron:")) &&
(params.preflight.isCronWake || event.contextKey?.startsWith("cron:")) &&
isCronSystemEvent(event.text),
)
.map((event) => event.text);
@@ -962,7 +995,7 @@ function selectSystemEventsConsumedByHeartbeat(params: {
if (params.hasCronEvents) {
return preflight.pendingEventEntries.filter(
(event) =>
(preflight.isCronEventReason || event.contextKey?.startsWith("cron:")) &&
(preflight.isCronWake || event.contextKey?.startsWith("cron:")) &&
isCronSystemEvent(event.text),
);
}
@@ -974,6 +1007,8 @@ export async function runHeartbeatOnce(opts: {
agentId?: string;
sessionKey?: string;
heartbeat?: HeartbeatConfig;
source?: HeartbeatWakeSource;
intent?: HeartbeatWakeIntent;
reason?: string;
deps?: HeartbeatDeps;
}): Promise<HeartbeatRunResult> {
@@ -1030,6 +1065,7 @@ export async function runHeartbeatOnce(opts: {
agentId,
heartbeat,
forcedSessionKey: opts.sessionKey,
source: opts.source,
reason: opts.reason,
nowMs: startedAt,
});
@@ -1157,7 +1193,7 @@ export async function runHeartbeatOnce(opts: {
// Wake-triggered events should stay queued when the run short-circuits:
// no reply turn ran, so there is nothing that actually consumed that wake payload.
const shouldConsumeInspectedEvents =
!preflight.isWakeReason && preflight.shouldInspectPendingEvents;
!preflight.isWakePayload && preflight.shouldInspectPendingEvents;
if (shouldConsumeInspectedEvents && inspectedSystemEventsToConsume.length > 0) {
consumeSelectedSystemEventEntries(sessionKey, inspectedSystemEventsToConsume);
}
@@ -1756,6 +1792,45 @@ export function startHeartbeatRunner(opts: {
now + agent.intervalMs;
};
// Centralized cooldown gate. Both targeted and broadcast dispatch branches
// call this before invoking `runOnce`. Manual wakes are never deferred.
// Everything else respects `nextDueMs`, the min-spacing floor, and the flood
// guard — see `heartbeat-cooldown.ts` for rationale and #75436.
const evaluateWakeDeferral = (
agent: HeartbeatAgentState,
now: number,
reason?: string,
intent: HeartbeatWakeIntent = "event",
): DeferDecision => {
const decision = shouldDeferWake({
intent,
reason,
now,
nextDueMs: agent.nextDueMs,
lastRunStartedAtMs: agent.lastRunStartedAtMs,
recentRunStarts: agent.recentRunStarts,
});
if (decision.defer && decision.reason === "flood") {
if (!agent.floodLoggedSinceLastRun) {
log.warn("heartbeat: flood guard tripped, deferring wake", {
agentId: agent.agentId,
reason: reason ?? "(none)",
recentRunCount: agent.recentRunStarts.length,
});
agent.floodLoggedSinceLastRun = true;
}
}
return decision;
};
// Called immediately before `runOnce` actually executes. Updates the
// bookkeeping that the cooldown gate consults on the next wake.
const recordRunBookkeeping = (agent: HeartbeatAgentState, now: number) => {
agent.lastRunStartedAtMs = now;
recordRunStart(agent.recentRunStarts, now);
agent.floodLoggedSinceLastRun = false;
};
const scheduleNext = () => {
if (state.stopped) {
return;
@@ -1788,7 +1863,12 @@ export function startHeartbeatRunner(opts: {
const delay = resolveSafeTimeoutDelayMs(rawDelay, { minMs: 0 });
state.timer = setTimeout(() => {
state.timer = null;
requestHeartbeatNow({ reason: "interval", coalesceMs: 0 });
requestHeartbeat({
source: "interval",
intent: "scheduled",
reason: "interval",
coalesceMs: 0,
});
}, delay);
state.timer.unref?.();
};
@@ -1821,6 +1901,9 @@ export function startHeartbeatRunner(opts: {
intervalMs,
phaseMs,
nextDueMs,
lastRunStartedAtMs: prevState?.lastRunStartedAtMs,
recentRunStarts: prevState?.recentRunStarts ?? [],
floodLoggedSinceLastRun: prevState?.floodLoggedSinceLastRun ?? false,
});
}
@@ -1866,6 +1949,7 @@ export function startHeartbeatRunner(opts: {
}
const reason = params?.reason;
const intent = params.intent;
const requestedAgentId = params?.agentId ? normalizeAgentId(params.agentId) : undefined;
const requestedSessionKey = normalizeOptionalString(params?.sessionKey);
const requestedHeartbeat = params?.heartbeat;
@@ -1886,19 +1970,32 @@ export function startHeartbeatRunner(opts: {
if (!targetAgent) {
return { status: "skipped", reason: "disabled" };
}
const deferral = evaluateWakeDeferral(targetAgent, now, reason, intent);
if (deferral.defer) {
return { status: "skipped", reason: deferral.reason };
}
try {
const res = await runOnce({
cfg: state.cfg,
agentId: targetAgent.agentId,
heartbeat: resolveRequestedHeartbeat(targetAgent.heartbeat),
source: params.source,
intent,
reason,
sessionKey: requestedSessionKey,
deps: { runtime: state.runtime },
});
if (res.status === "skipped" && isRetryableHeartbeatBusySkipReason(res.reason)) {
// Retryable busy — do NOT record run bookkeeping. The wake layer
// retries the same reason shortly; if we recorded `lastRunStartedAtMs`
// here, the retry would falsely defer with `not-due`/`min-spacing`
// because the cooldown would treat this skipped attempt as a real run.
retryableBusySkip = true;
return res;
}
// Non-retryable outcome (ran, disabled, failed-but-not-busy). Record
// bookkeeping so subsequent wakes within the cooldown window defer.
recordRunBookkeeping(targetAgent, now);
if (res.status !== "skipped" || res.reason !== "disabled") {
advanceAgentSchedule(targetAgent, now, reason);
}
@@ -1908,13 +2005,18 @@ export function startHeartbeatRunner(opts: {
log.error(`heartbeat runner: targeted runOnce threw unexpectedly: ${errMsg}`, {
error: errMsg,
});
// Throw counts as a non-retryable terminal attempt for cooldown
// purposes — record bookkeeping so the wake layer doesn't tight-loop
// on the same reason.
recordRunBookkeeping(targetAgent, now);
advanceAgentSchedule(targetAgent, now, reason);
return { status: "failed", reason: errMsg };
}
}
for (const agent of state.agents.values()) {
if (isInterval && now < agent.nextDueMs) {
const deferral = evaluateWakeDeferral(agent, now, reason, intent);
if (deferral.defer) {
continue;
}
@@ -1924,23 +2026,30 @@ export function startHeartbeatRunner(opts: {
cfg: state.cfg,
agentId: agent.agentId,
heartbeat: agent.heartbeat,
source: params.source,
intent,
reason,
deps: { runtime: state.runtime },
});
} catch (err) {
const errMsg = formatErrorMessage(err);
log.error(`heartbeat runner: runOnce threw unexpectedly: ${errMsg}`, { error: errMsg });
// Throw counts as a non-retryable terminal attempt — see comment in
// targeted branch above.
recordRunBookkeeping(agent, now);
advanceAgentSchedule(agent, now, reason);
continue;
}
if (res.status === "skipped" && isRetryableHeartbeatBusySkipReason(res.reason)) {
// Do not advance the schedule — the main lane is busy and the wake
// layer will retry shortly (DEFAULT_RETRY_MS = 1 s). Calling
// scheduleNext() here would register a 0 ms timer that races with
// the wake layer's 1 s retry and wins, bypassing the cooldown.
// Do not advance the schedule or record run bookkeeping — the main
// lane is busy and the wake layer will retry the same reason shortly
// (DEFAULT_RETRY_MS = 1 s). Recording here would convert the retry
// into a false `not-due`/`min-spacing` defer.
retryableBusySkip = true;
return res;
}
// Non-retryable outcome — record bookkeeping for cooldown gates.
recordRunBookkeeping(agent, now);
if (res.status !== "skipped" || res.reason !== "disabled") {
advanceAgentSchedule(agent, now, reason);
}
@@ -2014,6 +2123,8 @@ export function startHeartbeatRunner(opts: {
agentId: params.agentId,
sessionKey: params.sessionKey,
heartbeat: params.heartbeat,
source: params.source,
intent: params.intent,
});
const disposeWakeHandler = setHeartbeatWakeHandler(wakeHandler);
updateConfig(state.cfg);

105
src/infra/heartbeat-wake.test.ts
View File

@@ -5,12 +5,35 @@ import {
HEARTBEAT_SKIP_REQUESTS_IN_FLIGHT,
hasHeartbeatWakeHandler,
hasPendingHeartbeatWake,
requestHeartbeatNow,
requestHeartbeat,
resetHeartbeatWakeStateForTests,
setHeartbeatWakeHandler,
} from "./heartbeat-wake.js";
describe("heartbeat-wake", () => {
type WakeRequest = Parameters<typeof requestHeartbeat>[0];
function wake(reason: string, opts: Partial<WakeRequest> = {}): WakeRequest {
const source =
opts.source ??
(reason === "interval"
? "interval"
: reason === "manual"
? "manual"
: reason === "retry"
? "retry"
: reason === "exec-event"
? "exec-event"
: reason.startsWith("cron:")
? "cron"
: reason.startsWith("hook:")
? "hook"
: "other");
const intent =
opts.intent ??
(reason === "interval" ? "scheduled" : reason === "manual" ? "manual" : "event");
return { source, intent, reason, ...opts };
}
function setRetryOnceHeartbeatHandler() {
const handler = vi
.fn()
@@ -28,7 +51,7 @@ describe("heartbeat-wake", () => {
setHeartbeatWakeHandler(
params.handler as unknown as Parameters<typeof setHeartbeatWakeHandler>[0],
);
requestHeartbeatNow({ reason: params.initialReason, coalesceMs: 0 });
requestHeartbeat(wake(params.initialReason, { coalesceMs: 0 }));
await vi.advanceTimersByTimeAsync(1);
expect(params.handler).toHaveBeenCalledTimes(1);
@@ -38,7 +61,7 @@ describe("heartbeat-wake", () => {
await vi.advanceTimersByTimeAsync(500);
expect(params.handler).toHaveBeenCalledTimes(2);
expect(params.handler.mock.calls[1]?.[0]).toEqual({ reason: params.expectedRetryReason });
expect(params.handler.mock.calls[1]?.[0]).toEqual(wake(params.expectedRetryReason));
}
beforeEach(() => {
@@ -56,9 +79,9 @@ describe("heartbeat-wake", () => {
const handler = vi.fn().mockResolvedValue({ status: "skipped", reason: "disabled" });
setHeartbeatWakeHandler(handler);
requestHeartbeatNow({ reason: "interval", coalesceMs: 200 });
requestHeartbeatNow({ reason: "exec-event", coalesceMs: 200 });
requestHeartbeatNow({ reason: "retry", coalesceMs: 200 });
requestHeartbeat(wake("interval", { coalesceMs: 200 }));
requestHeartbeat(wake("exec-event", { coalesceMs: 200 }));
requestHeartbeat(wake("retry", { coalesceMs: 200 }));
expect(hasPendingHeartbeatWake()).toBe(true);
@@ -67,7 +90,7 @@ describe("heartbeat-wake", () => {
await vi.advanceTimersByTimeAsync(1);
expect(handler).toHaveBeenCalledTimes(1);
expect(handler).toHaveBeenCalledWith({ reason: "exec-event" });
expect(handler).toHaveBeenCalledWith(wake("exec-event"));
expect(hasPendingHeartbeatWake()).toBe(false);
});
@@ -104,18 +127,18 @@ describe("heartbeat-wake", () => {
vi.useFakeTimers();
const handler = setRetryOnceHeartbeatHandler();
requestHeartbeatNow({ reason: "interval", coalesceMs: 0 });
requestHeartbeat(wake("interval", { coalesceMs: 0 }));
await vi.advanceTimersByTimeAsync(1);
expect(handler).toHaveBeenCalledTimes(1);
// Retry is now waiting for 1000ms. This should not preempt cooldown.
requestHeartbeatNow({ reason: "hook:wake", coalesceMs: 0 });
requestHeartbeat(wake("hook:wake", { coalesceMs: 0 }));
await vi.advanceTimersByTimeAsync(998);
expect(handler).toHaveBeenCalledTimes(1);
await vi.advanceTimersByTimeAsync(1);
expect(handler).toHaveBeenCalledTimes(2);
expect(handler.mock.calls[1]?.[0]).toEqual({ reason: "hook:wake" });
expect(handler.mock.calls[1]?.[0]).toEqual(wake("hook:wake"));
});
it("retries thrown handler errors after the default retry delay", async () => {
@@ -147,7 +170,7 @@ describe("heartbeat-wake", () => {
expect(hasHeartbeatWakeHandler()).toBe(true);
// handlerB should still work
requestHeartbeatNow({ reason: "interval", coalesceMs: 0 });
requestHeartbeat(wake("interval", { coalesceMs: 0 }));
await vi.advanceTimersByTimeAsync(1);
expect(handlerB).toHaveBeenCalledTimes(1);
expect(handlerA).not.toHaveBeenCalled();
@@ -163,15 +186,15 @@ describe("heartbeat-wake", () => {
setHeartbeatWakeHandler(handler);
// Schedule for 5 seconds from now
requestHeartbeatNow({ reason: "slow", coalesceMs: 5000 });
requestHeartbeat(wake("slow", { coalesceMs: 5000 }));
// Schedule for 100ms from now — should preempt the 5s timer
requestHeartbeatNow({ reason: "fast", coalesceMs: 100 });
requestHeartbeat(wake("fast", { coalesceMs: 100 }));
await vi.advanceTimersByTimeAsync(100);
expect(handler).toHaveBeenCalledTimes(1);
// The reason should be "fast" since it was set last
expect(handler).toHaveBeenCalledWith({ reason: "fast" });
expect(handler).toHaveBeenCalledWith(wake("fast"));
});
it("keeps existing timer when later schedule is requested", async () => {
@@ -180,10 +203,10 @@ describe("heartbeat-wake", () => {
setHeartbeatWakeHandler(handler);
// Schedule for 100ms from now
requestHeartbeatNow({ reason: "fast", coalesceMs: 100 });
requestHeartbeat(wake("fast", { coalesceMs: 100 }));
// Schedule for 5 seconds from now — should NOT preempt
requestHeartbeatNow({ reason: "slow", coalesceMs: 5000 });
requestHeartbeat(wake("slow", { coalesceMs: 5000 }));
await vi.advanceTimersByTimeAsync(100);
expect(handler).toHaveBeenCalledTimes(1);
@@ -194,12 +217,12 @@ describe("heartbeat-wake", () => {
const handler = vi.fn().mockResolvedValue({ status: "ran", durationMs: 1 });
setHeartbeatWakeHandler(handler);
requestHeartbeatNow({ reason: "exec-event", coalesceMs: 100 });
requestHeartbeatNow({ reason: "retry", coalesceMs: 100 });
requestHeartbeat(wake("exec-event", { coalesceMs: 100 }));
requestHeartbeat(wake("retry", { coalesceMs: 100 }));
await vi.advanceTimersByTimeAsync(100);
expect(handler).toHaveBeenCalledTimes(1);
expect(handler).toHaveBeenCalledWith({ reason: "exec-event" });
expect(handler).toHaveBeenCalledWith(wake("exec-event"));
});
it("resets running/scheduled flags when new handler is registered", async () => {
@@ -217,7 +240,7 @@ describe("heartbeat-wake", () => {
setHeartbeatWakeHandler(handlerA);
// Trigger the handler — it starts running but never finishes
requestHeartbeatNow({ reason: "interval", coalesceMs: 0 });
requestHeartbeat(wake("interval", { coalesceMs: 0 }));
await vi.advanceTimersByTimeAsync(1);
expect(handlerA).toHaveBeenCalledTimes(1);
@@ -227,7 +250,7 @@ describe("heartbeat-wake", () => {
setHeartbeatWakeHandler(handlerB);
// handlerB should be able to fire (running was reset)
requestHeartbeatNow({ reason: "interval", coalesceMs: 0 });
requestHeartbeat(wake("interval", { coalesceMs: 0 }));
await vi.advanceTimersByTimeAsync(1);
expect(handlerB).toHaveBeenCalledTimes(1);
@@ -243,7 +266,7 @@ describe("heartbeat-wake", () => {
.mockResolvedValue({ status: "skipped", reason: HEARTBEAT_SKIP_REQUESTS_IN_FLIGHT });
setHeartbeatWakeHandler(handlerA);
requestHeartbeatNow({ reason: "interval", coalesceMs: 0 });
requestHeartbeat(wake("interval", { coalesceMs: 0 }));
await vi.advanceTimersByTimeAsync(1);
expect(handlerA).toHaveBeenCalledTimes(1);
@@ -251,16 +274,16 @@ describe("heartbeat-wake", () => {
const handlerB = vi.fn().mockResolvedValue({ status: "ran", durationMs: 1 });
setHeartbeatWakeHandler(handlerB);
requestHeartbeatNow({ reason: "manual", coalesceMs: 0 });
requestHeartbeat(wake("manual", { coalesceMs: 0 }));
await vi.advanceTimersByTimeAsync(1);
expect(handlerB).toHaveBeenCalledTimes(1);
expect(handlerB).toHaveBeenCalledWith({ reason: "manual" });
expect(handlerB).toHaveBeenCalledWith(wake("manual"));
});
it("drains pending wake once a handler is registered", async () => {
vi.useFakeTimers();
requestHeartbeatNow({ reason: "manual", coalesceMs: 0 });
requestHeartbeat(wake("manual", { coalesceMs: 0 }));
await vi.advanceTimersByTimeAsync(1);
expect(hasPendingHeartbeatWake()).toBe(true);
@@ -272,7 +295,7 @@ describe("heartbeat-wake", () => {
await vi.advanceTimersByTimeAsync(1);
expect(handler).toHaveBeenCalledTimes(1);
expect(handler).toHaveBeenCalledWith({ reason: "manual" });
expect(handler).toHaveBeenCalledWith(wake("manual"));
expect(hasPendingHeartbeatWake()).toBe(false);
});
@@ -280,7 +303,9 @@ describe("heartbeat-wake", () => {
vi.useFakeTimers();
const handler = setRetryOnceHeartbeatHandler();
requestHeartbeatNow({
requestHeartbeat({
source: "cron",
intent: "immediate",
reason: "cron:job-1",
agentId: "ops",
sessionKey: "agent:ops:guildchat:channel:alerts",
@@ -291,6 +316,8 @@ describe("heartbeat-wake", () => {
await vi.advanceTimersByTimeAsync(1);
expect(handler).toHaveBeenCalledTimes(1);
expect(handler.mock.calls[0]?.[0]).toEqual({
source: "cron",
intent: "immediate",
reason: "cron:job-1",
agentId: "ops",
sessionKey: "agent:ops:guildchat:channel:alerts",
@@ -300,6 +327,8 @@ describe("heartbeat-wake", () => {
await vi.advanceTimersByTimeAsync(1000);
expect(handler).toHaveBeenCalledTimes(2);
expect(handler.mock.calls[1]?.[0]).toEqual({
source: "cron",
intent: "immediate",
reason: "cron:job-1",
agentId: "ops",
sessionKey: "agent:ops:guildchat:channel:alerts",
@@ -312,14 +341,18 @@ describe("heartbeat-wake", () => {
const handler = vi.fn().mockResolvedValue({ status: "ran", durationMs: 1 });
setHeartbeatWakeHandler(handler);
requestHeartbeatNow({
requestHeartbeat({
source: "manual",
intent: "manual",
reason: "manual",
agentId: "ops",
sessionKey: "agent:ops:guildchat:channel:alerts",
heartbeat: { target: "last" },
coalesceMs: 100,
});
requestHeartbeatNow({
requestHeartbeat({
source: "manual",
intent: "manual",
reason: "manual",
agentId: "ops",
sessionKey: "agent:ops:guildchat:channel:alerts",
@@ -330,6 +363,8 @@ describe("heartbeat-wake", () => {
expect(handler).toHaveBeenCalledTimes(1);
expect(handler).toHaveBeenCalledWith({
source: "manual",
intent: "manual",
reason: "manual",
agentId: "ops",
sessionKey: "agent:ops:guildchat:channel:alerts",
@@ -342,13 +377,17 @@ describe("heartbeat-wake", () => {
const handler = vi.fn().mockResolvedValue({ status: "ran", durationMs: 1 });
setHeartbeatWakeHandler(handler);
requestHeartbeatNow({
requestHeartbeat({
source: "cron",
intent: "event",
reason: "cron:job-a",
agentId: "ops",
sessionKey: "agent:ops:guildchat:channel:alerts",
coalesceMs: 100,
});
requestHeartbeatNow({
requestHeartbeat({
source: "cron",
intent: "event",
reason: "cron:job-b",
agentId: "main",
sessionKey: "agent:main:forum:group:-1001",
@@ -361,11 +400,15 @@ describe("heartbeat-wake", () => {
expect(handler.mock.calls.map((call) => call[0])).toEqual(
expect.arrayContaining([
{
source: "cron",
intent: "event",
reason: "cron:job-a",
agentId: "ops",
sessionKey: "agent:ops:guildchat:channel:alerts",
},
{
source: "cron",
intent: "event",
reason: "cron:job-b",
agentId: "main",
sessionKey: "agent:main:forum:group:-1001",

92
src/infra/heartbeat-wake.ts
View File

@@ -1,9 +1,5 @@
import { normalizeOptionalString } from "../shared/string-coerce.js";
import {
isHeartbeatActionWakeReason,
normalizeHeartbeatWakeReason,
resolveHeartbeatReasonKind,
} from "./heartbeat-reason.js";
import { normalizeHeartbeatWakeReason } from "./heartbeat-reason.js";
export type HeartbeatRunResult =
| { status: "ran"; durationMs: number }
@@ -28,7 +24,26 @@ export function isRetryableHeartbeatBusySkipReason(reason: string): boolean {
return RETRYABLE_BUSY_SKIP_REASONS.has(reason);
}
export type HeartbeatWakeIntent = "scheduled" | "event" | "immediate" | "manual";
export type HeartbeatWakeSource =
| "interval"
| "manual"
| "exec-event"
| "notifications-event"
| "cron"
| "hook"
| "background-task"
| "background-task-blocked"
| "acp-spawn"
| "cli-watchdog"
| "restart-sentinel"
| "retry"
| "other";
export type HeartbeatWakeRequest = {
source: HeartbeatWakeSource;
intent: HeartbeatWakeIntent;
reason?: string;
agentId?: string;
sessionKey?: string;
@@ -49,6 +64,8 @@ export function areHeartbeatsEnabled(): boolean {
type WakeTimerKind = "normal" | "retry";
type PendingWakeReason = {
source: HeartbeatWakeSource;
intent: HeartbeatWakeIntent;
reason: string;
priority: number;
requestedAt: number;
@@ -75,17 +92,24 @@ const REASON_PRIORITY = {
ACTION: 3,
} as const;
function resolveReasonPriority(reason: string): number {
const kind = resolveHeartbeatReasonKind(reason);
if (kind === "retry") {
function resolveWakePriority(params: {
source: HeartbeatWakeSource;
intent: HeartbeatWakeIntent;
reason: string;
}): number {
if (params.intent === "manual" || params.intent === "immediate") {
return REASON_PRIORITY.ACTION;
}
if (params.source === "retry" || params.reason === "retry") {
return REASON_PRIORITY.RETRY;
}
if (kind === "interval") {
if (
params.intent === "scheduled" ||
params.source === "interval" ||
params.reason === "interval"
) {
return REASON_PRIORITY.INTERVAL;
}
if (isHeartbeatActionWakeReason(reason)) {
return REASON_PRIORITY.ACTION;
}
return REASON_PRIORITY.DEFAULT;
}
@@ -104,28 +128,36 @@ function getWakeTargetKey(params: { agentId?: string; sessionKey?: string }) {
return `${agentId ?? ""}::${sessionKey ?? ""}`;
}
function queuePendingWakeReason(params?: {
function queuePendingWakeReason(params: {
source: HeartbeatWakeSource;
intent: HeartbeatWakeIntent;
reason?: string;
requestedAt?: number;
agentId?: string;
sessionKey?: string;
heartbeat?: { target?: string };
}) {
const requestedAt = params?.requestedAt ?? Date.now();
const normalizedReason = normalizeWakeReason(params?.reason);
const normalizedAgentId = normalizeWakeTarget(params?.agentId);
const normalizedSessionKey = normalizeWakeTarget(params?.sessionKey);
const requestedAt = params.requestedAt ?? Date.now();
const normalizedReason = normalizeWakeReason(params.reason);
const normalizedAgentId = normalizeWakeTarget(params.agentId);
const normalizedSessionKey = normalizeWakeTarget(params.sessionKey);
const wakeTargetKey = getWakeTargetKey({
agentId: normalizedAgentId,
sessionKey: normalizedSessionKey,
});
const next: PendingWakeReason = {
source: params.source,
intent: params.intent,
reason: normalizedReason,
priority: resolveReasonPriority(normalizedReason),
priority: resolveWakePriority({
source: params.source,
intent: params.intent,
reason: normalizedReason,
}),
requestedAt,
agentId: normalizedAgentId,
sessionKey: normalizedSessionKey,
heartbeat: params?.heartbeat,
heartbeat: params.heartbeat,
};
const previous = pendingWakes.get(wakeTargetKey);
if (!previous) {
@@ -187,6 +219,8 @@ function schedule(coalesceMs: number, kind: WakeTimerKind = "normal") {
try {
for (const pendingWake of pendingBatch) {
const wakeOpts = {
source: pendingWake.source,
intent: pendingWake.intent,
reason: pendingWake.reason ?? undefined,
...(pendingWake.agentId ? { agentId: pendingWake.agentId } : {}),
...(pendingWake.sessionKey ? { sessionKey: pendingWake.sessionKey } : {}),
@@ -196,6 +230,8 @@ function schedule(coalesceMs: number, kind: WakeTimerKind = "normal") {
if (res.status === "skipped" && isRetryableHeartbeatBusySkipReason(res.reason)) {
// The target runtime is busy; retry this wake target soon.
queuePendingWakeReason({
source: pendingWake.source,
intent: pendingWake.intent,
reason: pendingWake.reason ?? "retry",
agentId: pendingWake.agentId,
sessionKey: pendingWake.sessionKey,
@@ -208,6 +244,8 @@ function schedule(coalesceMs: number, kind: WakeTimerKind = "normal") {
// Error is already logged by the heartbeat runner; schedule a retry.
for (const pendingWake of pendingBatch) {
queuePendingWakeReason({
source: pendingWake.source,
intent: pendingWake.intent,
reason: pendingWake.reason ?? "retry",
agentId: pendingWake.agentId,
sessionKey: pendingWake.sessionKey,
@@ -267,7 +305,9 @@ export function setHeartbeatWakeHandler(next: HeartbeatWakeHandler | null): () =
};
}
export function requestHeartbeatNow(opts?: {
export function requestHeartbeat(opts: {
source: HeartbeatWakeSource;
intent: HeartbeatWakeIntent;
reason?: string;
coalesceMs?: number;
agentId?: string;
@@ -275,12 +315,14 @@ export function requestHeartbeatNow(opts?: {
heartbeat?: { target?: string };
}) {
queuePendingWakeReason({
reason: opts?.reason,
agentId: opts?.agentId,
sessionKey: opts?.sessionKey,
heartbeat: opts?.heartbeat,
source: opts.source,
intent: opts.intent,
reason: opts.reason,
agentId: opts.agentId,
sessionKey: opts.sessionKey,
heartbeat: opts.heartbeat,
});
schedule(opts?.coalesceMs ?? DEFAULT_COALESCE_MS, "normal");
schedule(opts.coalesceMs ?? DEFAULT_COALESCE_MS, "normal");
}
export function hasHeartbeatWakeHandler() {

1
src/plugin-sdk/test-helpers/plugin-runtime-mock.ts
View File

@@ -370,6 +370,7 @@ export function createPluginRuntimeMock(overrides: DeepPartial<PluginRuntime> =
},
system: {
enqueueSystemEvent: vi.fn() as unknown as PluginRuntime["system"]["enqueueSystemEvent"],
requestHeartbeat: vi.fn() as unknown as PluginRuntime["system"]["requestHeartbeat"],
requestHeartbeatNow: vi.fn() as unknown as PluginRuntime["system"]["requestHeartbeatNow"],
runHeartbeatOnce: vi.fn(async () => ({
status: "ran" as const,

42
src/plugins/runtime/index.test.ts
View File

@@ -6,7 +6,11 @@ import {
type OpenClawConfig,
} from "../../config/config.js";
import { onAgentEvent } from "../../infra/agent-events.js";
import { requestHeartbeatNow } from "../../infra/heartbeat-wake.js";
import {
requestHeartbeat,
resetHeartbeatWakeStateForTests,
setHeartbeatWakeHandler,
} from "../../infra/heartbeat-wake.js";
import * as execModule from "../../process/exec.js";
import { onSessionTranscriptUpdate } from "../../sessions/transcript-events.js";
import { VERSION } from "../../version.js";
@@ -161,10 +165,16 @@ describe("plugin runtime command execution", () => {
expected: onSessionTranscriptUpdate,
},
{
name: "exposes runtime.system.requestHeartbeatNow",
name: "exposes runtime.system.requestHeartbeat",
readValue: (runtime: ReturnType<typeof createPluginRuntime>) =>
runtime.system.requestHeartbeatNow,
expected: requestHeartbeatNow,
runtime.system.requestHeartbeat,
expected: requestHeartbeat,
},
{
name: "exposes deprecated runtime.system.requestHeartbeatNow",
readValue: (runtime: ReturnType<typeof createPluginRuntime>) =>
typeof runtime.system.requestHeartbeatNow,
expected: "function",
},
{
name: "exposes runtime.version from the shared VERSION constant",
@@ -175,6 +185,30 @@ describe("plugin runtime command execution", () => {
expectRuntimeValue(readValue, expected);
});
it("maps deprecated runtime.system.requestHeartbeatNow to a structured event wake", async () => {
vi.useFakeTimers();
resetHeartbeatWakeStateForTests();
const handler = vi.fn(async () => ({ status: "skipped" as const, reason: "disabled" }));
setHeartbeatWakeHandler(handler);
try {
createPluginRuntime().system.requestHeartbeatNow({
reason: "legacy-plugin",
coalesceMs: 0,
});
await vi.advanceTimersByTimeAsync(1);
expect(handler).toHaveBeenCalledWith(
expect.objectContaining({
source: "other",
intent: "event",
reason: "legacy-plugin",
}),
);
} finally {
resetHeartbeatWakeStateForTests();
vi.useRealTimers();
}
});
it("resolves thinking policy with configured model compat from runtime config", () => {
setRuntimeConfigSnapshot({
models: {

14
src/plugins/runtime/runtime-system.ts
View File

@@ -1,4 +1,4 @@
import { requestHeartbeatNow } from "../../infra/heartbeat-wake.js";
import { requestHeartbeat } from "../../infra/heartbeat-wake.js";
import { enqueueSystemEvent } from "../../infra/system-events.js";
import { runCommandWithTimeout } from "../../process/exec.js";
import { createLazyRuntimeMethod, createLazyRuntimeModule } from "../../shared/lazy-runtime.js";
@@ -15,8 +15,20 @@ const runHeartbeatOnceInternal = createLazyRuntimeMethod(
);
export function createRuntimeSystem(): PluginRuntime["system"] {
const requestHeartbeatNow: PluginRuntime["system"]["requestHeartbeatNow"] = (opts) =>
requestHeartbeat({
source: opts?.source ?? "other",
intent: opts?.intent ?? "event",
reason: opts?.reason,
coalesceMs: opts?.coalesceMs,
agentId: opts?.agentId,
sessionKey: opts?.sessionKey,
heartbeat: opts?.heartbeat,
});
return {
enqueueSystemEvent,
requestHeartbeat,
requestHeartbeatNow,
runHeartbeatOnce: (opts?: RunHeartbeatOnceOptions) => {
// Destructure to forward only the plugin-safe subset; prevent cfg/deps injection at runtime.

17
src/plugins/runtime/types-core.ts
View File

@@ -10,6 +10,16 @@ import type { PluginRuntimeTaskFlows, PluginRuntimeTaskRuns } from "./runtime-ta
export type { HeartbeatRunResult };
export type RuntimeRequestHeartbeatOptions = Parameters<
typeof import("../../infra/heartbeat-wake.js").requestHeartbeat
>[0];
export type RuntimeRequestHeartbeatNowOptions = Omit<
RuntimeRequestHeartbeatOptions,
"source" | "intent"
> &
Partial<Pick<RuntimeRequestHeartbeatOptions, "source" | "intent">>;
type RuntimeWriteConfigOptions = {
envSnapshotForRestore?: Record<string, string | undefined>;
expectedConfigPath?: string;
@@ -154,7 +164,12 @@ export type PluginRuntimeCore = {
};
system: {
enqueueSystemEvent: typeof import("../../infra/system-events.js").enqueueSystemEvent;
requestHeartbeatNow: typeof import("../../infra/heartbeat-wake.js").requestHeartbeatNow;
requestHeartbeat: typeof import("../../infra/heartbeat-wake.js").requestHeartbeat;
/**
* @deprecated Use `requestHeartbeat({ source, intent, reason })` so wake producers declare
* scheduler intent explicitly.
*/
requestHeartbeatNow: (opts?: RuntimeRequestHeartbeatNowOptions) => void;
/**
* Run a single heartbeat cycle immediately (bypassing the coalesce timer).
* Accepts an optional `heartbeat` config override so callers can force

10
src/tasks/task-registry.ts
View File

@@ -3,7 +3,7 @@ import { createRequire } from "node:module";
import type { OpenClawConfig } from "../config/types.openclaw.js";
import { onAgentEvent } from "../infra/agent-events.js";
import { formatErrorMessage } from "../infra/errors.js";
import { requestHeartbeatNow } from "../infra/heartbeat-wake.js";
import { requestHeartbeat } from "../infra/heartbeat-wake.js";
import { enqueueSystemEvent } from "../infra/system-events.js";
import { createSubsystemLogger } from "../logging/subsystem.js";
import { parseAgentSessionKey } from "../routing/session-key.js";
@@ -1077,7 +1077,9 @@ function queueTaskSystemEvent(task: TaskRecord, text: string) {
deliveryContext: owner.requesterOrigin,
trusted: false,
});
requestHeartbeatNow({
requestHeartbeat({
source: "background-task",
intent: "immediate",
reason: "background-task",
sessionKey: ownerKey,
});
@@ -1100,7 +1102,9 @@ function queueBlockedTaskFollowup(task: TaskRecord) {
deliveryContext: owner.requesterOrigin,
trusted: false,
});
requestHeartbeatNow({
requestHeartbeat({
source: "background-task-blocked",
intent: "immediate",
reason: "background-task-blocked",
sessionKey: ownerKey,
});

2
test/helpers/cron/service-regression-fixtures.ts
View File

@@ -85,7 +85,7 @@ export function createRunningCronServiceState(params: {
log: params.log,
nowMs: params.nowMs,
enqueueSystemEvent: vi.fn(),
requestHeartbeatNow: vi.fn(),
requestHeartbeat: vi.fn(),
runIsolatedAgentJob: vi.fn().mockResolvedValue({ status: "ok", summary: "ok" }),
});
state.running = true;

9
test/scripts/audit-seams.test.ts
View File

@@ -96,12 +96,17 @@ describe("audit-seams subagent seam classification", () => {
it("detects parent-stream seams for ACP spawn relays", () => {
const source = `
import { onAgentEvent } from "../infra/agent-events.js";
import { requestHeartbeatNow } from "../infra/heartbeat-wake.js";
import { requestHeartbeat } from "../infra/heartbeat-wake.js";
import { enqueueSystemEvent } from "../infra/system-events.js";
export function startAcpSpawnParentStreamRelay() {
onAgentEvent("agent-output", () => {});
requestHeartbeatNow({ sessionKey: "agent:main" });
requestHeartbeat({
source: "acp-spawn",
intent: "event",
reason: "acp:spawn:stream",
sessionKey: "agent:main",
});
enqueueSystemEvent("progress", { sessionKey: "agent:main", contextKey: "stream" });
return { streamTo: "parent" };
}