diff --git a/src/gateway/server-channels.test.ts b/src/gateway/server-channels.test.ts index 3c9596234aac..cf775a8a02b6 100644 --- a/src/gateway/server-channels.test.ts +++ b/src/gateway/server-channels.test.ts @@ -31,7 +31,8 @@ const hoisted = vi.hoisted(() => { ); }); }); - return { computeBackoff, sleepWithAbort }; + const startChannelApprovalHandlerBootstrap = vi.fn(async () => async () => {}); + return { computeBackoff, sleepWithAbort, startChannelApprovalHandlerBootstrap }; }); vi.mock("../infra/backoff.js", () => ({ @@ -39,6 +40,10 @@ vi.mock("../infra/backoff.js", () => ({ sleepWithAbort: hoisted.sleepWithAbort, })); +vi.mock("../infra/approval-handler-bootstrap.js", () => ({ + startChannelApprovalHandlerBootstrap: hoisted.startChannelApprovalHandlerBootstrap, +})); + type TestAccount = { enabled?: boolean; configured?: boolean; @@ -218,6 +223,7 @@ function createManager(options?: { } describe("server-channels auto restart", () => { + const stableChannelRunMs = 5 * 60_000; let previousRegistry: PluginRegistry | null = null; beforeEach(() => { @@ -226,6 +232,8 @@ describe("server-channels auto restart", () => { vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout", "Date"] }); hoisted.computeBackoff.mockClear(); hoisted.sleepWithAbort.mockClear(); + hoisted.startChannelApprovalHandlerBootstrap.mockReset(); + hoisted.startChannelApprovalHandlerBootstrap.mockResolvedValue(async () => {}); }); afterEach(async () => { @@ -269,6 +277,76 @@ describe("server-channels auto restart", () => { expect(startAccount).toHaveBeenCalledTimes(11); }); + it.each(["resolve", "reject"] as const)( + "resets the restart counter after a stable run that ends with %s", + async (outcome) => { + const attemptsAtStart: number[] = []; + let calls = 0; + const startAccount = vi.fn(async (ctx: ChannelGatewayContext) => { + attemptsAtStart.push(ctx.getStatus().reconnectAttempts ?? 0); + calls += 1; + if (calls === 3) { + await new Promise((resolve) => { + setTimeout(resolve, stableChannelRunMs + 1_000); + }); + if (outcome === "reject") { + throw new Error("stable run ended"); + } + } + }); + installTestRegistry(createTestPlugin({ startAccount })); + const manager = createManager(); + + await manager.startChannels(); + // Two instant exits accumulate attempts 1 and 2; the third run is stable. + await advanceTimersUntil( + () => startAccount.mock.calls.length >= 3, + "expected two crash-loop restarts before the stable run", + { stepMs: 10, maxMs: 500 }, + ); + await advanceTimersUntil( + () => startAccount.mock.calls.length >= 4, + "expected an auto-restart after the stable run exited", + { stepMs: 30_000, maxMs: 4 * stableChannelRunMs }, + ); + + expect(attemptsAtStart[3]).toBe(1); + }, + ); + + it("does not count slow cleanup as a stable channel run", async () => { + const attemptsAtStart: number[] = []; + const startAccount = vi.fn(async (ctx: ChannelGatewayContext) => { + attemptsAtStart.push(ctx.getStatus().reconnectAttempts ?? 0); + }); + hoisted.startChannelApprovalHandlerBootstrap.mockImplementation(async () => { + const run = hoisted.startChannelApprovalHandlerBootstrap.mock.calls.length; + return async () => { + if (run === 3) { + await new Promise((resolve) => { + setTimeout(resolve, stableChannelRunMs + 1_000); + }); + } + }; + }); + installTestRegistry(createTestPlugin({ startAccount })); + const manager = createManager(); + + await manager.startChannels(); + await advanceTimersUntil( + () => startAccount.mock.calls.length >= 3, + "expected two crash-loop restarts before slow cleanup", + { stepMs: 10, maxMs: 500 }, + ); + await advanceTimersUntil( + () => startAccount.mock.calls.length >= 4, + "expected an auto-restart after slow cleanup", + { stepMs: 30_000, maxMs: 4 * stableChannelRunMs }, + ); + + expect(attemptsAtStart[3]).toBe(3); + }); + it("records a clean channel monitor exit before auto-restart", async () => { const startAccount = vi.fn(async () => {}); installTestRegistry(createTestPlugin({ startAccount })); diff --git a/src/gateway/server-channels.ts b/src/gateway/server-channels.ts index b8226c5ea11a..2d1a5b1ea846 100644 --- a/src/gateway/server-channels.ts +++ b/src/gateway/server-channels.ts @@ -36,6 +36,7 @@ const CHANNEL_RESTART_POLICY: BackoffPolicy = { jitter: 0.1, }; const MAX_RESTART_ATTEMPTS = 10; +const CHANNEL_STABLE_RUN_MS = CHANNEL_RESTART_POLICY.maxMs; const CHANNEL_STOP_ABORT_TIMEOUT_MS = 5_000; const CHANNEL_STARTUP_CONCURRENCY = 4; @@ -623,6 +624,7 @@ export function createChannelManager(opts: ChannelManagerOptions): ChannelManage } catch (error) { log.error?.(`[${id}] native approval bootstrap failed: ${formatErrorMessage(error)}`); } + let channelRunDurationMs: number | undefined; setRuntime(channelId, id, { accountId: id, enabled: true, @@ -647,21 +649,31 @@ export function createChannelManager(opts: ChannelManagerOptions): ChannelManage if (abort.signal.aborted || manuallyStopped.has(rKey)) { return; } - const runStartAccount = () => - startAccount({ - cfg, - accountId: id, - account, - runtime, - abortSignal: abort.signal, - log, - getStatus: () => getRuntime(channelId, id), - setStatus: (next) => - isCurrentTask() - ? setRuntimeFromTaskStatus(channelId, id, next, abort.signal) - : getRuntime(channelId, id), - ...(channelRuntimeForTask ? { channelRuntime: channelRuntimeForTask } : {}), - }); + const runStartAccount = () => { + const startedAt = Date.now(); + const recordDuration = () => { + channelRunDurationMs = Date.now() - startedAt; + }; + try { + return startAccount({ + cfg, + accountId: id, + account, + runtime, + abortSignal: abort.signal, + log, + getStatus: () => getRuntime(channelId, id), + setStatus: (next) => + isCurrentTask() + ? setRuntimeFromTaskStatus(channelId, id, next, abort.signal) + : getRuntime(channelId, id), + ...(channelRuntimeForTask ? { channelRuntime: channelRuntimeForTask } : {}), + }).finally(recordDuration); + } catch (error) { + recordDuration(); + throw error; + } + }; const routeRegistry = getPluginHttpRouteRegistry?.(); startAccountTask = routeRegistry ? withPluginHttpRouteRegistry(routeRegistry, runStartAccount) @@ -761,6 +773,14 @@ export function createChannelManager(opts: ChannelManagerOptions): ChannelManage } return; } + // Only plugin task lifetime counts. Deferred handoff and cleanup must not + // make a short crash look stable and erase crash-loop attempts. + if ( + channelRunDurationMs !== undefined && + channelRunDurationMs >= CHANNEL_STABLE_RUN_MS + ) { + restartAttempts.delete(rKey); + } const attempt = (restartAttempts.get(rKey) ?? 0) + 1; restartAttempts.set(rKey, attempt); if (attempt > MAX_RESTART_ATTEMPTS) {