fix(gateway): reset channel restart counter after a stable run (#101413)

* fix(gateway): reset channel restart counter after a stable run

* fix(gateway): measure stable channel runtime precisely

Co-authored-by: Ubuntu <azureuser@clinton-dev-vm.tail405bf7.ts.net>

* refactor(gateway): collapse stable-run timing state

Co-authored-by: Ubuntu <azureuser@clinton-dev-vm.tail405bf7.ts.net>

---------

Co-authored-by: Ubuntu <azureuser@clinton-dev-vm.tail405bf7.ts.net>
Co-authored-by: Peter Steinberger <steipete@gmail.com>

src/gateway/server-channels.test.ts

@@ -31,7 +31,8 @@ const hoisted = vi.hoisted(() => {
       );
     });
   });
-  return { computeBackoff, sleepWithAbort };
+  const startChannelApprovalHandlerBootstrap = vi.fn(async () => async () => {});
+  return { computeBackoff, sleepWithAbort, startChannelApprovalHandlerBootstrap };
 });
 
 vi.mock("../infra/backoff.js", () => ({
@@ -39,6 +40,10 @@ vi.mock("../infra/backoff.js", () => ({
   sleepWithAbort: hoisted.sleepWithAbort,
 }));
 
+vi.mock("../infra/approval-handler-bootstrap.js", () => ({
+  startChannelApprovalHandlerBootstrap: hoisted.startChannelApprovalHandlerBootstrap,
+}));
+
 type TestAccount = {
   enabled?: boolean;
   configured?: boolean;
@@ -218,6 +223,7 @@ function createManager(options?: {
 }
 
 describe("server-channels auto restart", () => {
+  const stableChannelRunMs = 5 * 60_000;
   let previousRegistry: PluginRegistry | null = null;
 
   beforeEach(() => {
@@ -226,6 +232,8 @@ describe("server-channels auto restart", () => {
     vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout", "Date"] });
     hoisted.computeBackoff.mockClear();
     hoisted.sleepWithAbort.mockClear();
+    hoisted.startChannelApprovalHandlerBootstrap.mockReset();
+    hoisted.startChannelApprovalHandlerBootstrap.mockResolvedValue(async () => {});
   });
 
   afterEach(async () => {
@@ -269,6 +277,76 @@ describe("server-channels auto restart", () => {
     expect(startAccount).toHaveBeenCalledTimes(11);
   });
 
+  it.each(["resolve", "reject"] as const)(
+    "resets the restart counter after a stable run that ends with %s",
+    async (outcome) => {
+      const attemptsAtStart: number[] = [];
+      let calls = 0;
+      const startAccount = vi.fn(async (ctx: ChannelGatewayContext<TestAccount>) => {
+        attemptsAtStart.push(ctx.getStatus().reconnectAttempts ?? 0);
+        calls += 1;
+        if (calls === 3) {
+          await new Promise<void>((resolve) => {
+            setTimeout(resolve, stableChannelRunMs + 1_000);
+          });
+          if (outcome === "reject") {
+            throw new Error("stable run ended");
+          }
+        }
+      });
+      installTestRegistry(createTestPlugin({ startAccount }));
+      const manager = createManager();
+
+      await manager.startChannels();
+      // Two instant exits accumulate attempts 1 and 2; the third run is stable.
+      await advanceTimersUntil(
+        () => startAccount.mock.calls.length >= 3,
+        "expected two crash-loop restarts before the stable run",
+        { stepMs: 10, maxMs: 500 },
+      );
+      await advanceTimersUntil(
+        () => startAccount.mock.calls.length >= 4,
+        "expected an auto-restart after the stable run exited",
+        { stepMs: 30_000, maxMs: 4 * stableChannelRunMs },
+      );
+
+      expect(attemptsAtStart[3]).toBe(1);
+    },
+  );
+
+  it("does not count slow cleanup as a stable channel run", async () => {
+    const attemptsAtStart: number[] = [];
+    const startAccount = vi.fn(async (ctx: ChannelGatewayContext<TestAccount>) => {
+      attemptsAtStart.push(ctx.getStatus().reconnectAttempts ?? 0);
+    });
+    hoisted.startChannelApprovalHandlerBootstrap.mockImplementation(async () => {
+      const run = hoisted.startChannelApprovalHandlerBootstrap.mock.calls.length;
+      return async () => {
+        if (run === 3) {
+          await new Promise<void>((resolve) => {
+            setTimeout(resolve, stableChannelRunMs + 1_000);
+          });
+        }
+      };
+    });
+    installTestRegistry(createTestPlugin({ startAccount }));
+    const manager = createManager();
+
+    await manager.startChannels();
+    await advanceTimersUntil(
+      () => startAccount.mock.calls.length >= 3,
+      "expected two crash-loop restarts before slow cleanup",
+      { stepMs: 10, maxMs: 500 },
+    );
+    await advanceTimersUntil(
+      () => startAccount.mock.calls.length >= 4,
+      "expected an auto-restart after slow cleanup",
+      { stepMs: 30_000, maxMs: 4 * stableChannelRunMs },
+    );
+
+    expect(attemptsAtStart[3]).toBe(3);
+  });
+
   it("records a clean channel monitor exit before auto-restart", async () => {
     const startAccount = vi.fn(async () => {});
     installTestRegistry(createTestPlugin({ startAccount }));

src/gateway/server-channels.ts

@@ -36,6 +36,7 @@ const CHANNEL_RESTART_POLICY: BackoffPolicy = {
   jitter: 0.1,
 };
 const MAX_RESTART_ATTEMPTS = 10;
+const CHANNEL_STABLE_RUN_MS = CHANNEL_RESTART_POLICY.maxMs;
 const CHANNEL_STOP_ABORT_TIMEOUT_MS = 5_000;
 const CHANNEL_STARTUP_CONCURRENCY = 4;
 
@@ -623,6 +624,7 @@ export function createChannelManager(opts: ChannelManagerOptions): ChannelManage
           } catch (error) {
             log.error?.(`[${id}] native approval bootstrap failed: ${formatErrorMessage(error)}`);
           }
+          let channelRunDurationMs: number | undefined;
           setRuntime(channelId, id, {
             accountId: id,
             enabled: true,
@@ -647,21 +649,31 @@ export function createChannelManager(opts: ChannelManagerOptions): ChannelManage
               if (abort.signal.aborted || manuallyStopped.has(rKey)) {
                 return;
               }
-              const runStartAccount = () =>
-                startAccount({
-                  cfg,
-                  accountId: id,
-                  account,
-                  runtime,
-                  abortSignal: abort.signal,
-                  log,
-                  getStatus: () => getRuntime(channelId, id),
-                  setStatus: (next) =>
-                    isCurrentTask()
-                      ? setRuntimeFromTaskStatus(channelId, id, next, abort.signal)
-                      : getRuntime(channelId, id),
-                  ...(channelRuntimeForTask ? { channelRuntime: channelRuntimeForTask } : {}),
-                });
+              const runStartAccount = () => {
+                const startedAt = Date.now();
+                const recordDuration = () => {
+                  channelRunDurationMs = Date.now() - startedAt;
+                };
+                try {
+                  return startAccount({
+                    cfg,
+                    accountId: id,
+                    account,
+                    runtime,
+                    abortSignal: abort.signal,
+                    log,
+                    getStatus: () => getRuntime(channelId, id),
+                    setStatus: (next) =>
+                      isCurrentTask()
+                        ? setRuntimeFromTaskStatus(channelId, id, next, abort.signal)
+                        : getRuntime(channelId, id),
+                    ...(channelRuntimeForTask ? { channelRuntime: channelRuntimeForTask } : {}),
+                  }).finally(recordDuration);
+                } catch (error) {
+                  recordDuration();
+                  throw error;
+                }
+              };
               const routeRegistry = getPluginHttpRouteRegistry?.();
               startAccountTask = routeRegistry
                 ? withPluginHttpRouteRegistry(routeRegistry, runStartAccount)
@@ -761,6 +773,14 @@ export function createChannelManager(opts: ChannelManagerOptions): ChannelManage
                 }
                 return;
               }
+              // Only plugin task lifetime counts. Deferred handoff and cleanup must not
+              // make a short crash look stable and erase crash-loop attempts.
+              if (
+                channelRunDurationMs !== undefined &&
+                channelRunDurationMs >= CHANNEL_STABLE_RUN_MS
+              ) {
+                restartAttempts.delete(rKey);
+              }
               const attempt = (restartAttempts.get(rKey) ?? 0) + 1;
               restartAttempts.set(rKey, attempt);
               if (attempt > MAX_RESTART_ATTEMPTS) {