From 025cecf1f9f4e2c98b050188641ec386bfeefe05 Mon Sep 17 00:00:00 2001 From: DaigoSoup Date: Sat, 18 Jul 2026 08:19:23 +0800 Subject: [PATCH] fix: avoid false macOS gateway restart failures (#109955) * fix: wait through launchd KeepAlive restart throttling * ci: retrigger checks * test: make update restart test platform-independent --------- Co-authored-by: Josh Lehman --- src/cli/daemon-cli/lifecycle.test.ts | 10 ++++ src/cli/daemon-cli/lifecycle.ts | 2 + .../daemon-cli/restart-health-wait.test.ts | 47 +++++++++++++++++++ .../daemon-cli/restart-health.test-helpers.ts | 7 ++- src/cli/daemon-cli/restart-health.ts | 8 +++- .../update-command-service.test-support.ts | 10 ++++ src/cli/update-cli/update-command-service.ts | 24 ++++++++++ src/cli/update-cli/update-command.test.ts | 33 +++++++++++++ 8 files changed, 139 insertions(+), 2 deletions(-) diff --git a/src/cli/daemon-cli/lifecycle.test.ts b/src/cli/daemon-cli/lifecycle.test.ts index 2ebe9760ce96..b329b4ee8905 100644 --- a/src/cli/daemon-cli/lifecycle.test.ts +++ b/src/cli/daemon-cli/lifecycle.test.ts @@ -404,6 +404,16 @@ describe("runDaemonRestart health checks", () => { expect(waitParams.env?.OPENCLAW_SYSTEMD_UNIT).toBe("openclaw-gateway-maintenance.service"); }); + it("carries launchd KeepAlive supervision into managed restart health", async () => { + vi.spyOn(process, "platform", "get").mockReturnValue("darwin"); + + await runDaemonRestart({ json: true }); + + expect(waitForGatewayHealthyRestart).toHaveBeenCalledWith( + expect.objectContaining({ supervisorKeepsAlive: true }), + ); + }); + it("re-reads the installed service environment after restart repair", async () => { service.readCommand .mockResolvedValueOnce({ diff --git a/src/cli/daemon-cli/lifecycle.ts b/src/cli/daemon-cli/lifecycle.ts index cbccf995ff63..05bef3199cd6 100644 --- a/src/cli/daemon-cli/lifecycle.ts +++ b/src/cli/daemon-cli/lifecycle.ts @@ -684,6 +684,7 @@ export async function runDaemonRestart(opts: DaemonLifecycleOptions = {}): Promi delayMs: POST_RESTART_HEALTH_DELAY_MS, env: managedRestartContext.env, includeUnknownListenersAsStale: process.platform === "win32", + supervisorKeepsAlive: process.platform === "darwin", }); if (!health.healthy && health.staleGatewayPids.length > 0) { @@ -708,6 +709,7 @@ export async function runDaemonRestart(opts: DaemonLifecycleOptions = {}): Promi delayMs: POST_RESTART_HEALTH_DELAY_MS, env: managedRestartContext.env, includeUnknownListenersAsStale: process.platform === "win32", + supervisorKeepsAlive: process.platform === "darwin", }); } diff --git a/src/cli/daemon-cli/restart-health-wait.test.ts b/src/cli/daemon-cli/restart-health-wait.test.ts index 6643d7844323..d7f1ff25003b 100644 --- a/src/cli/daemon-cli/restart-health-wait.test.ts +++ b/src/cli/daemon-cli/restart-health-wait.test.ts @@ -264,6 +264,53 @@ describe("restart health", () => { expect(sleep).toHaveBeenCalledTimes(25); }); + it("keeps waiting while a launchd KeepAlive supervisor can retry", async () => { + Object.defineProperty(process, "platform", { value: "darwin", configurable: true }); + + const snapshot = await waitForStoppedFreeGatewayRestart({ supervisorKeepsAlive: true }); + + expect(snapshot.healthy).toBe(false); + expect(snapshot.runtime.status).toBe("stopped"); + expect(snapshot.portUsage.status).toBe("free"); + expect(snapshot.waitOutcome).toBe("timeout"); + expect(snapshot.elapsedMs).toBe(60_000); + expect(sleep).toHaveBeenCalledTimes(120); + }); + + it("accepts a launchd KeepAlive restart after the stopped-free grace window", async () => { + let runtimeReads = 0; + let portInspections = 0; + const service = { + readRuntime: vi.fn(async () => + ++runtimeReads >= 27 ? { status: "running", pid: 8000 } : { status: "stopped" }, + ), + } as unknown as GatewayService; + inspectPortUsage.mockImplementation(async () => + ++portInspections >= 27 + ? { + port: 18789, + status: "busy", + listeners: [{ pid: 8000, commandLine: "openclaw-gateway" }], + hints: [], + } + : { port: 18789, status: "free", listeners: [], hints: [] }, + ); + probeGateway.mockResolvedValue({ ok: true, close: null }); + + const { waitForGatewayHealthyRestart } = await import("./restart-health.js"); + const snapshot = await waitForGatewayHealthyRestart({ + service, + port: 18789, + attempts: 120, + delayMs: 500, + supervisorKeepsAlive: true, + }); + + expect(snapshot.healthy).toBe(true); + expect(snapshot.waitOutcome).toBe("healthy"); + expect(snapshot.elapsedMs).toBe(13_000); + }); + it("waits longer before stopped-free early exit on Windows", async () => { Object.defineProperty(process, "platform", { value: "win32", configurable: true }); diff --git a/src/cli/daemon-cli/restart-health.test-helpers.ts b/src/cli/daemon-cli/restart-health.test-helpers.ts index 05aca795d1bc..70e7cc8567f3 100644 --- a/src/cli/daemon-cli/restart-health.test-helpers.ts +++ b/src/cli/daemon-cli/restart-health.test-helpers.ts @@ -156,7 +156,11 @@ export async function inspectAmbiguousOwnershipWithProbe( }); } -export async function waitForStoppedFreeGatewayRestart() { +export async function waitForStoppedFreeGatewayRestart( + params: { + supervisorKeepsAlive?: boolean; + } = {}, +) { const attempts = process.platform === "win32" ? 360 : 120; const service = makeGatewayService({ status: "stopped" }); inspectPortUsage.mockResolvedValue({ @@ -172,6 +176,7 @@ export async function waitForStoppedFreeGatewayRestart() { port: 18789, attempts, delayMs: 500, + supervisorKeepsAlive: params.supervisorKeepsAlive, }); } diff --git a/src/cli/daemon-cli/restart-health.ts b/src/cli/daemon-cli/restart-health.ts index c6ed04991e7a..34d9c8913a27 100644 --- a/src/cli/daemon-cli/restart-health.ts +++ b/src/cli/daemon-cli/restart-health.ts @@ -279,6 +279,7 @@ export async function waitForGatewayHealthyRestart(params: { expectedVersion?: string | null; includeUnknownListenersAsStale?: boolean; requireRunningService?: boolean; + supervisorKeepsAlive?: boolean; isStartupMigrationActive?: typeof hasActiveStartupMigrationLease; }): Promise { const startedAtMs = performance.now(); @@ -328,7 +329,12 @@ export async function waitForGatewayHealthyRestart(params: { if (snapshot.staleGatewayPids.length > 0 && snapshot.runtime.status !== "running") { return withWaitContext(snapshot, "stale-pids", elapsedMs); } - if (shouldEarlyExitStoppedFree(snapshot, attempt, minAttemptForEarlyExit)) { + // launchd KeepAlive can report a transient stopped state while its throttle window runs. + // Let the bounded standard deadline decide failure when the caller knows supervision persists. + if ( + !params.supervisorKeepsAlive && + shouldEarlyExitStoppedFree(snapshot, attempt, minAttemptForEarlyExit) + ) { consecutiveStoppedFreeCount += 1; if (consecutiveStoppedFreeCount >= STOPPED_FREE_THRESHOLD) { return withWaitContext(snapshot, "stopped-free", elapsedMs); diff --git a/src/cli/update-cli/update-command-service.test-support.ts b/src/cli/update-cli/update-command-service.test-support.ts index d44e0ffb5bb0..37d07863847e 100644 --- a/src/cli/update-cli/update-command-service.test-support.ts +++ b/src/cli/update-cli/update-command-service.test-support.ts @@ -51,6 +51,10 @@ type UpdateCommandServiceTestApi = { health: GatewayRestartSnapshot; launchAgentRecovery: PostUpdateLaunchAgentRecoveryResult | null; }>; + hasLoadedLaunchdKeepAliveSupervisor(params: { + service: GatewayService; + env?: NodeJS.ProcessEnv; + }): Promise; shouldUseLegacyProcessRestartAfterUpdate(params: { updateMode: UpdateRunResult["mode"]; }): boolean; @@ -81,6 +85,12 @@ export async function recoverLaunchAgentAndRecheckGatewayHealth( return await getTestApi().recoverLaunchAgentAndRecheckGatewayHealth(params); } +export async function hasLoadedLaunchdKeepAliveSupervisor( + params: Parameters[0], +): Promise { + return await getTestApi().hasLoadedLaunchdKeepAliveSupervisor(params); +} + export function shouldUseLegacyProcessRestartAfterUpdate( params: Parameters[0], ): boolean { diff --git a/src/cli/update-cli/update-command-service.ts b/src/cli/update-cli/update-command-service.ts index 5b0a64c2e123..4d825f600a9f 100644 --- a/src/cli/update-cli/update-command-service.ts +++ b/src/cli/update-cli/update-command-service.ts @@ -202,10 +202,23 @@ async function recoverLaunchAgentAndRecheckGatewayHealth(params: { port: params.port, expectedVersion: params.expectedVersion, env: params.env, + supervisorKeepsAlive: true, }); return { health, launchAgentRecovery }; } +async function hasLoadedLaunchdKeepAliveSupervisor(params: { + service: GatewayService; + env?: NodeJS.ProcessEnv; +}): Promise { + if (process.platform !== "darwin") { + return false; + } + // OpenClaw's loaded LaunchAgent has canonical KeepAlive policy. Read this once before + // polling so an unloaded agent can still reach the existing recovery path promptly. + return await params.service.isLoaded({ env: params.env }).catch(() => false); +} + function formatPostUpdateGatewayRecoveryLine(platform: NodeJS.Platform): string { const restartCommand = replaceCliName(formatCliCommand("openclaw gateway restart"), CLI_NAME); const installCommand = replaceCliName( @@ -248,6 +261,7 @@ if (process.env.VITEST || process.env.NODE_ENV === "test") { formatPostUpdateGatewayRecoveryInstructions, recoverInstalledLaunchAgentAfterUpdate, recoverLaunchAgentAndRecheckGatewayHealth, + hasLoadedLaunchdKeepAliveSupervisor, shouldUseLegacyProcessRestartAfterUpdate, }; } @@ -1206,12 +1220,17 @@ export async function maybeRestartService(params: { } }; const service = resolveGatewayService(); + let supervisorKeepsAlive = await hasLoadedLaunchdKeepAliveSupervisor({ + service, + env: params.serviceEnv, + }); let health = await waitForGatewayHealthyRestart({ service, port: params.gatewayPort, expectedVersion: expectedGatewayVersion, env: params.serviceEnv, requireRunningService: opts.requireRunningService, + supervisorKeepsAlive, }); if (!health.healthy && health.staleGatewayPids.length > 0) { if (!params.opts.json) { @@ -1223,12 +1242,17 @@ export async function maybeRestartService(params: { } await terminateStaleGatewayPids(health.staleGatewayPids); await restartAfterStaleCleanup(); + supervisorKeepsAlive = await hasLoadedLaunchdKeepAliveSupervisor({ + service, + env: params.serviceEnv, + }); health = await waitForGatewayHealthyRestart({ service, port: params.gatewayPort, expectedVersion: expectedGatewayVersion, env: params.serviceEnv, requireRunningService: opts.requireRunningService, + supervisorKeepsAlive, }); } diff --git a/src/cli/update-cli/update-command.test.ts b/src/cli/update-cli/update-command.test.ts index 3be97d493e0a..55aca8ba8f25 100644 --- a/src/cli/update-cli/update-command.test.ts +++ b/src/cli/update-cli/update-command.test.ts @@ -4,6 +4,7 @@ import os from "node:os"; import path from "node:path"; import { describe, expect, it, vi } from "vitest"; import { resolveGatewayInstallEntrypoint } from "../../daemon/gateway-entrypoint.js"; +import type { GatewayService } from "../../daemon/service.js"; import type { UpdateRunResult } from "../../infra/update-runner.js"; import { updatePluginsAfterCoreUpdate } from "./update-command-plugins.js"; import { @@ -20,6 +21,7 @@ import { } from "./update-command-service.js"; import { formatPostUpdateGatewayRecoveryInstructions, + hasLoadedLaunchdKeepAliveSupervisor, recoverInstalledLaunchAgentAfterUpdate, recoverLaunchAgentAndRecheckGatewayHealth, shouldUseLegacyProcessRestartAfterUpdate, @@ -760,6 +762,7 @@ describe("recoverLaunchAgentAndRecheckGatewayHealth", () => { port: 18790, expectedVersion: "2026.5.3", env: { OPENCLAW_PROFILE: "stomme", OPENCLAW_PORT: "18790" }, + supervisorKeepsAlive: true, }); }); @@ -798,6 +801,36 @@ describe("recoverLaunchAgentAndRecheckGatewayHealth", () => { }); }); +describe("hasLoadedLaunchdKeepAliveSupervisor", () => { + it("requires a loaded LaunchAgent before extending restart health", async () => { + const platformSpy = vi.spyOn(process, "platform", "get").mockReturnValue("darwin"); + const isLoaded = vi.fn().mockResolvedValue(false); + const service = { isLoaded } as unknown as GatewayService; + + await expect( + hasLoadedLaunchdKeepAliveSupervisor({ service, env: { OPENCLAW_PROFILE: "work" } }), + ).resolves.toBe(false); + isLoaded.mockResolvedValue(true); + await expect(hasLoadedLaunchdKeepAliveSupervisor({ service })).resolves.toBe(true); + + platformSpy.mockRestore(); + }); + + it("does not inspect KeepAlive supervision outside macOS", async () => { + const platformSpy = vi.spyOn(process, "platform", "get").mockReturnValue("linux"); + const isLoaded = vi.fn().mockResolvedValue(true); + + await expect( + hasLoadedLaunchdKeepAliveSupervisor({ + service: { isLoaded } as unknown as GatewayService, + }), + ).resolves.toBe(false); + expect(isLoaded).not.toHaveBeenCalled(); + + platformSpy.mockRestore(); + }); +}); + describe("resolvePostCoreUpdateChildStdio", () => { it('returns "pipe" on Windows so the child never inherits the parent console handles', () => { // On Windows, stdio:"inherit" passes the parent's console HANDLE to the child process.