fix: avoid false macOS gateway restart failures (#109955)

* fix: wait through launchd KeepAlive restart throttling

* ci: retrigger checks

* test: make update restart test platform-independent

---------

Co-authored-by: Josh Lehman <josh@martian.engineering>
This commit is contained in:
DaigoSoup
2026-07-18 08:19:23 +08:00
committed by GitHub
parent 13c0e7492a
commit 025cecf1f9
8 changed files with 139 additions and 2 deletions

View File

@@ -404,6 +404,16 @@ describe("runDaemonRestart health checks", () => {
expect(waitParams.env?.OPENCLAW_SYSTEMD_UNIT).toBe("openclaw-gateway-maintenance.service");
});
it("carries launchd KeepAlive supervision into managed restart health", async () => {
vi.spyOn(process, "platform", "get").mockReturnValue("darwin");
await runDaemonRestart({ json: true });
expect(waitForGatewayHealthyRestart).toHaveBeenCalledWith(
expect.objectContaining({ supervisorKeepsAlive: true }),
);
});
it("re-reads the installed service environment after restart repair", async () => {
service.readCommand
.mockResolvedValueOnce({

View File

@@ -684,6 +684,7 @@ export async function runDaemonRestart(opts: DaemonLifecycleOptions = {}): Promi
delayMs: POST_RESTART_HEALTH_DELAY_MS,
env: managedRestartContext.env,
includeUnknownListenersAsStale: process.platform === "win32",
supervisorKeepsAlive: process.platform === "darwin",
});
if (!health.healthy && health.staleGatewayPids.length > 0) {
@@ -708,6 +709,7 @@ export async function runDaemonRestart(opts: DaemonLifecycleOptions = {}): Promi
delayMs: POST_RESTART_HEALTH_DELAY_MS,
env: managedRestartContext.env,
includeUnknownListenersAsStale: process.platform === "win32",
supervisorKeepsAlive: process.platform === "darwin",
});
}

View File

@@ -264,6 +264,53 @@ describe("restart health", () => {
expect(sleep).toHaveBeenCalledTimes(25);
});
it("keeps waiting while a launchd KeepAlive supervisor can retry", async () => {
Object.defineProperty(process, "platform", { value: "darwin", configurable: true });
const snapshot = await waitForStoppedFreeGatewayRestart({ supervisorKeepsAlive: true });
expect(snapshot.healthy).toBe(false);
expect(snapshot.runtime.status).toBe("stopped");
expect(snapshot.portUsage.status).toBe("free");
expect(snapshot.waitOutcome).toBe("timeout");
expect(snapshot.elapsedMs).toBe(60_000);
expect(sleep).toHaveBeenCalledTimes(120);
});
it("accepts a launchd KeepAlive restart after the stopped-free grace window", async () => {
let runtimeReads = 0;
let portInspections = 0;
const service = {
readRuntime: vi.fn(async () =>
++runtimeReads >= 27 ? { status: "running", pid: 8000 } : { status: "stopped" },
),
} as unknown as GatewayService;
inspectPortUsage.mockImplementation(async () =>
++portInspections >= 27
? {
port: 18789,
status: "busy",
listeners: [{ pid: 8000, commandLine: "openclaw-gateway" }],
hints: [],
}
: { port: 18789, status: "free", listeners: [], hints: [] },
);
probeGateway.mockResolvedValue({ ok: true, close: null });
const { waitForGatewayHealthyRestart } = await import("./restart-health.js");
const snapshot = await waitForGatewayHealthyRestart({
service,
port: 18789,
attempts: 120,
delayMs: 500,
supervisorKeepsAlive: true,
});
expect(snapshot.healthy).toBe(true);
expect(snapshot.waitOutcome).toBe("healthy");
expect(snapshot.elapsedMs).toBe(13_000);
});
it("waits longer before stopped-free early exit on Windows", async () => {
Object.defineProperty(process, "platform", { value: "win32", configurable: true });

View File

@@ -156,7 +156,11 @@ export async function inspectAmbiguousOwnershipWithProbe(
});
}
export async function waitForStoppedFreeGatewayRestart() {
export async function waitForStoppedFreeGatewayRestart(
params: {
supervisorKeepsAlive?: boolean;
} = {},
) {
const attempts = process.platform === "win32" ? 360 : 120;
const service = makeGatewayService({ status: "stopped" });
inspectPortUsage.mockResolvedValue({
@@ -172,6 +176,7 @@ export async function waitForStoppedFreeGatewayRestart() {
port: 18789,
attempts,
delayMs: 500,
supervisorKeepsAlive: params.supervisorKeepsAlive,
});
}

View File

@@ -279,6 +279,7 @@ export async function waitForGatewayHealthyRestart(params: {
expectedVersion?: string | null;
includeUnknownListenersAsStale?: boolean;
requireRunningService?: boolean;
supervisorKeepsAlive?: boolean;
isStartupMigrationActive?: typeof hasActiveStartupMigrationLease;
}): Promise<GatewayRestartSnapshot> {
const startedAtMs = performance.now();
@@ -328,7 +329,12 @@ export async function waitForGatewayHealthyRestart(params: {
if (snapshot.staleGatewayPids.length > 0 && snapshot.runtime.status !== "running") {
return withWaitContext(snapshot, "stale-pids", elapsedMs);
}
if (shouldEarlyExitStoppedFree(snapshot, attempt, minAttemptForEarlyExit)) {
// launchd KeepAlive can report a transient stopped state while its throttle window runs.
// Let the bounded standard deadline decide failure when the caller knows supervision persists.
if (
!params.supervisorKeepsAlive &&
shouldEarlyExitStoppedFree(snapshot, attempt, minAttemptForEarlyExit)
) {
consecutiveStoppedFreeCount += 1;
if (consecutiveStoppedFreeCount >= STOPPED_FREE_THRESHOLD) {
return withWaitContext(snapshot, "stopped-free", elapsedMs);