mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-25 05:01:17 +00:00
fix: avoid false macOS gateway restart failures (#109955)
* fix: wait through launchd KeepAlive restart throttling * ci: retrigger checks * test: make update restart test platform-independent --------- Co-authored-by: Josh Lehman <josh@martian.engineering>
This commit is contained in:
@@ -404,6 +404,16 @@ describe("runDaemonRestart health checks", () => {
|
||||
expect(waitParams.env?.OPENCLAW_SYSTEMD_UNIT).toBe("openclaw-gateway-maintenance.service");
|
||||
});
|
||||
|
||||
it("carries launchd KeepAlive supervision into managed restart health", async () => {
|
||||
vi.spyOn(process, "platform", "get").mockReturnValue("darwin");
|
||||
|
||||
await runDaemonRestart({ json: true });
|
||||
|
||||
expect(waitForGatewayHealthyRestart).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ supervisorKeepsAlive: true }),
|
||||
);
|
||||
});
|
||||
|
||||
it("re-reads the installed service environment after restart repair", async () => {
|
||||
service.readCommand
|
||||
.mockResolvedValueOnce({
|
||||
|
||||
@@ -684,6 +684,7 @@ export async function runDaemonRestart(opts: DaemonLifecycleOptions = {}): Promi
|
||||
delayMs: POST_RESTART_HEALTH_DELAY_MS,
|
||||
env: managedRestartContext.env,
|
||||
includeUnknownListenersAsStale: process.platform === "win32",
|
||||
supervisorKeepsAlive: process.platform === "darwin",
|
||||
});
|
||||
|
||||
if (!health.healthy && health.staleGatewayPids.length > 0) {
|
||||
@@ -708,6 +709,7 @@ export async function runDaemonRestart(opts: DaemonLifecycleOptions = {}): Promi
|
||||
delayMs: POST_RESTART_HEALTH_DELAY_MS,
|
||||
env: managedRestartContext.env,
|
||||
includeUnknownListenersAsStale: process.platform === "win32",
|
||||
supervisorKeepsAlive: process.platform === "darwin",
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -264,6 +264,53 @@ describe("restart health", () => {
|
||||
expect(sleep).toHaveBeenCalledTimes(25);
|
||||
});
|
||||
|
||||
it("keeps waiting while a launchd KeepAlive supervisor can retry", async () => {
|
||||
Object.defineProperty(process, "platform", { value: "darwin", configurable: true });
|
||||
|
||||
const snapshot = await waitForStoppedFreeGatewayRestart({ supervisorKeepsAlive: true });
|
||||
|
||||
expect(snapshot.healthy).toBe(false);
|
||||
expect(snapshot.runtime.status).toBe("stopped");
|
||||
expect(snapshot.portUsage.status).toBe("free");
|
||||
expect(snapshot.waitOutcome).toBe("timeout");
|
||||
expect(snapshot.elapsedMs).toBe(60_000);
|
||||
expect(sleep).toHaveBeenCalledTimes(120);
|
||||
});
|
||||
|
||||
it("accepts a launchd KeepAlive restart after the stopped-free grace window", async () => {
|
||||
let runtimeReads = 0;
|
||||
let portInspections = 0;
|
||||
const service = {
|
||||
readRuntime: vi.fn(async () =>
|
||||
++runtimeReads >= 27 ? { status: "running", pid: 8000 } : { status: "stopped" },
|
||||
),
|
||||
} as unknown as GatewayService;
|
||||
inspectPortUsage.mockImplementation(async () =>
|
||||
++portInspections >= 27
|
||||
? {
|
||||
port: 18789,
|
||||
status: "busy",
|
||||
listeners: [{ pid: 8000, commandLine: "openclaw-gateway" }],
|
||||
hints: [],
|
||||
}
|
||||
: { port: 18789, status: "free", listeners: [], hints: [] },
|
||||
);
|
||||
probeGateway.mockResolvedValue({ ok: true, close: null });
|
||||
|
||||
const { waitForGatewayHealthyRestart } = await import("./restart-health.js");
|
||||
const snapshot = await waitForGatewayHealthyRestart({
|
||||
service,
|
||||
port: 18789,
|
||||
attempts: 120,
|
||||
delayMs: 500,
|
||||
supervisorKeepsAlive: true,
|
||||
});
|
||||
|
||||
expect(snapshot.healthy).toBe(true);
|
||||
expect(snapshot.waitOutcome).toBe("healthy");
|
||||
expect(snapshot.elapsedMs).toBe(13_000);
|
||||
});
|
||||
|
||||
it("waits longer before stopped-free early exit on Windows", async () => {
|
||||
Object.defineProperty(process, "platform", { value: "win32", configurable: true });
|
||||
|
||||
|
||||
@@ -156,7 +156,11 @@ export async function inspectAmbiguousOwnershipWithProbe(
|
||||
});
|
||||
}
|
||||
|
||||
export async function waitForStoppedFreeGatewayRestart() {
|
||||
export async function waitForStoppedFreeGatewayRestart(
|
||||
params: {
|
||||
supervisorKeepsAlive?: boolean;
|
||||
} = {},
|
||||
) {
|
||||
const attempts = process.platform === "win32" ? 360 : 120;
|
||||
const service = makeGatewayService({ status: "stopped" });
|
||||
inspectPortUsage.mockResolvedValue({
|
||||
@@ -172,6 +176,7 @@ export async function waitForStoppedFreeGatewayRestart() {
|
||||
port: 18789,
|
||||
attempts,
|
||||
delayMs: 500,
|
||||
supervisorKeepsAlive: params.supervisorKeepsAlive,
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -279,6 +279,7 @@ export async function waitForGatewayHealthyRestart(params: {
|
||||
expectedVersion?: string | null;
|
||||
includeUnknownListenersAsStale?: boolean;
|
||||
requireRunningService?: boolean;
|
||||
supervisorKeepsAlive?: boolean;
|
||||
isStartupMigrationActive?: typeof hasActiveStartupMigrationLease;
|
||||
}): Promise<GatewayRestartSnapshot> {
|
||||
const startedAtMs = performance.now();
|
||||
@@ -328,7 +329,12 @@ export async function waitForGatewayHealthyRestart(params: {
|
||||
if (snapshot.staleGatewayPids.length > 0 && snapshot.runtime.status !== "running") {
|
||||
return withWaitContext(snapshot, "stale-pids", elapsedMs);
|
||||
}
|
||||
if (shouldEarlyExitStoppedFree(snapshot, attempt, minAttemptForEarlyExit)) {
|
||||
// launchd KeepAlive can report a transient stopped state while its throttle window runs.
|
||||
// Let the bounded standard deadline decide failure when the caller knows supervision persists.
|
||||
if (
|
||||
!params.supervisorKeepsAlive &&
|
||||
shouldEarlyExitStoppedFree(snapshot, attempt, minAttemptForEarlyExit)
|
||||
) {
|
||||
consecutiveStoppedFreeCount += 1;
|
||||
if (consecutiveStoppedFreeCount >= STOPPED_FREE_THRESHOLD) {
|
||||
return withWaitContext(snapshot, "stopped-free", elapsedMs);
|
||||
|
||||
Reference in New Issue
Block a user