fix(daemon): keep launchd stop persistent without reinstall

This commit is contained in:
Nimrod Gutman
2026-04-10 21:14:17 +03:00
committed by Peter Steinberger
parent 31a0b7bd42
commit 23d9a100c4
5 changed files with 262 additions and 19 deletions

View File

@@ -40,6 +40,7 @@ describe("scheduleDetachedLaunchdRestartHandoff", () => {
expect(args[2]).toBe("openclaw-launchd-restart-handoff");
expect(args[6]).toBe("9876");
expect(args[1]).toContain('while kill -0 "$wait_pid" >/dev/null 2>&1; do');
expect(args[1]).toContain('launchctl enable "$service_target" >/dev/null 2>&1');
expect(args[1]).toContain('launchctl kickstart -k "$service_target" >/dev/null 2>&1');
expect(args[1]).not.toContain("sleep 1");
expect(unrefMock).toHaveBeenCalledTimes(1);

View File

@@ -79,8 +79,8 @@ fi
domain="$2"
plist_path="$3"
${waitForCallerPid}
launchctl enable "$service_target" >/dev/null 2>&1
if ! launchctl kickstart -k "$service_target" >/dev/null 2>&1; then
launchctl enable "$service_target" >/dev/null 2>&1
if launchctl bootstrap "$domain" "$plist_path" >/dev/null 2>&1; then
launchctl kickstart -k "$service_target" >/dev/null 2>&1 || true
fi
@@ -91,11 +91,12 @@ fi
return `service_target="$1"
domain="$2"
plist_path="$3"
label="$(basename "$service_target")"
${waitForCallerPid}
if ! launchctl start "$service_target" >/dev/null 2>&1; then
launchctl enable "$service_target" >/dev/null 2>&1
launchctl enable "$service_target" >/dev/null 2>&1
if ! launchctl start "$label" >/dev/null 2>&1; then
if launchctl bootstrap "$domain" "$plist_path" >/dev/null 2>&1; then
launchctl start "$service_target" >/dev/null 2>&1 || launchctl kickstart -k "$service_target" >/dev/null 2>&1 || true
launchctl start "$label" >/dev/null 2>&1 || launchctl kickstart -k "$service_target" >/dev/null 2>&1 || true
else
launchctl kickstart -k "$service_target" >/dev/null 2>&1 || true
fi

View File

@@ -10,6 +10,7 @@ import {
readLaunchAgentRuntime,
restartLaunchAgent,
resolveLaunchAgentPlistPath,
stopLaunchAgent,
uninstallLaunchAgent,
} from "./launchd.js";
import type { GatewayServiceEnv } from "./service-types.js";
@@ -85,6 +86,30 @@ async function waitForRunningRuntime(params: {
);
}
async function waitForNotRunningRuntime(params: {
env: GatewayServiceEnv;
timeoutMs?: number;
}): Promise<void> {
const timeoutMs = params.timeoutMs ?? WAIT_TIMEOUT_MS;
const deadline = Date.now() + timeoutMs;
let lastStatus = "unknown";
let lastPid: number | undefined;
while (Date.now() < deadline) {
const runtime = await readLaunchAgentRuntime(params.env);
lastStatus = runtime.status ?? "unknown";
lastPid = runtime.pid;
if (runtime.status !== "running" && runtime.pid === undefined) {
return;
}
await new Promise((resolve) => {
setTimeout(resolve, WAIT_INTERVAL_MS);
});
}
throw new Error(
`Timed out waiting for launchd runtime to stop (status=${lastStatus}, pid=${lastPid ?? "none"})`,
);
}
describeLaunchdIntegration("launchd integration", () => {
let env: GatewayServiceEnv | undefined;
let homeDir = "";
@@ -142,4 +167,36 @@ describeLaunchdIntegration("launchd integration", () => {
expect(after.pid).not.toBe(before.pid);
await fs.access(resolveLaunchAgentPlistPath(launchEnv));
}, 60_000);
it("stops persistently without reinstall and restarts later", async () => {
if (!env) {
throw new Error("launchd integration env was not initialized");
}
const launchEnv = env;
try {
await withTimeout({
run: async () => {
await installLaunchAgent({
env: launchEnv,
stdout,
programArguments: [process.execPath, "-e", "setInterval(() => {}, 1000);"],
});
await waitForRunningRuntime({ env: launchEnv });
},
timeoutMs: STARTUP_TIMEOUT_MS,
message: "Timed out initializing launchd integration runtime",
});
} catch {
return;
}
const before = await waitForRunningRuntime({ env: launchEnv });
await stopLaunchAgent({ env: launchEnv, stdout });
await waitForNotRunningRuntime({ env: launchEnv });
await restartLaunchAgent({ env: launchEnv, stdout });
const after = await waitForRunningRuntime({ env: launchEnv, pidNot: before.pid });
expect(after.pid).toBeGreaterThan(1);
expect(after.pid).not.toBe(before.pid);
await fs.access(resolveLaunchAgentPlistPath(launchEnv));
}, 60_000);
});

View File

@@ -11,6 +11,7 @@ import {
repairLaunchAgentBootstrap,
restartLaunchAgent,
resolveLaunchAgentPlistPath,
stopLaunchAgent,
} from "./launchd.js";
const state = vi.hoisted(() => ({
@@ -22,6 +23,15 @@ const state = vi.hoisted(() => ({
bootstrapCode: 1,
kickstartError: "",
kickstartFailuresRemaining: 0,
disableError: "",
disableCode: 1,
stopError: "",
stopCode: 1,
bootoutError: "",
bootoutCode: 1,
serviceLoaded: true,
serviceRunning: true,
stopLeavesRunning: false,
dirs: new Set<string>(),
dirModes: new Map<string, number>(),
files: new Map<string, string>(),
@@ -78,14 +88,56 @@ vi.mock("./exec-file.js", () => ({
state.printNotLoadedRemaining -= 1;
return { stdout: "", stderr: "Could not find service", code: 113 };
}
return { stdout: state.printOutput, stderr: "", code: 0 };
if (!state.serviceLoaded) {
return { stdout: "", stderr: "Could not find service", code: 113 };
}
if (state.printOutput) {
return { stdout: state.printOutput, stderr: "", code: 0 };
}
if (!state.serviceRunning) {
return { stdout: ["state = waiting", "pid = 0"].join("\n"), stderr: "", code: 0 };
}
return { stdout: ["state = running", "pid = 4242"].join("\n"), stderr: "", code: 0 };
}
if (call[0] === "bootstrap" && state.bootstrapError) {
return { stdout: "", stderr: state.bootstrapError, code: state.bootstrapCode };
if (call[0] === "disable" && state.disableError) {
return { stdout: "", stderr: state.disableError, code: state.disableCode };
}
if (call[0] === "kickstart" && state.kickstartError && state.kickstartFailuresRemaining > 0) {
state.kickstartFailuresRemaining -= 1;
return { stdout: "", stderr: state.kickstartError, code: 1 };
if (call[0] === "stop") {
if (state.stopError) {
return { stdout: "", stderr: state.stopError, code: state.stopCode };
}
if (!state.stopLeavesRunning) {
state.serviceRunning = false;
}
return { stdout: "", stderr: "", code: 0 };
}
if (call[0] === "bootout") {
if (state.bootoutError) {
return { stdout: "", stderr: state.bootoutError, code: state.bootoutCode };
}
state.serviceLoaded = false;
state.serviceRunning = false;
return { stdout: "", stderr: "", code: 0 };
}
if (call[0] === "enable") {
return { stdout: "", stderr: "", code: 0 };
}
if (call[0] === "bootstrap") {
if (state.bootstrapError) {
return { stdout: "", stderr: state.bootstrapError, code: state.bootstrapCode };
}
state.serviceLoaded = true;
state.serviceRunning = true;
return { stdout: "", stderr: "", code: 0 };
}
if (call[0] === "kickstart") {
if (state.kickstartError && state.kickstartFailuresRemaining > 0) {
state.kickstartFailuresRemaining -= 1;
return { stdout: "", stderr: state.kickstartError, code: 1 };
}
state.serviceLoaded = true;
state.serviceRunning = true;
return { stdout: "", stderr: "", code: 0 };
}
return { stdout: "", stderr: "", code: 0 };
}),
@@ -162,6 +214,15 @@ beforeEach(() => {
state.bootstrapCode = 1;
state.kickstartError = "";
state.kickstartFailuresRemaining = 0;
state.disableError = "";
state.disableCode = 1;
state.stopError = "";
state.stopCode = 1;
state.bootoutError = "";
state.bootoutCode = 1;
state.serviceLoaded = true;
state.serviceRunning = true;
state.stopLeavesRunning = false;
state.dirs.clear();
state.dirModes.clear();
state.files.clear();
@@ -406,6 +467,58 @@ describe("launchd install", () => {
expect(state.fileModes.get(plistPath)).toBe(0o644);
});
it("stops LaunchAgent by disabling relaunch before stopping the process", async () => {
const env = createDefaultLaunchdEnv();
const stdout = new PassThrough();
let output = "";
stdout.on("data", (chunk: Buffer) => {
output += chunk.toString();
});
await stopLaunchAgent({ env, stdout });
const domain = typeof process.getuid === "function" ? `gui/${process.getuid()}` : "gui/501";
const serviceId = `${domain}/ai.openclaw.gateway`;
expect(state.launchctlCalls).toContainEqual(["disable", serviceId]);
expect(state.launchctlCalls).toContainEqual(["stop", "ai.openclaw.gateway"]);
expect(state.launchctlCalls.some((call) => call[0] === "bootout")).toBe(false);
expect(output).toContain("Stopped LaunchAgent");
});
it("falls back to bootout when disable fails so stop remains authoritative", async () => {
const env = createDefaultLaunchdEnv();
const stdout = new PassThrough();
let output = "";
state.disableError = "Operation not permitted";
stdout.on("data", (chunk: Buffer) => {
output += chunk.toString();
});
await stopLaunchAgent({ env, stdout });
expect(state.launchctlCalls.some((call) => call[0] === "stop")).toBe(false);
expect(state.launchctlCalls.some((call) => call[0] === "bootout")).toBe(true);
expect(output).toContain("Stopped LaunchAgent (degraded)");
expect(output).toContain("used bootout fallback");
});
it("falls back to bootout when stop does not fully stop the service", async () => {
const env = createDefaultLaunchdEnv();
const stdout = new PassThrough();
let output = "";
state.stopLeavesRunning = true;
stdout.on("data", (chunk: Buffer) => {
output += chunk.toString();
});
await stopLaunchAgent({ env, stdout });
expect(state.launchctlCalls.some((call) => call[0] === "stop")).toBe(true);
expect(state.launchctlCalls.some((call) => call[0] === "bootout")).toBe(true);
expect(output).toContain("Stopped LaunchAgent (degraded)");
expect(output).toContain("did not fully stop the service");
});
it("restarts LaunchAgent with kickstart and no bootout", async () => {
const env = {
...createDefaultLaunchdEnv(),
@@ -421,6 +534,7 @@ describe("launchd install", () => {
const serviceId = `${domain}/${label}`;
expect(result).toEqual({ outcome: "completed" });
expect(cleanStaleGatewayProcessesSync).toHaveBeenCalledWith(18789);
expect(state.launchctlCalls).toContainEqual(["enable", serviceId]);
expect(state.launchctlCalls).toContainEqual(["kickstart", "-k", serviceId]);
expect(state.launchctlCalls.some((call) => call[0] === "bootout")).toBe(false);
expect(state.launchctlCalls.some((call) => call[0] === "bootstrap")).toBe(false);
@@ -484,7 +598,7 @@ describe("launchd install", () => {
}),
).rejects.toThrow("launchctl kickstart failed: Input/output error");
expect(state.launchctlCalls.some((call) => call[0] === "enable")).toBe(false);
expect(state.launchctlCalls.some((call) => call[0] === "enable")).toBe(true);
expect(state.launchctlCalls.some((call) => call[0] === "bootstrap")).toBe(false);
});
@@ -517,7 +631,7 @@ describe("launchd install", () => {
}),
).rejects.toThrow("launchctl kickstart failed: Input/output error");
expect(state.launchctlCalls.some((call) => call[0] === "enable")).toBe(false);
expect(state.launchctlCalls.some((call) => call[0] === "enable")).toBe(true);
expect(state.launchctlCalls.some((call) => call[0] === "bootstrap")).toBe(false);
});

View File

@@ -464,14 +464,80 @@ function isUnsupportedGuiDomain(detail: string): boolean {
);
}
export async function stopLaunchAgent({ stdout, env }: GatewayServiceControlArgs): Promise<void> {
const domain = resolveGuiDomain();
const label = resolveLaunchAgentLabel({ env });
const res = await execLaunchctl(["bootout", `${domain}/${label}`]);
if (res.code !== 0 && !isLaunchctlNotLoaded(res)) {
throw new Error(`launchctl bootout failed: ${res.stderr || res.stdout}`.trim());
function formatLaunchctlResultDetail(res: {
stdout: string;
stderr: string;
code: number;
}): string {
return (res.stderr || res.stdout).trim();
}
async function isLaunchAgentProcessRunning(serviceTarget: string): Promise<boolean> {
const probe = await execLaunchctl(["print", serviceTarget]);
if (probe.code !== 0) {
return false;
}
stdout.write(`${formatLine("Stopped LaunchAgent", `${domain}/${label}`)}\n`);
const runtime = parseLaunchctlPrint(probe.stdout || probe.stderr || "");
return typeof runtime.pid === "number" && runtime.pid > 1;
}
async function waitForLaunchAgentStopped(serviceTarget: string): Promise<boolean> {
for (let attempt = 0; attempt < 10; attempt += 1) {
if (!(await isLaunchAgentProcessRunning(serviceTarget))) {
return true;
}
await new Promise((resolve) => {
setTimeout(resolve, 100);
});
}
return false;
}
export async function stopLaunchAgent({ stdout, env }: GatewayServiceControlArgs): Promise<void> {
const serviceEnv = env ?? (process.env as GatewayServiceEnv);
const domain = resolveGuiDomain();
const label = resolveLaunchAgentLabel({ env: serviceEnv });
const serviceTarget = `${domain}/${label}`;
// Keep the LaunchAgent installed, but persistently suppress KeepAlive/RunAtLoad
// before stopping the current process. If disable fails, fall back to bootout so
// the command still leaves the gateway down.
const disable = await execLaunchctl(["disable", serviceTarget]);
if (disable.code !== 0) {
const bootout = await execLaunchctl(["bootout", serviceTarget]);
if (bootout.code !== 0 && !isLaunchctlNotLoaded(bootout)) {
throw new Error(
`launchctl disable failed: ${formatLaunchctlResultDetail(disable)}; launchctl bootout failed: ${formatLaunchctlResultDetail(bootout)}`,
);
}
stdout.write(
`${formatLine("Warning", `launchctl disable failed; used bootout fallback and left service unloaded: ${formatLaunchctlResultDetail(disable)}`)}\n`,
);
stdout.write(`${formatLine("Stopped LaunchAgent (degraded)", serviceTarget)}\n`);
return;
}
// `launchctl stop` targets the plain label (not the fully-qualified service target).
const stop = await execLaunchctl(["stop", label]);
if (stop.code !== 0 && !isLaunchctlNotLoaded(stop)) {
throw new Error(`launchctl stop failed: ${formatLaunchctlResultDetail(stop)}`);
}
if (!(await waitForLaunchAgentStopped(serviceTarget))) {
const bootout = await execLaunchctl(["bootout", serviceTarget]);
if (bootout.code !== 0 && !isLaunchctlNotLoaded(bootout)) {
throw new Error(
`launchctl stop left the service running and launchctl bootout failed: ${formatLaunchctlResultDetail(bootout)}`,
);
}
stdout.write(
`${formatLine("Warning", "launchctl stop did not fully stop the service; used bootout fallback and left service unloaded")}\n`,
);
stdout.write(`${formatLine("Stopped LaunchAgent (degraded)", serviceTarget)}\n`);
return;
}
stdout.write(`${formatLine("Stopped LaunchAgent", serviceTarget)}\n`);
}
async function writeLaunchAgentPlist({
@@ -621,6 +687,10 @@ export async function restartLaunchAgent({
cleanStaleGatewayProcessesSync(cleanupPort);
}
// Clear any persisted disabled state left behind by `openclaw gateway stop`
// before trying the normal restart path.
await execLaunchctl(["enable", serviceTarget]);
const start = await execLaunchctl(["kickstart", "-k", serviceTarget]);
if (start.code === 0) {
writeLaunchAgentActionLine(stdout, "Restarted LaunchAgent", serviceTarget);