mirror of
https://github.com/openclaw/openclaw.git
synced 2026-03-15 12:00:43 +00:00
fix: verify gateway restart health after daemon restart
This commit is contained in:
Peter Steinberger
@@ -10,14 +10,7 @@ import {
|
||||
resolveGatewayPort,
|
||||
writeConfigFile,
|
||||
} from "../../config/config.js";
|
||||
import type { GatewayServiceRuntime } from "../../daemon/service-runtime.js";
|
||||
import { resolveGatewayService } from "../../daemon/service.js";
|
||||
import {
|
||||
classifyPortListener,
|
||||
formatPortDiagnostics,
|
||||
inspectPortUsage,
|
||||
type PortUsage,
|
||||
} from "../../infra/ports.js";
|
||||
import {
|
||||
channelToNpmTag,
|
||||
DEFAULT_GIT_CHANNEL,
|
||||
@@ -40,11 +33,16 @@ import { runCommandWithTimeout } from "../../process/exec.js";
|
||||
import { defaultRuntime } from "../../runtime.js";
|
||||
import { stylePromptMessage } from "../../terminal/prompt-style.js";
|
||||
import { theme } from "../../terminal/theme.js";
|
||||
import { pathExists, sleep } from "../../utils.js";
|
||||
import { pathExists } from "../../utils.js";
|
||||
import { replaceCliName, resolveCliName } from "../cli-name.js";
|
||||
import { formatCliCommand } from "../command-format.js";
|
||||
import { installCompletion } from "../completion-cli.js";
|
||||
import { runDaemonInstall, runDaemonRestart } from "../daemon-cli.js";
|
||||
import {
|
||||
renderRestartDiagnostics,
|
||||
terminateStaleGatewayPids,
|
||||
waitForGatewayHealthyRestart,
|
||||
} from "../daemon-cli/restart-health.js";
|
||||
import { createUpdateProgress, printResult } from "./progress.js";
|
||||
import { prepareRestartScript, runRestartScript } from "./restart-helper.js";
|
||||
import {
|
||||
@@ -67,8 +65,6 @@ import { suppressDeprecations } from "./suppress-deprecations.js";
|
||||
|
||||
const CLI_NAME = resolveCliName();
|
||||
const SERVICE_REFRESH_TIMEOUT_MS = 60_000;
|
||||
const POST_RESTART_HEALTH_ATTEMPTS = 8;
|
||||
const POST_RESTART_HEALTH_DELAY_MS = 450;
|
||||
|
||||
const UPDATE_QUIPS = [
|
||||
"Leveled up! New skills unlocked. You're welcome.",
|
||||
@@ -97,13 +93,6 @@ function pickUpdateQuip(): string {
|
||||
return UPDATE_QUIPS[Math.floor(Math.random() * UPDATE_QUIPS.length)] ?? "Update complete.";
|
||||
}
|
||||
|
||||
type GatewayRestartSnapshot = {
|
||||
runtime: GatewayServiceRuntime;
|
||||
portUsage: PortUsage;
|
||||
healthy: boolean;
|
||||
staleGatewayPids: number[];
|
||||
};
|
||||
|
||||
function resolveGatewayInstallEntrypointCandidates(root?: string): string[] {
|
||||
if (!root) {
|
||||
return [];
|
||||
@@ -151,126 +140,6 @@ async function refreshGatewayServiceEnv(params: {
|
||||
await runDaemonInstall({ force: true, json: params.jsonMode || undefined });
|
||||
}
|
||||
|
||||
async function inspectGatewayRestart(port: number): Promise<GatewayRestartSnapshot> {
|
||||
const service = resolveGatewayService();
|
||||
let runtime: GatewayServiceRuntime = { status: "unknown" };
|
||||
try {
|
||||
runtime = await service.readRuntime(process.env);
|
||||
} catch (err) {
|
||||
runtime = { status: "unknown", detail: String(err) };
|
||||
}
|
||||
|
||||
let portUsage: PortUsage;
|
||||
try {
|
||||
portUsage = await inspectPortUsage(port);
|
||||
} catch (err) {
|
||||
portUsage = {
|
||||
port,
|
||||
status: "unknown",
|
||||
listeners: [],
|
||||
hints: [],
|
||||
errors: [String(err)],
|
||||
};
|
||||
}
|
||||
|
||||
const gatewayListeners =
|
||||
portUsage.status === "busy"
|
||||
? portUsage.listeners.filter((listener) => classifyPortListener(listener, port) === "gateway")
|
||||
: [];
|
||||
const running = runtime.status === "running";
|
||||
const ownsPort =
|
||||
runtime.pid != null
|
||||
? portUsage.listeners.some((listener) => listener.pid === runtime.pid)
|
||||
: gatewayListeners.length > 0 ||
|
||||
(portUsage.status === "busy" && portUsage.listeners.length === 0);
|
||||
const healthy = running && ownsPort;
|
||||
const staleGatewayPids = Array.from(
|
||||
new Set(
|
||||
gatewayListeners
|
||||
.map((listener) => listener.pid)
|
||||
.filter((pid): pid is number => Number.isFinite(pid))
|
||||
.filter((pid) => runtime.pid == null || pid !== runtime.pid || !running),
|
||||
),
|
||||
);
|
||||
|
||||
return {
|
||||
runtime,
|
||||
portUsage,
|
||||
healthy,
|
||||
staleGatewayPids,
|
||||
};
|
||||
}
|
||||
|
||||
async function waitForGatewayHealthyRestart(port: number): Promise<GatewayRestartSnapshot> {
|
||||
let snapshot = await inspectGatewayRestart(port);
|
||||
for (let attempt = 0; attempt < POST_RESTART_HEALTH_ATTEMPTS; attempt += 1) {
|
||||
if (snapshot.healthy) {
|
||||
return snapshot;
|
||||
}
|
||||
if (snapshot.staleGatewayPids.length > 0 && snapshot.runtime.status !== "running") {
|
||||
return snapshot;
|
||||
}
|
||||
await sleep(POST_RESTART_HEALTH_DELAY_MS);
|
||||
snapshot = await inspectGatewayRestart(port);
|
||||
}
|
||||
return snapshot;
|
||||
}
|
||||
|
||||
function renderRestartDiagnostics(snapshot: GatewayRestartSnapshot): string[] {
|
||||
const lines: string[] = [];
|
||||
const runtimeSummary = [
|
||||
snapshot.runtime.status ? `status=${snapshot.runtime.status}` : null,
|
||||
snapshot.runtime.state ? `state=${snapshot.runtime.state}` : null,
|
||||
snapshot.runtime.pid != null ? `pid=${snapshot.runtime.pid}` : null,
|
||||
snapshot.runtime.lastExitStatus != null ? `lastExit=${snapshot.runtime.lastExitStatus}` : null,
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join(", ");
|
||||
if (runtimeSummary) {
|
||||
lines.push(`Service runtime: ${runtimeSummary}`);
|
||||
}
|
||||
if (snapshot.portUsage.status === "busy") {
|
||||
lines.push(...formatPortDiagnostics(snapshot.portUsage));
|
||||
} else {
|
||||
lines.push(`Gateway port ${snapshot.portUsage.port} status: ${snapshot.portUsage.status}.`);
|
||||
}
|
||||
if (snapshot.portUsage.errors?.length) {
|
||||
lines.push(`Port diagnostics errors: ${snapshot.portUsage.errors.join("; ")}`);
|
||||
}
|
||||
return lines;
|
||||
}
|
||||
|
||||
async function terminateStaleGatewayPids(pids: number[]): Promise<number[]> {
|
||||
const killed: number[] = [];
|
||||
for (const pid of pids) {
|
||||
try {
|
||||
process.kill(pid, "SIGTERM");
|
||||
killed.push(pid);
|
||||
} catch (err) {
|
||||
const code = (err as NodeJS.ErrnoException)?.code;
|
||||
if (code !== "ESRCH") {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (killed.length === 0) {
|
||||
return killed;
|
||||
}
|
||||
await sleep(400);
|
||||
for (const pid of killed) {
|
||||
try {
|
||||
process.kill(pid, 0);
|
||||
process.kill(pid, "SIGKILL");
|
||||
} catch (err) {
|
||||
const code = (err as NodeJS.ErrnoException)?.code;
|
||||
if (code !== "ESRCH") {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
}
|
||||
return killed;
|
||||
}
|
||||
|
||||
async function tryInstallShellCompletion(opts: {
|
||||
jsonMode: boolean;
|
||||
skipPrompt: boolean;
|
||||
@@ -633,7 +502,11 @@ async function maybeRestartService(params: {
|
||||
}
|
||||
|
||||
if (!params.opts.json && restartInitiated) {
|
||||
let health = await waitForGatewayHealthyRestart(params.gatewayPort);
|
||||
const service = resolveGatewayService();
|
||||
let health = await waitForGatewayHealthyRestart({
|
||||
service,
|
||||
port: params.gatewayPort,
|
||||
});
|
||||
if (!health.healthy && health.staleGatewayPids.length > 0) {
|
||||
if (!params.opts.json) {
|
||||
defaultRuntime.log(
|
||||
@@ -644,7 +517,10 @@ async function maybeRestartService(params: {
|
||||
}
|
||||
await terminateStaleGatewayPids(health.staleGatewayPids);
|
||||
await runDaemonRestart();
|
||||
health = await waitForGatewayHealthyRestart(params.gatewayPort);
|
||||
health = await waitForGatewayHealthyRestart({
|
||||
service,
|
||||
port: params.gatewayPort,
|
||||
});
|
||||
}
|
||||
|
||||
if (health.healthy) {
|
||||
|
||||
Reference in New Issue
Block a user