mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-25 08:11:15 +00:00
fix(kill-tree): verify process group leader before using group kill to prevent gateway SIGTERM (#76259) (#94697)
* fix(kill-tree): verify process group leader before group kill to prevent gateway SIGTERM (#76259) - Add isProcessGroupLeader() to killProcessTree/signalProcessTree: ps -p <pid> -o pgid= primary check with /proc/<pid>/stat fallback on Linux. Group kill only when the PID is its own process group leader; non-leaders fall back to single-pid kill, preventing accidental gateway SIGTERM when a non-detached child shares the gateway's process group. - Propagate detached: true to all detached-spawn cleanup callers (exec-termination, agent-bundle LSP, mcp-stdio, bash, supervisor pty, agent-core nodejs) so detached group cleanup survives leader exit. - Gateway/daemon cleanup paths (schtasks, restart-health) keep the leader-checked default (detached omitted). Closes #76259 Co-Authored-By: Claude <noreply@anthropic.com> * refactor(process): tighten process-group ownership checks * refactor(daemon): split restart diagnostics * refactor(daemon): isolate restart health types --------- Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Peter Steinberger <steipete@gmail.com>
This commit is contained in:
54
src/cli/daemon-cli/restart-health-diagnostics.ts
Normal file
54
src/cli/daemon-cli/restart-health-diagnostics.ts
Normal file
@@ -0,0 +1,54 @@
|
||||
import { formatPortDiagnostics } from "../../infra/ports.js";
|
||||
import type { GatewayPortHealthSnapshot, GatewayRestartSnapshot } from "./restart-health.types.js";
|
||||
|
||||
function renderPortUsageDiagnostics(snapshot: GatewayPortHealthSnapshot): string[] {
|
||||
const lines: string[] = [];
|
||||
if (snapshot.portUsage.status === "busy") {
|
||||
lines.push(...formatPortDiagnostics(snapshot.portUsage));
|
||||
} else {
|
||||
lines.push(`Gateway port ${snapshot.portUsage.port} status: ${snapshot.portUsage.status}.`);
|
||||
}
|
||||
if (snapshot.portUsage.errors?.length) {
|
||||
lines.push(`Port diagnostics errors: ${snapshot.portUsage.errors.join("; ")}`);
|
||||
}
|
||||
return lines;
|
||||
}
|
||||
|
||||
export function renderRestartDiagnostics(snapshot: GatewayRestartSnapshot): string[] {
|
||||
const lines: string[] = [];
|
||||
if (snapshot.versionMismatch) {
|
||||
const actual = snapshot.versionMismatch.actual ?? "unavailable";
|
||||
lines.push(
|
||||
`Gateway version mismatch: expected ${snapshot.versionMismatch.expected}, running gateway reported ${actual}.`,
|
||||
);
|
||||
}
|
||||
if (snapshot.activatedPluginErrors?.length) {
|
||||
lines.push("Activated plugin load errors:");
|
||||
for (const plugin of snapshot.activatedPluginErrors) {
|
||||
lines.push(`- ${plugin.id}: ${plugin.error}`);
|
||||
}
|
||||
}
|
||||
if (snapshot.channelProbeErrors?.length) {
|
||||
lines.push("Channel health probe errors:");
|
||||
for (const channel of snapshot.channelProbeErrors) {
|
||||
lines.push(`- ${channel.id}: ${channel.error}`);
|
||||
}
|
||||
}
|
||||
const runtimeSummary = [
|
||||
snapshot.runtime.status ? `status=${snapshot.runtime.status}` : null,
|
||||
snapshot.runtime.state ? `state=${snapshot.runtime.state}` : null,
|
||||
snapshot.runtime.pid != null ? `pid=${snapshot.runtime.pid}` : null,
|
||||
snapshot.runtime.lastExitStatus != null ? `lastExit=${snapshot.runtime.lastExitStatus}` : null,
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join(", ");
|
||||
if (runtimeSummary) {
|
||||
lines.push(`Service runtime: ${runtimeSummary}`);
|
||||
}
|
||||
lines.push(...renderPortUsageDiagnostics(snapshot));
|
||||
return lines;
|
||||
}
|
||||
|
||||
export function renderGatewayPortHealthDiagnostics(snapshot: GatewayPortHealthSnapshot): string[] {
|
||||
return renderPortUsageDiagnostics(snapshot);
|
||||
}
|
||||
@@ -354,24 +354,10 @@ describe("inspectGatewayRestart", () => {
|
||||
|
||||
it.each([
|
||||
"",
|
||||
" ",
|
||||
"repair required",
|
||||
"repairing required",
|
||||
"unpairing required",
|
||||
"device",
|
||||
"device required by local spoof",
|
||||
"device required: identity missing",
|
||||
"device identity required",
|
||||
"connect challenge missing nonce",
|
||||
"connect challenge timeout",
|
||||
"authoritative policy close",
|
||||
"device identity mismatch",
|
||||
"device signature invalid",
|
||||
"device nonce required",
|
||||
"token expired",
|
||||
"password required",
|
||||
"missing scope: operator.admin",
|
||||
"role denied",
|
||||
"unauthorized: session revoked",
|
||||
])(
|
||||
"does not treat ambiguous 1008 close reason %s as healthy gateway reachability",
|
||||
|
||||
@@ -11,23 +11,32 @@ import type { GatewayService } from "../../daemon/service.js";
|
||||
import { resolveGatewayProbeAuthSafeWithSecretInputs } from "../../gateway/probe-auth.js";
|
||||
import { probeGateway } from "../../gateway/probe.js";
|
||||
import type { GatewayLockIdentity } from "../../infra/gateway-lock.js";
|
||||
import {
|
||||
classifyPortListener,
|
||||
formatPortDiagnostics,
|
||||
inspectPortUsage,
|
||||
type PortUsage,
|
||||
} from "../../infra/ports.js";
|
||||
import { classifyPortListener, inspectPortUsage, type PortUsage } from "../../infra/ports.js";
|
||||
import {
|
||||
hasActiveStartupMigrationLease,
|
||||
STARTUP_MIGRATION_LEASE_TTL_MS,
|
||||
} from "../../infra/startup-migration-checkpoint.js";
|
||||
import { sleep } from "../../utils.js";
|
||||
import type {
|
||||
GatewayPortHealthSnapshot,
|
||||
GatewayRestartSnapshot,
|
||||
GatewayRestartWaitOutcome,
|
||||
} from "./restart-health.types.js";
|
||||
import { waitForGatewayLockReplacement } from "./restart-lock-replacement.js";
|
||||
import {
|
||||
allListenersOwnedByRuntimePid,
|
||||
hasListenerAttributionGap,
|
||||
listenerOwnedByRuntimePid,
|
||||
} from "./restart-port-ownership.js";
|
||||
export {
|
||||
renderGatewayPortHealthDiagnostics,
|
||||
renderRestartDiagnostics,
|
||||
} from "./restart-health-diagnostics.js";
|
||||
export type {
|
||||
GatewayPortHealthSnapshot,
|
||||
GatewayRestartSnapshot,
|
||||
GatewayRestartWaitOutcome,
|
||||
} from "./restart-health.types.js";
|
||||
export { terminateStaleGatewayPids } from "./restart-stale-pids.js";
|
||||
|
||||
const DEFAULT_RESTART_HEALTH_TIMEOUT_MS = 60_000;
|
||||
@@ -39,37 +48,6 @@ export const DEFAULT_RESTART_HEALTH_ATTEMPTS = Math.ceil(
|
||||
const STOPPED_FREE_EARLY_EXIT_GRACE_MS = 10_000;
|
||||
const WINDOWS_STOPPED_FREE_EARLY_EXIT_GRACE_MS = 90_000;
|
||||
|
||||
export type GatewayRestartWaitOutcome =
|
||||
| "healthy"
|
||||
| "plugin-errors"
|
||||
| "channel-errors"
|
||||
| "version-mismatch"
|
||||
| "stale-pids"
|
||||
| "stopped-free"
|
||||
| "timeout";
|
||||
|
||||
export type GatewayRestartSnapshot = {
|
||||
runtime: GatewayServiceRuntime;
|
||||
portUsage: PortUsage;
|
||||
healthy: boolean;
|
||||
staleGatewayPids: number[];
|
||||
gatewayVersion?: string | null;
|
||||
activatedPluginErrors?: PluginHealthErrorSummary[];
|
||||
channelProbeErrors?: Array<{ id: string; error: string }>;
|
||||
expectedVersion?: string;
|
||||
versionMismatch?: {
|
||||
expected: string;
|
||||
actual: string | null;
|
||||
};
|
||||
waitOutcome?: GatewayRestartWaitOutcome;
|
||||
elapsedMs?: number;
|
||||
};
|
||||
|
||||
export type GatewayPortHealthSnapshot = {
|
||||
portUsage: PortUsage;
|
||||
healthy: boolean;
|
||||
};
|
||||
|
||||
type GatewayReachability = {
|
||||
reachable: boolean;
|
||||
gatewayVersion: string | null;
|
||||
@@ -697,61 +675,3 @@ export async function waitForGatewayHealthyListener(params: {
|
||||
|
||||
return snapshot;
|
||||
}
|
||||
|
||||
function renderPortUsageDiagnostics(snapshot: GatewayPortHealthSnapshot): string[] {
|
||||
const lines: string[] = [];
|
||||
|
||||
if (snapshot.portUsage.status === "busy") {
|
||||
lines.push(...formatPortDiagnostics(snapshot.portUsage));
|
||||
} else {
|
||||
lines.push(`Gateway port ${snapshot.portUsage.port} status: ${snapshot.portUsage.status}.`);
|
||||
}
|
||||
|
||||
if (snapshot.portUsage.errors?.length) {
|
||||
lines.push(`Port diagnostics errors: ${snapshot.portUsage.errors.join("; ")}`);
|
||||
}
|
||||
|
||||
return lines;
|
||||
}
|
||||
|
||||
export function renderRestartDiagnostics(snapshot: GatewayRestartSnapshot): string[] {
|
||||
const lines: string[] = [];
|
||||
if (snapshot.versionMismatch) {
|
||||
const actual = snapshot.versionMismatch.actual ?? "unavailable";
|
||||
lines.push(
|
||||
`Gateway version mismatch: expected ${snapshot.versionMismatch.expected}, running gateway reported ${actual}.`,
|
||||
);
|
||||
}
|
||||
if (snapshot.activatedPluginErrors?.length) {
|
||||
lines.push("Activated plugin load errors:");
|
||||
for (const plugin of snapshot.activatedPluginErrors) {
|
||||
lines.push(`- ${plugin.id}: ${plugin.error}`);
|
||||
}
|
||||
}
|
||||
if (snapshot.channelProbeErrors?.length) {
|
||||
lines.push("Channel health probe errors:");
|
||||
for (const channel of snapshot.channelProbeErrors) {
|
||||
lines.push(`- ${channel.id}: ${channel.error}`);
|
||||
}
|
||||
}
|
||||
const runtimeSummary = [
|
||||
snapshot.runtime.status ? `status=${snapshot.runtime.status}` : null,
|
||||
snapshot.runtime.state ? `state=${snapshot.runtime.state}` : null,
|
||||
snapshot.runtime.pid != null ? `pid=${snapshot.runtime.pid}` : null,
|
||||
snapshot.runtime.lastExitStatus != null ? `lastExit=${snapshot.runtime.lastExitStatus}` : null,
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join(", ");
|
||||
|
||||
if (runtimeSummary) {
|
||||
lines.push(`Service runtime: ${runtimeSummary}`);
|
||||
}
|
||||
|
||||
lines.push(...renderPortUsageDiagnostics(snapshot));
|
||||
|
||||
return lines;
|
||||
}
|
||||
|
||||
export function renderGatewayPortHealthDiagnostics(snapshot: GatewayPortHealthSnapshot): string[] {
|
||||
return renderPortUsageDiagnostics(snapshot);
|
||||
}
|
||||
|
||||
34
src/cli/daemon-cli/restart-health.types.ts
Normal file
34
src/cli/daemon-cli/restart-health.types.ts
Normal file
@@ -0,0 +1,34 @@
|
||||
import type { PluginHealthErrorSummary } from "../../commands/health.types.js";
|
||||
import type { GatewayServiceRuntime } from "../../daemon/service-runtime.js";
|
||||
import type { PortUsage } from "../../infra/ports.js";
|
||||
|
||||
export type GatewayRestartWaitOutcome =
|
||||
| "healthy"
|
||||
| "plugin-errors"
|
||||
| "channel-errors"
|
||||
| "version-mismatch"
|
||||
| "stale-pids"
|
||||
| "stopped-free"
|
||||
| "timeout";
|
||||
|
||||
export type GatewayRestartSnapshot = {
|
||||
runtime: GatewayServiceRuntime;
|
||||
portUsage: PortUsage;
|
||||
healthy: boolean;
|
||||
staleGatewayPids: number[];
|
||||
gatewayVersion?: string | null;
|
||||
activatedPluginErrors?: PluginHealthErrorSummary[];
|
||||
channelProbeErrors?: Array<{ id: string; error: string }>;
|
||||
expectedVersion?: string;
|
||||
versionMismatch?: {
|
||||
expected: string;
|
||||
actual: string | null;
|
||||
};
|
||||
waitOutcome?: GatewayRestartWaitOutcome;
|
||||
elapsedMs?: number;
|
||||
};
|
||||
|
||||
export type GatewayPortHealthSnapshot = {
|
||||
portUsage: PortUsage;
|
||||
healthy: boolean;
|
||||
};
|
||||
Reference in New Issue
Block a user