diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f264263054..f9b7a63f4d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ Docs: https://docs.openclaw.ai - CLI/media: accept HTTP(S) URLs in `openclaw infer image describe --file`, fetching remote images through the guarded media path instead of treating URLs as local files. Fixes #82837. (#82854) Thanks @neeravmakwana. - Agents/subagents: keep session-backed parent runs active when the child wait call times out before the child session has actually settled, so late subagent completions are reconciled instead of being lost. Fixes #82787. Thanks @ramitrkar-hash. - Control UI: advertise shared Gateway protocol constants in browser connect frames, fixing protocol mismatch handshakes after the protocol 5 bump. Fixes #82882. Thanks @galiniliev. +- Gateway: add rollback protocol-mismatch diagnostics, including client protocol ranges in Gateway logs and deep status/doctor hints for stale client processes. Fixes #82841. (#82908) - Agents/subagents: route group/channel subagent completions through message-tool-only handoffs when required and keep active-requester wake failures from dropping completion delivery. Fixes #82803. Thanks @galiniliev, @yozakura-ava, and @moeedahmed. - Memory-core: scan persisted memory source sessions on startup, comparing on-disk transcripts against the index and marking only missing/newer/resized files dirty for incremental sync. Fixes #82341. (#82341) Thanks @giodl73-repo. - Telegram: keep the top-level default account in the account list when named accounts or bindings are added alongside top-level credentials, preserving default polling while still letting named-only configs resolve to a single account. Fixes #82794. (#82794) Thanks @giodl73-repo. diff --git a/docs/gateway/troubleshooting.md b/docs/gateway/troubleshooting.md index 828790b4c58..ea032aef615 100644 --- a/docs/gateway/troubleshooting.md +++ b/docs/gateway/troubleshooting.md @@ -86,6 +86,32 @@ openclaw config get meta.lastTouchedVersion For intentional downgrade or emergency recovery only, set `OPENCLAW_ALLOW_OLDER_BINARY_DESTRUCTIVE_ACTIONS=1` for the single command. Leave it unset for normal operation. +## Protocol mismatch after rollback + +Use this when logs keep printing `protocol mismatch` after you downgrade or roll back OpenClaw. This means an older Gateway is running, but a newer local client process is still trying to reconnect with a protocol range that the older Gateway cannot speak. + +```bash +openclaw --version +which -a openclaw +openclaw gateway status --deep +openclaw doctor --deep +openclaw logs --follow +``` + +Look for: + +- `protocol mismatch ... client=... v min= max= expected=` in Gateway logs. +- `Established clients:` in `openclaw gateway status --deep` or `Gateway clients` in `openclaw doctor --deep`. This lists active TCP clients connected to the Gateway port, including PIDs and command lines when the OS allows it. +- A client process whose command line points at the newer OpenClaw install or wrapper you rolled back from. + +Fix: + +1. Stop or restart the stale OpenClaw client process shown by `gateway status --deep`. +2. Restart apps or wrappers that embed OpenClaw, such as local dashboards, editors, app-server helpers, or long-running `openclaw logs --follow` shells. +3. Re-run `openclaw gateway status --deep` or `openclaw doctor --deep` and confirm the stale client PID is gone. + +Do not make an older Gateway accept a newer incompatible protocol. Protocol bumps protect the wire contract; rollback recovery is a process/version cleanup problem. + ## Skill symlink skipped as path escape Use this when logs include: diff --git a/src/cli/daemon-cli.coverage.test.ts b/src/cli/daemon-cli.coverage.test.ts index 87d28da2f6a..b3ade826392 100644 --- a/src/cli/daemon-cli.coverage.test.ts +++ b/src/cli/daemon-cli.coverage.test.ts @@ -28,6 +28,10 @@ const inspectPortUsage = vi.fn(async (port: number) => ({ listeners: [], hints: [], })); +const inspectPortConnections = vi.fn(async (port: number) => ({ + port, + connections: [], +})); function collectMatching( items: readonly T[], @@ -114,6 +118,7 @@ vi.mock("../daemon/inspect.js", () => ({ })); vi.mock("../infra/ports.js", () => ({ + inspectPortConnections: (port: number) => inspectPortConnections(port), inspectPortUsage: (port: number) => inspectPortUsage(port), formatPortDiagnostics: () => ["Port 18789 is already in use."], })); @@ -192,6 +197,7 @@ describe("daemon-cli coverage", () => { serviceReadCommand.mockResolvedValue(null); resolveGatewayProbeAuthSafeWithSecretInputs.mockClear(); findExtraGatewayServices.mockClear(); + inspectPortConnections.mockClear(); buildGatewayInstallPlan.mockClear(); }); diff --git a/src/cli/daemon-cli/status.gather.test.ts b/src/cli/daemon-cli/status.gather.test.ts index 6e274c9bb0e..749da0be57d 100644 --- a/src/cli/daemon-cli/status.gather.test.ts +++ b/src/cli/daemon-cli/status.gather.test.ts @@ -4,6 +4,7 @@ import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import type { StaleOpenClawUpdateLaunchdJob } from "../../daemon/launchd.js"; import { createMockGatewayService } from "../../daemon/service.test-helpers.js"; +import type { PortConnections } from "../../infra/ports.js"; import type { GatewayRestartHandoff } from "../../infra/restart-handoff.js"; import { captureEnv } from "../../test-utils/env.js"; import { VERSION } from "../../version.js"; @@ -38,6 +39,12 @@ const inspectPortUsage = vi.fn(async (port: number) => ({ listeners: [], hints: [], })); +const inspectPortConnections = vi.fn<(port: number) => Promise>( + async (port: number) => ({ + port, + connections: [], + }), +); const readLastGatewayErrorLine = vi.fn(async (_env?: NodeJS.ProcessEnv) => null); const readGatewayRestartHandoffSync = vi.fn< (_env?: NodeJS.ProcessEnv) => GatewayRestartHandoff | null @@ -166,6 +173,7 @@ vi.mock("../../gateway/net.js", () => ({ })); vi.mock("../../infra/ports.js", () => ({ + inspectPortConnections: (port: number) => inspectPortConnections(port), inspectPortUsage: (port: number) => inspectPortUsage(port), formatPortDiagnostics: () => [], })); @@ -222,6 +230,7 @@ describe("gatherDaemonStatus", () => { findStaleOpenClawUpdateLaunchdJobs.mockResolvedValue([]); loadGatewayTlsRuntime.mockClear(); inspectGatewayRestart.mockClear(); + inspectPortConnections.mockClear(); readGatewayRestartHandoffSync.mockClear(); readConfigFileSnapshotCalls.mockClear(); loadConfigCalls.mockClear(); @@ -484,6 +493,59 @@ describe("gatherDaemonStatus", () => { expect(readGatewayRestartHandoffSync).not.toHaveBeenCalled(); expect(findStaleOpenClawUpdateLaunchdJobs).not.toHaveBeenCalled(); + expect(inspectPortConnections).not.toHaveBeenCalled(); + }); + + it("surfaces established gateway connections during deep status", async () => { + inspectPortConnections.mockResolvedValueOnce({ + port: 19001, + connections: [ + { + pid: 4242, + ppid: 1, + command: "node", + commandLine: "node /tmp/newer-openclaw/dist/index.js logs --follow", + address: "TCP 127.0.0.1:50123->127.0.0.1:19001 (ESTABLISHED)", + direction: "client", + }, + ], + }); + + const status = await gatherDaemonStatus({ + rpc: {}, + probe: false, + deep: true, + }); + + expect(inspectPortConnections).toHaveBeenCalledWith(19001); + expect(status.connections?.established).toEqual([ + { + pid: 4242, + ppid: 1, + command: "node", + commandLine: "node /tmp/newer-openclaw/dist/index.js logs --follow", + address: "TCP 127.0.0.1:50123->127.0.0.1:19001 (ESTABLISHED)", + direction: "client", + }, + ]); + }); + + it("skips established gateway connection scans for remote gateway status", async () => { + daemonLoadedConfig = { + gateway: { + mode: "remote", + remote: { url: "wss://gateway.example" }, + }, + }; + + const status = await gatherDaemonStatus({ + rpc: {}, + probe: false, + deep: true, + }); + + expect(inspectPortConnections).not.toHaveBeenCalled(); + expect(status.connections).toBeUndefined(); }); it("uses the fast config path for plain same-file status reads", async () => { diff --git a/src/cli/daemon-cli/status.gather.ts b/src/cli/daemon-cli/status.gather.ts index a45ba99f7f5..847c0fb63de 100644 --- a/src/cli/daemon-cli/status.gather.ts +++ b/src/cli/daemon-cli/status.gather.ts @@ -26,7 +26,9 @@ import { import { parseStrictPositiveInteger } from "../../infra/parse-finite-number.js"; import { formatPortDiagnostics, + inspectPortConnections, inspectPortUsage, + type PortConnection, type PortListener, type PortUsageStatus, } from "../../infra/ports.js"; @@ -294,6 +296,10 @@ export type DaemonStatus = { listeners: PortListener[]; hints: string[]; }; + connections?: { + port: number; + established: PortConnection[]; + }; lastError?: string; rpc?: { ok: boolean; @@ -460,6 +466,27 @@ async function inspectDaemonPortStatuses(params: { }; } +async function inspectEstablishedGatewayClients(params: { + daemonPort: number; + deep?: boolean; + gatewayMode?: string; +}): Promise { + if (params.deep !== true || params.gatewayMode === "remote") { + return undefined; + } + const result = await inspectPortConnections(params.daemonPort).catch(() => null); + const establishedClients = result?.connections.filter( + (connection) => connection.direction !== "server", + ); + if (!result || !establishedClients || establishedClients.length === 0) { + return undefined; + } + return { + port: result.port, + established: establishedClients, + }; +} + export async function gatherDaemonStatus( opts: { rpc: GatewayRpcOpts; @@ -508,6 +535,11 @@ export async function gatherDaemonStatus( daemonPort, cliPort, }); + const establishedClients = await inspectEstablishedGatewayClients({ + daemonPort, + deep: opts.deep, + gatewayMode: daemonCfg.gateway?.mode, + }); const extraServices = opts.deep ? await loadDaemonInspectModule() @@ -618,6 +650,7 @@ export async function gatherDaemonStatus( gateway, port: portStatus, ...(portCliStatus ? { portCli: portCliStatus } : {}), + ...(establishedClients ? { connections: establishedClients } : {}), lastError, ...(rpc ? { diff --git a/src/cli/daemon-cli/status.print.test.ts b/src/cli/daemon-cli/status.print.test.ts index db20593e386..13e34056029 100644 --- a/src/cli/daemon-cli/status.print.test.ts +++ b/src/cli/daemon-cli/status.print.test.ts @@ -131,6 +131,48 @@ describe("printDaemonStatus", () => { expectMockLineContains(runtime.error, formatCliCommand("openclaw gateway restart")); }); + it("prints established gateway client guidance gathered by deep status", () => { + printDaemonStatus( + { + service: { + label: "LaunchAgent", + loaded: true, + loadedText: "loaded", + notLoadedText: "not loaded", + runtime: { status: "running", pid: 8000 }, + }, + gateway: { + bindMode: "loopback", + bindHost: "127.0.0.1", + port: 18789, + portSource: "env/config", + probeUrl: "ws://127.0.0.1:18789", + }, + connections: { + port: 18789, + established: [ + { + pid: 4242, + ppid: 1, + command: "node", + commandLine: "/tmp/newer-openclaw/bin/openclaw logs --follow", + address: "TCP 127.0.0.1:50123->127.0.0.1:18789 (ESTABLISHED)", + direction: "client", + }, + ], + }, + extraServices: [], + }, + { json: false }, + ); + + expectMockLineContains(runtime.log, "Established clients: 1"); + expectMockLineContains(runtime.log, "pid=4242"); + expectMockLineContains(runtime.log, "newer-openclaw"); + expectMockLineContains(runtime.log, "client"); + expectMockLineContains(runtime.log, "protocol mismatch after rollback"); + }); + it("prints stale updater launchd job guidance", () => { printDaemonStatus( { diff --git a/src/cli/daemon-cli/status.print.ts b/src/cli/daemon-cli/status.print.ts index 3019a82680c..e3e54fa6f28 100644 --- a/src/cli/daemon-cli/status.print.ts +++ b/src/cli/daemon-cli/status.print.ts @@ -72,6 +72,20 @@ function formatCliVersionLine(cli: DaemonStatus["cli"]): string | null { return cli.entrypoint ? `${cli.version} (${shortenHomePath(cli.entrypoint)})` : cli.version; } +function formatConnectionLine( + connection: NonNullable["established"][number], +) { + const pid = connection.pid ? `pid=${connection.pid}` : "pid=?"; + const ppid = connection.ppid ? ` ppid=${connection.ppid}` : ""; + const direction = ` ${connection.direction}`; + const command = connection.command ? ` ${connection.command}` : ""; + const address = connection.address ? ` ${connection.address}` : ""; + const commandLine = connection.commandLine + ? ` cmd=${shortenHomePath(connection.commandLine)}` + : ""; + return `${pid}${ppid}${direction}${command}${address}${commandLine}`; +} + export function printDaemonStatus(status: DaemonStatus, opts: { json: boolean }) { if (opts.json) { const sanitized = sanitizeDaemonStatusForJson(status); @@ -285,6 +299,26 @@ export function printDaemonStatus(status: DaemonStatus, opts: { json: boolean }) spacer(); } + if (status.connections?.established.length) { + defaultRuntime.log( + `${label("Established clients:")} ${infoText(String(status.connections.established.length))}`, + ); + for (const connection of status.connections.established.slice(0, 8)) { + defaultRuntime.log(` ${infoText(formatConnectionLine(connection))}`); + } + if (status.connections.established.length > 8) { + defaultRuntime.log( + ` ${infoText(`... ${status.connections.established.length - 8} more connection(s)`)}`, + ); + } + defaultRuntime.log( + warnText( + "If logs show protocol mismatch after rollback, stop stale OpenClaw client processes listed here and re-run gateway status.", + ), + ); + spacer(); + } + const systemdUnavailable = process.platform === "linux" && isSystemdUnavailableDetail(service.runtime?.detail); if (systemdUnavailable) { diff --git a/src/commands/doctor-gateway-daemon-flow.test.ts b/src/commands/doctor-gateway-daemon-flow.test.ts index 7136da2275c..2c8fee35390 100644 --- a/src/commands/doctor-gateway-daemon-flow.test.ts +++ b/src/commands/doctor-gateway-daemon-flow.test.ts @@ -21,6 +21,7 @@ const service = vi.hoisted(() => ({ const note = vi.hoisted(() => vi.fn()); const sleep = vi.hoisted(() => vi.fn(async () => {})); const healthCommand = vi.hoisted(() => vi.fn(async () => {})); +const inspectPortConnections = vi.hoisted(() => vi.fn()); const inspectPortUsage = vi.hoisted(() => vi.fn()); const formatPortDiagnostics = vi.hoisted(() => vi.fn(() => ["Port 18789 is already in use."])); const isExpectedGatewayListeners = vi.hoisted(() => vi.fn(() => false)); @@ -88,6 +89,7 @@ vi.mock("../daemon/systemd.js", async () => { }); vi.mock("../infra/ports.js", () => ({ + inspectPortConnections, inspectPortUsage, formatPortDiagnostics, isExpectedGatewayListeners, @@ -164,6 +166,10 @@ describe("maybeRepairGatewayDaemon", () => { listeners: [], hints: [], }); + inspectPortConnections.mockResolvedValue({ + port: 18789, + connections: [], + }); isExpectedGatewayListeners.mockReturnValue(false); }); @@ -324,6 +330,72 @@ describe("maybeRepairGatewayDaemon", () => { await runNonInteractiveRepair(); expect(readGatewayRestartHandoffSync).not.toHaveBeenCalled(); + expect(inspectPortConnections).not.toHaveBeenCalled(); + }); + + it("reports established gateway clients during deep doctor", async () => { + setPlatform("linux"); + inspectPortConnections.mockResolvedValueOnce({ + port: 18789, + connections: [ + { + pid: 4242, + command: "node", + commandLine: "/tmp/newer-openclaw/bin/openclaw logs --follow", + address: "TCP 127.0.0.1:50123->127.0.0.1:18789 (ESTABLISHED)", + direction: "client", + }, + ], + }); + + await maybeRepairGatewayDaemon({ + cfg: { gateway: {} }, + runtime: { log: vi.fn(), error: vi.fn(), exit: vi.fn() }, + prompter: createDoctorPrompter({ + runtime: { log: vi.fn(), error: vi.fn(), exit: vi.fn() }, + options: { deep: true, nonInteractive: true }, + }), + options: { deep: true, nonInteractive: true }, + gatewayDetailsMessage: "details", + healthOk: false, + }); + + const gatewayClientNote = note.mock.calls.find(([, label]) => label === "Gateway clients"); + expect(gatewayClientNote?.[0]).toContain("pid=4242"); + expect(gatewayClientNote?.[0]).toContain("protocol mismatch after rollback"); + }); + + it("reports established gateway clients during healthy deep doctor", async () => { + setPlatform("linux"); + inspectPortConnections.mockResolvedValueOnce({ + port: 18789, + connections: [ + { + pid: 5151, + command: "node", + commandLine: "/tmp/newer-openclaw/bin/openclaw logs --follow", + address: "TCP 127.0.0.1:50123->127.0.0.1:18789 (ESTABLISHED)", + direction: "client", + }, + ], + }); + + await maybeRepairGatewayDaemon({ + cfg: { gateway: {} }, + runtime: { log: vi.fn(), error: vi.fn(), exit: vi.fn() }, + prompter: createDoctorPrompter({ + runtime: { log: vi.fn(), error: vi.fn(), exit: vi.fn() }, + options: { deep: true, nonInteractive: true }, + }), + options: { deep: true, nonInteractive: true }, + gatewayDetailsMessage: "details", + healthOk: true, + }); + + expect(inspectPortUsage).not.toHaveBeenCalled(); + const gatewayClientNote = note.mock.calls.find(([, label]) => label === "Gateway clients"); + expect(gatewayClientNote?.[0]).toContain("pid=5151"); + expect(gatewayClientNote?.[0]).toContain("protocol mismatch after rollback"); }); it("suppresses busy-port note for expected Gateway listeners", async () => { diff --git a/src/commands/doctor-gateway-daemon-flow.ts b/src/commands/doctor-gateway-daemon-flow.ts index 1556bf4b965..14b83e1de5d 100644 --- a/src/commands/doctor-gateway-daemon-flow.ts +++ b/src/commands/doctor-gateway-daemon-flow.ts @@ -17,8 +17,10 @@ import { renderSystemdUnavailableHints } from "../daemon/systemd-hints.js"; import { isSystemdUserServiceAvailable } from "../daemon/systemd.js"; import { formatPortDiagnostics, + inspectPortConnections, inspectPortUsage, isExpectedGatewayListeners, + type PortConnection, } from "../infra/ports.js"; import { formatGatewayRestartHandoffDiagnostic, @@ -111,6 +113,40 @@ function renderBlockingSystemGatewayServices(services: ExtraGatewayService[]): s ].join("\n"); } +function renderEstablishedGatewayConnections(connections: PortConnection[]): string { + return [ + "Established Gateway TCP clients detected:", + ...connections.slice(0, 8).map((connection) => { + const pid = connection.pid ? `pid=${connection.pid}` : "pid=?"; + const direction = connection.direction; + const command = connection.command ? ` ${connection.command}` : ""; + const address = connection.address ? ` ${connection.address}` : ""; + const commandLine = connection.commandLine ? ` cmd=${connection.commandLine}` : ""; + return `- ${pid} ${direction}${command}${address}${commandLine}`; + }), + ...(connections.length > 8 ? [`- ... ${connections.length - 8} more connection(s)`] : []), + "If logs show protocol mismatch after rollback, stop stale OpenClaw client processes listed here and rerun doctor.", + ].join("\n"); +} + +async function maybeReportEstablishedGatewayClients(params: { + cfg: OpenClawConfig; + deep: boolean; + port?: number; +}): Promise { + if (!params.deep || params.cfg.gateway?.mode === "remote") { + return; + } + const port = params.port ?? resolveGatewayPort(params.cfg, process.env); + const connections = await inspectPortConnections(port).catch(() => null); + const establishedClients = connections?.connections.filter( + (connection) => connection.direction !== "server", + ); + if (establishedClients && establishedClients.length > 0) { + note(renderEstablishedGatewayConnections(establishedClients), "Gateway clients"); + } +} + export async function maybeRepairGatewayDaemon(params: { cfg: OpenClawConfig; runtime: RuntimeEnv; @@ -120,6 +156,10 @@ export async function maybeRepairGatewayDaemon(params: { healthOk: boolean; }) { if (params.healthOk) { + await maybeReportEstablishedGatewayClients({ + cfg: params.cfg, + deep: params.options.deep ?? false, + }); return; } @@ -182,6 +222,11 @@ export async function maybeRepairGatewayDaemon(params: { if (params.cfg.gateway?.mode !== "remote") { const port = resolveGatewayPort(params.cfg, process.env); const diagnostics = await inspectPortUsage(port); + await maybeReportEstablishedGatewayClients({ + cfg: params.cfg, + deep: params.options.deep ?? false, + port, + }); if ( diagnostics.status === "busy" && !isExpectedGatewayListeners(diagnostics.listeners, diagnostics.port) diff --git a/src/gateway/server/ws-connection/message-handler.ts b/src/gateway/server/ws-connection/message-handler.ts index 13b12915bb6..01ca71a8c1e 100644 --- a/src/gateway/server/ws-connection/message-handler.ts +++ b/src/gateway/server/ws-connection/message-handler.ts @@ -548,7 +548,7 @@ export function attachGatewayWsMessageHandler(params: GatewayWsMessageHandlerPar minimumProbeProtocol: MIN_PROBE_PROTOCOL_VERSION, }); logWsControl.warn( - `protocol mismatch conn=${connId} remote=${remoteAddr ?? "?"} client=${clientLabel} ${connectParams.client.mode} v${connectParams.client.version}`, + `protocol mismatch conn=${connId} peer=${formatForLog(peerLabel)} remote=${remoteAddr ?? "?"} remotePort=${remotePort ?? "?"} client=${formatForLog(clientLabel)} ${connectParams.client.mode} v${formatForLog(connectParams.client.version)} min=${minProtocol} max=${maxProtocol} expected=${PROTOCOL_VERSION} probeMin=${MIN_PROBE_PROTOCOL_VERSION} instance=${formatForLog(connectParams.client.instanceId ?? "n/a")}`, ); sendHandshakeErrorResponse(ErrorCodes.INVALID_REQUEST, "protocol mismatch", { details: { diff --git a/src/infra/ports-inspect.ts b/src/infra/ports-inspect.ts index faed621d630..560b3a43912 100644 --- a/src/infra/ports-inspect.ts +++ b/src/infra/ports-inspect.ts @@ -1,10 +1,18 @@ +import os from "node:os"; import { runCommandWithTimeout } from "../process/exec.js"; import { normalizeLowercaseStringOrEmpty } from "../shared/string-coerce.js"; import { isErrno } from "./errors.js"; import { buildPortHints } from "./ports-format.js"; import { resolveLsofCommand } from "./ports-lsof.js"; import { tryListenOnPort } from "./ports-probe.js"; -import type { PortListener, PortUsage, PortUsageStatus } from "./ports-types.js"; +import type { + PortConnection, + PortConnectionDirection, + PortConnections, + PortListener, + PortUsage, + PortUsageStatus, +} from "./ports-types.js"; type CommandResult = { stdout: string; @@ -58,6 +66,148 @@ function parseLsofFieldOutput(output: string): PortListener[] { return listeners; } +function normalizeTcpHost(host: string): string { + const normalized = host.toLowerCase(); + return normalized.startsWith("::ffff:") ? normalized.slice("::ffff:".length) : normalized; +} + +function parseTcpEndpoint(raw: string): { host: string; port: number } | null { + const endpoint = raw.trim(); + const bracketMatch = endpoint.match(/^\[([^\]]+)\]:(\d+)$/); + if (bracketMatch) { + const port = Number.parseInt(bracketMatch[2], 10); + return Number.isFinite(port) ? { host: normalizeTcpHost(bracketMatch[1]), port } : null; + } + const lastColon = endpoint.lastIndexOf(":"); + if (lastColon <= 0 || lastColon >= endpoint.length - 1) { + return null; + } + const port = Number.parseInt(endpoint.slice(lastColon + 1), 10); + if (!Number.isFinite(port)) { + return null; + } + return { host: normalizeTcpHost(endpoint.slice(0, lastColon)), port }; +} + +function parseLsofTcpConnectionAddress( + address: string | undefined, +): { local: { host: string; port: number }; remote: { host: string; port: number } } | null { + const normalized = address + ?.replace(/^tcp\s+/i, "") + .replace(/\s*\([^)]*\)\s*$/i, "") + .trim(); + if (!normalized?.includes("->")) { + return null; + } + const [localRaw, remoteRaw] = normalized.split("->", 2); + const local = parseTcpEndpoint(localRaw ?? ""); + const remote = parseTcpEndpoint(remoteRaw ?? ""); + return local && remote ? { local, remote } : null; +} + +function resolveLocalNetworkAddresses(): Set { + const addresses = new Set(["127.0.0.1", "::1", "localhost", "0.0.0.0", "::"]); + for (const entries of Object.values(os.networkInterfaces())) { + for (const entry of entries ?? []) { + addresses.add(entry.address.toLowerCase()); + } + } + return addresses; +} + +function isGatewayConnectionAddress( + address: string | undefined, + port: number, + localAddresses: Set, +): boolean { + const parsed = parseLsofTcpConnectionAddress(address); + if (!parsed) { + return false; + } + if (parsed.local.port === port) { + return true; + } + return parsed.remote.port === port && localAddresses.has(parsed.remote.host); +} + +function resolveLsofTcpDirection( + address: string | undefined, + port: number, +): PortConnectionDirection { + const parsed = parseLsofTcpConnectionAddress(address); + if (!parsed) { + return "unknown"; + } + if (parsed.local.port === port) { + return "server"; + } + return parsed.remote.port === port ? "client" : "unknown"; +} + +function parseLsofConnectionFieldOutput(output: string, port: number): PortConnection[] { + const connections: PortConnection[] = []; + const localAddresses = resolveLocalNetworkAddresses(); + for (const entry of parseLsofFieldOutput(output)) { + if (!isGatewayConnectionAddress(entry.address, port, localAddresses)) { + continue; + } + const connection = entry as PortConnection; + connection.direction = resolveLsofTcpDirection(entry.address, port); + connections.push(connection); + } + return connections; +} + +function parseSsConnectionEndpoint(raw: string): string | null { + if (raw.startsWith("users:")) { + return null; + } + if (raw.includes(":")) { + return raw; + } + return null; +} + +function parseSsConnections(output: string, port: number): PortConnection[] { + const connections: PortConnection[] = []; + const localAddresses = resolveLocalNetworkAddresses(); + for (const rawLine of output.split(/\r?\n/)) { + const line = rawLine.trim(); + if (!line) { + continue; + } + const endpoints = line + .split(/\s+/) + .map(parseSsConnectionEndpoint) + .filter((endpoint): endpoint is string => Boolean(endpoint)); + if (endpoints.length < 2) { + continue; + } + const [local, remote] = endpoints.slice(-2); + const address = `TCP ${local}->${remote} (ESTABLISHED)`; + if (!isGatewayConnectionAddress(address, port, localAddresses)) { + continue; + } + const connection: PortConnection = { + address, + direction: resolveLsofTcpDirection(address, port), + }; + const pidMatch = line.match(/pid=(\d+)/); + if (pidMatch) { + const pid = Number.parseInt(pidMatch[1], 10); + if (Number.isFinite(pid)) { + connection.pid = pid; + } + } + const commandMatch = line.match(/users:\(\("([^"]+)"/); + if (commandMatch?.[1]) { + connection.command = commandMatch[1]; + } + connections.push(connection); + } + return connections; +} + async function enrichUnixListenerProcessInfo(listeners: PortListener[]): Promise { await Promise.all( listeners.map(async (listener) => { @@ -82,6 +232,71 @@ async function enrichUnixListenerProcessInfo(listeners: PortListener[]): Promise ); } +async function readUnixEstablishedConnectionsFromSs( + port: number, +): Promise<{ connections: PortConnection[]; detail?: string; errors: string[] }> { + const errors: string[] = []; + const res = await runCommandSafe([ + "ss", + "-H", + "-tnp", + "state", + "established", + `( sport = :${port} or dport = :${port} )`, + ]); + if (res.code === 0) { + const connections = parseSsConnections(res.stdout, port); + await enrichUnixListenerProcessInfo(connections); + return { connections, detail: res.stdout.trim() || undefined, errors }; + } + const stderr = res.stderr.trim(); + if (res.code === 1 && !res.error && !stderr) { + return { connections: [], detail: undefined, errors }; + } + if (res.error) { + errors.push(res.error); + } + const detail = [stderr, res.stdout.trim()].filter(Boolean).join("\n"); + if (detail) { + errors.push(detail); + } + return { connections: [], detail: undefined, errors }; +} + +async function readUnixEstablishedConnections( + port: number, +): Promise<{ connections: PortConnection[]; detail?: string; errors: string[] }> { + const lsof = await resolveLsofCommand(); + const res = await runCommandSafe([lsof, "-nP", `-iTCP:${port}`, "-sTCP:ESTABLISHED", "-FpFcn"]); + if (res.code === 0) { + const connections = parseLsofConnectionFieldOutput(res.stdout, port); + await enrichUnixListenerProcessInfo(connections); + return { connections, detail: res.stdout.trim() || undefined, errors: [] }; + } + const stderr = res.stderr.trim(); + if (res.code === 1 && !res.error && !stderr) { + return { connections: [], detail: undefined, errors: [] }; + } + const errors: string[] = []; + if (res.error) { + errors.push(res.error); + } + const detail = [stderr, res.stdout.trim()].filter(Boolean).join("\n"); + if (detail) { + errors.push(detail); + } + + const ssFallback = await readUnixEstablishedConnectionsFromSs(port); + if (ssFallback.connections.length > 0) { + return ssFallback; + } + return { + connections: [], + detail: undefined, + errors: [...errors, ...ssFallback.errors], + }; +} + async function resolveUnixCommandLine(pid: number): Promise { const res = await runCommandSafe(["ps", "-p", String(pid), "-o", "command="]); if (res.code !== 0) { @@ -233,6 +448,41 @@ function parseNetstatListeners(output: string, port: number): PortListener[] { return listeners; } +function parseNetstatConnections(output: string, port: number): PortConnection[] { + const connections: PortConnection[] = []; + const localAddresses = resolveLocalNetworkAddresses(); + for (const rawLine of output.split(/\r?\n/)) { + const line = rawLine.trim(); + if (!line || !normalizeLowercaseStringOrEmpty(line).includes("established")) { + continue; + } + const parts = line.split(/\s+/); + if (parts.length < 5) { + continue; + } + const local = parts[1]; + const remote = parts[2]; + const pidRaw = parts.at(-1); + if (!local || !remote || !pidRaw) { + continue; + } + const address = `TCP ${local}->${remote} (ESTABLISHED)`; + if (!isGatewayConnectionAddress(address, port, localAddresses)) { + continue; + } + const connection: PortConnection = { + address, + direction: resolveLsofTcpDirection(address, port), + }; + const pid = Number.parseInt(pidRaw, 10); + if (Number.isFinite(pid)) { + connection.pid = pid; + } + connections.push(connection); + } + return connections; +} + async function resolveWindowsImageName(pid: number): Promise { const res = await runCommandSafe(["tasklist", "/FI", `PID eq ${pid}`, "/FO", "LIST"]); if (res.code !== 0) { @@ -322,6 +572,42 @@ async function readWindowsListeners( return { listeners, detail: res.stdout.trim() || undefined, errors }; } +async function readWindowsEstablishedConnections( + port: number, +): Promise<{ connections: PortConnection[]; detail?: string; errors: string[] }> { + const errors: string[] = []; + const res = await runCommandSafe(["netstat", "-ano", "-p", "tcp"]); + if (res.code !== 0) { + if (res.error) { + errors.push(res.error); + } + const detail = [res.stderr.trim(), res.stdout.trim()].filter(Boolean).join("\n"); + if (detail) { + errors.push(detail); + } + return { connections: [], errors }; + } + const connections = parseNetstatConnections(res.stdout, port); + await Promise.all( + connections.map(async (connection) => { + if (!connection.pid) { + return; + } + const [imageName, commandLine] = await Promise.all([ + resolveWindowsImageName(connection.pid), + resolveWindowsCommandLine(connection.pid), + ]); + if (imageName) { + connection.command = imageName; + } + if (commandLine) { + connection.commandLine = commandLine; + } + }), + ); + return { connections, detail: res.stdout.trim() || undefined, errors }; +} + async function tryListenOnHost(port: number, host: string): Promise { try { await tryListenOnPort({ port, host, exclusive: true }); @@ -380,3 +666,16 @@ export async function inspectPortUsage(port: number): Promise { errors: errors.length > 0 ? errors : undefined, }; } + +export async function inspectPortConnections(port: number): Promise { + const result = + process.platform === "win32" + ? await readWindowsEstablishedConnections(port) + : await readUnixEstablishedConnections(port); + return { + port, + connections: result.connections, + detail: result.detail, + errors: result.errors.length > 0 ? result.errors : undefined, + }; +} diff --git a/src/infra/ports-types.ts b/src/infra/ports-types.ts index 827a5b3ade9..d34f64af3ee 100644 --- a/src/infra/ports-types.ts +++ b/src/infra/ports-types.ts @@ -7,6 +7,12 @@ export type PortListener = { address?: string; }; +export type PortConnectionDirection = "client" | "server" | "unknown"; + +export type PortConnection = PortListener & { + direction: PortConnectionDirection; +}; + export type PortUsageStatus = "free" | "busy" | "unknown"; export type PortUsage = { @@ -19,3 +25,10 @@ export type PortUsage = { }; export type PortListenerKind = "gateway" | "ssh" | "unknown"; + +export type PortConnections = { + port: number; + connections: PortConnection[]; + detail?: string; + errors?: string[]; +}; diff --git a/src/infra/ports.test.ts b/src/infra/ports.test.ts index f1c5b5e63d5..873e7a03cec 100644 --- a/src/infra/ports.test.ts +++ b/src/infra/ports.test.ts @@ -8,6 +8,7 @@ vi.mock("../process/exec.js", () => ({ runCommandWithTimeout: (...args: unknown[]) => runCommandWithTimeoutMock(...args), })); +let inspectPortConnections: typeof import("./ports-inspect.js").inspectPortConnections; let inspectPortUsage: typeof import("./ports-inspect.js").inspectPortUsage; let ensurePortAvailable: typeof import("./ports.js").ensurePortAvailable; let handlePortError: typeof import("./ports.js").handlePortError; @@ -53,7 +54,7 @@ async function listenServer( } beforeAll(async () => { - ({ inspectPortUsage } = await import("./ports-inspect.js")); + ({ inspectPortConnections, inspectPortUsage } = await import("./ports-inspect.js")); ({ ensurePortAvailable, handlePortError, PortInUseError } = await import("./ports.js")); }); @@ -197,9 +198,156 @@ describeUnix("inspectPortUsage", () => { await new Promise((resolve) => server.close(() => resolve())); } }); + + it("reports established gateway client connections from lsof", async () => { + runCommandWithTimeoutMock.mockImplementation(async (argv: string[]) => { + const command = argv[0]; + if (typeof command !== "string") { + return { stdout: "", stderr: "", code: 1 }; + } + if (command.includes("lsof")) { + return { + stdout: + "p111\ncnode\nnTCP 127.0.0.1:50123->127.0.0.1:18789 (ESTABLISHED)\n" + + "p222\ncnode\nnTCP 127.0.0.1:18789->127.0.0.1:50123 (ESTABLISHED)\n" + + "p444\ncnode\nnTCP 127.0.0.1:50125->[::ffff:127.0.0.1]:18789 (ESTABLISHED)\n" + + "p333\ncBrowser\nnTCP 127.0.0.1:50124->198.51.100.7:18789 (ESTABLISHED)\n", + stderr: "", + code: 0, + }; + } + if (command === "ps") { + const pid = argv[2]; + if (argv.includes("command=")) { + return { + stdout: + pid === "111" + ? "node /tmp/newer-openclaw/dist/index.js logs --follow\n" + : pid === "222" + ? "node /tmp/older-openclaw/dist/index.js gateway run\n" + : "browser https://example.invalid/\n", + stderr: "", + code: 0, + }; + } + if (argv.includes("user=")) { + return { stdout: "tester\n", stderr: "", code: 0 }; + } + if (argv.includes("ppid=")) { + return { stdout: "1\n", stderr: "", code: 0 }; + } + } + return { stdout: "", stderr: "", code: 1 }; + }); + + const result = await inspectPortConnections(18789); + + expect(result.connections).toHaveLength(3); + expect(result.connections[0]).toMatchObject({ + pid: 111, + direction: "client", + commandLine: "node /tmp/newer-openclaw/dist/index.js logs --follow", + }); + expect(result.connections[1]).toMatchObject({ + pid: 222, + direction: "server", + }); + expect(result.connections[2]).toMatchObject({ + pid: 444, + direction: "client", + }); + }); + + it("falls back to ss for established gateway client connections", async () => { + runCommandWithTimeoutMock.mockImplementation(async (argv: string[]) => { + const command = argv[0]; + if (typeof command !== "string") { + return { stdout: "", stderr: "", code: 1 }; + } + if (command.includes("lsof")) { + return { stdout: "", stderr: "lsof: not found\n", code: 1 }; + } + if (command === "ss") { + return { + stdout: + '0 0 127.0.0.1:50123 127.0.0.1:18789 users:(("node",pid=111,fd=12))\n' + + '0 0 127.0.0.1:50124 198.51.100.7:18789 users:(("browser",pid=333,fd=9))\n', + stderr: "", + code: 0, + }; + } + if (command === "ps") { + const pid = argv[2]; + if (argv.includes("command=")) { + return { + stdout: + pid === "111" + ? "node /tmp/newer-openclaw/dist/index.js logs --follow\n" + : "browser https://example.invalid/\n", + stderr: "", + code: 0, + }; + } + if (argv.includes("user=")) { + return { stdout: "tester\n", stderr: "", code: 0 }; + } + if (argv.includes("ppid=")) { + return { stdout: "1\n", stderr: "", code: 0 }; + } + } + return { stdout: "", stderr: "", code: 1 }; + }); + + const result = await inspectPortConnections(18789); + + expect(result.connections).toHaveLength(1); + expect(result.connections[0]).toMatchObject({ + pid: 111, + direction: "client", + commandLine: "node /tmp/newer-openclaw/dist/index.js logs --follow", + }); + }); }); describe("inspectPortUsage on Windows", () => { + it("reports established gateway client connections from netstat", async () => { + setPlatform("win32"); + runCommandWithTimeoutMock.mockImplementation(async (argv: string[]) => { + const [command] = argv; + if (command === "netstat") { + return { + stdout: + " TCP 127.0.0.1:50123 127.0.0.1:18789 ESTABLISHED 4242\r\n" + + " TCP 127.0.0.1:50124 198.51.100.7:18789 ESTABLISHED 5000\r\n", + stderr: "", + code: 0, + }; + } + if (command === "tasklist") { + return { stdout: "Image Name: node.exe\r\n", stderr: "", code: 0 }; + } + if (command === "powershell") { + return { + stdout: + '"C:\\Program Files\\nodejs\\node.exe" C:\\Users\\me\\AppData\\Roaming\\npm\\node_modules\\openclaw\\dist\\index.js logs --follow\r\n', + stderr: "", + code: 0, + }; + } + return { stdout: "", stderr: "", code: 1 }; + }); + + const result = await inspectPortConnections(18789); + + expect(result.connections).toHaveLength(1); + expect(result.connections[0]).toMatchObject({ + pid: 4242, + command: "node.exe", + direction: "client", + }); + expect(result.connections[0]?.commandLine).toContain("openclaw"); + }); + it("uses PowerShell process command lines to classify OpenClaw listeners", async () => { setPlatform("win32"); runCommandWithTimeoutMock.mockImplementation(async (argv: string[]) => { diff --git a/src/infra/ports.ts b/src/infra/ports.ts index 40430679a11..74764571545 100644 --- a/src/infra/ports.ts +++ b/src/infra/ports.ts @@ -6,7 +6,14 @@ import { isErrno } from "./errors.js"; import { formatPortDiagnostics } from "./ports-format.js"; import { inspectPortUsage } from "./ports-inspect.js"; import { tryListenOnPort } from "./ports-probe.js"; -import type { PortListener, PortListenerKind, PortUsage, PortUsageStatus } from "./ports-types.js"; +import type { + PortConnection, + PortConnections, + PortListener, + PortListenerKind, + PortUsage, + PortUsageStatus, +} from "./ports-types.js"; class PortInUseError extends Error { port: number; @@ -85,7 +92,14 @@ export async function handlePortError( } export { PortInUseError }; -export type { PortListener, PortListenerKind, PortUsage, PortUsageStatus }; +export type { + PortConnection, + PortConnections, + PortListener, + PortListenerKind, + PortUsage, + PortUsageStatus, +}; export { buildPortHints, classifyPortListener, @@ -94,4 +108,4 @@ export { isExpectedGatewayListeners, isSingleExpectedGatewayListener, } from "./ports-format.js"; -export { inspectPortUsage } from "./ports-inspect.js"; +export { inspectPortConnections, inspectPortUsage } from "./ports-inspect.js";