diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c720dd3edb..56b19d84dc5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,6 +45,7 @@ Docs: https://docs.openclaw.ai - Plugins/uninstall: remove empty managed git install parent directories after deleting cloned plugin repos and cover npm/git uninstall residue in Docker plugin lifecycle tests. Thanks @vincentkoc. - Plugins/install: resolve bare official external plugin IDs such as `brave` through the official catalog when no bundled source is available, so packaged installs fetch the intended scoped npm package instead of an unrelated unscoped package. Fixes #76373. Thanks @bek91 and @vincentkoc. - Plugins/install: require OpenClaw-owned install provenance before granting official npm plugin scanner trust, so direct npm package names no longer bypass launch-code scanning while catalog, onboarding, and doctor installs stay trusted. Thanks @fede-kamel and @vincentkoc. +- Network proxy: preserve target TLS hostname validation for Node HTTPS requests routed through the managed HTTP proxy, so Discord-style CONNECT traffic no longer validates certificates against the local proxy host. Fixes #74809. (#76442) Thanks @jesse-merhi and @abnershang. - Gateway/sessions: keep async `sessions.list` title and preview hydration bounded to transcript head/tail reads so Control UI polling cannot full-scan large session transcripts every refresh. Thanks @vincentkoc. - Gateway/sessions: keep agent runtime metadata on lightweight `sessions.list` rows so model-only session patches do not make Control UI lose runtime identity. Thanks @vincentkoc. - Gateway/sessions: keep bulk `sessions.list` rows lightweight by skipping per-row transcript usage fallback, display model inference, and plugin projection, avoiding event-loop stalls in large session stores. Thanks @Marvinthebored and @vincentkoc. diff --git a/src/infra/net/proxy/external-proxy.e2e.test.ts b/src/infra/net/proxy/external-proxy.e2e.test.ts index cbd0d4977c7..2b5daaa3e67 100644 --- a/src/infra/net/proxy/external-proxy.e2e.test.ts +++ b/src/infra/net/proxy/external-proxy.e2e.test.ts @@ -1,11 +1,106 @@ -import { spawn } from "node:child_process"; +import { execFileSync, spawn } from "node:child_process"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { createServer, request as httpRequest, type Server } from "node:http"; +import { createServer as createHttpsServer } from "node:https"; import * as net from "node:net"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { Duplex } from "node:stream"; import { afterEach, describe, expect, it } from "vitest"; import { WebSocketServer } from "ws"; const CHILD_PROCESS_TIMEOUT_MS = process.env.CI ? 45_000 : 15_000; const PROBE_TIMEOUT_MS = process.env.CI ? 15_000 : 5_000; +const PROXY_TUNNEL_SOCKETS = new WeakMap>(); +type DiscordTlsFixture = { + caPath: string; + cert: string; + cleanup: () => void; + key: string; +}; + +function createDiscordTlsFixture(): DiscordTlsFixture { + const dir = mkdtempSync(join(tmpdir(), "openclaw-discord-tls-")); + try { + const caKeyPath = join(dir, "ca-key.pem"); + const caCertPath = join(dir, "ca-cert.pem"); + const serverKeyPath = join(dir, "server-key.pem"); + const serverCsrPath = join(dir, "server.csr"); + const serverCertPath = join(dir, "server-cert.pem"); + const extPath = join(dir, "server-ext.cnf"); + + execFileSync( + "openssl", + [ + "req", + "-x509", + "-newkey", + "rsa:2048", + "-nodes", + "-keyout", + caKeyPath, + "-out", + caCertPath, + "-days", + "1", + "-subj", + "/CN=OpenClaw Proxy Test CA", + ], + { stdio: "ignore" }, + ); + execFileSync( + "openssl", + [ + "req", + "-newkey", + "rsa:2048", + "-nodes", + "-keyout", + serverKeyPath, + "-out", + serverCsrPath, + "-subj", + "/CN=discord.com", + ], + { stdio: "ignore" }, + ); + writeFileSync(extPath, "subjectAltName=DNS:discord.com\n"); + execFileSync( + "openssl", + [ + "x509", + "-req", + "-in", + serverCsrPath, + "-CA", + caCertPath, + "-CAkey", + caKeyPath, + "-CAcreateserial", + "-out", + serverCertPath, + "-days", + "1", + "-sha256", + "-extfile", + extPath, + ], + { stdio: "ignore" }, + ); + + return { + caPath: caCertPath, + cert: readFileSync(serverCertPath, "utf8"), + cleanup: () => { + rmSync(dir, { force: true, recursive: true }); + }, + key: readFileSync(serverKeyPath, "utf8"), + }; + } catch (err) { + rmSync(dir, { force: true, recursive: true }); + throw err; + } +} async function listenOnLoopback(server: Server): Promise { return new Promise((resolve, reject) => { @@ -26,6 +121,10 @@ async function closeServer(server: Server | null): Promise { if (server === null || !server.listening) { return; } + for (const socket of PROXY_TUNNEL_SOCKETS.get(server) ?? []) { + socket.destroy(); + } + server.closeAllConnections?.(); await new Promise((resolve, reject) => { server.close((err) => { if (err) { @@ -37,7 +136,16 @@ async function closeServer(server: Server | null): Promise { }); } -function createTunnelProxy(seenConnectTargets: string[]): Server { +type ConnectTargetOverride = { + hostname: string; + port: number; +}; + +function createTunnelProxy( + seenConnectTargets: string[], + connectTargetOverrides: Record = {}, +): Server { + const tunnelSockets = new Set(); const proxy = createServer((req, res) => { const target = req.url ?? ""; seenConnectTargets.push(target); @@ -71,6 +179,7 @@ function createTunnelProxy(seenConnectTargets: string[]): Server { }); req.pipe(upstream); }); + PROXY_TUNNEL_SOCKETS.set(proxy, tunnelSockets); proxy.on("connect", (req, clientSocket, head) => { const target = req.url ?? ""; @@ -84,13 +193,26 @@ function createTunnelProxy(seenConnectTargets: string[]): Server { return; } - const upstream = net.connect(Number(targetUrl.port), targetUrl.hostname, () => { - clientSocket.write("HTTP/1.1 200 Connection Established\r\n\r\n"); - if (head.length > 0) { - upstream.write(head); - } - upstream.pipe(clientSocket); - clientSocket.pipe(upstream); + const override = connectTargetOverrides[target]; + tunnelSockets.add(clientSocket); + clientSocket.once("close", () => { + tunnelSockets.delete(clientSocket); + }); + const upstream = net.connect( + override?.port ?? Number(targetUrl.port), + override?.hostname ?? targetUrl.hostname, + () => { + clientSocket.write("HTTP/1.1 200 Connection Established\r\n\r\n"); + if (head.length > 0) { + upstream.write(head); + } + upstream.pipe(clientSocket); + clientSocket.pipe(upstream); + }, + ); + tunnelSockets.add(upstream); + upstream.once("close", () => { + tunnelSockets.delete(upstream); }); upstream.on("error", () => { @@ -155,6 +277,7 @@ async function runNodeModule( describe("SSRF external proxy routing", () => { let target: Server | null = null; let httpsLikeTarget: Server | null = null; + let tlsTarget: Server | null = null; let proxy: Server | null = null; let wss: WebSocketServer | null = null; @@ -167,10 +290,12 @@ describe("SSRF external proxy routing", () => { wss.close(() => resolve()); }); await closeServer(proxy); + await closeServer(tlsTarget); await closeServer(httpsLikeTarget); await closeServer(target); wss = null; proxy = null; + tlsTarget = null; httpsLikeTarget = null; target = null; }); @@ -328,4 +453,75 @@ describe("SSRF external proxy routing", () => { expect(seenConnectTargets).toContain(`http://127.0.0.1:${targetPort}/websocket-proxied`); expect(seenConnectTargets).not.toContain(`http://127.0.0.1:${targetPort}/gateway-bypass`); }); + + it("preserves the target TLS hostname for Node HTTPS requests through the managed proxy", async () => { + const tlsFixture = createDiscordTlsFixture(); + try { + tlsTarget = createHttpsServer({ key: tlsFixture.key, cert: tlsFixture.cert }, (_req, res) => { + res.writeHead(209, { "content-type": "text/plain" }); + res.end("discord target tls ok"); + }); + const tlsTargetPort = await listenOnLoopback(tlsTarget); + + const seenConnectTargets: string[] = []; + proxy = createTunnelProxy(seenConnectTargets, { + [`discord.com:${tlsTargetPort}`]: { hostname: "127.0.0.1", port: tlsTargetPort }, + }); + const proxyPort = await listenOnLoopback(proxy); + + const child = await runNodeModule( + ` + import https from "node:https"; + import { startProxy, stopProxy } from "./src/infra/net/proxy/proxy-lifecycle.ts"; + + async function nodeHttpsGet(url) { + return new Promise((resolve, reject) => { + const req = https.get(url, (response) => { + let body = ""; + response.setEncoding("utf8"); + response.on("data", (chunk) => { + body += chunk; + }); + response.on("end", () => { + resolve({ status: response.statusCode, body }); + }); + }); + req.setTimeout(${PROBE_TIMEOUT_MS}, () => { + req.destroy(new Error("node:https request timed out")); + }); + req.on("error", reject); + }); + } + + const handle = await startProxy({ enabled: true }); + if (handle === null) { + throw new Error("expected external proxy routing to start"); + } + try { + const response = await nodeHttpsGet(process.env.OPENCLAW_TEST_DISCORD_TLS_URL); + console.log(JSON.stringify(response)); + } finally { + await stopProxy(handle); + } + `, + { + ...process.env, + NODE_EXTRA_CA_CERTS: tlsFixture.caPath, + OPENCLAW_PROXY_URL: `http://127.0.0.1:${proxyPort}`, + OPENCLAW_TEST_DISCORD_TLS_URL: `https://discord.com:${tlsTargetPort}/tls-proxy-proof`, + NO_PROXY: "127.0.0.1,localhost", + no_proxy: "localhost", + GLOBAL_AGENT_NO_PROXY: "localhost", + }, + ); + + expect(child.stderr).toBe(""); + expect(child.code).toBe(0); + expect(child.stdout).toContain('"status":209'); + expect(child.stdout).toContain('"body":"discord target tls ok"'); + expect(seenConnectTargets).toContain(`discord.com:${tlsTargetPort}`); + } finally { + tlsFixture.cleanup(); + } + }); }); diff --git a/src/infra/net/proxy/proxy-lifecycle.ts b/src/infra/net/proxy/proxy-lifecycle.ts index 91aca8f3bd9..605ee62e79b 100644 --- a/src/infra/net/proxy/proxy-lifecycle.ts +++ b/src/infra/net/proxy/proxy-lifecycle.ts @@ -9,6 +9,7 @@ import http from "node:http"; import https from "node:https"; +import { isIP } from "node:net"; import { bootstrap as bootstrapGlobalAgent } from "global-agent"; import type { ProxyConfig } from "../../../config/zod-schema.proxy.js"; import { logInfo, logWarn } from "../../../logger.js"; @@ -67,17 +68,23 @@ type ActiveProxyRegistration = { proxyUrl: string; stopped: boolean; }; +type GlobalAgentConnectConfiguration = Record & { + host: string; + tls: Record; +}; let globalAgentBootstrapped = false; let nodeHttpStackSnapshot: NodeHttpStackSnapshot | null = null; let activeProxyRegistrations: ActiveProxyRegistration[] = []; let baseProxyEnvSnapshot: ProxyEnvSnapshot | null = null; +let patchedGlobalAgentHttpsAgents = new WeakSet(); export function _resetGlobalAgentBootstrapForTests(): void { globalAgentBootstrapped = false; nodeHttpStackSnapshot = null; activeProxyRegistrations = []; baseProxyEnvSnapshot = null; + patchedGlobalAgentHttpsAgents = new WeakSet(); } function captureProxyEnv(): ProxyEnvSnapshot { @@ -212,6 +219,7 @@ function bootstrapNodeHttpStack(proxyUrl: string): void { if (!globalAgentBootstrapped) { nodeHttpStackSnapshot = captureNodeHttpStack(); bootstrapGlobalAgent(); + patchGlobalAgentHttpsConnectTlsTargetHost(); globalAgentBootstrapped = true; } @@ -226,6 +234,59 @@ function bootstrapNodeHttpStack(proxyUrl: string): void { } } +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null; +} + +function isGlobalAgentConnectConfiguration( + value: unknown, +): value is GlobalAgentConnectConfiguration { + if (!isRecord(value)) { + return false; + } + return typeof value["host"] === "string" && isRecord(value["tls"]); +} + +function withTlsTargetHost(configuration: unknown): unknown { + if (!isGlobalAgentConnectConfiguration(configuration)) { + return configuration; + } + + const tlsOptions: Record = { + ...configuration.tls, + host: configuration.host, + }; + if (tlsOptions["servername"] === undefined && isIP(configuration.host) === 0) { + tlsOptions["servername"] = configuration.host; + } + return { + ...configuration, + tls: tlsOptions, + }; +} + +function patchGlobalAgentHttpsConnectTlsTargetHost(): void { + const agent = https.globalAgent; + if (typeof agent !== "object" || agent === null || patchedGlobalAgentHttpsAgents.has(agent)) { + return; + } + + const agentRecord = agent as unknown as Record; + const createConnection = agentRecord["createConnection"]; + if (typeof createConnection !== "function") { + return; + } + + agentRecord["createConnection"] = function createConnectionWithTlsTargetHost( + this: unknown, + configuration: unknown, + callback: unknown, + ): unknown { + return createConnection.call(this, withTlsTargetHost(configuration), callback); + }; + patchedGlobalAgentHttpsAgents.add(agent); +} + function findTopActiveProxyRegistration(): ActiveProxyRegistration | null { for (let index = activeProxyRegistrations.length - 1; index >= 0; index -= 1) { const registration = activeProxyRegistrations[index];