diff --git a/extensions/diagnostics-otel/src/service.test.ts b/extensions/diagnostics-otel/src/service.test.ts index f834d2b15b9f..b2c2cc448f3d 100644 --- a/extensions/diagnostics-otel/src/service.test.ts +++ b/extensions/diagnostics-otel/src/service.test.ts @@ -1,4 +1,7 @@ // Diagnostics Otel tests cover service plugin behavior. +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; import { afterAll, afterEach, beforeEach, describe, expect, test, vi } from "vitest"; const telemetryState = vi.hoisted(() => { @@ -60,6 +63,8 @@ const traceExporterCtor = vi.hoisted(() => vi.fn()); const metricExporterCtor = vi.hoisted(() => vi.fn()); const logExporterCtor = vi.hoisted(() => vi.fn()); const spanProcessorCtor = vi.hoisted(() => vi.fn()); +const nodeProxyAgent = vi.hoisted(() => ({ kind: "node-proxy-agent" })); +const createNodeProxyAgentMock = vi.hoisted(() => vi.fn()); const unhandledRejectionHandlerState = vi.hoisted(() => { let handlers: Array<(reason: unknown) => boolean> = []; return { @@ -128,6 +133,10 @@ vi.mock("openclaw/plugin-sdk/runtime-env", () => ({ registerUnhandledRejectionHandler: unhandledRejectionHandlerState.register, })); +vi.mock("openclaw/plugin-sdk/fetch-runtime", () => ({ + createNodeProxyAgent: createNodeProxyAgentMock, +})); + vi.mock("@opentelemetry/sdk-logs", () => ({ BatchLogRecordProcessor: function BatchLogRecordProcessor() {}, LoggerProvider: class { @@ -206,6 +215,23 @@ const ORIGINAL_OTEL_EXPORTER_OTLP_METRICS_ENDPOINT = process.env.OTEL_EXPORTER_OTLP_METRICS_ENDPOINT; const ORIGINAL_OTEL_EXPORTER_OTLP_LOGS_ENDPOINT = process.env.OTEL_EXPORTER_OTLP_LOGS_ENDPOINT; const ORIGINAL_OTEL_SEMCONV_STABILITY_OPT_IN = process.env.OTEL_SEMCONV_STABILITY_OPT_IN; +const OTEL_CERT_ENV_KEYS = [ + "OTEL_EXPORTER_OTLP_CERTIFICATE", + "OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE", + "OTEL_EXPORTER_OTLP_CLIENT_KEY", + "OTEL_EXPORTER_OTLP_TRACES_CERTIFICATE", + "OTEL_EXPORTER_OTLP_TRACES_CLIENT_CERTIFICATE", + "OTEL_EXPORTER_OTLP_TRACES_CLIENT_KEY", + "OTEL_EXPORTER_OTLP_METRICS_CERTIFICATE", + "OTEL_EXPORTER_OTLP_METRICS_CLIENT_CERTIFICATE", + "OTEL_EXPORTER_OTLP_METRICS_CLIENT_KEY", + "OTEL_EXPORTER_OTLP_LOGS_CERTIFICATE", + "OTEL_EXPORTER_OTLP_LOGS_CLIENT_CERTIFICATE", + "OTEL_EXPORTER_OTLP_LOGS_CLIENT_KEY", +] as const; +const ORIGINAL_OTEL_CERT_ENV = Object.fromEntries( + OTEL_CERT_ENV_KEYS.map((key) => [key, process.env[key]]), +) as Record<(typeof OTEL_CERT_ENV_KEYS)[number], string | undefined>; function createLogger() { return { @@ -318,8 +344,46 @@ function mockCallArg(mock: { mock: { calls: unknown[][] } }, argIndex: number, c return mockCall(mock, callIndex)[argIndex]; } -function firstExporterOptions(mock: { mock: { calls: unknown[][] } }): { url?: string } { - return mockCallArg(mock, 0) as { url?: string }; +type TestExporterOptions = { + url?: string; + httpAgentOptions?: (protocol: string) => unknown; +}; + +function firstExporterOptions(mock: { mock: { calls: unknown[][] } }): TestExporterOptions { + return mockCallArg(mock, 0) as TestExporterOptions; +} + +function createNodeProxyAgentCalls(): Array<{ + mode?: string; + targetUrl?: string; + agentOptions?: { + keepAlive?: boolean; + ca?: Buffer; + cert?: Buffer; + key?: Buffer; + }; +}> { + return createNodeProxyAgentMock.mock.calls.map( + ([options]) => + options as { + mode?: string; + targetUrl?: string; + agentOptions?: { + keepAlive?: boolean; + ca?: Buffer; + cert?: Buffer; + key?: Buffer; + }; + }, + ); +} + +function findCreateNodeProxyAgentCall(targetUrl: string) { + const call = createNodeProxyAgentCalls().find((candidate) => candidate.targetUrl === targetUrl); + if (!call) { + throw new Error(`Expected createNodeProxyAgent call for ${targetUrl}`); + } + return call; } function firstSpanProcessorOptions(): { scheduledDelayMillis?: number } { @@ -489,6 +553,7 @@ afterAll(() => { vi.doUnmock("@opentelemetry/sdk-logs"); vi.doUnmock("@opentelemetry/sdk-metrics"); vi.doUnmock("@opentelemetry/sdk-trace-base"); + vi.doUnmock("openclaw/plugin-sdk/fetch-runtime"); vi.doUnmock("@opentelemetry/resources"); vi.doUnmock("@opentelemetry/semantic-conventions"); vi.resetModules(); @@ -514,11 +579,16 @@ describe("diagnostics-otel service", () => { metricExporterCtor.mockClear(); logExporterCtor.mockClear(); spanProcessorCtor.mockClear(); + createNodeProxyAgentMock.mockReset(); + createNodeProxyAgentMock.mockReturnValue(undefined); unhandledRejectionHandlerState.reset(); unhandledRejectionHandlerState.register.mockClear(); delete process.env.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT; delete process.env.OTEL_EXPORTER_OTLP_METRICS_ENDPOINT; delete process.env.OTEL_EXPORTER_OTLP_LOGS_ENDPOINT; + for (const key of OTEL_CERT_ENV_KEYS) { + delete process.env[key]; + } }); afterEach(() => { @@ -549,6 +619,14 @@ describe("diagnostics-otel service", () => { } else { process.env.OTEL_EXPORTER_OTLP_LOGS_ENDPOINT = ORIGINAL_OTEL_EXPORTER_OTLP_LOGS_ENDPOINT; } + for (const key of OTEL_CERT_ENV_KEYS) { + const value = ORIGINAL_OTEL_CERT_ENV[key]; + if (value === undefined) { + delete process.env[key]; + } else { + process.env[key] = value; + } + } }); test("drops camelCase and snake_case diagnostic id log attributes before export", async () => { @@ -1576,6 +1654,162 @@ describe("diagnostics-otel service", () => { await service.stop?.(ctx); }); + test("passes env proxy agents to OTLP HTTP exporters", async () => { + createNodeProxyAgentMock.mockReturnValue(nodeProxyAgent); + + const service = createDiagnosticsOtelService(); + const ctx = createOtelContext("https://collector.example.com/otlp", { + traces: true, + metrics: true, + logs: true, + }); + await service.start(ctx); + + const traceOptions = firstExporterOptions(traceExporterCtor); + const metricOptions = firstExporterOptions(metricExporterCtor); + const logOptions = firstExporterOptions(logExporterCtor); + expect(traceOptions.httpAgentOptions?.("https:")).toBe(nodeProxyAgent); + expect(metricOptions.httpAgentOptions?.("https:")).toBe(nodeProxyAgent); + expect(logOptions.httpAgentOptions?.("https:")).toBe(nodeProxyAgent); + expect(createNodeProxyAgentCalls()).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + mode: "env", + targetUrl: "https://collector.example.com/otlp/v1/traces", + agentOptions: expect.objectContaining({ keepAlive: true }), + }), + expect.objectContaining({ + mode: "env", + targetUrl: "https://collector.example.com/otlp/v1/metrics", + agentOptions: expect.objectContaining({ keepAlive: true }), + }), + expect.objectContaining({ + mode: "env", + targetUrl: "https://collector.example.com/otlp/v1/logs", + agentOptions: expect.objectContaining({ keepAlive: true }), + }), + ]), + ); + await service.stop?.(ctx); + }); + + test("preserves OTLP TLS env options when passing env proxy agents", async () => { + const certDir = mkdtempSync(path.join(tmpdir(), "openclaw-otel-tls-")); + try { + const rootCertificatePath = path.join(certDir, "root.pem"); + const clientCertificatePath = path.join(certDir, "client.pem"); + const clientKeyPath = path.join(certDir, "client-key.pem"); + writeFileSync(rootCertificatePath, "root-certificate"); + writeFileSync(clientCertificatePath, "trace-client-certificate"); + writeFileSync(clientKeyPath, "client-key"); + process.env.OTEL_EXPORTER_OTLP_CERTIFICATE = rootCertificatePath; + process.env.OTEL_EXPORTER_OTLP_TRACES_CLIENT_CERTIFICATE = clientCertificatePath; + process.env.OTEL_EXPORTER_OTLP_CLIENT_KEY = clientKeyPath; + createNodeProxyAgentMock.mockReturnValue(nodeProxyAgent); + + const service = createDiagnosticsOtelService(); + const ctx = createOtelContext("https://collector.example.com/otlp", { + traces: true, + metrics: true, + logs: true, + }); + await service.start(ctx); + + const traceCall = findCreateNodeProxyAgentCall( + "https://collector.example.com/otlp/v1/traces", + ); + const metricCall = findCreateNodeProxyAgentCall( + "https://collector.example.com/otlp/v1/metrics", + ); + expect(traceCall.agentOptions).toEqual({ + keepAlive: true, + ca: Buffer.from("root-certificate"), + cert: Buffer.from("trace-client-certificate"), + key: Buffer.from("client-key"), + }); + expect(metricCall.agentOptions).toEqual({ + keepAlive: true, + ca: Buffer.from("root-certificate"), + key: Buffer.from("client-key"), + }); + await service.stop?.(ctx); + } finally { + rmSync(certDir, { force: true, recursive: true }); + } + }); + + test("falls back to shared OTLP TLS env options when signal-specific values are empty", async () => { + const certDir = mkdtempSync(path.join(tmpdir(), "openclaw-otel-tls-")); + try { + const rootCertificatePath = path.join(certDir, "root.pem"); + writeFileSync(rootCertificatePath, "shared-root-certificate"); + process.env.OTEL_EXPORTER_OTLP_CERTIFICATE = rootCertificatePath; + process.env.OTEL_EXPORTER_OTLP_TRACES_CERTIFICATE = " "; + createNodeProxyAgentMock.mockReturnValue(nodeProxyAgent); + + const service = createDiagnosticsOtelService(); + const ctx = createOtelContext("https://collector.example.com/otlp", { + traces: true, + }); + await service.start(ctx); + + const traceCall = findCreateNodeProxyAgentCall( + "https://collector.example.com/otlp/v1/traces", + ); + expect(traceCall.agentOptions).toEqual({ + keepAlive: true, + ca: Buffer.from("shared-root-certificate"), + }); + await service.stop?.(ctx); + } finally { + rmSync(certDir, { force: true, recursive: true }); + } + }); + + test("falls back to default OTLP agents when env proxy agent creation fails", async () => { + createNodeProxyAgentMock.mockImplementation(() => { + throw new Error("unsupported proxy protocol"); + }); + + const service = createDiagnosticsOtelService(); + const ctx = createOtelContext("https://collector.example.com/otlp", { + traces: true, + metrics: true, + logs: true, + }); + await service.start(ctx); + + expect(firstExporterOptions(traceExporterCtor).httpAgentOptions).toBeUndefined(); + expect(firstExporterOptions(metricExporterCtor).httpAgentOptions).toBeUndefined(); + expect(firstExporterOptions(logExporterCtor).httpAgentOptions).toBeUndefined(); + expect(ctx.logger.warn).toHaveBeenCalledWith( + "diagnostics-otel: env proxy agent unavailable for OTLP traces exporter; falling back to default Node agent", + ); + expect(ctx.logger.warn).toHaveBeenCalledWith( + "diagnostics-otel: env proxy agent unavailable for OTLP metrics exporter; falling back to default Node agent", + ); + expect(ctx.logger.warn).toHaveBeenCalledWith( + "diagnostics-otel: env proxy agent unavailable for OTLP logs exporter; falling back to default Node agent", + ); + await service.stop?.(ctx); + }); + + test("leaves OTLP HTTP exporters on their default agents when env proxy is bypassed", async () => { + const service = createDiagnosticsOtelService(); + const ctx = createOtelContext("https://collector.example.com/otlp", { + traces: true, + metrics: true, + logs: true, + }); + await service.start(ctx); + + expect(firstExporterOptions(traceExporterCtor).httpAgentOptions).toBeUndefined(); + expect(firstExporterOptions(metricExporterCtor).httpAgentOptions).toBeUndefined(); + expect(firstExporterOptions(logExporterCtor).httpAgentOptions).toBeUndefined(); + expect(createNodeProxyAgentMock).toHaveBeenCalledTimes(3); + await service.stop?.(ctx); + }); + test("exports diagnostic logs as stdout JSONL without constructing the OTLP log exporter", async () => { const service = createDiagnosticsOtelService(); const ctx = createOtelContext("", { diff --git a/extensions/diagnostics-otel/src/service.ts b/extensions/diagnostics-otel/src/service.ts index 3cd8f0f504e1..6d8515293e54 100644 --- a/extensions/diagnostics-otel/src/service.ts +++ b/extensions/diagnostics-otel/src/service.ts @@ -1,4 +1,8 @@ // Diagnostics Otel plugin module implements service behavior. +import { readFileSync } from "node:fs"; +import type { Agent as HttpAgent } from "node:http"; +import type { Agent as HttpsAgent, AgentOptions as HttpsAgentOptions } from "node:https"; +import nodePath from "node:path"; import { context as otelContextApi, metrics, @@ -29,6 +33,7 @@ import { ATTR_GEN_AI_TOOL_DEFINITIONS, } from "@opentelemetry/semantic-conventions/incubating"; import { waitForDiagnosticEventsDrained } from "openclaw/plugin-sdk/diagnostic-runtime"; +import { createNodeProxyAgent } from "openclaw/plugin-sdk/fetch-runtime"; import { registerUnhandledRejectionHandler } from "openclaw/plugin-sdk/runtime-env"; import { isRecord } from "openclaw/plugin-sdk/string-coerce-runtime"; import type { @@ -36,6 +41,7 @@ import type { DiagnosticEventPayload, DiagnosticTraceContext, OpenClawPluginService, + OpenClawPluginServiceContext, } from "../api.js"; import { isValidDiagnosticSpanId, @@ -83,6 +89,9 @@ const OTEL_EXPORTER_OTLP_ENDPOINT_ENV = "OTEL_EXPORTER_OTLP_ENDPOINT"; const OTEL_EXPORTER_OTLP_TRACES_ENDPOINT_ENV = "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT"; const OTEL_EXPORTER_OTLP_METRICS_ENDPOINT_ENV = "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT"; const OTEL_EXPORTER_OTLP_LOGS_ENDPOINT_ENV = "OTEL_EXPORTER_OTLP_LOGS_ENDPOINT"; +const OTEL_EXPORTER_OTLP_CERTIFICATE_ENV = "OTEL_EXPORTER_OTLP_CERTIFICATE"; +const OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE_ENV = "OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE"; +const OTEL_EXPORTER_OTLP_CLIENT_KEY_ENV = "OTEL_EXPORTER_OTLP_CLIENT_KEY"; const OTEL_SEMCONV_STABILITY_OPT_IN_ENV = "OTEL_SEMCONV_STABILITY_OPT_IN"; const GEN_AI_LATEST_EXPERIMENTAL_OPT_IN = "gen_ai_latest_experimental"; const GEN_AI_TOKEN_USAGE_BUCKETS = [ @@ -105,6 +114,13 @@ type OtelContentCapturePolicy = { }; type OtelLogsExporter = "otlp" | "stdout" | "both"; +type OtelHttpAgent = HttpAgent | HttpsAgent; +type OtelHttpAgentFactory = (protocol: string) => OtelHttpAgent | Promise; +type OtelSignalIdentifier = "TRACES" | "METRICS" | "LOGS"; +type OtelHttpAgentOptions = HttpsAgentOptions & { + keepAlive: true; +}; +type OtelLogger = OpenClawPluginServiceContext["logger"]; type BuiltOtelLogRecord = { logRecord: LogRecord; @@ -198,6 +214,80 @@ function resolveSignalOtelUrl(params: { ); } +function readOtelEnvFile(params: { + signalIdentifier: OtelSignalIdentifier; + signalSuffix: "CERTIFICATE" | "CLIENT_CERTIFICATE" | "CLIENT_KEY"; + sharedEnvName: string; + logger: OtelLogger; + warning: string; +}): Buffer | undefined { + const signalEnvName = `OTEL_EXPORTER_OTLP_${params.signalIdentifier}_${params.signalSuffix}`; + const filePath = + normalizeOtelEnvValue(process.env[signalEnvName]) ?? + normalizeOtelEnvValue(process.env[params.sharedEnvName]); + if (!filePath) { + return undefined; + } + try { + return readFileSync(nodePath.resolve(process.cwd(), filePath)); + } catch { + params.logger.warn(`diagnostics-otel: ${params.warning}`); + return undefined; + } +} + +function normalizeOtelEnvValue(value: string | undefined): string | undefined { + const trimmed = value?.trim(); + return trimmed ? trimmed : undefined; +} + +function resolveOtelHttpAgentOptions(params: { + url: string | undefined; + signalIdentifier: OtelSignalIdentifier; + logger: OtelLogger; +}): OtelHttpAgentFactory | undefined { + const { url, signalIdentifier, logger } = params; + if (!url) { + return undefined; + } + const ca = readOtelEnvFile({ + signalIdentifier, + signalSuffix: "CERTIFICATE", + sharedEnvName: OTEL_EXPORTER_OTLP_CERTIFICATE_ENV, + logger, + warning: "failed to read root certificate file", + }); + const cert = readOtelEnvFile({ + signalIdentifier, + signalSuffix: "CLIENT_CERTIFICATE", + sharedEnvName: OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE_ENV, + logger, + warning: "failed to read client certificate chain file", + }); + const key = readOtelEnvFile({ + signalIdentifier, + signalSuffix: "CLIENT_KEY", + sharedEnvName: OTEL_EXPORTER_OTLP_CLIENT_KEY_ENV, + logger, + warning: "failed to read client certificate private key file", + }); + const agentOptions: OtelHttpAgentOptions = { + keepAlive: true, + ...(ca !== undefined ? { ca } : {}), + ...(cert !== undefined ? { cert } : {}), + ...(key !== undefined ? { key } : {}), + }; + try { + const agent = createNodeProxyAgent({ mode: "env", targetUrl: url, agentOptions }); + return agent ? () => agent : undefined; + } catch { + logger.warn( + `diagnostics-otel: env proxy agent unavailable for OTLP ${signalIdentifier.toLowerCase()} exporter; falling back to default Node agent`, + ); + return undefined; + } +} + function resolveSampleRate(value: number | undefined): number | undefined { if (typeof value !== "number" || !Number.isFinite(value)) { return undefined; @@ -1482,10 +1572,21 @@ export function createDiagnosticsOtelService(): OpenClawPluginService { endpoint, path: "v1/metrics", }); + const traceHttpAgentOptions = resolveOtelHttpAgentOptions({ + url: traceUrl, + signalIdentifier: "TRACES", + logger: ctx.logger, + }); + const metricHttpAgentOptions = resolveOtelHttpAgentOptions({ + url: metricUrl, + signalIdentifier: "METRICS", + logger: ctx.logger, + }); const traceExporter = tracesEnabled ? new OTLPTraceExporter({ ...(traceUrl ? { url: traceUrl } : {}), ...(headers ? { headers } : {}), + ...(traceHttpAgentOptions ? { httpAgentOptions: traceHttpAgentOptions } : {}), }) : undefined; const spanProcessors = @@ -1501,6 +1602,7 @@ export function createDiagnosticsOtelService(): OpenClawPluginService { ? new OTLPMetricExporter({ ...(metricUrl ? { url: metricUrl } : {}), ...(headers ? { headers } : {}), + ...(metricHttpAgentOptions ? { httpAgentOptions: metricHttpAgentOptions } : {}), }) : undefined; @@ -1904,9 +2006,15 @@ export function createDiagnosticsOtelService(): OpenClawPluginService { let otelLogger: { emit: (logRecord: LogRecord) => void } | undefined; if (logsToOtlp) { + const logHttpAgentOptions = resolveOtelHttpAgentOptions({ + url: logUrl, + signalIdentifier: "LOGS", + logger: ctx.logger, + }); const logExporter = new OTLPLogExporter({ ...(logUrl ? { url: logUrl } : {}), ...(headers ? { headers } : {}), + ...(logHttpAgentOptions ? { httpAgentOptions: logHttpAgentOptions } : {}), }); const logProcessor = new BatchLogRecordProcessor( logExporter, diff --git a/src/cli/gateway-cli/run-loop.ts b/src/cli/gateway-cli/run-loop.ts index 331be7e924dc..b1e7621fcaa3 100644 --- a/src/cli/gateway-cli/run-loop.ts +++ b/src/cli/gateway-cli/run-loop.ts @@ -179,7 +179,6 @@ export async function runGatewayLoop(params: { }; const reacquireLockForInProcessRestart = async (): Promise => { try { - startupStartedAt = Date.now(); lock = await acquireGatewayLock({ port: params.lockPort }); return true; } catch (err) { diff --git a/src/gateway/server-sidecar-startup-mode.ts b/src/gateway/server-sidecar-startup-mode.ts new file mode 100644 index 000000000000..1acbe9878e5d --- /dev/null +++ b/src/gateway/server-sidecar-startup-mode.ts @@ -0,0 +1,2 @@ +// Startup mode shared by the server implementation and post-ready sidecar scheduler. +export type GatewaySidecarStartupMode = "start" | "defer"; diff --git a/src/gateway/server-startup-post-attach.ts b/src/gateway/server-startup-post-attach.ts index 8fa38433d194..c4dec73015fe 100644 --- a/src/gateway/server-startup-post-attach.ts +++ b/src/gateway/server-startup-post-attach.ts @@ -23,6 +23,7 @@ import { } from "./events.js"; import { STARTUP_UNAVAILABLE_GATEWAY_METHODS } from "./methods/core-descriptors.js"; import type { refreshLatestUpdateRestartSentinel } from "./server-restart-sentinel.js"; +import type { GatewaySidecarStartupMode } from "./server-sidecar-startup-mode.js"; import type { logGatewayStartup } from "./server-startup-log.js"; import type { startGatewayTailscaleExposure } from "./server-tailscale.js"; @@ -71,8 +72,6 @@ export type GatewayPostReadySidecarHandle = { stop: () => Awaitable; }; -export type GatewaySidecarStartupMode = "start" | "defer"; - /** Stop sidecars immediately when shutdown has already started before they are reported. */ export function stopPostReadySidecarsAfterCloseStarted(params: { postReadySidecars: readonly GatewayPostReadySidecarHandle[]; diff --git a/src/gateway/server.impl.ts b/src/gateway/server.impl.ts index 056fe038a9fa..102712b29448 100644 --- a/src/gateway/server.impl.ts +++ b/src/gateway/server.impl.ts @@ -108,7 +108,7 @@ import { getRequiredSharedGatewaySessionGeneration, type SharedGatewaySessionGenerationState, } from "./server-shared-auth-generation.js"; -import type { GatewaySidecarStartupMode } from "./server-startup-post-attach.js"; +import type { GatewaySidecarStartupMode } from "./server-sidecar-startup-mode.js"; import { createWizardSessionTracker } from "./server-wizard-sessions.js"; import { createGatewayEventLoopHealthMonitor } from "./server/event-loop-health.js"; import { diff --git a/src/infra/net/node-proxy-agent.test.ts b/src/infra/net/node-proxy-agent.test.ts new file mode 100644 index 000000000000..1243689a8153 --- /dev/null +++ b/src/infra/net/node-proxy-agent.test.ts @@ -0,0 +1,62 @@ +// Node proxy agent tests cover shared Node HTTP(S) proxy agent construction. +import { describe, expect, it } from "vitest"; +import { withEnv } from "../../test-utils/env.js"; +import { createNodeProxyAgent } from "./node-proxy-agent.js"; + +const PROXY_ENV_KEYS = [ + "http_proxy", + "HTTP_PROXY", + "https_proxy", + "HTTPS_PROXY", + "all_proxy", + "ALL_PROXY", + "no_proxy", + "NO_PROXY", +] as const; + +function withProxyEnv( + env: Partial>, + fn: () => T, +): T { + const clearedEnv = Object.fromEntries(PROXY_ENV_KEYS.map((key) => [key, undefined])) as Record< + (typeof PROXY_ENV_KEYS)[number], + undefined + >; + return withEnv({ ...clearedEnv, ...env }, fn); +} + +describe("createNodeProxyAgent", () => { + it("preserves caller Node agent options on env proxy agents", () => { + withProxyEnv({ HTTPS_PROXY: "http://proxy.example:8080" }, () => { + const agent = createNodeProxyAgent({ + mode: "env", + targetUrl: "https://collector.example.test/v1/traces", + agentOptions: { + keepAlive: true, + ca: "collector-ca", + cert: "collector-cert", + key: "collector-key", + }, + }); + + const agentState = agent as + | { + options?: { + keepAlive?: boolean; + ca?: string; + cert?: string; + key?: string; + }; + keepAlive?: boolean; + } + | undefined; + expect(agentState?.options).toMatchObject({ + keepAlive: true, + ca: "collector-ca", + cert: "collector-cert", + key: "collector-key", + }); + expect(agentState?.keepAlive).toBe(true); + }); + }); +}); diff --git a/src/infra/net/node-proxy-agent.ts b/src/infra/net/node-proxy-agent.ts index 470ae54814d8..373721b3eefa 100644 --- a/src/infra/net/node-proxy-agent.ts +++ b/src/infra/net/node-proxy-agent.ts @@ -1,9 +1,10 @@ // Node proxy agent helpers adapt env or explicit proxy settings for libraries // that need node:http Agent instances. -import type { Agent as HttpAgent } from "node:http"; +import type { Agent as HttpAgent, AgentOptions as HttpAgentOptions } from "node:http"; +import type { AgentOptions as HttpsAgentOptions } from "node:https"; import { createRequire } from "node:module"; import { matchesNoProxy, resolveEnvHttpProxyAgentOptions } from "./proxy-env.js"; -import { resolveActiveManagedProxyTlsOptions } from "./proxy/managed-proxy-undici.js"; +import { resolveActiveManagedProxyTlsOptions } from "./proxy/active-managed-proxy-tls.js"; export const UNSUPPORTED_PROXY_PROTOCOL_MESSAGE = "Unsupported proxy protocol. SOCKS and PAC proxy URLs are not supported; use an HTTP or HTTPS proxy URL."; @@ -14,6 +15,17 @@ type ProxylineCreateAmbientNodeProxyAgent = type ProxylineAgentOptions = NonNullable[0]>; type ProxylineEnvSnapshot = NonNullable; type ProxylineTlsOptions = ProxylineAgentOptions["proxyTls"]; +type NodeProxyAgentOptions = HttpAgentOptions & HttpsAgentOptions; +type NodeProxyAgentWithOptions = HttpAgent & { + keepAlive: boolean; + keepAliveMsecs: number; + maxFreeSockets: number; + maxSockets: number; + maxTotalSockets: number; + options?: NodeProxyAgentOptions; + scheduling?: "fifo" | "lifo"; + timeout?: number; +}; const require = createRequire(import.meta.url); @@ -23,11 +35,13 @@ export type CreateNodeProxyAgentOptions = mode: "env"; targetUrl: string | URL; protocol?: NodeProxyProtocol; + agentOptions?: NodeProxyAgentOptions; } | { mode: "explicit"; proxyUrl: string | URL; protocol?: NodeProxyProtocol; + agentOptions?: NodeProxyAgentOptions; }; function inferTargetProtocol(targetUrl: string | URL): NodeProxyProtocol | undefined { @@ -110,6 +124,38 @@ function loadCreateAmbientNodeProxyAgent(): ProxylineCreateAmbientNodeProxyAgent .createAmbientNodeProxyAgent; } +function applyNodeAgentOptions(agent: HttpAgent, options: NodeProxyAgentOptions | undefined): void { + if (options === undefined) { + return; + } + const agentWithOptions = agent as NodeProxyAgentWithOptions; + agentWithOptions.options = { + ...agentWithOptions.options, + ...options, + }; + if (typeof options.keepAlive === "boolean") { + agentWithOptions.keepAlive = options.keepAlive; + } + if (typeof options.keepAliveMsecs === "number") { + agentWithOptions.keepAliveMsecs = options.keepAliveMsecs; + } + if (typeof options.maxFreeSockets === "number") { + agentWithOptions.maxFreeSockets = options.maxFreeSockets; + } + if (typeof options.maxSockets === "number") { + agentWithOptions.maxSockets = options.maxSockets; + } + if (typeof options.maxTotalSockets === "number") { + agentWithOptions.maxTotalSockets = options.maxTotalSockets; + } + if (options.scheduling === "fifo" || options.scheduling === "lifo") { + agentWithOptions.scheduling = options.scheduling; + } + if (typeof options.timeout === "number") { + agentWithOptions.timeout = options.timeout; + } +} + /** Resolves the env proxy URL that should be used for a specific Node target. */ export function resolveEnvNodeProxyUrlForTarget( targetUrl: string | URL, @@ -136,6 +182,7 @@ function createFixedNodeProxyAgent( options: { protocol?: NodeProxyProtocol; proxyTls?: ProxylineTlsOptions; + agentOptions?: NodeProxyAgentOptions; } = {}, ): HttpAgent { const parsedProxyUrl = @@ -150,6 +197,7 @@ function createFixedNodeProxyAgent( if (agent === undefined) { throw new Error(`${UNSUPPORTED_PROXY_PROTOCOL_MESSAGE} Got ${parsedProxyUrl.protocol}`); } + applyNodeAgentOptions(agent as HttpAgent, options.agentOptions); return agent as HttpAgent; } @@ -163,15 +211,22 @@ export function createNodeProxyAgent( ): HttpAgent | undefined; export function createNodeProxyAgent(options: CreateNodeProxyAgentOptions): HttpAgent | undefined { if (options.mode === "explicit") { - return createFixedNodeProxyAgent(options.proxyUrl, { protocol: options.protocol }); + return createFixedNodeProxyAgent(options.proxyUrl, { + protocol: options.protocol, + agentOptions: options.agentOptions, + }); } - return createEnvNodeProxyAgentForTarget(options.targetUrl, { protocol: options.protocol }); + return createEnvNodeProxyAgentForTarget(options.targetUrl, { + protocol: options.protocol, + agentOptions: options.agentOptions, + }); } function createEnvNodeProxyAgentForTarget( targetUrl: string | URL, options: { protocol?: NodeProxyProtocol; + agentOptions?: NodeProxyAgentOptions; } = {}, ): HttpAgent | undefined { const proxyUrl = resolveEnvNodeProxyUrlForTarget(targetUrl); @@ -181,6 +236,7 @@ function createEnvNodeProxyAgentForTarget( return createFixedNodeProxyAgent(proxyUrl, { protocol: options.protocol ?? inferTargetProtocol(targetUrl) ?? "https", proxyTls: resolveActiveManagedProxyTlsOptions({ proxyUrl: proxyUrl.href }), + agentOptions: options.agentOptions, }); } diff --git a/src/infra/net/proxy/active-managed-proxy-tls.ts b/src/infra/net/proxy/active-managed-proxy-tls.ts new file mode 100644 index 000000000000..0c550c8e367d --- /dev/null +++ b/src/infra/net/proxy/active-managed-proxy-tls.ts @@ -0,0 +1,71 @@ +// Resolves OpenClaw-managed proxy TLS trust for non-Undici transports. +import { resolveEnvHttpProxyUrl } from "../proxy-env.js"; +import { getActiveManagedProxyTlsOptions, getActiveManagedProxyUrl } from "./active-proxy-state.js"; +import { + loadManagedProxyTlsOptionsSync, + resolveManagedProxyCaFileForUrl, + type ManagedProxyTlsOptions, +} from "./proxy-tls.js"; + +type ManagedProxyTlsEnv = NodeJS.ProcessEnv; + +type ResolveActiveManagedProxyTlsOptionsParams = { + proxyUrl?: string; + env?: ManagedProxyTlsEnv; +}; + +const MANAGED_PROXY_ENV_PREFIX = ["OPENCLAW", "PROXY"].join("_"); +const MANAGED_PROXY_ACTIVE_ENV_KEY = `${MANAGED_PROXY_ENV_PREFIX}_ACTIVE`; +const MANAGED_PROXY_CA_FILE_ENV_KEY = `${MANAGED_PROXY_ENV_PREFIX}_CA_FILE`; + +function normalizeProxyUrl(value: string | undefined): string | undefined { + if (!value) { + return undefined; + } + try { + return new URL(value).href; + } catch { + return undefined; + } +} + +function resolveManagedProxyUrl(env: ManagedProxyTlsEnv = process.env): string | undefined { + const activeProxyUrl = getActiveManagedProxyUrl(); + if (activeProxyUrl) { + return activeProxyUrl.href; + } + if (env[MANAGED_PROXY_ACTIVE_ENV_KEY] !== "1") { + return undefined; + } + // Child processes inherit only env, so recover the managed proxy URL from + // HTTPS proxy settings when the active in-process registration is absent. + return normalizeProxyUrl(resolveEnvHttpProxyUrl("https", env)); +} + +/** Resolves managed proxy TLS trust only when the target proxy is OpenClaw's active proxy. */ +export function resolveActiveManagedProxyTlsOptions( + params?: ResolveActiveManagedProxyTlsOptionsParams, +): ManagedProxyTlsOptions | undefined { + const env = params?.env ?? process.env; + const managedProxyUrl = resolveManagedProxyUrl(env); + const targetProxyUrl = normalizeProxyUrl( + params?.proxyUrl ?? resolveEnvHttpProxyUrl("https", env), + ); + if (!managedProxyUrl || targetProxyUrl !== managedProxyUrl) { + return undefined; + } + const activeProxyTls = getActiveManagedProxyTlsOptions(); + if (activeProxyTls) { + return activeProxyTls; + } + const proxyCaFile = resolveManagedProxyCaFileForUrl({ + proxyUrl: managedProxyUrl, + caFileOverride: env[MANAGED_PROXY_CA_FILE_ENV_KEY], + }); + try { + return loadManagedProxyTlsOptionsSync(proxyCaFile); + } catch { + // Missing inherited CA files should not break non-managed or caller-owned proxies. + return undefined; + } +} diff --git a/src/infra/net/proxy/managed-proxy-undici.ts b/src/infra/net/proxy/managed-proxy-undici.ts index d8b63b521a41..219be2d68c79 100644 --- a/src/infra/net/proxy/managed-proxy-undici.ts +++ b/src/infra/net/proxy/managed-proxy-undici.ts @@ -2,13 +2,11 @@ // explicit ProxyAgent options without changing unrelated operator proxies. import { isRecord as isProxyTlsRecord } from "@openclaw/normalization-core/record-coerce"; import type { EnvHttpProxyAgent } from "undici"; -import { resolveEnvHttpProxyAgentOptions, resolveEnvHttpProxyUrl } from "../proxy-env.js"; -import { getActiveManagedProxyTlsOptions, getActiveManagedProxyUrl } from "./active-proxy-state.js"; -import { - loadManagedProxyTlsOptionsSync, - resolveManagedProxyCaFileForUrl, - type ManagedProxyTlsOptions, -} from "./proxy-tls.js"; +import { resolveEnvHttpProxyAgentOptions } from "../proxy-env.js"; +import { resolveActiveManagedProxyTlsOptions } from "./active-managed-proxy-tls.js"; +import type { ManagedProxyTlsOptions } from "./proxy-tls.js"; + +export { resolveActiveManagedProxyTlsOptions } from "./active-managed-proxy-tls.js"; type ManagedEnvHttpProxyAgentOptions = ConstructorParameters[0]; @@ -39,69 +37,12 @@ function readProxyUrlFromOptions(options: object | undefined): string | undefine return undefined; } -function normalizeProxyUrl(value: string | undefined): string | undefined { - if (!value) { - return undefined; - } - try { - return new URL(value).href; - } catch { - return undefined; - } -} - type ManagedProxyTlsEnv = NodeJS.ProcessEnv; -type ResolveActiveManagedProxyTlsOptionsParams = { - proxyUrl?: string; - env?: ManagedProxyTlsEnv; -}; - type AddActiveManagedProxyTlsOptionsParams = { env?: ManagedProxyTlsEnv; }; -function resolveManagedProxyUrl(env: ManagedProxyTlsEnv = process.env): string | undefined { - const activeProxyUrl = getActiveManagedProxyUrl(); - if (activeProxyUrl) { - return activeProxyUrl.href; - } - if (env["OPENCLAW_PROXY_ACTIVE"] !== "1") { - return undefined; - } - // Child processes inherit only env, so recover the managed proxy URL from - // HTTPS proxy settings when the active in-process registration is absent. - return normalizeProxyUrl(resolveEnvHttpProxyUrl("https", env)); -} - -/** Resolves managed proxy TLS trust only when the target proxy is OpenClaw's active proxy. */ -export function resolveActiveManagedProxyTlsOptions( - params?: ResolveActiveManagedProxyTlsOptionsParams, -): ManagedProxyTlsOptions | undefined { - const env = params?.env ?? process.env; - const managedProxyUrl = resolveManagedProxyUrl(env); - const targetProxyUrl = normalizeProxyUrl( - params?.proxyUrl ?? resolveEnvHttpProxyUrl("https", env), - ); - if (!managedProxyUrl || targetProxyUrl !== managedProxyUrl) { - return undefined; - } - const activeProxyTls = getActiveManagedProxyTlsOptions(); - if (activeProxyTls) { - return activeProxyTls; - } - const proxyCaFile = resolveManagedProxyCaFileForUrl({ - proxyUrl: managedProxyUrl, - caFileOverride: env["OPENCLAW_PROXY_CA_FILE"], - }); - try { - return loadManagedProxyTlsOptionsSync(proxyCaFile); - } catch { - // Missing inherited CA files should not break non-managed or caller-owned proxies. - return undefined; - } -} - /** Adds active managed proxy TLS options to env proxy agent options. */ export function addActiveManagedProxyTlsOptions( options: undefined,