diff --git a/packages/gateway-client/src/client.ts b/packages/gateway-client/src/client.ts index 2ffac5259c3..c247227b8e3 100644 --- a/packages/gateway-client/src/client.ts +++ b/packages/gateway-client/src/client.ts @@ -1,14 +1,10 @@ import { randomUUID } from "node:crypto"; -import { - type ConnectParams, - type EventFrame, - type HelloOk, - MIN_CLIENT_PROTOCOL_VERSION, - PROTOCOL_VERSION, - type RequestFrame, - validateEventFrame, - validateRequestFrame, - validateResponseFrame, +import type { + ConnectParams, + EventFrame, + HelloOk, + RequestFrame, + ResponseFrame, } from "@openclaw/gateway-protocol"; import { GATEWAY_CLIENT_MODES, @@ -25,6 +21,7 @@ import { type ConnectErrorRecoveryAdvice, } from "@openclaw/gateway-protocol/connect-error-details"; import { resolveGatewayStartupRetryAfterMs } from "@openclaw/gateway-protocol/startup-unavailable"; +import { MIN_CLIENT_PROTOCOL_VERSION, PROTOCOL_VERSION } from "@openclaw/gateway-protocol/version"; import ipaddr from "ipaddr.js"; import { WebSocket, type ClientOptions, type CertMeta } from "ws"; import { buildDeviceAuthPayloadV3 } from "./device-auth.js"; @@ -80,6 +77,63 @@ function normalizeOptionalString(value: unknown): string | undefined { return trimmed || undefined; } +function isRecord(value: unknown): value is Record { + return Boolean(value) && typeof value === "object" && !Array.isArray(value); +} + +function isNonEmptyString(value: unknown): value is string { + return typeof value === "string" && value.length > 0; +} + +function isNonNegativeInteger(value: unknown): value is number { + return typeof value === "number" && Number.isInteger(value) && value >= 0; +} + +function isGatewayClientErrorShape(value: unknown): boolean { + if (!isRecord(value)) { + return false; + } + if (!isNonEmptyString(value.code) || !isNonEmptyString(value.message)) { + return false; + } + if (value.retryable !== undefined && typeof value.retryable !== "boolean") { + return false; + } + if (value.retryAfterMs !== undefined && !isNonNegativeInteger(value.retryAfterMs)) { + return false; + } + return true; +} + +function isGatewayEventFrame(value: unknown): value is EventFrame { + if (!isRecord(value) || value.type !== "event" || !isNonEmptyString(value.event)) { + return false; + } + return value.seq === undefined || isNonNegativeInteger(value.seq); +} + +function isGatewayResponseFrame(value: unknown): value is ResponseFrame { + if ( + !isRecord(value) || + value.type !== "res" || + !isNonEmptyString(value.id) || + typeof value.ok !== "boolean" + ) { + return false; + } + return value.error === undefined || isGatewayClientErrorShape(value.error); +} + +function validateClientRequestFrame(frame: RequestFrame): string | null { + if (!isNonEmptyString(frame.id)) { + return "id must be a non-empty string"; + } + if (!isNonEmptyString(frame.method)) { + return "method must be a non-empty string"; + } + return null; +} + function normalizeLowercaseStringOrEmpty(value: unknown): string { return typeof value === "string" ? value.trim().toLowerCase() : ""; } @@ -1233,7 +1287,7 @@ export class GatewayClient { this.logDebug(`gateway client parse error: ${formatGatewayClientErrorForLog(err)}`); return; } - if (validateEventFrame(parsed)) { + if (isGatewayEventFrame(parsed)) { this.lastTick = Date.now(); const evt = parsed; if (evt.event === "connect.challenge") { @@ -1267,7 +1321,7 @@ export class GatewayClient { } return; } - if (validateResponseFrame(parsed)) { + if (isGatewayResponseFrame(parsed)) { this.lastTick = Date.now(); const pending = this.pending.get(parsed.id); if (!pending) { @@ -1454,10 +1508,9 @@ export class GatewayClient { } const id = randomUUID(); const frame: RequestFrame = { type: "req", id, method, params }; - if (!validateRequestFrame(frame)) { - throw new Error( - `invalid request frame: ${JSON.stringify(validateRequestFrame.errors, null, 2)}`, - ); + const requestFrameError = validateClientRequestFrame(frame); + if (requestFrameError) { + throw new Error(`invalid request frame: ${requestFrameError}`); } const expectFinal = opts?.expectFinal === true; const timeoutMs = diff --git a/src/cli/gateway-dispatch-dotenv.ts b/src/cli/gateway-dispatch-dotenv.ts new file mode 100644 index 00000000000..ef7174e0ef4 --- /dev/null +++ b/src/cli/gateway-dispatch-dotenv.ts @@ -0,0 +1,21 @@ +import fs from "node:fs"; +import path from "node:path"; +import { resolveStateDir } from "../config/paths.js"; +import { loadGlobalRuntimeDotEnvFiles } from "../infra/dotenv-global.js"; + +export async function loadGatewayDispatchCliDotEnv(opts?: { quiet?: boolean }) { + const quiet = opts?.quiet ?? true; + const cwdEnvPath = path.join(process.cwd(), ".env"); + if (fs.existsSync(cwdEnvPath)) { + const { loadCliDotEnv } = await import("./dotenv.js"); + loadCliDotEnv({ quiet }); + return; + } + + // Agent dispatch only needs trusted runtime env for gateway credentials. + // Workspace .env still falls back to the full provider-aware loader above. + loadGlobalRuntimeDotEnvFiles({ + quiet, + stateEnvPath: path.join(resolveStateDir(process.env), ".env"), + }); +} diff --git a/src/cli/run-main.ts b/src/cli/run-main.ts index 5aeb2a2d948..fae820c844c 100644 --- a/src/cli/run-main.ts +++ b/src/cli/run-main.ts @@ -111,6 +111,10 @@ function createGatewayCliMainStartupTrace(argv: string[]) { }; } +function isRemoteAgentDispatchInvocation(argv: string[], primary: string | null): boolean { + return primary === "agent" && !argv.includes("--local"); +} + export function isGatewayRunFastPathArgv(argv: string[]): boolean { const invocation = resolveCliArgvInvocation(argv); if (invocation.hasHelpOrVersion) { @@ -500,8 +504,13 @@ export async function runCli(argv: string[] = process.argv) { if (!isHelpOrVersionInvocation && shouldLoadCliDotEnv()) { await startupTrace.measure("dotenv", async () => { - const { loadCliDotEnv } = await import("./dotenv.js"); - loadCliDotEnv({ quiet: true }); + if (isRemoteAgentDispatchInvocation(normalizedArgv, normalizedInvocation.primary)) { + const { loadGatewayDispatchCliDotEnv } = await import("./gateway-dispatch-dotenv.js"); + await loadGatewayDispatchCliDotEnv({ quiet: true }); + } else { + const { loadCliDotEnv } = await import("./dotenv.js"); + loadCliDotEnv({ quiet: true }); + } }); } normalizeEnv(); diff --git a/src/commands/agent-via-gateway.test.ts b/src/commands/agent-via-gateway.test.ts index 4b58ed9b9f5..6ff4ef49f34 100644 --- a/src/commands/agent-via-gateway.test.ts +++ b/src/commands/agent-via-gateway.test.ts @@ -10,6 +10,8 @@ import { agentCliCommand, agentViaGatewayTesting } from "./agent-via-gateway.js" import type { agentCommand as AgentCommand } from "./agent.js"; const loadConfig = vi.hoisted(() => vi.fn()); +const loadConfigWithShellEnvFallback = vi.hoisted(() => vi.fn()); +const loadRuntimeConfig = vi.hoisted(() => vi.fn()); const callGateway = vi.hoisted(() => vi.fn()); const isGatewayCredentialsRequiredError = vi.hoisted(() => vi.fn( @@ -49,7 +51,7 @@ const jsonRuntime = { }; function mockConfig(storePath: string, overrides?: Partial) { - loadConfig.mockReturnValue({ + const config = { agents: { defaults: { timeoutSeconds: 600, @@ -63,7 +65,10 @@ function mockConfig(storePath: string, overrides?: Partial) { ...overrides?.session, }, gateway: overrides?.gateway, - }); + }; + loadConfig.mockReturnValue(config); + loadConfigWithShellEnvFallback.mockResolvedValue(config); + loadRuntimeConfig.mockReturnValue(config); } async function withTempStore( @@ -217,7 +222,14 @@ function createGatewayNormalCloseError() { }); } -vi.mock("../config/io.js", () => ({ getRuntimeConfig: loadConfig, loadConfig })); +vi.mock("../config/gateway-dispatch-config.js", () => ({ + readGatewayDispatchConfig: loadConfig, + readGatewayDispatchConfigWithShellEnvFallback: loadConfigWithShellEnvFallback, +})); +vi.mock("../config/io.js", () => ({ + getRuntimeConfig: loadRuntimeConfig, + loadConfig: loadRuntimeConfig, +})); vi.mock("../gateway/call.js", () => ({ callGateway, isGatewayCredentialsRequiredError, @@ -344,11 +356,7 @@ describe("agentCliCommand", () => { expect(params.sessionId).toBeUndefined(); expect(params.to).toBeUndefined(); expect(request.config).toBe(loadConfig.mock.results[0]?.value); - expect(loadConfig).toHaveBeenCalledWith({ - skipPluginValidation: true, - pin: false, - skipShellEnvFallback: true, - }); + expect(loadConfig).toHaveBeenCalledWith(); expect(agentCommand).not.toHaveBeenCalled(); expect(loadAgentSessionModuleMock).not.toHaveBeenCalled(); }); @@ -366,7 +374,8 @@ describe("agentCliCommand", () => { }; loadConfig.mockReset(); loadConfig.mockReturnValueOnce(fastConfig); - loadConfig.mockReturnValueOnce(shellEnvConfig); + loadConfigWithShellEnvFallback.mockReset(); + loadConfigWithShellEnvFallback.mockResolvedValueOnce(shellEnvConfig); const authError = new Error("gateway agent requires credentials"); authError.name = "GatewayCredentialsRequiredError"; callGateway.mockRejectedValueOnce(authError); @@ -374,10 +383,10 @@ describe("agentCliCommand", () => { await agentCliCommand({ message: "hi", sessionKey: "agent:main:incident-42" }, runtime); - expect(loadConfig.mock.calls).toEqual([ - [{ skipPluginValidation: true, pin: false, skipShellEnvFallback: true }], - [{ skipPluginValidation: true, pin: false, skipShellEnvFallback: false }], - ]); + expect(loadConfig).toHaveBeenCalledTimes(1); + expect(loadConfig).toHaveBeenCalledWith(); + expect(loadConfigWithShellEnvFallback).toHaveBeenCalledTimes(1); + expect(loadConfigWithShellEnvFallback).toHaveBeenCalledWith(); expect(callGateway).toHaveBeenCalledTimes(2); expect(requireRecord(callGateway.mock.calls[0]?.[0], "first gateway request").config).toBe( fastConfig, @@ -396,7 +405,8 @@ describe("agentCliCommand", () => { }; loadConfig.mockReset(); loadConfig.mockReturnValueOnce(fastConfig); - loadConfig.mockReturnValueOnce(fastConfig); + loadConfigWithShellEnvFallback.mockReset(); + loadConfigWithShellEnvFallback.mockResolvedValueOnce(fastConfig); const authError = new Error("gateway url override requires explicit credentials"); authError.name = "GatewayExplicitAuthRequiredError"; callGateway.mockRejectedValueOnce(authError); @@ -404,10 +414,10 @@ describe("agentCliCommand", () => { await agentCliCommand({ message: "hi", sessionKey: "agent:main:incident-42" }, runtime); - expect(loadConfig.mock.calls).toEqual([ - [{ skipPluginValidation: true, pin: false, skipShellEnvFallback: true }], - [{ skipPluginValidation: true, pin: false, skipShellEnvFallback: false }], - ]); + expect(loadConfig).toHaveBeenCalledTimes(1); + expect(loadConfig).toHaveBeenCalledWith(); + expect(loadConfigWithShellEnvFallback).toHaveBeenCalledTimes(1); + expect(loadConfigWithShellEnvFallback).toHaveBeenCalledWith(); expect(callGateway).toHaveBeenCalledTimes(2); }); }); @@ -1559,10 +1569,7 @@ describe("agentCliCommand", () => { }; expect(fallbackOpts.sessionId).toMatch(/^gateway-fallback-/); expect(fallbackOpts.sessionKey).toBe(`agent:ops:explicit:${fallbackOpts.sessionId}`); - expect(loadConfig.mock.calls).toEqual([ - [{ skipPluginValidation: true, pin: false, skipShellEnvFallback: true }], - [{ skipPluginValidation: true, pin: false, skipShellEnvFallback: true }], - ]); + expect(loadConfig.mock.calls).toEqual([[], []]); }, { agents: { list: [{ id: "ops", default: true }, { id: "main" }] } }, ); @@ -1750,7 +1757,7 @@ describe("agentCliCommand", () => { ); expect(localOpts.agentId).toBe("ops"); expect(localOpts.sessionKey).toBe("agent:ops:incident-42"); - expect(loadConfig).toHaveBeenCalledWith(); + expect(loadRuntimeConfig).toHaveBeenCalledWith(); }); }); diff --git a/src/commands/agent-via-gateway.ts b/src/commands/agent-via-gateway.ts index 4f4e620cd07..2dd4d550b8e 100644 --- a/src/commands/agent-via-gateway.ts +++ b/src/commands/agent-via-gateway.ts @@ -9,7 +9,10 @@ import { listAgentIds, resolveDefaultAgentId } from "../agents/agent-scope-confi import { formatCliCommand } from "../cli/command-format.js"; import type { CliDeps } from "../cli/deps.types.js"; import { withProgress } from "../cli/progress.js"; -import { getRuntimeConfig } from "../config/io.js"; +import { + readGatewayDispatchConfig, + readGatewayDispatchConfigWithShellEnvFallback, +} from "../config/gateway-dispatch-config.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import { callGateway, @@ -31,7 +34,7 @@ import { scopeLegacySessionKeyToAgent, } from "../routing/session-key.js"; import { type RuntimeEnv, writeRuntimeJson } from "../runtime.js"; -import { normalizeMessageChannel } from "../utils/message-channel.js"; +import { normalizeMessageChannel } from "../utils/message-channel-normalize.js"; type AgentGatewayResult = { payloads?: Array<{ @@ -96,6 +99,7 @@ type AgentGatewayCallIdentity = Pick< >; type EmbeddedAgentCommandModule = typeof import("./agent.js"); type AgentSessionModule = typeof import("./agent/session.js"); +type RuntimeConfigModule = typeof import("../config/io.js"); type AgentSessionModuleLoader = () => Promise; const AGENT_CLI_SIGNALS: readonly AgentCliSignal[] = ["SIGINT", "SIGTERM"]; @@ -108,6 +112,7 @@ const AGENT_CLI_SIGNAL_EXIT_CODES: Record = { let embeddedAgentCommandPromise: Promise | undefined; let agentSessionModulePromise: Promise | undefined; +let runtimeConfigModulePromise: Promise | undefined; let replyPayloadModulePromise: | Promise | undefined; @@ -130,6 +135,12 @@ function loadAgentSessionModule(): Promise { return agentSessionModulePromise; } +async function loadRuntimeConfig(): Promise { + runtimeConfigModulePromise ??= import("../config/io.js"); + const { getRuntimeConfig } = await runtimeConfigModulePromise; + return getRuntimeConfig(); +} + function loadReplyPayloadModule() { replyPayloadModulePromise ??= import("openclaw/plugin-sdk/reply-payload"); return replyPayloadModulePromise; @@ -139,6 +150,7 @@ export const agentViaGatewayTesting = { resetLazyImportsForTests(): void { embeddedAgentCommandPromise = undefined; agentSessionModulePromise = undefined; + runtimeConfigModulePromise = undefined; replyPayloadModulePromise = undefined; agentSessionModuleLoader = defaultAgentSessionModuleLoader; }, @@ -178,14 +190,15 @@ function resolveGatewayAgentTimeoutMs(timeoutSeconds: number): number { return resolveTimerTimeoutMs((timeoutSeconds + 30) * 1000, 10_000, 10_000); } -function getGatewayDispatchConfig(options?: { skipShellEnvFallback?: boolean }): OpenClawConfig { +async function getGatewayDispatchConfig(options?: { + skipShellEnvFallback?: boolean; +}): Promise { // Scoped gateway turns need core agent/session/gateway fields only. The // running gateway owns plugin validation and plugin metadata freshness. - return getRuntimeConfig({ - skipPluginValidation: true, - pin: false, - skipShellEnvFallback: options?.skipShellEnvFallback ?? true, - }); + if (options?.skipShellEnvFallback === false) { + return await readGatewayDispatchConfigWithShellEnvFallback(); + } + return readGatewayDispatchConfig(); } async function formatPayloadForLog(payload: { @@ -273,7 +286,7 @@ function validateExplicitSessionKeyForDispatch( } } -function normalizeSessionKeyOptsForDispatch(opts: AgentCliOpts): AgentCliOpts { +async function normalizeSessionKeyOptsForDispatch(opts: AgentCliOpts): Promise { const rawSessionKey = opts.sessionKey?.trim(); const isLegacySessionKey = rawSessionKey && classifySessionKeyShape(rawSessionKey) === "legacy_or_alias"; @@ -283,8 +296,8 @@ function normalizeSessionKeyOptsForDispatch(opts: AgentCliOpts): AgentCliOpts { const cfg = isLegacySessionKey && (agentIdRaw || shouldScopeDefaultAgentKey) ? opts.local === true - ? getRuntimeConfig() - : getGatewayDispatchConfig() + ? await loadRuntimeConfig() + : await getGatewayDispatchConfig() : undefined; const sessionKey = scopeLegacySessionKeyToAgent({ agentId: agentIdRaw ?? (shouldScopeDefaultAgentKey ? resolveDefaultAgentId(cfg!) : undefined), @@ -552,7 +565,7 @@ async function resolveAgentIdForGatewayTimeoutFallback( return resolveAgentIdFromSessionKey(explicitSessionKey); } if (isUnscopedSessionKeySentinel(explicitSessionKey)) { - return resolveDefaultAgentId(getGatewayDispatchConfig()); + return resolveDefaultAgentId(await getGatewayDispatchConfig()); } const agentIdRaw = opts.agent?.trim(); @@ -563,7 +576,7 @@ async function resolveAgentIdForGatewayTimeoutFallback( if (!opts.to && !opts.sessionId) { return undefined; } - const cfg = getGatewayDispatchConfig(); + const cfg = await getGatewayDispatchConfig(); const { resolveSessionKeyForRequest } = await loadAgentSessionModule(); const resolvedSessionKey = resolveSessionKeyForRequest({ cfg, @@ -615,7 +628,7 @@ async function agentViaGatewayCommand( ); } - let cfg = getGatewayDispatchConfig(); + let cfg = await getGatewayDispatchConfig(); const agentIdRaw = opts.agent?.trim(); const agentId = agentIdRaw ? normalizeAgentId(agentIdRaw) : undefined; if (agentId) { @@ -727,7 +740,7 @@ async function agentViaGatewayCommand( shouldRetryGatewayDispatchWithShellEnvFallback(err) ) { retriedWithShellEnvFallback = true; - cfg = getGatewayDispatchConfig({ skipShellEnvFallback: false }); + cfg = await getGatewayDispatchConfig({ skipShellEnvFallback: false }); continue; } if ( @@ -815,7 +828,7 @@ export async function agentCliCommand( deps?: AgentCliDeps, ) { protectJsonStdout(opts); - const dispatchOpts = normalizeSessionKeyOptsForDispatch(opts); + const dispatchOpts = await normalizeSessionKeyOptsForDispatch(opts); validateExplicitSessionKeyForDispatch(dispatchOpts); const gatewayDispatchOpts = dispatchOpts.runId ? dispatchOpts diff --git a/src/config/gateway-dispatch-config.test.ts b/src/config/gateway-dispatch-config.test.ts new file mode 100644 index 00000000000..90192efec91 --- /dev/null +++ b/src/config/gateway-dispatch-config.test.ts @@ -0,0 +1,103 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + readGatewayDispatchConfig, + readGatewayDispatchConfigWithShellEnvFallback, +} from "./gateway-dispatch-config.js"; + +const shellEnvMocks = vi.hoisted(() => ({ + loadShellEnvFallback: vi.fn(), + resolveShellEnvFallbackTimeoutMs: vi.fn(() => 50), + shouldDeferShellEnvFallback: vi.fn(() => false), + shouldEnableShellEnvFallback: vi.fn(() => false), +})); + +vi.mock("../infra/shell-env.js", () => shellEnvMocks); + +const tempDirs: string[] = []; + +function createTempConfig(files: Record): string { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-gateway-dispatch-config-")); + tempDirs.push(dir); + for (const [name, contents] of Object.entries(files)) { + fs.writeFileSync(path.join(dir, name), contents); + } + return path.join(dir, "openclaw.json5"); +} + +afterEach(() => { + vi.clearAllMocks(); + for (const dir of tempDirs.splice(0)) { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +describe("readGatewayDispatchConfig", () => { + it("reads only gateway dispatch fields from JSON5 config with includes and env vars", () => { + const configPath = createTempConfig({ + "gateway-base.json5": `{ + gateway: { + port: 18888, + auth: { mode: "token", token: "\${OPENCLAW_GATEWAY_TOKEN}" }, + }, + models: { providers: { expensive: { apiKey: "\${MISSING_MODEL_KEY}" } } }, + }`, + "openclaw.json5": `{ + $include: "./gateway-base.json5", + env: { vars: { OPENCLAW_GATEWAY_TOKEN: "inline-token" } }, + agents: { + defaults: { timeoutSeconds: 42 }, + list: [{ id: "ops", default: true }], + }, + plugins: { + allow: ["vault"], + entries: { vault: { enabled: true } }, + load: { paths: ["./plugins/vault"] }, + }, + session: { mainKey: "main-ops", store: "./sessions.json" }, + }`, + }); + const env = { OPENCLAW_CONFIG_PATH: configPath }; + + const config = readGatewayDispatchConfig({ env }); + + expect(config.gateway?.port).toBe(18888); + expect(config.gateway?.auth).toMatchObject({ mode: "token", token: "inline-token" }); + expect(config.agents?.defaults?.timeoutSeconds).toBe(42); + expect(config.agents?.list?.[0]?.id).toBe("ops"); + expect(config.plugins).toEqual({ + allow: ["vault"], + entries: { vault: { enabled: true } }, + load: { paths: ["./plugins/vault"] }, + }); + expect(config.session?.mainKey).toBe("main"); + expect((config as { models?: unknown }).models).toBeUndefined(); + expect(shellEnvMocks.loadShellEnvFallback).not.toHaveBeenCalled(); + }); + + it("loads only gateway credential shell env keys on explicit fallback", async () => { + const configPath = createTempConfig({ + "openclaw.json5": `{ + env: { shellEnv: { enabled: true, timeoutMs: 123 } }, + gateway: { auth: { mode: "token", token: "\${OPENCLAW_GATEWAY_TOKEN}" } }, + }`, + }); + const env: NodeJS.ProcessEnv = { OPENCLAW_CONFIG_PATH: configPath }; + shellEnvMocks.loadShellEnvFallback.mockImplementation(({ env: targetEnv }) => { + targetEnv.OPENCLAW_GATEWAY_TOKEN = "shell-token"; + }); + + const config = await readGatewayDispatchConfigWithShellEnvFallback({ env }); + + expect(shellEnvMocks.loadShellEnvFallback).toHaveBeenCalledWith({ + enabled: true, + env, + expectedKeys: ["OPENCLAW_GATEWAY_TOKEN", "OPENCLAW_GATEWAY_PASSWORD"], + logger: console, + timeoutMs: 123, + }); + expect(config.gateway?.auth).toMatchObject({ mode: "token", token: "shell-token" }); + }); +}); diff --git a/src/config/gateway-dispatch-config.ts b/src/config/gateway-dispatch-config.ts new file mode 100644 index 00000000000..6c364c8ddce --- /dev/null +++ b/src/config/gateway-dispatch-config.ts @@ -0,0 +1,150 @@ +import fs from "node:fs"; +import path from "node:path"; +import { parseJsonWithJson5Fallback } from "../utils/parse-json-compat.js"; +import { applyConfigEnvVars } from "./config-env-vars.js"; +import { resolveConfigEnvVars } from "./env-substitution.js"; +import { readConfigIncludeFileWithGuards, resolveConfigIncludes } from "./includes.js"; +import { resolveConfigPath, resolveIncludeRoots } from "./paths.js"; +import type { OpenClawConfig } from "./types.openclaw.js"; + +const GATEWAY_DISPATCH_SHELL_ENV_EXPECTED_KEYS = [ + "OPENCLAW_GATEWAY_TOKEN", + "OPENCLAW_GATEWAY_PASSWORD", +] as const; + +const GATEWAY_DISPATCH_TOP_LEVEL_KEYS = [ + "agents", + "env", + "gateway", + "plugins", + "secrets", + "session", +] as const; + +type GatewayDispatchConfigReadOptions = { + configPath?: string; + env?: NodeJS.ProcessEnv; + logger?: Pick; +}; + +function isPlainRecord(value: unknown): value is Record { + return Boolean(value) && typeof value === "object" && !Array.isArray(value); +} + +function cloneConfigValue(value: unknown): unknown { + if (Array.isArray(value)) { + return value.map((entry) => cloneConfigValue(entry)); + } + if (!isPlainRecord(value)) { + return value; + } + const out: Record = {}; + for (const [key, child] of Object.entries(value)) { + out[key] = cloneConfigValue(child); + } + return out; +} + +function projectGatewayDispatchConfig(value: unknown): OpenClawConfig { + if (!isPlainRecord(value)) { + return {}; + } + const projected: Record = {}; + for (const key of GATEWAY_DISPATCH_TOP_LEVEL_KEYS) { + if (Object.hasOwn(value, key)) { + projected[key] = cloneConfigValue(value[key]); + } + } + return projected as OpenClawConfig; +} + +function applyGatewayDispatchSessionDefaults(config: OpenClawConfig): OpenClawConfig { + if (config.session?.mainKey === undefined) { + return config; + } + return { + ...config, + session: { ...config.session, mainKey: "main" }, + }; +} + +function resolveIncludesForGatewayDispatch( + parsed: unknown, + configPath: string, + env: NodeJS.ProcessEnv, +): unknown { + return resolveConfigIncludes( + parsed, + configPath, + { + readFile: (candidate) => fs.readFileSync(candidate, "utf-8"), + readFileWithGuards: ({ includePath, resolvedPath, rootRealDir }) => + readConfigIncludeFileWithGuards({ + includePath, + resolvedPath, + rootRealDir, + ioFs: fs, + }), + parseJson: parseJsonWithJson5Fallback, + }, + { allowedRoots: resolveIncludeRoots(env) }, + ); +} + +function resolveGatewayDispatchEnvVars(config: unknown, env: NodeJS.ProcessEnv): unknown { + if (isPlainRecord(config) && Object.hasOwn(config, "env")) { + applyConfigEnvVars(config as OpenClawConfig, env); + } + return resolveConfigEnvVars(config, env, { onMissing: () => undefined }); +} + +function readRawGatewayDispatchConfig(options: GatewayDispatchConfigReadOptions = {}): { + config: OpenClawConfig; + configPath: string; +} { + const env = options.env ?? process.env; + const configPath = options.configPath ?? resolveConfigPath(env); + if (!fs.existsSync(configPath)) { + return { config: {}, configPath }; + } + + const raw = fs.readFileSync(configPath, "utf-8"); + const parsed = parseJsonWithJson5Fallback(raw); + const resolvedIncludes = resolveIncludesForGatewayDispatch(parsed, configPath, env); + const resolvedConfig = resolveGatewayDispatchEnvVars(resolvedIncludes, env); + return { + config: applyGatewayDispatchSessionDefaults(projectGatewayDispatchConfig(resolvedConfig)), + configPath, + }; +} + +export function readGatewayDispatchConfig( + options: GatewayDispatchConfigReadOptions = {}, +): OpenClawConfig { + return readRawGatewayDispatchConfig(options).config; +} + +export async function readGatewayDispatchConfigWithShellEnvFallback( + options: GatewayDispatchConfigReadOptions = {}, +): Promise { + const env = options.env ?? process.env; + const firstRead = readRawGatewayDispatchConfig(options); + const { + loadShellEnvFallback, + resolveShellEnvFallbackTimeoutMs, + shouldDeferShellEnvFallback, + shouldEnableShellEnvFallback, + } = await import("../infra/shell-env.js"); + const enabled = + shouldEnableShellEnvFallback(env) || firstRead.config.env?.shellEnv?.enabled === true; + if (enabled && !shouldDeferShellEnvFallback(env)) { + loadShellEnvFallback({ + enabled: true, + env, + expectedKeys: [...GATEWAY_DISPATCH_SHELL_ENV_EXPECTED_KEYS], + logger: options.logger ?? console, + timeoutMs: firstRead.config.env?.shellEnv?.timeoutMs ?? resolveShellEnvFallbackTimeoutMs(env), + }); + } + return readGatewayDispatchConfig({ ...options, configPath: path.resolve(firstRead.configPath) }); +} diff --git a/src/gateway/call.ts b/src/gateway/call.ts index bfd6852f8fe..f24c2e831fd 100644 --- a/src/gateway/call.ts +++ b/src/gateway/call.ts @@ -2,11 +2,17 @@ import { randomUUID } from "node:crypto"; import { isLoopbackIpAddress } from "@openclaw/net-policy/ip"; import { redactSensitiveUrlLikeString } from "@openclaw/net-policy/redact-sensitive-url"; import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce"; +import { + GATEWAY_CLIENT_MODES, + GATEWAY_CLIENT_NAMES, + type GatewayClientMode, + type GatewayClientName, +} from "../../packages/gateway-protocol/src/client-info.js"; import { MIN_CLIENT_PROTOCOL_VERSION, PROTOCOL_VERSION, -} from "../../packages/gateway-protocol/src/index.js"; -import { getRuntimeConfig } from "../config/io.js"; +} from "../../packages/gateway-protocol/src/version.js"; +import { readGatewayDispatchConfig } from "../config/gateway-dispatch-config.js"; import { resolveConfigPath as resolveConfigPathFromPaths, resolveGatewayPort as resolveGatewayPortFromPaths, @@ -16,12 +22,6 @@ import type { OpenClawConfig } from "../config/types.openclaw.js"; import { loadDeviceAuthToken } from "../infra/device-auth-store.js"; import { loadOrCreateDeviceIdentity, type DeviceIdentity } from "../infra/device-identity.js"; import { loadGatewayTlsRuntime } from "../infra/tls/gateway.js"; -import { - GATEWAY_CLIENT_MODES, - GATEWAY_CLIENT_NAMES, - type GatewayClientMode, - type GatewayClientName, -} from "../utils/message-channel.js"; import { resolveSafeTimeoutDelayMs } from "../utils/timer-delay.js"; import { VERSION } from "../version.js"; import { resolveGatewayAuth } from "./auth-resolve.js"; @@ -246,9 +246,21 @@ export function isGatewayExplicitAuthRequiredError( } const defaultCreateGatewayClient = (opts: GatewayClientOptions) => new GatewayClient(opts); -const defaultGatewayCallDeps = { +type GatewayRuntimeConfigLoader = () => OpenClawConfig | Promise; +const defaultGetRuntimeConfig = async (): Promise => + (await import("../config/io.js")).getRuntimeConfig(); +const defaultGatewayCallDeps: { + createGatewayClient: typeof defaultCreateGatewayClient; + getRuntimeConfig: GatewayRuntimeConfigLoader; + loadOrCreateDeviceIdentity: typeof loadOrCreateDeviceIdentity; + resolveGatewayPort: typeof resolveGatewayPortFromPaths; + resolveConfigPath: typeof resolveConfigPathFromPaths; + resolveStateDir: typeof resolveStateDirFromPaths; + loadGatewayTlsRuntime: typeof loadGatewayTlsRuntime; + loadDeviceAuthToken: typeof loadDeviceAuthToken; +} = { createGatewayClient: defaultCreateGatewayClient, - getRuntimeConfig, + getRuntimeConfig: defaultGetRuntimeConfig, loadOrCreateDeviceIdentity, resolveGatewayPort: resolveGatewayPortFromPaths, resolveConfigPath: resolveConfigPathFromPaths, @@ -281,14 +293,28 @@ function resolveGatewayClientDisplayName(opts: CallGatewayBaseOptions): string | return method ? `gateway:${method}` : "gateway:request"; } -function loadGatewayConfig(): OpenClawConfig { +async function loadGatewayConfig(): Promise { const loadConfigFn = typeof gatewayCallDeps.getRuntimeConfig === "function" ? gatewayCallDeps.getRuntimeConfig : typeof defaultGatewayCallDeps.getRuntimeConfig === "function" ? defaultGatewayCallDeps.getRuntimeConfig - : getRuntimeConfig; - return loadConfigFn(); + : defaultGetRuntimeConfig; + return await loadConfigFn(); +} + +function loadGatewayConfigForConnectionDetails(): OpenClawConfig { + if ( + gatewayCallDeps.getRuntimeConfig !== defaultGetRuntimeConfig && + typeof gatewayCallDeps.getRuntimeConfig === "function" + ) { + const config = gatewayCallDeps.getRuntimeConfig(); + if (config && typeof (config as Promise).then === "function") { + throw new Error("async gateway config loader is not supported for connection details"); + } + return config as OpenClawConfig; + } + return readGatewayDispatchConfig(); } function resolveGatewayStateDir(env: NodeJS.ProcessEnv): string { @@ -324,7 +350,7 @@ export function buildGatewayConnectionDetails( } = {}, ): GatewayConnectionDetails { return buildGatewayConnectionDetailsWithResolvers(options, { - getRuntimeConfig: () => loadGatewayConfig(), + getRuntimeConfig: () => loadGatewayConfigForConnectionDetails(), resolveConfigPath: (env) => resolveGatewayConfigPath(env), resolveGatewayPort: (config, env) => resolveGatewayPortValue(config, env), }); @@ -566,7 +592,9 @@ function resolveGatewayCallTimeout( return { timeoutMs, safeTimerTimeoutMs }; } -function resolveGatewayCallContext(opts: CallGatewayBaseOptions): ResolvedGatewayCallContext { +async function resolveGatewayCallContext( + opts: CallGatewayBaseOptions, +): Promise { const cliUrlOverride = trimToUndefined(opts.url); const explicitAuth = resolveExplicitGatewayAuth({ token: opts.token, password: opts.password }); const envUrlOverride = cliUrlOverride @@ -579,7 +607,8 @@ function resolveGatewayCallContext(opts: CallGatewayBaseOptions): ResolvedGatewa urlOverride, explicitAuth, }); - const config = opts.config ?? (canSkipConfigLoad ? ({} as OpenClawConfig) : loadGatewayConfig()); + const config = + opts.config ?? (canSkipConfigLoad ? ({} as OpenClawConfig) : await loadGatewayConfig()); const configPath = opts.configPath ?? resolveGatewayConfigPath(process.env); const isRemoteMode = config.gateway?.mode === "remote"; const remote = isRemoteMode @@ -965,7 +994,7 @@ async function callGatewayWithScopes>( opts: CallGatewayBaseOptions, scopes: OperatorScope[], ): Promise { - const context = resolveGatewayCallContext(opts); + const context = await resolveGatewayCallContext(opts); const { timeoutMs, safeTimerTimeoutMs } = resolveGatewayCallTimeout( opts.timeoutMs, context.config.gateway?.handshakeTimeoutMs, diff --git a/src/infra/device-auth-store.ts b/src/infra/device-auth-store.ts index 198d8b48e11..69f81c79659 100644 --- a/src/infra/device-auth-store.ts +++ b/src/infra/device-auth-store.ts @@ -1,6 +1,5 @@ import fs from "node:fs"; import path from "node:path"; -import { z } from "zod"; import { resolveStateDir } from "../config/paths.js"; import { clearDeviceAuthTokenFromStore, @@ -12,15 +11,28 @@ import type { DeviceAuthStore } from "../shared/device-auth.js"; import { privateFileStoreSync } from "./private-file-store.js"; const DEVICE_AUTH_FILE = "device-auth.json"; -const DeviceAuthStoreSchema = z.object({ - version: z.literal(1), - deviceId: z.string(), - tokens: z.record(z.string(), z.unknown()), -}) as z.ZodType; type StoreCacheEntry = { store: DeviceAuthStore | null; mtimeMs: number; size: number }; const storeReadCache = new Map(); +function isRecord(value: unknown): value is Record { + return Boolean(value) && typeof value === "object" && !Array.isArray(value); +} + +function parseDeviceAuthStore(value: unknown): DeviceAuthStore | null { + if (!isRecord(value) || value.version !== 1 || typeof value.deviceId !== "string") { + return null; + } + if (!isRecord(value.tokens)) { + return null; + } + return { + version: 1, + deviceId: value.deviceId, + tokens: value.tokens, + }; +} + function storeCacheHit( cached: StoreCacheEntry | undefined, stat: { mtimeMs: number; size: number }, @@ -52,8 +64,7 @@ function readStore(filePath: string): DeviceAuthStore | null { const parsed = privateFileStoreSync(path.dirname(filePath)).readJsonIfExists( path.basename(filePath), ); - const result = DeviceAuthStoreSchema.safeParse(parsed); - const store = result.success ? result.data : null; + const store = parseDeviceAuthStore(parsed); storeReadCache.set(filePath, { store, mtimeMs: stat.mtimeMs, size: stat.size }); return store; } catch { diff --git a/src/infra/dotenv-global.ts b/src/infra/dotenv-global.ts new file mode 100644 index 00000000000..85c21f34beb --- /dev/null +++ b/src/infra/dotenv-global.ts @@ -0,0 +1,132 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import dotenv from "dotenv"; +import { createSubsystemLogger } from "../logging/subsystem.js"; +import { resolveConfigDir } from "../utils.js"; +import { resolveRequiredHomeDir } from "./home-dir.js"; +import { normalizeEnvVarKey } from "./host-env-security.js"; + +const logger = createSubsystemLogger("infra:dotenv"); + +type DotEnvEntry = { + key: string; + value: string; +}; + +type LoadedDotEnvFile = { + filePath: string; + entries: DotEnvEntry[]; +}; + +function readGlobalRuntimeDotEnvFile(params: { + filePath: string; + quiet?: boolean; +}): LoadedDotEnvFile | null { + let content: string; + try { + content = fs.readFileSync(params.filePath, "utf8"); + } catch (error) { + if (!params.quiet) { + const code = + error && typeof error === "object" && "code" in error ? String(error.code) : undefined; + if (code !== "ENOENT") { + logger.warn(`Failed to read ${params.filePath}: ${String(error)}`, { error }); + } + } + return null; + } + + let parsed: Record; + try { + parsed = dotenv.parse(content); + } catch (error) { + if (!params.quiet) { + logger.warn(`Failed to parse ${params.filePath}: ${String(error)}`, { error }); + } + return null; + } + const entries: DotEnvEntry[] = []; + for (const [rawKey, value] of Object.entries(parsed)) { + const key = normalizeEnvVarKey(rawKey, { portable: true }); + if (key) { + entries.push({ key, value }); + } + } + return { filePath: params.filePath, entries }; +} + +function loadParsedDotEnvFiles(files: LoadedDotEnvFile[]) { + const preExistingKeys = new Set(Object.keys(process.env)); + const conflicts = new Map }>(); + const firstSeen = new Map(); + + for (const file of files) { + for (const { key, value } of file.entries) { + if (preExistingKeys.has(key)) { + continue; + } + const previous = firstSeen.get(key); + if (previous) { + if (previous.value !== value) { + const conflictKey = `${previous.filePath}\u0000${file.filePath}`; + const existing = conflicts.get(conflictKey); + if (existing) { + existing.keys.add(key); + } else { + conflicts.set(conflictKey, { + keptPath: previous.filePath, + ignoredPath: file.filePath, + keys: new Set([key]), + }); + } + } + continue; + } + firstSeen.set(key, { value, filePath: file.filePath }); + if (process.env[key] === undefined) { + process.env[key] = value; + } + } + } + + for (const conflict of conflicts.values()) { + const keys = [...conflict.keys].toSorted(); + if (keys.length === 0) { + continue; + } + logger.warn( + `Conflicting values in ${conflict.keptPath} and ${conflict.ignoredPath} for ${keys.join(", ")}; keeping ${conflict.keptPath}.`, + { keptPath: conflict.keptPath, ignoredPath: conflict.ignoredPath, keys }, + ); + } +} + +export function loadGlobalRuntimeDotEnvFiles(opts?: { quiet?: boolean; stateEnvPath?: string }) { + const quiet = opts?.quiet ?? true; + const stateEnvPath = opts?.stateEnvPath ?? path.join(resolveConfigDir(process.env), ".env"); + const defaultStateEnvPath = path.join( + resolveRequiredHomeDir(process.env, os.homedir), + ".openclaw", + ".env", + ); + const hasExplicitNonDefaultStateDir = + process.env.OPENCLAW_STATE_DIR?.trim() !== undefined && + path.resolve(stateEnvPath) !== path.resolve(defaultStateEnvPath); + const parsedFiles = [readGlobalRuntimeDotEnvFile({ filePath: stateEnvPath, quiet })]; + if (!hasExplicitNonDefaultStateDir) { + parsedFiles.push( + readGlobalRuntimeDotEnvFile({ + filePath: path.join( + resolveRequiredHomeDir(process.env, os.homedir), + ".config", + "openclaw", + "gateway.env", + ), + quiet, + }), + ); + } + const parsed = parsedFiles.filter((file): file is LoadedDotEnvFile => file !== null); + loadParsedDotEnvFiles(parsed); +} diff --git a/src/infra/dotenv.ts b/src/infra/dotenv.ts index bc5baae6559..ff5055a45f7 100644 --- a/src/infra/dotenv.ts +++ b/src/infra/dotenv.ts @@ -1,11 +1,9 @@ import fs from "node:fs"; -import os from "node:os"; import path from "node:path"; import dotenv from "dotenv"; import { createSubsystemLogger } from "../logging/subsystem.js"; import { listKnownProviderAuthEnvVarNames } from "../secrets/provider-env-vars.js"; -import { resolveConfigDir } from "../utils.js"; -import { resolveRequiredHomeDir } from "./home-dir.js"; +import { loadGlobalRuntimeDotEnvFiles } from "./dotenv-global.js"; import { isDangerousHostEnvOverrideVarName, isDangerousHostEnvVarName, @@ -196,15 +194,6 @@ function shouldBlockWorkspaceRuntimeDotEnvKey(key: string): boolean { return isDangerousHostEnvVarName(key) || isDangerousHostEnvOverrideVarName(key); } -function shouldBlockRuntimeDotEnvKey(key: string): boolean { - // The global ~/.openclaw/.env (or OPENCLAW_STATE_DIR/.env) is a trusted - // operator-controlled runtime surface. Workspace .env is untrusted and gets - // the strict blocklist, but the trusted global fallback is allowed to set - // runtime vars like proxy/base-url/auth values. - void key; - return false; -} - function buildProviderAuthWorkspaceDotEnvBlocklist(): ReadonlySet { const keys = new Set(BLOCKED_PROVIDER_AUTH_WORKSPACE_DOTENV_KEYS); for (const rawKey of listKnownProviderAuthEnvVarNames({ @@ -303,87 +292,7 @@ export function loadWorkspaceDotEnvFile(filePath: string, opts?: { quiet?: boole } } -function loadParsedDotEnvFiles(files: LoadedDotEnvFile[]) { - const preExistingKeys = new Set(Object.keys(process.env)); - const conflicts = new Map }>(); - const firstSeen = new Map(); - - for (const file of files) { - for (const { key, value } of file.entries) { - if (preExistingKeys.has(key)) { - continue; - } - const previous = firstSeen.get(key); - if (previous) { - if (previous.value !== value) { - const conflictKey = `${previous.filePath}\u0000${file.filePath}`; - const existing = conflicts.get(conflictKey); - if (existing) { - existing.keys.add(key); - } else { - conflicts.set(conflictKey, { - keptPath: previous.filePath, - ignoredPath: file.filePath, - keys: new Set([key]), - }); - } - } - continue; - } - firstSeen.set(key, { value, filePath: file.filePath }); - if (process.env[key] === undefined) { - process.env[key] = value; - } - } - } - - for (const conflict of conflicts.values()) { - const keys = [...conflict.keys].toSorted(); - if (keys.length === 0) { - continue; - } - logger.warn( - `Conflicting values in ${conflict.keptPath} and ${conflict.ignoredPath} for ${keys.join(", ")}; keeping ${conflict.keptPath}.`, - { keptPath: conflict.keptPath, ignoredPath: conflict.ignoredPath, keys }, - ); - } -} - -export function loadGlobalRuntimeDotEnvFiles(opts?: { quiet?: boolean; stateEnvPath?: string }) { - const quiet = opts?.quiet ?? true; - const stateEnvPath = opts?.stateEnvPath ?? path.join(resolveConfigDir(process.env), ".env"); - const defaultStateEnvPath = path.join( - resolveRequiredHomeDir(process.env, os.homedir), - ".openclaw", - ".env", - ); - const hasExplicitNonDefaultStateDir = - process.env.OPENCLAW_STATE_DIR?.trim() !== undefined && - path.resolve(stateEnvPath) !== path.resolve(defaultStateEnvPath); - const parsedFiles = [ - readDotEnvFile({ - filePath: stateEnvPath, - shouldBlockKey: shouldBlockRuntimeDotEnvKey, - quiet, - }), - ]; - if (!hasExplicitNonDefaultStateDir) { - parsedFiles.push( - readDotEnvFile({ - filePath: path.join( - resolveRequiredHomeDir(process.env, os.homedir), - ".config", - "openclaw", - "gateway.env", - ), - shouldBlockKey: shouldBlockRuntimeDotEnvKey, - quiet, - }), - ); - } - const parsed = parsedFiles.filter((file): file is LoadedDotEnvFile => file !== null); - loadParsedDotEnvFiles(parsed); -} +export { loadGlobalRuntimeDotEnvFiles }; export function loadDotEnv(opts?: { quiet?: boolean }) { const quiet = opts?.quiet ?? true;