From 0177a4b6c9cc04ae1cf3dc7d76e51bbeac470d8d Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sun, 17 May 2026 10:55:41 +0100 Subject: [PATCH] fix(gateway): speed up secrets startup Summary: - Split the lightweight secrets runtime state and auth-store cache from the full secrets runtime. - Use the startup fast path whenever gateway startup has no SecretRef values, while preserving cleanup and refresh semantics. - Add regression coverage for startup-only empty auth-store snapshots and update affected gateway/tool tests. Verification: - pnpm test src/secrets/runtime.fast-path.test.ts src/secrets/runtime-state.test.ts src/gateway/server-startup-config.secrets.test.ts src/gateway/server-import-boundary.test.ts src/gateway/server-aux-handlers.test.ts src/gateway/server-methods/config.shared-auth.test.ts src/agents/tools/web-tools.enabled-defaults.test.ts src/agents/tools/web-tool-runtime-context.test.ts -- --reporter=verbose - pnpm build - pnpm format:check -- src/agents/tools/web-tools.enabled-defaults.test.ts src/secrets/runtime-command-secrets.ts src/secrets/runtime-fast-path.ts src/secrets/runtime.fast-path.test.ts src/agents/auth-profiles/store.ts src/agents/auth-profiles/store-cache.ts src/secrets/runtime-state.ts src/secrets/runtime-state.test.ts src/gateway/server-startup-config.ts - codex-review --mode branch - isolated gateway token-auth smoke: openclaw gateway run + openclaw gateway health returned ok: true - GitHub CI on PR #83031 green; newer Real behavior proof run passed on current SHA f27ed3f7ce6fdd4e3ce7f36b8391ef8b95bad019. Co-authored-by: samzong --- src/agents/auth-profiles/store-cache.ts | 50 +++ src/agents/auth-profiles/store.ts | 57 +-- src/agents/openclaw-tools.ts | 6 +- .../tools/web-fetch.provider-fallback.test.ts | 2 +- src/agents/tools/web-search.late-bind.test.ts | 2 +- .../tools/web-tool-runtime-context.test.ts | 2 +- src/agents/tools/web-tool-runtime-context.ts | 2 +- .../tools/web-tools.enabled-defaults.test.ts | 4 +- src/gateway/server-aux-handlers.ts | 13 +- src/gateway/server-import-boundary.test.ts | 4 + .../server-methods/config-write-flow.ts | 2 +- .../server-methods/config.shared-auth.test.ts | 5 +- src/gateway/server-reload-handlers.ts | 15 +- .../server-startup-config.secrets.test.ts | 307 +++++++++++++++ src/gateway/server-startup-config.ts | 67 +++- src/gateway/server.impl.ts | 5 +- src/secrets/runtime-command-secrets.ts | 2 +- src/secrets/runtime-fast-path.ts | 310 +++++++++++++++ src/secrets/runtime-state.test.ts | 71 ++++ src/secrets/runtime-state.ts | 145 +++++++ src/secrets/runtime.fast-path.test.ts | 159 ++++++++ src/secrets/runtime.ts | 354 +++--------------- 22 files changed, 1202 insertions(+), 382 deletions(-) create mode 100644 src/agents/auth-profiles/store-cache.ts create mode 100644 src/secrets/runtime-fast-path.ts create mode 100644 src/secrets/runtime-state.test.ts create mode 100644 src/secrets/runtime-state.ts diff --git a/src/agents/auth-profiles/store-cache.ts b/src/agents/auth-profiles/store-cache.ts new file mode 100644 index 00000000000..c3100d4daa5 --- /dev/null +++ b/src/agents/auth-profiles/store-cache.ts @@ -0,0 +1,50 @@ +import { cloneAuthProfileStore } from "./clone.js"; +import { EXTERNAL_CLI_SYNC_TTL_MS } from "./constants.js"; +import type { AuthProfileStore } from "./types.js"; + +const loadedAuthStoreCache = new Map< + string, + { + authMtimeMs: number | null; + stateMtimeMs: number | null; + syncedAtMs: number; + store: AuthProfileStore; + } +>(); + +export function readCachedAuthProfileStore(params: { + authPath: string; + authMtimeMs: number | null; + stateMtimeMs: number | null; +}): AuthProfileStore | null { + const cached = loadedAuthStoreCache.get(params.authPath); + if ( + !cached || + cached.authMtimeMs !== params.authMtimeMs || + cached.stateMtimeMs !== params.stateMtimeMs + ) { + return null; + } + if (Date.now() - cached.syncedAtMs >= EXTERNAL_CLI_SYNC_TTL_MS) { + return null; + } + return cloneAuthProfileStore(cached.store); +} + +export function writeCachedAuthProfileStore(params: { + authPath: string; + authMtimeMs: number | null; + stateMtimeMs: number | null; + store: AuthProfileStore; +}): void { + loadedAuthStoreCache.set(params.authPath, { + authMtimeMs: params.authMtimeMs, + stateMtimeMs: params.stateMtimeMs, + syncedAtMs: Date.now(), + store: cloneAuthProfileStore(params.store), + }); +} + +export function clearLoadedAuthStoreCache(): void { + loadedAuthStoreCache.clear(); +} diff --git a/src/agents/auth-profiles/store.ts b/src/agents/auth-profiles/store.ts index bfe6f21b85e..b7e1f4f7ecf 100644 --- a/src/agents/auth-profiles/store.ts +++ b/src/agents/auth-profiles/store.ts @@ -5,12 +5,7 @@ import type { OpenClawConfig } from "../../config/types.openclaw.js"; import { withFileLock } from "../../infra/file-lock.js"; import { loadJsonFile, saveJsonFile } from "../../infra/json-file.js"; import { cloneAuthProfileStore } from "./clone.js"; -import { - AUTH_STORE_LOCK_OPTIONS, - AUTH_STORE_VERSION, - EXTERNAL_CLI_SYNC_TTL_MS, - log, -} from "./constants.js"; +import { AUTH_STORE_LOCK_OPTIONS, AUTH_STORE_VERSION, log } from "./constants.js"; import { overlayExternalAuthProfiles, shouldPersistExternalAuthProfile, @@ -40,6 +35,11 @@ import { setRuntimeAuthProfileStoreSnapshot, } from "./runtime-snapshots.js"; import { savePersistedAuthProfileState } from "./state.js"; +import { + clearLoadedAuthStoreCache, + readCachedAuthProfileStore, + writeCachedAuthProfileStore, +} from "./store-cache.js"; import type { AuthProfileStore } from "./types.js"; type LoadAuthProfileStoreOptions = { @@ -75,16 +75,6 @@ type ExternalCliSyncResult = { cacheable: boolean; }; -const loadedAuthStoreCache = new Map< - string, - { - authMtimeMs: number | null; - stateMtimeMs: number | null; - syncedAtMs: number; - store: AuthProfileStore; - } ->(); - function isInheritedMainOAuthCredential(params: { agentDir?: string; profileId: string; @@ -235,39 +225,6 @@ function acquireAuthStoreLockSync(authPath: string): (() => void) | null { } } -function readCachedAuthProfileStore(params: { - authPath: string; - authMtimeMs: number | null; - stateMtimeMs: number | null; -}): AuthProfileStore | null { - const cached = loadedAuthStoreCache.get(params.authPath); - if ( - !cached || - cached.authMtimeMs !== params.authMtimeMs || - cached.stateMtimeMs !== params.stateMtimeMs - ) { - return null; - } - if (Date.now() - cached.syncedAtMs >= EXTERNAL_CLI_SYNC_TTL_MS) { - return null; - } - return cloneAuthProfileStore(cached.store); -} - -function writeCachedAuthProfileStore(params: { - authPath: string; - authMtimeMs: number | null; - stateMtimeMs: number | null; - store: AuthProfileStore; -}): void { - loadedAuthStoreCache.set(params.authPath, { - authMtimeMs: params.authMtimeMs, - stateMtimeMs: params.stateMtimeMs, - syncedAtMs: Date.now(), - store: cloneAuthProfileStore(params.store), - }); -} - function resolveExternalCliOverlayOptions( options: LoadAuthProfileStoreOptions | undefined, ): ResolvedExternalCliOverlayOptions { @@ -735,7 +692,7 @@ export function replaceRuntimeAuthProfileStoreSnapshots( export function clearRuntimeAuthProfileStoreSnapshots(): void { clearRuntimeAuthProfileStoreSnapshotsImpl(); - loadedAuthStoreCache.clear(); + clearLoadedAuthStoreCache(); } export function saveAuthProfileStore( diff --git a/src/agents/openclaw-tools.ts b/src/agents/openclaw-tools.ts index 74324cdfbd2..b3408488b05 100644 --- a/src/agents/openclaw-tools.ts +++ b/src/agents/openclaw-tools.ts @@ -4,10 +4,8 @@ import { selectApplicableRuntimeConfig } from "../config/config.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import { callGateway } from "../gateway/call.js"; import { isEmbeddedMode } from "../infra/embedded-mode.js"; -import { - getActiveRuntimeWebToolsMetadata, - getActiveSecretsRuntimeSnapshot, -} from "../secrets/runtime.js"; +import { getActiveSecretsRuntimeSnapshot } from "../secrets/runtime-state.js"; +import { getActiveRuntimeWebToolsMetadata } from "../secrets/runtime-web-tools-state.js"; import { normalizeDeliveryContext } from "../utils/delivery-context.js"; import type { GatewayMessageChannel } from "../utils/message-channel.js"; import { resolveAgentWorkspaceDir, resolveSessionAgentIds } from "./agent-scope.js"; diff --git a/src/agents/tools/web-fetch.provider-fallback.test.ts b/src/agents/tools/web-fetch.provider-fallback.test.ts index e40d8ceaa9d..a8723126a29 100644 --- a/src/agents/tools/web-fetch.provider-fallback.test.ts +++ b/src/agents/tools/web-fetch.provider-fallback.test.ts @@ -14,7 +14,7 @@ const runtimeState = vi.hoisted(() => ({ vi.mock("../../web-fetch/runtime.js", () => ({ resolveWebFetchDefinition: resolveWebFetchDefinitionMock, })); -vi.mock("../../secrets/runtime.js", () => ({ +vi.mock("../../secrets/runtime-state.js", () => ({ getActiveSecretsRuntimeSnapshot: () => runtimeState.activeSecretsRuntimeSnapshot, })); vi.mock("../../secrets/runtime-web-tools-state.js", () => ({ diff --git a/src/agents/tools/web-search.late-bind.test.ts b/src/agents/tools/web-search.late-bind.test.ts index eb958a0e93a..246f15ce00a 100644 --- a/src/agents/tools/web-search.late-bind.test.ts +++ b/src/agents/tools/web-search.late-bind.test.ts @@ -21,7 +21,7 @@ vi.mock("../../secrets/runtime-web-tools-state.js", () => ({ getActiveRuntimeWebToolsMetadata: mocks.getActiveRuntimeWebToolsMetadata, })); -vi.mock("../../secrets/runtime.js", () => ({ +vi.mock("../../secrets/runtime-state.js", () => ({ getActiveSecretsRuntimeSnapshot: mocks.getActiveSecretsRuntimeSnapshot, })); diff --git a/src/agents/tools/web-tool-runtime-context.test.ts b/src/agents/tools/web-tool-runtime-context.test.ts index 2866ff74f8e..5eab5a6a560 100644 --- a/src/agents/tools/web-tool-runtime-context.test.ts +++ b/src/agents/tools/web-tool-runtime-context.test.ts @@ -18,7 +18,7 @@ vi.mock("../../secrets/runtime-web-tools-state.js", () => ({ getActiveRuntimeWebToolsMetadata: mocks.getActiveRuntimeWebToolsMetadata, })); -vi.mock("../../secrets/runtime.js", () => ({ +vi.mock("../../secrets/runtime-state.js", () => ({ getActiveSecretsRuntimeSnapshot: mocks.getActiveSecretsRuntimeSnapshot, })); diff --git a/src/agents/tools/web-tool-runtime-context.ts b/src/agents/tools/web-tool-runtime-context.ts index c2aa4472882..8431046dc45 100644 --- a/src/agents/tools/web-tool-runtime-context.ts +++ b/src/agents/tools/web-tool-runtime-context.ts @@ -1,11 +1,11 @@ import type { OpenClawConfig } from "../../config/types.openclaw.js"; import { resolveManifestContractOwnerPluginId } from "../../plugins/plugin-registry.js"; +import { getActiveSecretsRuntimeSnapshot } from "../../secrets/runtime-state.js"; import { getActiveRuntimeWebToolsMetadata } from "../../secrets/runtime-web-tools-state.js"; import type { RuntimeWebFetchMetadata, RuntimeWebSearchMetadata, } from "../../secrets/runtime-web-tools.types.js"; -import { getActiveSecretsRuntimeSnapshot } from "../../secrets/runtime.js"; type WebProviderKind = "fetch" | "search"; diff --git a/src/agents/tools/web-tools.enabled-defaults.test.ts b/src/agents/tools/web-tools.enabled-defaults.test.ts index ad88921dadb..4f1d84ff5db 100644 --- a/src/agents/tools/web-tools.enabled-defaults.test.ts +++ b/src/agents/tools/web-tools.enabled-defaults.test.ts @@ -39,7 +39,7 @@ function readConfiguredSearchProvider(config: unknown): string | undefined { return typeof provider === "string" ? provider : undefined; } -vi.mock("../../secrets/runtime.js", () => ({ +vi.mock("../../secrets/runtime-state.js", () => ({ getActiveSecretsRuntimeSnapshot: () => activeSecretsRuntimeSnapshot.current, })); @@ -166,7 +166,7 @@ describe("web tools defaults", () => { const result = await tool?.execute?.("call-runtime-provider", {}); - expect(tool?.description).toContain("Search the web"); + expect(tool?.description).toContain("Search web"); expect((result?.details as { ok?: boolean } | undefined)?.ok).toBe(true); }); diff --git a/src/gateway/server-aux-handlers.ts b/src/gateway/server-aux-handlers.ts index 359fd347f86..b626dc9a9b4 100644 --- a/src/gateway/server-aux-handlers.ts +++ b/src/gateway/server-aux-handlers.ts @@ -7,9 +7,9 @@ import { type CommandSecretAssignment, } from "../secrets/runtime-command-secrets.js"; import { - activateSecretsRuntimeSnapshot, getActiveSecretsRuntimeSnapshot, -} from "../secrets/runtime.js"; + type PreparedSecretsRuntimeSnapshot, +} from "../secrets/runtime-state.js"; import { diffConfigPaths } from "./config-diff.js"; import { buildGatewayReloadPlan, @@ -38,6 +38,13 @@ type ReloadSecretsResult = { warningCount: number; }; +async function activateSecretsRuntimeSnapshot( + snapshot: PreparedSecretsRuntimeSnapshot, +): Promise { + const runtime = await import("../secrets/runtime.js"); + runtime.activateSecretsRuntimeSnapshot(snapshot); +} + function createLazyHandler( method: string, loadHandlers: () => Promise, @@ -193,7 +200,7 @@ export function createGatewayAuxHandlers(params: { } return { warningCount: prepared.warnings.length }; } catch (err) { - activateSecretsRuntimeSnapshot(previousSnapshot); + await activateSecretsRuntimeSnapshot(previousSnapshot); params.sharedGatewaySessionGenerationState.current = previousSharedGatewaySessionGeneration; params.sharedGatewaySessionGenerationState.required = diff --git a/src/gateway/server-import-boundary.test.ts b/src/gateway/server-import-boundary.test.ts index 3f66eed5173..75d77a684c4 100644 --- a/src/gateway/server-import-boundary.test.ts +++ b/src/gateway/server-import-boundary.test.ts @@ -39,6 +39,10 @@ describe("gateway startup import boundaries", () => { expect(serverImpl).not.toContain('from "../tasks/task-registry.js"'); expect(serverImpl).not.toContain('from "../tasks/task-registry.maintenance.js"'); expect(serverImpl).toContain('import("../tasks/task-registry.maintenance.js")'); + expect(serverImpl).not.toContain('from "../secrets/runtime.js"'); + expect(readSource("src/gateway/server-reload-handlers.ts")).not.toContain( + 'from "../secrets/runtime.js"', + ); const wsConnection = readSource("src/gateway/server/ws-connection.ts"); expect(wsConnection).not.toMatch( /import\s+\{[^}]*attachGatewayWsMessageHandler[^}]*\}\s+from "\.\/ws-connection\/message-handler\.js"/s, diff --git a/src/gateway/server-methods/config-write-flow.ts b/src/gateway/server-methods/config-write-flow.ts index 6319e3dc494..1e519196158 100644 --- a/src/gateway/server-methods/config-write-flow.ts +++ b/src/gateway/server-methods/config-write-flow.ts @@ -12,7 +12,7 @@ import { writeRestartSentinel, } from "../../infra/restart-sentinel.js"; import { scheduleGatewaySigusr1Restart } from "../../infra/restart.js"; -import { getActiveSecretsRuntimeSnapshot } from "../../secrets/runtime.js"; +import { getActiveSecretsRuntimeSnapshot } from "../../secrets/runtime-state.js"; import { resolveEffectiveSharedGatewayAuth, resolveGatewayAuth } from "../auth.js"; import { buildGatewayReloadPlan } from "../config-reload-plan.js"; import { resolveGatewayReloadSettings } from "../config-reload-settings.js"; diff --git a/src/gateway/server-methods/config.shared-auth.test.ts b/src/gateway/server-methods/config.shared-auth.test.ts index 8a061cd3f16..cf1c8df8677 100644 --- a/src/gateway/server-methods/config.shared-auth.test.ts +++ b/src/gateway/server-methods/config.shared-auth.test.ts @@ -53,10 +53,13 @@ vi.mock("../../config/runtime-schema.js", () => ({ })); vi.mock("../../secrets/runtime.js", () => ({ - getActiveSecretsRuntimeSnapshot: () => null, prepareSecretsRuntimeSnapshot: prepareSecretsRuntimeSnapshotMock, })); +vi.mock("../../secrets/runtime-state.js", () => ({ + getActiveSecretsRuntimeSnapshot: () => null, +})); + vi.mock("../../infra/restart.js", () => ({ scheduleGatewaySigusr1Restart: scheduleGatewaySigusr1RestartMock, })); diff --git a/src/gateway/server-reload-handlers.ts b/src/gateway/server-reload-handlers.ts index ab4b5547792..c97606ffe7d 100644 --- a/src/gateway/server-reload-handlers.ts +++ b/src/gateway/server-reload-handlers.ts @@ -21,10 +21,10 @@ import { } from "../infra/restart.js"; import { getTotalQueueSize } from "../process/command-queue.js"; import { - activateSecretsRuntimeSnapshot, clearSecretsRuntimeSnapshot, getActiveSecretsRuntimeSnapshot, -} from "../secrets/runtime.js"; + type PreparedSecretsRuntimeSnapshot, +} from "../secrets/runtime-state.js"; import { getInspectableActiveTaskRestartBlockers, type ActiveTaskRestartBlocker, @@ -59,6 +59,13 @@ type GatewayHotReloadState = { channelHealthMonitor: ChannelHealthMonitor | null; }; +async function activateSecretsRuntimeSnapshot( + snapshot: PreparedSecretsRuntimeSnapshot, +): Promise { + const runtime = await import("../secrets/runtime.js"); + runtime.activateSecretsRuntimeSnapshot(snapshot); +} + type GatewayReloadLog = { info: (msg: string) => void; warn: (msg: string) => void; @@ -605,7 +612,7 @@ export function startManagedGatewayConfigReloader(params: ManagedGatewayConfigRe await applyHotReload(plan, prepared.config); } catch (err) { if (previousSnapshot) { - activateSecretsRuntimeSnapshot(previousSnapshot); + await activateSecretsRuntimeSnapshot(previousSnapshot); } else { clearSecretsRuntimeSnapshot(); } @@ -638,7 +645,7 @@ export function startManagedGatewayConfigReloader(params: ManagedGatewayConfigRe const restartQueued = requestGatewayRestart(plan, nextConfig); if (!restartQueued) { if (previousSharedGatewaySessionGeneration !== nextSharedGatewaySessionGeneration) { - activateSecretsRuntimeSnapshot(prepared); + await activateSecretsRuntimeSnapshot(prepared); setCurrentSharedGatewaySessionGeneration( params.sharedGatewaySessionGenerationState, nextSharedGatewaySessionGeneration, diff --git a/src/gateway/server-startup-config.secrets.test.ts b/src/gateway/server-startup-config.secrets.test.ts index fb90009013d..6803d71a3a5 100644 --- a/src/gateway/server-startup-config.secrets.test.ts +++ b/src/gateway/server-startup-config.secrets.test.ts @@ -1,3 +1,6 @@ +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; import { loadAuthProfileStoreWithoutExternalProfiles } from "../agents/auth-profiles.js"; import type { ConfigFileSnapshot, OpenClawConfig } from "../config/types.js"; @@ -501,4 +504,308 @@ describe("gateway startup config secret preflight", () => { expect(prepareRuntimeSecretsSnapshot).toHaveBeenCalledTimes(2); expect(activateRuntimeSecretsSnapshot).toHaveBeenCalledTimes(1); }); + + it("activates no-SecretRef startup config without importing the full secrets runtime", async () => { + vi.resetModules(); + const agentDir = mkdtempSync(path.join(tmpdir(), "openclaw-startup-fast-path-")); + const runtimeImport = vi.fn(); + const prepareRuntimeSecretsSnapshot = vi.fn(async ({ config }) => preparedSnapshot(config)); + const activateRuntimeSecretsSnapshot = vi.fn(); + const loadAuthProfileStoreWithoutExternalProfilesMock = vi.fn(() => ({ + version: 1, + profiles: {}, + })); + ( + globalThis as typeof globalThis & { + __gatewayStartupSecretsRuntimeMock?: { + runtimeImport: typeof runtimeImport; + prepareRuntimeSecretsSnapshot: typeof prepareRuntimeSecretsSnapshot; + activateRuntimeSecretsSnapshot: typeof activateRuntimeSecretsSnapshot; + }; + } + ).__gatewayStartupSecretsRuntimeMock = { + runtimeImport, + prepareRuntimeSecretsSnapshot, + activateRuntimeSecretsSnapshot, + }; + vi.doMock("../agents/auth-profiles.js", () => ({ + loadAuthProfileStoreWithoutExternalProfiles: loadAuthProfileStoreWithoutExternalProfilesMock, + })); + vi.doMock("../secrets/runtime.js", () => { + const state = ( + globalThis as typeof globalThis & { + __gatewayStartupSecretsRuntimeMock?: { + runtimeImport: typeof runtimeImport; + prepareRuntimeSecretsSnapshot: typeof prepareRuntimeSecretsSnapshot; + activateRuntimeSecretsSnapshot: typeof activateRuntimeSecretsSnapshot; + }; + } + ).__gatewayStartupSecretsRuntimeMock; + if (!state) { + throw new Error("missing gateway startup secrets runtime mock"); + } + state.runtimeImport(); + return { + prepareSecretsRuntimeSnapshot: state.prepareRuntimeSecretsSnapshot, + activateSecretsRuntimeSnapshot: state.activateRuntimeSecretsSnapshot, + }; + }); + + try { + const { createRuntimeSecretsActivator: createActivator } = + await import("./server-startup-config.js"); + const { clearSecretsRuntimeSnapshot, getActiveSecretsRuntimeSnapshot } = + await import("../secrets/runtime-state.js"); + const { getRuntimeConfigSnapshotRefreshHandler } = + await import("../config/runtime-snapshot.js"); + const result = await createActivator({ + logSecrets: { + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + }, + emitStateEvent: vi.fn(), + })( + gatewayTokenConfig( + asConfig({ + agents: { + list: [{ id: "default", agentDir }], + }, + }), + ), + { + reason: "startup", + activate: true, + }, + ); + + expect(runtimeImport).not.toHaveBeenCalled(); + expect(prepareRuntimeSecretsSnapshot).not.toHaveBeenCalled(); + expect(activateRuntimeSecretsSnapshot).not.toHaveBeenCalled(); + expect(loadAuthProfileStoreWithoutExternalProfilesMock).not.toHaveBeenCalled(); + expect(result.config.gateway?.auth?.token).toBe("startup-test-token"); + expect(getActiveSecretsRuntimeSnapshot()?.config.gateway?.auth?.token).toBe( + "startup-test-token", + ); + const refreshHandler = getRuntimeConfigSnapshotRefreshHandler(); + await expect( + refreshHandler?.refresh({ + sourceConfig: gatewayTokenConfig( + asConfig({ + agents: { + list: [{ id: "default", agentDir }], + }, + }), + ), + }), + ).resolves.toBe(true); + expect(runtimeImport).toHaveBeenCalledTimes(1); + const refreshInput = callArg<{ + loadAuthStore?: unknown; + }>(prepareRuntimeSecretsSnapshot); + expect(refreshInput.loadAuthStore).toBeUndefined(); + clearSecretsRuntimeSnapshot(); + } finally { + vi.doUnmock("../agents/auth-profiles.js"); + vi.doUnmock("../secrets/runtime.js"); + delete ( + globalThis as typeof globalThis & { + __gatewayStartupSecretsRuntimeMock?: unknown; + } + ).__gatewayStartupSecretsRuntimeMock; + rmSync(agentDir, { recursive: true, force: true }); + vi.resetModules(); + } + }); + + it("keeps the full secrets runtime path when startup config has a SecretRef", async () => { + vi.resetModules(); + const agentDir = mkdtempSync(path.join(tmpdir(), "openclaw-startup-secret-ref-")); + const runtimeImport = vi.fn(); + const prepareRuntimeSecretsSnapshot = vi.fn(async ({ config }) => preparedSnapshot(config)); + const activateRuntimeSecretsSnapshot = vi.fn(); + ( + globalThis as typeof globalThis & { + __gatewayStartupSecretsRuntimeMock?: { + runtimeImport: typeof runtimeImport; + prepareRuntimeSecretsSnapshot: typeof prepareRuntimeSecretsSnapshot; + activateRuntimeSecretsSnapshot: typeof activateRuntimeSecretsSnapshot; + }; + } + ).__gatewayStartupSecretsRuntimeMock = { + runtimeImport, + prepareRuntimeSecretsSnapshot, + activateRuntimeSecretsSnapshot, + }; + vi.doMock("../agents/auth-profiles.js", () => ({ + loadAuthProfileStoreWithoutExternalProfiles: vi.fn(() => ({ + version: 1, + profiles: {}, + })), + })); + vi.doMock("../secrets/runtime.js", () => { + const state = ( + globalThis as typeof globalThis & { + __gatewayStartupSecretsRuntimeMock?: { + runtimeImport: typeof runtimeImport; + prepareRuntimeSecretsSnapshot: typeof prepareRuntimeSecretsSnapshot; + activateRuntimeSecretsSnapshot: typeof activateRuntimeSecretsSnapshot; + }; + } + ).__gatewayStartupSecretsRuntimeMock; + if (!state) { + throw new Error("missing gateway startup secrets runtime mock"); + } + state.runtimeImport(); + return { + prepareSecretsRuntimeSnapshot: state.prepareRuntimeSecretsSnapshot, + activateSecretsRuntimeSnapshot: state.activateRuntimeSecretsSnapshot, + }; + }); + + try { + const { createRuntimeSecretsActivator: createActivator } = + await import("./server-startup-config.js"); + await createActivator({ + logSecrets: { + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + }, + emitStateEvent: vi.fn(), + })( + gatewayTokenConfig( + asConfig({ + agents: { + list: [{ id: "default", agentDir }], + }, + models: { + providers: { + openai: { + models: [], + apiKey: { source: "env", provider: "default", id: "OPENAI_API_KEY" }, + }, + }, + }, + }), + ), + { + reason: "startup", + activate: true, + }, + ); + + expect(runtimeImport).toHaveBeenCalledTimes(1); + expect(prepareRuntimeSecretsSnapshot).toHaveBeenCalledTimes(1); + expect(activateRuntimeSecretsSnapshot).toHaveBeenCalledTimes(1); + } finally { + vi.doUnmock("../agents/auth-profiles.js"); + vi.doUnmock("../secrets/runtime.js"); + delete ( + globalThis as typeof globalThis & { + __gatewayStartupSecretsRuntimeMock?: unknown; + } + ).__gatewayStartupSecretsRuntimeMock; + rmSync(agentDir, { recursive: true, force: true }); + vi.resetModules(); + } + }); + + it("keeps the full secrets runtime path when auth profile files are present", async () => { + vi.resetModules(); + const agentDir = mkdtempSync(path.join(tmpdir(), "openclaw-startup-auth-store-")); + writeFileSync( + path.join(agentDir, "auth-profiles.json"), + `${JSON.stringify({ + version: 1, + profiles: { + "openai:default": { + type: "api_key", + provider: "openai", + key: "sk-test", + }, + }, + })}\n`, + ); + const runtimeImport = vi.fn(); + const prepareRuntimeSecretsSnapshot = vi.fn(async ({ config }) => preparedSnapshot(config)); + const activateRuntimeSecretsSnapshot = vi.fn(); + ( + globalThis as typeof globalThis & { + __gatewayStartupSecretsRuntimeMock?: { + runtimeImport: typeof runtimeImport; + prepareRuntimeSecretsSnapshot: typeof prepareRuntimeSecretsSnapshot; + activateRuntimeSecretsSnapshot: typeof activateRuntimeSecretsSnapshot; + }; + } + ).__gatewayStartupSecretsRuntimeMock = { + runtimeImport, + prepareRuntimeSecretsSnapshot, + activateRuntimeSecretsSnapshot, + }; + vi.doMock("../agents/auth-profiles.js", () => ({ + loadAuthProfileStoreWithoutExternalProfiles: vi.fn(() => ({ + version: 1, + profiles: {}, + })), + })); + vi.doMock("../secrets/runtime.js", () => { + const state = ( + globalThis as typeof globalThis & { + __gatewayStartupSecretsRuntimeMock?: { + runtimeImport: typeof runtimeImport; + prepareRuntimeSecretsSnapshot: typeof prepareRuntimeSecretsSnapshot; + activateRuntimeSecretsSnapshot: typeof activateRuntimeSecretsSnapshot; + }; + } + ).__gatewayStartupSecretsRuntimeMock; + if (!state) { + throw new Error("missing gateway startup secrets runtime mock"); + } + state.runtimeImport(); + return { + prepareSecretsRuntimeSnapshot: state.prepareRuntimeSecretsSnapshot, + activateSecretsRuntimeSnapshot: state.activateRuntimeSecretsSnapshot, + }; + }); + + try { + const { createRuntimeSecretsActivator: createActivator } = + await import("./server-startup-config.js"); + await createActivator({ + logSecrets: { + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + }, + emitStateEvent: vi.fn(), + })( + gatewayTokenConfig( + asConfig({ + agents: { + list: [{ id: "default", agentDir }], + }, + }), + ), + { + reason: "startup", + activate: true, + }, + ); + + expect(runtimeImport).toHaveBeenCalledTimes(1); + expect(prepareRuntimeSecretsSnapshot).toHaveBeenCalledTimes(1); + expect(activateRuntimeSecretsSnapshot).toHaveBeenCalledTimes(1); + } finally { + vi.doUnmock("../agents/auth-profiles.js"); + vi.doUnmock("../secrets/runtime.js"); + delete ( + globalThis as typeof globalThis & { + __gatewayStartupSecretsRuntimeMock?: unknown; + } + ).__gatewayStartupSecretsRuntimeMock; + rmSync(agentDir, { recursive: true, force: true }); + vi.resetModules(); + } + }); }); diff --git a/src/gateway/server-startup-config.ts b/src/gateway/server-startup-config.ts index 7fb492a5d00..86d186bfbd4 100644 --- a/src/gateway/server-startup-config.ts +++ b/src/gateway/server-startup-config.ts @@ -12,10 +12,15 @@ import type { GatewayAuthConfig, GatewayTailscaleConfig } from "../config/types. import type { ConfigFileSnapshot, OpenClawConfig } from "../config/types.openclaw.js"; import { isTruthyEnvValue } from "../infra/env.js"; import type { PluginMetadataSnapshot } from "../plugins/plugin-metadata-snapshot.js"; +import { + prepareSecretsRuntimeFastPathSnapshot, + resolveRefreshAgentDirs, +} from "../secrets/runtime-fast-path.js"; import { GATEWAY_AUTH_SURFACE_PATHS, evaluateGatewayAuthSurfaceStates, } from "../secrets/runtime-gateway-auth-surfaces.js"; +import { activateSecretsRuntimeSnapshotState } from "../secrets/runtime-state.js"; import { resolveGatewayAuth } from "./auth.js"; import { assertGatewayAuthNotKnownWeak } from "./known-weak-gateway-secrets.js"; import { @@ -172,10 +177,14 @@ export function createRuntimeSecretsActivator(params: { const finishPreparedSnapshot = async ( prepared: PreparedRuntimeSecretsSnapshot, activationParams: RuntimeSecretsActivationParams, + options?: { + activateRuntimeSecretsSnapshot?: (snapshot: PreparedRuntimeSecretsSnapshot) => void; + }, ) => { assertRuntimeGatewayAuthNotKnownWeak(prepared.config); if (activationParams.activate) { - const activateRuntimeSecretsSnapshot = await loadActivateRuntimeSecretsSnapshot(); + const activateRuntimeSecretsSnapshot = + options?.activateRuntimeSecretsSnapshot ?? (await loadActivateRuntimeSecretsSnapshot()); activateRuntimeSecretsSnapshot(prepared); logGatewayAuthSurfaceDiagnostics(prepared, params.logSecrets); } @@ -222,17 +231,52 @@ export function createRuntimeSecretsActivator(params: { const activateRuntimeSecrets = (async (config, activationParams) => await runWithSecretsActivationLock(async () => { try { + const startupPreflight = + activationParams.reason === "startup" || activationParams.reason === "restart-check"; + if ( + activationParams.reason === "startup" && + activationParams.activate && + !params.prepareRuntimeSecretsSnapshot && + !params.activateRuntimeSecretsSnapshot + ) { + const fastPath = prepareSecretsRuntimeFastPathSnapshot({ + config: pruneSkippedStartupSecretSurfaces(config), + }); + if (fastPath) { + return await finishPreparedSnapshot(fastPath.snapshot, activationParams, { + activateRuntimeSecretsSnapshot: (snapshot) => + activateSecretsRuntimeSnapshotState({ + snapshot, + refreshContext: fastPath.refreshContext, + refreshHandler: { + refresh: async ({ sourceConfig }) => { + const secretsRuntime = await loadSecretsRuntime(); + const refreshed = await secretsRuntime.prepareSecretsRuntimeSnapshot({ + config: sourceConfig, + env: fastPath.refreshContext.env, + agentDirs: resolveRefreshAgentDirs(sourceConfig, fastPath.refreshContext), + loadablePluginOrigins: fastPath.refreshContext.loadablePluginOrigins, + ...(fastPath.usesAuthStoreFallback || !fastPath.refreshContext.loadAuthStore + ? {} + : { loadAuthStore: fastPath.refreshContext.loadAuthStore }), + }); + secretsRuntime.activateSecretsRuntimeSnapshot(refreshed); + return true; + }, + }, + }), + }); + } + } + const loadAuthStore = startupPreflight + ? (await loadAuthProfiles()).loadAuthProfileStoreWithoutExternalProfiles + : undefined; const secretsRuntime = params.prepareRuntimeSecretsSnapshot && params.activateRuntimeSecretsSnapshot ? null : await loadSecretsRuntime(); const prepareRuntimeSecretsSnapshot = params.prepareRuntimeSecretsSnapshot ?? secretsRuntime!.prepareSecretsRuntimeSnapshot; - const startupPreflight = - activationParams.reason === "startup" || activationParams.reason === "restart-check"; - const loadAuthStore = startupPreflight - ? (await loadAuthProfiles()).loadAuthProfileStoreWithoutExternalProfiles - : undefined; const prepared = await prepareRuntimeSecretsSnapshot({ config: pruneSkippedStartupSecretSurfaces(config), ...(loadAuthStore ? { loadAuthStore } : {}), @@ -309,11 +353,8 @@ export async function prepareGatewayStartupConfig(params: { const canReusePreflightPreparedSnapshot = (config: OpenClawConfig): boolean => Boolean( preflightPrepared && - params.activateRuntimeSecrets.activatePreparedSnapshot && - isDeepStrictEqual( - pruneSkippedStartupSecretSurfaces(config), - preflightPrepared.sourceConfig, - ), + params.activateRuntimeSecrets.activatePreparedSnapshot && + isDeepStrictEqual(pruneSkippedStartupSecretSurfaces(config), preflightPrepared.sourceConfig), ); const activateStartupSecrets = async (config: OpenClawConfig) => { if (preflightPrepared && canReusePreflightPreparedSnapshot(config)) { @@ -359,7 +400,9 @@ export async function prepareGatewayStartupConfig(params: { }), ); const activatedConfig = ( - await measure("config.auth.secrets-activate", () => activateStartupSecrets(runtimeStartupConfig)) + await measure("config.auth.secrets-activate", () => + activateStartupSecrets(runtimeStartupConfig), + ) ).config; return { ...authBootstrap, diff --git a/src/gateway/server.impl.ts b/src/gateway/server.impl.ts index 69c8ff3d27a..d1f38536a3f 100644 --- a/src/gateway/server.impl.ts +++ b/src/gateway/server.impl.ts @@ -54,7 +54,7 @@ import type { RuntimeEnv } from "../runtime.js"; import { clearSecretsRuntimeSnapshot, getActiveSecretsRuntimeSnapshot, -} from "../secrets/runtime.js"; +} from "../secrets/runtime-state.js"; import { createAuthRateLimiter, type AuthRateLimiter } from "./auth-rate-limit.js"; import { resolveGatewayAuth } from "./auth.js"; import { ADMIN_SCOPE } from "./method-scopes.js"; @@ -552,7 +552,6 @@ export async function startGatewayServer( } const startupTrace = createGatewayStartupTrace(); const startupConfigModulePromise = import("./server-startup-config.js"); - const reloadHandlersModulePromise = import("./server-reload-handlers.js"); let startupPluginsModulePromise: Promise | null = null; const loadStartupPluginsModule = () => { @@ -1553,7 +1552,7 @@ export async function startGatewayServer( postAttachRuntimeReturned = true; activateScheduledServicesWhenReady(); - const { startManagedGatewayConfigReloader } = await reloadHandlersModulePromise; + const { startManagedGatewayConfigReloader } = await import("./server-reload-handlers.js"); runtimeState.configReloader = startManagedGatewayConfigReloader({ minimalTestGateway, initialConfig: cfgAtStart, diff --git a/src/secrets/runtime-command-secrets.ts b/src/secrets/runtime-command-secrets.ts index a0cef950d2c..222b17a9a4a 100644 --- a/src/secrets/runtime-command-secrets.ts +++ b/src/secrets/runtime-command-secrets.ts @@ -11,8 +11,8 @@ import { import { getPath, setPathExistingStrict } from "./path-utils.js"; import { resolveSecretRefValue } from "./resolve.js"; import { createResolverContext } from "./runtime-shared.js"; +import { getActiveSecretsRuntimeEnv, getActiveSecretsRuntimeSnapshot } from "./runtime-state.js"; import { resolveRuntimeWebTools } from "./runtime-web-tools.js"; -import { getActiveSecretsRuntimeEnv, getActiveSecretsRuntimeSnapshot } from "./runtime.js"; import { assertExpectedResolvedSecretValue } from "./secret-value.js"; import { discoverConfigSecretTargetsByIds } from "./target-registry.js"; diff --git a/src/secrets/runtime-fast-path.ts b/src/secrets/runtime-fast-path.ts new file mode 100644 index 00000000000..f7ab727848d --- /dev/null +++ b/src/secrets/runtime-fast-path.ts @@ -0,0 +1,310 @@ +import { existsSync } from "node:fs"; +import path from "node:path"; +import { + listAgentIds, + resolveAgentDir, + resolveDefaultAgentDir, +} from "../agents/agent-scope-config.js"; +import { + AUTH_PROFILE_FILENAME, + AUTH_STATE_FILENAME, + LEGACY_AUTH_FILENAME, +} from "../agents/auth-profiles/path-constants.js"; +import type { AuthProfileStore } from "../agents/auth-profiles/types.js"; +import { resolveOAuthPath } from "../config/paths.js"; +import type { OpenClawConfig } from "../config/types.openclaw.js"; +import { coerceSecretRef } from "../config/types.secrets.js"; +import type { PluginOrigin } from "../plugins/plugin-origin.types.js"; +import { resolveUserPath } from "../utils.js"; +import type { + PreparedSecretsRuntimeSnapshot, + SecretsRuntimeRefreshContext, +} from "./runtime-state.js"; +import type { RuntimeWebToolsMetadata } from "./runtime-web-tools.types.js"; + +const RUNTIME_PATH_ENV_KEYS = [ + "HOME", + "USERPROFILE", + "HOMEDRIVE", + "HOMEPATH", + "OPENCLAW_HOME", + "OPENCLAW_STATE_DIR", + "OPENCLAW_CONFIG_PATH", + "OPENCLAW_AGENT_DIR", + "PI_CODING_AGENT_DIR", + "OPENCLAW_TEST_FAST", +] as const; + +export function mergeSecretsRuntimeEnv( + env: NodeJS.ProcessEnv | Record | undefined, +): Record { + const merged = { ...(env ?? process.env) } as Record; + for (const key of RUNTIME_PATH_ENV_KEYS) { + if (merged[key] !== undefined) { + continue; + } + const processValue = process.env[key]; + if (processValue !== undefined) { + merged[key] = processValue; + } + } + return merged; +} + +export function collectCandidateAgentDirs( + config: OpenClawConfig, + env: NodeJS.ProcessEnv | Record = process.env, +): string[] { + const dirs = new Set(); + dirs.add(resolveUserPath(resolveDefaultAgentDir(config, env), env)); + for (const agentId of listAgentIds(config)) { + dirs.add(resolveUserPath(resolveAgentDir(config, agentId, env), env)); + } + return [...dirs]; +} + +export function resolveRefreshAgentDirs( + config: OpenClawConfig, + context: SecretsRuntimeRefreshContext, +): string[] { + const configDerived = collectCandidateAgentDirs(config, context.env); + if (!context.explicitAgentDirs || context.explicitAgentDirs.length === 0) { + return configDerived; + } + return [...new Set([...context.explicitAgentDirs, ...configDerived])]; +} + +function resolveCandidateAgentDirs(params: { + config: OpenClawConfig; + env: NodeJS.ProcessEnv | Record; + agentDirs?: string[]; +}): string[] { + return params.agentDirs?.length + ? [...new Set(params.agentDirs.map((entry) => resolveUserPath(entry, params.env)))] + : collectCandidateAgentDirs(params.config, params.env); +} + +function hasCandidateAuthProfileStoreSource(agentDir: string): boolean { + return ( + existsSync(path.join(agentDir, AUTH_PROFILE_FILENAME)) || + existsSync(path.join(agentDir, AUTH_STATE_FILENAME)) || + existsSync(path.join(agentDir, LEGACY_AUTH_FILENAME)) + ); +} + +export function hasCandidateAuthProfileStoreSources(params: { + config: OpenClawConfig; + env: NodeJS.ProcessEnv | Record; + agentDirs?: string[]; +}): boolean { + const candidateDirs = resolveCandidateAgentDirs(params); + const mainAgentDir = resolveUserPath(resolveDefaultAgentDir({}, params.env), params.env); + return ( + candidateDirs.some((agentDir) => hasCandidateAuthProfileStoreSource(agentDir)) || + hasCandidateAuthProfileStoreSource(mainAgentDir) || + existsSync(resolveOAuthPath(params.env as NodeJS.ProcessEnv)) + ); +} + +export function createEmptyRuntimeWebToolsMetadata(): RuntimeWebToolsMetadata { + return { + search: { + providerSource: "none", + diagnostics: [], + }, + fetch: { + providerSource: "none", + diagnostics: [], + }, + diagnostics: [], + }; +} + +const WEB_FETCH_CREDENTIAL_FIELD_NAMES = new Set(["apikey", "key", "token", "secret", "password"]); + +function hasCredentialBearingWebFetchValue( + value: unknown, + defaults: Parameters[1], + seen = new WeakSet(), +): boolean { + if (coerceSecretRef(value, defaults)) { + return true; + } + if (!value || typeof value !== "object") { + return false; + } + if (seen.has(value)) { + return false; + } + seen.add(value); + if (Array.isArray(value)) { + return value.some((entry) => hasCredentialBearingWebFetchValue(entry, defaults, seen)); + } + return Object.entries(value as Record).some(([rawKey, entry]) => { + const key = rawKey.toLowerCase(); + if (WEB_FETCH_CREDENTIAL_FIELD_NAMES.has(key) && entry != null && entry !== "") { + return true; + } + return hasCredentialBearingWebFetchValue(entry, defaults, seen); + }); +} + +function hasActiveRuntimeWebFetchProviderSurface( + fetch: unknown, + defaults: Parameters[1], +): boolean { + if (!fetch || typeof fetch !== "object" || Array.isArray(fetch)) { + return false; + } + const fetchConfig = fetch as Record; + if (fetchConfig.enabled === false) { + return false; + } + if (typeof fetchConfig.provider === "string" && fetchConfig.provider.trim()) { + return true; + } + return hasCredentialBearingWebFetchValue(fetchConfig, defaults); +} + +function hasRuntimeWebToolConfigSurface(config: OpenClawConfig): boolean { + const web = config.tools?.web; + const defaults = config.secrets?.defaults; + const fetchExplicitlyDisabled = + web && + typeof web === "object" && + !Array.isArray(web) && + typeof (web as Record).fetch === "object" && + (web as { fetch?: { enabled?: unknown } }).fetch?.enabled === false; + if (web && typeof web === "object" && !Array.isArray(web)) { + const webRecord = web as Record; + if ("search" in webRecord || "x_search" in webRecord) { + return true; + } + if ( + "fetch" in webRecord && + hasActiveRuntimeWebFetchProviderSurface(webRecord.fetch, defaults) + ) { + return true; + } + } + const entries = config.plugins?.entries; + if (!entries || typeof entries !== "object" || Array.isArray(entries)) { + return false; + } + return Object.values(entries).some((entry) => { + if (!entry || typeof entry !== "object" || Array.isArray(entry)) { + return false; + } + const pluginConfig = (entry as { config?: unknown }).config; + return ( + !!pluginConfig && + typeof pluginConfig === "object" && + !Array.isArray(pluginConfig) && + ("webSearch" in pluginConfig || (!fetchExplicitlyDisabled && "webFetch" in pluginConfig)) + ); + }); +} + +function hasSecretRefCandidate( + value: unknown, + defaults: Parameters[1], + seen = new WeakSet(), +): boolean { + if (coerceSecretRef(value, defaults)) { + return true; + } + if (!value || typeof value !== "object") { + return false; + } + if (seen.has(value)) { + return false; + } + seen.add(value); + if (Array.isArray(value)) { + return value.some((entry) => hasSecretRefCandidate(entry, defaults, seen)); + } + return Object.values(value as Record).some((entry) => + hasSecretRefCandidate(entry, defaults, seen), + ); +} + +export function canUseSecretsRuntimeFastPath(params: { + sourceConfig: OpenClawConfig; + authStores: Array<{ agentDir: string; store: AuthProfileStore }>; +}): boolean { + if (hasRuntimeWebToolConfigSurface(params.sourceConfig)) { + return false; + } + const defaults = params.sourceConfig.secrets?.defaults; + if (hasSecretRefCandidate(params.sourceConfig, defaults)) { + return false; + } + return !params.authStores.some((entry) => hasSecretRefCandidate(entry.store, defaults)); +} + +export function prepareSecretsRuntimeFastPathSnapshot(params: { + config: OpenClawConfig; + env?: NodeJS.ProcessEnv; + agentDirs?: string[]; + includeAuthStoreRefs?: boolean; + loadAuthStore?: (agentDir?: string) => AuthProfileStore; + loadablePluginOrigins?: ReadonlyMap; +}): { + snapshot: PreparedSecretsRuntimeSnapshot; + refreshContext: SecretsRuntimeRefreshContext; + usesAuthStoreFallback: boolean; +} | null { + const runtimeEnv = mergeSecretsRuntimeEnv(params.env); + const sourceConfig = structuredClone(params.config); + const resolvedConfig = structuredClone(params.config); + const includeAuthStoreRefs = params.includeAuthStoreRefs ?? true; + const candidateDirs = resolveCandidateAgentDirs({ + config: resolvedConfig, + env: runtimeEnv, + agentDirs: params.agentDirs, + }); + let authStores: Array<{ agentDir: string; store: AuthProfileStore }> = []; + if (includeAuthStoreRefs) { + if (!params.loadAuthStore) { + if ( + hasCandidateAuthProfileStoreSources({ + config: resolvedConfig, + env: runtimeEnv, + agentDirs: candidateDirs, + }) + ) { + return null; + } + authStores = candidateDirs.map((agentDir) => ({ + agentDir, + store: { version: 1, profiles: {} }, + })); + } else { + const loadAuthStore = params.loadAuthStore; + authStores = candidateDirs.map((agentDir) => ({ + agentDir, + store: structuredClone(loadAuthStore(agentDir)), + })); + } + } + if (!canUseSecretsRuntimeFastPath({ sourceConfig, authStores })) { + return null; + } + const snapshot = { + sourceConfig, + config: resolvedConfig, + authStores, + warnings: [], + webTools: createEmptyRuntimeWebToolsMetadata(), + }; + return { + snapshot, + usesAuthStoreFallback: !params.loadAuthStore, + refreshContext: { + env: runtimeEnv, + explicitAgentDirs: params.agentDirs?.length ? [...candidateDirs] : null, + loadablePluginOrigins: params.loadablePluginOrigins ?? new Map(), + ...(params.loadAuthStore ? { loadAuthStore: params.loadAuthStore } : {}), + }, + }; +} diff --git a/src/secrets/runtime-state.test.ts b/src/secrets/runtime-state.test.ts new file mode 100644 index 00000000000..7004b212760 --- /dev/null +++ b/src/secrets/runtime-state.test.ts @@ -0,0 +1,71 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { resolveAuthStatePath, resolveAuthStorePath } from "../agents/auth-profiles/paths.js"; +import { writeCachedAuthProfileStore } from "../agents/auth-profiles/store-cache.js"; +import { loadAuthProfileStoreForRuntime } from "../agents/auth-profiles/store.js"; +import type { AuthProfileStore } from "../agents/auth-profiles/types.js"; +import { clearSecretsRuntimeSnapshot } from "./runtime-state.js"; + +function authStore(key: string): AuthProfileStore { + return { + version: 1, + profiles: { + "openai:default": { + type: "api_key", + provider: "openai", + key, + }, + }, + }; +} + +describe("secrets runtime state", () => { + const previousStateDir = process.env.OPENCLAW_STATE_DIR; + + afterEach(() => { + clearSecretsRuntimeSnapshot(); + if (previousStateDir === undefined) { + delete process.env.OPENCLAW_STATE_DIR; + } else { + process.env.OPENCLAW_STATE_DIR = previousStateDir; + } + }); + + it("clears loaded auth-profile cache without importing the full secrets runtime", () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-runtime-state-cache-")); + process.env.OPENCLAW_STATE_DIR = root; + const agentDir = path.join(root, "agents", "default", "agent"); + + try { + fs.mkdirSync(agentDir, { recursive: true }); + const authPath = resolveAuthStorePath(agentDir); + const statePath = resolveAuthStatePath(agentDir); + fs.writeFileSync(authPath, `${JSON.stringify(authStore("sk-new"))}\n`); + const stat = fs.statSync(authPath); + writeCachedAuthProfileStore({ + authPath, + authMtimeMs: stat.mtimeMs, + stateMtimeMs: fs.existsSync(statePath) ? fs.statSync(statePath).mtimeMs : null, + store: authStore("sk-old"), + }); + + expect( + loadAuthProfileStoreForRuntime(agentDir, { syncExternalCli: false }).profiles[ + "openai:default" + ], + ).toMatchObject({ key: "sk-old" }); + + clearSecretsRuntimeSnapshot(); + + expect( + loadAuthProfileStoreForRuntime(agentDir, { syncExternalCli: false }).profiles[ + "openai:default" + ], + ).toMatchObject({ key: "sk-new" }); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/src/secrets/runtime-state.ts b/src/secrets/runtime-state.ts new file mode 100644 index 00000000000..b3b2251253a --- /dev/null +++ b/src/secrets/runtime-state.ts @@ -0,0 +1,145 @@ +import { + clearRuntimeAuthProfileStoreSnapshots, + replaceRuntimeAuthProfileStoreSnapshots, +} from "../agents/auth-profiles/runtime-snapshots.js"; +import { clearLoadedAuthStoreCache } from "../agents/auth-profiles/store-cache.js"; +import type { AuthProfileStore } from "../agents/auth-profiles/types.js"; +import { + clearRuntimeConfigSnapshot, + setRuntimeConfigSnapshot, + setRuntimeConfigSnapshotRefreshHandler, + type RuntimeConfigSnapshotRefreshHandler, +} from "../config/runtime-snapshot.js"; +import type { OpenClawConfig } from "../config/types.openclaw.js"; +import type { PluginOrigin } from "../plugins/plugin-origin.types.js"; +import type { SecretResolverWarning } from "./runtime-shared.js"; +import { + clearActiveRuntimeWebToolsMetadata, + setActiveRuntimeWebToolsMetadata, +} from "./runtime-web-tools-state.js"; +import type { RuntimeWebToolsMetadata } from "./runtime-web-tools.types.js"; + +export type PreparedSecretsRuntimeSnapshot = { + sourceConfig: OpenClawConfig; + config: OpenClawConfig; + authStores: Array<{ agentDir: string; store: AuthProfileStore }>; + warnings: SecretResolverWarning[]; + webTools: RuntimeWebToolsMetadata; +}; + +export type SecretsRuntimeRefreshContext = { + env: Record; + explicitAgentDirs: string[] | null; + loadAuthStore?: (agentDir?: string) => AuthProfileStore; + loadablePluginOrigins: ReadonlyMap; +}; + +let activeSnapshot: PreparedSecretsRuntimeSnapshot | null = null; +let activeRefreshContext: SecretsRuntimeRefreshContext | null = null; +const clearHooks = new Set<() => void>(); +const preparedSnapshotRefreshContext = new WeakMap< + PreparedSecretsRuntimeSnapshot, + SecretsRuntimeRefreshContext +>(); + +export function cloneSecretsRuntimeRefreshContext( + context: SecretsRuntimeRefreshContext, +): SecretsRuntimeRefreshContext { + const cloned: SecretsRuntimeRefreshContext = { + env: { ...context.env }, + explicitAgentDirs: context.explicitAgentDirs ? [...context.explicitAgentDirs] : null, + loadablePluginOrigins: new Map(context.loadablePluginOrigins), + }; + if (context.loadAuthStore) { + cloned.loadAuthStore = context.loadAuthStore; + } + return cloned; +} + +function cloneSnapshot(snapshot: PreparedSecretsRuntimeSnapshot): PreparedSecretsRuntimeSnapshot { + return { + sourceConfig: structuredClone(snapshot.sourceConfig), + config: structuredClone(snapshot.config), + authStores: snapshot.authStores.map((entry) => ({ + agentDir: entry.agentDir, + store: structuredClone(entry.store), + })), + warnings: snapshot.warnings.map((warning) => ({ ...warning })), + webTools: structuredClone(snapshot.webTools), + }; +} + +export function setPreparedSecretsRuntimeSnapshotRefreshContext( + snapshot: PreparedSecretsRuntimeSnapshot, + context: SecretsRuntimeRefreshContext, +): void { + preparedSnapshotRefreshContext.set(snapshot, cloneSecretsRuntimeRefreshContext(context)); +} + +export function getPreparedSecretsRuntimeSnapshotRefreshContext( + snapshot: PreparedSecretsRuntimeSnapshot, +): SecretsRuntimeRefreshContext | null { + const context = preparedSnapshotRefreshContext.get(snapshot); + return context ? cloneSecretsRuntimeRefreshContext(context) : null; +} + +export function getActiveSecretsRuntimeRefreshContext(): SecretsRuntimeRefreshContext | null { + return activeRefreshContext ? cloneSecretsRuntimeRefreshContext(activeRefreshContext) : null; +} + +export function getActiveSecretsRuntimeEnv(): NodeJS.ProcessEnv { + return { + ...(activeRefreshContext?.env ?? process.env), + } as NodeJS.ProcessEnv; +} + +export function registerSecretsRuntimeStateClearHook(clearHook: () => void): void { + clearHooks.add(clearHook); +} + +export function activateSecretsRuntimeSnapshotState(params: { + snapshot: PreparedSecretsRuntimeSnapshot; + refreshContext: SecretsRuntimeRefreshContext | null; + refreshHandler: RuntimeConfigSnapshotRefreshHandler | null; +}): void { + const next = cloneSnapshot(params.snapshot); + const nextRefreshContext = params.refreshContext + ? cloneSecretsRuntimeRefreshContext(params.refreshContext) + : null; + setRuntimeConfigSnapshot(next.config, next.sourceConfig); + replaceRuntimeAuthProfileStoreSnapshots(next.authStores); + activeSnapshot = next; + activeRefreshContext = nextRefreshContext; + if (nextRefreshContext) { + preparedSnapshotRefreshContext.set(next, cloneSecretsRuntimeRefreshContext(nextRefreshContext)); + } + setActiveRuntimeWebToolsMetadata(next.webTools); + setRuntimeConfigSnapshotRefreshHandler(params.refreshHandler); +} + +export function getActiveSecretsRuntimeSnapshot(): PreparedSecretsRuntimeSnapshot | null { + if (!activeSnapshot) { + return null; + } + const snapshot = cloneSnapshot(activeSnapshot); + if (activeRefreshContext) { + preparedSnapshotRefreshContext.set( + snapshot, + cloneSecretsRuntimeRefreshContext(activeRefreshContext), + ); + } + return snapshot; +} + +export function clearSecretsRuntimeSnapshot(): void { + activeSnapshot = null; + activeRefreshContext = null; + clearActiveRuntimeWebToolsMetadata(); + setRuntimeConfigSnapshotRefreshHandler(null); + clearRuntimeConfigSnapshot(); + clearRuntimeAuthProfileStoreSnapshots(); + clearLoadedAuthStoreCache(); + for (const clearHook of clearHooks) { + clearHook(); + } +} diff --git a/src/secrets/runtime.fast-path.test.ts b/src/secrets/runtime.fast-path.test.ts index 253307cb95f..bfa6b050ad7 100644 --- a/src/secrets/runtime.fast-path.test.ts +++ b/src/secrets/runtime.fast-path.test.ts @@ -1,6 +1,12 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; +import { resolveDefaultAgentDir } from "../agents/agent-scope-config.js"; import type { AuthProfileStore } from "../agents/auth-profiles.js"; +import { AUTH_PROFILE_FILENAME } from "../agents/auth-profiles/path-constants.js"; import { clearConfigCache, clearRuntimeConfigSnapshot } from "../config/config.js"; +import { resolveOAuthPath } from "../config/paths.js"; import { createEmptyPluginRegistry } from "../plugins/registry.js"; import { setActivePluginRegistry } from "../plugins/runtime.js"; import { clearSecretsRuntimeSnapshot } from "./runtime.js"; @@ -48,6 +54,23 @@ function requireGatewayAuth( return auth; } +function writeAuthProfileStore(agentDir: string): void { + mkdirSync(agentDir, { recursive: true }); + writeFileSync( + path.join(agentDir, AUTH_PROFILE_FILENAME), + `${JSON.stringify({ + version: 1, + profiles: { + "openai:default": { + type: "api_key", + provider: "openai", + key: "sk-test", + }, + }, + })}\n`, + ); +} + describe("secrets runtime fast path", () => { afterEach(() => { runtimePrepareImportMock.mockClear(); @@ -179,4 +202,140 @@ describe("secrets runtime fast path", () => { expect(resolveRuntimeWebToolsMock).toHaveBeenCalledTimes(1); }); + + it.each([ + { + name: "oauth credentials file", + setup: (env: NodeJS.ProcessEnv, _mainAgentDir: string, _agentDir: string) => { + const credentialsPath = resolveOAuthPath(env); + mkdirSync(path.dirname(credentialsPath), { recursive: true }); + writeFileSync( + credentialsPath, + `${JSON.stringify({ + "openai-codex": { + access: "access-token", + refresh: "refresh-token", + expires: Date.now() + 60_000, + }, + })}\n`, + ); + }, + }, + { + name: "inherited main auth store", + setup: (_env: NodeJS.ProcessEnv, mainAgentDir: string, _agentDir: string) => { + writeAuthProfileStore(mainAgentDir); + }, + }, + ])("skips the startup-only fast path when $name exists", async ({ setup }) => { + const { prepareSecretsRuntimeFastPathSnapshot } = await import("./runtime-fast-path.js"); + const root = mkdtempSync(path.join(tmpdir(), "openclaw-runtime-fast-path-")); + const env: NodeJS.ProcessEnv = { + HOME: root, + OPENCLAW_STATE_DIR: root, + }; + const mainAgentDir = resolveDefaultAgentDir({}, env); + const agentDir = path.join(root, "custom-agent"); + mkdirSync(agentDir, { recursive: true }); + setup(env, mainAgentDir, agentDir); + + try { + const snapshot = prepareSecretsRuntimeFastPathSnapshot({ + config: asConfig({ + agents: { + list: [{ id: "default", agentDir }], + }, + }), + env, + }); + + expect(snapshot).toBeNull(); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it("refreshes startup-only fast-path snapshots from persisted auth stores after startup", async () => { + const { prepareSecretsRuntimeFastPathSnapshot } = await import("./runtime-fast-path.js"); + const { activateSecretsRuntimeSnapshotState, getActiveSecretsRuntimeSnapshot } = + await import("./runtime-state.js"); + const { refreshActiveSecretsRuntimeSnapshot } = await import("./runtime.js"); + const root = mkdtempSync(path.join(tmpdir(), "openclaw-runtime-fast-path-refresh-")); + const env: NodeJS.ProcessEnv = { + HOME: root, + OPENCLAW_STATE_DIR: root, + }; + const agentDir = path.join(root, "custom-agent"); + mkdirSync(agentDir, { recursive: true }); + + try { + const fastPath = prepareSecretsRuntimeFastPathSnapshot({ + config: asConfig({ + agents: { + list: [{ id: "default", agentDir }], + }, + }), + env, + }); + + expect(fastPath).not.toBeNull(); + activateSecretsRuntimeSnapshotState({ + snapshot: fastPath!.snapshot, + refreshContext: fastPath!.refreshContext, + refreshHandler: null, + }); + writeAuthProfileStore(agentDir); + + await expect(refreshActiveSecretsRuntimeSnapshot()).resolves.toBe(true); + const active = getActiveSecretsRuntimeSnapshot(); + expect(active?.authStores[0]?.agentDir).toBe(agentDir); + expect(active?.authStores[0]?.store.profiles["openai:default"]).toMatchObject({ + type: "api_key", + provider: "openai", + key: "sk-test", + }); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + it("pins empty auth stores on startup-only fast-path snapshots until refresh", async () => { + const { ensureAuthProfileStoreWithoutExternalProfiles } = + await import("../agents/auth-profiles/store.js"); + const { prepareSecretsRuntimeFastPathSnapshot } = await import("./runtime-fast-path.js"); + const { activateSecretsRuntimeSnapshotState } = await import("./runtime-state.js"); + const root = mkdtempSync(path.join(tmpdir(), "openclaw-runtime-fast-path-empty-store-")); + const env: NodeJS.ProcessEnv = { + HOME: root, + OPENCLAW_STATE_DIR: root, + }; + const agentDir = path.join(root, "custom-agent"); + mkdirSync(agentDir, { recursive: true }); + + try { + const fastPath = prepareSecretsRuntimeFastPathSnapshot({ + config: asConfig({ + agents: { + list: [{ id: "default", agentDir }], + }, + }), + env, + }); + + expect(fastPath).not.toBeNull(); + expect(fastPath!.snapshot.authStores).toEqual([{ agentDir, store: emptyAuthStore() }]); + activateSecretsRuntimeSnapshotState({ + snapshot: fastPath!.snapshot, + refreshContext: fastPath!.refreshContext, + refreshHandler: null, + }); + writeAuthProfileStore(agentDir); + + expect( + ensureAuthProfileStoreWithoutExternalProfiles(agentDir).profiles["openai:default"], + ).toBeUndefined(); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); }); diff --git a/src/secrets/runtime.ts b/src/secrets/runtime.ts index fcaebf37903..c86cf5e2564 100644 --- a/src/secrets/runtime.ts +++ b/src/secrets/runtime.ts @@ -1,70 +1,40 @@ -import { - listAgentIds, - resolveAgentDir, - resolveAgentWorkspaceDir, - resolveDefaultAgentDir, - resolveDefaultAgentId, -} from "../agents/agent-scope.js"; +import { resolveAgentWorkspaceDir, resolveDefaultAgentId } from "../agents/agent-scope-config.js"; import { clearRuntimeAuthProfileStoreSnapshots, loadAuthProfileStoreForSecretsRuntime, loadAuthProfileStoreWithoutExternalProfiles, - replaceRuntimeAuthProfileStoreSnapshots, } from "../agents/auth-profiles.js"; import type { AuthProfileStore } from "../agents/auth-profiles/types.js"; -import { - clearRuntimeConfigSnapshot, - setRuntimeConfigSnapshotRefreshHandler, - setRuntimeConfigSnapshot, - type OpenClawConfig, -} from "../config/config.js"; -import { coerceSecretRef } from "../config/types.secrets.js"; +import type { OpenClawConfig } from "../config/types.openclaw.js"; import type { PluginOrigin } from "../plugins/plugin-origin.types.js"; import { resolveUserPath } from "../utils.js"; -import { type SecretResolverWarning } from "./runtime-shared.js"; import { - clearActiveRuntimeWebToolsMetadata, - getActiveRuntimeWebToolsMetadata as getActiveRuntimeWebToolsMetadataFromState, - setActiveRuntimeWebToolsMetadata, -} from "./runtime-web-tools-state.js"; -import type { RuntimeWebToolsMetadata } from "./runtime-web-tools.js"; + canUseSecretsRuntimeFastPath, + collectCandidateAgentDirs, + createEmptyRuntimeWebToolsMetadata, + mergeSecretsRuntimeEnv, + resolveRefreshAgentDirs, +} from "./runtime-fast-path.js"; +import { + activateSecretsRuntimeSnapshotState, + clearSecretsRuntimeSnapshot as clearSecretsRuntimeSnapshotState, + getActiveSecretsRuntimeEnv as getActiveSecretsRuntimeEnvState, + getActiveSecretsRuntimeRefreshContext, + getActiveSecretsRuntimeSnapshot as getActiveSecretsRuntimeSnapshotState, + getPreparedSecretsRuntimeSnapshotRefreshContext, + registerSecretsRuntimeStateClearHook, + setPreparedSecretsRuntimeSnapshotRefreshContext, + type PreparedSecretsRuntimeSnapshot, + type SecretsRuntimeRefreshContext, +} from "./runtime-state.js"; +import { getActiveRuntimeWebToolsMetadata as getActiveRuntimeWebToolsMetadataFromState } from "./runtime-web-tools-state.js"; +import type { RuntimeWebToolsMetadata } from "./runtime-web-tools.types.js"; export type { SecretResolverWarning } from "./runtime-shared.js"; +export type { PreparedSecretsRuntimeSnapshot } from "./runtime-state.js"; -export type PreparedSecretsRuntimeSnapshot = { - sourceConfig: OpenClawConfig; - config: OpenClawConfig; - authStores: Array<{ agentDir: string; store: AuthProfileStore }>; - warnings: SecretResolverWarning[]; - webTools: RuntimeWebToolsMetadata; -}; +registerSecretsRuntimeStateClearHook(clearRuntimeAuthProfileStoreSnapshots); -type SecretsRuntimeRefreshContext = { - env: Record; - explicitAgentDirs: string[] | null; - loadAuthStore: (agentDir?: string) => AuthProfileStore; - loadablePluginOrigins: ReadonlyMap; -}; - -const RUNTIME_PATH_ENV_KEYS = [ - "HOME", - "USERPROFILE", - "HOMEDRIVE", - "HOMEPATH", - "OPENCLAW_HOME", - "OPENCLAW_STATE_DIR", - "OPENCLAW_CONFIG_PATH", - "OPENCLAW_AGENT_DIR", - "PI_CODING_AGENT_DIR", - "OPENCLAW_TEST_FAST", -] as const; - -let activeSnapshot: PreparedSecretsRuntimeSnapshot | null = null; -let activeRefreshContext: SecretsRuntimeRefreshContext | null = null; -const preparedSnapshotRefreshContext = new WeakMap< - PreparedSecretsRuntimeSnapshot, - SecretsRuntimeRefreshContext ->(); let runtimeManifestPromise: Promise | null = null; let runtimePreparePromise: Promise | null = null; @@ -78,60 +48,6 @@ function loadRuntimePrepareHelpers() { return runtimePreparePromise; } -function cloneSnapshot(snapshot: PreparedSecretsRuntimeSnapshot): PreparedSecretsRuntimeSnapshot { - return { - sourceConfig: structuredClone(snapshot.sourceConfig), - config: structuredClone(snapshot.config), - authStores: snapshot.authStores.map((entry) => ({ - agentDir: entry.agentDir, - store: structuredClone(entry.store), - })), - warnings: snapshot.warnings.map((warning) => ({ ...warning })), - webTools: structuredClone(snapshot.webTools), - }; -} - -function cloneRefreshContext(context: SecretsRuntimeRefreshContext): SecretsRuntimeRefreshContext { - return { - env: { ...context.env }, - explicitAgentDirs: context.explicitAgentDirs ? [...context.explicitAgentDirs] : null, - loadAuthStore: context.loadAuthStore, - loadablePluginOrigins: new Map(context.loadablePluginOrigins), - }; -} - -function clearActiveSecretsRuntimeState(): void { - activeSnapshot = null; - activeRefreshContext = null; - clearActiveRuntimeWebToolsMetadata(); - setRuntimeConfigSnapshotRefreshHandler(null); - clearRuntimeConfigSnapshot(); - clearRuntimeAuthProfileStoreSnapshots(); -} - -function collectCandidateAgentDirs( - config: OpenClawConfig, - env: NodeJS.ProcessEnv = process.env, -): string[] { - const dirs = new Set(); - dirs.add(resolveUserPath(resolveDefaultAgentDir(config, env), env)); - for (const agentId of listAgentIds(config)) { - dirs.add(resolveUserPath(resolveAgentDir(config, agentId, env), env)); - } - return [...dirs]; -} - -function resolveRefreshAgentDirs( - config: OpenClawConfig, - context: SecretsRuntimeRefreshContext, -): string[] { - const configDerived = collectCandidateAgentDirs(config, context.env); - if (!context.explicitAgentDirs || context.explicitAgentDirs.length === 0) { - return configDerived; - } - return [...new Set([...context.explicitAgentDirs, ...configDerived])]; -} - async function resolveLoadablePluginOrigins(params: { config: OpenClawConfig; env: NodeJS.ProcessEnv; @@ -150,22 +66,6 @@ async function resolveLoadablePluginOrigins(params: { return listPluginOriginsFromMetadataSnapshot(snapshot); } -function mergeSecretsRuntimeEnv( - env: NodeJS.ProcessEnv | Record | undefined, -): Record { - const merged = { ...(env ?? process.env) } as Record; - for (const key of RUNTIME_PATH_ENV_KEYS) { - if (merged[key] !== undefined) { - continue; - } - const processValue = process.env[key]; - if (processValue !== undefined) { - merged[key] = processValue; - } - } - return merged; -} - function hasConfiguredPluginEntries(config: OpenClawConfig): boolean { const entries = config.plugins?.entries; return ( @@ -186,142 +86,6 @@ function hasConfiguredChannelEntries(config: OpenClawConfig): boolean { ); } -function createEmptyRuntimeWebToolsMetadata(): RuntimeWebToolsMetadata { - return { - search: { - providerSource: "none", - diagnostics: [], - }, - fetch: { - providerSource: "none", - diagnostics: [], - }, - diagnostics: [], - }; -} - -const WEB_FETCH_CREDENTIAL_FIELD_NAMES = new Set(["apikey", "key", "token", "secret", "password"]); - -function hasCredentialBearingWebFetchValue( - value: unknown, - defaults: Parameters[1], - seen = new WeakSet(), -): boolean { - if (coerceSecretRef(value, defaults)) { - return true; - } - if (!value || typeof value !== "object") { - return false; - } - if (seen.has(value)) { - return false; - } - seen.add(value); - if (Array.isArray(value)) { - return value.some((entry) => hasCredentialBearingWebFetchValue(entry, defaults, seen)); - } - return Object.entries(value as Record).some(([rawKey, entry]) => { - const key = rawKey.toLowerCase(); - if (WEB_FETCH_CREDENTIAL_FIELD_NAMES.has(key) && entry != null && entry !== "") { - return true; - } - return hasCredentialBearingWebFetchValue(entry, defaults, seen); - }); -} - -function hasActiveRuntimeWebFetchProviderSurface( - fetch: unknown, - defaults: Parameters[1], -): boolean { - if (!fetch || typeof fetch !== "object" || Array.isArray(fetch)) { - return false; - } - const fetchConfig = fetch as Record; - if (fetchConfig.enabled === false) { - return false; - } - if (typeof fetchConfig.provider === "string" && fetchConfig.provider.trim()) { - return true; - } - return hasCredentialBearingWebFetchValue(fetchConfig, defaults); -} - -function hasRuntimeWebToolConfigSurface(config: OpenClawConfig): boolean { - const web = config.tools?.web; - const defaults = config.secrets?.defaults; - const fetchExplicitlyDisabled = - web && - typeof web === "object" && - !Array.isArray(web) && - typeof (web as Record).fetch === "object" && - (web as { fetch?: { enabled?: unknown } }).fetch?.enabled === false; - if (web && typeof web === "object" && !Array.isArray(web)) { - const webRecord = web as Record; - if ("search" in webRecord || "x_search" in webRecord) { - return true; - } - if ( - "fetch" in webRecord && - hasActiveRuntimeWebFetchProviderSurface(webRecord.fetch, defaults) - ) { - return true; - } - } - const entries = config.plugins?.entries; - if (!entries || typeof entries !== "object" || Array.isArray(entries)) { - return false; - } - return Object.values(entries).some((entry) => { - if (!entry || typeof entry !== "object" || Array.isArray(entry)) { - return false; - } - const pluginConfig = (entry as { config?: unknown }).config; - return ( - !!pluginConfig && - typeof pluginConfig === "object" && - !Array.isArray(pluginConfig) && - ("webSearch" in pluginConfig || (!fetchExplicitlyDisabled && "webFetch" in pluginConfig)) - ); - }); -} - -function hasSecretRefCandidate( - value: unknown, - defaults: Parameters[1], - seen = new WeakSet(), -): boolean { - if (coerceSecretRef(value, defaults)) { - return true; - } - if (!value || typeof value !== "object") { - return false; - } - if (seen.has(value)) { - return false; - } - seen.add(value); - if (Array.isArray(value)) { - return value.some((entry) => hasSecretRefCandidate(entry, defaults, seen)); - } - return Object.values(value as Record).some((entry) => - hasSecretRefCandidate(entry, defaults, seen), - ); -} - -function canUseSecretsRuntimeFastPath(params: { - sourceConfig: OpenClawConfig; - authStores: Array<{ agentDir: string; store: AuthProfileStore }>; -}): boolean { - if (hasRuntimeWebToolConfigSurface(params.sourceConfig)) { - return false; - } - const defaults = params.sourceConfig.secrets?.defaults; - if (hasSecretRefCandidate(params.sourceConfig, defaults)) { - return false; - } - return !params.authStores.some((entry) => hasSecretRefCandidate(entry.store, defaults)); -} - export async function prepareSecretsRuntimeSnapshot(params: { config: OpenClawConfig; env?: NodeJS.ProcessEnv; @@ -356,7 +120,7 @@ export async function prepareSecretsRuntimeSnapshot(params: { warnings: [], webTools: createEmptyRuntimeWebToolsMetadata(), }; - preparedSnapshotRefreshContext.set(snapshot, { + setPreparedSecretsRuntimeSnapshotRefreshContext(snapshot, { env: runtimeEnv, explicitAgentDirs: params.agentDirs?.length ? [...candidateDirs] : null, loadAuthStore: fastPathLoadAuthStore, @@ -430,7 +194,7 @@ export async function prepareSecretsRuntimeSnapshot(params: { context, }), }; - preparedSnapshotRefreshContext.set(snapshot, { + setPreparedSecretsRuntimeSnapshotRefreshContext(snapshot, { env: runtimeEnv, explicitAgentDirs: params.agentDirs?.length ? [...candidateDirs] : null, loadAuthStore: params.loadAuthStore ?? loadAuthProfileStoreForSecretsRuntime, @@ -440,40 +204,43 @@ export async function prepareSecretsRuntimeSnapshot(params: { } export function activateSecretsRuntimeSnapshot(snapshot: PreparedSecretsRuntimeSnapshot): void { - const next = cloneSnapshot(snapshot); const refreshContext = - preparedSnapshotRefreshContext.get(snapshot) ?? - activeRefreshContext ?? + getPreparedSecretsRuntimeSnapshotRefreshContext(snapshot) ?? + getActiveSecretsRuntimeRefreshContext() ?? ({ env: { ...process.env } as Record, explicitAgentDirs: null, loadAuthStore: loadAuthProfileStoreForSecretsRuntime, loadablePluginOrigins: new Map(), } satisfies SecretsRuntimeRefreshContext); - setRuntimeConfigSnapshot(next.config, next.sourceConfig); - replaceRuntimeAuthProfileStoreSnapshots(next.authStores); - activeSnapshot = next; - activeRefreshContext = cloneRefreshContext(refreshContext); - setActiveRuntimeWebToolsMetadata(next.webTools); - setRuntimeConfigSnapshotRefreshHandler({ - refresh: async ({ sourceConfig }) => { - if (!activeSnapshot || !activeRefreshContext) { - return false; - } - const refreshed = await prepareSecretsRuntimeSnapshot({ - config: sourceConfig, - env: activeRefreshContext.env, - agentDirs: resolveRefreshAgentDirs(sourceConfig, activeRefreshContext), - loadAuthStore: activeRefreshContext.loadAuthStore, - loadablePluginOrigins: activeRefreshContext.loadablePluginOrigins, - }); - activateSecretsRuntimeSnapshot(refreshed); - return true; + activateSecretsRuntimeSnapshotState({ + snapshot, + refreshContext, + refreshHandler: { + refresh: async ({ sourceConfig }) => { + const activeRefreshContext = getActiveSecretsRuntimeRefreshContext(); + if (!getActiveSecretsRuntimeSnapshotState() || !activeRefreshContext) { + return false; + } + const refreshed = await prepareSecretsRuntimeSnapshot({ + config: sourceConfig, + env: activeRefreshContext.env, + agentDirs: resolveRefreshAgentDirs(sourceConfig, activeRefreshContext), + loadablePluginOrigins: activeRefreshContext.loadablePluginOrigins, + ...(activeRefreshContext.loadAuthStore + ? { loadAuthStore: activeRefreshContext.loadAuthStore } + : {}), + }); + activateSecretsRuntimeSnapshot(refreshed); + return true; + }, }, }); } export async function refreshActiveSecretsRuntimeSnapshot(): Promise { + const activeSnapshot = getActiveSecretsRuntimeSnapshotState(); + const activeRefreshContext = getActiveSecretsRuntimeRefreshContext(); if (!activeSnapshot || !activeRefreshContext) { return false; } @@ -481,28 +248,21 @@ export async function refreshActiveSecretsRuntimeSnapshot(): Promise { config: activeSnapshot.sourceConfig, env: activeRefreshContext.env, agentDirs: resolveRefreshAgentDirs(activeSnapshot.sourceConfig, activeRefreshContext), - loadAuthStore: activeRefreshContext.loadAuthStore, loadablePluginOrigins: activeRefreshContext.loadablePluginOrigins, + ...(activeRefreshContext.loadAuthStore + ? { loadAuthStore: activeRefreshContext.loadAuthStore } + : {}), }); activateSecretsRuntimeSnapshot(refreshed); return true; } export function getActiveSecretsRuntimeSnapshot(): PreparedSecretsRuntimeSnapshot | null { - if (!activeSnapshot) { - return null; - } - const snapshot = cloneSnapshot(activeSnapshot); - if (activeRefreshContext) { - preparedSnapshotRefreshContext.set(snapshot, cloneRefreshContext(activeRefreshContext)); - } - return snapshot; + return getActiveSecretsRuntimeSnapshotState(); } export function getActiveSecretsRuntimeEnv(): NodeJS.ProcessEnv { - return { - ...(activeRefreshContext?.env ?? process.env), - } as NodeJS.ProcessEnv; + return getActiveSecretsRuntimeEnvState(); } export function getActiveRuntimeWebToolsMetadata(): RuntimeWebToolsMetadata | null { @@ -510,5 +270,5 @@ export function getActiveRuntimeWebToolsMetadata(): RuntimeWebToolsMetadata | nu } export function clearSecretsRuntimeSnapshot(): void { - clearActiveSecretsRuntimeState(); + clearSecretsRuntimeSnapshotState(); }