mirror of
https://github.com/openclaw/openclaw.git
synced 2026-05-06 13:10:43 +00:00
perf(gateway): fast path startup secrets
This commit is contained in:
@@ -71,7 +71,10 @@ describe("gateway startup config recovery", () => {
|
||||
minimalTestGateway: true,
|
||||
log,
|
||||
}),
|
||||
).resolves.toBe(recoveredSnapshot);
|
||||
).resolves.toEqual({
|
||||
snapshot: recoveredSnapshot,
|
||||
wroteConfig: true,
|
||||
});
|
||||
|
||||
expect(configIo.recoverConfigFromLastKnownGood).toHaveBeenCalledWith({
|
||||
snapshot: invalidSnapshot,
|
||||
|
||||
@@ -51,11 +51,17 @@ type GatewayStartupConfigOverrides = {
|
||||
tailscale?: GatewayTailscaleConfig;
|
||||
};
|
||||
|
||||
export type GatewayStartupConfigSnapshotLoadResult = {
|
||||
snapshot: ConfigFileSnapshot;
|
||||
wroteConfig: boolean;
|
||||
};
|
||||
|
||||
export async function loadGatewayStartupConfigSnapshot(params: {
|
||||
minimalTestGateway: boolean;
|
||||
log: GatewayStartupLog;
|
||||
}): Promise<ConfigFileSnapshot> {
|
||||
}): Promise<GatewayStartupConfigSnapshotLoadResult> {
|
||||
let configSnapshot = await readConfigFileSnapshot();
|
||||
let wroteConfig = false;
|
||||
if (configSnapshot.legacyIssues.length > 0 && isNixMode) {
|
||||
throw new Error(
|
||||
"Legacy config entries detected while running in Nix mode. Update your Nix config to the latest schema and restart.",
|
||||
@@ -68,6 +74,7 @@ export async function loadGatewayStartupConfigSnapshot(params: {
|
||||
reason: "startup-invalid-config",
|
||||
});
|
||||
if (recovered) {
|
||||
wroteConfig = true;
|
||||
params.log.warn(
|
||||
`gateway: invalid config was restored from last-known-good backup: ${configSnapshot.path}`,
|
||||
);
|
||||
@@ -89,11 +96,12 @@ export async function loadGatewayStartupConfigSnapshot(params: {
|
||||
? { config: configSnapshot.config, changes: [] as string[] }
|
||||
: applyPluginAutoEnable({ config: configSnapshot.config, env: process.env });
|
||||
if (autoEnable.changes.length === 0) {
|
||||
return configSnapshot;
|
||||
return { snapshot: configSnapshot, wroteConfig };
|
||||
}
|
||||
|
||||
try {
|
||||
await writeConfigFile(autoEnable.config);
|
||||
wroteConfig = true;
|
||||
configSnapshot = await readConfigFileSnapshot();
|
||||
assertValidGatewayStartupConfigSnapshot(configSnapshot);
|
||||
params.log.info(
|
||||
@@ -103,7 +111,7 @@ export async function loadGatewayStartupConfigSnapshot(params: {
|
||||
params.log.warn(`gateway: failed to persist plugin auto-enable changes: ${String(err)}`);
|
||||
}
|
||||
|
||||
return configSnapshot;
|
||||
return { snapshot: configSnapshot, wroteConfig };
|
||||
}
|
||||
|
||||
export function createRuntimeSecretsActivator(params: {
|
||||
|
||||
@@ -252,12 +252,13 @@ export async function startGatewayServer(
|
||||
});
|
||||
const startupTrace = createGatewayStartupTrace();
|
||||
|
||||
const configSnapshot = await startupTrace.measure("config.snapshot", () =>
|
||||
const startupConfigLoad = await startupTrace.measure("config.snapshot", () =>
|
||||
loadGatewayStartupConfigSnapshot({
|
||||
minimalTestGateway,
|
||||
log,
|
||||
}),
|
||||
);
|
||||
const configSnapshot = startupConfigLoad.snapshot;
|
||||
|
||||
const emitSecretsStateEvent = (
|
||||
code: "SECRETS_RELOADER_DEGRADED" | "SECRETS_RELOADER_RECOVERED",
|
||||
@@ -322,14 +323,14 @@ export async function startGatewayServer(
|
||||
}),
|
||||
);
|
||||
cfgAtStart = controlUiSeed.config;
|
||||
// Always capture the final config hash after all startup writes (plugin
|
||||
// auto-enable, auth token generation, control-UI origin seeding) so the
|
||||
// config reloader can recognize its own startup writes and suppress the
|
||||
// spurious hot-reload that would otherwise trigger a SIGUSR1 restart loop.
|
||||
// Previously the hash was only captured when auth or control-UI persisted
|
||||
// changes, missing the plugin auto-enable write performed earlier inside
|
||||
// loadGatewayStartupConfigSnapshot(). See #67436.
|
||||
{
|
||||
// Capture the final config hash only after startup writes (plugin auto-enable,
|
||||
// auth token generation, control-UI origin seeding) so the config reloader can
|
||||
// suppress its own persistence events without rereading config on every boot.
|
||||
if (
|
||||
startupConfigLoad.wroteConfig ||
|
||||
authBootstrap.persistedGeneratedToken ||
|
||||
controlUiSeed.persistedAllowedOriginsSeed
|
||||
) {
|
||||
const startupSnapshot = await startupTrace.measure("config.final-snapshot", () =>
|
||||
readConfigFileSnapshot(),
|
||||
);
|
||||
|
||||
96
src/secrets/runtime.fast-path.test.ts
Normal file
96
src/secrets/runtime.fast-path.test.ts
Normal file
@@ -0,0 +1,96 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import type { AuthProfileStore } from "../agents/auth-profiles.js";
|
||||
import { clearConfigCache, clearRuntimeConfigSnapshot } from "../config/config.js";
|
||||
import { createEmptyPluginRegistry } from "../plugins/registry.js";
|
||||
import { setActivePluginRegistry } from "../plugins/runtime.js";
|
||||
import { clearSecretsRuntimeSnapshot } from "./runtime.js";
|
||||
import { asConfig } from "./runtime.test-support.js";
|
||||
|
||||
const runtimePrepareImportMock = vi.hoisted(() => vi.fn());
|
||||
|
||||
vi.mock("./runtime-prepare.runtime.js", () => {
|
||||
runtimePrepareImportMock();
|
||||
return {
|
||||
createResolverContext: ({ sourceConfig, env }: { sourceConfig: unknown; env: unknown }) => ({
|
||||
sourceConfig,
|
||||
env,
|
||||
cache: {},
|
||||
warnings: [],
|
||||
warningKeys: new Set<string>(),
|
||||
assignments: [],
|
||||
}),
|
||||
collectConfigAssignments: () => undefined,
|
||||
collectAuthStoreAssignments: () => undefined,
|
||||
resolveSecretRefValues: async () => new Map(),
|
||||
applyResolvedAssignments: () => undefined,
|
||||
resolveRuntimeWebTools: async () => ({
|
||||
search: { providerSource: "none", diagnostics: [] },
|
||||
fetch: { providerSource: "none", diagnostics: [] },
|
||||
diagnostics: [],
|
||||
}),
|
||||
};
|
||||
});
|
||||
|
||||
function emptyAuthStore(): AuthProfileStore {
|
||||
return { version: 1, profiles: {} };
|
||||
}
|
||||
|
||||
describe("secrets runtime fast path", () => {
|
||||
afterEach(() => {
|
||||
runtimePrepareImportMock.mockClear();
|
||||
setActivePluginRegistry(createEmptyPluginRegistry());
|
||||
clearSecretsRuntimeSnapshot();
|
||||
clearRuntimeConfigSnapshot();
|
||||
clearConfigCache();
|
||||
vi.resetModules();
|
||||
});
|
||||
|
||||
it("skips heavy resolver loading when config and auth stores have no SecretRefs", async () => {
|
||||
const { prepareSecretsRuntimeSnapshot } = await import("./runtime.js");
|
||||
|
||||
const snapshot = await prepareSecretsRuntimeSnapshot({
|
||||
config: asConfig({
|
||||
gateway: {
|
||||
auth: {
|
||||
mode: "token",
|
||||
token: "plain-startup-token",
|
||||
},
|
||||
},
|
||||
}),
|
||||
env: {},
|
||||
agentDirs: ["/tmp/openclaw-agent-main"],
|
||||
loadAuthStore: emptyAuthStore,
|
||||
});
|
||||
|
||||
expect(runtimePrepareImportMock).not.toHaveBeenCalled();
|
||||
expect(snapshot.config.gateway?.auth?.token).toBe("plain-startup-token");
|
||||
expect(snapshot.authStores).toEqual([
|
||||
{
|
||||
agentDir: "/tmp/openclaw-agent-main",
|
||||
store: emptyAuthStore(),
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("uses the resolver path when an auth profile store contains a SecretRef", async () => {
|
||||
const { prepareSecretsRuntimeSnapshot } = await import("./runtime.js");
|
||||
|
||||
await prepareSecretsRuntimeSnapshot({
|
||||
config: asConfig({}),
|
||||
env: {},
|
||||
agentDirs: ["/tmp/openclaw-agent-main"],
|
||||
loadAuthStore: () => ({
|
||||
version: 1,
|
||||
profiles: {
|
||||
"openai:default": {
|
||||
type: "api_key",
|
||||
provider: "openai",
|
||||
keyRef: { source: "env", provider: "default", id: "OPENAI_API_KEY" },
|
||||
},
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
expect(runtimePrepareImportMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
@@ -8,6 +8,7 @@ import {
|
||||
import {
|
||||
clearRuntimeAuthProfileStoreSnapshots,
|
||||
loadAuthProfileStoreForSecretsRuntime,
|
||||
loadAuthProfileStoreWithoutExternalProfiles,
|
||||
replaceRuntimeAuthProfileStoreSnapshots,
|
||||
} from "../agents/auth-profiles.js";
|
||||
import type { AuthProfileStore } from "../agents/auth-profiles/types.js";
|
||||
@@ -17,6 +18,7 @@ import {
|
||||
setRuntimeConfigSnapshot,
|
||||
type OpenClawConfig,
|
||||
} from "../config/config.js";
|
||||
import { coerceSecretRef } from "../config/types.secrets.js";
|
||||
import type { PluginOrigin } from "../plugins/plugin-origin.types.js";
|
||||
import { resolveUserPath } from "../utils.js";
|
||||
import { type SecretResolverWarning } from "./runtime-shared.js";
|
||||
@@ -174,6 +176,80 @@ function hasConfiguredPluginEntries(config: OpenClawConfig): boolean {
|
||||
);
|
||||
}
|
||||
|
||||
function createEmptyRuntimeWebToolsMetadata(): RuntimeWebToolsMetadata {
|
||||
return {
|
||||
search: {
|
||||
providerSource: "none",
|
||||
diagnostics: [],
|
||||
},
|
||||
fetch: {
|
||||
providerSource: "none",
|
||||
diagnostics: [],
|
||||
},
|
||||
diagnostics: [],
|
||||
};
|
||||
}
|
||||
|
||||
function hasRuntimeWebToolConfigSurface(config: OpenClawConfig): boolean {
|
||||
const web = config.tools?.web;
|
||||
if (web && typeof web === "object" && ("search" in web || "fetch" in web || "x_search" in web)) {
|
||||
return true;
|
||||
}
|
||||
const entries = config.plugins?.entries;
|
||||
if (!entries || typeof entries !== "object" || Array.isArray(entries)) {
|
||||
return false;
|
||||
}
|
||||
return Object.values(entries).some((entry) => {
|
||||
if (!entry || typeof entry !== "object" || Array.isArray(entry)) {
|
||||
return false;
|
||||
}
|
||||
const pluginConfig = (entry as { config?: unknown }).config;
|
||||
return (
|
||||
!!pluginConfig &&
|
||||
typeof pluginConfig === "object" &&
|
||||
!Array.isArray(pluginConfig) &&
|
||||
("webSearch" in pluginConfig || "webFetch" in pluginConfig)
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
function hasSecretRefCandidate(
|
||||
value: unknown,
|
||||
defaults: Parameters<typeof coerceSecretRef>[1],
|
||||
seen = new WeakSet<object>(),
|
||||
): boolean {
|
||||
if (coerceSecretRef(value, defaults)) {
|
||||
return true;
|
||||
}
|
||||
if (!value || typeof value !== "object") {
|
||||
return false;
|
||||
}
|
||||
if (seen.has(value)) {
|
||||
return false;
|
||||
}
|
||||
seen.add(value);
|
||||
if (Array.isArray(value)) {
|
||||
return value.some((entry) => hasSecretRefCandidate(entry, defaults, seen));
|
||||
}
|
||||
return Object.values(value as Record<string, unknown>).some((entry) =>
|
||||
hasSecretRefCandidate(entry, defaults, seen),
|
||||
);
|
||||
}
|
||||
|
||||
function canUseSecretsRuntimeFastPath(params: {
|
||||
sourceConfig: OpenClawConfig;
|
||||
authStores: Array<{ agentDir: string; store: AuthProfileStore }>;
|
||||
}): boolean {
|
||||
if (hasRuntimeWebToolConfigSurface(params.sourceConfig)) {
|
||||
return false;
|
||||
}
|
||||
const defaults = params.sourceConfig.secrets?.defaults;
|
||||
if (hasSecretRefCandidate(params.sourceConfig, defaults)) {
|
||||
return false;
|
||||
}
|
||||
return !params.authStores.some((entry) => hasSecretRefCandidate(entry.store, defaults));
|
||||
}
|
||||
|
||||
export async function prepareSecretsRuntimeSnapshot(params: {
|
||||
config: OpenClawConfig;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
@@ -183,6 +259,40 @@ export async function prepareSecretsRuntimeSnapshot(params: {
|
||||
/** Test override for discovered loadable plugins and their origins. */
|
||||
loadablePluginOrigins?: ReadonlyMap<string, PluginOrigin>;
|
||||
}): Promise<PreparedSecretsRuntimeSnapshot> {
|
||||
const runtimeEnv = mergeSecretsRuntimeEnv(params.env);
|
||||
const sourceConfig = structuredClone(params.config);
|
||||
const resolvedConfig = structuredClone(params.config);
|
||||
const includeAuthStoreRefs = params.includeAuthStoreRefs ?? true;
|
||||
let authStores: Array<{ agentDir: string; store: AuthProfileStore }> = [];
|
||||
const fastPathLoadAuthStore = params.loadAuthStore ?? loadAuthProfileStoreWithoutExternalProfiles;
|
||||
const candidateDirs = params.agentDirs?.length
|
||||
? [...new Set(params.agentDirs.map((entry) => resolveUserPath(entry, runtimeEnv)))]
|
||||
: collectCandidateAgentDirs(resolvedConfig, runtimeEnv);
|
||||
if (includeAuthStoreRefs) {
|
||||
for (const agentDir of candidateDirs) {
|
||||
authStores.push({
|
||||
agentDir,
|
||||
store: structuredClone(fastPathLoadAuthStore(agentDir)),
|
||||
});
|
||||
}
|
||||
}
|
||||
if (canUseSecretsRuntimeFastPath({ sourceConfig, authStores })) {
|
||||
const snapshot = {
|
||||
sourceConfig,
|
||||
config: resolvedConfig,
|
||||
authStores,
|
||||
warnings: [],
|
||||
webTools: createEmptyRuntimeWebToolsMetadata(),
|
||||
};
|
||||
preparedSnapshotRefreshContext.set(snapshot, {
|
||||
env: runtimeEnv,
|
||||
explicitAgentDirs: params.agentDirs?.length ? [...candidateDirs] : null,
|
||||
loadAuthStore: fastPathLoadAuthStore,
|
||||
loadablePluginOrigins: params.loadablePluginOrigins ?? new Map<string, PluginOrigin>(),
|
||||
});
|
||||
return snapshot;
|
||||
}
|
||||
|
||||
const {
|
||||
applyResolvedAssignments,
|
||||
collectAuthStoreAssignments,
|
||||
@@ -191,9 +301,6 @@ export async function prepareSecretsRuntimeSnapshot(params: {
|
||||
resolveRuntimeWebTools,
|
||||
resolveSecretRefValues,
|
||||
} = await loadRuntimePrepareHelpers();
|
||||
const runtimeEnv = mergeSecretsRuntimeEnv(params.env);
|
||||
const sourceConfig = structuredClone(params.config);
|
||||
const resolvedConfig = structuredClone(params.config);
|
||||
const loadablePluginOrigins =
|
||||
params.loadablePluginOrigins ??
|
||||
(hasConfiguredPluginEntries(sourceConfig)
|
||||
@@ -210,21 +317,20 @@ export async function prepareSecretsRuntimeSnapshot(params: {
|
||||
loadablePluginOrigins,
|
||||
});
|
||||
|
||||
const includeAuthStoreRefs = params.includeAuthStoreRefs ?? true;
|
||||
const authStores: Array<{ agentDir: string; store: AuthProfileStore }> = [];
|
||||
const loadAuthStore = params.loadAuthStore ?? loadAuthProfileStoreForSecretsRuntime;
|
||||
const candidateDirs = params.agentDirs?.length
|
||||
? [...new Set(params.agentDirs.map((entry) => resolveUserPath(entry, runtimeEnv)))]
|
||||
: collectCandidateAgentDirs(resolvedConfig, runtimeEnv);
|
||||
if (includeAuthStoreRefs) {
|
||||
for (const agentDir of candidateDirs) {
|
||||
const store = structuredClone(loadAuthStore(agentDir));
|
||||
collectAuthStoreAssignments({
|
||||
store,
|
||||
context,
|
||||
const loadAuthStore = params.loadAuthStore ?? loadAuthProfileStoreForSecretsRuntime;
|
||||
if (!params.loadAuthStore) {
|
||||
authStores = candidateDirs.map((agentDir) => ({
|
||||
agentDir,
|
||||
store: structuredClone(loadAuthStore(agentDir)),
|
||||
}));
|
||||
}
|
||||
for (const entry of authStores) {
|
||||
collectAuthStoreAssignments({
|
||||
store: entry.store,
|
||||
context,
|
||||
agentDir: entry.agentDir,
|
||||
});
|
||||
authStores.push({ agentDir, store });
|
||||
}
|
||||
}
|
||||
|
||||
@@ -255,7 +361,7 @@ export async function prepareSecretsRuntimeSnapshot(params: {
|
||||
preparedSnapshotRefreshContext.set(snapshot, {
|
||||
env: runtimeEnv,
|
||||
explicitAgentDirs: params.agentDirs?.length ? [...candidateDirs] : null,
|
||||
loadAuthStore,
|
||||
loadAuthStore: params.loadAuthStore ?? loadAuthProfileStoreForSecretsRuntime,
|
||||
loadablePluginOrigins,
|
||||
});
|
||||
return snapshot;
|
||||
|
||||
Reference in New Issue
Block a user