From 4e7de4b5c9ed0c85aec430cbac54f8f8118e0d06 Mon Sep 17 00:00:00 2001 From: Shakker Date: Mon, 27 Apr 2026 17:18:12 +0100 Subject: [PATCH] feat: reuse current plugin metadata for provider discovery --- CHANGELOG.md | 1 + docs/plugins/architecture.md | 2 + ...els-config.applies-config-env-vars.test.ts | 64 +++++++++- src/agents/models-config.plan.ts | 19 ++- .../models-config.providers.implicit.ts | 8 ++ src/agents/models-config.ts | 15 +++ src/gateway/server.impl.ts | 9 +- .../current-plugin-metadata-snapshot.test.ts | 114 ++++++++++++++++++ .../current-plugin-metadata-snapshot.ts | 73 +++++++++++ src/plugins/installed-plugin-index-store.ts | 3 + 10 files changed, 305 insertions(+), 3 deletions(-) create mode 100644 src/plugins/current-plugin-metadata-snapshot.test.ts create mode 100644 src/plugins/current-plugin-metadata-snapshot.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 641405b1629..b7846c22e15 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ Docs: https://docs.openclaw.ai - Plugins/models: wire manifest `modelCatalog.aliases` and `modelCatalog.suppressions` into model-catalog planning and built-in model suppression, with OpenAI stale Spark suppression now declared in the plugin manifest before runtime fallback. Thanks @shakkernerd. - Channels/Yuanbao: register the Tencent Yuanbao external channel plugin (`openclaw-plugin-yuanbao`) in the official channel catalog, contract suites, and community plugin docs, with a new `docs/channels/yuanbao.md` quick-start guide for WebSocket bot DMs and group chats. (#72756) Thanks @loongfay. - Channels/QQBot: add full group chat support (history tracking, @-mention gating, activation modes, per-group config, FIFO message queue with deliver debounce), C2C `stream_messages` streaming with a `StreamingController` lifecycle manager, unified `sendMedia` with chunked upload for large files, and refactor the engine into pipeline stages, focused outbound submodules, builtin slash-command modules, and explicit DI ports via `createEngineAdapters()`. (#70624) Thanks @cxyhhhhh. +- Gateway/runtime: reuse the current plugin metadata snapshot for provider discovery so repeated model-provider discovery avoids rebuilding plugin manifest metadata. Thanks @shakkernerd. - Gateway/startup: pass the plugin metadata snapshot from config validation into plugin bootstrap so startup reuses one manifest product instead of rebuilding plugin metadata. Thanks @shakkernerd. ## 2026.4.26 diff --git a/docs/plugins/architecture.md b/docs/plugins/architecture.md index 21969c4a30f..faf364c68e9 100644 --- a/docs/plugins/architecture.md +++ b/docs/plugins/architecture.md @@ -151,6 +151,8 @@ Gateway startup builds one `PluginMetadataSnapshot` for the current config snaps Plugin-aware config validation, startup auto-enable, and Gateway plugin bootstrap consume that snapshot instead of rebuilding manifest/index metadata independently. `PluginLookUpTable` is derived from the same snapshot and adds the startup plugin plan for the current runtime config. +After startup, Gateway keeps the current metadata snapshot as a replaceable runtime product. Repeated runtime provider discovery can borrow that snapshot instead of reconstructing the installed index and manifest registry for each provider-catalog pass. The snapshot is cleared or replaced on Gateway shutdown, config/plugin inventory changes, and installed index writes; callers fall back to the cold manifest/index path when no compatible current snapshot exists. + The snapshot and lookup table keep repeated startup decisions on the fast path: - channel ownership diff --git a/src/agents/models-config.applies-config-env-vars.test.ts b/src/agents/models-config.applies-config-env-vars.test.ts index ffd6151af7f..ba6d4d46039 100644 --- a/src/agents/models-config.applies-config-env-vars.test.ts +++ b/src/agents/models-config.applies-config-env-vars.test.ts @@ -1,8 +1,12 @@ import { describe, expect, it } from "vitest"; import type { OpenClawConfig } from "../config/config.js"; import { createConfigRuntimeEnv } from "../config/env-vars.js"; +import type { PluginMetadataSnapshot } from "../plugins/plugin-metadata-snapshot.js"; import { unsetEnv, withTempEnv } from "./models-config.e2e-harness.js"; -import { resolveProvidersForModelsJsonWithDeps } from "./models-config.plan.js"; +import { + planOpenClawModelsJsonWithDeps, + resolveProvidersForModelsJsonWithDeps, +} from "./models-config.plan.js"; import type { ProviderConfig } from "./models-config.providers.secrets.js"; const TEST_ENV_VAR = "OPENCLAW_MODELS_CONFIG_TEST_ENV"; @@ -72,6 +76,64 @@ async function resolveProvidersAndCaptureDiscoveryEnv(cfg: OpenClawConfig) { } describe("models-config", () => { + it("threads plugin metadata snapshots into implicit provider discovery", async () => { + const pluginMetadataSnapshot = { + index: { plugins: [] }, + manifestRegistry: { plugins: [], diagnostics: [] }, + owners: { providers: new Map() }, + } as unknown as Pick; + let observedSnapshot: + | Pick + | undefined; + + await resolveProvidersForModelsJsonWithDeps( + { + cfg: { models: { providers: {} } }, + agentDir: "/tmp/openclaw-models-config-env-vars-test", + env: {}, + pluginMetadataSnapshot, + }, + { + resolveImplicitProviders: async ({ pluginMetadataSnapshot: receivedSnapshot }) => { + observedSnapshot = receivedSnapshot; + return {}; + }, + }, + ); + + expect(observedSnapshot).toBe(pluginMetadataSnapshot); + }); + + it("threads plugin metadata snapshots through models.json planning", async () => { + const pluginMetadataSnapshot = { + index: { plugins: [] }, + manifestRegistry: { plugins: [], diagnostics: [] }, + owners: { providers: new Map() }, + } as unknown as Pick; + let observedSnapshot: + | Pick + | undefined; + + await planOpenClawModelsJsonWithDeps( + { + cfg: { models: { providers: {} } }, + agentDir: "/tmp/openclaw-models-config-env-vars-test", + env: {}, + existingRaw: "", + existingParsed: null, + pluginMetadataSnapshot, + }, + { + resolveImplicitProviders: async ({ pluginMetadataSnapshot: receivedSnapshot }) => { + observedSnapshot = receivedSnapshot; + return {}; + }, + }, + ); + + expect(observedSnapshot).toBe(pluginMetadataSnapshot); + }); + it("uses config env.vars entries for implicit provider discovery without mutating process.env", async () => { await withTempEnv(["OPENROUTER_API_KEY", TEST_ENV_VAR], async () => { unsetEnv(["OPENROUTER_API_KEY", TEST_ENV_VAR]); diff --git a/src/agents/models-config.plan.ts b/src/agents/models-config.plan.ts index aedd8667759..34db89dca48 100644 --- a/src/agents/models-config.plan.ts +++ b/src/agents/models-config.plan.ts @@ -1,4 +1,5 @@ import type { OpenClawConfig } from "../config/types.openclaw.js"; +import type { PluginMetadataSnapshot } from "../plugins/plugin-metadata-snapshot.js"; import { isRecord } from "../utils.js"; import { mergeProviders, @@ -19,6 +20,7 @@ export type ResolveImplicitProvidersForModelsJson = (params: { config: OpenClawConfig; env: NodeJS.ProcessEnv; explicitProviders: Record; + pluginMetadataSnapshot?: Pick; }) => Promise>; export type ModelsJsonPlan = @@ -38,6 +40,7 @@ export async function resolveProvidersForModelsJsonWithDeps( cfg: OpenClawConfig; agentDir: string; env: NodeJS.ProcessEnv; + pluginMetadataSnapshot?: Pick; }, deps?: { resolveImplicitProviders?: ResolveImplicitProvidersForModelsJson; @@ -51,6 +54,9 @@ export async function resolveProvidersForModelsJsonWithDeps( config: cfg, env, explicitProviders, + ...(params.pluginMetadataSnapshot + ? { pluginMetadataSnapshot: params.pluginMetadataSnapshot } + : {}), }); return mergeProviders({ implicit: implicitProviders, @@ -90,13 +96,24 @@ export async function planOpenClawModelsJsonWithDeps( env: NodeJS.ProcessEnv; existingRaw: string; existingParsed: unknown; + pluginMetadataSnapshot?: Pick; }, deps?: { resolveImplicitProviders?: ResolveImplicitProvidersForModelsJson; }, ): Promise { const { cfg, agentDir, env } = params; - const providers = await resolveProvidersForModelsJsonWithDeps({ cfg, agentDir, env }, deps); + const providers = await resolveProvidersForModelsJsonWithDeps( + { + cfg, + agentDir, + env, + ...(params.pluginMetadataSnapshot + ? { pluginMetadataSnapshot: params.pluginMetadataSnapshot } + : {}), + }, + deps, + ); if (Object.keys(providers).length === 0) { return { action: "skip" }; diff --git a/src/agents/models-config.providers.implicit.ts b/src/agents/models-config.providers.implicit.ts index 635fd2fed74..0a40411997a 100644 --- a/src/agents/models-config.providers.implicit.ts +++ b/src/agents/models-config.providers.implicit.ts @@ -1,6 +1,7 @@ import type { OpenClawConfig } from "../config/types.openclaw.js"; import { formatErrorMessage } from "../infra/errors.js"; import { createSubsystemLogger } from "../logging/subsystem.js"; +import type { PluginMetadataSnapshot } from "../plugins/plugin-metadata-snapshot.js"; import { groupPluginDiscoveryProvidersByOrder, normalizePluginDiscoveryResult, @@ -43,6 +44,7 @@ type ImplicitProviderParams = { env?: NodeJS.ProcessEnv; workspaceDir?: string; explicitProviders?: Record | null; + pluginMetadataSnapshot?: Pick; }; type ImplicitProviderContext = ImplicitProviderParams & { @@ -367,7 +369,13 @@ export async function resolveImplicitProviders( config: params.config, workspaceDir: params.workspaceDir, env, + resolveOwners: params.pluginMetadataSnapshot + ? (provider) => params.pluginMetadataSnapshot?.owners.providers.get(provider) + : undefined, }), + ...(params.pluginMetadataSnapshot + ? { pluginMetadataSnapshot: params.pluginMetadataSnapshot } + : {}), }); for (const order of PLUGIN_DISCOVERY_ORDERS) { diff --git a/src/agents/models-config.ts b/src/agents/models-config.ts index 153be13fe1c..5535db704db 100644 --- a/src/agents/models-config.ts +++ b/src/agents/models-config.ts @@ -7,6 +7,9 @@ import { type OpenClawConfig, } from "../config/config.js"; import { createConfigRuntimeEnv } from "../config/env-vars.js"; +import { getCurrentPluginMetadataSnapshot } from "../plugins/current-plugin-metadata-snapshot.js"; +import { resolveInstalledManifestRegistryIndexFingerprint } from "../plugins/manifest-registry-installed.js"; +import type { PluginMetadataSnapshot } from "../plugins/plugin-metadata-snapshot.js"; import { resolveOpenClawAgentDir } from "./agent-paths.js"; import { MODELS_JSON_STATE } from "./models-config-state.js"; import { planOpenClawModelsJson } from "./models-config.plan.js"; @@ -41,18 +44,23 @@ async function buildModelsJsonFingerprint(params: { config: OpenClawConfig; sourceConfigForSecrets: OpenClawConfig; agentDir: string; + pluginMetadataSnapshot?: Pick; }): Promise { const authProfilesMtimeMs = await readFileMtimeMs( path.join(params.agentDir, "auth-profiles.json"), ); const modelsFileMtimeMs = await readFileMtimeMs(path.join(params.agentDir, "models.json")); const envShape = createConfigRuntimeEnv(params.config, {}); + const pluginMetadataSnapshotIndexFingerprint = params.pluginMetadataSnapshot + ? resolveInstalledManifestRegistryIndexFingerprint(params.pluginMetadataSnapshot.index) + : undefined; return stableStringify({ config: params.config, sourceConfigForSecrets: params.sourceConfigForSecrets, envShape, authProfilesMtimeMs, modelsFileMtimeMs, + pluginMetadataSnapshotIndexFingerprint, }); } @@ -138,15 +146,21 @@ async function withModelsJsonWriteLock(targetPath: string, run: () => Promise export async function ensureOpenClawModelsJson( config?: OpenClawConfig, agentDirOverride?: string, + options: { + pluginMetadataSnapshot?: Pick; + } = {}, ): Promise<{ agentDir: string; wrote: boolean }> { const resolved = resolveModelsConfigInput(config); const cfg = resolved.config; + const pluginMetadataSnapshot = + options.pluginMetadataSnapshot ?? getCurrentPluginMetadataSnapshot({ config: cfg }); const agentDir = agentDirOverride?.trim() ? agentDirOverride.trim() : resolveOpenClawAgentDir(); const targetPath = path.join(agentDir, "models.json"); const fingerprint = await buildModelsJsonFingerprint({ config: cfg, sourceConfigForSecrets: resolved.sourceConfigForSecrets, agentDir, + ...(pluginMetadataSnapshot ? { pluginMetadataSnapshot } : {}), }); const cached = MODELS_JSON_STATE.readyCache.get(targetPath); if (cached) { @@ -169,6 +183,7 @@ export async function ensureOpenClawModelsJson( env, existingRaw: existingModelsFile.raw, existingParsed: existingModelsFile.parsed, + ...(pluginMetadataSnapshot ? { pluginMetadataSnapshot } : {}), }); if (plan.action === "skip") { diff --git a/src/gateway/server.impl.ts b/src/gateway/server.impl.ts index 0a40c466da7..f26855d4371 100644 --- a/src/gateway/server.impl.ts +++ b/src/gateway/server.impl.ts @@ -31,6 +31,10 @@ import type { VoiceWakeRoutingConfig } from "../infra/voicewake-routing.js"; import { startDiagnosticHeartbeat, stopDiagnosticHeartbeat } from "../logging/diagnostic.js"; import { createSubsystemLogger, runtimeForLogger } from "../logging/subsystem.js"; import { getActiveBundledRuntimeDepsInstallCount } from "../plugins/bundled-runtime-deps-activity.js"; +import { + clearCurrentPluginMetadataSnapshot, + setCurrentPluginMetadataSnapshot, +} from "../plugins/current-plugin-metadata-snapshot.js"; import { runGlobalGatewayStopSafely } from "../plugins/hook-runner-global.js"; import type { PluginRuntime } from "../plugins/runtime/types.js"; import { getTotalQueueSize } from "../process/command-queue.js"; @@ -424,6 +428,7 @@ export async function startGatewayServer( pluginLookUpTable, baseMethods, } = pluginBootstrap; + setCurrentPluginMetadataSnapshot(pluginLookUpTable, { config: gatewayPluginConfigAtStart }); if (pluginLookUpTable) { const metrics = pluginLookUpTable.metrics; startupTrace.detail("plugins.lookup-table", [ @@ -640,7 +645,8 @@ export async function startGatewayServer( }); deps.cron = runtimeState.cronState.cron; - const runClosePrelude = async () => + const runClosePrelude = async () => { + clearCurrentPluginMetadataSnapshot(); await runGatewayClosePrelude({ ...(diagnosticsEnabled ? { stopDiagnostics: stopDiagnosticHeartbeat } : {}), clearSkillsRefreshTimer: () => { @@ -658,6 +664,7 @@ export async function startGatewayServer( clearSecretsRuntimeSnapshot, closeMcpServer: closeMcpLoopbackServerOnDemand, }); + }; const { getRuntimeSnapshot, startChannels, startChannel, stopChannel, markChannelLoggedOut } = channelManager; const createCloseHandler = () => diff --git a/src/plugins/current-plugin-metadata-snapshot.test.ts b/src/plugins/current-plugin-metadata-snapshot.test.ts new file mode 100644 index 00000000000..4a1a5bf2f0e --- /dev/null +++ b/src/plugins/current-plugin-metadata-snapshot.test.ts @@ -0,0 +1,114 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { + clearCurrentPluginMetadataSnapshot, + getCurrentPluginMetadataSnapshot, + setCurrentPluginMetadataSnapshot, +} from "./current-plugin-metadata-snapshot.js"; +import { resolveInstalledPluginIndexPolicyHash } from "./installed-plugin-index-policy.js"; +import { writePersistedInstalledPluginIndexSync } from "./installed-plugin-index-store.js"; +import type { PluginMetadataSnapshot } from "./plugin-metadata-snapshot.js"; + +function createSnapshot( + params: { + config?: Parameters[0]; + workspaceDir?: string; + } = {}, +): PluginMetadataSnapshot { + return { + policyHash: resolveInstalledPluginIndexPolicyHash(params.config), + ...(params.workspaceDir ? { workspaceDir: params.workspaceDir } : {}), + index: { + version: 1, + hostContractVersion: "test", + compatRegistryVersion: "test", + migrationVersion: 1, + policyHash: resolveInstalledPluginIndexPolicyHash(params.config), + generatedAtMs: 1, + installRecords: {}, + plugins: [], + diagnostics: [], + }, + registryDiagnostics: [], + manifestRegistry: { plugins: [], diagnostics: [] }, + plugins: [], + diagnostics: [], + byPluginId: new Map(), + normalizePluginId: (pluginId) => pluginId, + owners: { + channels: new Map(), + channelConfigs: new Map(), + providers: new Map(), + modelCatalogProviders: new Map(), + cliBackends: new Map(), + setupProviders: new Map(), + commandAliases: new Map(), + contracts: new Map(), + }, + metrics: { + registrySnapshotMs: 0, + manifestRegistryMs: 0, + ownerMapsMs: 0, + totalMs: 0, + indexPluginCount: 0, + manifestPluginCount: 0, + }, + }; +} + +describe("current plugin metadata snapshot", () => { + it("returns the current snapshot only for matching config policy and workspace", () => { + const config = { plugins: { allow: ["demo"] } }; + const snapshot = createSnapshot({ config, workspaceDir: "/workspace/a" }); + setCurrentPluginMetadataSnapshot(snapshot, { config }); + + expect(getCurrentPluginMetadataSnapshot({ config, workspaceDir: "/workspace/a" })).toBe( + snapshot, + ); + expect(getCurrentPluginMetadataSnapshot({ config })).toBe(snapshot); + expect( + getCurrentPluginMetadataSnapshot({ + config: { plugins: { allow: ["other"] } }, + workspaceDir: "/workspace/a", + }), + ).toBeUndefined(); + expect( + getCurrentPluginMetadataSnapshot({ config, workspaceDir: "/workspace/b" }), + ).toBeUndefined(); + }); + + it("rejects a current snapshot when plugin load paths change", () => { + const config = { plugins: { load: { paths: ["/plugins/one"] } } }; + const snapshot = createSnapshot({ config }); + setCurrentPluginMetadataSnapshot(snapshot, { config }); + + expect(getCurrentPluginMetadataSnapshot({ config })).toBe(snapshot); + expect( + getCurrentPluginMetadataSnapshot({ + config: { plugins: { load: { paths: ["/plugins/two"] } } }, + }), + ).toBeUndefined(); + }); + + it("clears the current snapshot", () => { + setCurrentPluginMetadataSnapshot(createSnapshot()); + clearCurrentPluginMetadataSnapshot(); + + expect(getCurrentPluginMetadataSnapshot()).toBeUndefined(); + }); + + it("clears the current snapshot when the persisted installed index changes", () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-plugin-metadata-")); + try { + setCurrentPluginMetadataSnapshot(createSnapshot()); + + writePersistedInstalledPluginIndexSync(createSnapshot().index, { stateDir: tempDir }); + + expect(getCurrentPluginMetadataSnapshot()).toBeUndefined(); + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + } + }); +}); diff --git a/src/plugins/current-plugin-metadata-snapshot.ts b/src/plugins/current-plugin-metadata-snapshot.ts new file mode 100644 index 00000000000..63d6ac98306 --- /dev/null +++ b/src/plugins/current-plugin-metadata-snapshot.ts @@ -0,0 +1,73 @@ +import type { OpenClawConfig } from "../config/types.openclaw.js"; +import { resolveInstalledPluginIndexPolicyHash } from "./installed-plugin-index-policy.js"; +import type { PluginMetadataSnapshot } from "./plugin-metadata-snapshot.js"; + +let currentPluginMetadataSnapshot: PluginMetadataSnapshot | undefined; +let currentPluginMetadataSnapshotConfigFingerprint: string | undefined; + +function normalizeLoadPaths(config: OpenClawConfig | undefined): readonly string[] { + const paths = config?.plugins?.load?.paths; + if (!Array.isArray(paths)) { + return []; + } + return paths.filter((entry) => typeof entry === "string"); +} + +export function resolvePluginMetadataSnapshotConfigFingerprint( + config: OpenClawConfig | undefined, +): string { + return JSON.stringify({ + policyHash: resolveInstalledPluginIndexPolicyHash(config), + pluginLoadPaths: normalizeLoadPaths(config), + }); +} + +// Single-slot Gateway-owned handoff. Replace or clear it at lifecycle boundaries; +// never accumulate historical metadata snapshots here. +export function setCurrentPluginMetadataSnapshot( + snapshot: PluginMetadataSnapshot | undefined, + options: { config?: OpenClawConfig } = {}, +): void { + currentPluginMetadataSnapshot = snapshot; + currentPluginMetadataSnapshotConfigFingerprint = snapshot + ? resolvePluginMetadataSnapshotConfigFingerprint(options.config) + : undefined; +} + +export function clearCurrentPluginMetadataSnapshot(): void { + currentPluginMetadataSnapshot = undefined; + currentPluginMetadataSnapshotConfigFingerprint = undefined; +} + +export function getCurrentPluginMetadataSnapshot( + params: { + config?: OpenClawConfig; + workspaceDir?: string; + } = {}, +): PluginMetadataSnapshot | undefined { + const snapshot = currentPluginMetadataSnapshot; + if (!snapshot) { + return undefined; + } + if ( + params.config && + snapshot.policyHash !== resolveInstalledPluginIndexPolicyHash(params.config) + ) { + return undefined; + } + if ( + params.config && + currentPluginMetadataSnapshotConfigFingerprint && + currentPluginMetadataSnapshotConfigFingerprint !== + resolvePluginMetadataSnapshotConfigFingerprint(params.config) + ) { + return undefined; + } + if ( + params.workspaceDir !== undefined && + (snapshot.workspaceDir ?? "") !== (params.workspaceDir ?? "") + ) { + return undefined; + } + return snapshot; +} diff --git a/src/plugins/installed-plugin-index-store.ts b/src/plugins/installed-plugin-index-store.ts index 45edc0201d3..f9230b7eaa9 100644 --- a/src/plugins/installed-plugin-index-store.ts +++ b/src/plugins/installed-plugin-index-store.ts @@ -3,6 +3,7 @@ import { saveJsonFile } from "../infra/json-file.js"; import { readJsonFile, readJsonFileSync, writeJsonAtomic } from "../infra/json-files.js"; import { isBlockedObjectKey } from "../infra/prototype-keys.js"; import { safeParseWithSchema } from "../utils/zod-parse.js"; +import { clearCurrentPluginMetadataSnapshot } from "./current-plugin-metadata-snapshot.js"; import { resolveInstalledPluginIndexStorePath, type InstalledPluginIndexStoreOptions, @@ -171,6 +172,7 @@ export async function writePersistedInstalledPluginIndex( mode: 0o600, }, ); + clearCurrentPluginMetadataSnapshot(); return filePath; } @@ -180,6 +182,7 @@ export function writePersistedInstalledPluginIndexSync( ): string { const filePath = resolveInstalledPluginIndexStorePath(options); saveJsonFile(filePath, { ...index, warning: INSTALLED_PLUGIN_INDEX_WARNING }); + clearCurrentPluginMetadataSnapshot(); return filePath; }