From 1a6d89113233434f5b0ed26ac582549de83e55fb Mon Sep 17 00:00:00 2001 From: Shakker Date: Fri, 1 May 2026 19:00:32 +0100 Subject: [PATCH] perf: use plugin metadata snapshot for media tool lookups --- CHANGELOG.md | 1 + src/agents/tools/image-generate-tool.ts | 1 + src/agents/tools/image-tool.ts | 5 ++ src/agents/tools/media-tool-shared.ts | 2 + src/agents/tools/music-generate-tool.ts | 1 + src/agents/tools/pdf-tool.model-config.ts | 20 ++++++- src/agents/tools/pdf-tool.ts | 1 + src/agents/tools/video-generate-tool.ts | 1 + src/media-understanding/defaults.ts | 18 ++++-- src/media-understanding/manifest-metadata.ts | 18 ++++-- .../capability-provider-runtime.test.ts | 59 ++++++++++++++++++- src/plugins/capability-provider-runtime.ts | 40 +++++++++---- 12 files changed, 143 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b549d80d5b2..e0c2a640907 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -159,6 +159,7 @@ Docs: https://docs.openclaw.ai ### Fixes +- Agents/tools: route media and generation capability lookups through the Gateway plugin metadata snapshot during reply tool registration, avoiding repeated manifest registry reloads on the live reply path. Thanks @shakkernerd. - Agents/tools: reuse the auth profile store already loaded for the active run when deciding media and generation tool availability, avoiding repeated provider-auth runtime discovery during reply startup. Thanks @shakkernerd. - Agents/tools: keep image, video, and music generation tool registration on manifest/auth control-plane checks instead of loading runtime provider registries during reply startup, reducing live-path tool-prep blocking while leaving provider runtime resolution for execution and list actions. Thanks @shakkernerd. - fix: block workspace CLOUDSDK_PYTHON override and always set trusted interpreter for gcloud. (#74492) Thanks @pgondhi987. diff --git a/src/agents/tools/image-generate-tool.ts b/src/agents/tools/image-generate-tool.ts index 8ac453b349e..fc0d3372a5f 100644 --- a/src/agents/tools/image-generate-tool.ts +++ b/src/agents/tools/image-generate-tool.ts @@ -576,6 +576,7 @@ export function createImageGenerateTool(options?: { !hasGenerationToolAvailability({ cfg, agentDir: options?.agentDir, + workspaceDir: options?.workspaceDir, authStore: options?.authProfileStore, modelConfig: cfg.agents?.defaults?.imageGenerationModel, providerKey: "imageGenerationProviders", diff --git a/src/agents/tools/image-tool.ts b/src/agents/tools/image-tool.ts index ba49ace7862..2f459701ffa 100644 --- a/src/agents/tools/image-tool.ts +++ b/src/agents/tools/image-tool.ts @@ -118,6 +118,7 @@ function resolveImageToolMaxTokens(modelMaxTokens: number | undefined, requested export function resolveImageModelConfigForTool(params: { cfg?: OpenClawConfig; agentDir: string; + workspaceDir?: string; authStore?: AuthProfileStore; }): ImageModelConfig | null { // Note: We intentionally do NOT gate based on primarySupportsImages here. @@ -144,6 +145,7 @@ export function resolveImageModelConfigForTool(params: { } const providerDefault = imageToolProviderDeps.resolveDefaultMediaModel({ cfg: params.cfg, + workspaceDir: params.workspaceDir, providerId: primary.provider, capability: "image", }); @@ -159,11 +161,13 @@ export function resolveImageModelConfigForTool(params: { const autoCandidates = imageToolProviderDeps .resolveAutoMediaKeyProviders({ cfg: params.cfg, + workspaceDir: params.workspaceDir, capability: "image", }) .map((providerId) => { const modelId = imageToolProviderDeps.resolveDefaultMediaModel({ cfg: params.cfg, + workspaceDir: params.workspaceDir, providerId, capability: "image", }); @@ -387,6 +391,7 @@ export function createImageTool(options?: { const imageModelConfig = resolveImageModelConfigForTool({ cfg: options?.config, agentDir, + workspaceDir: options?.workspaceDir, authStore: options?.authProfileStore, }); if (!imageModelConfig) { diff --git a/src/agents/tools/media-tool-shared.ts b/src/agents/tools/media-tool-shared.ts index 2abf4c5a31d..2e406340273 100644 --- a/src/agents/tools/media-tool-shared.ts +++ b/src/agents/tools/media-tool-shared.ts @@ -296,6 +296,7 @@ export function resolveCapabilityModelConfigForTool(params: { export function hasGenerationToolAvailability(params: { cfg?: OpenClawConfig; agentDir?: string; + workspaceDir?: string; authStore?: AuthProfileStore; modelConfig?: AgentModelConfig; providers?: CapabilityProvider[] | (() => CapabilityProvider[]); @@ -319,6 +320,7 @@ export function hasGenerationToolAvailability(params: { return resolveBundledCapabilityProviderIds({ key: params.providerKey, cfg: params.cfg, + workspaceDir: params.workspaceDir, }).some((providerId) => hasAuthForProvider({ provider: providerId, diff --git a/src/agents/tools/music-generate-tool.ts b/src/agents/tools/music-generate-tool.ts index e05eab9dedb..2ea14aa26b1 100644 --- a/src/agents/tools/music-generate-tool.ts +++ b/src/agents/tools/music-generate-tool.ts @@ -504,6 +504,7 @@ export function createMusicGenerateTool(options?: { !hasGenerationToolAvailability({ cfg, agentDir: options?.agentDir, + workspaceDir: options?.workspaceDir, authStore: options?.authProfileStore, modelConfig: cfg.agents?.defaults?.musicGenerationModel, providerKey: "musicGenerationProviders", diff --git a/src/agents/tools/pdf-tool.model-config.ts b/src/agents/tools/pdf-tool.model-config.ts index 398031f55d2..8a32a1da401 100644 --- a/src/agents/tools/pdf-tool.model-config.ts +++ b/src/agents/tools/pdf-tool.model-config.ts @@ -17,10 +17,15 @@ import { coercePdfModelConfig } from "./pdf-tool.helpers.js"; function resolveImageCandidateRefs(params: { cfg?: OpenClawConfig; agentDir: string; + workspaceDir?: string; authStore?: AuthProfileStore; filter?: (providerId: string) => boolean; }): string[] { - return resolveAutoMediaKeyProviders({ capability: "image", cfg: params.cfg }) + return resolveAutoMediaKeyProviders({ + capability: "image", + cfg: params.cfg, + workspaceDir: params.workspaceDir, + }) .filter((providerId) => !params.filter || params.filter(providerId)) .filter((providerId) => hasAuthForProvider({ @@ -37,6 +42,7 @@ function resolveImageCandidateRefs(params: { })?.split("/")[1] ?? resolveDefaultMediaModel({ cfg: params.cfg, + workspaceDir: params.workspaceDir, providerId, capability: "image", }); @@ -48,6 +54,7 @@ function resolveImageCandidateRefs(params: { export function resolvePdfModelConfigForTool(params: { cfg?: OpenClawConfig; agentDir: string; + workspaceDir?: string; authStore?: AuthProfileStore; }): ImageModelConfig | null { const explicitPdf = coercePdfModelConfig(params.cfg); @@ -96,22 +103,31 @@ export function resolvePdfModelConfigForTool(params: { providerVision?.split("/")[1] ?? resolveDefaultMediaModel({ cfg: params.cfg, + workspaceDir: params.workspaceDir, providerId: primary.provider, capability: "image", }); const primarySupportsNativePdf = providerSupportsNativePdfDocument({ cfg: params.cfg, + workspaceDir: params.workspaceDir, providerId: primary.provider, }); const nativePdfCandidates = resolveImageCandidateRefs({ cfg: params.cfg, agentDir: params.agentDir, + workspaceDir: params.workspaceDir, authStore: params.authStore, - filter: (providerId) => providerSupportsNativePdfDocument({ cfg: params.cfg, providerId }), + filter: (providerId) => + providerSupportsNativePdfDocument({ + cfg: params.cfg, + workspaceDir: params.workspaceDir, + providerId, + }), }); const genericImageCandidates = resolveImageCandidateRefs({ cfg: params.cfg, agentDir: params.agentDir, + workspaceDir: params.workspaceDir, authStore: params.authStore, }); diff --git a/src/agents/tools/pdf-tool.ts b/src/agents/tools/pdf-tool.ts index 261cb32773a..89c36b9dbdc 100644 --- a/src/agents/tools/pdf-tool.ts +++ b/src/agents/tools/pdf-tool.ts @@ -262,6 +262,7 @@ export function createPdfTool(options?: { const pdfModelConfig = resolvePdfModelConfigForTool({ cfg: options?.config, agentDir, + workspaceDir: options?.workspaceDir, authStore: options?.authProfileStore, }); if (!pdfModelConfig) { diff --git a/src/agents/tools/video-generate-tool.ts b/src/agents/tools/video-generate-tool.ts index 9d9612311ec..5048cae5c7a 100644 --- a/src/agents/tools/video-generate-tool.ts +++ b/src/agents/tools/video-generate-tool.ts @@ -811,6 +811,7 @@ export function createVideoGenerateTool(options?: { !hasGenerationToolAvailability({ cfg, agentDir: options?.agentDir, + workspaceDir: options?.workspaceDir, authStore: options?.authProfileStore, modelConfig: cfg.agents?.defaults?.videoGenerationModel, providerKey: "videoGenerationProviders", diff --git a/src/media-understanding/defaults.ts b/src/media-understanding/defaults.ts index f071fd36103..e101a3ffb55 100644 --- a/src/media-understanding/defaults.ts +++ b/src/media-understanding/defaults.ts @@ -38,17 +38,17 @@ function cacheConfigRegistry( return registry; } -function resolveDefaultRegistry(cfg?: OpenClawConfig) { +function resolveDefaultRegistry(cfg?: OpenClawConfig, workspaceDir?: string) { if (!cfg) { defaultRegistryCache ??= buildMediaUnderstandingManifestMetadataRegistry(); return defaultRegistryCache; } - const cacheKey = resolveRuntimeConfigCacheKey(cfg); + const cacheKey = `${resolveRuntimeConfigCacheKey(cfg)}:${workspaceDir ?? ""}`; const cached = configRegistryCache.get(cacheKey); if (cached) { return cached; } - const registry = buildMediaUnderstandingManifestMetadataRegistry(cfg); + const registry = buildMediaUnderstandingManifestMetadataRegistry(cfg, workspaceDir); return cacheConfigRegistry(cacheKey, registry); } @@ -112,6 +112,7 @@ export function resolveDefaultMediaModel(params: { providerId: string; capability: MediaUnderstandingCapability; cfg?: OpenClawConfig; + workspaceDir?: string; providerRegistry?: Map; }): string | undefined { if (!params.providerRegistry) { @@ -126,7 +127,8 @@ export function resolveDefaultMediaModel(params: { return configuredImageModel; } } - const registry = params.providerRegistry ?? resolveDefaultRegistry(params.cfg); + const registry = + params.providerRegistry ?? resolveDefaultRegistry(params.cfg, params.workspaceDir); const provider = registry.get(normalizeMediaProviderId(params.providerId)); return normalizeOptionalString(provider?.defaultModels?.[params.capability]); } @@ -134,9 +136,11 @@ export function resolveDefaultMediaModel(params: { export function resolveAutoMediaKeyProviders(params: { capability: MediaUnderstandingCapability; cfg?: OpenClawConfig; + workspaceDir?: string; providerRegistry?: Map; }): string[] { - const registry = params.providerRegistry ?? resolveDefaultRegistry(params.cfg); + const registry = + params.providerRegistry ?? resolveDefaultRegistry(params.cfg, params.workspaceDir); type AutoProviderEntry = { provider: MediaUnderstandingProvider; priority: number; @@ -167,9 +171,11 @@ export function resolveAutoMediaKeyProviders(params: { export function providerSupportsNativePdfDocument(params: { providerId: string; cfg?: OpenClawConfig; + workspaceDir?: string; providerRegistry?: Map; }): boolean { - const registry = params.providerRegistry ?? resolveDefaultRegistry(params.cfg); + const registry = + params.providerRegistry ?? resolveDefaultRegistry(params.cfg, params.workspaceDir); const provider = registry.get(normalizeMediaProviderId(params.providerId)); return provider?.nativeDocumentInputs?.includes("pdf") ?? false; } diff --git a/src/media-understanding/manifest-metadata.ts b/src/media-understanding/manifest-metadata.ts index 0df3b4cb149..be77e09c6e7 100644 --- a/src/media-understanding/manifest-metadata.ts +++ b/src/media-understanding/manifest-metadata.ts @@ -1,17 +1,27 @@ import type { OpenClawConfig } from "../config/types.js"; +import { getCurrentPluginMetadataSnapshot } from "../plugins/current-plugin-metadata-snapshot.js"; import { loadPluginManifestRegistryForPluginRegistry } from "../plugins/plugin-registry.js"; import { normalizeMediaProviderId } from "./provider-id.js"; import type { MediaUnderstandingProvider } from "./types.js"; export function buildMediaUnderstandingManifestMetadataRegistry( cfg?: OpenClawConfig, + workspaceDir?: string, ): Map { const registry = new Map(); - for (const plugin of loadPluginManifestRegistryForPluginRegistry({ + const snapshot = getCurrentPluginMetadataSnapshot({ config: cfg, - env: process.env, - includeDisabled: true, - }).plugins) { + ...(workspaceDir ? { workspaceDir } : {}), + }); + const plugins = + snapshot?.plugins ?? + loadPluginManifestRegistryForPluginRegistry({ + config: cfg, + env: process.env, + includeDisabled: true, + ...(workspaceDir ? { workspaceDir } : {}), + }).plugins; + for (const plugin of plugins) { const declaredProviders = new Set( (plugin.contracts?.mediaUnderstandingProviders ?? []).map((providerId) => normalizeMediaProviderId(providerId), diff --git a/src/plugins/capability-provider-runtime.test.ts b/src/plugins/capability-provider-runtime.test.ts index 3820b3df0f0..cd78e67f328 100644 --- a/src/plugins/capability-provider-runtime.test.ts +++ b/src/plugins/capability-provider-runtime.test.ts @@ -84,6 +84,9 @@ vi.mock("./bundled-compat.js", () => ({ let resolvePluginCapabilityProviders: typeof import("./capability-provider-runtime.js").resolvePluginCapabilityProviders; let resolvePluginCapabilityProvider: typeof import("./capability-provider-runtime.js").resolvePluginCapabilityProvider; +let resolveBundledCapabilityProviderIds: typeof import("./capability-provider-runtime.js").resolveBundledCapabilityProviderIds; +let clearCurrentPluginMetadataSnapshot: typeof import("./current-plugin-metadata-snapshot.js").clearCurrentPluginMetadataSnapshot; +let setCurrentPluginMetadataSnapshot: typeof import("./current-plugin-metadata-snapshot.js").setCurrentPluginMetadataSnapshot; function expectResolvedCapabilityProviderIds(providers: Array<{ id: string }>, expected: string[]) { expect(providers.map((provider) => provider.id)).toEqual(expected); @@ -192,11 +195,17 @@ function expectCompatChainApplied(params: { describe("resolvePluginCapabilityProviders", () => { beforeAll(async () => { - ({ resolvePluginCapabilityProvider, resolvePluginCapabilityProviders } = - await import("./capability-provider-runtime.js")); + ({ + resolveBundledCapabilityProviderIds, + resolvePluginCapabilityProvider, + resolvePluginCapabilityProviders, + } = await import("./capability-provider-runtime.js")); + ({ clearCurrentPluginMetadataSnapshot, setCurrentPluginMetadataSnapshot } = + await import("./current-plugin-metadata-snapshot.js")); }); beforeEach(() => { + clearCurrentPluginMetadataSnapshot(); mocks.resolveRuntimePluginRegistry.mockReset(); mocks.resolveRuntimePluginRegistry.mockReturnValue(undefined); mocks.resolvePluginRegistryLoadCacheKey.mockReset(); @@ -226,6 +235,52 @@ describe("resolvePluginCapabilityProviders", () => { mocks.withBundledPluginVitestCompat.mockImplementation(({ config }) => config); }); + it("resolves bundled capability ids from the current metadata snapshot", () => { + setCurrentPluginMetadataSnapshot({ + policyHash: "policy", + workspaceDir: "/workspace", + index: { plugins: [] }, + registryDiagnostics: [], + manifestRegistry: { plugins: [], diagnostics: [] }, + plugins: [ + { + id: "fal", + origin: "bundled", + contracts: { imageGenerationProviders: ["fal"] }, + }, + ], + diagnostics: [], + byPluginId: new Map(), + normalizePluginId: (id: string) => id, + owners: { + channels: new Map(), + channelConfigs: new Map(), + providers: new Map(), + modelCatalogProviders: new Map(), + cliBackends: new Map(), + setupProviders: new Map(), + commandAliases: new Map(), + contracts: new Map(), + }, + metrics: { + registrySnapshotMs: 0, + manifestRegistryMs: 0, + ownerMapsMs: 0, + totalMs: 0, + indexPluginCount: 0, + manifestPluginCount: 1, + }, + } as never); + + expect( + resolveBundledCapabilityProviderIds({ + key: "imageGenerationProviders", + workspaceDir: "/workspace", + }), + ).toEqual(["fal"]); + expect(mocks.loadPluginManifestRegistry).not.toHaveBeenCalled(); + }); + it("uses the active registry when capability providers are already loaded", () => { const active = createEmptyPluginRegistry(); active.speechProviders.push({ diff --git a/src/plugins/capability-provider-runtime.ts b/src/plugins/capability-provider-runtime.ts index de0daaabcce..a77d68132a2 100644 --- a/src/plugins/capability-provider-runtime.ts +++ b/src/plugins/capability-provider-runtime.ts @@ -5,6 +5,7 @@ import { withBundledPluginEnablementCompat, withBundledPluginVitestCompat, } from "./bundled-compat.js"; +import { getCurrentPluginMetadataSnapshot } from "./current-plugin-metadata-snapshot.js"; import { resolvePluginRegistryLoadCacheKey, resolveRuntimePluginRegistry, @@ -72,16 +73,25 @@ function shouldSkipCapabilityResolution(params: { function resolveBundledCapabilityCompatPluginIds(params: { key: CapabilityProviderRegistryKey; cfg?: OpenClawConfig; + workspaceDir?: string; providerId?: string; }): string[] { const env = process.env; const contractKey = CAPABILITY_CONTRACT_KEY[params.key]; - return loadPluginManifestRegistryForPluginRegistry({ + const snapshot = getCurrentPluginMetadataSnapshot({ config: params.cfg, - env, - includeDisabled: true, - }) - .plugins.filter( + ...(params.workspaceDir ? { workspaceDir: params.workspaceDir } : {}), + }); + const plugins = + snapshot?.plugins ?? + loadPluginManifestRegistryForPluginRegistry({ + config: params.cfg, + env, + includeDisabled: true, + ...(params.workspaceDir ? { workspaceDir: params.workspaceDir } : {}), + }).plugins; + return plugins + .filter( (plugin) => plugin.origin === "bundled" && (plugin.contracts?.[contractKey]?.length ?? 0) > 0 && @@ -94,16 +104,25 @@ function resolveBundledCapabilityCompatPluginIds(params: { export function resolveBundledCapabilityProviderIds(params: { key: CapabilityProviderRegistryKey; cfg?: OpenClawConfig; + workspaceDir?: string; }): string[] { const env = process.env; const contractKey = CAPABILITY_CONTRACT_KEY[params.key]; + const snapshot = getCurrentPluginMetadataSnapshot({ + config: params.cfg, + ...(params.workspaceDir ? { workspaceDir: params.workspaceDir } : {}), + }); + const plugins = + snapshot?.plugins ?? + loadPluginManifestRegistryForPluginRegistry({ + config: params.cfg, + env, + includeDisabled: true, + ...(params.workspaceDir ? { workspaceDir: params.workspaceDir } : {}), + }).plugins; return [ ...new Set( - loadPluginManifestRegistryForPluginRegistry({ - config: params.cfg, - env, - includeDisabled: true, - }).plugins.flatMap((plugin) => + plugins.flatMap((plugin) => plugin.origin === "bundled" ? (plugin.contracts?.[contractKey] ?? []) : [], ), ), @@ -113,6 +132,7 @@ export function resolveBundledCapabilityProviderIds(params: { function resolveCapabilityProviderConfig(params: { key: CapabilityProviderRegistryKey; cfg?: OpenClawConfig; + workspaceDir?: string; pluginIds?: string[]; }) { const pluginIds = params.pluginIds ?? resolveBundledCapabilityCompatPluginIds(params);