perf: use plugin metadata snapshot for media tool lookups

CHANGELOG.md

@@ -159,6 +159,7 @@ Docs: https://docs.openclaw.ai
 
 ### Fixes
 
+- Agents/tools: route media and generation capability lookups through the Gateway plugin metadata snapshot during reply tool registration, avoiding repeated manifest registry reloads on the live reply path. Thanks @shakkernerd.
 - Agents/tools: reuse the auth profile store already loaded for the active run when deciding media and generation tool availability, avoiding repeated provider-auth runtime discovery during reply startup. Thanks @shakkernerd.
 - Agents/tools: keep image, video, and music generation tool registration on manifest/auth control-plane checks instead of loading runtime provider registries during reply startup, reducing live-path tool-prep blocking while leaving provider runtime resolution for execution and list actions. Thanks @shakkernerd.
 - fix: block workspace CLOUDSDK_PYTHON override and always set trusted interpreter for gcloud. (#74492) Thanks @pgondhi987.

src/agents/tools/image-generate-tool.ts

@@ -576,6 +576,7 @@ export function createImageGenerateTool(options?: {
     !hasGenerationToolAvailability({
       cfg,
       agentDir: options?.agentDir,
+      workspaceDir: options?.workspaceDir,
       authStore: options?.authProfileStore,
       modelConfig: cfg.agents?.defaults?.imageGenerationModel,
       providerKey: "imageGenerationProviders",

src/agents/tools/image-tool.ts

@@ -118,6 +118,7 @@ function resolveImageToolMaxTokens(modelMaxTokens: number | undefined, requested
 export function resolveImageModelConfigForTool(params: {
   cfg?: OpenClawConfig;
   agentDir: string;
+  workspaceDir?: string;
   authStore?: AuthProfileStore;
 }): ImageModelConfig | null {
   // Note: We intentionally do NOT gate based on primarySupportsImages here.
@@ -144,6 +145,7 @@ export function resolveImageModelConfigForTool(params: {
     }
     const providerDefault = imageToolProviderDeps.resolveDefaultMediaModel({
       cfg: params.cfg,
+      workspaceDir: params.workspaceDir,
       providerId: primary.provider,
       capability: "image",
     });
@@ -159,11 +161,13 @@ export function resolveImageModelConfigForTool(params: {
   const autoCandidates = imageToolProviderDeps
     .resolveAutoMediaKeyProviders({
       cfg: params.cfg,
+      workspaceDir: params.workspaceDir,
       capability: "image",
     })
     .map((providerId) => {
       const modelId = imageToolProviderDeps.resolveDefaultMediaModel({
         cfg: params.cfg,
+        workspaceDir: params.workspaceDir,
         providerId,
         capability: "image",
       });
@@ -387,6 +391,7 @@ export function createImageTool(options?: {
   const imageModelConfig = resolveImageModelConfigForTool({
     cfg: options?.config,
     agentDir,
+    workspaceDir: options?.workspaceDir,
     authStore: options?.authProfileStore,
   });
   if (!imageModelConfig) {

src/agents/tools/media-tool-shared.ts

@@ -296,6 +296,7 @@ export function resolveCapabilityModelConfigForTool(params: {
 export function hasGenerationToolAvailability(params: {
   cfg?: OpenClawConfig;
   agentDir?: string;
+  workspaceDir?: string;
   authStore?: AuthProfileStore;
   modelConfig?: AgentModelConfig;
   providers?: CapabilityProvider[] | (() => CapabilityProvider[]);
@@ -319,6 +320,7 @@ export function hasGenerationToolAvailability(params: {
   return resolveBundledCapabilityProviderIds({
     key: params.providerKey,
     cfg: params.cfg,
+    workspaceDir: params.workspaceDir,
   }).some((providerId) =>
     hasAuthForProvider({
       provider: providerId,

src/agents/tools/music-generate-tool.ts

@@ -504,6 +504,7 @@ export function createMusicGenerateTool(options?: {
     !hasGenerationToolAvailability({
       cfg,
       agentDir: options?.agentDir,
+      workspaceDir: options?.workspaceDir,
       authStore: options?.authProfileStore,
       modelConfig: cfg.agents?.defaults?.musicGenerationModel,
       providerKey: "musicGenerationProviders",

src/agents/tools/pdf-tool.model-config.ts

@@ -17,10 +17,15 @@ import { coercePdfModelConfig } from "./pdf-tool.helpers.js";
 function resolveImageCandidateRefs(params: {
   cfg?: OpenClawConfig;
   agentDir: string;
+  workspaceDir?: string;
   authStore?: AuthProfileStore;
   filter?: (providerId: string) => boolean;
 }): string[] {
-  return resolveAutoMediaKeyProviders({ capability: "image", cfg: params.cfg })
+  return resolveAutoMediaKeyProviders({
+    capability: "image",
+    cfg: params.cfg,
+    workspaceDir: params.workspaceDir,
+  })
     .filter((providerId) => !params.filter || params.filter(providerId))
     .filter((providerId) =>
       hasAuthForProvider({
@@ -37,6 +42,7 @@ function resolveImageCandidateRefs(params: {
         })?.split("/")[1] ??
         resolveDefaultMediaModel({
           cfg: params.cfg,
+          workspaceDir: params.workspaceDir,
           providerId,
           capability: "image",
         });
@@ -48,6 +54,7 @@ function resolveImageCandidateRefs(params: {
 export function resolvePdfModelConfigForTool(params: {
   cfg?: OpenClawConfig;
   agentDir: string;
+  workspaceDir?: string;
   authStore?: AuthProfileStore;
 }): ImageModelConfig | null {
   const explicitPdf = coercePdfModelConfig(params.cfg);
@@ -96,22 +103,31 @@ export function resolvePdfModelConfigForTool(params: {
     providerVision?.split("/")[1] ??
     resolveDefaultMediaModel({
       cfg: params.cfg,
+      workspaceDir: params.workspaceDir,
       providerId: primary.provider,
       capability: "image",
     });
   const primarySupportsNativePdf = providerSupportsNativePdfDocument({
     cfg: params.cfg,
+    workspaceDir: params.workspaceDir,
     providerId: primary.provider,
   });
   const nativePdfCandidates = resolveImageCandidateRefs({
     cfg: params.cfg,
     agentDir: params.agentDir,
+    workspaceDir: params.workspaceDir,
     authStore: params.authStore,
-    filter: (providerId) => providerSupportsNativePdfDocument({ cfg: params.cfg, providerId }),
+    filter: (providerId) =>
+      providerSupportsNativePdfDocument({
+        cfg: params.cfg,
+        workspaceDir: params.workspaceDir,
+        providerId,
+      }),
   });
   const genericImageCandidates = resolveImageCandidateRefs({
     cfg: params.cfg,
     agentDir: params.agentDir,
+    workspaceDir: params.workspaceDir,
     authStore: params.authStore,
   });
 

src/agents/tools/pdf-tool.ts

@@ -262,6 +262,7 @@ export function createPdfTool(options?: {
   const pdfModelConfig = resolvePdfModelConfigForTool({
     cfg: options?.config,
     agentDir,
+    workspaceDir: options?.workspaceDir,
     authStore: options?.authProfileStore,
   });
   if (!pdfModelConfig) {

src/agents/tools/video-generate-tool.ts

@@ -811,6 +811,7 @@ export function createVideoGenerateTool(options?: {
     !hasGenerationToolAvailability({
       cfg,
       agentDir: options?.agentDir,
+      workspaceDir: options?.workspaceDir,
       authStore: options?.authProfileStore,
       modelConfig: cfg.agents?.defaults?.videoGenerationModel,
       providerKey: "videoGenerationProviders",

src/media-understanding/defaults.ts

@@ -38,17 +38,17 @@ function cacheConfigRegistry(
   return registry;
 }
 
-function resolveDefaultRegistry(cfg?: OpenClawConfig) {
+function resolveDefaultRegistry(cfg?: OpenClawConfig, workspaceDir?: string) {
   if (!cfg) {
     defaultRegistryCache ??= buildMediaUnderstandingManifestMetadataRegistry();
     return defaultRegistryCache;
   }
-  const cacheKey = resolveRuntimeConfigCacheKey(cfg);
+  const cacheKey = `${resolveRuntimeConfigCacheKey(cfg)}:${workspaceDir ?? ""}`;
   const cached = configRegistryCache.get(cacheKey);
   if (cached) {
     return cached;
   }
-  const registry = buildMediaUnderstandingManifestMetadataRegistry(cfg);
+  const registry = buildMediaUnderstandingManifestMetadataRegistry(cfg, workspaceDir);
   return cacheConfigRegistry(cacheKey, registry);
 }
 
@@ -112,6 +112,7 @@ export function resolveDefaultMediaModel(params: {
   providerId: string;
   capability: MediaUnderstandingCapability;
   cfg?: OpenClawConfig;
+  workspaceDir?: string;
   providerRegistry?: Map<string, MediaUnderstandingProvider>;
 }): string | undefined {
   if (!params.providerRegistry) {
@@ -126,7 +127,8 @@ export function resolveDefaultMediaModel(params: {
       return configuredImageModel;
     }
   }
-  const registry = params.providerRegistry ?? resolveDefaultRegistry(params.cfg);
+  const registry =
+    params.providerRegistry ?? resolveDefaultRegistry(params.cfg, params.workspaceDir);
   const provider = registry.get(normalizeMediaProviderId(params.providerId));
   return normalizeOptionalString(provider?.defaultModels?.[params.capability]);
 }
@@ -134,9 +136,11 @@ export function resolveDefaultMediaModel(params: {
 export function resolveAutoMediaKeyProviders(params: {
   capability: MediaUnderstandingCapability;
   cfg?: OpenClawConfig;
+  workspaceDir?: string;
   providerRegistry?: Map<string, MediaUnderstandingProvider>;
 }): string[] {
-  const registry = params.providerRegistry ?? resolveDefaultRegistry(params.cfg);
+  const registry =
+    params.providerRegistry ?? resolveDefaultRegistry(params.cfg, params.workspaceDir);
   type AutoProviderEntry = {
     provider: MediaUnderstandingProvider;
     priority: number;
@@ -167,9 +171,11 @@ export function resolveAutoMediaKeyProviders(params: {
 export function providerSupportsNativePdfDocument(params: {
   providerId: string;
   cfg?: OpenClawConfig;
+  workspaceDir?: string;
   providerRegistry?: Map<string, MediaUnderstandingProvider>;
 }): boolean {
-  const registry = params.providerRegistry ?? resolveDefaultRegistry(params.cfg);
+  const registry =
+    params.providerRegistry ?? resolveDefaultRegistry(params.cfg, params.workspaceDir);
   const provider = registry.get(normalizeMediaProviderId(params.providerId));
   return provider?.nativeDocumentInputs?.includes("pdf") ?? false;
 }

src/media-understanding/manifest-metadata.ts

@@ -1,17 +1,27 @@
 import type { OpenClawConfig } from "../config/types.js";
+import { getCurrentPluginMetadataSnapshot } from "../plugins/current-plugin-metadata-snapshot.js";
 import { loadPluginManifestRegistryForPluginRegistry } from "../plugins/plugin-registry.js";
 import { normalizeMediaProviderId } from "./provider-id.js";
 import type { MediaUnderstandingProvider } from "./types.js";
 
 export function buildMediaUnderstandingManifestMetadataRegistry(
   cfg?: OpenClawConfig,
+  workspaceDir?: string,
 ): Map<string, MediaUnderstandingProvider> {
   const registry = new Map<string, MediaUnderstandingProvider>();
-  for (const plugin of loadPluginManifestRegistryForPluginRegistry({
+  const snapshot = getCurrentPluginMetadataSnapshot({
     config: cfg,
-    env: process.env,
-    includeDisabled: true,
-  }).plugins) {
+    ...(workspaceDir ? { workspaceDir } : {}),
+  });
+  const plugins =
+    snapshot?.plugins ??
+    loadPluginManifestRegistryForPluginRegistry({
+      config: cfg,
+      env: process.env,
+      includeDisabled: true,
+      ...(workspaceDir ? { workspaceDir } : {}),
+    }).plugins;
+  for (const plugin of plugins) {
     const declaredProviders = new Set(
       (plugin.contracts?.mediaUnderstandingProviders ?? []).map((providerId) =>
         normalizeMediaProviderId(providerId),

src/plugins/capability-provider-runtime.test.ts

@@ -84,6 +84,9 @@ vi.mock("./bundled-compat.js", () => ({
 
 let resolvePluginCapabilityProviders: typeof import("./capability-provider-runtime.js").resolvePluginCapabilityProviders;
 let resolvePluginCapabilityProvider: typeof import("./capability-provider-runtime.js").resolvePluginCapabilityProvider;
+let resolveBundledCapabilityProviderIds: typeof import("./capability-provider-runtime.js").resolveBundledCapabilityProviderIds;
+let clearCurrentPluginMetadataSnapshot: typeof import("./current-plugin-metadata-snapshot.js").clearCurrentPluginMetadataSnapshot;
+let setCurrentPluginMetadataSnapshot: typeof import("./current-plugin-metadata-snapshot.js").setCurrentPluginMetadataSnapshot;
 
 function expectResolvedCapabilityProviderIds(providers: Array<{ id: string }>, expected: string[]) {
   expect(providers.map((provider) => provider.id)).toEqual(expected);
@@ -192,11 +195,17 @@ function expectCompatChainApplied(params: {
 
 describe("resolvePluginCapabilityProviders", () => {
   beforeAll(async () => {
-    ({ resolvePluginCapabilityProvider, resolvePluginCapabilityProviders } =
-      await import("./capability-provider-runtime.js"));
+    ({
+      resolveBundledCapabilityProviderIds,
+      resolvePluginCapabilityProvider,
+      resolvePluginCapabilityProviders,
+    } = await import("./capability-provider-runtime.js"));
+    ({ clearCurrentPluginMetadataSnapshot, setCurrentPluginMetadataSnapshot } =
+      await import("./current-plugin-metadata-snapshot.js"));
   });
 
   beforeEach(() => {
+    clearCurrentPluginMetadataSnapshot();
     mocks.resolveRuntimePluginRegistry.mockReset();
     mocks.resolveRuntimePluginRegistry.mockReturnValue(undefined);
     mocks.resolvePluginRegistryLoadCacheKey.mockReset();
@@ -226,6 +235,52 @@ describe("resolvePluginCapabilityProviders", () => {
     mocks.withBundledPluginVitestCompat.mockImplementation(({ config }) => config);
   });
 
+  it("resolves bundled capability ids from the current metadata snapshot", () => {
+    setCurrentPluginMetadataSnapshot({
+      policyHash: "policy",
+      workspaceDir: "/workspace",
+      index: { plugins: [] },
+      registryDiagnostics: [],
+      manifestRegistry: { plugins: [], diagnostics: [] },
+      plugins: [
+        {
+          id: "fal",
+          origin: "bundled",
+          contracts: { imageGenerationProviders: ["fal"] },
+        },
+      ],
+      diagnostics: [],
+      byPluginId: new Map(),
+      normalizePluginId: (id: string) => id,
+      owners: {
+        channels: new Map(),
+        channelConfigs: new Map(),
+        providers: new Map(),
+        modelCatalogProviders: new Map(),
+        cliBackends: new Map(),
+        setupProviders: new Map(),
+        commandAliases: new Map(),
+        contracts: new Map(),
+      },
+      metrics: {
+        registrySnapshotMs: 0,
+        manifestRegistryMs: 0,
+        ownerMapsMs: 0,
+        totalMs: 0,
+        indexPluginCount: 0,
+        manifestPluginCount: 1,
+      },
+    } as never);
+
+    expect(
+      resolveBundledCapabilityProviderIds({
+        key: "imageGenerationProviders",
+        workspaceDir: "/workspace",
+      }),
+    ).toEqual(["fal"]);
+    expect(mocks.loadPluginManifestRegistry).not.toHaveBeenCalled();
+  });
+
   it("uses the active registry when capability providers are already loaded", () => {
     const active = createEmptyPluginRegistry();
     active.speechProviders.push({

src/plugins/capability-provider-runtime.ts

@@ -5,6 +5,7 @@ import {
   withBundledPluginEnablementCompat,
   withBundledPluginVitestCompat,
 } from "./bundled-compat.js";
+import { getCurrentPluginMetadataSnapshot } from "./current-plugin-metadata-snapshot.js";
 import {
   resolvePluginRegistryLoadCacheKey,
   resolveRuntimePluginRegistry,
@@ -72,16 +73,25 @@ function shouldSkipCapabilityResolution(params: {
 function resolveBundledCapabilityCompatPluginIds(params: {
   key: CapabilityProviderRegistryKey;
   cfg?: OpenClawConfig;
+  workspaceDir?: string;
   providerId?: string;
 }): string[] {
   const env = process.env;
   const contractKey = CAPABILITY_CONTRACT_KEY[params.key];
-  return loadPluginManifestRegistryForPluginRegistry({
+  const snapshot = getCurrentPluginMetadataSnapshot({
     config: params.cfg,
-    env,
-    includeDisabled: true,
-  })
-    .plugins.filter(
+    ...(params.workspaceDir ? { workspaceDir: params.workspaceDir } : {}),
+  });
+  const plugins =
+    snapshot?.plugins ??
+    loadPluginManifestRegistryForPluginRegistry({
+      config: params.cfg,
+      env,
+      includeDisabled: true,
+      ...(params.workspaceDir ? { workspaceDir: params.workspaceDir } : {}),
+    }).plugins;
+  return plugins
+    .filter(
       (plugin) =>
         plugin.origin === "bundled" &&
         (plugin.contracts?.[contractKey]?.length ?? 0) > 0 &&
@@ -94,16 +104,25 @@ function resolveBundledCapabilityCompatPluginIds(params: {
 export function resolveBundledCapabilityProviderIds(params: {
   key: CapabilityProviderRegistryKey;
   cfg?: OpenClawConfig;
+  workspaceDir?: string;
 }): string[] {
   const env = process.env;
   const contractKey = CAPABILITY_CONTRACT_KEY[params.key];
+  const snapshot = getCurrentPluginMetadataSnapshot({
+    config: params.cfg,
+    ...(params.workspaceDir ? { workspaceDir: params.workspaceDir } : {}),
+  });
+  const plugins =
+    snapshot?.plugins ??
+    loadPluginManifestRegistryForPluginRegistry({
+      config: params.cfg,
+      env,
+      includeDisabled: true,
+      ...(params.workspaceDir ? { workspaceDir: params.workspaceDir } : {}),
+    }).plugins;
   return [
     ...new Set(
-      loadPluginManifestRegistryForPluginRegistry({
-        config: params.cfg,
-        env,
-        includeDisabled: true,
-      }).plugins.flatMap((plugin) =>
+      plugins.flatMap((plugin) =>
         plugin.origin === "bundled" ? (plugin.contracts?.[contractKey] ?? []) : [],
       ),
     ),
@@ -113,6 +132,7 @@ export function resolveBundledCapabilityProviderIds(params: {
 function resolveCapabilityProviderConfig(params: {
   key: CapabilityProviderRegistryKey;
   cfg?: OpenClawConfig;
+  workspaceDir?: string;
   pluginIds?: string[];
 }) {
   const pluginIds = params.pluginIds ?? resolveBundledCapabilityCompatPluginIds(params);