From 06de1d20804e260aae876bec92236bc21ab15d43 Mon Sep 17 00:00:00 2001 From: Shakker Date: Mon, 27 Apr 2026 10:57:56 +0100 Subject: [PATCH] fix: reuse web provider candidate manifests --- CHANGELOG.md | 1 + ...provider-public-artifacts.fallback.test.ts | 90 +++++++++++++++++++ src/plugins/web-provider-public-artifacts.ts | 68 ++++++++------ src/plugins/web-provider-resolution-shared.ts | 41 ++++++--- 4 files changed, 165 insertions(+), 35 deletions(-) create mode 100644 src/plugins/web-provider-public-artifacts.fallback.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index eb368f46356..0d0a1a9666b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -64,6 +64,7 @@ Docs: https://docs.openclaw.ai - Plugins/extractors: reuse one manifest registry pass while resolving bundled document and web-content extractor plugins instead of rereading manifests for compatibility and enablement filtering. Thanks @shakkernerd. - Plugins/providers: reuse one plugin registry snapshot and manifest registry while resolving provider discovery entries instead of rebuilding manifest metadata after provider owner discovery. Thanks @shakkernerd. - Plugins/registry: resolve lookup-table owner maps for providers, CLI backends, setup providers, command aliases, model catalogs, channel configs, and manifest contracts while preserving setup-only CLI backend ownership. Thanks @shakkernerd. +- Plugins/web: reuse manifest records already loaded for bundled web provider candidate discovery when falling back to public artifact provider loading. Thanks @shakkernerd. - Mattermost: keep direct-message replies top-level by suppressing reply roots for DM delivery while preserving channel and group thread roots, and derive inbound chat kind from the trusted channel lookup instead of the websocket event channel type. Carries forward #60115, #55186, #72305, and #72659; refs #59758, #59981, #59791, and #57565. Thanks @vincentkoc, @jwchmodx, and @hnykda. - Process/Windows: decode command stdout and stderr from raw bytes with console-codepage awareness, while preserving valid UTF-8 output and multibyte characters split across chunks. Fixes #50519. Thanks @iready, @kevinten10, @zhangyongjie1997, @knightplat-blip, @heiqishi666, and @slepybear. - Bonjour/Windows: hide the bundled mDNS advertiser's Windows ARP shell probe so Gateway startup no longer flashes command-prompt windows. Fixes #70238. Thanks @alexandre-leng, @PratikRai0101, @infinitypacific, and @tomerpeled. diff --git a/src/plugins/web-provider-public-artifacts.fallback.test.ts b/src/plugins/web-provider-public-artifacts.fallback.test.ts new file mode 100644 index 00000000000..7f71aba1ffc --- /dev/null +++ b/src/plugins/web-provider-public-artifacts.fallback.test.ts @@ -0,0 +1,90 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + loadPluginManifestRegistryForPluginRegistry: vi.fn(), + resolveBundledExplicitWebSearchProvidersFromPublicArtifacts: vi.fn(() => null), + resolveBundledExplicitWebFetchProvidersFromPublicArtifacts: vi.fn(() => null), + loadBundledWebSearchProviderEntriesFromDir: vi.fn(), + loadBundledWebFetchProviderEntriesFromDir: vi.fn(), +})); + +vi.mock("./plugin-registry.js", () => ({ + loadPluginManifestRegistryForPluginRegistry: mocks.loadPluginManifestRegistryForPluginRegistry, +})); + +vi.mock("./web-search-providers.shared.js", () => ({ + resolveBundledWebSearchResolutionConfig: (params: { config?: unknown }) => ({ + config: params.config, + }), +})); + +vi.mock("./web-fetch-providers.shared.js", () => ({ + resolveBundledWebFetchResolutionConfig: (params: { config?: unknown }) => ({ + config: params.config, + }), +})); + +vi.mock("./web-provider-public-artifacts.explicit.js", () => ({ + resolveBundledExplicitWebSearchProvidersFromPublicArtifacts: + mocks.resolveBundledExplicitWebSearchProvidersFromPublicArtifacts, + resolveBundledExplicitWebFetchProvidersFromPublicArtifacts: + mocks.resolveBundledExplicitWebFetchProvidersFromPublicArtifacts, + loadBundledWebSearchProviderEntriesFromDir: mocks.loadBundledWebSearchProviderEntriesFromDir, + loadBundledWebFetchProviderEntriesFromDir: mocks.loadBundledWebFetchProviderEntriesFromDir, +})); + +const { + resolveBundledWebFetchProvidersFromPublicArtifacts, + resolveBundledWebSearchProvidersFromPublicArtifacts, +} = await import("./web-provider-public-artifacts.js"); + +describe("web provider public artifact manifest fallback", () => { + beforeEach(() => { + vi.clearAllMocks(); + mocks.loadPluginManifestRegistryForPluginRegistry.mockReturnValue({ + diagnostics: [], + plugins: [ + { + id: "fallback-search", + origin: "bundled", + rootDir: "/tmp/fallback-search", + contracts: { webSearchProviders: ["fallback-search"] }, + }, + { + id: "fallback-fetch", + origin: "bundled", + rootDir: "/tmp/fallback-fetch", + contracts: { webFetchProviders: ["fallback-fetch"] }, + }, + ], + }); + mocks.loadBundledWebSearchProviderEntriesFromDir.mockReturnValue([ + { id: "fallback-search", pluginId: "fallback-search" }, + ]); + mocks.loadBundledWebFetchProviderEntriesFromDir.mockReturnValue([ + { id: "fallback-fetch", pluginId: "fallback-fetch" }, + ]); + }); + + it("reuses the candidate manifest registry for bundled web-search artifact fallback", () => { + const providers = resolveBundledWebSearchProvidersFromPublicArtifacts({ config: {} }); + + expect(providers).toEqual([{ id: "fallback-search", pluginId: "fallback-search" }]); + expect(mocks.loadPluginManifestRegistryForPluginRegistry).toHaveBeenCalledOnce(); + expect(mocks.loadBundledWebSearchProviderEntriesFromDir).toHaveBeenCalledWith({ + dirName: "fallback-search", + pluginId: "fallback-search", + }); + }); + + it("reuses the candidate manifest registry for bundled web-fetch artifact fallback", () => { + const providers = resolveBundledWebFetchProvidersFromPublicArtifacts({ config: {} }); + + expect(providers).toEqual([{ id: "fallback-fetch", pluginId: "fallback-fetch" }]); + expect(mocks.loadPluginManifestRegistryForPluginRegistry).toHaveBeenCalledOnce(); + expect(mocks.loadBundledWebFetchProviderEntriesFromDir).toHaveBeenCalledWith({ + dirName: "fallback-fetch", + pluginId: "fallback-fetch", + }); + }); +}); diff --git a/src/plugins/web-provider-public-artifacts.ts b/src/plugins/web-provider-public-artifacts.ts index 24db1f7fe1f..f7e85df3fbe 100644 --- a/src/plugins/web-provider-public-artifacts.ts +++ b/src/plugins/web-provider-public-artifacts.ts @@ -1,5 +1,6 @@ import path from "node:path"; import type { PluginLoadOptions } from "./loader.js"; +import type { PluginManifestRecord } from "./manifest-registry.js"; import { loadPluginManifestRegistryForPluginRegistry } from "./plugin-registry.js"; import type { PluginWebFetchProviderEntry, PluginWebSearchProviderEntry } from "./types.js"; import { resolveBundledWebFetchResolutionConfig } from "./web-fetch-providers.shared.js"; @@ -9,7 +10,7 @@ import { resolveBundledExplicitWebFetchProvidersFromPublicArtifacts, resolveBundledExplicitWebSearchProvidersFromPublicArtifacts, } from "./web-provider-public-artifacts.explicit.js"; -import { resolveManifestDeclaredWebProviderCandidatePluginIds } from "./web-provider-resolution-shared.js"; +import { resolveManifestDeclaredWebProviderCandidates } from "./web-provider-resolution-shared.js"; import { resolveBundledWebSearchResolutionConfig } from "./web-search-providers.shared.js"; type BundledWebProviderPublicArtifactParams = { @@ -20,6 +21,11 @@ type BundledWebProviderPublicArtifactParams = { onlyPluginIds?: readonly string[]; }; +type BundledCandidateResolution = { + pluginIds: string[]; + manifestRecords?: readonly PluginManifestRecord[]; +}; + function resolveBundledCandidatePluginIds(params: { contract: "webSearchProviders" | "webFetchProviders"; configKey: "webSearch" | "webFetch"; @@ -28,25 +34,31 @@ function resolveBundledCandidatePluginIds(params: { env?: PluginLoadOptions["env"]; bundledAllowlistCompat?: boolean; onlyPluginIds?: readonly string[]; -}): string[] { +}): BundledCandidateResolution { if (params.onlyPluginIds && params.onlyPluginIds.length > 0) { - return [...new Set(params.onlyPluginIds)].toSorted((left, right) => left.localeCompare(right)); + return { + pluginIds: [...new Set(params.onlyPluginIds)].toSorted((left, right) => + left.localeCompare(right), + ), + }; } const resolvedConfig = params.contract === "webSearchProviders" ? resolveBundledWebSearchResolutionConfig(params).config : resolveBundledWebFetchResolutionConfig(params).config; - return ( - resolveManifestDeclaredWebProviderCandidatePluginIds({ - contract: params.contract, - configKey: params.configKey, - config: resolvedConfig, - workspaceDir: params.workspaceDir, - env: params.env, - onlyPluginIds: params.onlyPluginIds, - origin: "bundled", - }) ?? [] - ); + const candidates = resolveManifestDeclaredWebProviderCandidates({ + contract: params.contract, + configKey: params.configKey, + config: resolvedConfig, + workspaceDir: params.workspaceDir, + env: params.env, + onlyPluginIds: params.onlyPluginIds, + origin: "bundled", + }); + return { + pluginIds: candidates.pluginIds ?? [], + ...(candidates.manifestRecords ? { manifestRecords: candidates.manifestRecords } : {}), + }; } function resolveBundledManifestRecordsByPluginId(params: { @@ -54,16 +66,20 @@ function resolveBundledManifestRecordsByPluginId(params: { workspaceDir?: string; env?: PluginLoadOptions["env"]; onlyPluginIds: readonly string[]; + manifestRecords?: readonly PluginManifestRecord[]; }) { const allowedPluginIds = new Set(params.onlyPluginIds); - return new Map( + const manifestRecords = + params.manifestRecords ?? loadPluginManifestRegistryForPluginRegistry({ config: params.config, workspaceDir: params.workspaceDir, env: params.env, includeDisabled: true, - }) - .plugins.filter((record) => record.origin === "bundled" && allowedPluginIds.has(record.id)) + }).plugins; + return new Map( + manifestRecords + .filter((record) => record.origin === "bundled" && allowedPluginIds.has(record.id)) .map((record) => [record.id, record] as const), ); } @@ -80,11 +96,11 @@ export function resolveBundledWebSearchProvidersFromPublicArtifacts( bundledAllowlistCompat: params.bundledAllowlistCompat, onlyPluginIds: params.onlyPluginIds, }); - if (pluginIds.length === 0) { + if (pluginIds.pluginIds.length === 0) { return []; } const directProviders = resolveBundledExplicitWebSearchProvidersFromPublicArtifacts({ - onlyPluginIds: pluginIds, + onlyPluginIds: pluginIds.pluginIds, }); if (directProviders) { return directProviders; @@ -93,10 +109,11 @@ export function resolveBundledWebSearchProvidersFromPublicArtifacts( config: params.config, workspaceDir: params.workspaceDir, env: params.env, - onlyPluginIds: pluginIds, + onlyPluginIds: pluginIds.pluginIds, + manifestRecords: pluginIds.manifestRecords, }); const providers: PluginWebSearchProviderEntry[] = []; - for (const pluginId of pluginIds) { + for (const pluginId of pluginIds.pluginIds) { const record = recordsByPluginId.get(pluginId); if (!record) { return null; @@ -125,11 +142,11 @@ export function resolveBundledWebFetchProvidersFromPublicArtifacts( bundledAllowlistCompat: params.bundledAllowlistCompat, onlyPluginIds: params.onlyPluginIds, }); - if (pluginIds.length === 0) { + if (pluginIds.pluginIds.length === 0) { return []; } const directProviders = resolveBundledExplicitWebFetchProvidersFromPublicArtifacts({ - onlyPluginIds: pluginIds, + onlyPluginIds: pluginIds.pluginIds, }); if (directProviders) { return directProviders; @@ -138,10 +155,11 @@ export function resolveBundledWebFetchProvidersFromPublicArtifacts( config: params.config, workspaceDir: params.workspaceDir, env: params.env, - onlyPluginIds: pluginIds, + onlyPluginIds: pluginIds.pluginIds, + manifestRecords: pluginIds.manifestRecords, }); const providers: PluginWebFetchProviderEntry[] = []; - for (const pluginId of pluginIds) { + for (const pluginId of pluginIds.pluginIds) { const record = recordsByPluginId.get(pluginId); if (!record) { return null; diff --git a/src/plugins/web-provider-resolution-shared.ts b/src/plugins/web-provider-resolution-shared.ts index 6f2ff99cf58..428c500f3b4 100644 --- a/src/plugins/web-provider-resolution-shared.ts +++ b/src/plugins/web-provider-resolution-shared.ts @@ -11,6 +11,11 @@ import { export type WebProviderContract = "webSearchProviders" | "webFetchProviders"; export type WebProviderConfigKey = "webSearch" | "webFetch"; +export type WebProviderCandidateResolution = { + pluginIds: string[] | undefined; + manifestRecords?: readonly PluginManifestRecord[]; +}; + type WebProviderSortEntry = { id: string; pluginId: string; @@ -83,17 +88,33 @@ export function resolveManifestDeclaredWebProviderCandidatePluginIds(params: { onlyPluginIds?: readonly string[]; origin?: PluginManifestRecord["origin"]; }): string[] | undefined { + return resolveManifestDeclaredWebProviderCandidates(params).pluginIds; +} + +export function resolveManifestDeclaredWebProviderCandidates(params: { + contract: WebProviderContract; + configKey: WebProviderConfigKey; + config?: PluginLoadOptions["config"]; + workspaceDir?: string; + env?: PluginLoadOptions["env"]; + onlyPluginIds?: readonly string[]; + origin?: PluginManifestRecord["origin"]; + manifestRecords?: readonly PluginManifestRecord[]; +}): WebProviderCandidateResolution { const scopedPluginIds = normalizePluginIdScope(params.onlyPluginIds); if (scopedPluginIds?.length === 0) { - return []; + return { pluginIds: [] }; } const onlyPluginIdSet = createPluginIdScopeSet(scopedPluginIds); - const ids = loadInstalledWebProviderManifestRecords({ - config: params.config, - workspaceDir: params.workspaceDir, - env: params.env, - pluginIds: scopedPluginIds, - }) + const manifestRecords = + params.manifestRecords ?? + loadInstalledWebProviderManifestRecords({ + config: params.config, + workspaceDir: params.workspaceDir, + env: params.env, + pluginIds: scopedPluginIds, + }); + const ids = manifestRecords .filter( (plugin) => (!params.origin || plugin.origin === params.origin) && @@ -103,12 +124,12 @@ export function resolveManifestDeclaredWebProviderCandidatePluginIds(params: { .map((plugin) => plugin.id) .toSorted((left, right) => left.localeCompare(right)); if (ids.length > 0) { - return ids; + return { pluginIds: ids, manifestRecords }; } if (params.origin || scopedPluginIds !== undefined) { - return []; + return { pluginIds: [], manifestRecords }; } - return undefined; + return { pluginIds: undefined, manifestRecords }; } function resolveBundledWebProviderCompatPluginIds(params: {