From 2baa07f62be378e92cef334fbbcaa74736a499f8 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 2 May 2026 09:54:47 +0100 Subject: [PATCH] refactor: streamline plugin cache helpers --- src/commands/agents.bindings.ts | 14 +---- src/commands/channel-setup/discovery.ts | 14 +---- src/commands/channels/logs.ts | 13 +--- src/media/document-extractors.runtime.ts | 28 ++------- src/plugins/manifest-channel-contributions.ts | 26 ++++++++ src/plugins/plugin-cache-primitives.test.ts | 62 +++++++++++++++++++ src/plugins/plugin-cache-primitives.ts | 44 +++++++++++++ src/plugins/provider-auth-choices.ts | 4 +- src/plugins/provider-discovery.runtime.ts | 4 +- src/plugins/setup-registry.runtime.ts | 4 +- src/plugins/web-provider-resolution-shared.ts | 4 +- src/plugins/web-search-credential-presence.ts | 4 +- src/web-fetch/content-extractors.runtime.ts | 30 ++------- 13 files changed, 161 insertions(+), 90 deletions(-) create mode 100644 src/plugins/manifest-channel-contributions.ts diff --git a/src/commands/agents.bindings.ts b/src/commands/agents.bindings.ts index 3d67e23fa52..25b873bcf19 100644 --- a/src/commands/agents.bindings.ts +++ b/src/commands/agents.bindings.ts @@ -6,10 +6,7 @@ import { normalizeChannelId as normalizeBundledChannelId } from "../channels/reg import { isRouteBinding, listRouteBindings } from "../config/bindings.js"; import type { AgentRouteBinding } from "../config/types.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; -import { - listPluginContributionIds, - loadPluginRegistrySnapshot, -} from "../plugins/plugin-registry.js"; +import { listManifestChannelContributionIds } from "../plugins/manifest-channel-contributions.js"; import { DEFAULT_ACCOUNT_ID, normalizeAgentId } from "../routing/session-key.js"; import { normalizeOptionalString } from "../shared/string-coerce.js"; import { normalizeStringEntries } from "../shared/string-normalization.js"; @@ -220,16 +217,11 @@ function resolveDefaultAccountId(cfg: OpenClawConfig, provider: ChannelId): stri } function listManifestChannelIds(config: OpenClawConfig): Set { - const index = loadPluginRegistrySnapshot({ - config, - env: process.env, - }); return new Set( - listPluginContributionIds({ - index, - contribution: "channels", + listManifestChannelContributionIds({ includeDisabled: true, config, + env: process.env, }), ); } diff --git a/src/commands/channel-setup/discovery.ts b/src/commands/channel-setup/discovery.ts index b991e39b70a..deb3e91e9b2 100644 --- a/src/commands/channel-setup/discovery.ts +++ b/src/commands/channel-setup/discovery.ts @@ -8,10 +8,7 @@ import type { ChannelMeta } from "../../channels/plugins/types.public.js"; import { isStaticallyChannelConfigured } from "../../config/channel-configured-shared.js"; import { applyPluginAutoEnable } from "../../config/plugin-auto-enable.js"; import type { OpenClawConfig } from "../../config/types.openclaw.js"; -import { - listPluginContributionIds, - loadPluginRegistrySnapshot, -} from "../../plugins/plugin-registry.js"; +import { listManifestChannelContributionIds } from "../../plugins/manifest-channel-contributions.js"; import type { ChannelChoice } from "../onboard-types.js"; import { listSetupDiscoveryChannelPluginCatalogEntries, @@ -51,15 +48,8 @@ export function listManifestInstalledChannelIds(params: { env: params.env ?? process.env, }).config; const workspaceDir = resolveWorkspaceDir(resolvedConfig, params.workspaceDir); - const index = loadPluginRegistrySnapshot({ - config: resolvedConfig, - workspaceDir, - env: params.env ?? process.env, - }); return new Set( - listPluginContributionIds({ - index, - contribution: "channels", + listManifestChannelContributionIds({ config: resolvedConfig, workspaceDir, env: params.env ?? process.env, diff --git a/src/commands/channels/logs.ts b/src/commands/channels/logs.ts index 38fdcfa038e..fb776622ad6 100644 --- a/src/commands/channels/logs.ts +++ b/src/commands/channels/logs.ts @@ -3,10 +3,7 @@ import { normalizeChannelId as normalizeBundledChannelId } from "../../channels/ import { getResolvedLoggerSettings } from "../../logging.js"; import { resolveLogFile } from "../../logging/log-tail.js"; import { parseLogLine } from "../../logging/parse-log-line.js"; -import { - listPluginContributionIds, - loadPluginRegistrySnapshot, -} from "../../plugins/plugin-registry.js"; +import { listManifestChannelContributionIds } from "../../plugins/manifest-channel-contributions.js"; import { defaultRuntime, type RuntimeEnv, writeRuntimeJson } from "../../runtime.js"; import { normalizeLowercaseStringOrEmpty } from "../../shared/string-coerce.js"; import { theme } from "../../terminal/theme.js"; @@ -23,14 +20,10 @@ const DEFAULT_LIMIT = 200; const MAX_BYTES = 1_000_000; function listManifestChannelIds(): Set { - const index = loadPluginRegistrySnapshot({ - env: process.env, - }); return new Set( - listPluginContributionIds({ - index, - contribution: "channels", + listManifestChannelContributionIds({ includeDisabled: true, + env: process.env, }), ); } diff --git a/src/media/document-extractors.runtime.ts b/src/media/document-extractors.runtime.ts index e7c0a426386..b1f28e40a31 100644 --- a/src/media/document-extractors.runtime.ts +++ b/src/media/document-extractors.runtime.ts @@ -4,30 +4,12 @@ import type { DocumentExtractionResult, } from "../plugins/document-extractor-types.js"; import { resolvePluginDocumentExtractors } from "../plugins/document-extractors.runtime.js"; +import { createConfigScopedPromiseLoader } from "../plugins/plugin-cache-primitives.js"; import { normalizeLowercaseStringOrEmpty } from "../shared/string-coerce.js"; -let extractorPromise: Promise> | undefined; -const extractorPromisesByConfig = new WeakMap< - OpenClawConfig, - Promise> ->(); - -async function loadDocumentExtractors(config?: OpenClawConfig) { - if (config) { - const cached = extractorPromisesByConfig.get(config); - if (cached) { - return await cached; - } - const promise = Promise.resolve().then(() => resolvePluginDocumentExtractors({ config })); - extractorPromisesByConfig.set(config, promise); - void promise.catch(() => { - extractorPromisesByConfig.delete(config); - }); - return await promise; - } - extractorPromise ??= Promise.resolve(resolvePluginDocumentExtractors()); - return await extractorPromise; -} +const documentExtractorLoader = createConfigScopedPromiseLoader((config?: OpenClawConfig) => + resolvePluginDocumentExtractors(config ? { config } : undefined), +); export async function extractDocumentContent( params: DocumentExtractionRequest & { @@ -35,7 +17,7 @@ export async function extractDocumentContent( }, ): Promise<(DocumentExtractionResult & { extractor: string }) | null> { const mimeType = normalizeLowercaseStringOrEmpty(params.mimeType); - const extractors = await loadDocumentExtractors(params.config); + const extractors = await documentExtractorLoader.load(params.config); const request: DocumentExtractionRequest = { buffer: params.buffer, mimeType: params.mimeType, diff --git a/src/plugins/manifest-channel-contributions.ts b/src/plugins/manifest-channel-contributions.ts new file mode 100644 index 00000000000..fe1c4e4d6ff --- /dev/null +++ b/src/plugins/manifest-channel-contributions.ts @@ -0,0 +1,26 @@ +import type { OpenClawConfig } from "../config/types.openclaw.js"; +import { listPluginContributionIds, loadPluginRegistrySnapshot } from "./plugin-registry.js"; + +export function listManifestChannelContributionIds( + params: { + config?: OpenClawConfig; + workspaceDir?: string; + env?: NodeJS.ProcessEnv; + includeDisabled?: boolean; + } = {}, +): readonly string[] { + const env = params.env ?? process.env; + const index = loadPluginRegistrySnapshot({ + config: params.config, + workspaceDir: params.workspaceDir, + env, + }); + return listPluginContributionIds({ + index, + contribution: "channels", + config: params.config, + workspaceDir: params.workspaceDir, + env, + includeDisabled: params.includeDisabled, + }); +} diff --git a/src/plugins/plugin-cache-primitives.test.ts b/src/plugins/plugin-cache-primitives.test.ts index 3f8e4245422..a32f0a274e7 100644 --- a/src/plugins/plugin-cache-primitives.test.ts +++ b/src/plugins/plugin-cache-primitives.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it, vi } from "vitest"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import { PluginLruCache, + createConfigScopedPromiseLoader, resolveConfigScopedRuntimeCacheValue, type ConfigScopedRuntimeCache, } from "./plugin-cache-primitives.js"; @@ -84,3 +85,64 @@ describe("resolveConfigScopedRuntimeCacheValue", () => { expect(load).toHaveBeenCalledOnce(); }); }); + +describe("createConfigScopedPromiseLoader", () => { + it("dedupes concurrent default loads", async () => { + let calls = 0; + const loader = createConfigScopedPromiseLoader(async () => `loaded-${++calls}`); + + await expect(Promise.all([loader.load(), loader.load()])).resolves.toEqual([ + "loaded-1", + "loaded-1", + ]); + await expect(loader.load()).resolves.toBe("loaded-1"); + expect(calls).toBe(1); + }); + + it("caches loads by config object", async () => { + const firstConfig = { plugins: { load: { disabled: true } } } as OpenClawConfig; + const secondConfig = { plugins: { load: { disabled: false } } } as OpenClawConfig; + const load = vi.fn(async (config?: OpenClawConfig) => + config === firstConfig ? "first" : "second", + ); + const loader = createConfigScopedPromiseLoader(load); + + await expect(loader.load(firstConfig)).resolves.toBe("first"); + await expect(loader.load(firstConfig)).resolves.toBe("first"); + await expect(loader.load(secondConfig)).resolves.toBe("second"); + + expect(load).toHaveBeenCalledTimes(2); + }); + + it("evicts rejected loads so retries can recover", async () => { + const config = {} as OpenClawConfig; + let calls = 0; + const loader = createConfigScopedPromiseLoader(async () => { + calls += 1; + if (calls === 1) { + throw new Error("transient"); + } + return "recovered"; + }); + + await expect(loader.load(config)).rejects.toThrow("transient"); + await expect(loader.load(config)).resolves.toBe("recovered"); + expect(calls).toBe(2); + }); + + it("clears default and config-scoped entries", async () => { + const config = {} as OpenClawConfig; + let calls = 0; + const loader = createConfigScopedPromiseLoader( + async (owner?: OpenClawConfig) => `${owner ? "config" : "default"}-${++calls}`, + ); + + await expect(loader.load()).resolves.toBe("default-1"); + await expect(loader.load(config)).resolves.toBe("config-2"); + + loader.clear(); + + await expect(loader.load()).resolves.toBe("default-3"); + await expect(loader.load(config)).resolves.toBe("config-4"); + }); +}); diff --git a/src/plugins/plugin-cache-primitives.ts b/src/plugins/plugin-cache-primitives.ts index d4af9f76ff6..412027eb244 100644 --- a/src/plugins/plugin-cache-primitives.ts +++ b/src/plugins/plugin-cache-primitives.ts @@ -68,6 +68,11 @@ export class PluginLruCache { export type ConfigScopedRuntimeCache = WeakMap>; +export type ConfigScopedPromiseLoader = { + load(config?: OpenClawConfig): Promise; + clear(): void; +}; + export function resolveConfigScopedRuntimeCacheValue(params: { cache: ConfigScopedRuntimeCache; config?: OpenClawConfig; @@ -94,6 +99,45 @@ export function createPluginCacheKey(parts: readonly unknown[]): string { return JSON.stringify(parts); } +export function createConfigScopedPromiseLoader( + load: (config?: OpenClawConfig) => T | Promise, +): ConfigScopedPromiseLoader { + let defaultPromise: Promise | undefined; + let promisesByConfig = new WeakMap>(); + + const createPromise = (config?: OpenClawConfig): Promise => { + const promise = Promise.resolve().then(() => load(config)); + void promise.catch(() => { + if (config) { + promisesByConfig.delete(config); + } else if (defaultPromise === promise) { + defaultPromise = undefined; + } + }); + return promise; + }; + + return { + async load(config?: OpenClawConfig): Promise { + if (!config) { + defaultPromise ??= createPromise(); + return await defaultPromise; + } + const cached = promisesByConfig.get(config); + if (cached) { + return await cached; + } + const promise = createPromise(config); + promisesByConfig.set(config, promise); + return await promise; + }, + clear(): void { + defaultPromise = undefined; + promisesByConfig = new WeakMap>(); + }, + }; +} + function normalizeMaxEntries(value: number, fallback: number): number { if (!Number.isFinite(value) || value <= 0) { return fallback; diff --git a/src/plugins/provider-auth-choices.ts b/src/plugins/provider-auth-choices.ts index 901c5d33fba..a1d3b21651a 100644 --- a/src/plugins/provider-auth-choices.ts +++ b/src/plugins/provider-auth-choices.ts @@ -2,8 +2,8 @@ import { resolveProviderIdForAuth } from "../agents/provider-auth-aliases.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import { sanitizeForLog } from "../terminal/ansi.js"; import { normalizePluginsConfig, resolveEffectiveEnableState } from "./config-state.js"; +import { loadManifestMetadataSnapshot } from "./manifest-contract-eligibility.js"; import type { PluginManifestRecord } from "./manifest-registry.js"; -import { loadPluginMetadataSnapshot } from "./plugin-metadata-snapshot.js"; import type { PluginOrigin } from "./plugin-origin.types.js"; export type ProviderAuthChoiceMetadata = { @@ -180,7 +180,7 @@ function resolveManifestProviderAuthChoiceCandidates(params?: { env?: NodeJS.ProcessEnv; includeUntrustedWorkspacePlugins?: boolean; }): ProviderAuthChoiceCandidate[] { - const metadataSnapshot = loadPluginMetadataSnapshot({ + const metadataSnapshot = loadManifestMetadataSnapshot({ config: params?.config ?? {}, workspaceDir: params?.workspaceDir, env: params?.env ?? process.env, diff --git a/src/plugins/provider-discovery.runtime.ts b/src/plugins/provider-discovery.runtime.ts index a2bcf157bcf..6be8f5ded04 100644 --- a/src/plugins/provider-discovery.runtime.ts +++ b/src/plugins/provider-discovery.runtime.ts @@ -1,6 +1,6 @@ import type { OpenClawConfig } from "../config/types.openclaw.js"; +import { loadManifestMetadataSnapshot } from "./manifest-contract-eligibility.js"; import type { PluginManifestRecord } from "./manifest-registry.js"; -import { loadPluginMetadataSnapshot } from "./plugin-metadata-snapshot.js"; import type { PluginMetadataRegistryView } from "./plugin-metadata-snapshot.types.js"; import { resolveDiscoveredProviderPluginIds } from "./providers.js"; import { resolvePluginProviders } from "./providers.runtime.js"; @@ -80,7 +80,7 @@ function resolveProviderDiscoveryEntryPlugins(params: { }): ProviderDiscoveryEntryResult { const metadataSnapshot = params.pluginMetadataSnapshot ?? - loadPluginMetadataSnapshot({ + loadManifestMetadataSnapshot({ config: params.config ?? {}, env: params.env ?? process.env, ...(params.workspaceDir ? { workspaceDir: params.workspaceDir } : {}), diff --git a/src/plugins/setup-registry.runtime.ts b/src/plugins/setup-registry.runtime.ts index 070d6542220..2e32c216b43 100644 --- a/src/plugins/setup-registry.runtime.ts +++ b/src/plugins/setup-registry.runtime.ts @@ -1,7 +1,7 @@ import { createRequire } from "node:module"; import { normalizeProviderId } from "../agents/provider-id.js"; import { isInstalledPluginEnabled } from "./installed-plugin-index.js"; -import { loadPluginMetadataSnapshot } from "./plugin-metadata-snapshot.js"; +import { loadManifestMetadataSnapshot } from "./manifest-contract-eligibility.js"; type SetupRegistryRuntimeModule = Pick< typeof import("./setup-registry.js"), @@ -30,7 +30,7 @@ export const __testing = { }; function resolveBundledSetupCliBackends(): SetupCliBackendRuntimeEntry[] { - const snapshot = loadPluginMetadataSnapshot({ config: {}, env: process.env }); + const snapshot = loadManifestMetadataSnapshot({ config: {}, env: process.env }); return snapshot.plugins.flatMap((plugin) => { if (plugin.origin !== "bundled" || !isInstalledPluginEnabled(snapshot.index, plugin.id)) { return []; diff --git a/src/plugins/web-provider-resolution-shared.ts b/src/plugins/web-provider-resolution-shared.ts index 38f5d43ebc7..ff647ed675a 100644 --- a/src/plugins/web-provider-resolution-shared.ts +++ b/src/plugins/web-provider-resolution-shared.ts @@ -1,7 +1,7 @@ import { resolveBundledPluginCompatibleLoadValues } from "./activation-context.js"; import type { PluginLoadOptions } from "./loader.js"; +import { loadManifestMetadataSnapshot } from "./manifest-contract-eligibility.js"; import type { PluginManifestRecord } from "./manifest-registry.js"; -import { loadPluginMetadataSnapshot } from "./plugin-metadata-snapshot.js"; import { createPluginIdScopeSet, normalizePluginIdScope } from "./plugin-scope.js"; export type WebProviderContract = "webSearchProviders" | "webFetchProviders"; @@ -66,7 +66,7 @@ function loadInstalledWebProviderManifestRecords(params: { env?: PluginLoadOptions["env"]; pluginIds?: readonly string[]; }): readonly PluginManifestRecord[] { - const records = loadPluginMetadataSnapshot({ + const records = loadManifestMetadataSnapshot({ config: params.config ?? {}, workspaceDir: params.workspaceDir, env: params.env ?? process.env, diff --git a/src/plugins/web-search-credential-presence.ts b/src/plugins/web-search-credential-presence.ts index f409a3bf09a..c5988f59d90 100644 --- a/src/plugins/web-search-credential-presence.ts +++ b/src/plugins/web-search-credential-presence.ts @@ -1,6 +1,6 @@ import type { OpenClawConfig } from "../config/types.openclaw.js"; +import { loadManifestMetadataSnapshot } from "./manifest-contract-eligibility.js"; import type { PluginManifestRecord } from "./manifest-registry.js"; -import { loadPluginMetadataSnapshot } from "./plugin-metadata-snapshot.js"; function hasConfiguredCredentialValue(value: unknown): boolean { if (typeof value === "string") { @@ -42,7 +42,7 @@ function hasManifestWebSearchEnvCredentialCandidate(params: { if (!env) { return false; } - return loadPluginMetadataSnapshot({ + return loadManifestMetadataSnapshot({ config: params.config, env, }).plugins.some((plugin) => { diff --git a/src/web-fetch/content-extractors.runtime.ts b/src/web-fetch/content-extractors.runtime.ts index d8295e11ab2..0e9c09ec75f 100644 --- a/src/web-fetch/content-extractors.runtime.ts +++ b/src/web-fetch/content-extractors.runtime.ts @@ -1,32 +1,14 @@ import type { OpenClawConfig } from "../config/types.openclaw.js"; +import { createConfigScopedPromiseLoader } from "../plugins/plugin-cache-primitives.js"; import type { WebContentExtractionResult, WebContentExtractMode, } from "../plugins/web-content-extractor-types.js"; import { resolvePluginWebContentExtractors } from "../plugins/web-content-extractors.runtime.js"; -let extractorPromise: Promise> | undefined; -const extractorPromisesByConfig = new WeakMap< - OpenClawConfig, - Promise> ->(); - -async function loadWebContentExtractors(config?: OpenClawConfig) { - if (config) { - const cached = extractorPromisesByConfig.get(config); - if (cached) { - return await cached; - } - const promise = Promise.resolve().then(() => resolvePluginWebContentExtractors({ config })); - extractorPromisesByConfig.set(config, promise); - void promise.catch(() => { - extractorPromisesByConfig.delete(config); - }); - return await promise; - } - extractorPromise ??= Promise.resolve(resolvePluginWebContentExtractors()); - return await extractorPromise; -} +const webContentExtractorLoader = createConfigScopedPromiseLoader((config?: OpenClawConfig) => + resolvePluginWebContentExtractors(config ? { config } : undefined), +); export async function extractReadableContent(params: { html: string; @@ -34,9 +16,9 @@ export async function extractReadableContent(params: { extractMode: WebContentExtractMode; config?: OpenClawConfig; }): Promise<(WebContentExtractionResult & { extractor: string }) | null> { - let extractors: Awaited>; + let extractors: Awaited>; try { - extractors = await loadWebContentExtractors(params.config); + extractors = await webContentExtractorLoader.load(params.config); } catch { return null; }