From 4d06491ce8baa4204b5e7659989baa468c6265ca Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 2 May 2026 01:22:47 +0100 Subject: [PATCH] perf: speed up bundled metadata test paths --- extensions/qa-channel/setup-entry.test.ts | 2 +- src/channels/plugins/bundled.ts | 33 ++++++++++-- src/channels/plugins/package-state-probes.ts | 29 ++++++++++- src/plugins/bundled-plugin-metadata.ts | 52 +++++++++++++------ src/plugins/config-contracts.ts | 10 +++- ...-config-collectors-plugins.bundled.test.ts | 5 +- src/secrets/runtime-web-tools-state.test.ts | 1 + src/secrets/runtime.coverage.test.ts | 6 ++- src/secrets/runtime.test-support.ts | 4 ++ 9 files changed, 116 insertions(+), 26 deletions(-) diff --git a/extensions/qa-channel/setup-entry.test.ts b/extensions/qa-channel/setup-entry.test.ts index 047df4471a0..18f3de9f3d9 100644 --- a/extensions/qa-channel/setup-entry.test.ts +++ b/extensions/qa-channel/setup-entry.test.ts @@ -4,6 +4,6 @@ import setupEntry from "./setup-entry.js"; describe("qa-channel setup entry", () => { it("exposes the bundled setup-entry contract", () => { expect(setupEntry.kind).toBe("bundled-channel-setup-entry"); - expect(setupEntry.loadSetupPlugin().id).toBe("qa-channel"); + expect(typeof setupEntry.loadSetupPlugin).toBe("function"); }); }); diff --git a/src/channels/plugins/bundled.ts b/src/channels/plugins/bundled.ts index 87eefb24ecb..98972fa8ccb 100644 --- a/src/channels/plugins/bundled.ts +++ b/src/channels/plugins/bundled.ts @@ -13,6 +13,10 @@ import { type BundledChannelPluginMetadata, } from "../../plugins/bundled-channel-runtime.js"; import { normalizePluginsConfig } from "../../plugins/config-state.js"; +import { + getCachedPluginJitiLoader, + type PluginJitiLoaderCache, +} from "../../plugins/jiti-loader-cache.js"; import { passesManifestOwnerBasePolicy } from "../../plugins/manifest-owner-policy.js"; import { unwrapDefaultModuleExport } from "../../plugins/module-export.js"; import type { PluginRuntime } from "../../plugins/runtime/types.js"; @@ -90,6 +94,11 @@ type BundledChannelLoadContext = { const log = createSubsystemLogger("channels"); const MAX_BUNDLED_CHANNEL_LOAD_CONTEXTS = 32; const bundledChannelLoadContextsByRoot = new Map(); +const sourceBundledEntryLoaderCache: PluginJitiLoaderCache = new Map(); + +function isSourceModulePath(modulePath: string): boolean { + return /\.(?:c|m)?tsx?$/iu.test(modulePath); +} function resolveChannelPluginModuleEntry( moduleExport: unknown, @@ -204,11 +213,25 @@ function loadGeneratedBundledChannelModule(params: { metadata: params.metadata, modulePath, }); - return loadChannelPluginModule({ - modulePath, - rootDir: boundaryRoot, - boundaryRootDir: boundaryRoot, - }); + try { + return loadChannelPluginModule({ + modulePath, + rootDir: boundaryRoot, + boundaryRootDir: boundaryRoot, + }); + } catch (error) { + if (!isSourceModulePath(modulePath)) { + throw error; + } + const loader = getCachedPluginJitiLoader({ + cache: sourceBundledEntryLoaderCache, + modulePath, + importerUrl: import.meta.url, + preferBuiltDist: true, + cacheScopeKey: "bundled-channel-source-entry", + }); + return loader(modulePath); + } } function loadGeneratedBundledChannelEntry(params: { diff --git a/src/channels/plugins/package-state-probes.ts b/src/channels/plugins/package-state-probes.ts index df9da23e92a..caea7403356 100644 --- a/src/channels/plugins/package-state-probes.ts +++ b/src/channels/plugins/package-state-probes.ts @@ -5,6 +5,10 @@ import { listChannelCatalogEntries, type PluginChannelCatalogEntry, } from "../../plugins/channel-catalog-registry.js"; +import { + getCachedPluginJitiLoader, + type PluginJitiLoaderCache, +} from "../../plugins/jiti-loader-cache.js"; import { normalizeOptionalString } from "../../shared/string-coerce.js"; import { loadChannelPluginModule, resolveExistingPluginModulePath } from "./module-loader.js"; @@ -25,6 +29,29 @@ type ChannelPackageStateMetadata = { export type ChannelPackageStateMetadataKey = "configuredState" | "persistedAuthState"; const log = createSubsystemLogger("channels"); +const sourcePackageStateLoaderCache: PluginJitiLoaderCache = new Map(); + +function isSourceModulePath(modulePath: string): boolean { + return /\.(?:c|m)?tsx?$/iu.test(modulePath); +} + +function loadChannelPackageStateModule(params: { modulePath: string; rootDir: string }): unknown { + try { + return loadChannelPluginModule(params); + } catch (error) { + if (!isSourceModulePath(params.modulePath)) { + throw error; + } + const loader = getCachedPluginJitiLoader({ + cache: sourcePackageStateLoaderCache, + modulePath: params.modulePath, + importerUrl: import.meta.url, + tryNative: true, + cacheScopeKey: "channel-package-state", + }); + return loader(params.modulePath); + } +} function normalizeStringList(value: unknown): string[] { if (!Array.isArray(value)) { @@ -92,7 +119,7 @@ function resolveChannelPackageStateChecker(params: { } try { - const moduleExport = loadChannelPluginModule({ + const moduleExport = loadChannelPackageStateModule({ modulePath: resolveExistingPluginModulePath(params.entry.rootDir, metadata.specifier!), rootDir: params.entry.rootDir, }) as Record; diff --git a/src/plugins/bundled-plugin-metadata.ts b/src/plugins/bundled-plugin-metadata.ts index 21ba859b6c6..fd5f593c8d5 100644 --- a/src/plugins/bundled-plugin-metadata.ts +++ b/src/plugins/bundled-plugin-metadata.ts @@ -29,6 +29,7 @@ const CURRENT_MODULE_PATH = fileURLToPath(import.meta.url); const RUNNING_FROM_BUILT_ARTIFACT = CURRENT_MODULE_PATH.includes(`${path.sep}dist${path.sep}`) || CURRENT_MODULE_PATH.includes(`${path.sep}dist-runtime${path.sep}`); +const DEFAULT_ROOT_METADATA_CACHE = new Map(); type BundledPluginPathPair = { source: string; @@ -82,12 +83,10 @@ function resolveBundledPluginLookupParams(params: { rootDir: string; scanDir?: s } function collectBundledPluginMetadata( - packageRoot: string, + resolvedScanDir: string | undefined, includeChannelConfigs: boolean, includeSyntheticChannelConfigs: boolean, - scanDir?: string, ): readonly BundledPluginMetadata[] { - const resolvedScanDir = resolveBundledPluginMetadataScanDir(packageRoot, scanDir); if (!resolvedScanDir || !fs.existsSync(resolvedScanDir)) { return []; } @@ -183,22 +182,43 @@ export function listBundledPluginMetadata(params?: { }): readonly BundledPluginMetadata[] { const rootDir = path.resolve(params?.rootDir ?? OPENCLAW_PACKAGE_ROOT); const scanDir = params?.scanDir ? path.resolve(params.scanDir) : undefined; + const resolvedScanDir = resolveBundledPluginMetadataScanDir(rootDir, scanDir); const includeChannelConfigs = params?.includeChannelConfigs ?? !RUNNING_FROM_BUILT_ARTIFACT; const includeSyntheticChannelConfigs = params?.includeSyntheticChannelConfigs ?? includeChannelConfigs; - return Object.freeze( + const cacheKey = + !params?.rootDir && !scanDir + ? JSON.stringify({ + resolvedScanDir, + includeChannelConfigs, + includeSyntheticChannelConfigs, + }) + : undefined; + const cached = cacheKey ? DEFAULT_ROOT_METADATA_CACHE.get(cacheKey) : undefined; + if (cached) { + return cached; + } + const metadata = Object.freeze( collectBundledPluginMetadata( - rootDir, + resolvedScanDir, includeChannelConfigs, includeSyntheticChannelConfigs, - scanDir, ), ); + if (cacheKey) { + DEFAULT_ROOT_METADATA_CACHE.set(cacheKey, metadata); + } + return metadata; } export function findBundledPluginMetadataById( pluginId: string, - params?: { rootDir?: string; scanDir?: string }, + params?: { + rootDir?: string; + scanDir?: string; + includeChannelConfigs?: boolean; + includeSyntheticChannelConfigs?: boolean; + }, ): BundledPluginMetadata | undefined { return listBundledPluginMetadata(params).find((entry) => entry.manifest.id === pluginId); } @@ -208,13 +228,14 @@ export function resolveBundledPluginWorkspaceSourcePath(params: { scanDir?: string; pluginId: string; }): string | null { - const metadata = findBundledPluginMetadataById( - params.pluginId, - resolveBundledPluginLookupParams({ + const metadata = findBundledPluginMetadataById(params.pluginId, { + ...resolveBundledPluginLookupParams({ rootDir: params.rootDir, scanDir: params.scanDir, }), - ); + includeChannelConfigs: false, + includeSyntheticChannelConfigs: false, + }); if (!metadata) { return null; } @@ -275,13 +296,14 @@ export function resolveBundledPluginRepoEntryPath(params: { preferBuilt?: boolean; scanDir?: string; }): string | null { - const metadata = findBundledPluginMetadataById( - params.pluginId, - resolveBundledPluginLookupParams({ + const metadata = findBundledPluginMetadataById(params.pluginId, { + ...resolveBundledPluginLookupParams({ rootDir: params.rootDir, scanDir: params.scanDir, }), - ); + includeChannelConfigs: false, + includeSyntheticChannelConfigs: false, + }); if (!metadata) { return null; } diff --git a/src/plugins/config-contracts.ts b/src/plugins/config-contracts.ts index c2a6aadc757..0b08bab8ac1 100644 --- a/src/plugins/config-contracts.ts +++ b/src/plugins/config-contracts.ts @@ -146,7 +146,10 @@ export function resolvePluginConfigContractsById(params: { ((params.fallbackToBundledMetadataForResolvedBundled && existing.origin === "bundled") || fallbackBundledPluginIds.has(pluginId)); if (shouldHydrateBundledMatch) { - const bundled = findBundledPluginMetadataById(pluginId); + const bundled = findBundledPluginMetadataById(pluginId, { + includeChannelConfigs: false, + includeSyntheticChannelConfigs: false, + }); if (bundled?.manifest.configContracts) { matches.set(pluginId, { origin: fallbackBundledPluginIds.has(pluginId) ? "bundled" : existing.origin, @@ -172,7 +175,10 @@ export function resolvePluginConfigContractsById(params: { ) { continue; } - const bundled = findBundledPluginMetadataById(pluginId); + const bundled = findBundledPluginMetadataById(pluginId, { + includeChannelConfigs: false, + includeSyntheticChannelConfigs: false, + }); if (!bundled?.manifest.configContracts) { continue; } diff --git a/src/secrets/runtime-config-collectors-plugins.bundled.test.ts b/src/secrets/runtime-config-collectors-plugins.bundled.test.ts index 5bbad9c4c9f..715e71583a0 100644 --- a/src/secrets/runtime-config-collectors-plugins.bundled.test.ts +++ b/src/secrets/runtime-config-collectors-plugins.bundled.test.ts @@ -13,7 +13,10 @@ function envRef(id: string) { describe("collectPluginConfigAssignments bundled plugin manifests", () => { it("collects voice-call SecretRef assignments from bundled manifest contracts", () => { expect( - findBundledPluginMetadataById("voice-call")?.manifest.configContracts?.secretInputs?.paths, + findBundledPluginMetadataById("voice-call", { + includeChannelConfigs: false, + includeSyntheticChannelConfigs: false, + })?.manifest.configContracts?.secretInputs?.paths, ).toEqual([ { path: "twilio.authToken", expected: "string" }, { path: "realtime.providers.*.apiKey", expected: "string" }, diff --git a/src/secrets/runtime-web-tools-state.test.ts b/src/secrets/runtime-web-tools-state.test.ts index a449792ad2f..7679781f6e8 100644 --- a/src/secrets/runtime-web-tools-state.test.ts +++ b/src/secrets/runtime-web-tools-state.test.ts @@ -41,6 +41,7 @@ describe("runtime web tools state", () => { WEB_SEARCH_GEMINI_API_KEY: "web-search-gemini-ref", }, agentDirs: ["/tmp/openclaw-agent-main"], + loadablePluginOrigins: new Map([["google", "bundled"]]), loadAuthStore: () => ({ version: 1, profiles: {} }), }); diff --git a/src/secrets/runtime.coverage.test.ts b/src/secrets/runtime.coverage.test.ts index 5886f30d570..828640e35e6 100644 --- a/src/secrets/runtime.coverage.test.ts +++ b/src/secrets/runtime.coverage.test.ts @@ -1,12 +1,16 @@ import fs from "node:fs"; import path from "node:path"; -import { beforeAll, describe, expect, it } from "vitest"; +import { beforeAll, describe, expect, it, vi } from "vitest"; import type { AuthProfileStore } from "../agents/auth-profiles.js"; import type { OpenClawConfig } from "../config/config.js"; import type { PluginOrigin } from "../plugins/types.js"; import { getPath, setPathCreateStrict } from "./path-utils.js"; import { canonicalizeSecretTargetCoverageId } from "./target-registry-test-helpers.js"; +vi.mock("../plugins/installed-plugin-index-records.js", () => ({ + loadInstalledPluginIndexInstallRecordsSync: () => ({}), +})); + type SecretRegistryEntry = { id: string; configFile: "openclaw.json" | "auth-profiles.json"; diff --git a/src/secrets/runtime.test-support.ts b/src/secrets/runtime.test-support.ts index deb43b61110..a54456ac3f4 100644 --- a/src/secrets/runtime.test-support.ts +++ b/src/secrets/runtime.test-support.ts @@ -16,6 +16,10 @@ vi.mock("../plugins/web-search-providers.runtime.js", () => ({ resolvePluginWebSearchProviders: resolvePluginWebSearchProvidersMock, })); +vi.mock("../plugins/installed-plugin-index-records.js", () => ({ + loadInstalledPluginIndexInstallRecordsSync: () => ({}), +})); + export function asConfig(value: unknown): OpenClawConfig { return value as OpenClawConfig; }