From b7a1bfd2d7d8100c54892c13a9300949cd4b6f97 Mon Sep 17 00:00:00 2001 From: Marcus Castro Date: Mon, 27 Apr 2026 01:56:06 -0300 Subject: [PATCH] fix(plugins): cache installed manifest registry --- .../manifest-registry-installed.test.ts | 70 ++++++- src/plugins/manifest-registry-installed.ts | 184 +++++++++++++++++- 2 files changed, 250 insertions(+), 4 deletions(-) diff --git a/src/plugins/manifest-registry-installed.test.ts b/src/plugins/manifest-registry-installed.test.ts index 5ffaad9b772..73c16b1cbd8 100644 --- a/src/plugins/manifest-registry-installed.test.ts +++ b/src/plugins/manifest-registry-installed.test.ts @@ -1,17 +1,22 @@ import fs from "node:fs"; import path from "node:path"; -import { afterEach, describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { readPersistedInstalledPluginIndex, writePersistedInstalledPluginIndex, } from "./installed-plugin-index-store.js"; import type { InstalledPluginIndex } from "./installed-plugin-index.js"; -import { loadPluginManifestRegistryForInstalledIndex } from "./manifest-registry-installed.js"; +import { + clearInstalledManifestRegistryCache, + loadPluginManifestRegistryForInstalledIndex, +} from "./manifest-registry-installed.js"; import { cleanupTrackedTempDirs, makeTrackedTempDir } from "./test-helpers/fs-fixtures.js"; const tempDirs: string[] = []; afterEach(() => { + clearInstalledManifestRegistryCache(); + vi.restoreAllMocks(); cleanupTrackedTempDirs(tempDirs); }); @@ -71,6 +76,67 @@ function createIndex(rootDir: string): InstalledPluginIndex { } describe("loadPluginManifestRegistryForInstalledIndex", () => { + it("reuses installed-index manifest registries for identical runtime lookups", () => { + const rootDir = makeTempDir(); + writePlugin(rootDir, "installed", "installed-"); + const index = createIndex(rootDir); + const readFileSync = vi.spyOn(fs, "readFileSync"); + const env = { + OPENCLAW_VERSION: "2026.4.25", + VITEST: "true", + }; + + const first = loadPluginManifestRegistryForInstalledIndex({ + index, + env, + includeDisabled: true, + }); + const readsAfterFirstLoad = readFileSync.mock.calls.length; + const second = loadPluginManifestRegistryForInstalledIndex({ + index, + env, + includeDisabled: true, + }); + + expect(second).toBe(first); + expect(readFileSync.mock.calls.length).toBe(readsAfterFirstLoad); + }); + + it("refreshes the installed-index manifest registry cache when manifest files change", () => { + const rootDir = makeTempDir(); + const manifestPath = path.join(rootDir, "openclaw.plugin.json"); + writePlugin(rootDir, "installed", "installed-"); + const index = createIndex(rootDir); + const env = { + OPENCLAW_VERSION: "2026.4.25", + VITEST: "true", + }; + + const first = loadPluginManifestRegistryForInstalledIndex({ + index, + env, + includeDisabled: true, + }); + expect(first.plugins[0]?.modelSupport).toEqual({ + modelPrefixes: ["installed-"], + }); + + writePlugin(rootDir, "installed", "updated-installed-"); + const nextMtime = new Date(Date.now() + 5000); + fs.utimesSync(manifestPath, nextMtime, nextMtime); + + const second = loadPluginManifestRegistryForInstalledIndex({ + index, + env, + includeDisabled: true, + }); + + expect(second).not.toBe(first); + expect(second.plugins[0]?.modelSupport).toEqual({ + modelPrefixes: ["updated-installed-"], + }); + }); + it("loads manifest metadata only for plugins present in the installed index", () => { const installedRoot = makeTempDir(); const unrelatedRoot = makeTempDir(); diff --git a/src/plugins/manifest-registry-installed.ts b/src/plugins/manifest-registry-installed.ts index 976f91768fd..8e75ec751cc 100644 --- a/src/plugins/manifest-registry-installed.ts +++ b/src/plugins/manifest-registry-installed.ts @@ -2,6 +2,8 @@ import fs from "node:fs"; import path from "node:path"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import type { PluginCandidate } from "./discovery.js"; +import { hashJson } from "./installed-plugin-index-hash.js"; +import { resolveInstalledPluginIndexPolicyHash } from "./installed-plugin-index-policy.js"; import type { InstalledPluginIndex, InstalledPluginIndexRecord } from "./installed-plugin-index.js"; import { extractPluginInstallRecordsFromInstalledPluginIndex } from "./installed-plugin-index.js"; import { loadPluginManifestRegistry, type PluginManifestRegistry } from "./manifest-registry.js"; @@ -13,6 +15,160 @@ import { type PackageManifest, } from "./manifest.js"; +const INSTALLED_MANIFEST_REGISTRY_CACHE_MAX_ENTRIES = 64; + +type InstalledManifestRegistryCacheEntry = { + registry: PluginManifestRegistry; + lastUsed: number; +}; + +const installedManifestRegistryCache = new Map(); +let installedManifestRegistryCacheTick = 0; + +function normalizePluginIdFilter(pluginIds: readonly string[] | undefined): string[] | undefined { + if (!pluginIds?.length) { + return undefined; + } + return [...new Set(pluginIds)].toSorted((left, right) => left.localeCompare(right)); +} + +function resolvePackageJsonPath(record: InstalledPluginIndexRecord): string | undefined { + if (!record.packageJson?.path) { + return undefined; + } + const rootDir = resolveInstalledPluginRootDir(record); + const packageJsonPath = path.resolve(rootDir, record.packageJson.path); + const relative = path.relative(rootDir, packageJsonPath); + if (relative.startsWith("..") || path.isAbsolute(relative)) { + return undefined; + } + return packageJsonPath; +} + +function safeFileSignature(filePath: string | undefined): string | undefined { + if (!filePath) { + return undefined; + } + try { + const stat = fs.statSync(filePath); + return `${filePath}:${stat.size}:${stat.mtimeMs}`; + } catch { + return `${filePath}:missing`; + } +} + +function shouldUseInstalledManifestRegistryCache(params: { + env: NodeJS.ProcessEnv; + bundledChannelConfigCollector?: BundledChannelConfigCollector; +}): boolean { + if (params.bundledChannelConfigCollector) { + return false; + } + if (params.env.OPENCLAW_DISABLE_INSTALLED_PLUGIN_MANIFEST_REGISTRY_CACHE?.trim()) { + return false; + } + return !params.env.OPENCLAW_DISABLE_PLUGIN_MANIFEST_CACHE?.trim(); +} + +function buildInstalledManifestRegistryCacheKey(params: { + index: InstalledPluginIndex; + config?: OpenClawConfig; + workspaceDir?: string; + env: NodeJS.ProcessEnv; + pluginIds?: readonly string[]; + includeDisabled?: boolean; +}): string { + return hashJson({ + index: { + version: params.index.version, + hostContractVersion: params.index.hostContractVersion, + compatRegistryVersion: params.index.compatRegistryVersion, + migrationVersion: params.index.migrationVersion, + policyHash: params.index.policyHash, + installRecords: params.index.installRecords, + diagnostics: params.index.diagnostics, + plugins: params.index.plugins.map((record) => { + const packageJsonPath = resolvePackageJsonPath(record); + return { + pluginId: record.pluginId, + packageName: record.packageName, + packageVersion: record.packageVersion, + installRecord: record.installRecord, + installRecordHash: record.installRecordHash, + packageInstall: record.packageInstall, + packageChannel: record.packageChannel, + manifestPath: record.manifestPath, + manifestHash: record.manifestHash, + manifestFile: safeFileSignature(record.manifestPath), + format: record.format, + bundleFormat: record.bundleFormat, + source: record.source, + setupSource: record.setupSource, + packageJson: record.packageJson, + packageJsonFile: safeFileSignature(packageJsonPath), + rootDir: record.rootDir, + origin: record.origin, + enabled: record.enabled, + enabledByDefault: record.enabledByDefault, + syntheticAuthRefs: record.syntheticAuthRefs, + startup: record.startup, + compat: record.compat, + }; + }), + }, + request: { + workspaceDir: params.workspaceDir, + pluginIds: normalizePluginIdFilter(params.pluginIds), + includeDisabled: params.includeDisabled === true, + configPolicyHash: resolveInstalledPluginIndexPolicyHash(params.config), + env: { + OPENCLAW_VERSION: params.env.OPENCLAW_VERSION, + HOME: params.env.HOME, + USERPROFILE: params.env.USERPROFILE, + }, + }, + }); +} + +function getCachedInstalledManifestRegistry(cacheKey: string): PluginManifestRegistry | undefined { + const cached = installedManifestRegistryCache.get(cacheKey); + if (!cached) { + return undefined; + } + cached.lastUsed = ++installedManifestRegistryCacheTick; + return cached.registry; +} + +function setCachedInstalledManifestRegistry( + cacheKey: string, + registry: PluginManifestRegistry, +): void { + if ( + !installedManifestRegistryCache.has(cacheKey) && + installedManifestRegistryCache.size >= INSTALLED_MANIFEST_REGISTRY_CACHE_MAX_ENTRIES + ) { + let oldestKey: string | undefined; + let oldestTick = Number.POSITIVE_INFINITY; + for (const [key, entry] of installedManifestRegistryCache) { + if (entry.lastUsed < oldestTick) { + oldestKey = key; + oldestTick = entry.lastUsed; + } + } + if (oldestKey) { + installedManifestRegistryCache.delete(oldestKey); + } + } + installedManifestRegistryCache.set(cacheKey, { + registry, + lastUsed: ++installedManifestRegistryCacheTick, + }); +} + +export function clearInstalledManifestRegistryCache(): void { + installedManifestRegistryCache.clear(); +} + function resolveInstalledPluginRootDir(record: InstalledPluginIndexRecord): string { return record.rootDir || path.dirname(record.manifestPath || process.cwd()); } @@ -94,6 +250,26 @@ export function loadPluginManifestRegistryForInstalledIndex(params: { if (params.pluginIds && params.pluginIds.length === 0) { return { plugins: [], diagnostics: [] }; } + const env = params.env ?? process.env; + const cacheKey = shouldUseInstalledManifestRegistryCache({ + env, + bundledChannelConfigCollector: params.bundledChannelConfigCollector, + }) + ? buildInstalledManifestRegistryCacheKey({ + index: params.index, + config: params.config, + workspaceDir: params.workspaceDir, + env, + pluginIds: params.pluginIds, + includeDisabled: params.includeDisabled, + }) + : undefined; + if (cacheKey) { + const cached = getCachedInstalledManifestRegistry(cacheKey); + if (cached) { + return cached; + } + } const pluginIdSet = params.pluginIds?.length ? new Set(params.pluginIds) : null; const diagnostics = pluginIdSet ? params.index.diagnostics.filter((diagnostic) => { @@ -105,10 +281,10 @@ export function loadPluginManifestRegistryForInstalledIndex(params: { .filter((plugin) => params.includeDisabled || plugin.enabled) .filter((plugin) => !pluginIdSet || pluginIdSet.has(plugin.pluginId)) .map(toPluginCandidate); - return loadPluginManifestRegistry({ + const registry = loadPluginManifestRegistry({ config: params.config, workspaceDir: params.workspaceDir, - env: params.env, + env, cache: false, candidates, diagnostics: [...diagnostics], @@ -117,4 +293,8 @@ export function loadPluginManifestRegistryForInstalledIndex(params: { ? { bundledChannelConfigCollector: params.bundledChannelConfigCollector } : {}), }); + if (cacheKey) { + setCachedInstalledManifestRegistry(cacheKey, registry); + } + return registry; }