fix: tighten plugin metadata cache invalidation

This commit is contained in:
Peter Steinberger
2026-05-02 03:42:32 +01:00
parent b16069cedc
commit 97a34e0f50
12 changed files with 405 additions and 36 deletions

View File

@@ -11,7 +11,7 @@ import { buildConfigSchema, type ConfigSchemaResponse } from "./schema.js";
function loadManifestRegistry(config: OpenClawConfig, env?: NodeJS.ProcessEnv) {
const workspaceDir = resolveAgentWorkspaceDir(config, resolveDefaultAgentId(config));
const currentSnapshot = getCurrentPluginMetadataSnapshot({ config, workspaceDir });
const currentSnapshot = getCurrentPluginMetadataSnapshot({ config, env, workspaceDir });
if (currentSnapshot) {
return currentSnapshot.manifestRegistry;
}

View File

@@ -100,6 +100,23 @@ describe("current plugin metadata snapshot", () => {
).toBeUndefined();
});
it("rejects a current snapshot when env-resolved plugin load paths change", () => {
const config = { plugins: { load: { paths: ["~/plugins"] } } };
const snapshot = createSnapshot({ config });
const snapshotEnv = {
HOME: "/home/snapshot",
OPENCLAW_HOME: undefined,
} as NodeJS.ProcessEnv;
const requestedEnv = {
HOME: "/home/requested",
OPENCLAW_HOME: undefined,
} as NodeJS.ProcessEnv;
setCurrentPluginMetadataSnapshot(snapshot, { config, env: snapshotEnv });
expect(getCurrentPluginMetadataSnapshot({ config, env: snapshotEnv })).toBe(snapshot);
expect(getCurrentPluginMetadataSnapshot({ config, env: requestedEnv })).toBeUndefined();
});
it("keeps source-policy compatibility when storing an auto-enabled runtime config", () => {
const sourceConfig = { channels: { telegram: { botToken: "token" } } };
const autoEnabledConfig = {

View File

@@ -5,36 +5,21 @@ import {
setCurrentPluginMetadataSnapshotState,
} from "./current-plugin-metadata-state.js";
import { resolveInstalledPluginIndexPolicyHash } from "./installed-plugin-index-policy.js";
import { resolvePluginMetadataSnapshotConfigFingerprint } from "./plugin-metadata-config-fingerprint.js";
import type { PluginMetadataSnapshot } from "./plugin-metadata-snapshot.types.js";
function normalizeLoadPaths(config: OpenClawConfig | undefined): readonly string[] {
const paths = config?.plugins?.load?.paths;
if (!Array.isArray(paths)) {
return [];
}
return paths.filter((entry) => typeof entry === "string");
}
export function resolvePluginMetadataSnapshotConfigFingerprint(
config: OpenClawConfig | undefined,
options: { policyHash?: string } = {},
): string {
return JSON.stringify({
policyHash: options.policyHash ?? resolveInstalledPluginIndexPolicyHash(config),
pluginLoadPaths: normalizeLoadPaths(config),
});
}
export { resolvePluginMetadataSnapshotConfigFingerprint } from "./plugin-metadata-config-fingerprint.js";
// Single-slot Gateway-owned handoff. Replace or clear it at lifecycle boundaries;
// never accumulate historical metadata snapshots here.
export function setCurrentPluginMetadataSnapshot(
snapshot: PluginMetadataSnapshot | undefined,
options: { config?: OpenClawConfig } = {},
options: { config?: OpenClawConfig; env?: NodeJS.ProcessEnv } = {},
): void {
setCurrentPluginMetadataSnapshotState(
snapshot,
snapshot
? resolvePluginMetadataSnapshotConfigFingerprint(options.config, {
env: options.env,
policyHash: snapshot.policyHash,
})
: undefined,
@@ -48,6 +33,7 @@ export function clearCurrentPluginMetadataSnapshot(): void {
export function getCurrentPluginMetadataSnapshot(
params: {
config?: OpenClawConfig;
env?: NodeJS.ProcessEnv;
workspaceDir?: string;
} = {},
): PluginMetadataSnapshot | undefined {
@@ -62,12 +48,19 @@ export function getCurrentPluginMetadataSnapshot(
) {
return undefined;
}
if (
params.config &&
configFingerprint &&
configFingerprint !== resolvePluginMetadataSnapshotConfigFingerprint(params.config)
) {
return undefined;
if (params.config) {
const requestedConfigFingerprint = resolvePluginMetadataSnapshotConfigFingerprint(
params.config,
{
env: params.env,
},
);
if (configFingerprint && configFingerprint !== requestedConfigFingerprint) {
return undefined;
}
if (snapshot.configFingerprint && snapshot.configFingerprint !== requestedConfigFingerprint) {
return undefined;
}
}
if (snapshot.workspaceDir !== undefined && params.workspaceDir === undefined) {
return undefined;

View File

@@ -303,6 +303,112 @@ describe("loadPluginLookUpTable", () => {
);
});
it("rebuilds when a provided metadata snapshot has stale plugin load paths", async () => {
const plugins = [
createManifestRecord({
id: "telegram",
origin: "bundled",
channels: ["telegram"],
}),
];
const snapshotConfig = {
plugins: {
load: { paths: ["/plugins/one"] },
},
} as OpenClawConfig;
const requestedConfig = {
plugins: {
load: { paths: ["/plugins/two"] },
},
} as OpenClawConfig;
const policyHash = resolveInstalledPluginIndexPolicyHash(snapshotConfig);
const index = createIndex(plugins, { policyHash });
const manifestRegistry: PluginManifestRegistry = {
plugins,
diagnostics: [],
};
loadPluginManifestRegistryForInstalledIndex.mockReturnValue(manifestRegistry);
const { loadPluginMetadataSnapshot } = await import("./plugin-metadata-snapshot.js");
const { loadPluginLookUpTable } = await import("./plugin-lookup-table.js");
const metadataSnapshot = loadPluginMetadataSnapshot({
config: snapshotConfig,
env: {},
index,
});
loadPluginManifestRegistryForInstalledIndex.mockClear();
loadPluginLookUpTable({
config: requestedConfig,
env: {},
index,
metadataSnapshot,
});
expect(loadPluginManifestRegistryForInstalledIndex).toHaveBeenCalledOnce();
expect(loadPluginManifestRegistryForInstalledIndex).toHaveBeenCalledWith(
expect.objectContaining({
index,
config: requestedConfig,
}),
);
});
it("rebuilds when a provided metadata snapshot has stale env-resolved plugin load paths", async () => {
const plugins = [
createManifestRecord({
id: "telegram",
origin: "bundled",
channels: ["telegram"],
}),
];
const config = {
plugins: {
load: { paths: ["~/plugins"] },
},
} as OpenClawConfig;
const snapshotEnv = {
HOME: "/home/snapshot",
OPENCLAW_HOME: undefined,
} as NodeJS.ProcessEnv;
const requestedEnv = {
HOME: "/home/requested",
OPENCLAW_HOME: undefined,
} as NodeJS.ProcessEnv;
const policyHash = resolveInstalledPluginIndexPolicyHash(config);
const index = createIndex(plugins, { policyHash });
const manifestRegistry: PluginManifestRegistry = {
plugins,
diagnostics: [],
};
loadPluginManifestRegistryForInstalledIndex.mockReturnValue(manifestRegistry);
const { loadPluginMetadataSnapshot } = await import("./plugin-metadata-snapshot.js");
const { loadPluginLookUpTable } = await import("./plugin-lookup-table.js");
const metadataSnapshot = loadPluginMetadataSnapshot({
config,
env: snapshotEnv,
index,
});
loadPluginManifestRegistryForInstalledIndex.mockClear();
loadPluginLookUpTable({
config,
env: requestedEnv,
index,
metadataSnapshot,
});
expect(loadPluginManifestRegistryForInstalledIndex).toHaveBeenCalledOnce();
expect(loadPluginManifestRegistryForInstalledIndex).toHaveBeenCalledWith(
expect.objectContaining({
index,
config,
env: requestedEnv,
}),
);
});
it("rebuilds when a provided metadata snapshot has stale plugin inventory", async () => {
const snapshotPlugins = [
createManifestRecord({

View File

@@ -59,6 +59,7 @@ export function loadPluginLookUpTable(params: LoadPluginLookUpTableParams): Plug
isPluginMetadataSnapshotCompatible({
snapshot: params.metadataSnapshot,
config: requestedSnapshotConfig,
env: params.env,
workspaceDir: params.workspaceDir,
index: params.index,
})

View File

@@ -0,0 +1,31 @@
import type { OpenClawConfig } from "../config/types.openclaw.js";
import { resolveHomeRelativePath } from "../infra/home-dir.js";
import { resolveInstalledPluginIndexPolicyHash } from "./installed-plugin-index-policy.js";
function normalizeResolvedLoadPaths(
config: OpenClawConfig | undefined,
env: NodeJS.ProcessEnv,
): readonly string[] {
const paths = config?.plugins?.load?.paths;
if (!Array.isArray(paths)) {
return [];
}
return paths.flatMap((entry) => {
if (typeof entry !== "string") {
return [];
}
const trimmed = entry.trim();
return trimmed ? [resolveHomeRelativePath(trimmed, { env })] : [];
});
}
export function resolvePluginMetadataSnapshotConfigFingerprint(
config: OpenClawConfig | undefined,
options: { env?: NodeJS.ProcessEnv; policyHash?: string } = {},
): string {
const env = options.env ?? process.env;
return JSON.stringify({
policyHash: options.policyHash ?? resolveInstalledPluginIndexPolicyHash(config),
pluginLoadPaths: normalizeResolvedLoadPaths(config, env),
});
}

View File

@@ -7,6 +7,7 @@ import {
resolveInstalledManifestRegistryIndexFingerprint,
} from "./manifest-registry-installed.js";
import type { PluginManifestRecord } from "./manifest-registry.js";
import { resolvePluginMetadataSnapshotConfigFingerprint } from "./plugin-metadata-config-fingerprint.js";
import type {
LoadPluginMetadataSnapshotParams,
PluginMetadataSnapshot,
@@ -36,13 +37,24 @@ function indexesMatch(
}
export function isPluginMetadataSnapshotCompatible(params: {
snapshot: Pick<PluginMetadataSnapshot, "index" | "policyHash" | "workspaceDir">;
snapshot: Pick<
PluginMetadataSnapshot,
"configFingerprint" | "index" | "policyHash" | "workspaceDir"
>;
config: OpenClawConfig;
env?: NodeJS.ProcessEnv;
workspaceDir?: string;
index?: InstalledPluginIndex;
}): boolean {
const env = params.env ?? process.env;
return (
params.snapshot.policyHash === resolveInstalledPluginIndexPolicyHash(params.config) &&
(!params.snapshot.configFingerprint ||
params.snapshot.configFingerprint ===
resolvePluginMetadataSnapshotConfigFingerprint(params.config, {
env,
policyHash: params.snapshot.policyHash,
})) &&
(params.snapshot.workspaceDir ?? "") === (params.workspaceDir ?? "") &&
indexesMatch(params.snapshot.index, params.index)
);
@@ -171,6 +183,10 @@ function loadPluginMetadataSnapshotImpl(
return {
policyHash: index.policyHash,
configFingerprint: resolvePluginMetadataSnapshotConfigFingerprint(params.config, {
env: params.env,
policyHash: index.policyHash,
}),
...(params.workspaceDir ? { workspaceDir: params.workspaceDir } : {}),
index,
registryDiagnostics: registryResult.diagnostics,

View File

@@ -35,6 +35,7 @@ export type PluginMetadataSnapshotRegistryDiagnostic = {
export type PluginMetadataSnapshot = {
policyHash: string;
configFingerprint?: string;
workspaceDir?: string;
index: InstalledPluginIndex;
registryDiagnostics: readonly PluginMetadataSnapshotRegistryDiagnostic[];

View File

@@ -137,7 +137,7 @@ describe("getCachedPluginModuleLoader", () => {
);
});
it("lets callers intentionally share loaders behind a custom cache scope key", async () => {
it("keeps cache scope keys separated by loader options", async () => {
const { createJiti, getCachedPluginModuleLoader } =
await loadCachedPluginModuleLoader("cache-scope-key");
@@ -165,6 +165,41 @@ describe("getCachedPluginModuleLoader", () => {
cacheScopeKey: "bundled:native",
});
expect(second).not.toBe(first);
first("/repo/dist/extensions/demo-a/api.js");
second("/repo/dist/extensions/demo-b/api.js");
expect(createJiti).toHaveBeenCalledTimes(2);
expect(cache.size).toBe(2);
});
it("lets callers explicitly share loaders behind an unsafe shared cache scope key", async () => {
const { createJiti, getCachedPluginModuleLoader } =
await loadCachedPluginModuleLoader("shared-cache-scope-key");
const cache = new Map();
const first = getCachedPluginModuleLoader({
cache,
modulePath: "/repo/dist/extensions/demo-a/api.js",
importerUrl: "file:///repo/src/plugins/public-surface-loader.ts",
loaderFilename: "file:///repo/src/plugins/public-surface-loader.ts",
aliasMap: {
demo: "/repo/demo-a.js",
},
tryNative: true,
sharedCacheScopeKey: "bundled:native",
});
const second = getCachedPluginModuleLoader({
cache,
modulePath: "/repo/dist/extensions/demo-b/api.js",
importerUrl: "file:///repo/src/plugins/public-surface-loader.ts",
loaderFilename: "file:///repo/src/plugins/public-surface-loader.ts",
aliasMap: {
demo: "/repo/demo-b.js",
},
tryNative: true,
sharedCacheScopeKey: "bundled:native",
});
expect(second).toBe(first);
second("/repo/dist/extensions/demo-b/api.js");
expect(createJiti).toHaveBeenCalledTimes(1);

View File

@@ -24,15 +24,9 @@ export function getCachedPluginModuleLoader(params: {
tryNative?: boolean;
pluginSdkResolution?: PluginSdkResolutionPreference;
cacheScopeKey?: string;
sharedCacheScopeKey?: string;
}): PluginModuleLoader {
const loaderFilename = toSafeImportPath(params.loaderFilename ?? params.modulePath);
if (params.cacheScopeKey) {
const scopedCacheKey = `${loaderFilename}::${params.cacheScopeKey}`;
const cached = params.cache.get(scopedCacheKey);
if (cached) {
return cached;
}
}
const hasAliasOverride = Boolean(params.aliasMap);
const hasTryNativeOverride = typeof params.tryNative === "boolean";
const defaultConfig =
@@ -71,7 +65,10 @@ export function getCachedPluginModuleLoader(params: {
tryNative,
aliasMap,
});
const scopedCacheKey = `${loaderFilename}::${params.cacheScopeKey ?? cacheKey}`;
const scopedCacheKey = `${loaderFilename}::${
params.sharedCacheScopeKey ??
(params.cacheScopeKey ? `${params.cacheScopeKey}::${cacheKey}` : cacheKey)
}`;
const cached = params.cache.get(scopedCacheKey);
if (cached) {
return cached;

View File

@@ -1,3 +1,4 @@
import crypto from "node:crypto";
import fs from "node:fs";
import path from "node:path";
import { resolveBundledPluginsDir } from "./bundled-dir.js";
@@ -109,6 +110,45 @@ function hasMismatchedPersistedBundledPluginRoot(
);
}
function hashExistingFile(filePath: string): string | null {
try {
return crypto.createHash("sha256").update(fs.readFileSync(filePath)).digest("hex");
} catch {
return null;
}
}
function resolveRecordPackageJsonPath(plugin: InstalledPluginIndexRecord): string | null {
const packageJsonPath = plugin.packageJson?.path;
if (!packageJsonPath) {
return null;
}
const rootDir = plugin.rootDir || path.dirname(plugin.manifestPath);
const resolved = path.resolve(rootDir, packageJsonPath);
const relative = path.relative(rootDir, resolved);
return relative.startsWith("..") || path.isAbsolute(relative) ? null : resolved;
}
function hasStalePersistedPluginMetadata(index: InstalledPluginIndex): boolean {
return index.plugins.some((plugin) => {
if (!hasOptionalMissingPluginManifestFile(plugin)) {
const manifestHash = hashExistingFile(plugin.manifestPath);
if (manifestHash && manifestHash !== plugin.manifestHash) {
return true;
}
}
const packageJsonPath = resolveRecordPackageJsonPath(plugin);
if (!plugin.packageJson?.hash) {
return false;
}
if (!packageJsonPath) {
return true;
}
const packageJsonHash = hashExistingFile(packageJsonPath);
return packageJsonHash !== plugin.packageJson.hash;
});
}
export function loadPluginRegistrySnapshotWithMetadata(
params: LoadPluginRegistryParams = {},
): PluginRegistrySnapshotResult {
@@ -154,6 +194,13 @@ export function loadPluginRegistrySnapshotWithMetadata(
message:
"Persisted plugin registry points at a different bundled plugin tree; using derived plugin index. Run `openclaw plugins registry --refresh` to update the persisted registry.",
});
} else if (hasStalePersistedPluginMetadata(persistedIndex)) {
diagnostics.push({
level: "warn",
code: "persisted-registry-stale-source",
message:
"Persisted plugin registry metadata no longer matches plugin manifest or package files; using derived plugin index. Run `openclaw plugins registry --refresh` to update the persisted registry.",
});
} else {
return {
snapshot: persistedIndex,

View File

@@ -1,3 +1,4 @@
import crypto from "node:crypto";
import fs from "node:fs";
import path from "node:path";
import { afterEach, describe, expect, it, vi } from "vitest";
@@ -54,6 +55,10 @@ function hermeticEnv(overrides: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv {
};
}
function hashFile(filePath: string): string {
return crypto.createHash("sha256").update(fs.readFileSync(filePath)).digest("hex");
}
function createCandidate(rootDir: string): PluginCandidate {
fs.writeFileSync(
path.join(rootDir, "index.ts"),
@@ -414,6 +419,7 @@ describe("plugin registry facade", () => {
{
...createIndex("persisted").plugins[0],
manifestPath: path.join(persistedRootDir, "openclaw.plugin.json"),
manifestHash: hashFile(path.join(persistedRootDir, "openclaw.plugin.json")),
source: path.join(persistedRootDir, "index.ts"),
rootDir: persistedRootDir,
},
@@ -464,6 +470,125 @@ describe("plugin registry facade", () => {
]);
});
it("falls back to the derived registry when persisted manifest metadata is stale", async () => {
const stateDir = makeTempDir();
const rootDir = makeTempDir();
const candidate = createCandidate(rootDir);
const config = {} as const;
const persisted = loadPluginRegistrySnapshot({
candidates: [candidate],
config,
env: hermeticEnv(),
preferPersisted: false,
});
await writePersistedInstalledPluginIndex(persisted, { stateDir });
fs.writeFileSync(
path.join(rootDir, "openclaw.plugin.json"),
JSON.stringify({
id: "demo",
name: "Demo",
configSchema: { type: "object" },
providers: ["demo", "demo-next"],
}),
"utf8",
);
const result = loadPluginRegistrySnapshotWithMetadata({
stateDir,
candidates: [candidate],
config,
env: hermeticEnv(),
});
expect(result.source).toBe("derived");
expect(result.diagnostics).toEqual([
expect.objectContaining({ code: "persisted-registry-stale-source" }),
]);
expect(result.snapshot.plugins[0]?.manifestHash).not.toBe(persisted.plugins[0]?.manifestHash);
});
it("falls back to the derived registry when persisted package metadata is stale", async () => {
const stateDir = makeTempDir();
const rootDir = makeTempDir();
fs.writeFileSync(
path.join(rootDir, "package.json"),
JSON.stringify({ name: "demo-plugin", version: "1.0.0" }),
"utf8",
);
const candidate = {
...createCandidate(rootDir),
packageDir: rootDir,
packageName: "demo-plugin",
packageVersion: "1.0.0",
} satisfies PluginCandidate;
const config = {} as const;
const persisted = loadPluginRegistrySnapshot({
candidates: [candidate],
config,
env: hermeticEnv(),
preferPersisted: false,
});
await writePersistedInstalledPluginIndex(persisted, { stateDir });
fs.writeFileSync(
path.join(rootDir, "package.json"),
JSON.stringify({ name: "demo-plugin", version: "1.0.1" }),
"utf8",
);
const result = loadPluginRegistrySnapshotWithMetadata({
stateDir,
candidates: [candidate],
config,
env: hermeticEnv(),
});
expect(result.source).toBe("derived");
expect(result.diagnostics).toEqual([
expect.objectContaining({ code: "persisted-registry-stale-source" }),
]);
expect(result.snapshot.plugins[0]?.packageJson?.hash).not.toBe(
persisted.plugins[0]?.packageJson?.hash,
);
});
it("falls back to the derived registry when persisted package metadata disappears", async () => {
const stateDir = makeTempDir();
const rootDir = makeTempDir();
fs.writeFileSync(
path.join(rootDir, "package.json"),
JSON.stringify({ name: "demo-plugin", version: "1.0.0" }),
"utf8",
);
const candidate = {
...createCandidate(rootDir),
packageDir: rootDir,
packageName: "demo-plugin",
packageVersion: "1.0.0",
} satisfies PluginCandidate;
const config = {} as const;
const persisted = loadPluginRegistrySnapshot({
candidates: [candidate],
config,
env: hermeticEnv(),
preferPersisted: false,
});
await writePersistedInstalledPluginIndex(persisted, { stateDir });
fs.rmSync(path.join(rootDir, "package.json"));
const result = loadPluginRegistrySnapshotWithMetadata({
stateDir,
candidates: [candidate],
config,
env: hermeticEnv(),
});
expect(result.source).toBe("derived");
expect(result.diagnostics).toEqual([
expect.objectContaining({ code: "persisted-registry-stale-source" }),
]);
expect(result.snapshot.plugins[0]?.packageJson).toBeUndefined();
});
it("falls back to the derived registry when persisted bundled roots point at another checkout", async () => {
const stateDir = makeTempDir();
const rootDir = makeTempDir();