diff --git a/CHANGELOG.md b/CHANGELOG.md index 72573c842e9..86330224cba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -140,6 +140,14 @@ Docs: https://docs.openclaw.ai - CLI/plugins: route lazy plugin command-registration chatter to stderr only during JSON-output command registration, keeping plugin-backed `--json` stdout parseable without changing parse-only or pass-through `--json` behavior. Fixes #81535. (#81536) Thanks @ScientificProgrammer and @vincentkoc. - Plugins: treat git plugin install refs as refs instead of checkout flags, so option-like selectors fail checkout instead of silently installing the default branch. Fixes #79898. (#79901) Thanks @afurm and @vincentkoc. - Doctor/memory: stop warning that no memory plugin is active when an enabled alternate memory plugin explicitly owns the memory slot, while preserving the warning for missing or disabled slot entries. Fixes #78540. (#78557) Thanks @carladams1299-lab and @vincentkoc. +- Plugins: keep process-local plugin metadata snapshot memo freshness tied to the cached registry snapshot so policy-stale derived plugin metadata edits invalidate the memo instead of returning stale owners or command aliases. (#81064) Thanks @Kaspre. +- Plugins: discover provider plugins from `setup.providers[].envVars` credentials during provider discovery while keeping the deprecated `providerAuthEnvVars` fallback. (#81542) Thanks @JARVIS-Glasses. +- Docs/Codex harness: clarify that per-agent `CODEX_HOME` isolates `~/.codex` while inherited `HOME` intentionally keeps `.agents` discovery and subprocess user-home state available. +- CLI/plugins: keep bare plugin and parent-command help on the lightweight path, avoiding plugin registry discovery before rendering help. +- Auth: reclaim dead-owner stale file locks before retrying locked writes, so crashed OAuth refreshes no longer wedge `auth-profiles.json` until manual cleanup. +- CLI tables: preserve muted/color styling on wrapped continuation lines after multiline cells, keeping `openclaw plugins list` descriptions readable. +- Process execution: collapse case-insensitive duplicate child environment keys on Windows so caller-provided overrides such as `PATH` cannot be shadowed by host `Path`. +- Browser CLI: request the existing `operator.admin` gateway scope explicitly for browser control commands, avoiding unnecessary scope-upgrade approval loops. Fixes #81555. (#81716) Thanks @joshavant. - Web: honor explicitly configured global `web_search` providers during provider ownership resolution while keeping sandboxed `web_fetch` limited to bundled providers. - Plugins/doctor: repair configured legacy npm declaration stubs by reinstalling their npm packages into the managed plugin root instead of loading workspace `node_modules`, and warn when discovery sees those stubs. Fixes #79632. Thanks @Dylanzhang1128 and @vincentkoc. - Channels: keep configured third-party channel plugins visible in `openclaw channels list` when their manifest declares `channels` but has not added `channelConfigs` metadata yet. Fixes #81334. (#81340) Thanks @AllynSheep and @vincentkoc. diff --git a/src/cli/update-cli.test.ts b/src/cli/update-cli.test.ts index d5861a6a78b..51ed62daa86 100644 --- a/src/cli/update-cli.test.ts +++ b/src/cli/update-cli.test.ts @@ -445,6 +445,7 @@ describe("update-cli", () => { previousHash: snapshot.hash ?? null, snapshot, nextConfig, + persistedHash: snapshot.hash ?? null, result: undefined, attempts: 1, afterWrite: { mode: "none", reason: "test" }, @@ -1190,6 +1191,7 @@ describe("update-cli", () => { previousHash: newerSnapshot.hash, snapshot: newerSnapshot, nextConfig, + persistedHash: newerSnapshot.hash, result: undefined, attempts: 2, afterWrite: { mode: "none", reason: "test" }, diff --git a/src/gateway/server.sessions.store-rpc.test.ts b/src/gateway/server.sessions.store-rpc.test.ts index f23309787a7..51fa391b80f 100644 --- a/src/gateway/server.sessions.store-rpc.test.ts +++ b/src/gateway/server.sessions.store-rpc.test.ts @@ -494,10 +494,16 @@ test("sessions.list configuredAgentsOnly hides disk-discovered unregistered agen "agent:main:main", ]); expect( - readFileSyncSpy.mock.calls.some( - ([file]) => - typeof file === "string" && fsSync.realpathSync.native(file) === realDiskOnlyStorePath, - ), + readFileSyncSpy.mock.calls.some(([file]) => { + if (typeof file !== "string") { + return false; + } + try { + return fsSync.realpathSync.native(file) === realDiskOnlyStorePath; + } catch { + return false; + } + }), ).toBe(false); } finally { readFileSyncSpy.mockRestore(); diff --git a/src/plugins/plugin-metadata-snapshot.memo.test.ts b/src/plugins/plugin-metadata-snapshot.memo.test.ts index f1ff75054a6..22180afe8d1 100644 --- a/src/plugins/plugin-metadata-snapshot.memo.test.ts +++ b/src/plugins/plugin-metadata-snapshot.memo.test.ts @@ -44,7 +44,107 @@ function touchPersistedIndex(stateDir: string, value = 1): void { fs.writeFileSync(indexPath, JSON.stringify({ value })); } -function makeIndex(pluginId = "demo"): InstalledPluginIndex { +function writeJson(filePath: string, value: unknown): void { + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.writeFileSync(filePath, `${JSON.stringify(value)}\n`); +} + +function writePersistedIndex(params: { + manifestPath?: string; + packageJsonPath?: string; + pluginId: string; + source?: string; + setupSource?: string; + stateDir: string; +}): void { + const pluginDir = path.join(params.stateDir, "extensions", params.pluginId); + const manifestPath = params.manifestPath ?? path.join(pluginDir, "openclaw.plugin.json"); + const packageJsonPath = params.packageJsonPath ?? path.join(pluginDir, "package.json"); + writeJson(path.join(params.stateDir, "plugins", "installs.json"), { + version: 1, + hostContractVersion: "test", + compatRegistryVersion: "test", + migrationVersion: 1, + policyHash: "test", + generatedAtMs: 1, + installRecords: {}, + diagnostics: [], + plugins: [ + { + pluginId: params.pluginId, + manifestPath, + manifestHash: `${params.pluginId}-manifest`, + rootDir: pluginDir, + ...(params.source ? { source: params.source } : {}), + ...(params.setupSource ? { setupSource: params.setupSource } : {}), + origin: "global", + enabled: true, + packageJson: { path: "package.json", hash: `${params.pluginId}-package` }, + startup: { + sidecar: false, + memory: false, + deferConfiguredChannelFullLoadUntilAfterListen: false, + agentHarnesses: [], + }, + compat: [], + }, + ], + }); + writeJson(manifestPath, { id: params.pluginId }); + writeJson(packageJsonPath, { name: params.pluginId }); +} + +function writeRecoverableNpmPlugin(params: { + packageName: string; + pluginId: string; + stateDir: string; + version: string; + writeRootManifest?: boolean; +}): void { + const packageDir = path.join(params.stateDir, "npm", "node_modules", params.packageName); + if (params.writeRootManifest !== false) { + writeJson(path.join(params.stateDir, "npm", "package.json"), { + dependencies: { + [params.packageName]: "1.0.0", + }, + }); + } + writeJson(path.join(packageDir, "package.json"), { + name: params.packageName, + version: params.version, + openclaw: { + extensions: ["."], + }, + }); + writeJson(path.join(packageDir, "openclaw.plugin.json"), { id: params.pluginId }); +} + +function writePersistedInstallRecords( + stateDir: string, + installRecords: Record>, +): void { + writeJson(path.join(stateDir, "plugins", "installs.json"), { + version: 1, + hostContractVersion: "test", + compatRegistryVersion: "test", + migrationVersion: 1, + policyHash: "test", + generatedAtMs: 1, + installRecords, + diagnostics: [], + plugins: [], + }); +} + +function makeIndex( + pluginId = "demo", + options: { + manifestPath?: string; + rootDir?: string; + } = {}, +): InstalledPluginIndex { + const rootDir = options.rootDir ?? `/plugins/${pluginId}`; + const manifestPath = options.manifestPath ?? path.join(rootDir, "openclaw.plugin.json"); return { version: 1, hostContractVersion: "test", @@ -57,9 +157,9 @@ function makeIndex(pluginId = "demo"): InstalledPluginIndex { plugins: [ { pluginId, - manifestPath: `/plugins/${pluginId}/openclaw.plugin.json`, + manifestPath, manifestHash: `${pluginId}-manifest`, - rootDir: `/plugins/${pluginId}`, + rootDir, origin: "global", enabled: true, startup: { @@ -175,6 +275,33 @@ describe("loadPluginMetadataSnapshot process memo", () => { expect(loadPluginRegistrySnapshotWithMetadata).toHaveBeenCalledOnce(); }); + it("refreshes policy-stale derived snapshots when derived plugin files change", () => { + const stateDir = tempStateDir(); + touchPersistedIndex(stateDir); + const pluginDir = path.join(stateDir, "current", "derived"); + const manifestPath = path.join(pluginDir, "openclaw.plugin.json"); + writeJson(manifestPath, { id: "derived", version: "1.0.0" }); + loadPluginRegistrySnapshotWithMetadata.mockReturnValue({ + source: "derived", + snapshot: makeIndex("derived", { manifestPath, rootDir: pluginDir }), + diagnostics: [ + { + level: "warn", + code: "persisted-registry-stale-policy", + message: "policy changed", + }, + ], + }); + loadPluginManifestRegistryForInstalledIndex.mockReturnValue(makeManifestRegistry("derived")); + + loadPluginMetadataSnapshot({ config: {}, env: {}, stateDir }); + writeJson(manifestPath, { id: "derived", version: "2.0.0", commandAliases: [{ name: "new" }] }); + loadPluginMetadataSnapshot({ config: {}, env: {}, stateDir }); + + expect(loadPluginRegistrySnapshotWithMetadata).toHaveBeenCalledTimes(2); + expect(loadPluginManifestRegistryForInstalledIndex).toHaveBeenCalledTimes(2); + }); + it.each([ ["persisted-registry-missing", undefined], ["persisted-registry-stale-source", undefined], @@ -210,4 +337,272 @@ describe("loadPluginMetadataSnapshot process memo", () => { expect(loadPluginRegistrySnapshotWithMetadata).toHaveBeenCalledTimes(2); }); + + it("reuses the expanded freshness fingerprint on hot cache hits", () => { + const stateDir = tempStateDir(); + const manifestPath = path.join(stateDir, "extensions", "demo", "openclaw.plugin.json"); + writePersistedIndex({ manifestPath, pluginId: "demo", stateDir }); + loadPluginRegistrySnapshotWithMetadata.mockReturnValue({ + source: "persisted", + snapshot: makeIndex(), + diagnostics: [], + }); + loadPluginMetadataSnapshot({ config: {}, env: {}, stateDir }); + const readSpy = vi.spyOn(fs, "readFileSync"); + + try { + loadPluginMetadataSnapshot({ config: {}, env: {}, stateDir }); + } finally { + readSpy.mockRestore(); + } + + expect(loadPluginRegistrySnapshotWithMetadata).toHaveBeenCalledTimes(1); + expect(loadPluginManifestRegistryForInstalledIndex).toHaveBeenCalledTimes(1); + expect(readSpy).not.toHaveBeenCalled(); + }); + + it.each([ + ["manifest", "openclaw.plugin.json", "manifestPath"], + ["source", "index.js", "source"], + ["setup source", "setup.js", "setupSource"], + ["package manifest", "package.json", "packageJsonPath"], + ])("refreshes when persisted plugin %s changes in the same process", (_, fileName, field) => { + const stateDir = tempStateDir(); + const filePath = path.join(stateDir, "extensions", "demo", fileName); + writePersistedIndex({ [field]: filePath, pluginId: "demo", stateDir }); + loadPluginRegistrySnapshotWithMetadata.mockReturnValue({ + source: "persisted", + snapshot: makeIndex(), + diagnostics: [], + }); + + loadPluginMetadataSnapshot({ config: {}, env: {}, stateDir }); + writeJson(filePath, { id: "demo", version: "0.2.0" }); + loadPluginMetadataSnapshot({ config: {}, env: {}, stateDir }); + + expect(loadPluginRegistrySnapshotWithMetadata).toHaveBeenCalledTimes(2); + expect(loadPluginManifestRegistryForInstalledIndex).toHaveBeenCalledTimes(2); + }); + + it.each([ + [ + "install path package manifest", + "~/tracked-plugin", + (recordPath: string) => ({ source: "path", installPath: recordPath }), + (homeDir: string) => path.join(homeDir, "tracked-plugin", "package.json"), + ], + [ + "source path package manifest", + "~/tracked-plugin", + (recordPath: string) => ({ source: "path", sourcePath: recordPath }), + (homeDir: string) => path.join(homeDir, "tracked-plugin", "package.json"), + ], + ])( + "refreshes when home-relative install record %s changes", + (_, recordPath, record, targetPath) => { + const stateDir = tempStateDir(); + const homeDir = path.join(stateDir, "home"); + const filePath = targetPath(homeDir); + writePersistedInstallRecords(stateDir, { demo: record(recordPath) }); + writeJson(filePath, { version: "1.0.0" }); + loadPluginRegistrySnapshotWithMetadata.mockReturnValue({ + source: "persisted", + snapshot: makeIndex(), + diagnostics: [], + }); + + loadPluginMetadataSnapshot({ config: {}, env: { HOME: homeDir }, stateDir }); + writeJson(filePath, { version: "1.0.1000" }); + loadPluginMetadataSnapshot({ config: {}, env: { HOME: homeDir }, stateDir }); + + expect(loadPluginRegistrySnapshotWithMetadata).toHaveBeenCalledTimes(2); + expect(loadPluginManifestRegistryForInstalledIndex).toHaveBeenCalledTimes(2); + }, + ); + + it("does not reuse home-relative install record watches across env changes", () => { + const stateDir = tempStateDir(); + const firstHomeDir = path.join(stateDir, "first-home"); + const secondHomeDir = path.join(stateDir, "second-home"); + const firstPackageJsonPath = path.join(firstHomeDir, "tracked-plugin", "package.json"); + const secondPackageJsonPath = path.join(secondHomeDir, "tracked-plugin", "package.json"); + writePersistedInstallRecords(stateDir, { + demo: { source: "path", installPath: "~/tracked-plugin" }, + }); + writeJson(firstPackageJsonPath, { version: "1.0.0" }); + writeJson(secondPackageJsonPath, { version: "1.0.0" }); + loadPluginRegistrySnapshotWithMetadata.mockReturnValue({ + source: "persisted", + snapshot: makeIndex(), + diagnostics: [], + }); + + loadPluginMetadataSnapshot({ config: {}, env: { HOME: firstHomeDir }, stateDir }); + loadPluginMetadataSnapshot({ config: {}, env: { HOME: secondHomeDir }, stateDir }); + writeJson(secondPackageJsonPath, { version: "1.0.1000" }); + loadPluginMetadataSnapshot({ config: {}, env: { HOME: secondHomeDir }, stateDir }); + + expect(loadPluginRegistrySnapshotWithMetadata).toHaveBeenCalledTimes(3); + expect(loadPluginManifestRegistryForInstalledIndex).toHaveBeenCalledTimes(3); + }); + + it("refreshes when recovered managed npm package metadata changes", () => { + const stateDir = tempStateDir(); + writeRecoverableNpmPlugin({ + packageName: "recovered-plugin", + pluginId: "recovered", + stateDir, + version: "1.0.0", + }); + loadPluginRegistrySnapshotWithMetadata.mockReturnValue({ + source: "persisted", + snapshot: makeIndex(), + diagnostics: [], + }); + + loadPluginMetadataSnapshot({ config: {}, env: {}, stateDir }); + writeRecoverableNpmPlugin({ + packageName: "recovered-plugin", + pluginId: "recovered", + stateDir, + version: "1.0.10", + writeRootManifest: false, + }); + loadPluginMetadataSnapshot({ config: {}, env: {}, stateDir }); + + expect(loadPluginRegistrySnapshotWithMetadata).toHaveBeenCalledTimes(2); + expect(loadPluginManifestRegistryForInstalledIndex).toHaveBeenCalledTimes(2); + }); + + it("refreshes when a declared recovered managed npm package appears", () => { + const stateDir = tempStateDir(); + writeJson(path.join(stateDir, "npm", "package.json"), { + dependencies: { + "late-plugin": "1.0.0", + }, + }); + loadPluginRegistrySnapshotWithMetadata.mockReturnValue({ + source: "persisted", + snapshot: makeIndex(), + diagnostics: [], + }); + + loadPluginMetadataSnapshot({ config: {}, env: {}, stateDir }); + writeRecoverableNpmPlugin({ + packageName: "late-plugin", + pluginId: "late-plugin", + stateDir, + version: "1.0.0", + writeRootManifest: false, + }); + loadPluginMetadataSnapshot({ config: {}, env: {}, stateDir }); + + expect(loadPluginRegistrySnapshotWithMetadata).toHaveBeenCalledTimes(2); + expect(loadPluginManifestRegistryForInstalledIndex).toHaveBeenCalledTimes(2); + }); + + it("refreshes when an in-root package manifest symlink target changes", () => { + const stateDir = tempStateDir(); + const pluginDir = path.join(stateDir, "extensions", "demo"); + const packageJsonPath = path.join(pluginDir, "package.json"); + const outsidePackageJsonPath = path.join(stateDir, "outside", "package.json"); + fs.mkdirSync(pluginDir, { recursive: true }); + writeJson(outsidePackageJsonPath, { name: "outside", version: "1.0.0" }); + fs.symlinkSync(outsidePackageJsonPath, packageJsonPath); + writePersistedIndex({ packageJsonPath, pluginId: "demo", stateDir }); + loadPluginRegistrySnapshotWithMetadata.mockReturnValue({ + source: "persisted", + snapshot: makeIndex(), + diagnostics: [], + }); + + loadPluginMetadataSnapshot({ config: {}, env: {}, stateDir }); + writeJson(outsidePackageJsonPath, { name: "outside", version: "1.0.1" }); + loadPluginMetadataSnapshot({ config: {}, env: {}, stateDir }); + + expect(loadPluginRegistrySnapshotWithMetadata).toHaveBeenCalledTimes(2); + expect(loadPluginManifestRegistryForInstalledIndex).toHaveBeenCalledTimes(2); + }); + + it("does not fingerprint persisted plugin paths outside the plugin root", () => { + const stateDir = tempStateDir(); + const outsideManifestPath = path.join(stateDir, "outside", "openclaw.plugin.json"); + const outsideSourcePath = path.join(stateDir, "outside", "index.js"); + writePersistedIndex({ + manifestPath: outsideManifestPath, + pluginId: "demo", + source: outsideSourcePath, + stateDir, + }); + loadPluginRegistrySnapshotWithMetadata.mockReturnValue({ + source: "persisted", + snapshot: makeIndex(), + diagnostics: [], + }); + const statSpy = vi.spyOn(fs, "statSync"); + const readSpy = vi.spyOn(fs, "readFileSync"); + + try { + loadPluginMetadataSnapshot({ config: {}, env: {}, stateDir }); + } finally { + statSpy.mockRestore(); + readSpy.mockRestore(); + } + + expect(statSpy.mock.calls.some(([filePath]) => filePath === outsideManifestPath)).toBe(false); + expect(statSpy.mock.calls.some(([filePath]) => filePath === outsideSourcePath)).toBe(false); + expect(readSpy.mock.calls.some(([filePath]) => filePath === outsideManifestPath)).toBe(false); + expect(readSpy.mock.calls.some(([filePath]) => filePath === outsideSourcePath)).toBe(false); + }); + + it("does not hash symlinked persisted plugin files that escape the plugin root", () => { + const stateDir = tempStateDir(); + const pluginDir = path.join(stateDir, "extensions", "demo"); + const manifestPath = path.join(pluginDir, "openclaw.plugin.json"); + const outsideManifestPath = path.join(stateDir, "outside", "openclaw.plugin.json"); + fs.mkdirSync(pluginDir, { recursive: true }); + writeJson(outsideManifestPath, { id: "outside" }); + fs.symlinkSync(outsideManifestPath, manifestPath); + writeJson(path.join(stateDir, "plugins", "installs.json"), { + version: 1, + hostContractVersion: "test", + compatRegistryVersion: "test", + migrationVersion: 1, + policyHash: "test", + generatedAtMs: 1, + installRecords: {}, + diagnostics: [], + plugins: [ + { + pluginId: "demo", + manifestPath, + manifestHash: "demo-manifest", + rootDir: pluginDir, + origin: "global", + enabled: true, + startup: { + sidecar: false, + memory: false, + deferConfiguredChannelFullLoadUntilAfterListen: false, + agentHarnesses: [], + }, + compat: [], + }, + ], + }); + loadPluginRegistrySnapshotWithMetadata.mockReturnValue({ + source: "persisted", + snapshot: makeIndex(), + diagnostics: [], + }); + const readSpy = vi.spyOn(fs, "readFileSync"); + + try { + loadPluginMetadataSnapshot({ config: {}, env: {}, stateDir }); + } finally { + readSpy.mockRestore(); + } + + expect(readSpy.mock.calls.some(([filePath]) => filePath === outsideManifestPath)).toBe(false); + }); }); diff --git a/src/plugins/plugin-metadata-snapshot.ts b/src/plugins/plugin-metadata-snapshot.ts index 1b8031e3dea..0051d975de3 100644 --- a/src/plugins/plugin-metadata-snapshot.ts +++ b/src/plugins/plugin-metadata-snapshot.ts @@ -1,3 +1,4 @@ +import fs from "node:fs"; import path from "node:path"; import { resolveIsNixMode } from "../config/paths.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; @@ -8,8 +9,9 @@ import { import { resolveUserPath } from "../utils.js"; import { resolveCompatibilityHostVersion } from "../version.js"; import { resolveDefaultPluginNpmDir } from "./install-paths.js"; -import { hashJson, safeFileSignature } from "./installed-plugin-index-hash.js"; +import { hashJson } from "./installed-plugin-index-hash.js"; import { resolveInstalledPluginIndexPolicyHash } from "./installed-plugin-index-policy.js"; +import { loadInstalledPluginIndexInstallRecordsSync } from "./installed-plugin-index-record-reader.js"; import { resolveInstalledPluginIndexStorePath } from "./installed-plugin-index-store-path.js"; import type { InstalledPluginIndex } from "./installed-plugin-index.js"; import { @@ -31,9 +33,18 @@ import { type PluginMetadataSnapshotMemo = { key: string; + registryState?: PersistedRegistryMemoState; snapshot: PluginMetadataSnapshot; }; +type PersistedRegistryMemoState = { + contextHash: string; + fastHash: string; + fingerprint: unknown; + watchedFilesHash: string; + watchedFiles: readonly string[]; +}; + let pluginMetadataSnapshotMemo: PluginMetadataSnapshotMemo | undefined; export function clearLoadPluginMetadataSnapshotMemo(): void { @@ -66,11 +77,200 @@ export type { } from "./plugin-metadata-snapshot.types.js"; function fileFingerprint(filePath: string): unknown { - const signature = safeFileSignature(filePath); - if (!signature) { + try { + const stat = fs.statSync(filePath, { bigint: true }); + const kind = stat.isFile() ? "file" : stat.isDirectory() ? "dir" : "other"; + return [filePath, kind, stat.size.toString(), stat.mtimeNs.toString(), stat.ctimeNs.toString()]; + } catch { return [filePath, "missing"]; } - return [filePath, signature.size, signature.mtimeMs, signature.ctimeMs]; +} + +function isRecord(value: unknown): value is Record { + return Boolean(value) && typeof value === "object" && !Array.isArray(value); +} + +function readJsonObject(filePath: string): Record | undefined { + try { + const parsed = JSON.parse(fs.readFileSync(filePath, "utf8")); + return isRecord(parsed) ? parsed : undefined; + } catch { + return undefined; + } +} + +function normalizeString(value: unknown): string | undefined { + return typeof value === "string" && value.trim() ? value.trim() : undefined; +} + +function stableMemoValue(value: unknown): unknown { + if (Array.isArray(value)) { + return value.map(stableMemoValue); + } + if (!isRecord(value)) { + return value; + } + return Object.fromEntries( + Object.entries(value) + .toSorted(([left], [right]) => left.localeCompare(right)) + .map(([key, entry]) => [key, stableMemoValue(entry)]), + ); +} + +function isPathInsideOrEqual(childPath: string, parentPath: string): boolean { + const relative = path.relative(parentPath, childPath); + return relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative)); +} + +function tryRealpath(filePath: string): string | null { + try { + return fs.realpathSync(filePath); + } catch { + return null; + } +} + +function resolvePluginFilePath( + pluginDir: string, + filePath: string | undefined, + options: { allowSymlinkOutsideRoot?: boolean } = {}, +): + | { status: "ok"; path: string } + | { status: "outside-root"; path: string } + | { status: "missing-root"; path: string } { + if (!filePath) { + return { status: "missing-root", path: "" }; + } + const rootDir = path.resolve(pluginDir); + const resolved = path.isAbsolute(filePath) + ? path.resolve(filePath) + : path.resolve(rootDir, filePath); + if (!isPathInsideOrEqual(resolved, rootDir)) { + return { status: "outside-root", path: resolved }; + } + const rootRealPath = tryRealpath(rootDir); + const targetRealPath = tryRealpath(resolved); + if ( + rootRealPath && + targetRealPath && + !isPathInsideOrEqual(targetRealPath, rootRealPath) && + !options.allowSymlinkOutsideRoot + ) { + return { status: "outside-root", path: resolved }; + } + return { status: "ok", path: resolved }; +} + +function persistedPluginFileFingerprint( + rootDir: string | undefined, + filePath: string | undefined, + options: { allowSymlinkOutsideRoot?: boolean; watchedFiles?: Set } = {}, +): unknown { + if (!filePath) { + return null; + } + if (!rootDir) { + return [filePath, "missing-root"]; + } + const resolved = resolvePluginFilePath(rootDir, filePath, { + allowSymlinkOutsideRoot: options.allowSymlinkOutsideRoot, + }); + if (resolved.status !== "ok") { + return [filePath, resolved.status]; + } + options.watchedFiles?.add(resolved.path); + return fileFingerprint(resolved.path); +} + +function watchedFileFingerprint(filePath: string | undefined, watchedFiles: Set): unknown { + if (!filePath) { + return null; + } + watchedFiles.add(filePath); + return fileFingerprint(filePath); +} + +function resolveInstallRecordPath(value: unknown, env: NodeJS.ProcessEnv): string | undefined { + const normalized = normalizeString(value); + return normalized ? resolveUserPath(normalized, env) : undefined; +} + +function installRecordPathFingerprints( + env: NodeJS.ProcessEnv, + records: unknown, + watchedFiles: Set, +): readonly unknown[] { + if (!isRecord(records)) { + return []; + } + return Object.entries(records) + .toSorted(([left], [right]) => left.localeCompare(right)) + .map(([pluginId, rawRecord]) => { + if (!isRecord(rawRecord)) { + return [pluginId, rawRecord]; + } + const installPath = normalizeString(rawRecord.installPath); + const sourcePath = normalizeString(rawRecord.sourcePath); + const resolvedInstallPath = resolveInstallRecordPath(rawRecord.installPath, env); + const resolvedSourcePath = resolveInstallRecordPath(rawRecord.sourcePath, env); + return [ + pluginId, + installPath, + sourcePath, + watchedFileFingerprint( + resolvedInstallPath ? path.join(resolvedInstallPath, "package.json") : undefined, + watchedFiles, + ), + watchedFileFingerprint( + resolvedInstallPath ? path.join(resolvedInstallPath, "openclaw.plugin.json") : undefined, + watchedFiles, + ), + watchedFileFingerprint(resolvedSourcePath, watchedFiles), + watchedFileFingerprint( + resolvedSourcePath ? path.join(resolvedSourcePath, "package.json") : undefined, + watchedFiles, + ), + watchedFileFingerprint( + resolvedSourcePath ? path.join(resolvedSourcePath, "openclaw.plugin.json") : undefined, + watchedFiles, + ), + ]; + }); +} + +function managedNpmDependencyMetadataFingerprints( + npmRoot: string, + watchedFiles: Set, +): readonly unknown[] { + const rootManifest = readJsonObject(path.join(npmRoot, "package.json")); + const dependencies = isRecord(rootManifest?.dependencies) ? rootManifest.dependencies : {}; + const nodeModulesRoot = path.join(npmRoot, "node_modules"); + return Object.entries(dependencies) + .toSorted(([left], [right]) => left.localeCompare(right)) + .map(([packageName, rawSpec]) => { + const dependencySpec = normalizeString(rawSpec); + if (!dependencySpec) { + return [packageName, rawSpec]; + } + const packageDir = path.resolve(nodeModulesRoot, packageName); + if (!isPathInsideOrEqual(packageDir, path.resolve(nodeModulesRoot))) { + return [packageName, dependencySpec, "outside-node-modules"]; + } + return [ + packageName, + dependencySpec, + watchedFileFingerprint(path.join(packageDir, "package.json"), watchedFiles), + watchedFileFingerprint(path.join(packageDir, "openclaw.plugin.json"), watchedFiles), + ]; + }); +} + +function resolveRecordPackageJsonPath(record: Record): string | undefined { + const packageJson = record.packageJson; + if (!isRecord(packageJson)) { + return undefined; + } + return normalizeString(packageJson.path); } function pickMemoRelevantEnv(env: NodeJS.ProcessEnv): Record { @@ -134,11 +334,21 @@ function clonePluginMetadataSnapshot(snapshot: PluginMetadataSnapshot): PluginMe }; } -function resolvePersistedRegistryMemoFingerprint(params: { +function resolvePersistedRegistryFastMemoFingerprint(params: { env: NodeJS.ProcessEnv; preferPersisted?: boolean; stateDir?: string; -}): unknown { +}): Record { + const disabledByEnv = params.env.OPENCLAW_DISABLE_PERSISTED_PLUGIN_REGISTRY?.trim().toLowerCase(); + const disabled = + params.preferPersisted === false || + (Boolean(disabledByEnv) && + disabledByEnv !== "0" && + disabledByEnv !== "false" && + disabledByEnv !== "no"); + if (disabled) { + return { disabled: true }; + } const indexPath = resolveInstalledPluginIndexStorePath({ env: params.env, ...(params.stateDir ? { stateDir: params.stateDir } : {}), @@ -147,24 +357,184 @@ function resolvePersistedRegistryMemoFingerprint(params: { ? path.join(params.stateDir, "npm") : resolveDefaultPluginNpmDir(params.env); return { - disabled: - params.preferPersisted === false || params.env.OPENCLAW_DISABLE_PERSISTED_PLUGIN_REGISTRY, index: fileFingerprint(indexPath), npmPackageJson: fileFingerprint(path.join(npmRoot, "package.json")), }; } -function computePluginMetadataSnapshotMemoKey(params: LoadPluginMetadataSnapshotParams): string { - const env = params.env ?? process.env; - const indexFingerprint = params.index - ? resolveInstalledManifestRegistryIndexFingerprint(params.index) +function resolvePersistedRegistryMemoContextHash(params: { + env: NodeJS.ProcessEnv; + fastFingerprint: unknown; + preferPersisted?: boolean; + stateDir?: string; +}): string { + return hashJson({ + env: pickMemoRelevantEnv(params.env), + fastFingerprint: params.fastFingerprint, + preferPersisted: params.preferPersisted ?? null, + stateDir: params.stateDir ?? null, + }); +} + +function hashWatchedFiles(watchedFiles: readonly string[]): string { + return hashJson(watchedFiles.map((filePath) => fileFingerprint(filePath))); +} + +function resolvePersistedRegistryMemoState(params: { + env: NodeJS.ProcessEnv; + index?: InstalledPluginIndex; + preferPersisted?: boolean; + stateDir?: string; +}): PersistedRegistryMemoState { + const fastFingerprint = resolvePersistedRegistryFastMemoFingerprint(params); + const fastHash = hashJson(fastFingerprint); + const contextHash = resolvePersistedRegistryMemoContextHash({ + ...params, + fastFingerprint, + }); + if (isRecord(fastFingerprint) && fastFingerprint.disabled === true) { + return { + contextHash, + fastHash, + fingerprint: fastFingerprint, + watchedFiles: [], + watchedFilesHash: hashJson([]), + }; + } + const indexPath = resolveInstalledPluginIndexStorePath({ + env: params.env, + ...(params.stateDir ? { stateDir: params.stateDir } : {}), + }); + const npmRoot = params.stateDir + ? path.join(params.stateDir, "npm") + : resolveDefaultPluginNpmDir(params.env); + const index = params.index ?? readJsonObject(indexPath); + const plugins = Array.isArray(index?.plugins) ? index.plugins : []; + const diagnostics = Array.isArray(index?.diagnostics) ? index.diagnostics : []; + const pluginRootById = new Map(); + const watchedFiles = new Set(); + for (const rawPlugin of plugins) { + if (!isRecord(rawPlugin)) { + continue; + } + const pluginId = normalizeString(rawPlugin.pluginId); + const rootDir = normalizeString(rawPlugin.rootDir); + if (pluginId && rootDir) { + pluginRootById.set(pluginId, rootDir); + } + } + const installRecords = + params.index?.installRecords ?? + loadInstalledPluginIndexInstallRecordsSync({ + env: params.env, + ...(params.stateDir ? { stateDir: params.stateDir } : {}), + }); + const watchedPlugins = plugins.map((rawPlugin) => { + if (!isRecord(rawPlugin)) { + return rawPlugin; + } + const rootDir = normalizeString(rawPlugin.rootDir); + const manifestPath = normalizeString(rawPlugin.manifestPath); + const packageJsonPath = resolveRecordPackageJsonPath(rawPlugin); + const source = normalizeString(rawPlugin.source); + const setupSource = normalizeString(rawPlugin.setupSource); + return [ + normalizeString(rawPlugin.pluginId), + rootDir, + rootDir ? fileFingerprint(rootDir) : null, + manifestPath, + persistedPluginFileFingerprint(rootDir, manifestPath, { watchedFiles }), + source, + persistedPluginFileFingerprint(rootDir, source, { watchedFiles }), + setupSource, + persistedPluginFileFingerprint(rootDir, setupSource, { watchedFiles }), + packageJsonPath, + persistedPluginFileFingerprint(rootDir, packageJsonPath, { + allowSymlinkOutsideRoot: true, + watchedFiles, + }), + ]; + }); + const watchedDiagnostics = diagnostics.map((rawDiagnostic) => { + if (!isRecord(rawDiagnostic)) { + return rawDiagnostic; + } + const pluginId = normalizeString(rawDiagnostic.pluginId); + const source = normalizeString(rawDiagnostic.source); + return [ + pluginId, + source, + persistedPluginFileFingerprint(pluginId ? pluginRootById.get(pluginId) : undefined, source, { + watchedFiles, + }), + ]; + }); + const installRecordFiles = installRecordPathFingerprints( + params.env, + installRecords, + watchedFiles, + ); + const managedNpmDependencyFiles = managedNpmDependencyMetadataFingerprints(npmRoot, watchedFiles); + const watchedFilesList = [...watchedFiles].toSorted(); + return { + contextHash, + fastHash, + fingerprint: { + ...fastFingerprint, + indexHash: hashJson(stableMemoValue(index) ?? null), + installRecords: hashJson(stableMemoValue(installRecords)), + installRecordFiles, + managedNpmDependencyFiles, + npmPackageJson: fileFingerprint(path.join(npmRoot, "package.json")), + plugins: watchedPlugins, + diagnostics: watchedDiagnostics, + }, + watchedFiles: watchedFilesList, + watchedFilesHash: hashWatchedFiles(watchedFilesList), + }; +} + +function resolvePersistedRegistryMemoStateForLookup( + params: { + env: NodeJS.ProcessEnv; + preferPersisted?: boolean; + stateDir?: string; + }, + memo: PluginMetadataSnapshotMemo | undefined, +): PersistedRegistryMemoState { + const fastFingerprint = resolvePersistedRegistryFastMemoFingerprint(params); + const fastHash = hashJson(fastFingerprint); + const contextHash = resolvePersistedRegistryMemoContextHash({ + ...params, + fastFingerprint, + }); + const registryState = memo?.registryState; + if ( + registryState && + registryState.contextHash === contextHash && + registryState.fastHash === fastHash && + hashWatchedFiles(registryState.watchedFiles) === registryState.watchedFilesHash + ) { + return registryState; + } + return resolvePersistedRegistryMemoState(params); +} + +function computePluginMetadataSnapshotMemoKey(params: { + params: LoadPluginMetadataSnapshotParams; + registryState: PersistedRegistryMemoState; +}): string { + const { params: snapshotParams, registryState } = params; + const env = snapshotParams.env ?? process.env; + const indexFingerprint = snapshotParams.index + ? resolveInstalledManifestRegistryIndexFingerprint(snapshotParams.index) : undefined; return hashJson({ controlPlane: resolvePluginControlPlaneFingerprint({ - config: params.config, + config: snapshotParams.config, env, - workspaceDir: params.workspaceDir, - policyHash: resolveInstalledPluginIndexPolicyHash(params.config), + workspaceDir: snapshotParams.workspaceDir, + policyHash: resolveInstalledPluginIndexPolicyHash(snapshotParams.config), ...(indexFingerprint ? { inventoryFingerprint: indexFingerprint } : {}), }), cwd: process.cwd(), @@ -174,14 +544,10 @@ function computePluginMetadataSnapshotMemoKey(params: LoadPluginMetadataSnapshot compatibilityHostVersion: resolveCompatibilityHostVersion(env), nixMode: resolveIsNixMode(env), }, - preferPersisted: params.preferPersisted ?? null, - registry: resolvePersistedRegistryMemoFingerprint({ - env, - ...(params.stateDir ? { stateDir: resolveUserPath(params.stateDir, env) } : {}), - ...(params.preferPersisted !== undefined ? { preferPersisted: params.preferPersisted } : {}), - }), - stateDir: params.stateDir ? resolveUserPath(params.stateDir, env) : null, - workspaceDir: params.workspaceDir ?? null, + preferPersisted: snapshotParams.preferPersisted ?? null, + registry: registryState.fingerprint, + stateDir: snapshotParams.stateDir ? resolveUserPath(snapshotParams.stateDir, env) : null, + workspaceDir: snapshotParams.workspaceDir ?? null, }); } @@ -330,16 +696,23 @@ export function listPluginOriginsFromMetadataSnapshot( return new Map(snapshot.plugins.map((record) => [record.id, record.origin])); } -// Process-local memoization is keyed by stable registry/config/env inputs. It -// intentionally does not watch arbitrary direct plugin file edits after a -// persisted registry has been accepted; registry refreshes and process restarts -// are the freshness boundaries for that broader edit flow. +// Process-local memoization keeps the hot snapshot work cached while checking +// the persisted metadata files that the installed-index loader consumes. export function loadPluginMetadataSnapshot( params: LoadPluginMetadataSnapshotParams, ): PluginMetadataSnapshot { const activeTimelineSpan = getActiveDiagnosticsTimelineSpan(); - const memoKey = computePluginMetadataSnapshotMemoKey(params); const memo = pluginMetadataSnapshotMemo; + const env = params.env ?? process.env; + const registryState = resolvePersistedRegistryMemoStateForLookup( + { + env, + ...(params.stateDir ? { stateDir: resolveUserPath(params.stateDir, env) } : {}), + ...(params.preferPersisted !== undefined ? { preferPersisted: params.preferPersisted } : {}), + }, + memo, + ); + const memoKey = computePluginMetadataSnapshotMemoKey({ params, registryState }); if (memo?.key === memoKey) { return measureDiagnosticsTimelineSpanSync( "plugins.metadata.scan", @@ -371,8 +744,20 @@ export function loadPluginMetadataSnapshot( }, ); if (canMemoizePluginMetadataSnapshotResult(result)) { + const cachedRegistryState = + result.registrySource === "derived" + ? resolvePersistedRegistryMemoState({ + env, + index: result.snapshot.index, + ...(params.stateDir ? { stateDir: resolveUserPath(params.stateDir, env) } : {}), + ...(params.preferPersisted !== undefined + ? { preferPersisted: params.preferPersisted } + : {}), + }) + : registryState; pluginMetadataSnapshotMemo = { - key: memoKey, + key: computePluginMetadataSnapshotMemoKey({ params, registryState: cachedRegistryState }), + registryState: cachedRegistryState, snapshot: clonePluginMetadataSnapshot(result.snapshot), }; }