From 8bb4dd7d08de38ca9b6d0e0271dfa2f3a0dc2a50 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Tue, 21 Apr 2026 05:34:32 +0100 Subject: [PATCH] fix: quiet bundled plugin runtime dep repairs --- CHANGELOG.md | 1 + src/plugins/bundled-runtime-deps.test.ts | 143 +++++++++++++++++++++- src/plugins/bundled-runtime-deps.ts | 146 ++++++++++++++++++++--- src/plugins/loader.test.ts | 6 +- src/plugins/loader.ts | 10 +- 5 files changed, 276 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 61ea035c2ee..a3906c2ebe1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ Docs: https://docs.openclaw.ai - OpenAI/Responses: resolve `/think` levels against each GPT model's supported reasoning efforts so `/think off` no longer becomes high reasoning or sends unsupported `reasoning.effort: "none"` payloads. - Lobster/TaskFlow: allow managed approval resumes to use `approvalId` without a resume token, and persist that id in approval wait state. (#69559) Thanks @kirkluokun. +- Plugins/startup: install bundled runtime dependencies into each plugin's own runtime directory, reuse source-checkout repair caches after rebuilds, and log only packages that were actually installed so repeated Gateway starts stay quiet once deps are present. - Setup/TUI: relaunch the setup hatch TUI in a fresh process while preserving the configured gateway target and auth source, so onboarding recovers terminal state cleanly without exposing gateway secrets on command-line args. (#69524) Thanks @shakkernerd. - Codex: avoid re-exposing the image-generation tool on native vision turns with inbound images, and keep bare image-model overrides on the configured image provider. (#65061) Thanks @zhulijin1991. - Sessions/reset: clear auto-sourced model, provider, and auth-profile overrides on `/new` and `/reset` while preserving explicit user selections, so channel sessions stop staying pinned to runtime fallback choices. (#69419) Thanks @sk7n4k3d. diff --git a/src/plugins/bundled-runtime-deps.test.ts b/src/plugins/bundled-runtime-deps.test.ts index d91ed800b6b..65ddbf75e31 100644 --- a/src/plugins/bundled-runtime-deps.test.ts +++ b/src/plugins/bundled-runtime-deps.test.ts @@ -7,6 +7,7 @@ import { ensureBundledPluginRuntimeDeps, installBundledRuntimeDeps, resolveBundledRuntimeDepsNpmRunner, + type BundledRuntimeDepsInstallParams, } from "./bundled-runtime-deps.js"; vi.mock("node:child_process", () => ({ @@ -168,7 +169,7 @@ describe("installBundledRuntimeDeps", () => { }); describe("ensureBundledPluginRuntimeDeps", () => { - it("installs all direct plugin runtime deps when one is missing", () => { + it("installs plugin-local runtime deps when one is missing", () => { const packageRoot = makeTempDir(); const extensionsRoot = path.join(packageRoot, "dist", "extensions"); const pluginRoot = path.join(extensionsRoot, "bedrock"); @@ -182,11 +183,11 @@ describe("ensureBundledPluginRuntimeDeps", () => { }, }), ); - fs.mkdirSync(path.join(extensionsRoot, "node_modules", "already-present"), { + fs.mkdirSync(path.join(pluginRoot, "node_modules", "already-present"), { recursive: true, }); fs.writeFileSync( - path.join(extensionsRoot, "node_modules", "already-present", "package.json"), + path.join(pluginRoot, "node_modules", "already-present", "package.json"), JSON.stringify({ name: "already-present", version: "1.0.0" }), ); @@ -196,7 +197,7 @@ describe("ensureBundledPluginRuntimeDeps", () => { installSpecs?: string[]; }> = []; - const retainedSpecs = ensureBundledPluginRuntimeDeps({ + const result = ensureBundledPluginRuntimeDeps({ env: {}, installDeps: (params) => { calls.push(params); @@ -206,13 +207,143 @@ describe("ensureBundledPluginRuntimeDeps", () => { retainSpecs: ["previous@3.0.0"], }); - expect(retainedSpecs).toEqual(["already-present@1.0.0", "missing@2.0.0", "previous@3.0.0"]); + expect(result).toEqual({ + installedSpecs: ["missing@2.0.0"], + retainSpecs: ["already-present@1.0.0", "missing@2.0.0", "previous@3.0.0"], + }); expect(calls).toEqual([ { - installRoot: extensionsRoot, + installRoot: pluginRoot, missingSpecs: ["missing@2.0.0"], installSpecs: ["already-present@1.0.0", "missing@2.0.0", "previous@3.0.0"], }, ]); }); + + it("skips install when staged plugin-local runtime deps are present", () => { + const packageRoot = makeTempDir(); + const extensionsRoot = path.join(packageRoot, "dist", "extensions"); + const pluginRoot = path.join(extensionsRoot, "discord"); + fs.mkdirSync(path.join(pluginRoot, "node_modules", "@buape", "carbon"), { + recursive: true, + }); + fs.writeFileSync( + path.join(pluginRoot, "package.json"), + JSON.stringify({ + dependencies: { + "@buape/carbon": "0.16.0", + }, + }), + ); + fs.writeFileSync( + path.join(pluginRoot, "node_modules", "@buape", "carbon", "package.json"), + JSON.stringify({ name: "@buape/carbon", version: "0.16.0" }), + ); + + const result = ensureBundledPluginRuntimeDeps({ + env: {}, + installDeps: () => { + throw new Error("staged plugin-local deps should not reinstall"); + }, + pluginId: "discord", + pluginRoot, + }); + + expect(result).toEqual({ installedSpecs: [], retainSpecs: [] }); + }); + + it("does not treat sibling extension runtime deps as satisfying a plugin", () => { + const packageRoot = makeTempDir(); + const extensionsRoot = path.join(packageRoot, "dist", "extensions"); + const pluginRoot = path.join(extensionsRoot, "codex"); + fs.mkdirSync(path.join(extensionsRoot, "discord", "node_modules", "zod"), { + recursive: true, + }); + fs.mkdirSync(pluginRoot, { recursive: true }); + fs.writeFileSync( + path.join(pluginRoot, "package.json"), + JSON.stringify({ + dependencies: { + zod: "^4.3.6", + }, + }), + ); + fs.writeFileSync( + path.join(extensionsRoot, "discord", "node_modules", "zod", "package.json"), + JSON.stringify({ name: "zod", version: "4.3.6" }), + ); + const calls: BundledRuntimeDepsInstallParams[] = []; + + const result = ensureBundledPluginRuntimeDeps({ + env: {}, + installDeps: (params) => { + calls.push(params); + }, + pluginId: "codex", + pluginRoot, + }); + + expect(result).toEqual({ + installedSpecs: ["zod@^4.3.6"], + retainSpecs: ["zod@^4.3.6"], + }); + expect(calls).toEqual([ + { + installRoot: pluginRoot, + missingSpecs: ["zod@^4.3.6"], + installSpecs: ["zod@^4.3.6"], + }, + ]); + }); + + it("rehydrates source-checkout dist deps from cache after rebuilds", () => { + const packageRoot = makeTempDir(); + fs.mkdirSync(path.join(packageRoot, ".git"), { recursive: true }); + fs.mkdirSync(path.join(packageRoot, "src"), { recursive: true }); + fs.mkdirSync(path.join(packageRoot, "extensions"), { recursive: true }); + const pluginRoot = path.join(packageRoot, "dist", "extensions", "codex"); + fs.mkdirSync(pluginRoot, { recursive: true }); + fs.writeFileSync( + path.join(pluginRoot, "package.json"), + JSON.stringify({ + dependencies: { + zod: "^4.3.6", + }, + }), + ); + const installCalls: BundledRuntimeDepsInstallParams[] = []; + + const first = ensureBundledPluginRuntimeDeps({ + env: {}, + installDeps: (params) => { + installCalls.push(params); + fs.mkdirSync(path.join(params.installRoot, "node_modules", "zod"), { recursive: true }); + fs.writeFileSync( + path.join(params.installRoot, "node_modules", "zod", "package.json"), + JSON.stringify({ name: "zod", version: "4.3.6" }), + ); + }, + pluginId: "codex", + pluginRoot, + }); + + fs.rmSync(path.join(pluginRoot, "node_modules"), { recursive: true, force: true }); + + const second = ensureBundledPluginRuntimeDeps({ + env: {}, + installDeps: () => { + throw new Error("cached runtime deps should not reinstall"); + }, + pluginId: "codex", + pluginRoot, + }); + + expect(first).toEqual({ + installedSpecs: ["zod@^4.3.6"], + retainSpecs: ["zod@^4.3.6"], + }); + expect(second).toEqual({ installedSpecs: [], retainSpecs: [] }); + expect(installCalls).toHaveLength(1); + expect(fs.existsSync(path.join(pluginRoot, "node_modules", "zod", "package.json"))).toBe(true); + }); }); diff --git a/src/plugins/bundled-runtime-deps.ts b/src/plugins/bundled-runtime-deps.ts index ff1a99d52e4..d58fcad7800 100644 --- a/src/plugins/bundled-runtime-deps.ts +++ b/src/plugins/bundled-runtime-deps.ts @@ -1,4 +1,5 @@ import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; import fs from "node:fs"; import path from "node:path"; import { normalizeChatChannelId } from "../channels/ids.js"; @@ -24,6 +25,11 @@ export type BundledRuntimeDepsInstallParams = { installSpecs?: string[]; }; +export type BundledRuntimeDepsEnsureResult = { + installedSpecs: string[]; + retainSpecs: string[]; +}; + type JsonObject = Record; export type BundledRuntimeDepsNpmRunner = { @@ -72,6 +78,107 @@ function isSourceCheckoutBundledPluginRoot(pluginRoot: string): boolean { return isSourceCheckoutRoot(path.dirname(extensionsDir)); } +function resolveSourceCheckoutDistPackageRoot(pluginRoot: string): string | null { + const extensionsDir = path.dirname(pluginRoot); + const buildDir = path.dirname(extensionsDir); + if ( + path.basename(extensionsDir) !== "extensions" || + (path.basename(buildDir) !== "dist" && path.basename(buildDir) !== "dist-runtime") + ) { + return null; + } + const packageRoot = path.dirname(buildDir); + return isSourceCheckoutRoot(packageRoot) ? packageRoot : null; +} + +function createRuntimeDepsCacheKey(pluginId: string, specs: readonly string[]): string { + return createHash("sha256") + .update(pluginId) + .update("\0") + .update(specs.join("\0")) + .digest("hex") + .slice(0, 16); +} + +function resolveSourceCheckoutRuntimeDepsCacheDir(params: { + pluginId: string; + pluginRoot: string; + installSpecs: readonly string[]; +}): string | null { + const packageRoot = resolveSourceCheckoutDistPackageRoot(params.pluginRoot); + if (!packageRoot) { + return null; + } + return path.join( + packageRoot, + ".local", + "bundled-plugin-runtime-deps", + `${params.pluginId}-${createRuntimeDepsCacheKey(params.pluginId, params.installSpecs)}`, + ); +} + +function hasAllDependencySentinels(rootDir: string, deps: readonly { name: string }[]): boolean { + return deps.every((dep) => fs.existsSync(path.join(rootDir, dependencySentinelPath(dep.name)))); +} + +function replaceNodeModulesDir(targetDir: string, sourceDir: string): void { + const parentDir = path.dirname(targetDir); + const tempDir = fs.mkdtempSync(path.join(parentDir, ".openclaw-runtime-deps-copy-")); + const stagedDir = path.join(tempDir, "node_modules"); + try { + fs.cpSync(sourceDir, stagedDir, { recursive: true }); + fs.rmSync(targetDir, { recursive: true, force: true }); + fs.renameSync(stagedDir, targetDir); + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + } +} + +function restoreSourceCheckoutRuntimeDepsFromCache(params: { + cacheDir: string | null; + deps: readonly { name: string }[]; + installRoot: string; +}): boolean { + if (!params.cacheDir) { + return false; + } + const cachedNodeModulesDir = path.join(params.cacheDir, "node_modules"); + if (!hasAllDependencySentinels(params.cacheDir, params.deps)) { + return false; + } + try { + replaceNodeModulesDir(path.join(params.installRoot, "node_modules"), cachedNodeModulesDir); + return true; + } catch { + return false; + } +} + +function storeSourceCheckoutRuntimeDepsCache(params: { + cacheDir: string | null; + installRoot: string; +}): void { + if (!params.cacheDir) { + return; + } + const nodeModulesDir = path.join(params.installRoot, "node_modules"); + if (!fs.existsSync(nodeModulesDir)) { + return; + } + let tempDir: string | null = null; + try { + fs.mkdirSync(path.dirname(params.cacheDir), { recursive: true }); + tempDir = fs.mkdtempSync(path.join(path.dirname(params.cacheDir), ".runtime-deps-cache-")); + fs.cpSync(nodeModulesDir, path.join(tempDir, "node_modules"), { recursive: true }); + fs.rmSync(params.cacheDir, { recursive: true, force: true }); + fs.renameSync(tempDir, params.cacheDir); + } catch { + if (tempDir) { + fs.rmSync(tempDir, { recursive: true, force: true }); + } + } +} + function createNestedNpmInstallEnv(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv { const nextEnv = { ...env }; delete nextEnv.npm_config_global; @@ -342,15 +449,7 @@ export function scanBundledPluginRuntimeDeps(params: { } export function resolveBundledRuntimeDependencyInstallRoot(pluginRoot: string): string { - const extensionsDir = path.dirname(pluginRoot); - const buildDir = path.dirname(extensionsDir); - if ( - path.basename(extensionsDir) === "extensions" && - (path.basename(buildDir) === "dist" || path.basename(buildDir) === "dist-runtime") - ) { - return extensionsDir; - } - return extensionsDir; + return pluginRoot; } export function installBundledRuntimeDeps(params: { @@ -395,9 +494,9 @@ export function ensureBundledPluginRuntimeDeps(params: { config?: OpenClawConfig; retainSpecs?: readonly string[]; installDeps?: (params: BundledRuntimeDepsInstallParams) => void; -}): string[] { +}): BundledRuntimeDepsEnsureResult { if (isSourceCheckoutBundledPluginRoot(params.pluginRoot)) { - return []; + return { installedSpecs: [], retainSpecs: [] }; } if ( params.config && @@ -407,11 +506,11 @@ export function ensureBundledPluginRuntimeDeps(params: { pluginDir: params.pluginRoot, }) ) { - return []; + return { installedSpecs: [], retainSpecs: [] }; } const packageJson = readJsonObject(path.join(params.pluginRoot, "package.json")); if (!packageJson) { - return []; + return { installedSpecs: [], retainSpecs: [] }; } const deps = Object.entries(collectRuntimeDeps(packageJson)) .map(([name, rawVersion]) => @@ -421,7 +520,7 @@ export function ensureBundledPluginRuntimeDeps(params: { ) .filter((entry): entry is { name: string; version: string } => Boolean(entry)); if (deps.length === 0) { - return []; + return { installedSpecs: [], retainSpecs: [] }; } const installRoot = resolveBundledRuntimeDependencyInstallRoot(params.pluginRoot); @@ -433,11 +532,25 @@ export function ensureBundledPluginRuntimeDeps(params: { .map((dep) => `${dep.name}@${dep.version}`) .toSorted((left, right) => left.localeCompare(right)); if (missingSpecs.length === 0) { - return []; + return { installedSpecs: [], retainSpecs: [] }; } const installSpecs = [...new Set([...(params.retainSpecs ?? []), ...dependencySpecs])].toSorted( (left, right) => left.localeCompare(right), ); + const cacheDir = resolveSourceCheckoutRuntimeDepsCacheDir({ + pluginId: params.pluginId, + pluginRoot: params.pluginRoot, + installSpecs, + }); + if ( + restoreSourceCheckoutRuntimeDepsFromCache({ + cacheDir, + deps, + installRoot, + }) + ) { + return { installedSpecs: [], retainSpecs: [] }; + } const install = params.installDeps ?? @@ -448,5 +561,6 @@ export function ensureBundledPluginRuntimeDeps(params: { env: params.env, })); install({ installRoot, missingSpecs, installSpecs }); - return installSpecs; + storeSourceCheckoutRuntimeDepsCache({ cacheDir, installRoot }); + return { installedSpecs: missingSpecs, retainSpecs: installSpecs }; } diff --git a/src/plugins/loader.test.ts b/src/plugins/loader.test.ts index 6ab65153ac6..4f876827e95 100644 --- a/src/plugins/loader.test.ts +++ b/src/plugins/loader.test.ts @@ -917,7 +917,7 @@ module.exports = { }, bundledRuntimeDepsInstaller: ({ installRoot, missingSpecs }) => { installedSpecs.push(...missingSpecs); - expect(fs.realpathSync(installRoot)).toBe(fs.realpathSync(bundledDir)); + expect(fs.realpathSync(installRoot)).toBe(fs.realpathSync(plugin.dir)); fs.mkdirSync(path.join(installRoot, "node_modules", "discord-runtime"), { recursive: true, }); @@ -1030,7 +1030,7 @@ module.exports = { expect(registry.plugins.find((entry) => entry.id === "openai")?.status).toBe("loaded"); }); - it("retains earlier bundled runtime deps across sequential repairs", () => { + it("installs bundled runtime deps into each plugin root", () => { const bundledDir = makeTempDir(); const alpha = writePlugin({ id: "alpha", @@ -1110,7 +1110,7 @@ module.exports = { }, { missingSpecs: ["beta-runtime@1.0.0"], - installSpecs: ["alpha-runtime@1.0.0", "beta-runtime@1.0.0"], + installSpecs: ["beta-runtime@1.0.0"], }, ]); }); diff --git a/src/plugins/loader.ts b/src/plugins/loader.ts index 8a53494e751..12288504cae 100644 --- a/src/plugins/loader.ts +++ b/src/plugins/loader.ts @@ -1738,7 +1738,7 @@ export function loadOpenClawPlugins(options: PluginLoadOptions = {}): PluginRegi try { const installRoot = resolveBundledRuntimeDependencyInstallRoot(pluginRoot); const retainSpecs = bundledRuntimeDepsRetainSpecsByInstallRoot.get(installRoot) ?? []; - const installedSpecs = ensureBundledPluginRuntimeDeps({ + const depsInstallResult = ensureBundledPluginRuntimeDeps({ pluginId: record.id, pluginRoot, env, @@ -1746,15 +1746,15 @@ export function loadOpenClawPlugins(options: PluginLoadOptions = {}): PluginRegi retainSpecs, installDeps: options.bundledRuntimeDepsInstaller, }); - if (installedSpecs.length > 0) { + if (depsInstallResult.installedSpecs.length > 0) { bundledRuntimeDepsRetainSpecsByInstallRoot.set( installRoot, - [...new Set([...retainSpecs, ...installedSpecs])].toSorted((left, right) => - left.localeCompare(right), + [...new Set([...retainSpecs, ...depsInstallResult.retainSpecs])].toSorted( + (left, right) => left.localeCompare(right), ), ); logger.info( - `[plugins] ${record.id} installed bundled runtime deps: ${installedSpecs.join(", ")}`, + `[plugins] ${record.id} installed bundled runtime deps: ${depsInstallResult.installedSpecs.join(", ")}`, ); } } catch (error) {