From 62f8cff33a308ed5603c32d0ffd5072ddbbab43e Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Tue, 28 Apr 2026 00:15:10 +0100 Subject: [PATCH] fix: avoid full runtime dependency restaging --- CHANGELOG.md | 1 + src/gateway/server-startup-plugins.test.ts | 28 ++++++ src/plugins/bundled-runtime-deps.test.ts | 101 ++++++++++++++++++++- src/plugins/bundled-runtime-deps.ts | 70 +++++++++++--- 4 files changed, 184 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c6728bd1195..c71e22c72b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,7 @@ Docs: https://docs.openclaw.ai - Gateway/startup: keep primary-model startup prewarm on scoped metadata preparation, let native approval bootstraps retry outside channel startup, and skip the global hook runner when no `gateway_start` hook is registered, so clean post-ready sidecar work stays off the critical path. Refs #72846. Thanks @RayWoo, @livekm0309, and @mrz1836. - Gateway/supervisor: exit cleanly when a supervised restart finds an existing healthy gateway and bound retries when the existing gateway stays unhealthy, so stale lock contention cannot loop indefinitely. Refs #72846. Thanks @azgardtek. - Gateway/startup: scope primary-model provider discovery during channel prewarm to the configured provider owner and add split startup trace timings, so boot avoids staging unrelated bundled provider dependencies while setup discovery remains broad. Fixes #73002. Thanks @Schnup03. +- Plugins/runtime deps: declare retained staged bundled plugin dependencies in the npm staging manifest while installing only newly missing packages, so Gateway restarts avoid reinstalling the full retained dependency set when one runtime dependency is absent. Fixes #73055. Thanks @GCorp2026. - Channels/Microsoft Teams: unwrap staged CommonJS JWT runtime dependencies before Bot Connector token validation so inbound Teams messages no longer 401 after the bundled runtime-deps move. Fixes #73026. Thanks @kbrown10000. - Gateway/auth: allow local direct callers in trusted-proxy mode to use the configured gateway password as an internal fallback while keeping token fallback rejected. Fixes #17761. Thanks @dashed, @vincentkoc, and @jetd1. - Channels/sessions: prevent guarded inbound session recording from creating route-only phantom sessions while still allowing last-route updates for sessions that already exist. Carries forward #73009. Thanks @jzakirov. diff --git a/src/gateway/server-startup-plugins.test.ts b/src/gateway/server-startup-plugins.test.ts index 5360daf8bf0..9c9dfdf763f 100644 --- a/src/gateway/server-startup-plugins.test.ts +++ b/src/gateway/server-startup-plugins.test.ts @@ -235,6 +235,34 @@ describe("prepareGatewayPluginBootstrap runtime-deps staging", () => { ); }); + it("pre-stages only missing runtime deps while retaining the full startup dependency set", async () => { + scanBundledPluginRuntimeDeps.mockReturnValueOnce({ + deps: [ + { name: "alpha-runtime", version: "1.0.0", pluginIds: ["telegram"] }, + { name: "grammy", version: "1.37.0", pluginIds: ["telegram"] }, + ], + missing: [{ name: "grammy", version: "1.37.0", pluginIds: ["telegram"] }], + conflicts: [], + }); + const log = createLog(); + const { prepareGatewayPluginBootstrap } = await import("./server-startup-plugins.js"); + + await prepareGatewayPluginBootstrap({ + cfgAtStart: {}, + startupRuntimeConfig: {}, + minimalTestGateway: false, + log, + }); + + expect(repairBundledRuntimeDepsInstallRootAsync).toHaveBeenCalledWith( + expect.objectContaining({ + installRoot: "/runtime", + missingSpecs: ["grammy@1.37.0"], + installSpecs: ["alpha-runtime@1.0.0", "grammy@1.37.0"], + }), + ); + }); + it("derives startup activation from source config instead of runtime plugin defaults", async () => { const sourceConfig = { channels: { diff --git a/src/plugins/bundled-runtime-deps.test.ts b/src/plugins/bundled-runtime-deps.test.ts index 2edbad7e16e..ce60ae1fcec 100644 --- a/src/plugins/bundled-runtime-deps.test.ts +++ b/src/plugins/bundled-runtime-deps.test.ts @@ -445,6 +445,10 @@ describe("installBundledRuntimeDeps", () => { expect(JSON.parse(fs.readFileSync(path.join(cwd, "package.json"), "utf8"))).toEqual({ name: "openclaw-runtime-deps-install", private: true, + dependencies: { + "@grammyjs/runner": "^2.0.3", + grammy: "1.37.0", + }, }); writeInstalledPackage(cwd, "@grammyjs/runner", "2.0.3"); return { @@ -460,6 +464,7 @@ describe("installBundledRuntimeDeps", () => { installBundledRuntimeDeps({ installRoot, missingSpecs: ["@grammyjs/runner@^2.0.3"], + installSpecs: ["@grammyjs/runner@^2.0.3", "grammy@1.37.0"], env: { HOME: parentRoot, }, @@ -467,13 +472,47 @@ describe("installBundledRuntimeDeps", () => { expect(spawnSyncMock).toHaveBeenCalledWith( expect.any(String), - expect.any(Array), + expect.not.arrayContaining(["grammy@1.37.0"]), expect.objectContaining({ cwd: installRoot, }), ); }); + it("repairs external install roots by installing only missing specs while retaining staged deps", async () => { + const installRoot = makeTempDir(); + writeInstalledPackage(installRoot, "alpha-runtime", "1.0.0"); + spawnMock.mockImplementation((_command, args, options) => { + const cwd = String(options?.cwd ?? ""); + expect(args.slice(-3)).toEqual(["install", "--ignore-scripts", "beta-runtime@2.0.0"]); + expect(JSON.parse(fs.readFileSync(path.join(cwd, "package.json"), "utf8"))).toEqual({ + name: "openclaw-runtime-deps-install", + private: true, + dependencies: { + "alpha-runtime": "1.0.0", + "beta-runtime": "2.0.0", + }, + }); + writeInstalledPackage(cwd, "beta-runtime", "2.0.0"); + const child = new EventEmitter() as ReturnType; + Object.assign(child, { + stdout: new EventEmitter(), + stderr: new EventEmitter(), + }); + queueMicrotask(() => child.emit("close", 0, null)); + return child; + }); + + await repairBundledRuntimeDepsInstallRootAsync({ + installRoot, + missingSpecs: ["beta-runtime@2.0.0"], + installSpecs: ["alpha-runtime@1.0.0", "beta-runtime@2.0.0"], + env: {}, + }); + + expect(spawnMock).toHaveBeenCalledOnce(); + }); + it("warns but still installs bundled runtime deps when disk space looks low", () => { const installRoot = makeTempDir(); const warn = vi.fn(); @@ -541,6 +580,9 @@ describe("installBundledRuntimeDeps", () => { ).toEqual({ name: "openclaw-runtime-deps-install", private: true, + dependencies: { + tokenjuice: "0.6.1", + }, }); expect( JSON.parse( @@ -562,6 +604,60 @@ describe("installBundledRuntimeDeps", () => { ); }); + it("installs the full retained set when plugin-root staging replaces node_modules", () => { + const pluginRoot = makeTempDir(); + fs.writeFileSync( + path.join(pluginRoot, "package.json"), + JSON.stringify({ + dependencies: { + "alpha-runtime": "1.0.0", + "beta-runtime": "2.0.0", + }, + }), + ); + writeInstalledPackage(pluginRoot, "alpha-runtime", "1.0.0"); + spawnSyncMock.mockImplementation((_command, args, options) => { + const cwd = String(options?.cwd ?? ""); + expect(cwd).toBe(path.join(pluginRoot, ".openclaw-install-stage")); + expect((args ?? []).slice(-4)).toEqual([ + "install", + "--ignore-scripts", + "alpha-runtime@1.0.0", + "beta-runtime@2.0.0", + ]); + writeInstalledPackage(cwd, "alpha-runtime", "1.0.0"); + writeInstalledPackage(cwd, "beta-runtime", "2.0.0"); + return { + pid: 123, + output: [], + stdout: "", + stderr: "", + signal: null, + status: 0, + }; + }); + + expect( + ensureBundledPluginRuntimeDeps({ + env: {}, + pluginId: "local-plugin", + pluginRoot, + }), + ).toEqual({ + installedSpecs: ["beta-runtime@2.0.0"], + retainSpecs: ["alpha-runtime@1.0.0", "beta-runtime@2.0.0"], + }); + expect(spawnSyncMock).toHaveBeenCalledOnce(); + expect( + JSON.parse( + fs.readFileSync( + path.join(pluginRoot, "node_modules", "alpha-runtime", "package.json"), + "utf8", + ), + ), + ).toEqual({ name: "alpha-runtime", version: "1.0.0" }); + }); + it("uses an OpenClaw-owned npm cache for runtime dependency installs", () => { const installRoot = makeTempDir(); spawnSyncMock.mockImplementation((_command, _args, options) => { @@ -595,6 +691,9 @@ describe("installBundledRuntimeDeps", () => { expect(JSON.parse(fs.readFileSync(path.join(installRoot, "package.json"), "utf8"))).toEqual({ name: "openclaw-runtime-deps-install", private: true, + dependencies: { + tokenjuice: "0.6.1", + }, }); expect(spawnSyncMock).toHaveBeenCalledWith( expect.any(String), diff --git a/src/plugins/bundled-runtime-deps.ts b/src/plugins/bundled-runtime-deps.ts index 816aa119f8e..3bc10c64b22 100644 --- a/src/plugins/bundled-runtime-deps.ts +++ b/src/plugins/bundled-runtime-deps.ts @@ -1757,16 +1757,33 @@ function shouldCleanBundledRuntimeDepsInstallExecutionRoot(params: { return installExecutionRoot.startsWith(`${installRoot}${path.sep}`); } -function ensureNpmInstallExecutionManifest(installExecutionRoot: string): void { +function createNpmInstallExecutionManifest(installSpecs: readonly string[]): JsonObject { + const dependencies: Record = {}; + for (const spec of installSpecs) { + const dep = parseInstallableRuntimeDepSpec(spec); + dependencies[dep.name] = dep.version; + } + const sortedDependencies = Object.fromEntries( + Object.entries(dependencies).toSorted(([left], [right]) => left.localeCompare(right)), + ); + return { + name: "openclaw-runtime-deps-install", + private: true, + ...(Object.keys(sortedDependencies).length > 0 ? { dependencies: sortedDependencies } : {}), + }; +} + +function ensureNpmInstallExecutionManifest( + installExecutionRoot: string, + installSpecs: readonly string[] = [], +): void { const manifestPath = path.join(installExecutionRoot, "package.json"); - if (fs.existsSync(manifestPath)) { + const manifest = createNpmInstallExecutionManifest(installSpecs); + const nextContents = `${JSON.stringify(manifest, null, 2)}\n`; + if (fs.existsSync(manifestPath) && fs.readFileSync(manifestPath, "utf8") === nextContents) { return; } - fs.writeFileSync( - manifestPath, - `${JSON.stringify({ name: "openclaw-runtime-deps-install", private: true }, null, 2)}\n`, - "utf8", - ); + fs.writeFileSync(manifestPath, nextContents, "utf8"); } function formatBundledRuntimeDepsInstallError(result: { @@ -1885,6 +1902,7 @@ export function installBundledRuntimeDeps(params: { installExecutionRoot?: string; linkNodeModulesFromExecutionRoot?: boolean; missingSpecs: string[]; + installSpecs?: string[]; env: NodeJS.ProcessEnv; warn?: (message: string) => void; }): void { @@ -1911,7 +1929,15 @@ export function installBundledRuntimeDeps(params: { // doctor repair path installs directly in the external stage dir; without a // manifest, npm can honor a user's global prefix config and write under // $HOME/node_modules instead of our managed stage. - ensureNpmInstallExecutionManifest(installExecutionRoot); + // + // The manifest also declares retained staged deps. npm may prune packages + // that are present in node_modules but absent from package dependencies + // while installing a new explicit spec, so keep retained deps in the + // manifest and pass only actually missing specs as install args. + ensureNpmInstallExecutionManifest( + installExecutionRoot, + params.installSpecs ?? params.missingSpecs, + ); const installEnv = createBundledRuntimeDepsInstallEnv(params.env, { cacheDir: path.join(installExecutionRoot, ".openclaw-npm-cache"), }); @@ -1955,6 +1981,7 @@ export async function installBundledRuntimeDepsAsync(params: { installExecutionRoot?: string; linkNodeModulesFromExecutionRoot?: boolean; missingSpecs: string[]; + installSpecs?: string[]; env: NodeJS.ProcessEnv; warn?: (message: string) => void; onProgress?: (message: string) => void; @@ -1978,7 +2005,10 @@ export async function installBundledRuntimeDepsAsync(params: { if (diskWarning) { params.warn?.(diskWarning); } - ensureNpmInstallExecutionManifest(installExecutionRoot); + ensureNpmInstallExecutionManifest( + installExecutionRoot, + params.installSpecs ?? params.missingSpecs, + ); const installEnv = createBundledRuntimeDepsInstallEnv(params.env, { cacheDir: path.join(installExecutionRoot, ".openclaw-npm-cache"), }); @@ -2035,7 +2065,8 @@ export function repairBundledRuntimeDepsInstallRoot(params: { ((installParams) => installBundledRuntimeDeps({ installRoot: installParams.installRoot, - missingSpecs: installParams.installSpecs ?? installParams.missingSpecs, + missingSpecs: installParams.missingSpecs, + installSpecs: installParams.installSpecs, env: params.env, warn: params.warn, })); @@ -2129,7 +2160,8 @@ export async function repairBundledRuntimeDepsInstallRootAsync(params: { ((installParams) => installBundledRuntimeDepsAsync({ installRoot: installParams.installRoot, - missingSpecs: installParams.installSpecs ?? installParams.missingSpecs, + missingSpecs: installParams.missingSpecs, + installSpecs: installParams.installSpecs, env: params.env, warn: params.warn, onProgress: params.onProgress, @@ -2268,14 +2300,22 @@ export function ensureBundledPluginRuntimeDeps(params: { const install = params.installDeps ?? - ((installParams) => - installBundledRuntimeDeps({ + ((installParams) => { + const isolatedExecutionRoot = + installParams.installExecutionRoot && + path.resolve(installParams.installExecutionRoot) !== + path.resolve(installParams.installRoot); + return installBundledRuntimeDeps({ installRoot: installParams.installRoot, installExecutionRoot: installParams.installExecutionRoot, linkNodeModulesFromExecutionRoot: installParams.linkNodeModulesFromExecutionRoot, - missingSpecs: installParams.installSpecs ?? installParams.missingSpecs, + missingSpecs: isolatedExecutionRoot + ? (installParams.installSpecs ?? installParams.missingSpecs) + : installParams.missingSpecs, + installSpecs: installParams.installSpecs, env: params.env, - })); + }); + }); const finishActivity = beginBundledRuntimeDepsInstall({ installRoot, missingSpecs,