diff --git a/CHANGELOG.md b/CHANGELOG.md index c1dda3b3927..cda0d18499e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -107,7 +107,10 @@ Docs: https://docs.openclaw.ai - Plugins: cache unchanged plugin manifest loads by file signature, reducing repeated JSON/JSON5 parsing and manifest normalization in bursty startup and runtime registry paths. Refs #73532 and #73647; carries forward #73678. Thanks @TheDutchRuler. - Plugins/runtime-deps: cache unchanged bundled runtime mirror dist-file materialization decisions and close file-lock handles on owner-write failures, reducing repeated startup chunk scans and avoiding FileHandle-GC recovery stalls. Refs #73532. Thanks @oadiazp and @bstanbury. - Plugins/runtime-deps: retry and defer transient cleanup failures for owned runtime staging directories so CLI startup no longer aborts after a successful bundled dependency swap. Refs #73903. Thanks @bobfreeman1989. -- Plugins/runtime-deps: cache bundled runtime-deps JSON/package files and root chunk import scans by file signature, reducing repeated staged-runtime scanning during bundled channel startup. Refs #73647 and #73705. Thanks @mattmcintyre and @bmilne1981. +- Plugins/runtime-deps: cache bundled runtime-deps JSON/package files by file signature, reducing repeated staged-runtime metadata reads during bundled channel startup. Refs #73647 and #73705. Thanks @mattmcintyre and @bmilne1981. +- Plugins/runtime-deps: delegate bundled plugin dependency staging to complete npm/pnpm install plans with durable runtime state, removing retained-manifest and source-checkout cache reconciliation from Gateway startup. Refs #73532. Thanks @oadiazp, @bstanbury, and @jmfraga. +- Plugins/runtime-deps: replace Gateway-start root chunk dependency inference with explicit mirrored-root dependency metadata, reducing staged runtime scans while preserving lazy per-plugin installs. Refs #73532. Thanks @oadiazp and @bstanbury. +- Plugins/runtime-deps: run pnpm staged installs outside the repository workspace and disable pnpm release-age gates for exact bundled runtime dependency materialization, so bundled plugin dependency repair writes packages into the generated stage without blocking fresh packaged dependencies. Refs #73532. Thanks @oadiazp and @bstanbury. - CLI/TUI: keep `chat.history` off model-catalog discovery so initial Gateway-backed TUI history loads cannot block behind slow provider/plugin model scans on low-core hosts. Refs #73524. Thanks @harshcatsystems-collab. - Channels/WhatsApp: flag recently reconnected linked accounts in channel status even when the socket is currently healthy, so flapping WhatsApp Web sessions no longer look clean after a brief reconnect. Refs #73602. Thanks @Vksh07. - Channels/WhatsApp: log shared dispatcher delivery failures with reply kind, message id, chat id, and connection id, so typing-without-send reports can identify whether the WhatsApp send path rejected a generated reply. Refs #74269. Thanks @tomcosta-git. diff --git a/docs/plugins/sdk-setup.md b/docs/plugins/sdk-setup.md index d7e8dde9e5f..6e86bd48868 100644 --- a/docs/plugins/sdk-setup.md +++ b/docs/plugins/sdk-setup.md @@ -520,6 +520,8 @@ For npm-sourced installs, `openclaw plugins install` runs project-local `npm ins Bundled OpenClaw-owned plugins are the only startup repair exception: when a packaged install sees one enabled by plugin config, legacy channel config, or its bundled default-enabled manifest, startup installs that plugin's missing runtime dependencies before import. Third-party plugins should not rely on startup installs; keep using the explicit plugin installer. +Bundled package-level runtime deps are explicit metadata, not inferred from built JavaScript at gateway startup. If a shared OpenClaw root dependency must be available inside the external bundled-plugin runtime mirror, declare it in `openclaw.bundle.mirroredRootRuntimeDependencies` in the root package manifest. + ## Related - [Building plugins](/plugins/building-plugins) — step-by-step getting started guide diff --git a/package.json b/package.json index 72eec728cf1..fdad0fbed59 100644 --- a/package.json +++ b/package.json @@ -1728,5 +1728,23 @@ "@whiskeysockets/baileys@7.0.0-rc.9": "patches/@whiskeysockets__baileys@7.0.0-rc.9.patch", "@agentclientprotocol/claude-agent-acp@0.31.0": "patches/@agentclientprotocol__claude-agent-acp@0.31.0.patch" } + }, + "openclaw": { + "bundle": { + "mirroredRootRuntimeDependencies": [ + "@agentclientprotocol/sdk", + "@lydell/node-pty", + "croner", + "dotenv", + "jiti", + "json5", + "jszip", + "markdown-it", + "semver", + "tar", + "tslog", + "web-push" + ] + } } } diff --git a/scripts/lib/bundled-plugin-root-runtime-mirrors.mjs b/scripts/lib/bundled-plugin-root-runtime-mirrors.mjs index c4c0cef46a2..4d2347ddaec 100644 --- a/scripts/lib/bundled-plugin-root-runtime-mirrors.mjs +++ b/scripts/lib/bundled-plugin-root-runtime-mirrors.mjs @@ -213,3 +213,28 @@ export function collectBundledPluginRootRuntimeMirrorErrors(params) { return errors.toSorted((left, right) => left.localeCompare(right)); } + +export function collectDeclaredRootRuntimeDependencyMetadataErrors(rootPackageJson) { + const declaredRootRuntimeDeps = collectRuntimeDependencySpecs(rootPackageJson); + const declaredMirrorDeps = + rootPackageJson?.openclaw?.bundle?.mirroredRootRuntimeDependencies ?? []; + if (!Array.isArray(declaredMirrorDeps)) { + return ["package.json openclaw.bundle.mirroredRootRuntimeDependencies must be an array."]; + } + + const errors = []; + for (const dependencyName of declaredMirrorDeps) { + if (typeof dependencyName !== "string" || dependencyName.trim().length === 0) { + errors.push( + "package.json openclaw.bundle.mirroredRootRuntimeDependencies entries must be non-empty strings.", + ); + continue; + } + if (!declaredRootRuntimeDeps.has(dependencyName)) { + errors.push( + `package.json openclaw.bundle.mirroredRootRuntimeDependencies declares '${dependencyName}' but package.json dependencies/optionalDependencies do not include it.`, + ); + } + } + return errors.toSorted((left, right) => left.localeCompare(right)); +} diff --git a/scripts/release-check.ts b/scripts/release-check.ts index 7ac7aa3ef43..758536d583c 100755 --- a/scripts/release-check.ts +++ b/scripts/release-check.ts @@ -35,6 +35,7 @@ import { collectBuiltBundledPluginStagedRuntimeDependencyErrors, collectBundledPluginRootRuntimeMirrorErrors, collectBundledPluginRuntimeDependencySpecs, + collectDeclaredRootRuntimeDependencyMetadataErrors, collectRootDistBundledRuntimeMirrors, } from "./lib/bundled-plugin-root-runtime-mirrors.mjs"; import { collectPackUnpackedSizeErrors as collectNpmPackUnpackedSizeErrors } from "./lib/npm-pack-budget.mjs"; @@ -52,6 +53,7 @@ export { collectBundledExtensionManifestErrors } from "./lib/bundled-extension-m export { collectBuiltBundledPluginStagedRuntimeDependencyErrors, collectBundledPluginRootRuntimeMirrorErrors, + collectDeclaredRootRuntimeDependencyMetadataErrors, collectRootDistBundledRuntimeMirrors, packageNameFromSpecifier, } from "./lib/bundled-plugin-root-runtime-mirrors.mjs"; @@ -162,10 +164,16 @@ function checkBundledExtensionMetadata() { requiredRootMirrors, rootPackageJson: rootPackage, }); + const rootMirrorMetadataErrors = collectDeclaredRootRuntimeDependencyMetadataErrors(rootPackage); const builtArtifactErrors = collectBuiltBundledPluginStagedRuntimeDependencyErrors({ bundledPluginsDir: resolve("dist/extensions"), }); - const errors = [...manifestErrors, ...rootMirrorErrors, ...builtArtifactErrors]; + const errors = [ + ...manifestErrors, + ...rootMirrorErrors, + ...rootMirrorMetadataErrors, + ...builtArtifactErrors, + ]; if (errors.length > 0) { console.error("release-check: bundled extension manifest validation failed:"); for (const error of errors) { diff --git a/scripts/test-built-bundled-runtime-deps.mjs b/scripts/test-built-bundled-runtime-deps.mjs index a2272e3cc4d..690d348c024 100644 --- a/scripts/test-built-bundled-runtime-deps.mjs +++ b/scripts/test-built-bundled-runtime-deps.mjs @@ -8,6 +8,7 @@ import { collectBuiltBundledPluginStagedRuntimeDependencyErrors, collectBundledPluginRootRuntimeMirrorErrors, collectBundledPluginRuntimeDependencySpecs, + collectDeclaredRootRuntimeDependencyMetadataErrors, collectRootDistBundledRuntimeMirrors, } from "./lib/bundled-plugin-root-runtime-mirrors.mjs"; import { parsePackageRootArg } from "./lib/package-root-args.mjs"; @@ -36,6 +37,7 @@ const errors = [ requiredRootMirrors, rootPackageJson, }), + ...collectDeclaredRootRuntimeDependencyMetadataErrors(rootPackageJson), ...collectBuiltBundledPluginStagedRuntimeDependencyErrors({ bundledPluginsDir: builtPluginsDir, }), diff --git a/src/channels/plugins/bundled.shape-guard.test.ts b/src/channels/plugins/bundled.shape-guard.test.ts index c23b266146c..b76e8fa8efd 100644 --- a/src/channels/plugins/bundled.shape-guard.test.ts +++ b/src/channels/plugins/bundled.shape-guard.test.ts @@ -1,6 +1,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; +import { pathToFileURL } from "node:url"; import { importFreshModule } from "openclaw/plugin-sdk/test-fixtures"; import { afterEach, describe, expect, it, vi } from "vitest"; import { loadPluginManifestRegistry } from "../../plugins/manifest-registry.js"; @@ -651,6 +652,102 @@ describe("bundled channel entry shape guards", () => { } }); + it("does not load bundled runtime entries through external staged runtime deps during discovery", async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-bundled-runtime-deps-")); + const stageRoot = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-bundled-stage-")); + const previousBundledPluginsDir = process.env.OPENCLAW_BUNDLED_PLUGINS_DIR; + const previousPluginStageDir = process.env.OPENCLAW_PLUGIN_STAGE_DIR; + const pluginDir = path.join(root, "dist", "extensions", "alpha"); + const testGlobal = globalThis as typeof globalThis & { + __bundledRuntimeDepMarker?: string; + }; + fs.mkdirSync(pluginDir, { recursive: true }); + fs.writeFileSync( + path.join(root, "package.json"), + JSON.stringify({ name: "openclaw", version: "2026.4.21" }), + "utf8", + ); + fs.writeFileSync( + path.join(pluginDir, "package.json"), + JSON.stringify({ + name: "@openclaw/alpha", + version: "2026.4.21", + type: "module", + dependencies: { + "alpha-runtime-dep": "1.0.0", + }, + }), + "utf8", + ); + fs.writeFileSync( + path.join(pluginDir, "plugin.js"), + [ + "import { marker } from 'alpha-runtime-dep';", + "globalThis.__bundledRuntimeDepMarker = marker;", + "export default { id: 'alpha', meta: { label: marker }, config: {} };", + "", + ].join("\n"), + "utf8", + ); + fs.writeFileSync( + path.join(pluginDir, "index.js"), + [ + `import { defineBundledChannelEntry } from ${JSON.stringify(pathToFileURL(path.resolve("src/plugin-sdk/channel-entry-contract.ts")).href)};`, + "export default defineBundledChannelEntry({", + " id: 'alpha',", + " name: 'Alpha',", + " description: 'Alpha',", + " importMetaUrl: import.meta.url,", + " plugin: { specifier: './plugin.js' },", + "});", + "", + ].join("\n"), + "utf8", + ); + + process.env.OPENCLAW_PLUGIN_STAGE_DIR = stageRoot; + const { resolveBundledRuntimeDependencyInstallRoot } = + await import("../../plugins/bundled-runtime-deps.js"); + const installRoot = resolveBundledRuntimeDependencyInstallRoot(pluginDir); + const depRoot = path.join(installRoot, "node_modules", "alpha-runtime-dep"); + fs.mkdirSync(depRoot, { recursive: true }); + fs.writeFileSync( + path.join(depRoot, "package.json"), + JSON.stringify({ + name: "alpha-runtime-dep", + version: "1.0.0", + type: "module", + main: "index.js", + }), + "utf8", + ); + fs.writeFileSync(path.join(depRoot, "index.js"), "export const marker = 'staged-alpha';\n"); + + mockAlphaDistExtensionRuntime(); + + try { + process.env.OPENCLAW_BUNDLED_PLUGINS_DIR = path.join(root, "dist", "extensions"); + + const bundled = await importFreshModule( + import.meta.url, + "./bundled.js?scope=bundled-runtime-deps", + ); + + expect(bundled.getBundledChannelPlugin("alpha")).toBeUndefined(); + expect(testGlobal.__bundledRuntimeDepMarker).toBeUndefined(); + } finally { + restoreBundledPluginsDir(previousBundledPluginsDir); + if (previousPluginStageDir === undefined) { + delete process.env.OPENCLAW_PLUGIN_STAGE_DIR; + } else { + process.env.OPENCLAW_PLUGIN_STAGE_DIR = previousPluginStageDir; + } + fs.rmSync(root, { recursive: true, force: true }); + fs.rmSync(stageRoot, { recursive: true, force: true }); + delete testGlobal.__bundledRuntimeDepMarker; + } + }); + it("swallows and caches bundled plugin and setup load failures", async () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-bundled-load-failure-")); const previousBundledPluginsDir = process.env.OPENCLAW_BUNDLED_PLUGINS_DIR; diff --git a/src/channels/plugins/bundled.ts b/src/channels/plugins/bundled.ts index 67af22f3298..d9a0923ce0c 100644 --- a/src/channels/plugins/bundled.ts +++ b/src/channels/plugins/bundled.ts @@ -36,9 +36,13 @@ type BundledChannelEntryRuntimeContract = { accountInspect?: boolean; }; register: (api: unknown) => void; - loadChannelPlugin: () => ChannelPlugin; - loadChannelSecrets?: () => ChannelPlugin["secrets"] | undefined; - loadChannelAccountInspector?: () => NonNullable; + loadChannelPlugin: (options?: BundledEntryModuleLoadOptions) => ChannelPlugin; + loadChannelSecrets?: ( + options?: BundledEntryModuleLoadOptions, + ) => ChannelPlugin["secrets"] | undefined; + loadChannelAccountInspector?: ( + options?: BundledEntryModuleLoadOptions, + ) => NonNullable; setChannelRuntime?: (runtime: PluginRuntime) => void; }; @@ -239,7 +243,7 @@ function loadGeneratedBundledChannelEntry(params: { rootScope: params.rootScope, metadata: params.metadata, entry: params.metadata.source, - installRuntimeDeps: true, + installRuntimeDeps: false, }), ); if (!entry) { @@ -586,7 +590,7 @@ function getBundledChannelSecretsForRoot( } try { const secrets = - entry.loadChannelSecrets?.() ?? + entry.loadChannelSecrets?.({ installRuntimeDeps: false }) ?? getBundledChannelPluginForRoot(id, rootScope, loadContext)?.secrets; loadContext.lazySecretsById.set(id, secrets ?? null); return secrets; @@ -612,7 +616,7 @@ function getBundledChannelAccountInspectorForRoot( return undefined; } try { - const inspector = entry.loadChannelAccountInspector(); + const inspector = entry.loadChannelAccountInspector({ installRuntimeDeps: false }); loadContext.lazyAccountInspectorsById.set(id, inspector); return inspector; } catch (error) { diff --git a/src/commands/doctor-bundled-plugin-runtime-deps.test.ts b/src/commands/doctor-bundled-plugin-runtime-deps.test.ts index 1297341d598..60989d91f28 100644 --- a/src/commands/doctor-bundled-plugin-runtime-deps.test.ts +++ b/src/commands/doctor-bundled-plugin-runtime-deps.test.ts @@ -56,12 +56,54 @@ function createInstalledRuntimeDeps(): InstalledRuntimeDeps { return []; } -function readRetainedRuntimeDepsManifest(installRoot: string): string[] { - const manifestPath = path.join(installRoot, ".openclaw-runtime-deps.json"); - const parsed = JSON.parse(fs.readFileSync(manifestPath, "utf8")) as { specs?: unknown }; - return Array.isArray(parsed.specs) - ? parsed.specs.filter((entry): entry is string => typeof entry === "string") - : []; +function parseInstallSpec(spec: string): { name: string; version: string } { + const versionSeparator = spec.startsWith("@") ? spec.indexOf("@", 1) : spec.lastIndexOf("@"); + if (versionSeparator <= 0) { + throw new Error(`Invalid install spec ${spec}`); + } + return { + name: spec.slice(0, versionSeparator), + version: spec.slice(versionSeparator + 1), + }; +} + +function materializeRuntimeDeps(params: BundledRuntimeDepsInstallParams): void { + for (const spec of params.installSpecs ?? params.missingSpecs) { + const { name, version } = parseInstallSpec(spec); + writeJson(path.join(params.installRoot, "node_modules", ...name.split("/"), "package.json"), { + name, + version: version.replace(/^[~^]/u, ""), + }); + } +} + +function readMaterializedRuntimeDepSpecs( + installRoot: string, + expectedSpecs: readonly string[], +): string[] { + return expectedSpecs.flatMap((spec) => { + const { name } = parseInstallSpec(spec); + const packageJsonPath = path.join( + installRoot, + "node_modules", + ...name.split("/"), + "package.json", + ); + if (!fs.existsSync(packageJsonPath)) { + return []; + } + const parsed = JSON.parse(fs.readFileSync(packageJsonPath, "utf8")) as { + name?: unknown; + version?: unknown; + }; + return typeof parsed.name === "string" && typeof parsed.version === "string" + ? [`${parsed.name}@${parsed.version}`] + : []; + }); +} + +function expectNoLegacyRuntimeDepsManifest(installRoot: string): void { + expect(fs.existsSync(path.join(installRoot, ".openclaw-runtime-deps.json"))).toBe(false); } function createNonInteractivePrompter( @@ -437,6 +479,7 @@ describe("doctor bundled plugin runtime deps", () => { }, installDeps: (params) => { installed.push(params); + materializeRuntimeDeps(params); }, }); @@ -472,6 +515,7 @@ describe("doctor bundled plugin runtime deps", () => { }, installDeps: (params) => { installed.push(params); + materializeRuntimeDeps(params); }, }); @@ -500,6 +544,7 @@ describe("doctor bundled plugin runtime deps", () => { }, installDeps: (params) => { installed.push(params); + materializeRuntimeDeps(params); }, }); @@ -512,7 +557,10 @@ describe("doctor bundled plugin runtime deps", () => { }, ]); expect(installRoot).not.toBe(root); - expect(readRetainedRuntimeDepsManifest(installRoot)).toEqual(["grammy@1.37.0"]); + expect(readMaterializedRuntimeDepSpecs(installRoot, ["grammy@1.37.0"])).toEqual([ + "grammy@1.37.0", + ]); + expectNoLegacyRuntimeDepsManifest(installRoot); }); it("logs runtime dependency repair progress before and after install", async () => { @@ -534,9 +582,7 @@ describe("doctor bundled plugin runtime deps", () => { expect(logs).toEqual( expect.arrayContaining([ - expect.stringContaining( - "Installing bundled plugin runtime deps (1 missing, 1 install specs): grammy@1.37.0", - ), + expect.stringContaining("Installing bundled plugin runtime deps (1 specs): grammy@1.37.0"), expect.stringContaining("Installed bundled plugin runtime deps in"), ]), ); @@ -622,6 +668,7 @@ describe("doctor bundled plugin runtime deps", () => { }, installDeps: (params) => { installed.push(params); + materializeRuntimeDeps(params); }, }); @@ -658,6 +705,7 @@ describe("doctor bundled plugin runtime deps", () => { }, installDeps: (params) => { installed.push(params); + materializeRuntimeDeps(params); }, }); @@ -740,6 +788,7 @@ describe("doctor bundled plugin runtime deps", () => { }, installDeps: (params) => { installed.push(params); + materializeRuntimeDeps(params); }, }); @@ -752,10 +801,13 @@ describe("doctor bundled plugin runtime deps", () => { }, ]); expect(installRoot).toContain(stageDir); - expect(readRetainedRuntimeDepsManifest(installRoot)).toEqual(["@slack/web-api@7.15.1"]); + expect(readMaterializedRuntimeDepSpecs(installRoot, ["@slack/web-api@7.15.1"])).toEqual([ + "@slack/web-api@7.15.1", + ]); + expectNoLegacyRuntimeDepsManifest(installRoot); }); - it("repairs only missing deps into the final layered stage dir", async () => { + it("repairs the complete dependency plan into the final layered stage dir", async () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-doctor-bundled-")); const baselineStageDir = fs.mkdtempSync( path.join(os.tmpdir(), "openclaw-doctor-bundled-baseline-"), @@ -797,14 +849,14 @@ describe("doctor bundled plugin runtime deps", () => { expect(installed).toEqual([ { installRoot, - missingSpecs: ["grammy@1.37.0"], - installSpecs: ["grammy@1.37.0"], + missingSpecs: ["@slack/web-api@7.15.1", "grammy@1.37.0"], + installSpecs: ["@slack/web-api@7.15.1", "grammy@1.37.0"], }, ]); - expect(readRetainedRuntimeDepsManifest(installRoot)).toEqual(["grammy@1.37.0"]); + expectNoLegacyRuntimeDepsManifest(installRoot); }); - it("drops stale retained bundled deps when repairing a subset", async () => { + it("drops stale legacy bundled deps manifests when repairing a subset", async () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-doctor-bundled-")); writeJson(path.join(root, "package.json"), { name: "openclaw" }); writeBundledChannelPlugin(root, "telegram", { grammy: "1.37.0" }); @@ -829,6 +881,7 @@ describe("doctor bundled plugin runtime deps", () => { }, installDeps: (params) => { installed.push(params); + materializeRuntimeDeps(params); }, }); @@ -840,6 +893,9 @@ describe("doctor bundled plugin runtime deps", () => { }, ]); expect(installRoot).not.toBe(root); - expect(readRetainedRuntimeDepsManifest(installRoot)).toEqual(["grammy@1.37.0"]); + expect(readMaterializedRuntimeDepSpecs(installRoot, ["grammy@1.37.0"])).toEqual([ + "grammy@1.37.0", + ]); + expectNoLegacyRuntimeDepsManifest(installRoot); }); }); diff --git a/src/commands/doctor-bundled-plugin-runtime-deps.ts b/src/commands/doctor-bundled-plugin-runtime-deps.ts index f019b675dcb..0c79d297ed7 100644 --- a/src/commands/doctor-bundled-plugin-runtime-deps.ts +++ b/src/commands/doctor-bundled-plugin-runtime-deps.ts @@ -4,7 +4,7 @@ import { formatCliCommand } from "../cli/command-format.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import { resolveOpenClawPackageRootSync } from "../infra/openclaw-root.js"; import { - createBundledRuntimeDepsWritableInstallSpecs, + createBundledRuntimeDepsInstallSpecs, repairBundledRuntimeDepsInstallRootAsync, resolveBundledRuntimeDependencyPackageInstallRootPlan, scanBundledPluginRuntimeDeps, @@ -164,18 +164,15 @@ export async function maybeRepairBundledPluginRuntimeDeps(params: { return; } - const missingSpecs = missing.map((dep) => `${dep.name}@${dep.version}`); const installRootPlan = resolveBundledRuntimeDependencyPackageInstallRootPlan(packageRoot, { env, }); - const installSpecs = createBundledRuntimeDepsWritableInstallSpecs({ + const installSpecs = createBundledRuntimeDepsInstallSpecs({ deps, - searchRoots: installRootPlan.searchRoots, - installRoot: installRootPlan.installRoot, }); note( [ - "Bundled plugin runtime deps are missing.", + "Bundled plugin runtime deps need staging.", ...missing.map((dep) => `- ${dep.name}@${dep.version} (used by ${dep.pluginIds.join(", ")})`), `Fix: run ${formatCliCommand("openclaw doctor --fix")} to install them.`, ].join("\n"), @@ -198,14 +195,14 @@ export async function maybeRepairBundledPluginRuntimeDeps(params: { try { const { createCliProgress } = await import("../cli/progress.js"); progress = createCliProgress({ - label: `Installing bundled plugin runtime deps (${missingSpecs.length})`, + label: `Installing bundled plugin runtime deps (${installSpecs.length})`, indeterminate: true, enabled: process.env.VITEST !== "true" || process.env.OPENCLAW_TEST_RUNTIME_LOG === "1", }); const installStartedAt = Date.now(); logRuntimeDepsInstallProgress( params.runtime, - `Installing bundled plugin runtime deps (${missingSpecs.length} missing, ${installSpecs.length} install specs): ${missingSpecs.join(", ")}`, + `Installing bundled plugin runtime deps (${installSpecs.length} specs): ${installSpecs.join(", ")}`, ); heartbeat = setInterval(() => { logRuntimeDepsInstallProgress( @@ -216,7 +213,7 @@ export async function maybeRepairBundledPluginRuntimeDeps(params: { heartbeat.unref?.(); const result = await repairBundledRuntimeDepsInstallRootAsync({ installRoot: installRootPlan.installRoot, - missingSpecs, + missingSpecs: installSpecs, installSpecs, env: params.env ?? process.env, installDeps: params.installDeps diff --git a/src/gateway/server-startup-plugins.test.ts b/src/gateway/server-startup-plugins.test.ts index 32bebc048eb..315af780c07 100644 --- a/src/gateway/server-startup-plugins.test.ts +++ b/src/gateway/server-startup-plugins.test.ts @@ -245,7 +245,7 @@ describe("prepareGatewayPluginBootstrap runtime-deps staging", () => { ); }); - it("pre-stages only missing runtime deps while retaining the full startup dependency set", async () => { + it("pre-stages the full startup dependency set", async () => { scanBundledPluginRuntimeDeps.mockReturnValueOnce({ deps: [ { name: "alpha-runtime", version: "1.0.0", pluginIds: ["telegram"] }, @@ -267,7 +267,7 @@ describe("prepareGatewayPluginBootstrap runtime-deps staging", () => { expect(repairBundledRuntimeDepsInstallRootAsync).toHaveBeenCalledWith( expect.objectContaining({ installRoot: "/runtime", - missingSpecs: ["grammy@1.37.0"], + missingSpecs: ["alpha-runtime@1.0.0", "grammy@1.37.0"], installSpecs: ["alpha-runtime@1.0.0", "grammy@1.37.0"], }), ); diff --git a/src/gateway/server-startup-plugins.ts b/src/gateway/server-startup-plugins.ts index 50c24947ff5..3ecfded12d5 100644 --- a/src/gateway/server-startup-plugins.ts +++ b/src/gateway/server-startup-plugins.ts @@ -87,19 +87,18 @@ async function prestageGatewayBundledRuntimeDeps(params: { if (missing.length === 0) { return; } - const missingSpecs = missing.map((dep) => `${dep.name}@${dep.version}`); const installSpecs = deps.map((dep) => `${dep.name}@${dep.version}`); const installRoot = resolveBundledRuntimeDependencyPackageInstallRoot(packageRoot, { env: process.env, }); const startedAt = Date.now(); params.log.info( - `[plugins] staging bundled runtime deps before gateway startup (${missingSpecs.length} missing, ${installSpecs.length} install specs): ${missingSpecs.join(", ")}`, + `[plugins] staging bundled runtime deps before gateway startup (${installSpecs.length} specs): ${installSpecs.join(", ")}`, ); try { await repairBundledRuntimeDepsInstallRootAsync({ installRoot, - missingSpecs, + missingSpecs: installSpecs, installSpecs, env: process.env, warn: (message) => params.log.warn(`[plugins] ${message}`), @@ -111,7 +110,7 @@ async function prestageGatewayBundledRuntimeDeps(params: { return; } params.log.info( - `[plugins] installed bundled runtime deps before gateway startup in ${Date.now() - startedAt}ms: ${missingSpecs.join(", ")}`, + `[plugins] installed bundled runtime deps before gateway startup in ${Date.now() - startedAt}ms: ${installSpecs.join(", ")}`, ); } diff --git a/src/plugin-sdk/channel-entry-contract.ts b/src/plugin-sdk/channel-entry-contract.ts index 309eb13f201..8d944e81a19 100644 --- a/src/plugin-sdk/channel-entry-contract.ts +++ b/src/plugin-sdk/channel-entry-contract.ts @@ -100,9 +100,13 @@ export type BundledChannelEntryContract = { configSchema: ChannelEntryConfigSchema; features?: BundledChannelEntryFeatures; register: (api: OpenClawPluginApi) => void; - loadChannelPlugin: () => TPlugin; - loadChannelSecrets?: () => ChannelPlugin["secrets"] | undefined; - loadChannelAccountInspector?: () => NonNullable; + loadChannelPlugin: (options?: BundledEntryModuleLoadOptions) => TPlugin; + loadChannelSecrets?: ( + options?: BundledEntryModuleLoadOptions, + ) => ChannelPlugin["secrets"] | undefined; + loadChannelAccountInspector?: ( + options?: BundledEntryModuleLoadOptions, + ) => NonNullable; setChannelRuntime?: (runtime: PluginRuntime) => void; }; @@ -448,15 +452,22 @@ export function defineBundledChannelEntry({ typeof configSchema === "function" ? configSchema() : ((configSchema ?? emptyChannelConfigSchema()) as ChannelEntryConfigSchema); - const loadChannelPlugin = () => loadBundledEntryExportSync(importMetaUrl, plugin); + const loadChannelPlugin = (options?: BundledEntryModuleLoadOptions) => + loadBundledEntryExportSync(importMetaUrl, plugin, options); const loadChannelSecrets = secrets - ? () => loadBundledEntryExportSync(importMetaUrl, secrets) + ? (options?: BundledEntryModuleLoadOptions) => + loadBundledEntryExportSync( + importMetaUrl, + secrets, + options, + ) : undefined; const loadChannelAccountInspector = accountInspect - ? () => + ? (options?: BundledEntryModuleLoadOptions) => loadBundledEntryExportSync>( importMetaUrl, accountInspect, + options, ) : undefined; const setChannelRuntime = runtime diff --git a/src/plugin-sdk/facade-loader.test.ts b/src/plugin-sdk/facade-loader.test.ts index 17b22812d16..c82f81c4bc7 100644 --- a/src/plugin-sdk/facade-loader.test.ts +++ b/src/plugin-sdk/facade-loader.test.ts @@ -4,6 +4,7 @@ import { fileURLToPath, pathToFileURL } from "node:url"; import { afterEach, describe, expect, it, vi } from "vitest"; import { clearBundledRuntimeDependencyNodePaths, + ensureBundledPluginRuntimeDeps, resolveBundledRuntimeDependencyInstallRoot, } from "../plugins/bundled-runtime-deps.js"; import { shouldExpectNativeJitiForJavaScriptTestRuntime } from "../test-utils/jiti-runtime.js"; @@ -180,6 +181,18 @@ function writeStagedRuntimeDepPackage(params: { fs.writeFileSync(path.join(depRoot, "index.js"), params.source ?? "export {};\n", "utf8"); } +function concreteRuntimeDepVersionForTest(version: string): string { + return version.startsWith("^") || version.startsWith("~") ? version.slice(1) : version; +} + +function parseRuntimeDepSpecForTest(spec: string): { name: string; version: string } { + const atIndex = spec.lastIndexOf("@"); + return { + name: spec.slice(0, atIndex), + version: spec.slice(atIndex + 1), + }; +} + function createPackagedBundledPluginDirWithStagedRuntimeDep(params: { marker: string; prefix: string; @@ -227,14 +240,24 @@ function createPackagedBundledPluginDirWithStagedRuntimeDep(params: { const installRoot = resolveBundledRuntimeDependencyInstallRoot(pluginRoot, { env, }); - writeStagedRuntimeDepPackage({ - installRoot, - name: STAGED_RUNTIME_DEP_NAME, - version: "1.0.0", - source: `export const marker = ${JSON.stringify(params.marker)};\n`, + ensureBundledPluginRuntimeDeps({ + env, + pluginId, + pluginRoot, + installDeps: ({ installRoot: runtimeInstallRoot, installSpecs = [] }) => { + for (const spec of installSpecs) { + const dep = parseRuntimeDepSpecForTest(spec); + writeStagedRuntimeDepPackage({ + installRoot: runtimeInstallRoot, + name: dep.name, + version: concreteRuntimeDepVersionForTest(dep.version), + ...(dep.name === STAGED_RUNTIME_DEP_NAME + ? { source: `export const marker = ${JSON.stringify(params.marker)};\n` } + : {}), + }); + } + }, }); - writeStagedRuntimeDepPackage({ installRoot, name: "semver", version: "7.7.4" }); - writeStagedRuntimeDepPackage({ installRoot, name: "tslog", version: "4.10.2" }); return { bundledPluginsDir, diff --git a/src/plugins/bundled-runtime-deps.test.ts b/src/plugins/bundled-runtime-deps.test.ts index c4e19f69100..2e3b70967ff 100644 --- a/src/plugins/bundled-runtime-deps.test.ts +++ b/src/plugins/bundled-runtime-deps.test.ts @@ -20,14 +20,13 @@ import { installBundledRuntimeDeps, installBundledRuntimeDepsAsync, isWritableDirectory, - materializeBundledRuntimeMirrorDistFile, pruneUnknownBundledRuntimeDepsRoots, repairBundledRuntimeDepsInstallRootAsync, + resolveBundledRuntimeDependencyPackageInstallRoot, resolveBundledRuntimeDependencyInstallRoot, resolveBundledRuntimeDependencyInstallRootPlan, resolveBundledRuntimeDepsNpmRunner, scanBundledPluginRuntimeDeps, - shouldMaterializeBundledRuntimeMirrorDistFile, type BundledRuntimeDepsInstallParams, } from "./bundled-runtime-deps.js"; @@ -57,6 +56,31 @@ function writeInstalledPackage(rootDir: string, packageName: string, version: st ); } +function writeGeneratedRuntimeDepsManifest(rootDir: string, specs: readonly string[]): void { + const dependencies = Object.fromEntries( + [...specs] + .toSorted((left, right) => left.localeCompare(right)) + .map((spec) => { + const atIndex = spec.lastIndexOf("@"); + return [spec.slice(0, atIndex), spec.slice(atIndex + 1)]; + }), + ); + fs.mkdirSync(rootDir, { recursive: true }); + fs.writeFileSync( + path.join(rootDir, "package.json"), + `${JSON.stringify( + { + name: "openclaw-runtime-deps-install", + private: true, + dependencies, + }, + null, + 2, + )}\n`, + "utf8", + ); +} + function writeBundledPluginPackage(params: { packageRoot: string; pluginId: string; @@ -102,42 +126,11 @@ afterEach(() => { spawnMock.mockReset(); spawnSyncMock.mockReset(); bundledRuntimeDepsActivityTesting.resetBundledRuntimeDepsInstallActivity(); - bundledRuntimeDepsTesting.clearBundledRuntimeMirrorMaterializeCache(); for (const dir of tempDirs.splice(0)) { fs.rmSync(dir, { recursive: true, force: true }); } }); -describe("shouldMaterializeBundledRuntimeMirrorDistFile", () => { - it("reuses unchanged root dist file decisions without rereading source", () => { - const root = makeTempDir(); - const sourcePath = path.join(root, "shared-runtime.js"); - fs.writeFileSync( - sourcePath, - [ - `//#region extensions/browser/src/runtime.ts`, - `export const marker = "shared-runtime";`, - `//#endregion`, - "", - ].join("\n"), - "utf8", - ); - const realReadFileSync = fs.readFileSync.bind(fs); - let sourceReads = 0; - vi.spyOn(fs, "readFileSync").mockImplementation(((target, options) => { - if (path.resolve(target.toString()) === path.resolve(sourcePath)) { - sourceReads += 1; - } - return realReadFileSync(target, options as never); - }) as typeof fs.readFileSync); - - expect(shouldMaterializeBundledRuntimeMirrorDistFile(sourcePath)).toBe(true); - expect(shouldMaterializeBundledRuntimeMirrorDistFile(sourcePath)).toBe(true); - - expect(sourceReads).toBe(1); - }); -}); - describe("resolveBundledRuntimeDepsNpmRunner", () => { it("ignores npm_execpath and uses the Node-adjacent npm CLI on Windows", () => { const execPath = "C:\\Program Files\\nodejs\\node.exe"; @@ -161,10 +154,12 @@ describe("resolveBundledRuntimeDepsNpmRunner", () => { }); it("uses package-manager-neutral install args with npm config env", () => { - expect(createBundledRuntimeDepsInstallArgs(["acpx@0.5.3"])).toEqual([ + expect(createBundledRuntimeDepsInstallArgs()).toEqual([ "install", "--ignore-scripts", - "acpx@0.5.3", + "--no-audit", + "--no-fund", + "--omit=dev", ]); expect( createBundledRuntimeDepsInstallEnv( @@ -186,16 +181,18 @@ describe("resolveBundledRuntimeDepsNpmRunner", () => { ), ).toEqual({ PATH: "/usr/bin:/bin", + npm_config_audit: "false", npm_config_cache: "/opt/openclaw/runtime-cache", npm_config_dry_run: "false", npm_config_fetch_retries: "5", npm_config_fetch_retry_maxtimeout: "120000", npm_config_fetch_retry_mintimeout: "10000", npm_config_fetch_timeout: "300000", + npm_config_fund: "false", npm_config_global: "false", npm_config_legacy_peer_deps: "true", npm_config_location: "project", - npm_config_package_lock: "false", + npm_config_package_lock: "true", npm_config_save: "false", }); }); @@ -270,6 +267,40 @@ describe("resolveBundledRuntimeDepsNpmRunner", () => { ).toThrow("Unable to resolve a safe npm executable on Windows"); }); + it("ignores Windows pnpm.cmd shims for shell-free installs", () => { + const execPath = "C:\\Program Files\\nodejs\\node.exe"; + const pnpmCmdPath = "C:\\Program Files\\nodejs\\pnpm.cmd"; + + expect( + bundledRuntimeDepsTesting.resolveBundledRuntimeDepsPnpmRunner({ + env: {}, + execPath, + existsSync: (candidate) => candidate === pnpmCmdPath, + platform: "win32", + pnpmArgs: ["install"], + }), + ).toBeNull(); + }); + + it("uses Windows pnpm.exe when available for shell-free installs", () => { + const execPath = "C:\\Program Files\\nodejs\\node.exe"; + const pnpmExePath = "C:\\Program Files\\nodejs\\pnpm.exe"; + + expect( + bundledRuntimeDepsTesting.resolveBundledRuntimeDepsPnpmRunner({ + env: {}, + execPath, + existsSync: (candidate) => candidate === pnpmExePath, + platform: "win32", + pnpmArgs: ["install"], + }), + ).toEqual({ + packageManager: "pnpm", + command: pnpmExePath, + args: ["install"], + }); + }); + it("refuses POSIX npm shim fallback when npm-cli.js is unavailable", () => { expect(() => resolveBundledRuntimeDepsNpmRunner({ @@ -286,50 +317,6 @@ describe("resolveBundledRuntimeDepsNpmRunner", () => { }); describe("installBundledRuntimeDeps", () => { - it("keeps already-materialized mirror chunks when source and target match", () => { - const tempDir = makeTempDir(); - const chunkPath = path.join(tempDir, "dist", "accounts.js"); - fs.mkdirSync(path.dirname(chunkPath), { recursive: true }); - fs.writeFileSync( - chunkPath, - [ - `//#region extensions/slack/src/accounts.ts`, - `export const marker = "same-file";`, - `//#endregion`, - "", - ].join("\n"), - "utf8", - ); - - materializeBundledRuntimeMirrorDistFile(chunkPath, chunkPath); - - expect(fs.readFileSync(chunkPath, "utf8")).toContain("same-file"); - }); - - it("replaces stale mirror symlinks when materializing chunks", () => { - const tempDir = makeTempDir(); - const sourcePath = path.join(tempDir, "dist", "accounts.js"); - const targetPath = path.join(tempDir, "stage", "dist", "accounts.js"); - fs.mkdirSync(path.dirname(sourcePath), { recursive: true }); - fs.mkdirSync(path.dirname(targetPath), { recursive: true }); - fs.writeFileSync( - sourcePath, - [ - `//#region extensions/slack/src/accounts.ts`, - `export const marker = "source";`, - `//#endregion`, - "", - ].join("\n"), - "utf8", - ); - fs.symlinkSync(sourcePath, targetPath, "file"); - - materializeBundledRuntimeMirrorDistFile(sourcePath, targetPath); - - expect(fs.lstatSync(targetPath).isSymbolicLink()).toBe(false); - expect(fs.readFileSync(targetPath, "utf8")).toContain("source"); - }); - it("uses a real write probe for runtime dependency roots", () => { const accessSpy = vi.spyOn(fs, "accessSync").mockImplementation(() => undefined); const mkdirSpy = vi.spyOn(fs, "mkdtempSync").mockImplementation(() => { @@ -353,8 +340,12 @@ describe("installBundledRuntimeDeps", () => { "node_modules/npm/bin/npm-cli.js", ); const attackerNpmCliPath = "C:\\repo\\evil\\npm-cli.js"; + const realExistsSync = fs.existsSync.bind(fs); vi.spyOn(fs, "existsSync").mockImplementation( - (candidate) => candidate === attackerNpmCliPath || candidate === safeNpmCliPath, + (candidate) => + candidate === attackerNpmCliPath || + candidate === safeNpmCliPath || + realExistsSync(candidate), ); spawnSyncMock.mockImplementation((_command, _args, options) => { writeInstalledPackage(String(options?.cwd ?? ""), "acpx", "0.5.3"); @@ -380,14 +371,14 @@ describe("installBundledRuntimeDeps", () => { expect(spawnSyncMock).toHaveBeenCalledWith( expect.any(String), - [safeNpmCliPath, "install", "--ignore-scripts", "acpx@0.5.3"], + [safeNpmCliPath, "install", "--ignore-scripts", "--no-audit", "--no-fund", "--omit=dev"], expect.objectContaining({ cwd: installRoot, windowsHide: true, env: expect.objectContaining({ npm_config_dry_run: "false", npm_config_legacy_peer_deps: "true", - npm_config_package_lock: "false", + npm_config_package_lock: "true", npm_config_save: "false", }), }), @@ -412,6 +403,47 @@ describe("installBundledRuntimeDeps", () => { ); }); + it("isolates pnpm installs from an enclosing workspace", () => { + const parentRoot = makeTempDir(); + const installRoot = path.join(parentRoot, "repo", "dist-runtime", "extensions", "qa-lab"); + const pnpmBinDir = path.join(parentRoot, "bin"); + fs.mkdirSync(pnpmBinDir, { recursive: true }); + fs.writeFileSync(path.join(pnpmBinDir, "pnpm"), "#!/bin/sh\n", "utf8"); + fs.mkdirSync(path.join(parentRoot, "repo"), { recursive: true }); + fs.writeFileSync( + path.join(parentRoot, "repo", "pnpm-workspace.yaml"), + "packages: []\n", + "utf8", + ); + spawnSyncMock.mockImplementation((_command, _args, options) => { + writeInstalledPackage(String(options?.cwd ?? ""), "zod", "4.3.6"); + return { + pid: 123, + output: [], + stdout: "", + stderr: "", + signal: null, + status: 0, + }; + }); + + installBundledRuntimeDeps({ + installRoot, + missingSpecs: ["zod@4.3.6"], + env: { + PATH: pnpmBinDir, + }, + }); + + expect(spawnSyncMock).toHaveBeenCalledWith( + expect.stringContaining("pnpm"), + expect.arrayContaining(["install", "--ignore-workspace", "--config.minimum-release-age=0"]), + expect.objectContaining({ + cwd: installRoot, + }), + ); + }); + it("hides async npm child windows for startup repair installs", async () => { const installRoot = makeTempDir(); spawnMock.mockImplementation((_command, _args, options) => { @@ -442,7 +474,7 @@ describe("installBundledRuntimeDeps", () => { ); }); - it("reports async npm output as install progress", async () => { + it("reports async package-manager output as install progress", async () => { const installRoot = makeTempDir(); const progress: string[] = []; spawnMock.mockImplementation((_command, _args, options) => { @@ -468,12 +500,18 @@ describe("installBundledRuntimeDeps", () => { onProgress: (message) => progress.push(message), }); - expect(progress).toContain("Starting npm install for bundled plugin runtime deps: acpx@0.5.3"); - expect(progress).toContain("npm stdout: added 1 package"); - expect(progress).toContain("npm stderr: npm notice"); + expect(progress).toEqual( + expect.arrayContaining([ + expect.stringMatching( + /^Starting (npm|pnpm) install for bundled plugin runtime deps: acpx@0\.5\.3$/, + ), + expect.stringMatching(/^(npm|pnpm) stdout: added 1 package$/), + expect.stringMatching(/^(npm|pnpm) stderr: npm notice$/), + ]), + ); }); - it("emits heartbeat progress while async npm is silent", async () => { + it("emits heartbeat progress while async package-manager install is silent", async () => { vi.useFakeTimers(); try { const installRoot = makeTempDir(); @@ -501,7 +539,11 @@ describe("installBundledRuntimeDeps", () => { }); await vi.advanceTimersByTimeAsync(5_000); - expect(progress).toContain("npm install still running (5s elapsed)"); + expect(progress).toEqual( + expect.arrayContaining([ + expect.stringMatching(/^(npm|pnpm) install still running \(5s elapsed\)$/), + ]), + ); closeChild(); await expect(install).resolves.toBeUndefined(); @@ -526,6 +568,7 @@ describe("installBundledRuntimeDeps", () => { }, }); writeInstalledPackage(cwd, "@grammyjs/runner", "2.0.3"); + writeInstalledPackage(cwd, "grammy", "1.37.0"); return { pid: 123, output: [], @@ -554,12 +597,12 @@ describe("installBundledRuntimeDeps", () => { ); }); - it("repairs external install roots by installing only missing specs while retaining staged deps", async () => { + it("repairs external install roots from the complete generated dependency plan", async () => { const installRoot = makeTempDir(); writeInstalledPackage(installRoot, "alpha-runtime", "1.0.0"); spawnMock.mockImplementation((_command, args, options) => { const cwd = String(options?.cwd ?? ""); - expect(args.slice(-3)).toEqual(["install", "--ignore-scripts", "beta-runtime@2.0.0"]); + expect(args).toEqual(expect.arrayContaining(["install", "--ignore-scripts"])); expect(JSON.parse(fs.readFileSync(path.join(cwd, "package.json"), "utf8"))).toEqual({ name: "openclaw-runtime-deps-install", private: true, @@ -588,7 +631,46 @@ describe("installBundledRuntimeDeps", () => { expect(spawnMock).toHaveBeenCalledOnce(); }); - it("prunes stale retained deps during package-level repair", async () => { + it("writes the requested package-manager install plan during startup repair", async () => { + const installRoot = makeTempDir(); + writeInstalledPackage(installRoot, "alpha-runtime", "1.0.0"); + writeGeneratedRuntimeDepsManifest(installRoot, ["alpha-runtime@1.0.0"]); + spawnMock.mockImplementation((_command, _args, options) => { + const cwd = String(options?.cwd ?? ""); + expect(JSON.parse(fs.readFileSync(path.join(cwd, "package.json"), "utf8"))).toEqual({ + name: "openclaw-runtime-deps-install", + private: true, + dependencies: { + "beta-runtime": "2.0.0", + }, + }); + writeInstalledPackage(cwd, "beta-runtime", "2.0.0"); + const child = new EventEmitter() as ReturnType; + Object.assign(child, { + stdout: new EventEmitter(), + stderr: new EventEmitter(), + }); + queueMicrotask(() => child.emit("close", 0, null)); + return child; + }); + + await repairBundledRuntimeDepsInstallRootAsync({ + installRoot, + missingSpecs: ["beta-runtime@2.0.0"], + installSpecs: ["beta-runtime@2.0.0"], + env: {}, + }); + + expect(JSON.parse(fs.readFileSync(path.join(installRoot, "package.json"), "utf8"))).toEqual({ + name: "openclaw-runtime-deps-install", + private: true, + dependencies: { + "beta-runtime": "2.0.0", + }, + }); + }); + + it("lets the package manager prune stale deps during package-level repair", async () => { const installRoot = makeTempDir(); writeInstalledPackage(installRoot, "alpha-runtime", "1.0.0"); fs.writeFileSync( @@ -597,6 +679,10 @@ describe("installBundledRuntimeDeps", () => { "utf8", ); spawnMock.mockImplementation((_command, _args, options) => { + fs.rmSync(path.join(installRoot, "node_modules", "alpha-runtime"), { + recursive: true, + force: true, + }); writeInstalledPackage(String(options?.cwd ?? ""), "beta-runtime", "2.0.0"); const child = new EventEmitter() as ReturnType; Object.assign(child, { @@ -617,9 +703,14 @@ describe("installBundledRuntimeDeps", () => { expect( fs.existsSync(path.join(installRoot, "node_modules", "alpha-runtime", "package.json")), ).toBe(false); - expect( - JSON.parse(fs.readFileSync(path.join(installRoot, ".openclaw-runtime-deps.json"), "utf8")), - ).toEqual({ specs: ["beta-runtime@2.0.0"] }); + expect(JSON.parse(fs.readFileSync(path.join(installRoot, "package.json"), "utf8"))).toEqual({ + name: "openclaw-runtime-deps-install", + private: true, + dependencies: { + "beta-runtime": "2.0.0", + }, + }); + expect(fs.existsSync(path.join(installRoot, ".openclaw-runtime-deps.json"))).toBe(false); }); it("warns but still installs bundled runtime deps when disk space looks low", () => { @@ -713,7 +804,7 @@ describe("installBundledRuntimeDeps", () => { ); }); - it("installs the full retained set when plugin-root staging replaces node_modules", () => { + it("installs the full generated plan when plugin-root staging replaces node_modules", () => { const pluginRoot = makeTempDir(); fs.writeFileSync( path.join(pluginRoot, "package.json"), @@ -728,12 +819,7 @@ describe("installBundledRuntimeDeps", () => { spawnSyncMock.mockImplementation((_command, args, options) => { const cwd = String(options?.cwd ?? ""); expect(cwd).toBe(path.join(pluginRoot, ".openclaw-install-stage")); - expect((args ?? []).slice(-4)).toEqual([ - "install", - "--ignore-scripts", - "alpha-runtime@1.0.0", - "beta-runtime@2.0.0", - ]); + expect(args).toEqual(expect.arrayContaining(["install", "--ignore-scripts"])); writeInstalledPackage(cwd, "alpha-runtime", "1.0.0"); writeInstalledPackage(cwd, "beta-runtime", "2.0.0"); return { @@ -753,8 +839,7 @@ describe("installBundledRuntimeDeps", () => { pluginRoot, }), ).toEqual({ - installedSpecs: ["beta-runtime@2.0.0"], - retainSpecs: ["alpha-runtime@1.0.0", "beta-runtime@2.0.0"], + installedSpecs: ["alpha-runtime@1.0.0", "beta-runtime@2.0.0"], }); expect(spawnSyncMock).toHaveBeenCalledOnce(); expect( @@ -849,7 +934,9 @@ describe("installBundledRuntimeDeps", () => { missingSpecs: ["tokenjuice@0.6.1"], env: {}, }), - ).toThrow(`npm install did not place bundled runtime deps in ${installRoot}: tokenjuice@0.6.1`); + ).toThrow( + `package manager install did not place bundled runtime deps in ${installRoot}: tokenjuice@0.6.1`, + ); }); it("cleans an owned isolated execution root after copying node_modules back", () => { @@ -952,7 +1039,11 @@ describe("installBundledRuntimeDeps", () => { it("rejects invalid install specs before spawning npm", () => { expect(() => - createBundledRuntimeDepsInstallArgs(["tokenjuice@https://evil.example/t.tgz"]), + installBundledRuntimeDeps({ + installRoot: makeTempDir(), + missingSpecs: ["tokenjuice@https://evil.example/t.tgz"], + env: {}, + }), ).toThrow("Unsupported bundled runtime dependency spec for tokenjuice"); }); @@ -1160,6 +1251,56 @@ describe("scanBundledPluginRuntimeDeps config policy", () => { expect(result.conflicts).toEqual([]); }); + it("does not stage explicitly disabled preselected channel deps", () => { + const result = scanBundledPluginRuntimeDeps({ + packageRoot: setupPolicyPackageRoot(), + selectedPluginIds: ["telegram"], + config: { + plugins: { allow: ["telegram"] }, + channels: { telegram: { enabled: false, botToken: "123:abc" } }, + }, + }); + + expect(result.deps).toEqual([]); + expect(result.missing).toEqual([]); + expect(result.conflicts).toEqual([]); + }); + + it("does not report already staged package-level runtime deps as missing", () => { + const packageRoot = setupPolicyPackageRoot(); + const env = { OPENCLAW_PLUGIN_STAGE_DIR: makeTempDir() }; + const installRoot = resolveBundledRuntimeDependencyPackageInstallRoot(packageRoot, { env }); + writeInstalledPackage(installRoot, "alpha-runtime", "1.0.0"); + + const result = scanBundledPluginRuntimeDeps({ + packageRoot, + config: {}, + env, + }); + + expect(result.deps.map((dep) => `${dep.name}@${dep.version}`)).toEqual(["alpha-runtime@1.0.0"]); + expect(result.missing).toEqual([]); + expect(result.conflicts).toEqual([]); + }); + + it("reports staged package-level runtime deps as missing when the version is stale", () => { + const packageRoot = setupPolicyPackageRoot(); + const env = { OPENCLAW_PLUGIN_STAGE_DIR: makeTempDir() }; + const installRoot = resolveBundledRuntimeDependencyPackageInstallRoot(packageRoot, { env }); + writeInstalledPackage(installRoot, "alpha-runtime", "0.9.0"); + + const result = scanBundledPluginRuntimeDeps({ + packageRoot, + config: {}, + env, + }); + + expect(result.missing.map((dep) => `${dep.name}@${dep.version}`)).toEqual([ + "alpha-runtime@1.0.0", + ]); + expect(result.conflicts).toEqual([]); + }); + it("reads each bundled plugin manifest once per runtime-deps scan", () => { const packageRoot = makeTempDir(); const pluginRoot = writeBundledPluginPackage({ @@ -1179,7 +1320,7 @@ describe("scanBundledPluginRuntimeDeps config policy", () => { ).toHaveLength(1); }); - it("reports missing mirrored core runtime deps for doctor repair", () => { + it("reports declared package mirror deps for doctor repair", () => { const packageRoot = makeTempDir(); const stageDir = makeTempDir(); fs.writeFileSync( @@ -1188,6 +1329,11 @@ describe("scanBundledPluginRuntimeDeps config policy", () => { name: "openclaw", version: "2026.4.25", dependencies: { semver: "7.7.4", tslog: "^4.10.2" }, + openclaw: { + bundle: { + mirroredRootRuntimeDependencies: ["semver", "tslog"], + }, + }, }), ); writeBundledPluginPackage({ @@ -1215,7 +1361,7 @@ describe("scanBundledPluginRuntimeDeps config policy", () => { ]); }); - it("reports missing root-dist mirror deps for selected bundled plugins", () => { + it("includes selected plugin deps that can be used by mirrored root chunks", () => { const packageRoot = makeTempDir(); const stageDir = makeTempDir(); fs.writeFileSync( @@ -1226,12 +1372,13 @@ describe("scanBundledPluginRuntimeDeps config policy", () => { dependencies: { chokidar: "^5.0.0" }, }), ); - writeBundledPluginPackage({ + const pluginRoot = writeBundledPluginPackage({ packageRoot, pluginId: "memory-core", deps: { chokidar: "^5.0.0" }, enabledByDefault: true, }); + fs.writeFileSync(path.join(pluginRoot, "index.js"), `import "../../refresh-CZ2n5WoB.js";\n`); fs.writeFileSync( path.join(packageRoot, "dist", "refresh-CZ2n5WoB.js"), `import chokidar from "chokidar";\n`, @@ -1248,7 +1395,7 @@ describe("scanBundledPluginRuntimeDeps config policy", () => { expect(result.missing.map((dep) => `${dep.name}@${dep.version}`)).toEqual(["chokidar@^5.0.0"]); }); - it("does not report root-dist mirror deps for inactive bundled plugin owners", () => { + it("does not include inactive bundled plugin deps", () => { const packageRoot = makeTempDir(); const stageDir = makeTempDir(); fs.writeFileSync( @@ -1259,11 +1406,12 @@ describe("scanBundledPluginRuntimeDeps config policy", () => { dependencies: { chokidar: "^5.0.0" }, }), ); - writeBundledPluginPackage({ + const memoryRoot = writeBundledPluginPackage({ packageRoot, pluginId: "memory-core", deps: { chokidar: "^5.0.0" }, }); + fs.writeFileSync(path.join(memoryRoot, "index.js"), `import "../../refresh-CZ2n5WoB.js";\n`); writeBundledPluginPackage({ packageRoot, pluginId: "slack", @@ -1288,7 +1436,7 @@ describe("scanBundledPluginRuntimeDeps config policy", () => { expect(result.missing).toEqual([]); }); - it("reports missing mirrored core runtime deps for startup plugins without own deps", () => { + it("reports declared root package deps for mirrored root chunks", () => { const packageRoot = makeTempDir(); const stageDir = makeTempDir(); fs.writeFileSync( @@ -1296,7 +1444,89 @@ describe("scanBundledPluginRuntimeDeps config policy", () => { JSON.stringify({ name: "openclaw", version: "2026.4.25", - dependencies: { tslog: "^4.10.2" }, + dependencies: { + chalk: "^5.6.2", + jiti: "^2.6.1", + json5: "^2.2.3", + }, + openclaw: { + bundle: { + mirroredRootRuntimeDependencies: ["chalk", "jiti", "json5"], + }, + }, + }), + ); + const pluginRoot = writeBundledPluginPackage({ + packageRoot, + pluginId: "whatsapp", + deps: { "@whiskeysockets/baileys": "7.0.0-rc.9" }, + channels: ["whatsapp"], + }); + writeBundledPluginPackage({ + packageRoot, + pluginId: "matrix", + deps: { jiti: "^2.6.1" }, + channels: ["matrix"], + }); + fs.writeFileSync( + path.join(pluginRoot, "setup-entry.js"), + `import "../../theme.js";\nimport "openclaw/plugin-sdk/setup";\n`, + ); + fs.mkdirSync(path.join(packageRoot, "dist", "plugin-sdk"), { recursive: true }); + fs.writeFileSync( + path.join(packageRoot, "dist", "plugin-sdk", "setup.js"), + `import "../bundled-plugin-metadata.js";\nimport "../redact.js";\n`, + ); + fs.writeFileSync( + path.join(packageRoot, "dist", "bundled-plugin-metadata.js"), + `import { createJiti } from "jiti";\nvoid createJiti;\n`, + ); + fs.writeFileSync(path.join(packageRoot, "dist", "redact.js"), `import JSON5 from "json5";\n`); + fs.writeFileSync(path.join(packageRoot, "dist", "theme.js"), `import chalk from "chalk";\n`); + + const result = scanBundledPluginRuntimeDeps({ + packageRoot, + selectedPluginIds: ["whatsapp"], + config: { + channels: { whatsapp: { enabled: true } }, + }, + env: { OPENCLAW_PLUGIN_STAGE_DIR: stageDir }, + }); + + expect(result.deps.map((dep) => `${dep.name}@${dep.version}`)).toEqual([ + "@whiskeysockets/baileys@7.0.0-rc.9", + "chalk@^5.6.2", + "jiti@^2.6.1", + "json5@^2.2.3", + ]); + expect(result.deps.map((dep) => dep.pluginIds)).toEqual([ + ["whatsapp"], + ["openclaw-core"], + ["openclaw-core"], + ["openclaw-core"], + ]); + expect(result.missing.map((dep) => `${dep.name}@${dep.version}`)).toEqual([ + "@whiskeysockets/baileys@7.0.0-rc.9", + "chalk@^5.6.2", + "jiti@^2.6.1", + "json5@^2.2.3", + ]); + }); + + it("reports declared package mirror deps for startup plugins without own deps", () => { + const packageRoot = makeTempDir(); + const stageDir = makeTempDir(); + fs.writeFileSync( + path.join(packageRoot, "package.json"), + JSON.stringify({ + name: "openclaw", + version: "2026.4.25", + dependencies: { semver: "7.7.4", tslog: "^4.10.2" }, + openclaw: { + bundle: { + mirroredRootRuntimeDependencies: ["semver", "tslog"], + }, + }, }), ); writeBundledPluginPackage({ @@ -1315,12 +1545,18 @@ describe("scanBundledPluginRuntimeDeps config policy", () => { env: { OPENCLAW_PLUGIN_STAGE_DIR: stageDir }, }); - expect(result.deps.map((dep) => `${dep.name}@${dep.version}`)).toEqual(["tslog@^4.10.2"]); - expect(result.deps[0]?.pluginIds).toEqual(["openclaw-core"]); - expect(result.missing.map((dep) => `${dep.name}@${dep.version}`)).toEqual(["tslog@^4.10.2"]); + expect(result.deps.map((dep) => `${dep.name}@${dep.version}`)).toEqual([ + "semver@7.7.4", + "tslog@^4.10.2", + ]); + expect(result.deps.map((dep) => dep.pluginIds)).toEqual([["openclaw-core"], ["openclaw-core"]]); + expect(result.missing.map((dep) => `${dep.name}@${dep.version}`)).toEqual([ + "semver@7.7.4", + "tslog@^4.10.2", + ]); }); - it("deduplicates mirrored core runtime deps already declared by a plugin", () => { + it("deduplicates declared package mirror deps already declared by a plugin", () => { const packageRoot = makeTempDir(); const stageDir = makeTempDir(); fs.writeFileSync( @@ -1329,6 +1565,11 @@ describe("scanBundledPluginRuntimeDeps config policy", () => { name: "openclaw", version: "2026.4.25", dependencies: { tslog: "^4.10.2" }, + openclaw: { + bundle: { + mirroredRootRuntimeDependencies: ["tslog"], + }, + }, }), ); writeBundledPluginPackage({ @@ -1349,7 +1590,50 @@ describe("scanBundledPluginRuntimeDeps config policy", () => { expect(result.missing.map((dep) => `${dep.name}@${dep.version}`)).toEqual(["tslog@^4.10.2"]); }); - it("resolves runtime deps from layered external stage dirs", () => { + it("keeps the complete staging plan without reporting present deps as missing", () => { + const packageRoot = makeTempDir(); + const stageDir = makeTempDir(); + fs.writeFileSync( + path.join(packageRoot, "package.json"), + JSON.stringify({ name: "openclaw", version: "2026.4.25" }), + ); + const pluginRoot = writeBundledPluginPackage({ + packageRoot, + pluginId: "memory-lancedb", + deps: { + "@lancedb/lancedb": "^0.27.2", + openai: "^6.34.0", + typebox: "1.1.33", + }, + enabledByDefault: true, + }); + const env = { OPENCLAW_PLUGIN_STAGE_DIR: stageDir }; + const installRoot = resolveBundledRuntimeDependencyInstallRoot(pluginRoot, { env }); + writeInstalledPackage(installRoot, "@lancedb/lancedb", "0.27.2"); + writeInstalledPackage(installRoot, "openai", "6.34.0"); + writeInstalledPackage(installRoot, "typebox", "1.1.33"); + writeGeneratedRuntimeDepsManifest(installRoot, [ + "@lancedb/lancedb@^0.27.2", + "openai@^6.34.0", + "typebox@1.1.33", + "@mariozechner/pi-ai@0.70.5", + ]); + + const result = scanBundledPluginRuntimeDeps({ + packageRoot, + config: {}, + env, + }); + + expect(result.deps.map((dep) => `${dep.name}@${dep.version}`)).toEqual([ + "@lancedb/lancedb@^0.27.2", + "openai@^6.34.0", + "typebox@1.1.33", + ]); + expect(result.missing).toEqual([]); + }); + + it("keeps a complete install plan while missing only absent deps", () => { const packageRoot = makeTempDir(); const baselineStageDir = makeTempDir(); const writableStageDir = makeTempDir(); @@ -1428,19 +1712,17 @@ describe("ensureBundledPluginRuntimeDeps", () => { }, pluginId: "bedrock", pluginRoot, - retainSpecs: ["previous@3.0.0"], }); expect(result).toEqual({ installedSpecs: ["already-present@1.0.0", "missing@2.0.0"], - retainSpecs: ["already-present@1.0.0", "missing@2.0.0", "previous@3.0.0"], }); const installRoot = resolveBundledRuntimeDependencyInstallRoot(pluginRoot, { env: {} }); expect(calls).toEqual([ { installRoot, missingSpecs: ["already-present@1.0.0", "missing@2.0.0"], - installSpecs: ["already-present@1.0.0", "missing@2.0.0", "previous@3.0.0"], + installSpecs: ["already-present@1.0.0", "missing@2.0.0"], }, ]); expect(installRoot).not.toBe(pluginRoot); @@ -1474,7 +1756,6 @@ describe("ensureBundledPluginRuntimeDeps", () => { expect(result).toEqual({ installedSpecs: ["external-runtime@^1.2.3"], - retainSpecs: ["external-runtime@^1.2.3"], }); const installRoot = resolveBundledRuntimeDependencyInstallRoot(pluginRoot, { env: {} }); expect(calls).toEqual([ @@ -1487,7 +1768,7 @@ describe("ensureBundledPluginRuntimeDeps", () => { expect(installRoot).not.toBe(pluginRoot); }); - it("installs mirrored core logger deps even when the plugin has no external deps", () => { + it("installs declared package mirror deps even when the plugin has no external deps", () => { const packageRoot = makeTempDir(); const stageDir = makeTempDir(); fs.writeFileSync( @@ -1496,6 +1777,11 @@ describe("ensureBundledPluginRuntimeDeps", () => { name: "openclaw", version: "2026.4.25", dependencies: { tslog: "^4.10.2" }, + openclaw: { + bundle: { + mirroredRootRuntimeDependencies: ["tslog"], + }, + }, }), ); const pluginRoot = path.join(packageRoot, "dist", "extensions", "slack"); @@ -1507,6 +1793,7 @@ describe("ensureBundledPluginRuntimeDeps", () => { env: { OPENCLAW_PLUGIN_STAGE_DIR: stageDir }, installDeps: (params) => { calls.push(params); + writeInstalledPackage(params.installRoot, "tokenjuice", "0.6.1"); }, pluginId: "slack", pluginRoot, @@ -1517,7 +1804,6 @@ describe("ensureBundledPluginRuntimeDeps", () => { }); expect(result).toEqual({ installedSpecs: ["tslog@^4.10.2"], - retainSpecs: ["tslog@^4.10.2"], }); expect(calls).toEqual([ { @@ -1560,7 +1846,6 @@ describe("ensureBundledPluginRuntimeDeps", () => { expect(result).toEqual({ installedSpecs: ["@anthropic-ai/sdk@^0.50.0"], - retainSpecs: ["@anthropic-ai/sdk@^0.50.0"], }); const installRoot = resolveBundledRuntimeDependencyInstallRoot(pluginRoot, { env: {} }); expect(calls).toEqual([ @@ -1612,7 +1897,6 @@ describe("ensureBundledPluginRuntimeDeps", () => { const installRoot = resolveBundledRuntimeDependencyInstallRoot(pluginRoot, { env }); expect(result).toEqual({ installedSpecs: ["@slack/web-api@7.15.1"], - retainSpecs: ["@slack/web-api@7.15.1"], }); expect(calls).toEqual([ { @@ -1635,10 +1919,10 @@ describe("ensureBundledPluginRuntimeDeps", () => { pluginId: "slack", pluginRoot, }); - expect(second).toEqual({ installedSpecs: [], retainSpecs: [] }); + expect(second).toEqual({ installedSpecs: [] }); }); - it("installs only missing deps into the final layered stage dir", () => { + it("installs the complete plan into the final layered stage dir", () => { const packageRoot = makeTempDir(); const baselineStageDir = makeTempDir(); const writableStageDir = makeTempDir(); @@ -1669,10 +1953,7 @@ describe("ensureBundledPluginRuntimeDeps", () => { env, installDeps: (params) => { calls.push(params); - fs.rmSync(path.join(params.installRoot, "node_modules", "@slack", "web-api"), { - recursive: true, - force: true, - }); + writeInstalledPackage(params.installRoot, "@slack/web-api", "7.15.1"); writeInstalledPackage(params.installRoot, "grammy", "1.37.0"); }, pluginId: "slack", @@ -1681,22 +1962,23 @@ describe("ensureBundledPluginRuntimeDeps", () => { expect(installRootPlan.installRoot).toContain(writableStageDir); expect(result).toEqual({ - installedSpecs: ["grammy@1.37.0"], - retainSpecs: ["grammy@1.37.0"], + installedSpecs: ["@slack/web-api@7.15.1", "grammy@1.37.0"], }); expect(calls).toEqual([ { installRoot: installRootPlan.installRoot, - missingSpecs: ["grammy@1.37.0"], - installSpecs: ["grammy@1.37.0"], + missingSpecs: ["@slack/web-api@7.15.1", "grammy@1.37.0"], + installSpecs: ["@slack/web-api@7.15.1", "grammy@1.37.0"], }, ]); expect( - fs.realpathSync(path.join(installRootPlan.installRoot, "node_modules", "@slack", "web-api")), - ).toBe(fs.realpathSync(path.join(baselineRoot, "node_modules", "@slack", "web-api"))); + fs.existsSync( + path.join(installRootPlan.installRoot, "node_modules", "@slack", "web-api", "package.json"), + ), + ).toBe(true); }); - it("retains external staged deps across separate loader passes", () => { + it("stages complete package-level deps once across separate loader passes", () => { const packageRoot = makeTempDir(); const stageDir = makeTempDir(); fs.writeFileSync( @@ -1731,12 +2013,28 @@ describe("ensureBundledPluginRuntimeDeps", () => { }; ensureBundledPluginRuntimeDeps({ + config: { + plugins: { + entries: { + alpha: { enabled: true }, + beta: { enabled: true }, + }, + }, + }, env, installDeps, pluginId: "alpha", pluginRoot: alphaRoot, }); ensureBundledPluginRuntimeDeps({ + config: { + plugins: { + entries: { + alpha: { enabled: true }, + beta: { enabled: true }, + }, + }, + }, env, installDeps, pluginId: "beta", @@ -1747,18 +2045,58 @@ describe("ensureBundledPluginRuntimeDeps", () => { expect(calls).toEqual([ { installRoot, - missingSpecs: ["alpha-runtime@1.0.0"], - installSpecs: ["alpha-runtime@1.0.0"], - }, - { - installRoot, - missingSpecs: ["beta-runtime@2.0.0"], + missingSpecs: ["alpha-runtime@1.0.0", "beta-runtime@2.0.0"], installSpecs: ["alpha-runtime@1.0.0", "beta-runtime@2.0.0"], }, ]); }); - it("does not retain already staged deps for disabled bundled channel owners", () => { + it("uses the complete package-level plan when no config is provided", () => { + const packageRoot = makeTempDir(); + const stageDir = makeTempDir(); + fs.writeFileSync( + path.join(packageRoot, "package.json"), + JSON.stringify({ name: "openclaw", version: "2026.4.22" }), + ); + const alphaRoot = path.join(packageRoot, "dist", "extensions", "alpha"); + const betaRoot = path.join(packageRoot, "dist", "extensions", "beta"); + fs.mkdirSync(alphaRoot, { recursive: true }); + fs.mkdirSync(betaRoot, { recursive: true }); + fs.writeFileSync( + path.join(alphaRoot, "package.json"), + JSON.stringify({ dependencies: { "alpha-runtime": "1.0.0" } }), + ); + fs.writeFileSync( + path.join(betaRoot, "package.json"), + JSON.stringify({ dependencies: { "beta-runtime": "2.0.0" } }), + ); + + const env = { OPENCLAW_PLUGIN_STAGE_DIR: stageDir }; + const calls: BundledRuntimeDepsInstallParams[] = []; + ensureBundledPluginRuntimeDeps({ + env, + installDeps: (params) => { + calls.push(params); + for (const spec of params.installSpecs ?? params.missingSpecs) { + const name = spec.slice(0, spec.lastIndexOf("@")); + writeInstalledPackage(params.installRoot, name, spec.slice(spec.lastIndexOf("@") + 1)); + } + }, + pluginId: "alpha", + pluginRoot: alphaRoot, + }); + + const installRoot = resolveBundledRuntimeDependencyInstallRoot(alphaRoot, { env }); + expect(calls).toEqual([ + { + installRoot, + missingSpecs: ["alpha-runtime@1.0.0", "beta-runtime@2.0.0"], + installSpecs: ["alpha-runtime@1.0.0", "beta-runtime@2.0.0"], + }, + ]); + }); + + it("excludes disabled bundled channel owners from the package-level plan", () => { const packageRoot = makeTempDir(); const stageDir = makeTempDir(); fs.writeFileSync( @@ -1781,11 +2119,7 @@ describe("ensureBundledPluginRuntimeDeps", () => { const installRoot = resolveBundledRuntimeDependencyInstallRoot(browserRoot, { env }); writeInstalledPackage(installRoot, "browser-runtime", "1.0.0"); writeInstalledPackage(installRoot, "grammy", "1.37.0"); - fs.writeFileSync( - path.join(installRoot, ".openclaw-runtime-deps.json"), - `${JSON.stringify({ specs: ["grammy@1.37.0"] }, null, 2)}\n`, - "utf8", - ); + writeGeneratedRuntimeDepsManifest(installRoot, ["browser-runtime@1.0.0"]); const result = ensureBundledPluginRuntimeDeps({ env, @@ -1802,10 +2136,220 @@ describe("ensureBundledPluginRuntimeDeps", () => { }, }); - expect(result).toEqual({ installedSpecs: [], retainSpecs: [] }); + expect(result).toEqual({ installedSpecs: [] }); + expect(fs.existsSync(path.join(installRoot, ".openclaw-runtime-deps.json"))).toBe(false); + }); + + it("does not install disabled channel deps during a package-level lazy plugin repair", () => { + const packageRoot = makeTempDir(); + const stageDir = makeTempDir(); + fs.writeFileSync( + path.join(packageRoot, "package.json"), + JSON.stringify({ name: "openclaw", version: "2026.4.27" }), + ); + const acpxRoot = writeBundledPluginPackage({ + packageRoot, + pluginId: "acpx", + deps: { "acpx-runtime": "1.0.0" }, + enabledByDefault: true, + }); + writeBundledPluginPackage({ + packageRoot, + pluginId: "feishu", + deps: { "@larksuiteoapi/node-sdk": "^1.62.0" }, + channels: ["feishu"], + }); + const env = { OPENCLAW_PLUGIN_STAGE_DIR: stageDir }; + const installRoot = resolveBundledRuntimeDependencyInstallRoot(acpxRoot, { env }); + const calls: BundledRuntimeDepsInstallParams[] = []; + + const result = ensureBundledPluginRuntimeDeps({ + env, + pluginId: "acpx", + pluginRoot: acpxRoot, + config: { + plugins: { enabled: true }, + channels: { + feishu: { enabled: false, appId: "disabled" }, + }, + }, + installDeps: (params) => { + calls.push(params); + writeInstalledPackage(params.installRoot, "acpx-runtime", "1.0.0"); + }, + }); + + expect(result).toEqual({ installedSpecs: ["acpx-runtime@1.0.0"] }); + expect(calls).toEqual([ + { + installRoot, + missingSpecs: ["acpx-runtime@1.0.0"], + installSpecs: ["acpx-runtime@1.0.0"], + }, + ]); expect( - JSON.parse(fs.readFileSync(path.join(installRoot, ".openclaw-runtime-deps.json"), "utf8")), - ).toEqual({ specs: ["browser-runtime@1.0.0"] }); + fs.existsSync( + path.join(installRoot, "node_modules", "@larksuiteoapi", "node-sdk", "package.json"), + ), + ).toBe(false); + }); + + it("uses the generated manifest for the complete package-level fast path", () => { + const packageRoot = makeTempDir(); + const stageDir = makeTempDir(); + fs.writeFileSync( + path.join(packageRoot, "package.json"), + JSON.stringify({ name: "openclaw", version: "2026.4.27" }), + ); + const alphaRoot = writeBundledPluginPackage({ + packageRoot, + pluginId: "alpha", + deps: { "alpha-runtime": "1.0.0" }, + enabledByDefault: true, + }); + writeBundledPluginPackage({ + packageRoot, + pluginId: "beta", + deps: { "beta-runtime": "2.0.0" }, + enabledByDefault: true, + }); + const env = { OPENCLAW_PLUGIN_STAGE_DIR: stageDir }; + const installRoot = resolveBundledRuntimeDependencyInstallRoot(alphaRoot, { env }); + writeInstalledPackage(installRoot, "alpha-runtime", "1.0.0"); + writeInstalledPackage(installRoot, "beta-runtime", "2.0.0"); + writeGeneratedRuntimeDepsManifest(installRoot, ["alpha-runtime@1.0.0", "beta-runtime@2.0.0"]); + + const result = ensureBundledPluginRuntimeDeps({ + env, + pluginId: "alpha", + pluginRoot: alphaRoot, + installDeps: () => { + throw new Error("current runtime deps should not reinstall"); + }, + }); + + expect(result).toEqual({ installedSpecs: [] }); + }); + + it("drops stale package versions from the next package-level plan", () => { + const packageRoot = makeTempDir(); + const stageDir = makeTempDir(); + fs.writeFileSync( + path.join(packageRoot, "package.json"), + JSON.stringify({ name: "openclaw", version: "2026.4.27" }), + ); + const pluginRoot = writeBundledPluginPackage({ + packageRoot, + pluginId: "alpha", + deps: { "alpha-runtime": "2.0.0" }, + enabledByDefault: true, + }); + const env = { OPENCLAW_PLUGIN_STAGE_DIR: stageDir }; + const installRoot = resolveBundledRuntimeDependencyInstallRoot(pluginRoot, { env }); + writeInstalledPackage(installRoot, "alpha-runtime", "1.0.0"); + writeInstalledPackage(installRoot, "beta-runtime", "1.0.0"); + writeGeneratedRuntimeDepsManifest(installRoot, ["alpha-runtime@1.0.0", "beta-runtime@1.0.0"]); + const calls: BundledRuntimeDepsInstallParams[] = []; + + const result = ensureBundledPluginRuntimeDeps({ + env, + pluginId: "alpha", + pluginRoot, + installDeps: (params) => { + calls.push(params); + writeInstalledPackage(params.installRoot, "alpha-runtime", "2.0.0"); + }, + }); + + expect(result).toEqual({ installedSpecs: ["alpha-runtime@2.0.0"] }); + expect(calls).toEqual([ + { + installRoot, + missingSpecs: ["alpha-runtime@2.0.0"], + installSpecs: ["alpha-runtime@2.0.0"], + }, + ]); + }); + + it("reinstalls when the generated manifest is current but the installed package version is stale", () => { + const packageRoot = makeTempDir(); + const stageDir = makeTempDir(); + fs.writeFileSync( + path.join(packageRoot, "package.json"), + JSON.stringify({ name: "openclaw", version: "2026.4.27" }), + ); + const pluginRoot = writeBundledPluginPackage({ + packageRoot, + pluginId: "alpha", + deps: { "alpha-runtime": "2.0.0" }, + enabledByDefault: true, + }); + const env = { OPENCLAW_PLUGIN_STAGE_DIR: stageDir }; + const installRoot = resolveBundledRuntimeDependencyInstallRoot(pluginRoot, { env }); + writeInstalledPackage(installRoot, "alpha-runtime", "1.0.0"); + writeGeneratedRuntimeDepsManifest(installRoot, ["alpha-runtime@2.0.0"]); + const calls: BundledRuntimeDepsInstallParams[] = []; + + const result = ensureBundledPluginRuntimeDeps({ + env, + pluginId: "alpha", + pluginRoot, + installDeps: (params) => { + calls.push(params); + writeInstalledPackage(params.installRoot, "alpha-runtime", "2.0.0"); + }, + }); + + expect(result).toEqual({ installedSpecs: ["alpha-runtime@2.0.0"] }); + expect(calls).toEqual([ + { + installRoot, + missingSpecs: ["alpha-runtime@2.0.0"], + installSpecs: ["alpha-runtime@2.0.0"], + }, + ]); + }); + + it("reinstalls when the generated runtime-deps manifest is stale", () => { + const packageRoot = makeTempDir(); + const stageDir = makeTempDir(); + fs.writeFileSync( + path.join(packageRoot, "package.json"), + JSON.stringify({ name: "openclaw", version: "2026.4.27" }), + ); + const pluginRoot = writeBundledPluginPackage({ + packageRoot, + pluginId: "memory-lancedb", + deps: { + "@lancedb/lancedb": "^0.27.2", + openai: "^6.34.0", + typebox: "1.1.33", + }, + enabledByDefault: true, + }); + const env = { OPENCLAW_PLUGIN_STAGE_DIR: stageDir }; + const installRoot = resolveBundledRuntimeDependencyInstallRoot(pluginRoot, { env }); + writeInstalledPackage(installRoot, "@lancedb/lancedb", "0.27.2"); + writeInstalledPackage(installRoot, "openai", "6.34.0"); + writeInstalledPackage(installRoot, "typebox", "1.1.33"); + writeGeneratedRuntimeDepsManifest(installRoot, ["@mariozechner/pi-ai@0.70.5"]); + + const calls: BundledRuntimeDepsInstallParams[] = []; + const result = ensureBundledPluginRuntimeDeps({ + env, + pluginId: "memory-lancedb", + pluginRoot, + installDeps: (params) => { + calls.push(params); + }, + }); + + expect(result.installedSpecs).toEqual([ + "@lancedb/lancedb@^0.27.2", + "openai@^6.34.0", + "typebox@1.1.33", + ]); + expect(calls).toHaveLength(1); }); it("does not derive a second-generation stage root from external runtime mirrors", () => { @@ -1834,6 +2378,7 @@ describe("ensureBundledPluginRuntimeDeps", () => { path.join(installRoot, "node_modules", "grammy", "package.json"), JSON.stringify({ name: "grammy", version: "1.42.0" }), ); + writeGeneratedRuntimeDepsManifest(installRoot, ["grammy@^1.42.0"]); const nestedUnknownRoot = path.join( stageDir, @@ -1855,7 +2400,7 @@ describe("ensureBundledPluginRuntimeDeps", () => { pluginId: "telegram", pluginRoot: mirroredPluginRoot, }), - ).toEqual({ installedSpecs: [], retainSpecs: [] }); + ).toEqual({ installedSpecs: [] }); }); it("resolves nested cache pluginRoot to enclosing versioned cache", () => { @@ -1927,7 +2472,7 @@ describe("ensureBundledPluginRuntimeDeps", () => { expect(fs.existsSync(versioned)).toBe(true); }); - it("links source-checkout runtime deps from the cache instead of copying them", () => { + it("uses the plugin-local stage for source-checkout runtime deps", () => { const packageRoot = makeTempDir(); fs.writeFileSync( path.join(packageRoot, "package.json"), @@ -1944,7 +2489,7 @@ describe("ensureBundledPluginRuntimeDeps", () => { ); spawnSyncMock.mockImplementation((_command, _args, options) => { const cwd = String(options?.cwd); - expect(cwd).toContain(path.join(".local", "bundled-plugin-runtime-deps")); + expect(cwd).toBe(path.join(pluginRoot, ".openclaw-install-stage")); const depRoot = path.join(cwd, "node_modules", "voice-runtime"); fs.mkdirSync(depRoot, { recursive: true }); fs.writeFileSync( @@ -1962,26 +2507,28 @@ describe("ensureBundledPluginRuntimeDeps", () => { }), ).toEqual({ installedSpecs: ["voice-runtime@1.0.0"], - retainSpecs: ["voice-runtime@1.0.0"], }); expect(spawnSyncMock).toHaveBeenCalledTimes(1); - expect(fs.lstatSync(path.join(pluginRoot, "node_modules")).isSymbolicLink()).toBe(true); + expect(fs.lstatSync(path.join(pluginRoot, "node_modules")).isSymbolicLink()).toBe(false); fs.rmSync(path.join(pluginRoot, "node_modules"), { recursive: true, force: true }); + spawnSyncMock.mockImplementation((_command, _args, options) => { + writeInstalledPackage(String(options?.cwd), "voice-runtime", "1.0.0"); + return { status: 0, stdout: "", stderr: "" } as ReturnType; + }); expect( ensureBundledPluginRuntimeDeps({ env: {}, - installDeps: () => { - throw new Error("cache restore should not reinstall"); - }, pluginId: "voice-call", pluginRoot, }), - ).toEqual({ installedSpecs: [], retainSpecs: [] }); - expect(fs.lstatSync(path.join(pluginRoot, "node_modules")).isSymbolicLink()).toBe(true); + ).toEqual({ + installedSpecs: ["voice-runtime@1.0.0"], + }); + expect(fs.lstatSync(path.join(pluginRoot, "node_modules")).isSymbolicLink()).toBe(false); }); - it("retains existing staged deps without a retained manifest before shared installs", () => { + it("keeps the complete package-level install plan for configured plugins", () => { const packageRoot = makeTempDir(); const stageDir = makeTempDir(); fs.writeFileSync( @@ -2008,10 +2555,19 @@ describe("ensureBundledPluginRuntimeDeps", () => { path.join(installRoot, "node_modules", "alpha-runtime", "package.json"), JSON.stringify({ name: "alpha-runtime", version: "1.0.0" }), ); + writeGeneratedRuntimeDepsManifest(installRoot, ["alpha-runtime@1.0.0"]); expect(fs.existsSync(path.join(installRoot, ".openclaw-runtime-deps.json"))).toBe(false); const calls: BundledRuntimeDepsInstallParams[] = []; const result = ensureBundledPluginRuntimeDeps({ + config: { + plugins: { + entries: { + alpha: { enabled: true }, + beta: { enabled: true }, + }, + }, + }, env, installDeps: (params) => { calls.push(params); @@ -2021,13 +2577,12 @@ describe("ensureBundledPluginRuntimeDeps", () => { }); expect(result).toEqual({ - installedSpecs: ["beta-runtime@2.0.0"], - retainSpecs: ["alpha-runtime@1.0.0", "beta-runtime@2.0.0"], + installedSpecs: ["alpha-runtime@1.0.0", "beta-runtime@2.0.0"], }); expect(calls).toEqual([ { installRoot, - missingSpecs: ["beta-runtime@2.0.0"], + missingSpecs: ["alpha-runtime@1.0.0", "beta-runtime@2.0.0"], installSpecs: ["alpha-runtime@1.0.0", "beta-runtime@2.0.0"], }, ]); @@ -2057,7 +2612,6 @@ describe("ensureBundledPluginRuntimeDeps", () => { expect(result).toEqual({ installedSpecs: ["browser-runtime@1.0.0"], - retainSpecs: ["browser-runtime@1.0.0"], }); expect(getActiveBundledRuntimeDepsInstallCount()).toBe(0); await expect(idleWait).resolves.toEqual({ drained: true, active: 0 }); @@ -2273,7 +2827,6 @@ describe("ensureBundledPluginRuntimeDeps", () => { expect(result).toEqual({ installedSpecs: ["@mariozechner/pi-ai@0.70.2"], - retainSpecs: ["@mariozechner/pi-ai@0.70.2"], }); expect(calls).toHaveLength(1); expect(fs.existsSync(lockDir)).toBe(false); @@ -2318,7 +2871,6 @@ describe("ensureBundledPluginRuntimeDeps", () => { expect(result).toEqual({ installedSpecs: ["browser-runtime@1.0.0"], - retainSpecs: ["browser-runtime@1.0.0"], }); expect(calls).toHaveLength(1); expect(fs.existsSync(lockDir)).toBe(false); @@ -2366,7 +2918,6 @@ describe("ensureBundledPluginRuntimeDeps", () => { expect(result).toEqual({ installedSpecs: ["browser-runtime@1.0.0"], - retainSpecs: ["browser-runtime@1.0.0"], }); expect(calls).toHaveLength(1); expect(fs.existsSync(lockDir)).toBe(false); @@ -2397,7 +2948,7 @@ describe("ensureBundledPluginRuntimeDeps", () => { pluginRoot, }); - expect(result).toEqual({ installedSpecs: [], retainSpecs: [] }); + expect(result).toEqual({ installedSpecs: [] }); }); it("installs missing runtime deps for source-checkout bundled plugins", () => { @@ -2421,6 +2972,7 @@ describe("ensureBundledPluginRuntimeDeps", () => { env: { OPENCLAW_PLUGIN_STAGE_DIR: stageDir }, installDeps: (params) => { calls.push(params); + writeInstalledPackage(params.installRoot, "tokenjuice", "0.6.1"); }, pluginId: "tokenjuice", pluginRoot, @@ -2428,7 +2980,6 @@ describe("ensureBundledPluginRuntimeDeps", () => { expect(result).toEqual({ installedSpecs: ["tokenjuice@0.6.1"], - retainSpecs: ["tokenjuice@0.6.1"], }); const installRoot = resolveBundledRuntimeDependencyInstallRoot(pluginRoot, { env: { OPENCLAW_PLUGIN_STAGE_DIR: stageDir }, @@ -2443,8 +2994,8 @@ describe("ensureBundledPluginRuntimeDeps", () => { expect(installRoot).toContain(stageDir); expect(installRoot).not.toBe(pluginRoot); expect( - JSON.parse(fs.readFileSync(path.join(installRoot, ".openclaw-runtime-deps.json"), "utf8")), - ).toEqual({ specs: ["tokenjuice@0.6.1"] }); + fs.existsSync(path.join(installRoot, "node_modules", "tokenjuice", "package.json")), + ).toBe(true); }); it("keeps source-checkout bundled runtime deps in the plugin root without manifest churn", () => { @@ -2457,6 +3008,7 @@ describe("ensureBundledPluginRuntimeDeps", () => { path.join(pluginRoot, ".openclaw-runtime-deps.json"), JSON.stringify({ specs: ["stale@9.9.9"] }), ); + writeGeneratedRuntimeDepsManifest(pluginRoot, ["tokenjuice@0.6.1"]); fs.writeFileSync( path.join(pluginRoot, "package.json"), JSON.stringify({ @@ -2478,15 +3030,11 @@ describe("ensureBundledPluginRuntimeDeps", () => { expect(result).toEqual({ installedSpecs: ["tokenjuice@0.6.1"], - retainSpecs: ["tokenjuice@0.6.1"], }); expect(calls).toEqual([ { installRoot: pluginRoot, - installExecutionRoot: expect.stringContaining( - path.join(".local", "bundled-plugin-runtime-deps"), - ), - linkNodeModulesFromExecutionRoot: true, + installExecutionRoot: path.join(pluginRoot, ".openclaw-install-stage"), missingSpecs: ["tokenjuice@0.6.1"], installSpecs: ["tokenjuice@0.6.1"], }, @@ -2517,6 +3065,7 @@ describe("ensureBundledPluginRuntimeDeps", () => { path.join(pluginRoot, ".openclaw-runtime-deps.json"), JSON.stringify({ specs: ["stale@9.9.9"] }), ); + writeGeneratedRuntimeDepsManifest(pluginRoot, ["tokenjuice@0.6.1"]); const result = ensureBundledPluginRuntimeDeps({ env: {}, @@ -2527,7 +3076,7 @@ describe("ensureBundledPluginRuntimeDeps", () => { pluginRoot, }); - expect(result).toEqual({ installedSpecs: [], retainSpecs: [] }); + expect(result).toEqual({ installedSpecs: [] }); expect(fs.existsSync(path.join(pluginRoot, ".openclaw-runtime-deps.json"))).toBe(false); }); @@ -2561,15 +3110,11 @@ describe("ensureBundledPluginRuntimeDeps", () => { expect(result).toEqual({ installedSpecs: ["acpx@0.5.3"], - retainSpecs: ["acpx@0.5.3"], }); expect(calls).toEqual([ { installRoot: pluginRoot, - installExecutionRoot: expect.stringContaining( - path.join(".local", "bundled-plugin-runtime-deps"), - ), - linkNodeModulesFromExecutionRoot: true, + installExecutionRoot: path.join(pluginRoot, ".openclaw-install-stage"), missingSpecs: ["acpx@0.5.3"], installSpecs: ["acpx@0.5.3"], }, @@ -2611,7 +3156,6 @@ describe("ensureBundledPluginRuntimeDeps", () => { expect(result).toEqual({ installedSpecs: ["tokenjuice@0.6.1"], - retainSpecs: ["tokenjuice@0.6.1"], }); expect(calls).toEqual([ { @@ -2659,7 +3203,6 @@ describe("ensureBundledPluginRuntimeDeps", () => { expect(result).toEqual({ installedSpecs: ["tokenjuice@0.6.1"], - retainSpecs: ["tokenjuice@0.6.1"], }); const installRoot = resolveBundledRuntimeDependencyInstallRoot(pluginRoot, { env: { OPENCLAW_PLUGIN_STAGE_DIR: stageDir }, @@ -2696,7 +3239,6 @@ describe("ensureBundledPluginRuntimeDeps", () => { expect(result).toEqual({ installedSpecs: ["chokidar@^5.0.0"], - retainSpecs: ["chokidar@^5.0.0"], }); const installRoot = resolveBundledRuntimeDependencyInstallRoot(pluginRoot, { env: {} }); expect(calls).toEqual([ @@ -2749,7 +3291,6 @@ describe("ensureBundledPluginRuntimeDeps", () => { const installRoot = resolveBundledRuntimeDependencyInstallRoot(pluginRoot, { env: {} }); expect(result).toEqual({ installedSpecs: ["@discordjs/voice@0.19.2"], - retainSpecs: ["@discordjs/voice@0.19.2"], }); expect(calls).toEqual([ { @@ -2793,7 +3334,6 @@ describe("ensureBundledPluginRuntimeDeps", () => { expect(result).toEqual({ installedSpecs: ["@mariozechner/pi-ai@0.68.1"], - retainSpecs: ["@mariozechner/pi-ai@0.68.1"], }); const installRoot = resolveBundledRuntimeDependencyInstallRoot(pluginRoot, { env: {} }); expect(calls).toEqual([ @@ -2837,7 +3377,6 @@ describe("ensureBundledPluginRuntimeDeps", () => { expect(result).toEqual({ installedSpecs: ["ws@^8.20.0", "zod@^4.3.6"], - retainSpecs: ["ws@^8.20.0", "zod@^4.3.6"], }); const installRoot = resolveBundledRuntimeDependencyInstallRoot(pluginRoot, { env: {} }); expect(calls).toEqual([ @@ -2883,7 +3422,6 @@ describe("ensureBundledPluginRuntimeDeps", () => { expect(result).toEqual({ installedSpecs: ["zod@^4.3.6"], - retainSpecs: ["zod@^4.3.6"], }); const installRoot = resolveBundledRuntimeDependencyInstallRoot(pluginRoot, { env: {} }); expect(calls).toEqual([ @@ -2946,7 +3484,7 @@ describe("ensureBundledPluginRuntimeDeps", () => { ).toThrow("Invalid bundled runtime dependency name"); }); - it("rehydrates source-checkout dist deps from cache after rebuilds", () => { + it("reinstalls source-checkout dist deps after rebuilds remove node_modules", () => { const packageRoot = makeTempDir(); fs.mkdirSync(path.join(packageRoot, ".git"), { recursive: true }); fs.mkdirSync(path.join(packageRoot, "src"), { recursive: true }); @@ -2981,8 +3519,9 @@ describe("ensureBundledPluginRuntimeDeps", () => { const second = ensureBundledPluginRuntimeDeps({ env: {}, - installDeps: () => { - throw new Error("cached runtime deps should not reinstall"); + installDeps: (params) => { + installCalls.push(params); + writeInstalledPackage(params.installRoot, "zod", "4.3.6"); }, pluginId: "codex", pluginRoot, @@ -2990,15 +3529,16 @@ describe("ensureBundledPluginRuntimeDeps", () => { expect(first).toEqual({ installedSpecs: ["zod@^4.3.6"], - retainSpecs: ["zod@^4.3.6"], }); - expect(second).toEqual({ installedSpecs: [], retainSpecs: [] }); - expect(installCalls).toHaveLength(1); + expect(second).toEqual({ + installedSpecs: ["zod@^4.3.6"], + }); + expect(installCalls).toHaveLength(2); expect(fs.existsSync(path.join(pluginRoot, "node_modules", "zod", "package.json"))).toBe(true); }); }); -describe("MIRRORED_CORE_RUNTIME_DEP_NAMES drift guard", () => { +describe("mirrored root runtime dependency drift guard", () => { // Intentionally not mirrored at runtime: build-only / type-only / TUI-only // tooling and packages that resolve transitively through other mirrored deps. // If you change this set, document why in the comment beside the entry. @@ -3059,6 +3599,18 @@ describe("MIRRORED_CORE_RUNTIME_DEP_NAMES drift guard", () => { return out; } + function readMirroredRootRuntimeDeps(repoRoot: string): Set { + const parsed = JSON.parse(fs.readFileSync(path.join(repoRoot, "package.json"), "utf8")) as { + openclaw?: { + bundle?: { + mirroredRootRuntimeDependencies?: unknown; + }; + }; + }; + const deps = parsed.openclaw?.bundle?.mirroredRootRuntimeDependencies; + return new Set(Array.isArray(deps) ? deps.filter((dep) => typeof dep === "string") : []); + } + function collectExtensionOwnedDeps(repoRoot: string): Set { const out = new Set(); const extensionsDir = path.join(repoRoot, "extensions"); @@ -3143,20 +3695,7 @@ describe("MIRRORED_CORE_RUNTIME_DEP_NAMES drift guard", () => { const repoRoot = locateRepoRoot(); const rootDeps = readPackageJsonDeps(path.join(repoRoot, "package.json")); const extensionDeps = collectExtensionOwnedDeps(repoRoot); - const mirroredCore = new Set([ - "@agentclientprotocol/sdk", - "@lydell/node-pty", - "croner", - "dotenv", - "jiti", - "json5", - "jszip", - "markdown-it", - "semver", - "tar", - "tslog", - "web-push", - ]); + const mirroredCore = readMirroredRootRuntimeDeps(repoRoot); const nodeBuiltins = new Set(Module.builtinModules); const violations = new Map(); @@ -3206,9 +3745,9 @@ describe("MIRRORED_CORE_RUNTIME_DEP_NAMES drift guard", () => { throw new Error( [ "Bare imports found in src/ that are root-package runtime deps but are neither", - "in MIRRORED_CORE_RUNTIME_DEP_NAMES nor declared by any extension's package.json.", + "in package.json openclaw.bundle.mirroredRootRuntimeDependencies nor declared by any extension's package.json.", "These will be missing from the runtime-deps mirror at gateway start and Node", - "will fail to resolve them. Either add the package to MIRRORED_CORE_RUNTIME_DEP_NAMES,", + "will fail to resolve them. Either add the package to openclaw.bundle.mirroredRootRuntimeDependencies,", "declare it under an owning extension's dependencies, or add it to", "KNOWN_UNMIRRORED_BARE_IMPORTS in this test with a comment explaining why.", "", diff --git a/src/plugins/bundled-runtime-deps.ts b/src/plugins/bundled-runtime-deps.ts index cb362e00c06..aef5afa7534 100644 --- a/src/plugins/bundled-runtime-deps.ts +++ b/src/plugins/bundled-runtime-deps.ts @@ -15,7 +15,7 @@ import { sanitizeTerminalText } from "../terminal/safe-text.js"; import { beginBundledRuntimeDepsInstall } from "./bundled-runtime-deps-activity.js"; import { normalizePluginsConfig } from "./config-state.js"; import { passesManifestOwnerBasePolicy } from "./manifest-owner-policy.js"; -import { satisfies, validRange, validSemver } from "./semver.runtime.js"; +import { satisfies, validSemver } from "./semver.runtime.js"; export type RuntimeDepEntry = { name: string; @@ -32,7 +32,6 @@ export type RuntimeDepConflict = { export type BundledRuntimeDepsInstallParams = { installRoot: string; installExecutionRoot?: string; - linkNodeModulesFromExecutionRoot?: boolean; missingSpecs: string[]; installSpecs?: string[]; warn?: (message: string) => void; @@ -40,7 +39,6 @@ export type BundledRuntimeDepsInstallParams = { export type BundledRuntimeDepsEnsureResult = { installedSpecs: string[]; - retainSpecs: string[]; }; export type BundledRuntimeDepsInstallRoot = { @@ -53,14 +51,13 @@ export type BundledRuntimeDepsInstallRootPlan = BundledRuntimeDepsInstallRoot & }; type JsonObject = Record; -const RETAINED_RUNTIME_DEPS_MANIFEST = ".openclaw-runtime-deps.json"; +const LEGACY_RETAINED_RUNTIME_DEPS_MANIFEST = ".openclaw-runtime-deps.json"; // Packaged bundled plugins (Docker image, npm global install) keep their // `package.json` next to their entry point; running `npm install ` with // `cwd: pluginRoot` would make npm resolve the plugin's own `workspace:*` // dependencies and fail with `EUNSUPPORTEDPROTOCOL`. To avoid that, stage the // install inside this sub-directory and move the produced `node_modules/` back -// to the plugin root. Source-checkout installs already have their own cache -// path and keep using it. +// to the plugin root. const PLUGIN_ROOT_INSTALL_STAGE_DIR = ".openclaw-install-stage"; const BUNDLED_RUNTIME_DEPS_LOCK_DIR = ".openclaw-runtime-deps.lock"; const BUNDLED_RUNTIME_DEPS_LOCK_OWNER_FILE = "owner.json"; @@ -71,47 +68,16 @@ const BUNDLED_RUNTIME_DEPS_OWNERLESS_LOCK_STALE_MS = 30_000; const DEFAULT_UNKNOWN_RUNTIME_DEPS_ROOTS_TO_KEEP = 20; const DEFAULT_UNKNOWN_RUNTIME_DEPS_MIN_AGE_MS = 10 * 60_000; const BUNDLED_RUNTIME_DEPS_INSTALL_PROGRESS_INTERVAL_MS = 5_000; -const BUNDLED_RUNTIME_MIRROR_MATERIALIZED_EXTENSIONS = new Set([".cjs", ".js", ".mjs"]); -const BUNDLED_EXTENSION_DIST_DIR = "extensions"; -const MIRRORED_CORE_RUNTIME_DEP_NAMES = [ - "@agentclientprotocol/sdk", - "@lydell/node-pty", - "croner", - "dotenv", - "jiti", - "json5", - "jszip", - "markdown-it", - "semver", - "tar", - "tslog", - "web-push", -] as const; const MIRRORED_PACKAGE_RUNTIME_DEP_PLUGIN_ID = "openclaw-core"; -const BUNDLED_RUNTIME_MIRROR_PLUGIN_REGION_RE = /(?:^|\n)\/\/#region extensions\/[^/\s]+(?:\/|$)/u; -const BUNDLED_RUNTIME_MIRROR_IMPORT_SPECIFIER_RE = - /(?:^|[;\n])\s*(?:import|export)\s+(?:[^'"()]+?\s+from\s+)?["']([^"']+)["']|\bimport\(\s*["']([^"']+)["']\s*\)|\brequire\(\s*["']([^"']+)["']\s*\)/g; const NPM_EXECPATH_ENV_KEY = "npm_execpath"; const MAX_RUNTIME_DEPS_FILE_CACHE_ENTRIES = 2048; const registeredBundledRuntimeDepNodePaths = new Set(); -const bundledRuntimeMirrorMaterializeCache = new Map< - string, - { signature: string; materialize: boolean } ->(); const runtimeDepsTextFileCache = new Map(); const runtimeDepsJsonObjectCache = new Map< string, { signature: string; value: JsonObject | null } >(); -const runtimeDepsImportSpecifierCache = new Map< - string, - { signature: string; value: readonly string[] } ->(); -const runtimeMirrorMaterializeImportSpecifierCache = new Map< - string, - { signature: string; value: readonly string[] } ->(); export type BundledRuntimeDepsNpmRunner = { command: string; @@ -119,93 +85,16 @@ export type BundledRuntimeDepsNpmRunner = { env?: NodeJS.ProcessEnv; }; -function clearBundledRuntimeMirrorMaterializeCache(): void { - bundledRuntimeMirrorMaterializeCache.clear(); -} +type BundledRuntimeDepsPackageManager = "pnpm" | "npm"; -function statSignature(stat: Pick): string { - return `${stat.dev}:${stat.ino}:${stat.size}:${stat.mtimeMs}`; -} +type BundledRuntimeDepsPackageManagerRunner = BundledRuntimeDepsNpmRunner & { + packageManager: BundledRuntimeDepsPackageManager; +}; -function computeBundledRuntimeMirrorDistFileMaterialization(sourcePath: string): boolean { - const signature = getRuntimeDepsFileSignature(sourcePath); - const source = readRuntimeDepsTextFile(sourcePath, signature); - if (source === null) { - return false; - } - if (BUNDLED_RUNTIME_MIRROR_PLUGIN_REGION_RE.test(source)) { - return true; - } - for (const specifier of readRuntimeMirrorMaterializeImportSpecifiers( - sourcePath, - signature, - source, - )) { - if ( - specifier !== "" && - !specifier.startsWith(".") && - !specifier.startsWith("/") && - !specifier.startsWith("node:") && - !specifier.includes(":") - ) { - return false; - } - } - return true; -} - -export function shouldMaterializeBundledRuntimeMirrorDistFile(sourcePath: string): boolean { - if (!BUNDLED_RUNTIME_MIRROR_MATERIALIZED_EXTENSIONS.has(path.extname(sourcePath))) { - return false; - } - const cacheKey = path.resolve(sourcePath); - let signature: string; - try { - signature = statSignature(fs.statSync(sourcePath)); - } catch { - bundledRuntimeMirrorMaterializeCache.delete(cacheKey); - return false; - } - const cached = bundledRuntimeMirrorMaterializeCache.get(cacheKey); - if (cached?.signature === signature) { - return cached.materialize; - } - const materialize = computeBundledRuntimeMirrorDistFileMaterialization(sourcePath); - bundledRuntimeMirrorMaterializeCache.set(cacheKey, { signature, materialize }); - return materialize; -} - -export function materializeBundledRuntimeMirrorDistFile( - sourcePath: string, - targetPath: string, -): void { - if (path.resolve(sourcePath) === path.resolve(targetPath)) { - return; - } - try { - if ( - fs.realpathSync(sourcePath) === fs.realpathSync(targetPath) && - !fs.lstatSync(targetPath).isSymbolicLink() - ) { - return; - } - } catch { - // Missing targets are expected before the mirror file is materialized. - } - fs.mkdirSync(path.dirname(targetPath), { recursive: true, mode: 0o755 }); - fs.rmSync(targetPath, { recursive: true, force: true }); - try { - fs.linkSync(sourcePath, targetPath); - return; - } catch { - fs.copyFileSync(sourcePath, targetPath); - } - try { - const sourceMode = fs.statSync(sourcePath).mode; - fs.chmodSync(targetPath, sourceMode | 0o600); - } catch { - // Readable materialized chunks are enough for ESM loading. - } +function createBundledRuntimeDepsEnsureResult( + installedSpecs: string[], +): BundledRuntimeDepsEnsureResult { + return { installedSpecs }; } const BUNDLED_RUNTIME_DEP_SEGMENT_RE = /^[a-z0-9][a-z0-9._-]*$/; @@ -304,12 +193,6 @@ function resolveDependencySentinelAbsolutePath(rootDir: string, depName: string) return sentinelPath; } -function readInstalledDependencyVersion(rootDir: string, depName: string): string | null { - const parsed = readJsonObject(resolveDependencySentinelAbsolutePath(rootDir, depName)); - const version = parsed && typeof parsed.version === "string" ? parsed.version.trim() : ""; - return version || null; -} - function readJsonObject(filePath: string): JsonObject | null { const signature = getRuntimeDepsFileSignature(filePath); const cached = signature ? runtimeDepsJsonObjectCache.get(filePath) : undefined; @@ -356,52 +239,6 @@ function readRuntimeDepsTextFile(filePath: string, signature?: string | null): s } } -function readRuntimeDepsImportSpecifiers( - filePath: string, - signature: string | null, - source: string, -): readonly string[] { - const cached = signature ? runtimeDepsImportSpecifierCache.get(filePath) : undefined; - if (cached?.signature === signature) { - return cached.value; - } - const value = extractStaticRuntimeImportSpecifiers(source); - if (signature) { - rememberRuntimeDepsCacheEntry(runtimeDepsImportSpecifierCache, filePath, { signature, value }); - } - return value; -} - -function readRuntimeMirrorMaterializeImportSpecifiers( - filePath: string, - signature: string | null, - source: string, -): readonly string[] { - const cached = signature ? runtimeMirrorMaterializeImportSpecifierCache.get(filePath) : undefined; - if (cached?.signature === signature) { - return cached.value; - } - const value = extractRuntimeMirrorMaterializeImportSpecifiers(source); - if (signature) { - rememberRuntimeDepsCacheEntry(runtimeMirrorMaterializeImportSpecifierCache, filePath, { - signature, - value, - }); - } - return value; -} - -function extractRuntimeMirrorMaterializeImportSpecifiers(source: string): string[] { - const specifiers = new Set(); - for (const match of source.matchAll(BUNDLED_RUNTIME_MIRROR_IMPORT_SPECIFIER_RE)) { - const specifier = match[1] ?? match[2] ?? match[3]; - if (specifier) { - specifiers.add(specifier); - } - } - return [...specifiers]; -} - function getRuntimeDepsFileSignature(filePath: string): string | null { try { const stat = fs.statSync(filePath, { bigint: true }); @@ -605,8 +442,8 @@ function formatRuntimeDepsLockTimeoutMessage(params: { } export const __testing = { - clearBundledRuntimeMirrorMaterializeCache, formatRuntimeDepsLockTimeoutMessage, + resolveBundledRuntimeDepsPnpmRunner, shouldRemoveRuntimeDepsLock, }; @@ -698,10 +535,37 @@ function collectRuntimeDeps(packageJson: JsonObject): Record { }; } -function collectMirroredPackageRuntimeDeps( - packageRoot: string | null, - ownerPluginIds?: ReadonlySet, -): { +function collectDeclaredMirroredRootRuntimeDepNames(packageJson: JsonObject): string[] { + const openclaw = packageJson.openclaw; + const bundle = + openclaw && typeof openclaw === "object" && !Array.isArray(openclaw) + ? (openclaw as JsonObject).bundle + : undefined; + const rawNames = + bundle && typeof bundle === "object" && !Array.isArray(bundle) + ? (bundle as JsonObject).mirroredRootRuntimeDependencies + : undefined; + if (rawNames === undefined) { + return []; + } + if (!Array.isArray(rawNames)) { + throw new Error("openclaw.bundle.mirroredRootRuntimeDependencies must be an array"); + } + const names = new Set(); + for (const rawName of rawNames) { + if (typeof rawName !== "string") { + throw new Error("openclaw.bundle.mirroredRootRuntimeDependencies must contain strings"); + } + const normalizedName = normalizeInstallableRuntimeDepName(rawName); + if (!normalizedName) { + throw new Error(`Invalid mirrored bundled runtime dependency name: ${rawName}`); + } + names.add(normalizedName); + } + return [...names].toSorted((left, right) => left.localeCompare(right)); +} + +function collectMirroredPackageRuntimeDeps(packageRoot: string | null): { name: string; version: string; pluginIds: string[]; @@ -714,190 +578,20 @@ function collectMirroredPackageRuntimeDeps( return []; } const runtimeDeps = collectRuntimeDeps(packageJson); - const coreRuntimeDeps = MIRRORED_CORE_RUNTIME_DEP_NAMES.flatMap((name) => { + const deps: RuntimeDepEntry[] = []; + for (const name of collectDeclaredMirroredRootRuntimeDepNames(packageJson)) { const dep = parseInstallableRuntimeDep(name, runtimeDeps[name]); - return dep ? [{ ...dep, pluginIds: [MIRRORED_PACKAGE_RUNTIME_DEP_PLUGIN_ID] }] : []; - }); - return mergeRuntimeDepEntries([ - ...coreRuntimeDeps, - ...collectRootDistMirroredRuntimeDeps({ - packageRoot, - runtimeDeps, - ownerPluginIds, - }), - ]); -} - -function packageNameFromSpecifier(specifier: string): string | null { - if ( - specifier.startsWith(".") || - specifier.startsWith("/") || - specifier.startsWith("node:") || - specifier.startsWith("#") - ) { - return null; - } - const [first, second] = specifier.split("/"); - if (!first) { - return null; - } - return first.startsWith("@") && second ? `${first}/${second}` : first; -} - -function extractStaticRuntimeImportSpecifiers(source: string): string[] { - const specifiers = new Set(); - const patterns = [ - /\bfrom\s*["']([^"']+)["']/g, - /\bimport\s*["']([^"']+)["']/g, - /\bimport\s*\(\s*["']([^"']+)["']\s*\)/g, - /\brequire\s*\(\s*["']([^"']+)["']\s*\)/g, - ]; - for (const pattern of patterns) { - for (const match of source.matchAll(pattern)) { - if (match[1]) { - specifiers.add(match[1]); - } + if (!dep) { + throw new Error( + `Declared mirrored bundled runtime dependency ${name} is missing from package dependencies`, + ); } + deps.push({ + ...dep, + pluginIds: [MIRRORED_PACKAGE_RUNTIME_DEP_PLUGIN_ID], + }); } - return [...specifiers]; -} - -function walkRuntimeDistJavaScriptFiles(params: { - rootDir: string; - skipTopLevelDirs?: ReadonlySet; -}): string[] { - if (!fs.existsSync(params.rootDir)) { - return []; - } - const files: string[] = []; - const queue = [params.rootDir]; - while (queue.length > 0) { - const current = queue.shift(); - if (!current) { - continue; - } - for (const entry of fs.readdirSync(current, { withFileTypes: true })) { - const fullPath = path.join(current, entry.name); - if (entry.isDirectory()) { - const isSkippedTopLevelDir = - path.resolve(current) === path.resolve(params.rootDir) && - params.skipTopLevelDirs?.has(entry.name); - if (entry.name !== "node_modules" && !isSkippedTopLevelDir) { - queue.push(fullPath); - } - continue; - } - if ( - entry.isFile() && - BUNDLED_RUNTIME_MIRROR_MATERIALIZED_EXTENSIONS.has(path.extname(entry.name)) - ) { - files.push(fullPath); - } - } - } - return files.toSorted((left, right) => left.localeCompare(right)); -} - -function isPluginOwnedDistImporter(params: { - relativePath: string; - source: string; - pluginIds: readonly string[]; -}): boolean { - return params.pluginIds.some((pluginId) => { - const pluginPathPrefix = `${BUNDLED_EXTENSION_DIST_DIR}/${pluginId}/`; - return ( - params.relativePath.startsWith(pluginPathPrefix) || - params.source.includes(`//#region ${pluginPathPrefix}`) - ); - }); -} - -function collectBundledRuntimeDependencyOwners(packageRoot: string): Map< - string, - { - name: string; - version: string; - pluginIds: string[]; - } -> { - const extensionsDir = path.join(packageRoot, "dist", "extensions"); - if (!fs.existsSync(extensionsDir)) { - return new Map(); - } - const owners = new Map(); - for (const entry of fs.readdirSync(extensionsDir, { withFileTypes: true })) { - if (!entry.isDirectory()) { - continue; - } - const pluginId = entry.name; - const packageJson = readJsonObject(path.join(extensionsDir, pluginId, "package.json")); - if (!packageJson) { - continue; - } - for (const [name, rawVersion] of Object.entries(collectRuntimeDeps(packageJson))) { - const dep = parseInstallableRuntimeDep(name, rawVersion); - if (!dep) { - continue; - } - const existing = owners.get(dep.name); - if (existing) { - existing.pluginIds = [...new Set([...existing.pluginIds, pluginId])].toSorted( - (left, right) => left.localeCompare(right), - ); - continue; - } - owners.set(dep.name, { ...dep, pluginIds: [pluginId] }); - } - } - return owners; -} - -function collectRootDistMirroredRuntimeDeps(params: { - packageRoot: string; - runtimeDeps: Record; - ownerPluginIds?: ReadonlySet; -}): { name: string; version: string; pluginIds: string[] }[] { - const dependencyOwners = collectBundledRuntimeDependencyOwners(params.packageRoot); - if (dependencyOwners.size === 0) { - return []; - } - const mirrored = new Map(); - const distDir = path.join(params.packageRoot, "dist"); - for (const filePath of walkRuntimeDistJavaScriptFiles({ - rootDir: distDir, - skipTopLevelDirs: new Set(["extensions"]), - })) { - const signature = getRuntimeDepsFileSignature(filePath); - const source = readRuntimeDepsTextFile(filePath, signature); - if (source === null) { - continue; - } - const relativePath = path.relative(distDir, filePath).replaceAll(path.sep, "/"); - for (const specifier of readRuntimeDepsImportSpecifiers(filePath, signature, source)) { - const dependencyName = packageNameFromSpecifier(specifier); - if (!dependencyName) { - continue; - } - const owner = dependencyOwners.get(dependencyName); - if (!owner) { - continue; - } - if ( - params.ownerPluginIds && - !owner.pluginIds.some((pluginId) => params.ownerPluginIds?.has(pluginId)) - ) { - continue; - } - if (isPluginOwnedDistImporter({ relativePath, source, pluginIds: owner.pluginIds })) { - continue; - } - const dep = parseInstallableRuntimeDep(dependencyName, params.runtimeDeps[dependencyName]); - if (dep) { - mirrored.set(dep.name, { ...dep, pluginIds: owner.pluginIds }); - } - } - } - return [...mirrored.values()].toSorted((left, right) => { + return deps.toSorted((left, right) => { const nameOrder = left.name.localeCompare(right.name); return nameOrder === 0 ? left.version.localeCompare(right.version) : nameOrder; }); @@ -944,35 +638,6 @@ function isSourceCheckoutRoot(packageRoot: string): boolean { ); } -function resolveSourceCheckoutBundledPluginPackageRoot(pluginRoot: string): string | null { - const extensionsDir = path.dirname(path.resolve(pluginRoot)); - if (path.basename(extensionsDir) !== "extensions") { - return null; - } - const packageRoot = path.dirname(extensionsDir); - return isSourceCheckoutRoot(packageRoot) ? packageRoot : null; -} - -function resolveSourceCheckoutDistPackageRoot(pluginRoot: string): string | null { - const extensionsDir = path.dirname(pluginRoot); - const buildDir = path.dirname(extensionsDir); - if ( - path.basename(extensionsDir) !== "extensions" || - (path.basename(buildDir) !== "dist" && path.basename(buildDir) !== "dist-runtime") - ) { - return null; - } - const packageRoot = path.dirname(buildDir); - return isSourceCheckoutRoot(packageRoot) ? packageRoot : null; -} - -function resolveSourceCheckoutPackageRoot(pluginRoot: string): string | null { - return ( - resolveSourceCheckoutBundledPluginPackageRoot(pluginRoot) ?? - resolveSourceCheckoutDistPackageRoot(pluginRoot) - ); -} - function resolveBundledPluginPackageRoot(pluginRoot: string): string | null { const extensionsDir = path.dirname(path.resolve(pluginRoot)); const buildDir = path.dirname(extensionsDir); @@ -1027,15 +692,6 @@ function isPackagedBundledPluginRoot(pluginRoot: string): boolean { return Boolean(packageRoot && !isSourceCheckoutRoot(packageRoot)); } -function createRuntimeDepsCacheKey(pluginId: string, specs: readonly string[]): string { - return createHash("sha256") - .update(pluginId) - .update("\0") - .update(specs.join("\0")) - .digest("hex") - .slice(0, 16); -} - function createPathHash(value: string): string { return createHash("sha256").update(path.resolve(value)).digest("hex").slice(0, 12); } @@ -1050,97 +706,105 @@ function readPackageVersion(packageRoot: string): string { return version || "unknown"; } -function readRetainedRuntimeDepsManifest(installRoot: string): string[] { - const parsed = readJsonObject(path.join(installRoot, RETAINED_RUNTIME_DEPS_MANIFEST)); - const specs = parsed?.specs; - if (!Array.isArray(specs)) { - return []; - } - return specs - .filter((entry): entry is string => typeof entry === "string" && entry.trim().length > 0) - .toSorted((left, right) => left.localeCompare(right)); -} - -function writeRetainedRuntimeDepsManifest(installRoot: string, specs: readonly string[]): void { - fs.mkdirSync(installRoot, { recursive: true }); - fs.writeFileSync( - path.join(installRoot, RETAINED_RUNTIME_DEPS_MANIFEST), - `${JSON.stringify({ specs: [...specs].toSorted((left, right) => left.localeCompare(right)) }, null, 2)}\n`, - "utf8", - ); -} - -function removeRetainedRuntimeDepsManifest(installRoot: string): void { - fs.rmSync(path.join(installRoot, RETAINED_RUNTIME_DEPS_MANIFEST), { force: true }); -} - -function removeRuntimeDepPackageDir(rootDir: string, depName: string): void { - const packageDir = path.dirname(resolveDependencySentinelAbsolutePath(rootDir, depName)); - fs.rmSync(packageDir, { recursive: true, force: true }); - if (depName.startsWith("@")) { - try { - const scopeDir = path.dirname(packageDir); - if (fs.existsSync(scopeDir) && fs.readdirSync(scopeDir).length === 0) { - fs.rmdirSync(scopeDir); - } - } catch { - // Empty scope cleanup is best-effort; removing the package dir is enough. - } - } -} - -function pruneRetainedRuntimeDepsManifestSpecs(params: { - installRoot: string; - previousSpecs: readonly string[]; - nextSpecs: readonly string[]; -}): void { - if (params.previousSpecs.length === 0) { - return; - } - const nextNames = new Set( - params.nextSpecs.map((spec) => parseInstallableRuntimeDepSpec(spec).name), - ); - for (const spec of params.previousSpecs) { - const dep = parseInstallableRuntimeDepSpec(spec); - if (!nextNames.has(dep.name)) { - removeRuntimeDepPackageDir(params.installRoot, dep.name); - } - } -} - -function collectAlreadyStagedBundledRuntimeDepSpecs(params: { - pluginRoot: string; - installRoot: string; - config?: OpenClawConfig; -}): string[] { - const packageRoot = resolveBundledPluginPackageRoot(params.pluginRoot); - if (!packageRoot) { - return []; - } - const extensionsDir = path.join(packageRoot, "dist", "extensions"); - if (!fs.existsSync(extensionsDir)) { - return []; - } - const { deps, pluginIds } = collectBundledPluginRuntimeDeps({ - extensionsDir, - config: params.config, +function normalizeRuntimeDepSpecs(specs: readonly string[]): string[] { + specs.forEach((spec) => { + parseInstallableRuntimeDepSpec(spec); }); - const packageRuntimeDeps = - pluginIds.length > 0 ? collectMirroredPackageRuntimeDeps(packageRoot, new Set(pluginIds)) : []; - return mergeRuntimeDepEntries([...deps, ...packageRuntimeDeps]) - .filter((dep) => hasDependencySentinel([params.installRoot], dep)) - .map((dep) => `${dep.name}@${dep.version}`) - .toSorted((left, right) => left.localeCompare(right)); + return [...new Set(specs)].toSorted((left, right) => left.localeCompare(right)); } -function shouldPersistRetainedRuntimeDepsManifest(params: { - pluginRoot: string; - installRoot: string; -}): boolean { - if (path.resolve(params.installRoot) !== path.resolve(params.pluginRoot)) { - return true; +function readGeneratedInstallManifestSpecs(installRoot: string): string[] | null { + const parsed = readJsonObject(path.join(installRoot, "package.json")); + if (parsed?.name !== "openclaw-runtime-deps-install") { + return null; } - return !resolveSourceCheckoutPackageRoot(params.pluginRoot); + const dependencies = parsed.dependencies; + if (!dependencies || typeof dependencies !== "object" || Array.isArray(dependencies)) { + return []; + } + const specs: string[] = []; + for (const [name, version] of Object.entries(dependencies as Record)) { + const dep = parseInstallableRuntimeDep(name, version); + if (dep) { + specs.push(`${dep.name}@${dep.version}`); + } + } + return normalizeRuntimeDepSpecs(specs); +} + +function readPackageRuntimeDepSpecs(packageRoot: string): string[] | null { + const parsed = readJsonObject(path.join(packageRoot, "package.json")); + if (!parsed || parsed.name === "openclaw-runtime-deps-install") { + return null; + } + const specs = Object.entries(collectRuntimeDeps(parsed)) + .map(([name, rawVersion]) => parseInstallableRuntimeDep(name, rawVersion)) + .filter((dep): dep is { name: string; version: string } => Boolean(dep)) + .map((dep) => `${dep.name}@${dep.version}`); + return normalizeRuntimeDepSpecs(specs); +} + +function sameRuntimeDepSpecs(left: readonly string[], right: readonly string[]): boolean { + const normalizedLeft = normalizeRuntimeDepSpecs(left); + const normalizedRight = normalizeRuntimeDepSpecs(right); + return ( + normalizedLeft.length === normalizedRight.length && + normalizedLeft.every((entry, index) => entry === normalizedRight[index]) + ); +} + +function readInstalledRuntimeDepVersion(rootDir: string, depName: string): string | null { + try { + const parsed = JSON.parse( + fs.readFileSync(resolveDependencySentinelAbsolutePath(rootDir, depName), "utf8"), + ) as unknown; + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { + return null; + } + const version = (parsed as JsonObject).version; + return typeof version === "string" && version.trim() ? version.trim() : null; + } catch { + return null; + } +} + +function isRuntimeDepSatisfied(rootDir: string, dep: { name: string; version: string }): boolean { + const installedVersion = readInstalledRuntimeDepVersion(rootDir, dep.name); + return Boolean(installedVersion && satisfies(installedVersion, dep.version)); +} + +function isRuntimeDepSatisfiedInAnyRoot( + dep: { name: string; version: string }, + roots: readonly string[], +): boolean { + return roots.some((root) => isRuntimeDepSatisfied(root, dep)); +} + +function hasSatisfiedInstallSpecPackages(rootDir: string, specs: readonly string[]): boolean { + return specs + .map(parseInstallableRuntimeDepSpec) + .every((dep) => isRuntimeDepSatisfied(rootDir, dep)); +} + +function isRuntimeDepsPlanMaterialized( + installRoot: string, + installSpecs: readonly string[], +): boolean { + const generatedManifestSpecs = readGeneratedInstallManifestSpecs(installRoot); + const packageManifestSpecs = + generatedManifestSpecs !== null ? null : readPackageRuntimeDepSpecs(installRoot); + return ( + ((generatedManifestSpecs !== null && + sameRuntimeDepSpecs(generatedManifestSpecs, installSpecs)) || + (packageManifestSpecs !== null && sameRuntimeDepSpecs(packageManifestSpecs, installSpecs))) && + hasSatisfiedInstallSpecPackages(installRoot, installSpecs) + ); +} + +function removeLegacyRuntimeDepsManifest(installRoot: string): void { + fs.rmSync(path.join(installRoot, LEGACY_RETAINED_RUNTIME_DEPS_MANIFEST), { + force: true, + }); } export function isWritableDirectory(dir: string): boolean { @@ -1320,74 +984,6 @@ function realpathOrResolve(targetPath: string): string { } } -function resolveSourceCheckoutRuntimeDepsCacheDir(params: { - pluginId: string; - pluginRoot: string; - installSpecs: readonly string[]; -}): string | null { - const packageRoot = resolveSourceCheckoutPackageRoot(params.pluginRoot); - if (!packageRoot) { - return null; - } - return path.join( - packageRoot, - ".local", - "bundled-plugin-runtime-deps", - `${params.pluginId}-${createRuntimeDepsCacheKey(params.pluginId, params.installSpecs)}`, - ); -} - -function hasAllDependencySentinels(rootDir: string, deps: readonly { name: string }[]): boolean { - return deps.every((dep) => fs.existsSync(path.join(rootDir, dependencySentinelPath(dep.name)))); -} - -function isInstalledDependencyVersionSatisfied(installedVersion: string, spec: string): boolean { - const normalizedInstalledVersion = validSemver(installedVersion); - const normalizedRange = validRange(spec); - if (normalizedInstalledVersion && normalizedRange) { - return satisfies(normalizedInstalledVersion, normalizedRange, { - includePrerelease: true, - }); - } - return installedVersion === spec; -} - -function hasDependencySentinel( - searchRoots: readonly string[], - dep: { name: string; version: string }, -): boolean { - return searchRoots.some((rootDir) => { - const installedVersion = readInstalledDependencyVersion(rootDir, dep.name); - return ( - typeof installedVersion === "string" && - isInstalledDependencyVersionSatisfied(installedVersion, dep.version) - ); - }); -} - -function findDependencySentinelRoot( - searchRoots: readonly string[], - dep: { name: string; version: string }, -): string | null { - return ( - searchRoots.find((rootDir) => { - const installedVersion = readInstalledDependencyVersion(rootDir, dep.name); - return ( - typeof installedVersion === "string" && - isInstalledDependencyVersionSatisfied(installedVersion, dep.version) - ); - }) ?? null - ); -} - -function dependencyPackageDir(rootDir: string, depName: string): string { - const normalizedDepName = normalizeInstallableRuntimeDepName(depName); - if (!normalizedDepName) { - throw new Error(`Invalid bundled runtime dependency name: ${depName}`); - } - return path.join(rootDir, "node_modules", ...normalizedDepName.split("/")); -} - function createBundledRuntimeDepsInstallRootPlan(params: { installRoot: string; searchRoots: readonly string[]; @@ -1410,55 +1006,24 @@ function createBundledRuntimeDepsInstallRootPlan(params: { }; } -export function createBundledRuntimeDepsWritableInstallSpecs(params: { +export function createBundledRuntimeDepsInstallSpecs(params: { deps: readonly { name: string; version: string }[]; - searchRoots: readonly string[]; - installRoot: string; }): string[] { - const readOnlyRoots = params.searchRoots.filter( - (rootDir) => path.resolve(rootDir) !== path.resolve(params.installRoot), - ); return params.deps - .filter((dep) => !hasDependencySentinel(readOnlyRoots, dep)) .map((dep) => `${dep.name}@${dep.version}`) .toSorted((left, right) => left.localeCompare(right)); } -function linkBundledRuntimeDepsFromSearchRoots(params: { - deps: readonly { name: string; version: string }[]; - searchRoots: readonly string[]; - installRoot: string; -}): void { - for (const dep of params.deps) { - if (hasDependencySentinel([params.installRoot], dep)) { - continue; - } - const sourceRoot = findDependencySentinelRoot(params.searchRoots, dep); - if (!sourceRoot || path.resolve(sourceRoot) === path.resolve(params.installRoot)) { - continue; - } - const sourceDir = dependencyPackageDir(sourceRoot, dep.name); - const targetDir = dependencyPackageDir(params.installRoot, dep.name); - fs.mkdirSync(path.dirname(targetDir), { recursive: true }); - fs.rmSync(targetDir, { recursive: true, force: true }); - try { - fs.symlinkSync(sourceDir, targetDir, process.platform === "win32" ? "junction" : "dir"); - } catch { - fs.cpSync(sourceDir, targetDir, { recursive: true }); - } - } -} - function assertBundledRuntimeDepsInstalled(rootDir: string, specs: readonly string[]): void { const missingSpecs = specs.filter((spec) => { const dep = parseInstallableRuntimeDepSpec(spec); - return !hasDependencySentinel([rootDir], dep); + return !isRuntimeDepSatisfied(rootDir, dep); }); if (missingSpecs.length === 0) { return; } throw new Error( - `npm install did not place bundled runtime deps in ${rootDir}: ${missingSpecs.join(", ")}`, + `package manager install did not place bundled runtime deps in ${rootDir}: ${missingSpecs.join(", ")}`, ); } @@ -1480,86 +1045,16 @@ function replaceNodeModulesDir(targetDir: string, sourceDir: string): void { } } -function linkNodeModulesDir(targetDir: string, sourceDir: string): boolean { - const parentDir = path.dirname(targetDir); - const tempLink = path.join(parentDir, `.openclaw-runtime-deps-link-${process.pid}-${Date.now()}`); - try { - fs.symlinkSync(sourceDir, tempLink, process.platform === "win32" ? "junction" : "dir"); - fs.rmSync(targetDir, { recursive: true, force: true }); - fs.renameSync(tempLink, targetDir); - return true; - } catch { - try { - fs.rmSync(tempLink, { recursive: true, force: true }); - } catch { - // Best-effort cleanup; caller falls back to copying. - } - return false; - } -} - -function replaceNodeModulesDirFromCache(targetDir: string, sourceDir: string): void { - if (linkNodeModulesDir(targetDir, sourceDir)) { - return; - } - replaceNodeModulesDir(targetDir, sourceDir); -} - -function restoreSourceCheckoutRuntimeDepsFromCache(params: { - cacheDir: string | null; - deps: readonly { name: string }[]; - installRoot: string; -}): boolean { - if (!params.cacheDir) { - return false; - } - const cachedNodeModulesDir = path.join(params.cacheDir, "node_modules"); - if (!hasAllDependencySentinels(params.cacheDir, params.deps)) { - return false; - } - try { - replaceNodeModulesDirFromCache( - path.join(params.installRoot, "node_modules"), - cachedNodeModulesDir, - ); - return true; - } catch { - return false; - } -} - -function storeSourceCheckoutRuntimeDepsCache(params: { - cacheDir: string | null; - installRoot: string; -}): void { - if (!params.cacheDir) { - return; - } - const nodeModulesDir = path.join(params.installRoot, "node_modules"); - if (!fs.existsSync(nodeModulesDir)) { - return; - } - let tempDir: string | null = null; - try { - fs.mkdirSync(path.dirname(params.cacheDir), { recursive: true }); - tempDir = fs.mkdtempSync(path.join(path.dirname(params.cacheDir), ".runtime-deps-cache-")); - fs.cpSync(nodeModulesDir, path.join(tempDir, "node_modules"), { recursive: true }); - fs.rmSync(params.cacheDir, { recursive: true, force: true }); - fs.renameSync(tempDir, params.cacheDir); - } catch { - if (tempDir) { - fs.rmSync(tempDir, { recursive: true, force: true }); - } - } -} - export function createBundledRuntimeDepsInstallEnv( env: NodeJS.ProcessEnv, options: { cacheDir?: string } = {}, ): NodeJS.ProcessEnv { const nextEnv: NodeJS.ProcessEnv = { ...createNpmProjectInstallEnv(env, options), + npm_config_audit: "false", + npm_config_fund: "false", npm_config_legacy_peer_deps: "true", + npm_config_package_lock: "true", }; for (const key of Object.keys(nextEnv)) { if (key.toLowerCase() === NPM_EXECPATH_ENV_KEY) { @@ -1569,11 +1064,22 @@ export function createBundledRuntimeDepsInstallEnv( return nextEnv; } -export function createBundledRuntimeDepsInstallArgs(missingSpecs: readonly string[]): string[] { - missingSpecs.forEach((spec) => { - parseInstallableRuntimeDepSpec(spec); - }); - return ["install", "--ignore-scripts", ...missingSpecs]; +export function createBundledRuntimeDepsInstallArgs(): string[] { + return ["install", "--ignore-scripts", "--no-audit", "--no-fund", "--omit=dev"]; +} + +function createBundledRuntimeDepsPnpmInstallArgs(params: { storeDir: string }): string[] { + return [ + "install", + "--prod", + "--ignore-scripts", + "--ignore-workspace", + "--config.frozen-lockfile=false", + "--config.minimum-release-age=0", + `--config.store-dir=${params.storeDir}`, + "--config.node-linker=hoisted", + "--config.virtual-store-dir=.pnpm", + ]; } export function resolveBundledRuntimeDepsNpmRunner(params: { @@ -1624,6 +1130,67 @@ export function resolveBundledRuntimeDepsNpmRunner(params: { throw new Error("Unable to resolve a safe npm executable"); } + +function pathEntries(env: NodeJS.ProcessEnv, platform: NodeJS.Platform): string[] { + const pathKey = Object.keys(env).find((key) => key.toLowerCase() === "path") ?? "PATH"; + return (env[pathKey] ?? "") + .split(platform === "win32" ? ";" : path.delimiter) + .map((entry) => entry.trim()) + .filter((entry) => entry.length > 0); +} + +function resolveBundledRuntimeDepsPnpmRunner(params: { + pnpmArgs: string[]; + env?: NodeJS.ProcessEnv; + execPath?: string; + existsSync?: typeof fs.existsSync; + platform?: NodeJS.Platform; +}): BundledRuntimeDepsPackageManagerRunner | null { + const env = params.env ?? process.env; + const execPath = params.execPath ?? process.execPath; + const existsSync = params.existsSync ?? fs.existsSync; + const platform = params.platform ?? process.platform; + const pathImpl = platform === "win32" ? path.win32 : path.posix; + const nodeDir = pathImpl.dirname(execPath); + const names = platform === "win32" ? ["pnpm.exe"] : ["pnpm"]; + const candidateDirs = [nodeDir, ...pathEntries(env, platform)]; + for (const dir of candidateDirs) { + for (const name of names) { + const candidate = pathImpl.resolve(dir, name); + if (pathImpl.isAbsolute(candidate) && existsSync(candidate)) { + return { + packageManager: "pnpm", + command: candidate, + args: params.pnpmArgs, + }; + } + } + } + return null; +} + +function resolveBundledRuntimeDepsPackageManagerRunner(params: { + installExecutionRoot: string; + env: NodeJS.ProcessEnv; + npmArgs: string[]; +}): BundledRuntimeDepsPackageManagerRunner { + const pnpmRunner = resolveBundledRuntimeDepsPnpmRunner({ + env: params.env, + pnpmArgs: createBundledRuntimeDepsPnpmInstallArgs({ + storeDir: path.join(params.installExecutionRoot, ".openclaw-pnpm-store"), + }), + }); + if (pnpmRunner) { + return pnpmRunner; + } + return { + packageManager: "npm", + ...resolveBundledRuntimeDepsNpmRunner({ + env: params.env, + npmArgs: params.npmArgs, + }), + }; +} type BundledPluginRuntimeDepsManifest = { channels: string[]; enabledByDefault: boolean; @@ -1724,6 +1291,34 @@ function isBundledPluginConfiguredForRuntimeDeps(params: { return manifest.enabledByDefault; } +function isBundledPluginExplicitlyDisabledForRuntimeDeps(params: { + config: OpenClawConfig; + pluginId: string; + pluginDir: string; + manifestCache?: BundledPluginRuntimeDepsManifestCache; +}): boolean { + const plugins = normalizePluginsConfig(params.config.plugins); + if (plugins.entries[params.pluginId]?.enabled === false) { + return true; + } + const manifest = readBundledPluginRuntimeDepsManifest(params.pluginDir, params.manifestCache); + return manifest.channels.some((channelId) => { + const normalizedChannelId = normalizeOptionalLowercaseString(channelId); + if (!normalizedChannelId) { + return false; + } + const channelConfig = (params.config.channels as Record | undefined)?.[ + normalizedChannelId + ]; + return ( + channelConfig && + typeof channelConfig === "object" && + !Array.isArray(channelConfig) && + (channelConfig as { enabled?: unknown }).enabled === false + ); + }); +} + function shouldIncludeBundledPluginRuntimeDeps(params: { config?: OpenClawConfig; pluginIds?: ReadonlySet; @@ -1734,7 +1329,18 @@ function shouldIncludeBundledPluginRuntimeDeps(params: { manifestCache?: BundledPluginRuntimeDepsManifestCache; }): boolean { if (params.selectedPluginIds) { - return params.selectedPluginIds.has(params.pluginId); + return ( + params.selectedPluginIds.has(params.pluginId) && + !( + params.config && + isBundledPluginExplicitlyDisabledForRuntimeDeps({ + config: params.config, + pluginId: params.pluginId, + pluginDir: params.pluginDir, + manifestCache: params.manifestCache, + }) + ) + ); } const scopedToPluginIds = Boolean(params.pluginIds); if (params.pluginIds) { @@ -1892,31 +1498,17 @@ export function scanBundledPluginRuntimeDeps(params: { includeConfiguredChannels: params.includeConfiguredChannels, }); const packageRuntimeDeps = - pluginIds.length > 0 - ? collectMirroredPackageRuntimeDeps(params.packageRoot, new Set(pluginIds)) - : []; + pluginIds.length > 0 ? collectMirroredPackageRuntimeDeps(params.packageRoot) : []; const allDeps = mergeRuntimeDepEntries([...deps, ...packageRuntimeDeps]); - const packageInstallRootPlan = resolveBundledRuntimeDependencyPackageInstallRootPlan( + const installRootPlan = resolveBundledRuntimeDependencyPackageInstallRootPlan( params.packageRoot, { env: params.env, }, ); - const missing = allDeps.filter((dep) => { - if (hasDependencySentinel(packageInstallRootPlan.searchRoots, dep)) { - return false; - } - if (dep.pluginIds.includes(MIRRORED_PACKAGE_RUNTIME_DEP_PLUGIN_ID)) { - return true; - } - return dep.pluginIds.every((pluginId) => { - const pluginRoot = path.join(extensionsDir, pluginId); - const installRootPlan = resolveBundledRuntimeDependencyInstallRootPlan(pluginRoot, { - env: params.env, - }); - return !hasDependencySentinel(installRootPlan.searchRoots, dep); - }); - }); + const missing = allDeps.filter( + (dep) => !isRuntimeDepSatisfiedInAnyRoot(dep, installRootPlan.searchRoots), + ); return { deps: allDeps, missing, conflicts }; } @@ -2132,6 +1724,7 @@ function formatBundledRuntimeDepsInstallElapsed(ms: number): string { function emitBundledRuntimeDepsOutputProgress( chunk: Buffer, stream: "stdout" | "stderr", + packageManager: BundledRuntimeDepsPackageManager, onProgress: ((message: string) => void) | undefined, ): void { if (!onProgress) { @@ -2144,7 +1737,86 @@ function emitBundledRuntimeDepsOutputProgress( .filter((line) => line.length > 0) .slice(-3); for (const line of lines) { - onProgress(`npm ${stream}: ${line}`); + onProgress(`${packageManager} ${stream}: ${line}`); + } +} + +type BundledRuntimeDepsInstallContext = { + installExecutionRoot: string; + installSpecs: string[]; + installEnv: NodeJS.ProcessEnv; + runner: BundledRuntimeDepsPackageManagerRunner; + isolatedExecutionRoot: boolean; + cleanInstallExecutionRoot: boolean; +}; + +function createBundledRuntimeDepsInstallContext(params: { + installRoot: string; + installExecutionRoot?: string; + installSpecs: readonly string[]; + env: NodeJS.ProcessEnv; + warn?: (message: string) => void; +}): BundledRuntimeDepsInstallContext { + const installExecutionRoot = params.installExecutionRoot ?? params.installRoot; + const isolatedExecutionRoot = + path.resolve(installExecutionRoot) !== path.resolve(params.installRoot); + const cleanInstallExecutionRoot = + isolatedExecutionRoot && + shouldCleanBundledRuntimeDepsInstallExecutionRoot({ + installRoot: params.installRoot, + installExecutionRoot, + }); + + fs.mkdirSync(params.installRoot, { recursive: true }); + fs.mkdirSync(installExecutionRoot, { recursive: true }); + const diskWarning = createLowDiskSpaceWarning({ + targetPath: installExecutionRoot, + purpose: "bundled plugin runtime dependency staging", + }); + if (diskWarning) { + params.warn?.(diskWarning); + } + ensureNpmInstallExecutionManifest(installExecutionRoot, params.installSpecs); + const installEnv = createBundledRuntimeDepsInstallEnv(params.env, { + cacheDir: path.join(installExecutionRoot, ".openclaw-npm-cache"), + }); + const runner = resolveBundledRuntimeDepsPackageManagerRunner({ + installExecutionRoot, + env: installEnv, + npmArgs: createBundledRuntimeDepsInstallArgs(), + }); + + return { + installExecutionRoot, + installSpecs: normalizeRuntimeDepSpecs(params.installSpecs), + installEnv, + runner, + isolatedExecutionRoot, + cleanInstallExecutionRoot, + }; +} + +function finalizeBundledRuntimeDepsInstall(params: { + installRoot: string; + context: BundledRuntimeDepsInstallContext; +}): void { + const { context } = params; + assertBundledRuntimeDepsInstalled(context.installExecutionRoot, context.installSpecs); + if (context.isolatedExecutionRoot) { + const stagedNodeModulesDir = path.join(context.installExecutionRoot, "node_modules"); + if (!fs.existsSync(stagedNodeModulesDir)) { + throw new Error(`${context.runner.packageManager} install did not produce node_modules`); + } + const targetNodeModulesDir = path.join(params.installRoot, "node_modules"); + replaceNodeModulesDir(targetNodeModulesDir, stagedNodeModulesDir); + assertBundledRuntimeDepsInstalled(params.installRoot, context.installSpecs); + } + removeLegacyRuntimeDepsManifest(params.installRoot); +} + +function cleanupBundledRuntimeDepsInstallContext(context: BundledRuntimeDepsInstallContext): void { + if (context.cleanInstallExecutionRoot) { + fs.rmSync(context.installExecutionRoot, { recursive: true, force: true }); } } @@ -2153,6 +1825,7 @@ async function spawnBundledRuntimeDepsInstall(params: { args: string[]; cwd: string; env: NodeJS.ProcessEnv; + packageManager: BundledRuntimeDepsPackageManager; onProgress?: (message: string) => void; }): Promise { await new Promise((resolve, reject) => { @@ -2161,7 +1834,7 @@ async function spawnBundledRuntimeDepsInstall(params: { params.onProgress && setInterval(() => { params.onProgress?.( - `npm install still running (${formatBundledRuntimeDepsInstallElapsed(Date.now() - startedAtMs)} elapsed)`, + `${params.packageManager} install still running (${formatBundledRuntimeDepsInstallElapsed(Date.now() - startedAtMs)} elapsed)`, ); }, BUNDLED_RUNTIME_DEPS_INSTALL_PROGRESS_INTERVAL_MS); heartbeat?.unref?.(); @@ -2181,11 +1854,21 @@ async function spawnBundledRuntimeDepsInstall(params: { const stderr: Buffer[] = []; child.stdout?.on("data", (chunk: Buffer) => { stdout.push(chunk); - emitBundledRuntimeDepsOutputProgress(chunk, "stdout", params.onProgress); + emitBundledRuntimeDepsOutputProgress( + chunk, + "stdout", + params.packageManager, + params.onProgress, + ); }); child.stderr?.on("data", (chunk: Buffer) => { stderr.push(chunk); - emitBundledRuntimeDepsOutputProgress(chunk, "stderr", params.onProgress); + emitBundledRuntimeDepsOutputProgress( + chunk, + "stderr", + params.packageManager, + params.onProgress, + ); }); child.on("error", (error) => { settle(() => reject(new Error(formatBundledRuntimeDepsInstallError({ error })))); @@ -2214,150 +1897,82 @@ async function spawnBundledRuntimeDepsInstall(params: { export function installBundledRuntimeDeps(params: { installRoot: string; installExecutionRoot?: string; - linkNodeModulesFromExecutionRoot?: boolean; missingSpecs: string[]; installSpecs?: string[]; env: NodeJS.ProcessEnv; warn?: (message: string) => void; }): void { - const installExecutionRoot = params.installExecutionRoot ?? params.installRoot; - const isolatedExecutionRoot = - path.resolve(installExecutionRoot) !== path.resolve(params.installRoot); - const cleanInstallExecutionRoot = - isolatedExecutionRoot && - shouldCleanBundledRuntimeDepsInstallExecutionRoot({ - installRoot: params.installRoot, - installExecutionRoot, - }); + const installSpecs = normalizeRuntimeDepSpecs(params.installSpecs ?? params.missingSpecs); + if (installSpecs.length === 0) { + return; + } + if (isRuntimeDepsPlanMaterialized(params.installRoot, installSpecs)) { + removeLegacyRuntimeDepsManifest(params.installRoot); + return; + } + const context = createBundledRuntimeDepsInstallContext({ + installRoot: params.installRoot, + installExecutionRoot: params.installExecutionRoot, + installSpecs, + env: params.env, + warn: params.warn, + }); try { - fs.mkdirSync(params.installRoot, { recursive: true }); - fs.mkdirSync(installExecutionRoot, { recursive: true }); - const diskWarning = createLowDiskSpaceWarning({ - targetPath: installExecutionRoot, - purpose: "bundled plugin runtime dependency staging", - }); - if (diskWarning) { - params.warn?.(diskWarning); - } - // Always make npm see an OpenClaw-owned package root. The package-level - // doctor repair path installs directly in the external stage dir; without a - // manifest, npm can honor a user's global prefix config and write under - // $HOME/node_modules instead of our managed stage. - // - // The manifest also declares retained staged deps. npm may prune packages - // that are present in node_modules but absent from package dependencies - // while installing a new explicit spec, so keep retained deps in the - // manifest and pass only actually missing specs as install args. - ensureNpmInstallExecutionManifest( - installExecutionRoot, - params.installSpecs ?? params.missingSpecs, - ); - const installEnv = createBundledRuntimeDepsInstallEnv(params.env, { - cacheDir: path.join(installExecutionRoot, ".openclaw-npm-cache"), - }); - const npmRunner = resolveBundledRuntimeDepsNpmRunner({ - env: installEnv, - npmArgs: createBundledRuntimeDepsInstallArgs(params.missingSpecs), - }); - const result = spawnSync(npmRunner.command, npmRunner.args, { - cwd: installExecutionRoot, + const result = spawnSync(context.runner.command, context.runner.args, { + cwd: context.installExecutionRoot, encoding: "utf8", - env: npmRunner.env ?? installEnv, + env: context.runner.env ?? context.installEnv, stdio: "pipe", windowsHide: true, }); if (result.status !== 0 || result.error) { throw new Error(formatBundledRuntimeDepsInstallError(result)); } - assertBundledRuntimeDepsInstalled(installExecutionRoot, params.missingSpecs); - if (isolatedExecutionRoot) { - const stagedNodeModulesDir = path.join(installExecutionRoot, "node_modules"); - if (!fs.existsSync(stagedNodeModulesDir)) { - throw new Error("npm install did not produce node_modules"); - } - const targetNodeModulesDir = path.join(params.installRoot, "node_modules"); - if (params.linkNodeModulesFromExecutionRoot) { - replaceNodeModulesDirFromCache(targetNodeModulesDir, stagedNodeModulesDir); - } else { - replaceNodeModulesDir(targetNodeModulesDir, stagedNodeModulesDir); - } - assertBundledRuntimeDepsInstalled(params.installRoot, params.missingSpecs); - } + finalizeBundledRuntimeDepsInstall({ installRoot: params.installRoot, context }); } finally { - if (cleanInstallExecutionRoot) { - fs.rmSync(installExecutionRoot, { recursive: true, force: true }); - } + cleanupBundledRuntimeDepsInstallContext(context); } } export async function installBundledRuntimeDepsAsync(params: { installRoot: string; installExecutionRoot?: string; - linkNodeModulesFromExecutionRoot?: boolean; missingSpecs: string[]; installSpecs?: string[]; env: NodeJS.ProcessEnv; warn?: (message: string) => void; onProgress?: (message: string) => void; }): Promise { - const installExecutionRoot = params.installExecutionRoot ?? params.installRoot; - const isolatedExecutionRoot = - path.resolve(installExecutionRoot) !== path.resolve(params.installRoot); - const cleanInstallExecutionRoot = - isolatedExecutionRoot && - shouldCleanBundledRuntimeDepsInstallExecutionRoot({ - installRoot: params.installRoot, - installExecutionRoot, - }); + const installSpecs = normalizeRuntimeDepSpecs(params.installSpecs ?? params.missingSpecs); + if (installSpecs.length === 0) { + return; + } + if (isRuntimeDepsPlanMaterialized(params.installRoot, installSpecs)) { + removeLegacyRuntimeDepsManifest(params.installRoot); + return; + } + const context = createBundledRuntimeDepsInstallContext({ + installRoot: params.installRoot, + installExecutionRoot: params.installExecutionRoot, + installSpecs, + env: params.env, + warn: params.warn, + }); try { - fs.mkdirSync(params.installRoot, { recursive: true }); - fs.mkdirSync(installExecutionRoot, { recursive: true }); - const diskWarning = createLowDiskSpaceWarning({ - targetPath: installExecutionRoot, - purpose: "bundled plugin runtime dependency staging", - }); - if (diskWarning) { - params.warn?.(diskWarning); - } - ensureNpmInstallExecutionManifest( - installExecutionRoot, - params.installSpecs ?? params.missingSpecs, - ); - const installEnv = createBundledRuntimeDepsInstallEnv(params.env, { - cacheDir: path.join(installExecutionRoot, ".openclaw-npm-cache"), - }); - const npmRunner = resolveBundledRuntimeDepsNpmRunner({ - env: installEnv, - npmArgs: createBundledRuntimeDepsInstallArgs(params.missingSpecs), - }); params.onProgress?.( - `Starting npm install for bundled plugin runtime deps: ${params.missingSpecs.join(", ")}`, + `Starting ${context.runner.packageManager} install for bundled plugin runtime deps: ${installSpecs.join(", ")}`, ); await spawnBundledRuntimeDepsInstall({ - command: npmRunner.command, - args: npmRunner.args, - cwd: installExecutionRoot, - env: npmRunner.env ?? installEnv, + command: context.runner.command, + args: context.runner.args, + cwd: context.installExecutionRoot, + env: context.runner.env ?? context.installEnv, + packageManager: context.runner.packageManager, onProgress: params.onProgress, }); - assertBundledRuntimeDepsInstalled(installExecutionRoot, params.missingSpecs); - if (isolatedExecutionRoot) { - const stagedNodeModulesDir = path.join(installExecutionRoot, "node_modules"); - if (!fs.existsSync(stagedNodeModulesDir)) { - throw new Error("npm install did not produce node_modules"); - } - const targetNodeModulesDir = path.join(params.installRoot, "node_modules"); - if (params.linkNodeModulesFromExecutionRoot) { - replaceNodeModulesDirFromCache(targetNodeModulesDir, stagedNodeModulesDir); - } else { - replaceNodeModulesDir(targetNodeModulesDir, stagedNodeModulesDir); - } - assertBundledRuntimeDepsInstalled(params.installRoot, params.missingSpecs); - } + finalizeBundledRuntimeDepsInstall({ installRoot: params.installRoot, context }); } finally { - if (cleanInstallExecutionRoot) { - fs.rmSync(installExecutionRoot, { recursive: true, force: true }); - } + cleanupBundledRuntimeDepsInstallContext(context); } } @@ -2370,10 +1985,7 @@ export function repairBundledRuntimeDepsInstallRoot(params: { warn?: (message: string) => void; }): { installSpecs: string[] } { return withBundledRuntimeDepsInstallRootLock(params.installRoot, () => { - const previousRetainedManifestSpecs = readRetainedRuntimeDepsManifest(params.installRoot); - const installSpecs = [...new Set(params.installSpecs)].toSorted((left, right) => - left.localeCompare(right), - ); + const installSpecs = normalizeRuntimeDepSpecs(params.installSpecs); const install = params.installDeps ?? ((installParams) => @@ -2386,24 +1998,20 @@ export function repairBundledRuntimeDepsInstallRoot(params: { })); const finishActivity = beginBundledRuntimeDepsInstall({ installRoot: params.installRoot, - missingSpecs: params.missingSpecs, + missingSpecs: installSpecs, installSpecs, }); + ensureNpmInstallExecutionManifest(params.installRoot, installSpecs); try { install({ installRoot: params.installRoot, - missingSpecs: params.missingSpecs, + missingSpecs: installSpecs, installSpecs, }); } finally { finishActivity(); } - pruneRetainedRuntimeDepsManifestSpecs({ - installRoot: params.installRoot, - previousSpecs: previousRetainedManifestSpecs, - nextSpecs: installSpecs, - }); - writeRetainedRuntimeDepsManifest(params.installRoot, installSpecs); + removeLegacyRuntimeDepsManifest(params.installRoot); return { installSpecs }; }); } @@ -2480,10 +2088,7 @@ export async function repairBundledRuntimeDepsInstallRootAsync(params: { onProgress?: (message: string) => void; }): Promise<{ installSpecs: string[] }> { return await withBundledRuntimeDepsInstallRootLockAsync(params.installRoot, async () => { - const previousRetainedManifestSpecs = readRetainedRuntimeDepsManifest(params.installRoot); - const installSpecs = [...new Set(params.installSpecs)].toSorted((left, right) => - left.localeCompare(right), - ); + const installSpecs = normalizeRuntimeDepSpecs(params.installSpecs); const install = params.installDeps ?? ((installParams) => @@ -2497,24 +2102,21 @@ export async function repairBundledRuntimeDepsInstallRootAsync(params: { })); const finishActivity = beginBundledRuntimeDepsInstall({ installRoot: params.installRoot, - missingSpecs: params.missingSpecs, + missingSpecs: installSpecs, installSpecs, }); + removeLegacyRuntimeDepsManifest(params.installRoot); + ensureNpmInstallExecutionManifest(params.installRoot, installSpecs); try { await install({ installRoot: params.installRoot, - missingSpecs: params.missingSpecs, + missingSpecs: installSpecs, installSpecs, }); } finally { finishActivity(); } - pruneRetainedRuntimeDepsManifestSpecs({ - installRoot: params.installRoot, - previousSpecs: previousRetainedManifestSpecs, - nextSpecs: installSpecs, - }); - writeRetainedRuntimeDepsManifest(params.installRoot, installSpecs); + removeLegacyRuntimeDepsManifest(params.installRoot); return { installSpecs }; }); } @@ -2524,7 +2126,6 @@ export function ensureBundledPluginRuntimeDeps(params: { pluginRoot: string; env: NodeJS.ProcessEnv; config?: OpenClawConfig; - retainSpecs?: readonly string[]; installDeps?: (params: BundledRuntimeDepsInstallParams) => void; }): BundledRuntimeDepsEnsureResult { if ( @@ -2535,11 +2136,11 @@ export function ensureBundledPluginRuntimeDeps(params: { pluginDir: params.pluginRoot, }) ) { - return { installedSpecs: [], retainSpecs: [] }; + return createBundledRuntimeDepsEnsureResult([]); } const packageJson = readJsonObject(path.join(params.pluginRoot, "package.json")); if (!packageJson) { - return { installedSpecs: [], retainSpecs: [] }; + return createBundledRuntimeDepsEnsureResult([]); } const pluginDeps = Object.entries(collectRuntimeDeps(packageJson)) .map(([name, rawVersion]) => parseInstallableRuntimeDep(name, rawVersion)) @@ -2550,141 +2151,74 @@ export function ensureBundledPluginRuntimeDeps(params: { }); const installRoot = installRootPlan.installRoot; const packageRoot = resolveBundledRuntimeDependencyPackageRoot(params.pluginRoot); - const packageRuntimeDeps = - packageRoot && path.resolve(installRoot) !== path.resolve(params.pluginRoot) - ? collectMirroredPackageRuntimeDeps(packageRoot, new Set([params.pluginId])) - : []; - const deps = mergeInstallableRuntimeDeps([...pluginDeps, ...packageRuntimeDeps]); + const usePackageLevelPlan = + packageRoot && path.resolve(installRoot) !== path.resolve(params.pluginRoot); + let deps = pluginDeps; + if (usePackageLevelPlan && packageRoot) { + const packagePlan = collectBundledPluginRuntimeDeps({ + extensionsDir: path.dirname(params.pluginRoot), + ...(params.config ? { config: params.config } : {}), + }); + if (packagePlan.conflicts.length === 0 && packagePlan.deps.length > 0) { + deps = mergeInstallableRuntimeDeps([ + ...packagePlan.deps.map((dep) => ({ name: dep.name, version: dep.version })), + ...collectMirroredPackageRuntimeDeps(packageRoot), + ]); + } else { + deps = mergeInstallableRuntimeDeps([ + ...pluginDeps, + ...collectMirroredPackageRuntimeDeps(packageRoot), + ]); + } + } if (deps.length === 0) { - return { installedSpecs: [], retainSpecs: [] }; + return createBundledRuntimeDepsEnsureResult([]); } return withBundledRuntimeDepsInstallRootLock(installRoot, () => { - const persistRetainedManifest = shouldPersistRetainedRuntimeDepsManifest({ - pluginRoot: params.pluginRoot, - installRoot, - }); - if (!persistRetainedManifest) { - removeRetainedRuntimeDepsManifest(installRoot); - } - linkBundledRuntimeDepsFromSearchRoots({ + const installSpecs = createBundledRuntimeDepsInstallSpecs({ deps, - searchRoots: installRootPlan.searchRoots, - installRoot, }); - const dependencySpecs = createBundledRuntimeDepsWritableInstallSpecs({ - deps, - searchRoots: installRootPlan.searchRoots, - installRoot, - }); - const retainedManifestSpecs = persistRetainedManifest - ? readRetainedRuntimeDepsManifest(installRoot) - : []; - const readonlySearchRoots = installRootPlan.searchRoots.filter( - (rootDir) => path.resolve(rootDir) !== path.resolve(installRoot), - ); - const alreadyStagedSpecs = persistRetainedManifest - ? collectAlreadyStagedBundledRuntimeDepSpecs({ - pluginRoot: params.pluginRoot, - installRoot, - config: params.config, - }).filter( - (spec) => - !hasDependencySentinel(readonlySearchRoots, parseInstallableRuntimeDepSpec(spec)), - ) - : []; - const retainedAllowedSpecs = new Set([...alreadyStagedSpecs, ...dependencySpecs]); - const retainSpecIfActive = (spec: string) => - params.config === undefined || retainedAllowedSpecs.has(spec); - const installSpecs = [ - ...new Set([ - ...(params.retainSpecs ?? []).filter(retainSpecIfActive), - ...retainedManifestSpecs.filter(retainSpecIfActive), - ...alreadyStagedSpecs, - ...dependencySpecs, - ]), - ].toSorted((left, right) => left.localeCompare(right)); - const missingSpecs = deps - .filter((dep) => !hasDependencySentinel(installRootPlan.searchRoots, dep)) - .map((dep) => `${dep.name}@${dep.version}`) - .toSorted((left, right) => left.localeCompare(right)); - if (missingSpecs.length === 0) { - if (params.config !== undefined && persistRetainedManifest && installSpecs.length > 0) { - writeRetainedRuntimeDepsManifest(installRoot, installSpecs); - } - return { installedSpecs: [], retainSpecs: [] }; + if (isRuntimeDepsPlanMaterialized(installRoot, installSpecs)) { + removeLegacyRuntimeDepsManifest(installRoot); + return createBundledRuntimeDepsEnsureResult([]); } - const cacheDir = resolveSourceCheckoutRuntimeDepsCacheDir({ - pluginId: params.pluginId, - pluginRoot: params.pluginRoot, - installSpecs, - }); const isPluginRootInstall = path.resolve(installRoot) === path.resolve(params.pluginRoot); - const sourceCheckoutCacheStage = - cacheDir && isPluginRootInstall && resolveSourceCheckoutPackageRoot(params.pluginRoot) - ? cacheDir - : undefined; - const installExecutionRoot = - sourceCheckoutCacheStage ?? - (isPluginRootInstall ? path.join(installRoot, PLUGIN_ROOT_INSTALL_STAGE_DIR) : undefined); - if ( - restoreSourceCheckoutRuntimeDepsFromCache({ - cacheDir, - deps, - installRoot, - }) - ) { - return { installedSpecs: [], retainSpecs: [] }; - } + const installExecutionRoot = isPluginRootInstall + ? path.join(installRoot, PLUGIN_ROOT_INSTALL_STAGE_DIR) + : undefined; + removeLegacyRuntimeDepsManifest(installRoot); const install = params.installDeps ?? ((installParams) => { - const isolatedExecutionRoot = - installParams.installExecutionRoot && - path.resolve(installParams.installExecutionRoot) !== - path.resolve(installParams.installRoot); return installBundledRuntimeDeps({ installRoot: installParams.installRoot, installExecutionRoot: installParams.installExecutionRoot, - linkNodeModulesFromExecutionRoot: installParams.linkNodeModulesFromExecutionRoot, - missingSpecs: isolatedExecutionRoot - ? (installParams.installSpecs ?? installParams.missingSpecs) - : installParams.missingSpecs, + missingSpecs: installParams.installSpecs ?? installParams.missingSpecs, installSpecs: installParams.installSpecs, env: params.env, }); }); const finishActivity = beginBundledRuntimeDepsInstall({ installRoot, - missingSpecs, + missingSpecs: installSpecs, installSpecs, pluginId: params.pluginId, }); + if (!installExecutionRoot) { + ensureNpmInstallExecutionManifest(installRoot, installSpecs); + } try { install({ installRoot, - installExecutionRoot, - ...(sourceCheckoutCacheStage ? { linkNodeModulesFromExecutionRoot: true } : {}), - missingSpecs, + ...(installExecutionRoot ? { installExecutionRoot } : {}), + missingSpecs: installSpecs, installSpecs, }); } finally { finishActivity(); } - linkBundledRuntimeDepsFromSearchRoots({ - deps, - searchRoots: installRootPlan.searchRoots, - installRoot, - }); - const cacheAlreadyPopulated = Boolean( - sourceCheckoutCacheStage && hasAllDependencySentinels(sourceCheckoutCacheStage, deps), - ); - if (persistRetainedManifest) { - writeRetainedRuntimeDepsManifest(installRoot, installSpecs); - } - if (!cacheAlreadyPopulated) { - storeSourceCheckoutRuntimeDepsCache({ cacheDir, installRoot }); - } - return { installedSpecs: missingSpecs, retainSpecs: installSpecs }; + removeLegacyRuntimeDepsManifest(installRoot); + return createBundledRuntimeDepsEnsureResult(installSpecs); }); } diff --git a/src/plugins/bundled-runtime-mirror.ts b/src/plugins/bundled-runtime-mirror.ts index 0dcfb8a83be..7ec5ade9c27 100644 --- a/src/plugins/bundled-runtime-mirror.ts +++ b/src/plugins/bundled-runtime-mirror.ts @@ -71,16 +71,45 @@ export function copyBundledPluginRuntimeRoot(sourceRoot: string, targetRoot: str } removeBundledRuntimeMirrorPathIfTypeChanged(targetPath, "file"); copyBundledRuntimeMirrorFileAtomic(sourcePath, targetPath); - try { - const sourceMode = fs.statSync(sourcePath).mode; - fs.chmodSync(targetPath, sourceMode | 0o600); - } catch { - // Readable copied files are enough for plugin loading. - } + chmodBundledRuntimeMirrorFileReadable(sourcePath, targetPath); } pruneStaleBundledRuntimeMirrorEntries(targetRoot, mirroredNames); } +export function materializeBundledRuntimeMirrorFile(sourcePath: string, targetPath: string): void { + if (path.resolve(sourcePath) === path.resolve(targetPath)) { + return; + } + try { + if ( + fs.realpathSync(sourcePath) === fs.realpathSync(targetPath) && + !fs.lstatSync(targetPath).isSymbolicLink() + ) { + return; + } + } catch { + // Missing targets are expected before the mirror file is materialized. + } + fs.mkdirSync(path.dirname(targetPath), { recursive: true, mode: 0o755 }); + fs.rmSync(targetPath, { recursive: true, force: true }); + try { + fs.linkSync(sourcePath, targetPath); + return; + } catch { + fs.copyFileSync(sourcePath, targetPath); + } + chmodBundledRuntimeMirrorFileReadable(sourcePath, targetPath); +} + +function chmodBundledRuntimeMirrorFileReadable(sourcePath: string, targetPath: string): void { + try { + const sourceMode = fs.statSync(sourcePath).mode; + fs.chmodSync(targetPath, sourceMode | 0o600); + } catch { + // Readable mirrored files are enough for plugin loading. + } +} + function pruneStaleBundledRuntimeMirrorEntries( targetRoot: string, mirroredNames: Set, diff --git a/src/plugins/bundled-runtime-root.test.ts b/src/plugins/bundled-runtime-root.test.ts index a07b547a374..9e3edc0fae5 100644 --- a/src/plugins/bundled-runtime-root.test.ts +++ b/src/plugins/bundled-runtime-root.test.ts @@ -35,6 +35,21 @@ function isBigIntStatOptions(options: unknown): boolean { ); } +function writeGeneratedRuntimeDepsManifest(rootDir: string, specs: readonly string[]): void { + const dependencies = Object.fromEntries( + specs.map((spec) => { + const atIndex = spec.lastIndexOf("@"); + return [spec.slice(0, atIndex), spec.slice(atIndex + 1)]; + }), + ); + fs.mkdirSync(rootDir, { recursive: true }); + fs.writeFileSync( + path.join(rootDir, "package.json"), + JSON.stringify({ name: "openclaw-runtime-deps-install", private: true, dependencies }), + "utf8", + ); +} + describe("prepareBundledPluginRuntimeRoot", () => { it("materializes root JavaScript chunks in external mirrors", () => { const packageRoot = makeTempRoot(); @@ -110,6 +125,7 @@ describe("prepareBundledPluginRuntimeRoot", () => { "utf8", ); fs.writeFileSync(path.join(depRoot, "index.js"), "export const marker = 'stage-ok';\n", "utf8"); + writeGeneratedRuntimeDepsManifest(installRoot, ["playwright-core@1.0.0"]); const staleMirrorChunk = path.join(installRoot, "dist", "pw-ai.js"); fs.mkdirSync(path.dirname(staleMirrorChunk), { recursive: true }); @@ -141,14 +157,14 @@ describe("prepareBundledPluginRuntimeRoot", () => { false, ); expect(fs.lstatSync(path.join(installRoot, "dist", "config-runtime.js")).isSymbolicLink()).toBe( - true, + false, ); expect(fs.lstatSync(path.join(installRoot, "dist", "string-runtime.js")).isSymbolicLink()).toBe( false, ); }); - it("reuses root chunk materialization decisions across bundled plugin mirrors", () => { + it("reuses prepared root mirrors across bundled plugins", () => { const packageRoot = makeTempRoot(); const stageDir = makeTempRoot(); const env = { ...process.env, OPENCLAW_PLUGIN_STAGE_DIR: stageDir }; @@ -162,6 +178,10 @@ describe("prepareBundledPluginRuntimeRoot", () => { ); fs.writeFileSync(rootChunk, "export const shared = 'root';\n", "utf8"); fs.writeFileSync(externalChunk, "import zod from 'zod'; export const schema = zod;\n", "utf8"); + const installRoot = resolveBundledRuntimeDependencyInstallRoot( + path.join(packageRoot, "dist", "extensions", "alpha"), + { env }, + ); for (const pluginId of ["alpha", "beta"]) { const pluginRoot = path.join(packageRoot, "dist", "extensions", pluginId); @@ -186,28 +206,20 @@ describe("prepareBundledPluginRuntimeRoot", () => { ), "utf8", ); - const installRoot = resolveBundledRuntimeDependencyInstallRoot(pluginRoot, { env }); - fs.mkdirSync(path.join(installRoot, "node_modules", `${pluginId}-runtime`), { + const pluginInstallRoot = resolveBundledRuntimeDependencyInstallRoot(pluginRoot, { env }); + fs.mkdirSync(path.join(pluginInstallRoot, "node_modules", `${pluginId}-runtime`), { recursive: true, }); fs.writeFileSync( - path.join(installRoot, "node_modules", `${pluginId}-runtime`, "package.json"), + path.join(pluginInstallRoot, "node_modules", `${pluginId}-runtime`, "package.json"), JSON.stringify({ name: `${pluginId}-runtime`, version: "1.0.0", type: "module" }), "utf8", ); } + writeGeneratedRuntimeDepsManifest(installRoot, ["alpha-runtime@1.0.0", "beta-runtime@1.0.0"]); - const realReadFileSync = fs.readFileSync.bind(fs); const realReaddirSync = fs.readdirSync.bind(fs); - const readPaths: string[] = []; const readdirPaths: string[] = []; - vi.spyOn(fs, "readFileSync").mockImplementation(((target, options) => { - const targetPath = target.toString(); - if (targetPath === rootChunk || targetPath === externalChunk) { - readPaths.push(targetPath); - } - return realReadFileSync(target, options as never); - }) as typeof fs.readFileSync); vi.spyOn(fs, "readdirSync").mockImplementation(((target, options) => { const targetPath = target.toString(); if ( @@ -229,8 +241,12 @@ describe("prepareBundledPluginRuntimeRoot", () => { }); } - expect(readPaths.filter((entry) => entry === rootChunk)).toHaveLength(1); - expect(readPaths.filter((entry) => entry === externalChunk)).toHaveLength(1); + expect(fs.lstatSync(path.join(installRoot, "dist", "shared-runtime.js")).isSymbolicLink()).toBe( + false, + ); + expect( + fs.lstatSync(path.join(installRoot, "dist", "external-runtime.js")).isSymbolicLink(), + ).toBe(false); expect(readdirPaths).toHaveLength(1); }); @@ -276,6 +292,7 @@ describe("prepareBundledPluginRuntimeRoot", () => { JSON.stringify({ name: "alpha-runtime", version: "1.0.0", type: "module" }), "utf8", ); + writeGeneratedRuntimeDepsManifest(installRoot, ["alpha-runtime@1.0.0"]); const realReaddirSync = fs.readdirSync.bind(fs); const readdirPaths: string[] = []; @@ -341,6 +358,7 @@ describe("prepareBundledPluginRuntimeRoot", () => { JSON.stringify({ name: "qqbot-runtime", version: "1.0.0", type: "module" }), "utf8", ); + writeGeneratedRuntimeDepsManifest(installRoot, ["qqbot-runtime@1.0.0"]); const prepared = prepareBundledPluginRuntimeRoot({ pluginId: "qqbot", @@ -424,6 +442,7 @@ describe("prepareBundledPluginRuntimeRoot", () => { JSON.stringify({ name: "qqbot-runtime", version: "1.0.0", type: "module" }), "utf8", ); + writeGeneratedRuntimeDepsManifest(installRoot, ["qqbot-runtime@1.0.0"]); const prepared = prepareBundledPluginRuntimeRoot({ pluginId: "qqbot", @@ -489,6 +508,7 @@ describe("prepareBundledPluginRuntimeRoot", () => { JSON.stringify({ name: "qqbot-runtime", version: "1.0.0", type: "module" }), "utf8", ); + writeGeneratedRuntimeDepsManifest(installRoot, ["qqbot-runtime@1.0.0"]); const lockPath = path.join(installRoot, ".openclaw-runtime-mirror.lock"); const fingerprintLockStates: Array<{ source: "runtime" | "canonical"; locked: boolean }> = []; @@ -551,6 +571,7 @@ describe("prepareBundledPluginRuntimeRoot", () => { JSON.stringify({ name: "whatsapp-runtime", version: "1.0.0", type: "module" }), "utf8", ); + writeGeneratedRuntimeDepsManifest(installRoot, ["whatsapp-runtime@1.0.0"]); const prepared = prepareBundledPluginRuntimeRoot({ pluginId: "whatsapp", @@ -610,6 +631,7 @@ describe("prepareBundledPluginRuntimeRoot", () => { JSON.stringify({ name: "whatsapp-runtime", version: "1.0.0", type: "module" }), "utf8", ); + writeGeneratedRuntimeDepsManifest(installRoot, ["whatsapp-runtime@1.0.0"]); const prepared = prepareBundledPluginRuntimeRoot({ pluginId: "whatsapp", diff --git a/src/plugins/bundled-runtime-root.ts b/src/plugins/bundled-runtime-root.ts index 5e90d010f17..50ccbb7af93 100644 --- a/src/plugins/bundled-runtime-root.ts +++ b/src/plugins/bundled-runtime-root.ts @@ -1,28 +1,33 @@ import fs from "node:fs"; import path from "node:path"; +import type { OpenClawConfig } from "../config/types.openclaw.js"; import { ensureBundledPluginRuntimeDeps, - materializeBundledRuntimeMirrorDistFile, resolveBundledRuntimeDependencyInstallRootPlan, resolveBundledRuntimeDependencyPackageRoot, registerBundledRuntimeDependencyNodePath, - shouldMaterializeBundledRuntimeMirrorDistFile, withBundledRuntimeDepsFilesystemLock, + type BundledRuntimeDepsInstallParams, } from "./bundled-runtime-deps.js"; import { markBundledRuntimeDistMirrorPrepared, shouldReusePreparedBundledRuntimeDistMirror, } from "./bundled-runtime-dist-mirror-cache.js"; import { - copyBundledPluginRuntimeRoot, + materializeBundledRuntimeMirrorFile, precomputeBundledRuntimeMirrorMetadata, refreshBundledPluginRuntimeMirrorRoot, type PrecomputedBundledRuntimeMirrorMetadata, } from "./bundled-runtime-mirror.js"; -const bundledRuntimeDepsRetainSpecsByInstallRoot = new Map(); const BUNDLED_RUNTIME_MIRROR_LOCK_DIR = ".openclaw-runtime-mirror.lock"; +export type PreparedBundledPluginRuntimeLoadRoot = { + pluginRoot: string; + modulePath: string; + setupModulePath?: string; +}; + export function isBuiltBundledPluginRuntimeRoot(pluginRoot: string): boolean { const extensionsDir = path.dirname(pluginRoot); const buildDir = path.dirname(extensionsDir); @@ -39,36 +44,51 @@ export function prepareBundledPluginRuntimeRoot(params: { env?: NodeJS.ProcessEnv; logInstalled?: (installedSpecs: readonly string[]) => void; }): { pluginRoot: string; modulePath: string } { + return prepareBundledPluginRuntimeLoadRoot(params); +} + +export function prepareBundledPluginRuntimeLoadRoot(params: { + pluginId: string; + pluginRoot: string; + modulePath: string; + setupModulePath?: string; + env?: NodeJS.ProcessEnv; + config?: OpenClawConfig; + installDeps?: (params: BundledRuntimeDepsInstallParams) => void; + registerRuntimeAliasRoot?: (rootDir: string) => void; + logInstalled?: (installedSpecs: readonly string[]) => void; +}): PreparedBundledPluginRuntimeLoadRoot { const env = params.env ?? process.env; const installRootPlan = resolveBundledRuntimeDependencyInstallRootPlan(params.pluginRoot, { env, }); const installRoot = installRootPlan.installRoot; - const retainSpecs = bundledRuntimeDepsRetainSpecsByInstallRoot.get(installRoot) ?? []; const depsInstallResult = ensureBundledPluginRuntimeDeps({ pluginId: params.pluginId, pluginRoot: params.pluginRoot, env, - retainSpecs, + config: params.config, + installDeps: params.installDeps, }); if (depsInstallResult.installedSpecs.length > 0) { - bundledRuntimeDepsRetainSpecsByInstallRoot.set( - installRoot, - [...new Set([...retainSpecs, ...depsInstallResult.retainSpecs])].toSorted((left, right) => - left.localeCompare(right), - ), - ); params.logInstalled?.(depsInstallResult.installedSpecs); } if (path.resolve(installRoot) === path.resolve(params.pluginRoot)) { - return { pluginRoot: params.pluginRoot, modulePath: params.modulePath }; + ensureOpenClawPluginSdkAlias(path.dirname(path.dirname(params.pluginRoot))); + return { + pluginRoot: params.pluginRoot, + modulePath: params.modulePath, + ...(params.setupModulePath ? { setupModulePath: params.setupModulePath } : {}), + }; } const packageRoot = resolveBundledRuntimeDependencyPackageRoot(params.pluginRoot); if (packageRoot) { registerBundledRuntimeDependencyNodePath(packageRoot); + params.registerRuntimeAliasRoot?.(packageRoot); } for (const searchRoot of installRootPlan.searchRoots) { registerBundledRuntimeDependencyNodePath(searchRoot); + params.registerRuntimeAliasRoot?.(searchRoot); } const mirrorRoot = mirrorBundledPluginRuntimeRoot({ pluginId: params.pluginId, @@ -82,6 +102,15 @@ export function prepareBundledPluginRuntimeRoot(params: { pluginRoot: params.pluginRoot, mirroredRoot: mirrorRoot, }), + ...(params.setupModulePath + ? { + setupModulePath: remapBundledPluginRuntimePath({ + source: params.setupModulePath, + pluginRoot: params.pluginRoot, + mirroredRoot: mirrorRoot, + }), + } + : {}), }; } @@ -200,10 +229,18 @@ function ensureBundledRuntimeMirrorDirectory(targetRoot: string): void { fs.mkdirSync(targetRoot, { recursive: true, mode: 0o755 }); } +function isPathInsideDirectory(childPath: string, parentPath: string): boolean { + const relative = path.relative(path.resolve(parentPath), path.resolve(childPath)); + return relative !== "" && !relative.startsWith("..") && !path.isAbsolute(relative); +} + function mirrorBundledRuntimeDistRootEntries(params: { sourceDistRoot: string; mirrorDistRoot: string; }): void { + const mirrorRootDirectories = + path.basename(params.sourceDistRoot) === "dist" || + path.basename(params.sourceDistRoot) === "dist-runtime"; for (const entry of fs.readdirSync(params.sourceDistRoot, { withFileTypes: true })) { if (entry.name === "extensions") { continue; @@ -213,24 +250,25 @@ function mirrorBundledRuntimeDistRootEntries(params: { if (path.resolve(sourcePath) === path.resolve(targetPath)) { continue; } - if (entry.isFile() && shouldMaterializeBundledRuntimeMirrorDistFile(sourcePath)) { - materializeBundledRuntimeMirrorDistFile(sourcePath, targetPath); + if (entry.isDirectory() && isPathInsideDirectory(targetPath, sourcePath)) { continue; } - if (fs.existsSync(targetPath)) { - continue; - } - try { - fs.symlinkSync(sourcePath, targetPath, entry.isDirectory() ? "junction" : "file"); - } catch { - if (fs.existsSync(targetPath)) { + const sourceStat = fs.statSync(sourcePath); + if (sourceStat.isDirectory()) { + if (!mirrorRootDirectories) { continue; } - if (entry.isDirectory()) { - copyBundledPluginRuntimeRoot(sourcePath, targetPath); - } else if (entry.isFile()) { - fs.copyFileSync(sourcePath, targetPath); - } + refreshBundledPluginRuntimeMirrorRoot({ + pluginId: `openclaw-dist:${entry.name}`, + sourceRoot: sourcePath, + targetRoot: targetPath, + tempDirParent: params.mirrorDistRoot, + }); + continue; + } + if (sourceStat.isFile()) { + materializeBundledRuntimeMirrorFile(sourcePath, targetPath); + continue; } } } @@ -354,7 +392,7 @@ function writeRuntimeModuleWrapper(sourcePath: string, targetPath: string): void fs.writeFileSync(targetPath, content, "utf8"); } -function ensureOpenClawPluginSdkAlias(distRoot: string): void { +export function ensureOpenClawPluginSdkAlias(distRoot: string): void { const pluginSdkDir = path.join(distRoot, "plugin-sdk"); if (!fs.existsSync(pluginSdkDir)) { return; diff --git a/src/plugins/loader.test.ts b/src/plugins/loader.test.ts index 0b3900178b7..b1c1150e820 100644 --- a/src/plugins/loader.test.ts +++ b/src/plugins/loader.test.ts @@ -22,7 +22,11 @@ import { type DetachedTaskLifecycleRuntime, } from "../tasks/detached-task-runtime-state.js"; import { withEnv } from "../test-utils/env.js"; -import { resolveBundledRuntimeDependencyInstallRootPlan } from "./bundled-runtime-deps.js"; +import { + resolveBundledRuntimeDependencyInstallRootPlan, + type BundledRuntimeDepsInstallParams, +} from "./bundled-runtime-deps.js"; +import { ensureOpenClawPluginSdkAlias } from "./bundled-runtime-root.js"; import { clearPluginCommands } from "./command-registry-state.js"; import { getPluginCommandSpecs } from "./command-specs.js"; import { listCompactionProviderIds } from "./compaction-provider.js"; @@ -947,9 +951,9 @@ describe("loadOpenClawPlugins", () => { fs.writeFileSync(path.join(pluginSdkDir, "core.js"), "export const core = 1;\n", "utf8"); fs.writeFileSync(path.join(aliasDir, "sentinel.txt"), "keep\n", "utf8"); - __testing.ensureOpenClawPluginSdkAlias(distRoot); + ensureOpenClawPluginSdkAlias(distRoot); fs.writeFileSync(path.join(pluginSdkDir, "core.js"), "export const core = 2;\n", "utf8"); - __testing.ensureOpenClawPluginSdkAlias(distRoot); + ensureOpenClawPluginSdkAlias(distRoot); expect(fs.existsSync(path.join(aliasDir, "sentinel.txt"))).toBe(true); expect(fs.readFileSync(path.join(aliasDir, "core.js"), "utf8")).toContain("core.js"); @@ -1047,7 +1051,7 @@ module.exports = { }, bundledRuntimeDepsInstaller: ({ installRoot, missingSpecs }) => { expect(logger.info).toHaveBeenCalledWith( - "[plugins] discord staging bundled runtime deps (1 missing, 1 install specs): discord-runtime@1.0.0", + "[plugins] discord staging bundled runtime deps (1 specs): discord-runtime@1.0.0", ); installedSpecs.push(...missingSpecs); expect(fs.realpathSync(installRoot)).toBe(fs.realpathSync(plugin.dir)); @@ -1142,7 +1146,7 @@ module.exports = { "[plugins] discord installed bundled runtime deps: discord-runtime@1.0.0", ); expect(logger.info).not.toHaveBeenCalledWith( - "[plugins] discord staging bundled runtime deps (1 missing, 1 install specs): discord-runtime@1.0.0", + "[plugins] discord staging bundled runtime deps (1 specs): discord-runtime@1.0.0", ); }); @@ -1926,7 +1930,7 @@ module.exports = { ).toBe(false); expect( fs.lstatSync(path.join(actualInstallRoot, "dist", "config-runtime.js")).isSymbolicLink(), - ).toBe(true); + ).toBe(false); }); it("loads bundled plugins with plugin-sdk imports from an external stage dir", () => { @@ -2224,6 +2228,26 @@ module.exports = { try { let actualInstallRoot = ""; + const installExternalRuntime = ({ installRoot }: BundledRuntimeDepsInstallParams) => { + actualInstallRoot = installRoot; + const depRoot = path.join(installRoot, "node_modules", "external-runtime"); + fs.mkdirSync(depRoot, { recursive: true }); + fs.writeFileSync( + path.join(depRoot, "package.json"), + JSON.stringify({ + name: "external-runtime", + version: "1.0.0", + type: "module", + exports: "./index.js", + }), + "utf-8", + ); + fs.writeFileSync( + path.join(depRoot, "index.js"), + "export default { marker: 'dist-runtime-ok' };\n", + "utf-8", + ); + }; const registry = loadOpenClawPlugins({ cache: false, config: { @@ -2231,26 +2255,7 @@ module.exports = { enabled: true, }, }, - bundledRuntimeDepsInstaller: ({ installRoot }) => { - actualInstallRoot = installRoot; - const depRoot = path.join(installRoot, "node_modules", "external-runtime"); - fs.mkdirSync(depRoot, { recursive: true }); - fs.writeFileSync( - path.join(depRoot, "package.json"), - JSON.stringify({ - name: "external-runtime", - version: "1.0.0", - type: "module", - exports: "./index.js", - }), - "utf-8", - ); - fs.writeFileSync( - path.join(depRoot, "index.js"), - "export default { marker: 'dist-runtime-ok' };\n", - "utf-8", - ); - }, + bundledRuntimeDepsInstaller: installExternalRuntime, }); const record = registry.plugins.find((entry) => entry.id === "acpx"); @@ -2277,6 +2282,7 @@ module.exports = { enabled: true, }, }, + bundledRuntimeDepsInstaller: installExternalRuntime, }); const reloadedRecord = reloadedRegistry.plugins.find((entry) => entry.id === "acpx"); @@ -2294,7 +2300,7 @@ module.exports = { } }); - it("loads native ESM deps from a layered baseline stage dir", () => { + it("loads native ESM deps from the writable stage dir without reusing a layered baseline", () => { const packageRoot = makeTempDir(); const baselineStageDir = makeTempDir(); const writableStageDir = makeTempDir(); @@ -2417,17 +2423,39 @@ module.exports = { enabled: true, }, }, - bundledRuntimeDepsInstaller: () => { - throw new Error("baseline deps should not reinstall"); + bundledRuntimeDepsInstaller: ({ installRoot }) => { + const depRoot = path.join(installRoot, "node_modules", "external-runtime"); + fs.mkdirSync(depRoot, { recursive: true }); + fs.writeFileSync( + path.join(depRoot, "package.json"), + JSON.stringify({ + name: "external-runtime", + version: "1.0.0", + type: "module", + exports: "./index.js", + }), + "utf-8", + ); + fs.writeFileSync( + path.join(depRoot, "index.js"), + "export default { marker: 'writable-ok' };\n", + "utf-8", + ); }, }); const layeredRecord = registry.plugins.find((entry) => entry.id === "acpx"); expect(layeredRecord?.error).toBeUndefined(); expect(layeredRecord?.status).toBe("loaded"); + expect(fs.readFileSync(path.join(baselineDepRoot, "index.js"), "utf-8")).toContain( + "baseline-ok", + ); expect( - fs.realpathSync(path.join(installRootPlan.installRoot, "node_modules", "external-runtime")), - ).toBe(fs.realpathSync(baselineDepRoot)); + fs.readFileSync( + path.join(installRootPlan.installRoot, "node_modules", "external-runtime", "index.js"), + "utf-8", + ), + ).toContain("writable-ok"); }); it("loads source-checkout bundled runtime deps without mirroring the repo tree", () => { diff --git a/src/plugins/loader.ts b/src/plugins/loader.ts index 8d0696f5ac0..0434afd1a52 100644 --- a/src/plugins/loader.ts +++ b/src/plugins/loader.ts @@ -33,27 +33,14 @@ import { buildPluginApi } from "./api-builder.js"; import { inspectBundleMcpRuntimeSupport } from "./bundle-mcp.js"; import { clearBundledRuntimeDependencyNodePaths, - ensureBundledPluginRuntimeDeps, installBundledRuntimeDeps, - materializeBundledRuntimeMirrorDistFile, - resolveBundledRuntimeDependencyInstallRootPlan, - resolveBundledRuntimeDependencyPackageRoot, - registerBundledRuntimeDependencyNodePath, - shouldMaterializeBundledRuntimeMirrorDistFile, - withBundledRuntimeDepsFilesystemLock, type BundledRuntimeDepsInstallParams, } from "./bundled-runtime-deps.js"; +import { clearBundledRuntimeDistMirrorPreparationCache } from "./bundled-runtime-dist-mirror-cache.js"; import { - clearBundledRuntimeDistMirrorPreparationCache, - markBundledRuntimeDistMirrorPrepared, - shouldReusePreparedBundledRuntimeDistMirror, -} from "./bundled-runtime-dist-mirror-cache.js"; -import { - copyBundledPluginRuntimeRoot, - precomputeBundledRuntimeMirrorMetadata, - refreshBundledPluginRuntimeMirrorRoot, - type PrecomputedBundledRuntimeMirrorMetadata, -} from "./bundled-runtime-mirror.js"; + ensureOpenClawPluginSdkAlias, + prepareBundledPluginRuntimeLoadRoot, +} from "./bundled-runtime-root.js"; import { clearPluginCommands, listRegisteredPluginCommands, @@ -297,7 +284,6 @@ export function clearPluginLoaderCache(): void { } const defaultLogger = () => createSubsystemLogger("plugins"); -const BUNDLED_RUNTIME_MIRROR_LOCK_DIR = ".openclaw-runtime-mirror.lock"; function isPromiseLike(value: unknown): value is PromiseLike { return ( @@ -707,314 +693,6 @@ function resolveCanonicalDistRuntimeSource(source: string): string { return fs.existsSync(candidate) ? candidate : source; } -function mirrorBundledPluginRuntimeRoot(params: { - pluginId: string; - pluginRoot: string; - installRoot: string; -}): string { - const sourceDistRoot = path.dirname(path.dirname(params.pluginRoot)); - const mirrorParent = path.join(params.installRoot, path.basename(sourceDistRoot), "extensions"); - const mirrorRoot = path.join(mirrorParent, params.pluginId); - const precomputedPluginRootMetadata = - path.resolve(mirrorRoot) === path.resolve(params.pluginRoot) - ? undefined - : precomputeBundledRuntimeMirrorMetadata({ sourceRoot: params.pluginRoot }); - const precomputedCanonicalPluginRootMetadata = - precomputeCanonicalBundledRuntimeDistPluginMetadata({ - pluginRoot: params.pluginRoot, - sourceDistRoot, - }); - - return withBundledRuntimeDepsFilesystemLock( - params.installRoot, - BUNDLED_RUNTIME_MIRROR_LOCK_DIR, - () => { - const preparedMirrorParent = prepareBundledPluginRuntimeDistMirror({ - installRoot: params.installRoot, - pluginRoot: params.pluginRoot, - precomputedCanonicalPluginRootMetadata, - }); - const preparedMirrorRoot = path.join(preparedMirrorParent, params.pluginId); - fs.mkdirSync(params.installRoot, { recursive: true }); - try { - fs.chmodSync(params.installRoot, 0o755); - } catch { - // Best-effort only: staged roots may live on filesystems that reject chmod. - } - fs.mkdirSync(preparedMirrorParent, { recursive: true }); - try { - fs.chmodSync(preparedMirrorParent, 0o755); - } catch { - // Best-effort only: the access check below will surface non-writable dirs. - } - fs.accessSync(preparedMirrorParent, fs.constants.W_OK); - if (path.resolve(preparedMirrorRoot) === path.resolve(params.pluginRoot)) { - return preparedMirrorRoot; - } - refreshBundledPluginRuntimeMirrorRoot({ - pluginId: params.pluginId, - sourceRoot: params.pluginRoot, - targetRoot: preparedMirrorRoot, - tempDirParent: preparedMirrorParent, - precomputedSourceMetadata: precomputedPluginRootMetadata, - }); - return preparedMirrorRoot; - }, - ); -} - -function prepareBundledPluginRuntimeDistMirror(params: { - installRoot: string; - pluginRoot: string; - precomputedCanonicalPluginRootMetadata?: PrecomputedBundledRuntimeMirrorMetadata; -}): string { - const sourceExtensionsRoot = path.dirname(params.pluginRoot); - const sourceDistRoot = path.dirname(sourceExtensionsRoot); - const sourceDistRootName = path.basename(sourceDistRoot); - const mirrorDistRoot = path.join(params.installRoot, sourceDistRootName); - const mirrorExtensionsRoot = path.join(mirrorDistRoot, "extensions"); - ensureBundledRuntimeMirrorDirectory(mirrorDistRoot); - fs.mkdirSync(mirrorExtensionsRoot, { recursive: true, mode: 0o755 }); - ensureBundledRuntimeDistPackageJson(mirrorDistRoot); - if (!shouldReusePreparedBundledRuntimeDistMirror({ sourceDistRoot, mirrorDistRoot })) { - mirrorBundledRuntimeDistRootEntries({ - sourceDistRoot, - mirrorDistRoot, - }); - markBundledRuntimeDistMirrorPrepared({ sourceDistRoot, mirrorDistRoot }); - } - if (sourceDistRootName === "dist-runtime") { - mirrorCanonicalBundledRuntimeDistRoot({ - installRoot: params.installRoot, - pluginRoot: params.pluginRoot, - sourceRuntimeDistRoot: sourceDistRoot, - precomputedSourceMetadata: params.precomputedCanonicalPluginRootMetadata, - }); - } - ensureOpenClawPluginSdkAlias(mirrorDistRoot); - return mirrorExtensionsRoot; -} - -function ensureBundledRuntimeMirrorDirectory(targetRoot: string): void { - try { - const stat = fs.lstatSync(targetRoot); - if (stat.isDirectory() && !stat.isSymbolicLink()) { - return; - } - fs.rmSync(targetRoot, { recursive: true, force: true }); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "ENOENT") { - throw error; - } - } - fs.mkdirSync(targetRoot, { recursive: true, mode: 0o755 }); -} - -function mirrorBundledRuntimeDistRootEntries(params: { - sourceDistRoot: string; - mirrorDistRoot: string; -}): void { - for (const entry of fs.readdirSync(params.sourceDistRoot, { withFileTypes: true })) { - if (entry.name === "extensions") { - continue; - } - const sourcePath = path.join(params.sourceDistRoot, entry.name); - const targetPath = path.join(params.mirrorDistRoot, entry.name); - if (path.resolve(sourcePath) === path.resolve(targetPath)) { - continue; - } - if (entry.isFile() && shouldMaterializeBundledRuntimeMirrorDistFile(sourcePath)) { - materializeBundledRuntimeMirrorDistFile(sourcePath, targetPath); - continue; - } - if (fs.existsSync(targetPath)) { - continue; - } - try { - fs.symlinkSync(sourcePath, targetPath, entry.isDirectory() ? "junction" : "file"); - } catch { - if (fs.existsSync(targetPath)) { - continue; - } - if (entry.isDirectory()) { - copyBundledPluginRuntimeRoot(sourcePath, targetPath); - } else if (entry.isFile()) { - fs.copyFileSync(sourcePath, targetPath); - } - } - } -} - -function mirrorCanonicalBundledRuntimeDistRoot(params: { - installRoot: string; - pluginRoot: string; - sourceRuntimeDistRoot: string; - precomputedSourceMetadata?: PrecomputedBundledRuntimeMirrorMetadata; -}): void { - const sourceCanonicalDistRoot = path.join(path.dirname(params.sourceRuntimeDistRoot), "dist"); - if (!fs.existsSync(sourceCanonicalDistRoot)) { - return; - } - const targetCanonicalDistRoot = path.join(params.installRoot, "dist"); - ensureBundledRuntimeMirrorDirectory(targetCanonicalDistRoot); - fs.mkdirSync(path.join(targetCanonicalDistRoot, "extensions"), { recursive: true, mode: 0o755 }); - ensureBundledRuntimeDistPackageJson(targetCanonicalDistRoot); - if ( - !shouldReusePreparedBundledRuntimeDistMirror({ - sourceDistRoot: sourceCanonicalDistRoot, - mirrorDistRoot: targetCanonicalDistRoot, - }) - ) { - mirrorBundledRuntimeDistRootEntries({ - sourceDistRoot: sourceCanonicalDistRoot, - mirrorDistRoot: targetCanonicalDistRoot, - }); - markBundledRuntimeDistMirrorPrepared({ - sourceDistRoot: sourceCanonicalDistRoot, - mirrorDistRoot: targetCanonicalDistRoot, - }); - } - ensureOpenClawPluginSdkAlias(targetCanonicalDistRoot); - - const pluginId = path.basename(params.pluginRoot); - const sourceCanonicalPluginRoot = path.join(sourceCanonicalDistRoot, "extensions", pluginId); - if (!fs.existsSync(sourceCanonicalPluginRoot)) { - return; - } - const targetCanonicalPluginRoot = path.join(targetCanonicalDistRoot, "extensions", pluginId); - refreshBundledPluginRuntimeMirrorRoot({ - pluginId, - sourceRoot: sourceCanonicalPluginRoot, - targetRoot: targetCanonicalPluginRoot, - tempDirParent: path.dirname(targetCanonicalPluginRoot), - precomputedSourceMetadata: params.precomputedSourceMetadata, - }); -} - -function precomputeCanonicalBundledRuntimeDistPluginMetadata(params: { - pluginRoot: string; - sourceDistRoot: string; -}): PrecomputedBundledRuntimeMirrorMetadata | undefined { - if (path.basename(params.sourceDistRoot) !== "dist-runtime") { - return undefined; - } - const pluginId = path.basename(params.pluginRoot); - const sourceCanonicalPluginRoot = path.join( - path.dirname(params.sourceDistRoot), - "dist", - "extensions", - pluginId, - ); - if (!fs.existsSync(sourceCanonicalPluginRoot)) { - return undefined; - } - return precomputeBundledRuntimeMirrorMetadata({ sourceRoot: sourceCanonicalPluginRoot }); -} - -function ensureBundledRuntimeDistPackageJson(mirrorDistRoot: string): void { - const packageJsonPath = path.join(mirrorDistRoot, "package.json"); - if (fs.existsSync(packageJsonPath)) { - return; - } - writeRuntimeJsonFile(packageJsonPath, { type: "module" }); -} - -function writeRuntimeJsonFile(targetPath: string, value: unknown): void { - fs.mkdirSync(path.dirname(targetPath), { recursive: true }); - fs.writeFileSync(targetPath, `${JSON.stringify(value, null, 2)}\n`, "utf8"); -} - -function hasRuntimeDefaultExport(sourcePath: string): boolean { - const text = fs.readFileSync(sourcePath, "utf8"); - return /\bexport\s+default\b/u.test(text) || /\bas\s+default\b/u.test(text); -} - -function writeRuntimeModuleWrapper(sourcePath: string, targetPath: string): void { - const specifier = path.relative(path.dirname(targetPath), sourcePath).replaceAll(path.sep, "/"); - const normalizedSpecifier = specifier.startsWith(".") ? specifier : `./${specifier}`; - const defaultForwarder = hasRuntimeDefaultExport(sourcePath) - ? [ - `import defaultModule from ${JSON.stringify(normalizedSpecifier)};`, - `let defaultExport = defaultModule;`, - `for (let index = 0; index < 4 && defaultExport && typeof defaultExport === "object" && "default" in defaultExport; index += 1) {`, - ` defaultExport = defaultExport.default;`, - `}`, - ] - : [ - `import * as module from ${JSON.stringify(normalizedSpecifier)};`, - `let defaultExport = "default" in module ? module.default : module;`, - `for (let index = 0; index < 4 && defaultExport && typeof defaultExport === "object" && "default" in defaultExport; index += 1) {`, - ` defaultExport = defaultExport.default;`, - `}`, - ]; - const content = [ - `export * from ${JSON.stringify(normalizedSpecifier)};`, - ...defaultForwarder, - "export { defaultExport as default };", - "", - ].join("\n"); - try { - if (fs.readFileSync(targetPath, "utf8") === content) { - return; - } - } catch { - // Missing or unreadable wrapper; rewrite below. - } - fs.mkdirSync(path.dirname(targetPath), { recursive: true }); - fs.writeFileSync(targetPath, content, "utf8"); -} - -function ensureOpenClawPluginSdkAlias(distRoot: string): void { - const pluginSdkDir = path.join(distRoot, "plugin-sdk"); - if (!fs.existsSync(pluginSdkDir)) { - return; - } - - const aliasDir = path.join(distRoot, "extensions", "node_modules", "openclaw"); - const pluginSdkAliasDir = path.join(aliasDir, "plugin-sdk"); - writeRuntimeJsonFile(path.join(aliasDir, "package.json"), { - name: "openclaw", - type: "module", - exports: { - "./plugin-sdk": "./plugin-sdk/index.js", - "./plugin-sdk/*": "./plugin-sdk/*.js", - }, - }); - try { - if (fs.existsSync(pluginSdkAliasDir) && !fs.lstatSync(pluginSdkAliasDir).isDirectory()) { - fs.rmSync(pluginSdkAliasDir, { recursive: true, force: true }); - } - } catch { - // Another process may be creating the alias at the same time; mkdir/write - // below will either converge or surface the real filesystem error. - } - fs.mkdirSync(pluginSdkAliasDir, { recursive: true }); - for (const entry of fs.readdirSync(pluginSdkDir, { withFileTypes: true })) { - if (!entry.isFile() || path.extname(entry.name) !== ".js") { - continue; - } - writeRuntimeModuleWrapper( - path.join(pluginSdkDir, entry.name), - path.join(pluginSdkAliasDir, entry.name), - ); - } -} - -function remapBundledPluginRuntimePath(params: { - source: string | undefined; - pluginRoot: string; - mirroredRoot: string; -}): string | undefined { - if (!params.source) { - return undefined; - } - const relative = path.relative(params.pluginRoot, params.source); - if (relative.startsWith("..") || path.isAbsolute(relative)) { - return params.source; - } - return path.join(params.mirroredRoot, relative); -} - export const __testing = { buildPluginLoaderJitiOptions, buildPluginLoaderAliasMap, @@ -2468,7 +2146,6 @@ export function loadOpenClawPlugins(options: PluginLoadOptions = {}): PluginRegi }); const seenIds = new Map(); - const bundledRuntimeDepsRetainSpecsByInstallRoot = new Map(); const memorySlot = normalized.slots.memory; let selectedMemoryPluginId: string | null = null; let memorySlotMatched = false; @@ -2627,24 +2304,21 @@ export function loadOpenClawPlugins(options: PluginLoadOptions = {}): PluginRegi let runtimeDepsInstallStartedAt: number | null = null; let runtimeDepsInstallSpecs: string[] = []; try { - const installRootPlan = resolveBundledRuntimeDependencyInstallRootPlan(pluginRoot, { - env, - }); - const installRoot = installRootPlan.installRoot; - const retainSpecs = bundledRuntimeDepsRetainSpecsByInstallRoot.get(installRoot) ?? []; - const depsInstallResult = ensureBundledPluginRuntimeDeps({ + const preparedRuntimeRoot = prepareBundledPluginRuntimeLoadRoot({ pluginId: record.id, pluginRoot, + modulePath: runtimeCandidateSource, + ...(runtimeSetupSource ? { setupModulePath: runtimeSetupSource } : {}), env, config: cfg, - retainSpecs, + registerRuntimeAliasRoot: registerBundledRuntimeDependencyJitiAliases, installDeps: (installParams) => { const installSpecs = installParams.installSpecs ?? installParams.missingSpecs; runtimeDepsInstallStartedAt = Date.now(); - runtimeDepsInstallSpecs = installParams.missingSpecs; + runtimeDepsInstallSpecs = installSpecs; if (shouldActivate) { logger.info( - `[plugins] ${record.id} staging bundled runtime deps (${installParams.missingSpecs.length} missing, ${installSpecs.length} install specs): ${installParams.missingSpecs.join(", ")}`, + `[plugins] ${record.id} staging bundled runtime deps (${installSpecs.length} specs): ${installSpecs.join(", ")}`, ); } const installer = @@ -2654,58 +2328,27 @@ export function loadOpenClawPlugins(options: PluginLoadOptions = {}): PluginRegi installRoot: params.installRoot, installExecutionRoot: params.installExecutionRoot, missingSpecs: params.installSpecs ?? params.missingSpecs, + installSpecs: params.installSpecs, env, warn: (message) => logger.warn(`[plugins] ${record.id}: ${message}`), })); installer(installParams); }, + logInstalled: (installedSpecs) => { + if (shouldActivate) { + const elapsed = + runtimeDepsInstallStartedAt === null + ? "" + : ` in ${Date.now() - runtimeDepsInstallStartedAt}ms`; + logger.info( + `[plugins] ${record.id} installed bundled runtime deps${elapsed}: ${installedSpecs.join(", ")}`, + ); + } + }, }); - if (depsInstallResult.installedSpecs.length > 0) { - bundledRuntimeDepsRetainSpecsByInstallRoot.set( - installRoot, - [...new Set([...retainSpecs, ...depsInstallResult.retainSpecs])].toSorted( - (left, right) => left.localeCompare(right), - ), - ); - if (shouldActivate) { - const elapsed = - runtimeDepsInstallStartedAt === null - ? "" - : ` in ${Date.now() - runtimeDepsInstallStartedAt}ms`; - logger.info( - `[plugins] ${record.id} installed bundled runtime deps${elapsed}: ${depsInstallResult.installedSpecs.join(", ")}`, - ); - } - } - if (path.resolve(installRoot) !== path.resolve(pluginRoot)) { - const packageRoot = resolveBundledRuntimeDependencyPackageRoot(pluginRoot); - if (packageRoot) { - registerBundledRuntimeDependencyNodePath(packageRoot); - registerBundledRuntimeDependencyJitiAliases(packageRoot); - } - for (const searchRoot of installRootPlan.searchRoots) { - registerBundledRuntimeDependencyNodePath(searchRoot); - registerBundledRuntimeDependencyJitiAliases(searchRoot); - } - runtimePluginRoot = mirrorBundledPluginRuntimeRoot({ - pluginId: record.id, - pluginRoot, - installRoot, - }); - runtimeCandidateSource = - remapBundledPluginRuntimePath({ - source: runtimeCandidateSource, - pluginRoot, - mirroredRoot: runtimePluginRoot, - }) ?? runtimeCandidateSource; - runtimeSetupSource = remapBundledPluginRuntimePath({ - source: runtimeSetupSource, - pluginRoot, - mirroredRoot: runtimePluginRoot, - }); - } else { - ensureOpenClawPluginSdkAlias(path.dirname(path.dirname(pluginRoot))); - } + runtimePluginRoot = preparedRuntimeRoot.pluginRoot; + runtimeCandidateSource = preparedRuntimeRoot.modulePath; + runtimeSetupSource = preparedRuntimeRoot.setupModulePath; } catch (error) { if (shouldActivate && runtimeDepsInstallStartedAt !== null) { logger.error( diff --git a/src/plugins/provider-validation.test.ts b/src/plugins/provider-validation.test.ts index ac212434538..4b7a0d8ca07 100644 --- a/src/plugins/provider-validation.test.ts +++ b/src/plugins/provider-validation.test.ts @@ -101,6 +101,7 @@ describe("normalizeRegisteredProvider", () => { modelAllowlist: { allowedKeys: [" demo/model ", "demo/model"], initialSelections: [" demo/model "], + loadCatalog: true, message: " Demo models ", }, }, @@ -140,6 +141,7 @@ describe("normalizeRegisteredProvider", () => { modelAllowlist: { allowedKeys: ["demo/model"], initialSelections: ["demo/model"], + loadCatalog: true, message: "Demo models", }, }, diff --git a/src/plugins/provider-validation.ts b/src/plugins/provider-validation.ts index 411bc1db9b6..9b25af3b851 100644 --- a/src/plugins/provider-validation.ts +++ b/src/plugins/provider-validation.ts @@ -82,13 +82,15 @@ function buildNormalizedModelAllowlist( } const allowedKeys = normalizeTextList(modelAllowlist.allowedKeys); const initialSelections = normalizeTextList(modelAllowlist.initialSelections); + const loadCatalog = modelAllowlist.loadCatalog === true; const message = normalizeOptionalString(modelAllowlist.message); - if (!allowedKeys && !initialSelections && !message) { + if (!allowedKeys && !initialSelections && !loadCatalog && !message) { return undefined; } return { ...(allowedKeys ? { allowedKeys } : {}), ...(initialSelections ? { initialSelections } : {}), + ...(loadCatalog ? { loadCatalog } : {}), ...(message ? { message } : {}), }; } diff --git a/src/plugins/types.ts b/src/plugins/types.ts index 76e0daf49a3..70ca1c3b714 100644 --- a/src/plugins/types.ts +++ b/src/plugins/types.ts @@ -1101,6 +1101,7 @@ export type ProviderPluginWizardSetup = { modelAllowlist?: { allowedKeys?: string[]; initialSelections?: string[]; + loadCatalog?: boolean; message?: string; }; /** diff --git a/test/release-check.test.ts b/test/release-check.test.ts index 527e72bd694..2a0f8bfbb4f 100644 --- a/test/release-check.test.ts +++ b/test/release-check.test.ts @@ -11,6 +11,7 @@ import { collectAppcastSparkleVersionErrors, collectBundledExtensionManifestErrors, collectBundledPluginRootRuntimeMirrorErrors, + collectDeclaredRootRuntimeDependencyMetadataErrors, collectForbiddenPackContentPaths, collectInstalledBundledPluginRuntimeDepErrors, bundledRuntimeDependencySentinelCandidates, @@ -262,6 +263,34 @@ describe("bundled plugin root runtime mirrors", () => { ]); }); + it("flags mirrored root runtime metadata without root deps", () => { + expect( + collectDeclaredRootRuntimeDependencyMetadataErrors({ + dependencies: { semver: "7.7.4" }, + openclaw: { + bundle: { + mirroredRootRuntimeDependencies: ["json5", "semver"], + }, + }, + }), + ).toEqual([ + "package.json openclaw.bundle.mirroredRootRuntimeDependencies declares 'json5' but package.json dependencies/optionalDependencies do not include it.", + ]); + }); + + it("accepts mirrored root runtime metadata backed by root deps", () => { + expect( + collectDeclaredRootRuntimeDependencyMetadataErrors({ + dependencies: { json5: "^2.2.3", semver: "7.7.4" }, + openclaw: { + bundle: { + mirroredRootRuntimeDependencies: ["json5", "semver"], + }, + }, + }), + ).toEqual([]); + }); + it("does not derive root mirrors for root chunks sourced from the owning plugin", () => { const tempRoot = mkdtempSync(join(tmpdir(), "openclaw-root-mirror-owned-"));