diff --git a/CHANGELOG.md b/CHANGELOG.md index be9849b719f..5cc38b38a97 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ Docs: https://docs.openclaw.ai ### Changes +- Plugins/install: allow `OPENCLAW_PLUGIN_STAGE_DIR` to contain layered runtime-dependency roots, resolving read-only preinstalled deps before installing missing deps into the final writable root. Fixes #72396. Thanks @liorb-mountapps. - Control UI: polish the quick settings dashboard grid so common cards align across desktop, tablet, and mobile layouts without wasting horizontal space. Thanks @BunsDev. - Matrix/E2EE: add `openclaw matrix encryption setup` to enable Matrix encryption, bootstrap recovery, and print verification status from one setup flow. Thanks @gumadeiras. - Agents/compaction: add an opt-in `agents.defaults.compaction.maxActiveTranscriptBytes` preflight trigger that runs normal local compaction when the active JSONL grows too large, requiring transcript rotation so successful compaction moves future turns onto a smaller successor file instead of raw byte-splitting history. Thanks @vincentkoc. diff --git a/docs/cli/doctor.md b/docs/cli/doctor.md index c3ac4c0c16b..f622d6bc30f 100644 --- a/docs/cli/doctor.md +++ b/docs/cli/doctor.md @@ -43,7 +43,7 @@ Notes: - `--fix` (alias for `--repair`) writes a backup to `~/.openclaw/openclaw.json.bak` and drops unknown config keys, listing each removal. - State integrity checks now detect orphan transcript files in the sessions directory and can archive them as `.deleted.` to reclaim space safely. - Doctor also scans `~/.openclaw/cron/jobs.json` (or `cron.store`) for legacy cron job shapes and can rewrite them in place before the scheduler has to auto-normalize them at runtime. -- Doctor repairs missing bundled plugin runtime dependencies without writing into packaged global installs. For root-owned npm installs or hardened systemd units, set `OPENCLAW_PLUGIN_STAGE_DIR` to a writable directory such as `/var/lib/openclaw/plugin-runtime-deps`. +- Doctor repairs missing bundled plugin runtime dependencies without writing into packaged global installs. For root-owned npm installs or hardened systemd units, set `OPENCLAW_PLUGIN_STAGE_DIR` to a writable directory such as `/var/lib/openclaw/plugin-runtime-deps`; it can also be a path-list such as `/opt/openclaw/plugin-runtime-deps:/var/lib/openclaw/plugin-runtime-deps`, where earlier roots are read-only lookup layers and the final root is the repair target. - Set `OPENCLAW_SERVICE_REPAIR_POLICY=external` when another supervisor owns the gateway lifecycle. Doctor still reports gateway/service health and applies non-service repairs, but skips service install/start/restart/bootstrap and legacy service cleanup. - Doctor auto-migrates legacy flat Talk config (`talk.voiceId`, `talk.modelId`, and friends) into `talk.provider` + `talk.providers.`. - Repeat `doctor --fix` runs no longer report/apply Talk normalization when the only difference is object key order. diff --git a/docs/install/updating.md b/docs/install/updating.md index 2916791de1c..fedba9fe24a 100644 --- a/docs/install/updating.md +++ b/docs/install/updating.md @@ -118,6 +118,13 @@ bun add -g openclaw@latest ReadWritePaths=/var/lib/openclaw /home/openclaw/.openclaw /tmp ``` + `OPENCLAW_PLUGIN_STAGE_DIR` also accepts a path list. OpenClaw resolves bundled plugin runtime dependencies left-to-right across the listed roots, treats earlier roots as read-only preinstalled layers, and installs or repairs only into the final writable root: + + ```ini + Environment=OPENCLAW_PLUGIN_STAGE_DIR=/opt/openclaw/plugin-runtime-deps:/var/lib/openclaw/plugin-runtime-deps + ReadWritePaths=/var/lib/openclaw /home/openclaw/.openclaw /tmp + ``` + If `OPENCLAW_PLUGIN_STAGE_DIR` is not set, OpenClaw uses `$STATE_DIRECTORY` when systemd provides it, then falls back to `~/.openclaw/plugin-runtime-deps`. The repair step treats that stage as an OpenClaw-owned local package root and ignores user npm prefix and global settings, so global-install npm config does not redirect bundled plugin dependencies into `~/node_modules` or the global package tree. diff --git a/src/commands/doctor-bundled-plugin-runtime-deps.test.ts b/src/commands/doctor-bundled-plugin-runtime-deps.test.ts index 39e1a381705..6ebc6ae1e36 100644 --- a/src/commands/doctor-bundled-plugin-runtime-deps.test.ts +++ b/src/commands/doctor-bundled-plugin-runtime-deps.test.ts @@ -616,6 +616,55 @@ describe("doctor bundled plugin runtime deps", () => { expect(readRetainedRuntimeDepsManifest(installRoot)).toEqual(["@slack/web-api@7.15.1"]); }); + it("repairs only missing deps into the final layered stage dir", async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-doctor-bundled-")); + const baselineStageDir = fs.mkdtempSync( + path.join(os.tmpdir(), "openclaw-doctor-bundled-baseline-"), + ); + const writableStageDir = fs.mkdtempSync( + path.join(os.tmpdir(), "openclaw-doctor-bundled-writable-"), + ); + writeJson(path.join(root, "package.json"), { name: "openclaw", version: "2026.4.25" }); + writeBundledChannelPlugin(root, "slack", { + "@slack/web-api": "7.15.1", + grammy: "1.37.0", + }); + const env = { + OPENCLAW_PLUGIN_STAGE_DIR: [baselineStageDir, writableStageDir].join(path.delimiter), + }; + const installRoot = resolveBundledRuntimeDependencyPackageInstallRoot(root, { env }); + const baselineRoot = installRoot.replace(writableStageDir, baselineStageDir); + writeJson(path.join(baselineRoot, "node_modules", "@slack", "web-api", "package.json"), { + name: "@slack/web-api", + version: "7.15.1", + }); + const installed = createInstalledRuntimeDeps(); + + await maybeRepairBundledPluginRuntimeDeps({ + runtime: { error: () => {} } as never, + prompter: createNonInteractivePrompter(), + env, + packageRoot: root, + config: { + plugins: { enabled: true }, + channels: { slack: { enabled: true } }, + }, + installDeps: (params) => { + installed.push(params); + }, + }); + + expect(installRoot).toContain(writableStageDir); + expect(installed).toEqual([ + { + installRoot, + missingSpecs: ["grammy@1.37.0"], + installSpecs: ["grammy@1.37.0"], + }, + ]); + expect(readRetainedRuntimeDepsManifest(installRoot)).toEqual(["grammy@1.37.0"]); + }); + it("retains already staged bundled deps when repairing a subset", async () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-doctor-bundled-")); writeJson(path.join(root, "package.json"), { name: "openclaw" }); diff --git a/src/commands/doctor-bundled-plugin-runtime-deps.ts b/src/commands/doctor-bundled-plugin-runtime-deps.ts index 4918f22011c..5a21bec18e8 100644 --- a/src/commands/doctor-bundled-plugin-runtime-deps.ts +++ b/src/commands/doctor-bundled-plugin-runtime-deps.ts @@ -3,8 +3,9 @@ import { formatCliCommand } from "../cli/command-format.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import { resolveOpenClawPackageRootSync } from "../infra/openclaw-root.js"; import { + createBundledRuntimeDepsWritableInstallSpecs, repairBundledRuntimeDepsInstallRoot, - resolveBundledRuntimeDependencyPackageInstallRoot, + resolveBundledRuntimeDependencyPackageInstallRootPlan, scanBundledPluginRuntimeDeps, type BundledRuntimeDepsInstallParams, } from "../plugins/bundled-runtime-deps.js"; @@ -75,7 +76,14 @@ export async function maybeRepairBundledPluginRuntimeDeps(params: { } const missingSpecs = missing.map((dep) => `${dep.name}@${dep.version}`); - const installSpecs = deps.map((dep) => `${dep.name}@${dep.version}`); + const installRootPlan = resolveBundledRuntimeDependencyPackageInstallRootPlan(packageRoot, { + env, + }); + const installSpecs = createBundledRuntimeDepsWritableInstallSpecs({ + deps, + searchRoots: installRootPlan.searchRoots, + installRoot: installRootPlan.installRoot, + }); note( [ "Bundled plugin runtime deps are missing.", @@ -97,11 +105,8 @@ export async function maybeRepairBundledPluginRuntimeDeps(params: { } try { - const installRoot = resolveBundledRuntimeDependencyPackageInstallRoot(packageRoot, { - env: params.env ?? process.env, - }); const result = repairBundledRuntimeDepsInstallRoot({ - installRoot, + installRoot: installRootPlan.installRoot, missingSpecs, installSpecs, env: params.env ?? process.env, diff --git a/src/plugins/bundled-runtime-deps.test.ts b/src/plugins/bundled-runtime-deps.test.ts index 12d5ce6e792..8e3d977322e 100644 --- a/src/plugins/bundled-runtime-deps.test.ts +++ b/src/plugins/bundled-runtime-deps.test.ts @@ -21,6 +21,7 @@ import { materializeBundledRuntimeMirrorDistFile, repairBundledRuntimeDepsInstallRootAsync, resolveBundledRuntimeDependencyInstallRoot, + resolveBundledRuntimeDependencyInstallRootPlan, resolveBundledRuntimeDepsNpmRunner, scanBundledPluginRuntimeDeps, type BundledRuntimeDepsInstallParams, @@ -989,6 +990,47 @@ describe("scanBundledPluginRuntimeDeps config policy", () => { expect(result.deps[0]?.pluginIds).toEqual(["logger-plugin", "openclaw-core"]); expect(result.missing.map((dep) => `${dep.name}@${dep.version}`)).toEqual(["tslog@^4.10.2"]); }); + + it("resolves runtime deps from layered external stage dirs", () => { + const packageRoot = makeTempDir(); + const baselineStageDir = makeTempDir(); + const writableStageDir = makeTempDir(); + fs.writeFileSync( + path.join(packageRoot, "package.json"), + JSON.stringify({ name: "openclaw", version: "2026.4.25" }), + ); + const pluginRoot = writeBundledPluginPackage({ + packageRoot, + pluginId: "slack", + deps: { + "@slack/web-api": "7.15.1", + grammy: "1.37.0", + }, + enabledByDefault: true, + }); + const env = { + OPENCLAW_PLUGIN_STAGE_DIR: [baselineStageDir, writableStageDir].join(path.delimiter), + }; + const installRootPlan = resolveBundledRuntimeDependencyInstallRootPlan(pluginRoot, { env }); + writeInstalledPackage( + installRootPlan.searchRoots[0] ?? baselineStageDir, + "@slack/web-api", + "7.15.1", + ); + + const result = scanBundledPluginRuntimeDeps({ + packageRoot, + config: {}, + env, + }); + + expect(installRootPlan.installRoot).toContain(writableStageDir); + expect(result.deps.map((dep) => `${dep.name}@${dep.version}`)).toEqual([ + "@slack/web-api@7.15.1", + "grammy@1.37.0", + ]); + expect(result.missing.map((dep) => `${dep.name}@${dep.version}`)).toEqual(["grammy@1.37.0"]); + }); }); describe("ensureBundledPluginRuntimeDeps", () => { @@ -1238,6 +1280,64 @@ describe("ensureBundledPluginRuntimeDeps", () => { expect(second).toEqual({ installedSpecs: [], retainSpecs: [] }); }); + it("installs only missing deps into the final layered stage dir", () => { + const packageRoot = makeTempDir(); + const baselineStageDir = makeTempDir(); + const writableStageDir = makeTempDir(); + fs.writeFileSync( + path.join(packageRoot, "package.json"), + JSON.stringify({ name: "openclaw", version: "2026.4.25" }), + ); + const pluginRoot = path.join(packageRoot, "dist", "extensions", "slack"); + fs.mkdirSync(pluginRoot, { recursive: true }); + fs.writeFileSync( + path.join(pluginRoot, "package.json"), + JSON.stringify({ + dependencies: { + "@slack/web-api": "7.15.1", + grammy: "1.37.0", + }, + }), + ); + const env = { + OPENCLAW_PLUGIN_STAGE_DIR: [baselineStageDir, writableStageDir].join(path.delimiter), + }; + const installRootPlan = resolveBundledRuntimeDependencyInstallRootPlan(pluginRoot, { env }); + const baselineRoot = installRootPlan.searchRoots[0] ?? baselineStageDir; + writeInstalledPackage(baselineRoot, "@slack/web-api", "7.15.1"); + + const calls: BundledRuntimeDepsInstallParams[] = []; + const result = ensureBundledPluginRuntimeDeps({ + env, + installDeps: (params) => { + calls.push(params); + fs.rmSync(path.join(params.installRoot, "node_modules", "@slack", "web-api"), { + recursive: true, + force: true, + }); + writeInstalledPackage(params.installRoot, "grammy", "1.37.0"); + }, + pluginId: "slack", + pluginRoot, + }); + + expect(installRootPlan.installRoot).toContain(writableStageDir); + expect(result).toEqual({ + installedSpecs: ["grammy@1.37.0"], + retainSpecs: ["grammy@1.37.0"], + }); + expect(calls).toEqual([ + { + installRoot: installRootPlan.installRoot, + missingSpecs: ["grammy@1.37.0"], + installSpecs: ["grammy@1.37.0"], + }, + ]); + expect( + fs.realpathSync(path.join(installRootPlan.installRoot, "node_modules", "@slack", "web-api")), + ).toBe(fs.realpathSync(path.join(baselineRoot, "node_modules", "@slack", "web-api"))); + }); + it("retains external staged deps across separate loader passes", () => { const packageRoot = makeTempDir(); const stageDir = makeTempDir(); diff --git a/src/plugins/bundled-runtime-deps.ts b/src/plugins/bundled-runtime-deps.ts index 6e8eff55cb0..447f3884464 100644 --- a/src/plugins/bundled-runtime-deps.ts +++ b/src/plugins/bundled-runtime-deps.ts @@ -45,6 +45,10 @@ export type BundledRuntimeDepsInstallRoot = { external: boolean; }; +export type BundledRuntimeDepsInstallRootPlan = BundledRuntimeDepsInstallRoot & { + searchRoots: string[]; +}; + type JsonObject = Record; const RETAINED_RUNTIME_DEPS_MANIFEST = ".openclaw-runtime-deps.json"; // Packaged bundled plugins (Docker image, npm global install) keep their @@ -705,51 +709,83 @@ function resolveSystemdStateDirectory(env: NodeJS.ProcessEnv): string | null { return first ? path.resolve(first) : null; } -function resolveBundledRuntimeDepsExternalBaseDir(env: NodeJS.ProcessEnv): string { +function resolveBundledRuntimeDepsExternalBaseDirs(env: NodeJS.ProcessEnv): string[] { const explicit = env.OPENCLAW_PLUGIN_STAGE_DIR?.trim(); if (explicit) { - return resolveHomeRelativePath(explicit, { env, homedir: os.homedir }); + const roots = explicit + .split(path.delimiter) + .map((entry) => entry.trim()) + .filter((entry) => entry.length > 0) + .map((entry) => path.resolve(resolveHomeRelativePath(entry, { env, homedir: os.homedir }))); + if (roots.length > 0) { + const uniqueRoots: string[] = []; + for (const root of roots) { + const existingIndex = uniqueRoots.findIndex( + (entry) => path.resolve(entry) === path.resolve(root), + ); + if (existingIndex >= 0) { + uniqueRoots.splice(existingIndex, 1); + } + uniqueRoots.push(root); + } + return uniqueRoots; + } } const systemdStateDir = resolveSystemdStateDirectory(env); if (systemdStateDir) { - return path.join(systemdStateDir, "plugin-runtime-deps"); + return [path.join(systemdStateDir, "plugin-runtime-deps")]; } - return path.join(resolveStateDir(env, os.homedir), "plugin-runtime-deps"); + return [path.join(resolveStateDir(env, os.homedir), "plugin-runtime-deps")]; } function resolveExternalBundledRuntimeDepsInstallRoot(params: { pluginRoot: string; env: NodeJS.ProcessEnv; }): string { + return resolveExternalBundledRuntimeDepsInstallRoots(params).at(-1)!; +} + +function resolveExternalBundledRuntimeDepsInstallRoots(params: { + pluginRoot: string; + env: NodeJS.ProcessEnv; +}): string[] { const packageRoot = resolveBundledPluginPackageRoot(params.pluginRoot) ?? params.pluginRoot; - const existingExternalRoot = resolveExistingExternalBundledRuntimeDepsRoot({ + const existingExternalRoots = resolveExistingExternalBundledRuntimeDepsRoots({ packageRoot, env: params.env, }); - if (existingExternalRoot) { - return existingExternalRoot; + if (existingExternalRoots) { + return existingExternalRoots; } const version = sanitizePathSegment(readPackageVersion(packageRoot)); const packageKey = `openclaw-${version}-${createPathHash(packageRoot)}`; - return path.join(resolveBundledRuntimeDepsExternalBaseDir(params.env), packageKey); + return resolveBundledRuntimeDepsExternalBaseDirs(params.env).map((baseDir) => + path.join(baseDir, packageKey), + ); } -function resolveExistingExternalBundledRuntimeDepsRoot(params: { +function resolveExistingExternalBundledRuntimeDepsRoots(params: { packageRoot: string; env: NodeJS.ProcessEnv; -}): string | null { - const externalBaseDir = path.resolve(resolveBundledRuntimeDepsExternalBaseDir(params.env)); +}): string[] | null { const packageRoot = path.resolve(params.packageRoot); - const relative = path.relative(externalBaseDir, packageRoot); - if ( - relative === "" || - relative.startsWith("..") || - path.isAbsolute(relative) || - relative.includes(path.sep) - ) { - return null; + const externalBaseDirs = resolveBundledRuntimeDepsExternalBaseDirs(params.env); + for (const externalBaseDir of externalBaseDirs) { + const relative = path.relative(path.resolve(externalBaseDir), packageRoot); + if ( + relative === "" || + relative.startsWith("..") || + path.isAbsolute(relative) || + relative.includes(path.sep) + ) { + continue; + } + const packageKey = path.basename(packageRoot); + return packageKey.startsWith("openclaw-") + ? externalBaseDirs.map((baseDir) => path.join(baseDir, packageKey)) + : null; } - return path.basename(packageRoot).startsWith("openclaw-") ? packageRoot : null; + return null; } function resolveSourceCheckoutRuntimeDepsCacheDir(params: { @@ -797,6 +833,90 @@ function hasDependencySentinel( }); } +function findDependencySentinelRoot( + searchRoots: readonly string[], + dep: { name: string; version: string }, +): string | null { + return ( + searchRoots.find((rootDir) => { + const installedVersion = readInstalledDependencyVersion(rootDir, dep.name); + return ( + typeof installedVersion === "string" && + isInstalledDependencyVersionSatisfied(installedVersion, dep.version) + ); + }) ?? null + ); +} + +function dependencyPackageDir(rootDir: string, depName: string): string { + const normalizedDepName = normalizeInstallableRuntimeDepName(depName); + if (!normalizedDepName) { + throw new Error(`Invalid bundled runtime dependency name: ${depName}`); + } + return path.join(rootDir, "node_modules", ...normalizedDepName.split("/")); +} + +function createBundledRuntimeDepsInstallRootPlan(params: { + installRoot: string; + searchRoots: readonly string[]; + external: boolean; +}): BundledRuntimeDepsInstallRootPlan { + const searchRoots: string[] = []; + for (const root of params.searchRoots) { + const resolved = path.resolve(root); + if (!searchRoots.some((entry) => path.resolve(entry) === resolved)) { + searchRoots.push(root); + } + } + if (!searchRoots.some((entry) => path.resolve(entry) === path.resolve(params.installRoot))) { + searchRoots.push(params.installRoot); + } + return { + installRoot: params.installRoot, + searchRoots, + external: params.external, + }; +} + +export function createBundledRuntimeDepsWritableInstallSpecs(params: { + deps: readonly { name: string; version: string }[]; + searchRoots: readonly string[]; + installRoot: string; +}): string[] { + const readOnlyRoots = params.searchRoots.filter( + (rootDir) => path.resolve(rootDir) !== path.resolve(params.installRoot), + ); + return params.deps + .filter((dep) => !hasDependencySentinel(readOnlyRoots, dep)) + .map((dep) => `${dep.name}@${dep.version}`) + .toSorted((left, right) => left.localeCompare(right)); +} + +function linkBundledRuntimeDepsFromSearchRoots(params: { + deps: readonly { name: string; version: string }[]; + searchRoots: readonly string[]; + installRoot: string; +}): void { + for (const dep of params.deps) { + if (hasDependencySentinel([params.installRoot], dep)) { + continue; + } + const sourceRoot = findDependencySentinelRoot(params.searchRoots, dep); + if (!sourceRoot || path.resolve(sourceRoot) === path.resolve(params.installRoot)) { + continue; + } + const sourceDir = dependencyPackageDir(sourceRoot, dep.name); + const targetDir = dependencyPackageDir(params.installRoot, dep.name); + fs.mkdirSync(path.dirname(targetDir), { recursive: true }); + fs.rmSync(targetDir, { recursive: true, force: true }); + try { + fs.symlinkSync(sourceDir, targetDir, process.platform === "win32" ? "junction" : "dir"); + } catch { + fs.cpSync(sourceDir, targetDir, { recursive: true }); + } + } +} + function assertBundledRuntimeDepsInstalled(rootDir: string, specs: readonly string[]): void { const missingSpecs = specs.filter((spec) => { const dep = parseInstallableRuntimeDepSpec(spec); @@ -1261,12 +1381,14 @@ export function scanBundledPluginRuntimeDeps(params: { })) : []; const allDeps = mergeRuntimeDepEntries([...deps, ...packageRuntimeDeps]); - const packageInstallRoot = resolveBundledRuntimeDependencyPackageInstallRoot(params.packageRoot, { - env: params.env, - }); - const packageSearchRoots = [packageInstallRoot]; + const packageInstallRootPlan = resolveBundledRuntimeDependencyPackageInstallRootPlan( + params.packageRoot, + { + env: params.env, + }, + ); const missing = allDeps.filter((dep) => { - if (hasDependencySentinel(packageSearchRoots, dep)) { + if (hasDependencySentinel(packageInstallRootPlan.searchRoots, dep)) { return false; } if (dep.pluginIds.includes(MIRRORED_PACKAGE_RUNTIME_DEP_PLUGIN_ID)) { @@ -1274,21 +1396,21 @@ export function scanBundledPluginRuntimeDeps(params: { } return dep.pluginIds.every((pluginId) => { const pluginRoot = path.join(extensionsDir, pluginId); - const installRoot = resolveBundledRuntimeDependencyInstallRoot(pluginRoot, { + const installRootPlan = resolveBundledRuntimeDependencyInstallRootPlan(pluginRoot, { env: params.env, }); - return !hasDependencySentinel([installRoot], dep); + return !hasDependencySentinel(installRootPlan.searchRoots, dep); }); }); return { deps: allDeps, missing, conflicts }; } -export function resolveBundledRuntimeDependencyPackageInstallRoot( +export function resolveBundledRuntimeDependencyPackageInstallRootPlan( packageRoot: string, options: { env?: NodeJS.ProcessEnv; forceExternal?: boolean } = {}, -): string { +): BundledRuntimeDepsInstallRootPlan { const env = options.env ?? process.env; - const externalRoot = resolveExternalBundledRuntimeDepsInstallRoot({ + const externalRoots = resolveExternalBundledRuntimeDepsInstallRoots({ pluginRoot: path.join(packageRoot, "dist", "extensions", "__package__"), env, }); @@ -1298,36 +1420,103 @@ export function resolveBundledRuntimeDependencyPackageInstallRoot( env.STATE_DIRECTORY?.trim() || !isSourceCheckoutRoot(packageRoot) ) { - return externalRoot; + return createBundledRuntimeDepsInstallRootPlan({ + installRoot: + externalRoots.at(-1) ?? + resolveExternalBundledRuntimeDepsInstallRoot({ + pluginRoot: path.join(packageRoot, "dist", "extensions", "__package__"), + env, + }), + searchRoots: externalRoots, + external: true, + }); } - return isWritableDirectory(packageRoot) ? packageRoot : externalRoot; + if (isWritableDirectory(packageRoot)) { + return createBundledRuntimeDepsInstallRootPlan({ + installRoot: packageRoot, + searchRoots: [packageRoot], + external: false, + }); + } + return createBundledRuntimeDepsInstallRootPlan({ + installRoot: + externalRoots.at(-1) ?? + resolveExternalBundledRuntimeDepsInstallRoot({ + pluginRoot: path.join(packageRoot, "dist", "extensions", "__package__"), + env, + }), + searchRoots: externalRoots, + external: true, + }); } -export function resolveBundledRuntimeDependencyInstallRoot( - pluginRoot: string, +export function resolveBundledRuntimeDependencyPackageInstallRoot( + packageRoot: string, options: { env?: NodeJS.ProcessEnv; forceExternal?: boolean } = {}, ): string { + return resolveBundledRuntimeDependencyPackageInstallRootPlan(packageRoot, options).installRoot; +} + +export function resolveBundledRuntimeDependencyInstallRootPlan( + pluginRoot: string, + options: { env?: NodeJS.ProcessEnv; forceExternal?: boolean } = {}, +): BundledRuntimeDepsInstallRootPlan { const env = options.env ?? process.env; - const externalRoot = resolveExternalBundledRuntimeDepsInstallRoot({ pluginRoot, env }); + const externalRoots = resolveExternalBundledRuntimeDepsInstallRoots({ pluginRoot, env }); if ( options.forceExternal || env.OPENCLAW_PLUGIN_STAGE_DIR?.trim() || env.STATE_DIRECTORY?.trim() || isPackagedBundledPluginRoot(pluginRoot) ) { - return externalRoot; + return createBundledRuntimeDepsInstallRootPlan({ + installRoot: + externalRoots.at(-1) ?? + resolveExternalBundledRuntimeDepsInstallRoot({ + pluginRoot, + env, + }), + searchRoots: externalRoots, + external: true, + }); } - return isWritableDirectory(pluginRoot) ? pluginRoot : externalRoot; + if (isWritableDirectory(pluginRoot)) { + return createBundledRuntimeDepsInstallRootPlan({ + installRoot: pluginRoot, + searchRoots: [pluginRoot], + external: false, + }); + } + return createBundledRuntimeDepsInstallRootPlan({ + installRoot: + externalRoots.at(-1) ?? + resolveExternalBundledRuntimeDepsInstallRoot({ + pluginRoot, + env, + }), + searchRoots: externalRoots, + external: true, + }); +} + +export function resolveBundledRuntimeDependencyInstallRoot( + pluginRoot: string, + options: { env?: NodeJS.ProcessEnv; forceExternal?: boolean } = {}, +): string { + return resolveBundledRuntimeDependencyInstallRootPlan(pluginRoot, options).installRoot; } export function resolveBundledRuntimeDependencyInstallRootInfo( pluginRoot: string, options: { env?: NodeJS.ProcessEnv; forceExternal?: boolean } = {}, ): BundledRuntimeDepsInstallRoot { - const installRoot = resolveBundledRuntimeDependencyInstallRoot(pluginRoot, options); + const { installRoot, external } = resolveBundledRuntimeDependencyInstallRootPlan( + pluginRoot, + options, + ); return { installRoot, - external: path.resolve(installRoot) !== path.resolve(pluginRoot), + external, }; } @@ -1729,9 +1918,10 @@ export function ensureBundledPluginRuntimeDeps(params: { .map(([name, rawVersion]) => parseInstallableRuntimeDep(name, rawVersion)) .filter((entry): entry is { name: string; version: string } => Boolean(entry)); - const installRoot = resolveBundledRuntimeDependencyInstallRoot(params.pluginRoot, { + const installRootPlan = resolveBundledRuntimeDependencyInstallRootPlan(params.pluginRoot, { env: params.env, }); + const installRoot = installRootPlan.installRoot; const packageRoot = resolveBundledRuntimeDependencyPackageRoot(params.pluginRoot); const packageRuntimeDeps = packageRoot && path.resolve(installRoot) !== path.resolve(params.pluginRoot) @@ -1749,17 +1939,30 @@ export function ensureBundledPluginRuntimeDeps(params: { if (!persistRetainedManifest) { removeRetainedRuntimeDepsManifest(installRoot); } - const dependencySpecs = deps - .map((dep) => `${dep.name}@${dep.version}`) - .toSorted((left, right) => left.localeCompare(right)); + linkBundledRuntimeDepsFromSearchRoots({ + deps, + searchRoots: installRootPlan.searchRoots, + installRoot, + }); + const dependencySpecs = createBundledRuntimeDepsWritableInstallSpecs({ + deps, + searchRoots: installRootPlan.searchRoots, + installRoot, + }); const retainedManifestSpecs = persistRetainedManifest ? readRetainedRuntimeDepsManifest(installRoot) : []; + const readonlySearchRoots = installRootPlan.searchRoots.filter( + (rootDir) => path.resolve(rootDir) !== path.resolve(installRoot), + ); const alreadyStagedSpecs = persistRetainedManifest ? collectAlreadyStagedBundledRuntimeDepSpecs({ pluginRoot: params.pluginRoot, installRoot, - }) + }).filter( + (spec) => + !hasDependencySentinel(readonlySearchRoots, parseInstallableRuntimeDepSpec(spec)), + ) : []; const installSpecs = [ ...new Set([ @@ -1770,7 +1973,7 @@ export function ensureBundledPluginRuntimeDeps(params: { ]), ].toSorted((left, right) => left.localeCompare(right)); const missingSpecs = deps - .filter((dep) => !hasDependencySentinel([installRoot], dep)) + .filter((dep) => !hasDependencySentinel(installRootPlan.searchRoots, dep)) .map((dep) => `${dep.name}@${dep.version}`) .toSorted((left, right) => left.localeCompare(right)); if (missingSpecs.length === 0) { @@ -1829,6 +2032,11 @@ export function ensureBundledPluginRuntimeDeps(params: { } finally { finishActivity(); } + linkBundledRuntimeDepsFromSearchRoots({ + deps, + searchRoots: installRootPlan.searchRoots, + installRoot, + }); const cacheAlreadyPopulated = Boolean( sourceCheckoutCacheStage && hasAllDependencySentinels(sourceCheckoutCacheStage, deps), ); diff --git a/src/plugins/bundled-runtime-root.ts b/src/plugins/bundled-runtime-root.ts index 8ade46a01f8..b76651f8e45 100644 --- a/src/plugins/bundled-runtime-root.ts +++ b/src/plugins/bundled-runtime-root.ts @@ -3,7 +3,7 @@ import path from "node:path"; import { ensureBundledPluginRuntimeDeps, materializeBundledRuntimeMirrorDistFile, - resolveBundledRuntimeDependencyInstallRoot, + resolveBundledRuntimeDependencyInstallRootPlan, resolveBundledRuntimeDependencyPackageRoot, registerBundledRuntimeDependencyNodePath, shouldMaterializeBundledRuntimeMirrorDistFile, @@ -30,7 +30,10 @@ export function prepareBundledPluginRuntimeRoot(params: { logInstalled?: (installedSpecs: readonly string[]) => void; }): { pluginRoot: string; modulePath: string } { const env = params.env ?? process.env; - const installRoot = resolveBundledRuntimeDependencyInstallRoot(params.pluginRoot, { env }); + const installRootPlan = resolveBundledRuntimeDependencyInstallRootPlan(params.pluginRoot, { + env, + }); + const installRoot = installRootPlan.installRoot; const retainSpecs = bundledRuntimeDepsRetainSpecsByInstallRoot.get(installRoot) ?? []; const depsInstallResult = ensureBundledPluginRuntimeDeps({ pluginId: params.pluginId, @@ -54,7 +57,9 @@ export function prepareBundledPluginRuntimeRoot(params: { if (packageRoot) { registerBundledRuntimeDependencyNodePath(packageRoot); } - registerBundledRuntimeDependencyNodePath(installRoot); + for (const searchRoot of installRootPlan.searchRoots) { + registerBundledRuntimeDependencyNodePath(searchRoot); + } const mirrorRoot = mirrorBundledPluginRuntimeRoot({ pluginId: params.pluginId, pluginRoot: params.pluginRoot, diff --git a/src/plugins/loader.test.ts b/src/plugins/loader.test.ts index 1e0b3a8e925..3c081944c38 100644 --- a/src/plugins/loader.test.ts +++ b/src/plugins/loader.test.ts @@ -22,6 +22,7 @@ import { type DetachedTaskLifecycleRuntime, } from "../tasks/detached-task-runtime-state.js"; import { withEnv } from "../test-utils/env.js"; +import { resolveBundledRuntimeDependencyInstallRootPlan } from "./bundled-runtime-deps.js"; import { clearPluginCommands } from "./command-registry-state.js"; import { getPluginCommandSpecs } from "./command-specs.js"; import { @@ -2128,13 +2129,151 @@ module.exports = { }, }); - expect(registry.plugins.find((entry) => entry.id === "acpx")?.status).toBe("loaded"); + const record = registry.plugins.find((entry) => entry.id === "acpx"); + expect(record?.error).toBeUndefined(); + expect(record?.status).toBe("loaded"); expect(fs.lstatSync(path.join(actualInstallRoot, "dist")).isSymbolicLink()).toBe(false); expect(fs.lstatSync(path.join(actualInstallRoot, "dist", "pw-ai.js")).isSymbolicLink()).toBe( false, ); }); + it("loads native ESM deps from a layered baseline stage dir", () => { + const packageRoot = makeTempDir(); + const baselineStageDir = makeTempDir(); + const writableStageDir = makeTempDir(); + const bundledDir = path.join(packageRoot, "dist-runtime", "extensions"); + const pluginRoot = path.join(bundledDir, "acpx"); + const canonicalPluginRoot = path.join(packageRoot, "dist", "extensions", "acpx"); + const canonicalEntryImport = path.posix.join( + "..", + "..", + "..", + "dist", + "extensions", + "acpx", + "index.js", + ); + fs.mkdirSync(pluginRoot, { recursive: true }); + fs.mkdirSync(canonicalPluginRoot, { recursive: true }); + fs.writeFileSync( + path.join(packageRoot, "package.json"), + JSON.stringify({ name: "openclaw", version: "2026.4.25", type: "module" }), + "utf-8", + ); + fs.writeFileSync( + path.join(packageRoot, "dist", "pw-ai.js"), + [ + `//#region extensions/acpx/src/pw-ai.ts`, + `import runtimeDep from "external-runtime";`, + `export const marker = runtimeDep.marker;`, + `//#endregion`, + "", + ].join("\n"), + "utf-8", + ); + fs.writeFileSync( + path.join(pluginRoot, "index.js"), + [ + `export * from ${JSON.stringify(canonicalEntryImport)};`, + `import defaultModule from ${JSON.stringify(canonicalEntryImport)};`, + `export default defaultModule;`, + "", + ].join("\n"), + "utf-8", + ); + fs.writeFileSync( + path.join(canonicalPluginRoot, "index.js"), + [ + `import { marker } from "../../pw-ai.js";`, + `export default {`, + ` id: "acpx",`, + ` register(api) {`, + ` api.registerCommand({ name: "external-runtime", handler: () => marker });`, + ` },`, + `};`, + "", + ].join("\n"), + "utf-8", + ); + fs.writeFileSync( + path.join(pluginRoot, "package.json"), + JSON.stringify( + { + name: "@openclaw/acpx", + version: "1.0.0", + type: "module", + dependencies: { + "external-runtime": "1.0.0", + }, + openclaw: { extensions: ["./index.js"] }, + }, + null, + 2, + ), + "utf-8", + ); + fs.writeFileSync( + path.join(pluginRoot, "openclaw.plugin.json"), + JSON.stringify( + { + id: "acpx", + enabledByDefault: true, + configSchema: EMPTY_PLUGIN_SCHEMA, + }, + null, + 2, + ), + "utf-8", + ); + const env = { + OPENCLAW_PLUGIN_STAGE_DIR: [baselineStageDir, writableStageDir].join(path.delimiter), + }; + const installRootPlan = resolveBundledRuntimeDependencyInstallRootPlan( + fs.realpathSync(pluginRoot), + { env }, + ); + const baselineRoot = installRootPlan.searchRoots[0] ?? baselineStageDir; + const baselineDepRoot = path.join(baselineRoot, "node_modules", "external-runtime"); + fs.mkdirSync(baselineDepRoot, { recursive: true }); + fs.writeFileSync( + path.join(baselineDepRoot, "package.json"), + JSON.stringify({ + name: "external-runtime", + version: "1.0.0", + type: "module", + exports: "./index.js", + }), + "utf-8", + ); + fs.writeFileSync( + path.join(baselineDepRoot, "index.js"), + "export default { marker: 'baseline-ok' };\n", + "utf-8", + ); + process.env.OPENCLAW_BUNDLED_PLUGINS_DIR = bundledDir; + process.env.OPENCLAW_PLUGIN_STAGE_DIR = env.OPENCLAW_PLUGIN_STAGE_DIR; + + const registry = loadOpenClawPlugins({ + cache: false, + config: { + plugins: { + enabled: true, + }, + }, + bundledRuntimeDepsInstaller: () => { + throw new Error("baseline deps should not reinstall"); + }, + }); + + const layeredRecord = registry.plugins.find((entry) => entry.id === "acpx"); + expect(layeredRecord?.error).toBeUndefined(); + expect(layeredRecord?.status).toBe("loaded"); + expect( + fs.realpathSync(path.join(installRootPlan.installRoot, "node_modules", "external-runtime")), + ).toBe(fs.realpathSync(baselineDepRoot)); + }); + it("loads source-checkout bundled runtime deps without mirroring the repo tree", () => { const packageRoot = makeTempDir(); fs.mkdirSync(path.join(packageRoot, ".git"), { recursive: true }); diff --git a/src/plugins/loader.ts b/src/plugins/loader.ts index 9258271f927..514059aab26 100644 --- a/src/plugins/loader.ts +++ b/src/plugins/loader.ts @@ -36,7 +36,7 @@ import { ensureBundledPluginRuntimeDeps, installBundledRuntimeDeps, materializeBundledRuntimeMirrorDistFile, - resolveBundledRuntimeDependencyInstallRoot, + resolveBundledRuntimeDependencyInstallRootPlan, resolveBundledRuntimeDependencyPackageRoot, registerBundledRuntimeDependencyNodePath, shouldMaterializeBundledRuntimeMirrorDistFile, @@ -2552,7 +2552,10 @@ export function loadOpenClawPlugins(options: PluginLoadOptions = {}): PluginRegi let runtimeDepsInstallStartedAt: number | null = null; let runtimeDepsInstallSpecs: string[] = []; try { - const installRoot = resolveBundledRuntimeDependencyInstallRoot(pluginRoot, { env }); + const installRootPlan = resolveBundledRuntimeDependencyInstallRootPlan(pluginRoot, { + env, + }); + const installRoot = installRootPlan.installRoot; const retainSpecs = bundledRuntimeDepsRetainSpecsByInstallRoot.get(installRoot) ?? []; const depsInstallResult = ensureBundledPluginRuntimeDeps({ pluginId: record.id, @@ -2605,8 +2608,10 @@ export function loadOpenClawPlugins(options: PluginLoadOptions = {}): PluginRegi registerBundledRuntimeDependencyNodePath(packageRoot); registerBundledRuntimeDependencyJitiAliases(packageRoot); } - registerBundledRuntimeDependencyNodePath(installRoot); - registerBundledRuntimeDependencyJitiAliases(installRoot); + for (const searchRoot of installRootPlan.searchRoots) { + registerBundledRuntimeDependencyNodePath(searchRoot); + registerBundledRuntimeDependencyJitiAliases(searchRoot); + } runtimePluginRoot = mirrorBundledPluginRuntimeRoot({ pluginId: record.id, pluginRoot,