From a99490fba4af1dc52bde0a8c2c2916e1757d9662 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Wed, 22 Apr 2026 05:01:54 +0100 Subject: [PATCH] fix(plugins): support root-owned bundled runtime deps --- docs/cli/doctor.md | 1 + docs/install/updating.md | 19 ++ .../bundled-channel-runtime-deps-docker.sh | 202 +++++++++++++++++- ...doctor-bundled-plugin-runtime-deps.test.ts | 58 ++++- .../doctor-bundled-plugin-runtime-deps.ts | 7 +- src/plugins/bundled-runtime-deps.test.ts | 129 +++++++++++ src/plugins/bundled-runtime-deps.ts | 200 +++++++++++++++-- src/plugins/loader.test-fixtures.ts | 6 + src/plugins/loader.test.ts | 75 +++++++ src/plugins/loader.ts | 185 +++++++++++++++- 10 files changed, 857 insertions(+), 25 deletions(-) diff --git a/docs/cli/doctor.md b/docs/cli/doctor.md index 5f4c00322e1..ba27f736208 100644 --- a/docs/cli/doctor.md +++ b/docs/cli/doctor.md @@ -42,6 +42,7 @@ Notes: - `--fix` (alias for `--repair`) writes a backup to `~/.openclaw/openclaw.json.bak` and drops unknown config keys, listing each removal. - State integrity checks now detect orphan transcript files in the sessions directory and can archive them as `.deleted.` to reclaim space safely. - Doctor also scans `~/.openclaw/cron/jobs.json` (or `cron.store`) for legacy cron job shapes and can rewrite them in place before the scheduler has to auto-normalize them at runtime. +- Doctor repairs missing bundled plugin runtime dependencies without requiring write access to the installed OpenClaw package. For root-owned npm installs or hardened systemd units, set `OPENCLAW_PLUGIN_STAGE_DIR` to a writable directory such as `/var/lib/openclaw/plugin-runtime-deps`. - Doctor auto-migrates legacy flat Talk config (`talk.voiceId`, `talk.modelId`, and friends) into `talk.provider` + `talk.providers.`. - Repeat `doctor --fix` runs no longer report/apply Talk normalization when the only difference is object key order. - Doctor includes a memory-search readiness check and can recommend `openclaw configure --section model` when embedding credentials are missing. diff --git a/docs/install/updating.md b/docs/install/updating.md index d12309c3e6d..ce34bb2937c 100644 --- a/docs/install/updating.md +++ b/docs/install/updating.md @@ -54,6 +54,25 @@ pnpm add -g openclaw@latest bun add -g openclaw@latest ``` +### Root-owned global npm installs + +Some Linux npm setups install global packages under root-owned directories such as +`/usr/lib/node_modules/openclaw`. OpenClaw supports that layout: the installed +package is treated as read-only at runtime, and bundled plugin runtime +dependencies are staged into a writable runtime directory instead of mutating the +package tree. + +For hardened systemd units, set a writable stage directory that is included in +`ReadWritePaths`: + +```ini +Environment=OPENCLAW_PLUGIN_STAGE_DIR=/var/lib/openclaw/plugin-runtime-deps +ReadWritePaths=/var/lib/openclaw /home/openclaw/.openclaw /tmp +``` + +If `OPENCLAW_PLUGIN_STAGE_DIR` is not set, OpenClaw uses `$STATE_DIRECTORY` when +systemd provides it, then falls back to `~/.openclaw/plugin-runtime-deps`. + ## Auto-updater The auto-updater is off by default. Enable it in `~/.openclaw/openclaw.json`: diff --git a/scripts/e2e/bundled-channel-runtime-deps-docker.sh b/scripts/e2e/bundled-channel-runtime-deps-docker.sh index 48ee70c2ecc..1c7bd1d6e16 100644 --- a/scripts/e2e/bundled-channel-runtime-deps-docker.sh +++ b/scripts/e2e/bundled-channel-runtime-deps-docker.sh @@ -6,7 +6,9 @@ source "$ROOT_DIR/scripts/lib/docker-e2e-logs.sh" IMAGE_NAME="${OPENCLAW_BUNDLED_CHANNEL_DEPS_E2E_IMAGE:-openclaw-bundled-channel-deps-e2e}" UPDATE_BASELINE_VERSION="${OPENCLAW_BUNDLED_CHANNEL_UPDATE_BASELINE_VERSION:-2026.4.20}" +RUN_CHANNEL_SCENARIOS="${OPENCLAW_BUNDLED_CHANNEL_SCENARIOS:-1}" RUN_UPDATE_SCENARIO="${OPENCLAW_BUNDLED_CHANNEL_UPDATE_SCENARIO:-1}" +RUN_ROOT_OWNED_SCENARIO="${OPENCLAW_BUNDLED_CHANNEL_ROOT_OWNED_SCENARIO:-1}" echo "Building Docker image..." run_logged bundled-channel-deps-build docker build -t "$IMAGE_NAME" -f "$ROOT_DIR/scripts/e2e/Dockerfile" "$ROOT_DIR" @@ -299,6 +301,195 @@ EOF rm -f "$run_log" } +run_root_owned_global_scenario() { + local run_log + run_log="$(mktemp "${TMPDIR:-/tmp}/openclaw-bundled-channel-root-owned.XXXXXX")" + + echo "Running bundled channel root-owned global install Docker E2E..." + if ! docker run --rm --user root \ + -e COREPACK_ENABLE_DOWNLOAD_PROMPT=0 \ + -i "$IMAGE_NAME" bash -s >"$run_log" 2>&1 <<'EOF' +set -euo pipefail + +export HOME="/root" +export OPENAI_API_KEY="sk-openclaw-bundled-channel-root-owned-e2e" +export OPENCLAW_NO_ONBOARD=1 +export OPENCLAW_PLUGIN_STAGE_DIR="/var/lib/openclaw/plugin-runtime-deps" + +TOKEN="bundled-channel-root-owned-token" +PORT="18791" +CHANNEL="slack" +DEP_SENTINEL="@slack/web-api" +gateway_pid="" + +package_root() { + printf "%s/openclaw" "$(npm root -g)" +} + +cleanup() { + if [ -n "${gateway_pid:-}" ] && kill -0 "$gateway_pid" 2>/dev/null; then + kill "$gateway_pid" 2>/dev/null || true + wait "$gateway_pid" 2>/dev/null || true + fi +} +trap cleanup EXIT + +echo "Packing and installing current OpenClaw build into root-owned global npm..." +pack_dir="$(mktemp -d "/tmp/openclaw-root-owned-pack.XXXXXX")" +npm pack --ignore-scripts --pack-destination "$pack_dir" >/tmp/openclaw-root-owned-pack.log 2>&1 +package_tgz="$(find "$pack_dir" -maxdepth 1 -name 'openclaw-*.tgz' -print -quit)" +if [ -z "$package_tgz" ]; then + cat /tmp/openclaw-root-owned-pack.log + echo "missing packed OpenClaw tarball" >&2 + exit 1 +fi +npm install -g "$package_tgz" --no-fund --no-audit >/tmp/openclaw-root-owned-install.log 2>&1 + +root="$(package_root)" +test -d "$root/dist/extensions/$CHANNEL" +rm -rf "$root/dist/extensions/$CHANNEL/node_modules" +chmod -R a-w "$root" +mkdir -p "$OPENCLAW_PLUGIN_STAGE_DIR" /home/appuser/.openclaw +chown -R appuser:appuser /home/appuser/.openclaw /var/lib/openclaw + +if runuser -u appuser -- test -w "$root"; then + echo "expected package root to be unwritable for appuser" >&2 + exit 1 +fi + +node - <<'NODE' "$TOKEN" "$PORT" +const fs = require("node:fs"); +const path = require("node:path"); +const token = process.argv[2]; +const port = Number(process.argv[3]); +const configPath = "/home/appuser/.openclaw/openclaw.json"; +const config = { + gateway: { + port, + auth: { mode: "token", token }, + controlUi: { enabled: false }, + }, + agents: { + defaults: { + model: { primary: "openai/gpt-4.1-mini" }, + }, + }, + models: { + providers: { + openai: { + apiKey: process.env.OPENAI_API_KEY, + baseUrl: "https://api.openai.com/v1", + models: [], + }, + }, + }, + plugins: { enabled: true }, + channels: { + slack: { + enabled: true, + botToken: "xoxb-bundled-channel-root-owned-token", + appToken: "xapp-bundled-channel-root-owned-token", + }, + }, +}; +fs.mkdirSync(path.dirname(configPath), { recursive: true }); +fs.writeFileSync(configPath, `${JSON.stringify(config, null, 2)}\n`, "utf8"); +NODE +chown appuser:appuser /home/appuser/.openclaw/openclaw.json + +start_gateway() { + local log_file="$1" + : >"$log_file" + chown appuser:appuser "$log_file" + runuser -u appuser -- env \ + HOME=/home/appuser \ + OPENAI_API_KEY="$OPENAI_API_KEY" \ + OPENCLAW_NO_ONBOARD=1 \ + OPENCLAW_PLUGIN_STAGE_DIR="$OPENCLAW_PLUGIN_STAGE_DIR" \ + npm_config_cache=/tmp/openclaw-root-owned-npm-cache \ + openclaw gateway --port "$PORT" --bind loopback --allow-unconfigured >"$log_file" 2>&1 & + gateway_pid="$!" + + for _ in $(seq 1 240); do + if grep -Eq "listening on ws://|\\[gateway\\] ready \\(" "$log_file"; then + return 0 + fi + if ! kill -0 "$gateway_pid" 2>/dev/null; then + echo "gateway exited unexpectedly" >&2 + cat "$log_file" >&2 + exit 1 + fi + sleep 0.25 + done + + echo "timed out waiting for gateway" >&2 + cat "$log_file" >&2 + exit 1 +} + +wait_for_gateway_health() { + for _ in $(seq 1 120); do + if runuser -u appuser -- env HOME=/home/appuser openclaw gateway health --url "ws://127.0.0.1:$PORT" --token "$TOKEN" --json >/dev/null 2>&1; then + return 0 + fi + sleep 0.25 + done + echo "timed out waiting for gateway health" >&2 + return 1 +} + +assert_channel_status() { + local out="/tmp/openclaw-root-owned-channel-status.json" + runuser -u appuser -- env HOME=/home/appuser openclaw gateway call channels.status \ + --url "ws://127.0.0.1:$PORT" \ + --token "$TOKEN" \ + --timeout 30000 \ + --json \ + --params '{"probe":false}' >"$out" + if ! node - <<'NODE' "$out" "$CHANNEL" +const fs = require("node:fs"); +const raw = JSON.parse(fs.readFileSync(process.argv[2], "utf8")); +const payload = raw.result ?? raw.data ?? raw; +const channel = process.argv[3]; +if (!payload.channels || !payload.channels[channel]) { + throw new Error(`missing channels.${channel}\n${JSON.stringify(raw, null, 2).slice(0, 4000)}`); +} +console.log(`${channel} channel plugin visible`); +NODE + then + cat /tmp/openclaw-root-owned-gateway.log >&2 + exit 1 + fi +} + +start_gateway /tmp/openclaw-root-owned-gateway.log +wait_for_gateway_health +assert_channel_status + +if [ -e "$root/dist/extensions/$CHANNEL/node_modules/$DEP_SENTINEL/package.json" ]; then + echo "root-owned package tree was mutated" >&2 + find "$root/dist/extensions/$CHANNEL/node_modules" -maxdepth 4 -type f | sort | head -80 >&2 || true + exit 1 +fi +if ! find "$OPENCLAW_PLUGIN_STAGE_DIR" -maxdepth 12 -path "*/node_modules/$DEP_SENTINEL/package.json" -type f | grep -q .; then + echo "missing external staged dependency sentinel for $DEP_SENTINEL" >&2 + find "$OPENCLAW_PLUGIN_STAGE_DIR" -maxdepth 12 -type f | sort | head -120 >&2 || true + cat /tmp/openclaw-root-owned-gateway.log >&2 + exit 1 +fi + +echo "root-owned global install Docker E2E passed" +EOF + then + cat "$run_log" + rm -f "$run_log" + exit 1 + fi + + cat "$run_log" + rm -f "$run_log" +} + run_update_scenario() { local run_log run_log="$(mktemp "${TMPDIR:-/tmp}/openclaw-bundled-channel-update.XXXXXX")" @@ -600,9 +791,14 @@ EOF rm -f "$run_log" } -run_channel_scenario telegram grammy -run_channel_scenario discord discord-api-types -run_channel_scenario slack @slack/web-api +if [ "$RUN_CHANNEL_SCENARIOS" != "0" ]; then + run_channel_scenario telegram grammy + run_channel_scenario discord discord-api-types + run_channel_scenario slack @slack/web-api +fi if [ "$RUN_UPDATE_SCENARIO" != "0" ]; then run_update_scenario fi +if [ "$RUN_ROOT_OWNED_SCENARIO" != "0" ]; then + run_root_owned_global_scenario +fi diff --git a/src/commands/doctor-bundled-plugin-runtime-deps.test.ts b/src/commands/doctor-bundled-plugin-runtime-deps.test.ts index 040858fe413..9201b27f339 100644 --- a/src/commands/doctor-bundled-plugin-runtime-deps.test.ts +++ b/src/commands/doctor-bundled-plugin-runtime-deps.test.ts @@ -2,7 +2,10 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { describe, expect, it } from "vitest"; -import { scanBundledPluginRuntimeDeps } from "../plugins/bundled-runtime-deps.js"; +import { + resolveBundledRuntimeDependencyPackageInstallRoot, + scanBundledPluginRuntimeDeps, +} from "../plugins/bundled-runtime-deps.js"; import { maybeRepairBundledPluginRuntimeDeps } from "./doctor-bundled-plugin-runtime-deps.js"; import type { DoctorPrompter } from "./doctor-prompter.js"; @@ -216,6 +219,59 @@ describe("doctor bundled plugin runtime deps", () => { ]); }); + it("repairs missing deps into an external stage dir when configured", async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-doctor-bundled-")); + const stageDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-doctor-bundled-stage-")); + writeJson(path.join(root, "package.json"), { name: "openclaw", version: "2026.4.22" }); + writeBundledChannelPlugin(root, "slack", { "@slack/web-api": "7.15.1" }); + const env = { OPENCLAW_PLUGIN_STAGE_DIR: stageDir }; + const installed: Array<{ + installRoot: string; + missingSpecs: string[]; + installSpecs: string[]; + }> = []; + const prompter = { + shouldRepair: false, + shouldForce: false, + repairMode: { + shouldRepair: false, + shouldForce: false, + nonInteractive: true, + canPrompt: false, + updateInProgress: false, + }, + confirm: async () => false, + confirmAutoFix: async () => false, + confirmAggressiveAutoFix: async () => false, + confirmRuntimeRepair: async () => false, + select: async (_params: unknown, fallback: unknown) => fallback, + } as DoctorPrompter; + + await maybeRepairBundledPluginRuntimeDeps({ + runtime: { error: () => {} } as never, + prompter, + env, + packageRoot: root, + config: { + plugins: { enabled: true }, + channels: { slack: { enabled: true } }, + }, + installDeps: (params) => { + installed.push(params); + }, + }); + + const installRoot = resolveBundledRuntimeDependencyPackageInstallRoot(root, { env }); + expect(installed).toEqual([ + { + installRoot, + missingSpecs: ["@slack/web-api@7.15.1"], + installSpecs: ["@slack/web-api@7.15.1"], + }, + ]); + expect(installRoot).toContain(stageDir); + }); + it("retains configured bundled deps when repairing a subset", async () => { const root = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-doctor-bundled-")); writeJson(path.join(root, "package.json"), { name: "openclaw" }); diff --git a/src/commands/doctor-bundled-plugin-runtime-deps.ts b/src/commands/doctor-bundled-plugin-runtime-deps.ts index 42baa45180b..31b9b8a0983 100644 --- a/src/commands/doctor-bundled-plugin-runtime-deps.ts +++ b/src/commands/doctor-bundled-plugin-runtime-deps.ts @@ -3,6 +3,7 @@ import type { OpenClawConfig } from "../config/types.openclaw.js"; import { resolveOpenClawPackageRootSync } from "../infra/openclaw-root.js"; import { installBundledRuntimeDeps, + resolveBundledRuntimeDependencyPackageInstallRoot, scanBundledPluginRuntimeDeps, } from "../plugins/bundled-runtime-deps.js"; import type { RuntimeEnv } from "../runtime.js"; @@ -37,6 +38,7 @@ export async function maybeRepairBundledPluginRuntimeDeps(params: { packageRoot, config: params.config, includeConfiguredChannels: params.includeConfiguredChannels, + env: params.env ?? process.env, }); if (conflicts.length > 0) { const conflictLines = conflicts.flatMap((conflict) => @@ -84,6 +86,9 @@ export async function maybeRepairBundledPluginRuntimeDeps(params: { } try { + const installRoot = resolveBundledRuntimeDependencyPackageInstallRoot(packageRoot, { + env: params.env ?? process.env, + }); const install = params.installDeps ?? ((installParams) => @@ -92,7 +97,7 @@ export async function maybeRepairBundledPluginRuntimeDeps(params: { missingSpecs: installParams.installSpecs, env: params.env ?? process.env, })); - install({ installRoot: packageRoot, missingSpecs, installSpecs }); + install({ installRoot, missingSpecs, installSpecs }); note(`Installed bundled plugin deps: ${installSpecs.join(", ")}`, "Bundled plugins"); } catch (error) { params.runtime.error(`Failed to install bundled plugin runtime deps: ${String(error)}`); diff --git a/src/plugins/bundled-runtime-deps.test.ts b/src/plugins/bundled-runtime-deps.test.ts index 6fbc43358af..2012e506276 100644 --- a/src/plugins/bundled-runtime-deps.test.ts +++ b/src/plugins/bundled-runtime-deps.test.ts @@ -4,10 +4,12 @@ import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; import { + createBundledRuntimeDependencyAliasMap, createBundledRuntimeDepsInstallArgs, createBundledRuntimeDepsInstallEnv, ensureBundledPluginRuntimeDeps, installBundledRuntimeDeps, + resolveBundledRuntimeDependencyInstallRoot, resolveBundledRuntimeDepsNpmRunner, type BundledRuntimeDepsInstallParams, } from "./bundled-runtime-deps.js"; @@ -301,6 +303,133 @@ describe("ensureBundledPluginRuntimeDeps", () => { ]); }); + it("installs runtime deps into an external stage dir and exposes loader aliases", () => { + const packageRoot = makeTempDir(); + const stageDir = makeTempDir(); + fs.writeFileSync( + path.join(packageRoot, "package.json"), + JSON.stringify({ name: "openclaw", version: "2026.4.22" }), + ); + const pluginRoot = path.join(packageRoot, "dist", "extensions", "slack"); + fs.mkdirSync(pluginRoot, { recursive: true }); + fs.writeFileSync( + path.join(pluginRoot, "package.json"), + JSON.stringify({ + dependencies: { + "@slack/web-api": "7.15.1", + }, + }), + ); + + const env = { OPENCLAW_PLUGIN_STAGE_DIR: stageDir }; + const calls: BundledRuntimeDepsInstallParams[] = []; + const result = ensureBundledPluginRuntimeDeps({ + env, + installDeps: (params) => { + calls.push(params); + fs.mkdirSync(path.join(params.installRoot, "node_modules", "@slack", "web-api"), { + recursive: true, + }); + fs.writeFileSync( + path.join(params.installRoot, "node_modules", "@slack", "web-api", "package.json"), + JSON.stringify({ name: "@slack/web-api", version: "7.15.1" }), + ); + }, + pluginId: "slack", + pluginRoot, + }); + + const installRoot = resolveBundledRuntimeDependencyInstallRoot(pluginRoot, { env }); + expect(result).toEqual({ + installedSpecs: ["@slack/web-api@7.15.1"], + retainSpecs: ["@slack/web-api@7.15.1"], + }); + expect(calls).toEqual([ + { + installRoot, + missingSpecs: ["@slack/web-api@7.15.1"], + installSpecs: ["@slack/web-api@7.15.1"], + }, + ]); + expect(installRoot).toContain(stageDir); + expect(installRoot).not.toBe(pluginRoot); + expect(createBundledRuntimeDependencyAliasMap({ pluginRoot, installRoot })).toEqual({ + "@slack/web-api": path.join(installRoot, "node_modules", "@slack", "web-api"), + }); + + const second = ensureBundledPluginRuntimeDeps({ + env, + installDeps: () => { + throw new Error("external staged deps should not reinstall"); + }, + pluginId: "slack", + pluginRoot, + }); + expect(second).toEqual({ installedSpecs: [], retainSpecs: [] }); + }); + + it("retains external staged deps across separate loader passes", () => { + const packageRoot = makeTempDir(); + const stageDir = makeTempDir(); + fs.writeFileSync( + path.join(packageRoot, "package.json"), + JSON.stringify({ name: "openclaw", version: "2026.4.22" }), + ); + const alphaRoot = path.join(packageRoot, "dist", "extensions", "alpha"); + const betaRoot = path.join(packageRoot, "dist", "extensions", "beta"); + fs.mkdirSync(alphaRoot, { recursive: true }); + fs.mkdirSync(betaRoot, { recursive: true }); + fs.writeFileSync( + path.join(alphaRoot, "package.json"), + JSON.stringify({ dependencies: { "alpha-runtime": "1.0.0" } }), + ); + fs.writeFileSync( + path.join(betaRoot, "package.json"), + JSON.stringify({ dependencies: { "beta-runtime": "2.0.0" } }), + ); + + const env = { OPENCLAW_PLUGIN_STAGE_DIR: stageDir }; + const calls: BundledRuntimeDepsInstallParams[] = []; + const installDeps = (params: BundledRuntimeDepsInstallParams) => { + calls.push(params); + for (const spec of params.installSpecs ?? params.missingSpecs) { + const name = spec.slice(0, spec.lastIndexOf("@")); + fs.mkdirSync(path.join(params.installRoot, "node_modules", name), { recursive: true }); + fs.writeFileSync( + path.join(params.installRoot, "node_modules", name, "package.json"), + JSON.stringify({ name, version: spec.slice(spec.lastIndexOf("@") + 1) }), + ); + } + }; + + ensureBundledPluginRuntimeDeps({ + env, + installDeps, + pluginId: "alpha", + pluginRoot: alphaRoot, + }); + ensureBundledPluginRuntimeDeps({ + env, + installDeps, + pluginId: "beta", + pluginRoot: betaRoot, + }); + + const installRoot = resolveBundledRuntimeDependencyInstallRoot(alphaRoot, { env }); + expect(calls).toEqual([ + { + installRoot, + missingSpecs: ["alpha-runtime@1.0.0"], + installSpecs: ["alpha-runtime@1.0.0"], + }, + { + installRoot, + missingSpecs: ["beta-runtime@2.0.0"], + installSpecs: ["alpha-runtime@1.0.0", "beta-runtime@2.0.0"], + }, + ]); + }); + it("does not install when runtime deps are only workspace links", () => { const packageRoot = makeTempDir(); const extensionsRoot = path.join(packageRoot, "dist", "extensions"); diff --git a/src/plugins/bundled-runtime-deps.ts b/src/plugins/bundled-runtime-deps.ts index 25294cb3c18..4f29816df3b 100644 --- a/src/plugins/bundled-runtime-deps.ts +++ b/src/plugins/bundled-runtime-deps.ts @@ -1,9 +1,12 @@ import { spawnSync } from "node:child_process"; import { createHash } from "node:crypto"; import fs from "node:fs"; +import os from "node:os"; import path from "node:path"; import { normalizeChatChannelId } from "../channels/ids.js"; +import { resolveStateDir } from "../config/paths.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; +import { resolveHomeRelativePath } from "../infra/home-dir.js"; import { normalizeOptionalLowercaseString } from "../shared/string-coerce.js"; import { normalizePluginsConfig } from "./config-state.js"; @@ -30,7 +33,13 @@ export type BundledRuntimeDepsEnsureResult = { retainSpecs: string[]; }; +export type BundledRuntimeDepsInstallRoot = { + installRoot: string; + external: boolean; +}; + type JsonObject = Record; +const RETAINED_RUNTIME_DEPS_MANIFEST = ".openclaw-runtime-deps.json"; export type BundledRuntimeDepsNpmRunner = { command: string; @@ -122,6 +131,18 @@ function resolveBundledRuntimeDependencySearchRoots(params: { return [...roots]; } +function resolveBundledPluginPackageRoot(pluginRoot: string): string | null { + const extensionsDir = path.dirname(path.resolve(pluginRoot)); + const buildDir = path.dirname(extensionsDir); + if ( + path.basename(extensionsDir) !== "extensions" || + (path.basename(buildDir) !== "dist" && path.basename(buildDir) !== "dist-runtime") + ) { + return null; + } + return path.dirname(buildDir); +} + function createRuntimeDepsCacheKey(pluginId: string, specs: readonly string[]): string { return createHash("sha256") .update(pluginId) @@ -131,6 +152,79 @@ function createRuntimeDepsCacheKey(pluginId: string, specs: readonly string[]): .slice(0, 16); } +function createPathHash(value: string): string { + return createHash("sha256").update(path.resolve(value)).digest("hex").slice(0, 12); +} + +function sanitizePathSegment(value: string): string { + return value.replace(/[^A-Za-z0-9._-]+/g, "-").replace(/^-+|-+$/g, "") || "unknown"; +} + +function readPackageVersion(packageRoot: string): string { + const parsed = readJsonObject(path.join(packageRoot, "package.json")); + const version = parsed && typeof parsed.version === "string" ? parsed.version.trim() : ""; + return version || "unknown"; +} + +function readRetainedRuntimeDepsManifest(installRoot: string): string[] { + const parsed = readJsonObject(path.join(installRoot, RETAINED_RUNTIME_DEPS_MANIFEST)); + const specs = parsed?.specs; + if (!Array.isArray(specs)) { + return []; + } + return specs + .filter((entry): entry is string => typeof entry === "string" && entry.trim().length > 0) + .toSorted((left, right) => left.localeCompare(right)); +} + +function writeRetainedRuntimeDepsManifest(installRoot: string, specs: readonly string[]): void { + fs.writeFileSync( + path.join(installRoot, RETAINED_RUNTIME_DEPS_MANIFEST), + `${JSON.stringify({ specs: [...specs].toSorted((left, right) => left.localeCompare(right)) }, null, 2)}\n`, + "utf8", + ); +} + +function isWritableDirectory(dir: string): boolean { + try { + fs.accessSync(dir, fs.constants.W_OK); + return true; + } catch { + return false; + } +} + +function resolveSystemdStateDirectory(env: NodeJS.ProcessEnv): string | null { + const raw = env.STATE_DIRECTORY?.trim(); + if (!raw) { + return null; + } + const first = raw.split(path.delimiter).find((entry) => entry.trim().length > 0); + return first ? path.resolve(first) : null; +} + +function resolveBundledRuntimeDepsExternalBaseDir(env: NodeJS.ProcessEnv): string { + const explicit = env.OPENCLAW_PLUGIN_STAGE_DIR?.trim(); + if (explicit) { + return resolveHomeRelativePath(explicit, { env, homedir: os.homedir }); + } + const systemdStateDir = resolveSystemdStateDirectory(env); + if (systemdStateDir) { + return path.join(systemdStateDir, "plugin-runtime-deps"); + } + return path.join(resolveStateDir(env, os.homedir), "plugin-runtime-deps"); +} + +function resolveExternalBundledRuntimeDepsInstallRoot(params: { + pluginRoot: string; + env: NodeJS.ProcessEnv; +}): string { + const packageRoot = resolveBundledPluginPackageRoot(params.pluginRoot) ?? params.pluginRoot; + const version = sanitizePathSegment(readPackageVersion(packageRoot)); + const packageKey = `openclaw-${version}-${createPathHash(packageRoot)}`; + return path.join(resolveBundledRuntimeDepsExternalBaseDir(params.env), packageKey); +} + function resolveSourceCheckoutRuntimeDepsCacheDir(params: { pluginId: string; pluginRoot: string; @@ -483,6 +577,7 @@ export function scanBundledPluginRuntimeDeps(params: { config?: OpenClawConfig; pluginIds?: readonly string[]; includeConfiguredChannels?: boolean; + env?: NodeJS.ProcessEnv; }): { deps: RuntimeDepEntry[]; missing: RuntimeDepEntry[]; @@ -501,20 +596,96 @@ export function scanBundledPluginRuntimeDeps(params: { pluginIds: normalizePluginIdSet(params.pluginIds), includeConfiguredChannels: params.includeConfiguredChannels, }); + const packageInstallRoot = resolveBundledRuntimeDependencyPackageInstallRoot(params.packageRoot, { + env: params.env, + }); + const packageSearchRoots = [packageInstallRoot, params.packageRoot, extensionsDir]; const missing = deps.filter( (dep) => - !fs.existsSync(path.join(params.packageRoot, dependencySentinelPath(dep.name))) && - !fs.existsSync(path.join(extensionsDir, dependencySentinelPath(dep.name))) && - dep.pluginIds.every( - (pluginId) => - !fs.existsSync(path.join(extensionsDir, pluginId, dependencySentinelPath(dep.name))), - ), + !hasDependencySentinel(packageSearchRoots, dep) && + dep.pluginIds.every((pluginId) => { + const pluginRoot = path.join(extensionsDir, pluginId); + const installRoot = resolveBundledRuntimeDependencyInstallRoot(pluginRoot, { + env: params.env, + }); + return !hasDependencySentinel([installRoot, pluginRoot], dep); + }), ); return { deps, missing, conflicts }; } -export function resolveBundledRuntimeDependencyInstallRoot(pluginRoot: string): string { - return pluginRoot; +export function resolveBundledRuntimeDependencyPackageInstallRoot( + packageRoot: string, + options: { env?: NodeJS.ProcessEnv; forceExternal?: boolean } = {}, +): string { + const env = options.env ?? process.env; + if ( + options.forceExternal || + env.OPENCLAW_PLUGIN_STAGE_DIR?.trim() || + env.STATE_DIRECTORY?.trim() + ) { + return resolveExternalBundledRuntimeDepsInstallRoot({ + pluginRoot: path.join(packageRoot, "dist", "extensions", "__package__"), + env, + }); + } + return isWritableDirectory(packageRoot) + ? packageRoot + : resolveExternalBundledRuntimeDepsInstallRoot({ + pluginRoot: path.join(packageRoot, "dist", "extensions", "__package__"), + env, + }); +} + +export function resolveBundledRuntimeDependencyInstallRoot( + pluginRoot: string, + options: { env?: NodeJS.ProcessEnv; forceExternal?: boolean } = {}, +): string { + const env = options.env ?? process.env; + if ( + options.forceExternal || + env.OPENCLAW_PLUGIN_STAGE_DIR?.trim() || + env.STATE_DIRECTORY?.trim() + ) { + return resolveExternalBundledRuntimeDepsInstallRoot({ pluginRoot, env }); + } + return isWritableDirectory(pluginRoot) + ? pluginRoot + : resolveExternalBundledRuntimeDepsInstallRoot({ pluginRoot, env }); +} + +export function resolveBundledRuntimeDependencyInstallRootInfo( + pluginRoot: string, + options: { env?: NodeJS.ProcessEnv; forceExternal?: boolean } = {}, +): BundledRuntimeDepsInstallRoot { + const installRoot = resolveBundledRuntimeDependencyInstallRoot(pluginRoot, options); + return { + installRoot, + external: path.resolve(installRoot) !== path.resolve(pluginRoot), + }; +} + +export function createBundledRuntimeDependencyAliasMap(params: { + pluginRoot: string; + installRoot: string; +}): Record { + if (path.resolve(params.installRoot) === path.resolve(params.pluginRoot)) { + return {}; + } + const packageJson = readJsonObject(path.join(params.pluginRoot, "package.json")); + if (!packageJson) { + return {}; + } + const aliases: Record = {}; + for (const name of Object.keys(collectRuntimeDeps(packageJson)).toSorted((a, b) => + a.localeCompare(b), + )) { + const target = path.join(params.installRoot, "node_modules", ...name.split("/")); + if (fs.existsSync(path.join(target, "package.json"))) { + aliases[name] = target; + } + } + return aliases; } export function installBundledRuntimeDeps(params: { @@ -522,6 +693,7 @@ export function installBundledRuntimeDeps(params: { missingSpecs: string[]; env: NodeJS.ProcessEnv; }): void { + fs.mkdirSync(params.installRoot, { recursive: true }); const installEnv = createBundledRuntimeDepsInstallEnv(params.env); const npmRunner = resolveBundledRuntimeDepsNpmRunner({ env: installEnv, @@ -578,7 +750,9 @@ export function ensureBundledPluginRuntimeDeps(params: { return { installedSpecs: [], retainSpecs: [] }; } - const installRoot = resolveBundledRuntimeDependencyInstallRoot(params.pluginRoot); + const installRoot = resolveBundledRuntimeDependencyInstallRoot(params.pluginRoot, { + env: params.env, + }); const dependencySearchRoots = resolveBundledRuntimeDependencySearchRoots({ installRoot, pluginRoot: params.pluginRoot, @@ -593,9 +767,10 @@ export function ensureBundledPluginRuntimeDeps(params: { if (missingSpecs.length === 0) { return { installedSpecs: [], retainSpecs: [] }; } - const installSpecs = [...new Set([...(params.retainSpecs ?? []), ...dependencySpecs])].toSorted( - (left, right) => left.localeCompare(right), - ); + const retainedManifestSpecs = readRetainedRuntimeDepsManifest(installRoot); + const installSpecs = [ + ...new Set([...(params.retainSpecs ?? []), ...retainedManifestSpecs, ...dependencySpecs]), + ].toSorted((left, right) => left.localeCompare(right)); const cacheDir = resolveSourceCheckoutRuntimeDepsCacheDir({ pluginId: params.pluginId, pluginRoot: params.pluginRoot, @@ -620,6 +795,7 @@ export function ensureBundledPluginRuntimeDeps(params: { env: params.env, })); install({ installRoot, missingSpecs, installSpecs }); + writeRetainedRuntimeDepsManifest(installRoot, installSpecs); storeSourceCheckoutRuntimeDepsCache({ cacheDir, installRoot }); return { installedSpecs: missingSpecs, retainSpecs: installSpecs }; } diff --git a/src/plugins/loader.test-fixtures.ts b/src/plugins/loader.test-fixtures.ts index c4d47b4c232..49f53dbd507 100644 --- a/src/plugins/loader.test-fixtures.ts +++ b/src/plugins/loader.test-fixtures.ts @@ -33,6 +33,7 @@ export function mkdirSafe(dir: string) { const fixtureRoot = mkdtempSafe(path.join(os.tmpdir(), "openclaw-plugin-")); let tempDirIndex = 0; const prevBundledDir = process.env.OPENCLAW_BUNDLED_PLUGINS_DIR; +const prevPluginStageDir = process.env.OPENCLAW_PLUGIN_STAGE_DIR; export const EMPTY_PLUGIN_SCHEMA = { type: "object", @@ -143,6 +144,11 @@ export function resetPluginLoaderTestStateForTest() { } else { process.env.OPENCLAW_BUNDLED_PLUGINS_DIR = prevBundledDir; } + if (prevPluginStageDir === undefined) { + delete process.env.OPENCLAW_PLUGIN_STAGE_DIR; + } else { + process.env.OPENCLAW_PLUGIN_STAGE_DIR = prevPluginStageDir; + } } export function cleanupPluginLoaderFixturesForTest() { diff --git a/src/plugins/loader.test.ts b/src/plugins/loader.test.ts index 4f876827e95..df3a0e2e73b 100644 --- a/src/plugins/loader.test.ts +++ b/src/plugins/loader.test.ts @@ -1115,6 +1115,81 @@ module.exports = { ]); }); + it("loads bundled runtime deps from an external stage dir", () => { + const bundledDir = makeTempDir(); + const stageDir = makeTempDir(); + const plugin = writePlugin({ + id: "alpha", + dir: path.join(bundledDir, "alpha"), + filename: "index.cjs", + body: ` + const runtimeDep = require("external-runtime"); + module.exports = { + id: "alpha", + register(api) { + api.registerCommand({ name: "external-runtime", handler: () => runtimeDep.marker }); + } + }; + `, + }); + process.env.OPENCLAW_BUNDLED_PLUGINS_DIR = bundledDir; + process.env.OPENCLAW_PLUGIN_STAGE_DIR = stageDir; + fs.writeFileSync( + path.join(plugin.dir, "package.json"), + JSON.stringify( + { + name: "@openclaw/alpha", + version: "1.0.0", + dependencies: { + "external-runtime": "1.0.0", + }, + openclaw: { extensions: ["./index.cjs"] }, + }, + null, + 2, + ), + "utf-8", + ); + fs.writeFileSync( + path.join(plugin.dir, "openclaw.plugin.json"), + JSON.stringify( + { + id: "alpha", + enabledByDefault: true, + configSchema: EMPTY_PLUGIN_SCHEMA, + }, + null, + 2, + ), + "utf-8", + ); + + const registry = loadOpenClawPlugins({ + cache: false, + config: { + plugins: { + enabled: true, + }, + }, + bundledRuntimeDepsInstaller: ({ installRoot }) => { + const depRoot = path.join(installRoot, "node_modules", "external-runtime"); + fs.mkdirSync(depRoot, { recursive: true }); + fs.writeFileSync( + path.join(depRoot, "package.json"), + JSON.stringify({ name: "external-runtime", version: "1.0.0", main: "index.cjs" }), + "utf-8", + ); + fs.writeFileSync( + path.join(depRoot, "index.cjs"), + "module.exports = { marker: 'external-ok' };\n", + "utf-8", + ); + }, + }); + + expect(registry.plugins.find((entry) => entry.id === "alpha")?.status).toBe("loaded"); + }); + it("registers standalone text transforms", () => { useNoBundledPlugins(); const plugin = writePlugin({ diff --git a/src/plugins/loader.ts b/src/plugins/loader.ts index b474648dcb5..15efe858606 100644 --- a/src/plugins/loader.ts +++ b/src/plugins/loader.ts @@ -1,5 +1,6 @@ import { createHash } from "node:crypto"; import fs from "node:fs"; +import { Module } from "node:module"; import path from "node:path"; import { clearAgentHarnesses, @@ -238,6 +239,7 @@ export function clearPluginLoaderCache(): void { registryCache.clear(); inFlightPluginRegistryLoads.clear(); openAllowlistWarningCache.clear(); + clearBundledRuntimeDependencyNodePaths(); clearAgentHarnesses(); clearCompactionProviders(); clearDetachedTaskLifecycleRuntimeRegistration(); @@ -448,6 +450,151 @@ function createPluginJitiLoader(options: Pick(); + +function registerBundledRuntimeDependencyNodePath(installRoot: string): void { + const nodeModulesDir = path.join(installRoot, "node_modules"); + if (registeredBundledRuntimeDepNodePaths.has(nodeModulesDir) || !fs.existsSync(nodeModulesDir)) { + return; + } + const currentPaths = (process.env.NODE_PATH ?? "") + .split(path.delimiter) + .map((entry) => entry.trim()) + .filter((entry) => entry.length > 0); + process.env.NODE_PATH = [ + nodeModulesDir, + ...currentPaths.filter((entry) => entry !== nodeModulesDir), + ].join(path.delimiter); + (Module as unknown as { _initPaths?: () => void })._initPaths?.(); + registeredBundledRuntimeDepNodePaths.add(nodeModulesDir); +} + +function clearBundledRuntimeDependencyNodePaths(): void { + if (registeredBundledRuntimeDepNodePaths.size === 0) { + return; + } + const retainedPaths = (process.env.NODE_PATH ?? "") + .split(path.delimiter) + .filter((entry) => entry.length > 0 && !registeredBundledRuntimeDepNodePaths.has(entry)); + if (retainedPaths.length > 0) { + process.env.NODE_PATH = retainedPaths.join(path.delimiter); + } else { + delete process.env.NODE_PATH; + } + registeredBundledRuntimeDepNodePaths.clear(); + (Module as unknown as { _initPaths?: () => void })._initPaths?.(); +} + +function mirrorBundledPluginRuntimeRoot(params: { + pluginId: string; + pluginRoot: string; + installRoot: string; +}): string { + const mirrorParent = prepareBundledPluginRuntimeDistMirror({ + installRoot: params.installRoot, + pluginRoot: params.pluginRoot, + }); + const mirrorRoot = path.join(mirrorParent, params.pluginId); + fs.mkdirSync(params.installRoot, { recursive: true }); + try { + fs.chmodSync(params.installRoot, 0o755); + } catch { + // Best-effort only: staged roots may live on filesystems that reject chmod. + } + fs.mkdirSync(mirrorParent, { recursive: true }); + try { + fs.chmodSync(mirrorParent, 0o755); + } catch { + // Best-effort only: the access check below will surface non-writable dirs. + } + fs.accessSync(mirrorParent, fs.constants.W_OK); + const tempDir = fs.mkdtempSync(path.join(mirrorParent, `.plugin-${params.pluginId}-`)); + const stagedRoot = path.join(tempDir, "plugin"); + try { + copyBundledPluginRuntimeRoot(params.pluginRoot, stagedRoot); + fs.rmSync(mirrorRoot, { recursive: true, force: true }); + fs.renameSync(stagedRoot, mirrorRoot); + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + } + return mirrorRoot; +} + +function prepareBundledPluginRuntimeDistMirror(params: { + installRoot: string; + pluginRoot: string; +}): string { + const sourceExtensionsRoot = path.dirname(params.pluginRoot); + const sourceDistRoot = path.dirname(sourceExtensionsRoot); + const mirrorDistRoot = path.join(params.installRoot, "dist"); + const mirrorExtensionsRoot = path.join(mirrorDistRoot, "extensions"); + fs.mkdirSync(mirrorExtensionsRoot, { recursive: true, mode: 0o755 }); + for (const entry of fs.readdirSync(sourceDistRoot, { withFileTypes: true })) { + if (entry.name === "extensions") { + continue; + } + const sourcePath = path.join(sourceDistRoot, entry.name); + const targetPath = path.join(mirrorDistRoot, entry.name); + if (fs.existsSync(targetPath)) { + continue; + } + try { + fs.symlinkSync(sourcePath, targetPath, entry.isDirectory() ? "junction" : "file"); + } catch { + if (entry.isDirectory()) { + copyBundledPluginRuntimeRoot(sourcePath, targetPath); + } else if (entry.isFile()) { + fs.copyFileSync(sourcePath, targetPath); + } + } + } + return mirrorExtensionsRoot; +} + +function copyBundledPluginRuntimeRoot(sourceRoot: string, targetRoot: string): void { + fs.mkdirSync(targetRoot, { recursive: true, mode: 0o755 }); + for (const entry of fs.readdirSync(sourceRoot, { withFileTypes: true })) { + if (entry.name === "node_modules") { + continue; + } + const sourcePath = path.join(sourceRoot, entry.name); + const targetPath = path.join(targetRoot, entry.name); + if (entry.isDirectory()) { + copyBundledPluginRuntimeRoot(sourcePath, targetPath); + continue; + } + if (entry.isSymbolicLink()) { + fs.symlinkSync(fs.readlinkSync(sourcePath), targetPath); + continue; + } + if (!entry.isFile()) { + continue; + } + fs.copyFileSync(sourcePath, targetPath); + try { + const sourceMode = fs.statSync(sourcePath).mode; + fs.chmodSync(targetPath, sourceMode | 0o600); + } catch { + // Readable copied files are enough for plugin loading. + } + } +} + +function remapBundledPluginRuntimePath(params: { + source: string | undefined; + pluginRoot: string; + mirroredRoot: string; +}): string | undefined { + if (!params.source) { + return undefined; + } + const relative = path.relative(params.pluginRoot, params.source); + if (relative.startsWith("..") || path.isAbsolute(relative)) { + return params.source; + } + return path.join(params.mirroredRoot, relative); +} + export const __testing = { buildPluginLoaderJitiOptions, buildPluginLoaderAliasMap, @@ -1742,10 +1889,13 @@ export function loadOpenClawPlugins(options: PluginLoadOptions = {}): PluginRegi }); }; const pluginRoot = safeRealpathOrResolve(candidate.rootDir); + let runtimePluginRoot = pluginRoot; + let runtimeCandidateSource = candidate.source; + let runtimeSetupSource = manifestRecord.setupSource; if (shouldLoadModules && candidate.origin === "bundled" && enableState.enabled) { try { - const installRoot = resolveBundledRuntimeDependencyInstallRoot(pluginRoot); + const installRoot = resolveBundledRuntimeDependencyInstallRoot(pluginRoot, { env }); const retainSpecs = bundledRuntimeDepsRetainSpecsByInstallRoot.get(installRoot) ?? []; const depsInstallResult = ensureBundledPluginRuntimeDeps({ pluginId: record.id, @@ -1766,6 +1916,25 @@ export function loadOpenClawPlugins(options: PluginLoadOptions = {}): PluginRegi `[plugins] ${record.id} installed bundled runtime deps: ${depsInstallResult.installedSpecs.join(", ")}`, ); } + if (path.resolve(installRoot) !== path.resolve(pluginRoot)) { + registerBundledRuntimeDependencyNodePath(installRoot); + runtimePluginRoot = mirrorBundledPluginRuntimeRoot({ + pluginId: record.id, + pluginRoot, + installRoot, + }); + runtimeCandidateSource = + remapBundledPluginRuntimePath({ + source: candidate.source, + pluginRoot, + mirroredRoot: runtimePluginRoot, + }) ?? candidate.source; + runtimeSetupSource = remapBundledPluginRuntimePath({ + source: manifestRecord.setupSource, + pluginRoot, + mirroredRoot: runtimePluginRoot, + }); + } } catch (error) { pushPluginLoadError(`failed to install bundled runtime deps: ${String(error)}`); continue; @@ -1955,12 +2124,12 @@ export function loadOpenClawPlugins(options: PluginLoadOptions = {}): PluginRegi const loadSource = (registrationMode === "setup-only" || registrationMode === "setup-runtime") && - manifestRecord.setupSource - ? manifestRecord.setupSource - : candidate.source; + runtimeSetupSource + ? runtimeSetupSource + : runtimeCandidateSource; const opened = openBoundaryFileSync({ absolutePath: loadSource, - rootPath: pluginRoot, + rootPath: runtimePluginRoot, boundaryLabel: "plugin root", rejectHardlinks: candidate.origin !== "bundled", skipLexicalRootCheck: true, @@ -2043,11 +2212,11 @@ export function loadOpenClawPlugins(options: PluginLoadOptions = {}): PluginRegi if ( registrationMode === "setup-runtime" && setupRegistration.usesBundledSetupContract && - candidate.source !== safeSource + runtimeCandidateSource !== safeSource ) { const runtimeOpened = openBoundaryFileSync({ - absolutePath: candidate.source, - rootPath: pluginRoot, + absolutePath: runtimeCandidateSource, + rootPath: runtimePluginRoot, boundaryLabel: "plugin root", rejectHardlinks: candidate.origin !== "bundled", skipLexicalRootCheck: true,