From 19cb9ca6bf1db855f055e5805d400e9ee9000a62 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 27 Apr 2026 07:08:22 +0100 Subject: [PATCH] fix: materialize staged plugin runtime chunks --- CHANGELOG.md | 2 +- src/plugins/bundled-runtime-deps.ts | 33 ++++++ src/plugins/bundled-runtime-root.test.ts | 98 ++++++++++++++++++ src/plugins/bundled-runtime-root.ts | 6 ++ src/plugins/loader.test.ts | 125 ++++++++++++++++++++++- src/plugins/loader.ts | 103 +++++++++++++++---- 6 files changed, 342 insertions(+), 25 deletions(-) create mode 100644 src/plugins/bundled-runtime-root.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 421fe064c50..3a49b02b375 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,7 @@ Docs: https://docs.openclaw.ai ### Fixes - macOS Gateway: detect installed-but-unloaded LaunchAgent split-brain states during status, doctor, and restart, and re-bootstrap launchd supervision before falling back to unmanaged listener restarts. Fixes #67335, #53475, and #71060; refs #58890, #60885, and #70801. Thanks @ze1tgeist88, @dafacto, and @vishutdhar. -- Plugins/install: stage bundled plugin runtime dependencies before Gateway startup and drain update restarts while preserving per-plugin isolation when pre-stage scan or install fails. Thanks @codex. +- Plugins/install: stage bundled plugin runtime dependencies before Gateway startup, drain update restarts, and materialize plugin-owned root chunks in external mirrors so staged deps resolve under native ESM. Fixes #72058; supersedes #72084. Thanks @amnesia106 and @drvoss. - TTS/SecretRef: resolve `messages.tts.providers.*.apiKey` from the active runtime snapshot so SecretRef-backed MiniMax and other TTS provider keys work in runtime reply/audio paths. Fixes #68690. Thanks @joshavant. - CLI/startup: read generated startup metadata from the bundled `dist` layout before falling back to live help rendering, so root/browser help and channel-option bootstrap stay on the fast path. Thanks @vincentkoc. - CLI/help: treat positional `help` invocations like `openclaw channels help` as help paths for startup gating, avoiding model/auth warmup while preserving positional arguments such as `openclaw docs help`. Thanks @gumadeiras. diff --git a/src/plugins/bundled-runtime-deps.ts b/src/plugins/bundled-runtime-deps.ts index e79cb58be71..101bf068e82 100644 --- a/src/plugins/bundled-runtime-deps.ts +++ b/src/plugins/bundled-runtime-deps.ts @@ -61,6 +61,8 @@ const BUNDLED_RUNTIME_DEPS_LOCK_WAIT_MS = 100; const BUNDLED_RUNTIME_DEPS_LOCK_TIMEOUT_MS = 5 * 60_000; const BUNDLED_RUNTIME_DEPS_LOCK_STALE_MS = 10 * 60_000; const BUNDLED_RUNTIME_DEPS_OWNERLESS_LOCK_STALE_MS = 30_000; +const BUNDLED_RUNTIME_MIRROR_MATERIALIZED_EXTENSIONS = new Set([".cjs", ".js", ".mjs"]); +const BUNDLED_RUNTIME_MIRROR_PLUGIN_REGION_RE = /(?:^|\n)\/\/#region extensions\/[^/\s]+(?:\/|$)/u; const registeredBundledRuntimeDepNodePaths = new Set(); @@ -70,6 +72,37 @@ export type BundledRuntimeDepsNpmRunner = { env?: NodeJS.ProcessEnv; }; +export function shouldMaterializeBundledRuntimeMirrorDistFile(sourcePath: string): boolean { + if (!BUNDLED_RUNTIME_MIRROR_MATERIALIZED_EXTENSIONS.has(path.extname(sourcePath))) { + return false; + } + try { + return BUNDLED_RUNTIME_MIRROR_PLUGIN_REGION_RE.test(fs.readFileSync(sourcePath, "utf8")); + } catch { + return false; + } +} + +export function materializeBundledRuntimeMirrorDistFile( + sourcePath: string, + targetPath: string, +): void { + fs.mkdirSync(path.dirname(targetPath), { recursive: true, mode: 0o755 }); + fs.rmSync(targetPath, { recursive: true, force: true }); + try { + fs.linkSync(sourcePath, targetPath); + return; + } catch { + fs.copyFileSync(sourcePath, targetPath); + } + try { + const sourceMode = fs.statSync(sourcePath).mode; + fs.chmodSync(targetPath, sourceMode | 0o600); + } catch { + // Readable materialized chunks are enough for ESM loading. + } +} + const BUNDLED_RUNTIME_DEP_SEGMENT_RE = /^[a-z0-9][a-z0-9._-]*$/; function normalizeInstallableRuntimeDepName(rawName: string): string | null { diff --git a/src/plugins/bundled-runtime-root.test.ts b/src/plugins/bundled-runtime-root.test.ts new file mode 100644 index 00000000000..2d59e3ccd44 --- /dev/null +++ b/src/plugins/bundled-runtime-root.test.ts @@ -0,0 +1,98 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { resolveBundledRuntimeDependencyInstallRoot } from "./bundled-runtime-deps.js"; +import { prepareBundledPluginRuntimeRoot } from "./bundled-runtime-root.js"; + +const tempRoots: string[] = []; + +function makeTempRoot(): string { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-bundled-runtime-root-")); + tempRoots.push(root); + return root; +} + +afterEach(() => { + for (const root of tempRoots.splice(0)) { + fs.rmSync(root, { recursive: true, force: true }); + } +}); + +describe("prepareBundledPluginRuntimeRoot", () => { + it("materializes plugin-owned root chunks in external mirrors", () => { + const packageRoot = makeTempRoot(); + const stageDir = makeTempRoot(); + const pluginRoot = path.join(packageRoot, "dist", "extensions", "browser"); + const env = { ...process.env, OPENCLAW_PLUGIN_STAGE_DIR: stageDir }; + fs.mkdirSync(pluginRoot, { recursive: true }); + fs.writeFileSync( + path.join(packageRoot, "package.json"), + JSON.stringify({ name: "openclaw", version: "2026.4.24", type: "module" }), + "utf8", + ); + fs.writeFileSync( + path.join(packageRoot, "dist", "pw-ai.js"), + [ + `//#region extensions/browser/src/pw-ai.ts`, + `import { marker } from "playwright-core";`, + `export { marker };`, + `//#endregion`, + "", + ].join("\n"), + "utf8", + ); + fs.writeFileSync( + path.join(pluginRoot, "index.js"), + `import { marker } from "../../pw-ai.js"; export default { id: "browser", marker };\n`, + "utf8", + ); + fs.writeFileSync( + path.join(pluginRoot, "package.json"), + JSON.stringify( + { + name: "@openclaw/browser", + version: "1.0.0", + type: "module", + dependencies: { + "playwright-core": "1.0.0", + }, + openclaw: { extensions: ["./index.js"] }, + }, + null, + 2, + ), + "utf8", + ); + + const installRoot = resolveBundledRuntimeDependencyInstallRoot(pluginRoot, { env }); + const depRoot = path.join(installRoot, "node_modules", "playwright-core"); + fs.mkdirSync(depRoot, { recursive: true }); + fs.writeFileSync( + path.join(depRoot, "package.json"), + JSON.stringify({ + name: "playwright-core", + version: "1.0.0", + type: "module", + exports: "./index.js", + }), + "utf8", + ); + fs.writeFileSync(path.join(depRoot, "index.js"), "export const marker = 'stage-ok';\n", "utf8"); + + const staleMirrorChunk = path.join(installRoot, "dist", "pw-ai.js"); + fs.mkdirSync(path.dirname(staleMirrorChunk), { recursive: true }); + fs.symlinkSync(path.join(packageRoot, "dist", "pw-ai.js"), staleMirrorChunk, "file"); + + const prepared = prepareBundledPluginRuntimeRoot({ + pluginId: "browser", + pluginRoot, + modulePath: path.join(pluginRoot, "index.js"), + env, + }); + + expect(prepared.pluginRoot).toBe(path.join(installRoot, "dist", "extensions", "browser")); + expect(prepared.modulePath).toBe(path.join(prepared.pluginRoot, "index.js")); + expect(fs.lstatSync(staleMirrorChunk).isSymbolicLink()).toBe(false); + }); +}); diff --git a/src/plugins/bundled-runtime-root.ts b/src/plugins/bundled-runtime-root.ts index a042744d33e..354b7bd179a 100644 --- a/src/plugins/bundled-runtime-root.ts +++ b/src/plugins/bundled-runtime-root.ts @@ -2,9 +2,11 @@ import fs from "node:fs"; import path from "node:path"; import { ensureBundledPluginRuntimeDeps, + materializeBundledRuntimeMirrorDistFile, resolveBundledRuntimeDependencyInstallRoot, resolveBundledRuntimeDependencyPackageRoot, registerBundledRuntimeDependencyNodePath, + shouldMaterializeBundledRuntimeMirrorDistFile, withBundledRuntimeDepsFilesystemLock, } from "./bundled-runtime-deps.js"; @@ -137,6 +139,10 @@ function prepareBundledPluginRuntimeDistMirror(params: { } const sourcePath = path.join(sourceDistRoot, entry.name); const targetPath = path.join(mirrorDistRoot, entry.name); + if (entry.isFile() && shouldMaterializeBundledRuntimeMirrorDistFile(sourcePath)) { + materializeBundledRuntimeMirrorDistFile(sourcePath, targetPath); + continue; + } if (fs.existsSync(targetPath)) { continue; } diff --git a/src/plugins/loader.test.ts b/src/plugins/loader.test.ts index 2d89d9b37b3..1cbadd65317 100644 --- a/src/plugins/loader.test.ts +++ b/src/plugins/loader.test.ts @@ -1714,6 +1714,110 @@ module.exports = { expect(registry?.plugins.find((entry) => entry.id === "alpha")?.status).toBe("loaded"); }); + it("materializes plugin-owned root chunks in external runtime mirrors", () => { + const packageRoot = makeTempDir(); + const stageDir = makeTempDir(); + const bundledDir = path.join(packageRoot, "dist", "extensions"); + const pluginRoot = path.join(bundledDir, "browser"); + fs.mkdirSync(pluginRoot, { recursive: true }); + fs.writeFileSync( + path.join(packageRoot, "package.json"), + JSON.stringify({ name: "openclaw", version: "2026.4.24", type: "module" }), + "utf-8", + ); + fs.writeFileSync( + path.join(packageRoot, "dist", "pw-ai.js"), + [ + `//#region extensions/browser/src/pw-ai.ts`, + `import { marker } from "playwright-core";`, + `export { marker };`, + `//#endregion`, + "", + ].join("\n"), + "utf-8", + ); + fs.writeFileSync( + path.join(pluginRoot, "index.js"), + [ + `import { marker } from "../../pw-ai.js";`, + `export default {`, + ` id: "browser",`, + ` register(api) {`, + ` api.registerCommand({ name: "browser-marker", handler: () => marker });`, + ` },`, + `};`, + "", + ].join("\n"), + "utf-8", + ); + fs.writeFileSync( + path.join(pluginRoot, "package.json"), + JSON.stringify( + { + name: "@openclaw/browser", + version: "1.0.0", + type: "module", + dependencies: { + "playwright-core": "1.0.0", + }, + openclaw: { extensions: ["./index.js"] }, + }, + null, + 2, + ), + "utf-8", + ); + fs.writeFileSync( + path.join(pluginRoot, "openclaw.plugin.json"), + JSON.stringify( + { + id: "browser", + enabledByDefault: true, + configSchema: EMPTY_PLUGIN_SCHEMA, + }, + null, + 2, + ), + "utf-8", + ); + process.env.OPENCLAW_BUNDLED_PLUGINS_DIR = bundledDir; + process.env.OPENCLAW_PLUGIN_STAGE_DIR = stageDir; + + let actualInstallRoot = ""; + let stagedMirrorChunk = ""; + const registry = loadOpenClawPlugins({ + cache: false, + config: { + plugins: { + enabled: true, + }, + }, + bundledRuntimeDepsInstaller: ({ installRoot }) => { + actualInstallRoot = installRoot; + stagedMirrorChunk = path.join(installRoot, "dist", "pw-ai.js"); + fs.mkdirSync(path.dirname(stagedMirrorChunk), { recursive: true }); + fs.symlinkSync(path.join(packageRoot, "dist", "pw-ai.js"), stagedMirrorChunk, "file"); + const depRoot = path.join(installRoot, "node_modules", "playwright-core"); + fs.mkdirSync(depRoot, { recursive: true }); + fs.writeFileSync( + path.join(depRoot, "package.json"), + JSON.stringify({ + name: "playwright-core", + version: "1.0.0", + type: "module", + exports: "./index.js", + }), + "utf-8", + ); + fs.writeFileSync(path.join(depRoot, "index.js"), "export const marker = 'stage-ok';\n"); + }, + }); + + expect(actualInstallRoot).not.toBe(""); + expect(registry.plugins.find((entry) => entry.id === "browser")?.status).toBe("loaded"); + expect(fs.lstatSync(stagedMirrorChunk).isSymbolicLink()).toBe(false); + }); + it("loads bundled plugins with plugin-sdk imports from an external stage dir", () => { const packageRoot = makeTempDir(); const stageDir = makeTempDir(); @@ -1913,6 +2017,17 @@ module.exports = { ); fs.mkdirSync(pluginRoot, { recursive: true }); fs.mkdirSync(canonicalPluginRoot, { recursive: true }); + fs.writeFileSync( + path.join(packageRoot, "dist", "pw-ai.js"), + [ + `//#region extensions/acpx/src/pw-ai.ts`, + `import runtimeDep from "external-runtime";`, + `export const marker = runtimeDep.marker;`, + `//#endregion`, + "", + ].join("\n"), + "utf-8", + ); fs.writeFileSync( path.join(pluginRoot, "index.js"), [ @@ -1926,11 +2041,11 @@ module.exports = { fs.writeFileSync( path.join(canonicalPluginRoot, "index.js"), [ - `import runtimeDep from "external-runtime";`, + `import { marker } from "../../pw-ai.js";`, `export default {`, ` id: "acpx",`, ` register(api) {`, - ` api.registerCommand({ name: "external-runtime", handler: () => runtimeDep.marker });`, + ` api.registerCommand({ name: "external-runtime", handler: () => marker });`, ` },`, `};`, "", @@ -1970,6 +2085,7 @@ module.exports = { "utf-8", ); + let actualInstallRoot = ""; const registry = loadOpenClawPlugins({ cache: false, config: { @@ -1978,6 +2094,7 @@ module.exports = { }, }, bundledRuntimeDepsInstaller: ({ installRoot }) => { + actualInstallRoot = installRoot; const depRoot = path.join(installRoot, "node_modules", "external-runtime"); fs.mkdirSync(depRoot, { recursive: true }); fs.writeFileSync( @@ -1999,6 +2116,10 @@ module.exports = { }); expect(registry.plugins.find((entry) => entry.id === "acpx")?.status).toBe("loaded"); + expect(fs.lstatSync(path.join(actualInstallRoot, "dist")).isSymbolicLink()).toBe(false); + expect(fs.lstatSync(path.join(actualInstallRoot, "dist", "pw-ai.js")).isSymbolicLink()).toBe( + false, + ); }); it("loads source-checkout bundled runtime deps without mirroring the repo tree", () => { diff --git a/src/plugins/loader.ts b/src/plugins/loader.ts index 61a1f49782e..9e0f4e14abd 100644 --- a/src/plugins/loader.ts +++ b/src/plugins/loader.ts @@ -35,9 +35,11 @@ import { clearBundledRuntimeDependencyNodePaths, ensureBundledPluginRuntimeDeps, installBundledRuntimeDeps, + materializeBundledRuntimeMirrorDistFile, resolveBundledRuntimeDependencyInstallRoot, resolveBundledRuntimeDependencyPackageRoot, registerBundledRuntimeDependencyNodePath, + shouldMaterializeBundledRuntimeMirrorDistFile, withBundledRuntimeDepsFilesystemLock, type BundledRuntimeDepsInstallParams, } from "./bundled-runtime-deps.js"; @@ -743,14 +745,53 @@ function prepareBundledPluginRuntimeDistMirror(params: { const sourceDistRootName = path.basename(sourceDistRoot); const mirrorDistRoot = path.join(params.installRoot, sourceDistRootName); const mirrorExtensionsRoot = path.join(mirrorDistRoot, "extensions"); + ensureBundledRuntimeMirrorDirectory(mirrorDistRoot); fs.mkdirSync(mirrorExtensionsRoot, { recursive: true, mode: 0o755 }); ensureBundledRuntimeDistPackageJson(mirrorDistRoot); - for (const entry of fs.readdirSync(sourceDistRoot, { withFileTypes: true })) { + mirrorBundledRuntimeDistRootEntries({ + sourceDistRoot, + mirrorDistRoot, + }); + if (sourceDistRootName === "dist-runtime") { + mirrorCanonicalBundledRuntimeDistRoot({ + installRoot: params.installRoot, + pluginRoot: params.pluginRoot, + sourceRuntimeDistRoot: sourceDistRoot, + }); + } + ensureOpenClawPluginSdkAlias(mirrorDistRoot); + return mirrorExtensionsRoot; +} + +function ensureBundledRuntimeMirrorDirectory(targetRoot: string): void { + try { + const stat = fs.lstatSync(targetRoot); + if (stat.isDirectory() && !stat.isSymbolicLink()) { + return; + } + fs.rmSync(targetRoot, { recursive: true, force: true }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") { + throw error; + } + } + fs.mkdirSync(targetRoot, { recursive: true, mode: 0o755 }); +} + +function mirrorBundledRuntimeDistRootEntries(params: { + sourceDistRoot: string; + mirrorDistRoot: string; +}): void { + for (const entry of fs.readdirSync(params.sourceDistRoot, { withFileTypes: true })) { if (entry.name === "extensions") { continue; } - const sourcePath = path.join(sourceDistRoot, entry.name); - const targetPath = path.join(mirrorDistRoot, entry.name); + const sourcePath = path.join(params.sourceDistRoot, entry.name); + const targetPath = path.join(params.mirrorDistRoot, entry.name); + if (entry.isFile() && shouldMaterializeBundledRuntimeMirrorDistFile(sourcePath)) { + materializeBundledRuntimeMirrorDistFile(sourcePath, targetPath); + continue; + } if (fs.existsSync(targetPath)) { continue; } @@ -767,26 +808,44 @@ function prepareBundledPluginRuntimeDistMirror(params: { } } } - if (sourceDistRootName === "dist-runtime") { - const sourceCanonicalDistRoot = path.join(path.dirname(sourceDistRoot), "dist"); - const targetCanonicalDistRoot = path.join(params.installRoot, "dist"); - if (fs.existsSync(sourceCanonicalDistRoot)) { - const targetMatchesSource = - fs.existsSync(targetCanonicalDistRoot) && - safeRealpathOrResolve(targetCanonicalDistRoot) === - safeRealpathOrResolve(sourceCanonicalDistRoot); - if (!targetMatchesSource) { - fs.rmSync(targetCanonicalDistRoot, { recursive: true, force: true }); - try { - fs.symlinkSync(sourceCanonicalDistRoot, targetCanonicalDistRoot, "junction"); - } catch { - copyBundledPluginRuntimeRoot(sourceCanonicalDistRoot, targetCanonicalDistRoot); - } - } - } +} + +function mirrorCanonicalBundledRuntimeDistRoot(params: { + installRoot: string; + pluginRoot: string; + sourceRuntimeDistRoot: string; +}): void { + const sourceCanonicalDistRoot = path.join(path.dirname(params.sourceRuntimeDistRoot), "dist"); + if (!fs.existsSync(sourceCanonicalDistRoot)) { + return; + } + const targetCanonicalDistRoot = path.join(params.installRoot, "dist"); + ensureBundledRuntimeMirrorDirectory(targetCanonicalDistRoot); + fs.mkdirSync(path.join(targetCanonicalDistRoot, "extensions"), { recursive: true, mode: 0o755 }); + ensureBundledRuntimeDistPackageJson(targetCanonicalDistRoot); + mirrorBundledRuntimeDistRootEntries({ + sourceDistRoot: sourceCanonicalDistRoot, + mirrorDistRoot: targetCanonicalDistRoot, + }); + ensureOpenClawPluginSdkAlias(targetCanonicalDistRoot); + + const pluginId = path.basename(params.pluginRoot); + const sourceCanonicalPluginRoot = path.join(sourceCanonicalDistRoot, "extensions", pluginId); + if (!fs.existsSync(sourceCanonicalPluginRoot)) { + return; + } + const targetCanonicalPluginRoot = path.join(targetCanonicalDistRoot, "extensions", pluginId); + const tempDir = fs.mkdtempSync( + path.join(path.dirname(targetCanonicalPluginRoot), `.plugin-${pluginId}-`), + ); + const stagedRoot = path.join(tempDir, "plugin"); + try { + copyBundledPluginRuntimeRoot(sourceCanonicalPluginRoot, stagedRoot); + fs.rmSync(targetCanonicalPluginRoot, { recursive: true, force: true }); + fs.renameSync(stagedRoot, targetCanonicalPluginRoot); + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }); } - ensureOpenClawPluginSdkAlias(mirrorDistRoot); - return mirrorExtensionsRoot; } function ensureBundledRuntimeDistPackageJson(mirrorDistRoot: string): void {