From 09df232f3937c044d6b2a6ac7fc9fd5c69441d0f Mon Sep 17 00:00:00 2001 From: Gustavo Madeira Santana Date: Mon, 16 Mar 2026 16:43:23 +0000 Subject: [PATCH] Plugins: stage local bundled runtime tree --- .gitignore | 1 + scripts/release-check.ts | 2 +- scripts/runtime-postbuild.mjs | 2 + scripts/stage-bundled-plugin-runtime.d.mts | 1 + scripts/stage-bundled-plugin-runtime.mjs | 100 +++++++++++++++++ src/plugins/bundled-dir.test.ts | 15 +-- src/plugins/bundled-dir.ts | 27 ++--- .../stage-bundled-plugin-runtime.test.ts | 101 ++++++++++++++++++ 8 files changed, 225 insertions(+), 24 deletions(-) create mode 100644 scripts/stage-bundled-plugin-runtime.d.mts create mode 100644 scripts/stage-bundled-plugin-runtime.mjs create mode 100644 src/plugins/stage-bundled-plugin-runtime.test.ts diff --git a/.gitignore b/.gitignore index 0eabcb6843c..a0da79d14ef 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ node_modules docker-compose.override.yml docker-compose.extra.yml dist +dist-runtime pnpm-lock.yaml bun.lock bun.lockb diff --git a/scripts/release-check.ts b/scripts/release-check.ts index 7eedc970103..9b67303b4a6 100755 --- a/scripts/release-check.ts +++ b/scripts/release-check.ts @@ -25,7 +25,7 @@ const requiredPathGroups = [ "dist/plugin-sdk/root-alias.cjs", "dist/build-info.json", ]; -const forbiddenPrefixes = ["dist/OpenClaw.app/"]; +const forbiddenPrefixes = ["dist-runtime/", "dist/OpenClaw.app/"]; // 2026.3.12 ballooned to ~213.6 MiB unpacked and correlated with low-memory // startup/doctor OOM reports. Keep enough headroom for the current pack while // failing fast if duplicate/shim content sneaks back into the release artifact. diff --git a/scripts/runtime-postbuild.mjs b/scripts/runtime-postbuild.mjs index 884ba7af036..32dc6a31171 100644 --- a/scripts/runtime-postbuild.mjs +++ b/scripts/runtime-postbuild.mjs @@ -1,10 +1,12 @@ import { pathToFileURL } from "node:url"; import { copyBundledPluginMetadata } from "./copy-bundled-plugin-metadata.mjs"; import { copyPluginSdkRootAlias } from "./copy-plugin-sdk-root-alias.mjs"; +import { stageBundledPluginRuntime } from "./stage-bundled-plugin-runtime.mjs"; export function runRuntimePostBuild(params = {}) { copyPluginSdkRootAlias(params); copyBundledPluginMetadata(params); + stageBundledPluginRuntime(params); } if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { diff --git a/scripts/stage-bundled-plugin-runtime.d.mts b/scripts/stage-bundled-plugin-runtime.d.mts new file mode 100644 index 00000000000..718cac12a8e --- /dev/null +++ b/scripts/stage-bundled-plugin-runtime.d.mts @@ -0,0 +1 @@ +export function stageBundledPluginRuntime(params?: { cwd?: string; repoRoot?: string }): void; diff --git a/scripts/stage-bundled-plugin-runtime.mjs b/scripts/stage-bundled-plugin-runtime.mjs new file mode 100644 index 00000000000..134c76699c9 --- /dev/null +++ b/scripts/stage-bundled-plugin-runtime.mjs @@ -0,0 +1,100 @@ +import fs from "node:fs"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { removePathIfExists } from "./runtime-postbuild-shared.mjs"; + +function linkOrCopyFile(sourcePath, targetPath) { + try { + fs.linkSync(sourcePath, targetPath); + } catch (error) { + if (error && typeof error === "object" && "code" in error) { + const code = error.code; + if (code === "EXDEV" || code === "EPERM" || code === "EMLINK") { + fs.copyFileSync(sourcePath, targetPath); + return; + } + } + throw error; + } +} + +function mirrorTreeWithHardlinks(sourceRoot, targetRoot) { + fs.mkdirSync(targetRoot, { recursive: true }); + const queue = [{ sourceDir: sourceRoot, targetDir: targetRoot }]; + + while (queue.length > 0) { + const current = queue.pop(); + if (!current) { + continue; + } + + for (const dirent of fs.readdirSync(current.sourceDir, { withFileTypes: true })) { + const sourcePath = path.join(current.sourceDir, dirent.name); + const targetPath = path.join(current.targetDir, dirent.name); + + if (dirent.isDirectory()) { + fs.mkdirSync(targetPath, { recursive: true }); + queue.push({ sourceDir: sourcePath, targetDir: targetPath }); + continue; + } + + if (dirent.isSymbolicLink()) { + fs.symlinkSync(fs.readlinkSync(sourcePath), targetPath); + continue; + } + + if (!dirent.isFile()) { + continue; + } + + linkOrCopyFile(sourcePath, targetPath); + } + } +} + +function symlinkType() { + return process.platform === "win32" ? "junction" : "dir"; +} + +function linkPluginNodeModules(params) { + const runtimeNodeModulesDir = path.join(params.runtimePluginDir, "node_modules"); + removePathIfExists(runtimeNodeModulesDir); + if (!fs.existsSync(params.sourcePluginNodeModulesDir)) { + return; + } + fs.symlinkSync(params.sourcePluginNodeModulesDir, runtimeNodeModulesDir, symlinkType()); +} + +export function stageBundledPluginRuntime(params = {}) { + const repoRoot = params.cwd ?? params.repoRoot ?? process.cwd(); + const distRoot = path.join(repoRoot, "dist"); + const runtimeRoot = path.join(repoRoot, "dist-runtime"); + const sourceExtensionsRoot = path.join(repoRoot, "extensions"); + const distExtensionsRoot = path.join(distRoot, "extensions"); + const runtimeExtensionsRoot = path.join(runtimeRoot, "extensions"); + + if (!fs.existsSync(distExtensionsRoot)) { + removePathIfExists(runtimeRoot); + return; + } + + removePathIfExists(runtimeRoot); + mirrorTreeWithHardlinks(distRoot, runtimeRoot); + + for (const dirent of fs.readdirSync(distExtensionsRoot, { withFileTypes: true })) { + if (!dirent.isDirectory()) { + continue; + } + const runtimePluginDir = path.join(runtimeExtensionsRoot, dirent.name); + const sourcePluginNodeModulesDir = path.join(sourceExtensionsRoot, dirent.name, "node_modules"); + + linkPluginNodeModules({ + runtimePluginDir, + sourcePluginNodeModulesDir, + }); + } +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { + stageBundledPluginRuntime(); +} diff --git a/src/plugins/bundled-dir.test.ts b/src/plugins/bundled-dir.test.ts index 8bc1fc9cf76..93f53acaf75 100644 --- a/src/plugins/bundled-dir.test.ts +++ b/src/plugins/bundled-dir.test.ts @@ -7,7 +7,6 @@ import { resolveBundledPluginsDir } from "./bundled-dir.js"; const tempDirs: string[] = []; const originalCwd = process.cwd(); const originalBundledDir = process.env.OPENCLAW_BUNDLED_PLUGINS_DIR; -const originalWatchMode = process.env.OPENCLAW_WATCH_MODE; function makeRepoRoot(prefix: string): string { const repoRoot = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); @@ -22,20 +21,15 @@ afterEach(() => { } else { process.env.OPENCLAW_BUNDLED_PLUGINS_DIR = originalBundledDir; } - if (originalWatchMode === undefined) { - delete process.env.OPENCLAW_WATCH_MODE; - } else { - process.env.OPENCLAW_WATCH_MODE = originalWatchMode; - } for (const dir of tempDirs.splice(0, tempDirs.length)) { fs.rmSync(dir, { recursive: true, force: true }); } }); describe("resolveBundledPluginsDir", () => { - it("prefers source extensions from the package root in watch mode", () => { - const repoRoot = makeRepoRoot("openclaw-bundled-dir-watch-"); - fs.mkdirSync(path.join(repoRoot, "extensions"), { recursive: true }); + it("prefers the staged runtime bundled plugin tree from the package root", () => { + const repoRoot = makeRepoRoot("openclaw-bundled-dir-runtime-"); + fs.mkdirSync(path.join(repoRoot, "dist-runtime", "extensions"), { recursive: true }); fs.mkdirSync(path.join(repoRoot, "dist", "extensions"), { recursive: true }); fs.writeFileSync( path.join(repoRoot, "package.json"), @@ -44,10 +38,9 @@ describe("resolveBundledPluginsDir", () => { ); process.chdir(repoRoot); - process.env.OPENCLAW_WATCH_MODE = "1"; expect(fs.realpathSync(resolveBundledPluginsDir() ?? "")).toBe( - fs.realpathSync(path.join(repoRoot, "extensions")), + fs.realpathSync(path.join(repoRoot, "dist-runtime", "extensions")), ); }); }); diff --git a/src/plugins/bundled-dir.ts b/src/plugins/bundled-dir.ts index 7fa25092f42..b69da702a7e 100644 --- a/src/plugins/bundled-dir.ts +++ b/src/plugins/bundled-dir.ts @@ -10,20 +10,23 @@ export function resolveBundledPluginsDir(env: NodeJS.ProcessEnv = process.env): return resolveUserPath(override, env); } - if (env.OPENCLAW_WATCH_MODE === "1") { - try { - const packageRoot = resolveOpenClawPackageRootSync({ cwd: process.cwd() }); - if (packageRoot) { - // In watch mode, prefer source plugin roots so plugin-local runtime deps - // resolve from extensions//node_modules instead of stripped dist copies. - const sourceExtensionsDir = path.join(packageRoot, "extensions"); - if (fs.existsSync(sourceExtensionsDir)) { - return sourceExtensionsDir; - } + try { + const packageRoots = [ + resolveOpenClawPackageRootSync({ cwd: process.cwd() }), + resolveOpenClawPackageRootSync({ moduleUrl: import.meta.url }), + ].filter( + (entry, index, all): entry is string => Boolean(entry) && all.indexOf(entry) === index, + ); + for (const packageRoot of packageRoots) { + // Local source checkouts stage a runtime-complete bundled plugin tree under + // dist-runtime/. Prefer that over release-shaped dist/extensions. + const runtimeExtensionsDir = path.join(packageRoot, "dist-runtime", "extensions"); + if (fs.existsSync(runtimeExtensionsDir)) { + return runtimeExtensionsDir; } - } catch { - // ignore } + } catch { + // ignore } // bun --compile: ship a sibling `extensions/` next to the executable. diff --git a/src/plugins/stage-bundled-plugin-runtime.test.ts b/src/plugins/stage-bundled-plugin-runtime.test.ts new file mode 100644 index 00000000000..f96a2408c6a --- /dev/null +++ b/src/plugins/stage-bundled-plugin-runtime.test.ts @@ -0,0 +1,101 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { stageBundledPluginRuntime } from "../../scripts/stage-bundled-plugin-runtime.mjs"; + +const tempDirs: string[] = []; + +function makeRepoRoot(prefix: string): string { + const repoRoot = fs.mkdtempSync(path.join(os.tmpdir(), prefix)); + tempDirs.push(repoRoot); + return repoRoot; +} + +afterEach(() => { + for (const dir of tempDirs.splice(0, tempDirs.length)) { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +describe("stageBundledPluginRuntime", () => { + it("hard-links bundled dist plugins into dist-runtime and links plugin-local node_modules", () => { + const repoRoot = makeRepoRoot("openclaw-stage-bundled-runtime-"); + const distPluginDir = path.join(repoRoot, "dist", "extensions", "diffs"); + fs.mkdirSync(path.join(repoRoot, "dist"), { recursive: true }); + const sourcePluginNodeModulesDir = path.join(repoRoot, "extensions", "diffs", "node_modules"); + fs.mkdirSync(distPluginDir, { recursive: true }); + fs.mkdirSync(path.join(sourcePluginNodeModulesDir, "@pierre", "diffs"), { + recursive: true, + }); + fs.writeFileSync(path.join(distPluginDir, "index.js"), "export default {}\n", "utf8"); + fs.writeFileSync( + path.join(sourcePluginNodeModulesDir, "@pierre", "diffs", "index.js"), + "export default {}\n", + "utf8", + ); + + stageBundledPluginRuntime({ repoRoot }); + + const runtimePluginDir = path.join(repoRoot, "dist-runtime", "extensions", "diffs"); + expect(fs.existsSync(path.join(runtimePluginDir, "index.js"))).toBe(true); + expect(fs.statSync(path.join(runtimePluginDir, "index.js")).nlink).toBeGreaterThan(1); + expect(fs.lstatSync(path.join(runtimePluginDir, "node_modules")).isSymbolicLink()).toBe(true); + expect(fs.realpathSync(path.join(runtimePluginDir, "node_modules"))).toBe( + fs.realpathSync(sourcePluginNodeModulesDir), + ); + }); + + it("hard-links top-level dist chunks so staged bundled plugins keep relative imports working", () => { + const repoRoot = makeRepoRoot("openclaw-stage-bundled-runtime-chunks-"); + fs.mkdirSync(path.join(repoRoot, "dist", "extensions", "diffs"), { recursive: true }); + fs.writeFileSync( + path.join(repoRoot, "dist", "chunk-abc.js"), + "export const value = 1;\n", + "utf8", + ); + fs.writeFileSync( + path.join(repoRoot, "dist", "extensions", "diffs", "index.js"), + "export { value } from '../../chunk-abc.js';\n", + "utf8", + ); + + stageBundledPluginRuntime({ repoRoot }); + + const runtimeChunkPath = path.join(repoRoot, "dist-runtime", "chunk-abc.js"); + expect(fs.readFileSync(runtimeChunkPath, "utf8")).toContain("value = 1"); + expect(fs.statSync(runtimeChunkPath).nlink).toBeGreaterThan(1); + expect( + fs.readFileSync( + path.join(repoRoot, "dist-runtime", "extensions", "diffs", "index.js"), + "utf8", + ), + ).toContain("../../chunk-abc.js"); + const distChunkStats = fs.statSync(path.join(repoRoot, "dist", "chunk-abc.js")); + const runtimeChunkStats = fs.statSync(runtimeChunkPath); + expect(runtimeChunkStats.ino).toBe(distChunkStats.ino); + expect(runtimeChunkStats.dev).toBe(distChunkStats.dev); + }); + + it("removes stale runtime plugin directories that are no longer in dist", () => { + const repoRoot = makeRepoRoot("openclaw-stage-bundled-runtime-stale-"); + const staleRuntimeDir = path.join(repoRoot, "dist-runtime", "extensions", "stale"); + fs.mkdirSync(staleRuntimeDir, { recursive: true }); + fs.writeFileSync(path.join(staleRuntimeDir, "index.js"), "stale\n", "utf8"); + fs.mkdirSync(path.join(repoRoot, "dist", "extensions"), { recursive: true }); + + stageBundledPluginRuntime({ repoRoot }); + + expect(fs.existsSync(staleRuntimeDir)).toBe(false); + }); + + it("removes dist-runtime when the built bundled plugin tree is absent", () => { + const repoRoot = makeRepoRoot("openclaw-stage-bundled-runtime-missing-"); + const runtimeRoot = path.join(repoRoot, "dist-runtime", "extensions", "diffs"); + fs.mkdirSync(runtimeRoot, { recursive: true }); + + stageBundledPluginRuntime({ repoRoot }); + + expect(fs.existsSync(path.join(repoRoot, "dist-runtime"))).toBe(false); + }); +});