fix(plugins): localize bundled runtime deps to extensions (#67099)

* fix(plugins): localize bundled runtime deps to extensions

* fix(plugins): move staged runtime deps out of root

* fix(packaging): harden prepack and runtime dep staging

* fix(packaging): preserve optional runtime dep staging

* Update CHANGELOG.md

* fix(packaging): harden runtime staging filesystem writes

* fix(docker): ship preinstall warning in bootstrap layers

* fix(packaging): exclude staged plugin node_modules from npm pack
This commit is contained in:
Vincent Koc
2026-04-15 12:04:31 +01:00
committed by GitHub
parent a780151fd1
commit c727388f93
29 changed files with 1335 additions and 277 deletions

View File

@@ -20,7 +20,7 @@ COPY ui/package.json ./ui/package.json
COPY packages ./packages
COPY extensions ./extensions
COPY patches ./patches
COPY scripts/postinstall-bundled-plugins.mjs scripts/npm-runner.mjs scripts/windows-cmd-helpers.mjs ./scripts/
COPY scripts/postinstall-bundled-plugins.mjs scripts/preinstall-package-manager-warning.mjs scripts/npm-runner.mjs scripts/windows-cmd-helpers.mjs ./scripts/
RUN --mount=type=cache,id=openclaw-pnpm-store,target=/root/.local/share/pnpm/store,sharing=locked \
corepack enable \
&& if ! pnpm install --frozen-lockfile >/tmp/openclaw-cleanup-pnpm-install.log 2>&1; then \

View File

@@ -22,7 +22,7 @@ COPY --chown=appuser:appuser package.json pnpm-lock.yaml pnpm-workspace.yaml .np
COPY --chown=appuser:appuser ui/package.json ./ui/package.json
COPY --chown=appuser:appuser extensions ./extensions
COPY --chown=appuser:appuser patches ./patches
COPY --chown=appuser:appuser scripts/postinstall-bundled-plugins.mjs scripts/npm-runner.mjs scripts/windows-cmd-helpers.mjs ./scripts/
COPY --chown=appuser:appuser scripts/postinstall-bundled-plugins.mjs scripts/preinstall-package-manager-warning.mjs scripts/npm-runner.mjs scripts/windows-cmd-helpers.mjs ./scripts/
RUN --mount=type=cache,id=openclaw-pnpm-store,target=/home/appuser/.local/share/pnpm/store,sharing=locked \
pnpm install --frozen-lockfile

View File

@@ -2,6 +2,10 @@ import fs from "node:fs";
import path from "node:path";
const JS_EXTENSIONS = new Set([".cjs", ".js", ".mjs"]);
const CURATED_ROOT_RUNTIME_MIRRORS = new Set([
"@matrix-org/matrix-sdk-crypto-nodejs",
"@matrix-org/matrix-sdk-crypto-wasm",
]);
export function collectRuntimeDependencySpecs(packageJson = {}) {
return new Map(
@@ -152,6 +156,18 @@ export function collectRootDistBundledRuntimeMirrors(params) {
const bundledSpecs = params.bundledRuntimeDependencySpecs;
const mirrors = new Map();
for (const dependencyName of CURATED_ROOT_RUNTIME_MIRRORS) {
const bundledSpec = bundledSpecs.get(dependencyName);
if (!bundledSpec) {
continue;
}
mirrors.set(dependencyName, {
importers: new Set(["<curated root runtime surface>"]),
pluginIds: bundledSpec.pluginIds,
spec: bundledSpec.spec,
});
}
for (const filePath of walkJavaScriptFiles(distDir)) {
const source = fs.readFileSync(filePath, "utf8");
const relativePath = path.relative(distDir, filePath).replaceAll(path.sep, "/");

View File

@@ -5,8 +5,6 @@ import { existsSync, readdirSync } from "node:fs";
import { pathToFileURL } from "node:url";
import { formatErrorMessage } from "../src/infra/errors.ts";
import { writePackageDistInventory } from "../src/infra/package-dist-inventory.ts";
const skipPrepackPreparedEnv = "OPENCLAW_PREPACK_PREPARED";
const requiredPreparedPathGroups = [
["dist/index.js", "dist/index.mjs"],
["dist/control-ui/index.html"],
@@ -22,14 +20,6 @@ function normalizeFiles(files: Iterable<string>): Set<string> {
return new Set(Array.from(files, (file) => file.replace(/\\/g, "/")));
}
export function shouldSkipPrepack(env = process.env): boolean {
const raw = env[skipPrepackPreparedEnv];
if (!raw) {
return false;
}
return !/^(0|false)$/i.test(raw);
}
export function collectPreparedPrepackErrors(
files: Iterable<string>,
assetPaths: Iterable<string>,
@@ -83,9 +73,7 @@ function ensurePreparedArtifacts(): void {
const preparedFiles = collectPreparedFilePaths();
const errors = collectPreparedPrepackErrors(preparedFiles.files, preparedFiles.assets);
if (errors.length === 0) {
console.error(
`prepack: using prepared artifacts from ${skipPrepackPreparedEnv}; skipping rebuild.`,
);
console.error("prepack: using existing prepared artifacts.");
return;
}
for (const error of errors) {
@@ -97,7 +85,7 @@ function ensurePreparedArtifacts(): void {
}
console.error(
`prepack: ${skipPrepackPreparedEnv}=1 requires an existing build and Control UI bundle. Run \`pnpm build && pnpm ui:build\` first or unset ${skipPrepackPreparedEnv}.`,
"prepack: requires an existing build and Control UI bundle. Run `pnpm build && pnpm ui:build` before packing or publishing.",
);
process.exit(1);
}
@@ -123,14 +111,9 @@ async function writeDistInventory(): Promise<void> {
async function main(): Promise<void> {
const pnpmCommand = process.platform === "win32" ? "pnpm.cmd" : "pnpm";
if (shouldSkipPrepack()) {
ensurePreparedArtifacts();
await writeDistInventory();
runBuildSmoke();
return;
}
run(pnpmCommand, ["build"]);
run(pnpmCommand, ["ui:build"]);
ensurePreparedArtifacts();
await writeDistInventory();
runBuildSmoke();
}

View File

@@ -0,0 +1,64 @@
import { pathToFileURL } from "node:url";
const allowedLifecyclePackageManagers = new Set(["pnpm", "npm", "yarn", "bun"]);
function normalizeEnvValue(value) {
return typeof value === "string" ? value.trim() : "";
}
function normalizeLifecyclePackageManagerName(value) {
const normalized = normalizeEnvValue(value).toLowerCase();
if (!/^[a-z0-9][a-z0-9._-]*$/u.test(normalized)) {
return null;
}
return allowedLifecyclePackageManagers.has(normalized) ? normalized : null;
}
export function detectLifecyclePackageManager(env = process.env) {
const userAgent = normalizeEnvValue(env.npm_config_user_agent);
const userAgentMatch = /^([A-Za-z0-9._-]+)\//u.exec(userAgent);
if (userAgentMatch) {
return normalizeLifecyclePackageManagerName(userAgentMatch[1]);
}
const execPath = normalizeEnvValue(env.npm_execpath).toLowerCase();
if (execPath.includes("pnpm")) {
return "pnpm";
}
if (execPath.includes("npm")) {
return "npm";
}
if (execPath.includes("yarn")) {
return "yarn";
}
if (execPath.includes("bun")) {
return "bun";
}
return null;
}
export function createPackageManagerWarningMessage(packageManager) {
if (!packageManager || packageManager === "pnpm") {
return null;
}
return [
`[openclaw] warning: detected ${packageManager} for install lifecycle.`,
"[openclaw] this repo works best with pnpm; npm-compatible installs are slower and much larger here.",
"[openclaw] prefer: corepack pnpm install",
].join("\n");
}
export function warnIfNonPnpmLifecycle(env = process.env, warn = console.warn) {
const message = createPackageManagerWarningMessage(detectLifecyclePackageManager(env));
if (!message) {
return false;
}
warn(message);
return true;
}
if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) {
warnIfNonPnpmLifecycle();
}

View File

@@ -46,6 +46,7 @@ const requiredPathGroups = [
...WORKSPACE_TEMPLATE_PACK_PATHS,
...listRequiredQaScenarioPackPaths(),
"scripts/npm-runner.mjs",
"scripts/preinstall-package-manager-warning.mjs",
"scripts/postinstall-bundled-plugins.mjs",
"dist/plugin-sdk/compat.js",
"dist/plugin-sdk/root-alias.cjs",
@@ -260,13 +261,10 @@ export function collectMissingPackPaths(paths: Iterable<string>): string[] {
}
export function collectForbiddenPackPaths(paths: Iterable<string>): string[] {
const isAllowedBundledPluginNodeModulesPath = (path: string) =>
/^dist\/extensions\/[^/]+\/node_modules\//.test(path);
return [...paths]
.filter(
(path) =>
forbiddenPrefixes.some((prefix) => path.startsWith(prefix)) ||
(/node_modules\//.test(path) && !isAllowedBundledPluginNodeModulesPath(path)),
forbiddenPrefixes.some((prefix) => path.startsWith(prefix)) || /node_modules\//.test(path),
)
.toSorted((left, right) => left.localeCompare(right));
}

View File

@@ -0,0 +1,305 @@
#!/usr/bin/env node
import fs from "node:fs";
import path from "node:path";
import { pathToFileURL } from "node:url";
import {
collectBundledPluginRuntimeDependencySpecs,
collectRootDistBundledRuntimeMirrors,
packageNameFromSpecifier,
} from "./lib/bundled-plugin-root-runtime-mirrors.mjs";
const DEFAULT_SCAN_ROOTS = ["src", "extensions", "packages", "ui", "scripts", "test"];
const SCANNED_EXTENSIONS = new Set([".cjs", ".cts", ".js", ".jsx", ".mjs", ".mts", ".ts", ".tsx"]);
const IMPORT_PATTERNS = [
/\bfrom\s*["']([^"']+)["']/g,
/\bimport\s*\(\s*["']([^"']+)["']\s*\)/g,
/\brequire\s*\(\s*["']([^"']+)["']\s*\)/g,
/\b(?:require|[_$A-Za-z][\w$]*require[\w$]*)\.resolve\s*\(\s*["']([^"']+)["']\s*\)/gi,
];
function readJson(filePath) {
return JSON.parse(fs.readFileSync(filePath, "utf8"));
}
function isScannableSourceFile(fileName) {
return SCANNED_EXTENSIONS.has(path.extname(fileName));
}
function shouldSkipDir(dirName) {
return dirName === "dist" || dirName === "node_modules" || dirName === ".git";
}
function walkFiles(rootDir) {
if (!fs.existsSync(rootDir)) {
return [];
}
const files = [];
const queue = [rootDir];
while (queue.length > 0) {
const current = queue.shift();
for (const entry of fs.readdirSync(current, { withFileTypes: true })) {
const fullPath = path.join(current, entry.name);
if (entry.isDirectory()) {
if (shouldSkipDir(entry.name)) {
continue;
}
queue.push(fullPath);
continue;
}
if (entry.isFile() && isScannableSourceFile(entry.name)) {
files.push(fullPath);
}
}
}
return files.toSorted((left, right) => left.localeCompare(right));
}
function normalizeRelativePath(filePath, repoRoot) {
return path.relative(repoRoot, filePath).replaceAll(path.sep, "/");
}
function sectionFor(relativePath) {
const [section = "other"] = relativePath.split("/");
return section;
}
export function collectModuleSpecifiers(source) {
const specifiers = new Set();
for (const pattern of IMPORT_PATTERNS) {
for (const match of source.matchAll(pattern)) {
if (match[1]) {
specifiers.add(match[1]);
}
}
}
return specifiers;
}
function collectExtensionDependencyDeclarations(repoRoot) {
const declarations = new Map();
const extensionsRoot = path.join(repoRoot, "extensions");
if (!fs.existsSync(extensionsRoot)) {
return declarations;
}
for (const entry of fs.readdirSync(extensionsRoot, { withFileTypes: true })) {
if (!entry.isDirectory()) {
continue;
}
const packageJsonPath = path.join(extensionsRoot, entry.name, "package.json");
if (!fs.existsSync(packageJsonPath)) {
continue;
}
const packageJson = readJson(packageJsonPath);
for (const section of [
"dependencies",
"optionalDependencies",
"devDependencies",
"peerDependencies",
]) {
for (const depName of Object.keys(packageJson[section] ?? {})) {
const existing = declarations.get(depName) ?? [];
existing.push(`${entry.name}:${section}`);
declarations.set(depName, existing);
}
}
}
for (const values of declarations.values()) {
values.sort((left, right) => left.localeCompare(right));
}
return declarations;
}
function sectionSetContainsCore(sectionSet) {
return sectionSet.has("src") || sectionSet.has("packages") || sectionSet.has("ui");
}
function sectionSetIsSubsetOf(sectionSet, allowed) {
for (const value of sectionSet) {
if (!allowed.has(value)) {
return false;
}
}
return sectionSet.size > 0;
}
export function classifyRootDependencyOwnership(record) {
const sections = new Set(record.sections);
if (record.rootMirrorImporters.length > 0) {
return {
category: "extension_only_root_mirror",
recommendation:
"blocked by packaged host graph: remove root mirror only after bundled runtime resolution stops importing it from root dist",
};
}
if (sections.size === 0) {
return {
category: "unreferenced",
recommendation: "investigate removal; no direct source imports found in scanned files",
};
}
if (sectionSetIsSubsetOf(sections, new Set(["scripts", "test"]))) {
return {
category: "script_or_test_only",
recommendation: "consider moving from dependencies to devDependencies",
};
}
if (sectionSetContainsCore(sections)) {
if (sections.has("extensions")) {
return {
category: "shared_core_and_extension",
recommendation:
"keep at root until shared code is split or extension/core boundary changes",
};
}
return {
category: "core_runtime",
recommendation: "keep at root",
};
}
if (sectionSetIsSubsetOf(sections, new Set(["extensions", "test"]))) {
return {
category: "extension_only_localizable",
recommendation:
"candidate to remove from root package.json and rely on owning extension manifests",
};
}
return {
category: "mixed_noncore",
recommendation: "inspect manually; usage spans non-core surfaces",
};
}
export function collectRootDependencyOwnershipAudit(params = {}) {
const repoRoot = path.resolve(params.repoRoot ?? process.cwd());
const rootPackageJson = readJson(path.join(repoRoot, "package.json"));
const rootDependencies = {
...rootPackageJson.dependencies,
...rootPackageJson.optionalDependencies,
};
const records = new Map(
Object.keys(rootDependencies).map((depName) => [
depName,
{
depName,
sections: new Set(),
files: new Set(),
declaredInExtensions: [],
rootMirrorImporters: [],
spec: rootDependencies[depName],
},
]),
);
const scanRoots = params.scanRoots ?? DEFAULT_SCAN_ROOTS;
for (const scanRoot of scanRoots) {
for (const filePath of walkFiles(path.join(repoRoot, scanRoot))) {
const relativePath = normalizeRelativePath(filePath, repoRoot);
const source = fs.readFileSync(filePath, "utf8");
for (const specifier of collectModuleSpecifiers(source)) {
const depName = packageNameFromSpecifier(specifier);
if (!depName || !records.has(depName)) {
continue;
}
const record = records.get(depName);
record.sections.add(sectionFor(relativePath));
record.files.add(relativePath);
}
}
}
const extensionDeclarations = collectExtensionDependencyDeclarations(repoRoot);
for (const [depName, declarations] of extensionDeclarations) {
const record = records.get(depName);
if (record) {
record.declaredInExtensions = declarations;
}
}
const distDir = path.join(repoRoot, "dist");
if (fs.existsSync(distDir)) {
const bundledSpecs = collectBundledPluginRuntimeDependencySpecs(
path.join(repoRoot, "extensions"),
);
const rootMirrors = collectRootDistBundledRuntimeMirrors({
bundledRuntimeDependencySpecs: bundledSpecs,
distDir,
});
for (const [depName, mirror] of rootMirrors) {
const record = records.get(depName);
if (!record) {
continue;
}
record.rootMirrorImporters = [...mirror.importers].toSorted((left, right) =>
left.localeCompare(right),
);
}
}
return [...records.values()]
.map((record) => {
const classification = classifyRootDependencyOwnership({
...record,
sections: [...record.sections].toSorted((left, right) => left.localeCompare(right)),
});
return {
depName: record.depName,
spec: record.spec,
sections: [...record.sections].toSorted((left, right) => left.localeCompare(right)),
fileCount: record.files.size,
sampleFiles: [...record.files].slice(0, 5),
declaredInExtensions: record.declaredInExtensions,
rootMirrorImporters: record.rootMirrorImporters,
category: classification.category,
recommendation: classification.recommendation,
};
})
.toSorted((left, right) => left.depName.localeCompare(right.depName));
}
function printTextReport(records) {
const grouped = new Map();
for (const record of records) {
const existing = grouped.get(record.category) ?? [];
existing.push(record);
grouped.set(record.category, existing);
}
for (const category of [...grouped.keys()].toSorted((left, right) => left.localeCompare(right))) {
console.log(`\n## ${category}`);
for (const record of grouped.get(category)) {
const details = [`sections=${record.sections.join(",") || "-"}`, `files=${record.fileCount}`];
if (record.declaredInExtensions.length > 0) {
details.push(`extensions=${record.declaredInExtensions.join(",")}`);
}
if (record.rootMirrorImporters.length > 0) {
details.push(`rootDist=${record.rootMirrorImporters.join(",")}`);
}
console.log(`- ${record.depName}@${record.spec} :: ${details.join(" | ")}`);
console.log(` ${record.recommendation}`);
}
}
}
function main(argv = process.argv.slice(2)) {
const asJson = argv.includes("--json");
const records = collectRootDependencyOwnershipAudit();
if (asJson) {
console.log(JSON.stringify(records, null, 2));
return;
}
printTextReport(records);
}
if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) {
main();
}

View File

@@ -1,7 +1,6 @@
import { spawnSync } from "node:child_process";
import { createHash } from "node:crypto";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { pathToFileURL } from "node:url";
import semverSatisfies from "semver/functions/satisfies.js";
@@ -35,18 +34,67 @@ function sanitizeTempPrefixSegment(value) {
return normalized.length > 0 ? normalized : "plugin";
}
function replaceDir(targetPath, sourcePath) {
removePathIfExists(targetPath);
function makePluginOwnedTempDir(pluginDir, label) {
return makeTempDir(pluginDir, `.openclaw-runtime-deps-${label}-`);
}
function assertPathIsNotSymlink(targetPath, label) {
try {
fs.renameSync(sourcePath, targetPath);
return;
} catch (error) {
if (error?.code !== "EXDEV") {
throw error;
if (fs.lstatSync(targetPath).isSymbolicLink()) {
throw new Error(`refusing to ${label} via symlinked path: ${targetPath}`);
}
} catch (error) {
if (error?.code === "ENOENT") {
return;
}
throw error;
}
}
function replaceDirAtomically(targetPath, sourcePath) {
assertPathIsNotSymlink(targetPath, "replace runtime deps");
const targetParentDir = path.dirname(targetPath);
fs.mkdirSync(targetParentDir, { recursive: true });
const backupPath = makeTempDir(
targetParentDir,
`.openclaw-runtime-deps-backup-${sanitizeTempPrefixSegment(path.basename(targetPath))}-`,
);
removePathIfExists(backupPath);
let movedExistingTarget = false;
try {
if (fs.existsSync(targetPath)) {
fs.renameSync(targetPath, backupPath);
movedExistingTarget = true;
}
fs.renameSync(sourcePath, targetPath);
removePathIfExists(backupPath);
} catch (error) {
if (movedExistingTarget && !fs.existsSync(targetPath) && fs.existsSync(backupPath)) {
fs.renameSync(backupPath, targetPath);
}
throw error;
}
}
function writeJsonAtomically(targetPath, value) {
assertPathIsNotSymlink(targetPath, "write runtime deps stamp");
const targetParentDir = path.dirname(targetPath);
fs.mkdirSync(targetParentDir, { recursive: true });
const tempDir = makeTempDir(
targetParentDir,
`.openclaw-runtime-deps-stamp-${sanitizeTempPrefixSegment(path.basename(targetPath))}-`,
);
const tempPath = path.join(tempDir, path.basename(targetPath));
try {
fs.writeFileSync(tempPath, `${JSON.stringify(value, null, 2)}\n`, {
encoding: "utf8",
flag: "wx",
});
fs.renameSync(tempPath, targetPath);
} finally {
removePathIfExists(tempDir);
}
fs.cpSync(sourcePath, targetPath, { recursive: true, force: true });
removePathIfExists(sourcePath);
}
function dependencyPathSegments(depName) {
@@ -80,19 +128,6 @@ function dependencyNodeModulesPath(nodeModulesDir, depName) {
return segments ? path.join(nodeModulesDir, ...segments) : null;
}
function readInstalledDependencyVersion(nodeModulesDir, depName) {
const depRoot = dependencyNodeModulesPath(nodeModulesDir, depName);
if (depRoot === null) {
return null;
}
const packageJsonPath = path.join(depRoot, "package.json");
if (!fs.existsSync(packageJsonPath)) {
return null;
}
const version = readJson(packageJsonPath).version;
return typeof version === "string" ? version : null;
}
function dependencyVersionSatisfied(spec, installedVersion) {
return semverSatisfies(installedVersion, spec, { includePrerelease: false });
}
@@ -147,7 +182,8 @@ const defaultStagedRuntimeDepPruneRules = new Map([
["@jimp/plugin-quantize", { paths: ["src/__image_snapshots__"] }],
["@jimp/plugin-threshold", { paths: ["src/__image_snapshots__"] }],
]);
const runtimeDepsStagingVersion = 3;
const runtimeDepsStagingVersion = 5;
const exactVersionSpecRe = /^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?$/u;
function resolveRuntimeDepPruneConfig(params = {}) {
return {
@@ -175,7 +211,10 @@ function resolveInstalledDependencyRoot(params) {
for (const depRoot of candidates) {
const installedVersion = readInstalledDependencyVersionFromRoot(depRoot);
if (installedVersion !== null && dependencyVersionSatisfied(params.spec, installedVersion)) {
if (installedVersion === null) {
continue;
}
if (params.enforceSpec === false || dependencyVersionSatisfied(params.spec, installedVersion)) {
return depRoot;
}
}
@@ -183,14 +222,18 @@ function resolveInstalledDependencyRoot(params) {
return null;
}
function collectInstalledRuntimeDependencyRoots(rootNodeModulesDir, dependencySpecs) {
function collectInstalledRuntimeDependencyRoots(
rootNodeModulesDir,
dependencySpecs,
directDependencyPackageRoot = null,
) {
const packageCache = new Map();
const directRoots = [];
const allRoots = [];
const queue = Object.entries(dependencySpecs).map(([depName, spec]) => ({
depName,
spec,
parentPackageRoot: null,
parentPackageRoot: directDependencyPackageRoot,
direct: true,
}));
const seen = new Set();
@@ -200,6 +243,7 @@ function collectInstalledRuntimeDependencyRoots(rootNodeModulesDir, dependencySp
const depRoot = resolveInstalledDependencyRoot({
depName: current.depName,
spec: current.spec,
enforceSpec: current.direct,
parentPackageRoot: current.parentPackageRoot,
rootNodeModulesDir,
});
@@ -328,10 +372,23 @@ function selectRuntimeDependencyRootsToCopy(resolution) {
return rootsToCopy;
}
function resolveInstalledDirectDependencyNames(rootNodeModulesDir, dependencySpecs) {
function resolveInstalledDirectDependencyNames(
rootNodeModulesDir,
dependencySpecs,
directDependencyPackageRoot = null,
) {
const directDependencyNames = [];
for (const [depName, spec] of Object.entries(dependencySpecs)) {
const installedVersion = readInstalledDependencyVersion(rootNodeModulesDir, depName);
const depRoot = resolveInstalledDependencyRoot({
depName,
spec,
parentPackageRoot: directDependencyPackageRoot,
rootNodeModulesDir,
});
if (depRoot === null) {
return null;
}
const installedVersion = readInstalledDependencyVersionFromRoot(depRoot);
if (installedVersion === null || !dependencyVersionSatisfied(spec, installedVersion)) {
return null;
}
@@ -390,6 +447,7 @@ function resolveInstalledRuntimeClosureFingerprint(params) {
const resolution = collectInstalledRuntimeDependencyRoots(
params.rootNodeModulesDir,
dependencySpecs,
params.directDependencyPackageRoot,
);
if (resolution === null) {
return null;
@@ -486,6 +544,32 @@ function listBundledPluginRuntimeDirs(repoRoot) {
.filter((pluginDir) => fs.existsSync(path.join(pluginDir, "package.json")));
}
function resolveInstalledWorkspacePluginRoot(repoRoot, pluginId) {
const currentPluginRoot = path.join(repoRoot, "extensions", pluginId);
if (fs.existsSync(path.join(currentPluginRoot, "node_modules"))) {
return currentPluginRoot;
}
const nodeModulesDir = path.join(repoRoot, "node_modules");
if (!fs.existsSync(nodeModulesDir)) {
return currentPluginRoot;
}
let installedWorkspaceRoot;
try {
installedWorkspaceRoot = path.dirname(fs.realpathSync(nodeModulesDir));
} catch {
return currentPluginRoot;
}
const installedPluginRoot = path.join(installedWorkspaceRoot, "extensions", pluginId);
if (fs.existsSync(path.join(installedPluginRoot, "package.json"))) {
return installedPluginRoot;
}
return currentPluginRoot;
}
function hasRuntimeDeps(packageJson) {
return (
Object.keys(packageJson.dependencies ?? {}).length > 0 ||
@@ -524,6 +608,168 @@ function sanitizeBundledManifestForRuntimeInstall(pluginDir) {
return packageJson;
}
function isSafeRuntimeDependencySpec(spec) {
if (typeof spec !== "string") {
return false;
}
const normalized = spec.trim();
if (normalized.length === 0) {
return false;
}
const lower = normalized.toLowerCase();
if (
lower.startsWith("file:") ||
lower.startsWith("link:") ||
lower.startsWith("workspace:") ||
lower.startsWith("git:") ||
lower.startsWith("git+") ||
lower.startsWith("ssh:") ||
lower.startsWith("http:") ||
lower.startsWith("https:")
) {
return false;
}
if (normalized.includes("://")) {
return false;
}
if (
normalized.startsWith("/") ||
normalized.startsWith("\\") ||
normalized.startsWith("../") ||
normalized.startsWith("..\\") ||
normalized.includes("/../") ||
normalized.includes("\\..\\")
) {
return false;
}
return true;
}
function assertSafeRuntimeDependencySpec(depName, spec) {
if (!isSafeRuntimeDependencySpec(spec)) {
throw new Error(`disallowed runtime dependency spec for ${depName}: ${spec}`);
}
}
function resolveInstalledPinnedDependencyVersion(params) {
const depRoot = resolveInstalledDependencyRoot({
depName: params.depName,
enforceSpec: true,
parentPackageRoot: params.parentPackageRoot,
rootNodeModulesDir: params.rootNodeModulesDir,
spec: params.spec,
});
if (depRoot === null) {
return null;
}
return readInstalledDependencyVersionFromRoot(depRoot);
}
function resolvePinnedRuntimeDependencyVersion(params) {
assertSafeRuntimeDependencySpec(params.depName, params.spec);
if (exactVersionSpecRe.test(params.spec)) {
return params.spec;
}
const installedVersion = resolveInstalledPinnedDependencyVersion(params);
if (typeof installedVersion === "string" && exactVersionSpecRe.test(installedVersion)) {
return installedVersion;
}
throw new Error(
`runtime dependency ${params.depName} must resolve to an exact installed version, got: ${params.spec}`,
);
}
function collectRuntimeDependencyGroups(packageJson) {
const readRuntimeGroup = (group) =>
Object.fromEntries(
Object.entries(group ?? {}).filter(
(entry) => typeof entry[0] === "string" && typeof entry[1] === "string",
),
);
return {
dependencies: readRuntimeGroup(packageJson.dependencies),
optionalDependencies: readRuntimeGroup(packageJson.optionalDependencies),
};
}
function resolvePinnedRuntimeDependencyGroup(group, params = {}) {
return Object.fromEntries(
Object.entries(group).map(([name, version]) => {
const pinnedVersion = resolvePinnedRuntimeDependencyVersion({
depName: name,
parentPackageRoot: params.directDependencyPackageRoot ?? null,
rootNodeModulesDir: params.rootNodeModulesDir ?? path.join(process.cwd(), "node_modules"),
spec: version,
});
return [name, pinnedVersion];
}),
);
}
function resolvePinnedRuntimeDependencyGroups(packageJson, params = {}) {
const runtimeGroups = collectRuntimeDependencyGroups(packageJson);
return {
dependencies: resolvePinnedRuntimeDependencyGroup(runtimeGroups.dependencies, params),
optionalDependencies: resolvePinnedRuntimeDependencyGroup(
runtimeGroups.optionalDependencies,
params,
),
};
}
export function collectRuntimeDependencyInstallManifest(packageJson, params = {}) {
const pinnedGroups = resolvePinnedRuntimeDependencyGroups(packageJson, params);
return createRuntimeInstallManifest(params.pluginId ?? "runtime-deps", pinnedGroups);
}
export function collectRuntimeDependencyInstallSpecs(packageJson, params = {}) {
const manifest = collectRuntimeDependencyInstallManifest(packageJson, params);
const buildSpecs = (group) =>
Object.entries(group ?? {}).map(([name, version]) => `${name}@${String(version)}`);
return {
dependencies: buildSpecs(manifest.dependencies),
optionalDependencies: buildSpecs(manifest.optionalDependencies),
};
}
function createRuntimeInstallManifest(pluginId, pinnedGroups) {
const manifest = {
name: `openclaw-runtime-deps-${sanitizeTempPrefixSegment(pluginId)}`,
private: true,
version: "0.0.0",
};
if (Object.keys(pinnedGroups.dependencies).length > 0) {
manifest.dependencies = pinnedGroups.dependencies;
}
if (Object.keys(pinnedGroups.optionalDependencies).length > 0) {
manifest.optionalDependencies = pinnedGroups.optionalDependencies;
}
return manifest;
}
function runNpmInstall(params) {
const npmEnv = {
...(params.npmRunner.env ?? process.env),
CI: "1",
npm_config_loglevel: "error",
npm_config_yes: "true",
};
const result = spawnSync(params.npmRunner.command, params.npmRunner.args, {
cwd: params.cwd,
encoding: "utf8",
env: npmEnv,
shell: params.npmRunner.shell,
stdio: ["ignore", "pipe", "pipe"],
timeout: params.timeoutMs ?? 5 * 60 * 1000,
windowsVerbatimArguments: params.npmRunner.windowsVerbatimArguments,
});
if (result.status === 0) {
return;
}
const output = [result.stderr, result.stdout].filter(Boolean).join("\n").trim();
throw new Error(output || "npm install failed");
}
function resolveRuntimeDepsStampPath(pluginDir) {
return path.join(pluginDir, ".openclaw-runtime-deps-stamp.json");
}
@@ -561,7 +807,14 @@ function readRuntimeDepsStamp(stampPath) {
}
function stageInstalledRootRuntimeDeps(params) {
const { fingerprint, packageJson, pluginDir, pruneConfig, repoRoot } = params;
const {
directDependencyPackageRoot = null,
fingerprint,
packageJson,
pluginDir,
pruneConfig,
repoRoot,
} = params;
const dependencySpecs = {
...packageJson.dependencies,
...packageJson.optionalDependencies,
@@ -574,11 +827,16 @@ function stageInstalledRootRuntimeDeps(params) {
const directDependencyNames = resolveInstalledDirectDependencyNames(
rootNodeModulesDir,
dependencySpecs,
directDependencyPackageRoot,
);
if (directDependencyNames === null) {
return false;
}
const resolution = collectInstalledRuntimeDependencyRoots(rootNodeModulesDir, dependencySpecs);
const resolution = collectInstalledRuntimeDependencyRoots(
rootNodeModulesDir,
dependencySpecs,
directDependencyPackageRoot,
);
if (resolution === null) {
return false;
}
@@ -588,10 +846,7 @@ function stageInstalledRootRuntimeDeps(params) {
const nodeModulesDir = path.join(pluginDir, "node_modules");
const stampPath = resolveRuntimeDepsStampPath(pluginDir);
const stagedNodeModulesDir = path.join(
makeTempDir(
os.tmpdir(),
`openclaw-runtime-deps-${sanitizeTempPrefixSegment(path.basename(pluginDir))}-`,
),
makePluginOwnedTempDir(pluginDir, "stage"),
"node_modules",
);
@@ -620,8 +875,8 @@ function stageInstalledRootRuntimeDeps(params) {
}
pruneStagedRuntimeDependencyCargo(stagedNodeModulesDir, pruneConfig);
replaceDir(nodeModulesDir, stagedNodeModulesDir);
writeJson(stampPath, {
replaceDirAtomically(nodeModulesDir, stagedNodeModulesDir);
writeJsonAtomically(stampPath, {
fingerprint,
generatedAt: new Date().toISOString(),
});
@@ -631,66 +886,6 @@ function stageInstalledRootRuntimeDeps(params) {
}
}
function installPluginRuntimeDeps(params) {
const { fingerprint, packageJson, pluginDir, pluginId, pruneConfig, repoRoot } = params;
if (
repoRoot &&
stageInstalledRootRuntimeDeps({ fingerprint, packageJson, pluginDir, pruneConfig, repoRoot })
) {
return;
}
const nodeModulesDir = path.join(pluginDir, "node_modules");
const stampPath = resolveRuntimeDepsStampPath(pluginDir);
const tempInstallDir = makeTempDir(
os.tmpdir(),
`openclaw-runtime-deps-${sanitizeTempPrefixSegment(pluginId)}-`,
);
const npmRunner = resolveNpmRunner({
npmArgs: [
"install",
"--omit=dev",
"--silent",
"--ignore-scripts",
"--legacy-peer-deps",
"--package-lock=false",
],
});
try {
writeJson(path.join(tempInstallDir, "package.json"), packageJson);
const result = spawnSync(npmRunner.command, npmRunner.args, {
cwd: tempInstallDir,
encoding: "utf8",
env: npmRunner.env,
stdio: "pipe",
shell: npmRunner.shell,
windowsVerbatimArguments: npmRunner.windowsVerbatimArguments,
});
if (result.status !== 0) {
const output = [result.stderr, result.stdout].filter(Boolean).join("\n").trim();
throw new Error(
`failed to stage bundled runtime deps for ${pluginId}: ${output || "npm install failed"}`,
);
}
const stagedNodeModulesDir = path.join(tempInstallDir, "node_modules");
if (!fs.existsSync(stagedNodeModulesDir)) {
throw new Error(
`failed to stage bundled runtime deps for ${pluginId}: npm install produced no node_modules directory`,
);
}
pruneStagedRuntimeDependencyCargo(stagedNodeModulesDir, pruneConfig);
replaceDir(nodeModulesDir, stagedNodeModulesDir);
writeJson(stampPath, {
fingerprint,
generatedAt: new Date().toISOString(),
});
} finally {
removePathIfExists(tempInstallDir);
}
}
function installPluginRuntimeDepsWithRetries(params) {
const { attempts = 3 } = params;
let lastError;
@@ -708,6 +903,86 @@ function installPluginRuntimeDepsWithRetries(params) {
throw lastError;
}
function createRootRuntimeStagingError(params) {
const runtimeDependencyNames = [
...Object.keys(params.packageJson.dependencies ?? {}),
...Object.keys(params.packageJson.optionalDependencies ?? {}),
].toSorted((left, right) => left.localeCompare(right));
const dependencyLabel =
runtimeDependencyNames.length > 0 ? runtimeDependencyNames.join(", ") : "<none>";
const causeMessage =
params.cause instanceof Error && typeof params.cause.message === "string"
? ` Cause: ${params.cause.message}`
: "";
return new Error(
`failed to stage bundled runtime deps for ${params.pluginId}: ` +
`runtime dependency closure must resolve from the installed root workspace graph. ` +
`Could not materialize: ${dependencyLabel}. ` +
"Run `pnpm install` and rebuild from a trusted workspace checkout, or provide a hardened fallback installer." +
causeMessage,
);
}
function installPluginRuntimeDeps(params) {
const {
directDependencyPackageRoot = null,
fingerprint,
packageJson,
pluginDir,
pluginId,
pruneConfig,
repoRoot,
} = params;
const nodeModulesDir = path.join(pluginDir, "node_modules");
const stampPath = resolveRuntimeDepsStampPath(pluginDir);
const tempInstallDir = makePluginOwnedTempDir(pluginDir, "install");
const pinnedGroups = resolvePinnedRuntimeDependencyGroups(packageJson, {
directDependencyPackageRoot,
rootNodeModulesDir: path.join(repoRoot, "node_modules"),
});
const requiredDependencyCount = Object.keys(pinnedGroups.dependencies).length;
try {
writeJson(
path.join(tempInstallDir, "package.json"),
createRuntimeInstallManifest(pluginId, pinnedGroups),
);
if (requiredDependencyCount > 0 || Object.keys(pinnedGroups.optionalDependencies).length > 0) {
runNpmInstall({
cwd: tempInstallDir,
npmRunner: resolveNpmRunner({
npmArgs: [
"install",
"--omit=dev",
"--ignore-scripts",
"--legacy-peer-deps",
"--package-lock=false",
"--silent",
],
}),
});
}
const stagedNodeModulesDir = path.join(tempInstallDir, "node_modules");
if (requiredDependencyCount > 0 && !fs.existsSync(stagedNodeModulesDir)) {
throw new Error(
`failed to stage bundled runtime deps for ${pluginId}: explicit npm install produced no node_modules directory`,
);
}
if (fs.existsSync(stagedNodeModulesDir)) {
pruneStagedRuntimeDependencyCargo(stagedNodeModulesDir, pruneConfig);
replaceDirAtomically(nodeModulesDir, stagedNodeModulesDir);
} else {
assertPathIsNotSymlink(nodeModulesDir, "remove runtime deps");
removePathIfExists(nodeModulesDir);
}
writeJsonAtomically(stampPath, {
fingerprint,
generatedAt: new Date().toISOString(),
});
} finally {
removePathIfExists(tempInstallDir);
}
}
export function stageBundledPluginRuntimeDeps(params = {}) {
const repoRoot = params.cwd ?? params.repoRoot ?? process.cwd();
const installPluginRuntimeDepsImpl =
@@ -716,6 +991,10 @@ export function stageBundledPluginRuntimeDeps(params = {}) {
const pruneConfig = resolveRuntimeDepPruneConfig(params);
for (const pluginDir of listBundledPluginRuntimeDirs(repoRoot)) {
const pluginId = path.basename(pluginDir);
const sourcePluginRoot = resolveInstalledWorkspacePluginRoot(repoRoot, pluginId);
const directDependencyPackageRoot = fs.existsSync(path.join(sourcePluginRoot, "package.json"))
? sourcePluginRoot
: null;
const packageJson = sanitizeBundledManifestForRuntimeInstall(pluginDir);
const nodeModulesDir = path.join(pluginDir, "node_modules");
const stampPath = resolveRuntimeDepsStampPath(pluginDir);
@@ -725,6 +1004,7 @@ export function stageBundledPluginRuntimeDeps(params = {}) {
continue;
}
const rootInstalledRuntimeFingerprint = resolveInstalledRuntimeClosureFingerprint({
directDependencyPackageRoot,
packageJson,
rootNodeModulesDir: path.join(repoRoot, "node_modules"),
});
@@ -736,18 +1016,35 @@ export function stageBundledPluginRuntimeDeps(params = {}) {
if (fs.existsSync(nodeModulesDir) && stamp?.fingerprint === fingerprint) {
continue;
}
installPluginRuntimeDepsWithRetries({
attempts: installAttempts,
install: installPluginRuntimeDepsImpl,
installParams: {
if (
stageInstalledRootRuntimeDeps({
directDependencyPackageRoot,
fingerprint,
packageJson,
pluginDir,
pluginId,
pruneConfig,
repoRoot,
},
});
})
) {
continue;
}
try {
installPluginRuntimeDepsWithRetries({
attempts: installAttempts,
install: installPluginRuntimeDepsImpl,
installParams: {
directDependencyPackageRoot,
fingerprint,
packageJson,
pluginDir,
pluginId,
pruneConfig,
repoRoot,
},
});
} catch (error) {
throw createRootRuntimeStagingError({ packageJson, pluginId, cause: error });
}
}
}