Files
openclaw/scripts/stage-bundled-plugin-runtime-deps.mjs
Frank Yang d86527d8c6 fix(whatsapp): harden Baileys media upload hotfix (#65966)
Merged via squash.

Prepared head SHA: b5db59b8fe
Co-authored-by: frankekn <4488090+frankekn@users.noreply.github.com>
Co-authored-by: frankekn <4488090+frankekn@users.noreply.github.com>
Reviewed-by: @frankekn
2026-04-14 21:34:23 +08:00

757 lines
22 KiB
JavaScript

import { spawnSync } from "node:child_process";
import { createHash } from "node:crypto";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { pathToFileURL } from "node:url";
import semverSatisfies from "semver/functions/satisfies.js";
import { resolveNpmRunner } from "./npm-runner.mjs";
function readJson(filePath) {
return JSON.parse(fs.readFileSync(filePath, "utf8"));
}
function writeJson(filePath, value) {
fs.writeFileSync(filePath, `${JSON.stringify(value, null, 2)}\n`, "utf8");
}
function readOptionalUtf8(filePath) {
if (!fs.existsSync(filePath)) {
return null;
}
return fs.readFileSync(filePath, "utf8");
}
function removePathIfExists(targetPath) {
fs.rmSync(targetPath, { recursive: true, force: true });
}
function makeTempDir(parentDir, prefix) {
return fs.mkdtempSync(path.join(parentDir, prefix));
}
function sanitizeTempPrefixSegment(value) {
const normalized = value.replace(/[^A-Za-z0-9._-]+/g, "-").replace(/-+/g, "-");
return normalized.length > 0 ? normalized : "plugin";
}
function replaceDir(targetPath, sourcePath) {
removePathIfExists(targetPath);
try {
fs.renameSync(sourcePath, targetPath);
return;
} catch (error) {
if (error?.code !== "EXDEV") {
throw error;
}
}
fs.cpSync(sourcePath, targetPath, { recursive: true, force: true });
removePathIfExists(sourcePath);
}
function dependencyPathSegments(depName) {
if (typeof depName !== "string" || depName.length === 0) {
return null;
}
const segments = depName.split("/");
if (depName.startsWith("@")) {
if (segments.length !== 2) {
return null;
}
const [scope, name] = segments;
if (
!/^@[A-Za-z0-9._-]+$/.test(scope) ||
!/^[A-Za-z0-9._-]+$/.test(name) ||
scope === "@." ||
scope === "@.."
) {
return null;
}
return [scope, name];
}
if (segments.length !== 1 || !/^[A-Za-z0-9._-]+$/.test(segments[0])) {
return null;
}
return segments;
}
function dependencyNodeModulesPath(nodeModulesDir, depName) {
const segments = dependencyPathSegments(depName);
return segments ? path.join(nodeModulesDir, ...segments) : null;
}
function readInstalledDependencyVersion(nodeModulesDir, depName) {
const depRoot = dependencyNodeModulesPath(nodeModulesDir, depName);
if (depRoot === null) {
return null;
}
const packageJsonPath = path.join(depRoot, "package.json");
if (!fs.existsSync(packageJsonPath)) {
return null;
}
const version = readJson(packageJsonPath).version;
return typeof version === "string" ? version : null;
}
function dependencyVersionSatisfied(spec, installedVersion) {
return semverSatisfies(installedVersion, spec, { includePrerelease: false });
}
function readInstalledDependencyVersionFromRoot(depRoot) {
const packageJsonPath = path.join(depRoot, "package.json");
if (!fs.existsSync(packageJsonPath)) {
return null;
}
const version = readJson(packageJsonPath).version;
return typeof version === "string" ? version : null;
}
const defaultStagedRuntimeDepGlobalPruneSuffixes = [".d.ts", ".map"];
const defaultStagedRuntimeDepPruneRules = new Map([
// Type declarations only; runtime resolves through lib/es entrypoints.
["@larksuiteoapi/node-sdk", { paths: ["types"] }],
[
"@matrix-org/matrix-sdk-crypto-nodejs",
{
paths: ["index.d.ts", "README.md", "CHANGELOG.md", "RELEASING.md", ".node-version"],
},
],
[
"@matrix-org/matrix-sdk-crypto-wasm",
{
paths: [
"index.d.ts",
"pkg/matrix_sdk_crypto_wasm.d.ts",
"pkg/matrix_sdk_crypto_wasm_bg.wasm.d.ts",
"README.md",
],
},
],
[
"matrix-js-sdk",
{
paths: ["src", "CHANGELOG.md", "CONTRIBUTING.rst", "README.md", "release.sh"],
suffixes: [".d.ts"],
},
],
["matrix-widget-api", { paths: ["src"], suffixes: [".d.ts"] }],
["oidc-client-ts", { paths: ["README.md"], suffixes: [".d.ts"] }],
["music-metadata", { paths: ["README.md"], suffixes: [".d.ts"] }],
["@cloudflare/workers-types", { paths: ["."] }],
["gifwrap", { paths: ["test"] }],
["playwright-core", { paths: ["types"], suffixes: [".d.ts"] }],
["@jimp/plugin-blit", { paths: ["src/__image_snapshots__"] }],
["@jimp/plugin-blur", { paths: ["src/__image_snapshots__"] }],
["@jimp/plugin-color", { paths: ["src/__image_snapshots__"] }],
["@jimp/plugin-print", { paths: ["src/__image_snapshots__"] }],
["@jimp/plugin-quantize", { paths: ["src/__image_snapshots__"] }],
["@jimp/plugin-threshold", { paths: ["src/__image_snapshots__"] }],
]);
const runtimeDepsStagingVersion = 3;
function resolveRuntimeDepPruneConfig(params = {}) {
return {
globalPruneSuffixes:
params.stagedRuntimeDepGlobalPruneSuffixes ?? defaultStagedRuntimeDepGlobalPruneSuffixes,
pruneRules: params.stagedRuntimeDepPruneRules ?? defaultStagedRuntimeDepPruneRules,
};
}
function resolveInstalledDependencyRoot(params) {
const candidates = [];
if (params.parentPackageRoot) {
const nestedDepRoot = dependencyNodeModulesPath(
path.join(params.parentPackageRoot, "node_modules"),
params.depName,
);
if (nestedDepRoot !== null) {
candidates.push(nestedDepRoot);
}
}
const rootDepRoot = dependencyNodeModulesPath(params.rootNodeModulesDir, params.depName);
if (rootDepRoot !== null) {
candidates.push(rootDepRoot);
}
for (const depRoot of candidates) {
const installedVersion = readInstalledDependencyVersionFromRoot(depRoot);
if (installedVersion !== null && dependencyVersionSatisfied(params.spec, installedVersion)) {
return depRoot;
}
}
return null;
}
function collectInstalledRuntimeDependencyRoots(rootNodeModulesDir, dependencySpecs) {
const packageCache = new Map();
const directRoots = [];
const allRoots = [];
const queue = Object.entries(dependencySpecs).map(([depName, spec]) => ({
depName,
spec,
parentPackageRoot: null,
direct: true,
}));
const seen = new Set();
while (queue.length > 0) {
const current = queue.shift();
const depRoot = resolveInstalledDependencyRoot({
depName: current.depName,
spec: current.spec,
parentPackageRoot: current.parentPackageRoot,
rootNodeModulesDir,
});
if (depRoot === null) {
return null;
}
const canonicalDepRoot = fs.realpathSync(depRoot);
const seenKey = `${current.depName}\0${canonicalDepRoot}`;
if (seen.has(seenKey)) {
continue;
}
seen.add(seenKey);
const record = { name: current.depName, root: depRoot, realRoot: canonicalDepRoot };
allRoots.push(record);
if (current.direct) {
directRoots.push(record);
}
const packageJson =
packageCache.get(canonicalDepRoot) ?? readJson(path.join(depRoot, "package.json"));
packageCache.set(canonicalDepRoot, packageJson);
for (const [childName, childSpec] of Object.entries(packageJson.dependencies ?? {})) {
queue.push({
depName: childName,
spec: childSpec,
parentPackageRoot: depRoot,
direct: false,
});
}
for (const [childName, childSpec] of Object.entries(packageJson.optionalDependencies ?? {})) {
queue.push({
depName: childName,
spec: childSpec,
parentPackageRoot: depRoot,
direct: false,
});
}
}
return { allRoots, directRoots };
}
function pathIsInsideCopiedRoot(candidateRoot, copiedRoot) {
return candidateRoot === copiedRoot || candidateRoot.startsWith(`${copiedRoot}${path.sep}`);
}
function findContainingRealRoot(candidatePath, allowedRealRoots) {
return (
allowedRealRoots.find((rootPath) => pathIsInsideCopiedRoot(candidatePath, rootPath)) ?? null
);
}
function copyMaterializedDependencyTree(params) {
const { activeRoots, allowedRealRoots, sourcePath, targetPath } = params;
const sourceStats = fs.lstatSync(sourcePath);
if (sourceStats.isSymbolicLink()) {
let resolvedPath;
try {
resolvedPath = fs.realpathSync(sourcePath);
} catch {
return false;
}
const containingRoot = findContainingRealRoot(resolvedPath, allowedRealRoots);
if (containingRoot === null) {
return false;
}
if (activeRoots.has(containingRoot)) {
return true;
}
const nextActiveRoots = new Set(activeRoots);
nextActiveRoots.add(containingRoot);
return copyMaterializedDependencyTree({
activeRoots: nextActiveRoots,
allowedRealRoots,
sourcePath: resolvedPath,
targetPath,
});
}
if (sourceStats.isDirectory()) {
fs.mkdirSync(targetPath, { recursive: true });
for (const entry of fs
.readdirSync(sourcePath, { withFileTypes: true })
.toSorted((left, right) => left.name.localeCompare(right.name))) {
if (
!copyMaterializedDependencyTree({
activeRoots,
allowedRealRoots,
sourcePath: path.join(sourcePath, entry.name),
targetPath: path.join(targetPath, entry.name),
})
) {
return false;
}
}
return true;
}
if (sourceStats.isFile()) {
fs.mkdirSync(path.dirname(targetPath), { recursive: true });
fs.copyFileSync(sourcePath, targetPath);
fs.chmodSync(targetPath, sourceStats.mode);
return true;
}
return true;
}
function selectRuntimeDependencyRootsToCopy(resolution) {
const rootsToCopy = [];
for (const record of resolution.directRoots) {
rootsToCopy.push(record);
}
for (const record of resolution.allRoots) {
if (rootsToCopy.some((entry) => pathIsInsideCopiedRoot(record.realRoot, entry.realRoot))) {
continue;
}
rootsToCopy.push(record);
}
return rootsToCopy;
}
function resolveInstalledDirectDependencyNames(rootNodeModulesDir, dependencySpecs) {
const directDependencyNames = [];
for (const [depName, spec] of Object.entries(dependencySpecs)) {
const installedVersion = readInstalledDependencyVersion(rootNodeModulesDir, depName);
if (installedVersion === null || !dependencyVersionSatisfied(spec, installedVersion)) {
return null;
}
directDependencyNames.push(depName);
}
return directDependencyNames;
}
function appendDirectoryFingerprint(hash, rootDir, currentDir = rootDir) {
const entries = fs
.readdirSync(currentDir, { withFileTypes: true })
.toSorted((left, right) => left.name.localeCompare(right.name));
for (const entry of entries) {
const fullPath = path.join(currentDir, entry.name);
const relativePath = path.relative(rootDir, fullPath).replace(/\\/g, "/");
if (entry.isSymbolicLink()) {
hash.update(`symlink:${relativePath}->${fs.readlinkSync(fullPath).replace(/\\/g, "/")}\n`);
continue;
}
if (entry.isDirectory()) {
hash.update(`dir:${relativePath}\n`);
appendDirectoryFingerprint(hash, rootDir, fullPath);
continue;
}
if (!entry.isFile()) {
continue;
}
const stat = fs.statSync(fullPath);
hash.update(`file:${relativePath}:${stat.size}\n`);
hash.update(fs.readFileSync(fullPath));
}
}
function createInstalledRuntimeClosureFingerprint(rootNodeModulesDir, dependencyNames) {
const hash = createHash("sha256");
for (const depName of [...dependencyNames].toSorted((left, right) => left.localeCompare(right))) {
const depRoot = dependencyNodeModulesPath(rootNodeModulesDir, depName);
if (depRoot === null || !fs.existsSync(depRoot)) {
return null;
}
hash.update(`package:${depName}\n`);
appendDirectoryFingerprint(hash, depRoot);
}
return hash.digest("hex");
}
function resolveInstalledRuntimeClosureFingerprint(params) {
const dependencySpecs = {
...params.packageJson.dependencies,
...params.packageJson.optionalDependencies,
};
if (Object.keys(dependencySpecs).length === 0 || !fs.existsSync(params.rootNodeModulesDir)) {
return null;
}
const resolution = collectInstalledRuntimeDependencyRoots(
params.rootNodeModulesDir,
dependencySpecs,
);
if (resolution === null) {
return null;
}
return createInstalledRuntimeClosureFingerprint(
params.rootNodeModulesDir,
selectRuntimeDependencyRootsToCopy(resolution).map((record) => record.name),
);
}
function walkFiles(rootDir, visitFile) {
if (!fs.existsSync(rootDir)) {
return;
}
const queue = [rootDir];
while (queue.length > 0) {
const currentDir = queue.shift();
for (const entry of fs.readdirSync(currentDir, { withFileTypes: true })) {
const fullPath = path.join(currentDir, entry.name);
if (entry.isDirectory()) {
queue.push(fullPath);
continue;
}
if (entry.isFile()) {
visitFile(fullPath);
}
}
}
}
function pruneDependencyFilesBySuffixes(depRoot, suffixes) {
if (!suffixes || suffixes.length === 0 || !fs.existsSync(depRoot)) {
return;
}
walkFiles(depRoot, (fullPath) => {
if (suffixes.some((suffix) => fullPath.endsWith(suffix))) {
removePathIfExists(fullPath);
}
});
}
function pruneStagedInstalledDependencyCargo(nodeModulesDir, depName, pruneConfig) {
const depRoot = dependencyNodeModulesPath(nodeModulesDir, depName);
if (depRoot === null) {
return;
}
const pruneRule = pruneConfig.pruneRules.get(depName);
for (const relativePath of pruneRule?.paths ?? []) {
removePathIfExists(path.join(depRoot, relativePath));
}
pruneDependencyFilesBySuffixes(depRoot, pruneConfig.globalPruneSuffixes);
pruneDependencyFilesBySuffixes(depRoot, pruneRule?.suffixes ?? []);
}
function listInstalledDependencyNames(nodeModulesDir) {
if (!fs.existsSync(nodeModulesDir)) {
return [];
}
const names = [];
for (const entry of fs.readdirSync(nodeModulesDir, { withFileTypes: true })) {
if (!entry.isDirectory()) {
continue;
}
if (entry.name.startsWith("@")) {
const scopeDir = path.join(nodeModulesDir, entry.name);
for (const scopedEntry of fs.readdirSync(scopeDir, { withFileTypes: true })) {
if (scopedEntry.isDirectory()) {
names.push(`${entry.name}/${scopedEntry.name}`);
}
}
continue;
}
names.push(entry.name);
}
return names;
}
function pruneStagedRuntimeDependencyCargo(nodeModulesDir, pruneConfig) {
for (const depName of listInstalledDependencyNames(nodeModulesDir)) {
pruneStagedInstalledDependencyCargo(nodeModulesDir, depName, pruneConfig);
}
}
function listBundledPluginRuntimeDirs(repoRoot) {
const extensionsRoot = path.join(repoRoot, "dist", "extensions");
if (!fs.existsSync(extensionsRoot)) {
return [];
}
return fs
.readdirSync(extensionsRoot, { withFileTypes: true })
.filter((dirent) => dirent.isDirectory())
.map((dirent) => path.join(extensionsRoot, dirent.name))
.filter((pluginDir) => fs.existsSync(path.join(pluginDir, "package.json")));
}
function hasRuntimeDeps(packageJson) {
return (
Object.keys(packageJson.dependencies ?? {}).length > 0 ||
Object.keys(packageJson.optionalDependencies ?? {}).length > 0
);
}
function shouldStageRuntimeDeps(packageJson) {
return packageJson.openclaw?.bundle?.stageRuntimeDependencies === true;
}
function sanitizeBundledManifestForRuntimeInstall(pluginDir) {
const manifestPath = path.join(pluginDir, "package.json");
const packageJson = readJson(manifestPath);
let changed = false;
if (packageJson.peerDependencies) {
delete packageJson.peerDependencies;
changed = true;
}
if (packageJson.peerDependenciesMeta) {
delete packageJson.peerDependenciesMeta;
changed = true;
}
if (packageJson.devDependencies) {
delete packageJson.devDependencies;
changed = true;
}
if (changed) {
writeJson(manifestPath, packageJson);
}
return packageJson;
}
function resolveRuntimeDepsStampPath(pluginDir) {
return path.join(pluginDir, ".openclaw-runtime-deps-stamp.json");
}
function createRuntimeDepsFingerprint(packageJson, pruneConfig, params = {}) {
const repoRoot = params.repoRoot;
const lockfilePath =
typeof repoRoot === "string" && repoRoot.length > 0
? path.join(repoRoot, "pnpm-lock.yaml")
: null;
const rootLockfile = lockfilePath ? readOptionalUtf8(lockfilePath) : null;
return createHash("sha256")
.update(
JSON.stringify({
globalPruneSuffixes: pruneConfig.globalPruneSuffixes,
packageJson,
pruneRules: [...pruneConfig.pruneRules.entries()],
rootInstalledRuntimeFingerprint: params.rootInstalledRuntimeFingerprint ?? null,
rootLockfile,
version: runtimeDepsStagingVersion,
}),
)
.digest("hex");
}
function readRuntimeDepsStamp(stampPath) {
if (!fs.existsSync(stampPath)) {
return null;
}
try {
return readJson(stampPath);
} catch {
return null;
}
}
function stageInstalledRootRuntimeDeps(params) {
const { fingerprint, packageJson, pluginDir, pruneConfig, repoRoot } = params;
const dependencySpecs = {
...packageJson.dependencies,
...packageJson.optionalDependencies,
};
const rootNodeModulesDir = path.join(repoRoot, "node_modules");
if (Object.keys(dependencySpecs).length === 0 || !fs.existsSync(rootNodeModulesDir)) {
return false;
}
const directDependencyNames = resolveInstalledDirectDependencyNames(
rootNodeModulesDir,
dependencySpecs,
);
if (directDependencyNames === null) {
return false;
}
const resolution = collectInstalledRuntimeDependencyRoots(rootNodeModulesDir, dependencySpecs);
if (resolution === null) {
return false;
}
const rootsToCopy = selectRuntimeDependencyRootsToCopy(resolution);
const allowedRealRoots = rootsToCopy.map((record) => record.realRoot);
const nodeModulesDir = path.join(pluginDir, "node_modules");
const stampPath = resolveRuntimeDepsStampPath(pluginDir);
const stagedNodeModulesDir = path.join(
makeTempDir(
os.tmpdir(),
`openclaw-runtime-deps-${sanitizeTempPrefixSegment(path.basename(pluginDir))}-`,
),
"node_modules",
);
try {
for (const record of rootsToCopy.toSorted((left, right) =>
left.name.localeCompare(right.name),
)) {
const sourcePath = record.realRoot;
const targetPath = dependencyNodeModulesPath(stagedNodeModulesDir, record.name);
if (targetPath === null) {
return false;
}
fs.mkdirSync(path.dirname(targetPath), { recursive: true });
const sourceRootReal = findContainingRealRoot(sourcePath, allowedRealRoots);
if (
sourceRootReal === null ||
!copyMaterializedDependencyTree({
activeRoots: new Set([sourceRootReal]),
allowedRealRoots,
sourcePath,
targetPath,
})
) {
return false;
}
}
pruneStagedRuntimeDependencyCargo(stagedNodeModulesDir, pruneConfig);
replaceDir(nodeModulesDir, stagedNodeModulesDir);
writeJson(stampPath, {
fingerprint,
generatedAt: new Date().toISOString(),
});
return true;
} finally {
removePathIfExists(path.dirname(stagedNodeModulesDir));
}
}
function installPluginRuntimeDeps(params) {
const { fingerprint, packageJson, pluginDir, pluginId, pruneConfig, repoRoot } = params;
if (
repoRoot &&
stageInstalledRootRuntimeDeps({ fingerprint, packageJson, pluginDir, pruneConfig, repoRoot })
) {
return;
}
const nodeModulesDir = path.join(pluginDir, "node_modules");
const stampPath = resolveRuntimeDepsStampPath(pluginDir);
const tempInstallDir = makeTempDir(
os.tmpdir(),
`openclaw-runtime-deps-${sanitizeTempPrefixSegment(pluginId)}-`,
);
const npmRunner = resolveNpmRunner({
npmArgs: [
"install",
"--omit=dev",
"--silent",
"--ignore-scripts",
"--legacy-peer-deps",
"--package-lock=false",
],
});
try {
writeJson(path.join(tempInstallDir, "package.json"), packageJson);
const result = spawnSync(npmRunner.command, npmRunner.args, {
cwd: tempInstallDir,
encoding: "utf8",
env: npmRunner.env,
stdio: "pipe",
shell: npmRunner.shell,
windowsVerbatimArguments: npmRunner.windowsVerbatimArguments,
});
if (result.status !== 0) {
const output = [result.stderr, result.stdout].filter(Boolean).join("\n").trim();
throw new Error(
`failed to stage bundled runtime deps for ${pluginId}: ${output || "npm install failed"}`,
);
}
const stagedNodeModulesDir = path.join(tempInstallDir, "node_modules");
if (!fs.existsSync(stagedNodeModulesDir)) {
throw new Error(
`failed to stage bundled runtime deps for ${pluginId}: npm install produced no node_modules directory`,
);
}
pruneStagedRuntimeDependencyCargo(stagedNodeModulesDir, pruneConfig);
replaceDir(nodeModulesDir, stagedNodeModulesDir);
writeJson(stampPath, {
fingerprint,
generatedAt: new Date().toISOString(),
});
} finally {
removePathIfExists(tempInstallDir);
}
}
function installPluginRuntimeDepsWithRetries(params) {
const { attempts = 3 } = params;
let lastError;
for (let attempt = 1; attempt <= attempts; attempt += 1) {
try {
params.install({ ...params.installParams, attempt });
return;
} catch (error) {
lastError = error;
if (attempt === attempts) {
break;
}
}
}
throw lastError;
}
export function stageBundledPluginRuntimeDeps(params = {}) {
const repoRoot = params.cwd ?? params.repoRoot ?? process.cwd();
const installPluginRuntimeDepsImpl =
params.installPluginRuntimeDepsImpl ?? installPluginRuntimeDeps;
const installAttempts = params.installAttempts ?? 3;
const pruneConfig = resolveRuntimeDepPruneConfig(params);
for (const pluginDir of listBundledPluginRuntimeDirs(repoRoot)) {
const pluginId = path.basename(pluginDir);
const packageJson = sanitizeBundledManifestForRuntimeInstall(pluginDir);
const nodeModulesDir = path.join(pluginDir, "node_modules");
const stampPath = resolveRuntimeDepsStampPath(pluginDir);
if (!hasRuntimeDeps(packageJson) || !shouldStageRuntimeDeps(packageJson)) {
removePathIfExists(nodeModulesDir);
removePathIfExists(stampPath);
continue;
}
const rootInstalledRuntimeFingerprint = resolveInstalledRuntimeClosureFingerprint({
packageJson,
rootNodeModulesDir: path.join(repoRoot, "node_modules"),
});
const fingerprint = createRuntimeDepsFingerprint(packageJson, pruneConfig, {
repoRoot,
rootInstalledRuntimeFingerprint,
});
const stamp = readRuntimeDepsStamp(stampPath);
if (fs.existsSync(nodeModulesDir) && stamp?.fingerprint === fingerprint) {
continue;
}
installPluginRuntimeDepsWithRetries({
attempts: installAttempts,
install: installPluginRuntimeDepsImpl,
installParams: {
fingerprint,
packageJson,
pluginDir,
pluginId,
pruneConfig,
repoRoot,
},
});
}
}
if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) {
stageBundledPluginRuntimeDeps();
}