From b53ec93ed9d36aee5ca232de60e390f9a3251730 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Wed, 29 Apr 2026 17:43:42 +0100 Subject: [PATCH] refactor(plugins): split bundled runtime deps staging script --- .../lib/bundled-runtime-deps-materialize.mjs | 209 ++++ .../lib/bundled-runtime-deps-package-tree.mjs | 273 +++++ scripts/lib/bundled-runtime-deps-prune.mjs | 198 ++++ .../lib/bundled-runtime-deps-stage-state.mjs | 188 ++++ scripts/lib/bundled-runtime-deps-stamp.mjs | 76 ++ scripts/stage-bundled-plugin-runtime-deps.mjs | 944 +----------------- 6 files changed, 977 insertions(+), 911 deletions(-) create mode 100644 scripts/lib/bundled-runtime-deps-materialize.mjs create mode 100644 scripts/lib/bundled-runtime-deps-package-tree.mjs create mode 100644 scripts/lib/bundled-runtime-deps-prune.mjs create mode 100644 scripts/lib/bundled-runtime-deps-stage-state.mjs create mode 100644 scripts/lib/bundled-runtime-deps-stamp.mjs diff --git a/scripts/lib/bundled-runtime-deps-materialize.mjs b/scripts/lib/bundled-runtime-deps-materialize.mjs new file mode 100644 index 00000000000..d3004b6751e --- /dev/null +++ b/scripts/lib/bundled-runtime-deps-materialize.mjs @@ -0,0 +1,209 @@ +import fs from "node:fs"; +import path from "node:path"; +import { + collectInstalledRuntimeDependencyRoots, + dependencyNodeModulesPath, + findContainingRealRoot, + resolveInstalledDirectDependencyNames, + selectRuntimeDependencyRootsToCopy, +} from "./bundled-runtime-deps-package-tree.mjs"; +import { pruneStagedRuntimeDependencyCargo } from "./bundled-runtime-deps-prune.mjs"; +import { + assertPathIsNotSymlink, + makePluginOwnedTempDir, + removeOwnedTempPathBestEffort, + removePathIfExists, + replaceDirAtomically, + writeJsonAtomically, +} from "./bundled-runtime-deps-stage-state.mjs"; + +function copyMaterializedDependencyTree(params) { + const { activeRoots, allowedRealRoots, sourcePath, targetPath } = params; + const sourceStats = fs.lstatSync(sourcePath); + + if (sourceStats.isSymbolicLink()) { + let resolvedPath; + try { + resolvedPath = fs.realpathSync(sourcePath); + } catch { + return false; + } + const containingRoot = findContainingRealRoot(resolvedPath, allowedRealRoots); + if (containingRoot === null) { + return false; + } + if (activeRoots.has(containingRoot)) { + return true; + } + const nextActiveRoots = new Set(activeRoots); + nextActiveRoots.add(containingRoot); + return copyMaterializedDependencyTree({ + activeRoots: nextActiveRoots, + allowedRealRoots, + sourcePath: resolvedPath, + targetPath, + }); + } + + if (sourceStats.isDirectory()) { + fs.mkdirSync(targetPath, { recursive: true }); + for (const entry of fs + .readdirSync(sourcePath, { withFileTypes: true }) + .toSorted((left, right) => left.name.localeCompare(right.name))) { + if ( + !copyMaterializedDependencyTree({ + activeRoots, + allowedRealRoots, + sourcePath: path.join(sourcePath, entry.name), + targetPath: path.join(targetPath, entry.name), + }) + ) { + return false; + } + } + return true; + } + + if (sourceStats.isFile()) { + fs.mkdirSync(path.dirname(targetPath), { recursive: true }); + fs.copyFileSync(sourcePath, targetPath); + fs.chmodSync(targetPath, sourceStats.mode); + return true; + } + + return true; +} + +export function listBundledPluginRuntimeDirs(repoRoot) { + const extensionsRoot = path.join(repoRoot, "dist", "extensions"); + if (!fs.existsSync(extensionsRoot)) { + return []; + } + + return fs + .readdirSync(extensionsRoot, { withFileTypes: true }) + .filter((dirent) => dirent.isDirectory()) + .map((dirent) => path.join(extensionsRoot, dirent.name)) + .filter((pluginDir) => fs.existsSync(path.join(pluginDir, "package.json"))); +} + +export function resolveInstalledWorkspacePluginRoot(repoRoot, pluginId) { + const currentPluginRoot = path.join(repoRoot, "extensions", pluginId); + if (fs.existsSync(path.join(currentPluginRoot, "node_modules"))) { + return currentPluginRoot; + } + + const nodeModulesDir = path.join(repoRoot, "node_modules"); + if (!fs.existsSync(nodeModulesDir)) { + return currentPluginRoot; + } + + let installedWorkspaceRoot; + try { + installedWorkspaceRoot = path.dirname(fs.realpathSync(nodeModulesDir)); + } catch { + return currentPluginRoot; + } + + const installedPluginRoot = path.join(installedWorkspaceRoot, "extensions", pluginId); + if (fs.existsSync(path.join(installedPluginRoot, "package.json"))) { + return installedPluginRoot; + } + + return currentPluginRoot; +} + +export function stageInstalledRootRuntimeDeps(params) { + const { + directDependencyPackageRoot = null, + cheapFingerprint, + fingerprint, + packageJson, + pluginDir, + pruneConfig, + repoRoot, + stampPath, + } = params; + const dependencySpecs = { + ...packageJson.dependencies, + ...packageJson.optionalDependencies, + }; + const optionalDependencyNames = new Set(Object.keys(packageJson.optionalDependencies ?? {})); + const rootNodeModulesDir = path.join(repoRoot, "node_modules"); + if (Object.keys(dependencySpecs).length === 0 || !fs.existsSync(rootNodeModulesDir)) { + return false; + } + + const directDependencyNames = resolveInstalledDirectDependencyNames( + rootNodeModulesDir, + dependencySpecs, + directDependencyPackageRoot, + optionalDependencyNames, + ); + if (directDependencyNames === null) { + return false; + } + const resolution = collectInstalledRuntimeDependencyRoots( + rootNodeModulesDir, + dependencySpecs, + directDependencyPackageRoot, + optionalDependencyNames, + ); + if (resolution === null) { + return false; + } + const rootsToCopy = selectRuntimeDependencyRootsToCopy(resolution); + const nodeModulesDir = path.join(pluginDir, "node_modules"); + if (rootsToCopy.length === 0) { + assertPathIsNotSymlink(nodeModulesDir, "remove runtime deps"); + removePathIfExists(nodeModulesDir); + writeJsonAtomically(stampPath, { + cheapFingerprint, + fingerprint, + generatedAt: new Date().toISOString(), + }); + return true; + } + const allowedRealRoots = rootsToCopy.map((record) => record.realRoot); + + const stagedNodeModulesDir = path.join( + makePluginOwnedTempDir(pluginDir, "stage"), + "node_modules", + ); + + try { + for (const record of rootsToCopy.toSorted((left, right) => + left.name.localeCompare(right.name), + )) { + const sourcePath = record.realRoot; + const targetPath = dependencyNodeModulesPath(stagedNodeModulesDir, record.name); + if (targetPath === null) { + return false; + } + fs.mkdirSync(path.dirname(targetPath), { recursive: true }); + const sourceRootReal = findContainingRealRoot(sourcePath, allowedRealRoots); + if ( + sourceRootReal === null || + !copyMaterializedDependencyTree({ + activeRoots: new Set([sourceRootReal]), + allowedRealRoots, + sourcePath, + targetPath, + }) + ) { + return false; + } + } + pruneStagedRuntimeDependencyCargo(stagedNodeModulesDir, pruneConfig); + + replaceDirAtomically(nodeModulesDir, stagedNodeModulesDir); + writeJsonAtomically(stampPath, { + cheapFingerprint, + fingerprint, + generatedAt: new Date().toISOString(), + }); + return true; + } finally { + removeOwnedTempPathBestEffort(path.dirname(stagedNodeModulesDir)); + } +} diff --git a/scripts/lib/bundled-runtime-deps-package-tree.mjs b/scripts/lib/bundled-runtime-deps-package-tree.mjs new file mode 100644 index 00000000000..766f63ecbf4 --- /dev/null +++ b/scripts/lib/bundled-runtime-deps-package-tree.mjs @@ -0,0 +1,273 @@ +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; +import semverSatisfies from "semver/functions/satisfies.js"; + +function readJson(filePath) { + return JSON.parse(fs.readFileSync(filePath, "utf8")); +} + +function dependencyPathSegments(depName) { + if (typeof depName !== "string" || depName.length === 0) { + return null; + } + const segments = depName.split("/"); + if (depName.startsWith("@")) { + if (segments.length !== 2) { + return null; + } + const [scope, name] = segments; + if ( + !/^@[A-Za-z0-9._-]+$/.test(scope) || + !/^[A-Za-z0-9._-]+$/.test(name) || + scope === "@." || + scope === "@.." + ) { + return null; + } + return [scope, name]; + } + if (segments.length !== 1 || !/^[A-Za-z0-9._-]+$/.test(segments[0])) { + return null; + } + return segments; +} + +export function dependencyNodeModulesPath(nodeModulesDir, depName) { + const segments = dependencyPathSegments(depName); + return segments ? path.join(nodeModulesDir, ...segments) : null; +} + +function dependencyVersionSatisfied(spec, installedVersion) { + return semverSatisfies(installedVersion, spec, { includePrerelease: false }); +} + +export function readInstalledDependencyVersionFromRoot(depRoot) { + const packageJsonPath = path.join(depRoot, "package.json"); + if (!fs.existsSync(packageJsonPath)) { + return null; + } + const version = readJson(packageJsonPath).version; + return typeof version === "string" ? version : null; +} + +export function resolveInstalledDependencyRoot(params) { + const candidates = []; + if (params.parentPackageRoot) { + const nestedDepRoot = dependencyNodeModulesPath( + path.join(params.parentPackageRoot, "node_modules"), + params.depName, + ); + if (nestedDepRoot !== null) { + candidates.push(nestedDepRoot); + } + } + const rootDepRoot = dependencyNodeModulesPath(params.rootNodeModulesDir, params.depName); + if (rootDepRoot !== null) { + candidates.push(rootDepRoot); + } + + for (const depRoot of candidates) { + const installedVersion = readInstalledDependencyVersionFromRoot(depRoot); + if (installedVersion === null) { + continue; + } + if (params.enforceSpec === false || dependencyVersionSatisfied(params.spec, installedVersion)) { + return depRoot; + } + } + + return null; +} + +export function collectInstalledRuntimeDependencyRoots( + rootNodeModulesDir, + dependencySpecs, + directDependencyPackageRoot = null, + optionalDependencyNames = new Set(), +) { + const packageCache = new Map(); + const directRoots = []; + const allRoots = []; + const queue = Object.entries(dependencySpecs).map(([depName, spec]) => ({ + depName, + optional: optionalDependencyNames.has(depName), + spec, + parentPackageRoot: directDependencyPackageRoot, + direct: true, + })); + const seen = new Set(); + + while (queue.length > 0) { + const current = queue.shift(); + const depRoot = resolveInstalledDependencyRoot({ + depName: current.depName, + spec: current.spec, + enforceSpec: current.direct, + parentPackageRoot: current.parentPackageRoot, + rootNodeModulesDir, + }); + if (depRoot === null) { + if (current.optional) { + continue; + } + return null; + } + const canonicalDepRoot = fs.realpathSync(depRoot); + + const seenKey = `${current.depName}\0${canonicalDepRoot}`; + if (seen.has(seenKey)) { + continue; + } + seen.add(seenKey); + + const record = { name: current.depName, root: depRoot, realRoot: canonicalDepRoot }; + allRoots.push(record); + if (current.direct) { + directRoots.push(record); + } + + const packageJson = + packageCache.get(canonicalDepRoot) ?? readJson(path.join(depRoot, "package.json")); + packageCache.set(canonicalDepRoot, packageJson); + for (const [childName, childSpec] of Object.entries(packageJson.dependencies ?? {})) { + queue.push({ + depName: childName, + optional: false, + spec: childSpec, + parentPackageRoot: depRoot, + direct: false, + }); + } + for (const [childName, childSpec] of Object.entries(packageJson.optionalDependencies ?? {})) { + queue.push({ + depName: childName, + optional: true, + spec: childSpec, + parentPackageRoot: depRoot, + direct: false, + }); + } + } + + return { allRoots, directRoots }; +} + +function pathIsInsideCopiedRoot(candidateRoot, copiedRoot) { + return candidateRoot === copiedRoot || candidateRoot.startsWith(`${copiedRoot}${path.sep}`); +} + +export function findContainingRealRoot(candidatePath, allowedRealRoots) { + return ( + allowedRealRoots.find((rootPath) => pathIsInsideCopiedRoot(candidatePath, rootPath)) ?? null + ); +} + +export function selectRuntimeDependencyRootsToCopy(resolution) { + const rootsToCopy = []; + + for (const record of resolution.directRoots) { + rootsToCopy.push(record); + } + + for (const record of resolution.allRoots) { + if (rootsToCopy.some((entry) => pathIsInsideCopiedRoot(record.realRoot, entry.realRoot))) { + continue; + } + rootsToCopy.push(record); + } + + return rootsToCopy; +} + +export function resolveInstalledDirectDependencyNames( + rootNodeModulesDir, + dependencySpecs, + directDependencyPackageRoot = null, + optionalDependencyNames = new Set(), +) { + const directDependencyNames = []; + for (const [depName, spec] of Object.entries(dependencySpecs)) { + const depRoot = resolveInstalledDependencyRoot({ + depName, + spec, + parentPackageRoot: directDependencyPackageRoot, + rootNodeModulesDir, + }); + if (depRoot === null) { + if (optionalDependencyNames.has(depName)) { + continue; + } + return null; + } + const installedVersion = readInstalledDependencyVersionFromRoot(depRoot); + if (installedVersion === null || !dependencyVersionSatisfied(spec, installedVersion)) { + return null; + } + directDependencyNames.push(depName); + } + return directDependencyNames; +} + +function appendDirectoryFingerprint(hash, rootDir, currentDir = rootDir) { + const entries = fs + .readdirSync(currentDir, { withFileTypes: true }) + .toSorted((left, right) => left.name.localeCompare(right.name)); + + for (const entry of entries) { + const fullPath = path.join(currentDir, entry.name); + const relativePath = path.relative(rootDir, fullPath).replace(/\\/g, "/"); + const stats = fs.lstatSync(fullPath); + if (stats.isSymbolicLink()) { + hash.update(`symlink:${relativePath}->${fs.readlinkSync(fullPath).replace(/\\/g, "/")}\n`); + continue; + } + if (stats.isDirectory()) { + hash.update(`dir:${relativePath}\n`); + appendDirectoryFingerprint(hash, rootDir, fullPath); + continue; + } + if (!stats.isFile()) { + continue; + } + const stat = fs.statSync(fullPath); + hash.update(`file:${relativePath}:${stat.size}\n`); + hash.update(fs.readFileSync(fullPath)); + } +} + +function createInstalledRuntimeClosureFingerprint(rootNodeModulesDir, dependencyNames) { + const hash = createHash("sha256"); + for (const depName of [...dependencyNames].toSorted((left, right) => left.localeCompare(right))) { + const depRoot = dependencyNodeModulesPath(rootNodeModulesDir, depName); + if (depRoot === null || !fs.existsSync(depRoot)) { + return null; + } + hash.update(`package:${depName}:${fs.realpathSync(depRoot)}\n`); + appendDirectoryFingerprint(hash, depRoot); + } + return hash.digest("hex"); +} + +export function resolveInstalledRuntimeClosureFingerprint(params) { + const dependencySpecs = { + ...params.packageJson.dependencies, + ...params.packageJson.optionalDependencies, + }; + if (Object.keys(dependencySpecs).length === 0 || !fs.existsSync(params.rootNodeModulesDir)) { + return null; + } + const resolution = collectInstalledRuntimeDependencyRoots( + params.rootNodeModulesDir, + dependencySpecs, + params.directDependencyPackageRoot, + new Set(Object.keys(params.packageJson.optionalDependencies ?? {})), + ); + if (resolution === null) { + return null; + } + return createInstalledRuntimeClosureFingerprint( + params.rootNodeModulesDir, + selectRuntimeDependencyRootsToCopy(resolution).map((record) => record.name), + ); +} diff --git a/scripts/lib/bundled-runtime-deps-prune.mjs b/scripts/lib/bundled-runtime-deps-prune.mjs new file mode 100644 index 00000000000..ee9bb64ecd4 --- /dev/null +++ b/scripts/lib/bundled-runtime-deps-prune.mjs @@ -0,0 +1,198 @@ +import fs from "node:fs"; +import path from "node:path"; +import { dependencyNodeModulesPath } from "./bundled-runtime-deps-package-tree.mjs"; +import { removePathIfExists } from "./bundled-runtime-deps-stage-state.mjs"; + +const defaultStagedRuntimeDepGlobalPruneSuffixes = [".d.ts", ".map"]; +const defaultStagedRuntimeDepGlobalPruneDirectories = [ + "__snapshots__", + "__tests__", + "test", + "tests", +]; +const defaultStagedRuntimeDepGlobalPruneFilePatterns = [ + /(?:^|\/)[^/]+\.(?:test|spec)\.(?:[cm]?[jt]sx?)$/u, +]; +const defaultStagedRuntimeDepPruneRules = new Map([ + ["@larksuiteoapi/node-sdk", { paths: ["types"] }], + [ + "@matrix-org/matrix-sdk-crypto-nodejs", + { + paths: ["index.d.ts", "README.md", "CHANGELOG.md", "RELEASING.md", ".node-version"], + }, + ], + [ + "@matrix-org/matrix-sdk-crypto-wasm", + { + paths: [ + "index.d.ts", + "pkg/matrix_sdk_crypto_wasm.d.ts", + "pkg/matrix_sdk_crypto_wasm_bg.wasm.d.ts", + "README.md", + ], + }, + ], + [ + "matrix-js-sdk", + { + paths: ["src", "CHANGELOG.md", "CONTRIBUTING.rst", "README.md", "release.sh"], + suffixes: [".d.ts"], + }, + ], + ["matrix-widget-api", { paths: ["src"], suffixes: [".d.ts"] }], + ["oidc-client-ts", { paths: ["README.md"], suffixes: [".d.ts"] }], + ["music-metadata", { paths: ["README.md"], suffixes: [".d.ts"] }], + ["@cloudflare/workers-types", { paths: ["."] }], + ["gifwrap", { paths: ["test"] }], + ["playwright-core", { paths: ["types"], suffixes: [".d.ts"] }], + ["@jimp/plugin-blit", { paths: ["src/__image_snapshots__"] }], + ["@jimp/plugin-blur", { paths: ["src/__image_snapshots__"] }], + ["@jimp/plugin-color", { paths: ["src/__image_snapshots__"] }], + ["@jimp/plugin-print", { paths: ["src/__image_snapshots__"] }], + ["@jimp/plugin-quantize", { paths: ["src/__image_snapshots__"] }], + ["@jimp/plugin-threshold", { paths: ["src/__image_snapshots__"] }], + ["tokenjuice", { keepDirectories: ["dist/rules/tests"] }], +]); + +export function resolveRuntimeDepPruneConfig(params = {}) { + return { + globalPruneDirectories: + params.stagedRuntimeDepGlobalPruneDirectories ?? + defaultStagedRuntimeDepGlobalPruneDirectories, + globalPruneFilePatterns: + params.stagedRuntimeDepGlobalPruneFilePatterns ?? + defaultStagedRuntimeDepGlobalPruneFilePatterns, + globalPruneSuffixes: + params.stagedRuntimeDepGlobalPruneSuffixes ?? defaultStagedRuntimeDepGlobalPruneSuffixes, + pruneRules: params.stagedRuntimeDepPruneRules ?? defaultStagedRuntimeDepPruneRules, + }; +} + +function walkFiles(rootDir, visitFile) { + if (!fs.existsSync(rootDir)) { + return; + } + const queue = [rootDir]; + for (let index = 0; index < queue.length; index += 1) { + const currentDir = queue[index]; + for (const entry of fs.readdirSync(currentDir, { withFileTypes: true })) { + const fullPath = path.join(currentDir, entry.name); + if (entry.isDirectory()) { + queue.push(fullPath); + continue; + } + if (entry.isFile()) { + visitFile(fullPath); + } + } + } +} + +function pruneDependencyFilesBySuffixes(depRoot, suffixes) { + if (!suffixes || suffixes.length === 0 || !fs.existsSync(depRoot)) { + return; + } + walkFiles(depRoot, (fullPath) => { + if (suffixes.some((suffix) => fullPath.endsWith(suffix))) { + removePathIfExists(fullPath); + } + }); +} + +function relativePathSegments(rootDir, fullPath) { + return path.relative(rootDir, fullPath).split(path.sep).filter(Boolean); +} + +function isNodeModulesPackageRoot(segments, index) { + const parent = segments[index - 1]; + if (parent === "node_modules") { + return true; + } + return parent?.startsWith("@") === true && segments[index - 2] === "node_modules"; +} + +function pruneDependencyDirectoriesByBasename(depRoot, basenames, keepDirs = new Set()) { + if (!basenames || basenames.length === 0 || !fs.existsSync(depRoot)) { + return; + } + const basenameSet = new Set(basenames); + const queue = [depRoot]; + for (let index = 0; index < queue.length; index += 1) { + const currentDir = queue[index]; + for (const entry of fs.readdirSync(currentDir, { withFileTypes: true })) { + if (!entry.isDirectory()) { + continue; + } + const fullPath = path.join(currentDir, entry.name); + const segments = relativePathSegments(depRoot, fullPath); + if (basenameSet.has(entry.name) && !isNodeModulesPackageRoot(segments, segments.length - 1)) { + if (keepDirs.has(fullPath)) { + queue.push(fullPath); + continue; + } + removePathIfExists(fullPath); + continue; + } + queue.push(fullPath); + } + } +} + +function pruneDependencyFilesByPatterns(depRoot, patterns) { + if (!patterns || patterns.length === 0 || !fs.existsSync(depRoot)) { + return; + } + walkFiles(depRoot, (fullPath) => { + const relativePath = relativePathSegments(depRoot, fullPath).join("/"); + if (patterns.some((pattern) => pattern.test(relativePath))) { + removePathIfExists(fullPath); + } + }); +} + +function pruneStagedInstalledDependencyCargo(nodeModulesDir, depName, pruneConfig) { + const depRoot = dependencyNodeModulesPath(nodeModulesDir, depName); + if (depRoot === null) { + return; + } + const pruneRule = pruneConfig.pruneRules.get(depName); + for (const relativePath of pruneRule?.paths ?? []) { + removePathIfExists(path.join(depRoot, relativePath)); + } + const keepDirs = new Set( + (pruneRule?.keepDirectories ?? []).map((relativePath) => path.resolve(depRoot, relativePath)), + ); + pruneDependencyDirectoriesByBasename(depRoot, pruneConfig.globalPruneDirectories, keepDirs); + pruneDependencyFilesByPatterns(depRoot, pruneConfig.globalPruneFilePatterns); + pruneDependencyFilesBySuffixes(depRoot, pruneConfig.globalPruneSuffixes); + pruneDependencyFilesBySuffixes(depRoot, pruneRule?.suffixes ?? []); +} + +function listInstalledDependencyNames(nodeModulesDir) { + if (!fs.existsSync(nodeModulesDir)) { + return []; + } + const names = []; + for (const entry of fs.readdirSync(nodeModulesDir, { withFileTypes: true })) { + if (!entry.isDirectory()) { + continue; + } + if (entry.name.startsWith("@")) { + const scopeDir = path.join(nodeModulesDir, entry.name); + for (const scopedEntry of fs.readdirSync(scopeDir, { withFileTypes: true })) { + if (scopedEntry.isDirectory()) { + names.push(`${entry.name}/${scopedEntry.name}`); + } + } + continue; + } + names.push(entry.name); + } + return names; +} + +export function pruneStagedRuntimeDependencyCargo(nodeModulesDir, pruneConfig) { + for (const depName of listInstalledDependencyNames(nodeModulesDir)) { + pruneStagedInstalledDependencyCargo(nodeModulesDir, depName, pruneConfig); + } +} diff --git a/scripts/lib/bundled-runtime-deps-stage-state.mjs b/scripts/lib/bundled-runtime-deps-stage-state.mjs new file mode 100644 index 00000000000..1349b8baaea --- /dev/null +++ b/scripts/lib/bundled-runtime-deps-stage-state.mjs @@ -0,0 +1,188 @@ +import fs from "node:fs"; +import path from "node:path"; + +const TRANSIENT_TEMP_REMOVE_ERROR_CODES = new Set(["EBUSY", "ENOTEMPTY", "EPERM"]); +const TEMP_REMOVE_RETRY_DELAYS_MS = [10, 25, 50]; +const TEMP_OWNER_FILE = "owner.json"; + +function readJson(filePath) { + return JSON.parse(fs.readFileSync(filePath, "utf8")); +} + +function writeJson(filePath, value) { + fs.writeFileSync(filePath, `${JSON.stringify(value, null, 2)}\n`, "utf8"); +} + +export function removePathIfExists(targetPath, options = {}) { + const retryDelays = options.retryTransient ? TEMP_REMOVE_RETRY_DELAYS_MS : []; + for (let attempt = 0; attempt <= retryDelays.length; attempt += 1) { + try { + fs.rmSync(targetPath, { recursive: true, force: true }); + return true; + } catch (error) { + if (!isTransientTempRemoveError(error)) { + throw error; + } + const delay = retryDelays[attempt]; + if (delay === undefined) { + if (options.ignoreTransient) { + return false; + } + throw error; + } + sleepSync(delay); + } + } + return true; +} + +export function removeOwnedTempPathBestEffort(targetPath) { + return removePathIfExists(targetPath, { retryTransient: true, ignoreTransient: true }); +} + +function isTransientTempRemoveError(error) { + return ( + !!error && + typeof error === "object" && + typeof error.code === "string" && + TRANSIENT_TEMP_REMOVE_ERROR_CODES.has(error.code) + ); +} + +function sleepSync(ms) { + if (!Number.isFinite(ms) || ms <= 0) { + return; + } + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms); +} + +function makeTempDir(parentDir, prefix) { + return fs.mkdtempSync(path.join(parentDir, prefix)); +} + +export function writeRuntimeDepsTempOwner(tempDir) { + writeJson(path.join(tempDir, TEMP_OWNER_FILE), { + pid: process.pid, + createdAtMs: Date.now(), + }); +} + +function makeOwnedTempDir(parentDir, prefix) { + const tempDir = makeTempDir(parentDir, prefix); + writeRuntimeDepsTempOwner(tempDir); + return tempDir; +} + +export function sanitizeTempPrefixSegment(value) { + const normalized = value.replace(/[^A-Za-z0-9._-]+/g, "-").replace(/-+/g, "-"); + return normalized.length > 0 ? normalized : "plugin"; +} + +export function makePluginOwnedTempDir(pluginDir, label) { + return makeOwnedTempDir(pluginDir, `.openclaw-runtime-deps-${label}-`); +} + +export function assertPathIsNotSymlink(targetPath, label) { + try { + if (fs.lstatSync(targetPath).isSymbolicLink()) { + throw new Error(`refusing to ${label} via symlinked path: ${targetPath}`); + } + } catch (error) { + if (error?.code === "ENOENT") { + return; + } + throw error; + } +} + +export function replaceDirAtomically(targetPath, sourcePath) { + assertPathIsNotSymlink(targetPath, "replace runtime deps"); + const targetParentDir = path.dirname(targetPath); + fs.mkdirSync(targetParentDir, { recursive: true }); + const backupPath = makeTempDir( + targetParentDir, + `.openclaw-runtime-deps-backup-${sanitizeTempPrefixSegment(path.basename(targetPath))}-`, + ); + removePathIfExists(backupPath, { retryTransient: true }); + + let movedExistingTarget = false; + try { + if (fs.existsSync(targetPath)) { + fs.renameSync(targetPath, backupPath); + writeRuntimeDepsTempOwner(backupPath); + movedExistingTarget = true; + } + fs.renameSync(sourcePath, targetPath); + removeOwnedTempPathBestEffort(backupPath); + } catch (error) { + if (movedExistingTarget && !fs.existsSync(targetPath) && fs.existsSync(backupPath)) { + fs.renameSync(backupPath, targetPath); + removePathIfExists(path.join(targetPath, TEMP_OWNER_FILE)); + } + throw error; + } +} + +export function writeJsonAtomically(targetPath, value) { + assertPathIsNotSymlink(targetPath, "write runtime deps stamp"); + const targetParentDir = path.dirname(targetPath); + fs.mkdirSync(targetParentDir, { recursive: true }); + const tempDir = makeOwnedTempDir( + targetParentDir, + `.openclaw-runtime-deps-stamp-${sanitizeTempPrefixSegment(path.basename(targetPath))}-`, + ); + const tempPath = path.join(tempDir, path.basename(targetPath)); + try { + fs.writeFileSync(tempPath, `${JSON.stringify(value, null, 2)}\n`, { + encoding: "utf8", + flag: "wx", + }); + fs.renameSync(tempPath, targetPath); + } finally { + removeOwnedTempPathBestEffort(tempDir); + } +} + +function readRuntimeDepsTempOwner(tempDir) { + try { + const owner = readJson(path.join(tempDir, TEMP_OWNER_FILE)); + return owner && typeof owner === "object" ? owner : null; + } catch { + return null; + } +} + +function isLiveProcess(pid) { + if (!Number.isInteger(pid) || pid <= 0) { + return false; + } + try { + process.kill(pid, 0); + return true; + } catch (error) { + return error?.code === "EPERM"; + } +} + +function shouldRemoveRuntimeDepsTempDir(tempDir) { + const owner = readRuntimeDepsTempOwner(tempDir); + if (!owner || typeof owner.pid !== "number") { + return true; + } + return !isLiveProcess(owner.pid); +} + +export function removeStaleRuntimeDepsTempDirs(pluginDir) { + if (!fs.existsSync(pluginDir)) { + return; + } + for (const entry of fs.readdirSync(pluginDir, { withFileTypes: true })) { + if (entry.name.startsWith(".openclaw-runtime-deps-")) { + const targetPath = path.join(pluginDir, entry.name); + if (!shouldRemoveRuntimeDepsTempDir(targetPath)) { + continue; + } + removeOwnedTempPathBestEffort(targetPath); + } + } +} diff --git a/scripts/lib/bundled-runtime-deps-stamp.mjs b/scripts/lib/bundled-runtime-deps-stamp.mjs new file mode 100644 index 00000000000..52d2d5b89da --- /dev/null +++ b/scripts/lib/bundled-runtime-deps-stamp.mjs @@ -0,0 +1,76 @@ +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; +import { sanitizeTempPrefixSegment } from "./bundled-runtime-deps-stage-state.mjs"; + +const runtimeDepsStagingVersion = 7; + +function readJson(filePath) { + return JSON.parse(fs.readFileSync(filePath, "utf8")); +} + +function readOptionalUtf8(filePath) { + if (!fs.existsSync(filePath)) { + return null; + } + return fs.readFileSync(filePath, "utf8"); +} + +export function resolveLegacyRuntimeDepsStampPath(pluginDir) { + return path.join(pluginDir, ".openclaw-runtime-deps-stamp.json"); +} + +export function resolveRuntimeDepsStampPath(repoRoot, pluginId) { + return path.join( + repoRoot, + ".artifacts", + "bundled-runtime-deps-stamps", + `${sanitizeTempPrefixSegment(pluginId)}.json`, + ); +} + +export function createRuntimeDepsFingerprint(packageJson, pruneConfig, params = {}) { + return createHash("sha256") + .update( + JSON.stringify({ + cheapFingerprint: createRuntimeDepsCheapFingerprint(packageJson, pruneConfig, params), + rootInstalledRuntimeFingerprint: params.rootInstalledRuntimeFingerprint ?? null, + }), + ) + .digest("hex"); +} + +export function createRuntimeDepsCheapFingerprint(packageJson, pruneConfig, params = {}) { + const repoRoot = params.repoRoot; + const lockfilePath = + typeof repoRoot === "string" && repoRoot.length > 0 + ? path.join(repoRoot, "pnpm-lock.yaml") + : null; + const rootLockfile = lockfilePath ? readOptionalUtf8(lockfilePath) : null; + return createHash("sha256") + .update( + JSON.stringify({ + globalPruneDirectories: pruneConfig.globalPruneDirectories, + globalPruneFilePatterns: pruneConfig.globalPruneFilePatterns.map((pattern) => + pattern.toString(), + ), + globalPruneSuffixes: pruneConfig.globalPruneSuffixes, + packageJson, + pruneRules: [...pruneConfig.pruneRules.entries()], + rootLockfile, + version: runtimeDepsStagingVersion, + }), + ) + .digest("hex"); +} + +export function readRuntimeDepsStamp(stampPath) { + if (!fs.existsSync(stampPath)) { + return null; + } + try { + return readJson(stampPath); + } catch { + return null; + } +} diff --git a/scripts/stage-bundled-plugin-runtime-deps.mjs b/scripts/stage-bundled-plugin-runtime-deps.mjs index 02a37e0a289..87f7756d903 100644 --- a/scripts/stage-bundled-plugin-runtime-deps.mjs +++ b/scripts/stage-bundled-plugin-runtime-deps.mjs @@ -1,19 +1,47 @@ -import { createHash } from "node:crypto"; import fs from "node:fs"; import path from "node:path"; import { performance } from "node:perf_hooks"; import { pathToFileURL } from "node:url"; -import semverSatisfies from "semver/functions/satisfies.js"; import { createBundledRuntimeDependencyInstallArgs, createBundledRuntimeDependencyInstallEnv, runBundledRuntimeDependencyNpmInstall, } from "./lib/bundled-runtime-deps-install.mjs"; +import { + listBundledPluginRuntimeDirs, + resolveInstalledWorkspacePluginRoot, + stageInstalledRootRuntimeDeps, +} from "./lib/bundled-runtime-deps-materialize.mjs"; +import { + readInstalledDependencyVersionFromRoot, + resolveInstalledDependencyRoot, + resolveInstalledRuntimeClosureFingerprint, +} from "./lib/bundled-runtime-deps-package-tree.mjs"; +import { + pruneStagedRuntimeDependencyCargo, + resolveRuntimeDepPruneConfig, +} from "./lib/bundled-runtime-deps-prune.mjs"; +import { + assertPathIsNotSymlink, + makePluginOwnedTempDir, + removeOwnedTempPathBestEffort, + removePathIfExists, + removeStaleRuntimeDepsTempDirs, + replaceDirAtomically, + sanitizeTempPrefixSegment, + writeJsonAtomically, + writeRuntimeDepsTempOwner, +} from "./lib/bundled-runtime-deps-stage-state.mjs"; +import { + createRuntimeDepsCheapFingerprint, + createRuntimeDepsFingerprint, + readRuntimeDepsStamp, + resolveLegacyRuntimeDepsStampPath, + resolveRuntimeDepsStampPath, +} from "./lib/bundled-runtime-deps-stamp.mjs"; import { resolveNpmRunner } from "./npm-runner.mjs"; -const TRANSIENT_TEMP_REMOVE_ERROR_CODES = new Set(["EBUSY", "ENOTEMPTY", "EPERM"]); -const TEMP_REMOVE_RETRY_DELAYS_MS = [10, 25, 50]; -const TEMP_OWNER_FILE = "owner.json"; +const exactVersionSpecRe = /^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?$/u; function readJson(filePath) { return JSON.parse(fs.readFileSync(filePath, "utf8")); @@ -23,714 +51,6 @@ function writeJson(filePath, value) { fs.writeFileSync(filePath, `${JSON.stringify(value, null, 2)}\n`, "utf8"); } -function readOptionalUtf8(filePath) { - if (!fs.existsSync(filePath)) { - return null; - } - return fs.readFileSync(filePath, "utf8"); -} - -function removePathIfExists(targetPath, options = {}) { - const retryDelays = options.retryTransient ? TEMP_REMOVE_RETRY_DELAYS_MS : []; - for (let attempt = 0; attempt <= retryDelays.length; attempt += 1) { - try { - fs.rmSync(targetPath, { recursive: true, force: true }); - return true; - } catch (error) { - if (!isTransientTempRemoveError(error)) { - throw error; - } - const delay = retryDelays[attempt]; - if (delay === undefined) { - if (options.ignoreTransient) { - return false; - } - throw error; - } - sleepSync(delay); - } - } - return true; -} - -function removeOwnedTempPathBestEffort(targetPath) { - return removePathIfExists(targetPath, { retryTransient: true, ignoreTransient: true }); -} - -function isTransientTempRemoveError(error) { - return ( - !!error && - typeof error === "object" && - typeof error.code === "string" && - TRANSIENT_TEMP_REMOVE_ERROR_CODES.has(error.code) - ); -} - -function sleepSync(ms) { - if (!Number.isFinite(ms) || ms <= 0) { - return; - } - Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms); -} - -function makeTempDir(parentDir, prefix) { - return fs.mkdtempSync(path.join(parentDir, prefix)); -} - -function writeRuntimeDepsTempOwner(tempDir) { - writeJson(path.join(tempDir, TEMP_OWNER_FILE), { - pid: process.pid, - createdAtMs: Date.now(), - }); -} - -function makeOwnedTempDir(parentDir, prefix) { - const tempDir = makeTempDir(parentDir, prefix); - writeRuntimeDepsTempOwner(tempDir); - return tempDir; -} - -function sanitizeTempPrefixSegment(value) { - const normalized = value.replace(/[^A-Za-z0-9._-]+/g, "-").replace(/-+/g, "-"); - return normalized.length > 0 ? normalized : "plugin"; -} - -function makePluginOwnedTempDir(pluginDir, label) { - return makeOwnedTempDir(pluginDir, `.openclaw-runtime-deps-${label}-`); -} - -function assertPathIsNotSymlink(targetPath, label) { - try { - if (fs.lstatSync(targetPath).isSymbolicLink()) { - throw new Error(`refusing to ${label} via symlinked path: ${targetPath}`); - } - } catch (error) { - if (error?.code === "ENOENT") { - return; - } - throw error; - } -} - -function replaceDirAtomically(targetPath, sourcePath) { - assertPathIsNotSymlink(targetPath, "replace runtime deps"); - const targetParentDir = path.dirname(targetPath); - fs.mkdirSync(targetParentDir, { recursive: true }); - const backupPath = makeTempDir( - targetParentDir, - `.openclaw-runtime-deps-backup-${sanitizeTempPrefixSegment(path.basename(targetPath))}-`, - ); - removePathIfExists(backupPath, { retryTransient: true }); - - let movedExistingTarget = false; - try { - if (fs.existsSync(targetPath)) { - fs.renameSync(targetPath, backupPath); - writeRuntimeDepsTempOwner(backupPath); - movedExistingTarget = true; - } - fs.renameSync(sourcePath, targetPath); - removeOwnedTempPathBestEffort(backupPath); - } catch (error) { - if (movedExistingTarget && !fs.existsSync(targetPath) && fs.existsSync(backupPath)) { - fs.renameSync(backupPath, targetPath); - removePathIfExists(path.join(targetPath, TEMP_OWNER_FILE)); - } - throw error; - } -} - -function writeJsonAtomically(targetPath, value) { - assertPathIsNotSymlink(targetPath, "write runtime deps stamp"); - const targetParentDir = path.dirname(targetPath); - fs.mkdirSync(targetParentDir, { recursive: true }); - const tempDir = makeOwnedTempDir( - targetParentDir, - `.openclaw-runtime-deps-stamp-${sanitizeTempPrefixSegment(path.basename(targetPath))}-`, - ); - const tempPath = path.join(tempDir, path.basename(targetPath)); - try { - fs.writeFileSync(tempPath, `${JSON.stringify(value, null, 2)}\n`, { - encoding: "utf8", - flag: "wx", - }); - fs.renameSync(tempPath, targetPath); - } finally { - removeOwnedTempPathBestEffort(tempDir); - } -} - -function dependencyPathSegments(depName) { - if (typeof depName !== "string" || depName.length === 0) { - return null; - } - const segments = depName.split("/"); - if (depName.startsWith("@")) { - if (segments.length !== 2) { - return null; - } - const [scope, name] = segments; - if ( - !/^@[A-Za-z0-9._-]+$/.test(scope) || - !/^[A-Za-z0-9._-]+$/.test(name) || - scope === "@." || - scope === "@.." - ) { - return null; - } - return [scope, name]; - } - if (segments.length !== 1 || !/^[A-Za-z0-9._-]+$/.test(segments[0])) { - return null; - } - return segments; -} - -function dependencyNodeModulesPath(nodeModulesDir, depName) { - const segments = dependencyPathSegments(depName); - return segments ? path.join(nodeModulesDir, ...segments) : null; -} - -function dependencyVersionSatisfied(spec, installedVersion) { - return semverSatisfies(installedVersion, spec, { includePrerelease: false }); -} - -function readInstalledDependencyVersionFromRoot(depRoot) { - const packageJsonPath = path.join(depRoot, "package.json"); - if (!fs.existsSync(packageJsonPath)) { - return null; - } - const version = readJson(packageJsonPath).version; - return typeof version === "string" ? version : null; -} - -const defaultStagedRuntimeDepGlobalPruneSuffixes = [".d.ts", ".map"]; -const defaultStagedRuntimeDepGlobalPruneDirectories = [ - "__snapshots__", - "__tests__", - "test", - "tests", -]; -const defaultStagedRuntimeDepGlobalPruneFilePatterns = [ - /(?:^|\/)[^/]+\.(?:test|spec)\.(?:[cm]?[jt]sx?)$/u, -]; -const defaultStagedRuntimeDepPruneRules = new Map([ - // Type declarations only; runtime resolves through lib/es entrypoints. - ["@larksuiteoapi/node-sdk", { paths: ["types"] }], - [ - "@matrix-org/matrix-sdk-crypto-nodejs", - { - paths: ["index.d.ts", "README.md", "CHANGELOG.md", "RELEASING.md", ".node-version"], - }, - ], - [ - "@matrix-org/matrix-sdk-crypto-wasm", - { - paths: [ - "index.d.ts", - "pkg/matrix_sdk_crypto_wasm.d.ts", - "pkg/matrix_sdk_crypto_wasm_bg.wasm.d.ts", - "README.md", - ], - }, - ], - [ - "matrix-js-sdk", - { - paths: ["src", "CHANGELOG.md", "CONTRIBUTING.rst", "README.md", "release.sh"], - suffixes: [".d.ts"], - }, - ], - ["matrix-widget-api", { paths: ["src"], suffixes: [".d.ts"] }], - ["oidc-client-ts", { paths: ["README.md"], suffixes: [".d.ts"] }], - ["music-metadata", { paths: ["README.md"], suffixes: [".d.ts"] }], - ["@cloudflare/workers-types", { paths: ["."] }], - ["gifwrap", { paths: ["test"] }], - ["playwright-core", { paths: ["types"], suffixes: [".d.ts"] }], - ["@jimp/plugin-blit", { paths: ["src/__image_snapshots__"] }], - ["@jimp/plugin-blur", { paths: ["src/__image_snapshots__"] }], - ["@jimp/plugin-color", { paths: ["src/__image_snapshots__"] }], - ["@jimp/plugin-print", { paths: ["src/__image_snapshots__"] }], - ["@jimp/plugin-quantize", { paths: ["src/__image_snapshots__"] }], - ["@jimp/plugin-threshold", { paths: ["src/__image_snapshots__"] }], - // tokenjuice ships built-in rules as JSON data under `dist/rules/tests/*.json` - // (e.g. `bun-test.json`, `jest.json`, `pytest.json`). These are NOT test - // fixtures — they are the runtime-loaded rule definitions consumed by - // `dist/core/builtin-rules.generated.js`. The global `tests` basename prune - // would strip them, and the plugin then fails to load with - // `Cannot find module '../rules/tests/bun-test.json'`. Keep them staged. - ["tokenjuice", { keepDirectories: ["dist/rules/tests"] }], -]); -const runtimeDepsStagingVersion = 7; -const exactVersionSpecRe = /^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?$/u; - -function resolveRuntimeDepPruneConfig(params = {}) { - return { - globalPruneDirectories: - params.stagedRuntimeDepGlobalPruneDirectories ?? - defaultStagedRuntimeDepGlobalPruneDirectories, - globalPruneFilePatterns: - params.stagedRuntimeDepGlobalPruneFilePatterns ?? - defaultStagedRuntimeDepGlobalPruneFilePatterns, - globalPruneSuffixes: - params.stagedRuntimeDepGlobalPruneSuffixes ?? defaultStagedRuntimeDepGlobalPruneSuffixes, - pruneRules: params.stagedRuntimeDepPruneRules ?? defaultStagedRuntimeDepPruneRules, - }; -} - -function resolveInstalledDependencyRoot(params) { - const candidates = []; - if (params.parentPackageRoot) { - const nestedDepRoot = dependencyNodeModulesPath( - path.join(params.parentPackageRoot, "node_modules"), - params.depName, - ); - if (nestedDepRoot !== null) { - candidates.push(nestedDepRoot); - } - } - const rootDepRoot = dependencyNodeModulesPath(params.rootNodeModulesDir, params.depName); - if (rootDepRoot !== null) { - candidates.push(rootDepRoot); - } - - for (const depRoot of candidates) { - const installedVersion = readInstalledDependencyVersionFromRoot(depRoot); - if (installedVersion === null) { - continue; - } - if (params.enforceSpec === false || dependencyVersionSatisfied(params.spec, installedVersion)) { - return depRoot; - } - } - - return null; -} - -function collectInstalledRuntimeDependencyRoots( - rootNodeModulesDir, - dependencySpecs, - directDependencyPackageRoot = null, - optionalDependencyNames = new Set(), -) { - const packageCache = new Map(); - const directRoots = []; - const allRoots = []; - const queue = Object.entries(dependencySpecs).map(([depName, spec]) => ({ - depName, - optional: optionalDependencyNames.has(depName), - spec, - parentPackageRoot: directDependencyPackageRoot, - direct: true, - })); - const seen = new Set(); - - while (queue.length > 0) { - const current = queue.shift(); - const depRoot = resolveInstalledDependencyRoot({ - depName: current.depName, - spec: current.spec, - enforceSpec: current.direct, - parentPackageRoot: current.parentPackageRoot, - rootNodeModulesDir, - }); - if (depRoot === null) { - if (current.optional) { - continue; - } - return null; - } - const canonicalDepRoot = fs.realpathSync(depRoot); - - const seenKey = `${current.depName}\0${canonicalDepRoot}`; - if (seen.has(seenKey)) { - continue; - } - seen.add(seenKey); - - const record = { name: current.depName, root: depRoot, realRoot: canonicalDepRoot }; - allRoots.push(record); - if (current.direct) { - directRoots.push(record); - } - - const packageJson = - packageCache.get(canonicalDepRoot) ?? readJson(path.join(depRoot, "package.json")); - packageCache.set(canonicalDepRoot, packageJson); - for (const [childName, childSpec] of Object.entries(packageJson.dependencies ?? {})) { - queue.push({ - depName: childName, - optional: false, - spec: childSpec, - parentPackageRoot: depRoot, - direct: false, - }); - } - for (const [childName, childSpec] of Object.entries(packageJson.optionalDependencies ?? {})) { - queue.push({ - depName: childName, - optional: true, - spec: childSpec, - parentPackageRoot: depRoot, - direct: false, - }); - } - } - - return { allRoots, directRoots }; -} - -function pathIsInsideCopiedRoot(candidateRoot, copiedRoot) { - return candidateRoot === copiedRoot || candidateRoot.startsWith(`${copiedRoot}${path.sep}`); -} - -function findContainingRealRoot(candidatePath, allowedRealRoots) { - return ( - allowedRealRoots.find((rootPath) => pathIsInsideCopiedRoot(candidatePath, rootPath)) ?? null - ); -} - -function copyMaterializedDependencyTree(params) { - const { activeRoots, allowedRealRoots, sourcePath, targetPath } = params; - const sourceStats = fs.lstatSync(sourcePath); - - if (sourceStats.isSymbolicLink()) { - let resolvedPath; - try { - resolvedPath = fs.realpathSync(sourcePath); - } catch { - return false; - } - const containingRoot = findContainingRealRoot(resolvedPath, allowedRealRoots); - if (containingRoot === null) { - return false; - } - if (activeRoots.has(containingRoot)) { - return true; - } - const nextActiveRoots = new Set(activeRoots); - nextActiveRoots.add(containingRoot); - return copyMaterializedDependencyTree({ - activeRoots: nextActiveRoots, - allowedRealRoots, - sourcePath: resolvedPath, - targetPath, - }); - } - - if (sourceStats.isDirectory()) { - fs.mkdirSync(targetPath, { recursive: true }); - for (const entry of fs - .readdirSync(sourcePath, { withFileTypes: true }) - .toSorted((left, right) => left.name.localeCompare(right.name))) { - if ( - !copyMaterializedDependencyTree({ - activeRoots, - allowedRealRoots, - sourcePath: path.join(sourcePath, entry.name), - targetPath: path.join(targetPath, entry.name), - }) - ) { - return false; - } - } - return true; - } - - if (sourceStats.isFile()) { - fs.mkdirSync(path.dirname(targetPath), { recursive: true }); - fs.copyFileSync(sourcePath, targetPath); - fs.chmodSync(targetPath, sourceStats.mode); - return true; - } - - return true; -} - -function selectRuntimeDependencyRootsToCopy(resolution) { - const rootsToCopy = []; - - for (const record of resolution.directRoots) { - rootsToCopy.push(record); - } - - for (const record of resolution.allRoots) { - if (rootsToCopy.some((entry) => pathIsInsideCopiedRoot(record.realRoot, entry.realRoot))) { - continue; - } - rootsToCopy.push(record); - } - - return rootsToCopy; -} - -function resolveInstalledDirectDependencyNames( - rootNodeModulesDir, - dependencySpecs, - directDependencyPackageRoot = null, - optionalDependencyNames = new Set(), -) { - const directDependencyNames = []; - for (const [depName, spec] of Object.entries(dependencySpecs)) { - const depRoot = resolveInstalledDependencyRoot({ - depName, - spec, - parentPackageRoot: directDependencyPackageRoot, - rootNodeModulesDir, - }); - if (depRoot === null) { - if (optionalDependencyNames.has(depName)) { - continue; - } - return null; - } - const installedVersion = readInstalledDependencyVersionFromRoot(depRoot); - if (installedVersion === null || !dependencyVersionSatisfied(spec, installedVersion)) { - return null; - } - directDependencyNames.push(depName); - } - return directDependencyNames; -} - -function appendDirectoryFingerprint(hash, rootDir, currentDir = rootDir) { - const entries = fs - .readdirSync(currentDir, { withFileTypes: true }) - .toSorted((left, right) => left.name.localeCompare(right.name)); - - for (const entry of entries) { - const fullPath = path.join(currentDir, entry.name); - const relativePath = path.relative(rootDir, fullPath).replace(/\\/g, "/"); - const stats = fs.lstatSync(fullPath); - if (stats.isSymbolicLink()) { - hash.update(`symlink:${relativePath}->${fs.readlinkSync(fullPath).replace(/\\/g, "/")}\n`); - continue; - } - if (stats.isDirectory()) { - hash.update(`dir:${relativePath}\n`); - appendDirectoryFingerprint(hash, rootDir, fullPath); - continue; - } - if (!stats.isFile()) { - continue; - } - const stat = fs.statSync(fullPath); - hash.update(`file:${relativePath}:${stat.size}\n`); - hash.update(fs.readFileSync(fullPath)); - } -} - -function createInstalledRuntimeClosureFingerprint(rootNodeModulesDir, dependencyNames) { - const hash = createHash("sha256"); - for (const depName of [...dependencyNames].toSorted((left, right) => left.localeCompare(right))) { - const depRoot = dependencyNodeModulesPath(rootNodeModulesDir, depName); - if (depRoot === null || !fs.existsSync(depRoot)) { - return null; - } - hash.update(`package:${depName}:${fs.realpathSync(depRoot)}\n`); - appendDirectoryFingerprint(hash, depRoot); - } - return hash.digest("hex"); -} - -function resolveInstalledRuntimeClosureFingerprint(params) { - const dependencySpecs = { - ...params.packageJson.dependencies, - ...params.packageJson.optionalDependencies, - }; - if (Object.keys(dependencySpecs).length === 0 || !fs.existsSync(params.rootNodeModulesDir)) { - return null; - } - const resolution = collectInstalledRuntimeDependencyRoots( - params.rootNodeModulesDir, - dependencySpecs, - params.directDependencyPackageRoot, - new Set(Object.keys(params.packageJson.optionalDependencies ?? {})), - ); - if (resolution === null) { - return null; - } - return createInstalledRuntimeClosureFingerprint( - params.rootNodeModulesDir, - selectRuntimeDependencyRootsToCopy(resolution).map((record) => record.name), - ); -} - -function walkFiles(rootDir, visitFile) { - if (!fs.existsSync(rootDir)) { - return; - } - const queue = [rootDir]; - for (let index = 0; index < queue.length; index += 1) { - const currentDir = queue[index]; - for (const entry of fs.readdirSync(currentDir, { withFileTypes: true })) { - const fullPath = path.join(currentDir, entry.name); - if (entry.isDirectory()) { - queue.push(fullPath); - continue; - } - if (entry.isFile()) { - visitFile(fullPath); - } - } - } -} - -function pruneDependencyFilesBySuffixes(depRoot, suffixes) { - if (!suffixes || suffixes.length === 0 || !fs.existsSync(depRoot)) { - return; - } - walkFiles(depRoot, (fullPath) => { - if (suffixes.some((suffix) => fullPath.endsWith(suffix))) { - removePathIfExists(fullPath); - } - }); -} - -function relativePathSegments(rootDir, fullPath) { - return path.relative(rootDir, fullPath).split(path.sep).filter(Boolean); -} - -function isNodeModulesPackageRoot(segments, index) { - const parent = segments[index - 1]; - if (parent === "node_modules") { - return true; - } - return parent?.startsWith("@") === true && segments[index - 2] === "node_modules"; -} - -function pruneDependencyDirectoriesByBasename(depRoot, basenames, keepDirs = new Set()) { - if (!basenames || basenames.length === 0 || !fs.existsSync(depRoot)) { - return; - } - const basenameSet = new Set(basenames); - const queue = [depRoot]; - for (let index = 0; index < queue.length; index += 1) { - const currentDir = queue[index]; - for (const entry of fs.readdirSync(currentDir, { withFileTypes: true })) { - if (!entry.isDirectory()) { - continue; - } - const fullPath = path.join(currentDir, entry.name); - const segments = relativePathSegments(depRoot, fullPath); - if (basenameSet.has(entry.name) && !isNodeModulesPackageRoot(segments, segments.length - 1)) { - // Per-package opt-out: a pruneRule may keep specific directories that - // would otherwise match a global basename prune (e.g. a data/asset - // directory named `tests/` that is NOT test code). Descend into kept - // directories so their contents are still subject to suffix/pattern - // pruning, but do not remove the directory itself. - if (keepDirs.has(fullPath)) { - queue.push(fullPath); - continue; - } - removePathIfExists(fullPath); - continue; - } - queue.push(fullPath); - } - } -} - -function pruneDependencyFilesByPatterns(depRoot, patterns) { - if (!patterns || patterns.length === 0 || !fs.existsSync(depRoot)) { - return; - } - walkFiles(depRoot, (fullPath) => { - const relativePath = relativePathSegments(depRoot, fullPath).join("/"); - if (patterns.some((pattern) => pattern.test(relativePath))) { - removePathIfExists(fullPath); - } - }); -} - -function pruneStagedInstalledDependencyCargo(nodeModulesDir, depName, pruneConfig) { - const depRoot = dependencyNodeModulesPath(nodeModulesDir, depName); - if (depRoot === null) { - return; - } - const pruneRule = pruneConfig.pruneRules.get(depName); - for (const relativePath of pruneRule?.paths ?? []) { - removePathIfExists(path.join(depRoot, relativePath)); - } - // Resolve per-package keepDirectories (opt-out of global basename prune) - // against depRoot up front so the walk can skip them cheaply. - const keepDirs = new Set( - (pruneRule?.keepDirectories ?? []).map((relativePath) => path.resolve(depRoot, relativePath)), - ); - pruneDependencyDirectoriesByBasename(depRoot, pruneConfig.globalPruneDirectories, keepDirs); - pruneDependencyFilesByPatterns(depRoot, pruneConfig.globalPruneFilePatterns); - pruneDependencyFilesBySuffixes(depRoot, pruneConfig.globalPruneSuffixes); - pruneDependencyFilesBySuffixes(depRoot, pruneRule?.suffixes ?? []); -} - -function listInstalledDependencyNames(nodeModulesDir) { - if (!fs.existsSync(nodeModulesDir)) { - return []; - } - const names = []; - for (const entry of fs.readdirSync(nodeModulesDir, { withFileTypes: true })) { - if (!entry.isDirectory()) { - continue; - } - if (entry.name.startsWith("@")) { - const scopeDir = path.join(nodeModulesDir, entry.name); - for (const scopedEntry of fs.readdirSync(scopeDir, { withFileTypes: true })) { - if (scopedEntry.isDirectory()) { - names.push(`${entry.name}/${scopedEntry.name}`); - } - } - continue; - } - names.push(entry.name); - } - return names; -} - -function pruneStagedRuntimeDependencyCargo(nodeModulesDir, pruneConfig) { - for (const depName of listInstalledDependencyNames(nodeModulesDir)) { - pruneStagedInstalledDependencyCargo(nodeModulesDir, depName, pruneConfig); - } -} - -function listBundledPluginRuntimeDirs(repoRoot) { - const extensionsRoot = path.join(repoRoot, "dist", "extensions"); - if (!fs.existsSync(extensionsRoot)) { - return []; - } - - return fs - .readdirSync(extensionsRoot, { withFileTypes: true }) - .filter((dirent) => dirent.isDirectory()) - .map((dirent) => path.join(extensionsRoot, dirent.name)) - .filter((pluginDir) => fs.existsSync(path.join(pluginDir, "package.json"))); -} - -function resolveInstalledWorkspacePluginRoot(repoRoot, pluginId) { - const currentPluginRoot = path.join(repoRoot, "extensions", pluginId); - if (fs.existsSync(path.join(currentPluginRoot, "node_modules"))) { - return currentPluginRoot; - } - - const nodeModulesDir = path.join(repoRoot, "node_modules"); - if (!fs.existsSync(nodeModulesDir)) { - return currentPluginRoot; - } - - let installedWorkspaceRoot; - try { - installedWorkspaceRoot = path.dirname(fs.realpathSync(nodeModulesDir)); - } catch { - return currentPluginRoot; - } - - const installedPluginRoot = path.join(installedWorkspaceRoot, "extensions", pluginId); - if (fs.existsSync(path.join(installedPluginRoot, "package.json"))) { - return installedPluginRoot; - } - - return currentPluginRoot; -} - function hasRuntimeDeps(packageJson) { return ( Object.keys(packageJson.dependencies ?? {}).length > 0 || @@ -922,204 +242,6 @@ function runNpmInstall(params) { }); } -function resolveLegacyRuntimeDepsStampPath(pluginDir) { - return path.join(pluginDir, ".openclaw-runtime-deps-stamp.json"); -} - -function resolveRuntimeDepsStampPath(repoRoot, pluginId) { - return path.join( - repoRoot, - ".artifacts", - "bundled-runtime-deps-stamps", - `${sanitizeTempPrefixSegment(pluginId)}.json`, - ); -} - -function createRuntimeDepsFingerprint(packageJson, pruneConfig, params = {}) { - return createHash("sha256") - .update( - JSON.stringify({ - cheapFingerprint: createRuntimeDepsCheapFingerprint(packageJson, pruneConfig, params), - rootInstalledRuntimeFingerprint: params.rootInstalledRuntimeFingerprint ?? null, - }), - ) - .digest("hex"); -} - -function createRuntimeDepsCheapFingerprint(packageJson, pruneConfig, params = {}) { - const repoRoot = params.repoRoot; - const lockfilePath = - typeof repoRoot === "string" && repoRoot.length > 0 - ? path.join(repoRoot, "pnpm-lock.yaml") - : null; - const rootLockfile = lockfilePath ? readOptionalUtf8(lockfilePath) : null; - return createHash("sha256") - .update( - JSON.stringify({ - globalPruneDirectories: pruneConfig.globalPruneDirectories, - globalPruneFilePatterns: pruneConfig.globalPruneFilePatterns.map((pattern) => - pattern.toString(), - ), - globalPruneSuffixes: pruneConfig.globalPruneSuffixes, - packageJson, - pruneRules: [...pruneConfig.pruneRules.entries()], - rootLockfile, - version: runtimeDepsStagingVersion, - }), - ) - .digest("hex"); -} - -function readRuntimeDepsStamp(stampPath) { - if (!fs.existsSync(stampPath)) { - return null; - } - try { - return readJson(stampPath); - } catch { - return null; - } -} - -function readRuntimeDepsTempOwner(tempDir) { - try { - const owner = readJson(path.join(tempDir, TEMP_OWNER_FILE)); - return owner && typeof owner === "object" ? owner : null; - } catch { - return null; - } -} - -function isLiveProcess(pid) { - if (!Number.isInteger(pid) || pid <= 0) { - return false; - } - try { - process.kill(pid, 0); - return true; - } catch (error) { - return error?.code === "EPERM"; - } -} - -function shouldRemoveRuntimeDepsTempDir(tempDir) { - const owner = readRuntimeDepsTempOwner(tempDir); - if (!owner || typeof owner.pid !== "number") { - return true; - } - return !isLiveProcess(owner.pid); -} - -function removeStaleRuntimeDepsTempDirs(pluginDir) { - if (!fs.existsSync(pluginDir)) { - return; - } - for (const entry of fs.readdirSync(pluginDir, { withFileTypes: true })) { - if (entry.name.startsWith(".openclaw-runtime-deps-")) { - const targetPath = path.join(pluginDir, entry.name); - if (!shouldRemoveRuntimeDepsTempDir(targetPath)) { - continue; - } - removeOwnedTempPathBestEffort(targetPath); - } - } -} - -function stageInstalledRootRuntimeDeps(params) { - const { - directDependencyPackageRoot = null, - cheapFingerprint, - fingerprint, - packageJson, - pluginDir, - pruneConfig, - repoRoot, - stampPath, - } = params; - const dependencySpecs = { - ...packageJson.dependencies, - ...packageJson.optionalDependencies, - }; - const optionalDependencyNames = new Set(Object.keys(packageJson.optionalDependencies ?? {})); - const rootNodeModulesDir = path.join(repoRoot, "node_modules"); - if (Object.keys(dependencySpecs).length === 0 || !fs.existsSync(rootNodeModulesDir)) { - return false; - } - - const directDependencyNames = resolveInstalledDirectDependencyNames( - rootNodeModulesDir, - dependencySpecs, - directDependencyPackageRoot, - optionalDependencyNames, - ); - if (directDependencyNames === null) { - return false; - } - const resolution = collectInstalledRuntimeDependencyRoots( - rootNodeModulesDir, - dependencySpecs, - directDependencyPackageRoot, - optionalDependencyNames, - ); - if (resolution === null) { - return false; - } - const rootsToCopy = selectRuntimeDependencyRootsToCopy(resolution); - const nodeModulesDir = path.join(pluginDir, "node_modules"); - if (rootsToCopy.length === 0) { - assertPathIsNotSymlink(nodeModulesDir, "remove runtime deps"); - removePathIfExists(nodeModulesDir); - writeJsonAtomically(stampPath, { - cheapFingerprint, - fingerprint, - generatedAt: new Date().toISOString(), - }); - return true; - } - const allowedRealRoots = rootsToCopy.map((record) => record.realRoot); - - const stagedNodeModulesDir = path.join( - makePluginOwnedTempDir(pluginDir, "stage"), - "node_modules", - ); - - try { - for (const record of rootsToCopy.toSorted((left, right) => - left.name.localeCompare(right.name), - )) { - const sourcePath = record.realRoot; - const targetPath = dependencyNodeModulesPath(stagedNodeModulesDir, record.name); - if (targetPath === null) { - return false; - } - fs.mkdirSync(path.dirname(targetPath), { recursive: true }); - const sourceRootReal = findContainingRealRoot(sourcePath, allowedRealRoots); - if ( - sourceRootReal === null || - !copyMaterializedDependencyTree({ - activeRoots: new Set([sourceRootReal]), - allowedRealRoots, - sourcePath, - targetPath, - }) - ) { - return false; - } - } - pruneStagedRuntimeDependencyCargo(stagedNodeModulesDir, pruneConfig); - - replaceDirAtomically(nodeModulesDir, stagedNodeModulesDir); - writeJsonAtomically(stampPath, { - cheapFingerprint, - fingerprint, - generatedAt: new Date().toISOString(), - }); - return true; - } finally { - removeOwnedTempPathBestEffort(path.dirname(stagedNodeModulesDir)); - } -} - function installPluginRuntimeDepsWithRetries(params) { const { attempts = 3 } = params; let lastError;