From c160bec3d62a23a5f675535cb73bde704ebf0042 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Wed, 29 Apr 2026 21:46:20 +0100 Subject: [PATCH] refactor(plugins): split runtime deps planner --- .../bundled-runtime-deps-drift.test.ts | 224 +++ src/plugins/bundled-runtime-deps-install.ts | 444 ++++++ src/plugins/bundled-runtime-deps-roots.ts | 381 +++++ src/plugins/bundled-runtime-deps-selection.ts | 489 ++++++ src/plugins/bundled-runtime-deps.test.ts | 221 --- src/plugins/bundled-runtime-deps.ts | 1331 +---------------- 6 files changed, 1589 insertions(+), 1501 deletions(-) create mode 100644 src/plugins/bundled-runtime-deps-drift.test.ts create mode 100644 src/plugins/bundled-runtime-deps-install.ts create mode 100644 src/plugins/bundled-runtime-deps-roots.ts create mode 100644 src/plugins/bundled-runtime-deps-selection.ts diff --git a/src/plugins/bundled-runtime-deps-drift.test.ts b/src/plugins/bundled-runtime-deps-drift.test.ts new file mode 100644 index 00000000000..b68ee52a206 --- /dev/null +++ b/src/plugins/bundled-runtime-deps-drift.test.ts @@ -0,0 +1,224 @@ +import fs from "node:fs"; +import { Module } from "node:module"; +import path from "node:path"; +import { describe, it } from "vitest"; + +describe("mirrored root runtime dependency drift guard", () => { + // Intentionally not mirrored at runtime: build-only / type-only / TUI-only + // tooling and packages that resolve transitively through other mirrored deps. + // If you change this set, document why in the comment beside the entry. + const KNOWN_UNMIRRORED_BARE_IMPORTS = new Set([ + "@mariozechner/pi-tui", // TUI mode runs from npm-global, not the gateway runtime mirror + "chalk", // available transitively via mirrored deps + "file-type", // available transitively via mirrored deps + "global-agent", // proxy bootstrap, only loaded when HTTP_PROXY is set + "ipaddr.js", // available transitively via mirrored deps + "proxy-agent", // available transitively via mirrored deps + "qrcode", // type-only import in src/media/qr-runtime.ts + "typescript", // CLI/dev only (api-baseline, jiti-runtime-api) + ]); + + function locateRepoRoot(): string { + let dir = path.resolve(import.meta.dirname); + for (let depth = 0; depth < 10; depth += 1) { + const candidate = path.join(dir, "package.json"); + if (fs.existsSync(candidate)) { + try { + const data = JSON.parse(fs.readFileSync(candidate, "utf8")) as { name?: string }; + if (data.name === "openclaw") { + return dir; + } + } catch { + // fall through + } + } + const parent = path.dirname(dir); + if (parent === dir) { + break; + } + dir = parent; + } + throw new Error("could not locate openclaw repo root from test file"); + } + + function readPackageJsonDeps(packageJsonPath: string): Set { + const out = new Set(); + if (!fs.existsSync(packageJsonPath)) { + return out; + } + let parsed: { + dependencies?: Record; + optionalDependencies?: Record; + }; + try { + parsed = JSON.parse(fs.readFileSync(packageJsonPath, "utf8")); + } catch { + return out; + } + for (const name of Object.keys(parsed.dependencies ?? {})) { + out.add(name); + } + for (const name of Object.keys(parsed.optionalDependencies ?? {})) { + out.add(name); + } + return out; + } + + function readMirroredRootRuntimeDeps(repoRoot: string): Set { + const parsed = JSON.parse(fs.readFileSync(path.join(repoRoot, "package.json"), "utf8")) as { + openclaw?: { + bundle?: { + mirroredRootRuntimeDependencies?: unknown; + }; + }; + }; + const deps = parsed.openclaw?.bundle?.mirroredRootRuntimeDependencies; + return new Set(Array.isArray(deps) ? deps.filter((dep) => typeof dep === "string") : []); + } + + function collectExtensionOwnedDeps(repoRoot: string): Set { + const out = new Set(); + const extensionsDir = path.join(repoRoot, "extensions"); + if (!fs.existsSync(extensionsDir)) { + return out; + } + for (const entry of fs.readdirSync(extensionsDir, { withFileTypes: true })) { + if (!entry.isDirectory()) { + continue; + } + for (const name of readPackageJsonDeps( + path.join(extensionsDir, entry.name, "package.json"), + )) { + out.add(name); + } + } + return out; + } + + function walkCoreSourceFiles(repoRoot: string): string[] { + const srcDir = path.join(repoRoot, "src"); + const files: string[] = []; + const queue: string[] = [srcDir]; + while (queue.length > 0) { + const current = queue.shift(); + if (!current) { + continue; + } + for (const entry of fs.readdirSync(current, { withFileTypes: true })) { + const full = path.join(current, entry.name); + if (entry.isDirectory()) { + if (entry.name === "node_modules" || entry.name.startsWith(".")) { + continue; + } + queue.push(full); + continue; + } + if (!entry.isFile()) { + continue; + } + if ( + /\.test\.tsx?$/u.test(entry.name) || + /\.e2e\.test\.tsx?$/u.test(entry.name) || + /\.test-helpers?\.tsx?$/u.test(entry.name) || + /\.test-fixture\.tsx?$/u.test(entry.name) || + entry.name.endsWith(".d.ts") || + !/\.(?:ts|tsx|cjs|mjs|js)$/u.test(entry.name) + ) { + continue; + } + files.push(full); + } + } + return files; + } + + function packageNameFromBareSpecifier(specifier: string): string | null { + if ( + specifier.startsWith(".") || + specifier.startsWith("/") || + specifier.startsWith("node:") || + specifier.startsWith("#") + ) { + return null; + } + const [first, second] = specifier.split("/"); + if (!first) { + return null; + } + return first.startsWith("@") && second ? `${first}/${second}` : first; + } + + // Match value imports (`import x from 'y'`, `import 'y'`, `require('y')`, + // `import('y')`) but skip `import type` to avoid noise from type-only imports. + const VALUE_IMPORT_PATTERNS = [ + /(?:^|[;\n])\s*import\s+(?!type\b)(?:[^'"()]+?\s+from\s+)?["']([^"']+)["']/g, + /\brequire\s*\(\s*["']([^"']+)["']\s*\)/g, + /\bimport\s*\(\s*["']([^"']+)["']\s*\)/g, + ] as const; + + it("every value-imported root-package dep in src/ is mirrored or owned by an extension", () => { + const repoRoot = locateRepoRoot(); + const rootDeps = readPackageJsonDeps(path.join(repoRoot, "package.json")); + const extensionDeps = collectExtensionOwnedDeps(repoRoot); + const mirroredCore = readMirroredRootRuntimeDeps(repoRoot); + const nodeBuiltins = new Set(Module.builtinModules); + + const violations = new Map(); + for (const file of walkCoreSourceFiles(repoRoot)) { + const source = fs.readFileSync(file, "utf8"); + const specifiers = new Set(); + for (const pattern of VALUE_IMPORT_PATTERNS) { + for (const match of source.matchAll(pattern)) { + if (match[1]) { + specifiers.add(match[1]); + } + } + } + for (const specifier of specifiers) { + const packageName = packageNameFromBareSpecifier(specifier); + if (!packageName) { + continue; + } + if (nodeBuiltins.has(packageName)) { + continue; + } + if (packageName === "openclaw" || packageName.startsWith("@openclaw/")) { + continue; + } + if (mirroredCore.has(packageName) || extensionDeps.has(packageName)) { + continue; + } + if (KNOWN_UNMIRRORED_BARE_IMPORTS.has(packageName)) { + continue; + } + if (!rootDeps.has(packageName)) { + // Not a root runtime dep; not our concern (could be a peer/dev import + // that resolves through some other path; the mirror does not own it). + continue; + } + if (!violations.has(packageName)) { + violations.set(packageName, path.relative(repoRoot, file).replaceAll(path.sep, "/")); + } + } + } + + if (violations.size > 0) { + const summary = [...violations.entries()] + .toSorted(([left], [right]) => left.localeCompare(right)) + .map(([packageName, filePath]) => ` - ${packageName} (e.g. ${filePath})`) + .join("\n"); + throw new Error( + [ + "Bare imports found in src/ that are root-package runtime deps but are neither", + "in package.json openclaw.bundle.mirroredRootRuntimeDependencies nor declared by any extension's package.json.", + "These will be missing from the runtime-deps mirror at gateway start and Node", + "will fail to resolve them. Either add the package to openclaw.bundle.mirroredRootRuntimeDependencies,", + "declare it under an owning extension's dependencies, or add it to", + "KNOWN_UNMIRRORED_BARE_IMPORTS in this test with a comment explaining why.", + "", + summary, + ].join("\n"), + ); + } + }); +}); diff --git a/src/plugins/bundled-runtime-deps-install.ts b/src/plugins/bundled-runtime-deps-install.ts new file mode 100644 index 00000000000..6b0916441ad --- /dev/null +++ b/src/plugins/bundled-runtime-deps-install.ts @@ -0,0 +1,444 @@ +import { spawn, spawnSync } from "node:child_process"; +import fs from "node:fs"; +import path from "node:path"; +import { createLowDiskSpaceWarning } from "../infra/disk-space.js"; +import { sanitizeTerminalText } from "../terminal/safe-text.js"; +import { beginBundledRuntimeDepsInstall } from "./bundled-runtime-deps-activity.js"; +import { + BUNDLED_RUNTIME_DEPS_LOCK_DIR, + withBundledRuntimeDepsFilesystemLock, + withBundledRuntimeDepsFilesystemLockAsync, +} from "./bundled-runtime-deps-lock.js"; +import { + assertBundledRuntimeDepsInstalled, + ensureNpmInstallExecutionManifest, + isRuntimeDepsPlanMaterialized, + removeLegacyRuntimeDepsManifest, +} from "./bundled-runtime-deps-materialization.js"; +import { + createBundledRuntimeDepsInstallArgs, + createBundledRuntimeDepsInstallEnv, + resolveBundledRuntimeDepsPackageManagerRunner, + type BundledRuntimeDepsPackageManager, + type BundledRuntimeDepsPackageManagerRunner, +} from "./bundled-runtime-deps-package-manager.js"; +import { normalizeRuntimeDepSpecs } from "./bundled-runtime-deps-specs.js"; + +const BUNDLED_RUNTIME_DEPS_INSTALL_PROGRESS_INTERVAL_MS = 5_000; + +export type BundledRuntimeDepsInstallParams = { + installRoot: string; + installExecutionRoot?: string; + missingSpecs: string[]; + installSpecs?: string[]; + warn?: (message: string) => void; +}; + +function withBundledRuntimeDepsInstallRootLock(installRoot: string, run: () => T): T { + return withBundledRuntimeDepsFilesystemLock(installRoot, BUNDLED_RUNTIME_DEPS_LOCK_DIR, run); +} + +async function withBundledRuntimeDepsInstallRootLockAsync( + installRoot: string, + run: () => Promise, +): Promise { + return await withBundledRuntimeDepsFilesystemLockAsync( + installRoot, + BUNDLED_RUNTIME_DEPS_LOCK_DIR, + run, + ); +} + +function replaceNodeModulesDir(targetDir: string, sourceDir: string): void { + const parentDir = path.dirname(targetDir); + const tempDir = fs.mkdtempSync(path.join(parentDir, ".openclaw-runtime-deps-copy-")); + const stagedDir = path.join(tempDir, "node_modules"); + try { + fs.cpSync(sourceDir, stagedDir, { recursive: true }); + fs.rmSync(targetDir, { recursive: true, force: true }); + fs.renameSync(stagedDir, targetDir); + } finally { + try { + fs.rmSync(tempDir, { recursive: true, force: true }); + } catch { + // Stale temp dirs are swept at the next runtime-deps pass. Do not fail + // a node_modules replacement on a transient cleanup race. + } + } +} + +function shouldCleanBundledRuntimeDepsInstallExecutionRoot(params: { + installRoot: string; + installExecutionRoot: string; +}): boolean { + const installRoot = path.resolve(params.installRoot); + const installExecutionRoot = path.resolve(params.installExecutionRoot); + return installExecutionRoot.startsWith(`${installRoot}${path.sep}`); +} + +function formatBundledRuntimeDepsInstallError(result: { + error?: Error; + signal?: NodeJS.Signals | null; + status?: number | null; + stderr?: string | Buffer | null; + stdout?: string | Buffer | null; +}): string { + const output = [ + result.error?.message, + result.signal ? `terminated by ${result.signal}` : null, + result.stderr, + result.stdout, + ] + .filter(Boolean) + .join("\n") + .trim(); + return output || "npm install failed"; +} + +function formatBundledRuntimeDepsInstallElapsed(ms: number): string { + const seconds = Math.max(0, Math.round(ms / 1000)); + if (seconds < 60) { + return `${seconds}s`; + } + const minutes = Math.floor(seconds / 60); + const remainingSeconds = seconds % 60; + return remainingSeconds > 0 ? `${minutes}m ${remainingSeconds}s` : `${minutes}m`; +} + +function emitBundledRuntimeDepsOutputProgress( + chunk: Buffer, + stream: "stdout" | "stderr", + packageManager: BundledRuntimeDepsPackageManager, + onProgress: ((message: string) => void) | undefined, +): void { + if (!onProgress) { + return; + } + const lines = chunk + .toString("utf8") + .split(/\r\n|\n|\r/u) + .map((line) => sanitizeTerminalText(line).trim()) + .filter((line) => line.length > 0) + .slice(-3); + for (const line of lines) { + onProgress(`${packageManager} ${stream}: ${line}`); + } +} + +type BundledRuntimeDepsInstallContext = { + installExecutionRoot: string; + installSpecs: string[]; + installEnv: NodeJS.ProcessEnv; + runner: BundledRuntimeDepsPackageManagerRunner; + isolatedExecutionRoot: boolean; + cleanInstallExecutionRoot: boolean; +}; + +function createBundledRuntimeDepsInstallContext(params: { + installRoot: string; + installExecutionRoot?: string; + installSpecs: readonly string[]; + env: NodeJS.ProcessEnv; + warn?: (message: string) => void; +}): BundledRuntimeDepsInstallContext { + const installExecutionRoot = params.installExecutionRoot ?? params.installRoot; + const isolatedExecutionRoot = + path.resolve(installExecutionRoot) !== path.resolve(params.installRoot); + const cleanInstallExecutionRoot = + isolatedExecutionRoot && + shouldCleanBundledRuntimeDepsInstallExecutionRoot({ + installRoot: params.installRoot, + installExecutionRoot, + }); + + fs.mkdirSync(params.installRoot, { recursive: true }); + fs.mkdirSync(installExecutionRoot, { recursive: true }); + const diskWarning = createLowDiskSpaceWarning({ + targetPath: installExecutionRoot, + purpose: "bundled plugin runtime dependency staging", + }); + if (diskWarning) { + params.warn?.(diskWarning); + } + ensureNpmInstallExecutionManifest(installExecutionRoot, params.installSpecs); + const installEnv = createBundledRuntimeDepsInstallEnv(params.env, { + cacheDir: path.join(installExecutionRoot, ".openclaw-npm-cache"), + }); + const runner = resolveBundledRuntimeDepsPackageManagerRunner({ + installExecutionRoot, + env: installEnv, + npmArgs: createBundledRuntimeDepsInstallArgs(), + }); + + return { + installExecutionRoot, + installSpecs: normalizeRuntimeDepSpecs(params.installSpecs), + installEnv, + runner, + isolatedExecutionRoot, + cleanInstallExecutionRoot, + }; +} + +function finalizeBundledRuntimeDepsInstall(params: { + installRoot: string; + context: BundledRuntimeDepsInstallContext; +}): void { + const { context } = params; + assertBundledRuntimeDepsInstalled(context.installExecutionRoot, context.installSpecs); + if (context.isolatedExecutionRoot) { + const stagedNodeModulesDir = path.join(context.installExecutionRoot, "node_modules"); + if (!fs.existsSync(stagedNodeModulesDir)) { + throw new Error(`${context.runner.packageManager} install did not produce node_modules`); + } + const targetNodeModulesDir = path.join(params.installRoot, "node_modules"); + replaceNodeModulesDir(targetNodeModulesDir, stagedNodeModulesDir); + assertBundledRuntimeDepsInstalled(params.installRoot, context.installSpecs); + } + removeLegacyRuntimeDepsManifest(params.installRoot); +} + +function cleanupBundledRuntimeDepsInstallContext(context: BundledRuntimeDepsInstallContext): void { + if (context.cleanInstallExecutionRoot) { + fs.rmSync(context.installExecutionRoot, { recursive: true, force: true }); + } +} + +async function spawnBundledRuntimeDepsInstall(params: { + command: string; + args: string[]; + cwd: string; + env: NodeJS.ProcessEnv; + packageManager: BundledRuntimeDepsPackageManager; + onProgress?: (message: string) => void; +}): Promise { + await new Promise((resolve, reject) => { + const startedAtMs = Date.now(); + const heartbeat = + params.onProgress && + setInterval(() => { + params.onProgress?.( + `${params.packageManager} install still running (${formatBundledRuntimeDepsInstallElapsed(Date.now() - startedAtMs)} elapsed)`, + ); + }, BUNDLED_RUNTIME_DEPS_INSTALL_PROGRESS_INTERVAL_MS); + heartbeat?.unref?.(); + const settle = (fn: () => void) => { + if (heartbeat) { + clearInterval(heartbeat); + } + fn(); + }; + const child = spawn(params.command, params.args, { + cwd: params.cwd, + env: params.env, + stdio: ["ignore", "pipe", "pipe"], + windowsHide: true, + }); + const stdout: Buffer[] = []; + const stderr: Buffer[] = []; + child.stdout?.on("data", (chunk: Buffer) => { + stdout.push(chunk); + emitBundledRuntimeDepsOutputProgress( + chunk, + "stdout", + params.packageManager, + params.onProgress, + ); + }); + child.stderr?.on("data", (chunk: Buffer) => { + stderr.push(chunk); + emitBundledRuntimeDepsOutputProgress( + chunk, + "stderr", + params.packageManager, + params.onProgress, + ); + }); + child.on("error", (error) => { + settle(() => reject(new Error(formatBundledRuntimeDepsInstallError({ error })))); + }); + child.on("close", (status, signal) => { + if (status === 0 && !signal) { + settle(resolve); + return; + } + settle(() => + reject( + new Error( + formatBundledRuntimeDepsInstallError({ + status, + signal, + stdout: Buffer.concat(stdout).toString("utf8"), + stderr: Buffer.concat(stderr).toString("utf8"), + }), + ), + ), + ); + }); + }); +} + +export function installBundledRuntimeDeps(params: { + installRoot: string; + installExecutionRoot?: string; + missingSpecs: string[]; + installSpecs?: string[]; + env: NodeJS.ProcessEnv; + warn?: (message: string) => void; +}): void { + const installSpecs = normalizeRuntimeDepSpecs(params.installSpecs ?? params.missingSpecs); + if (installSpecs.length === 0) { + return; + } + if (isRuntimeDepsPlanMaterialized(params.installRoot, installSpecs)) { + removeLegacyRuntimeDepsManifest(params.installRoot); + return; + } + const context = createBundledRuntimeDepsInstallContext({ + installRoot: params.installRoot, + installExecutionRoot: params.installExecutionRoot, + installSpecs, + env: params.env, + warn: params.warn, + }); + try { + const result = spawnSync(context.runner.command, context.runner.args, { + cwd: context.installExecutionRoot, + encoding: "utf8", + env: context.runner.env ?? context.installEnv, + stdio: "pipe", + windowsHide: true, + }); + if (result.status !== 0 || result.error) { + throw new Error(formatBundledRuntimeDepsInstallError(result)); + } + finalizeBundledRuntimeDepsInstall({ installRoot: params.installRoot, context }); + } finally { + cleanupBundledRuntimeDepsInstallContext(context); + } +} + +export async function installBundledRuntimeDepsAsync(params: { + installRoot: string; + installExecutionRoot?: string; + missingSpecs: string[]; + installSpecs?: string[]; + env: NodeJS.ProcessEnv; + warn?: (message: string) => void; + onProgress?: (message: string) => void; +}): Promise { + const installSpecs = normalizeRuntimeDepSpecs(params.installSpecs ?? params.missingSpecs); + if (installSpecs.length === 0) { + return; + } + if (isRuntimeDepsPlanMaterialized(params.installRoot, installSpecs)) { + removeLegacyRuntimeDepsManifest(params.installRoot); + return; + } + const context = createBundledRuntimeDepsInstallContext({ + installRoot: params.installRoot, + installExecutionRoot: params.installExecutionRoot, + installSpecs, + env: params.env, + warn: params.warn, + }); + try { + params.onProgress?.( + `Starting ${context.runner.packageManager} install for bundled plugin runtime deps: ${installSpecs.join(", ")}`, + ); + await spawnBundledRuntimeDepsInstall({ + command: context.runner.command, + args: context.runner.args, + cwd: context.installExecutionRoot, + env: context.runner.env ?? context.installEnv, + packageManager: context.runner.packageManager, + onProgress: params.onProgress, + }); + finalizeBundledRuntimeDepsInstall({ installRoot: params.installRoot, context }); + } finally { + cleanupBundledRuntimeDepsInstallContext(context); + } +} + +export function repairBundledRuntimeDepsInstallRoot(params: { + installRoot: string; + missingSpecs: string[]; + installSpecs: string[]; + env: NodeJS.ProcessEnv; + installDeps?: (params: BundledRuntimeDepsInstallParams) => void; + warn?: (message: string) => void; +}): { installSpecs: string[] } { + return withBundledRuntimeDepsInstallRootLock(params.installRoot, () => { + const installSpecs = normalizeRuntimeDepSpecs(params.installSpecs); + const install = + params.installDeps ?? + ((installParams) => + installBundledRuntimeDeps({ + installRoot: installParams.installRoot, + missingSpecs: installParams.missingSpecs, + installSpecs: installParams.installSpecs, + env: params.env, + warn: params.warn, + })); + const finishActivity = beginBundledRuntimeDepsInstall({ + installRoot: params.installRoot, + missingSpecs: installSpecs, + installSpecs, + }); + ensureNpmInstallExecutionManifest(params.installRoot, installSpecs); + try { + install({ + installRoot: params.installRoot, + missingSpecs: installSpecs, + installSpecs, + }); + } finally { + finishActivity(); + } + removeLegacyRuntimeDepsManifest(params.installRoot); + return { installSpecs }; + }); +} + +export async function repairBundledRuntimeDepsInstallRootAsync(params: { + installRoot: string; + missingSpecs: string[]; + installSpecs: string[]; + env: NodeJS.ProcessEnv; + installDeps?: (params: BundledRuntimeDepsInstallParams) => Promise; + warn?: (message: string) => void; + onProgress?: (message: string) => void; +}): Promise<{ installSpecs: string[] }> { + return await withBundledRuntimeDepsInstallRootLockAsync(params.installRoot, async () => { + const installSpecs = normalizeRuntimeDepSpecs(params.installSpecs); + const install = + params.installDeps ?? + ((installParams) => + installBundledRuntimeDepsAsync({ + installRoot: installParams.installRoot, + missingSpecs: installParams.missingSpecs, + installSpecs: installParams.installSpecs, + env: params.env, + warn: params.warn, + onProgress: params.onProgress, + })); + const finishActivity = beginBundledRuntimeDepsInstall({ + installRoot: params.installRoot, + missingSpecs: installSpecs, + installSpecs, + }); + removeLegacyRuntimeDepsManifest(params.installRoot); + ensureNpmInstallExecutionManifest(params.installRoot, installSpecs); + try { + await install({ + installRoot: params.installRoot, + missingSpecs: installSpecs, + installSpecs, + }); + } finally { + finishActivity(); + } + removeLegacyRuntimeDepsManifest(params.installRoot); + return { installSpecs }; + }); +} diff --git a/src/plugins/bundled-runtime-deps-roots.ts b/src/plugins/bundled-runtime-deps-roots.ts new file mode 100644 index 00000000000..cdd0c8264d8 --- /dev/null +++ b/src/plugins/bundled-runtime-deps-roots.ts @@ -0,0 +1,381 @@ +import { createHash } from "node:crypto"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { resolveStateDir } from "../config/paths.js"; +import { resolveHomeRelativePath } from "../infra/home-dir.js"; +import { readRuntimeDepsJsonObject } from "./bundled-runtime-deps-json.js"; +import { + BUNDLED_RUNTIME_DEPS_LOCK_DIR, + removeRuntimeDepsLockIfStale, +} from "./bundled-runtime-deps-lock.js"; + +const DEFAULT_UNKNOWN_RUNTIME_DEPS_ROOTS_TO_KEEP = 20; +const DEFAULT_UNKNOWN_RUNTIME_DEPS_MIN_AGE_MS = 10 * 60_000; + +export type BundledRuntimeDepsInstallRoot = { + installRoot: string; + external: boolean; +}; + +export type BundledRuntimeDepsInstallRootPlan = BundledRuntimeDepsInstallRoot & { + searchRoots: string[]; +}; + +export function isSourceCheckoutRoot(packageRoot: string): boolean { + return ( + (fs.existsSync(path.join(packageRoot, ".git")) || + fs.existsSync(path.join(packageRoot, "pnpm-workspace.yaml"))) && + fs.existsSync(path.join(packageRoot, "src")) && + fs.existsSync(path.join(packageRoot, "extensions")) + ); +} + +function resolveBundledPluginPackageRoot(pluginRoot: string): string | null { + const extensionsDir = path.dirname(path.resolve(pluginRoot)); + const buildDir = path.dirname(extensionsDir); + if ( + path.basename(extensionsDir) !== "extensions" || + (path.basename(buildDir) !== "dist" && path.basename(buildDir) !== "dist-runtime") + ) { + return null; + } + return path.dirname(buildDir); +} + +export function resolveBundledRuntimeDependencyPackageRoot(pluginRoot: string): string | null { + return resolveBundledPluginPackageRoot(pluginRoot); +} + +function isPackagedBundledPluginRoot(pluginRoot: string): boolean { + const packageRoot = resolveBundledPluginPackageRoot(pluginRoot); + return Boolean(packageRoot && !isSourceCheckoutRoot(packageRoot)); +} + +function createPathHash(value: string): string { + return createHash("sha256").update(path.resolve(value)).digest("hex").slice(0, 12); +} + +function sanitizePathSegment(value: string): string { + return value.replace(/[^A-Za-z0-9._-]+/g, "-").replace(/^-+|-+$/g, "") || "unknown"; +} + +function readPackageVersion(packageRoot: string): string { + const parsed = readRuntimeDepsJsonObject(path.join(packageRoot, "package.json")); + const version = parsed && typeof parsed.version === "string" ? parsed.version.trim() : ""; + return version || "unknown"; +} + +export function isWritableDirectory(dir: string): boolean { + let probeDir: string | null = null; + try { + probeDir = fs.mkdtempSync(path.join(dir, ".openclaw-write-probe-")); + fs.writeFileSync(path.join(probeDir, "probe"), "", "utf8"); + return true; + } catch { + return false; + } finally { + if (probeDir) { + try { + fs.rmSync(probeDir, { recursive: true, force: true }); + } catch { + // Best-effort cleanup. A failed cleanup should not turn a writable + // probe into a hard runtime-dependency failure. + } + } + } +} + +function resolveSystemdStateDirectory(env: NodeJS.ProcessEnv): string | null { + const raw = env.STATE_DIRECTORY?.trim(); + if (!raw) { + return null; + } + const first = raw.split(path.delimiter).find((entry) => entry.trim().length > 0); + return first ? path.resolve(first) : null; +} + +function resolveBundledRuntimeDepsExternalBaseDirs(env: NodeJS.ProcessEnv): string[] { + const explicit = env.OPENCLAW_PLUGIN_STAGE_DIR?.trim(); + if (explicit) { + const roots = explicit + .split(path.delimiter) + .map((entry) => entry.trim()) + .filter((entry) => entry.length > 0) + .map((entry) => path.resolve(resolveHomeRelativePath(entry, { env, homedir: os.homedir }))); + if (roots.length > 0) { + const uniqueRoots: string[] = []; + for (const root of roots) { + const existingIndex = uniqueRoots.findIndex( + (entry) => path.resolve(entry) === path.resolve(root), + ); + if (existingIndex >= 0) { + uniqueRoots.splice(existingIndex, 1); + } + uniqueRoots.push(root); + } + return uniqueRoots; + } + } + const systemdStateDir = resolveSystemdStateDirectory(env); + if (systemdStateDir) { + return [path.join(systemdStateDir, "plugin-runtime-deps")]; + } + return [path.join(resolveStateDir(env, os.homedir), "plugin-runtime-deps")]; +} + +export function pruneUnknownBundledRuntimeDepsRoots( + params: { + env?: NodeJS.ProcessEnv; + nowMs?: number; + maxRootsToKeep?: number; + minAgeMs?: number; + warn?: (message: string) => void; + } = {}, +): { scanned: number; removed: number; skippedLocked: number } { + const env = params.env ?? process.env; + const nowMs = params.nowMs ?? Date.now(); + const maxRootsToKeep = Math.max( + 0, + params.maxRootsToKeep ?? DEFAULT_UNKNOWN_RUNTIME_DEPS_ROOTS_TO_KEEP, + ); + const minAgeMs = Math.max(0, params.minAgeMs ?? DEFAULT_UNKNOWN_RUNTIME_DEPS_MIN_AGE_MS); + let scanned = 0; + let removed = 0; + let skippedLocked = 0; + + for (const baseDir of resolveBundledRuntimeDepsExternalBaseDirs(env)) { + let entries: fs.Dirent[]; + try { + entries = fs.readdirSync(baseDir, { withFileTypes: true }); + } catch { + continue; + } + const unknownRoots = entries + .filter((entry) => entry.isDirectory() && entry.name.startsWith("openclaw-unknown-")) + .map((entry) => { + const root = path.join(baseDir, entry.name); + try { + return { root, mtimeMs: fs.statSync(root).mtimeMs }; + } catch { + return null; + } + }) + .filter((entry): entry is { root: string; mtimeMs: number } => entry !== null) + .toSorted((left, right) => right.mtimeMs - left.mtimeMs); + scanned += unknownRoots.length; + + for (const [index, entry] of unknownRoots.entries()) { + const ageMs = nowMs - entry.mtimeMs; + if (index < maxRootsToKeep && ageMs < minAgeMs) { + continue; + } + const lockDir = path.join(entry.root, BUNDLED_RUNTIME_DEPS_LOCK_DIR); + if (fs.existsSync(lockDir) && !removeRuntimeDepsLockIfStale(lockDir, nowMs)) { + skippedLocked += 1; + continue; + } + try { + fs.rmSync(entry.root, { recursive: true, force: true }); + removed += 1; + } catch (error) { + params.warn?.( + `failed to remove stale bundled runtime deps root ${entry.root}: ${String(error)}`, + ); + } + } + } + + return { scanned, removed, skippedLocked }; +} + +function resolveExternalBundledRuntimeDepsInstallRoot(params: { + pluginRoot: string; + env: NodeJS.ProcessEnv; +}): string { + return resolveExternalBundledRuntimeDepsInstallRoots(params).at(-1)!; +} + +function resolveExternalBundledRuntimeDepsInstallRoots(params: { + pluginRoot: string; + env: NodeJS.ProcessEnv; +}): string[] { + const packageRoot = resolveBundledPluginPackageRoot(params.pluginRoot) ?? params.pluginRoot; + const existingExternalRoots = resolveExistingExternalBundledRuntimeDepsRoots({ + packageRoot, + env: params.env, + }); + if (existingExternalRoots) { + return existingExternalRoots; + } + const version = sanitizePathSegment(readPackageVersion(packageRoot)); + const packageKey = `openclaw-${version}-${createPathHash(packageRoot)}`; + return resolveBundledRuntimeDepsExternalBaseDirs(params.env).map((baseDir) => + path.join(baseDir, packageKey), + ); +} + +function resolveExistingExternalBundledRuntimeDepsRoots(params: { + packageRoot: string; + env: NodeJS.ProcessEnv; +}): string[] | null { + const packageRoot = realpathOrResolve(params.packageRoot); + const externalBaseDirs = resolveBundledRuntimeDepsExternalBaseDirs(params.env); + for (const externalBaseDir of externalBaseDirs) { + const relative = path.relative(realpathOrResolve(externalBaseDir), packageRoot); + if (relative === "" || relative.startsWith("..") || path.isAbsolute(relative)) { + continue; + } + const packageKey = relative.split(path.sep)[0]; + if (!packageKey || !packageKey.startsWith("openclaw-")) { + continue; + } + return externalBaseDirs.map((baseDir) => path.join(baseDir, packageKey)); + } + return null; +} + +function realpathOrResolve(targetPath: string): string { + try { + return fs.realpathSync.native(targetPath); + } catch { + return path.resolve(targetPath); + } +} + +function createBundledRuntimeDepsInstallRootPlan(params: { + installRoot: string; + searchRoots: readonly string[]; + external: boolean; +}): BundledRuntimeDepsInstallRootPlan { + const searchRoots: string[] = []; + for (const root of params.searchRoots) { + const resolved = path.resolve(root); + if (!searchRoots.some((entry) => path.resolve(entry) === resolved)) { + searchRoots.push(root); + } + } + if (!searchRoots.some((entry) => path.resolve(entry) === path.resolve(params.installRoot))) { + searchRoots.push(params.installRoot); + } + return { + installRoot: params.installRoot, + searchRoots, + external: params.external, + }; +} + +export function resolveBundledRuntimeDependencyPackageInstallRootPlan( + packageRoot: string, + options: { env?: NodeJS.ProcessEnv; forceExternal?: boolean } = {}, +): BundledRuntimeDepsInstallRootPlan { + const env = options.env ?? process.env; + const externalRoots = resolveExternalBundledRuntimeDepsInstallRoots({ + pluginRoot: path.join(packageRoot, "dist", "extensions", "__package__"), + env, + }); + if ( + options.forceExternal || + env.OPENCLAW_PLUGIN_STAGE_DIR?.trim() || + env.STATE_DIRECTORY?.trim() || + !isSourceCheckoutRoot(packageRoot) + ) { + return createBundledRuntimeDepsInstallRootPlan({ + installRoot: + externalRoots.at(-1) ?? + resolveExternalBundledRuntimeDepsInstallRoot({ + pluginRoot: path.join(packageRoot, "dist", "extensions", "__package__"), + env, + }), + searchRoots: externalRoots, + external: true, + }); + } + if (isWritableDirectory(packageRoot)) { + return createBundledRuntimeDepsInstallRootPlan({ + installRoot: packageRoot, + searchRoots: [packageRoot], + external: false, + }); + } + return createBundledRuntimeDepsInstallRootPlan({ + installRoot: + externalRoots.at(-1) ?? + resolveExternalBundledRuntimeDepsInstallRoot({ + pluginRoot: path.join(packageRoot, "dist", "extensions", "__package__"), + env, + }), + searchRoots: externalRoots, + external: true, + }); +} + +export function resolveBundledRuntimeDependencyPackageInstallRoot( + packageRoot: string, + options: { env?: NodeJS.ProcessEnv; forceExternal?: boolean } = {}, +): string { + return resolveBundledRuntimeDependencyPackageInstallRootPlan(packageRoot, options).installRoot; +} + +export function resolveBundledRuntimeDependencyInstallRootPlan( + pluginRoot: string, + options: { env?: NodeJS.ProcessEnv; forceExternal?: boolean } = {}, +): BundledRuntimeDepsInstallRootPlan { + const env = options.env ?? process.env; + const externalRoots = resolveExternalBundledRuntimeDepsInstallRoots({ pluginRoot, env }); + if ( + options.forceExternal || + env.OPENCLAW_PLUGIN_STAGE_DIR?.trim() || + env.STATE_DIRECTORY?.trim() || + isPackagedBundledPluginRoot(pluginRoot) + ) { + return createBundledRuntimeDepsInstallRootPlan({ + installRoot: + externalRoots.at(-1) ?? + resolveExternalBundledRuntimeDepsInstallRoot({ + pluginRoot, + env, + }), + searchRoots: externalRoots, + external: true, + }); + } + if (isWritableDirectory(pluginRoot)) { + return createBundledRuntimeDepsInstallRootPlan({ + installRoot: pluginRoot, + searchRoots: [pluginRoot], + external: false, + }); + } + return createBundledRuntimeDepsInstallRootPlan({ + installRoot: + externalRoots.at(-1) ?? + resolveExternalBundledRuntimeDepsInstallRoot({ + pluginRoot, + env, + }), + searchRoots: externalRoots, + external: true, + }); +} + +export function resolveBundledRuntimeDependencyInstallRoot( + pluginRoot: string, + options: { env?: NodeJS.ProcessEnv; forceExternal?: boolean } = {}, +): string { + return resolveBundledRuntimeDependencyInstallRootPlan(pluginRoot, options).installRoot; +} + +export function resolveBundledRuntimeDependencyInstallRootInfo( + pluginRoot: string, + options: { env?: NodeJS.ProcessEnv; forceExternal?: boolean } = {}, +): BundledRuntimeDepsInstallRoot { + const { installRoot, external } = resolveBundledRuntimeDependencyInstallRootPlan( + pluginRoot, + options, + ); + return { + installRoot, + external, + }; +} diff --git a/src/plugins/bundled-runtime-deps-selection.ts b/src/plugins/bundled-runtime-deps-selection.ts new file mode 100644 index 00000000000..cef954cc381 --- /dev/null +++ b/src/plugins/bundled-runtime-deps-selection.ts @@ -0,0 +1,489 @@ +import fs from "node:fs"; +import path from "node:path"; +import type { OpenClawConfig } from "../config/types.openclaw.js"; +import { normalizeOptionalLowercaseString } from "../shared/string-coerce.js"; +import { readRuntimeDepsJsonObject, type JsonObject } from "./bundled-runtime-deps-json.js"; +import { + collectPackageRuntimeDeps, + normalizeInstallableRuntimeDepName, + parseInstallableRuntimeDep, + type RuntimeDepEntry, +} from "./bundled-runtime-deps-specs.js"; +import { + normalizePluginsConfigWithResolver, + type NormalizedPluginsConfig, + type NormalizePluginId, +} from "./config-normalization-shared.js"; + +const MIRRORED_PACKAGE_RUNTIME_DEP_PLUGIN_ID = "openclaw-core"; + +export type RuntimeDepConflict = { + name: string; + versions: string[]; + pluginIdsByVersion: Map; +}; + +export type BundledPluginRuntimeDepsManifest = { + channels: string[]; + enabledByDefault: boolean; + id?: string; + legacyPluginIds: string[]; + providers: string[]; +}; + +export type BundledPluginRuntimeDepsManifestCache = Map; + +function collectDeclaredMirroredRootRuntimeDepNames(packageJson: JsonObject): string[] { + const openclaw = packageJson.openclaw; + const bundle = + openclaw && typeof openclaw === "object" && !Array.isArray(openclaw) + ? (openclaw as JsonObject).bundle + : undefined; + const rawNames = + bundle && typeof bundle === "object" && !Array.isArray(bundle) + ? (bundle as JsonObject).mirroredRootRuntimeDependencies + : undefined; + if (rawNames === undefined) { + return []; + } + if (!Array.isArray(rawNames)) { + throw new Error("openclaw.bundle.mirroredRootRuntimeDependencies must be an array"); + } + const names = new Set(); + for (const rawName of rawNames) { + if (typeof rawName !== "string") { + throw new Error("openclaw.bundle.mirroredRootRuntimeDependencies must contain strings"); + } + const normalizedName = normalizeInstallableRuntimeDepName(rawName); + if (!normalizedName) { + throw new Error(`Invalid mirrored bundled runtime dependency name: ${rawName}`); + } + names.add(normalizedName); + } + return [...names].toSorted((left, right) => left.localeCompare(right)); +} + +export function collectMirroredPackageRuntimeDeps(packageRoot: string | null): RuntimeDepEntry[] { + if (!packageRoot) { + return []; + } + const packageJson = readRuntimeDepsJsonObject(path.join(packageRoot, "package.json")); + if (!packageJson) { + return []; + } + const runtimeDeps = collectPackageRuntimeDeps(packageJson); + const deps: RuntimeDepEntry[] = []; + for (const name of collectDeclaredMirroredRootRuntimeDepNames(packageJson)) { + const dep = parseInstallableRuntimeDep(name, runtimeDeps[name]); + if (!dep) { + throw new Error( + `Declared mirrored bundled runtime dependency ${name} is missing from package dependencies`, + ); + } + deps.push({ + ...dep, + pluginIds: [MIRRORED_PACKAGE_RUNTIME_DEP_PLUGIN_ID], + }); + } + return deps.toSorted((left, right) => { + const nameOrder = left.name.localeCompare(right.name); + return nameOrder === 0 ? left.version.localeCompare(right.version) : nameOrder; + }); +} + +function readBundledPluginRuntimeDepsManifest( + pluginDir: string, + cache?: BundledPluginRuntimeDepsManifestCache, +): BundledPluginRuntimeDepsManifest { + const cached = cache?.get(pluginDir); + if (cached) { + return cached; + } + const manifest = readRuntimeDepsJsonObject(path.join(pluginDir, "openclaw.plugin.json")); + const channels = manifest?.channels; + const legacyPluginIds = manifest?.legacyPluginIds; + const providers = manifest?.providers; + const runtimeDepsManifest = { + channels: Array.isArray(channels) + ? channels.filter((entry): entry is string => typeof entry === "string" && entry !== "") + : [], + enabledByDefault: manifest?.enabledByDefault === true, + ...(typeof manifest?.id === "string" && manifest.id.trim() ? { id: manifest.id } : {}), + legacyPluginIds: Array.isArray(legacyPluginIds) + ? legacyPluginIds.filter( + (entry): entry is string => typeof entry === "string" && entry !== "", + ) + : [], + providers: Array.isArray(providers) + ? providers.filter((entry): entry is string => typeof entry === "string" && entry !== "") + : [], + }; + cache?.set(pluginDir, runtimeDepsManifest); + return runtimeDepsManifest; +} + +const BUILT_IN_RUNTIME_DEPS_PLUGIN_ALIAS_FALLBACKS: ReadonlyArray< + readonly [alias: string, pluginId: string] +> = [ + ["openai-codex", "openai"], + ["google-gemini-cli", "google"], + ["minimax-portal", "minimax"], + ["minimax-portal-auth", "minimax"], +] as const; + +function addBundledRuntimeDepsPluginAlias( + lookup: Map, + alias: string | undefined, + pluginId: string, +): void { + const normalizedAlias = normalizeOptionalLowercaseString(alias); + if (normalizedAlias) { + lookup.set(normalizedAlias, pluginId); + } +} + +export function createBundledRuntimeDepsPluginIdNormalizer(params: { + extensionsDir: string; + manifestCache: BundledPluginRuntimeDepsManifestCache; +}): NormalizePluginId { + const lookup = new Map(); + for (const [alias, pluginId] of BUILT_IN_RUNTIME_DEPS_PLUGIN_ALIAS_FALLBACKS) { + lookup.set(alias, pluginId); + lookup.set(pluginId, pluginId); + } + if (!fs.existsSync(params.extensionsDir)) { + return (id) => { + const trimmed = id.trim(); + const normalized = normalizeOptionalLowercaseString(trimmed); + return (normalized && lookup.get(normalized)) || trimmed; + }; + } + for (const entry of fs.readdirSync(params.extensionsDir, { withFileTypes: true })) { + if (!entry.isDirectory()) { + continue; + } + const fallbackPluginId = entry.name; + const pluginDir = path.join(params.extensionsDir, fallbackPluginId); + const manifest = readBundledPluginRuntimeDepsManifest(pluginDir, params.manifestCache); + const pluginId = manifest.id ?? fallbackPluginId; + addBundledRuntimeDepsPluginAlias(lookup, pluginId, pluginId); + addBundledRuntimeDepsPluginAlias(lookup, fallbackPluginId, pluginId); + for (const providerId of manifest.providers) { + addBundledRuntimeDepsPluginAlias(lookup, providerId, pluginId); + } + for (const legacyPluginId of manifest.legacyPluginIds) { + addBundledRuntimeDepsPluginAlias(lookup, legacyPluginId, pluginId); + } + } + return (id) => { + const trimmed = id.trim(); + const normalized = normalizeOptionalLowercaseString(trimmed); + return (normalized && lookup.get(normalized)) || trimmed; + }; +} + +function passesRuntimeDepsPluginPolicy(params: { + pluginId: string; + plugins: NormalizedPluginsConfig; + allowExplicitlyDisabled?: boolean; + allowRestrictiveAllowlistBypass?: boolean; +}): boolean { + if (!params.plugins.enabled) { + return false; + } + if (params.plugins.deny.includes(params.pluginId)) { + return false; + } + if ( + params.plugins.entries[params.pluginId]?.enabled === false && + params.allowExplicitlyDisabled !== true + ) { + return false; + } + return ( + params.allowRestrictiveAllowlistBypass === true || + params.plugins.allow.length === 0 || + params.plugins.allow.includes(params.pluginId) + ); +} + +export function isBundledPluginConfiguredForRuntimeDeps(params: { + config: OpenClawConfig; + plugins: NormalizedPluginsConfig; + pluginId: string; + pluginDir: string; + includeConfiguredChannels?: boolean; + manifestCache?: BundledPluginRuntimeDepsManifestCache; +}): boolean { + if ( + !passesRuntimeDepsPluginPolicy({ + pluginId: params.pluginId, + plugins: params.plugins, + allowRestrictiveAllowlistBypass: true, + }) + ) { + return false; + } + const entry = params.plugins.entries[params.pluginId]; + const manifest = readBundledPluginRuntimeDepsManifest(params.pluginDir, params.manifestCache); + if ( + params.plugins.slots.memory === params.pluginId || + params.plugins.slots.contextEngine === params.pluginId + ) { + return true; + } + let hasExplicitChannelDisable = false; + let hasConfiguredChannel = false; + for (const channelId of manifest.channels) { + const normalizedChannelId = normalizeOptionalLowercaseString(channelId); + if (!normalizedChannelId) { + continue; + } + const channelConfig = (params.config.channels as Record | undefined)?.[ + normalizedChannelId + ]; + if ( + channelConfig && + typeof channelConfig === "object" && + !Array.isArray(channelConfig) && + (channelConfig as { enabled?: unknown }).enabled === false + ) { + hasExplicitChannelDisable = true; + continue; + } + if ( + channelConfig && + typeof channelConfig === "object" && + !Array.isArray(channelConfig) && + (channelConfig as { enabled?: unknown }).enabled === true + ) { + return true; + } + if ( + channelConfig && + typeof channelConfig === "object" && + !Array.isArray(channelConfig) && + params.includeConfiguredChannels + ) { + hasConfiguredChannel = true; + } + } + if (hasExplicitChannelDisable) { + return false; + } + if (params.plugins.allow.length > 0 && !params.plugins.allow.includes(params.pluginId)) { + return false; + } + if (entry?.enabled === true) { + return true; + } + if (hasConfiguredChannel) { + return true; + } + return manifest.enabledByDefault; +} + +function isBundledPluginExplicitlyDisabledForRuntimeDeps(params: { + config: OpenClawConfig; + plugins: NormalizedPluginsConfig; + pluginId: string; + pluginDir: string; + manifestCache?: BundledPluginRuntimeDepsManifestCache; +}): boolean { + if (params.plugins.entries[params.pluginId]?.enabled === false) { + return true; + } + const manifest = readBundledPluginRuntimeDepsManifest(params.pluginDir, params.manifestCache); + return manifest.channels.some((channelId) => { + const normalizedChannelId = normalizeOptionalLowercaseString(channelId); + if (!normalizedChannelId) { + return false; + } + const channelConfig = (params.config.channels as Record | undefined)?.[ + normalizedChannelId + ]; + return ( + channelConfig && + typeof channelConfig === "object" && + !Array.isArray(channelConfig) && + (channelConfig as { enabled?: unknown }).enabled === false + ); + }); +} + +function shouldIncludeBundledPluginRuntimeDeps(params: { + config?: OpenClawConfig; + plugins?: NormalizedPluginsConfig; + pluginIds?: ReadonlySet; + selectedPluginIds?: ReadonlySet; + pluginId: string; + pluginDir: string; + includeConfiguredChannels?: boolean; + manifestCache?: BundledPluginRuntimeDepsManifestCache; +}): boolean { + if (params.selectedPluginIds) { + return ( + params.selectedPluginIds.has(params.pluginId) && + !( + params.config && + params.plugins && + isBundledPluginExplicitlyDisabledForRuntimeDeps({ + config: params.config, + plugins: params.plugins, + pluginId: params.pluginId, + pluginDir: params.pluginDir, + manifestCache: params.manifestCache, + }) + ) + ); + } + const scopedToPluginIds = Boolean(params.pluginIds); + if (params.pluginIds) { + if (!params.pluginIds.has(params.pluginId)) { + return false; + } + if (!params.config) { + return true; + } + } + if (!params.config) { + return true; + } + if (scopedToPluginIds) { + if (!params.plugins) { + return true; + } + return passesRuntimeDepsPluginPolicy({ + pluginId: params.pluginId, + plugins: params.plugins, + allowRestrictiveAllowlistBypass: true, + }); + } + if (!params.plugins) { + return false; + } + return isBundledPluginConfiguredForRuntimeDeps({ + config: params.config, + plugins: params.plugins, + pluginId: params.pluginId, + pluginDir: params.pluginDir, + includeConfiguredChannels: params.includeConfiguredChannels, + manifestCache: params.manifestCache, + }); +} + +export function collectBundledPluginRuntimeDeps(params: { + extensionsDir: string; + config?: OpenClawConfig; + pluginIds?: ReadonlySet; + selectedPluginIds?: ReadonlySet; + includeConfiguredChannels?: boolean; + manifestCache?: BundledPluginRuntimeDepsManifestCache; + normalizePluginId?: NormalizePluginId; +}): { + deps: RuntimeDepEntry[]; + conflicts: RuntimeDepConflict[]; + pluginIds: string[]; +} { + const versionMap = new Map>>(); + const manifestCache: BundledPluginRuntimeDepsManifestCache = params.manifestCache ?? new Map(); + const needsPluginIdNormalizer = Boolean(params.config); + const normalizePluginId = + params.normalizePluginId ?? + (needsPluginIdNormalizer + ? createBundledRuntimeDepsPluginIdNormalizer({ + extensionsDir: params.extensionsDir, + manifestCache, + }) + : undefined); + const plugins = params.config + ? normalizePluginsConfigWithResolver(params.config.plugins, normalizePluginId) + : undefined; + const includedPluginIds = new Set(); + + for (const entry of fs.readdirSync(params.extensionsDir, { withFileTypes: true })) { + if (!entry.isDirectory()) { + continue; + } + const pluginId = entry.name; + const pluginDir = path.join(params.extensionsDir, pluginId); + if ( + !shouldIncludeBundledPluginRuntimeDeps({ + config: params.config, + plugins, + pluginIds: params.pluginIds, + selectedPluginIds: params.selectedPluginIds, + pluginId, + pluginDir, + includeConfiguredChannels: params.includeConfiguredChannels, + manifestCache, + }) + ) { + continue; + } + includedPluginIds.add(pluginId); + const packageJson = readRuntimeDepsJsonObject(path.join(pluginDir, "package.json")); + if (!packageJson) { + continue; + } + for (const [name, rawVersion] of Object.entries(collectPackageRuntimeDeps(packageJson))) { + const dep = parseInstallableRuntimeDep(name, rawVersion); + if (!dep) { + continue; + } + const byVersion = versionMap.get(dep.name) ?? new Map>(); + const pluginIds = byVersion.get(dep.version) ?? new Set(); + pluginIds.add(pluginId); + byVersion.set(dep.version, pluginIds); + versionMap.set(dep.name, byVersion); + } + } + + const deps: RuntimeDepEntry[] = []; + const conflicts: RuntimeDepConflict[] = []; + for (const [name, byVersion] of versionMap.entries()) { + if (byVersion.size === 1) { + const [version, pluginIds] = [...byVersion.entries()][0] ?? []; + if (version) { + deps.push({ + name, + version, + pluginIds: [...pluginIds].toSorted((a, b) => a.localeCompare(b)), + }); + } + continue; + } + const versions = [...byVersion.keys()].toSorted((a, b) => a.localeCompare(b)); + const pluginIdsByVersion = new Map(); + for (const [version, pluginIds] of byVersion.entries()) { + pluginIdsByVersion.set( + version, + [...pluginIds].toSorted((a, b) => a.localeCompare(b)), + ); + } + conflicts.push({ + name, + versions, + pluginIdsByVersion, + }); + } + + return { + deps: deps.toSorted((a, b) => a.name.localeCompare(b.name)), + conflicts: conflicts.toSorted((a, b) => a.name.localeCompare(b.name)), + pluginIds: [...includedPluginIds].toSorted((a, b) => a.localeCompare(b)), + }; +} + +export function normalizePluginIdSet( + pluginIds: readonly string[] | undefined, + normalizePluginId: NormalizePluginId = (id) => normalizeOptionalLowercaseString(id) ?? "", +): ReadonlySet | undefined { + if (!pluginIds) { + return undefined; + } + const normalized = pluginIds + .map((entry) => normalizePluginId(entry)) + .filter((entry): entry is string => Boolean(entry)); + return new Set(normalized); +} diff --git a/src/plugins/bundled-runtime-deps.test.ts b/src/plugins/bundled-runtime-deps.test.ts index f085f445b84..e9186cdf4e5 100644 --- a/src/plugins/bundled-runtime-deps.test.ts +++ b/src/plugins/bundled-runtime-deps.test.ts @@ -2,7 +2,6 @@ import { spawn, spawnSync } from "node:child_process"; import { createHash } from "node:crypto"; import { EventEmitter } from "node:events"; import fs from "node:fs"; -import { Module } from "node:module"; import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; @@ -3483,223 +3482,3 @@ describe("ensureBundledPluginRuntimeDeps", () => { expect(fs.existsSync(path.join(pluginRoot, "node_modules", "zod", "package.json"))).toBe(true); }); }); - -describe("mirrored root runtime dependency drift guard", () => { - // Intentionally not mirrored at runtime: build-only / type-only / TUI-only - // tooling and packages that resolve transitively through other mirrored deps. - // If you change this set, document why in the comment beside the entry. - const KNOWN_UNMIRRORED_BARE_IMPORTS = new Set([ - "@mariozechner/pi-tui", // TUI mode runs from npm-global, not the gateway runtime mirror - "chalk", // available transitively via mirrored deps - "file-type", // available transitively via mirrored deps - "global-agent", // proxy bootstrap, only loaded when HTTP_PROXY is set - "ipaddr.js", // available transitively via mirrored deps - "proxy-agent", // available transitively via mirrored deps - "qrcode", // type-only import in src/media/qr-runtime.ts - "typescript", // CLI/dev only (api-baseline, jiti-runtime-api) - ]); - - function locateRepoRoot(): string { - let dir = path.resolve(import.meta.dirname); - for (let depth = 0; depth < 10; depth += 1) { - const candidate = path.join(dir, "package.json"); - if (fs.existsSync(candidate)) { - try { - const data = JSON.parse(fs.readFileSync(candidate, "utf8")) as { name?: string }; - if (data.name === "openclaw") { - return dir; - } - } catch { - // fall through - } - } - const parent = path.dirname(dir); - if (parent === dir) { - break; - } - dir = parent; - } - throw new Error("could not locate openclaw repo root from test file"); - } - - function readPackageJsonDeps(packageJsonPath: string): Set { - const out = new Set(); - if (!fs.existsSync(packageJsonPath)) { - return out; - } - let parsed: { - dependencies?: Record; - optionalDependencies?: Record; - }; - try { - parsed = JSON.parse(fs.readFileSync(packageJsonPath, "utf8")); - } catch { - return out; - } - for (const name of Object.keys(parsed.dependencies ?? {})) { - out.add(name); - } - for (const name of Object.keys(parsed.optionalDependencies ?? {})) { - out.add(name); - } - return out; - } - - function readMirroredRootRuntimeDeps(repoRoot: string): Set { - const parsed = JSON.parse(fs.readFileSync(path.join(repoRoot, "package.json"), "utf8")) as { - openclaw?: { - bundle?: { - mirroredRootRuntimeDependencies?: unknown; - }; - }; - }; - const deps = parsed.openclaw?.bundle?.mirroredRootRuntimeDependencies; - return new Set(Array.isArray(deps) ? deps.filter((dep) => typeof dep === "string") : []); - } - - function collectExtensionOwnedDeps(repoRoot: string): Set { - const out = new Set(); - const extensionsDir = path.join(repoRoot, "extensions"); - if (!fs.existsSync(extensionsDir)) { - return out; - } - for (const entry of fs.readdirSync(extensionsDir, { withFileTypes: true })) { - if (!entry.isDirectory()) { - continue; - } - for (const name of readPackageJsonDeps( - path.join(extensionsDir, entry.name, "package.json"), - )) { - out.add(name); - } - } - return out; - } - - function walkCoreSourceFiles(repoRoot: string): string[] { - const srcDir = path.join(repoRoot, "src"); - const files: string[] = []; - const queue: string[] = [srcDir]; - while (queue.length > 0) { - const current = queue.shift(); - if (!current) { - continue; - } - for (const entry of fs.readdirSync(current, { withFileTypes: true })) { - const full = path.join(current, entry.name); - if (entry.isDirectory()) { - if (entry.name === "node_modules" || entry.name.startsWith(".")) { - continue; - } - queue.push(full); - continue; - } - if (!entry.isFile()) { - continue; - } - if ( - /\.test\.tsx?$/u.test(entry.name) || - /\.e2e\.test\.tsx?$/u.test(entry.name) || - /\.test-helpers?\.tsx?$/u.test(entry.name) || - /\.test-fixture\.tsx?$/u.test(entry.name) || - entry.name.endsWith(".d.ts") || - !/\.(?:ts|tsx|cjs|mjs|js)$/u.test(entry.name) - ) { - continue; - } - files.push(full); - } - } - return files; - } - - function packageNameFromBareSpecifier(specifier: string): string | null { - if ( - specifier.startsWith(".") || - specifier.startsWith("/") || - specifier.startsWith("node:") || - specifier.startsWith("#") - ) { - return null; - } - const [first, second] = specifier.split("/"); - if (!first) { - return null; - } - return first.startsWith("@") && second ? `${first}/${second}` : first; - } - - // Match value imports (`import x from 'y'`, `import 'y'`, `require('y')`, - // `import('y')`) but skip `import type` to avoid noise from type-only imports. - const VALUE_IMPORT_PATTERNS = [ - /(?:^|[;\n])\s*import\s+(?!type\b)(?:[^'"()]+?\s+from\s+)?["']([^"']+)["']/g, - /\brequire\s*\(\s*["']([^"']+)["']\s*\)/g, - /\bimport\s*\(\s*["']([^"']+)["']\s*\)/g, - ] as const; - - it("every value-imported root-package dep in src/ is mirrored or owned by an extension", () => { - const repoRoot = locateRepoRoot(); - const rootDeps = readPackageJsonDeps(path.join(repoRoot, "package.json")); - const extensionDeps = collectExtensionOwnedDeps(repoRoot); - const mirroredCore = readMirroredRootRuntimeDeps(repoRoot); - const nodeBuiltins = new Set(Module.builtinModules); - - const violations = new Map(); - for (const file of walkCoreSourceFiles(repoRoot)) { - const source = fs.readFileSync(file, "utf8"); - const specifiers = new Set(); - for (const pattern of VALUE_IMPORT_PATTERNS) { - for (const match of source.matchAll(pattern)) { - if (match[1]) { - specifiers.add(match[1]); - } - } - } - for (const specifier of specifiers) { - const packageName = packageNameFromBareSpecifier(specifier); - if (!packageName) { - continue; - } - if (nodeBuiltins.has(packageName)) { - continue; - } - if (packageName === "openclaw" || packageName.startsWith("@openclaw/")) { - continue; - } - if (mirroredCore.has(packageName) || extensionDeps.has(packageName)) { - continue; - } - if (KNOWN_UNMIRRORED_BARE_IMPORTS.has(packageName)) { - continue; - } - if (!rootDeps.has(packageName)) { - // Not a root runtime dep; not our concern (could be a peer/dev import - // that resolves through some other path; the mirror does not own it). - continue; - } - if (!violations.has(packageName)) { - violations.set(packageName, path.relative(repoRoot, file).replaceAll(path.sep, "/")); - } - } - } - - if (violations.size > 0) { - const summary = [...violations.entries()] - .toSorted(([left], [right]) => left.localeCompare(right)) - .map(([packageName, filePath]) => ` - ${packageName} (e.g. ${filePath})`) - .join("\n"); - throw new Error( - [ - "Bare imports found in src/ that are root-package runtime deps but are neither", - "in package.json openclaw.bundle.mirroredRootRuntimeDependencies nor declared by any extension's package.json.", - "These will be missing from the runtime-deps mirror at gateway start and Node", - "will fail to resolve them. Either add the package to openclaw.bundle.mirroredRootRuntimeDependencies,", - "declare it under an owning extension's dependencies, or add it to", - "KNOWN_UNMIRRORED_BARE_IMPORTS in this test with a comment explaining why.", - "", - summary, - ].join("\n"), - ); - } - }); -}); diff --git a/src/plugins/bundled-runtime-deps.ts b/src/plugins/bundled-runtime-deps.ts index 2560b216ece..7b0ca4f2649 100644 --- a/src/plugins/bundled-runtime-deps.ts +++ b/src/plugins/bundled-runtime-deps.ts @@ -1,27 +1,23 @@ -import { spawn, spawnSync } from "node:child_process"; -import { createHash } from "node:crypto"; import fs from "node:fs"; import { Module } from "node:module"; -import os from "node:os"; import path from "node:path"; -import { resolveStateDir } from "../config/paths.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; -import { createLowDiskSpaceWarning } from "../infra/disk-space.js"; -import { resolveHomeRelativePath } from "../infra/home-dir.js"; -import { normalizeOptionalLowercaseString } from "../shared/string-coerce.js"; -import { sanitizeTerminalText } from "../terminal/safe-text.js"; import { beginBundledRuntimeDepsInstall } from "./bundled-runtime-deps-activity.js"; -import { readRuntimeDepsJsonObject, type JsonObject } from "./bundled-runtime-deps-json.js"; +import { + installBundledRuntimeDeps, + installBundledRuntimeDepsAsync, + repairBundledRuntimeDepsInstallRoot, + repairBundledRuntimeDepsInstallRootAsync, + type BundledRuntimeDepsInstallParams, +} from "./bundled-runtime-deps-install.js"; +import { readRuntimeDepsJsonObject } from "./bundled-runtime-deps-json.js"; import { BUNDLED_RUNTIME_DEPS_LOCK_DIR, formatRuntimeDepsLockTimeoutMessage, - removeRuntimeDepsLockIfStale, shouldRemoveRuntimeDepsLock, withBundledRuntimeDepsFilesystemLock, - withBundledRuntimeDepsFilesystemLockAsync, } from "./bundled-runtime-deps-lock.js"; import { - assertBundledRuntimeDepsInstalled, ensureNpmInstallExecutionManifest, isRuntimeDepSatisfiedInAnyRoot, isRuntimeDepsPlanMaterialized, @@ -31,33 +27,67 @@ import { createBundledRuntimeDepsInstallArgs, createBundledRuntimeDepsInstallEnv, resolveBundledRuntimeDepsNpmRunner, - resolveBundledRuntimeDepsPackageManagerRunner, resolveBundledRuntimeDepsPnpmRunner, type BundledRuntimeDepsNpmRunner, - type BundledRuntimeDepsPackageManager, - type BundledRuntimeDepsPackageManagerRunner, } from "./bundled-runtime-deps-package-manager.js"; +import { + isSourceCheckoutRoot, + isWritableDirectory, + pruneUnknownBundledRuntimeDepsRoots, + resolveBundledRuntimeDependencyInstallRoot, + resolveBundledRuntimeDependencyInstallRootInfo, + resolveBundledRuntimeDependencyInstallRootPlan, + resolveBundledRuntimeDependencyPackageInstallRoot, + resolveBundledRuntimeDependencyPackageInstallRootPlan, + resolveBundledRuntimeDependencyPackageRoot, + type BundledRuntimeDepsInstallRoot, + type BundledRuntimeDepsInstallRootPlan, +} from "./bundled-runtime-deps-roots.js"; +import { + collectBundledPluginRuntimeDeps, + collectMirroredPackageRuntimeDeps, + createBundledRuntimeDepsPluginIdNormalizer, + isBundledPluginConfiguredForRuntimeDeps, + normalizePluginIdSet, + type BundledPluginRuntimeDepsManifestCache, + type RuntimeDepConflict, +} from "./bundled-runtime-deps-selection.js"; import { collectPackageRuntimeDeps, normalizeInstallableRuntimeDepName, - normalizeRuntimeDepSpecs, parseInstallableRuntimeDep, type RuntimeDepEntry, } from "./bundled-runtime-deps-specs.js"; -import { - normalizePluginsConfigWithResolver, - type NormalizedPluginsConfig, - type NormalizePluginId, -} from "./config-normalization-shared.js"; +import { normalizePluginsConfigWithResolver } from "./config-normalization-shared.js"; export { createBundledRuntimeDepsInstallArgs, createBundledRuntimeDepsInstallEnv, + installBundledRuntimeDeps, + installBundledRuntimeDepsAsync, + repairBundledRuntimeDepsInstallRoot, + repairBundledRuntimeDepsInstallRootAsync, resolveBundledRuntimeDepsNpmRunner, withBundledRuntimeDepsFilesystemLock, }; export type { BundledRuntimeDepsNpmRunner }; +export type { BundledRuntimeDepsInstallParams } from "./bundled-runtime-deps-install.js"; export type { RuntimeDepEntry } from "./bundled-runtime-deps-specs.js"; +export { + isWritableDirectory, + pruneUnknownBundledRuntimeDepsRoots, + resolveBundledRuntimeDependencyInstallRoot, + resolveBundledRuntimeDependencyInstallRootInfo, + resolveBundledRuntimeDependencyInstallRootPlan, + resolveBundledRuntimeDependencyPackageInstallRoot, + resolveBundledRuntimeDependencyPackageInstallRootPlan, + resolveBundledRuntimeDependencyPackageRoot, +}; +export type { + BundledRuntimeDepsInstallRoot, + BundledRuntimeDepsInstallRootPlan, +} from "./bundled-runtime-deps-roots.js"; +export type { RuntimeDepConflict } from "./bundled-runtime-deps-selection.js"; export const __testing = { formatRuntimeDepsLockTimeoutMessage, @@ -65,33 +95,10 @@ export const __testing = { shouldRemoveRuntimeDepsLock, }; -export type RuntimeDepConflict = { - name: string; - versions: string[]; - pluginIdsByVersion: Map; -}; - -export type BundledRuntimeDepsInstallParams = { - installRoot: string; - installExecutionRoot?: string; - missingSpecs: string[]; - installSpecs?: string[]; - warn?: (message: string) => void; -}; - export type BundledRuntimeDepsEnsureResult = { installedSpecs: string[]; }; -export type BundledRuntimeDepsInstallRoot = { - installRoot: string; - external: boolean; -}; - -export type BundledRuntimeDepsInstallRootPlan = BundledRuntimeDepsInstallRoot & { - searchRoots: string[]; -}; - export type BundledRuntimeDepsPlan = { deps: RuntimeDepEntry[]; missing: RuntimeDepEntry[]; @@ -107,10 +114,6 @@ export type BundledRuntimeDepsPlan = { // install inside this sub-directory and move the produced `node_modules/` back // to the plugin root. const PLUGIN_ROOT_INSTALL_STAGE_DIR = ".openclaw-install-stage"; -const DEFAULT_UNKNOWN_RUNTIME_DEPS_ROOTS_TO_KEEP = 20; -const DEFAULT_UNKNOWN_RUNTIME_DEPS_MIN_AGE_MS = 10 * 60_000; -const BUNDLED_RUNTIME_DEPS_INSTALL_PROGRESS_INTERVAL_MS = 5_000; -const MIRRORED_PACKAGE_RUNTIME_DEP_PLUGIN_ID = "openclaw-core"; const registeredBundledRuntimeDepNodePaths = new Set(); @@ -124,79 +127,6 @@ function withBundledRuntimeDepsInstallRootLock(installRoot: string, run: () = return withBundledRuntimeDepsFilesystemLock(installRoot, BUNDLED_RUNTIME_DEPS_LOCK_DIR, run); } -async function withBundledRuntimeDepsInstallRootLockAsync( - installRoot: string, - run: () => Promise, -): Promise { - return await withBundledRuntimeDepsFilesystemLockAsync( - installRoot, - BUNDLED_RUNTIME_DEPS_LOCK_DIR, - run, - ); -} - -function collectDeclaredMirroredRootRuntimeDepNames(packageJson: JsonObject): string[] { - const openclaw = packageJson.openclaw; - const bundle = - openclaw && typeof openclaw === "object" && !Array.isArray(openclaw) - ? (openclaw as JsonObject).bundle - : undefined; - const rawNames = - bundle && typeof bundle === "object" && !Array.isArray(bundle) - ? (bundle as JsonObject).mirroredRootRuntimeDependencies - : undefined; - if (rawNames === undefined) { - return []; - } - if (!Array.isArray(rawNames)) { - throw new Error("openclaw.bundle.mirroredRootRuntimeDependencies must be an array"); - } - const names = new Set(); - for (const rawName of rawNames) { - if (typeof rawName !== "string") { - throw new Error("openclaw.bundle.mirroredRootRuntimeDependencies must contain strings"); - } - const normalizedName = normalizeInstallableRuntimeDepName(rawName); - if (!normalizedName) { - throw new Error(`Invalid mirrored bundled runtime dependency name: ${rawName}`); - } - names.add(normalizedName); - } - return [...names].toSorted((left, right) => left.localeCompare(right)); -} - -function collectMirroredPackageRuntimeDeps(packageRoot: string | null): { - name: string; - version: string; - pluginIds: string[]; -}[] { - if (!packageRoot) { - return []; - } - const packageJson = readRuntimeDepsJsonObject(path.join(packageRoot, "package.json")); - if (!packageJson) { - return []; - } - const runtimeDeps = collectPackageRuntimeDeps(packageJson); - const deps: RuntimeDepEntry[] = []; - for (const name of collectDeclaredMirroredRootRuntimeDepNames(packageJson)) { - const dep = parseInstallableRuntimeDep(name, runtimeDeps[name]); - if (!dep) { - throw new Error( - `Declared mirrored bundled runtime dependency ${name} is missing from package dependencies`, - ); - } - deps.push({ - ...dep, - pluginIds: [MIRRORED_PACKAGE_RUNTIME_DEP_PLUGIN_ID], - }); - } - return deps.toSorted((left, right) => { - const nameOrder = left.name.localeCompare(right.name); - return nameOrder === 0 ? left.version.localeCompare(right.version) : nameOrder; - }); -} - function mergeRuntimeDepEntries(deps: readonly RuntimeDepEntry[]): RuntimeDepEntry[] { const bySpec = new Map(); for (const dep of deps) { @@ -216,31 +146,6 @@ function mergeRuntimeDepEntries(deps: readonly RuntimeDepEntry[]): RuntimeDepEnt }); } -function isSourceCheckoutRoot(packageRoot: string): boolean { - return ( - (fs.existsSync(path.join(packageRoot, ".git")) || - fs.existsSync(path.join(packageRoot, "pnpm-workspace.yaml"))) && - fs.existsSync(path.join(packageRoot, "src")) && - fs.existsSync(path.join(packageRoot, "extensions")) - ); -} - -function resolveBundledPluginPackageRoot(pluginRoot: string): string | null { - const extensionsDir = path.dirname(path.resolve(pluginRoot)); - const buildDir = path.dirname(extensionsDir); - if ( - path.basename(extensionsDir) !== "extensions" || - (path.basename(buildDir) !== "dist" && path.basename(buildDir) !== "dist-runtime") - ) { - return null; - } - return path.dirname(buildDir); -} - -export function resolveBundledRuntimeDependencyPackageRoot(pluginRoot: string): string | null { - return resolveBundledPluginPackageRoot(pluginRoot); -} - export function registerBundledRuntimeDependencyNodePath(rootDir: string): void { const nodeModulesDir = path.join(rootDir, "node_modules"); if (registeredBundledRuntimeDepNodePaths.has(nodeModulesDir) || !fs.existsSync(nodeModulesDir)) { @@ -274,224 +179,6 @@ export function clearBundledRuntimeDependencyNodePaths(): void { (Module as unknown as { _initPaths?: () => void })._initPaths?.(); } -function isPackagedBundledPluginRoot(pluginRoot: string): boolean { - const packageRoot = resolveBundledPluginPackageRoot(pluginRoot); - return Boolean(packageRoot && !isSourceCheckoutRoot(packageRoot)); -} - -function createPathHash(value: string): string { - return createHash("sha256").update(path.resolve(value)).digest("hex").slice(0, 12); -} - -function sanitizePathSegment(value: string): string { - return value.replace(/[^A-Za-z0-9._-]+/g, "-").replace(/^-+|-+$/g, "") || "unknown"; -} - -function readPackageVersion(packageRoot: string): string { - const parsed = readRuntimeDepsJsonObject(path.join(packageRoot, "package.json")); - const version = parsed && typeof parsed.version === "string" ? parsed.version.trim() : ""; - return version || "unknown"; -} - -export function isWritableDirectory(dir: string): boolean { - let probeDir: string | null = null; - try { - probeDir = fs.mkdtempSync(path.join(dir, ".openclaw-write-probe-")); - fs.writeFileSync(path.join(probeDir, "probe"), "", "utf8"); - return true; - } catch { - return false; - } finally { - if (probeDir) { - try { - fs.rmSync(probeDir, { recursive: true, force: true }); - } catch { - // Best-effort cleanup. A failed cleanup should not turn a writable - // probe into a hard runtime-dependency failure. - } - } - } -} - -function resolveSystemdStateDirectory(env: NodeJS.ProcessEnv): string | null { - const raw = env.STATE_DIRECTORY?.trim(); - if (!raw) { - return null; - } - const first = raw.split(path.delimiter).find((entry) => entry.trim().length > 0); - return first ? path.resolve(first) : null; -} - -function resolveBundledRuntimeDepsExternalBaseDirs(env: NodeJS.ProcessEnv): string[] { - const explicit = env.OPENCLAW_PLUGIN_STAGE_DIR?.trim(); - if (explicit) { - const roots = explicit - .split(path.delimiter) - .map((entry) => entry.trim()) - .filter((entry) => entry.length > 0) - .map((entry) => path.resolve(resolveHomeRelativePath(entry, { env, homedir: os.homedir }))); - if (roots.length > 0) { - const uniqueRoots: string[] = []; - for (const root of roots) { - const existingIndex = uniqueRoots.findIndex( - (entry) => path.resolve(entry) === path.resolve(root), - ); - if (existingIndex >= 0) { - uniqueRoots.splice(existingIndex, 1); - } - uniqueRoots.push(root); - } - return uniqueRoots; - } - } - const systemdStateDir = resolveSystemdStateDirectory(env); - if (systemdStateDir) { - return [path.join(systemdStateDir, "plugin-runtime-deps")]; - } - return [path.join(resolveStateDir(env, os.homedir), "plugin-runtime-deps")]; -} - -export function pruneUnknownBundledRuntimeDepsRoots( - params: { - env?: NodeJS.ProcessEnv; - nowMs?: number; - maxRootsToKeep?: number; - minAgeMs?: number; - warn?: (message: string) => void; - } = {}, -): { scanned: number; removed: number; skippedLocked: number } { - const env = params.env ?? process.env; - const nowMs = params.nowMs ?? Date.now(); - const maxRootsToKeep = Math.max( - 0, - params.maxRootsToKeep ?? DEFAULT_UNKNOWN_RUNTIME_DEPS_ROOTS_TO_KEEP, - ); - const minAgeMs = Math.max(0, params.minAgeMs ?? DEFAULT_UNKNOWN_RUNTIME_DEPS_MIN_AGE_MS); - let scanned = 0; - let removed = 0; - let skippedLocked = 0; - - for (const baseDir of resolveBundledRuntimeDepsExternalBaseDirs(env)) { - let entries: fs.Dirent[]; - try { - entries = fs.readdirSync(baseDir, { withFileTypes: true }); - } catch { - continue; - } - const unknownRoots = entries - .filter((entry) => entry.isDirectory() && entry.name.startsWith("openclaw-unknown-")) - .map((entry) => { - const root = path.join(baseDir, entry.name); - try { - return { root, mtimeMs: fs.statSync(root).mtimeMs }; - } catch { - return null; - } - }) - .filter((entry): entry is { root: string; mtimeMs: number } => entry !== null) - .toSorted((left, right) => right.mtimeMs - left.mtimeMs); - scanned += unknownRoots.length; - - for (const [index, entry] of unknownRoots.entries()) { - const ageMs = nowMs - entry.mtimeMs; - if (index < maxRootsToKeep && ageMs < minAgeMs) { - continue; - } - const lockDir = path.join(entry.root, BUNDLED_RUNTIME_DEPS_LOCK_DIR); - if (fs.existsSync(lockDir) && !removeRuntimeDepsLockIfStale(lockDir, nowMs)) { - skippedLocked += 1; - continue; - } - try { - fs.rmSync(entry.root, { recursive: true, force: true }); - removed += 1; - } catch (error) { - params.warn?.( - `failed to remove stale bundled runtime deps root ${entry.root}: ${String(error)}`, - ); - } - } - } - - return { scanned, removed, skippedLocked }; -} - -function resolveExternalBundledRuntimeDepsInstallRoot(params: { - pluginRoot: string; - env: NodeJS.ProcessEnv; -}): string { - return resolveExternalBundledRuntimeDepsInstallRoots(params).at(-1)!; -} - -function resolveExternalBundledRuntimeDepsInstallRoots(params: { - pluginRoot: string; - env: NodeJS.ProcessEnv; -}): string[] { - const packageRoot = resolveBundledPluginPackageRoot(params.pluginRoot) ?? params.pluginRoot; - const existingExternalRoots = resolveExistingExternalBundledRuntimeDepsRoots({ - packageRoot, - env: params.env, - }); - if (existingExternalRoots) { - return existingExternalRoots; - } - const version = sanitizePathSegment(readPackageVersion(packageRoot)); - const packageKey = `openclaw-${version}-${createPathHash(packageRoot)}`; - return resolveBundledRuntimeDepsExternalBaseDirs(params.env).map((baseDir) => - path.join(baseDir, packageKey), - ); -} - -function resolveExistingExternalBundledRuntimeDepsRoots(params: { - packageRoot: string; - env: NodeJS.ProcessEnv; -}): string[] | null { - const packageRoot = realpathOrResolve(params.packageRoot); - const externalBaseDirs = resolveBundledRuntimeDepsExternalBaseDirs(params.env); - for (const externalBaseDir of externalBaseDirs) { - const relative = path.relative(realpathOrResolve(externalBaseDir), packageRoot); - if (relative === "" || relative.startsWith("..") || path.isAbsolute(relative)) { - continue; - } - const packageKey = relative.split(path.sep)[0]; - if (!packageKey || !packageKey.startsWith("openclaw-")) { - continue; - } - return externalBaseDirs.map((baseDir) => path.join(baseDir, packageKey)); - } - return null; -} - -function realpathOrResolve(targetPath: string): string { - try { - return fs.realpathSync.native(targetPath); - } catch { - return path.resolve(targetPath); - } -} - -function createBundledRuntimeDepsInstallRootPlan(params: { - installRoot: string; - searchRoots: readonly string[]; - external: boolean; -}): BundledRuntimeDepsInstallRootPlan { - const searchRoots: string[] = []; - for (const root of params.searchRoots) { - const resolved = path.resolve(root); - if (!searchRoots.some((entry) => path.resolve(entry) === resolved)) { - searchRoots.push(root); - } - } - if (!searchRoots.some((entry) => path.resolve(entry) === path.resolve(params.installRoot))) { - searchRoots.push(params.installRoot); - } - return { - installRoot: params.installRoot, - searchRoots, - external: params.external, - }; -} - export function createBundledRuntimeDepsInstallSpecs(params: { deps: readonly { name: string; version: string }[]; }): string[] { @@ -517,431 +204,6 @@ function createBundledRuntimeDepsPlan(params: { }; } -function replaceNodeModulesDir(targetDir: string, sourceDir: string): void { - const parentDir = path.dirname(targetDir); - const tempDir = fs.mkdtempSync(path.join(parentDir, ".openclaw-runtime-deps-copy-")); - const stagedDir = path.join(tempDir, "node_modules"); - try { - fs.cpSync(sourceDir, stagedDir, { recursive: true }); - fs.rmSync(targetDir, { recursive: true, force: true }); - fs.renameSync(stagedDir, targetDir); - } finally { - try { - fs.rmSync(tempDir, { recursive: true, force: true }); - } catch { - // Stale temp dirs are swept at the next runtime-deps pass. Do not fail - // a node_modules replacement on a transient cleanup race. - } - } -} - -type BundledPluginRuntimeDepsManifest = { - channels: string[]; - enabledByDefault: boolean; - id?: string; - legacyPluginIds: string[]; - providers: string[]; -}; - -type BundledPluginRuntimeDepsManifestCache = Map; - -function readBundledPluginRuntimeDepsManifest( - pluginDir: string, - cache?: BundledPluginRuntimeDepsManifestCache, -): BundledPluginRuntimeDepsManifest { - const cached = cache?.get(pluginDir); - if (cached) { - return cached; - } - const manifest = readRuntimeDepsJsonObject(path.join(pluginDir, "openclaw.plugin.json")); - const channels = manifest?.channels; - const legacyPluginIds = manifest?.legacyPluginIds; - const providers = manifest?.providers; - const runtimeDepsManifest = { - channels: Array.isArray(channels) - ? channels.filter((entry): entry is string => typeof entry === "string" && entry !== "") - : [], - enabledByDefault: manifest?.enabledByDefault === true, - ...(typeof manifest?.id === "string" && manifest.id.trim() ? { id: manifest.id } : {}), - legacyPluginIds: Array.isArray(legacyPluginIds) - ? legacyPluginIds.filter( - (entry): entry is string => typeof entry === "string" && entry !== "", - ) - : [], - providers: Array.isArray(providers) - ? providers.filter((entry): entry is string => typeof entry === "string" && entry !== "") - : [], - }; - cache?.set(pluginDir, runtimeDepsManifest); - return runtimeDepsManifest; -} - -const BUILT_IN_RUNTIME_DEPS_PLUGIN_ALIAS_FALLBACKS: ReadonlyArray< - readonly [alias: string, pluginId: string] -> = [ - ["openai-codex", "openai"], - ["google-gemini-cli", "google"], - ["minimax-portal", "minimax"], - ["minimax-portal-auth", "minimax"], -] as const; - -function addBundledRuntimeDepsPluginAlias( - lookup: Map, - alias: string | undefined, - pluginId: string, -): void { - const normalizedAlias = normalizeOptionalLowercaseString(alias); - if (normalizedAlias) { - lookup.set(normalizedAlias, pluginId); - } -} - -function createBundledRuntimeDepsPluginIdNormalizer(params: { - extensionsDir: string; - manifestCache: BundledPluginRuntimeDepsManifestCache; -}): NormalizePluginId { - const lookup = new Map(); - for (const [alias, pluginId] of BUILT_IN_RUNTIME_DEPS_PLUGIN_ALIAS_FALLBACKS) { - lookup.set(alias, pluginId); - lookup.set(pluginId, pluginId); - } - if (!fs.existsSync(params.extensionsDir)) { - return (id) => { - const trimmed = id.trim(); - const normalized = normalizeOptionalLowercaseString(trimmed); - return (normalized && lookup.get(normalized)) || trimmed; - }; - } - for (const entry of fs.readdirSync(params.extensionsDir, { withFileTypes: true })) { - if (!entry.isDirectory()) { - continue; - } - const fallbackPluginId = entry.name; - const pluginDir = path.join(params.extensionsDir, fallbackPluginId); - const manifest = readBundledPluginRuntimeDepsManifest(pluginDir, params.manifestCache); - const pluginId = manifest.id ?? fallbackPluginId; - addBundledRuntimeDepsPluginAlias(lookup, pluginId, pluginId); - addBundledRuntimeDepsPluginAlias(lookup, fallbackPluginId, pluginId); - for (const providerId of manifest.providers) { - addBundledRuntimeDepsPluginAlias(lookup, providerId, pluginId); - } - for (const legacyPluginId of manifest.legacyPluginIds) { - addBundledRuntimeDepsPluginAlias(lookup, legacyPluginId, pluginId); - } - } - return (id) => { - const trimmed = id.trim(); - const normalized = normalizeOptionalLowercaseString(trimmed); - return (normalized && lookup.get(normalized)) || trimmed; - }; -} - -function passesRuntimeDepsPluginPolicy(params: { - pluginId: string; - plugins: NormalizedPluginsConfig; - allowExplicitlyDisabled?: boolean; - allowRestrictiveAllowlistBypass?: boolean; -}): boolean { - if (!params.plugins.enabled) { - return false; - } - if (params.plugins.deny.includes(params.pluginId)) { - return false; - } - if ( - params.plugins.entries[params.pluginId]?.enabled === false && - params.allowExplicitlyDisabled !== true - ) { - return false; - } - return ( - params.allowRestrictiveAllowlistBypass === true || - params.plugins.allow.length === 0 || - params.plugins.allow.includes(params.pluginId) - ); -} - -function isBundledPluginConfiguredForRuntimeDeps(params: { - config: OpenClawConfig; - plugins: NormalizedPluginsConfig; - pluginId: string; - pluginDir: string; - includeConfiguredChannels?: boolean; - manifestCache?: BundledPluginRuntimeDepsManifestCache; -}): boolean { - if ( - !passesRuntimeDepsPluginPolicy({ - pluginId: params.pluginId, - plugins: params.plugins, - allowRestrictiveAllowlistBypass: true, - }) - ) { - return false; - } - const entry = params.plugins.entries[params.pluginId]; - const manifest = readBundledPluginRuntimeDepsManifest(params.pluginDir, params.manifestCache); - if ( - params.plugins.slots.memory === params.pluginId || - params.plugins.slots.contextEngine === params.pluginId - ) { - return true; - } - let hasExplicitChannelDisable = false; - let hasConfiguredChannel = false; - for (const channelId of manifest.channels) { - const normalizedChannelId = normalizeOptionalLowercaseString(channelId); - if (!normalizedChannelId) { - continue; - } - const channelConfig = (params.config.channels as Record | undefined)?.[ - normalizedChannelId - ]; - if ( - channelConfig && - typeof channelConfig === "object" && - !Array.isArray(channelConfig) && - (channelConfig as { enabled?: unknown }).enabled === false - ) { - hasExplicitChannelDisable = true; - continue; - } - if ( - channelConfig && - typeof channelConfig === "object" && - !Array.isArray(channelConfig) && - (channelConfig as { enabled?: unknown }).enabled === true - ) { - return true; - } - if ( - channelConfig && - typeof channelConfig === "object" && - !Array.isArray(channelConfig) && - params.includeConfiguredChannels - ) { - hasConfiguredChannel = true; - } - } - if (hasExplicitChannelDisable) { - return false; - } - if (params.plugins.allow.length > 0 && !params.plugins.allow.includes(params.pluginId)) { - return false; - } - if (entry?.enabled === true) { - return true; - } - if (hasConfiguredChannel) { - return true; - } - return manifest.enabledByDefault; -} - -function isBundledPluginExplicitlyDisabledForRuntimeDeps(params: { - config: OpenClawConfig; - plugins: NormalizedPluginsConfig; - pluginId: string; - pluginDir: string; - manifestCache?: BundledPluginRuntimeDepsManifestCache; -}): boolean { - if (params.plugins.entries[params.pluginId]?.enabled === false) { - return true; - } - const manifest = readBundledPluginRuntimeDepsManifest(params.pluginDir, params.manifestCache); - return manifest.channels.some((channelId) => { - const normalizedChannelId = normalizeOptionalLowercaseString(channelId); - if (!normalizedChannelId) { - return false; - } - const channelConfig = (params.config.channels as Record | undefined)?.[ - normalizedChannelId - ]; - return ( - channelConfig && - typeof channelConfig === "object" && - !Array.isArray(channelConfig) && - (channelConfig as { enabled?: unknown }).enabled === false - ); - }); -} - -function shouldIncludeBundledPluginRuntimeDeps(params: { - config?: OpenClawConfig; - plugins?: NormalizedPluginsConfig; - pluginIds?: ReadonlySet; - selectedPluginIds?: ReadonlySet; - pluginId: string; - pluginDir: string; - includeConfiguredChannels?: boolean; - manifestCache?: BundledPluginRuntimeDepsManifestCache; -}): boolean { - if (params.selectedPluginIds) { - return ( - params.selectedPluginIds.has(params.pluginId) && - !( - params.config && - params.plugins && - isBundledPluginExplicitlyDisabledForRuntimeDeps({ - config: params.config, - plugins: params.plugins, - pluginId: params.pluginId, - pluginDir: params.pluginDir, - manifestCache: params.manifestCache, - }) - ) - ); - } - const scopedToPluginIds = Boolean(params.pluginIds); - if (params.pluginIds) { - if (!params.pluginIds.has(params.pluginId)) { - return false; - } - if (!params.config) { - return true; - } - } - if (!params.config) { - return true; - } - if (scopedToPluginIds) { - if (!params.plugins) { - return true; - } - return passesRuntimeDepsPluginPolicy({ - pluginId: params.pluginId, - plugins: params.plugins, - allowRestrictiveAllowlistBypass: true, - }); - } - if (!params.plugins) { - return false; - } - return isBundledPluginConfiguredForRuntimeDeps({ - config: params.config, - plugins: params.plugins, - pluginId: params.pluginId, - pluginDir: params.pluginDir, - includeConfiguredChannels: params.includeConfiguredChannels, - manifestCache: params.manifestCache, - }); -} - -function collectBundledPluginRuntimeDeps(params: { - extensionsDir: string; - config?: OpenClawConfig; - pluginIds?: ReadonlySet; - selectedPluginIds?: ReadonlySet; - includeConfiguredChannels?: boolean; - manifestCache?: BundledPluginRuntimeDepsManifestCache; - normalizePluginId?: NormalizePluginId; -}): { - deps: RuntimeDepEntry[]; - conflicts: RuntimeDepConflict[]; - pluginIds: string[]; -} { - const versionMap = new Map>>(); - const manifestCache: BundledPluginRuntimeDepsManifestCache = params.manifestCache ?? new Map(); - const needsPluginIdNormalizer = Boolean(params.config); - const normalizePluginId = - params.normalizePluginId ?? - (needsPluginIdNormalizer - ? createBundledRuntimeDepsPluginIdNormalizer({ - extensionsDir: params.extensionsDir, - manifestCache, - }) - : undefined); - const plugins = params.config - ? normalizePluginsConfigWithResolver(params.config.plugins, normalizePluginId) - : undefined; - const includedPluginIds = new Set(); - - for (const entry of fs.readdirSync(params.extensionsDir, { withFileTypes: true })) { - if (!entry.isDirectory()) { - continue; - } - const pluginId = entry.name; - const pluginDir = path.join(params.extensionsDir, pluginId); - if ( - !shouldIncludeBundledPluginRuntimeDeps({ - config: params.config, - plugins, - pluginIds: params.pluginIds, - selectedPluginIds: params.selectedPluginIds, - pluginId, - pluginDir, - includeConfiguredChannels: params.includeConfiguredChannels, - manifestCache, - }) - ) { - continue; - } - includedPluginIds.add(pluginId); - const packageJson = readRuntimeDepsJsonObject(path.join(pluginDir, "package.json")); - if (!packageJson) { - continue; - } - for (const [name, rawVersion] of Object.entries(collectPackageRuntimeDeps(packageJson))) { - const dep = parseInstallableRuntimeDep(name, rawVersion); - if (!dep) { - continue; - } - const byVersion = versionMap.get(dep.name) ?? new Map>(); - const pluginIds = byVersion.get(dep.version) ?? new Set(); - pluginIds.add(pluginId); - byVersion.set(dep.version, pluginIds); - versionMap.set(dep.name, byVersion); - } - } - - const deps: RuntimeDepEntry[] = []; - const conflicts: RuntimeDepConflict[] = []; - for (const [name, byVersion] of versionMap.entries()) { - if (byVersion.size === 1) { - const [version, pluginIds] = [...byVersion.entries()][0] ?? []; - if (version) { - deps.push({ - name, - version, - pluginIds: [...pluginIds].toSorted((a, b) => a.localeCompare(b)), - }); - } - continue; - } - const versions = [...byVersion.keys()].toSorted((a, b) => a.localeCompare(b)); - const pluginIdsByVersion = new Map(); - for (const [version, pluginIds] of byVersion.entries()) { - pluginIdsByVersion.set( - version, - [...pluginIds].toSorted((a, b) => a.localeCompare(b)), - ); - } - conflicts.push({ - name, - versions, - pluginIdsByVersion, - }); - } - - return { - deps: deps.toSorted((a, b) => a.name.localeCompare(b.name)), - conflicts: conflicts.toSorted((a, b) => a.name.localeCompare(b.name)), - pluginIds: [...includedPluginIds].toSorted((a, b) => a.localeCompare(b)), - }; -} - -function normalizePluginIdSet( - pluginIds: readonly string[] | undefined, - normalizePluginId: NormalizePluginId = (id) => normalizeOptionalLowercaseString(id) ?? "", -): ReadonlySet | undefined { - if (!pluginIds) { - return undefined; - } - const normalized = pluginIds - .map((entry) => normalizePluginId(entry)) - .filter((entry): entry is string => Boolean(entry)); - return new Set(normalized); -} - export function scanBundledPluginRuntimeDeps(params: { packageRoot: string; config?: OpenClawConfig; @@ -994,121 +256,6 @@ export function scanBundledPluginRuntimeDeps(params: { return { deps: plan.deps, missing: plan.missing, conflicts: plan.conflicts }; } -export function resolveBundledRuntimeDependencyPackageInstallRootPlan( - packageRoot: string, - options: { env?: NodeJS.ProcessEnv; forceExternal?: boolean } = {}, -): BundledRuntimeDepsInstallRootPlan { - const env = options.env ?? process.env; - const externalRoots = resolveExternalBundledRuntimeDepsInstallRoots({ - pluginRoot: path.join(packageRoot, "dist", "extensions", "__package__"), - env, - }); - if ( - options.forceExternal || - env.OPENCLAW_PLUGIN_STAGE_DIR?.trim() || - env.STATE_DIRECTORY?.trim() || - !isSourceCheckoutRoot(packageRoot) - ) { - return createBundledRuntimeDepsInstallRootPlan({ - installRoot: - externalRoots.at(-1) ?? - resolveExternalBundledRuntimeDepsInstallRoot({ - pluginRoot: path.join(packageRoot, "dist", "extensions", "__package__"), - env, - }), - searchRoots: externalRoots, - external: true, - }); - } - if (isWritableDirectory(packageRoot)) { - return createBundledRuntimeDepsInstallRootPlan({ - installRoot: packageRoot, - searchRoots: [packageRoot], - external: false, - }); - } - return createBundledRuntimeDepsInstallRootPlan({ - installRoot: - externalRoots.at(-1) ?? - resolveExternalBundledRuntimeDepsInstallRoot({ - pluginRoot: path.join(packageRoot, "dist", "extensions", "__package__"), - env, - }), - searchRoots: externalRoots, - external: true, - }); -} - -export function resolveBundledRuntimeDependencyPackageInstallRoot( - packageRoot: string, - options: { env?: NodeJS.ProcessEnv; forceExternal?: boolean } = {}, -): string { - return resolveBundledRuntimeDependencyPackageInstallRootPlan(packageRoot, options).installRoot; -} - -export function resolveBundledRuntimeDependencyInstallRootPlan( - pluginRoot: string, - options: { env?: NodeJS.ProcessEnv; forceExternal?: boolean } = {}, -): BundledRuntimeDepsInstallRootPlan { - const env = options.env ?? process.env; - const externalRoots = resolveExternalBundledRuntimeDepsInstallRoots({ pluginRoot, env }); - if ( - options.forceExternal || - env.OPENCLAW_PLUGIN_STAGE_DIR?.trim() || - env.STATE_DIRECTORY?.trim() || - isPackagedBundledPluginRoot(pluginRoot) - ) { - return createBundledRuntimeDepsInstallRootPlan({ - installRoot: - externalRoots.at(-1) ?? - resolveExternalBundledRuntimeDepsInstallRoot({ - pluginRoot, - env, - }), - searchRoots: externalRoots, - external: true, - }); - } - if (isWritableDirectory(pluginRoot)) { - return createBundledRuntimeDepsInstallRootPlan({ - installRoot: pluginRoot, - searchRoots: [pluginRoot], - external: false, - }); - } - return createBundledRuntimeDepsInstallRootPlan({ - installRoot: - externalRoots.at(-1) ?? - resolveExternalBundledRuntimeDepsInstallRoot({ - pluginRoot, - env, - }), - searchRoots: externalRoots, - external: true, - }); -} - -export function resolveBundledRuntimeDependencyInstallRoot( - pluginRoot: string, - options: { env?: NodeJS.ProcessEnv; forceExternal?: boolean } = {}, -): string { - return resolveBundledRuntimeDependencyInstallRootPlan(pluginRoot, options).installRoot; -} - -export function resolveBundledRuntimeDependencyInstallRootInfo( - pluginRoot: string, - options: { env?: NodeJS.ProcessEnv; forceExternal?: boolean } = {}, -): BundledRuntimeDepsInstallRoot { - const { installRoot, external } = resolveBundledRuntimeDependencyInstallRootPlan( - pluginRoot, - options, - ); - return { - installRoot, - external, - }; -} - export function createBundledRuntimeDependencyAliasMap(params: { pluginRoot: string; installRoot: string; @@ -1136,382 +283,6 @@ export function createBundledRuntimeDependencyAliasMap(params: { return aliases; } -function shouldCleanBundledRuntimeDepsInstallExecutionRoot(params: { - installRoot: string; - installExecutionRoot: string; -}): boolean { - const installRoot = path.resolve(params.installRoot); - const installExecutionRoot = path.resolve(params.installExecutionRoot); - return installExecutionRoot.startsWith(`${installRoot}${path.sep}`); -} - -function formatBundledRuntimeDepsInstallError(result: { - error?: Error; - signal?: NodeJS.Signals | null; - status?: number | null; - stderr?: string | Buffer | null; - stdout?: string | Buffer | null; -}): string { - const output = [ - result.error?.message, - result.signal ? `terminated by ${result.signal}` : null, - result.stderr, - result.stdout, - ] - .filter(Boolean) - .join("\n") - .trim(); - return output || "npm install failed"; -} - -function formatBundledRuntimeDepsInstallElapsed(ms: number): string { - const seconds = Math.max(0, Math.round(ms / 1000)); - if (seconds < 60) { - return `${seconds}s`; - } - const minutes = Math.floor(seconds / 60); - const remainingSeconds = seconds % 60; - return remainingSeconds > 0 ? `${minutes}m ${remainingSeconds}s` : `${minutes}m`; -} - -function emitBundledRuntimeDepsOutputProgress( - chunk: Buffer, - stream: "stdout" | "stderr", - packageManager: BundledRuntimeDepsPackageManager, - onProgress: ((message: string) => void) | undefined, -): void { - if (!onProgress) { - return; - } - const lines = chunk - .toString("utf8") - .split(/\r\n|\n|\r/u) - .map((line) => sanitizeTerminalText(line).trim()) - .filter((line) => line.length > 0) - .slice(-3); - for (const line of lines) { - onProgress(`${packageManager} ${stream}: ${line}`); - } -} - -type BundledRuntimeDepsInstallContext = { - installExecutionRoot: string; - installSpecs: string[]; - installEnv: NodeJS.ProcessEnv; - runner: BundledRuntimeDepsPackageManagerRunner; - isolatedExecutionRoot: boolean; - cleanInstallExecutionRoot: boolean; -}; - -function createBundledRuntimeDepsInstallContext(params: { - installRoot: string; - installExecutionRoot?: string; - installSpecs: readonly string[]; - env: NodeJS.ProcessEnv; - warn?: (message: string) => void; -}): BundledRuntimeDepsInstallContext { - const installExecutionRoot = params.installExecutionRoot ?? params.installRoot; - const isolatedExecutionRoot = - path.resolve(installExecutionRoot) !== path.resolve(params.installRoot); - const cleanInstallExecutionRoot = - isolatedExecutionRoot && - shouldCleanBundledRuntimeDepsInstallExecutionRoot({ - installRoot: params.installRoot, - installExecutionRoot, - }); - - fs.mkdirSync(params.installRoot, { recursive: true }); - fs.mkdirSync(installExecutionRoot, { recursive: true }); - const diskWarning = createLowDiskSpaceWarning({ - targetPath: installExecutionRoot, - purpose: "bundled plugin runtime dependency staging", - }); - if (diskWarning) { - params.warn?.(diskWarning); - } - ensureNpmInstallExecutionManifest(installExecutionRoot, params.installSpecs); - const installEnv = createBundledRuntimeDepsInstallEnv(params.env, { - cacheDir: path.join(installExecutionRoot, ".openclaw-npm-cache"), - }); - const runner = resolveBundledRuntimeDepsPackageManagerRunner({ - installExecutionRoot, - env: installEnv, - npmArgs: createBundledRuntimeDepsInstallArgs(), - }); - - return { - installExecutionRoot, - installSpecs: normalizeRuntimeDepSpecs(params.installSpecs), - installEnv, - runner, - isolatedExecutionRoot, - cleanInstallExecutionRoot, - }; -} - -function finalizeBundledRuntimeDepsInstall(params: { - installRoot: string; - context: BundledRuntimeDepsInstallContext; -}): void { - const { context } = params; - assertBundledRuntimeDepsInstalled(context.installExecutionRoot, context.installSpecs); - if (context.isolatedExecutionRoot) { - const stagedNodeModulesDir = path.join(context.installExecutionRoot, "node_modules"); - if (!fs.existsSync(stagedNodeModulesDir)) { - throw new Error(`${context.runner.packageManager} install did not produce node_modules`); - } - const targetNodeModulesDir = path.join(params.installRoot, "node_modules"); - replaceNodeModulesDir(targetNodeModulesDir, stagedNodeModulesDir); - assertBundledRuntimeDepsInstalled(params.installRoot, context.installSpecs); - } - removeLegacyRuntimeDepsManifest(params.installRoot); -} - -function cleanupBundledRuntimeDepsInstallContext(context: BundledRuntimeDepsInstallContext): void { - if (context.cleanInstallExecutionRoot) { - fs.rmSync(context.installExecutionRoot, { recursive: true, force: true }); - } -} - -async function spawnBundledRuntimeDepsInstall(params: { - command: string; - args: string[]; - cwd: string; - env: NodeJS.ProcessEnv; - packageManager: BundledRuntimeDepsPackageManager; - onProgress?: (message: string) => void; -}): Promise { - await new Promise((resolve, reject) => { - const startedAtMs = Date.now(); - const heartbeat = - params.onProgress && - setInterval(() => { - params.onProgress?.( - `${params.packageManager} install still running (${formatBundledRuntimeDepsInstallElapsed(Date.now() - startedAtMs)} elapsed)`, - ); - }, BUNDLED_RUNTIME_DEPS_INSTALL_PROGRESS_INTERVAL_MS); - heartbeat?.unref?.(); - const settle = (fn: () => void) => { - if (heartbeat) { - clearInterval(heartbeat); - } - fn(); - }; - const child = spawn(params.command, params.args, { - cwd: params.cwd, - env: params.env, - stdio: ["ignore", "pipe", "pipe"], - windowsHide: true, - }); - const stdout: Buffer[] = []; - const stderr: Buffer[] = []; - child.stdout?.on("data", (chunk: Buffer) => { - stdout.push(chunk); - emitBundledRuntimeDepsOutputProgress( - chunk, - "stdout", - params.packageManager, - params.onProgress, - ); - }); - child.stderr?.on("data", (chunk: Buffer) => { - stderr.push(chunk); - emitBundledRuntimeDepsOutputProgress( - chunk, - "stderr", - params.packageManager, - params.onProgress, - ); - }); - child.on("error", (error) => { - settle(() => reject(new Error(formatBundledRuntimeDepsInstallError({ error })))); - }); - child.on("close", (status, signal) => { - if (status === 0 && !signal) { - settle(resolve); - return; - } - settle(() => - reject( - new Error( - formatBundledRuntimeDepsInstallError({ - status, - signal, - stdout: Buffer.concat(stdout).toString("utf8"), - stderr: Buffer.concat(stderr).toString("utf8"), - }), - ), - ), - ); - }); - }); -} - -export function installBundledRuntimeDeps(params: { - installRoot: string; - installExecutionRoot?: string; - missingSpecs: string[]; - installSpecs?: string[]; - env: NodeJS.ProcessEnv; - warn?: (message: string) => void; -}): void { - const installSpecs = normalizeRuntimeDepSpecs(params.installSpecs ?? params.missingSpecs); - if (installSpecs.length === 0) { - return; - } - if (isRuntimeDepsPlanMaterialized(params.installRoot, installSpecs)) { - removeLegacyRuntimeDepsManifest(params.installRoot); - return; - } - const context = createBundledRuntimeDepsInstallContext({ - installRoot: params.installRoot, - installExecutionRoot: params.installExecutionRoot, - installSpecs, - env: params.env, - warn: params.warn, - }); - try { - const result = spawnSync(context.runner.command, context.runner.args, { - cwd: context.installExecutionRoot, - encoding: "utf8", - env: context.runner.env ?? context.installEnv, - stdio: "pipe", - windowsHide: true, - }); - if (result.status !== 0 || result.error) { - throw new Error(formatBundledRuntimeDepsInstallError(result)); - } - finalizeBundledRuntimeDepsInstall({ installRoot: params.installRoot, context }); - } finally { - cleanupBundledRuntimeDepsInstallContext(context); - } -} - -export async function installBundledRuntimeDepsAsync(params: { - installRoot: string; - installExecutionRoot?: string; - missingSpecs: string[]; - installSpecs?: string[]; - env: NodeJS.ProcessEnv; - warn?: (message: string) => void; - onProgress?: (message: string) => void; -}): Promise { - const installSpecs = normalizeRuntimeDepSpecs(params.installSpecs ?? params.missingSpecs); - if (installSpecs.length === 0) { - return; - } - if (isRuntimeDepsPlanMaterialized(params.installRoot, installSpecs)) { - removeLegacyRuntimeDepsManifest(params.installRoot); - return; - } - const context = createBundledRuntimeDepsInstallContext({ - installRoot: params.installRoot, - installExecutionRoot: params.installExecutionRoot, - installSpecs, - env: params.env, - warn: params.warn, - }); - try { - params.onProgress?.( - `Starting ${context.runner.packageManager} install for bundled plugin runtime deps: ${installSpecs.join(", ")}`, - ); - await spawnBundledRuntimeDepsInstall({ - command: context.runner.command, - args: context.runner.args, - cwd: context.installExecutionRoot, - env: context.runner.env ?? context.installEnv, - packageManager: context.runner.packageManager, - onProgress: params.onProgress, - }); - finalizeBundledRuntimeDepsInstall({ installRoot: params.installRoot, context }); - } finally { - cleanupBundledRuntimeDepsInstallContext(context); - } -} - -export function repairBundledRuntimeDepsInstallRoot(params: { - installRoot: string; - missingSpecs: string[]; - installSpecs: string[]; - env: NodeJS.ProcessEnv; - installDeps?: (params: BundledRuntimeDepsInstallParams) => void; - warn?: (message: string) => void; -}): { installSpecs: string[] } { - return withBundledRuntimeDepsInstallRootLock(params.installRoot, () => { - const installSpecs = normalizeRuntimeDepSpecs(params.installSpecs); - const install = - params.installDeps ?? - ((installParams) => - installBundledRuntimeDeps({ - installRoot: installParams.installRoot, - missingSpecs: installParams.missingSpecs, - installSpecs: installParams.installSpecs, - env: params.env, - warn: params.warn, - })); - const finishActivity = beginBundledRuntimeDepsInstall({ - installRoot: params.installRoot, - missingSpecs: installSpecs, - installSpecs, - }); - ensureNpmInstallExecutionManifest(params.installRoot, installSpecs); - try { - install({ - installRoot: params.installRoot, - missingSpecs: installSpecs, - installSpecs, - }); - } finally { - finishActivity(); - } - removeLegacyRuntimeDepsManifest(params.installRoot); - return { installSpecs }; - }); -} - -export async function repairBundledRuntimeDepsInstallRootAsync(params: { - installRoot: string; - missingSpecs: string[]; - installSpecs: string[]; - env: NodeJS.ProcessEnv; - installDeps?: (params: BundledRuntimeDepsInstallParams) => Promise; - warn?: (message: string) => void; - onProgress?: (message: string) => void; -}): Promise<{ installSpecs: string[] }> { - return await withBundledRuntimeDepsInstallRootLockAsync(params.installRoot, async () => { - const installSpecs = normalizeRuntimeDepSpecs(params.installSpecs); - const install = - params.installDeps ?? - ((installParams) => - installBundledRuntimeDepsAsync({ - installRoot: installParams.installRoot, - missingSpecs: installParams.missingSpecs, - installSpecs: installParams.installSpecs, - env: params.env, - warn: params.warn, - onProgress: params.onProgress, - })); - const finishActivity = beginBundledRuntimeDepsInstall({ - installRoot: params.installRoot, - missingSpecs: installSpecs, - installSpecs, - }); - removeLegacyRuntimeDepsManifest(params.installRoot); - ensureNpmInstallExecutionManifest(params.installRoot, installSpecs); - try { - await install({ - installRoot: params.installRoot, - missingSpecs: installSpecs, - installSpecs, - }); - } finally { - finishActivity(); - } - removeLegacyRuntimeDepsManifest(params.installRoot); - return { installSpecs }; - }); -} - export function ensureBundledPluginRuntimeDeps(params: { pluginId: string; pluginRoot: string;