mirror of
https://github.com/openclaw/openclaw.git
synced 2026-05-27 17:55:58 +00:00
564 lines
16 KiB
JavaScript
564 lines
16 KiB
JavaScript
#!/usr/bin/env node
|
|
|
|
import { spawn, spawnSync } from "node:child_process";
|
|
import fs from "node:fs";
|
|
import path from "node:path";
|
|
import { pathToFileURL } from "node:url";
|
|
import { BUNDLED_PLUGIN_PATH_PREFIX } from "./lib/bundled-plugin-paths.mjs";
|
|
import { resolvePnpmRunner } from "./pnpm-runner.mjs";
|
|
import {
|
|
isSourceCheckoutRoot,
|
|
pruneBundledPluginSourceNodeModules,
|
|
} from "./postinstall-bundled-plugins.mjs";
|
|
|
|
const logLevel = process.env.OPENCLAW_BUILD_VERBOSE ? "info" : "warn";
|
|
const extraArgs = process.argv.slice(2);
|
|
const INEFFECTIVE_DYNAMIC_IMPORT_MARKER = "[INEFFECTIVE_DYNAMIC_IMPORT]";
|
|
const UNRESOLVED_IMPORT_RE = /\[UNRESOLVED_IMPORT\]/;
|
|
const ANSI_ESCAPE_RE = new RegExp(String.raw`\u001B\[[0-9;]*m`, "g");
|
|
const DEPENDENCY_PATH_MARKERS = ["node_modules/", "openclaw-pnpm-node-modules/"];
|
|
const HASHED_ROOT_JS_RE = /^(?<base>.+)-[A-Za-z0-9_-]+\.js$/u;
|
|
const DEFAULT_CAPTURE_BYTES = 8 * 1024 * 1024;
|
|
const DEFAULT_HEARTBEAT_MS = 30_000;
|
|
const DEFAULT_TSDOWN_MAX_OLD_SPACE_MB = 8192;
|
|
const MIN_TSDOWN_MAX_OLD_SPACE_MB = 2048;
|
|
const TSDOWN_CGROUP_MEMORY_HEADROOM_MB = 768;
|
|
const CGROUP_MEMORY_LIMIT_PATHS = [
|
|
"/sys/fs/cgroup/memory.max",
|
|
"/sys/fs/cgroup/memory/memory.limit_in_bytes",
|
|
];
|
|
const PROC_MEMINFO_PATH = "/proc/meminfo";
|
|
const TERMINATION_GRACE_MS = 5_000;
|
|
const TSDOWN_OUTPUT_ROOTS = ["dist", "dist-runtime"];
|
|
const GENERATED_SOURCE_DECLARATION_PATHSPEC = ":(glob)extensions/**/*.d.ts";
|
|
const SOURCE_DECLARATION_SOURCE_EXTENSIONS = [".ts", ".tsx", ".mts", ".cts", ".js", ".mjs", ".cjs"];
|
|
|
|
function removeDistPluginNodeModulesSymlinks(rootDir) {
|
|
const extensionsDir = path.join(rootDir, "extensions");
|
|
if (!fs.existsSync(extensionsDir)) {
|
|
return;
|
|
}
|
|
|
|
for (const dirent of fs.readdirSync(extensionsDir, { withFileTypes: true })) {
|
|
if (!dirent.isDirectory()) {
|
|
continue;
|
|
}
|
|
const nodeModulesPath = path.join(extensionsDir, dirent.name, "node_modules");
|
|
try {
|
|
if (fs.lstatSync(nodeModulesPath).isSymbolicLink()) {
|
|
fs.rmSync(nodeModulesPath, { force: true, recursive: true });
|
|
}
|
|
} catch {
|
|
// Skip missing or unreadable paths so the build can proceed.
|
|
}
|
|
}
|
|
}
|
|
|
|
function pruneStaleRuntimeSymlinks() {
|
|
const cwd = process.cwd();
|
|
// runtime-postbuild stages plugin-owned node_modules into dist/ and links the
|
|
// dist-runtime overlay back to that tree. Remove only those symlinks up front
|
|
// so tsdown's clean step cannot traverse stale runtime overlays on rebuilds.
|
|
removeDistPluginNodeModulesSymlinks(path.join(cwd, "dist"));
|
|
removeDistPluginNodeModulesSymlinks(path.join(cwd, "dist-runtime"));
|
|
}
|
|
|
|
export function cleanTsdownOutputRoots(params = {}) {
|
|
const cwd = params.cwd ?? process.cwd();
|
|
const fsImpl = params.fs ?? fs;
|
|
for (const root of TSDOWN_OUTPUT_ROOTS) {
|
|
const rootPath = path.join(cwd, root);
|
|
try {
|
|
fsImpl.rmSync(rootPath, { force: true, recursive: true });
|
|
} catch {
|
|
// Best-effort cleanup. tsdown will recreate the output tree it needs.
|
|
}
|
|
}
|
|
}
|
|
|
|
export function pruneStaleRootChunkFiles(params = {}) {
|
|
const cwd = params.cwd ?? process.cwd();
|
|
const fsImpl = params.fs ?? fs;
|
|
const roots = TSDOWN_OUTPUT_ROOTS.map((root) => path.join(cwd, root));
|
|
for (const root of roots) {
|
|
let entries = [];
|
|
try {
|
|
entries = fsImpl.readdirSync(root, { withFileTypes: true });
|
|
} catch {
|
|
continue;
|
|
}
|
|
|
|
for (const entry of entries) {
|
|
if (!entry.isFile()) {
|
|
continue;
|
|
}
|
|
if (!HASHED_ROOT_JS_RE.test(entry.name)) {
|
|
continue;
|
|
}
|
|
try {
|
|
fsImpl.rmSync(path.join(root, entry.name), { force: true });
|
|
} catch {
|
|
// Best-effort cleanup. The subsequent build will overwrite any stragglers.
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
export function pruneUntrackedGeneratedSourceDeclarations(params = {}) {
|
|
const cwd = params.cwd ?? process.cwd();
|
|
const fsImpl = params.fs ?? fs;
|
|
const spawnSyncImpl = params.spawnSync ?? spawnSync;
|
|
let result;
|
|
try {
|
|
result = spawnSyncImpl(
|
|
"git",
|
|
["ls-files", "--others", "--exclude-standard", "--", GENERATED_SOURCE_DECLARATION_PATHSPEC],
|
|
{
|
|
cwd,
|
|
encoding: "utf8",
|
|
stdio: ["ignore", "pipe", "ignore"],
|
|
},
|
|
);
|
|
} catch {
|
|
return 0;
|
|
}
|
|
if (result.status !== 0 || typeof result.stdout !== "string") {
|
|
return 0;
|
|
}
|
|
|
|
let removed = 0;
|
|
for (const rawPath of result.stdout.split(/\r?\n/u)) {
|
|
const relativePath = rawPath.trim().replaceAll("\\", "/");
|
|
if (!relativePath.startsWith("extensions/") || !relativePath.endsWith(".d.ts")) {
|
|
continue;
|
|
}
|
|
const declarationPath = path.join(cwd, relativePath);
|
|
const sourceBase = declarationPath.slice(0, -".d.ts".length);
|
|
const hasMatchingSource = SOURCE_DECLARATION_SOURCE_EXTENSIONS.some((extension) =>
|
|
fsImpl.existsSync(`${sourceBase}${extension}`),
|
|
);
|
|
if (!hasMatchingSource) {
|
|
continue;
|
|
}
|
|
try {
|
|
fsImpl.rmSync(declarationPath, { force: true });
|
|
removed += 1;
|
|
} catch {
|
|
// Best-effort cleanup; tsdown will still report any remaining stale files.
|
|
}
|
|
}
|
|
return removed;
|
|
}
|
|
|
|
export function pruneSourceCheckoutBundledPluginNodeModules(params = {}) {
|
|
const cwd = params.cwd ?? process.cwd();
|
|
const logger = params.logger ?? console;
|
|
if (!isSourceCheckoutRoot({ packageRoot: cwd, existsSync: fs.existsSync })) {
|
|
return;
|
|
}
|
|
try {
|
|
pruneBundledPluginSourceNodeModules({
|
|
extensionsDir: path.join(cwd, "extensions"),
|
|
existsSync: fs.existsSync,
|
|
readdirSync: fs.readdirSync,
|
|
rmSync: fs.rmSync,
|
|
});
|
|
} catch (error) {
|
|
logger.warn(`tsdown: could not prune bundled plugin source node_modules: ${String(error)}`);
|
|
}
|
|
}
|
|
|
|
function findFatalUnresolvedImport(lines) {
|
|
for (const line of lines) {
|
|
if (!UNRESOLVED_IMPORT_RE.test(line)) {
|
|
continue;
|
|
}
|
|
|
|
const normalizedLine = line.replace(ANSI_ESCAPE_RE, "");
|
|
if (
|
|
!normalizedLine.includes(BUNDLED_PLUGIN_PATH_PREFIX) &&
|
|
!DEPENDENCY_PATH_MARKERS.some((marker) => normalizedLine.includes(marker))
|
|
) {
|
|
return normalizedLine;
|
|
}
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
function parsePositiveInteger(value) {
|
|
if (typeof value !== "string" || value.trim() === "") {
|
|
return null;
|
|
}
|
|
const parsed = Number(value);
|
|
if (!Number.isFinite(parsed) || parsed <= 0) {
|
|
return null;
|
|
}
|
|
return Math.trunc(parsed);
|
|
}
|
|
|
|
function parseNonNegativeInteger(value) {
|
|
if (typeof value !== "string" || value.trim() === "") {
|
|
return null;
|
|
}
|
|
const parsed = Number(value);
|
|
if (!Number.isFinite(parsed) || parsed < 0) {
|
|
return null;
|
|
}
|
|
return Math.trunc(parsed);
|
|
}
|
|
|
|
function parseCgroupMemoryLimitBytes(value) {
|
|
if (typeof value !== "string") {
|
|
return null;
|
|
}
|
|
const trimmed = value.trim();
|
|
if (trimmed === "" || trimmed === "max" || !/^\d+$/u.test(trimmed)) {
|
|
return null;
|
|
}
|
|
const parsed = BigInt(trimmed);
|
|
if (parsed <= 0n || parsed > BigInt(Number.MAX_SAFE_INTEGER)) {
|
|
return null;
|
|
}
|
|
return Number(parsed);
|
|
}
|
|
|
|
function readCgroupMemoryLimitBytes(params = {}) {
|
|
if (Number.isFinite(params.cgroupMemoryLimitBytes) && params.cgroupMemoryLimitBytes > 0) {
|
|
return Math.trunc(params.cgroupMemoryLimitBytes);
|
|
}
|
|
|
|
const fsImpl = params.fs ?? fs;
|
|
const paths = params.cgroupMemoryLimitPaths ?? CGROUP_MEMORY_LIMIT_PATHS;
|
|
for (const limitPath of paths) {
|
|
try {
|
|
const limitBytes = parseCgroupMemoryLimitBytes(fsImpl.readFileSync(limitPath, "utf8"));
|
|
if (limitBytes !== null) {
|
|
return limitBytes;
|
|
}
|
|
} catch {
|
|
// Missing cgroup files are expected outside Linux containers.
|
|
}
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
function parseProcMemTotalBytes(value) {
|
|
if (typeof value !== "string") {
|
|
return null;
|
|
}
|
|
const match = value.match(/^MemTotal:\s+(\d+)\s+kB$/imu);
|
|
if (!match) {
|
|
return null;
|
|
}
|
|
const parsed = BigInt(match[1]) * 1024n;
|
|
if (parsed <= 0n || parsed > BigInt(Number.MAX_SAFE_INTEGER)) {
|
|
return null;
|
|
}
|
|
return Number(parsed);
|
|
}
|
|
|
|
function readProcMemTotalBytes(params = {}) {
|
|
if (Number.isFinite(params.procMemTotalBytes) && params.procMemTotalBytes > 0) {
|
|
return Math.trunc(params.procMemTotalBytes);
|
|
}
|
|
|
|
const fsImpl = params.fs ?? fs;
|
|
try {
|
|
return parseProcMemTotalBytes(
|
|
fsImpl.readFileSync(params.procMeminfoPath ?? PROC_MEMINFO_PATH, "utf8"),
|
|
);
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
function resolveTsdownMaxOldSpaceMb(params = {}) {
|
|
const limitBytes = readCgroupMemoryLimitBytes(params) ?? readProcMemTotalBytes(params);
|
|
if (limitBytes === null) {
|
|
return DEFAULT_TSDOWN_MAX_OLD_SPACE_MB;
|
|
}
|
|
|
|
const limitMb = Math.floor(limitBytes / 1024 / 1024);
|
|
if (limitMb <= 0) {
|
|
return DEFAULT_TSDOWN_MAX_OLD_SPACE_MB;
|
|
}
|
|
|
|
const cgroupCap = Math.max(
|
|
MIN_TSDOWN_MAX_OLD_SPACE_MB,
|
|
limitMb - TSDOWN_CGROUP_MEMORY_HEADROOM_MB,
|
|
);
|
|
return Math.min(DEFAULT_TSDOWN_MAX_OLD_SPACE_MB, cgroupCap);
|
|
}
|
|
|
|
function parseMaxOldSpaceSizeMb(value, fallbackMb) {
|
|
const parsed = Number(value);
|
|
if (!Number.isFinite(parsed) || parsed <= 0) {
|
|
return fallbackMb;
|
|
}
|
|
return Math.trunc(parsed);
|
|
}
|
|
|
|
function normalizeTsdownNodeOptions(nodeOptions, params = {}) {
|
|
const maxOldSpaceMb = resolveTsdownMaxOldSpaceMb(params);
|
|
const parts = nodeOptions.trim().split(/\s+/u).filter(Boolean);
|
|
const normalized = [];
|
|
let foundMaxOldSpaceSize = false;
|
|
|
|
for (let index = 0; index < parts.length; index += 1) {
|
|
const part = parts[index];
|
|
const inlineMatch = part.match(/^--max-old-space-size=(\d+)$/u);
|
|
if (inlineMatch) {
|
|
foundMaxOldSpaceSize = true;
|
|
const value = Math.min(parseMaxOldSpaceSizeMb(inlineMatch[1], maxOldSpaceMb), maxOldSpaceMb);
|
|
normalized.push(`--max-old-space-size=${value}`);
|
|
continue;
|
|
}
|
|
|
|
if (part === "--max-old-space-size") {
|
|
foundMaxOldSpaceSize = true;
|
|
const next = parts[index + 1];
|
|
const value = Math.min(parseMaxOldSpaceSizeMb(next, maxOldSpaceMb), maxOldSpaceMb);
|
|
normalized.push(`--max-old-space-size=${value}`);
|
|
if (next !== undefined) {
|
|
index += 1;
|
|
}
|
|
continue;
|
|
}
|
|
|
|
normalized.push(part);
|
|
}
|
|
|
|
if (!foundMaxOldSpaceSize) {
|
|
normalized.push(`--max-old-space-size=${maxOldSpaceMb}`);
|
|
}
|
|
|
|
return normalized.join(" ");
|
|
}
|
|
|
|
function resolveTsdownEnv(env, params = {}) {
|
|
const nodeOptions = env.NODE_OPTIONS?.trim() ?? "";
|
|
return {
|
|
...env,
|
|
NODE_OPTIONS: normalizeTsdownNodeOptions(nodeOptions, params),
|
|
};
|
|
}
|
|
|
|
export function createTsdownOutputScanner(params = {}) {
|
|
const maxCaptureBytes = params.maxCaptureBytes ?? DEFAULT_CAPTURE_BYTES;
|
|
let captured = "";
|
|
let pendingLine = "";
|
|
let hasIneffectiveDynamicImport = false;
|
|
let fatalUnresolvedImport = null;
|
|
|
|
function scanLines(text) {
|
|
const combined = pendingLine + text;
|
|
const lines = combined.split(/\r?\n/u);
|
|
pendingLine = lines.pop() ?? "";
|
|
for (const line of lines) {
|
|
fatalUnresolvedImport ??= findFatalUnresolvedImport([line]);
|
|
}
|
|
}
|
|
|
|
return {
|
|
append(chunk) {
|
|
const text = Buffer.isBuffer(chunk) ? chunk.toString("utf8") : String(chunk);
|
|
if (text.includes(INEFFECTIVE_DYNAMIC_IMPORT_MARKER)) {
|
|
hasIneffectiveDynamicImport = true;
|
|
}
|
|
scanLines(text);
|
|
captured += text;
|
|
if (captured.length > maxCaptureBytes) {
|
|
captured = captured.slice(-maxCaptureBytes);
|
|
}
|
|
},
|
|
finish() {
|
|
if (pendingLine) {
|
|
fatalUnresolvedImport ??= findFatalUnresolvedImport([pendingLine]);
|
|
pendingLine = "";
|
|
}
|
|
return {
|
|
captured,
|
|
hasIneffectiveDynamicImport,
|
|
fatalUnresolvedImport,
|
|
};
|
|
},
|
|
};
|
|
}
|
|
|
|
export function resolveTsdownBuildInvocation(params = {}) {
|
|
const env = resolveTsdownEnv(params.env ?? process.env, params);
|
|
const tsdownArgs = [
|
|
"--config-loader",
|
|
"unrun",
|
|
"--logLevel",
|
|
logLevel,
|
|
"--no-clean",
|
|
...extraArgs,
|
|
];
|
|
if (env.OPENCLAW_BUILD_ALL_NO_PNPM === "1") {
|
|
return {
|
|
command: params.nodeExecPath ?? process.execPath,
|
|
args: ["node_modules/tsdown/dist/run.mjs", ...tsdownArgs],
|
|
options: {
|
|
stdio: ["ignore", "pipe", "pipe"],
|
|
shell: false,
|
|
windowsVerbatimArguments: undefined,
|
|
env,
|
|
},
|
|
};
|
|
}
|
|
const runner = resolvePnpmRunner({
|
|
pnpmArgs: ["exec", "tsdown", ...tsdownArgs],
|
|
nodeExecPath: params.nodeExecPath ?? process.execPath,
|
|
npmExecPath: params.npmExecPath ?? env.npm_execpath,
|
|
comSpec: params.comSpec ?? env.ComSpec,
|
|
platform: params.platform ?? process.platform,
|
|
});
|
|
return {
|
|
command: runner.command,
|
|
args: runner.args,
|
|
options: {
|
|
stdio: ["ignore", "pipe", "pipe"],
|
|
shell: runner.shell,
|
|
windowsVerbatimArguments: runner.windowsVerbatimArguments,
|
|
env,
|
|
},
|
|
};
|
|
}
|
|
|
|
export async function runTsdownBuildInvocation(invocation, params = {}) {
|
|
const stdout = params.stdout ?? process.stdout;
|
|
const stderr = params.stderr ?? process.stderr;
|
|
const env = params.env ?? process.env;
|
|
const scanner = params.scanner ?? createTsdownOutputScanner();
|
|
const timeoutMs = parsePositiveInteger(env.OPENCLAW_TSDOWN_TIMEOUT_MS);
|
|
const heartbeatMs =
|
|
parseNonNegativeInteger(env.OPENCLAW_TSDOWN_HEARTBEAT_MS) ?? DEFAULT_HEARTBEAT_MS;
|
|
let timedOut = false;
|
|
let settled = false;
|
|
let lastOutputAt = Date.now();
|
|
|
|
const child = spawn(invocation.command, invocation.args, invocation.options);
|
|
const pidText = child.pid ? ` pid=${child.pid}` : "";
|
|
|
|
function markOutput() {
|
|
lastOutputAt = Date.now();
|
|
}
|
|
|
|
child.stdout?.on("data", (chunk) => {
|
|
markOutput();
|
|
scanner.append(chunk);
|
|
stdout.write(chunk);
|
|
});
|
|
child.stderr?.on("data", (chunk) => {
|
|
markOutput();
|
|
scanner.append(chunk);
|
|
stderr.write(chunk);
|
|
});
|
|
|
|
const heartbeat =
|
|
heartbeatMs > 0
|
|
? setInterval(() => {
|
|
if (settled) {
|
|
return;
|
|
}
|
|
const silentForMs = Date.now() - lastOutputAt;
|
|
if (silentForMs < heartbeatMs) {
|
|
return;
|
|
}
|
|
stderr.write(
|
|
`[tsdown-build] still running${pidText}; no output for ${Math.round(
|
|
silentForMs / 1000,
|
|
)}s\n`,
|
|
);
|
|
lastOutputAt = Date.now();
|
|
}, heartbeatMs).unref()
|
|
: null;
|
|
|
|
const timeout =
|
|
timeoutMs !== null
|
|
? setTimeout(() => {
|
|
timedOut = true;
|
|
stderr.write(`[tsdown-build] timeout after ${timeoutMs}ms${pidText}; sending SIGTERM\n`);
|
|
child.kill("SIGTERM");
|
|
setTimeout(() => {
|
|
if (!settled) {
|
|
stderr.write(`[tsdown-build] forcing SIGKILL${pidText}\n`);
|
|
child.kill("SIGKILL");
|
|
}
|
|
}, TERMINATION_GRACE_MS).unref();
|
|
}, timeoutMs).unref()
|
|
: null;
|
|
|
|
return new Promise((resolve) => {
|
|
child.once("error", (error) => {
|
|
settled = true;
|
|
clearInterval(heartbeat);
|
|
clearTimeout(timeout);
|
|
stderr.write(`[tsdown-build] failed to start: ${String(error)}\n`);
|
|
resolve({
|
|
status: 1,
|
|
signal: null,
|
|
timedOut,
|
|
error,
|
|
...scanner.finish(),
|
|
});
|
|
});
|
|
child.once("close", (status, signal) => {
|
|
settled = true;
|
|
clearInterval(heartbeat);
|
|
clearTimeout(timeout);
|
|
resolve({
|
|
status,
|
|
signal,
|
|
timedOut,
|
|
error: null,
|
|
...scanner.finish(),
|
|
});
|
|
});
|
|
});
|
|
}
|
|
|
|
function isMainModule() {
|
|
const argv1 = process.argv[1];
|
|
if (!argv1) {
|
|
return false;
|
|
}
|
|
return import.meta.url === pathToFileURL(argv1).href;
|
|
}
|
|
|
|
if (isMainModule()) {
|
|
pruneSourceCheckoutBundledPluginNodeModules();
|
|
pruneUntrackedGeneratedSourceDeclarations();
|
|
pruneStaleRuntimeSymlinks();
|
|
cleanTsdownOutputRoots();
|
|
const invocation = resolveTsdownBuildInvocation();
|
|
const result = await runTsdownBuildInvocation(invocation);
|
|
|
|
if (result.status === 0 && result.hasIneffectiveDynamicImport) {
|
|
console.error(
|
|
"Build emitted [INEFFECTIVE_DYNAMIC_IMPORT]. Replace transparent runtime re-export facades with real runtime boundaries.",
|
|
);
|
|
process.exit(1);
|
|
}
|
|
|
|
if (result.status === 0 && result.fatalUnresolvedImport) {
|
|
console.error(
|
|
`Build emitted [UNRESOLVED_IMPORT] outside extensions: ${result.fatalUnresolvedImport}`,
|
|
);
|
|
process.exit(1);
|
|
}
|
|
|
|
if (result.timedOut) {
|
|
process.exit(124);
|
|
}
|
|
|
|
if (typeof result.status === "number") {
|
|
process.exit(result.status);
|
|
}
|
|
|
|
process.exit(1);
|
|
}
|