diff --git a/scripts/check-openclaw-package-tarball.mjs b/scripts/check-openclaw-package-tarball.mjs index 576a8a62169..54d4a14ec8d 100644 --- a/scripts/check-openclaw-package-tarball.mjs +++ b/scripts/check-openclaw-package-tarball.mjs @@ -6,8 +6,10 @@ import { spawnSync } from "node:child_process"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; +import { performance } from "node:perf_hooks"; import { LOCAL_BUILD_METADATA_DIST_PATHS } from "./lib/local-build-metadata-paths.mjs"; import { + collectPackageDistImports, collectPackageDistImportErrors, expandPackageDistImportClosure, } from "./lib/package-dist-imports.mjs"; @@ -29,20 +31,37 @@ if (!fs.existsSync(tarball)) { fail(`OpenClaw package tarball does not exist: ${tarball}`); } -const list = spawnSync("tar", ["-tf", tarball], { - encoding: "utf8", - stdio: ["ignore", "pipe", "pipe"], -}); +const phaseTimingsEnabled = process.env.OPENCLAW_PACKAGE_TARBALL_CHECK_TIMINGS !== "0"; +function runPhase(label, action) { + const startedAt = performance.now(); + try { + return action(); + } finally { + if (phaseTimingsEnabled) { + const durationMs = Math.round(performance.now() - startedAt); + console.error(`check-openclaw-package-tarball: ${label} completed in ${durationMs}ms`); + } + } +} + +const list = runPhase("tar list", () => + spawnSync("tar", ["-tf", tarball], { + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + }), +); if (list.status !== 0) { fail(`tar -tf failed for ${tarball}: ${list.stderr || list.status}`); } const extractDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-package-tarball-")); try { - const extract = spawnSync("tar", ["-xf", tarball, "-C", extractDir], { - encoding: "utf8", - stdio: ["ignore", "pipe", "pipe"], - }); + const extract = runPhase("tar extract", () => + spawnSync("tar", ["-xf", tarball, "-C", extractDir], { + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + }), + ); if (extract.status !== 0) { fail(`tar -xf failed for ${tarball}: ${extract.stderr || extract.status}`); } @@ -181,6 +200,7 @@ for (const forbiddenEntry of FORBIDDEN_LOCAL_BUILD_METADATA_FILES) { if (!entrySet.has("dist/postinstall-inventory.json")) { errors.push("missing dist/postinstall-inventory.json"); } +let packageDistImports = null; if (entrySet.has("dist/postinstall-inventory.json")) { try { const allowLegacyPrivateQaInventoryOmissions = @@ -191,6 +211,12 @@ if (entrySet.has("dist/postinstall-inventory.json")) { } else { const normalizedInventory = inventory.map((entry) => entry.replace(/\\/gu, "/")); const normalizedInventorySet = new Set(normalizedInventory); + packageDistImports = runPhase("dist import graph", () => + collectPackageDistImports({ + files: normalized, + readText: readTarEntry, + }), + ); for (const inventoryEntry of inventory) { const normalizedEntry = inventoryEntry.replace(/\\/gu, "/"); if (!entrySet.has(normalizedEntry)) { @@ -210,6 +236,7 @@ if (entrySet.has("dist/postinstall-inventory.json")) { files: normalized, seedFiles: normalizedInventory, readText: readTarEntry, + imports: packageDistImports, }); for (const importedEntry of expandedInventory) { if (!normalizedInventorySet.has(importedEntry)) { @@ -230,6 +257,7 @@ errors.push( ...collectPackageDistImportErrors({ files: normalized, readText: readTarEntry, + imports: packageDistImports ?? undefined, }), ); diff --git a/scripts/lib/package-dist-imports.mjs b/scripts/lib/package-dist-imports.mjs index 35ca7bac370..fb1589d9755 100644 --- a/scripts/lib/package-dist-imports.mjs +++ b/scripts/lib/package-dist-imports.mjs @@ -112,11 +112,9 @@ export function collectPackageDistImportErrors(params) { const files = [...new Set(params.files.map(normalizePackagePath))]; const fileSet = new Set(files); const errors = []; + const imports = params.imports ?? collectPackageDistImports({ files, readText: params.readText }); - for (const { importerPath, importedPath } of collectPackageDistImports({ - files, - readText: params.readText, - })) { + for (const { importerPath, importedPath } of imports) { if (!fileSet.has(importedPath)) { errors.push(`${importerPath} imports missing ${importedPath}`); } @@ -150,19 +148,22 @@ export function expandPackageDistImportClosure(params) { const files = [...new Set(params.files.map(normalizePackagePath))]; const fileSet = new Set(files); const expectedSet = new Set(params.seedFiles.map(normalizePackagePath)); - let changed = true; + const imports = params.imports ?? collectPackageDistImports({ files, readText: params.readText }); + const importsByImporter = new Map(); + for (const { importerPath, importedPath } of imports) { + const importerImports = importsByImporter.get(importerPath) ?? []; + importerImports.push(importedPath); + importsByImporter.set(importerPath, importerImports); + } - while (changed) { - changed = false; - for (const { importedPath } of collectPackageDistImports({ - files: [...expectedSet].filter((file) => fileSet.has(file)), - readText: params.readText, - })) { - if (!fileSet.has(importedPath) || expectedSet.has(importedPath)) { - continue; + const queue = [...expectedSet].filter((file) => fileSet.has(file)); + for (let index = 0; index < queue.length; index += 1) { + const importerPath = queue[index]; + for (const importedPath of importsByImporter.get(importerPath) ?? []) { + if (fileSet.has(importedPath) && !expectedSet.has(importedPath)) { + expectedSet.add(importedPath); + queue.push(importedPath); } - expectedSet.add(importedPath); - changed = true; } } diff --git a/scripts/runtime-postbuild.mjs b/scripts/runtime-postbuild.mjs index 9b1c4973206..34e7661dfac 100644 --- a/scripts/runtime-postbuild.mjs +++ b/scripts/runtime-postbuild.mjs @@ -1,5 +1,6 @@ import fs from "node:fs"; import path from "node:path"; +import { performance } from "node:perf_hooks"; import { fileURLToPath, pathToFileURL } from "node:url"; import { copyBundledPluginMetadata } from "./copy-bundled-plugin-metadata.mjs"; import { copyPluginSdkRootAlias } from "./copy-plugin-sdk-root-alias.mjs"; @@ -31,6 +32,14 @@ export const STATIC_EXTENSION_ASSETS = [ src: "extensions/acpx/src/runtime-internals/mcp-proxy.mjs", dest: "dist/extensions/acpx/mcp-proxy.mjs", }, + { + src: "extensions/acpx/src/runtime-internals/error-format.mjs", + dest: "dist/extensions/acpx/error-format.mjs", + }, + { + src: "extensions/acpx/src/runtime-internals/mcp-command-line.mjs", + dest: "dist/extensions/acpx/mcp-command-line.mjs", + }, // diffs viewer runtime bundle — co-deployed inside the plugin package so the // built bundle can resolve `./assets/viewer-runtime.js` from dist. { @@ -96,14 +105,26 @@ export function writeLegacyCliExitCompatChunks(params = {}) { } export function runRuntimePostBuild(params = {}) { - copyPluginSdkRootAlias(params); - copyBundledPluginMetadata(params); - writeOfficialChannelCatalog(params); - stageBundledPluginRuntimeDeps(params); - stageBundledPluginRuntime(params); - writeStableRootRuntimeAliases(params); - writeLegacyCliExitCompatChunks(params); - copyStaticExtensionAssets(params); + const timingsEnabled = params.timings ?? process.env.OPENCLAW_RUNTIME_POSTBUILD_TIMINGS !== "0"; + const runPhase = (label, action) => { + const startedAt = performance.now(); + try { + return action(); + } finally { + if (timingsEnabled) { + const durationMs = Math.round(performance.now() - startedAt); + console.error(`runtime-postbuild: ${label} completed in ${durationMs}ms`); + } + } + }; + runPhase("plugin SDK root alias", () => copyPluginSdkRootAlias(params)); + runPhase("bundled plugin metadata", () => copyBundledPluginMetadata(params)); + runPhase("official channel catalog", () => writeOfficialChannelCatalog(params)); + runPhase("bundled plugin runtime deps", () => stageBundledPluginRuntimeDeps(params)); + runPhase("bundled plugin runtime overlay", () => stageBundledPluginRuntime(params)); + runPhase("stable root runtime aliases", () => writeStableRootRuntimeAliases(params)); + runPhase("legacy CLI exit compat chunks", () => writeLegacyCliExitCompatChunks(params)); + runPhase("static extension assets", () => copyStaticExtensionAssets(params)); } if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { diff --git a/scripts/stage-bundled-plugin-runtime-deps.mjs b/scripts/stage-bundled-plugin-runtime-deps.mjs index b93b74cb4c7..d6385e521ee 100644 --- a/scripts/stage-bundled-plugin-runtime-deps.mjs +++ b/scripts/stage-bundled-plugin-runtime-deps.mjs @@ -2,6 +2,7 @@ import { spawnSync } from "node:child_process"; import { createHash } from "node:crypto"; import fs from "node:fs"; import path from "node:path"; +import { performance } from "node:perf_hooks"; import { pathToFileURL } from "node:url"; import semverSatisfies from "semver/functions/satisfies.js"; import { resolveNpmRunner } from "./npm-runner.mjs"; @@ -488,33 +489,6 @@ function resolveInstalledDirectDependencyNames( return directDependencyNames; } -function appendDirectoryFingerprint(hash, rootDir, currentDir = rootDir) { - const entries = fs - .readdirSync(currentDir, { withFileTypes: true }) - .toSorted((left, right) => left.name.localeCompare(right.name)); - - for (const entry of entries) { - const fullPath = path.join(currentDir, entry.name); - const relativePath = path.relative(rootDir, fullPath).replace(/\\/g, "/"); - const stats = fs.lstatSync(fullPath); - if (stats.isSymbolicLink()) { - hash.update(`symlink:${relativePath}->${fs.readlinkSync(fullPath).replace(/\\/g, "/")}\n`); - continue; - } - if (stats.isDirectory()) { - hash.update(`dir:${relativePath}\n`); - appendDirectoryFingerprint(hash, rootDir, fullPath); - continue; - } - if (!stats.isFile()) { - continue; - } - const stat = fs.statSync(fullPath); - hash.update(`file:${relativePath}:${stat.size}\n`); - hash.update(fs.readFileSync(fullPath)); - } -} - function createInstalledRuntimeClosureFingerprint(rootNodeModulesDir, dependencyNames) { const hash = createHash("sha256"); for (const depName of [...dependencyNames].toSorted((left, right) => left.localeCompare(right))) { @@ -522,8 +496,19 @@ function createInstalledRuntimeClosureFingerprint(rootNodeModulesDir, dependency if (depRoot === null || !fs.existsSync(depRoot)) { return null; } - hash.update(`package:${depName}\n`); - appendDirectoryFingerprint(hash, depRoot); + const packageJsonPath = path.join(depRoot, "package.json"); + const installedVersion = readInstalledDependencyVersionFromRoot(depRoot); + const realRoot = fs.realpathSync(depRoot); + const packageJsonStat = fs.statSync(packageJsonPath); + hash.update( + JSON.stringify({ + depName, + installedVersion, + packageJsonMtimeMs: packageJsonStat.mtimeMs, + packageJsonSize: packageJsonStat.size, + realRoot, + }), + ); } return hash.digest("hex"); } @@ -1235,69 +1220,102 @@ export function stageBundledPluginRuntimeDeps(params = {}) { params.installPluginRuntimeDepsImpl ?? installPluginRuntimeDeps; const installAttempts = params.installAttempts ?? 3; const pruneConfig = resolveRuntimeDepPruneConfig(params); + const timingsEnabled = + params.timings ?? process.env.OPENCLAW_RUNTIME_DEPS_STAGING_TIMINGS === "1"; + const runPluginPhase = (pluginId, label, action) => { + const startedAt = performance.now(); + try { + return action(); + } finally { + if (timingsEnabled) { + const durationMs = Math.round(performance.now() - startedAt); + console.error( + `stage-bundled-plugin-runtime-deps: ${pluginId} ${label} completed in ${durationMs}ms`, + ); + } + } + }; for (const pluginDir of listBundledPluginRuntimeDirs(repoRoot)) { const pluginId = path.basename(pluginDir); const sourcePluginRoot = resolveInstalledWorkspacePluginRoot(repoRoot, pluginId); const directDependencyPackageRoot = fs.existsSync(path.join(sourcePluginRoot, "package.json")) ? sourcePluginRoot : null; - const packageJson = sanitizeBundledManifestForRuntimeInstall(pluginDir); + const packageJson = runPluginPhase(pluginId, "sanitize manifest", () => + sanitizeBundledManifestForRuntimeInstall(pluginDir), + ); const nodeModulesDir = path.join(pluginDir, "node_modules"); const stampPath = resolveRuntimeDepsStampPath(repoRoot, pluginId); const legacyStampPath = resolveLegacyRuntimeDepsStampPath(pluginDir); - removePathIfExists(legacyStampPath); - removeStaleRuntimeDepsTempDirs(pluginDir); + runPluginPhase(pluginId, "cleanup stale runtime dirs", () => { + removePathIfExists(legacyStampPath); + removeStaleRuntimeDepsTempDirs(pluginDir); + }); if (!hasRuntimeDeps(packageJson) || !shouldStageRuntimeDeps(packageJson)) { - removePathIfExists(nodeModulesDir); - removePathIfExists(stampPath); + runPluginPhase(pluginId, "remove unstaged runtime deps", () => { + removePathIfExists(nodeModulesDir); + removePathIfExists(stampPath); + }); continue; } - const cheapFingerprint = createRuntimeDepsCheapFingerprint(packageJson, pruneConfig, { - repoRoot, - }); + const cheapFingerprint = runPluginPhase(pluginId, "cheap fingerprint", () => + createRuntimeDepsCheapFingerprint(packageJson, pruneConfig, { + repoRoot, + }), + ); const stamp = readRuntimeDepsStamp(stampPath); - const rootInstalledRuntimeFingerprint = resolveInstalledRuntimeClosureFingerprint({ - directDependencyPackageRoot, - packageJson, - rootNodeModulesDir: path.join(repoRoot, "node_modules"), - }); + const rootInstalledRuntimeFingerprint = runPluginPhase( + pluginId, + "installed runtime fingerprint", + () => + resolveInstalledRuntimeClosureFingerprint({ + directDependencyPackageRoot, + packageJson, + rootNodeModulesDir: path.join(repoRoot, "node_modules"), + }), + ); const fingerprint = createRuntimeDepsFingerprint(packageJson, pruneConfig, { repoRoot, rootInstalledRuntimeFingerprint, }); if (fs.existsSync(nodeModulesDir) && stamp?.fingerprint === fingerprint) { + runPluginPhase(pluginId, "reuse staged runtime deps", () => {}); continue; } if ( - stageInstalledRootRuntimeDeps({ - directDependencyPackageRoot, - fingerprint, - cheapFingerprint, - packageJson, - pluginDir, - pruneConfig, - repoRoot, - stampPath, - }) - ) { - continue; - } - try { - installPluginRuntimeDepsWithRetries({ - attempts: installAttempts, - install: installPluginRuntimeDepsImpl, - installParams: { + runPluginPhase(pluginId, "stage installed root runtime deps", () => + stageInstalledRootRuntimeDeps({ directDependencyPackageRoot, fingerprint, cheapFingerprint, packageJson, pluginDir, - pluginId, pruneConfig, repoRoot, stampPath, - }, - }); + }), + ) + ) { + continue; + } + try { + runPluginPhase(pluginId, "fallback install runtime deps", () => + installPluginRuntimeDepsWithRetries({ + attempts: installAttempts, + install: installPluginRuntimeDepsImpl, + installParams: { + directDependencyPackageRoot, + fingerprint, + cheapFingerprint, + packageJson, + pluginDir, + pluginId, + pruneConfig, + repoRoot, + stampPath, + }, + }), + ); } catch (error) { throw createRootRuntimeStagingError({ packageJson, pluginId, cause: error }); } diff --git a/test/scripts/runtime-postbuild.test.ts b/test/scripts/runtime-postbuild.test.ts index b430d00e8a1..3c3ccae3d33 100644 --- a/test/scripts/runtime-postbuild.test.ts +++ b/test/scripts/runtime-postbuild.test.ts @@ -13,8 +13,13 @@ const { createTempDir } = createScriptTestHarness(); describe("runtime postbuild static assets", () => { it("tracks plugin-owned static assets that release packaging must ship", () => { - expect(listStaticExtensionAssetOutputs()).toContain( - "dist/extensions/diffs/assets/viewer-runtime.js", + expect(listStaticExtensionAssetOutputs()).toEqual( + expect.arrayContaining([ + "dist/extensions/acpx/error-format.mjs", + "dist/extensions/acpx/mcp-command-line.mjs", + "dist/extensions/acpx/mcp-proxy.mjs", + "dist/extensions/diffs/assets/viewer-runtime.js", + ]), ); });