ci: speed up package tarball validation

scripts/check-openclaw-package-tarball.mjs

@@ -6,8 +6,10 @@ import { spawnSync } from "node:child_process";
 import fs from "node:fs";
 import os from "node:os";
 import path from "node:path";
+import { performance } from "node:perf_hooks";
 import { LOCAL_BUILD_METADATA_DIST_PATHS } from "./lib/local-build-metadata-paths.mjs";
 import {
+  collectPackageDistImports,
   collectPackageDistImportErrors,
   expandPackageDistImportClosure,
 } from "./lib/package-dist-imports.mjs";
@@ -29,20 +31,37 @@ if (!fs.existsSync(tarball)) {
   fail(`OpenClaw package tarball does not exist: ${tarball}`);
 }
 
-const list = spawnSync("tar", ["-tf", tarball], {
-  encoding: "utf8",
-  stdio: ["ignore", "pipe", "pipe"],
-});
+const phaseTimingsEnabled = process.env.OPENCLAW_PACKAGE_TARBALL_CHECK_TIMINGS !== "0";
+function runPhase(label, action) {
+  const startedAt = performance.now();
+  try {
+    return action();
+  } finally {
+    if (phaseTimingsEnabled) {
+      const durationMs = Math.round(performance.now() - startedAt);
+      console.error(`check-openclaw-package-tarball: ${label} completed in ${durationMs}ms`);
+    }
+  }
+}
+
+const list = runPhase("tar list", () =>
+  spawnSync("tar", ["-tf", tarball], {
+    encoding: "utf8",
+    stdio: ["ignore", "pipe", "pipe"],
+  }),
+);
 if (list.status !== 0) {
   fail(`tar -tf failed for ${tarball}: ${list.stderr || list.status}`);
 }
 
 const extractDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-package-tarball-"));
 try {
-  const extract = spawnSync("tar", ["-xf", tarball, "-C", extractDir], {
-    encoding: "utf8",
-    stdio: ["ignore", "pipe", "pipe"],
-  });
+  const extract = runPhase("tar extract", () =>
+    spawnSync("tar", ["-xf", tarball, "-C", extractDir], {
+      encoding: "utf8",
+      stdio: ["ignore", "pipe", "pipe"],
+    }),
+  );
   if (extract.status !== 0) {
     fail(`tar -xf failed for ${tarball}: ${extract.stderr || extract.status}`);
   }
@@ -181,6 +200,7 @@ for (const forbiddenEntry of FORBIDDEN_LOCAL_BUILD_METADATA_FILES) {
 if (!entrySet.has("dist/postinstall-inventory.json")) {
   errors.push("missing dist/postinstall-inventory.json");
 }
+let packageDistImports = null;
 if (entrySet.has("dist/postinstall-inventory.json")) {
   try {
     const allowLegacyPrivateQaInventoryOmissions =
@@ -191,6 +211,12 @@ if (entrySet.has("dist/postinstall-inventory.json")) {
     } else {
       const normalizedInventory = inventory.map((entry) => entry.replace(/\\/gu, "/"));
       const normalizedInventorySet = new Set(normalizedInventory);
+      packageDistImports = runPhase("dist import graph", () =>
+        collectPackageDistImports({
+          files: normalized,
+          readText: readTarEntry,
+        }),
+      );
       for (const inventoryEntry of inventory) {
         const normalizedEntry = inventoryEntry.replace(/\\/gu, "/");
         if (!entrySet.has(normalizedEntry)) {
@@ -210,6 +236,7 @@ if (entrySet.has("dist/postinstall-inventory.json")) {
         files: normalized,
         seedFiles: normalizedInventory,
         readText: readTarEntry,
+        imports: packageDistImports,
       });
       for (const importedEntry of expandedInventory) {
         if (!normalizedInventorySet.has(importedEntry)) {
@@ -230,6 +257,7 @@ errors.push(
   ...collectPackageDistImportErrors({
     files: normalized,
     readText: readTarEntry,
+    imports: packageDistImports ?? undefined,
   }),
 );
 

scripts/lib/package-dist-imports.mjs

@@ -112,11 +112,9 @@ export function collectPackageDistImportErrors(params) {
   const files = [...new Set(params.files.map(normalizePackagePath))];
   const fileSet = new Set(files);
   const errors = [];
+  const imports = params.imports ?? collectPackageDistImports({ files, readText: params.readText });
 
-  for (const { importerPath, importedPath } of collectPackageDistImports({
-    files,
-    readText: params.readText,
-  })) {
+  for (const { importerPath, importedPath } of imports) {
     if (!fileSet.has(importedPath)) {
       errors.push(`${importerPath} imports missing ${importedPath}`);
     }
@@ -150,19 +148,22 @@ export function expandPackageDistImportClosure(params) {
   const files = [...new Set(params.files.map(normalizePackagePath))];
   const fileSet = new Set(files);
   const expectedSet = new Set(params.seedFiles.map(normalizePackagePath));
-  let changed = true;
+  const imports = params.imports ?? collectPackageDistImports({ files, readText: params.readText });
+  const importsByImporter = new Map();
+  for (const { importerPath, importedPath } of imports) {
+    const importerImports = importsByImporter.get(importerPath) ?? [];
+    importerImports.push(importedPath);
+    importsByImporter.set(importerPath, importerImports);
+  }
 
-  while (changed) {
-    changed = false;
-    for (const { importedPath } of collectPackageDistImports({
-      files: [...expectedSet].filter((file) => fileSet.has(file)),
-      readText: params.readText,
-    })) {
-      if (!fileSet.has(importedPath) || expectedSet.has(importedPath)) {
-        continue;
+  const queue = [...expectedSet].filter((file) => fileSet.has(file));
+  for (let index = 0; index < queue.length; index += 1) {
+    const importerPath = queue[index];
+    for (const importedPath of importsByImporter.get(importerPath) ?? []) {
+      if (fileSet.has(importedPath) && !expectedSet.has(importedPath)) {
+        expectedSet.add(importedPath);
+        queue.push(importedPath);
       }
-      expectedSet.add(importedPath);
-      changed = true;
     }
   }
 

scripts/runtime-postbuild.mjs

@@ -1,5 +1,6 @@
 import fs from "node:fs";
 import path from "node:path";
+import { performance } from "node:perf_hooks";
 import { fileURLToPath, pathToFileURL } from "node:url";
 import { copyBundledPluginMetadata } from "./copy-bundled-plugin-metadata.mjs";
 import { copyPluginSdkRootAlias } from "./copy-plugin-sdk-root-alias.mjs";
@@ -31,6 +32,14 @@ export const STATIC_EXTENSION_ASSETS = [
     src: "extensions/acpx/src/runtime-internals/mcp-proxy.mjs",
     dest: "dist/extensions/acpx/mcp-proxy.mjs",
   },
+  {
+    src: "extensions/acpx/src/runtime-internals/error-format.mjs",
+    dest: "dist/extensions/acpx/error-format.mjs",
+  },
+  {
+    src: "extensions/acpx/src/runtime-internals/mcp-command-line.mjs",
+    dest: "dist/extensions/acpx/mcp-command-line.mjs",
+  },
   // diffs viewer runtime bundle — co-deployed inside the plugin package so the
   // built bundle can resolve `./assets/viewer-runtime.js` from dist.
   {
@@ -96,14 +105,26 @@ export function writeLegacyCliExitCompatChunks(params = {}) {
 }
 
 export function runRuntimePostBuild(params = {}) {
-  copyPluginSdkRootAlias(params);
-  copyBundledPluginMetadata(params);
-  writeOfficialChannelCatalog(params);
-  stageBundledPluginRuntimeDeps(params);
-  stageBundledPluginRuntime(params);
-  writeStableRootRuntimeAliases(params);
-  writeLegacyCliExitCompatChunks(params);
-  copyStaticExtensionAssets(params);
+  const timingsEnabled = params.timings ?? process.env.OPENCLAW_RUNTIME_POSTBUILD_TIMINGS !== "0";
+  const runPhase = (label, action) => {
+    const startedAt = performance.now();
+    try {
+      return action();
+    } finally {
+      if (timingsEnabled) {
+        const durationMs = Math.round(performance.now() - startedAt);
+        console.error(`runtime-postbuild: ${label} completed in ${durationMs}ms`);
+      }
+    }
+  };
+  runPhase("plugin SDK root alias", () => copyPluginSdkRootAlias(params));
+  runPhase("bundled plugin metadata", () => copyBundledPluginMetadata(params));
+  runPhase("official channel catalog", () => writeOfficialChannelCatalog(params));
+  runPhase("bundled plugin runtime deps", () => stageBundledPluginRuntimeDeps(params));
+  runPhase("bundled plugin runtime overlay", () => stageBundledPluginRuntime(params));
+  runPhase("stable root runtime aliases", () => writeStableRootRuntimeAliases(params));
+  runPhase("legacy CLI exit compat chunks", () => writeLegacyCliExitCompatChunks(params));
+  runPhase("static extension assets", () => copyStaticExtensionAssets(params));
 }
 
 if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) {

scripts/stage-bundled-plugin-runtime-deps.mjs

@@ -2,6 +2,7 @@ import { spawnSync } from "node:child_process";
 import { createHash } from "node:crypto";
 import fs from "node:fs";
 import path from "node:path";
+import { performance } from "node:perf_hooks";
 import { pathToFileURL } from "node:url";
 import semverSatisfies from "semver/functions/satisfies.js";
 import { resolveNpmRunner } from "./npm-runner.mjs";
@@ -488,33 +489,6 @@ function resolveInstalledDirectDependencyNames(
   return directDependencyNames;
 }
 
-function appendDirectoryFingerprint(hash, rootDir, currentDir = rootDir) {
-  const entries = fs
-    .readdirSync(currentDir, { withFileTypes: true })
-    .toSorted((left, right) => left.name.localeCompare(right.name));
-
-  for (const entry of entries) {
-    const fullPath = path.join(currentDir, entry.name);
-    const relativePath = path.relative(rootDir, fullPath).replace(/\\/g, "/");
-    const stats = fs.lstatSync(fullPath);
-    if (stats.isSymbolicLink()) {
-      hash.update(`symlink:${relativePath}->${fs.readlinkSync(fullPath).replace(/\\/g, "/")}\n`);
-      continue;
-    }
-    if (stats.isDirectory()) {
-      hash.update(`dir:${relativePath}\n`);
-      appendDirectoryFingerprint(hash, rootDir, fullPath);
-      continue;
-    }
-    if (!stats.isFile()) {
-      continue;
-    }
-    const stat = fs.statSync(fullPath);
-    hash.update(`file:${relativePath}:${stat.size}\n`);
-    hash.update(fs.readFileSync(fullPath));
-  }
-}
-
 function createInstalledRuntimeClosureFingerprint(rootNodeModulesDir, dependencyNames) {
   const hash = createHash("sha256");
   for (const depName of [...dependencyNames].toSorted((left, right) => left.localeCompare(right))) {
@@ -522,8 +496,19 @@ function createInstalledRuntimeClosureFingerprint(rootNodeModulesDir, dependency
     if (depRoot === null || !fs.existsSync(depRoot)) {
       return null;
     }
-    hash.update(`package:${depName}\n`);
-    appendDirectoryFingerprint(hash, depRoot);
+    const packageJsonPath = path.join(depRoot, "package.json");
+    const installedVersion = readInstalledDependencyVersionFromRoot(depRoot);
+    const realRoot = fs.realpathSync(depRoot);
+    const packageJsonStat = fs.statSync(packageJsonPath);
+    hash.update(
+      JSON.stringify({
+        depName,
+        installedVersion,
+        packageJsonMtimeMs: packageJsonStat.mtimeMs,
+        packageJsonSize: packageJsonStat.size,
+        realRoot,
+      }),
+    );
   }
   return hash.digest("hex");
 }
@@ -1235,69 +1220,102 @@ export function stageBundledPluginRuntimeDeps(params = {}) {
     params.installPluginRuntimeDepsImpl ?? installPluginRuntimeDeps;
   const installAttempts = params.installAttempts ?? 3;
   const pruneConfig = resolveRuntimeDepPruneConfig(params);
+  const timingsEnabled =
+    params.timings ?? process.env.OPENCLAW_RUNTIME_DEPS_STAGING_TIMINGS === "1";
+  const runPluginPhase = (pluginId, label, action) => {
+    const startedAt = performance.now();
+    try {
+      return action();
+    } finally {
+      if (timingsEnabled) {
+        const durationMs = Math.round(performance.now() - startedAt);
+        console.error(
+          `stage-bundled-plugin-runtime-deps: ${pluginId} ${label} completed in ${durationMs}ms`,
+        );
+      }
+    }
+  };
   for (const pluginDir of listBundledPluginRuntimeDirs(repoRoot)) {
     const pluginId = path.basename(pluginDir);
     const sourcePluginRoot = resolveInstalledWorkspacePluginRoot(repoRoot, pluginId);
     const directDependencyPackageRoot = fs.existsSync(path.join(sourcePluginRoot, "package.json"))
       ? sourcePluginRoot
       : null;
-    const packageJson = sanitizeBundledManifestForRuntimeInstall(pluginDir);
+    const packageJson = runPluginPhase(pluginId, "sanitize manifest", () =>
+      sanitizeBundledManifestForRuntimeInstall(pluginDir),
+    );
     const nodeModulesDir = path.join(pluginDir, "node_modules");
     const stampPath = resolveRuntimeDepsStampPath(repoRoot, pluginId);
     const legacyStampPath = resolveLegacyRuntimeDepsStampPath(pluginDir);
-    removePathIfExists(legacyStampPath);
-    removeStaleRuntimeDepsTempDirs(pluginDir);
+    runPluginPhase(pluginId, "cleanup stale runtime dirs", () => {
+      removePathIfExists(legacyStampPath);
+      removeStaleRuntimeDepsTempDirs(pluginDir);
+    });
     if (!hasRuntimeDeps(packageJson) || !shouldStageRuntimeDeps(packageJson)) {
-      removePathIfExists(nodeModulesDir);
-      removePathIfExists(stampPath);
+      runPluginPhase(pluginId, "remove unstaged runtime deps", () => {
+        removePathIfExists(nodeModulesDir);
+        removePathIfExists(stampPath);
+      });
       continue;
     }
-    const cheapFingerprint = createRuntimeDepsCheapFingerprint(packageJson, pruneConfig, {
-      repoRoot,
-    });
+    const cheapFingerprint = runPluginPhase(pluginId, "cheap fingerprint", () =>
+      createRuntimeDepsCheapFingerprint(packageJson, pruneConfig, {
+        repoRoot,
+      }),
+    );
     const stamp = readRuntimeDepsStamp(stampPath);
-    const rootInstalledRuntimeFingerprint = resolveInstalledRuntimeClosureFingerprint({
-      directDependencyPackageRoot,
-      packageJson,
-      rootNodeModulesDir: path.join(repoRoot, "node_modules"),
-    });
+    const rootInstalledRuntimeFingerprint = runPluginPhase(
+      pluginId,
+      "installed runtime fingerprint",
+      () =>
+        resolveInstalledRuntimeClosureFingerprint({
+          directDependencyPackageRoot,
+          packageJson,
+          rootNodeModulesDir: path.join(repoRoot, "node_modules"),
+        }),
+    );
     const fingerprint = createRuntimeDepsFingerprint(packageJson, pruneConfig, {
       repoRoot,
       rootInstalledRuntimeFingerprint,
     });
     if (fs.existsSync(nodeModulesDir) && stamp?.fingerprint === fingerprint) {
+      runPluginPhase(pluginId, "reuse staged runtime deps", () => {});
       continue;
     }
     if (
-      stageInstalledRootRuntimeDeps({
-        directDependencyPackageRoot,
-        fingerprint,
-        cheapFingerprint,
-        packageJson,
-        pluginDir,
-        pruneConfig,
-        repoRoot,
-        stampPath,
-      })
-    ) {
-      continue;
-    }
-    try {
-      installPluginRuntimeDepsWithRetries({
-        attempts: installAttempts,
-        install: installPluginRuntimeDepsImpl,
-        installParams: {
+      runPluginPhase(pluginId, "stage installed root runtime deps", () =>
+        stageInstalledRootRuntimeDeps({
           directDependencyPackageRoot,
           fingerprint,
           cheapFingerprint,
           packageJson,
           pluginDir,
-          pluginId,
           pruneConfig,
           repoRoot,
           stampPath,
-        },
-      });
+        }),
+      )
+    ) {
+      continue;
+    }
+    try {
+      runPluginPhase(pluginId, "fallback install runtime deps", () =>
+        installPluginRuntimeDepsWithRetries({
+          attempts: installAttempts,
+          install: installPluginRuntimeDepsImpl,
+          installParams: {
+            directDependencyPackageRoot,
+            fingerprint,
+            cheapFingerprint,
+            packageJson,
+            pluginDir,
+            pluginId,
+            pruneConfig,
+            repoRoot,
+            stampPath,
+          },
+        }),
+      );
     } catch (error) {
       throw createRootRuntimeStagingError({ packageJson, pluginId, cause: error });
     }