perf(ci): balance node test shards and gate heavy CI lanes by changed scope (#108091)

scripts/ci-run-node-test-shard.d.mts

@@ -0,0 +1,41 @@
+export type ShardTargetPlan = {
+  kind: "target";
+  name: string;
+  target: string;
+};
+
+export type ShardGroupPlan = {
+  kind: "group";
+  name: string;
+  plan: {
+    configs: string[];
+    env?: Record<string, unknown> | null;
+    includePatterns?: string[] | null;
+    shard_name?: string;
+  };
+};
+
+export type ShardPlan = ShardTargetPlan | ShardGroupPlan;
+
+export function resolveShardPlans(env?: Record<string, string | undefined>): ShardPlan[];
+
+export function buildChildEnv(
+  entry: ShardPlan,
+  baseEnv: Record<string, string | undefined>,
+  scratchDir: string,
+  index: number,
+): Record<string, string | undefined>;
+
+export function runShardPlans(
+  plans: ShardPlan[],
+  options?: {
+    concurrency?: number;
+    env?: Record<string, string | undefined>;
+    runChild?: (
+      args: string[],
+      childEnv: Record<string, string | undefined>,
+      label: string,
+    ) => Promise<number>;
+    scratchDir?: string;
+  },
+): Promise<number>;

scripts/ci-run-node-test-shard.mjs

@@ -0,0 +1,191 @@
+// Runs one CI node test shard job: either explicit changed-test targets or a
+// list of packed group plans. Extracted from .github/workflows/ci.yml so the
+// execution policy is unit-testable and plans can run concurrently.
+import { spawn } from "node:child_process";
+import { mkdtempSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { StringDecoder } from "node:string_decoder";
+import { isDirectRunUrl } from "./lib/direct-run.mjs";
+import { acquireLocalHeavyCheckLockSync } from "./lib/local-heavy-check-runtime.mjs";
+
+// Two concurrent plans halve the serial tail of packed jobs. Children run with
+// inner test-projects parallelism 1 so a job never exceeds two Vitest runs;
+// stacking outer and inner parallelism oversubscribes the 4 vCPU runner class.
+const PLAN_CONCURRENCY = 2;
+const FS_MODULE_CACHE_PATH_ENV_KEY = "OPENCLAW_VITEST_FS_MODULE_CACHE_PATH";
+
+function parseJsonEnv(env, name, fallback = null) {
+  try {
+    return JSON.parse(env[name] ?? "null") ?? fallback;
+  } catch {
+    return fallback;
+  }
+}
+
+export function resolveShardPlans(env = process.env) {
+  const targets = parseJsonEnv(env, "OPENCLAW_NODE_TEST_TARGETS_JSON");
+  if (Array.isArray(targets) && targets.length > 0) {
+    // One target per child process preserves the isolation boundaries encoded
+    // by full-suite include-pattern shards while keeping one runner job.
+    return targets.map((target) => ({ kind: "target", name: target, target }));
+  }
+
+  const groups = parseJsonEnv(env, "OPENCLAW_NODE_TEST_GROUPS_JSON");
+  const plans =
+    Array.isArray(groups) && groups.length > 0
+      ? groups
+      : [
+          {
+            configs: parseJsonEnv(env, "OPENCLAW_NODE_TEST_CONFIGS_JSON", []),
+            env: parseJsonEnv(env, "OPENCLAW_NODE_TEST_ENV_JSON"),
+            includePatterns: parseJsonEnv(env, "OPENCLAW_NODE_TEST_INCLUDE_PATTERNS_JSON"),
+            shard_name: env.OPENCLAW_VITEST_SHARD_NAME,
+          },
+        ];
+  return plans.map((plan) => ({
+    kind: "group",
+    name: plan.shard_name ?? plan.configs?.[0] ?? "group",
+    plan,
+  }));
+}
+
+export function buildChildEnv(entry, baseEnv, scratchDir, index) {
+  const childEnv = {
+    ...baseEnv,
+    // Concurrent children must not share a Vitest module cache directory;
+    // shared caches race with ENOTEMPTY when two runs rewrite the same entries.
+    [FS_MODULE_CACHE_PATH_ENV_KEY]: join(scratchDir, `vitest-cache-${index}`),
+    OPENCLAW_TEST_PROJECTS_PARALLEL: "1",
+    // This wrapper holds the repo heavy-check lock; children skipping it is
+    // what lets two plans run concurrently instead of serializing on the lock.
+    OPENCLAW_TEST_HEAVY_CHECK_LOCK_HELD: "1",
+  };
+  if (entry.kind === "target") {
+    return childEnv;
+  }
+  const plan = entry.plan;
+  if (plan.shard_name) {
+    childEnv.OPENCLAW_VITEST_SHARD_NAME = plan.shard_name;
+  }
+  if (plan.env && typeof plan.env === "object" && !Array.isArray(plan.env)) {
+    for (const [key, value] of Object.entries(plan.env)) {
+      if (typeof value === "string") {
+        childEnv[key] = value;
+      }
+    }
+  }
+  if (Array.isArray(plan.includePatterns) && plan.includePatterns.length > 0) {
+    const includeFile = join(scratchDir, `node-test-include-${index}.json`);
+    writeFileSync(includeFile, JSON.stringify(plan.includePatterns), "utf8");
+    childEnv.OPENCLAW_VITEST_INCLUDE_FILE = includeFile;
+  } else {
+    delete childEnv.OPENCLAW_VITEST_INCLUDE_FILE;
+  }
+  return childEnv;
+}
+
+const MAX_PENDING_LINE_CHARS = 1_000_000;
+
+function relayChildStream(stream, label) {
+  const decoder = new StringDecoder("utf8");
+  let pending = "";
+  const writeLine = (line) => {
+    if (!process.stdout.write(`[shard:${label}] ${line}\n`)) {
+      stream.pause();
+      process.stdout.once("drain", () => stream.resume());
+    }
+  };
+  stream.on("data", (chunk) => {
+    pending += decoder.write(chunk);
+    const lines = pending.split("\n");
+    pending = lines.pop() ?? "";
+    for (const line of lines) {
+      writeLine(line);
+    }
+    if (pending.length > MAX_PENDING_LINE_CHARS) {
+      writeLine(pending);
+      pending = "";
+    }
+  });
+  return () => {
+    pending += decoder.end();
+    if (pending !== "") {
+      writeLine(pending);
+      pending = "";
+    }
+  };
+}
+
+function runChild(args, childEnv, label) {
+  return new Promise((resolve) => {
+    const child = spawn("pnpm", ["exec", "node", "scripts/test-projects.mjs", ...args], {
+      env: childEnv,
+      stdio: ["ignore", "pipe", "pipe"],
+    });
+    // Stream with a per-line label instead of buffering: children can run
+    // whole suites for hours and verbose output must not accumulate on the
+    // wrapper heap. Backpressure pauses the child stream while stdout drains,
+    // and an oversized newline-free tail is force-flushed so the pending
+    // partial line stays bounded too.
+    const flushers = [child.stdout, child.stderr].map((stream) => relayChildStream(stream, label));
+    process.stdout.write(`[shard:${label}] begin\n`);
+    child.on("close", (code) => {
+      for (const flush of flushers) {
+        flush();
+      }
+      process.stdout.write(`[shard:${label}] end (exit ${code ?? 1})\n`);
+      resolve(code ?? 1);
+    });
+    child.on("error", (error) => {
+      process.stdout.write(`[shard:${label}] failed to spawn: ${error}\n`);
+      resolve(1);
+    });
+  });
+}
+
+export async function runShardPlans(plans, options = {}) {
+  const baseEnv = options.env ?? process.env;
+  const concurrency = Math.max(1, options.concurrency ?? PLAN_CONCURRENCY);
+  const runner = options.runChild ?? runChild;
+  const scratchDir = options.scratchDir ?? mkdtempSync(join(tmpdir(), "openclaw-node-shard-"));
+
+  let nextIndex = 0;
+  let exitCode = 0;
+  const workers = Array.from({ length: concurrency }, async () => {
+    while (nextIndex < plans.length && exitCode === 0) {
+      const index = nextIndex;
+      nextIndex += 1;
+      const entry = plans[index];
+      const args = entry.kind === "target" ? [entry.target] : entry.plan.configs;
+      if (!Array.isArray(args) || args.length === 0) {
+        console.error(`Missing node test shard configs for ${entry.name}`);
+        exitCode = exitCode || 1;
+        return;
+      }
+      const childEnv = buildChildEnv(entry, baseEnv, scratchDir, index);
+      const code = await runner(args, childEnv, entry.name);
+      if (code !== 0) {
+        // Stop scheduling new plans after a failure; the in-flight sibling
+        // finishes so its buffered output still lands in the job log.
+        exitCode = exitCode || code;
+      }
+    }
+  });
+  await Promise.all(workers);
+  return exitCode;
+}
+
+if (isDirectRunUrl(process.argv[1], import.meta.url)) {
+  const plans = resolveShardPlans();
+  const releaseLock = acquireLocalHeavyCheckLockSync({
+    cwd: process.cwd(),
+    env: process.env,
+    toolName: "test",
+  });
+  try {
+    process.exitCode = await runShardPlans(plans);
+  } finally {
+    releaseLock();
+  }
+}

scripts/lib/ci-changed-node-test-plan.d.mts

@@ -4,10 +4,14 @@ export type ChangedNodeTestShard = {
   requiresDist: boolean;
   runner: string;
   shardName: string;
-  targets: string[];
+  targets?: string[];
 };
 
 export function createChangedNodeTestShards(
   changedPaths: string[],
   options?: { cwd?: string },
 ): ChangedNodeTestShard[] | null;
+
+export function hasBuildArtifactAffectingChange(changedPaths: string[]): boolean;
+
+export function hasQaSmokeAffectingChange(changedPaths: string[]): boolean;

scripts/lib/ci-changed-node-test-plan.mjs

@@ -12,8 +12,11 @@ import { createNodeTestShards } from "./ci-node-test-plan.mjs";
 import { buildPluginSdkEntrySources, publicPluginSdkEntrypoints } from "./plugin-sdk-entries.mjs";
 
 const DEFAULT_NODE_TEST_RUNNER = "blacksmith-8vcpu-ubuntu-2404";
-const DIST_DEPENDENT_CONFIGS = new Set(["test/vitest/vitest.boundary.config.ts"]);
-const MAX_CHANGED_NODE_TEST_TARGETS = 20;
+const MAX_CHANGED_NODE_TEST_TARGETS = 96;
+// Each target runs in its own child process (isolation contract), so bound the
+// serial tail per job; the shard runner overlaps two children at a time.
+const CHANGED_NODE_TEST_TARGETS_PER_JOB = 12;
+const BOUNDARY_NODE_TEST_CONFIG = "test/vitest/vitest.boundary.config.ts";
 const publicPluginSdkEntrySources = Object.values(
   buildPluginSdkEntrySources(publicPluginSdkEntrypoints),
 );
@@ -28,8 +31,49 @@ const splitNodeTestConfigs = new Set(
   fullNodeTestShards.filter((shard) => shard.includePatterns).flatMap((shard) => shard.configs),
 );
 
+function isTestOnlyPath(changedPath) {
+  return isTestFileTarget(changedPath) || changedPath.startsWith("test/");
+}
+
 /**
- * Builds one bounded PR job from precise changed-test targets.
+ * True when any changed path can influence built dist/packaging bytes.
+ * Test-only diffs cannot change what `build:ci-artifacts` produces, so the
+ * manifest may skip the build-artifacts lane for them.
+ */
+export function hasBuildArtifactAffectingChange(changedPaths) {
+  return changedPaths.some((changedPath) => !isTestOnlyPath(changedPath));
+}
+
+const QA_SMOKE_CRITICAL_RE =
+  /^(?:extensions\/qa-lab|qa)\/|^scripts\/(?:build-all\.mjs|package-openclaw-for-docker\.mjs)$|^(?:package\.json|pnpm-lock\.yaml|npm-shrinkwrap\.json)$|^ui\//u;
+
+/**
+ * True when a changed path touches the QA smoke packaging/scenario surface.
+ * Deliberate product tradeoff: targeted PRs outside this surface skip QA smoke
+ * and rely on import-selected shards plus build-artifact CLI smokes. Targeting
+ * only fires for narrow non-SDK-impacting diffs, and QA smoke still runs on
+ * every node-scoped main push, so a missed regression breaks main visibly
+ * instead of shipping silently.
+ */
+export function hasQaSmokeAffectingChange(changedPaths) {
+  return changedPaths.some((changedPath) => QA_SMOKE_CRITICAL_RE.test(changedPath));
+}
+
+function createBoundaryShard() {
+  // Boundary tests scan the source tree (including test files) and build
+  // their own fixtures; they do not consume the built dist artifact. When the
+  // build-artifacts lane is skipped, this shard keeps that coverage.
+  return {
+    checkName: "checks-node-changed-boundary",
+    configs: [BOUNDARY_NODE_TEST_CONFIG],
+    requiresDist: false,
+    runner: DEFAULT_NODE_TEST_RUNNER,
+    shardName: "changed-boundary",
+  };
+}
+
+/**
+ * Builds bounded PR jobs from precise changed-test targets.
  * Null means the caller must fail safe to the compact full-suite plan.
  */
 export function createChangedNodeTestShards(changedPaths, options = {}) {
@@ -38,8 +82,16 @@ export function createChangedNodeTestShards(changedPaths, options = {}) {
     return null;
   }
 
-  // Deletions cannot be reconstructed reliably from the post-merge tree.
-  if (changedPaths.some((changedPath) => !existsSync(path.join(cwd, changedPath)))) {
+  const livePaths = [];
+  const deletedPaths = [];
+  for (const changedPath of changedPaths) {
+    (existsSync(path.join(cwd, changedPath)) ? livePaths : deletedPaths).push(changedPath);
+  }
+  // Deleted test files cannot regress runtime behavior, so they never block
+  // targeting. Deleted source files cannot be import-graphed from the merged
+  // tree and no live-path heuristic proves their consumers are covered, so
+  // any source deletion keeps the full-suite plan.
+  if (deletedPaths.some((deletedPath) => !isTestFileTarget(deletedPath))) {
     return null;
   }
 
@@ -53,8 +105,8 @@ export function createChangedNodeTestShards(changedPaths, options = {}) {
   // Fail safe when a core change reaches a public SDK entrypoint indirectly.
   if (
     detectChangedLanes(changedPaths).extensionImpactFromCore ||
-    (changedPaths.some((changedPath) => changedPath.startsWith("src/")) &&
-      hasImportGraphImpactOnTargets(changedPaths, publicPluginSdkEntrySources, cwd))
+    (livePaths.some((changedPath) => changedPath.startsWith("src/")) &&
+      hasImportGraphImpactOnTargets(livePaths, publicPluginSdkEntrySources, cwd))
   ) {
     return null;
   }
@@ -67,22 +119,25 @@ export function createChangedNodeTestShards(changedPaths, options = {}) {
       forceFullImportGraph: true,
       includeExtensionImpact: false,
     });
-  const plan = resolveTargetPlan(changedPaths);
+  const plan =
+    livePaths.length > 0 ? resolveTargetPlan(livePaths) : { mode: "targets", targets: [] };
   // Aggregate resolution must not let one precise path hide another path that
   // contributes no tests. Partial plans silently drop coverage.
   if (
-    changedPaths.some((changedPath) => {
+    livePaths.some((changedPath) => {
       const changedPathPlan = resolveTargetPlan([changedPath]);
       return changedPathPlan.mode !== "targets" || changedPathPlan.targets.length === 0;
     })
   ) {
     return null;
   }
+  if (plan.mode !== "targets") {
+    return null;
+  }
+  const targets = [...new Set(plan.targets)];
   if (
-    plan.mode !== "targets" ||
-    plan.targets.length === 0 ||
-    plan.targets.length > MAX_CHANGED_NODE_TEST_TARGETS ||
-    plan.targets.some(
+    targets.length > MAX_CHANGED_NODE_TEST_TARGETS ||
+    targets.some(
       (target) =>
         /^test\/vitest\/vitest\.full-.*\.config\.ts$/u.test(target) ||
         splitNodeTestConfigs.has(target),
@@ -92,7 +147,7 @@ export function createChangedNodeTestShards(changedPaths, options = {}) {
   }
 
   if (
-    plan.targets.some(
+    targets.some(
       (target) =>
         !isTestFileTarget(target) || findUnmatchedExplicitTestTargets([target], cwd).length > 0,
     )
@@ -100,7 +155,7 @@ export function createChangedNodeTestShards(changedPaths, options = {}) {
     return null;
   }
 
-  const targetPlans = plan.targets.map((target) => ({
+  const targetPlans = targets.map((target) => ({
     plans: buildVitestRunPlans([target], cwd),
     target,
   }));
@@ -121,37 +176,30 @@ export function createChangedNodeTestShards(changedPaths, options = {}) {
     return null;
   }
 
-  const nonDistTargets = targetPlans
-    .filter(({ plans }) => plans.some(({ config }) => !DIST_DEPENDENT_CONFIGS.has(config)))
-    .map(({ target }) => target);
-  const distTargets = targetPlans
-    .filter(({ plans }) => plans.some(({ config }) => DIST_DEPENDENT_CONFIGS.has(config)))
-    .map(({ target }) => target);
-
-  return [
-    ...(nonDistTargets.length > 0
-      ? [
-          {
-            checkName: "checks-node-changed",
-            configs: [],
-            requiresDist: false,
-            runner: DEFAULT_NODE_TEST_RUNNER,
-            shardName: "changed",
-            targets: nonDistTargets,
-          },
-        ]
-      : []),
-    ...(distTargets.length > 0
-      ? [
-          {
-            checkName: "checks-node-changed-dist",
-            configs: ["test/vitest/vitest.boundary.config.ts"],
-            requiresDist: true,
-            runner: DEFAULT_NODE_TEST_RUNNER,
-            shardName: "changed-dist",
-            targets: distTargets,
-          },
-        ]
-      : []),
+  // Boundary-config targets run as regular nondist targets: the boundary
+  // suite scans the checked-out tree and never consumes the built dist.
+  const orderedTargets = targetPlans.map(({ target }) => target);
+  const targetChunks = [];
+  for (
+    let offset = 0;
+    offset < orderedTargets.length;
+    offset += CHANGED_NODE_TEST_TARGETS_PER_JOB
+  ) {
+    targetChunks.push(orderedTargets.slice(offset, offset + CHANGED_NODE_TEST_TARGETS_PER_JOB));
+  }
+  const shards = [
+    ...targetChunks.map((chunk, index) => {
+      const suffix = targetChunks.length === 1 ? "" : `-${index + 1}`;
+      return {
+        checkName: `checks-node-changed${suffix}`,
+        configs: [],
+        requiresDist: false,
+        runner: DEFAULT_NODE_TEST_RUNNER,
+        shardName: `changed${suffix}`,
+        targets: chunk,
+      };
+    }),
+    ...(hasBuildArtifactAffectingChange(changedPaths) ? [] : [createBoundaryShard()]),
   ];
+  return shards.length > 0 ? shards : null;
 }

scripts/lib/ci-node-test-plan.mjs

@@ -28,11 +28,107 @@ const GATEWAY_STARTUP_HEALTH_RUNTIME_ENV = {
 const MAX_BUNDLED_NODE_TEST_PATTERNS = 64;
 // PR-only bundles trade a little serial work for fewer ephemeral runner registrations.
 // Keep runner classes and subprocess isolation intact while bounding each combined job.
-const COMPACT_NODE_TEST_JOB_WEIGHT = 256;
+const COMPACT_NODE_TEST_JOB_SECONDS = 260;
 const COMPACT_NODE_TEST_JOB_GROUPS = 10;
-const COMPACT_TOOLING_NODE_TEST_GROUPS = 3;
-const COMPACT_WHOLE_NODE_TEST_JOB_GROUPS = 8;
+const COMPACT_TOOLING_NODE_TEST_GROUPS = 4;
 const COMPACT_WHOLE_NODE_TEST_TIMEOUT_MINUTES = 120;
+const AUTO_REPLY_COMMANDS_STRIPES = 3;
+// Advisory runtime estimates (seconds) per split shard, measured from a
+// Blacksmith compact PR run under two-plans-per-job concurrency (run
+// 29395969440). Packing only: a stale entry skews job balance but never
+// correctness. Unknown shards fall back to a per-file estimate.
+const COMPACT_GROUP_SECONDS_HINTS = new Map([
+  ["agentic-agents-core-auth", 28],
+  ["agentic-agents-core-isolated", 12],
+  ["agentic-agents-core-models", 48],
+  ["agentic-agents-core-runner-cli", 120],
+  ["agentic-agents-core-runner-commands", 38],
+  ["agentic-agents-core-runner-embedded", 27],
+  ["agentic-agents-core-runner-sessions", 120],
+  ["agentic-agents-core-runtime", 81],
+  ["agentic-agents-core-subagents", 30],
+  ["agentic-agents-core-tools", 41],
+  ["agentic-agents-embedded", 69],
+  ["agentic-agents-support", 104],
+  ["agentic-agents-tools", 43],
+  ["agentic-cli", 70],
+  ["agentic-command-support", 30],
+  ["agentic-commands-agent-channel", 43],
+  ["agentic-commands-doctor", 26],
+  ["agentic-commands-doctor-auth", 8],
+  ["agentic-commands-doctor-config-state", 58],
+  ["agentic-commands-doctor-gateway", 6],
+  ["agentic-commands-doctor-plugins-tools", 14],
+  ["agentic-commands-doctor-sessions-cron", 22],
+  ["agentic-commands-models", 17],
+  ["agentic-commands-onboard-config", 25],
+  ["agentic-commands-status-tools", 19],
+  ["agentic-control-plane-agent-chat", 74],
+  ["agentic-control-plane-auth-node", 105],
+  ["agentic-control-plane-http-models", 34],
+  ["agentic-control-plane-http-plugin-ws", 28],
+  ["agentic-control-plane-runtime-config", 32],
+  ["agentic-control-plane-runtime-cron", 22],
+  ["agentic-control-plane-runtime-server", 40],
+  ["agentic-control-plane-runtime-state", 29],
+  ["agentic-control-plane-runtime-ui-tools", 23],
+  ["agentic-control-plane-startup-core", 85],
+  ["agentic-control-plane-startup-health-runtime", 20],
+  ["agentic-control-plane-startup-restart-close", 19],
+  ["agentic-gateway-core", 112],
+  ["agentic-gateway-methods", 90],
+  ["agentic-plugin-sdk", 52],
+  ["auto-reply-core-top-level", 26],
+  ["auto-reply-reply-agent-runner", 33],
+  ["auto-reply-reply-commands-1", 179],
+  ["auto-reply-reply-commands-2", 39],
+  ["auto-reply-reply-commands-3", 28],
+  ["auto-reply-reply-dispatch", 41],
+  ["auto-reply-reply-session", 38],
+  ["auto-reply-reply-state-routing", 22],
+  ["core-runtime-cron-core", 29],
+  ["core-runtime-cron-isolated-agent", 55],
+  ["core-runtime-cron-service", 13],
+  ["core-runtime-hooks", 11],
+  ["core-runtime-infra-approval-exec", 35],
+  ["core-runtime-infra-channel-plugin", 20],
+  ["core-runtime-infra-heartbeat-runner", 44],
+  ["core-runtime-infra-net-install", 15],
+  ["core-runtime-infra-outbound-actions", 15],
+  ["core-runtime-infra-outbound-core", 37],
+  ["core-runtime-infra-process", 75],
+  ["core-runtime-infra-provider-push", 21],
+  ["core-runtime-infra-storage-state", 41],
+  ["core-runtime-infra-system-runtime", 29],
+  ["core-runtime-media-ui", 123],
+  ["core-runtime-secrets", 30],
+  ["core-runtime-shared", 49],
+  // PTY timing tests inflate badly next to co-runners; keep this group in a
+  // lightly packed bin so its lane stays close to solo runtime.
+  ["core-runtime-tui-pty", 200],
+  ["core-tooling-1", 88],
+  ["core-tooling-2", 84],
+  ["core-tooling-3", 100],
+  ["core-tooling-4", 79],
+  ["core-tooling-docker", 6],
+  ["core-tooling-isolated", 50],
+  ["core-unit-fast", 157],
+  ["core-unit-src-security", 149],
+  ["core-unit-support", 28],
+]);
+const DEFAULT_WHOLE_GROUP_SECONDS = 25;
+const DEFAULT_SECONDS_PER_TEST_FILE = 0.5;
+
+function estimateCompactGroupSeconds(group) {
+  const hint = COMPACT_GROUP_SECONDS_HINTS.get(group.shard_name);
+  if (hint !== undefined) {
+    return hint;
+  }
+  if (Array.isArray(group.includePatterns)) {
+    return Math.max(3, Math.round(group.includePatterns.length * DEFAULT_SECONDS_PER_TEST_FILE));
+  }
+  return DEFAULT_WHOLE_GROUP_SECONDS;
+}
 const TOOLING_CONFIG = "test/vitest/vitest.tooling.config.ts";
 const TOOLING_DOCKER_TEST_FILE = "test/scripts/docker-build-helper.test.ts";
 const TOOLING_ISOLATED_CONFIG = "test/vitest/vitest.tooling-isolated.config.ts";
@@ -40,7 +136,9 @@ const TOOLING_ISOLATED_CONFIG = "test/vitest/vitest.tooling-isolated.config.ts";
 // shards first so short alphabetical groups cannot leave them on the tail.
 const FULL_NODE_TEST_ADMISSION_PRIORITY = new Map([
   ["core-tooling", 0],
-  ["auto-reply-reply-commands", 1],
+  ["auto-reply-reply-commands-1", 1],
+  ["auto-reply-reply-commands-2", 1],
+  ["auto-reply-reply-commands-3", 1],
 ]);
 // Commands and cron run non-isolated, so keep their split shards as separate
 // processes. Combining their include lists can retain test state across groups.
@@ -52,7 +150,10 @@ const KEEP_LARGE_NODE_TEST_RUNNER = new Set([
   "agentic-agents-core-subagents",
   "agentic-agents-embedded",
   "agentic-agents-support",
-  "agentic-agents-core-runner",
+  "agentic-agents-core-runner-cli",
+  "agentic-agents-core-runner-commands",
+  "agentic-agents-core-runner-embedded",
+  "agentic-agents-core-runner-sessions",
   "agentic-agents-core-tools",
   "agentic-control-plane-startup-core",
   "agentic-gateway-core",
@@ -103,12 +204,28 @@ function createAutoReplyReplySplitShards() {
   }
 
   return Object.entries(groups)
-    .map(([groupName, includePatterns]) => ({
-      configs: ["test/vitest/vitest.auto-reply-reply.config.ts"],
-      includePatterns,
-      requiresDist: false,
-      shardName: groupName,
-    }))
+    .flatMap(([groupName, includePatterns]) => {
+      // The commands bucket alone serializes ~3 minutes; stripe it so packing
+      // can spread that runtime across jobs.
+      if (groupName === "auto-reply-reply-commands") {
+        return createStripedBatches(includePatterns, AUTO_REPLY_COMMANDS_STRIPES).map(
+          (batch, index) => ({
+            configs: ["test/vitest/vitest.auto-reply-reply.config.ts"],
+            includePatterns: batch,
+            requiresDist: false,
+            shardName: `${groupName}-${index + 1}`,
+          }),
+        );
+      }
+      return [
+        {
+          configs: ["test/vitest/vitest.auto-reply-reply.config.ts"],
+          includePatterns,
+          requiresDist: false,
+          shardName: groupName,
+        },
+      ];
+    })
     .filter((shard) => shard.includePatterns.length > 0);
 }
 
@@ -267,15 +384,23 @@ function resolveAgentCoreShardName(file) {
   ) {
     return "agentic-agents-core-subagents";
   }
+  // The former single "core-runner" bucket serialized ~3 minutes of tests in
+  // one group; keep these three slices separate so packing can balance them.
+  if (name.startsWith("embedded-agent-runner")) {
+    return "agentic-agents-core-runner-embedded";
+  }
   if (
-    name.startsWith("embedded-agent-runner") ||
-    name.startsWith("cli-runner") ||
     name.startsWith("agent-command") ||
     name.startsWith("command") ||
-    name.includes("compaction") ||
-    name.includes("session")
+    name.includes("compaction")
   ) {
-    return "agentic-agents-core-runner";
+    return "agentic-agents-core-runner-commands";
+  }
+  if (name.startsWith("cli-runner")) {
+    return "agentic-agents-core-runner-cli";
+  }
+  if (name.includes("session")) {
+    return "agentic-agents-core-runner-sessions";
   }
   return "agentic-agents-core-runtime";
 }
@@ -297,7 +422,10 @@ function createAgentCoreSplitShards() {
     "agentic-agents-core-models",
     "agentic-agents-core-tools",
     "agentic-agents-core-subagents",
-    "agentic-agents-core-runner",
+    "agentic-agents-core-runner-cli",
+    "agentic-agents-core-runner-commands",
+    "agentic-agents-core-runner-embedded",
+    "agentic-agents-core-runner-sessions",
     "agentic-agents-core-runtime",
   ]
     .map((shardName) => ({
@@ -1196,62 +1324,31 @@ function createCompactNodeTestShardBundles(options = {}) {
 
   const compactJobs = [];
   for (const groups of groupsByRunner.values()) {
+    // First-fit decreasing on estimated serial seconds keeps every job near
+    // the same runtime; the old per-file weights let one 3-minute group land
+    // next to nine trivial ones and own the PR wall clock.
     const bins = [];
     const sortedGroups = groups.toSorted(
       (a, b) =>
-        (b.includePatterns?.length ?? 1) - (a.includePatterns?.length ?? 1) ||
+        estimateCompactGroupSeconds(b) - estimateCompactGroupSeconds(a) ||
         a.shard_name.localeCompare(b.shard_name),
     );
-    for (const group of sortedGroups.filter((candidate) => candidate.includePatterns)) {
-      const weight = group.includePatterns.length;
+    for (const group of sortedGroups) {
+      const weight = estimateCompactGroupSeconds(group);
       const bin = bins.find(
         (candidate) =>
           candidate.groups.length < COMPACT_NODE_TEST_JOB_GROUPS &&
-          candidate.weight + weight <= COMPACT_NODE_TEST_JOB_WEIGHT,
+          candidate.weight + weight <= COMPACT_NODE_TEST_JOB_SECONDS,
       );
       if (bin) {
         bin.groups.push(group);
         bin.weight += weight;
+        bin.hasWholeConfigGroup ||= !group.includePatterns;
       } else {
-        bins.push({ groups: [group], weight });
+        bins.push({ groups: [group], hasWholeConfigGroup: !group.includePatterns, weight });
       }
     }
 
-    const wholeGroups = sortedGroups.filter((candidate) => !candidate.includePatterns);
-    const wholeJobCount = Math.ceil(wholeGroups.length / COMPACT_WHOLE_NODE_TEST_JOB_GROUPS);
-    // A lone whole-config job serializes every fixed suite and owns PR wall time.
-    // Fold it into same-runner jobs when caps allow, retaining the whole-config timeout.
-    const canSpreadWholeGroups =
-      wholeJobCount === 1 &&
-      bins.length > 1 &&
-      bins.every(
-        (bin) =>
-          bin.groups.length + Math.ceil(wholeGroups.length / bins.length) <=
-          COMPACT_NODE_TEST_JOB_GROUPS,
-      );
-    const wholeGroupBatches = canSpreadWholeGroups
-      ? []
-      : createStripedBatches(wholeGroups, wholeJobCount);
-    if (canSpreadWholeGroups) {
-      for (const [index, group] of wholeGroups.entries()) {
-        const bin = bins[index % bins.length];
-        bin.groups.push(group);
-        bin.timeoutMinutes = COMPACT_WHOLE_NODE_TEST_TIMEOUT_MINUTES;
-      }
-    }
-    for (const [index, groupBatch] of wholeGroupBatches.entries()) {
-      const runnerClass = groupBatch[0].runner.includes("-8vcpu-") ? "large" : "small";
-      const distSuffix = groupBatch[0].requiresDist ? "-dist" : "";
-      compactJobs.push({
-        checkName: `checks-node-compact-${runnerClass}${distSuffix}-whole-${index + 1}`,
-        groups: groupBatch,
-        requiresDist: groupBatch[0].requiresDist,
-        runner: groupBatch[0].runner,
-        shardName: `compact-${runnerClass}${distSuffix}-whole-${index + 1}`,
-        timeoutMinutes: COMPACT_WHOLE_NODE_TEST_TIMEOUT_MINUTES,
-      });
-    }
-
     for (const [index, bin] of bins.entries()) {
       const runnerClass = bin.groups[0].runner.includes("-8vcpu-") ? "large" : "small";
       const distSuffix = bin.groups[0].requiresDist ? "-dist" : "";
@@ -1260,8 +1357,11 @@ function createCompactNodeTestShardBundles(options = {}) {
         groups: bin.groups,
         requiresDist: bin.groups[0].requiresDist,
         runner: bin.groups[0].runner,
-        shardName: `compact-${runnerClass}-${index + 1}`,
-        ...(bin.timeoutMinutes ? { timeoutMinutes: bin.timeoutMinutes } : {}),
+        shardName: `compact-${runnerClass}${distSuffix}-${index + 1}`,
+        // Whole-config groups run entire suites; keep their generous timeout.
+        ...(bin.hasWholeConfigGroup
+          ? { timeoutMinutes: COMPACT_WHOLE_NODE_TEST_TIMEOUT_MINUTES }
+          : {}),
       });
     }
   }