ci: split auto-reply shard timing

scripts/ci-run-timings.mjs

@@ -18,6 +18,11 @@ function formatSeconds(value) {
   return value === null ? "" : `${value}s`;
 }
 
+function parseRunList(raw) {
+  const parsed = JSON.parse(raw);
+  return Array.isArray(parsed) ? parsed : [];
+}
+
 function collectRunTimingContext(run) {
   const created = parseTime(run.createdAt);
   const updated = parseTime(run.updatedAt);
@@ -64,6 +69,17 @@ export function summarizeRunTimings(run, limit = 15) {
   };
 }
 
+export function selectLatestMainPushCiRun(runs, headSha = null) {
+  const pushRuns = runs.filter((run) => run.event === "push");
+  if (headSha) {
+    const matchingRun = pushRuns.find((run) => run.headSha === headSha);
+    if (matchingRun) {
+      return matchingRun;
+    }
+  }
+  return pushRuns[0] ?? null;
+}
+
 function getLatestCiRunId() {
   const raw = execFileSync(
     "gh",
@@ -78,6 +94,40 @@ function getLatestCiRunId() {
   return String(runId);
 }
 
+function getRemoteMainSha() {
+  const raw = execFileSync("git", ["ls-remote", "origin", "main"], { encoding: "utf8" }).trim();
+  const [sha] = raw.split(/\s+/u);
+  if (!sha) {
+    throw new Error("Could not resolve origin/main");
+  }
+  return sha;
+}
+
+function getLatestMainPushCiRunId() {
+  const headSha = getRemoteMainSha();
+  const raw = execFileSync(
+    "gh",
+    [
+      "run",
+      "list",
+      "--branch",
+      "main",
+      "--workflow",
+      "CI",
+      "--limit",
+      "20",
+      "--json",
+      "databaseId,headSha,event,status,conclusion",
+    ],
+    { encoding: "utf8" },
+  );
+  const run = selectLatestMainPushCiRun(parseRunList(raw), headSha);
+  if (!run?.databaseId) {
+    throw new Error(`No push CI run found for origin/main ${headSha.slice(0, 10)}`);
+  }
+  return String(run.databaseId);
+}
+
 function listRecentSuccessfulCiRuns(limit) {
   const raw = execFileSync(
     "gh",
@@ -161,11 +211,15 @@ function printSection(title, jobs, metric) {
   }
 }
 
-async function main() {
-  const args = process.argv.slice(2);
+export function parseRunTimingArgs(args) {
   const recentIndex = args.indexOf("--recent");
   const limitIndex = args.indexOf("--limit");
   const ignoredArgIndexes = new Set();
+  for (const [index, arg] of args.entries()) {
+    if (arg === "--" || arg === "--latest-main") {
+      ignoredArgIndexes.add(index);
+    }
+  }
   if (limitIndex !== -1) {
     ignoredArgIndexes.add(limitIndex);
     ignoredArgIndexes.add(limitIndex + 1);
@@ -176,8 +230,21 @@ async function main() {
   }
   const limit =
     limitIndex === -1 ? 15 : Math.max(1, Number.parseInt(args[limitIndex + 1] ?? "", 10) || 15);
-  if (recentIndex !== -1) {
-    const recentLimit = Math.max(1, Number.parseInt(args[recentIndex + 1] ?? "", 10) || 10);
+  const recentLimit =
+    recentIndex === -1 ? null : Math.max(1, Number.parseInt(args[recentIndex + 1] ?? "", 10) || 10);
+  return {
+    explicitRunId: args.find((_arg, index) => !ignoredArgIndexes.has(index)),
+    limit,
+    recentLimit,
+    useLatestMain: args.includes("--latest-main"),
+  };
+}
+
+async function main() {
+  const { explicitRunId, limit, recentLimit, useLatestMain } = parseRunTimingArgs(
+    process.argv.slice(2),
+  );
+  if (recentLimit !== null) {
     for (const run of listRecentSuccessfulCiRuns(recentLimit)) {
       const summary = summarizeJobs(loadRun(run.databaseId));
       console.log(
@@ -197,7 +264,7 @@ async function main() {
     }
     return;
   }
-  const runId = args.find((_arg, index) => !ignoredArgIndexes.has(index)) ?? getLatestCiRunId();
+  const runId = explicitRunId ?? (useLatestMain ? getLatestMainPushCiRunId() : getLatestCiRunId());
   const summary = summarizeRunTimings(loadRun(runId), limit);
 
   console.log(

scripts/lib/ci-node-test-plan.mjs

@@ -66,10 +66,8 @@ function createAutoReplyReplySplitShards() {
   }
 
   const mergedGroups = {
-    "auto-reply-reply-agent-dispatch": [
-      ...groups["auto-reply-reply-agent-runner"],
-      ...groups["auto-reply-reply-dispatch"],
-    ],
+    "auto-reply-reply-agent-runner": groups["auto-reply-reply-agent-runner"],
+    "auto-reply-reply-dispatch": groups["auto-reply-reply-dispatch"],
     "auto-reply-reply-commands-state-routing": [
       ...groups["auto-reply-reply-commands"],
       ...groups["auto-reply-reply-state-routing"],

scripts/lib/vitest-shard-timings.mjs

@@ -0,0 +1,126 @@
+import { createHash } from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+
+const TIMINGS_FILE_ENV_KEY = "OPENCLAW_TEST_PROJECTS_TIMINGS_PATH";
+const TIMINGS_DISABLE_ENV_KEY = "OPENCLAW_TEST_PROJECTS_TIMINGS";
+const SHARD_NAME_ENV_KEY = "OPENCLAW_VITEST_SHARD_NAME";
+
+function sanitizeTimingLabel(value) {
+  return String(value)
+    .trim()
+    .replace(/[^a-zA-Z0-9_.-]+/g, "-")
+    .replace(/^-+|-+$/g, "");
+}
+
+function hashIncludePatterns(includePatterns) {
+  return createHash("sha1").update(JSON.stringify(includePatterns)).digest("hex").slice(0, 12);
+}
+
+export function shouldUseShardTimings(env = process.env) {
+  return env[TIMINGS_DISABLE_ENV_KEY] !== "0";
+}
+
+export function resolveShardTimingsPath(cwd = process.cwd(), env = process.env) {
+  return env[TIMINGS_FILE_ENV_KEY] || path.join(cwd, ".artifacts", "vitest-shard-timings.json");
+}
+
+export function resolveShardTimingKey(spec) {
+  if (!Array.isArray(spec.includePatterns) || spec.includePatterns.length === 0) {
+    return spec.config;
+  }
+
+  const shardName = sanitizeTimingLabel(spec.env?.[SHARD_NAME_ENV_KEY] ?? "");
+  if (shardName) {
+    return `${spec.config}#${shardName}`;
+  }
+
+  return `${spec.config}#include-${spec.includePatterns.length}-${hashIncludePatterns(
+    spec.includePatterns,
+  )}`;
+}
+
+export function createShardTimingSample(spec, durationMs) {
+  if (spec.watchMode || !Number.isFinite(durationMs) || durationMs <= 0) {
+    return null;
+  }
+
+  const includePatternCount = Array.isArray(spec.includePatterns) ? spec.includePatterns.length : 0;
+  return {
+    baseConfig: spec.config,
+    config: resolveShardTimingKey(spec),
+    durationMs,
+    includePatternCount,
+  };
+}
+
+export function readShardTimings(cwd = process.cwd(), env = process.env) {
+  if (!shouldUseShardTimings(env)) {
+    return new Map();
+  }
+  try {
+    const raw = fs.readFileSync(resolveShardTimingsPath(cwd, env), "utf8");
+    const parsed = JSON.parse(raw);
+    const configs = parsed && typeof parsed === "object" ? parsed.configs : null;
+    if (!configs || typeof configs !== "object") {
+      return new Map();
+    }
+    return new Map(
+      Object.entries(configs)
+        .map(([config, value]) => {
+          const durationMs = Number(value?.averageMs ?? value?.durationMs);
+          return Number.isFinite(durationMs) && durationMs > 0 ? [config, durationMs] : null;
+        })
+        .filter(Boolean),
+    );
+  } catch {
+    return new Map();
+  }
+}
+
+export function writeShardTimings(samples, cwd = process.cwd(), env = process.env) {
+  if (!shouldUseShardTimings(env) || samples.length === 0) {
+    return;
+  }
+
+  const outputPath = resolveShardTimingsPath(cwd, env);
+  let current = { version: 1, configs: {} };
+  try {
+    current = JSON.parse(fs.readFileSync(outputPath, "utf8"));
+  } catch {
+    // First run, or a corrupt local artifact. Rewrite below.
+  }
+
+  const configs =
+    current && typeof current === "object" && current.configs && typeof current.configs === "object"
+      ? { ...current.configs }
+      : {};
+  const updatedAt = new Date().toISOString();
+  for (const sample of samples) {
+    if (!sample.config || !Number.isFinite(sample.durationMs) || sample.durationMs <= 0) {
+      continue;
+    }
+    const previous = configs[sample.config];
+    const previousAverage = Number(previous?.averageMs ?? previous?.durationMs);
+    const sampleCount = Math.max(0, Number(previous?.sampleCount) || 0) + 1;
+    const averageMs =
+      Number.isFinite(previousAverage) && previousAverage > 0
+        ? Math.round(previousAverage * 0.7 + sample.durationMs * 0.3)
+        : Math.round(sample.durationMs);
+    configs[sample.config] = {
+      averageMs,
+      lastMs: Math.round(sample.durationMs),
+      sampleCount,
+      updatedAt,
+      ...(sample.baseConfig && sample.baseConfig !== sample.config
+        ? { baseConfig: sample.baseConfig }
+        : {}),
+      ...(sample.includePatternCount ? { includePatternCount: sample.includePatternCount } : {}),
+    };
+  }
+
+  fs.mkdirSync(path.dirname(outputPath), { recursive: true });
+  const tempPath = `${outputPath}.${process.pid}.tmp`;
+  fs.writeFileSync(tempPath, `${JSON.stringify({ version: 1, configs }, null, 2)}\n`, "utf8");
+  fs.renameSync(tempPath, outputPath);
+}

scripts/test-projects.mjs

@@ -1,5 +1,4 @@
 import fs from "node:fs";
-import path from "node:path";
 import { performance } from "node:perf_hooks";
 import { acquireLocalHeavyCheckLockSync } from "./lib/local-heavy-check-runtime.mjs";
 import {
@@ -7,6 +6,11 @@ import {
   resolveLocalFullSuiteProfile,
   resolveLocalVitestEnv,
 } from "./lib/vitest-local-scheduling.mjs";
+import {
+  createShardTimingSample,
+  readShardTimings,
+  writeShardTimings,
+} from "./lib/vitest-shard-timings.mjs";
 import {
   resolveVitestCliEntry,
   resolveVitestNodeArgs,
@@ -94,8 +98,6 @@ const FULL_SUITE_CONFIG_WEIGHT = new Map([
   ["test/vitest/vitest.extension-memory.config.ts", 6],
   ["test/vitest/vitest.extension-msteams.config.ts", 4],
 ]);
-const TIMINGS_FILE_ENV_KEY = "OPENCLAW_TEST_PROJECTS_TIMINGS_PATH";
-const TIMINGS_DISABLE_ENV_KEY = "OPENCLAW_TEST_PROJECTS_TIMINGS";
 const releaseLockOnce = () => {
   if (lockReleased) {
     return;
@@ -104,81 +106,6 @@ const releaseLockOnce = () => {
   releaseLock();
 };
 
-function shouldUseShardTimings(env = process.env) {
-  return env[TIMINGS_DISABLE_ENV_KEY] !== "0";
-}
-
-function resolveShardTimingsPath(cwd = process.cwd(), env = process.env) {
-  return env[TIMINGS_FILE_ENV_KEY] || path.join(cwd, ".artifacts", "vitest-shard-timings.json");
-}
-
-function readShardTimings(cwd = process.cwd(), env = process.env) {
-  if (!shouldUseShardTimings(env)) {
-    return new Map();
-  }
-  try {
-    const raw = fs.readFileSync(resolveShardTimingsPath(cwd, env), "utf8");
-    const parsed = JSON.parse(raw);
-    const configs = parsed && typeof parsed === "object" ? parsed.configs : null;
-    if (!configs || typeof configs !== "object") {
-      return new Map();
-    }
-    return new Map(
-      Object.entries(configs)
-        .map(([config, value]) => {
-          const durationMs = Number(value?.averageMs ?? value?.durationMs);
-          return Number.isFinite(durationMs) && durationMs > 0 ? [config, durationMs] : null;
-        })
-        .filter(Boolean),
-    );
-  } catch {
-    return new Map();
-  }
-}
-
-function writeShardTimings(samples, cwd = process.cwd(), env = process.env) {
-  if (!shouldUseShardTimings(env) || samples.length === 0) {
-    return;
-  }
-
-  const outputPath = resolveShardTimingsPath(cwd, env);
-  let current = { version: 1, configs: {} };
-  try {
-    current = JSON.parse(fs.readFileSync(outputPath, "utf8"));
-  } catch {
-    // First run, or a corrupt local artifact. Rewrite below.
-  }
-
-  const configs =
-    current && typeof current === "object" && current.configs && typeof current.configs === "object"
-      ? { ...current.configs }
-      : {};
-  const updatedAt = new Date().toISOString();
-  for (const sample of samples) {
-    if (!sample.config || !Number.isFinite(sample.durationMs) || sample.durationMs <= 0) {
-      continue;
-    }
-    const previous = configs[sample.config];
-    const previousAverage = Number(previous?.averageMs ?? previous?.durationMs);
-    const sampleCount = Math.max(0, Number(previous?.sampleCount) || 0) + 1;
-    const averageMs =
-      Number.isFinite(previousAverage) && previousAverage > 0
-        ? Math.round(previousAverage * 0.7 + sample.durationMs * 0.3)
-        : Math.round(sample.durationMs);
-    configs[sample.config] = {
-      averageMs,
-      lastMs: Math.round(sample.durationMs),
-      sampleCount,
-      updatedAt,
-    };
-  }
-
-  fs.mkdirSync(path.dirname(outputPath), { recursive: true });
-  const tempPath = `${outputPath}.${process.pid}.tmp`;
-  fs.writeFileSync(tempPath, `${JSON.stringify({ version: 1, configs }, null, 2)}\n`, "utf8");
-  fs.renameSync(tempPath, outputPath);
-}
-
 function cleanupVitestRunSpec(spec) {
   if (!spec.includeFilePath) {
     return;
@@ -263,8 +190,7 @@ async function runLoggedVitestSpec(spec) {
   }
   return {
     ...result,
-    timing:
-      !spec.watchMode && spec.includePatterns === null ? { config: spec.config, durationMs } : null,
+    timing: createShardTimingSample(spec, durationMs),
   };
 }
 
@@ -288,6 +214,7 @@ function interleaveSlowAndFastSpecs(sortedSpecs) {
 }
 
 function orderFullSuiteSpecsForParallelRun(specs, shardTimings = new Map()) {
+  const hasMatchingShardTiming = specs.some((spec) => shardTimings.has(spec.config));
   const sortedSpecs = specs.toSorted((a, b) => {
     const weightDelta =
       resolveConfigSortWeight(b.config, shardTimings) -
@@ -297,7 +224,7 @@ function orderFullSuiteSpecsForParallelRun(specs, shardTimings = new Map()) {
     }
     return a.config.localeCompare(b.config);
   });
-  return shardTimings.size > 0 ? interleaveSlowAndFastSpecs(sortedSpecs) : sortedSpecs;
+  return hasMatchingShardTiming ? interleaveSlowAndFastSpecs(sortedSpecs) : sortedSpecs;
 }
 
 function isFullExtensionsProjectRun(specs) {

scripts/test-projects.test-support.mjs

@@ -237,9 +237,12 @@ const TOOLING_SOURCE_TEST_TARGETS = new Map([
     ],
   ],
   ["scripts/run-oxlint.mjs", ["test/scripts/run-oxlint.test.ts"]],
+  ["scripts/ci-run-timings.mjs", ["test/scripts/ci-run-timings.test.ts"]],
   ["scripts/test-extension-batch.mjs", ["test/scripts/test-extension.test.ts"]],
   ["scripts/lib/extension-test-plan.mjs", ["test/scripts/test-extension.test.ts"]],
   ["scripts/lib/vitest-batch-runner.mjs", ["test/scripts/test-extension.test.ts"]],
+  ["scripts/lib/ci-node-test-plan.mjs", ["test/scripts/ci-node-test-plan.test.ts"]],
+  ["scripts/lib/vitest-shard-timings.mjs", ["test/scripts/vitest-shard-timings.test.ts"]],
   ["scripts/test-projects.mjs", ["test/scripts/test-projects.test.ts"]],
   ["scripts/test-projects.test-support.d.mts", ["test/scripts/test-projects.test.ts"]],
   ["scripts/test-projects.test-support.mjs", ["test/scripts/test-projects.test.ts"]],