From 5eab61b45df61f8dcd4dc2c791215cd798f08db1 Mon Sep 17 00:00:00 2001
From: Peter Steinberger <steipete@gmail.com>
Date: Wed, 8 Apr 2026 05:28:55 +0100
Subject: [PATCH] test: add opt-in leaf project scheduler

---
 scripts/test-projects.mjs              | 126 +++++++++++++++++++
 scripts/test-projects.test-support.mjs |  16 ++-
 test/scripts/test-projects.test.ts     | 165 ++++++++++++++-----------
 3 files changed, 228 insertions(+), 79 deletions(-)

diff --git a/scripts/test-projects.mjs b/scripts/test-projects.mjs
index 914ecaf24ac..c76d07e55bd 100644
--- a/scripts/test-projects.mjs
+++ b/scripts/test-projects.mjs
@@ -23,6 +23,52 @@ const releaseLock = acquireLocalHeavyCheckLockSync({
 });
 let lockReleased = false;
 
+const FULL_SUITE_CONFIG_WEIGHT = new Map([
+  ["vitest.gateway.config.ts", 180],
+  ["vitest.commands.config.ts", 175],
+  ["vitest.agents.config.ts", 170],
+  ["vitest.extensions.config.ts", 168],
+  ["vitest.tasks.config.ts", 165],
+  ["vitest.unit-fast.config.ts", 160],
+  ["vitest.auto-reply-reply.config.ts", 155],
+  ["vitest.infra.config.ts", 145],
+  ["vitest.secrets.config.ts", 140],
+  ["vitest.cron.config.ts", 135],
+  ["vitest.wizard.config.ts", 130],
+  ["vitest.unit-src.config.ts", 125],
+  ["vitest.extension-channels.config.ts", 100],
+  ["vitest.extension-matrix.config.ts", 98],
+  ["vitest.extension-providers.config.ts", 96],
+  ["vitest.extension-telegram.config.ts", 94],
+  ["vitest.extension-whatsapp.config.ts", 92],
+  ["vitest.auto-reply-core.config.ts", 90],
+  ["vitest.cli.config.ts", 86],
+  ["vitest.channels.config.ts", 84],
+  ["vitest.plugins.config.ts", 82],
+  ["vitest.bundled.config.ts", 80],
+  ["vitest.commands-light.config.ts", 48],
+  ["vitest.plugin-sdk.config.ts", 46],
+  ["vitest.auto-reply-top-level.config.ts", 45],
+  ["vitest.unit-ui.config.ts", 40],
+  ["vitest.plugin-sdk-light.config.ts", 38],
+  ["vitest.daemon.config.ts", 36],
+  ["vitest.boundary.config.ts", 34],
+  ["vitest.tooling.config.ts", 32],
+  ["vitest.unit-security.config.ts", 30],
+  ["vitest.unit-support.config.ts", 28],
+  ["vitest.contracts.config.ts", 26],
+  ["vitest.extension-zalo.config.ts", 24],
+  ["vitest.extension-bluebubbles.config.ts", 22],
+  ["vitest.extension-irc.config.ts", 20],
+  ["vitest.extension-feishu.config.ts", 18],
+  ["vitest.extension-mattermost.config.ts", 16],
+  ["vitest.extension-messaging.config.ts", 14],
+  ["vitest.extension-acpx.config.ts", 10],
+  ["vitest.extension-diffs.config.ts", 8],
+  ["vitest.extension-memory.config.ts", 6],
+  ["vitest.extension-msteams.config.ts", 4],
+  ["vitest.extension-voice-call.config.ts", 2],
+]);
 const releaseLockOnce = () => {
   if (lockReleased) {
     return;
@@ -69,6 +115,66 @@ function runVitestSpec(spec) {
   });
 }
 
+function parsePositiveInt(value) {
+  const parsed = Number.parseInt(value ?? "", 10);
+  return Number.isFinite(parsed) && parsed > 0 ? parsed : null;
+}
+
+function resolveParallelFullSuiteConcurrency(specCount, env) {
+  const override = parsePositiveInt(env.OPENCLAW_TEST_PROJECTS_PARALLEL);
+  if (override !== null) {
+    return Math.min(override, specCount);
+  }
+  if (
+    env.OPENCLAW_TEST_PROJECTS_LEAF_SHARDS !== "1" ||
+    env.CI === "true" ||
+    env.GITHUB_ACTIONS === "true"
+  ) {
+    return 1;
+  }
+  return Math.min(5, specCount);
+}
+
+function orderFullSuiteSpecsForParallelRun(specs) {
+  return specs.toSorted((a, b) => {
+    const weightDelta =
+      (FULL_SUITE_CONFIG_WEIGHT.get(b.config) ?? 0) - (FULL_SUITE_CONFIG_WEIGHT.get(a.config) ?? 0);
+    if (weightDelta !== 0) {
+      return weightDelta;
+    }
+    return a.config.localeCompare(b.config);
+  });
+}
+
+async function runVitestSpecsParallel(specs, concurrency) {
+  let nextIndex = 0;
+  let exitCode = 0;
+
+  const runWorker = async () => {
+    for (;;) {
+      const index = nextIndex;
+      nextIndex += 1;
+      const spec = specs[index];
+      if (!spec) {
+        return;
+      }
+      console.error(`[test] starting ${spec.config}`);
+      const result = await runVitestSpec(spec);
+      if (result.signal) {
+        releaseLockOnce();
+        process.kill(process.pid, result.signal);
+        return;
+      }
+      if (result.code !== 0) {
+        exitCode = exitCode || result.code;
+      }
+    }
+  };
+
+  await Promise.all(Array.from({ length: concurrency }, () => runWorker()));
+  return exitCode;
+}
+
 async function main() {
   const args = process.argv.slice(2);
   const { targetArgs } = parseTestProjectsArgs(args, process.cwd());
@@ -99,6 +205,26 @@ async function main() {
           cwd: process.cwd(),
         });
 
+  const isFullSuiteRun =
+    targetArgs.length === 0 &&
+    changedTargetArgs === null &&
+    !runSpecs.some((spec) => spec.watchMode);
+  if (isFullSuiteRun) {
+    const concurrency = resolveParallelFullSuiteConcurrency(runSpecs.length, process.env);
+    if (concurrency > 1) {
+      const parallelSpecs = orderFullSuiteSpecsForParallelRun(runSpecs);
+      console.error(
+        `[test] running ${parallelSpecs.length} Vitest shards with parallelism ${concurrency}`,
+      );
+      const parallelExitCode = await runVitestSpecsParallel(parallelSpecs, concurrency);
+      releaseLockOnce();
+      if (parallelExitCode !== 0) {
+        process.exit(parallelExitCode);
+      }
+      return;
+    }
+  }
+
   let exitCode = 0;
   for (const spec of runSpecs) {
     const result = await runVitestSpec(spec);
diff --git a/scripts/test-projects.test-support.mjs b/scripts/test-projects.test-support.mjs
index 8665670c56a..f155f41d2e9 100644
--- a/scripts/test-projects.test-support.mjs
+++ b/scripts/test-projects.test-support.mjs
@@ -612,12 +612,16 @@ export function buildFullSuiteVitestRunPlans(args, cwd = process.cwd()) {
       },
     ];
   }
-  return fullSuiteVitestShards.map((shard) => ({
-    config: shard.config,
-    forwardedArgs,
-    includePatterns: null,
-    watchMode: false,
-  }));
+  const expandToProjectConfigs = process.env.OPENCLAW_TEST_PROJECTS_LEAF_SHARDS === "1";
+  return fullSuiteVitestShards.flatMap((shard) => {
+    const configs = expandToProjectConfigs ? shard.projects : [shard.config];
+    return configs.map((config) => ({
+      config,
+      forwardedArgs,
+      includePatterns: null,
+      watchMode: false,
+    }));
+  });
 }
 
 export function createVitestRunSpecs(args, params = {}) {
diff --git a/test/scripts/test-projects.test.ts b/test/scripts/test-projects.test.ts
index d10d1b4e82f..64741dc37a9 100644
--- a/test/scripts/test-projects.test.ts
+++ b/test/scripts/test-projects.test.ts
@@ -178,82 +178,101 @@ describe("scripts/test-projects changed-target routing", () => {
 
 describe("scripts/test-projects full-suite sharding", () => {
   it("splits untargeted runs into fixed shard configs", () => {
-    expect(buildFullSuiteVitestRunPlans([], process.cwd())).toEqual([
-      {
-        config: "vitest.full-core-unit-fast.config.ts",
-        forwardedArgs: [],
-        includePatterns: null,
-        watchMode: false,
-      },
-      {
-        config: "vitest.full-core-unit-src.config.ts",
-        forwardedArgs: [],
-        includePatterns: null,
-        watchMode: false,
-      },
-      {
-        config: "vitest.full-core-unit-security.config.ts",
-        forwardedArgs: [],
-        includePatterns: null,
-        watchMode: false,
-      },
-      {
-        config: "vitest.full-core-unit-ui.config.ts",
-        forwardedArgs: [],
-        includePatterns: null,
-        watchMode: false,
-      },
-      {
-        config: "vitest.full-core-unit-support.config.ts",
-        forwardedArgs: [],
-        includePatterns: null,
-        watchMode: false,
-      },
-      {
-        config: "vitest.full-core-support-boundary.config.ts",
-        forwardedArgs: [],
-        includePatterns: null,
-        watchMode: false,
-      },
-      {
-        config: "vitest.full-core-contracts.config.ts",
-        forwardedArgs: [],
-        includePatterns: null,
-        watchMode: false,
-      },
-      {
-        config: "vitest.full-core-bundled.config.ts",
-        forwardedArgs: [],
-        includePatterns: null,
-        watchMode: false,
-      },
-      {
-        config: "vitest.full-core-runtime.config.ts",
-        forwardedArgs: [],
-        includePatterns: null,
-        watchMode: false,
-      },
-      {
-        config: "vitest.full-agentic.config.ts",
-        forwardedArgs: [],
-        includePatterns: null,
-        watchMode: false,
-      },
-      {
-        config: "vitest.full-auto-reply.config.ts",
-        forwardedArgs: [],
-        includePatterns: null,
-        watchMode: false,
-      },
-      {
-        config: "vitest.full-extensions.config.ts",
-        forwardedArgs: [],
-        includePatterns: null,
-        watchMode: false,
-      },
+    delete process.env.OPENCLAW_TEST_PROJECTS_LEAF_SHARDS;
+
+    expect(buildFullSuiteVitestRunPlans([], process.cwd()).map((plan) => plan.config)).toEqual([
+      "vitest.full-core-unit-fast.config.ts",
+      "vitest.full-core-unit-src.config.ts",
+      "vitest.full-core-unit-security.config.ts",
+      "vitest.full-core-unit-ui.config.ts",
+      "vitest.full-core-unit-support.config.ts",
+      "vitest.full-core-support-boundary.config.ts",
+      "vitest.full-core-contracts.config.ts",
+      "vitest.full-core-bundled.config.ts",
+      "vitest.full-core-runtime.config.ts",
+      "vitest.full-agentic.config.ts",
+      "vitest.full-auto-reply.config.ts",
+      "vitest.full-extensions.config.ts",
     ]);
   });
 
+  it("can expand full-suite shards to project configs for perf experiments", () => {
+    const previous = process.env.OPENCLAW_TEST_PROJECTS_LEAF_SHARDS;
+    process.env.OPENCLAW_TEST_PROJECTS_LEAF_SHARDS = "1";
+    const plans = buildFullSuiteVitestRunPlans([], process.cwd());
+    if (previous === undefined) {
+      delete process.env.OPENCLAW_TEST_PROJECTS_LEAF_SHARDS;
+    } else {
+      process.env.OPENCLAW_TEST_PROJECTS_LEAF_SHARDS = previous;
+    }
+
+    expect(plans.map((plan) => plan.config)).toEqual([
+      "vitest.unit-fast.config.ts",
+      "vitest.unit-src.config.ts",
+      "vitest.unit-security.config.ts",
+      "vitest.unit-ui.config.ts",
+      "vitest.unit-support.config.ts",
+      "vitest.boundary.config.ts",
+      "vitest.tooling.config.ts",
+      "vitest.contracts.config.ts",
+      "vitest.bundled.config.ts",
+      "vitest.infra.config.ts",
+      "vitest.hooks.config.ts",
+      "vitest.acp.config.ts",
+      "vitest.runtime-config.config.ts",
+      "vitest.secrets.config.ts",
+      "vitest.logging.config.ts",
+      "vitest.process.config.ts",
+      "vitest.cron.config.ts",
+      "vitest.media.config.ts",
+      "vitest.media-understanding.config.ts",
+      "vitest.shared-core.config.ts",
+      "vitest.tasks.config.ts",
+      "vitest.tui.config.ts",
+      "vitest.ui.config.ts",
+      "vitest.utils.config.ts",
+      "vitest.wizard.config.ts",
+      "vitest.gateway.config.ts",
+      "vitest.cli.config.ts",
+      "vitest.commands-light.config.ts",
+      "vitest.commands.config.ts",
+      "vitest.agents.config.ts",
+      "vitest.daemon.config.ts",
+      "vitest.plugin-sdk-light.config.ts",
+      "vitest.plugin-sdk.config.ts",
+      "vitest.plugins.config.ts",
+      "vitest.channels.config.ts",
+      "vitest.auto-reply-core.config.ts",
+      "vitest.auto-reply-top-level.config.ts",
+      "vitest.auto-reply-reply.config.ts",
+      "vitest.extension-acpx.config.ts",
+      "vitest.extension-bluebubbles.config.ts",
+      "vitest.extension-channels.config.ts",
+      "vitest.extension-diffs.config.ts",
+      "vitest.extension-feishu.config.ts",
+      "vitest.extension-irc.config.ts",
+      "vitest.extension-mattermost.config.ts",
+      "vitest.extension-matrix.config.ts",
+      "vitest.extension-memory.config.ts",
+      "vitest.extension-messaging.config.ts",
+      "vitest.extension-msteams.config.ts",
+      "vitest.extension-providers.config.ts",
+      "vitest.extension-telegram.config.ts",
+      "vitest.extension-voice-call.config.ts",
+      "vitest.extension-whatsapp.config.ts",
+      "vitest.extension-zalo.config.ts",
+      "vitest.extensions.config.ts",
+    ]);
+    expect(plans).toEqual(
+      plans.map((plan) => ({
+        config: plan.config,
+        forwardedArgs: [],
+        includePatterns: null,
+        watchMode: false,
+      })),
+    );
+  });
+
   it("keeps untargeted watch mode on the native root config", () => {
     expect(buildFullSuiteVitestRunPlans(["--watch"], process.cwd())).toEqual([
       {