test: interleave cold full-suite shards

scripts/test-projects.mjs

@@ -25,6 +25,7 @@ import {
   buildFullSuiteVitestRunPlans,
   createVitestRunSpecs,
   listFullExtensionVitestProjectConfigs,
+  orderFullSuiteSpecsForParallelRun,
   parseTestProjectsArgs,
   resolveParallelFullSuiteConcurrency,
   resolveChangedTargetArgs,
@@ -38,70 +39,6 @@ import {
 let releaseLock = () => {};
 let lockReleased = false;
 
-const FULL_SUITE_CONFIG_WEIGHT = new Map([
-  ["test/vitest/vitest.gateway.config.ts", 180],
-  ["test/vitest/vitest.gateway-server.config.ts", 180],
-  ["test/vitest/vitest.gateway-core.config.ts", 179],
-  ["test/vitest/vitest.gateway-client.config.ts", 178],
-  ["test/vitest/vitest.gateway-methods.config.ts", 177],
-  ["test/vitest/vitest.commands.config.ts", 175],
-  ["test/vitest/vitest.agents-core.config.ts", 170],
-  ["test/vitest/vitest.agents-pi-embedded.config.ts", 169],
-  ["test/vitest/vitest.agents-support.config.ts", 168],
-  ["test/vitest/vitest.agents-tools.config.ts", 167],
-  ["test/vitest/vitest.extension-voice-call.config.ts", 169],
-  ["test/vitest/vitest.extensions.config.ts", 168],
-  ["test/vitest/vitest.extension-provider-openai.config.ts", 167],
-  ["test/vitest/vitest.runtime-config.config.ts", 166],
-  ["test/vitest/vitest.contracts-channel-config.config.ts", 85],
-  ["test/vitest/vitest.contracts-channel-surface.config.ts", 60],
-  ["test/vitest/vitest.contracts-channel-session.config.ts", 50],
-  ["test/vitest/vitest.contracts-channel-registry.config.ts", 35],
-  ["test/vitest/vitest.contracts-plugin.config.ts", 20],
-  ["test/vitest/vitest.tasks.config.ts", 165],
-  ["test/vitest/vitest.channels.config.ts", 164],
-  ["test/vitest/vitest.unit-fast.config.ts", 160],
-  ["test/vitest/vitest.auto-reply-reply.config.ts", 155],
-  ["test/vitest/vitest.infra.config.ts", 145],
-  ["test/vitest/vitest.secrets.config.ts", 140],
-  ["test/vitest/vitest.cron.config.ts", 135],
-  ["test/vitest/vitest.wizard.config.ts", 130],
-  ["test/vitest/vitest.unit-src.config.ts", 125],
-  ["test/vitest/vitest.extension-matrix.config.ts", 100],
-  ["test/vitest/vitest.extension-discord.config.ts", 98],
-  ["test/vitest/vitest.extension-providers.config.ts", 96],
-  ["test/vitest/vitest.extension-telegram.config.ts", 94],
-  ["test/vitest/vitest.extension-whatsapp.config.ts", 92],
-  ["test/vitest/vitest.auto-reply-core.config.ts", 90],
-  ["test/vitest/vitest.cli.config.ts", 86],
-  ["test/vitest/vitest.media.config.ts", 84],
-  ["test/vitest/vitest.plugins.config.ts", 82],
-  ["test/vitest/vitest.bundled.config.ts", 80],
-  ["test/vitest/vitest.extension-slack.config.ts", 78],
-  ["test/vitest/vitest.commands-light.config.ts", 48],
-  ["test/vitest/vitest.plugin-sdk.config.ts", 46],
-  ["test/vitest/vitest.auto-reply-top-level.config.ts", 45],
-  ["test/vitest/vitest.unit-ui.config.ts", 40],
-  ["test/vitest/vitest.plugin-sdk-light.config.ts", 38],
-  ["test/vitest/vitest.daemon.config.ts", 36],
-  ["test/vitest/vitest.boundary.config.ts", 34],
-  ["test/vitest/vitest.tooling.config.ts", 32],
-  ["test/vitest/vitest.unit-security.config.ts", 30],
-  ["test/vitest/vitest.unit-support.config.ts", 28],
-  ["test/vitest/vitest.extension-zalo.config.ts", 24],
-  ["test/vitest/vitest.extension-bluebubbles.config.ts", 22],
-  ["test/vitest/vitest.extension-irc.config.ts", 20],
-  ["test/vitest/vitest.extension-feishu.config.ts", 18],
-  ["test/vitest/vitest.extension-mattermost.config.ts", 16],
-  ["test/vitest/vitest.extension-messaging.config.ts", 14],
-  ["test/vitest/vitest.extension-imessage.config.ts", 13],
-  ["test/vitest/vitest.extension-line.config.ts", 12],
-  ["test/vitest/vitest.extension-signal.config.ts", 11],
-  ["test/vitest/vitest.extension-acpx.config.ts", 10],
-  ["test/vitest/vitest.extension-diffs.config.ts", 8],
-  ["test/vitest/vitest.extension-memory.config.ts", 6],
-  ["test/vitest/vitest.extension-msteams.config.ts", 4],
-]);
 const releaseLockOnce = () => {
   if (lockReleased) {
     return;
@@ -198,39 +135,6 @@ async function runLoggedVitestSpec(spec) {
   };
 }
 
-function resolveConfigSortWeight(config, shardTimings) {
-  return shardTimings.get(config) ?? (FULL_SUITE_CONFIG_WEIGHT.get(config) ?? 0) * 1000;
-}
-
-function interleaveSlowAndFastSpecs(sortedSpecs) {
-  const ordered = [];
-  let slowIndex = 0;
-  let fastIndex = sortedSpecs.length - 1;
-  while (slowIndex <= fastIndex) {
-    ordered.push(sortedSpecs[slowIndex]);
-    slowIndex += 1;
-    if (slowIndex <= fastIndex) {
-      ordered.push(sortedSpecs[fastIndex]);
-      fastIndex -= 1;
-    }
-  }
-  return ordered;
-}
-
-function orderFullSuiteSpecsForParallelRun(specs, shardTimings = new Map()) {
-  const hasMatchingShardTiming = specs.some((spec) => shardTimings.has(spec.config));
-  const sortedSpecs = specs.toSorted((a, b) => {
-    const weightDelta =
-      resolveConfigSortWeight(b.config, shardTimings) -
-      resolveConfigSortWeight(a.config, shardTimings);
-    if (weightDelta !== 0) {
-      return weightDelta;
-    }
-    return a.config.localeCompare(b.config);
-  });
-  return hasMatchingShardTiming ? interleaveSlowAndFastSpecs(sortedSpecs) : sortedSpecs;
-}
-
 function isFullExtensionsProjectRun(specs) {
   const fullExtensionProjectConfigs = new Set(listFullExtensionVitestProjectConfigs());
   return (

scripts/test-projects.test-support.mjs

@@ -118,6 +118,103 @@ const UNIT_SECURITY_VITEST_CONFIG = "test/vitest/vitest.unit-security.config.ts"
 const UNIT_SRC_VITEST_CONFIG = "test/vitest/vitest.unit-src.config.ts";
 const UNIT_SUPPORT_VITEST_CONFIG = "test/vitest/vitest.unit-support.config.ts";
 const UNIT_UI_VITEST_CONFIG = "test/vitest/vitest.unit-ui.config.ts";
+
+const FULL_SUITE_CONFIG_WEIGHT = new Map([
+  [GATEWAY_VITEST_CONFIG, 180],
+  [GATEWAY_SERVER_VITEST_CONFIG, 180],
+  [GATEWAY_CORE_VITEST_CONFIG, 179],
+  [GATEWAY_CLIENT_VITEST_CONFIG, 178],
+  [GATEWAY_METHODS_VITEST_CONFIG, 177],
+  [COMMANDS_VITEST_CONFIG, 175],
+  ["test/vitest/vitest.agents-core.config.ts", 170],
+  ["test/vitest/vitest.agents-pi-embedded.config.ts", 169],
+  ["test/vitest/vitest.agents-support.config.ts", 168],
+  ["test/vitest/vitest.agents-tools.config.ts", 167],
+  [EXTENSION_VOICE_CALL_VITEST_CONFIG, 169],
+  [EXTENSIONS_VITEST_CONFIG, 168],
+  [EXTENSION_PROVIDER_OPENAI_VITEST_CONFIG, 167],
+  ["test/vitest/vitest.runtime-config.config.ts", 166],
+  [CONTRACTS_CHANNEL_CONFIG_VITEST_CONFIG, 85],
+  [CONTRACTS_CHANNEL_SURFACE_VITEST_CONFIG, 60],
+  [CONTRACTS_CHANNEL_SESSION_VITEST_CONFIG, 50],
+  [CONTRACTS_CHANNEL_REGISTRY_VITEST_CONFIG, 35],
+  [CONTRACTS_PLUGIN_VITEST_CONFIG, 20],
+  ["test/vitest/vitest.tasks.config.ts", 165],
+  [CHANNEL_VITEST_CONFIG, 164],
+  [UNIT_FAST_VITEST_CONFIG, 160],
+  [AUTO_REPLY_REPLY_VITEST_CONFIG, 155],
+  [INFRA_VITEST_CONFIG, 145],
+  ["test/vitest/vitest.secrets.config.ts", 140],
+  [CRON_VITEST_CONFIG, 135],
+  ["test/vitest/vitest.wizard.config.ts", 130],
+  [UNIT_SRC_VITEST_CONFIG, 125],
+  [EXTENSION_MATRIX_VITEST_CONFIG, 100],
+  [EXTENSION_DISCORD_VITEST_CONFIG, 98],
+  [EXTENSION_PROVIDERS_VITEST_CONFIG, 96],
+  [EXTENSION_TELEGRAM_VITEST_CONFIG, 94],
+  [EXTENSION_WHATSAPP_VITEST_CONFIG, 92],
+  [AUTO_REPLY_CORE_VITEST_CONFIG, 90],
+  [CLI_VITEST_CONFIG, 86],
+  [MEDIA_VITEST_CONFIG, 84],
+  [PLUGINS_VITEST_CONFIG, 82],
+  [BUNDLED_VITEST_CONFIG, 80],
+  [EXTENSION_SLACK_VITEST_CONFIG, 78],
+  [COMMANDS_LIGHT_VITEST_CONFIG, 48],
+  [PLUGIN_SDK_VITEST_CONFIG, 46],
+  [AUTO_REPLY_TOP_LEVEL_VITEST_CONFIG, 45],
+  [UNIT_UI_VITEST_CONFIG, 40],
+  [PLUGIN_SDK_LIGHT_VITEST_CONFIG, 38],
+  [DAEMON_VITEST_CONFIG, 36],
+  [BOUNDARY_VITEST_CONFIG, 34],
+  ["test/vitest/vitest.tooling.config.ts", 32],
+  [UNIT_SECURITY_VITEST_CONFIG, 30],
+  [UNIT_SUPPORT_VITEST_CONFIG, 28],
+  [EXTENSION_ZALO_VITEST_CONFIG, 24],
+  [EXTENSION_BLUEBUBBLES_VITEST_CONFIG, 22],
+  [EXTENSION_IRC_VITEST_CONFIG, 20],
+  [EXTENSION_FEISHU_VITEST_CONFIG, 18],
+  [EXTENSION_MATTERMOST_VITEST_CONFIG, 16],
+  [EXTENSION_MESSAGING_VITEST_CONFIG, 14],
+  [EXTENSION_IMESSAGE_VITEST_CONFIG, 13],
+  [EXTENSION_LINE_VITEST_CONFIG, 12],
+  [EXTENSION_SIGNAL_VITEST_CONFIG, 11],
+  [EXTENSION_ACPX_VITEST_CONFIG, 10],
+  [EXTENSION_DIFFS_VITEST_CONFIG, 8],
+  [EXTENSION_MEMORY_VITEST_CONFIG, 6],
+  [EXTENSION_MSTEAMS_VITEST_CONFIG, 4],
+]);
+
+function resolveConfigSortWeight(config, shardTimings) {
+  return shardTimings.get(config) ?? (FULL_SUITE_CONFIG_WEIGHT.get(config) ?? 0) * 1000;
+}
+
+function interleaveSlowAndFastSpecs(sortedSpecs) {
+  const ordered = [];
+  let slowIndex = 0;
+  let fastIndex = sortedSpecs.length - 1;
+  while (slowIndex <= fastIndex) {
+    ordered.push(sortedSpecs[slowIndex]);
+    slowIndex += 1;
+    if (slowIndex <= fastIndex) {
+      ordered.push(sortedSpecs[fastIndex]);
+      fastIndex -= 1;
+    }
+  }
+  return ordered;
+}
+
+export function orderFullSuiteSpecsForParallelRun(specs, shardTimings = new Map()) {
+  const sortedSpecs = specs.toSorted((a, b) => {
+    const weightDelta =
+      resolveConfigSortWeight(b.config, shardTimings) -
+      resolveConfigSortWeight(a.config, shardTimings);
+    if (weightDelta !== 0) {
+      return weightDelta;
+    }
+    return a.config.localeCompare(b.config);
+  });
+  return interleaveSlowAndFastSpecs(sortedSpecs);
+}
 const PROCESS_VITEST_CONFIG = "test/vitest/vitest.process.config.ts";
 const RUNTIME_CONFIG_VITEST_CONFIG = "test/vitest/vitest.runtime-config.config.ts";
 const SECRETS_VITEST_CONFIG = "test/vitest/vitest.secrets.config.ts";

test/scripts/test-projects.test.ts

@@ -9,6 +9,7 @@ import {
   buildFullSuiteVitestRunPlans,
   buildVitestRunPlans,
   listFullExtensionVitestProjectConfigs,
+  orderFullSuiteSpecsForParallelRun,
   shouldAcquireLocalHeavyCheckLock,
   resolveChangedTestTargetPlan,
   resolveChangedTargetArgs,
@@ -933,6 +934,24 @@ describe("scripts/test-projects local heavy-check lock", () => {
 });
 
 describe("scripts/test-projects full-suite sharding", () => {
+  it("interleaves heavy and light configs for cold parallel full-suite runs", () => {
+    const specs = [
+      "test/vitest/vitest.gateway.config.ts",
+      "test/vitest/vitest.gateway-server.config.ts",
+      "test/vitest/vitest.commands.config.ts",
+      "test/vitest/vitest.extension-memory.config.ts",
+      "test/vitest/vitest.extension-msteams.config.ts",
+    ].map((config) => ({ config }));
+
+    expect(orderFullSuiteSpecsForParallelRun(specs).map((spec) => spec.config)).toEqual([
+      "test/vitest/vitest.gateway-server.config.ts",
+      "test/vitest/vitest.extension-msteams.config.ts",
+      "test/vitest/vitest.gateway.config.ts",
+      "test/vitest/vitest.extension-memory.config.ts",
+      "test/vitest/vitest.commands.config.ts",
+    ]);
+  });
+
   it("covers each normal full-suite test file exactly once", async () => {
     const matches = await listFullSuiteTestFileMatches();
     const e2eNamedIntegrationTests = new Set([