diff --git a/scripts/test-projects.mjs b/scripts/test-projects.mjs index 4fdab27f432..a062f544483 100644 --- a/scripts/test-projects.mjs +++ b/scripts/test-projects.mjs @@ -25,6 +25,7 @@ import { buildFullSuiteVitestRunPlans, createVitestRunSpecs, listFullExtensionVitestProjectConfigs, + orderFullSuiteSpecsForParallelRun, parseTestProjectsArgs, resolveParallelFullSuiteConcurrency, resolveChangedTargetArgs, @@ -38,70 +39,6 @@ import { let releaseLock = () => {}; let lockReleased = false; -const FULL_SUITE_CONFIG_WEIGHT = new Map([ - ["test/vitest/vitest.gateway.config.ts", 180], - ["test/vitest/vitest.gateway-server.config.ts", 180], - ["test/vitest/vitest.gateway-core.config.ts", 179], - ["test/vitest/vitest.gateway-client.config.ts", 178], - ["test/vitest/vitest.gateway-methods.config.ts", 177], - ["test/vitest/vitest.commands.config.ts", 175], - ["test/vitest/vitest.agents-core.config.ts", 170], - ["test/vitest/vitest.agents-pi-embedded.config.ts", 169], - ["test/vitest/vitest.agents-support.config.ts", 168], - ["test/vitest/vitest.agents-tools.config.ts", 167], - ["test/vitest/vitest.extension-voice-call.config.ts", 169], - ["test/vitest/vitest.extensions.config.ts", 168], - ["test/vitest/vitest.extension-provider-openai.config.ts", 167], - ["test/vitest/vitest.runtime-config.config.ts", 166], - ["test/vitest/vitest.contracts-channel-config.config.ts", 85], - ["test/vitest/vitest.contracts-channel-surface.config.ts", 60], - ["test/vitest/vitest.contracts-channel-session.config.ts", 50], - ["test/vitest/vitest.contracts-channel-registry.config.ts", 35], - ["test/vitest/vitest.contracts-plugin.config.ts", 20], - ["test/vitest/vitest.tasks.config.ts", 165], - ["test/vitest/vitest.channels.config.ts", 164], - ["test/vitest/vitest.unit-fast.config.ts", 160], - ["test/vitest/vitest.auto-reply-reply.config.ts", 155], - ["test/vitest/vitest.infra.config.ts", 145], - ["test/vitest/vitest.secrets.config.ts", 140], - ["test/vitest/vitest.cron.config.ts", 135], - ["test/vitest/vitest.wizard.config.ts", 130], - ["test/vitest/vitest.unit-src.config.ts", 125], - ["test/vitest/vitest.extension-matrix.config.ts", 100], - ["test/vitest/vitest.extension-discord.config.ts", 98], - ["test/vitest/vitest.extension-providers.config.ts", 96], - ["test/vitest/vitest.extension-telegram.config.ts", 94], - ["test/vitest/vitest.extension-whatsapp.config.ts", 92], - ["test/vitest/vitest.auto-reply-core.config.ts", 90], - ["test/vitest/vitest.cli.config.ts", 86], - ["test/vitest/vitest.media.config.ts", 84], - ["test/vitest/vitest.plugins.config.ts", 82], - ["test/vitest/vitest.bundled.config.ts", 80], - ["test/vitest/vitest.extension-slack.config.ts", 78], - ["test/vitest/vitest.commands-light.config.ts", 48], - ["test/vitest/vitest.plugin-sdk.config.ts", 46], - ["test/vitest/vitest.auto-reply-top-level.config.ts", 45], - ["test/vitest/vitest.unit-ui.config.ts", 40], - ["test/vitest/vitest.plugin-sdk-light.config.ts", 38], - ["test/vitest/vitest.daemon.config.ts", 36], - ["test/vitest/vitest.boundary.config.ts", 34], - ["test/vitest/vitest.tooling.config.ts", 32], - ["test/vitest/vitest.unit-security.config.ts", 30], - ["test/vitest/vitest.unit-support.config.ts", 28], - ["test/vitest/vitest.extension-zalo.config.ts", 24], - ["test/vitest/vitest.extension-bluebubbles.config.ts", 22], - ["test/vitest/vitest.extension-irc.config.ts", 20], - ["test/vitest/vitest.extension-feishu.config.ts", 18], - ["test/vitest/vitest.extension-mattermost.config.ts", 16], - ["test/vitest/vitest.extension-messaging.config.ts", 14], - ["test/vitest/vitest.extension-imessage.config.ts", 13], - ["test/vitest/vitest.extension-line.config.ts", 12], - ["test/vitest/vitest.extension-signal.config.ts", 11], - ["test/vitest/vitest.extension-acpx.config.ts", 10], - ["test/vitest/vitest.extension-diffs.config.ts", 8], - ["test/vitest/vitest.extension-memory.config.ts", 6], - ["test/vitest/vitest.extension-msteams.config.ts", 4], -]); const releaseLockOnce = () => { if (lockReleased) { return; @@ -198,39 +135,6 @@ async function runLoggedVitestSpec(spec) { }; } -function resolveConfigSortWeight(config, shardTimings) { - return shardTimings.get(config) ?? (FULL_SUITE_CONFIG_WEIGHT.get(config) ?? 0) * 1000; -} - -function interleaveSlowAndFastSpecs(sortedSpecs) { - const ordered = []; - let slowIndex = 0; - let fastIndex = sortedSpecs.length - 1; - while (slowIndex <= fastIndex) { - ordered.push(sortedSpecs[slowIndex]); - slowIndex += 1; - if (slowIndex <= fastIndex) { - ordered.push(sortedSpecs[fastIndex]); - fastIndex -= 1; - } - } - return ordered; -} - -function orderFullSuiteSpecsForParallelRun(specs, shardTimings = new Map()) { - const hasMatchingShardTiming = specs.some((spec) => shardTimings.has(spec.config)); - const sortedSpecs = specs.toSorted((a, b) => { - const weightDelta = - resolveConfigSortWeight(b.config, shardTimings) - - resolveConfigSortWeight(a.config, shardTimings); - if (weightDelta !== 0) { - return weightDelta; - } - return a.config.localeCompare(b.config); - }); - return hasMatchingShardTiming ? interleaveSlowAndFastSpecs(sortedSpecs) : sortedSpecs; -} - function isFullExtensionsProjectRun(specs) { const fullExtensionProjectConfigs = new Set(listFullExtensionVitestProjectConfigs()); return ( diff --git a/scripts/test-projects.test-support.mjs b/scripts/test-projects.test-support.mjs index 8854cf3b908..9d5d078bbcb 100644 --- a/scripts/test-projects.test-support.mjs +++ b/scripts/test-projects.test-support.mjs @@ -118,6 +118,103 @@ const UNIT_SECURITY_VITEST_CONFIG = "test/vitest/vitest.unit-security.config.ts" const UNIT_SRC_VITEST_CONFIG = "test/vitest/vitest.unit-src.config.ts"; const UNIT_SUPPORT_VITEST_CONFIG = "test/vitest/vitest.unit-support.config.ts"; const UNIT_UI_VITEST_CONFIG = "test/vitest/vitest.unit-ui.config.ts"; + +const FULL_SUITE_CONFIG_WEIGHT = new Map([ + [GATEWAY_VITEST_CONFIG, 180], + [GATEWAY_SERVER_VITEST_CONFIG, 180], + [GATEWAY_CORE_VITEST_CONFIG, 179], + [GATEWAY_CLIENT_VITEST_CONFIG, 178], + [GATEWAY_METHODS_VITEST_CONFIG, 177], + [COMMANDS_VITEST_CONFIG, 175], + ["test/vitest/vitest.agents-core.config.ts", 170], + ["test/vitest/vitest.agents-pi-embedded.config.ts", 169], + ["test/vitest/vitest.agents-support.config.ts", 168], + ["test/vitest/vitest.agents-tools.config.ts", 167], + [EXTENSION_VOICE_CALL_VITEST_CONFIG, 169], + [EXTENSIONS_VITEST_CONFIG, 168], + [EXTENSION_PROVIDER_OPENAI_VITEST_CONFIG, 167], + ["test/vitest/vitest.runtime-config.config.ts", 166], + [CONTRACTS_CHANNEL_CONFIG_VITEST_CONFIG, 85], + [CONTRACTS_CHANNEL_SURFACE_VITEST_CONFIG, 60], + [CONTRACTS_CHANNEL_SESSION_VITEST_CONFIG, 50], + [CONTRACTS_CHANNEL_REGISTRY_VITEST_CONFIG, 35], + [CONTRACTS_PLUGIN_VITEST_CONFIG, 20], + ["test/vitest/vitest.tasks.config.ts", 165], + [CHANNEL_VITEST_CONFIG, 164], + [UNIT_FAST_VITEST_CONFIG, 160], + [AUTO_REPLY_REPLY_VITEST_CONFIG, 155], + [INFRA_VITEST_CONFIG, 145], + ["test/vitest/vitest.secrets.config.ts", 140], + [CRON_VITEST_CONFIG, 135], + ["test/vitest/vitest.wizard.config.ts", 130], + [UNIT_SRC_VITEST_CONFIG, 125], + [EXTENSION_MATRIX_VITEST_CONFIG, 100], + [EXTENSION_DISCORD_VITEST_CONFIG, 98], + [EXTENSION_PROVIDERS_VITEST_CONFIG, 96], + [EXTENSION_TELEGRAM_VITEST_CONFIG, 94], + [EXTENSION_WHATSAPP_VITEST_CONFIG, 92], + [AUTO_REPLY_CORE_VITEST_CONFIG, 90], + [CLI_VITEST_CONFIG, 86], + [MEDIA_VITEST_CONFIG, 84], + [PLUGINS_VITEST_CONFIG, 82], + [BUNDLED_VITEST_CONFIG, 80], + [EXTENSION_SLACK_VITEST_CONFIG, 78], + [COMMANDS_LIGHT_VITEST_CONFIG, 48], + [PLUGIN_SDK_VITEST_CONFIG, 46], + [AUTO_REPLY_TOP_LEVEL_VITEST_CONFIG, 45], + [UNIT_UI_VITEST_CONFIG, 40], + [PLUGIN_SDK_LIGHT_VITEST_CONFIG, 38], + [DAEMON_VITEST_CONFIG, 36], + [BOUNDARY_VITEST_CONFIG, 34], + ["test/vitest/vitest.tooling.config.ts", 32], + [UNIT_SECURITY_VITEST_CONFIG, 30], + [UNIT_SUPPORT_VITEST_CONFIG, 28], + [EXTENSION_ZALO_VITEST_CONFIG, 24], + [EXTENSION_BLUEBUBBLES_VITEST_CONFIG, 22], + [EXTENSION_IRC_VITEST_CONFIG, 20], + [EXTENSION_FEISHU_VITEST_CONFIG, 18], + [EXTENSION_MATTERMOST_VITEST_CONFIG, 16], + [EXTENSION_MESSAGING_VITEST_CONFIG, 14], + [EXTENSION_IMESSAGE_VITEST_CONFIG, 13], + [EXTENSION_LINE_VITEST_CONFIG, 12], + [EXTENSION_SIGNAL_VITEST_CONFIG, 11], + [EXTENSION_ACPX_VITEST_CONFIG, 10], + [EXTENSION_DIFFS_VITEST_CONFIG, 8], + [EXTENSION_MEMORY_VITEST_CONFIG, 6], + [EXTENSION_MSTEAMS_VITEST_CONFIG, 4], +]); + +function resolveConfigSortWeight(config, shardTimings) { + return shardTimings.get(config) ?? (FULL_SUITE_CONFIG_WEIGHT.get(config) ?? 0) * 1000; +} + +function interleaveSlowAndFastSpecs(sortedSpecs) { + const ordered = []; + let slowIndex = 0; + let fastIndex = sortedSpecs.length - 1; + while (slowIndex <= fastIndex) { + ordered.push(sortedSpecs[slowIndex]); + slowIndex += 1; + if (slowIndex <= fastIndex) { + ordered.push(sortedSpecs[fastIndex]); + fastIndex -= 1; + } + } + return ordered; +} + +export function orderFullSuiteSpecsForParallelRun(specs, shardTimings = new Map()) { + const sortedSpecs = specs.toSorted((a, b) => { + const weightDelta = + resolveConfigSortWeight(b.config, shardTimings) - + resolveConfigSortWeight(a.config, shardTimings); + if (weightDelta !== 0) { + return weightDelta; + } + return a.config.localeCompare(b.config); + }); + return interleaveSlowAndFastSpecs(sortedSpecs); +} const PROCESS_VITEST_CONFIG = "test/vitest/vitest.process.config.ts"; const RUNTIME_CONFIG_VITEST_CONFIG = "test/vitest/vitest.runtime-config.config.ts"; const SECRETS_VITEST_CONFIG = "test/vitest/vitest.secrets.config.ts"; diff --git a/test/scripts/test-projects.test.ts b/test/scripts/test-projects.test.ts index e156f6c2279..b21d5e3de31 100644 --- a/test/scripts/test-projects.test.ts +++ b/test/scripts/test-projects.test.ts @@ -9,6 +9,7 @@ import { buildFullSuiteVitestRunPlans, buildVitestRunPlans, listFullExtensionVitestProjectConfigs, + orderFullSuiteSpecsForParallelRun, shouldAcquireLocalHeavyCheckLock, resolveChangedTestTargetPlan, resolveChangedTargetArgs, @@ -933,6 +934,24 @@ describe("scripts/test-projects local heavy-check lock", () => { }); describe("scripts/test-projects full-suite sharding", () => { + it("interleaves heavy and light configs for cold parallel full-suite runs", () => { + const specs = [ + "test/vitest/vitest.gateway.config.ts", + "test/vitest/vitest.gateway-server.config.ts", + "test/vitest/vitest.commands.config.ts", + "test/vitest/vitest.extension-memory.config.ts", + "test/vitest/vitest.extension-msteams.config.ts", + ].map((config) => ({ config })); + + expect(orderFullSuiteSpecsForParallelRun(specs).map((spec) => spec.config)).toEqual([ + "test/vitest/vitest.gateway-server.config.ts", + "test/vitest/vitest.extension-msteams.config.ts", + "test/vitest/vitest.gateway.config.ts", + "test/vitest/vitest.extension-memory.config.ts", + "test/vitest/vitest.commands.config.ts", + ]); + }); + it("covers each normal full-suite test file exactly once", async () => { const matches = await listFullSuiteTestFileMatches(); const e2eNamedIntegrationTests = new Set([