ci: reduce node runner fanout

This commit is contained in:
Peter Steinberger
2026-04-23 15:25:28 +01:00
parent b4d1992338
commit 6532ee0c39
6 changed files with 83 additions and 22 deletions

View File

@@ -746,6 +746,7 @@ jobs:
timeout-minutes: 60
strategy:
fail-fast: false
max-parallel: 3
matrix: ${{ fromJson(needs.preflight.outputs.channel_contracts_matrix) }}
steps:
- name: Checkout
@@ -1118,6 +1119,7 @@ jobs:
timeout-minutes: 60
strategy:
fail-fast: false
max-parallel: 8
matrix: ${{ fromJson(needs.preflight.outputs.checks_node_core_nondist_matrix) }}
steps:
- name: Checkout
@@ -1355,6 +1357,7 @@ jobs:
timeout-minutes: 20
strategy:
fail-fast: false
max-parallel: 3
matrix:
include:
- check_name: check-preflight-guards
@@ -1495,6 +1498,7 @@ jobs:
timeout-minutes: 20
strategy:
fail-fast: false
max-parallel: 3
matrix:
include:
- check_name: check-additional-boundaries

View File

@@ -62,7 +62,7 @@ Local changed-lane logic lives in `scripts/changed-lanes.mjs` and is executed by
On pushes, the `checks` matrix adds the push-only `compat-node22` lane. On pull requests, that lane is skipped and the matrix stays focused on the normal test/channel lanes.
The slowest Node test families are split or balanced so each job stays small: channel contracts split registry and core coverage into six weighted shards total, bundled plugin tests balance across six extension workers, auto-reply runs as three balanced workers instead of six tiny workers, and agentic gateway/plugin configs are spread across the existing source-only agentic Node jobs instead of waiting on built artifacts. Broad browser, QA, media, and miscellaneous plugin tests use their dedicated Vitest configs instead of the shared plugin catch-all. The broad agents lane uses the shared Vitest file-parallel scheduler because it is import/scheduling dominated rather than owned by a single slow test file. `runtime-config` runs with the infra core-runtime shard to keep the shared runtime shard from owning the tail. `check-additional` keeps package-boundary compile/canary work together and separates runtime topology architecture from gateway watch coverage; the boundary guard shard runs its small independent guards concurrently inside one job. Gateway watch, channel tests, and the core support-boundary shard run concurrently inside `build-artifacts` after `dist/` and `dist-runtime/` are already built, keeping their old check names as lightweight verifier jobs while avoiding two extra Blacksmith workers and a second artifact-consumer queue.
The slowest Node test families are split or balanced so each job stays small without over-reserving runners: channel contracts run as three weighted shards, bundled plugin tests balance across six extension workers, small core unit lanes are paired, auto-reply runs as three balanced workers instead of six tiny workers, and agentic gateway/plugin configs are spread across the existing source-only agentic Node jobs instead of waiting on built artifacts. Broad browser, QA, media, and miscellaneous plugin tests use their dedicated Vitest configs instead of the shared plugin catch-all. The broad agents lane uses the shared Vitest file-parallel scheduler because it is import/scheduling dominated rather than owned by a single slow test file. `runtime-config` runs with the infra core-runtime shard to keep the shared runtime shard from owning the tail. `check-additional` keeps package-boundary compile/canary work together and separates runtime topology architecture from gateway watch coverage; the boundary guard shard runs its small independent guards concurrently inside one job. Gateway watch, channel tests, and the core support-boundary shard run concurrently inside `build-artifacts` after `dist/` and `dist-runtime/` are already built, keeping their old check names as lightweight verifier jobs while avoiding two extra Blacksmith workers and a second artifact-consumer queue.
Android CI runs both `testPlayDebugUnitTest` and `testThirdPartyDebugUnitTest`, then builds the Play debug APK. The third-party flavor has no separate source set or manifest; its unit-test lane still compiles that flavor with the SMS/call-log BuildConfig flags, while avoiding a duplicate debug APK packaging job on every Android-relevant push.
`extension-fast` is PR-only because push runs already execute the full bundled plugin shards. That keeps changed-plugin feedback for reviews without reserving an extra Blacksmith worker on `main` for coverage already present in `checks-node-extensions`.

View File

@@ -41,14 +41,9 @@ export function createChannelContractTestShards() {
const rootDir = "src/channels/plugins/contracts";
const suffixes = ["a", "b", "c"];
const groups = Object.fromEntries(
["registry", "core"].flatMap((family) =>
suffixes.map((suffix) => [`checks-fast-contracts-channels-${family}-${suffix}`, []]),
),
suffixes.map((suffix) => [`checks-fast-contracts-channels-${suffix}`, []]),
);
const groupKeys = {
core: suffixes.map((suffix) => `checks-fast-contracts-channels-core-${suffix}`),
registry: suffixes.map((suffix) => `checks-fast-contracts-channels-registry-${suffix}`),
};
const groupKeys = suffixes.map((suffix) => `checks-fast-contracts-channels-${suffix}`);
const weights = Object.fromEntries(Object.keys(groups).map((key) => [key, 0]));
const pushBalanced = (keys, file) => {
const target = keys.toSorted((a, b) => weights[a] - weights[b] || a.localeCompare(b))[0];
@@ -71,10 +66,10 @@ export function createChannelContractTestShards() {
return delta === 0 ? left.localeCompare(right) : delta;
};
for (const file of registryFiles.toSorted(byDescendingWeight)) {
pushBalanced(groupKeys.registry, file);
pushBalanced(groupKeys, file);
}
for (const file of coreFiles.toSorted(byDescendingWeight)) {
pushBalanced(groupKeys.core, file);
pushBalanced(groupKeys, file);
}
return Object.entries(groups).map(([checkName, includePatterns]) => ({

View File

@@ -87,6 +87,36 @@ function createAutoReplyReplySplitShards() {
}
const SPLIT_NODE_SHARDS = new Map([
[
"core-unit-fast",
[
{
shardName: "core-unit-fast-support",
configs: [
"test/vitest/vitest.unit-fast.config.ts",
"test/vitest/vitest.unit-support.config.ts",
],
includeExternalConfigs: true,
requiresDist: false,
},
],
],
[
"core-unit-src",
[
{
shardName: "core-unit-src-security",
configs: [
"test/vitest/vitest.unit-src.config.ts",
"test/vitest/vitest.unit-security.config.ts",
],
includeExternalConfigs: true,
requiresDist: false,
},
],
],
["core-unit-security", []],
["core-unit-support", []],
[
"core-runtime",
[
@@ -205,7 +235,9 @@ export function createNodeTestShards() {
const splitShards = SPLIT_NODE_SHARDS.get(shard.name);
if (splitShards) {
return splitShards.flatMap((splitShard) => {
const splitConfigs = splitShard.configs.filter((config) => configs.includes(config));
const splitConfigs = splitShard.includeExternalConfigs
? splitShard.configs
: splitShard.configs.filter((config) => configs.includes(config));
if (splitConfigs.length === 0) {
return [];
}

View File

@@ -25,13 +25,11 @@ describe("scripts/lib/channel-contract-test-plan.mjs", () => {
task: shard.task,
})),
).toEqual(
["registry", "core"].flatMap((family) =>
suffixes.map((suffix) => ({
checkName: `checks-fast-contracts-channels-${family}-${suffix}`,
runtime: "node",
task: "contracts-channels",
})),
),
suffixes.map((suffix) => ({
checkName: `checks-fast-contracts-channels-${suffix}`,
runtime: "node",
task: "contracts-channels",
})),
);
});
@@ -45,13 +43,11 @@ describe("scripts/lib/channel-contract-test-plan.mjs", () => {
});
it("keeps registry-backed surface shards spread across checks", () => {
for (const shard of createChannelContractTestShards().filter((entry) =>
entry.checkName.includes("-registry-"),
)) {
for (const shard of createChannelContractTestShards()) {
const surfaceRegistryFiles = shard.includePatterns.filter((pattern) =>
pattern.includes("/surfaces-only.registry-backed-shard-"),
);
expect(surfaceRegistryFiles.length).toBeLessThanOrEqual(3);
expect(surfaceRegistryFiles.length).toBeLessThanOrEqual(4);
}
});
});

View File

@@ -25,6 +25,40 @@ function listTestFiles(rootDir: string): string[] {
}
describe("scripts/lib/ci-node-test-plan.mjs", () => {
it("combines the small core unit shards to reduce CI runner fanout", () => {
const coreUnitShards = createNodeTestShards()
.filter((shard) => shard.shardName.startsWith("core-unit-"))
.map((shard) => ({
configs: shard.configs,
requiresDist: shard.requiresDist,
shardName: shard.shardName,
}));
expect(coreUnitShards).toEqual([
{
configs: [
"test/vitest/vitest.unit-fast.config.ts",
"test/vitest/vitest.unit-support.config.ts",
],
requiresDist: false,
shardName: "core-unit-fast-support",
},
{
configs: [
"test/vitest/vitest.unit-src.config.ts",
"test/vitest/vitest.unit-security.config.ts",
],
requiresDist: false,
shardName: "core-unit-src-security",
},
{
configs: ["test/vitest/vitest.unit-ui.config.ts"],
requiresDist: false,
shardName: "core-unit-ui",
},
]);
});
it("names the node shard checks as core test lanes", () => {
const shards = createNodeTestShards();