mirror of
https://github.com/openclaw/openclaw.git
synced 2026-05-06 07:40:44 +00:00
ci: reduce node runner fanout
This commit is contained in:
4
.github/workflows/ci.yml
vendored
4
.github/workflows/ci.yml
vendored
@@ -746,6 +746,7 @@ jobs:
|
||||
timeout-minutes: 60
|
||||
strategy:
|
||||
fail-fast: false
|
||||
max-parallel: 3
|
||||
matrix: ${{ fromJson(needs.preflight.outputs.channel_contracts_matrix) }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
@@ -1118,6 +1119,7 @@ jobs:
|
||||
timeout-minutes: 60
|
||||
strategy:
|
||||
fail-fast: false
|
||||
max-parallel: 8
|
||||
matrix: ${{ fromJson(needs.preflight.outputs.checks_node_core_nondist_matrix) }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
@@ -1355,6 +1357,7 @@ jobs:
|
||||
timeout-minutes: 20
|
||||
strategy:
|
||||
fail-fast: false
|
||||
max-parallel: 3
|
||||
matrix:
|
||||
include:
|
||||
- check_name: check-preflight-guards
|
||||
@@ -1495,6 +1498,7 @@ jobs:
|
||||
timeout-minutes: 20
|
||||
strategy:
|
||||
fail-fast: false
|
||||
max-parallel: 3
|
||||
matrix:
|
||||
include:
|
||||
- check_name: check-additional-boundaries
|
||||
|
||||
@@ -62,7 +62,7 @@ Local changed-lane logic lives in `scripts/changed-lanes.mjs` and is executed by
|
||||
|
||||
On pushes, the `checks` matrix adds the push-only `compat-node22` lane. On pull requests, that lane is skipped and the matrix stays focused on the normal test/channel lanes.
|
||||
|
||||
The slowest Node test families are split or balanced so each job stays small: channel contracts split registry and core coverage into six weighted shards total, bundled plugin tests balance across six extension workers, auto-reply runs as three balanced workers instead of six tiny workers, and agentic gateway/plugin configs are spread across the existing source-only agentic Node jobs instead of waiting on built artifacts. Broad browser, QA, media, and miscellaneous plugin tests use their dedicated Vitest configs instead of the shared plugin catch-all. The broad agents lane uses the shared Vitest file-parallel scheduler because it is import/scheduling dominated rather than owned by a single slow test file. `runtime-config` runs with the infra core-runtime shard to keep the shared runtime shard from owning the tail. `check-additional` keeps package-boundary compile/canary work together and separates runtime topology architecture from gateway watch coverage; the boundary guard shard runs its small independent guards concurrently inside one job. Gateway watch, channel tests, and the core support-boundary shard run concurrently inside `build-artifacts` after `dist/` and `dist-runtime/` are already built, keeping their old check names as lightweight verifier jobs while avoiding two extra Blacksmith workers and a second artifact-consumer queue.
|
||||
The slowest Node test families are split or balanced so each job stays small without over-reserving runners: channel contracts run as three weighted shards, bundled plugin tests balance across six extension workers, small core unit lanes are paired, auto-reply runs as three balanced workers instead of six tiny workers, and agentic gateway/plugin configs are spread across the existing source-only agentic Node jobs instead of waiting on built artifacts. Broad browser, QA, media, and miscellaneous plugin tests use their dedicated Vitest configs instead of the shared plugin catch-all. The broad agents lane uses the shared Vitest file-parallel scheduler because it is import/scheduling dominated rather than owned by a single slow test file. `runtime-config` runs with the infra core-runtime shard to keep the shared runtime shard from owning the tail. `check-additional` keeps package-boundary compile/canary work together and separates runtime topology architecture from gateway watch coverage; the boundary guard shard runs its small independent guards concurrently inside one job. Gateway watch, channel tests, and the core support-boundary shard run concurrently inside `build-artifacts` after `dist/` and `dist-runtime/` are already built, keeping their old check names as lightweight verifier jobs while avoiding two extra Blacksmith workers and a second artifact-consumer queue.
|
||||
Android CI runs both `testPlayDebugUnitTest` and `testThirdPartyDebugUnitTest`, then builds the Play debug APK. The third-party flavor has no separate source set or manifest; its unit-test lane still compiles that flavor with the SMS/call-log BuildConfig flags, while avoiding a duplicate debug APK packaging job on every Android-relevant push.
|
||||
`extension-fast` is PR-only because push runs already execute the full bundled plugin shards. That keeps changed-plugin feedback for reviews without reserving an extra Blacksmith worker on `main` for coverage already present in `checks-node-extensions`.
|
||||
|
||||
|
||||
@@ -41,14 +41,9 @@ export function createChannelContractTestShards() {
|
||||
const rootDir = "src/channels/plugins/contracts";
|
||||
const suffixes = ["a", "b", "c"];
|
||||
const groups = Object.fromEntries(
|
||||
["registry", "core"].flatMap((family) =>
|
||||
suffixes.map((suffix) => [`checks-fast-contracts-channels-${family}-${suffix}`, []]),
|
||||
),
|
||||
suffixes.map((suffix) => [`checks-fast-contracts-channels-${suffix}`, []]),
|
||||
);
|
||||
const groupKeys = {
|
||||
core: suffixes.map((suffix) => `checks-fast-contracts-channels-core-${suffix}`),
|
||||
registry: suffixes.map((suffix) => `checks-fast-contracts-channels-registry-${suffix}`),
|
||||
};
|
||||
const groupKeys = suffixes.map((suffix) => `checks-fast-contracts-channels-${suffix}`);
|
||||
const weights = Object.fromEntries(Object.keys(groups).map((key) => [key, 0]));
|
||||
const pushBalanced = (keys, file) => {
|
||||
const target = keys.toSorted((a, b) => weights[a] - weights[b] || a.localeCompare(b))[0];
|
||||
@@ -71,10 +66,10 @@ export function createChannelContractTestShards() {
|
||||
return delta === 0 ? left.localeCompare(right) : delta;
|
||||
};
|
||||
for (const file of registryFiles.toSorted(byDescendingWeight)) {
|
||||
pushBalanced(groupKeys.registry, file);
|
||||
pushBalanced(groupKeys, file);
|
||||
}
|
||||
for (const file of coreFiles.toSorted(byDescendingWeight)) {
|
||||
pushBalanced(groupKeys.core, file);
|
||||
pushBalanced(groupKeys, file);
|
||||
}
|
||||
|
||||
return Object.entries(groups).map(([checkName, includePatterns]) => ({
|
||||
|
||||
@@ -87,6 +87,36 @@ function createAutoReplyReplySplitShards() {
|
||||
}
|
||||
|
||||
const SPLIT_NODE_SHARDS = new Map([
|
||||
[
|
||||
"core-unit-fast",
|
||||
[
|
||||
{
|
||||
shardName: "core-unit-fast-support",
|
||||
configs: [
|
||||
"test/vitest/vitest.unit-fast.config.ts",
|
||||
"test/vitest/vitest.unit-support.config.ts",
|
||||
],
|
||||
includeExternalConfigs: true,
|
||||
requiresDist: false,
|
||||
},
|
||||
],
|
||||
],
|
||||
[
|
||||
"core-unit-src",
|
||||
[
|
||||
{
|
||||
shardName: "core-unit-src-security",
|
||||
configs: [
|
||||
"test/vitest/vitest.unit-src.config.ts",
|
||||
"test/vitest/vitest.unit-security.config.ts",
|
||||
],
|
||||
includeExternalConfigs: true,
|
||||
requiresDist: false,
|
||||
},
|
||||
],
|
||||
],
|
||||
["core-unit-security", []],
|
||||
["core-unit-support", []],
|
||||
[
|
||||
"core-runtime",
|
||||
[
|
||||
@@ -205,7 +235,9 @@ export function createNodeTestShards() {
|
||||
const splitShards = SPLIT_NODE_SHARDS.get(shard.name);
|
||||
if (splitShards) {
|
||||
return splitShards.flatMap((splitShard) => {
|
||||
const splitConfigs = splitShard.configs.filter((config) => configs.includes(config));
|
||||
const splitConfigs = splitShard.includeExternalConfigs
|
||||
? splitShard.configs
|
||||
: splitShard.configs.filter((config) => configs.includes(config));
|
||||
if (splitConfigs.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
@@ -25,13 +25,11 @@ describe("scripts/lib/channel-contract-test-plan.mjs", () => {
|
||||
task: shard.task,
|
||||
})),
|
||||
).toEqual(
|
||||
["registry", "core"].flatMap((family) =>
|
||||
suffixes.map((suffix) => ({
|
||||
checkName: `checks-fast-contracts-channels-${family}-${suffix}`,
|
||||
runtime: "node",
|
||||
task: "contracts-channels",
|
||||
})),
|
||||
),
|
||||
suffixes.map((suffix) => ({
|
||||
checkName: `checks-fast-contracts-channels-${suffix}`,
|
||||
runtime: "node",
|
||||
task: "contracts-channels",
|
||||
})),
|
||||
);
|
||||
});
|
||||
|
||||
@@ -45,13 +43,11 @@ describe("scripts/lib/channel-contract-test-plan.mjs", () => {
|
||||
});
|
||||
|
||||
it("keeps registry-backed surface shards spread across checks", () => {
|
||||
for (const shard of createChannelContractTestShards().filter((entry) =>
|
||||
entry.checkName.includes("-registry-"),
|
||||
)) {
|
||||
for (const shard of createChannelContractTestShards()) {
|
||||
const surfaceRegistryFiles = shard.includePatterns.filter((pattern) =>
|
||||
pattern.includes("/surfaces-only.registry-backed-shard-"),
|
||||
);
|
||||
expect(surfaceRegistryFiles.length).toBeLessThanOrEqual(3);
|
||||
expect(surfaceRegistryFiles.length).toBeLessThanOrEqual(4);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
@@ -25,6 +25,40 @@ function listTestFiles(rootDir: string): string[] {
|
||||
}
|
||||
|
||||
describe("scripts/lib/ci-node-test-plan.mjs", () => {
|
||||
it("combines the small core unit shards to reduce CI runner fanout", () => {
|
||||
const coreUnitShards = createNodeTestShards()
|
||||
.filter((shard) => shard.shardName.startsWith("core-unit-"))
|
||||
.map((shard) => ({
|
||||
configs: shard.configs,
|
||||
requiresDist: shard.requiresDist,
|
||||
shardName: shard.shardName,
|
||||
}));
|
||||
|
||||
expect(coreUnitShards).toEqual([
|
||||
{
|
||||
configs: [
|
||||
"test/vitest/vitest.unit-fast.config.ts",
|
||||
"test/vitest/vitest.unit-support.config.ts",
|
||||
],
|
||||
requiresDist: false,
|
||||
shardName: "core-unit-fast-support",
|
||||
},
|
||||
{
|
||||
configs: [
|
||||
"test/vitest/vitest.unit-src.config.ts",
|
||||
"test/vitest/vitest.unit-security.config.ts",
|
||||
],
|
||||
requiresDist: false,
|
||||
shardName: "core-unit-src-security",
|
||||
},
|
||||
{
|
||||
configs: ["test/vitest/vitest.unit-ui.config.ts"],
|
||||
requiresDist: false,
|
||||
shardName: "core-unit-ui",
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("names the node shard checks as core test lanes", () => {
|
||||
const shards = createNodeTestShards();
|
||||
|
||||
|
||||
Reference in New Issue
Block a user