vultr/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-07-23 21:51:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

perf(ci): balance node test shards and gate heavy CI lanes by changed scope (#108091)

Browse Source
This commit is contained in:
Peter Steinberger
2026-07-15 01:16:47 -07:00
committed by GitHub
parent 9b6361a50c
commit 57761ebe8c
10 changed files with 849 additions and 244 deletions
Download Patch File Download Diff File Expand all files Collapse all files

104
test/scripts/ci-changed-node-test-plan.test.ts
View File

@@ -2,7 +2,12 @@ import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import path from "node:path";
import { describe, expect, it } from "vitest";
import { createChangedNodeTestShards } from "../../scripts/lib/ci-changed-node-test-plan.mjs";
import {
createChangedNodeTestShards,
hasBuildArtifactAffectingChange,
hasQaSmokeAffectingChange,
} from "../../scripts/lib/ci-changed-node-test-plan.mjs";
import { listGitTrackedFiles } from "../../src/test-utils/repo-files.js";
describe("CI changed Node test plan", () => {
it("routes a focused source change into one targeted job", () => {
@@ -22,22 +27,77 @@ describe("CI changed Node test plan", () => {
]);
});
it("routes built-artifact boundary tests through the dist gate", () => {
it("keeps boundary coverage on test-only diffs without the build-artifacts lane", () => {
// Test-only diffs skip build-artifacts (which hosts the full boundary
// gate), so the plan carries its own nondist boundary shard instead.
expect(createChangedNodeTestShards(["test/extension-import-boundaries.test.ts"])).toEqual([
{
checkName: "checks-node-changed-dist",
configs: ["test/vitest/vitest.boundary.config.ts"],
requiresDist: true,
checkName: "checks-node-changed",
configs: [],
requiresDist: false,
runner: "blacksmith-8vcpu-ubuntu-2404",
shardName: "changed-dist",
shardName: "changed",
targets: ["test/extension-import-boundaries.test.ts"],
},
{
checkName: "checks-node-changed-boundary",
configs: ["test/vitest/vitest.boundary.config.ts"],
requiresDist: false,
runner: "blacksmith-8vcpu-ubuntu-2404",
shardName: "changed-boundary",
},
]);
});
it("fails safe to the full plan for broad or deleted changes", () => {
it("classifies build-artifact and QA smoke impact by changed surface", () => {
expect(hasBuildArtifactAffectingChange(["src/agents/foo.test.ts", "test/helpers/x.ts"])).toBe(
false,
);
expect(hasBuildArtifactAffectingChange(["src/agents/foo.ts"])).toBe(true);
expect(hasQaSmokeAffectingChange(["extensions/qa-lab/src/ci-smoke-plan.ts"])).toBe(true);
expect(hasQaSmokeAffectingChange(["ui/src/app.ts"])).toBe(true);
expect(hasQaSmokeAffectingChange(["src/infra/retry.ts"])).toBe(false);
});
it("fails safe to the full plan for broad changes", () => {
expect(createChangedNodeTestShards(["package.json"])).toBeNull();
expect(createChangedNodeTestShards(["src/removed-module.ts"])).toBeNull();
});
it("fails safe whenever a diff deletes source files", () => {
expect(createChangedNodeTestShards(["src/infra/format-time/deleted-helper.ts"])).toBeNull();
expect(
createChangedNodeTestShards([
"src/infra/format-time/deleted-helper.ts",
"src/agents/live-model-filter.ts",
]),
).toBeNull();
});
it("keeps targeting when a diff only deletes test files alongside live source", () => {
const shards = createChangedNodeTestShards([
"src/agents/deleted-obsolete.test.ts",
"src/agents/live-model-filter.ts",
]);
expect(shards).not.toBeNull();
const targets = shards?.flatMap((shard) => shard.targets ?? []) ?? [];
expect(targets).toContain("src/agents/live-model-filter.test.ts");
});
it("runs only the boundary shard when a diff deletes test files", () => {
const cwd = mkdtempSync(path.join(tmpdir(), "openclaw-ci-deleted-test-"));
try {
expect(createChangedNodeTestShards(["src/gone.test.ts"], { cwd })).toEqual([
{
checkName: "checks-node-changed-boundary",
configs: ["test/vitest/vitest.boundary.config.ts"],
requiresDist: false,
runner: "blacksmith-8vcpu-ubuntu-2404",
shardName: "changed-boundary",
},
]);
} finally {
rmSync(cwd, { force: true, recursive: true });
}
});
it("fails safe when an unresolved path is mixed with a precise source change", () => {
@@ -97,16 +157,22 @@ describe("CI changed Node test plan", () => {
).toBeNull();
});
it("fails safe before serial target processes can dominate job time", () => {
const cwd = mkdtempSync(path.join(tmpdir(), "openclaw-ci-many-targets-"));
try {
writeFileSync(path.join(cwd, "value.ts"), "export const value = 1;\n");
for (let index = 0; index < 21; index += 1) {
writeFileSync(path.join(cwd, `consumer-${index}.test.ts`), 'import "./value.js";\n');
}
expect(createChangedNodeTestShards(["value.ts"], { cwd })).toBeNull();
} finally {
rmSync(cwd, { force: true, recursive: true });
}
it("chunks many targets into bounded parallel jobs", () => {
// A wide test-file diff exercises the multi-chunk path against the real
// tree; the cron suite has well over one chunk's worth of test files.
const changedTests = listGitTrackedFiles({ pathspecs: "src/cron" })
?.filter((file) => file.endsWith(".test.ts") && !/\.(?:e2e|live)\.test\.ts$/u.test(file))
.slice(0, 15);
expect(changedTests?.length).toBe(15);
const shards = createChangedNodeTestShards(changedTests ?? []);
expect(shards).not.toBeNull();
const targetShards = shards?.filter((shard) => shard.targets) ?? [];
expect(targetShards.length).toBeGreaterThan(1);
expect(
targetShards.every((shard, index) => shard.checkName === `checks-node-changed-${index + 1}`),
).toBe(true);
expect(targetShards.every((shard) => (shard.targets?.length ?? 0) <= 12)).toBe(true);
const targets = targetShards.flatMap((shard) => shard.targets ?? []);
expect(new Set(targets).size).toBe(targets.length);
});
});

93
test/scripts/ci-node-test-plan.test.ts
View File

@@ -145,9 +145,11 @@ describe("scripts/lib/ci-node-test-plan.mjs", () => {
).toBe(true);
expect(bundled.every((shard) => shard.runner?.startsWith("blacksmith-"))).toBe(true);
expect(bundled).toEqual(createNodeTestShardBundles({ includeReleaseOnlyPluginShards: false }));
expect(bundled.slice(0, 2).map((shard) => shard.shardName)).toEqual([
expect(bundled.slice(0, 4).map((shard) => shard.shardName)).toEqual([
"core-tooling",
"auto-reply-reply-commands",
"auto-reply-reply-commands-1",
"auto-reply-reply-commands-2",
"auto-reply-reply-commands-3",
]);
expect(bundled.find((shard) => shard.shardName === "core-unit-fast")?.runner).toBe(
DEFAULT_NODE_TEST_RUNNER,
@@ -184,15 +186,23 @@ describe("scripts/lib/ci-node-test-plan.mjs", () => {
compact: true,
});
expect(compact).toHaveLength(15);
expect(compact.filter((shard) => !shard.requiresDist)).toHaveLength(14);
expect(compact.length).toBeGreaterThanOrEqual(12);
expect(compact.length).toBeLessThanOrEqual(24);
expect(compact.every((shard) => Array.isArray(shard.groups))).toBe(true);
expect(compact.every((shard) => shard.groups.length <= 10)).toBe(true);
expect(compact.some((shard) => shard.requiresDist)).toBe(true);
expect(
compact.every((shard) =>
shard.groups.every((group) => group.requiresDist === shard.requiresDist),
),
).toBe(true);
// Runtime-balanced packing must keep the two heaviest measured groups in
// different jobs; regressing to per-file weights recombines them.
const jobOf = (name: string) =>
compact.findIndex((shard) => shard.groups.some((group) => group.shard_name === name));
expect(jobOf("agentic-agents-core-runner-embedded")).toBeGreaterThanOrEqual(0);
expect(jobOf("core-unit-fast")).toBeGreaterThanOrEqual(0);
expect(jobOf("agentic-agents-core-runner-embedded")).not.toBe(jobOf("core-unit-fast"));
expect(
compact
.flatMap((shard) =>
@@ -233,27 +243,14 @@ describe("scripts/lib/ci-node-test-plan.mjs", () => {
.filter((shard) => shard.groups.some((group) => !group.includePatterns))
.every((shard) => shard.timeoutMinutes === 120),
).toBe(true);
const largeJobs = compact.filter((shard) =>
shard.checkName.startsWith("checks-node-compact-large-"),
);
// Whole-config groups now pack into the same runtime-balanced bins as
// include-pattern groups; the separate "-whole-" job class is gone.
expect(compact.some((shard) => shard.checkName.includes("-whole-"))).toBe(false);
expect(
largeJobs.map((shard) => shard.groups.filter((group) => !group.includePatterns).length),
).toEqual([2, 2, 2]);
expect(
compact.some((shard) => shard.checkName.startsWith("checks-node-compact-large-whole-")),
).toBe(false);
const smallWholeJobs = compact.filter((shard) =>
shard.checkName.startsWith("checks-node-compact-small-whole-"),
);
const wholeGroupCounts = smallWholeJobs.map((shard) => shard.groups.length);
expect(Math.max(...wholeGroupCounts) - Math.min(...wholeGroupCounts)).toBeLessThanOrEqual(1);
expect(
smallWholeJobs.some((shard) =>
shard.groups.some((group) => group.shard_name === "core-tooling"),
),
compact.some((shard) => shard.groups.some((group) => group.shard_name === "core-tooling")),
).toBe(false);
expect(
smallWholeJobs
compact
.flatMap((shard) => shard.groups)
.find((group) => group.shard_name === "core-tooling-isolated"),
).toEqual(
@@ -262,7 +259,7 @@ describe("scripts/lib/ci-node-test-plan.mjs", () => {
}),
);
expect(
smallWholeJobs
compact
.flatMap((shard) => shard.groups)
.find((group) => group.shard_name === "core-tooling-docker"),
).toEqual(
@@ -274,7 +271,7 @@ describe("scripts/lib/ci-node-test-plan.mjs", () => {
.flatMap((shard) => shard.groups)
.filter((group) => /^core-tooling-\d+$/u.test(group.shard_name));
const toolingFiles = toolingGroups.flatMap((group) => group.includePatterns ?? []);
expect(toolingGroups).toHaveLength(3);
expect(toolingGroups).toHaveLength(4);
expect(
toolingGroups.every((group) => group.configs[0] === "test/vitest/vitest.tooling.config.ts"),
).toBe(true);
@@ -859,17 +856,41 @@ describe("scripts/lib/ci-node-test-plan.mjs", () => {
shardName: "agentic-agents-core-subagents",
},
{
checkName: "checks-node-agentic-agents-core-runner",
checkName: "checks-node-agentic-agents-core-runner-cli",
configs: ["test/vitest/vitest.agents-core.config.ts"],
includePatterns: agentShards[4]?.includePatterns,
requiresDist: false,
runner: DEFAULT_NODE_TEST_RUNNER,
shardName: "agentic-agents-core-runner",
shardName: "agentic-agents-core-runner-cli",
},
{
checkName: "checks-node-agentic-agents-core-runner-commands",
configs: ["test/vitest/vitest.agents-core.config.ts"],
includePatterns: agentShards[5]?.includePatterns,
requiresDist: false,
runner: DEFAULT_NODE_TEST_RUNNER,
shardName: "agentic-agents-core-runner-commands",
},
{
checkName: "checks-node-agentic-agents-core-runner-embedded",
configs: ["test/vitest/vitest.agents-core.config.ts"],
includePatterns: agentShards[6]?.includePatterns,
requiresDist: false,
runner: DEFAULT_NODE_TEST_RUNNER,
shardName: "agentic-agents-core-runner-embedded",
},
{
checkName: "checks-node-agentic-agents-core-runner-sessions",
configs: ["test/vitest/vitest.agents-core.config.ts"],
includePatterns: agentShards[7]?.includePatterns,
requiresDist: false,
runner: DEFAULT_NODE_TEST_RUNNER,
shardName: "agentic-agents-core-runner-sessions",
},
{
checkName: "checks-node-agentic-agents-core-runtime",
configs: ["test/vitest/vitest.agents-core.config.ts"],
includePatterns: agentShards[5]?.includePatterns,
includePatterns: agentShards[8]?.includePatterns,
requiresDist: false,
runner: DEFAULT_NODE_TEST_RUNNER,
shardName: "agentic-agents-core-runtime",
@@ -877,7 +898,7 @@ describe("scripts/lib/ci-node-test-plan.mjs", () => {
{
checkName: "checks-node-agentic-agents-core-isolated",
configs: ["test/vitest/vitest.agents-core-isolated.config.ts"],
includePatterns: agentShards[6]?.includePatterns,
includePatterns: agentShards[9]?.includePatterns,
requiresDist: false,
runner: DEFAULT_NODE_TEST_RUNNER,
shardName: "agentic-agents-core-isolated",
@@ -1011,10 +1032,22 @@ describe("scripts/lib/ci-node-test-plan.mjs", () => {
shardName: "auto-reply-reply-agent-runner",
},
{
checkName: "checks-node-auto-reply-reply-commands",
checkName: "checks-node-auto-reply-reply-commands-1",
configs: ["test/vitest/vitest.auto-reply-reply.config.ts"],
requiresDist: false,
shardName: "auto-reply-reply-commands",
shardName: "auto-reply-reply-commands-1",
},
{
checkName: "checks-node-auto-reply-reply-commands-2",
configs: ["test/vitest/vitest.auto-reply-reply.config.ts"],
requiresDist: false,
shardName: "auto-reply-reply-commands-2",
},
{
checkName: "checks-node-auto-reply-reply-commands-3",
configs: ["test/vitest/vitest.auto-reply-reply.config.ts"],
requiresDist: false,
shardName: "auto-reply-reply-commands-3",
},
{
checkName: "checks-node-auto-reply-reply-dispatch",

176
test/scripts/ci-run-node-test-shard.test.ts Normal file
View File

@@ -0,0 +1,176 @@
// Covers the CI node test shard runner: plan resolution from job env and
// bounded-concurrency execution with per-child Vitest cache isolation.
import { mkdtempSync, readFileSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import path from "node:path";
import { afterEach, describe, expect, it } from "vitest";
import {
buildChildEnv,
resolveShardPlans,
runShardPlans,
} from "../../scripts/ci-run-node-test-shard.mjs";
const scratchDirs: string[] = [];
function makeScratchDir(): string {
const dir = mkdtempSync(path.join(tmpdir(), "openclaw-shard-test-"));
scratchDirs.push(dir);
return dir;
}
afterEach(() => {
for (const dir of scratchDirs.splice(0)) {
rmSync(dir, { force: true, recursive: true });
}
});
describe("scripts/ci-run-node-test-shard.mjs", () => {
it("prefers explicit targets and keeps one target per child", () => {
const plans = resolveShardPlans({
OPENCLAW_NODE_TEST_TARGETS_JSON: JSON.stringify(["a.test.ts", "b.test.ts"]),
OPENCLAW_NODE_TEST_GROUPS_JSON: JSON.stringify([{ configs: ["c.config.ts"] }]),
});
expect(plans).toEqual([
{ kind: "target", name: "a.test.ts", target: "a.test.ts" },
{ kind: "target", name: "b.test.ts", target: "b.test.ts" },
]);
});
it("falls back from groups to the single-shard matrix envelope", () => {
const groupPlans = resolveShardPlans({
OPENCLAW_NODE_TEST_GROUPS_JSON: JSON.stringify([
{ configs: ["one.config.ts"], shard_name: "one" },
{ configs: ["two.config.ts"], shard_name: "two" },
]),
});
expect(groupPlans.map((plan) => plan.name)).toEqual(["one", "two"]);
const singlePlans = resolveShardPlans({
OPENCLAW_NODE_TEST_CONFIGS_JSON: JSON.stringify(["solo.config.ts"]),
OPENCLAW_VITEST_SHARD_NAME: "solo",
});
expect(singlePlans).toHaveLength(1);
expect(singlePlans[0]).toMatchObject({ kind: "group", name: "solo" });
});
it("builds child env with per-plan cache isolation, includes, and env overlays", () => {
const scratchDir = makeScratchDir();
const entry = {
kind: "group" as const,
name: "g",
plan: {
configs: ["cfg.ts"],
env: { EXTRA: "yes", IGNORED: 42 },
includePatterns: ["src/a.test.ts"],
shard_name: "g",
},
};
const childEnv = buildChildEnv(
entry,
{ BASE: "1", OPENCLAW_VITEST_INCLUDE_FILE: "stale.json" },
scratchDir,
3,
);
expect(childEnv.BASE).toBe("1");
expect(childEnv.EXTRA).toBe("yes");
expect(childEnv.IGNORED).toBeUndefined();
expect(childEnv.OPENCLAW_VITEST_SHARD_NAME).toBe("g");
expect(childEnv.OPENCLAW_TEST_PROJECTS_PARALLEL).toBe("1");
expect(childEnv.OPENCLAW_TEST_HEAVY_CHECK_LOCK_HELD).toBe("1");
expect(childEnv.OPENCLAW_VITEST_FS_MODULE_CACHE_PATH).toBe(
path.join(scratchDir, "vitest-cache-3"),
);
expect(childEnv.OPENCLAW_VITEST_INCLUDE_FILE).toBe(
path.join(scratchDir, "node-test-include-3.json"),
);
expect(JSON.parse(readFileSync(childEnv.OPENCLAW_VITEST_INCLUDE_FILE ?? "", "utf8"))).toEqual([
"src/a.test.ts",
]);
const bare = buildChildEnv(
{ kind: "group" as const, name: "bare", plan: { configs: ["cfg.ts"] } },
{ OPENCLAW_VITEST_INCLUDE_FILE: "stale.json" },
scratchDir,
0,
);
expect(bare.OPENCLAW_VITEST_INCLUDE_FILE).toBeUndefined();
});
it("runs plans with bounded concurrency and distinct cache paths", async () => {
const scratchDir = makeScratchDir();
const seen: Array<{ args: string[]; cache: string | undefined; label: string }> = [];
let active = 0;
let peakActive = 0;
const exitCode = await runShardPlans(
resolveShardPlans({
OPENCLAW_NODE_TEST_GROUPS_JSON: JSON.stringify([
{ configs: ["a.config.ts"], shard_name: "a" },
{ configs: ["b.config.ts"], shard_name: "b" },
{ configs: ["c.config.ts"], shard_name: "c" },
]),
}),
{
concurrency: 2,
env: {},
runChild: async (
args: string[],
childEnv: Record<string, string | undefined>,
label: string,
) => {
active += 1;
peakActive = Math.max(peakActive, active);
await new Promise((resolve) => {
setTimeout(resolve, 10);
});
seen.push({ args, cache: childEnv.OPENCLAW_VITEST_FS_MODULE_CACHE_PATH, label });
active -= 1;
return 0;
},
scratchDir,
},
);
expect(exitCode).toBe(0);
expect(peakActive).toBeLessThanOrEqual(2);
expect(seen.map((run) => run.label).toSorted()).toEqual(["a", "b", "c"]);
expect(new Set(seen.map((run) => run.cache)).size).toBe(3);
});
it("stops scheduling new plans after a failure and reports the first failing code", async () => {
const scratchDir = makeScratchDir();
const started: string[] = [];
const exitCode = await runShardPlans(
resolveShardPlans({
OPENCLAW_NODE_TEST_GROUPS_JSON: JSON.stringify([
{ configs: ["a.config.ts"], shard_name: "a" },
{ configs: ["b.config.ts"], shard_name: "b" },
{ configs: ["c.config.ts"], shard_name: "c" },
{ configs: ["d.config.ts"], shard_name: "d" },
]),
}),
{
concurrency: 1,
env: {},
runChild: async (
_args: string[],
_env: Record<string, string | undefined>,
label: string,
) => {
started.push(label);
return label === "b" ? 7 : 0;
},
scratchDir,
},
);
expect(exitCode).toBe(7);
expect(started).toEqual(["a", "b"]);
});
it("fails plans that carry no configs", async () => {
const scratchDir = makeScratchDir();
const exitCode = await runShardPlans(
[{ kind: "group" as const, name: "broken", plan: { configs: [] } }],
{ concurrency: 1, env: {}, runChild: async () => 0, scratchDir },
);
expect(exitCode).toBe(1);
});
});

9
test/scripts/ci-workflow-guards.test.ts
View File

@@ -2641,13 +2641,12 @@ describe("ci workflow guards", () => {
expect(nodeTestJob["timeout-minutes"]).toBe("${{ matrix.timeout_minutes || 60 }}");
expect(runStep.env.OPENCLAW_VITEST_NO_OUTPUT_TIMEOUT_MS).toBe("300000");
expect(runStep.env.OPENCLAW_VITEST_NO_OUTPUT_RETRY).toBe("1");
expect(runStep.env.OPENCLAW_TEST_PROJECTS_PARALLEL).toBe("2");
expect(runStep.env.OPENCLAW_NODE_TEST_ENV_JSON).toBe("${{ toJson(matrix.env) }}");
expect(runStep.env.OPENCLAW_NODE_TEST_TARGETS_JSON).toBe("${{ toJson(matrix.targets) }}");
expect(runStep.run).toContain("env: JSON.parse(process.env.OPENCLAW_NODE_TEST_ENV_JSON");
expect(runStep.run).toContain('["exec", "node", "scripts/test-projects.mjs", target]');
expect(runStep.run).toContain('if (plan.env && typeof plan.env === "object"');
expect(runStep.run).toContain("childEnv[key] = value");
// Shard execution policy lives in the unit-tested wrapper script, not in
// inline workflow JavaScript.
expect(runStep.run).toBe("node scripts/ci-run-node-test-shard.mjs");
expect(existsSync("scripts/ci-run-node-test-shard.mjs")).toBe(true);
});
it("keeps the CI timing summary parked for timing optimization work", () => {