From ebdade0efc568ab0656a0812d0e540c16be3c67b Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 3 Apr 2026 18:26:14 +0100 Subject: [PATCH] ci: shard extension fast checks --- .github/workflows/ci.yml | 73 ++++++++++++++++--- package.json | 1 + scripts/lib/extension-test-plan.mjs | 99 ++++++++++++++++++++++++++ scripts/test-extension-batch.mjs | 105 ++++++++++++++++++++++++++++ test/scripts/test-extension.test.ts | 52 +++++++++++++- 5 files changed, 321 insertions(+), 9 deletions(-) create mode 100644 scripts/test-extension-batch.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1b4842dfa6b..96dacff1bce 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,7 +36,8 @@ jobs: changed_extensions_matrix: ${{ steps.manifest.outputs.changed_extensions_matrix }} run_build_artifacts: ${{ steps.manifest.outputs.run_build_artifacts }} run_checks_fast: ${{ steps.manifest.outputs.run_checks_fast }} - checks_fast_matrix: ${{ steps.manifest.outputs.checks_fast_matrix }} + checks_fast_core_matrix: ${{ steps.manifest.outputs.checks_fast_core_matrix }} + checks_fast_extensions_matrix: ${{ steps.manifest.outputs.checks_fast_extensions_matrix }} run_checks: ${{ steps.manifest.outputs.run_checks }} checks_matrix: ${{ steps.manifest.outputs.checks_matrix }} run_extension_fast: ${{ steps.manifest.outputs.run_extension_fast }} @@ -132,6 +133,10 @@ jobs: run: | node --input-type=module <<'EOF' import { appendFileSync } from "node:fs"; + import { + createExtensionTestShards, + DEFAULT_EXTENSION_TEST_SHARD_COUNT, + } from "./scripts/lib/extension-test-plan.mjs"; const parseBoolean = (value, fallback = false) => { if (value === undefined) return fallback; @@ -165,6 +170,18 @@ jobs: const changedExtensionsMatrix = hasChangedExtensions ? parseJson(process.env.OPENCLAW_CI_CHANGED_EXTENSIONS_MATRIX, { include: [] }) : { include: [] }; + const extensionShardMatrix = createMatrix( + runNode + ? createExtensionTestShards({ + shardCount: DEFAULT_EXTENSION_TEST_SHARD_COUNT, + }).map((shard) => ({ + check_name: shard.checkName, + extensions_csv: shard.extensionIds.join(","), + shard_index: shard.index + 1, + task: "extensions-batch", + })) + : [], + ); const manifest = { docs_only: docsOnly, @@ -178,11 +195,10 @@ jobs: changed_extensions_matrix: changedExtensionsMatrix, run_build_artifacts: runNode, run_checks_fast: runNode, - checks_fast_matrix: createMatrix( + checks_fast_core_matrix: createMatrix( runNode ? [ { check_name: "checks-fast-bundled", runtime: "node", task: "bundled" }, - { check_name: "checks-fast-extensions", runtime: "node", task: "extensions" }, { check_name: "checks-fast-contracts-protocol", runtime: "node", @@ -191,6 +207,7 @@ jobs: ] : [], ), + checks_fast_extensions_matrix: extensionShardMatrix, run_checks: runNode, checks_matrix: createMatrix( runNode @@ -404,7 +421,7 @@ jobs: include-hidden-files: true retention-days: 1 - checks-fast: + checks-fast-core: name: ${{ matrix.check_name }} needs: [preflight] if: needs.preflight.outputs.run_checks_fast == 'true' @@ -412,7 +429,7 @@ jobs: timeout-minutes: 60 strategy: fail-fast: false - matrix: ${{ fromJson(needs.preflight.outputs.checks_fast_matrix) }} + matrix: ${{ fromJson(needs.preflight.outputs.checks_fast_core_matrix) }} steps: - name: Checkout uses: actions/checkout@v6 @@ -436,9 +453,6 @@ jobs: bundled) pnpm test:bundled ;; - extensions) - pnpm test:extensions - ;; contracts|contracts-protocol) pnpm build pnpm test:contracts @@ -450,6 +464,49 @@ jobs: ;; esac + checks-fast-extensions-shard: + name: ${{ matrix.check_name }} + needs: [preflight] + if: needs.preflight.outputs.run_checks_fast == 'true' + runs-on: blacksmith-16vcpu-ubuntu-2404 + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.preflight.outputs.checks_fast_extensions_matrix) }} + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + persist-credentials: false + submodules: false + + - name: Setup Node environment + uses: ./.github/actions/setup-node-env + with: + install-bun: "false" + use-sticky-disk: "false" + + - name: Run extension shard + env: + OPENCLAW_EXTENSION_BATCH: ${{ matrix.extensions_csv }} + run: pnpm test:extensions:batch -- "$OPENCLAW_EXTENSION_BATCH" + + checks-fast-extensions: + name: checks-fast-extensions + needs: [preflight, checks-fast-extensions-shard] + if: always() && needs.preflight.outputs.run_checks_fast == 'true' + runs-on: blacksmith-16vcpu-ubuntu-2404 + timeout-minutes: 5 + steps: + - name: Verify extension shards + env: + SHARD_RESULT: ${{ needs.checks-fast-extensions-shard.result }} + run: | + if [ "$SHARD_RESULT" != "success" ]; then + echo "Extension shard checks failed: $SHARD_RESULT" >&2 + exit 1 + fi + checks: name: ${{ matrix.check_name }} needs: [preflight, build-artifacts] diff --git a/package.json b/package.json index 6a54c5fdf3f..38447410a03 100644 --- a/package.json +++ b/package.json @@ -1086,6 +1086,7 @@ "test:e2e:openshell": "OPENCLAW_E2E_OPENSHELL=1 vitest run --config vitest.e2e.config.ts test/openshell-sandbox.e2e.test.ts", "test:extension": "node scripts/test-extension.mjs", "test:extensions": "vitest run --config vitest.extensions.config.ts", + "test:extensions:batch": "node scripts/test-extension-batch.mjs", "test:extensions:memory": "node scripts/profile-extension-memory.mjs", "test:fast": "vitest run --config vitest.unit.config.ts", "test:force": "node --import tsx scripts/test-force.ts", diff --git a/scripts/lib/extension-test-plan.mjs b/scripts/lib/extension-test-plan.mjs index 8f350db10da..d5f728e0b56 100644 --- a/scripts/lib/extension-test-plan.mjs +++ b/scripts/lib/extension-test-plan.mjs @@ -2,8 +2,10 @@ import fs from "node:fs"; import path from "node:path"; import { channelTestRoots } from "../../vitest.channel-paths.mjs"; import { BUNDLED_PLUGIN_PATH_PREFIX, BUNDLED_PLUGIN_ROOT_DIR } from "./bundled-plugin-paths.mjs"; +import { listAvailableExtensionIds } from "./changed-extensions.mjs"; const repoRoot = path.resolve(import.meta.dirname, "..", ".."); +export const DEFAULT_EXTENSION_TEST_SHARD_COUNT = 6; function normalizeRelative(inputPath) { return inputPath.split(path.sep).join("/"); @@ -102,3 +104,100 @@ export function resolveExtensionTestPlan(params = {}) { testFileCount, }; } + +function mergeTestPlans(plans) { + const groupsByConfig = new Map(); + + for (const plan of plans) { + const current = groupsByConfig.get(plan.config) ?? { + config: plan.config, + extensionIds: [], + roots: [], + testFileCount: 0, + }; + + current.extensionIds.push(plan.extensionId); + current.roots.push(...plan.roots); + current.testFileCount += plan.testFileCount; + groupsByConfig.set(plan.config, current); + } + + const planGroups = [...groupsByConfig.values()] + .map((group) => ({ + ...group, + extensionIds: group.extensionIds.toSorted((left, right) => left.localeCompare(right)), + roots: [...new Set(group.roots)], + })) + .toSorted((left, right) => left.config.localeCompare(right.config)); + + return { + extensionCount: plans.length, + extensionIds: plans + .map((plan) => plan.extensionId) + .toSorted((left, right) => left.localeCompare(right)), + hasTests: plans.length > 0, + planGroups, + testFileCount: plans.reduce((sum, plan) => sum + plan.testFileCount, 0), + }; +} + +export function resolveExtensionBatchPlan(params = {}) { + const cwd = params.cwd ?? process.cwd(); + const extensionIds = params.extensionIds ?? listAvailableExtensionIds(); + const plans = extensionIds + .map((extensionId) => resolveExtensionTestPlan({ cwd, targetArg: extensionId })) + .filter((plan) => plan.hasTests); + + return mergeTestPlans(plans); +} + +function pickLeastLoadedShard(shards) { + return shards.reduce((bestIndex, shard, index) => { + if (bestIndex === -1) { + return index; + } + const best = shards[bestIndex]; + if (shard.testFileCount !== best.testFileCount) { + return shard.testFileCount < best.testFileCount ? index : bestIndex; + } + if (shard.plans.length !== best.plans.length) { + return shard.plans.length < best.plans.length ? index : bestIndex; + } + return index < bestIndex ? index : bestIndex; + }, -1); +} + +export function createExtensionTestShards(params = {}) { + const cwd = params.cwd ?? process.cwd(); + const extensionIds = params.extensionIds ?? listAvailableExtensionIds(); + const shardCount = Math.max(1, Number.parseInt(String(params.shardCount ?? ""), 10) || 1); + const plans = extensionIds + .map((extensionId) => resolveExtensionTestPlan({ cwd, targetArg: extensionId })) + .filter((plan) => plan.hasTests) + .toSorted((left, right) => { + if (left.testFileCount !== right.testFileCount) { + return right.testFileCount - left.testFileCount; + } + return left.extensionId.localeCompare(right.extensionId); + }); + + const effectiveShardCount = Math.min(shardCount, Math.max(1, plans.length)); + const shards = Array.from({ length: effectiveShardCount }, () => ({ + plans: [], + testFileCount: 0, + })); + + for (const plan of plans) { + const targetIndex = pickLeastLoadedShard(shards); + shards[targetIndex].plans.push(plan); + shards[targetIndex].testFileCount += plan.testFileCount; + } + + return shards + .map((shard, index) => ({ + index, + checkName: `checks-fast-extensions-shard-${index + 1}`, + ...mergeTestPlans(shard.plans), + })) + .filter((shard) => shard.hasTests); +} diff --git a/scripts/test-extension-batch.mjs b/scripts/test-extension-batch.mjs new file mode 100644 index 00000000000..1122adae3d0 --- /dev/null +++ b/scripts/test-extension-batch.mjs @@ -0,0 +1,105 @@ +#!/usr/bin/env node + +import { spawn } from "node:child_process"; +import path from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { resolveExtensionBatchPlan } from "./lib/extension-test-plan.mjs"; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); +const repoRoot = path.resolve(__dirname, ".."); +const pnpm = "pnpm"; + +async function runVitestBatch(params) { + return await new Promise((resolve, reject) => { + const child = spawn( + pnpm, + ["exec", "vitest", "run", "--config", params.config, ...params.targets, ...params.args], + { + cwd: repoRoot, + stdio: "inherit", + shell: process.platform === "win32", + env: params.env, + }, + ); + + child.on("error", reject); + child.on("exit", (code, signal) => { + if (signal) { + process.kill(process.pid, signal); + return; + } + resolve(code ?? 1); + }); + }); +} + +function printUsage() { + console.error("Usage: pnpm test:extensions:batch [vitest args...]"); + console.error( + " node scripts/test-extension-batch.mjs [vitest args...]", + ); +} + +function parseExtensionIds(rawArgs) { + const args = [...rawArgs]; + const extensionIds = []; + + while (args[0] && !args[0].startsWith("-")) { + extensionIds.push( + ...args + .shift() + .split(",") + .map((value) => value.trim()) + .filter(Boolean), + ); + } + + return { extensionIds, passthroughArgs: args }; +} + +async function run() { + const rawArgs = process.argv.slice(2); + if (rawArgs.includes("--help") || rawArgs.includes("-h")) { + printUsage(); + return; + } + + const passthroughArgs = rawArgs.filter((arg) => arg !== "--"); + const { extensionIds, passthroughArgs: vitestArgs } = parseExtensionIds(passthroughArgs); + if (extensionIds.length === 0) { + printUsage(); + process.exit(1); + } + + const batchPlan = resolveExtensionBatchPlan({ cwd: process.cwd(), extensionIds }); + if (!batchPlan.hasTests) { + console.log("[test-extension-batch] No tests found for the requested extensions. Skipping."); + return; + } + + console.log( + `[test-extension-batch] Running ${batchPlan.testFileCount} test files across ${batchPlan.extensionCount} extensions`, + ); + + for (const group of batchPlan.planGroups) { + console.log( + `[test-extension-batch] ${group.config}: ${group.extensionIds.join(", ")} (${group.testFileCount} files)`, + ); + const exitCode = await runVitestBatch({ + args: vitestArgs, + config: group.config, + env: process.env, + targets: group.roots, + }); + if (exitCode !== 0) { + process.exit(exitCode); + } + } +} + +const entryHref = process.argv[1] ? pathToFileURL(path.resolve(process.argv[1])).href : ""; + +if (import.meta.url === entryHref) { + await run(); +} diff --git a/test/scripts/test-extension.test.ts b/test/scripts/test-extension.test.ts index 3afa2e348b4..9d0bc68ee76 100644 --- a/test/scripts/test-extension.test.ts +++ b/test/scripts/test-extension.test.ts @@ -6,7 +6,12 @@ import { listAvailableExtensionIds, listChangedExtensionIds, } from "../../scripts/lib/changed-extensions.mjs"; -import { resolveExtensionTestPlan } from "../../scripts/lib/extension-test-plan.mjs"; +import { + DEFAULT_EXTENSION_TEST_SHARD_COUNT, + createExtensionTestShards, + resolveExtensionBatchPlan, + resolveExtensionTestPlan, +} from "../../scripts/lib/extension-test-plan.mjs"; import { bundledPluginFile, bundledPluginRoot } from "../helpers/bundled-plugin-paths.js"; const scriptPath = path.join(process.cwd(), "scripts", "test-extension.mjs"); @@ -103,6 +108,51 @@ describe("scripts/test-extension.mjs", () => { expect(plan.testFileCount).toBe(0); }); + it("batches extensions into config-specific vitest invocations", () => { + const batch = resolveExtensionBatchPlan({ + cwd: process.cwd(), + extensionIds: ["slack", "firecrawl", "line"], + }); + + expect(batch.extensionIds).toEqual(["firecrawl", "line", "slack"]); + expect(batch.planGroups).toEqual([ + { + config: "vitest.channels.config.ts", + extensionIds: ["slack"], + roots: [bundledPluginRoot("slack")], + testFileCount: expect.any(Number), + }, + { + config: "vitest.extensions.config.ts", + extensionIds: ["firecrawl", "line"], + roots: [bundledPluginRoot("firecrawl"), bundledPluginRoot("line")], + testFileCount: expect.any(Number), + }, + ]); + }); + + it("balances extension test shards by test file count", () => { + const shards = createExtensionTestShards({ + cwd: process.cwd(), + shardCount: DEFAULT_EXTENSION_TEST_SHARD_COUNT, + }); + + expect(shards).toHaveLength(DEFAULT_EXTENSION_TEST_SHARD_COUNT); + + const assigned = shards.flatMap((shard) => shard.extensionIds); + const uniqueAssigned = [...new Set(assigned)]; + const expected = listAvailableExtensionIds().filter( + (extensionId) => + resolveExtensionTestPlan({ cwd: process.cwd(), targetArg: extensionId }).hasTests, + ); + + expect(uniqueAssigned).toEqual(expected); + expect(assigned).toHaveLength(expected.length); + + const totals = shards.map((shard) => shard.testFileCount); + expect(Math.max(...totals) - Math.min(...totals)).toBeLessThanOrEqual(1); + }); + it("treats extensions without tests as a no-op by default", () => { const extensionId = findExtensionWithoutTests(); const stdout = runScript([extensionId]);