From ab37d8810dbee45296e4e87ad5bd58d5ef02e61c Mon Sep 17 00:00:00 2001 From: Tak Hoffman <781889+Takhoffman@users.noreply.github.com> Date: Wed, 25 Mar 2026 18:11:58 -0500 Subject: [PATCH] test: introduce planner-backed test runner, stabilize local builds (#54650) * test: stabilize ci and local vitest workers * test: introduce planner-backed test runner * test: address planner review follow-ups * test: derive planner budgets from host capabilities * test: restore planner filter helper import * test: align planner explain output with execution * test: keep low profile as serial alias * test: restrict explicit planner file targets * test: clean planner exits and pnpm launch * test: tighten wrapper flag validation * ci: gate heavy fanout on check * test: key shard assignments by unit identity * ci(bun): shard vitest lanes further * test: restore ci overlap and stabilize planner tests * test: relax planner output worker assertions * test: reset plugin runtime state in optional tools suite * ci: split macos node and swift jobs * test: honor no-isolate top-level concurrency budgets * ci: fix macos swift format lint * test: cap max-profile top-level concurrency * ci: shard macos node checks * ci: use four macos node shards * test: normalize explain targets before classification --- .github/workflows/ci-bun.yml | 14 +- .github/workflows/ci.yml | 167 +- AGENTS.md | 2 +- .../Sources/OpenClaw/OpenClawConfigFile.swift | 8 +- .../Sources/OpenClaw/SkillsSettings.swift | 18 +- package.json | 10 +- scripts/test-parallel.mjs | 1795 ++--------------- scripts/test-planner/catalog.mjs | 187 ++ scripts/test-planner/executor.mjs | 668 ++++++ scripts/test-planner/planner.mjs | 1023 ++++++++++ scripts/test-planner/runtime-profile.mjs | 348 ++++ scripts/test-planner/vitest-args.mjs | 74 + src/plugins/tools.optional.test.ts | 3 + test/scripts/test-parallel.test.ts | 192 +- test/scripts/test-planner.test.ts | 287 +++ test/vitest-config.test.ts | 156 ++ vitest.channels.config.ts | 1 - vitest.config.ts | 10 +- vitest.extensions.config.ts | 1 - 19 files changed, 3243 insertions(+), 1721 deletions(-) create mode 100644 scripts/test-planner/catalog.mjs create mode 100644 scripts/test-planner/executor.mjs create mode 100644 scripts/test-planner/planner.mjs create mode 100644 scripts/test-planner/runtime-profile.mjs create mode 100644 scripts/test-planner/vitest-args.mjs create mode 100644 test/scripts/test-planner.test.ts create mode 100644 test/vitest-config.test.ts diff --git a/.github/workflows/ci-bun.yml b/.github/workflows/ci-bun.yml index f8eb3b012e9..aeb20874274 100644 --- a/.github/workflows/ci-bun.yml +++ b/.github/workflows/ci-bun.yml @@ -45,11 +45,17 @@ jobs: matrix: include: - shard_index: 1 - shard_count: 2 - command: OPENCLAW_TEST_ISOLATE=1 bunx vitest run --config vitest.unit.config.ts --shard 1/2 + shard_count: 4 + command: OPENCLAW_TEST_ISOLATE=1 bunx vitest run --config vitest.unit.config.ts --shard 1/4 - shard_index: 2 - shard_count: 2 - command: OPENCLAW_TEST_ISOLATE=1 bunx vitest run --config vitest.unit.config.ts --shard 2/2 + shard_count: 4 + command: OPENCLAW_TEST_ISOLATE=1 bunx vitest run --config vitest.unit.config.ts --shard 2/4 + - shard_index: 3 + shard_count: 4 + command: OPENCLAW_TEST_ISOLATE=1 bunx vitest run --config vitest.unit.config.ts --shard 3/4 + - shard_index: 4 + shard_count: 4 + command: OPENCLAW_TEST_ISOLATE=1 bunx vitest run --config vitest.unit.config.ts --shard 4/4 steps: - name: Checkout uses: actions/checkout@v6 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 572a787b421..7f4e8af5785 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -201,9 +201,9 @@ jobs: - name: Audit production dependencies run: pre-commit run --config "${PRE_COMMIT_CONFIG_PATH:-.pre-commit-config.yaml}" --all-files pnpm-audit-prod - # Fanout: downstream lanes branch from preflight outputs instead of waiting - # on unrelated Linux checks. # Build dist once for Node-relevant changes and share it with downstream jobs. + # Keep this overlapping with the fast correctness lanes so green PRs get heavy + # test/build feedback sooner instead of waiting behind a full `check` pass. build-artifacts: needs: [scope] if: needs.scope.outputs.docs_only != 'true' && needs.scope.outputs.run_node == 'true' @@ -280,7 +280,7 @@ jobs: checks-fast: needs: [scope] - if: always() && needs.scope.outputs.docs_only != 'true' && needs.scope.outputs.run_node == 'true' + if: needs.scope.outputs.docs_only != 'true' && needs.scope.outputs.run_node == 'true' runs-on: blacksmith-16vcpu-ubuntu-2404 timeout-minutes: 20 strategy: @@ -820,13 +820,9 @@ jobs: - name: Run ${{ matrix.task }} (${{ matrix.runtime }}) run: ${{ matrix.command }} - # Consolidated macOS job: runs TS tests + Swift lint/build/test sequentially - # on a single runner. GitHub limits macOS concurrent jobs to 5 per org; - # running 4 separate jobs per PR (as before) starved the queue. One job - # per PR allows 5 PRs to run macOS checks simultaneously. - macos: - needs: [scope] - if: github.event_name == 'pull_request' && needs.scope.outputs.docs_only != 'true' && needs.scope.outputs.run_macos == 'true' + macos-node-1: + needs: [scope, build-artifacts] + if: always() && github.event_name == 'pull_request' && needs.scope.outputs.docs_only != 'true' && needs.scope.outputs.run_macos == 'true' && needs.build-artifacts.result == 'success' runs-on: macos-latest timeout-minutes: 20 steps: @@ -841,16 +837,157 @@ jobs: with: install-bun: "false" - - name: Build dist (macOS) - run: pnpm build + - name: Download dist artifact + uses: actions/download-artifact@v8 + with: + name: dist-build + path: dist/ - # --- Run all checks sequentially (fast gates first) --- - - name: TS tests (macOS) + - name: Download A2UI bundle artifact + uses: actions/download-artifact@v8 + with: + name: canvas-a2ui-bundle + path: src/canvas-host/a2ui/ + + - name: Configure test shard (macOS 1/4) + run: | + echo "OPENCLAW_TEST_SHARDS=4" >> "$GITHUB_ENV" + echo "OPENCLAW_TEST_SHARD_INDEX=1" >> "$GITHUB_ENV" + + - name: TS tests (macOS 1/4) env: NODE_OPTIONS: --max-old-space-size=4096 run: pnpm test - # --- Xcode/Swift setup --- + macos-node-2: + needs: [scope, build-artifacts] + if: always() && github.event_name == 'pull_request' && needs.scope.outputs.docs_only != 'true' && needs.scope.outputs.run_macos == 'true' && needs.build-artifacts.result == 'success' + runs-on: macos-latest + timeout-minutes: 20 + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + persist-credentials: false + submodules: false + + - name: Setup Node environment + uses: ./.github/actions/setup-node-env + with: + install-bun: "false" + + - name: Download dist artifact + uses: actions/download-artifact@v8 + with: + name: dist-build + path: dist/ + + - name: Download A2UI bundle artifact + uses: actions/download-artifact@v8 + with: + name: canvas-a2ui-bundle + path: src/canvas-host/a2ui/ + + - name: Configure test shard (macOS 2/4) + run: | + echo "OPENCLAW_TEST_SHARDS=4" >> "$GITHUB_ENV" + echo "OPENCLAW_TEST_SHARD_INDEX=2" >> "$GITHUB_ENV" + + - name: TS tests (macOS 2/4) + env: + NODE_OPTIONS: --max-old-space-size=4096 + run: pnpm test + + macos-node-3: + needs: [scope, build-artifacts] + if: always() && github.event_name == 'pull_request' && needs.scope.outputs.docs_only != 'true' && needs.scope.outputs.run_macos == 'true' && needs.build-artifacts.result == 'success' + runs-on: macos-latest + timeout-minutes: 20 + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + persist-credentials: false + submodules: false + + - name: Setup Node environment + uses: ./.github/actions/setup-node-env + with: + install-bun: "false" + + - name: Download dist artifact + uses: actions/download-artifact@v8 + with: + name: dist-build + path: dist/ + + - name: Download A2UI bundle artifact + uses: actions/download-artifact@v8 + with: + name: canvas-a2ui-bundle + path: src/canvas-host/a2ui/ + + - name: Configure test shard (macOS 3/4) + run: | + echo "OPENCLAW_TEST_SHARDS=4" >> "$GITHUB_ENV" + echo "OPENCLAW_TEST_SHARD_INDEX=3" >> "$GITHUB_ENV" + + - name: TS tests (macOS 3/4) + env: + NODE_OPTIONS: --max-old-space-size=4096 + run: pnpm test + + macos-node-4: + needs: [scope, build-artifacts] + if: always() && github.event_name == 'pull_request' && needs.scope.outputs.docs_only != 'true' && needs.scope.outputs.run_macos == 'true' && needs.build-artifacts.result == 'success' + runs-on: macos-latest + timeout-minutes: 20 + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + persist-credentials: false + submodules: false + + - name: Setup Node environment + uses: ./.github/actions/setup-node-env + with: + install-bun: "false" + + - name: Download dist artifact + uses: actions/download-artifact@v8 + with: + name: dist-build + path: dist/ + + - name: Download A2UI bundle artifact + uses: actions/download-artifact@v8 + with: + name: canvas-a2ui-bundle + path: src/canvas-host/a2ui/ + + - name: Configure test shard (macOS 4/4) + run: | + echo "OPENCLAW_TEST_SHARDS=4" >> "$GITHUB_ENV" + echo "OPENCLAW_TEST_SHARD_INDEX=4" >> "$GITHUB_ENV" + + - name: TS tests (macOS 4/4) + env: + NODE_OPTIONS: --max-old-space-size=4096 + run: pnpm test + + macos-swift: + needs: [scope] + if: github.event_name == 'pull_request' && needs.scope.outputs.docs_only != 'true' && needs.scope.outputs.run_macos == 'true' + runs-on: macos-latest + timeout-minutes: 20 + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + persist-credentials: false + submodules: false + - name: Select Xcode 26.1 run: | sudo xcode-select -s /Applications/Xcode_26.1.app diff --git a/AGENTS.md b/AGENTS.md index eba43e2ed72..40094095f4e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -120,7 +120,7 @@ - For targeted/local debugging, keep using the wrapper: `pnpm test -- [vitest args...]` (for example `pnpm test -- src/commands/onboard-search.test.ts -t "shows registered plugin providers"`); do not default to raw `pnpm vitest run ...` because it bypasses wrapper config/profile/pool routing. - Do not set test workers above 16; tried already. - Keep Vitest on `forks` only. Do not introduce or reintroduce any non-`forks` Vitest pool or alternate execution mode in configs, wrapper scripts, or default test commands without explicit approval in this chat. This includes `threads`, `vmThreads`, `vmForks`, and any future/nonstandard pool variant. -- If local Vitest runs cause memory pressure (common on non-Mac-Studio hosts), use `OPENCLAW_TEST_PROFILE=low OPENCLAW_TEST_SERIAL_GATEWAY=1 pnpm test` for land/gate runs. +- If local Vitest runs cause memory pressure, the wrapper now derives budgets from host capabilities (CPU, memory band, current load). For a conservative explicit override during land/gate runs, use `OPENCLAW_TEST_PROFILE=serial OPENCLAW_TEST_SERIAL_GATEWAY=1 pnpm test`. - Live tests (real keys): `OPENCLAW_LIVE_TEST=1 pnpm test:live` (OpenClaw-only) or `LIVE=1 pnpm test:live` (includes provider live tests). Docker: `pnpm test:docker:live-models`, `pnpm test:docker:live-gateway`. Onboarding Docker E2E: `pnpm test:docker:onboard`. - Full kit + what’s covered: `docs/help/testing.md`. - Changelog: user-facing changes only; no internal/meta notes (version alignment, appcast reminders, release process). diff --git a/apps/macos/Sources/OpenClaw/OpenClawConfigFile.swift b/apps/macos/Sources/OpenClaw/OpenClawConfigFile.swift index 8bc2e620881..32d79040e3a 100644 --- a/apps/macos/Sources/OpenClaw/OpenClawConfigFile.swift +++ b/apps/macos/Sources/OpenClaw/OpenClawConfigFile.swift @@ -493,8 +493,12 @@ enum OpenClawConfigFile { return } - let backup = self.readConfigFingerprint(at: configURL.deletingLastPathComponent().appendingPathComponent("\(configURL.lastPathComponent).bak")) - let clobberedPath = self.persistClobberedSnapshot(data: data, configURL: configURL, observedAt: observedAt) + let backup = self.readConfigFingerprint( + at: configURL.deletingLastPathComponent().appendingPathComponent("\(configURL.lastPathComponent).bak")) + let clobberedPath = self.persistClobberedSnapshot( + data: data, + configURL: configURL, + observedAt: observedAt) self.logger.warning("config observe anomaly (\(suspicious.joined(separator: ", "))) at \(configURL.path)") self.appendConfigObserveAudit([ "phase": "read", diff --git a/apps/macos/Sources/OpenClaw/SkillsSettings.swift b/apps/macos/Sources/OpenClaw/SkillsSettings.swift index d3733d77b54..6f6cfed1915 100644 --- a/apps/macos/Sources/OpenClaw/SkillsSettings.swift +++ b/apps/macos/Sources/OpenClaw/SkillsSettings.swift @@ -259,9 +259,12 @@ private struct SkillRow: View { guard let raw = self.skill.homepage?.trimmingCharacters(in: .whitespacesAndNewlines) else { return nil } - guard !raw.isEmpty, let url = URL(string: raw), - let scheme = url.scheme?.lowercased(), - scheme == "http" || scheme == "https" else { + guard + !raw.isEmpty, + let url = URL(string: raw), + let scheme = url.scheme?.lowercased(), + scheme == "http" || scheme == "https" + else { return nil } return url @@ -481,9 +484,12 @@ private struct EnvEditorView: View { guard let raw = self.editor.homepage?.trimmingCharacters(in: .whitespacesAndNewlines) else { return nil } - guard !raw.isEmpty, let url = URL(string: raw), - let scheme = url.scheme?.lowercased(), - scheme == "http" || scheme == "https" else { + guard + !raw.isEmpty, + let url = URL(string: raw), + let scheme = url.scheme?.lowercased(), + scheme == "http" || scheme == "https" + else { return nil } return url diff --git a/package.json b/package.json index 5bb8ac95d6e..762f40bf3fd 100644 --- a/package.json +++ b/package.json @@ -701,10 +701,10 @@ "test:auth:compat": "vitest run --config vitest.gateway.config.ts src/gateway/server.auth.compat-baseline.test.ts src/gateway/client.test.ts src/gateway/reconnect-gating.test.ts src/gateway/protocol/connect-error-details.test.ts", "test:build:singleton": "node scripts/test-built-plugin-singleton.mjs", "test:changed": "pnpm test -- --changed origin/main", - "test:channels": "OPENCLAW_TEST_SKIP_DEFAULT=1 OPENCLAW_TEST_INCLUDE_CHANNELS=1 node scripts/test-parallel.mjs", + "test:channels": "node scripts/test-parallel.mjs --surface channels", "test:contracts": "pnpm test:contracts:channels && pnpm test:contracts:plugins", - "test:contracts:channels": "OPENCLAW_TEST_PROFILE=low pnpm test -- src/channels/plugins/contracts", - "test:contracts:plugins": "OPENCLAW_TEST_PROFILE=low pnpm test -- src/plugins/contracts", + "test:contracts:channels": "OPENCLAW_TEST_PROFILE=serial pnpm test -- src/channels/plugins/contracts", + "test:contracts:plugins": "OPENCLAW_TEST_PROFILE=serial pnpm test -- src/plugins/contracts", "test:coverage": "vitest run --config vitest.unit.config.ts --coverage", "test:coverage:changed": "vitest run --config vitest.unit.config.ts --coverage --changed origin/main", "test:docker:all": "pnpm test:docker:live-models && pnpm test:docker:live-gateway && pnpm test:docker:openwebui && pnpm test:docker:onboard && pnpm test:docker:gateway-network && pnpm test:docker:qr && pnpm test:docker:doctor-switch && pnpm test:docker:plugins && pnpm test:docker:cleanup", @@ -720,7 +720,7 @@ "test:e2e": "vitest run --config vitest.e2e.config.ts", "test:e2e:openshell": "OPENCLAW_E2E_OPENSHELL=1 vitest run --config vitest.e2e.config.ts test/openshell-sandbox.e2e.test.ts", "test:extension": "node scripts/test-extension.mjs", - "test:extensions": "OPENCLAW_TEST_SKIP_DEFAULT=1 OPENCLAW_TEST_INCLUDE_EXTENSIONS=1 node scripts/test-parallel.mjs", + "test:extensions": "node scripts/test-parallel.mjs --surface extensions", "test:extensions:memory": "node scripts/profile-extension-memory.mjs", "test:fast": "vitest run --config vitest.unit.config.ts", "test:force": "node --import tsx scripts/test-force.ts", @@ -731,7 +731,6 @@ "test:install:e2e:openai": "OPENCLAW_E2E_MODELS=openai bash scripts/test-install-sh-e2e-docker.sh", "test:install:smoke": "bash scripts/test-install-sh-docker.sh", "test:live": "OPENCLAW_LIVE_TEST=1 vitest run --config vitest.live.config.ts", - "test:macmini": "OPENCLAW_TEST_PROFILE=macmini node scripts/test-parallel.mjs", "test:parallels:linux": "bash scripts/e2e/parallels-linux-smoke.sh", "test:parallels:macos": "bash scripts/e2e/parallels-macos-smoke.sh", "test:parallels:npm-update": "bash scripts/e2e/parallels-npm-update-smoke.sh", @@ -745,6 +744,7 @@ "test:perf:update-memory-hotspots": "node scripts/test-update-memory-hotspots.mjs", "test:perf:update-timings": "node scripts/test-update-timings.mjs", "test:sectriage": "pnpm exec vitest run --config vitest.gateway.config.ts && vitest run --config vitest.unit.config.ts --exclude src/daemon/launchd.integration.test.ts --exclude src/process/exec.test.ts", + "test:serial": "node scripts/test-parallel.mjs --profile serial", "test:startup:memory": "node scripts/check-cli-startup-memory.mjs", "test:ui": "pnpm lint:ui:no-raw-window-open && pnpm --dir ui test", "test:voicecall:closedloop": "vitest run extensions/voice-call/src/manager.test.ts extensions/voice-call/src/media-stream.test.ts src/plugins/voice-call.plugin.test.ts --maxWorkers=1", diff --git a/scripts/test-parallel.mjs b/scripts/test-parallel.mjs index bd41f49bf17..c530c3fcbf5 100644 --- a/scripts/test-parallel.mjs +++ b/scripts/test-parallel.mjs @@ -1,1703 +1,152 @@ -import { spawn } from "node:child_process"; -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; -import { channelTestPrefixes } from "../vitest.channel-paths.mjs"; -import { isUnitConfigTestFile } from "../vitest.unit-paths.mjs"; import { - getProcessTreeRecords, - parseCompletedTestFileLines, - sampleProcessTreeRssKb, -} from "./test-parallel-memory.mjs"; -import { - appendCapturedOutput, - formatCapturedOutputTail, - hasFatalTestRunOutput, - resolveTestRunExitCode, -} from "./test-parallel-utils.mjs"; -import { - dedupeFilesPreserveOrder, - loadChannelTimingManifest, - loadUnitMemoryHotspotManifest, - loadTestRunnerBehavior, - loadUnitTimingManifest, - selectUnitHeavyFileGroups, - packFilesByDuration, - packFilesByDurationWithBaseLoads, -} from "./test-runner-manifest.mjs"; + createExecutionArtifacts, + executePlan, + formatExplanation, + formatPlanOutput, +} from "./test-planner/executor.mjs"; +import { buildExecutionPlan, explainExecutionTarget } from "./test-planner/planner.mjs"; -// On Windows, `.cmd` launchers can fail with `spawn EINVAL` when invoked without a shell -// (especially under GitHub Actions + Git Bash). Use `shell: true` and let the shell resolve pnpm. -const pnpm = "pnpm"; -const behaviorManifest = loadTestRunnerBehavior(); -const existingFiles = (entries) => - entries.map((entry) => entry.file).filter((file) => fs.existsSync(file)); -let tempArtifactDir = null; -const ensureTempArtifactDir = () => { - if (tempArtifactDir === null) { - tempArtifactDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-test-parallel-")); - } - return tempArtifactDir; -}; -const writeTempJsonArtifact = (name, value) => { - const filePath = path.join(ensureTempArtifactDir(), `${name}.json`); - fs.writeFileSync(filePath, `${JSON.stringify(value)}\n`, "utf8"); - return filePath; -}; -const sanitizeArtifactName = (value) => { - const normalized = value - .trim() - .replace(/[^a-z0-9._-]+/giu, "-") - .replace(/^-+|-+$/gu, ""); - return normalized || "artifact"; -}; -const cleanupTempArtifacts = () => { - if (tempArtifactDir === null) { - return; - } - if (process.env.OPENCLAW_TEST_KEEP_TEMP_ARTIFACTS === "1") { - console.error(`[test-parallel] keeping temp artifacts at ${tempArtifactDir}`); - return; - } - fs.rmSync(tempArtifactDir, { recursive: true, force: true }); - tempArtifactDir = null; -}; -const existingUnitConfigFiles = (entries) => existingFiles(entries).filter(isUnitConfigTestFile); -const baseThreadPinnedFiles = existingFiles(behaviorManifest.base?.threadPinned ?? []); -const channelIsolatedManifestFiles = existingFiles(behaviorManifest.channels?.isolated ?? []); -const channelIsolatedPrefixes = behaviorManifest.channels?.isolatedPrefixes ?? []; -const extensionForkIsolatedFiles = existingFiles(behaviorManifest.extensions?.isolated ?? []); -const unitForkIsolatedFiles = existingUnitConfigFiles(behaviorManifest.unit.isolated); -const unitThreadPinnedFiles = existingUnitConfigFiles(behaviorManifest.unit.threadPinned); -const unitBehaviorOverrideSet = new Set([...unitForkIsolatedFiles, ...unitThreadPinnedFiles]); - -const children = new Set(); -const isCI = process.env.CI === "true" || process.env.GITHUB_ACTIONS === "true"; -const isMacOS = process.platform === "darwin" || process.env.RUNNER_OS === "macOS"; -const isWindows = process.platform === "win32" || process.env.RUNNER_OS === "Windows"; -const isWindowsCi = isCI && isWindows; -const hostCpuCount = os.cpus().length; -const hostMemoryGiB = Math.floor(os.totalmem() / 1024 ** 3); -// Keep aggressive local defaults for high-memory workstations (Mac Studio class). -const highMemLocalHost = !isCI && hostMemoryGiB >= 96; -const lowMemLocalHost = !isCI && hostMemoryGiB < 64; -const nodeMajor = Number.parseInt(process.versions.node.split(".")[0] ?? "", 10); -const rawTestProfile = process.env.OPENCLAW_TEST_PROFILE?.trim().toLowerCase(); -const testProfile = - rawTestProfile === "low" || - rawTestProfile === "macmini" || - rawTestProfile === "max" || - rawTestProfile === "normal" || - rawTestProfile === "serial" - ? rawTestProfile - : "normal"; -const isMacMiniProfile = testProfile === "macmini"; -// Vitest executes Node tests through Vite's SSR/module-runner pipeline, so the -// shared unit lane still retains transformed ESM/module state even when the -// tests themselves are not "server rendering" a website. Keep forks as the -// only active pool so local and CI behavior stay aligned. -const forceIsolation = - process.env.OPENCLAW_TEST_ISOLATE === "1" || process.env.OPENCLAW_TEST_ISOLATE === "true"; -const disableIsolation = - !forceIsolation && - process.env.OPENCLAW_TEST_NO_ISOLATE !== "0" && - process.env.OPENCLAW_TEST_NO_ISOLATE !== "false"; -const includeGatewaySuite = process.env.OPENCLAW_TEST_INCLUDE_GATEWAY === "1"; -const includeChannelsSuite = process.env.OPENCLAW_TEST_INCLUDE_CHANNELS === "1"; -const includeExtensionsSuite = process.env.OPENCLAW_TEST_INCLUDE_EXTENSIONS === "1"; -const noIsolateArgs = disableIsolation ? ["--isolate=false"] : []; -const skipDefaultRuns = process.env.OPENCLAW_TEST_SKIP_DEFAULT === "1"; -// Even on low-memory or fully serial hosts, keep the unit lane split so -// long-lived workers do not accumulate the whole unit transform graph. -const shouldSplitUnitRuns = true; -const useLowProfileUnitSchedulingDefaults = testProfile === "low" || testProfile === "serial"; -let runs = []; -const shardOverride = Number.parseInt(process.env.OPENCLAW_TEST_SHARDS ?? "", 10); -const configuredShardCount = - Number.isFinite(shardOverride) && shardOverride > 1 ? shardOverride : null; -const shardCount = configuredShardCount ?? (isWindowsCi ? 2 : 1); -const shardIndexOverride = (() => { - const parsed = Number.parseInt(process.env.OPENCLAW_TEST_SHARD_INDEX ?? "", 10); - return Number.isFinite(parsed) && parsed > 0 ? parsed : null; -})(); -const OPTION_TAKES_VALUE = new Set([ - "-t", - "-c", - "-r", - "--testNamePattern", - "--config", - "--root", - "--dir", - "--reporter", - "--outputFile", - "--pool", - "--execArgv", - "--vmMemoryLimit", - "--maxWorkers", - "--environment", - "--shard", - "--changed", - "--sequence", - "--inspect", - "--inspectBrk", - "--testTimeout", - "--hookTimeout", - "--bail", - "--retry", - "--diff", - "--exclude", - "--project", - "--slowTestThreshold", - "--teardownTimeout", - "--attachmentsDir", - "--mode", - "--api", - "--browser", - "--maxConcurrency", - "--mergeReports", - "--configLoader", - "--experimental", -]); -const SINGLE_RUN_ONLY_FLAGS = new Set(["--coverage", "--outputFile", "--mergeReports"]); - -if (shardIndexOverride !== null && shardCount <= 1) { - console.error( - `[test-parallel] OPENCLAW_TEST_SHARD_INDEX=${String( - shardIndexOverride, - )} requires OPENCLAW_TEST_SHARDS>1.`, - ); - process.exit(2); -} - -if (shardIndexOverride !== null && shardIndexOverride > shardCount) { - console.error( - `[test-parallel] OPENCLAW_TEST_SHARD_INDEX=${String( - shardIndexOverride, - )} exceeds OPENCLAW_TEST_SHARDS=${String(shardCount)}.`, - ); - process.exit(2); -} -const windowsCiArgs = isWindowsCi ? ["--dangerouslyIgnoreUnhandledErrors"] : []; -const silentArgs = - process.env.OPENCLAW_TEST_SHOW_PASSED_LOGS === "1" ? [] : ["--silent=passed-only"]; -const rawPassthroughArgs = process.argv.slice(2); -const passthroughArgs = - rawPassthroughArgs[0] === "--" ? rawPassthroughArgs.slice(1) : rawPassthroughArgs; -const parsePassthroughArgs = (args) => { - const fileFilters = []; - const optionArgs = []; - let consumeNextAsOptionValue = false; - - for (const arg of args) { - if (consumeNextAsOptionValue) { - optionArgs.push(arg); - consumeNextAsOptionValue = false; +const parseCliArgs = (args) => { + const wrapper = { + plan: false, + explain: null, + mode: null, + profile: null, + surfaces: [], + files: [], + passthroughArgs: [], + showHelp: false, + }; + let passthroughMode = false; + for (let index = 0; index < args.length; index += 1) { + const arg = args[index]; + if (passthroughMode) { + wrapper.passthroughArgs.push(arg); continue; } if (arg === "--") { - optionArgs.push(arg); + passthroughMode = true; continue; } - if (arg.startsWith("-")) { - optionArgs.push(arg); - consumeNextAsOptionValue = !arg.includes("=") && OPTION_TAKES_VALUE.has(arg); + if (arg === "--plan") { + wrapper.plan = true; continue; } - fileFilters.push(arg); - } - - return { fileFilters, optionArgs }; -}; -const { fileFilters: passthroughFileFilters, optionArgs: passthroughOptionArgs } = - parsePassthroughArgs(passthroughArgs); -const passthroughMetadataFlags = new Set(["-h", "--help", "--listTags", "--clearCache"]); -const passthroughMetadataOnly = - passthroughArgs.length > 0 && - passthroughFileFilters.length === 0 && - passthroughOptionArgs.every((arg) => { - if (!arg.startsWith("-")) { - return false; - } - const [flag] = arg.split("=", 1); - return passthroughMetadataFlags.has(flag); - }); -const countExplicitEntryFilters = (entryArgs) => { - const { fileFilters } = parsePassthroughArgs(entryArgs.slice(2)); - return fileFilters.length > 0 ? fileFilters.length : null; -}; -const getExplicitEntryFilters = (entryArgs) => parsePassthroughArgs(entryArgs.slice(2)).fileFilters; -const passthroughRequiresSingleRun = passthroughOptionArgs.some((arg) => { - if (!arg.startsWith("-")) { - return false; - } - const [flag] = arg.split("=", 1); - return SINGLE_RUN_ONLY_FLAGS.has(flag); -}); -const baseConfigPrefixes = ["src/agents/", "src/auto-reply/", "src/commands/", "test/", "ui/"]; -const normalizeRepoPath = (value) => value.split(path.sep).join("/"); -const walkTestFiles = (rootDir) => { - if (!fs.existsSync(rootDir)) { - return []; - } - const entries = fs.readdirSync(rootDir, { withFileTypes: true }); - const files = []; - for (const entry of entries) { - const fullPath = path.join(rootDir, entry.name); - if (entry.isDirectory()) { - files.push(...walkTestFiles(fullPath)); + if (arg === "--help") { + wrapper.showHelp = true; continue; } - if (!entry.isFile()) { + if (arg === "--mode") { + const nextValue = args[index + 1] ?? null; + if (nextValue === "ci" || nextValue === "local") { + wrapper.mode = nextValue; + index += 1; + continue; + } + } + if (arg === "--profile") { + const nextValue = args[index + 1] ?? ""; + if (!nextValue || nextValue === "--" || nextValue.startsWith("-")) { + throw new Error(`Invalid --profile value: ${String(nextValue || "")}`); + } + wrapper.profile = nextValue; + index += 1; continue; } - if ( - fullPath.endsWith(".test.ts") || - fullPath.endsWith(".live.test.ts") || - fullPath.endsWith(".e2e.test.ts") - ) { - files.push(normalizeRepoPath(fullPath)); - } - } - return files; -}; -const allKnownTestFiles = [ - ...new Set([ - ...walkTestFiles("src"), - ...walkTestFiles("extensions"), - ...walkTestFiles("test"), - ...walkTestFiles(path.join("ui", "src", "ui")), - ]), -]; -const channelIsolatedFiles = dedupeFilesPreserveOrder([ - ...channelIsolatedManifestFiles, - ...allKnownTestFiles.filter((file) => - channelIsolatedPrefixes.some((prefix) => file.startsWith(prefix)), - ), -]); -const channelIsolatedFileSet = new Set(channelIsolatedFiles); -const isTargetedIsolatedUnitFile = (fileFilter) => - unitForkIsolatedFiles.includes(fileFilter) || unitMemoryIsolatedFiles.includes(fileFilter); -const isLegacyBasePinnedFile = (fileFilter) => baseThreadPinnedFiles.includes(fileFilter); -const inferTarget = (fileFilter) => { - const isolated = - isTargetedIsolatedUnitFile(fileFilter) || - extensionForkIsolatedFiles.includes(fileFilter) || - channelIsolatedFileSet.has(fileFilter); - if (isUnitConfigTestFile(fileFilter)) { - return { owner: "unit", isolated }; - } - if (fileFilter.endsWith(".live.test.ts")) { - return { owner: "live", isolated }; - } - if (fileFilter.endsWith(".e2e.test.ts")) { - return { owner: "e2e", isolated }; - } - if (channelTestPrefixes.some((prefix) => fileFilter.startsWith(prefix))) { - return { owner: "channels", isolated }; - } - if (fileFilter.startsWith("extensions/")) { - return { owner: "extensions", isolated }; - } - if (fileFilter.startsWith("src/gateway/")) { - return { owner: "gateway", isolated }; - } - if (baseConfigPrefixes.some((prefix) => fileFilter.startsWith(prefix))) { - return { owner: "base", isolated }; - } - if (fileFilter.startsWith("src/")) { - return { owner: "unit", isolated }; - } - return { owner: "base", isolated }; -}; -const unitTimingManifest = loadUnitTimingManifest(); -const channelTimingManifest = loadChannelTimingManifest(); -const unitMemoryHotspotManifest = loadUnitMemoryHotspotManifest(); -const parseEnvNumber = (name, fallback) => { - const parsed = Number.parseInt(process.env[name] ?? "", 10); - return Number.isFinite(parsed) && parsed >= 0 ? parsed : fallback; -}; -const allKnownUnitFiles = allKnownTestFiles.filter((file) => { - return isUnitConfigTestFile(file); -}); -const defaultHeavyUnitFileLimit = isMacMiniProfile - ? 90 - : useLowProfileUnitSchedulingDefaults - ? 36 - : highMemLocalHost - ? 80 - : 60; -const defaultHeavyUnitLaneCount = isMacMiniProfile - ? 6 - : useLowProfileUnitSchedulingDefaults - ? 4 - : highMemLocalHost - ? 5 - : 4; -const heavyUnitFileLimit = parseEnvNumber( - "OPENCLAW_TEST_HEAVY_UNIT_FILE_LIMIT", - defaultHeavyUnitFileLimit, -); -const heavyUnitLaneCount = parseEnvNumber( - "OPENCLAW_TEST_HEAVY_UNIT_LANES", - defaultHeavyUnitLaneCount, -); -const heavyUnitMinDurationMs = parseEnvNumber("OPENCLAW_TEST_HEAVY_UNIT_MIN_MS", 1200); -const defaultMemoryHeavyUnitFileLimit = isCI ? 64 : useLowProfileUnitSchedulingDefaults ? 8 : 16; -const memoryHeavyUnitFileLimit = parseEnvNumber( - "OPENCLAW_TEST_MEMORY_HEAVY_UNIT_FILE_LIMIT", - defaultMemoryHeavyUnitFileLimit, -); -const memoryHeavyUnitMinDeltaKb = parseEnvNumber( - "OPENCLAW_TEST_MEMORY_HEAVY_UNIT_MIN_KB", - unitMemoryHotspotManifest.defaultMinDeltaKb, -); -const { memoryHeavyFiles: memoryHeavyUnitFiles, timedHeavyFiles: timedHeavyUnitFiles } = - shouldSplitUnitRuns - ? selectUnitHeavyFileGroups({ - candidates: allKnownUnitFiles, - behaviorOverrides: unitBehaviorOverrideSet, - timedLimit: heavyUnitFileLimit, - timedMinDurationMs: heavyUnitMinDurationMs, - memoryLimit: memoryHeavyUnitFileLimit, - memoryMinDeltaKb: memoryHeavyUnitMinDeltaKb, - timings: unitTimingManifest, - hotspots: unitMemoryHotspotManifest, - }) - : { - memoryHeavyFiles: [], - timedHeavyFiles: [], - }; -const unitMemoryIsolatedFiles = dedupeFilesPreserveOrder( - memoryHeavyUnitFiles, - unitBehaviorOverrideSet, -); -const unitSchedulingOverrideSet = new Set([...unitBehaviorOverrideSet, ...memoryHeavyUnitFiles]); -const unitFastExcludedFiles = [ - ...new Set([...unitSchedulingOverrideSet, ...timedHeavyUnitFiles, ...channelIsolatedFiles]), -]; -const estimateUnitDurationMs = (file) => - unitTimingManifest.files[file]?.durationMs ?? unitTimingManifest.defaultDurationMs; -const estimateChannelDurationMs = (file) => - channelTimingManifest.files[file]?.durationMs ?? channelTimingManifest.defaultDurationMs; -const resolveEntryTimingEstimator = (entry) => { - const configIndex = entry.args.findIndex((arg) => arg === "--config"); - const config = configIndex >= 0 ? (entry.args[configIndex + 1] ?? "") : ""; - if (config === "vitest.unit.config.ts") { - return estimateUnitDurationMs; - } - if (config === "vitest.channels.config.ts") { - return estimateChannelDurationMs; - } - if (config === "vitest.extensions.config.ts") { - return estimateChannelDurationMs; - } - return null; -}; -const estimateEntryFilesDurationMs = (entry, files) => { - const estimateDurationMs = resolveEntryTimingEstimator(entry); - if (!estimateDurationMs) { - return files.length * 1_000; - } - return files.reduce((totalMs, file) => totalMs + estimateDurationMs(file), 0); -}; -const splitFilesByDurationBudget = (files, targetDurationMs, estimateDurationMs) => { - if (!Number.isFinite(targetDurationMs) || targetDurationMs <= 0 || files.length <= 1) { - return [files]; - } - - const batches = []; - let currentBatch = []; - let currentDurationMs = 0; - - for (const file of files) { - const durationMs = estimateDurationMs(file); - if (currentBatch.length > 0 && currentDurationMs + durationMs > targetDurationMs) { - batches.push(currentBatch); - currentBatch = []; - currentDurationMs = 0; - } - currentBatch.push(file); - currentDurationMs += durationMs; - } - - if (currentBatch.length > 0) { - batches.push(currentBatch); - } - - return batches; -}; -const unitFastExcludedFileSet = new Set(unitFastExcludedFiles); -const unitFastCandidateFiles = allKnownUnitFiles.filter( - (file) => !unitFastExcludedFileSet.has(file), -); -const extensionForkIsolatedFileSet = new Set(extensionForkIsolatedFiles); -const extensionSharedCandidateFiles = allKnownTestFiles.filter( - (file) => file.startsWith("extensions/") && !extensionForkIsolatedFileSet.has(file), -); -const channelSharedCandidateFiles = allKnownTestFiles.filter( - (file) => - channelTestPrefixes.some((prefix) => file.startsWith(prefix)) && - !channelIsolatedFileSet.has(file), -); -const defaultExtensionsBatchTargetMs = isCI && !isWindows ? 30_000 : 0; -const extensionsBatchTargetMs = parseEnvNumber( - "OPENCLAW_TEST_EXTENSIONS_BATCH_TARGET_MS", - defaultExtensionsBatchTargetMs, -); -const extensionIsolatedEntries = extensionForkIsolatedFiles.map((file) => ({ - name: `extensions-${path.basename(file, ".test.ts")}-isolated`, - args: ["vitest", "run", "--config", "vitest.extensions.config.ts", "--pool=forks", file], -})); -// Shared extensions workers can retain a very large transform graph across -// hundreds of plugin files on forks/non-isolated runs. Recycle that lane in -// bounded batches so teardown happens before the worker reaches CI memory-cliff -// territory and starts surfacing spurious worker-shutdown errors. -const extensionsSharedBatches = splitFilesByDurationBudget( - extensionSharedCandidateFiles, - extensionsBatchTargetMs, - estimateChannelDurationMs, -); -const extensionsSharedEntries = extensionsSharedBatches - .filter((batch) => batch.length > 0) - .map((batch, batchIndex) => ({ - name: - extensionsSharedBatches.length === 1 - ? "extensions" - : `extensions-batch-${String(batchIndex + 1)}`, - serialPhase: "extensions", - includeFiles: batch, - estimatedDurationMs: estimateEntryFilesDurationMs( - { args: ["vitest", "run", "--config", "vitest.extensions.config.ts"] }, - batch, - ), - env: { - OPENCLAW_VITEST_INCLUDE_FILE: writeTempJsonArtifact( - `vitest-extensions-include-${String(batchIndex + 1)}`, - batch, - ), - }, - args: ["vitest", "run", "--config", "vitest.extensions.config.ts", ...noIsolateArgs], - })); -const channelIsolatedEntries = channelIsolatedFiles.map((file) => ({ - name: `${path.basename(file, ".test.ts")}-channels-isolated`, - args: ["vitest", "run", "--config", "vitest.channels.config.ts", "--pool=forks", file], -})); -const defaultUnitFastLaneCount = testProfile === "low" ? 8 : isCI && !isWindows ? 3 : 1; -const unitFastLaneCount = Math.max( - 1, - parseEnvNumber("OPENCLAW_TEST_UNIT_FAST_LANES", defaultUnitFastLaneCount), -); -const defaultUnitFastBatchTargetMs = useLowProfileUnitSchedulingDefaults - ? 10_000 - : isCI && !isWindows - ? 45_000 - : highMemLocalHost - ? 45_000 - : 0; -const unitFastBatchTargetMs = parseEnvNumber( - "OPENCLAW_TEST_UNIT_FAST_BATCH_TARGET_MS", - defaultUnitFastBatchTargetMs, -); -const defaultChannelsBatchTargetMs = isCI && !isWindows ? 30_000 : 0; -const channelsBatchTargetMs = parseEnvNumber( - "OPENCLAW_TEST_CHANNELS_BATCH_TARGET_MS", - defaultChannelsBatchTargetMs, -); -// Heap snapshots on current main show long-lived unit-fast workers retaining -// transformed Vitest/Vite module graphs rather than app objects. Multiple -// bounded unit-fast lanes only help if we also recycle them serially instead -// of keeping several transform-heavy workers resident at the same time. -const unitFastBuckets = - unitFastLaneCount > 1 - ? packFilesByDuration(unitFastCandidateFiles, unitFastLaneCount, estimateUnitDurationMs) - : [unitFastCandidateFiles]; -const unitFastEntries = unitFastBuckets.flatMap((files, index) => { - const laneName = unitFastBuckets.length === 1 ? "unit-fast" : `unit-fast-${String(index + 1)}`; - const recycledBatches = splitFilesByDurationBudget( - files, - unitFastBatchTargetMs, - estimateUnitDurationMs, - ); - return recycledBatches - .filter((batch) => batch.length > 0) - .map((batch, batchIndex) => ({ - name: recycledBatches.length === 1 ? laneName : `${laneName}-batch-${String(batchIndex + 1)}`, - serialPhase: "unit-fast", - includeFiles: batch, - estimatedDurationMs: estimateEntryFilesDurationMs( - { args: ["vitest", "run", "--config", "vitest.unit.config.ts"] }, - batch, - ), - env: { - OPENCLAW_VITEST_INCLUDE_FILE: writeTempJsonArtifact( - `vitest-unit-fast-include-${String(index + 1)}-${String(batchIndex + 1)}`, - batch, - ), - }, - args: [ - "vitest", - "run", - "--config", - "vitest.unit.config.ts", - "--pool=forks", - ...noIsolateArgs, - ], - })); -}); -// Shared channel workers retain large transformed module graphs across files on -// non-isolated runs. Recycle that lane in bounded batches so the -// process gets torn down before unrelated channel files inherit the full graph. -const channelsSharedBatches = splitFilesByDurationBudget( - channelSharedCandidateFiles, - channelsBatchTargetMs, - estimateChannelDurationMs, -); -const channelsSharedEntries = channelsSharedBatches - .filter((batch) => batch.length > 0) - .map((batch, batchIndex) => ({ - name: - channelsSharedBatches.length === 1 ? "channels" : `channels-batch-${String(batchIndex + 1)}`, - serialPhase: "channels", - includeFiles: batch, - estimatedDurationMs: estimateEntryFilesDurationMs( - { args: ["vitest", "run", "--config", "vitest.channels.config.ts"] }, - batch, - ), - env: { - OPENCLAW_VITEST_INCLUDE_FILE: writeTempJsonArtifact( - `vitest-channels-include-${String(batchIndex + 1)}`, - batch, - ), - }, - args: ["vitest", "run", "--config", "vitest.channels.config.ts", ...noIsolateArgs], - })); -const heavyUnitBuckets = packFilesByDuration( - timedHeavyUnitFiles, - heavyUnitLaneCount, - estimateUnitDurationMs, -); -const unitHeavyEntries = heavyUnitBuckets.map((files, index) => ({ - name: `unit-heavy-${String(index + 1)}`, - args: [ - "vitest", - "run", - "--config", - "vitest.unit.config.ts", - "--pool=forks", - ...noIsolateArgs, - ...files, - ], -})); -const unitPinnedEntries = - unitThreadPinnedFiles.length > 0 - ? [ - { - name: "unit-pinned", - args: [ - "vitest", - "run", - "--config", - "vitest.unit.config.ts", - "--pool=forks", - ...noIsolateArgs, - ...unitThreadPinnedFiles, - ], - }, - ] - : []; -const unitIsolatedEntries = unitForkIsolatedFiles.map((file) => ({ - name: `unit-${path.basename(file, ".test.ts")}-isolated`, - args: [ - "vitest", - "run", - "--config", - "vitest.unit.config.ts", - "--pool=forks", - ...noIsolateArgs, - file, - ], -})); -const baseRuns = [ - ...(skipDefaultRuns - ? [] - : shouldSplitUnitRuns - ? [ - ...unitFastEntries, - ...unitIsolatedEntries, - ...unitHeavyEntries, - ...unitMemoryIsolatedFiles.map((file) => ({ - name: `unit-${path.basename(file, ".test.ts")}-memory-isolated`, - args: [ - "vitest", - "run", - "--config", - "vitest.unit.config.ts", - "--pool=forks", - ...noIsolateArgs, - file, - ], - })), - ...unitPinnedEntries, - ] - : [ - { - name: "unit", - args: [ - "vitest", - "run", - "--config", - "vitest.unit.config.ts", - "--pool=forks", - ...noIsolateArgs, - ], - }, - ]), - ...(includeExtensionsSuite ? [...extensionIsolatedEntries, ...extensionsSharedEntries] : []), - ...(includeChannelsSuite - ? [ - ...channelIsolatedEntries.map((entry) => ({ - ...entry, - args: [...entry.args.slice(0, 5), ...noIsolateArgs, ...entry.args.slice(5)], - })), - ...channelsSharedEntries, - ] - : []), - ...(includeGatewaySuite - ? [ - { - name: "gateway", - args: [ - "vitest", - "run", - "--config", - "vitest.gateway.config.ts", - "--pool=forks", - ...noIsolateArgs, - ], - }, - ] - : []), -]; -runs = baseRuns; -const formatEntrySummary = (entry) => { - const explicitFilters = countExplicitEntryFilters(entry.args) ?? 0; - return `${entry.name} filters=${String(explicitFilters || "all")} maxWorkers=${String( - maxWorkersForRun(entry.name) ?? "default", - )}`; -}; -const resolveFilterMatches = (fileFilter) => { - const normalizedFilter = normalizeRepoPath(fileFilter); - if (fs.existsSync(fileFilter)) { - const stats = fs.statSync(fileFilter); - if (stats.isFile()) { - return [normalizedFilter]; - } - if (stats.isDirectory()) { - const prefix = normalizedFilter.endsWith("/") ? normalizedFilter : `${normalizedFilter}/`; - return allKnownTestFiles.filter((file) => file.startsWith(prefix)); - } - } - if (/[*?[\]{}]/.test(normalizedFilter)) { - return allKnownTestFiles.filter((file) => path.matchesGlob(file, normalizedFilter)); - } - return allKnownTestFiles.filter((file) => file.includes(normalizedFilter)); -}; -const createTargetedEntry = (owner, isolated, filters) => { - const name = isolated ? `${owner}-isolated` : owner; - const forceForks = isolated; - if (owner === "unit") { - return { - name, - args: [ - "vitest", - "run", - "--config", - "vitest.unit.config.ts", - "--pool=forks", - ...noIsolateArgs, - ...filters, - ], - }; - } - if (owner === "base-pinned") { - return { - name, - args: [ - "vitest", - "run", - "--config", - "vitest.config.ts", - "--pool=forks", - ...noIsolateArgs, - ...filters, - ], - }; - } - if (owner === "extensions") { - return { - name, - args: [ - "vitest", - "run", - "--config", - "vitest.extensions.config.ts", - ...(forceForks ? ["--pool=forks"] : []), - ...noIsolateArgs, - ...filters, - ], - }; - } - if (owner === "gateway") { - return { - name, - args: [ - "vitest", - "run", - "--config", - "vitest.gateway.config.ts", - "--pool=forks", - ...noIsolateArgs, - ...filters, - ], - }; - } - if (owner === "channels") { - return { - name, - args: [ - "vitest", - "run", - "--config", - "vitest.channels.config.ts", - ...(forceForks ? ["--pool=forks"] : []), - ...noIsolateArgs, - ...filters, - ], - }; - } - if (owner === "live") { - return { - name, - args: ["vitest", "run", "--config", "vitest.live.config.ts", ...noIsolateArgs, ...filters], - }; - } - if (owner === "e2e") { - return { - name, - args: ["vitest", "run", "--config", "vitest.e2e.config.ts", ...noIsolateArgs, ...filters], - }; - } - return { - name, - args: [ - "vitest", - "run", - "--config", - "vitest.config.ts", - ...noIsolateArgs, - ...(forceForks ? ["--pool=forks"] : []), - ...filters, - ], - }; -}; -const formatPerFileEntryName = (owner, file) => { - const baseName = path - .basename(file) - .replace(/\.live\.test\.ts$/u, "") - .replace(/\.e2e\.test\.ts$/u, "") - .replace(/\.test\.ts$/u, ""); - return `${owner}-${baseName}`; -}; -const createPerFileTargetedEntry = (file) => { - const target = inferTarget(file); - const owner = isLegacyBasePinnedFile(file) ? "base-pinned" : target.owner; - return { - ...createTargetedEntry(owner, target.isolated, [file]), - name: `${formatPerFileEntryName(owner, file)}${target.isolated ? "-isolated" : ""}`, - }; -}; -const rebuildEntryArgsWithFilters = (entryArgs, filters) => { - const baseArgs = entryArgs.slice(0, 2); - const { optionArgs } = parsePassthroughArgs(entryArgs.slice(2)); - return [...baseArgs, ...optionArgs, ...filters]; -}; -const createPinnedShardEntry = (entry, files, fixedShardIndex) => { - const nextEntry = { - ...entry, - name: `${entry.name}-shard-${String(fixedShardIndex)}`, - fixedShardIndex, - estimatedDurationMs: estimateEntryFilesDurationMs(entry, files), - }; - if (Array.isArray(entry.includeFiles) && entry.includeFiles.length > 0) { - return { - ...nextEntry, - includeFiles: files, - env: { - ...entry.env, - OPENCLAW_VITEST_INCLUDE_FILE: writeTempJsonArtifact( - `${sanitizeArtifactName(entry.name)}-shard-${String(fixedShardIndex)}-include`, - files, - ), - }, - args: rebuildEntryArgsWithFilters(entry.args, []), - }; - } - return { - ...nextEntry, - args: rebuildEntryArgsWithFilters(entry.args, files), - }; -}; -const expandEntryAcrossTopLevelShards = (entry) => { - if (configuredShardCount === null || shardCount <= 1 || entry.fixedShardIndex !== undefined) { - return [entry]; - } - const estimateDurationMs = resolveEntryTimingEstimator(entry); - if (!estimateDurationMs) { - return [entry]; - } - const candidateFiles = - Array.isArray(entry.includeFiles) && entry.includeFiles.length > 0 - ? entry.includeFiles - : getExplicitEntryFilters(entry.args); - if (candidateFiles.length <= 1) { - return [entry]; - } - const effectiveShardCount = Math.min(shardCount, Math.max(1, candidateFiles.length - 1)); - if (effectiveShardCount <= 1) { - return [entry]; - } - const buckets = packFilesByDurationWithBaseLoads( - candidateFiles, - effectiveShardCount, - estimateDurationMs, - ); - return buckets.flatMap((files, bucketIndex) => - files.length > 0 ? [createPinnedShardEntry(entry, files, bucketIndex + 1)] : [], - ); -}; -const targetedEntries = (() => { - if (passthroughFileFilters.length === 0) { - return []; - } - const groups = passthroughFileFilters.reduce((acc, fileFilter) => { - const matchedFiles = resolveFilterMatches(fileFilter); - if (matchedFiles.length === 0) { - const normalizedFile = normalizeRepoPath(fileFilter); - const target = inferTarget(normalizedFile); - const owner = isLegacyBasePinnedFile(normalizedFile) ? "base-pinned" : target.owner; - const key = `${owner}:${target.isolated ? "isolated" : "default"}`; - const files = acc.get(key) ?? []; - files.push(normalizedFile); - acc.set(key, files); - return acc; - } - for (const matchedFile of matchedFiles) { - const target = inferTarget(matchedFile); - const owner = isLegacyBasePinnedFile(matchedFile) ? "base-pinned" : target.owner; - const key = `${owner}:${target.isolated ? "isolated" : "default"}`; - const files = acc.get(key) ?? []; - files.push(matchedFile); - acc.set(key, files); - } - return acc; - }, new Map()); - return Array.from(groups, ([key, filters]) => { - const [owner, mode] = key.split(":"); - const uniqueFilters = [...new Set(filters)]; - if (mode === "isolated" || owner === "base-pinned") { - return uniqueFilters.map((file) => createPerFileTargetedEntry(file)); - } - return [createTargetedEntry(owner, false, uniqueFilters)]; - }).flat(); -})(); -if (configuredShardCount !== null && shardCount > 1) { - runs = runs.flatMap((entry) => expandEntryAcrossTopLevelShards(entry)); -} -const estimateTopLevelEntryDurationMs = (entry) => { - if (Number.isFinite(entry.estimatedDurationMs) && entry.estimatedDurationMs > 0) { - return entry.estimatedDurationMs; - } - const filters = getExplicitEntryFilters(entry.args); - if (filters.length === 0) { - return unitTimingManifest.defaultDurationMs; - } - return filters.reduce((totalMs, file) => { - if (isUnitConfigTestFile(file)) { - return totalMs + estimateUnitDurationMs(file); - } - if (channelTestPrefixes.some((prefix) => file.startsWith(prefix))) { - return totalMs + 3_000; - } - if (file.startsWith("extensions/")) { - return totalMs + 2_000; - } - return totalMs + 1_000; - }, 0); -}; -const topLevelSingleShardAssignments = (() => { - if (shardIndexOverride === null || shardCount <= 1) { - return new Map(); - } - - // Single-file and other non-shardable explicit lanes would otherwise run on - // every shard. Assign them to one top-level shard instead. - const entriesNeedingAssignment = runs.filter((entry) => { - if (entry.fixedShardIndex !== undefined) { - return false; - } - const explicitFilterCount = countExplicitEntryFilters(entry.args); - if (explicitFilterCount === null) { - return false; - } - const effectiveShardCount = Math.min(shardCount, Math.max(1, explicitFilterCount - 1)); - return effectiveShardCount <= 1; - }); - - const assignmentMap = new Map(); - const pinnedShardLoadsMs = Array.from({ length: shardCount }, () => 0); - for (const entry of runs) { - if (entry.fixedShardIndex === undefined) { + if (arg === "--surface") { + const nextValue = args[index + 1] ?? ""; + if (!nextValue || nextValue === "--" || nextValue.startsWith("-")) { + throw new Error(`Invalid --surface value: ${String(nextValue || "")}`); + } + wrapper.surfaces.push(nextValue); + index += 1; continue; } - const shardArrayIndex = entry.fixedShardIndex - 1; - if (shardArrayIndex < 0 || shardArrayIndex >= pinnedShardLoadsMs.length) { + if (arg === "--files") { + const nextValue = args[index + 1] ?? ""; + if (!nextValue || nextValue === "--" || nextValue.startsWith("-")) { + throw new Error(`Invalid --files value: ${String(nextValue || "")}`); + } + wrapper.files.push(nextValue); + index += 1; continue; } - pinnedShardLoadsMs[shardArrayIndex] += estimateTopLevelEntryDurationMs(entry); - } - const buckets = packFilesByDurationWithBaseLoads( - entriesNeedingAssignment, - shardCount, - estimateTopLevelEntryDurationMs, - pinnedShardLoadsMs, - ); - for (const [bucketIndex, bucket] of buckets.entries()) { - for (const entry of bucket) { - assignmentMap.set(entry, bucketIndex + 1); - } - } - return assignmentMap; -})(); -// Node 25 local runs still show cross-process worker shutdown contention even -// after moving the known heavy files into singleton lanes. -const topLevelParallelEnabled = - testProfile !== "low" && - testProfile !== "serial" && - !(!isCI && nodeMajor >= 25) && - !isMacMiniProfile; -const defaultTopLevelParallelLimit = disableIsolation - ? isCI - ? isWindows - ? 2 - : 4 - : highMemLocalHost - ? Math.min(16, hostCpuCount) - : lowMemLocalHost - ? Math.min(8, hostCpuCount) - : Math.min(12, hostCpuCount) - : testProfile === "serial" - ? 1 - : testProfile === "low" - ? lowMemLocalHost - ? 2 - : 3 - : testProfile === "max" - ? 5 - : highMemLocalHost - ? 4 - : lowMemLocalHost - ? 2 - : 3; -const topLevelParallelLimit = Math.max( - 1, - parseEnvNumber("OPENCLAW_TEST_TOP_LEVEL_CONCURRENCY", defaultTopLevelParallelLimit), -); -const overrideWorkers = Number.parseInt(process.env.OPENCLAW_TEST_WORKERS ?? "", 10); -const resolvedOverride = - Number.isFinite(overrideWorkers) && overrideWorkers > 0 ? overrideWorkers : null; -const parallelGatewayEnabled = - !isMacMiniProfile && - (process.env.OPENCLAW_TEST_PARALLEL_GATEWAY === "1" || (!isCI && highMemLocalHost)); -// Keep gateway serial by default except when explicitly requested or on high-memory local hosts. -const keepGatewaySerial = - isWindowsCi || - process.env.OPENCLAW_TEST_SERIAL_GATEWAY === "1" || - testProfile === "serial" || - !parallelGatewayEnabled; -const parallelRuns = keepGatewaySerial ? runs.filter((entry) => entry.name !== "gateway") : runs; -const serialRuns = keepGatewaySerial ? runs.filter((entry) => entry.name === "gateway") : []; -const serialPrefixRuns = parallelRuns.filter((entry) => entry.serialPhase); -const deferredParallelRuns = parallelRuns.filter((entry) => !entry.serialPhase); -const baseLocalWorkers = Math.max(4, Math.min(16, hostCpuCount)); -const loadAwareDisabledRaw = process.env.OPENCLAW_TEST_LOAD_AWARE?.trim().toLowerCase(); -const loadAwareDisabled = loadAwareDisabledRaw === "0" || loadAwareDisabledRaw === "false"; -const loadRatio = - !isCI && !loadAwareDisabled && process.platform !== "win32" && hostCpuCount > 0 - ? os.loadavg()[0] / hostCpuCount - : 0; -// Keep the fast-path unchanged on normal load; only throttle under extreme host pressure. -const extremeLoadScale = loadRatio >= 1.1 ? 0.75 : loadRatio >= 1 ? 0.85 : 1; -const localWorkers = Math.max(4, Math.min(16, Math.floor(baseLocalWorkers * extremeLoadScale))); -const defaultWorkerBudget = - testProfile === "low" - ? { - unit: 2, - unitIsolated: 1, - extensions: 4, - gateway: 1, + if (arg === "--explain") { + const nextValue = args[index + 1] ?? ""; + if (!nextValue || nextValue === "--" || nextValue.startsWith("-")) { + throw new Error(`Invalid --explain value: ${String(nextValue || "")}`); } - : isMacMiniProfile - ? { - unit: 3, - unitIsolated: 1, - extensions: 1, - gateway: 1, - } - : testProfile === "serial" - ? { - unit: 1, - unitIsolated: 1, - extensions: 1, - gateway: 1, - } - : testProfile === "max" - ? { - unit: localWorkers, - unitIsolated: Math.min(4, localWorkers), - extensions: Math.max(1, Math.min(6, Math.floor(localWorkers / 2))), - gateway: Math.max(1, Math.min(2, Math.floor(localWorkers / 4))), - } - : highMemLocalHost - ? { - // After peeling measured hotspots into dedicated lanes, the shared - // unit-fast lane shuts down more reliably with a slightly smaller - // worker fan-out than the old "max it out" local default. - unit: Math.max(4, Math.min(10, Math.floor((localWorkers * 5) / 8))), - unitIsolated: Math.max(1, Math.min(2, Math.floor(localWorkers / 6) || 1)), - extensions: Math.max(1, Math.min(4, Math.floor(localWorkers / 4))), - gateway: Math.max(2, Math.min(6, Math.floor(localWorkers / 2))), - } - : lowMemLocalHost - ? { - // Sub-64 GiB local hosts are prone to OOM with large vmFork runs. - unit: 2, - unitIsolated: 1, - extensions: 4, - gateway: 1, - } - : { - // 64-95 GiB local hosts: conservative split with some parallel headroom. - unit: Math.max(2, Math.min(8, Math.floor(localWorkers / 2))), - unitIsolated: 1, - extensions: Math.max(1, Math.min(4, Math.floor(localWorkers / 4))), - gateway: 1, - }; - -// Keep worker counts predictable for local runs; trim macOS CI workers to avoid worker crashes/OOM. -// In CI on linux/windows, prefer Vitest defaults to avoid cross-test interference from lower worker counts. -const maxWorkersForRun = (name) => { - if (resolvedOverride) { - return resolvedOverride; + wrapper.explain = nextValue; + index += 1; + continue; + } + wrapper.passthroughArgs.push(arg); } - if (isCI && !isMacOS) { - return null; - } - if (isCI && isMacOS) { - return 1; - } - if (name.endsWith("-isolated")) { - return 1; - } - if (name.startsWith("unit-heavy-")) { - return defaultWorkerBudget.unitIsolated; - } - if (name === "extensions") { - return defaultWorkerBudget.extensions; - } - if (name === "gateway") { - return defaultWorkerBudget.gateway; - } - return defaultWorkerBudget.unit; + return wrapper; }; -const WARNING_SUPPRESSION_FLAGS = [ - "--disable-warning=ExperimentalWarning", - "--disable-warning=DEP0040", - "--disable-warning=DEP0060", - "--disable-warning=MaxListenersExceededWarning", -]; - -const DEFAULT_CI_MAX_OLD_SPACE_SIZE_MB = 4096; -const maxOldSpaceSizeMb = (() => { - // CI can hit Node heap limits (especially on large suites). Allow override, default to 4GB. - const raw = process.env.OPENCLAW_TEST_MAX_OLD_SPACE_SIZE_MB ?? ""; - const parsed = Number.parseInt(raw, 10); - if (Number.isFinite(parsed) && parsed > 0) { - return parsed; - } - if (isCI && !isWindows) { - return DEFAULT_CI_MAX_OLD_SPACE_SIZE_MB; - } - return null; -})(); -const formatElapsedMs = (elapsedMs) => - elapsedMs >= 1000 ? `${(elapsedMs / 1000).toFixed(1)}s` : `${Math.round(elapsedMs)}ms`; -const formatMemoryKb = (rssKb) => - rssKb >= 1024 ** 2 - ? `${(rssKb / 1024 ** 2).toFixed(2)}GiB` - : rssKb >= 1024 - ? `${(rssKb / 1024).toFixed(1)}MiB` - : `${rssKb}KiB`; -const formatMemoryDeltaKb = (rssKb) => - `${rssKb >= 0 ? "+" : "-"}${formatMemoryKb(Math.abs(rssKb))}`; -const rawMemoryTrace = process.env.OPENCLAW_TEST_MEMORY_TRACE?.trim().toLowerCase(); -const memoryTraceEnabled = - process.platform !== "win32" && - (rawMemoryTrace === "1" || - rawMemoryTrace === "true" || - (rawMemoryTrace !== "0" && rawMemoryTrace !== "false" && isCI)); -const memoryTracePollMs = Math.max(250, parseEnvNumber("OPENCLAW_TEST_MEMORY_TRACE_POLL_MS", 1000)); -const memoryTraceTopCount = Math.max(1, parseEnvNumber("OPENCLAW_TEST_MEMORY_TRACE_TOP_COUNT", 6)); -const requestedHeapSnapshotIntervalMs = Math.max( - 0, - parseEnvNumber("OPENCLAW_TEST_HEAPSNAPSHOT_INTERVAL_MS", 0), -); -const heapSnapshotMinIntervalMs = 1000; -const heapSnapshotIntervalMs = - requestedHeapSnapshotIntervalMs > 0 - ? Math.max(heapSnapshotMinIntervalMs, requestedHeapSnapshotIntervalMs) - : 0; -const heapSnapshotEnabled = - process.platform !== "win32" && heapSnapshotIntervalMs >= heapSnapshotMinIntervalMs; -const heapSnapshotSignal = process.env.OPENCLAW_TEST_HEAPSNAPSHOT_SIGNAL?.trim() || "SIGUSR2"; -const heapSnapshotBaseDir = heapSnapshotEnabled - ? path.resolve( - process.env.OPENCLAW_TEST_HEAPSNAPSHOT_DIR?.trim() || - path.join(os.tmpdir(), `openclaw-heapsnapshots-${Date.now()}`), - ) - : null; -const ensureNodeOptionFlag = (nodeOptions, flagPrefix, nextValue) => - nodeOptions.includes(flagPrefix) ? nodeOptions : `${nodeOptions} ${nextValue}`.trim(); -const isNodeLikeProcess = (command) => /(?:^|\/)node(?:$|\.exe$)/iu.test(command); -const getShardLabel = (args) => { - const shardIndex = args.findIndex((arg) => arg === "--shard"); - if (shardIndex < 0) { - return ""; - } - return typeof args[shardIndex + 1] === "string" ? args[shardIndex + 1] : ""; +const exitWithCleanup = (artifacts, code) => { + artifacts?.cleanupTempArtifacts?.(); + process.exit(code); }; -const runOnce = (entry, extraArgs = []) => - new Promise((resolve) => { - const startedAt = Date.now(); - const maxWorkers = maxWorkersForRun(entry.name); - const entryArgs = entry.args; - const explicitEntryFilters = getExplicitEntryFilters(entryArgs); - const args = maxWorkers - ? [ - ...entryArgs, - "--maxWorkers", - String(maxWorkers), - ...silentArgs, - ...windowsCiArgs, - ...extraArgs, - ] - : [...entryArgs, ...silentArgs, ...windowsCiArgs, ...extraArgs]; - const shardLabel = getShardLabel(extraArgs); - const artifactStem = [ - sanitizeArtifactName(entry.name), - shardLabel ? `shard-${sanitizeArtifactName(shardLabel)}` : "", - String(startedAt), - ] - .filter(Boolean) - .join("-"); - const laneLogPath = path.join(ensureTempArtifactDir(), `${artifactStem}.log`); - const laneLogStream = fs.createWriteStream(laneLogPath, { flags: "w" }); - laneLogStream.write(`[test-parallel] entry=${entry.name}\n`); - laneLogStream.write(`[test-parallel] cwd=${process.cwd()}\n`); - laneLogStream.write(`[test-parallel] command=${[pnpm, ...args].join(" ")}\n\n`); - console.log( - `[test-parallel] start ${entry.name} workers=${maxWorkers ?? "default"} filters=${String( - countExplicitEntryFilters(entryArgs) ?? "all", - )}`, - ); - const nodeOptions = process.env.NODE_OPTIONS ?? ""; - const nextNodeOptions = WARNING_SUPPRESSION_FLAGS.reduce( - (acc, flag) => (acc.includes(flag) ? acc : `${acc} ${flag}`.trim()), - nodeOptions, - ); - const heapSnapshotDir = - heapSnapshotBaseDir === null ? null : path.join(heapSnapshotBaseDir, entry.name); - let resolvedNodeOptions = - maxOldSpaceSizeMb && !nextNodeOptions.includes("--max-old-space-size=") - ? `${nextNodeOptions} --max-old-space-size=${maxOldSpaceSizeMb}`.trim() - : nextNodeOptions; - if (heapSnapshotEnabled && heapSnapshotDir) { - try { - fs.mkdirSync(heapSnapshotDir, { recursive: true }); - } catch (err) { - console.error( - `[test-parallel] failed to create heap snapshot dir ${heapSnapshotDir}: ${String(err)}`, - ); - resolve(1); - return; - } - resolvedNodeOptions = ensureNodeOptionFlag( - resolvedNodeOptions, - "--diagnostic-dir=", - `--diagnostic-dir=${heapSnapshotDir}`, - ); - resolvedNodeOptions = ensureNodeOptionFlag( - resolvedNodeOptions, - "--heapsnapshot-signal=", - `--heapsnapshot-signal=${heapSnapshotSignal}`, - ); - } - let output = ""; - let fatalSeen = false; - let childError = null; - let child; - let pendingLine = ""; - let memoryPollTimer = null; - let heapSnapshotTimer = null; - const memoryFileRecords = []; - let initialTreeSample = null; - let latestTreeSample = null; - let peakTreeSample = null; - let heapSnapshotSequence = 0; - const updatePeakTreeSample = (sample, reason) => { - if (!sample) { - return; - } - if (!peakTreeSample || sample.rssKb > peakTreeSample.rssKb) { - peakTreeSample = { ...sample, reason }; - } - }; - const triggerHeapSnapshot = (reason) => { - if (!heapSnapshotEnabled || !child?.pid || !heapSnapshotDir) { - return; - } - const records = getProcessTreeRecords(child.pid) ?? []; - const targetPids = records - .filter((record) => record.pid !== process.pid && isNodeLikeProcess(record.command)) - .map((record) => record.pid); - if (targetPids.length === 0) { - return; - } - heapSnapshotSequence += 1; - let signaledCount = 0; - for (const pid of targetPids) { - try { - process.kill(pid, heapSnapshotSignal); - signaledCount += 1; - } catch { - // Process likely exited between ps sampling and signal delivery. - } - } - if (signaledCount > 0) { - console.log( - `[test-parallel][heap] ${entry.name} seq=${String(heapSnapshotSequence)} reason=${reason} signaled=${String( - signaledCount, - )}/${String(targetPids.length)} dir=${heapSnapshotDir}`, - ); - } - }; - const captureTreeSample = (reason) => { - if (!memoryTraceEnabled || !child?.pid) { - return null; - } - const sample = sampleProcessTreeRssKb(child.pid); - if (!sample) { - return null; - } - latestTreeSample = sample; - if (!initialTreeSample) { - initialTreeSample = sample; - } - updatePeakTreeSample(sample, reason); - return sample; - }; - const logMemoryTraceForText = (text) => { - if (!memoryTraceEnabled) { - return; - } - const combined = `${pendingLine}${text}`; - const lines = combined.split(/\r?\n/u); - pendingLine = lines.pop() ?? ""; - const completedFiles = parseCompletedTestFileLines(lines.join("\n")); - for (const completedFile of completedFiles) { - const sample = captureTreeSample(completedFile.file); - if (!sample) { - continue; - } - const previousRssKb = - memoryFileRecords.length > 0 - ? (memoryFileRecords.at(-1)?.rssKb ?? initialTreeSample?.rssKb ?? sample.rssKb) - : (initialTreeSample?.rssKb ?? sample.rssKb); - const deltaKb = sample.rssKb - previousRssKb; - const record = { - ...completedFile, - rssKb: sample.rssKb, - processCount: sample.processCount, - deltaKb, - }; - memoryFileRecords.push(record); - console.log( - `[test-parallel][mem] ${entry.name} file=${record.file} rss=${formatMemoryKb( - record.rssKb, - )} delta=${formatMemoryDeltaKb(record.deltaKb)} peak=${formatMemoryKb( - peakTreeSample?.rssKb ?? record.rssKb, - )} procs=${record.processCount}${record.durationMs ? ` duration=${formatElapsedMs(record.durationMs)}` : ""}`, - ); - } - }; - const logMemoryTraceSummary = () => { - if (!memoryTraceEnabled) { - return; - } - captureTreeSample("close"); - const fallbackRecord = - memoryFileRecords.length === 0 && - explicitEntryFilters.length === 1 && - latestTreeSample && - initialTreeSample - ? [ - { - file: explicitEntryFilters[0], - deltaKb: latestTreeSample.rssKb - initialTreeSample.rssKb, - }, - ] - : []; - const totalDeltaKb = - initialTreeSample && latestTreeSample - ? latestTreeSample.rssKb - initialTreeSample.rssKb - : 0; - const topGrowthFiles = [...memoryFileRecords, ...fallbackRecord] - .filter((record) => record.deltaKb > 0 && typeof record.file === "string") - .toSorted((left, right) => right.deltaKb - left.deltaKb) - .slice(0, memoryTraceTopCount) - .map((record) => `${record.file}:${formatMemoryDeltaKb(record.deltaKb)}`); - console.log( - `[test-parallel][mem] summary ${entry.name} files=${memoryFileRecords.length} peak=${formatMemoryKb( - peakTreeSample?.rssKb ?? 0, - )} totalDelta=${formatMemoryDeltaKb(totalDeltaKb)} peakAt=${ - peakTreeSample?.reason ?? "n/a" - } top=${topGrowthFiles.length > 0 ? topGrowthFiles.join(", ") : "none"}`, - ); - }; - try { - child = spawn(pnpm, args, { - stdio: ["inherit", "pipe", "pipe"], - env: { - ...process.env, - ...entry.env, - VITEST_GROUP: entry.name, - NODE_OPTIONS: resolvedNodeOptions, - }, - shell: isWindows, - }); - captureTreeSample("spawn"); - if (memoryTraceEnabled) { - memoryPollTimer = setInterval(() => { - captureTreeSample("poll"); - }, memoryTracePollMs); - } - if (heapSnapshotEnabled) { - heapSnapshotTimer = setInterval(() => { - triggerHeapSnapshot("interval"); - }, heapSnapshotIntervalMs); - } - } catch (err) { - laneLogStream.end(); - console.error(`[test-parallel] spawn failed: ${String(err)}`); - resolve(1); - return; - } - children.add(child); - child.stdout?.on("data", (chunk) => { - const text = chunk.toString(); - fatalSeen ||= hasFatalTestRunOutput(`${output}${text}`); - output = appendCapturedOutput(output, text); - laneLogStream.write(text); - logMemoryTraceForText(text); - process.stdout.write(chunk); - }); - child.stderr?.on("data", (chunk) => { - const text = chunk.toString(); - fatalSeen ||= hasFatalTestRunOutput(`${output}${text}`); - output = appendCapturedOutput(output, text); - laneLogStream.write(text); - logMemoryTraceForText(text); - process.stderr.write(chunk); - }); - child.on("error", (err) => { - childError = err; - laneLogStream.write(`\n[test-parallel] child error: ${String(err)}\n`); - console.error(`[test-parallel] child error: ${String(err)}`); - }); - child.on("close", (code, signal) => { - if (memoryPollTimer) { - clearInterval(memoryPollTimer); - } - if (heapSnapshotTimer) { - clearInterval(heapSnapshotTimer); - } - children.delete(child); - const resolvedCode = resolveTestRunExitCode({ code, signal, output, fatalSeen, childError }); - const elapsedMs = Date.now() - startedAt; - logMemoryTraceSummary(); - if (resolvedCode !== 0) { - const failureTail = formatCapturedOutputTail(output); - const failureArtifactPath = writeTempJsonArtifact(`${artifactStem}-failure`, { - entry: entry.name, - command: [pnpm, ...args], - elapsedMs, - error: childError ? String(childError) : null, - exitCode: resolvedCode, - fatalSeen, - logPath: laneLogPath, - outputTail: failureTail, - signal: signal ?? null, - }); - if (failureTail) { - console.error(`[test-parallel] failure tail ${entry.name}\n${failureTail}`); - } - console.error( - `[test-parallel] failure artifacts ${entry.name} log=${laneLogPath} meta=${failureArtifactPath}`, - ); - } - laneLogStream.write( - `\n[test-parallel] done ${entry.name} code=${String(resolvedCode)} signal=${ - signal ?? "none" - } elapsed=${formatElapsedMs(elapsedMs)}\n`, - ); - laneLogStream.end(); - console.log( - `[test-parallel] done ${entry.name} code=${String(resolvedCode)} elapsed=${formatElapsedMs(elapsedMs)}`, - ); - resolve(resolvedCode); - }); - }); - -const run = async (entry, extraArgs = []) => { - if (entry.fixedShardIndex !== undefined) { - if (shardIndexOverride !== null && shardIndexOverride !== entry.fixedShardIndex) { - return 0; - } - return runOnce(entry, extraArgs); - } - const explicitFilterCount = countExplicitEntryFilters(entry.args); - const topLevelAssignedShard = topLevelSingleShardAssignments.get(entry); - if (topLevelAssignedShard !== undefined) { - if (shardIndexOverride !== null && shardIndexOverride !== topLevelAssignedShard) { - return 0; - } - return runOnce(entry, extraArgs); - } - // Vitest requires the shard count to stay strictly below the number of - // resolved test files, so explicit-filter lanes need a `< fileCount` cap. - const effectiveShardCount = - explicitFilterCount === null - ? shardCount - : Math.min(shardCount, Math.max(1, explicitFilterCount - 1)); - - if (effectiveShardCount <= 1) { - if (shardIndexOverride !== null && shardIndexOverride > effectiveShardCount) { - return 0; - } - return runOnce(entry, extraArgs); - } - if (shardIndexOverride !== null) { - if (shardIndexOverride > effectiveShardCount) { - return 0; - } - return runOnce(entry, [ - "--shard", - `${shardIndexOverride}/${effectiveShardCount}`, - ...extraArgs, - ]); - } - for (let shardIndex = 1; shardIndex <= effectiveShardCount; shardIndex += 1) { - // eslint-disable-next-line no-await-in-loop - const code = await runOnce(entry, [ - "--shard", - `${shardIndex}/${effectiveShardCount}`, - ...extraArgs, - ]); - if (code !== 0) { - return code; - } - } - return 0; -}; - -const runEntriesWithLimit = async (entries, extraArgs = [], concurrency = 1) => { - if (entries.length === 0) { - return undefined; - } - - const normalizedConcurrency = Math.max(1, Math.floor(concurrency)); - if (normalizedConcurrency <= 1) { - for (const entry of entries) { - // eslint-disable-next-line no-await-in-loop - const code = await run(entry, extraArgs); - if (code !== 0) { - return code; - } - } - - return undefined; - } - - let nextIndex = 0; - let firstFailure; - const worker = async () => { - while (firstFailure === undefined) { - const entryIndex = nextIndex; - nextIndex += 1; - if (entryIndex >= entries.length) { - return; - } - const code = await run(entries[entryIndex], extraArgs); - if (code !== 0 && firstFailure === undefined) { - firstFailure = code; - } - } - }; - - const workerCount = Math.min(normalizedConcurrency, entries.length); - await Promise.all(Array.from({ length: workerCount }, () => worker())); - return firstFailure; -}; - -const runEntries = async (entries, extraArgs = []) => { - if (topLevelParallelEnabled) { - // Keep a bounded number of top-level Vitest processes in flight. As the - // singleton lane list grows, unbounded Promise.all scheduling turns - // isolation into cross-process contention and can reintroduce timeouts. - return runEntriesWithLimit(entries, extraArgs, topLevelParallelLimit); - } - - return runEntriesWithLimit(entries, extraArgs); -}; - -const shutdown = (signal) => { - for (const child of children) { - child.kill(signal); - } -}; - -process.on("SIGINT", () => shutdown("SIGINT")); -process.on("SIGTERM", () => shutdown("SIGTERM")); -process.on("exit", cleanupTempArtifacts); - -if (process.env.OPENCLAW_TEST_LIST_LANES === "1") { - const entriesToPrint = targetedEntries.length > 0 ? targetedEntries : runs; - for (const entry of entriesToPrint) { - console.log(formatEntrySummary(entry)); - } - process.exit(0); -} - -if (passthroughMetadataOnly) { - const exitCode = await runOnce( - { - name: "vitest-meta", - args: ["vitest", "run"], - }, - passthroughOptionArgs, - ); - process.exit(exitCode); -} - -if (targetedEntries.length > 0) { - if (passthroughRequiresSingleRun && targetedEntries.length > 1) { - console.error( - "[test-parallel] The provided Vitest args require a single run, but the selected test filters span multiple wrapper configs. Run one target/config at a time.", - ); - process.exit(2); - } - const targetedParallelRuns = keepGatewaySerial - ? targetedEntries.filter((entry) => entry.name !== "gateway") - : targetedEntries; - const targetedSerialRuns = keepGatewaySerial - ? targetedEntries.filter((entry) => entry.name === "gateway") - : []; - const failedTargetedParallel = await runEntries(targetedParallelRuns, passthroughOptionArgs); - if (failedTargetedParallel !== undefined) { - process.exit(failedTargetedParallel); - } - for (const entry of targetedSerialRuns) { - // eslint-disable-next-line no-await-in-loop - const code = await run(entry, passthroughOptionArgs); - if (code !== 0) { - process.exit(code); - } - } - process.exit(0); -} - -if (passthroughRequiresSingleRun && passthroughOptionArgs.length > 0) { - console.error( - "[test-parallel] The provided Vitest args require a single run. Use the dedicated npm script for that workflow (for example `pnpm test:coverage`) or target a single test file/filter.", - ); +let rawCli; +try { + rawCli = parseCliArgs(process.argv.slice(2)); +} catch (error) { + console.error(`[test-parallel] ${error instanceof Error ? error.message : String(error)}`); process.exit(2); } - -if (serialPrefixRuns.length > 0) { - const failedSerialPrefix = await runEntriesWithLimit(serialPrefixRuns, passthroughOptionArgs, 1); - if (failedSerialPrefix !== undefined) { - process.exit(failedSerialPrefix); - } - // Low-profile runs favor stability over overlap once we leave the shared - // unit-fast batches; the isolated memory-heavy lanes can still trip over - // each other when two singleton Vitest processes overlap. - const deferredRunConcurrency = isMacMiniProfile ? 3 : testProfile === "low" ? 1 : undefined; - const failedDeferredParallel = isMacMiniProfile - ? await runEntriesWithLimit(deferredParallelRuns, passthroughOptionArgs, deferredRunConcurrency) - : deferredRunConcurrency - ? await runEntriesWithLimit( - deferredParallelRuns, - passthroughOptionArgs, - deferredRunConcurrency, - ) - : await runEntries(deferredParallelRuns, passthroughOptionArgs); - if (failedDeferredParallel !== undefined) { - process.exit(failedDeferredParallel); - } -} else if (isMacMiniProfile && targetedEntries.length === 0) { - const unitFastEntriesForMacMini = parallelRuns.filter((entry) => - entry.name.startsWith("unit-fast"), +if (rawCli.showHelp) { + console.log( + [ + "Usage: node scripts/test-parallel.mjs [wrapper flags] [-- vitest args]", + "", + "Wrapper flags:", + " --plan Print the resolved execution plan", + " --explain Explain how a file is classified and run", + " --surface Select a surface (repeatable or comma-separated)", + " --files Add targeted files/patterns (repeatable)", + " --mode Override runtime mode", + " --profile Override execution intent (normal|max|serial)", + ].join("\n"), ); - for (const entry of unitFastEntriesForMacMini) { - // eslint-disable-next-line no-await-in-loop - const unitFastCode = await run(entry, passthroughOptionArgs); - if (unitFastCode !== 0) { - process.exit(unitFastCode); - } - } - const deferredEntries = parallelRuns.filter((entry) => !entry.name.startsWith("unit-fast")); - const failedMacMiniParallel = await runEntriesWithLimit( - deferredEntries, - passthroughOptionArgs, - 3, - ); - if (failedMacMiniParallel !== undefined) { - process.exit(failedMacMiniParallel); - } -} else { - const failedParallel = await runEntries(parallelRuns, passthroughOptionArgs); - if (failedParallel !== undefined) { - process.exit(failedParallel); - } + process.exit(0); } -for (const entry of serialRuns) { - // eslint-disable-next-line no-await-in-loop - const code = await run(entry, passthroughOptionArgs); - if (code !== 0) { - process.exit(code); - } +const request = { + mode: rawCli.mode, + profile: rawCli.profile, + surfaces: rawCli.surfaces, + fileFilters: rawCli.files, + passthroughArgs: rawCli.passthroughArgs, +}; + +if (rawCli.explain) { + const explanation = explainExecutionTarget( + { ...request, passthroughArgs: [], fileFilters: [rawCli.explain] }, + { env: process.env }, + ); + console.log(formatExplanation(explanation)); + process.exit(0); } -process.exit(0); +const artifacts = createExecutionArtifacts(process.env); +let plan; +try { + plan = buildExecutionPlan(request, { + env: process.env, + writeTempJsonArtifact: artifacts.writeTempJsonArtifact, + }); +} catch (error) { + console.error(`[test-parallel] ${error instanceof Error ? error.message : String(error)}`); + exitWithCleanup(artifacts, 2); +} + +if (process.env.OPENCLAW_TEST_LIST_LANES === "1" || rawCli.plan) { + console.log(formatPlanOutput(plan)); + exitWithCleanup(artifacts, 0); +} + +const exitCode = await executePlan(plan, { env: process.env, artifacts }); +process.exit(exitCode); diff --git a/scripts/test-planner/catalog.mjs b/scripts/test-planner/catalog.mjs new file mode 100644 index 00000000000..1638a97a486 --- /dev/null +++ b/scripts/test-planner/catalog.mjs @@ -0,0 +1,187 @@ +import fs from "node:fs"; +import path from "node:path"; +import { channelTestPrefixes } from "../../vitest.channel-paths.mjs"; +import { isUnitConfigTestFile } from "../../vitest.unit-paths.mjs"; +import { dedupeFilesPreserveOrder, loadTestRunnerBehavior } from "../test-runner-manifest.mjs"; + +const baseConfigPrefixes = ["src/agents/", "src/auto-reply/", "src/commands/", "test/", "ui/"]; + +export const normalizeRepoPath = (value) => value.split(path.sep).join("/"); + +const toRepoRelativePath = (value) => { + const relativePath = normalizeRepoPath(path.relative(process.cwd(), path.resolve(value))); + return relativePath.startsWith("../") || relativePath === ".." ? null : relativePath; +}; + +const walkTestFiles = (rootDir) => { + if (!fs.existsSync(rootDir)) { + return []; + } + const entries = fs.readdirSync(rootDir, { withFileTypes: true }); + const files = []; + for (const entry of entries) { + const fullPath = path.join(rootDir, entry.name); + if (entry.isDirectory()) { + files.push(...walkTestFiles(fullPath)); + continue; + } + if (!entry.isFile()) { + continue; + } + if ( + fullPath.endsWith(".test.ts") || + fullPath.endsWith(".live.test.ts") || + fullPath.endsWith(".e2e.test.ts") + ) { + files.push(normalizeRepoPath(fullPath)); + } + } + return files; +}; + +export function loadTestCatalog() { + const behaviorManifest = loadTestRunnerBehavior(); + const existingFiles = (entries) => + entries.map((entry) => entry.file).filter((file) => fs.existsSync(file)); + const existingUnitConfigFiles = (entries) => existingFiles(entries).filter(isUnitConfigTestFile); + const baseThreadPinnedFiles = existingFiles(behaviorManifest.base?.threadPinned ?? []); + const channelIsolatedManifestFiles = existingFiles(behaviorManifest.channels?.isolated ?? []); + const channelIsolatedPrefixes = behaviorManifest.channels?.isolatedPrefixes ?? []; + const extensionForkIsolatedFiles = existingFiles(behaviorManifest.extensions?.isolated ?? []); + const unitForkIsolatedFiles = existingUnitConfigFiles(behaviorManifest.unit.isolated); + const unitThreadPinnedFiles = existingUnitConfigFiles(behaviorManifest.unit.threadPinned); + const unitBehaviorOverrideSet = new Set([...unitForkIsolatedFiles, ...unitThreadPinnedFiles]); + const allKnownTestFiles = [ + ...new Set([ + ...walkTestFiles("src"), + ...walkTestFiles("extensions"), + ...walkTestFiles("test"), + ...walkTestFiles(path.join("ui", "src", "ui")), + ]), + ]; + const channelIsolatedFiles = dedupeFilesPreserveOrder([ + ...channelIsolatedManifestFiles, + ...allKnownTestFiles.filter((file) => + channelIsolatedPrefixes.some((prefix) => file.startsWith(prefix)), + ), + ]); + const channelIsolatedFileSet = new Set(channelIsolatedFiles); + const extensionForkIsolatedFileSet = new Set(extensionForkIsolatedFiles); + const baseThreadPinnedFileSet = new Set(baseThreadPinnedFiles); + const unitThreadPinnedFileSet = new Set(unitThreadPinnedFiles); + const unitForkIsolatedFileSet = new Set(unitForkIsolatedFiles); + + const classifyTestFile = (fileFilter, options = {}) => { + const normalizedFile = normalizeRepoPath(fileFilter); + const reasons = []; + const isolated = + options.unitMemoryIsolatedFiles?.includes(normalizedFile) || + unitForkIsolatedFileSet.has(normalizedFile) || + extensionForkIsolatedFileSet.has(normalizedFile) || + channelIsolatedFileSet.has(normalizedFile); + if (options.unitMemoryIsolatedFiles?.includes(normalizedFile)) { + reasons.push("unit-memory-isolated"); + } + if (unitForkIsolatedFileSet.has(normalizedFile)) { + reasons.push("unit-isolated-manifest"); + } + if (extensionForkIsolatedFileSet.has(normalizedFile)) { + reasons.push("extensions-isolated-manifest"); + } + if (channelIsolatedFileSet.has(normalizedFile)) { + reasons.push("channels-isolated-rule"); + } + + let surface = "base"; + if (isUnitConfigTestFile(normalizedFile)) { + surface = "unit"; + } else if (normalizedFile.endsWith(".live.test.ts")) { + surface = "live"; + } else if (normalizedFile.endsWith(".e2e.test.ts")) { + surface = "e2e"; + } else if (channelTestPrefixes.some((prefix) => normalizedFile.startsWith(prefix))) { + surface = "channels"; + } else if (normalizedFile.startsWith("extensions/")) { + surface = "extensions"; + } else if (normalizedFile.startsWith("src/gateway/")) { + surface = "gateway"; + } else if (baseConfigPrefixes.some((prefix) => normalizedFile.startsWith(prefix))) { + surface = "base"; + } else if (normalizedFile.startsWith("src/")) { + surface = "unit"; + } + if (surface === "unit") { + reasons.push("unit-surface"); + } else if (surface !== "base") { + reasons.push(`${surface}-surface`); + } else { + reasons.push("base-surface"); + } + + const legacyBasePinned = baseThreadPinnedFileSet.has(normalizedFile); + if (legacyBasePinned) { + reasons.push("base-pinned-manifest"); + } + if (unitThreadPinnedFileSet.has(normalizedFile)) { + reasons.push("unit-pinned-manifest"); + } + + return { + file: normalizedFile, + surface, + isolated, + legacyBasePinned, + reasons, + }; + }; + + const resolveFilterMatches = (fileFilter) => { + const normalizedFilter = normalizeRepoPath(fileFilter); + const repoRelativeFilter = toRepoRelativePath(fileFilter); + if (fs.existsSync(fileFilter)) { + const stats = fs.statSync(fileFilter); + if (stats.isFile()) { + if (repoRelativeFilter && allKnownTestFiles.includes(repoRelativeFilter)) { + return [repoRelativeFilter]; + } + throw new Error(`Explicit path ${fileFilter} is not a known test file.`); + } + if (stats.isDirectory()) { + if (!repoRelativeFilter) { + throw new Error(`Explicit path ${fileFilter} is outside the repo test roots.`); + } + const prefix = repoRelativeFilter.endsWith("/") + ? repoRelativeFilter + : `${repoRelativeFilter}/`; + const matches = allKnownTestFiles.filter((file) => file.startsWith(prefix)); + if (matches.length === 0) { + throw new Error(`Explicit path ${fileFilter} does not contain known test files.`); + } + return matches; + } + } + if (/[*?[\]{}]/.test(normalizedFilter)) { + return allKnownTestFiles.filter((file) => path.matchesGlob(file, normalizedFilter)); + } + return allKnownTestFiles.filter((file) => file.includes(normalizedFilter)); + }; + + return { + allKnownTestFiles, + allKnownUnitFiles: allKnownTestFiles.filter((file) => isUnitConfigTestFile(file)), + baseThreadPinnedFiles, + channelIsolatedFiles, + channelIsolatedFileSet, + channelTestPrefixes, + extensionForkIsolatedFiles, + extensionForkIsolatedFileSet, + unitBehaviorOverrideSet, + unitForkIsolatedFiles, + unitThreadPinnedFiles, + baseThreadPinnedFileSet, + classifyTestFile, + resolveFilterMatches, + }; +} + +export const testSurfaces = ["unit", "extensions", "channels", "gateway", "live", "e2e", "base"]; diff --git a/scripts/test-planner/executor.mjs b/scripts/test-planner/executor.mjs new file mode 100644 index 00000000000..5eee7a3d4f8 --- /dev/null +++ b/scripts/test-planner/executor.mjs @@ -0,0 +1,668 @@ +import { spawn } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { + getProcessTreeRecords, + parseCompletedTestFileLines, + sampleProcessTreeRssKb, +} from "../test-parallel-memory.mjs"; +import { + appendCapturedOutput, + formatCapturedOutputTail, + hasFatalTestRunOutput, + resolveTestRunExitCode, +} from "../test-parallel-utils.mjs"; +import { countExplicitEntryFilters, getExplicitEntryFilters } from "./vitest-args.mjs"; + +export function resolvePnpmCommandInvocation(options = {}) { + const npmExecPath = typeof options.npmExecPath === "string" ? options.npmExecPath.trim() : ""; + if (npmExecPath && path.isAbsolute(npmExecPath)) { + const npmExecBase = path.basename(npmExecPath).toLowerCase(); + if (npmExecBase.startsWith("pnpm")) { + return { + command: options.nodeExecPath || process.execPath, + args: [npmExecPath], + }; + } + } + + if (options.platform === "win32") { + return { + command: options.comSpec || "cmd.exe", + args: ["/d", "/s", "/c", "pnpm.cmd"], + }; + } + + return { + command: "pnpm", + args: [], + }; +} + +const sanitizeArtifactName = (value) => { + const normalized = value + .trim() + .replace(/[^a-z0-9._-]+/giu, "-") + .replace(/^-+|-+$/gu, ""); + return normalized || "artifact"; +}; + +const DEFAULT_CI_MAX_OLD_SPACE_SIZE_MB = 4096; +const WARNING_SUPPRESSION_FLAGS = [ + "--disable-warning=ExperimentalWarning", + "--disable-warning=DEP0040", + "--disable-warning=DEP0060", + "--disable-warning=MaxListenersExceededWarning", +]; + +const formatElapsedMs = (elapsedMs) => + elapsedMs >= 1000 ? `${(elapsedMs / 1000).toFixed(1)}s` : `${Math.round(elapsedMs)}ms`; +const formatMemoryKb = (rssKb) => + rssKb >= 1024 ** 2 + ? `${(rssKb / 1024 ** 2).toFixed(2)}GiB` + : rssKb >= 1024 + ? `${(rssKb / 1024).toFixed(1)}MiB` + : `${rssKb}KiB`; +const formatMemoryDeltaKb = (rssKb) => + `${rssKb >= 0 ? "+" : "-"}${formatMemoryKb(Math.abs(rssKb))}`; + +export function createExecutionArtifacts(env = process.env) { + let tempArtifactDir = null; + const ensureTempArtifactDir = () => { + if (tempArtifactDir === null) { + tempArtifactDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-test-parallel-")); + } + return tempArtifactDir; + }; + const writeTempJsonArtifact = (name, value) => { + const filePath = path.join(ensureTempArtifactDir(), `${sanitizeArtifactName(name)}.json`); + fs.writeFileSync(filePath, `${JSON.stringify(value)}\n`, "utf8"); + return filePath; + }; + const cleanupTempArtifacts = () => { + if (tempArtifactDir === null) { + return; + } + if (env.OPENCLAW_TEST_KEEP_TEMP_ARTIFACTS === "1") { + console.error(`[test-parallel] keeping temp artifacts at ${tempArtifactDir}`); + return; + } + fs.rmSync(tempArtifactDir, { recursive: true, force: true }); + tempArtifactDir = null; + }; + return { ensureTempArtifactDir, writeTempJsonArtifact, cleanupTempArtifacts }; +} + +const ensureNodeOptionFlag = (nodeOptions, flagPrefix, nextValue) => + nodeOptions.includes(flagPrefix) ? nodeOptions : `${nodeOptions} ${nextValue}`.trim(); + +const isNodeLikeProcess = (command) => /(?:^|\/)node(?:$|\.exe$)/iu.test(command); + +const getShardLabel = (args) => { + const shardIndex = args.findIndex((arg) => arg === "--shard"); + if (shardIndex < 0) { + return ""; + } + return typeof args[shardIndex + 1] === "string" ? args[shardIndex + 1] : ""; +}; + +export function formatPlanOutput(plan) { + return [ + `runtime=${plan.runtimeCapabilities.runtimeProfileName} mode=${plan.runtimeCapabilities.mode} intent=${plan.runtimeCapabilities.intentProfile} memoryBand=${plan.runtimeCapabilities.memoryBand} loadBand=${plan.runtimeCapabilities.loadBand} vitestMaxWorkers=${String(plan.executionBudget.vitestMaxWorkers ?? "default")} topLevelParallel=${plan.topLevelParallelEnabled ? String(plan.topLevelParallelLimit) : "off"}`, + ...plan.selectedUnits.map( + (unit) => + `${unit.id} filters=${String(countExplicitEntryFilters(unit.args) ?? "all")} maxWorkers=${String( + unit.maxWorkers ?? "default", + )} surface=${unit.surface} isolate=${unit.isolate ? "yes" : "no"} pool=${unit.pool}`, + ), + ].join("\n"); +} + +export function formatExplanation(explanation) { + return [ + `file=${explanation.file}`, + `runtime=${explanation.runtimeProfile} intent=${explanation.intentProfile} memoryBand=${explanation.memoryBand} loadBand=${explanation.loadBand}`, + `surface=${explanation.surface}`, + `isolate=${explanation.isolate ? "yes" : "no"}`, + `pool=${explanation.pool}`, + `maxWorkers=${String(explanation.maxWorkers ?? "default")}`, + `reasons=${explanation.reasons.join(",")}`, + `command=${explanation.args.join(" ")}`, + ].join("\n"); +} + +export async function executePlan(plan, options = {}) { + const env = options.env ?? process.env; + const artifacts = options.artifacts ?? createExecutionArtifacts(env); + const pnpmInvocation = resolvePnpmCommandInvocation({ + npmExecPath: env.npm_execpath, + nodeExecPath: process.execPath, + platform: process.platform, + comSpec: env.ComSpec, + }); + const children = new Set(); + const windowsCiArgs = plan.runtimeCapabilities.isWindowsCi + ? ["--dangerouslyIgnoreUnhandledErrors"] + : []; + const silentArgs = env.OPENCLAW_TEST_SHOW_PASSED_LOGS === "1" ? [] : ["--silent=passed-only"]; + const rawMemoryTrace = env.OPENCLAW_TEST_MEMORY_TRACE?.trim().toLowerCase(); + const memoryTraceEnabled = + process.platform !== "win32" && + (rawMemoryTrace === "1" || + rawMemoryTrace === "true" || + (rawMemoryTrace !== "0" && rawMemoryTrace !== "false" && plan.runtimeCapabilities.isCI)); + const parseEnvNumber = (name, fallback) => { + const parsed = Number.parseInt(env[name] ?? "", 10); + return Number.isFinite(parsed) && parsed >= 0 ? parsed : fallback; + }; + const memoryTracePollMs = Math.max( + 250, + parseEnvNumber("OPENCLAW_TEST_MEMORY_TRACE_POLL_MS", 1000), + ); + const memoryTraceTopCount = Math.max( + 1, + parseEnvNumber("OPENCLAW_TEST_MEMORY_TRACE_TOP_COUNT", 6), + ); + const requestedHeapSnapshotIntervalMs = Math.max( + 0, + parseEnvNumber("OPENCLAW_TEST_HEAPSNAPSHOT_INTERVAL_MS", 0), + ); + const heapSnapshotMinIntervalMs = 1000; + const heapSnapshotIntervalMs = + requestedHeapSnapshotIntervalMs > 0 + ? Math.max(heapSnapshotMinIntervalMs, requestedHeapSnapshotIntervalMs) + : 0; + const heapSnapshotEnabled = + process.platform !== "win32" && heapSnapshotIntervalMs >= heapSnapshotMinIntervalMs; + const heapSnapshotSignal = env.OPENCLAW_TEST_HEAPSNAPSHOT_SIGNAL?.trim() || "SIGUSR2"; + const heapSnapshotBaseDir = heapSnapshotEnabled + ? path.resolve( + env.OPENCLAW_TEST_HEAPSNAPSHOT_DIR?.trim() || + path.join(os.tmpdir(), `openclaw-heapsnapshots-${Date.now()}`), + ) + : null; + const maxOldSpaceSizeMb = (() => { + const raw = env.OPENCLAW_TEST_MAX_OLD_SPACE_SIZE_MB ?? ""; + const parsed = Number.parseInt(raw, 10); + if (Number.isFinite(parsed) && parsed > 0) { + return parsed; + } + if (plan.runtimeCapabilities.isCI && !plan.runtimeCapabilities.isWindows) { + return DEFAULT_CI_MAX_OLD_SPACE_SIZE_MB; + } + return null; + })(); + + const shutdown = (signal) => { + for (const child of children) { + child.kill(signal); + } + }; + process.on("SIGINT", () => shutdown("SIGINT")); + process.on("SIGTERM", () => shutdown("SIGTERM")); + process.on("exit", artifacts.cleanupTempArtifacts); + + const runOnce = (unit, extraArgs = []) => + new Promise((resolve) => { + const startedAt = Date.now(); + const entryArgs = unit.args; + const explicitEntryFilters = getExplicitEntryFilters(entryArgs); + const args = unit.maxWorkers + ? [ + ...entryArgs, + "--maxWorkers", + String(unit.maxWorkers), + ...silentArgs, + ...windowsCiArgs, + ...extraArgs, + ] + : [...entryArgs, ...silentArgs, ...windowsCiArgs, ...extraArgs]; + const spawnArgs = [...pnpmInvocation.args, ...args]; + const shardLabel = getShardLabel(extraArgs); + const artifactStem = [ + sanitizeArtifactName(unit.id), + shardLabel ? `shard-${sanitizeArtifactName(shardLabel)}` : "", + String(startedAt), + ] + .filter(Boolean) + .join("-"); + const laneLogPath = path.join(artifacts.ensureTempArtifactDir(), `${artifactStem}.log`); + const laneLogStream = fs.createWriteStream(laneLogPath, { flags: "w" }); + laneLogStream.write(`[test-parallel] entry=${unit.id}\n`); + laneLogStream.write(`[test-parallel] cwd=${process.cwd()}\n`); + laneLogStream.write( + `[test-parallel] command=${[pnpmInvocation.command, ...spawnArgs].join(" ")}\n\n`, + ); + console.log( + `[test-parallel] start ${unit.id} workers=${unit.maxWorkers ?? "default"} filters=${String( + countExplicitEntryFilters(entryArgs) ?? "all", + )}`, + ); + const nodeOptions = env.NODE_OPTIONS ?? ""; + const nextNodeOptions = WARNING_SUPPRESSION_FLAGS.reduce( + (acc, flag) => (acc.includes(flag) ? acc : `${acc} ${flag}`.trim()), + nodeOptions, + ); + const heapSnapshotDir = + heapSnapshotBaseDir === null ? null : path.join(heapSnapshotBaseDir, unit.id); + let resolvedNodeOptions = + maxOldSpaceSizeMb && !nextNodeOptions.includes("--max-old-space-size=") + ? `${nextNodeOptions} --max-old-space-size=${maxOldSpaceSizeMb}`.trim() + : nextNodeOptions; + if (heapSnapshotEnabled && heapSnapshotDir) { + try { + fs.mkdirSync(heapSnapshotDir, { recursive: true }); + } catch (err) { + console.error( + `[test-parallel] failed to create heap snapshot dir ${heapSnapshotDir}: ${String(err)}`, + ); + resolve(1); + return; + } + resolvedNodeOptions = ensureNodeOptionFlag( + resolvedNodeOptions, + "--diagnostic-dir=", + `--diagnostic-dir=${heapSnapshotDir}`, + ); + resolvedNodeOptions = ensureNodeOptionFlag( + resolvedNodeOptions, + "--heapsnapshot-signal=", + `--heapsnapshot-signal=${heapSnapshotSignal}`, + ); + } + let output = ""; + let fatalSeen = false; + let childError = null; + let child; + let pendingLine = ""; + let memoryPollTimer = null; + let heapSnapshotTimer = null; + const memoryFileRecords = []; + let initialTreeSample = null; + let latestTreeSample = null; + let peakTreeSample = null; + let heapSnapshotSequence = 0; + const updatePeakTreeSample = (sample, reason) => { + if (!sample) { + return; + } + if (!peakTreeSample || sample.rssKb > peakTreeSample.rssKb) { + peakTreeSample = { ...sample, reason }; + } + }; + const triggerHeapSnapshot = (reason) => { + if (!heapSnapshotEnabled || !child?.pid || !heapSnapshotDir) { + return; + } + const records = getProcessTreeRecords(child.pid) ?? []; + const targetPids = records + .filter((record) => record.pid !== process.pid && isNodeLikeProcess(record.command)) + .map((record) => record.pid); + if (targetPids.length === 0) { + return; + } + heapSnapshotSequence += 1; + let signaledCount = 0; + for (const pid of targetPids) { + try { + process.kill(pid, heapSnapshotSignal); + signaledCount += 1; + } catch {} + } + if (signaledCount > 0) { + console.log( + `[test-parallel][heap] ${unit.id} seq=${String(heapSnapshotSequence)} reason=${reason} signaled=${String( + signaledCount, + )}/${String(targetPids.length)} dir=${heapSnapshotDir}`, + ); + } + }; + const captureTreeSample = (reason) => { + if (!memoryTraceEnabled || !child?.pid) { + return null; + } + const sample = sampleProcessTreeRssKb(child.pid); + if (!sample) { + return null; + } + latestTreeSample = sample; + if (!initialTreeSample) { + initialTreeSample = sample; + } + updatePeakTreeSample(sample, reason); + return sample; + }; + const logMemoryTraceForText = (text) => { + if (!memoryTraceEnabled) { + return; + } + const combined = `${pendingLine}${text}`; + const lines = combined.split(/\r?\n/u); + pendingLine = lines.pop() ?? ""; + const completedFiles = parseCompletedTestFileLines(lines.join("\n")); + for (const completedFile of completedFiles) { + const sample = captureTreeSample(completedFile.file); + if (!sample) { + continue; + } + const previousRssKb = + memoryFileRecords.length > 0 + ? (memoryFileRecords.at(-1)?.rssKb ?? initialTreeSample?.rssKb ?? sample.rssKb) + : (initialTreeSample?.rssKb ?? sample.rssKb); + const deltaKb = sample.rssKb - previousRssKb; + const record = { + ...completedFile, + rssKb: sample.rssKb, + processCount: sample.processCount, + deltaKb, + }; + memoryFileRecords.push(record); + console.log( + `[test-parallel][mem] ${unit.id} file=${record.file} rss=${formatMemoryKb( + record.rssKb, + )} delta=${formatMemoryDeltaKb(record.deltaKb)} peak=${formatMemoryKb( + peakTreeSample?.rssKb ?? record.rssKb, + )} procs=${record.processCount}${record.durationMs ? ` duration=${formatElapsedMs(record.durationMs)}` : ""}`, + ); + } + }; + const logMemoryTraceSummary = () => { + if (!memoryTraceEnabled) { + return; + } + captureTreeSample("close"); + const fallbackRecord = + memoryFileRecords.length === 0 && + explicitEntryFilters.length === 1 && + latestTreeSample && + initialTreeSample + ? [ + { + file: explicitEntryFilters[0], + deltaKb: latestTreeSample.rssKb - initialTreeSample.rssKb, + }, + ] + : []; + const totalDeltaKb = + initialTreeSample && latestTreeSample + ? latestTreeSample.rssKb - initialTreeSample.rssKb + : 0; + const topGrowthFiles = [...memoryFileRecords, ...fallbackRecord] + .filter((record) => record.deltaKb > 0 && typeof record.file === "string") + .toSorted((left, right) => right.deltaKb - left.deltaKb) + .slice(0, memoryTraceTopCount) + .map((record) => `${record.file}:${formatMemoryDeltaKb(record.deltaKb)}`); + console.log( + `[test-parallel][mem] summary ${unit.id} files=${memoryFileRecords.length} peak=${formatMemoryKb( + peakTreeSample?.rssKb ?? 0, + )} totalDelta=${formatMemoryDeltaKb(totalDeltaKb)} peakAt=${ + peakTreeSample?.reason ?? "n/a" + } top=${topGrowthFiles.length > 0 ? topGrowthFiles.join(", ") : "none"}`, + ); + }; + try { + child = spawn(pnpmInvocation.command, spawnArgs, { + stdio: ["inherit", "pipe", "pipe"], + env: { + ...env, + ...unit.env, + VITEST_GROUP: unit.id, + NODE_OPTIONS: resolvedNodeOptions, + }, + shell: false, + }); + captureTreeSample("spawn"); + if (memoryTraceEnabled) { + memoryPollTimer = setInterval(() => { + captureTreeSample("poll"); + }, memoryTracePollMs); + } + if (heapSnapshotEnabled) { + heapSnapshotTimer = setInterval(() => { + triggerHeapSnapshot("interval"); + }, heapSnapshotIntervalMs); + } + } catch (err) { + laneLogStream.end(); + console.error(`[test-parallel] spawn failed: ${String(err)}`); + resolve(1); + return; + } + children.add(child); + child.stdout?.on("data", (chunk) => { + const text = chunk.toString(); + fatalSeen ||= hasFatalTestRunOutput(`${output}${text}`); + output = appendCapturedOutput(output, text); + laneLogStream.write(text); + logMemoryTraceForText(text); + process.stdout.write(chunk); + }); + child.stderr?.on("data", (chunk) => { + const text = chunk.toString(); + fatalSeen ||= hasFatalTestRunOutput(`${output}${text}`); + output = appendCapturedOutput(output, text); + laneLogStream.write(text); + logMemoryTraceForText(text); + process.stderr.write(chunk); + }); + child.on("error", (err) => { + childError = err; + laneLogStream.write(`\n[test-parallel] child error: ${String(err)}\n`); + console.error(`[test-parallel] child error: ${String(err)}`); + }); + child.on("close", (code, signal) => { + if (memoryPollTimer) { + clearInterval(memoryPollTimer); + } + if (heapSnapshotTimer) { + clearInterval(heapSnapshotTimer); + } + children.delete(child); + const resolvedCode = resolveTestRunExitCode({ + code, + signal, + output, + fatalSeen, + childError, + }); + const elapsedMs = Date.now() - startedAt; + logMemoryTraceSummary(); + if (resolvedCode !== 0) { + const failureTail = formatCapturedOutputTail(output); + const failureArtifactPath = artifacts.writeTempJsonArtifact(`${artifactStem}-failure`, { + entry: unit.id, + command: [pnpmInvocation.command, ...spawnArgs], + elapsedMs, + error: childError ? String(childError) : null, + exitCode: resolvedCode, + fatalSeen, + logPath: laneLogPath, + outputTail: failureTail, + signal: signal ?? null, + }); + if (failureTail) { + console.error(`[test-parallel] failure tail ${unit.id}\n${failureTail}`); + } + console.error( + `[test-parallel] failure artifacts ${unit.id} log=${laneLogPath} meta=${failureArtifactPath}`, + ); + } + laneLogStream.write( + `\n[test-parallel] done ${unit.id} code=${String(resolvedCode)} signal=${ + signal ?? "none" + } elapsed=${formatElapsedMs(elapsedMs)}\n`, + ); + laneLogStream.end(); + console.log( + `[test-parallel] done ${unit.id} code=${String(resolvedCode)} elapsed=${formatElapsedMs(elapsedMs)}`, + ); + resolve(resolvedCode); + }); + }); + + const runUnit = async (unit, extraArgs = []) => { + if (unit.fixedShardIndex !== undefined) { + if (plan.shardIndexOverride !== null && plan.shardIndexOverride !== unit.fixedShardIndex) { + return 0; + } + return runOnce(unit, extraArgs); + } + const explicitFilterCount = countExplicitEntryFilters(unit.args); + const topLevelAssignedShard = plan.topLevelSingleShardAssignments.get(unit); + if (topLevelAssignedShard !== undefined) { + if (plan.shardIndexOverride !== null && plan.shardIndexOverride !== topLevelAssignedShard) { + return 0; + } + return runOnce(unit, extraArgs); + } + const effectiveShardCount = + explicitFilterCount === null + ? plan.shardCount + : Math.min(plan.shardCount, Math.max(1, explicitFilterCount - 1)); + if (effectiveShardCount <= 1) { + if (plan.shardIndexOverride !== null && plan.shardIndexOverride > effectiveShardCount) { + return 0; + } + return runOnce(unit, extraArgs); + } + if (plan.shardIndexOverride !== null) { + if (plan.shardIndexOverride > effectiveShardCount) { + return 0; + } + return runOnce(unit, [ + "--shard", + `${plan.shardIndexOverride}/${effectiveShardCount}`, + ...extraArgs, + ]); + } + for (let shardIndex = 1; shardIndex <= effectiveShardCount; shardIndex += 1) { + // eslint-disable-next-line no-await-in-loop + const code = await runOnce(unit, [ + "--shard", + `${shardIndex}/${effectiveShardCount}`, + ...extraArgs, + ]); + if (code !== 0) { + return code; + } + } + return 0; + }; + + const runUnitsWithLimit = async (units, extraArgs = [], concurrency = 1) => { + if (units.length === 0) { + return undefined; + } + const normalizedConcurrency = Math.max(1, Math.floor(concurrency)); + if (normalizedConcurrency <= 1) { + for (const unit of units) { + // eslint-disable-next-line no-await-in-loop + const code = await runUnit(unit, extraArgs); + if (code !== 0) { + return code; + } + } + return undefined; + } + let nextIndex = 0; + let firstFailure; + const worker = async () => { + while (firstFailure === undefined) { + const unitIndex = nextIndex; + nextIndex += 1; + if (unitIndex >= units.length) { + return; + } + const code = await runUnit(units[unitIndex], extraArgs); + if (code !== 0 && firstFailure === undefined) { + firstFailure = code; + } + } + }; + const workerCount = Math.min(normalizedConcurrency, units.length); + await Promise.all(Array.from({ length: workerCount }, () => worker())); + return firstFailure; + }; + + const runUnits = async (units, extraArgs = []) => { + if (plan.topLevelParallelEnabled) { + return runUnitsWithLimit(units, extraArgs, plan.topLevelParallelLimit); + } + return runUnitsWithLimit(units, extraArgs); + }; + + if (plan.passthroughMetadataOnly) { + return runOnce( + { + id: "vitest-meta", + args: ["vitest", "run"], + maxWorkers: null, + }, + plan.passthroughOptionArgs, + ); + } + + if (plan.targetedUnits.length > 0) { + if (plan.passthroughRequiresSingleRun && plan.targetedUnits.length > 1) { + console.error( + "[test-parallel] The provided Vitest args require a single run, but the selected test filters span multiple wrapper configs. Run one target/config at a time.", + ); + return 2; + } + const failedTargetedParallel = await runUnits(plan.parallelUnits, plan.passthroughOptionArgs); + if (failedTargetedParallel !== undefined) { + return failedTargetedParallel; + } + for (const unit of plan.serialUnits) { + // eslint-disable-next-line no-await-in-loop + const code = await runUnit(unit, plan.passthroughOptionArgs); + if (code !== 0) { + return code; + } + } + return 0; + } + + if (plan.passthroughRequiresSingleRun && plan.passthroughOptionArgs.length > 0) { + console.error( + "[test-parallel] The provided Vitest args require a single run. Use the dedicated npm script for that workflow (for example `pnpm test:coverage`) or target a single test file/filter.", + ); + return 2; + } + + if (plan.serialPrefixUnits.length > 0) { + const failedSerialPrefix = await runUnitsWithLimit( + plan.serialPrefixUnits, + plan.passthroughOptionArgs, + 1, + ); + if (failedSerialPrefix !== undefined) { + return failedSerialPrefix; + } + const failedDeferredParallel = plan.deferredRunConcurrency + ? await runUnitsWithLimit( + plan.deferredParallelUnits, + plan.passthroughOptionArgs, + plan.deferredRunConcurrency, + ) + : await runUnits(plan.deferredParallelUnits, plan.passthroughOptionArgs); + if (failedDeferredParallel !== undefined) { + return failedDeferredParallel; + } + } else { + const failedParallel = await runUnits(plan.parallelUnits, plan.passthroughOptionArgs); + if (failedParallel !== undefined) { + return failedParallel; + } + } + + for (const unit of plan.serialUnits) { + // eslint-disable-next-line no-await-in-loop + const code = await runUnit(unit, plan.passthroughOptionArgs); + if (code !== 0) { + return code; + } + } + return 0; +} diff --git a/scripts/test-planner/planner.mjs b/scripts/test-planner/planner.mjs new file mode 100644 index 00000000000..bdc9e677ee5 --- /dev/null +++ b/scripts/test-planner/planner.mjs @@ -0,0 +1,1023 @@ +import path from "node:path"; +import { isUnitConfigTestFile } from "../../vitest.unit-paths.mjs"; +import { + loadChannelTimingManifest, + loadUnitMemoryHotspotManifest, + loadUnitTimingManifest, + packFilesByDuration, + packFilesByDurationWithBaseLoads, + selectUnitHeavyFileGroups, +} from "../test-runner-manifest.mjs"; +import { loadTestCatalog, normalizeRepoPath } from "./catalog.mjs"; +import { resolveExecutionBudget, resolveRuntimeCapabilities } from "./runtime-profile.mjs"; +import { + countExplicitEntryFilters, + getExplicitEntryFilters, + parsePassthroughArgs, + SINGLE_RUN_ONLY_FLAGS, +} from "./vitest-args.mjs"; + +const parseEnvNumber = (env, name, fallback) => { + const parsed = Number.parseInt(env[name] ?? "", 10); + return Number.isFinite(parsed) && parsed >= 0 ? parsed : fallback; +}; + +const normalizeSurfaces = (values = []) => [ + ...new Set( + values + .flatMap((value) => String(value).split(",")) + .map((value) => value.trim()) + .filter(Boolean), + ), +]; + +const EXPLICIT_PLAN_SURFACES = new Set(["unit", "extensions", "channels", "gateway"]); + +const validateExplicitSurfaces = (surfaces) => { + const invalidSurfaces = surfaces.filter((surface) => !EXPLICIT_PLAN_SURFACES.has(surface)); + if (invalidSurfaces.length > 0) { + throw new Error( + `Unsupported --surface value(s): ${invalidSurfaces.join(", ")}. Supported surfaces: unit, extensions, channels, gateway.`, + ); + } +}; + +const buildRequestedSurfaces = (request, env) => { + const explicit = normalizeSurfaces(request.surfaces ?? []); + if (explicit.length > 0) { + validateExplicitSurfaces(explicit); + return explicit; + } + const surfaces = []; + const skipDefaultRuns = env.OPENCLAW_TEST_SKIP_DEFAULT === "1"; + if (!skipDefaultRuns) { + surfaces.push("unit"); + } + if (env.OPENCLAW_TEST_INCLUDE_EXTENSIONS === "1") { + surfaces.push("extensions"); + } + if (env.OPENCLAW_TEST_INCLUDE_CHANNELS === "1") { + surfaces.push("channels"); + } + if (env.OPENCLAW_TEST_INCLUDE_GATEWAY === "1") { + surfaces.push("gateway"); + } + return surfaces; +}; + +const createPlannerContext = (request, options = {}) => { + const env = options.env ?? process.env; + const runtime = resolveRuntimeCapabilities(env, { + mode: request.mode ?? null, + profile: request.profile ?? null, + cpuCount: options.cpuCount, + totalMemoryBytes: options.totalMemoryBytes, + platform: options.platform, + loadAverage: options.loadAverage, + nodeVersion: options.nodeVersion, + }); + const executionBudget = resolveExecutionBudget(runtime); + const catalog = options.catalog ?? loadTestCatalog(); + const unitTimingManifest = loadUnitTimingManifest(); + const channelTimingManifest = loadChannelTimingManifest(); + const unitMemoryHotspotManifest = loadUnitMemoryHotspotManifest(); + return { + env, + runtime, + executionBudget, + catalog, + unitTimingManifest, + channelTimingManifest, + unitMemoryHotspotManifest, + }; +}; + +const estimateEntryFilesDurationMs = (entry, files, context) => { + const estimateDurationMs = resolveEntryTimingEstimator(entry, context); + if (!estimateDurationMs) { + return files.length * 1_000; + } + return files.reduce((totalMs, file) => totalMs + estimateDurationMs(file), 0); +}; + +const resolveEntryTimingEstimator = (entry, context) => { + const configIndex = entry.args.findIndex((arg) => arg === "--config"); + const config = configIndex >= 0 ? (entry.args[configIndex + 1] ?? "") : ""; + if (config === "vitest.unit.config.ts") { + return (file) => + context.unitTimingManifest.files[file]?.durationMs ?? + context.unitTimingManifest.defaultDurationMs; + } + if (config === "vitest.channels.config.ts" || config === "vitest.extensions.config.ts") { + return (file) => + context.channelTimingManifest.files[file]?.durationMs ?? + context.channelTimingManifest.defaultDurationMs; + } + return null; +}; + +const splitFilesByDurationBudget = (files, targetDurationMs, estimateDurationMs) => { + if (!Number.isFinite(targetDurationMs) || targetDurationMs <= 0 || files.length <= 1) { + return [files]; + } + + const batches = []; + let currentBatch = []; + let currentDurationMs = 0; + + for (const file of files) { + const durationMs = estimateDurationMs(file); + if (currentBatch.length > 0 && currentDurationMs + durationMs > targetDurationMs) { + batches.push(currentBatch); + currentBatch = []; + currentDurationMs = 0; + } + currentBatch.push(file); + currentDurationMs += durationMs; + } + + if (currentBatch.length > 0) { + batches.push(currentBatch); + } + + return batches; +}; + +const resolveMaxWorkersForUnit = (unit, context) => { + const overrideWorkers = Number.parseInt(context.env.OPENCLAW_TEST_WORKERS ?? "", 10); + const resolvedOverride = + Number.isFinite(overrideWorkers) && overrideWorkers > 0 ? overrideWorkers : null; + if (resolvedOverride) { + return resolvedOverride; + } + const budget = context.executionBudget; + if (unit.isolate) { + return budget.unitIsolatedWorkers; + } + if (unit.id.startsWith("unit-heavy-")) { + return budget.unitHeavyWorkers; + } + if (unit.surface === "extensions") { + return budget.extensionWorkers; + } + if (unit.surface === "gateway") { + return budget.gatewayWorkers; + } + return budget.unitSharedWorkers; +}; + +const formatPerFileEntryName = (owner, file) => { + const baseName = path + .basename(file) + .replace(/\.live\.test\.ts$/u, "") + .replace(/\.e2e\.test\.ts$/u, "") + .replace(/\.test\.ts$/u, ""); + return `${owner}-${baseName}`; +}; + +const createExecutionUnit = (context, config) => { + const unit = { + id: config.id, + surface: config.surface, + isolate: Boolean(config.isolate), + pool: config.pool ?? "forks", + args: config.args, + env: config.env, + includeFiles: config.includeFiles, + serialPhase: config.serialPhase, + fixedShardIndex: config.fixedShardIndex, + estimatedDurationMs: config.estimatedDurationMs, + timeoutMs: config.timeoutMs, + reasons: config.reasons ?? [], + }; + unit.maxWorkers = resolveMaxWorkersForUnit(unit, context); + return unit; +}; + +const withIncludeFileEnv = (context, unitId, files) => ({ + OPENCLAW_VITEST_INCLUDE_FILE: context.writeTempJsonArtifact(unitId, files), +}); + +const resolveUnitHeavyFileGroups = (context) => { + const { env, runtime, executionBudget, catalog, unitTimingManifest, unitMemoryHotspotManifest } = + context; + const heavyUnitFileLimit = parseEnvNumber( + env, + "OPENCLAW_TEST_HEAVY_UNIT_FILE_LIMIT", + runtime.intentProfile === "max" + ? Math.max(executionBudget.heavyUnitFileLimit, 90) + : executionBudget.heavyUnitFileLimit, + ); + const heavyUnitLaneCount = parseEnvNumber( + env, + "OPENCLAW_TEST_HEAVY_UNIT_LANES", + runtime.intentProfile === "max" + ? Math.max(executionBudget.heavyUnitLaneCount, 6) + : executionBudget.heavyUnitLaneCount, + ); + const heavyUnitMinDurationMs = parseEnvNumber(env, "OPENCLAW_TEST_HEAVY_UNIT_MIN_MS", 1200); + const memoryHeavyUnitFileLimit = parseEnvNumber( + env, + "OPENCLAW_TEST_MEMORY_HEAVY_UNIT_FILE_LIMIT", + executionBudget.memoryHeavyUnitFileLimit, + ); + const memoryHeavyUnitMinDeltaKb = parseEnvNumber( + env, + "OPENCLAW_TEST_MEMORY_HEAVY_UNIT_MIN_KB", + unitMemoryHotspotManifest.defaultMinDeltaKb, + ); + return { + heavyUnitLaneCount, + ...selectUnitHeavyFileGroups({ + candidates: catalog.allKnownUnitFiles, + behaviorOverrides: catalog.unitBehaviorOverrideSet, + timedLimit: heavyUnitFileLimit, + timedMinDurationMs: heavyUnitMinDurationMs, + memoryLimit: memoryHeavyUnitFileLimit, + memoryMinDeltaKb: memoryHeavyUnitMinDeltaKb, + timings: unitTimingManifest, + hotspots: unitMemoryHotspotManifest, + }), + }; +}; + +const buildDefaultUnits = (context, request) => { + const { env, executionBudget, catalog, unitTimingManifest, channelTimingManifest } = context; + const noIsolateArgs = context.noIsolateArgs; + const selectedSurfaces = buildRequestedSurfaces(request, env); + const selectedSurfaceSet = new Set(selectedSurfaces); + + const { + heavyUnitLaneCount, + memoryHeavyFiles: memoryHeavyUnitFiles, + timedHeavyFiles: timedHeavyUnitFiles, + } = resolveUnitHeavyFileGroups(context); + const unitMemoryIsolatedFiles = [...memoryHeavyUnitFiles].filter( + (file) => !catalog.unitBehaviorOverrideSet.has(file), + ); + const unitSchedulingOverrideSet = new Set([ + ...catalog.unitBehaviorOverrideSet, + ...memoryHeavyUnitFiles, + ]); + const unitFastExcludedFiles = [ + ...new Set([ + ...unitSchedulingOverrideSet, + ...timedHeavyUnitFiles, + ...catalog.channelIsolatedFiles, + ]), + ]; + const estimateUnitDurationMs = (file) => + unitTimingManifest.files[file]?.durationMs ?? unitTimingManifest.defaultDurationMs; + const estimateChannelDurationMs = (file) => + channelTimingManifest.files[file]?.durationMs ?? channelTimingManifest.defaultDurationMs; + const unitFastCandidateFiles = catalog.allKnownUnitFiles.filter( + (file) => !new Set(unitFastExcludedFiles).has(file), + ); + const extensionSharedCandidateFiles = catalog.allKnownTestFiles.filter( + (file) => file.startsWith("extensions/") && !catalog.extensionForkIsolatedFileSet.has(file), + ); + const channelSharedCandidateFiles = catalog.allKnownTestFiles.filter( + (file) => + catalog.channelTestPrefixes.some((prefix) => file.startsWith(prefix)) && + !catalog.channelIsolatedFileSet.has(file), + ); + const defaultExtensionsBatchTargetMs = executionBudget.extensionsBatchTargetMs; + const extensionsBatchTargetMs = parseEnvNumber( + env, + "OPENCLAW_TEST_EXTENSIONS_BATCH_TARGET_MS", + defaultExtensionsBatchTargetMs, + ); + const defaultUnitFastLaneCount = executionBudget.unitFastLaneCount; + const unitFastLaneCount = Math.max( + 1, + parseEnvNumber(env, "OPENCLAW_TEST_UNIT_FAST_LANES", defaultUnitFastLaneCount), + ); + const defaultUnitFastBatchTargetMs = executionBudget.unitFastBatchTargetMs; + const unitFastBatchTargetMs = parseEnvNumber( + env, + "OPENCLAW_TEST_UNIT_FAST_BATCH_TARGET_MS", + defaultUnitFastBatchTargetMs, + ); + const defaultChannelsBatchTargetMs = executionBudget.channelsBatchTargetMs; + const channelsBatchTargetMs = parseEnvNumber( + env, + "OPENCLAW_TEST_CHANNELS_BATCH_TARGET_MS", + defaultChannelsBatchTargetMs, + ); + const unitFastBuckets = + unitFastLaneCount > 1 + ? packFilesByDuration(unitFastCandidateFiles, unitFastLaneCount, estimateUnitDurationMs) + : [unitFastCandidateFiles]; + const units = []; + + if (selectedSurfaceSet.has("unit")) { + for (const [laneIndex, files] of unitFastBuckets.entries()) { + const laneName = + unitFastBuckets.length === 1 ? "unit-fast" : `unit-fast-${String(laneIndex + 1)}`; + const recycledBatches = splitFilesByDurationBudget( + files, + unitFastBatchTargetMs, + estimateUnitDurationMs, + ); + for (const [batchIndex, batch] of recycledBatches.entries()) { + if (batch.length === 0) { + continue; + } + const unitId = + recycledBatches.length === 1 ? laneName : `${laneName}-batch-${String(batchIndex + 1)}`; + units.push( + createExecutionUnit(context, { + id: unitId, + surface: "unit", + isolate: false, + serialPhase: "unit-fast", + includeFiles: batch, + estimatedDurationMs: estimateEntryFilesDurationMs( + { args: ["vitest", "run", "--config", "vitest.unit.config.ts"] }, + batch, + context, + ), + env: withIncludeFileEnv( + context, + `vitest-unit-fast-include-${String(laneIndex + 1)}-${String(batchIndex + 1)}`, + batch, + ), + args: [ + "vitest", + "run", + "--config", + "vitest.unit.config.ts", + "--pool=forks", + ...noIsolateArgs, + ], + reasons: ["unit-fast-shared"], + }), + ); + } + } + + for (const file of catalog.unitForkIsolatedFiles) { + units.push( + createExecutionUnit(context, { + id: `unit-${path.basename(file, ".test.ts")}-isolated`, + surface: "unit", + isolate: true, + args: [ + "vitest", + "run", + "--config", + "vitest.unit.config.ts", + "--pool=forks", + ...noIsolateArgs, + file, + ], + reasons: ["unit-isolated-manifest"], + }), + ); + } + + const heavyUnitBuckets = packFilesByDuration( + timedHeavyUnitFiles, + heavyUnitLaneCount, + estimateUnitDurationMs, + ); + for (const [index, files] of heavyUnitBuckets.entries()) { + units.push( + createExecutionUnit(context, { + id: `unit-heavy-${String(index + 1)}`, + surface: "unit", + isolate: false, + args: [ + "vitest", + "run", + "--config", + "vitest.unit.config.ts", + "--pool=forks", + ...noIsolateArgs, + ...files, + ], + reasons: ["unit-timed-heavy"], + }), + ); + } + + for (const file of unitMemoryIsolatedFiles) { + units.push( + createExecutionUnit(context, { + id: `unit-${path.basename(file, ".test.ts")}-memory-isolated`, + surface: "unit", + isolate: true, + args: [ + "vitest", + "run", + "--config", + "vitest.unit.config.ts", + "--pool=forks", + ...noIsolateArgs, + file, + ], + reasons: ["unit-memory-isolated"], + }), + ); + } + + if (catalog.unitThreadPinnedFiles.length > 0) { + units.push( + createExecutionUnit(context, { + id: "unit-pinned", + surface: "unit", + isolate: false, + args: [ + "vitest", + "run", + "--config", + "vitest.unit.config.ts", + "--pool=forks", + ...noIsolateArgs, + ...catalog.unitThreadPinnedFiles, + ], + reasons: ["unit-pinned-manifest"], + }), + ); + } + } + + if (selectedSurfaceSet.has("extensions")) { + for (const file of catalog.extensionForkIsolatedFiles) { + units.push( + createExecutionUnit(context, { + id: `extensions-${path.basename(file, ".test.ts")}-isolated`, + surface: "extensions", + isolate: true, + args: ["vitest", "run", "--config", "vitest.extensions.config.ts", "--pool=forks", file], + reasons: ["extensions-isolated-manifest"], + }), + ); + } + const extensionBatches = splitFilesByDurationBudget( + extensionSharedCandidateFiles, + extensionsBatchTargetMs, + estimateChannelDurationMs, + ); + for (const [batchIndex, batch] of extensionBatches.entries()) { + if (batch.length === 0) { + continue; + } + const unitId = + extensionBatches.length === 1 ? "extensions" : `extensions-batch-${String(batchIndex + 1)}`; + units.push( + createExecutionUnit(context, { + id: unitId, + surface: "extensions", + isolate: false, + serialPhase: "extensions", + includeFiles: batch, + estimatedDurationMs: estimateEntryFilesDurationMs( + { args: ["vitest", "run", "--config", "vitest.extensions.config.ts"] }, + batch, + context, + ), + env: withIncludeFileEnv( + context, + `vitest-extensions-include-${String(batchIndex + 1)}`, + batch, + ), + args: ["vitest", "run", "--config", "vitest.extensions.config.ts", ...noIsolateArgs], + reasons: ["extensions-shared"], + }), + ); + } + } + + if (selectedSurfaceSet.has("channels")) { + for (const file of catalog.channelIsolatedFiles) { + units.push( + createExecutionUnit(context, { + id: `${path.basename(file, ".test.ts")}-channels-isolated`, + surface: "channels", + isolate: true, + args: [ + "vitest", + "run", + "--config", + "vitest.channels.config.ts", + "--pool=forks", + ...noIsolateArgs, + file, + ], + reasons: ["channels-isolated-rule"], + }), + ); + } + const channelBatches = splitFilesByDurationBudget( + channelSharedCandidateFiles, + channelsBatchTargetMs, + estimateChannelDurationMs, + ); + for (const [batchIndex, batch] of channelBatches.entries()) { + if (batch.length === 0) { + continue; + } + const unitId = + channelBatches.length === 1 ? "channels" : `channels-batch-${String(batchIndex + 1)}`; + units.push( + createExecutionUnit(context, { + id: unitId, + surface: "channels", + isolate: false, + serialPhase: "channels", + includeFiles: batch, + estimatedDurationMs: estimateEntryFilesDurationMs( + { args: ["vitest", "run", "--config", "vitest.channels.config.ts"] }, + batch, + context, + ), + env: withIncludeFileEnv( + context, + `vitest-channels-include-${String(batchIndex + 1)}`, + batch, + ), + args: ["vitest", "run", "--config", "vitest.channels.config.ts", ...noIsolateArgs], + reasons: ["channels-shared"], + }), + ); + } + } + + if (selectedSurfaceSet.has("gateway")) { + units.push( + createExecutionUnit(context, { + id: "gateway", + surface: "gateway", + isolate: false, + args: [ + "vitest", + "run", + "--config", + "vitest.gateway.config.ts", + "--pool=forks", + ...noIsolateArgs, + ], + reasons: ["gateway-surface"], + }), + ); + } + + return { units, unitMemoryIsolatedFiles }; +}; + +const createTargetedUnit = (context, classification, filters) => { + const owner = classification.legacyBasePinned ? "base-pinned" : classification.surface; + const unitId = + filters.length === 1 && (classification.isolated || owner === "base-pinned") + ? `${formatPerFileEntryName(owner, filters[0])}${classification.isolated ? "-isolated" : ""}` + : classification.isolated + ? `${owner}-isolated` + : owner; + const args = (() => { + if (owner === "unit") { + return [ + "vitest", + "run", + "--config", + "vitest.unit.config.ts", + "--pool=forks", + ...context.noIsolateArgs, + ...filters, + ]; + } + if (owner === "base-pinned") { + return [ + "vitest", + "run", + "--config", + "vitest.config.ts", + "--pool=forks", + ...context.noIsolateArgs, + ...filters, + ]; + } + if (owner === "extensions") { + return [ + "vitest", + "run", + "--config", + "vitest.extensions.config.ts", + ...(classification.isolated ? ["--pool=forks"] : []), + ...context.noIsolateArgs, + ...filters, + ]; + } + if (owner === "gateway") { + return [ + "vitest", + "run", + "--config", + "vitest.gateway.config.ts", + "--pool=forks", + ...context.noIsolateArgs, + ...filters, + ]; + } + if (owner === "channels") { + return [ + "vitest", + "run", + "--config", + "vitest.channels.config.ts", + ...(classification.isolated ? ["--pool=forks"] : []), + ...context.noIsolateArgs, + ...filters, + ]; + } + if (owner === "live") { + return [ + "vitest", + "run", + "--config", + "vitest.live.config.ts", + ...context.noIsolateArgs, + ...filters, + ]; + } + if (owner === "e2e") { + return [ + "vitest", + "run", + "--config", + "vitest.e2e.config.ts", + ...context.noIsolateArgs, + ...filters, + ]; + } + return [ + "vitest", + "run", + "--config", + "vitest.config.ts", + ...context.noIsolateArgs, + ...(classification.isolated ? ["--pool=forks"] : []), + ...filters, + ]; + })(); + return createExecutionUnit(context, { + id: unitId, + surface: classification.legacyBasePinned ? "base" : classification.surface, + isolate: classification.isolated || owner === "base-pinned", + args, + reasons: classification.reasons, + }); +}; + +const buildTargetedUnits = (context, request) => { + if (request.fileFilters.length === 0) { + return []; + } + const unitMemoryIsolatedFiles = request.unitMemoryIsolatedFiles ?? []; + const groups = request.fileFilters.reduce((acc, fileFilter) => { + const matchedFiles = context.catalog.resolveFilterMatches(fileFilter); + if (matchedFiles.length === 0) { + const classification = context.catalog.classifyTestFile(normalizeRepoPath(fileFilter), { + unitMemoryIsolatedFiles, + }); + const key = `${classification.legacyBasePinned ? "base-pinned" : classification.surface}:${ + classification.isolated ? "isolated" : "default" + }`; + const files = acc.get(key) ?? { classification, files: [] }; + files.files.push(normalizeRepoPath(fileFilter)); + acc.set(key, files); + return acc; + } + for (const matchedFile of matchedFiles) { + const classification = context.catalog.classifyTestFile(matchedFile, { + unitMemoryIsolatedFiles, + }); + const key = `${classification.legacyBasePinned ? "base-pinned" : classification.surface}:${ + classification.isolated ? "isolated" : "default" + }`; + const files = acc.get(key) ?? { classification, files: [] }; + files.files.push(matchedFile); + acc.set(key, files); + } + return acc; + }, new Map()); + return Array.from(groups.values()).flatMap(({ classification, files }) => { + const uniqueFilters = [...new Set(files)]; + if (classification.isolated || classification.legacyBasePinned) { + return uniqueFilters.map((file) => + createTargetedUnit( + context, + context.catalog.classifyTestFile(file, { + unitMemoryIsolatedFiles, + }), + [file], + ), + ); + } + return [createTargetedUnit(context, classification, uniqueFilters)]; + }); +}; + +const rebuildEntryArgsWithFilters = (entryArgs, filters) => { + const baseArgs = entryArgs.slice(0, 2); + const { optionArgs } = parsePassthroughArgs(entryArgs.slice(2)); + return [...baseArgs, ...optionArgs, ...filters]; +}; + +const createPinnedShardUnit = (context, unit, files, fixedShardIndex) => { + const nextUnit = createExecutionUnit(context, { + ...unit, + id: `${unit.id}-shard-${String(fixedShardIndex)}`, + fixedShardIndex, + estimatedDurationMs: estimateEntryFilesDurationMs(unit, files, context), + includeFiles: + Array.isArray(unit.includeFiles) && unit.includeFiles.length > 0 ? files : undefined, + env: + Array.isArray(unit.includeFiles) && unit.includeFiles.length > 0 + ? { + ...unit.env, + OPENCLAW_VITEST_INCLUDE_FILE: context.writeTempJsonArtifact( + `${unit.id}-shard-${String(fixedShardIndex)}-include`, + files, + ), + } + : unit.env, + args: + Array.isArray(unit.includeFiles) && unit.includeFiles.length > 0 + ? rebuildEntryArgsWithFilters(unit.args, []) + : rebuildEntryArgsWithFilters(unit.args, files), + }); + nextUnit.fixedShardIndex = fixedShardIndex; + return nextUnit; +}; + +const expandUnitsAcrossTopLevelShards = (context, units) => { + if (context.configuredShardCount === null || context.shardCount <= 1) { + return units; + } + return units.flatMap((unit) => { + const estimateDurationMs = resolveEntryTimingEstimator(unit, context); + if (!estimateDurationMs || unit.fixedShardIndex !== undefined) { + return [unit]; + } + const candidateFiles = + Array.isArray(unit.includeFiles) && unit.includeFiles.length > 0 + ? unit.includeFiles + : getExplicitEntryFilters(unit.args); + if (candidateFiles.length <= 1) { + return [unit]; + } + const effectiveShardCount = Math.min( + context.shardCount, + Math.max(1, candidateFiles.length - 1), + ); + if (effectiveShardCount <= 1) { + return [unit]; + } + const buckets = packFilesByDurationWithBaseLoads( + candidateFiles, + effectiveShardCount, + estimateDurationMs, + ); + return buckets.flatMap((files, bucketIndex) => + files.length > 0 ? [createPinnedShardUnit(context, unit, files, bucketIndex + 1)] : [], + ); + }); +}; + +const estimateTopLevelEntryDurationMs = (unit, context) => { + if (Number.isFinite(unit.estimatedDurationMs) && unit.estimatedDurationMs > 0) { + return unit.estimatedDurationMs; + } + const filters = getExplicitEntryFilters(unit.args); + if (filters.length === 0) { + return context.unitTimingManifest.defaultDurationMs; + } + return filters.reduce((totalMs, file) => { + if (isUnitConfigTestFile(file)) { + return ( + totalMs + + (context.unitTimingManifest.files[file]?.durationMs ?? + context.unitTimingManifest.defaultDurationMs) + ); + } + if (context.catalog.channelTestPrefixes.some((prefix) => file.startsWith(prefix))) { + return totalMs + 3_000; + } + if (file.startsWith("extensions/")) { + return totalMs + 2_000; + } + return totalMs + 1_000; + }, 0); +}; + +const buildTopLevelSingleShardAssignments = (context, units) => { + if (context.shardIndexOverride === null || context.shardCount <= 1) { + return new WeakMap(); + } + + const entriesNeedingAssignment = units.filter((unit) => { + if (unit.fixedShardIndex !== undefined) { + return false; + } + const explicitFilterCount = countExplicitEntryFilters(unit.args); + if (explicitFilterCount === null) { + return false; + } + const effectiveShardCount = Math.min(context.shardCount, Math.max(1, explicitFilterCount - 1)); + return effectiveShardCount <= 1; + }); + + const assignmentMap = new WeakMap(); + const pinnedShardLoadsMs = Array.from({ length: context.shardCount }, () => 0); + for (const unit of units) { + if (unit.fixedShardIndex === undefined) { + continue; + } + const shardArrayIndex = unit.fixedShardIndex - 1; + if (shardArrayIndex < 0 || shardArrayIndex >= pinnedShardLoadsMs.length) { + continue; + } + pinnedShardLoadsMs[shardArrayIndex] += estimateTopLevelEntryDurationMs(unit, context); + } + const buckets = packFilesByDurationWithBaseLoads( + entriesNeedingAssignment, + context.shardCount, + (unit) => estimateTopLevelEntryDurationMs(unit, context), + pinnedShardLoadsMs, + ); + for (const [bucketIndex, bucket] of buckets.entries()) { + for (const unit of bucket) { + assignmentMap.set(unit, bucketIndex + 1); + } + } + return assignmentMap; +}; + +export const formatExecutionUnitSummary = (unit) => + `${unit.id} filters=${String(countExplicitEntryFilters(unit.args) || "all")} maxWorkers=${String( + unit.maxWorkers ?? "default", + )} surface=${unit.surface} isolate=${unit.isolate ? "yes" : "no"} pool=${unit.pool}`; + +export function explainExecutionTarget(request, options = {}) { + const context = createPlannerContext(request, options); + context.noIsolateArgs = + context.env.OPENCLAW_TEST_ISOLATE === "1" || context.env.OPENCLAW_TEST_ISOLATE === "true" + ? [] + : context.env.OPENCLAW_TEST_NO_ISOLATE !== "0" && + context.env.OPENCLAW_TEST_NO_ISOLATE !== "false" + ? ["--isolate=false"] + : []; + const [target] = request.fileFilters; + const matchedFiles = context.catalog.resolveFilterMatches(target); + const normalizedTarget = matchedFiles[0] ?? normalizeRepoPath(target); + const { memoryHeavyFiles } = resolveUnitHeavyFileGroups(context); + const unitMemoryIsolatedFiles = [...memoryHeavyFiles].filter( + (file) => !context.catalog.unitBehaviorOverrideSet.has(file), + ); + const classification = context.catalog.classifyTestFile(normalizedTarget, { + unitMemoryIsolatedFiles, + }); + const targetedUnit = createTargetedUnit(context, classification, [normalizedTarget]); + return { + runtimeProfile: context.runtime.runtimeProfileName, + intentProfile: context.runtime.intentProfile, + memoryBand: context.runtime.memoryBand, + loadBand: context.runtime.loadBand, + file: classification.file, + surface: classification.legacyBasePinned ? "base" : classification.surface, + isolate: targetedUnit.isolate, + pool: targetedUnit.pool, + maxWorkers: targetedUnit.maxWorkers, + reasons: classification.reasons, + args: targetedUnit.args, + }; +} + +export function buildExecutionPlan(request, options = {}) { + const env = options.env ?? process.env; + const explicitFileFilters = (request.fileFilters ?? []).map((value) => normalizeRepoPath(value)); + const { fileFilters: passthroughFileFilters, optionArgs } = parsePassthroughArgs( + request.passthroughArgs ?? [], + ); + const fileFilters = [...explicitFileFilters, ...passthroughFileFilters]; + const passthroughMetadataFlags = new Set(["-h", "--help", "--listTags", "--clearCache"]); + const passthroughMetadataOnly = + (request.passthroughArgs ?? []).length > 0 && + fileFilters.length === 0 && + optionArgs.every((arg) => { + if (!arg.startsWith("-")) { + return false; + } + const [flag] = arg.split("=", 1); + return passthroughMetadataFlags.has(flag); + }); + const passthroughRequiresSingleRun = optionArgs.some((arg) => { + if (!arg.startsWith("-")) { + return false; + } + const [flag] = arg.split("=", 1); + return SINGLE_RUN_ONLY_FLAGS.has(flag); + }); + const context = createPlannerContext( + { + ...request, + fileFilters, + passthroughOptionArgs: optionArgs, + }, + options, + ); + context.noIsolateArgs = + env.OPENCLAW_TEST_ISOLATE === "1" || env.OPENCLAW_TEST_ISOLATE === "true" + ? [] + : env.OPENCLAW_TEST_NO_ISOLATE !== "0" && env.OPENCLAW_TEST_NO_ISOLATE !== "false" + ? ["--isolate=false"] + : []; + context.writeTempJsonArtifact = + options.writeTempJsonArtifact ?? + (() => { + throw new Error("buildExecutionPlan requires writeTempJsonArtifact for include-file units"); + }); + + const shardOverride = Number.parseInt(env.OPENCLAW_TEST_SHARDS ?? "", 10); + context.configuredShardCount = + Number.isFinite(shardOverride) && shardOverride > 1 ? shardOverride : null; + context.shardCount = context.configuredShardCount ?? (context.runtime.isWindowsCi ? 2 : 1); + const shardIndexOverride = Number.parseInt(env.OPENCLAW_TEST_SHARD_INDEX ?? "", 10); + context.shardIndexOverride = + Number.isFinite(shardIndexOverride) && shardIndexOverride > 0 ? shardIndexOverride : null; + + if (context.shardIndexOverride !== null && context.shardCount <= 1) { + throw new Error( + `OPENCLAW_TEST_SHARD_INDEX=${String(context.shardIndexOverride)} requires OPENCLAW_TEST_SHARDS>1.`, + ); + } + if (context.shardIndexOverride !== null && context.shardIndexOverride > context.shardCount) { + throw new Error( + `OPENCLAW_TEST_SHARD_INDEX=${String(context.shardIndexOverride)} exceeds OPENCLAW_TEST_SHARDS=${String(context.shardCount)}.`, + ); + } + + const defaultPlanning = buildDefaultUnits(context, { ...request, fileFilters }); + let units = defaultPlanning.units; + const targetedUnits = buildTargetedUnits(context, { + ...request, + fileFilters, + unitMemoryIsolatedFiles: defaultPlanning.unitMemoryIsolatedFiles, + }); + if (context.configuredShardCount !== null && context.shardCount > 1) { + units = expandUnitsAcrossTopLevelShards(context, units); + } + const selectedUnits = targetedUnits.length > 0 ? targetedUnits : units; + const topLevelSingleShardAssignments = buildTopLevelSingleShardAssignments(context, units); + const parallelGatewayEnabled = + env.OPENCLAW_TEST_PARALLEL_GATEWAY === "1" || + (!context.runtime.isCI && context.executionBudget.gatewayWorkers > 1); + const keepGatewaySerial = + context.runtime.isWindowsCi || + env.OPENCLAW_TEST_SERIAL_GATEWAY === "1" || + context.runtime.intentProfile === "serial" || + !parallelGatewayEnabled; + const parallelUnits = keepGatewaySerial + ? selectedUnits.filter((unit) => unit.surface !== "gateway") + : selectedUnits; + const serialUnits = keepGatewaySerial + ? selectedUnits.filter((unit) => unit.surface === "gateway") + : []; + const serialPrefixUnits = parallelUnits.filter((unit) => unit.serialPhase); + const deferredParallelUnits = parallelUnits.filter((unit) => !unit.serialPhase); + const topLevelParallelEnabled = context.executionBudget.topLevelParallelEnabled; + const baseTopLevelParallelLimit = + context.noIsolateArgs.length > 0 + ? context.executionBudget.topLevelParallelLimitNoIsolate + : context.executionBudget.topLevelParallelLimitIsolated; + const defaultTopLevelParallelLimit = baseTopLevelParallelLimit; + const topLevelParallelLimit = Math.max( + 1, + parseEnvNumber(env, "OPENCLAW_TEST_TOP_LEVEL_CONCURRENCY", defaultTopLevelParallelLimit), + ); + const deferredRunConcurrency = context.executionBudget.deferredRunConcurrency; + + return { + runtimeCapabilities: context.runtime, + executionBudget: context.executionBudget, + passthroughOptionArgs: optionArgs, + passthroughRequiresSingleRun, + passthroughMetadataOnly, + fileFilters, + allUnits: units, + selectedUnits, + targetedUnits, + parallelUnits, + serialUnits, + serialPrefixUnits, + deferredParallelUnits, + topLevelParallelEnabled, + topLevelParallelLimit, + deferredRunConcurrency, + keepGatewaySerial, + shardCount: context.shardCount, + shardIndexOverride: context.shardIndexOverride, + topLevelSingleShardAssignments, + }; +} diff --git a/scripts/test-planner/runtime-profile.mjs b/scripts/test-planner/runtime-profile.mjs new file mode 100644 index 00000000000..b059eb73f15 --- /dev/null +++ b/scripts/test-planner/runtime-profile.mjs @@ -0,0 +1,348 @@ +import os from "node:os"; + +export const TEST_PROFILES = new Set(["normal", "serial", "max"]); + +export const parsePositiveInt = (value) => { + const parsed = Number.parseInt(value ?? "", 10); + return Number.isFinite(parsed) && parsed > 0 ? parsed : null; +}; + +export const resolveVitestMode = (env = process.env, explicitMode = null) => { + if (explicitMode === "ci" || explicitMode === "local") { + return explicitMode; + } + return env.CI === "true" || env.GITHUB_ACTIONS === "true" ? "ci" : "local"; +}; + +const clamp = (value, min, max) => Math.max(min, Math.min(max, value)); + +const parseProfile = (rawProfile) => { + if (!rawProfile) { + return "normal"; + } + const normalized = rawProfile.trim().toLowerCase(); + if (normalized === "low") { + return "serial"; + } + if (!TEST_PROFILES.has(normalized)) { + throw new Error( + `Unsupported test profile "${normalized}". Supported profiles: normal, serial, max.`, + ); + } + return normalized; +}; + +const resolveLoadRatio = (env, cpuCount, platform, loadAverage) => { + const loadAwareDisabledRaw = env.OPENCLAW_TEST_LOAD_AWARE?.trim().toLowerCase(); + const loadAwareDisabled = loadAwareDisabledRaw === "0" || loadAwareDisabledRaw === "false"; + if (loadAwareDisabled || platform === "win32" || cpuCount <= 0) { + return 0; + } + const source = Array.isArray(loadAverage) ? loadAverage : os.loadavg(); + return source.length > 0 ? source[0] / cpuCount : 0; +}; + +const resolveMemoryBand = (memoryGiB) => { + if (memoryGiB < 24) { + return "constrained"; + } + if (memoryGiB < 48) { + return "moderate"; + } + if (memoryGiB < 96) { + return "mid"; + } + return "high"; +}; + +const resolveLoadBand = (isLoadAware, loadRatio) => { + if (!isLoadAware) { + return "normal"; + } + if (loadRatio < 0.5) { + return "idle"; + } + if (loadRatio < 0.9) { + return "normal"; + } + if (loadRatio < 1.1) { + return "busy"; + } + return "saturated"; +}; + +const scaleForLoad = (value, loadBand) => { + if (value === null || value === undefined) { + return value; + } + const scale = loadBand === "busy" ? 0.75 : loadBand === "saturated" ? 0.5 : 1; + return Math.max(1, Math.floor(value * scale)); +}; + +const scaleConcurrencyForLoad = (value, loadBand) => { + if (value === null || value === undefined) { + return value; + } + const scale = loadBand === "busy" ? 0.8 : loadBand === "saturated" ? 0.5 : 1; + return Math.max(1, Math.floor(value * scale)); +}; + +const LOCAL_MEMORY_BUDGETS = { + constrained: { + vitestCap: 2, + unitShared: 2, + unitIsolated: 1, + unitHeavy: 1, + extensions: 1, + gateway: 1, + topLevelNoIsolate: 4, + topLevelIsolated: 2, + deferred: 1, + heavyFileLimit: 36, + heavyLaneCount: 3, + memoryHeavyFileLimit: 8, + unitFastBatchTargetMs: 10_000, + }, + moderate: { + vitestCap: 3, + unitShared: 3, + unitIsolated: 1, + unitHeavy: 1, + extensions: 2, + gateway: 1, + topLevelNoIsolate: 6, + topLevelIsolated: 2, + deferred: 1, + heavyFileLimit: 48, + heavyLaneCount: 4, + memoryHeavyFileLimit: 12, + unitFastBatchTargetMs: 15_000, + }, + mid: { + vitestCap: 4, + unitShared: 4, + unitIsolated: 1, + unitHeavy: 1, + extensions: 3, + gateway: 1, + topLevelNoIsolate: 8, + topLevelIsolated: 3, + deferred: 2, + heavyFileLimit: 60, + heavyLaneCount: 4, + memoryHeavyFileLimit: 16, + unitFastBatchTargetMs: 0, + }, + high: { + vitestCap: 6, + unitShared: 6, + unitIsolated: 2, + unitHeavy: 2, + extensions: 4, + gateway: 3, + topLevelNoIsolate: 12, + topLevelIsolated: 4, + deferred: 3, + heavyFileLimit: 80, + heavyLaneCount: 5, + memoryHeavyFileLimit: 16, + unitFastBatchTargetMs: 45_000, + }, +}; + +const withIntentBudgetAdjustments = (budget, intentProfile, cpuCount) => { + if (intentProfile === "serial") { + return { + ...budget, + vitestMaxWorkers: 1, + unitSharedWorkers: 1, + unitIsolatedWorkers: 1, + unitHeavyWorkers: 1, + extensionWorkers: 1, + gatewayWorkers: 1, + topLevelParallelEnabled: false, + topLevelParallelLimit: 1, + topLevelParallelLimitNoIsolate: 1, + topLevelParallelLimitIsolated: 1, + deferredRunConcurrency: 1, + }; + } + + if (intentProfile === "max") { + const maxTopLevelParallelLimit = clamp( + Math.max(budget.topLevelParallelLimitNoIsolate ?? budget.topLevelParallelLimit ?? 1, 5), + 1, + 8, + ); + return { + ...budget, + vitestMaxWorkers: clamp(Math.max(budget.vitestMaxWorkers, Math.min(8, cpuCount)), 1, 16), + unitSharedWorkers: clamp(Math.max(budget.unitSharedWorkers, Math.min(8, cpuCount)), 1, 16), + unitIsolatedWorkers: clamp(Math.max(budget.unitIsolatedWorkers, Math.min(4, cpuCount)), 1, 4), + unitHeavyWorkers: clamp(Math.max(budget.unitHeavyWorkers, Math.min(4, cpuCount)), 1, 4), + extensionWorkers: clamp(Math.max(budget.extensionWorkers, Math.min(6, cpuCount)), 1, 6), + gatewayWorkers: clamp(Math.max(budget.gatewayWorkers, Math.min(2, cpuCount)), 1, 6), + topLevelParallelEnabled: true, + topLevelParallelLimit: maxTopLevelParallelLimit, + topLevelParallelLimitNoIsolate: maxTopLevelParallelLimit, + topLevelParallelLimitIsolated: clamp( + Math.max(budget.topLevelParallelLimitIsolated ?? budget.topLevelParallelLimit ?? 1, 4), + 1, + 8, + ), + deferredRunConcurrency: Math.max(budget.deferredRunConcurrency ?? 1, 3), + }; + } + + return budget; +}; + +export function resolveRuntimeCapabilities(env = process.env, options = {}) { + const mode = resolveVitestMode(env, options.mode ?? null); + const isCI = mode === "ci"; + const platform = options.platform ?? process.platform; + const runnerOs = env.RUNNER_OS ?? ""; + const isMacOS = platform === "darwin" || runnerOs === "macOS"; + const isWindows = platform === "win32" || runnerOs === "Windows"; + const isWindowsCi = isCI && isWindows; + const hostCpuCount = + parsePositiveInt(env.OPENCLAW_TEST_HOST_CPU_COUNT) ?? options.cpuCount ?? os.cpus().length; + const totalMemoryBytes = options.totalMemoryBytes ?? os.totalmem(); + const hostMemoryGiB = + parsePositiveInt(env.OPENCLAW_TEST_HOST_MEMORY_GIB) ?? Math.floor(totalMemoryBytes / 1024 ** 3); + const nodeMajor = Number.parseInt( + (options.nodeVersion ?? process.versions.node).split(".")[0] ?? "", + 10, + ); + const intentProfile = parseProfile(options.profile ?? env.OPENCLAW_TEST_PROFILE ?? "normal"); + const loadRatio = !isCI ? resolveLoadRatio(env, hostCpuCount, platform, options.loadAverage) : 0; + const loadAware = !isCI && platform !== "win32"; + const memoryBand = resolveMemoryBand(hostMemoryGiB); + const loadBand = resolveLoadBand(loadAware, loadRatio); + const runtimeProfileName = isCI + ? isWindows + ? "ci-windows" + : isMacOS + ? "ci-macos" + : "ci-linux" + : isWindows + ? "local-windows" + : isMacOS + ? "local-darwin" + : "local-linux"; + + return { + mode, + runtimeProfileName, + isCI, + isMacOS, + isWindows, + isWindowsCi, + platform, + hostCpuCount, + hostMemoryGiB, + nodeMajor, + intentProfile, + memoryBand, + loadAware, + loadRatio, + loadBand, + }; +} + +export function resolveExecutionBudget(runtimeCapabilities) { + const runtime = runtimeCapabilities; + const cpuCount = clamp(runtime.hostCpuCount, 1, 16); + + if (runtime.isCI) { + const macCiWorkers = runtime.isMacOS ? 1 : null; + return { + vitestMaxWorkers: runtime.isWindows ? 2 : runtime.isMacOS ? 1 : 3, + unitSharedWorkers: macCiWorkers, + unitIsolatedWorkers: macCiWorkers, + unitHeavyWorkers: macCiWorkers, + extensionWorkers: macCiWorkers, + gatewayWorkers: macCiWorkers, + topLevelParallelEnabled: runtime.intentProfile !== "serial" && !runtime.isWindows, + topLevelParallelLimit: runtime.isWindows ? 2 : 4, + topLevelParallelLimitNoIsolate: runtime.isWindows ? 2 : 4, + topLevelParallelLimitIsolated: runtime.isWindows ? 2 : 4, + deferredRunConcurrency: null, + heavyUnitFileLimit: 64, + heavyUnitLaneCount: 4, + memoryHeavyUnitFileLimit: 64, + unitFastLaneCount: runtime.isWindows ? 1 : 3, + unitFastBatchTargetMs: runtime.isWindows ? 0 : 45_000, + channelsBatchTargetMs: runtime.isWindows ? 0 : 30_000, + extensionsBatchTargetMs: runtime.isWindows ? 0 : 30_000, + }; + } + + const bandBudget = LOCAL_MEMORY_BUDGETS[runtime.memoryBand]; + const baseBudget = { + vitestMaxWorkers: Math.min(cpuCount, bandBudget.vitestCap), + unitSharedWorkers: Math.min(cpuCount, bandBudget.unitShared), + unitIsolatedWorkers: Math.min(cpuCount, bandBudget.unitIsolated), + unitHeavyWorkers: Math.min(cpuCount, bandBudget.unitHeavy), + extensionWorkers: Math.min(cpuCount, bandBudget.extensions), + gatewayWorkers: Math.min(cpuCount, bandBudget.gateway), + topLevelParallelEnabled: runtime.nodeMajor < 25, + topLevelParallelLimit: Math.min(cpuCount, bandBudget.topLevelIsolated), + topLevelParallelLimitNoIsolate: Math.min(cpuCount, bandBudget.topLevelNoIsolate), + topLevelParallelLimitIsolated: Math.min(cpuCount, bandBudget.topLevelIsolated), + deferredRunConcurrency: bandBudget.deferred, + heavyUnitFileLimit: bandBudget.heavyFileLimit, + heavyUnitLaneCount: bandBudget.heavyLaneCount, + memoryHeavyUnitFileLimit: bandBudget.memoryHeavyFileLimit, + unitFastLaneCount: 1, + unitFastBatchTargetMs: bandBudget.unitFastBatchTargetMs, + channelsBatchTargetMs: 0, + extensionsBatchTargetMs: 0, + }; + + const loadAdjustedBudget = { + ...baseBudget, + vitestMaxWorkers: scaleForLoad(baseBudget.vitestMaxWorkers, runtime.loadBand), + unitSharedWorkers: scaleForLoad(baseBudget.unitSharedWorkers, runtime.loadBand), + unitHeavyWorkers: scaleForLoad(baseBudget.unitHeavyWorkers, runtime.loadBand), + extensionWorkers: scaleForLoad(baseBudget.extensionWorkers, runtime.loadBand), + gatewayWorkers: scaleForLoad(baseBudget.gatewayWorkers, runtime.loadBand), + topLevelParallelLimit: scaleConcurrencyForLoad( + baseBudget.topLevelParallelLimit, + runtime.loadBand, + ), + topLevelParallelLimitNoIsolate: scaleConcurrencyForLoad( + baseBudget.topLevelParallelLimitNoIsolate, + runtime.loadBand, + ), + topLevelParallelLimitIsolated: scaleConcurrencyForLoad( + baseBudget.topLevelParallelLimitIsolated, + runtime.loadBand, + ), + deferredRunConcurrency: + runtime.loadBand === "busy" + ? Math.max(1, (baseBudget.deferredRunConcurrency ?? 1) - 1) + : runtime.loadBand === "saturated" + ? 1 + : baseBudget.deferredRunConcurrency, + }; + + return withIntentBudgetAdjustments(loadAdjustedBudget, runtime.intentProfile, cpuCount); +} + +export function resolveLocalVitestMaxWorkers(env = process.env, options = {}) { + const explicit = parsePositiveInt(env.OPENCLAW_VITEST_MAX_WORKERS); + if (explicit !== null) { + return explicit; + } + + const runtimeCapabilities = resolveRuntimeCapabilities(env, { + cpuCount: options.cpuCount, + totalMemoryBytes: options.totalMemoryBytes, + platform: options.platform, + mode: "local", + loadAverage: options.loadAverage, + profile: options.profile, + }); + return resolveExecutionBudget(runtimeCapabilities).vitestMaxWorkers; +} diff --git a/scripts/test-planner/vitest-args.mjs b/scripts/test-planner/vitest-args.mjs new file mode 100644 index 00000000000..ca385b77b59 --- /dev/null +++ b/scripts/test-planner/vitest-args.mjs @@ -0,0 +1,74 @@ +export const OPTION_TAKES_VALUE = new Set([ + "-t", + "-c", + "-r", + "--testNamePattern", + "--config", + "--root", + "--dir", + "--reporter", + "--outputFile", + "--pool", + "--execArgv", + "--vmMemoryLimit", + "--maxWorkers", + "--environment", + "--shard", + "--changed", + "--sequence", + "--inspect", + "--inspectBrk", + "--testTimeout", + "--hookTimeout", + "--bail", + "--retry", + "--diff", + "--exclude", + "--project", + "--slowTestThreshold", + "--teardownTimeout", + "--attachmentsDir", + "--mode", + "--api", + "--browser", + "--maxConcurrency", + "--mergeReports", + "--configLoader", + "--experimental", +]); + +export const SINGLE_RUN_ONLY_FLAGS = new Set(["--coverage", "--outputFile", "--mergeReports"]); + +export const parsePassthroughArgs = (args = []) => { + const fileFilters = []; + const optionArgs = []; + let consumeNextAsOptionValue = false; + + for (const arg of args) { + if (consumeNextAsOptionValue) { + optionArgs.push(arg); + consumeNextAsOptionValue = false; + continue; + } + if (arg === "--") { + optionArgs.push(arg); + continue; + } + if (typeof arg === "string" && arg.startsWith("-")) { + optionArgs.push(arg); + consumeNextAsOptionValue = !arg.includes("=") && OPTION_TAKES_VALUE.has(arg); + continue; + } + fileFilters.push(arg); + } + + return { fileFilters, optionArgs }; +}; + +export const countExplicitEntryFilters = (entryArgs) => { + const { fileFilters } = parsePassthroughArgs(entryArgs.slice(2)); + return fileFilters.length > 0 ? fileFilters.length : null; +}; + +export const getExplicitEntryFilters = (entryArgs) => + parsePassthroughArgs(entryArgs.slice(2)).fileFilters; diff --git a/src/plugins/tools.optional.test.ts b/src/plugins/tools.optional.test.ts index c18f5008c31..938c0f817f8 100644 --- a/src/plugins/tools.optional.test.ts +++ b/src/plugins/tools.optional.test.ts @@ -14,6 +14,7 @@ vi.mock("./loader.js", () => ({ })); let resolvePluginTools: typeof import("./tools.js").resolvePluginTools; +let resetPluginRuntimeStateForTest: typeof import("./runtime.js").resetPluginRuntimeStateForTest; function makeTool(name: string) { return { @@ -94,6 +95,8 @@ describe("resolvePluginTools optional tools", () => { beforeEach(async () => { vi.resetModules(); loadOpenClawPluginsMock.mockClear(); + ({ resetPluginRuntimeStateForTest } = await import("./runtime.js")); + resetPluginRuntimeStateForTest(); ({ resolvePluginTools } = await import("./tools.js")); }); diff --git a/test/scripts/test-parallel.test.ts b/test/scripts/test-parallel.test.ts index 4cd78613f8e..276d7c91d40 100644 --- a/test/scripts/test-parallel.test.ts +++ b/test/scripts/test-parallel.test.ts @@ -1,4 +1,6 @@ import { execFileSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; import path from "node:path"; import { describe, expect, it } from "vitest"; import { @@ -12,6 +14,13 @@ import { resolveTestRunExitCode, } from "../../scripts/test-parallel-utils.mjs"; +const clearPlannerShardEnv = (env) => { + const nextEnv = { ...env }; + delete nextEnv.OPENCLAW_TEST_SHARDS; + delete nextEnv.OPENCLAW_TEST_SHARD_INDEX; + return nextEnv; +}; + describe("scripts/test-parallel fatal output guard", () => { it("fails a zero exit when V8 reports an out-of-memory fatal", () => { const output = [ @@ -114,11 +123,10 @@ describe("scripts/test-parallel memory trace parsing", () => { describe("scripts/test-parallel lane planning", () => { it("keeps serial profile on split unit lanes instead of one giant unit worker", () => { const repoRoot = path.resolve(import.meta.dirname, "../.."); - const output = execFileSync("node", ["scripts/test-parallel.mjs"], { + const output = execFileSync("node", ["scripts/test-parallel.mjs", "--plan"], { cwd: repoRoot, env: { - ...process.env, - OPENCLAW_TEST_LIST_LANES: "1", + ...clearPlannerShardEnv(process.env), OPENCLAW_TEST_PROFILE: "serial", }, encoding: "utf8", @@ -130,12 +138,11 @@ describe("scripts/test-parallel lane planning", () => { it("recycles default local unit-fast runs into bounded batches", () => { const repoRoot = path.resolve(import.meta.dirname, "../.."); - const output = execFileSync("node", ["scripts/test-parallel.mjs"], { + const output = execFileSync("node", ["scripts/test-parallel.mjs", "--plan"], { cwd: repoRoot, env: { - ...process.env, + ...clearPlannerShardEnv(process.env), CI: "", - OPENCLAW_TEST_LIST_LANES: "1", OPENCLAW_TEST_UNIT_FAST_LANES: "1", OPENCLAW_TEST_UNIT_FAST_BATCH_TARGET_MS: "1", }, @@ -150,13 +157,15 @@ describe("scripts/test-parallel lane planning", () => { const repoRoot = path.resolve(import.meta.dirname, "../.."); const output = execFileSync( "node", - ["scripts/test-parallel.mjs", "src/auto-reply/reply/followup-runner.test.ts"], + [ + "scripts/test-parallel.mjs", + "--plan", + "--files", + "src/auto-reply/reply/followup-runner.test.ts", + ], { cwd: repoRoot, - env: { - ...process.env, - OPENCLAW_TEST_LIST_LANES: "1", - }, + env: clearPlannerShardEnv(process.env), encoding: "utf8", }, ); @@ -164,4 +173,165 @@ describe("scripts/test-parallel lane planning", () => { expect(output).toContain("base-pinned-followup-runner"); expect(output).not.toContain("base-followup-runner"); }); + + it("reports capability-derived output for mid-memory local macOS hosts", () => { + const repoRoot = path.resolve(import.meta.dirname, "../.."); + const output = execFileSync( + "node", + ["scripts/test-parallel.mjs", "--plan", "--surface", "unit", "--surface", "extensions"], + { + cwd: repoRoot, + env: { + ...clearPlannerShardEnv(process.env), + CI: "", + GITHUB_ACTIONS: "", + RUNNER_OS: "macOS", + OPENCLAW_TEST_HOST_CPU_COUNT: "10", + OPENCLAW_TEST_HOST_MEMORY_GIB: "64", + OPENCLAW_TEST_LOAD_AWARE: "0", + }, + encoding: "utf8", + }, + ); + + expect(output).toContain("mode=local intent=normal memoryBand=mid"); + expect(output).toContain("unit-fast filters=all maxWorkers="); + expect(output).toContain("extensions filters=all maxWorkers="); + }); + + it("explains targeted file ownership and execution policy", () => { + const repoRoot = path.resolve(import.meta.dirname, "../.."); + const output = execFileSync( + "node", + ["scripts/test-parallel.mjs", "--explain", "src/auto-reply/reply/followup-runner.test.ts"], + { + cwd: repoRoot, + env: clearPlannerShardEnv(process.env), + encoding: "utf8", + }, + ); + + expect(output).toContain("surface=base"); + expect(output).toContain("reasons=base-surface,base-pinned-manifest"); + expect(output).toContain("pool=forks"); + }); + + it("passes through vitest --mode values that are not wrapper runtime overrides", () => { + const repoRoot = path.resolve(import.meta.dirname, "../.."); + const output = execFileSync( + "node", + [ + "scripts/test-parallel.mjs", + "--plan", + "--mode", + "development", + "src/infra/outbound/deliver.test.ts", + ], + { + cwd: repoRoot, + env: { + ...clearPlannerShardEnv(process.env), + CI: "", + GITHUB_ACTIONS: "", + RUNNER_OS: "Linux", + OPENCLAW_TEST_HOST_CPU_COUNT: "16", + OPENCLAW_TEST_HOST_MEMORY_GIB: "128", + }, + encoding: "utf8", + }, + ); + + expect(output).toContain("mode=local intent=normal memoryBand=high"); + expect(output).toContain("unit-deliver-isolated filters=1"); + }); + + it("rejects removed machine-name profiles", () => { + const repoRoot = path.resolve(import.meta.dirname, "../.."); + + expect(() => + execFileSync("node", ["scripts/test-parallel.mjs", "--plan", "--profile", "macmini"], { + cwd: repoRoot, + env: clearPlannerShardEnv(process.env), + encoding: "utf8", + }), + ).toThrowError(/Unsupported test profile "macmini"/u); + }); + + it("rejects unknown explicit surface names", () => { + const repoRoot = path.resolve(import.meta.dirname, "../.."); + + expect(() => + execFileSync("node", ["scripts/test-parallel.mjs", "--plan", "--surface", "channel"], { + cwd: repoRoot, + env: clearPlannerShardEnv(process.env), + encoding: "utf8", + }), + ).toThrowError(/Unsupported --surface value\(s\): channel/u); + }); + + it("rejects wrapper --files values that look like options", () => { + const repoRoot = path.resolve(import.meta.dirname, "../.."); + + expect(() => + execFileSync("node", ["scripts/test-parallel.mjs", "--plan", "--files", "--config"], { + cwd: repoRoot, + env: clearPlannerShardEnv(process.env), + encoding: "utf8", + }), + ).toThrowError(/Invalid --files value/u); + }); + + it("rejects missing --profile values", () => { + const repoRoot = path.resolve(import.meta.dirname, "../.."); + + expect(() => + execFileSync("node", ["scripts/test-parallel.mjs", "--plan", "--profile"], { + cwd: repoRoot, + env: clearPlannerShardEnv(process.env), + encoding: "utf8", + }), + ).toThrowError(/Invalid --profile value/u); + }); + + it("rejects missing --surface values", () => { + const repoRoot = path.resolve(import.meta.dirname, "../.."); + + expect(() => + execFileSync("node", ["scripts/test-parallel.mjs", "--plan", "--surface"], { + cwd: repoRoot, + env: clearPlannerShardEnv(process.env), + encoding: "utf8", + }), + ).toThrowError(/Invalid --surface value/u); + }); + + it("rejects missing --explain values", () => { + const repoRoot = path.resolve(import.meta.dirname, "../.."); + + expect(() => + execFileSync("node", ["scripts/test-parallel.mjs", "--explain"], { + cwd: repoRoot, + env: clearPlannerShardEnv(process.env), + encoding: "utf8", + }), + ).toThrowError(/Invalid --explain value/u); + }); + + it("rejects explicit existing files that are not known test files", () => { + const repoRoot = path.resolve(import.meta.dirname, "../.."); + const tempFilePath = path.join(os.tmpdir(), `openclaw-non-test-${Date.now()}.ts`); + fs.writeFileSync(tempFilePath, "export const notATest = true;\n", "utf8"); + + try { + expect(() => + execFileSync("node", ["scripts/test-parallel.mjs", "--plan", "--files", tempFilePath], { + cwd: repoRoot, + env: clearPlannerShardEnv(process.env), + encoding: "utf8", + }), + ).toThrowError(/is not a known test file/u); + } finally { + fs.rmSync(tempFilePath, { force: true }); + } + }); }); diff --git a/test/scripts/test-planner.test.ts b/test/scripts/test-planner.test.ts new file mode 100644 index 00000000000..f16ce03eeba --- /dev/null +++ b/test/scripts/test-planner.test.ts @@ -0,0 +1,287 @@ +import fs from "node:fs"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { + createExecutionArtifacts, + resolvePnpmCommandInvocation, +} from "../../scripts/test-planner/executor.mjs"; +import { buildExecutionPlan, explainExecutionTarget } from "../../scripts/test-planner/planner.mjs"; + +describe("test planner", () => { + it("builds a capability-aware plan for mid-memory local runs", () => { + const artifacts = createExecutionArtifacts({ + RUNNER_OS: "macOS", + OPENCLAW_TEST_HOST_CPU_COUNT: "10", + OPENCLAW_TEST_HOST_MEMORY_GIB: "64", + OPENCLAW_TEST_LOAD_AWARE: "0", + }); + const plan = buildExecutionPlan( + { + profile: null, + mode: "local", + surfaces: ["unit", "extensions"], + passthroughArgs: [], + }, + { + env: { + RUNNER_OS: "macOS", + OPENCLAW_TEST_HOST_CPU_COUNT: "10", + OPENCLAW_TEST_HOST_MEMORY_GIB: "64", + OPENCLAW_TEST_LOAD_AWARE: "0", + }, + platform: "darwin", + writeTempJsonArtifact: artifacts.writeTempJsonArtifact, + }, + ); + + expect(plan.runtimeCapabilities.runtimeProfileName).toBe("local-darwin"); + expect(plan.runtimeCapabilities.memoryBand).toBe("mid"); + expect(plan.executionBudget.unitSharedWorkers).toBe(4); + expect(plan.executionBudget.topLevelParallelLimitNoIsolate).toBe(8); + expect(plan.executionBudget.topLevelParallelLimitIsolated).toBe(3); + expect(plan.selectedUnits.some((unit) => unit.id.startsWith("unit-fast"))).toBe(true); + expect(plan.selectedUnits.some((unit) => unit.id.startsWith("extensions"))).toBe(true); + expect(plan.topLevelParallelLimit).toBe(8); + artifacts.cleanupTempArtifacts(); + }); + + it("scales down mid-tier local concurrency under saturated load", () => { + const artifacts = createExecutionArtifacts({ + RUNNER_OS: "Linux", + OPENCLAW_TEST_HOST_CPU_COUNT: "10", + OPENCLAW_TEST_HOST_MEMORY_GIB: "64", + }); + const plan = buildExecutionPlan( + { + profile: null, + mode: "local", + surfaces: ["unit", "extensions"], + passthroughArgs: [], + }, + { + env: { + RUNNER_OS: "Linux", + OPENCLAW_TEST_HOST_CPU_COUNT: "10", + OPENCLAW_TEST_HOST_MEMORY_GIB: "64", + }, + platform: "linux", + loadAverage: [11.5, 11.5, 11.5], + writeTempJsonArtifact: artifacts.writeTempJsonArtifact, + }, + ); + + expect(plan.runtimeCapabilities.memoryBand).toBe("mid"); + expect(plan.runtimeCapabilities.loadBand).toBe("saturated"); + expect(plan.executionBudget.unitSharedWorkers).toBe(2); + expect(plan.executionBudget.topLevelParallelLimitNoIsolate).toBe(4); + expect(plan.executionBudget.topLevelParallelLimitIsolated).toBe(1); + expect(plan.topLevelParallelLimit).toBe(4); + expect(plan.deferredRunConcurrency).toBe(1); + artifacts.cleanupTempArtifacts(); + }); + + it("honors the max-profile top-level no-isolate cap without adding extra lanes", () => { + const artifacts = createExecutionArtifacts({ + RUNNER_OS: "Linux", + OPENCLAW_TEST_HOST_CPU_COUNT: "16", + OPENCLAW_TEST_HOST_MEMORY_GIB: "128", + OPENCLAW_TEST_LOAD_AWARE: "0", + OPENCLAW_TEST_PROFILE: "max", + }); + const plan = buildExecutionPlan( + { + profile: "max", + mode: "local", + surfaces: ["unit", "extensions"], + passthroughArgs: [], + }, + { + env: { + RUNNER_OS: "Linux", + OPENCLAW_TEST_HOST_CPU_COUNT: "16", + OPENCLAW_TEST_HOST_MEMORY_GIB: "128", + OPENCLAW_TEST_LOAD_AWARE: "0", + OPENCLAW_TEST_PROFILE: "max", + }, + platform: "linux", + writeTempJsonArtifact: artifacts.writeTempJsonArtifact, + }, + ); + + expect(plan.runtimeCapabilities.intentProfile).toBe("max"); + expect(plan.executionBudget.topLevelParallelLimitNoIsolate).toBe(8); + expect(plan.topLevelParallelLimit).toBe(8); + artifacts.cleanupTempArtifacts(); + }); + + it("splits mixed targeted file selections across surfaces", () => { + const artifacts = createExecutionArtifacts({}); + const plan = buildExecutionPlan( + { + mode: "local", + surfaces: [], + passthroughArgs: [ + "src/auto-reply/reply/followup-runner.test.ts", + "extensions/discord/src/monitor/message-handler.preflight.acp-bindings.test.ts", + ], + }, + { + env: {}, + writeTempJsonArtifact: artifacts.writeTempJsonArtifact, + }, + ); + + expect(plan.targetedUnits).toHaveLength(2); + expect( + plan.targetedUnits + .map((unit) => unit.surface) + .toSorted((left, right) => left.localeCompare(right)), + ).toEqual(["base", "channels"]); + artifacts.cleanupTempArtifacts(); + }); + + it("explains runtime truth using the same catalog and worker policy", () => { + const explanation = explainExecutionTarget( + { + mode: "local", + fileFilters: ["src/auto-reply/reply/followup-runner.test.ts"], + }, + { + env: {}, + }, + ); + + expect(explanation.surface).toBe("base"); + expect(explanation.pool).toBe("forks"); + expect(explanation.reasons).toContain("base-pinned-manifest"); + expect(explanation.intentProfile).toBe("normal"); + }); + + it("uses hotspot-backed memory isolation when explaining unit tests", () => { + const explanation = explainExecutionTarget( + { + mode: "local", + fileFilters: ["src/infra/outbound/targets.channel-resolution.test.ts"], + }, + { + env: { + OPENCLAW_TEST_LOAD_AWARE: "0", + }, + }, + ); + + expect(explanation.isolate).toBe(true); + expect(explanation.reasons).toContain("unit-memory-isolated"); + }); + + it("normalizes absolute explain targets before classification", () => { + const relativeExplanation = explainExecutionTarget( + { + mode: "local", + fileFilters: ["src/infra/outbound/targets.channel-resolution.test.ts"], + }, + { + env: { + OPENCLAW_TEST_LOAD_AWARE: "0", + }, + }, + ); + const absoluteExplanation = explainExecutionTarget( + { + mode: "local", + fileFilters: [ + path.join(process.cwd(), "src/infra/outbound/targets.channel-resolution.test.ts"), + ], + }, + { + env: { + OPENCLAW_TEST_LOAD_AWARE: "0", + }, + }, + ); + + expect(absoluteExplanation.file).toBe(relativeExplanation.file); + expect(absoluteExplanation.surface).toBe(relativeExplanation.surface); + expect(absoluteExplanation.pool).toBe(relativeExplanation.pool); + expect(absoluteExplanation.isolate).toBe(relativeExplanation.isolate); + expect(absoluteExplanation.reasons).toEqual(relativeExplanation.reasons); + }); + + it("does not leak default-plan shard assignments into targeted units with the same id", () => { + const artifacts = createExecutionArtifacts({}); + const plan = buildExecutionPlan( + { + mode: "local", + fileFilters: ["src/cli/qr-dashboard.integration.test.ts"], + passthroughArgs: [], + }, + { + env: { + OPENCLAW_TEST_SHARDS: "4", + OPENCLAW_TEST_SHARD_INDEX: "2", + OPENCLAW_TEST_LOAD_AWARE: "0", + }, + writeTempJsonArtifact: artifacts.writeTempJsonArtifact, + }, + ); + + const targetedUnit = plan.targetedUnits.at(0); + const defaultUnitWithSameId = plan.allUnits.find((unit) => unit.id === targetedUnit?.id); + + expect(targetedUnit).toBeTruthy(); + expect(defaultUnitWithSameId).toBeTruthy(); + expect(defaultUnitWithSameId).not.toBe(targetedUnit); + expect(plan.topLevelSingleShardAssignments.get(targetedUnit)).toBeUndefined(); + expect(plan.topLevelSingleShardAssignments.get(defaultUnitWithSameId)).toBeDefined(); + + artifacts.cleanupTempArtifacts(); + }); + + it("removes planner temp artifacts when cleanup runs after planning", () => { + const artifacts = createExecutionArtifacts({}); + buildExecutionPlan( + { + mode: "local", + surfaces: ["unit"], + passthroughArgs: [], + }, + { + env: {}, + writeTempJsonArtifact: artifacts.writeTempJsonArtifact, + }, + ); + + const artifactDir = artifacts.ensureTempArtifactDir(); + expect(fs.existsSync(artifactDir)).toBe(true); + artifacts.cleanupTempArtifacts(); + expect(fs.existsSync(artifactDir)).toBe(false); + }); +}); + +describe("resolvePnpmCommandInvocation", () => { + it("prefers the parent pnpm CLI path when npm_execpath points to pnpm", () => { + expect( + resolvePnpmCommandInvocation({ + npmExecPath: "/opt/homebrew/lib/node_modules/corepack/dist/pnpm.cjs", + nodeExecPath: "/usr/local/bin/node", + platform: "linux", + }), + ).toEqual({ + command: "/usr/local/bin/node", + args: ["/opt/homebrew/lib/node_modules/corepack/dist/pnpm.cjs"], + }); + }); + + it("falls back to cmd.exe mediation on Windows when npm_execpath is unavailable", () => { + expect( + resolvePnpmCommandInvocation({ + npmExecPath: "", + platform: "win32", + comSpec: "C:\\Windows\\System32\\cmd.exe", + }), + ).toEqual({ + command: "C:\\Windows\\System32\\cmd.exe", + args: ["/d", "/s", "/c", "pnpm.cmd"], + }); + }); +}); diff --git a/test/vitest-config.test.ts b/test/vitest-config.test.ts new file mode 100644 index 00000000000..b34f9ab4878 --- /dev/null +++ b/test/vitest-config.test.ts @@ -0,0 +1,156 @@ +import { describe, expect, it } from "vitest"; +import { + resolveExecutionBudget, + resolveRuntimeCapabilities, +} from "../scripts/test-planner/runtime-profile.mjs"; +import { resolveLocalVitestMaxWorkers } from "../vitest.config.ts"; + +describe("resolveLocalVitestMaxWorkers", () => { + it("derives a mid-tier local cap for 64 GiB hosts", () => { + expect( + resolveLocalVitestMaxWorkers( + { + RUNNER_OS: "macOS", + }, + { + cpuCount: 10, + totalMemoryBytes: 64 * 1024 ** 3, + platform: "darwin", + loadAverage: [0.1, 0.1, 0.1], + }, + ), + ).toBe(4); + }); + + it("lets OPENCLAW_VITEST_MAX_WORKERS override the inferred cap", () => { + expect( + resolveLocalVitestMaxWorkers( + { + OPENCLAW_VITEST_MAX_WORKERS: "2", + }, + { + cpuCount: 10, + totalMemoryBytes: 128 * 1024 ** 3, + platform: "darwin", + }, + ), + ).toBe(2); + }); + + it("maps the legacy low profile to serial intent for compatibility", () => { + const runtime = resolveRuntimeCapabilities( + { + OPENCLAW_TEST_PROFILE: "low", + RUNNER_OS: "Linux", + }, + { + cpuCount: 8, + totalMemoryBytes: 32 * 1024 ** 3, + platform: "linux", + mode: "local", + }, + ); + + expect(runtime.intentProfile).toBe("serial"); + }); + + it("classifies 64 GiB local macOS hosts as mid-memory capabilities", () => { + const runtime = resolveRuntimeCapabilities( + { + RUNNER_OS: "macOS", + }, + { + cpuCount: 10, + totalMemoryBytes: 64 * 1024 ** 3, + platform: "darwin", + mode: "local", + loadAverage: [0.2, 0.2, 0.2], + }, + ); + + expect(runtime.runtimeProfileName).toBe("local-darwin"); + expect(runtime.memoryBand).toBe("mid"); + expect(runtime.loadBand).toBe("idle"); + }); + + it("does not classify 64 GiB non-macOS hosts as constrained locals", () => { + const runtime = resolveRuntimeCapabilities( + { + RUNNER_OS: "Linux", + }, + { + cpuCount: 16, + totalMemoryBytes: 64 * 1024 ** 3, + platform: "linux", + mode: "local", + loadAverage: [0.2, 0.2, 0.2], + }, + ); + + expect(runtime.memoryBand).toBe("mid"); + expect(runtime.runtimeProfileName).toBe("local-linux"); + }); + + it("reduces local budgets when the host is busy", () => { + const runtime = resolveRuntimeCapabilities( + { + RUNNER_OS: "Linux", + }, + { + cpuCount: 10, + totalMemoryBytes: 16 * 1024 ** 3, + platform: "linux", + mode: "local", + loadAverage: [9.5, 9.5, 9.5], + }, + ); + const budget = resolveExecutionBudget(runtime); + + expect(runtime.memoryBand).toBe("constrained"); + expect(runtime.loadBand).toBe("busy"); + expect(budget.vitestMaxWorkers).toBe(1); + expect(budget.topLevelParallelLimit).toBe(1); + }); + + it("keeps 64 GiB hosts mid-tier but scales them down under saturation", () => { + const runtime = resolveRuntimeCapabilities( + { + RUNNER_OS: "Linux", + }, + { + cpuCount: 10, + totalMemoryBytes: 64 * 1024 ** 3, + platform: "linux", + mode: "local", + loadAverage: [11.5, 11.5, 11.5], + }, + ); + const budget = resolveExecutionBudget(runtime); + + expect(runtime.memoryBand).toBe("mid"); + expect(runtime.loadBand).toBe("saturated"); + expect(budget.vitestMaxWorkers).toBe(2); + expect(budget.deferredRunConcurrency).toBe(1); + }); + + it("keeps CI windows policy constrained independently of host load", () => { + const runtime = resolveRuntimeCapabilities( + { + CI: "true", + RUNNER_OS: "Windows", + }, + { + cpuCount: 32, + totalMemoryBytes: 128 * 1024 ** 3, + platform: "win32", + mode: "ci", + loadAverage: [0, 0, 0], + }, + ); + const budget = resolveExecutionBudget(runtime); + + expect(runtime.runtimeProfileName).toBe("ci-windows"); + expect(budget.vitestMaxWorkers).toBe(2); + expect(budget.topLevelParallelLimit).toBe(2); + }); +}); diff --git a/vitest.channels.config.ts b/vitest.channels.config.ts index 308ab2484a5..29cb6270fea 100644 --- a/vitest.channels.config.ts +++ b/vitest.channels.config.ts @@ -11,7 +11,6 @@ export function loadIncludePatternsFromEnv( export function createChannelsVitestConfig(env?: Record) { return createScopedVitestConfig(loadIncludePatternsFromEnv(env) ?? channelTestInclude, { env, - pool: "forks", exclude: ["src/gateway/**"], passWithNoTests: true, }); diff --git a/vitest.config.ts b/vitest.config.ts index 1f96ef339ff..bb68f57d618 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -1,8 +1,8 @@ -import os from "node:os"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { defineConfig } from "vitest/config"; import { pluginSdkSubpaths } from "./scripts/lib/plugin-sdk-entries.mjs"; +import { resolveLocalVitestMaxWorkers } from "./scripts/test-planner/runtime-profile.mjs"; import { behaviorManifestPath, unitMemoryHotspotManifestPath, @@ -10,10 +10,12 @@ import { } from "./scripts/test-runner-manifest.mjs"; import { loadVitestExperimentalConfig } from "./vitest.performance-config.ts"; +export { resolveLocalVitestMaxWorkers }; + const repoRoot = path.dirname(fileURLToPath(import.meta.url)); const isCI = process.env.CI === "true" || process.env.GITHUB_ACTIONS === "true"; const isWindows = process.platform === "win32"; -const localWorkers = Math.max(4, Math.min(16, os.cpus().length)); +const localWorkers = resolveLocalVitestMaxWorkers(); const ciWorkers = isWindows ? 2 : 3; export default defineConfig({ resolve: { @@ -48,6 +50,10 @@ export default defineConfig({ "pnpm-lock.yaml", "test/setup.ts", "scripts/test-parallel.mjs", + "scripts/test-planner/catalog.mjs", + "scripts/test-planner/executor.mjs", + "scripts/test-planner/planner.mjs", + "scripts/test-planner/runtime-profile.mjs", "scripts/test-runner-manifest.mjs", "vitest.channel-paths.mjs", "vitest.channels.config.ts", diff --git a/vitest.extensions.config.ts b/vitest.extensions.config.ts index 86338ffa664..28b9cbed691 100644 --- a/vitest.extensions.config.ts +++ b/vitest.extensions.config.ts @@ -14,7 +14,6 @@ export function createExtensionsVitestConfig( return createScopedVitestConfig(loadIncludePatternsFromEnv(env) ?? ["extensions/**/*.test.ts"], { dir: "extensions", env, - pool: "forks", passWithNoTests: true, // Channel implementations live under extensions/ but are tested by // vitest.channels.config.ts (pnpm test:channels) which provides