diff --git a/docs/help/testing.md b/docs/help/testing.md index 2d7e9664176..6fb91982f1d 100644 --- a/docs/help/testing.md +++ b/docs/help/testing.md @@ -52,6 +52,10 @@ Think of the suites as “increasing realism” (and increasing flakiness/cost): - Runs in CI - No real keys required - Should be fast and stable +- Scheduler note: + - `pnpm test` now keeps a small checked-in behavioral manifest for true pool/isolation overrides and a separate timing snapshot for the slowest unit files. + - Shared unit coverage stays on, but the wrapper peels the heaviest measured files into dedicated lanes instead of relying on a growing hand-maintained exclusion list. + - Refresh the timing snapshot with `pnpm test:perf:update-timings` after major suite shape changes. - Embedded runner note: - When you change message-tool discovery inputs or compaction runtime context, keep both levels of coverage. diff --git a/docs/reference/test.md b/docs/reference/test.md index 378789f6d6e..e337e963e1d 100644 --- a/docs/reference/test.md +++ b/docs/reference/test.md @@ -12,9 +12,10 @@ title: "Tests" - `pnpm test:force`: Kills any lingering gateway process holding the default control port, then runs the full Vitest suite with an isolated gateway port so server tests don’t collide with a running instance. Use this when a prior gateway run left port 18789 occupied. - `pnpm test:coverage`: Runs the unit suite with V8 coverage (via `vitest.unit.config.ts`). Global thresholds are 70% lines/branches/functions/statements. Coverage excludes integration-heavy entrypoints (CLI wiring, gateway/telegram bridges, webchat static server) to keep the target focused on unit-testable logic. - `pnpm test` on Node 22, 23, and 24 uses Vitest `vmForks` by default for faster startup. Node 25+ falls back to `forks` until re-validated. You can force behavior with `OPENCLAW_TEST_VM_FORKS=0|1`. -- `pnpm test`: runs the fast core unit lane by default for quick local feedback. +- `pnpm test`: runs the full wrapper. It keeps only a small behavioral override manifest in git, then uses a checked-in timing snapshot to peel the heaviest measured unit files into dedicated lanes. - `pnpm test:channels`: runs channel-heavy suites. - `pnpm test:extensions`: runs extension/plugin suites. +- `pnpm test:perf:update-timings`: refreshes the checked-in slow-file timing snapshot used by `scripts/test-parallel.mjs`. - Gateway integration: opt-in via `OPENCLAW_TEST_INCLUDE_GATEWAY=1 pnpm test` or `pnpm test:gateway`. - `pnpm test:e2e`: Runs gateway end-to-end smoke tests (multi-instance WS/HTTP/node pairing). Defaults to `vmForks` + adaptive workers in `vitest.e2e.config.ts`; tune with `OPENCLAW_E2E_WORKERS=` and set `OPENCLAW_E2E_VERBOSE=1` for verbose logs. - `pnpm test:live`: Runs provider live tests (minimax/zai). Requires API keys and `LIVE=1` (or provider-specific `*_LIVE_TEST=1`) to unskip. diff --git a/package.json b/package.json index f752857492f..413fee96094 100644 --- a/package.json +++ b/package.json @@ -642,6 +642,7 @@ "test:parallels:windows": "bash scripts/e2e/parallels-windows-smoke.sh", "test:perf:budget": "node scripts/test-perf-budget.mjs", "test:perf:hotspots": "node scripts/test-hotspots.mjs", + "test:perf:update-timings": "node scripts/test-update-timings.mjs", "test:sectriage": "pnpm exec vitest run --config vitest.gateway.config.ts && vitest run --config vitest.unit.config.ts --exclude src/daemon/launchd.integration.test.ts --exclude src/process/exec.test.ts", "test:startup:memory": "node scripts/check-cli-startup-memory.mjs", "test:ui": "pnpm lint:ui:no-raw-window-open && pnpm --dir ui test", diff --git a/scripts/test-parallel.mjs b/scripts/test-parallel.mjs index dc7158a4cb7..68361a6b094 100644 --- a/scripts/test-parallel.mjs +++ b/scripts/test-parallel.mjs @@ -3,127 +3,30 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { channelTestPrefixes } from "../vitest.channel-paths.mjs"; +import { + loadTestRunnerBehavior, + loadUnitTimingManifest, + packFilesByDuration, + selectTimedHeavyFiles, +} from "./test-runner-manifest.mjs"; // On Windows, `.cmd` launchers can fail with `spawn EINVAL` when invoked without a shell // (especially under GitHub Actions + Git Bash). Use `shell: true` and let the shell resolve pnpm. const pnpm = "pnpm"; - -const unitIsolatedFilesRaw = [ - "src/plugins/loader.test.ts", - "src/plugins/tools.optional.test.ts", - "src/agents/session-tool-result-guard.tool-result-persist-hook.test.ts", - "src/security/fix.test.ts", - // Runtime source guard scans are sensitive to filesystem contention. - "src/security/temp-path-guard.test.ts", - "src/security/audit.test.ts", - "src/utils.test.ts", - "src/auto-reply/tool-meta.test.ts", - "src/auto-reply/envelope.test.ts", - "src/commands/auth-choice.test.ts", - // Provider runtime contract imports plugin runtimes plus async ESM mocks; - // keep it off the shared fast lane to avoid teardown stalls on this host. - "src/plugins/contracts/runtime.contract.test.ts", - // Process supervision + docker setup suites are stable but setup-heavy. - "src/process/supervisor/supervisor.test.ts", - "src/docker-setup.test.ts", - // Filesystem-heavy skills sync suite. - "src/agents/skills.build-workspace-skills-prompt.syncs-merged-skills-into-target-workspace.test.ts", - // Real git hook integration test; keep signal, move off unit-fast critical path. - "test/git-hooks-pre-commit.test.ts", - // Setup-heavy doctor command suites; keep them off the unit-fast critical path. - "src/commands/doctor.warns-state-directory-is-missing.test.ts", - "src/commands/doctor.warns-per-agent-sandbox-docker-browser-prune.test.ts", - "src/commands/doctor.runs-legacy-state-migrations-yes-mode-without.test.ts", - // Setup-heavy CLI update flow suite; move off unit-fast critical path. - "src/cli/update-cli.test.ts", - // Uses temp repos + module cache resets; keep it off vmForks to avoid ref-resolution flakes. - "src/infra/git-commit.test.ts", - // Expensive schema build/bootstrap checks; keep coverage but run in isolated lane. - "src/config/schema.test.ts", - "src/config/schema.tags.test.ts", - // CLI smoke/agent flows are stable but setup-heavy. - "src/cli/program.smoke.test.ts", - "src/commands/agent.test.ts", - "src/media/store.test.ts", - "src/media/store.header-ext.test.ts", - "extensions/whatsapp/src/media.test.ts", - "extensions/whatsapp/src/auto-reply.web-auto-reply.falls-back-text-media-send-fails.test.ts", - "src/browser/server.covers-additional-endpoint-branches.test.ts", - "src/browser/server.post-tabs-open-profile-unknown-returns-404.test.ts", - "src/browser/server.agent-contract-snapshot-endpoints.test.ts", - "src/browser/server.agent-contract-form-layout-act-commands.test.ts", - "src/browser/server.skips-default-maxchars-explicitly-set-zero.test.ts", - "src/browser/server.auth-token-gates-http.test.ts", - // Keep this high-variance heavy file off the unit-fast critical path. - "src/auto-reply/reply.block-streaming.test.ts", - // Archive extraction/fixture-heavy suite; keep off unit-fast critical path. - "src/hooks/install.test.ts", - // Download/extraction safety cases can spike under unit-fast contention. - "src/agents/skills-install.download.test.ts", - // Skills discovery/snapshot suites are filesystem-heavy and high-variance in vmForks lanes. - "src/agents/skills.test.ts", - "src/agents/skills.buildworkspaceskillsnapshot.test.ts", - "extensions/acpx/src/runtime.test.ts", - // Shell-heavy script harness can contend under vmForks startup bursts. - "test/scripts/ios-team-id.test.ts", - // Heavy runner/exec/archive suites are stable but contend on shared resources under vmForks. - "src/agents/pi-embedded-runner.test.ts", - "src/agents/bash-tools.test.ts", - "src/agents/openclaw-tools.subagents.sessions-spawn.lifecycle.test.ts", - "src/agents/bash-tools.exec.background-abort.test.ts", - "src/agents/subagent-announce.format.test.ts", - "src/infra/archive.test.ts", - "src/cli/daemon-cli.coverage.test.ts", - // Model normalization test imports config/model discovery stack; keep off unit-fast critical path. - "src/agents/models-config.normalizes-gemini-3-ids-preview-google-providers.test.ts", - // Auth profile rotation suite is retry-heavy and high-variance under vmForks contention. - "src/agents/pi-embedded-runner.run-embedded-pi-agent.auth-profile-rotation.test.ts", - // Heavy trigger command scenarios; keep off unit-fast critical path to reduce contention noise. - "src/auto-reply/reply.triggers.trigger-handling.filters-usage-summary-current-model-provider.test.ts", - "src/auto-reply/reply.triggers.trigger-handling.targets-active-session-native-stop.test.ts", - "src/auto-reply/reply.triggers.group-intro-prompts.test.ts", - "src/auto-reply/reply.triggers.trigger-handling.handles-inline-commands-strips-it-before-agent.test.ts", - "extensions/whatsapp/src/auto-reply.web-auto-reply.compresses-common-formats-jpeg-cap.test.ts", - // Setup-heavy bot bootstrap suite. - "extensions/telegram/src/bot.create-telegram-bot.test.ts", - // Medium-heavy bot behavior suite; move off unit-fast critical path. - "extensions/telegram/src/bot.test.ts", - // Slack slash registration tests are setup-heavy and can bottleneck unit-fast. - "extensions/slack/src/monitor/slash.test.ts", - // Uses process-level unhandledRejection listeners; keep it off vmForks to avoid cross-file leakage. - "extensions/imessage/src/monitor.shutdown.unhandled-rejection.test.ts", - // Mutates process.cwd() and mocks core module loaders; isolate from the shared fast lane. - "src/infra/git-commit.test.ts", -]; -const unitIsolatedFiles = unitIsolatedFilesRaw.filter((file) => fs.existsSync(file)); -const unitSingletonIsolatedFilesRaw = [ - // These pass clean in isolation but can hang on fork shutdown after sharing - // the broad unit-fast lane on this host; keep them in dedicated processes. - "src/cli/command-secret-gateway.test.ts", -]; -const unitSingletonIsolatedFiles = unitSingletonIsolatedFilesRaw.filter((file) => - fs.existsSync(file), -); -const unitThreadSingletonFilesRaw = [ - // These suites terminate cleanly under the threads pool but can hang during - // forks worker shutdown on this host. - "src/channels/plugins/actions/actions.test.ts", - "src/infra/outbound/deliver.test.ts", - "src/infra/outbound/deliver.lifecycle.test.ts", - "src/infra/outbound/message.channels.test.ts", - "src/infra/outbound/message-action-runner.poll.test.ts", - "src/tts/tts.test.ts", -]; -const unitThreadSingletonFiles = unitThreadSingletonFilesRaw.filter((file) => fs.existsSync(file)); -const unitVmForkSingletonFilesRaw = [ - "src/channels/plugins/contracts/inbound.telegram.contract.test.ts", -]; -const unitVmForkSingletonFiles = unitVmForkSingletonFilesRaw.filter((file) => fs.existsSync(file)); -const groupedUnitIsolatedFiles = unitIsolatedFiles.filter( - (file) => !unitSingletonIsolatedFiles.includes(file) && !unitThreadSingletonFiles.includes(file), -); -const channelSingletonFilesRaw = []; -const channelSingletonFiles = channelSingletonFilesRaw.filter((file) => fs.existsSync(file)); +const behaviorManifest = loadTestRunnerBehavior(); +const existingFiles = (entries) => + entries.map((entry) => entry.file).filter((file) => fs.existsSync(file)); +const unitBehaviorIsolatedFiles = existingFiles(behaviorManifest.unit.isolated); +const unitSingletonIsolatedFiles = existingFiles(behaviorManifest.unit.singletonIsolated); +const unitThreadSingletonFiles = existingFiles(behaviorManifest.unit.threadSingleton); +const unitVmForkSingletonFiles = existingFiles(behaviorManifest.unit.vmForkSingleton); +const unitBehaviorOverrideSet = new Set([ + ...unitBehaviorIsolatedFiles, + ...unitSingletonIsolatedFiles, + ...unitThreadSingletonFiles, + ...unitVmForkSingletonFiles, +]); +const channelSingletonFiles = []; const children = new Set(); const isCI = process.env.CI === "true" || process.env.GITHUB_ACTIONS === "true"; @@ -158,117 +61,7 @@ const testProfile = // Even on low-memory hosts, keep the isolated lane split so files like // git-commit.test.ts still get the worker/process isolation they require. const shouldSplitUnitRuns = testProfile !== "serial"; -const runs = [ - ...(shouldSplitUnitRuns - ? [ - { - name: "unit-fast", - args: [ - "vitest", - "run", - "--config", - "vitest.unit.config.ts", - `--pool=${useVmForks ? "vmForks" : "forks"}`, - ...(disableIsolation ? ["--isolate=false"] : []), - ...[ - ...unitIsolatedFiles, - ...unitSingletonIsolatedFiles, - ...unitThreadSingletonFiles, - ...unitVmForkSingletonFiles, - ].flatMap((file) => ["--exclude", file]), - ], - }, - ...(groupedUnitIsolatedFiles.length > 0 - ? [ - { - name: "unit-isolated", - args: [ - "vitest", - "run", - "--config", - "vitest.unit.config.ts", - "--pool=forks", - ...groupedUnitIsolatedFiles, - ], - }, - ] - : []), - ...unitSingletonIsolatedFiles.map((file) => ({ - name: `${path.basename(file, ".test.ts")}-isolated`, - args: [ - "vitest", - "run", - "--config", - "vitest.unit.config.ts", - `--pool=${useVmForks ? "vmForks" : "forks"}`, - file, - ], - })), - ...unitThreadSingletonFiles.map((file) => ({ - name: `${path.basename(file, ".test.ts")}-threads`, - args: ["vitest", "run", "--config", "vitest.unit.config.ts", "--pool=threads", file], - })), - ...unitVmForkSingletonFiles.map((file) => ({ - name: `${path.basename(file, ".test.ts")}-vmforks`, - args: [ - "vitest", - "run", - "--config", - "vitest.unit.config.ts", - `--pool=${useVmForks ? "vmForks" : "forks"}`, - ...(disableIsolation ? ["--isolate=false"] : []), - file, - ], - })), - ...channelSingletonFiles.map((file) => ({ - name: `${path.basename(file, ".test.ts")}-channels-isolated`, - args: ["vitest", "run", "--config", "vitest.channels.config.ts", "--pool=forks", file], - })), - ] - : [ - { - name: "unit", - args: [ - "vitest", - "run", - "--config", - "vitest.unit.config.ts", - `--pool=${useVmForks ? "vmForks" : "forks"}`, - ...(disableIsolation ? ["--isolate=false"] : []), - ], - }, - ]), - ...(includeExtensionsSuite - ? [ - { - name: "extensions", - args: [ - "vitest", - "run", - "--config", - "vitest.extensions.config.ts", - ...(useVmForks ? ["--pool=vmForks"] : []), - ], - }, - ] - : []), - ...(includeGatewaySuite - ? [ - { - name: "gateway", - args: [ - "vitest", - "run", - "--config", - "vitest.gateway.config.ts", - // Gateway tests are sensitive to vmForks behavior (global state + env stubs). - // Keep them on process forks for determinism even when other suites use vmForks. - "--pool=forks", - ], - }, - ] - : []), -]; +let runs = []; const shardOverride = Number.parseInt(process.env.OPENCLAW_TEST_SHARDS ?? "", 10); const configuredShardCount = Number.isFinite(shardOverride) && shardOverride > 1 ? shardOverride : null; @@ -414,7 +207,7 @@ const allKnownTestFiles = [ ]), ]; const inferTarget = (fileFilter) => { - const isolated = unitIsolatedFiles.includes(fileFilter); + const isolated = unitBehaviorIsolatedFiles.includes(fileFilter); if (fileFilter.endsWith(".live.test.ts")) { return { owner: "live", isolated }; } @@ -438,6 +231,155 @@ const inferTarget = (fileFilter) => { } return { owner: "base", isolated }; }; +const unitTimingManifest = loadUnitTimingManifest(); +const parseEnvNumber = (name, fallback) => { + const parsed = Number.parseInt(process.env[name] ?? "", 10); + return Number.isFinite(parsed) && parsed >= 0 ? parsed : fallback; +}; +const allKnownUnitFiles = allKnownTestFiles.filter((file) => inferTarget(file).owner === "unit"); +const defaultHeavyUnitFileLimit = + testProfile === "serial" ? 0 : testProfile === "low" ? 8 : highMemLocalHost ? 24 : 16; +const defaultHeavyUnitLaneCount = + testProfile === "serial" ? 0 : testProfile === "low" ? 1 : highMemLocalHost ? 3 : 2; +const heavyUnitFileLimit = parseEnvNumber( + "OPENCLAW_TEST_HEAVY_UNIT_FILE_LIMIT", + defaultHeavyUnitFileLimit, +); +const heavyUnitLaneCount = parseEnvNumber( + "OPENCLAW_TEST_HEAVY_UNIT_LANES", + defaultHeavyUnitLaneCount, +); +const heavyUnitMinDurationMs = parseEnvNumber("OPENCLAW_TEST_HEAVY_UNIT_MIN_MS", 1200); +const timedHeavyUnitFiles = + shouldSplitUnitRuns && heavyUnitFileLimit > 0 + ? selectTimedHeavyFiles({ + candidates: allKnownUnitFiles, + limit: heavyUnitFileLimit, + minDurationMs: heavyUnitMinDurationMs, + exclude: unitBehaviorOverrideSet, + timings: unitTimingManifest, + }) + : []; +const unitFastExcludedFiles = [ + ...new Set([...unitBehaviorOverrideSet, ...timedHeavyUnitFiles, ...channelSingletonFiles]), +]; +const estimateUnitDurationMs = (file) => + unitTimingManifest.files[file]?.durationMs ?? unitTimingManifest.defaultDurationMs; +const heavyUnitBuckets = packFilesByDuration( + timedHeavyUnitFiles, + heavyUnitLaneCount, + estimateUnitDurationMs, +); +const unitHeavyEntries = heavyUnitBuckets.map((files, index) => ({ + name: `unit-heavy-${String(index + 1)}`, + args: ["vitest", "run", "--config", "vitest.unit.config.ts", "--pool=forks", ...files], +})); +const baseRuns = [ + ...(shouldSplitUnitRuns + ? [ + { + name: "unit-fast", + args: [ + "vitest", + "run", + "--config", + "vitest.unit.config.ts", + `--pool=${useVmForks ? "vmForks" : "forks"}`, + ...(disableIsolation ? ["--isolate=false"] : []), + ...unitFastExcludedFiles.flatMap((file) => ["--exclude", file]), + ], + }, + ...(unitBehaviorIsolatedFiles.length > 0 + ? [ + { + name: "unit-isolated", + args: [ + "vitest", + "run", + "--config", + "vitest.unit.config.ts", + "--pool=forks", + ...unitBehaviorIsolatedFiles, + ], + }, + ] + : []), + ...unitHeavyEntries, + ...unitSingletonIsolatedFiles.map((file) => ({ + name: `${path.basename(file, ".test.ts")}-isolated`, + args: [ + "vitest", + "run", + "--config", + "vitest.unit.config.ts", + `--pool=${useVmForks ? "vmForks" : "forks"}`, + file, + ], + })), + ...unitThreadSingletonFiles.map((file) => ({ + name: `${path.basename(file, ".test.ts")}-threads`, + args: ["vitest", "run", "--config", "vitest.unit.config.ts", "--pool=threads", file], + })), + ...unitVmForkSingletonFiles.map((file) => ({ + name: `${path.basename(file, ".test.ts")}-vmforks`, + args: [ + "vitest", + "run", + "--config", + "vitest.unit.config.ts", + `--pool=${useVmForks ? "vmForks" : "forks"}`, + ...(disableIsolation ? ["--isolate=false"] : []), + file, + ], + })), + ...channelSingletonFiles.map((file) => ({ + name: `${path.basename(file, ".test.ts")}-channels-isolated`, + args: ["vitest", "run", "--config", "vitest.channels.config.ts", "--pool=forks", file], + })), + ] + : [ + { + name: "unit", + args: [ + "vitest", + "run", + "--config", + "vitest.unit.config.ts", + `--pool=${useVmForks ? "vmForks" : "forks"}`, + ...(disableIsolation ? ["--isolate=false"] : []), + ], + }, + ]), + ...(includeExtensionsSuite + ? [ + { + name: "extensions", + args: [ + "vitest", + "run", + "--config", + "vitest.extensions.config.ts", + ...(useVmForks ? ["--pool=vmForks"] : []), + ], + }, + ] + : []), + ...(includeGatewaySuite + ? [ + { + name: "gateway", + args: ["vitest", "run", "--config", "vitest.gateway.config.ts", "--pool=forks"], + }, + ] + : []), +]; +runs = baseRuns; +const formatEntrySummary = (entry) => { + const explicitFilters = countExplicitEntryFilters(entry.args) ?? 0; + return `${entry.name} filters=${String(explicitFilters || "all")} maxWorkers=${String( + maxWorkersForRun(entry.name) ?? "default", + )}`; +}; const resolveFilterMatches = (fileFilter) => { const normalizedFilter = normalizeRepoPath(fileFilter); if (fs.existsSync(fileFilter)) { @@ -674,7 +616,13 @@ const maxWorkersForRun = (name) => { if (isCI && isMacOS) { return 1; } - if (name === "unit-isolated" || name.endsWith("-isolated")) { + if (name.endsWith("-threads") || name.endsWith("-vmforks")) { + return 1; + } + if (name.endsWith("-isolated") && name !== "unit-isolated") { + return 1; + } + if (name === "unit-isolated" || name.startsWith("unit-heavy-")) { return defaultWorkerBudget.unitIsolated; } if (name === "extensions") { @@ -706,9 +654,12 @@ const maxOldSpaceSizeMb = (() => { } return null; })(); +const formatElapsedMs = (elapsedMs) => + elapsedMs >= 1000 ? `${(elapsedMs / 1000).toFixed(1)}s` : `${Math.round(elapsedMs)}ms`; const runOnce = (entry, extraArgs = []) => new Promise((resolve) => { + const startedAt = Date.now(); const maxWorkers = maxWorkersForRun(entry.name); // vmForks with a single worker has shown cross-file leakage in extension suites. // Fall back to process forks when we intentionally clamp that lane to one worker. @@ -726,6 +677,11 @@ const runOnce = (entry, extraArgs = []) => ...extraArgs, ] : [...entryArgs, ...silentArgs, ...windowsCiArgs, ...extraArgs]; + console.log( + `[test-parallel] start ${entry.name} workers=${maxWorkers ?? "default"} filters=${String( + countExplicitEntryFilters(entryArgs) ?? "all", + )}`, + ); const nodeOptions = process.env.NODE_OPTIONS ?? ""; const nextNodeOptions = WARNING_SUPPRESSION_FLAGS.reduce( (acc, flag) => (acc.includes(flag) ? acc : `${acc} ${flag}`.trim()), @@ -756,6 +712,11 @@ const runOnce = (entry, extraArgs = []) => }); child.on("exit", (code, signal) => { children.delete(child); + console.log( + `[test-parallel] done ${entry.name} code=${String(code ?? (signal ? 1 : 0))} elapsed=${formatElapsedMs( + Date.now() - startedAt, + )}`, + ); resolve(code ?? (signal ? 1 : 0)); }); }); @@ -823,6 +784,14 @@ const shutdown = (signal) => { process.on("SIGINT", () => shutdown("SIGINT")); process.on("SIGTERM", () => shutdown("SIGTERM")); +if (process.env.OPENCLAW_TEST_LIST_LANES === "1") { + const entriesToPrint = targetedEntries.length > 0 ? targetedEntries : runs; + for (const entry of entriesToPrint) { + console.log(formatEntrySummary(entry)); + } + process.exit(0); +} + if (targetedEntries.length > 0) { if (passthroughRequiresSingleRun && targetedEntries.length > 1) { console.error( diff --git a/scripts/test-runner-manifest.mjs b/scripts/test-runner-manifest.mjs new file mode 100644 index 00000000000..30b4414acc7 --- /dev/null +++ b/scripts/test-runner-manifest.mjs @@ -0,0 +1,129 @@ +import fs from "node:fs"; +import path from "node:path"; + +export const behaviorManifestPath = "test/fixtures/test-parallel.behavior.json"; +export const unitTimingManifestPath = "test/fixtures/test-timings.unit.json"; + +const defaultTimingManifest = { + config: "vitest.unit.config.ts", + defaultDurationMs: 250, + files: {}, +}; + +const readJson = (filePath, fallback) => { + try { + return JSON.parse(fs.readFileSync(filePath, "utf8")); + } catch { + return fallback; + } +}; + +const normalizeRepoPath = (value) => value.split(path.sep).join("/"); + +const normalizeManifestEntries = (entries) => + entries + .map((entry) => + typeof entry === "string" + ? { file: normalizeRepoPath(entry), reason: "" } + : { + file: normalizeRepoPath(String(entry?.file ?? "")), + reason: typeof entry?.reason === "string" ? entry.reason : "", + }, + ) + .filter((entry) => entry.file.length > 0); + +export function loadTestRunnerBehavior() { + const raw = readJson(behaviorManifestPath, {}); + const unit = raw.unit ?? {}; + return { + unit: { + isolated: normalizeManifestEntries(unit.isolated ?? []), + singletonIsolated: normalizeManifestEntries(unit.singletonIsolated ?? []), + threadSingleton: normalizeManifestEntries(unit.threadSingleton ?? []), + vmForkSingleton: normalizeManifestEntries(unit.vmForkSingleton ?? []), + }, + }; +} + +export function loadUnitTimingManifest() { + const raw = readJson(unitTimingManifestPath, defaultTimingManifest); + const defaultDurationMs = + Number.isFinite(raw.defaultDurationMs) && raw.defaultDurationMs > 0 + ? raw.defaultDurationMs + : defaultTimingManifest.defaultDurationMs; + const files = Object.fromEntries( + Object.entries(raw.files ?? {}) + .map(([file, value]) => { + const normalizedFile = normalizeRepoPath(file); + const durationMs = + Number.isFinite(value?.durationMs) && value.durationMs >= 0 ? value.durationMs : null; + const testCount = + Number.isFinite(value?.testCount) && value.testCount >= 0 ? value.testCount : null; + if (!durationMs) { + return [normalizedFile, null]; + } + return [ + normalizedFile, + { + durationMs, + ...(testCount !== null ? { testCount } : {}), + }, + ]; + }) + .filter(([, value]) => value !== null), + ); + + return { + config: + typeof raw.config === "string" && raw.config ? raw.config : defaultTimingManifest.config, + generatedAt: typeof raw.generatedAt === "string" ? raw.generatedAt : "", + defaultDurationMs, + files, + }; +} + +export function selectTimedHeavyFiles({ + candidates, + limit, + minDurationMs, + exclude = new Set(), + timings, +}) { + return candidates + .filter((file) => !exclude.has(file)) + .map((file) => ({ + file, + durationMs: timings.files[file]?.durationMs ?? timings.defaultDurationMs, + known: Boolean(timings.files[file]), + })) + .filter((entry) => entry.known && entry.durationMs >= minDurationMs) + .toSorted((a, b) => b.durationMs - a.durationMs) + .slice(0, limit) + .map((entry) => entry.file); +} + +export function packFilesByDuration(files, bucketCount, estimateDurationMs) { + const normalizedBucketCount = Math.max(0, Math.floor(bucketCount)); + if (normalizedBucketCount <= 0 || files.length === 0) { + return []; + } + + const buckets = Array.from({ length: Math.min(normalizedBucketCount, files.length) }, () => ({ + totalMs: 0, + files: [], + })); + + const sortedFiles = [...files].toSorted((left, right) => { + return estimateDurationMs(right) - estimateDurationMs(left); + }); + + for (const file of sortedFiles) { + const bucket = buckets.reduce((lightest, current) => + current.totalMs < lightest.totalMs ? current : lightest, + ); + bucket.files.push(file); + bucket.totalMs += estimateDurationMs(file); + } + + return buckets.map((bucket) => bucket.files).filter((bucket) => bucket.length > 0); +} diff --git a/scripts/test-update-timings.mjs b/scripts/test-update-timings.mjs new file mode 100644 index 00000000000..722d3539f7a --- /dev/null +++ b/scripts/test-update-timings.mjs @@ -0,0 +1,109 @@ +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { unitTimingManifestPath } from "./test-runner-manifest.mjs"; + +function parseArgs(argv) { + const args = { + config: "vitest.unit.config.ts", + out: unitTimingManifestPath, + reportPath: "", + limit: 128, + defaultDurationMs: 250, + }; + for (let i = 0; i < argv.length; i += 1) { + const arg = argv[i]; + if (arg === "--config") { + args.config = argv[i + 1] ?? args.config; + i += 1; + continue; + } + if (arg === "--out") { + args.out = argv[i + 1] ?? args.out; + i += 1; + continue; + } + if (arg === "--report") { + args.reportPath = argv[i + 1] ?? ""; + i += 1; + continue; + } + if (arg === "--limit") { + const parsed = Number.parseInt(argv[i + 1] ?? "", 10); + if (Number.isFinite(parsed) && parsed > 0) { + args.limit = parsed; + } + i += 1; + continue; + } + if (arg === "--default-duration-ms") { + const parsed = Number.parseInt(argv[i + 1] ?? "", 10); + if (Number.isFinite(parsed) && parsed > 0) { + args.defaultDurationMs = parsed; + } + i += 1; + continue; + } + } + return args; +} + +const normalizeRepoPath = (value) => value.split(path.sep).join("/"); + +const opts = parseArgs(process.argv.slice(2)); +const reportPath = + opts.reportPath || path.join(os.tmpdir(), `openclaw-vitest-timings-${Date.now()}.json`); + +if (!(opts.reportPath && fs.existsSync(reportPath))) { + const run = spawnSync( + "pnpm", + ["vitest", "run", "--config", opts.config, "--reporter=json", "--outputFile", reportPath], + { + stdio: "inherit", + env: process.env, + }, + ); + + if (run.status !== 0) { + process.exit(run.status ?? 1); + } +} + +const report = JSON.parse(fs.readFileSync(reportPath, "utf8")); +const files = Object.fromEntries( + (report.testResults ?? []) + .map((result) => { + const file = typeof result.name === "string" ? normalizeRepoPath(result.name) : ""; + const start = typeof result.startTime === "number" ? result.startTime : 0; + const end = typeof result.endTime === "number" ? result.endTime : 0; + const testCount = Array.isArray(result.assertionResults) ? result.assertionResults.length : 0; + return { + file, + durationMs: Math.max(0, end - start), + testCount, + }; + }) + .filter((entry) => entry.file.length > 0 && entry.durationMs > 0) + .toSorted((a, b) => b.durationMs - a.durationMs) + .slice(0, opts.limit) + .map((entry) => [ + entry.file, + { + durationMs: entry.durationMs, + testCount: entry.testCount, + }, + ]), +); + +const output = { + config: opts.config, + generatedAt: new Date().toISOString(), + defaultDurationMs: opts.defaultDurationMs, + files, +}; + +fs.writeFileSync(opts.out, `${JSON.stringify(output, null, 2)}\n`); +console.log( + `[test-update-timings] wrote ${String(Object.keys(files).length)} timings to ${opts.out}`, +); diff --git a/test/fixtures/test-parallel.behavior.json b/test/fixtures/test-parallel.behavior.json new file mode 100644 index 00000000000..b1ed463612e --- /dev/null +++ b/test/fixtures/test-parallel.behavior.json @@ -0,0 +1,60 @@ +{ + "unit": { + "isolated": [ + { + "file": "src/plugins/contracts/runtime.contract.test.ts", + "reason": "Async runtime imports + provider refresh seams; keep out of the shared lane." + }, + { + "file": "src/security/temp-path-guard.test.ts", + "reason": "Filesystem guard scans are sensitive to contention." + }, + { + "file": "src/infra/git-commit.test.ts", + "reason": "Mutates process.cwd() and core loader seams." + }, + { + "file": "extensions/imessage/src/monitor.shutdown.unhandled-rejection.test.ts", + "reason": "Touches process-level unhandledRejection listeners." + } + ], + "singletonIsolated": [ + { + "file": "src/cli/command-secret-gateway.test.ts", + "reason": "Clean in isolation, but can hang after sharing the broad lane." + } + ], + "threadSingleton": [ + { + "file": "src/channels/plugins/actions/actions.test.ts", + "reason": "Terminates cleanly under threads, but not process forks on this host." + }, + { + "file": "src/infra/outbound/deliver.test.ts", + "reason": "Terminates cleanly under threads, but not process forks on this host." + }, + { + "file": "src/infra/outbound/deliver.lifecycle.test.ts", + "reason": "Terminates cleanly under threads, but not process forks on this host." + }, + { + "file": "src/infra/outbound/message.channels.test.ts", + "reason": "Terminates cleanly under threads, but not process forks on this host." + }, + { + "file": "src/infra/outbound/message-action-runner.poll.test.ts", + "reason": "Terminates cleanly under threads, but not process forks on this host." + }, + { + "file": "src/tts/tts.test.ts", + "reason": "Terminates cleanly under threads, but not process forks on this host." + } + ], + "vmForkSingleton": [ + { + "file": "src/channels/plugins/contracts/inbound.telegram.contract.test.ts", + "reason": "Needs the vmForks lane when targeted." + } + ] + } +} diff --git a/test/fixtures/test-timings.unit.json b/test/fixtures/test-timings.unit.json new file mode 100644 index 00000000000..2199276bc5b --- /dev/null +++ b/test/fixtures/test-timings.unit.json @@ -0,0 +1,135 @@ +{ + "config": "vitest.unit.config.ts", + "generatedAt": "2026-03-18T17:10:00.000Z", + "defaultDurationMs": 250, + "files": { + "src/security/audit.test.ts": { + "durationMs": 6200, + "testCount": 380 + }, + "src/plugins/loader.test.ts": { + "durationMs": 6100, + "testCount": 260 + }, + "src/cli/update-cli.test.ts": { + "durationMs": 5400, + "testCount": 210 + }, + "src/agents/pi-embedded-runner.test.ts": { + "durationMs": 5200, + "testCount": 140 + }, + "src/process/supervisor/supervisor.test.ts": { + "durationMs": 5000, + "testCount": 120 + }, + "src/agents/bash-tools.test.ts": { + "durationMs": 4700, + "testCount": 150 + }, + "src/cli/program.smoke.test.ts": { + "durationMs": 4500, + "testCount": 95 + }, + "src/hooks/install.test.ts": { + "durationMs": 4300, + "testCount": 95 + }, + "src/agents/skills.test.ts": { + "durationMs": 4200, + "testCount": 135 + }, + "src/config/schema.test.ts": { + "durationMs": 4000, + "testCount": 110 + }, + "src/media/store.test.ts": { + "durationMs": 3900, + "testCount": 120 + }, + "src/commands/agent.test.ts": { + "durationMs": 3700, + "testCount": 110 + }, + "extensions/telegram/src/bot.create-telegram-bot.test.ts": { + "durationMs": 3600, + "testCount": 80 + }, + "extensions/telegram/src/bot.test.ts": { + "durationMs": 3400, + "testCount": 95 + }, + "src/agents/openclaw-tools.subagents.sessions-spawn.lifecycle.test.ts": { + "durationMs": 3300, + "testCount": 85 + }, + "src/infra/archive.test.ts": { + "durationMs": 3200, + "testCount": 75 + }, + "src/auto-reply/reply.block-streaming.test.ts": { + "durationMs": 3100, + "testCount": 60 + }, + "src/agents/pi-embedded-runner.run-embedded-pi-agent.auth-profile-rotation.test.ts": { + "durationMs": 3000, + "testCount": 55 + }, + "src/agents/skills.buildworkspaceskillsnapshot.test.ts": { + "durationMs": 2900, + "testCount": 70 + }, + "src/docker-setup.test.ts": { + "durationMs": 2800, + "testCount": 65 + }, + "src/agents/skills-install.download.test.ts": { + "durationMs": 2700, + "testCount": 60 + }, + "src/config/schema.tags.test.ts": { + "durationMs": 2600, + "testCount": 70 + }, + "src/cli/daemon-cli.coverage.test.ts": { + "durationMs": 2500, + "testCount": 50 + }, + "extensions/slack/src/monitor/slash.test.ts": { + "durationMs": 2400, + "testCount": 55 + }, + "test/git-hooks-pre-commit.test.ts": { + "durationMs": 2300, + "testCount": 20 + }, + "src/commands/doctor.warns-state-directory-is-missing.test.ts": { + "durationMs": 2200, + "testCount": 35 + }, + "src/commands/doctor.warns-per-agent-sandbox-docker-browser-prune.test.ts": { + "durationMs": 2100, + "testCount": 30 + }, + "src/commands/doctor.runs-legacy-state-migrations-yes-mode-without.test.ts": { + "durationMs": 2000, + "testCount": 28 + }, + "src/browser/server.agent-contract-snapshot-endpoints.test.ts": { + "durationMs": 1900, + "testCount": 45 + }, + "src/browser/server.agent-contract-form-layout-act-commands.test.ts": { + "durationMs": 1800, + "testCount": 40 + }, + "src/agents/models-config.normalizes-gemini-3-ids-preview-google-providers.test.ts": { + "durationMs": 1700, + "testCount": 25 + }, + "src/agents/session-tool-result-guard.tool-result-persist-hook.test.ts": { + "durationMs": 1600, + "testCount": 22 + } + } +}