fix: throttle vitest under local contention

2026-04-05 22:32:12 +00:00 · 2026-04-04 05:49:41 +01:00
parent 9afaec1b0c
commit 55812eaf14
5 changed files with 260 additions and 23 deletions
--- a/package.json
+++ b/package.json
@@ -970,7 +970,7 @@
    "android:run": "cd apps/android && ./gradlew :app:installPlayDebug && adb shell am start -n ai.openclaw.app/.MainActivity",
    "android:run:third-party": "cd apps/android && ./gradlew :app:installThirdPartyDebug && adb shell am start -n ai.openclaw.app/.MainActivity",
    "android:test": "cd apps/android && ./gradlew :app:testPlayDebugUnitTest",
-    "android:test:integration": "OPENCLAW_LIVE_TEST=1 OPENCLAW_LIVE_ANDROID_NODE=1 vitest run --config vitest.live.config.ts src/gateway/android-node.capabilities.live.test.ts",
+    "android:test:integration": "OPENCLAW_LIVE_TEST=1 OPENCLAW_LIVE_ANDROID_NODE=1 node scripts/run-vitest.mjs run --config vitest.live.config.ts src/gateway/android-node.capabilities.live.test.ts",
    "android:test:third-party": "cd apps/android && ./gradlew :app:testThirdPartyDebugUnitTest",
    "audit:seams": "node scripts/audit-seams.mjs",
    "build": "pnpm canvas:a2ui:bundle && node scripts/tsdown-build.mjs && node scripts/runtime-postbuild.mjs && node scripts/build-stamp.mjs && pnpm build:plugin-sdk:dts && node --import tsx scripts/write-plugin-sdk-entry-dts.ts && node scripts/check-plugin-sdk-exports.mjs && node --import tsx scripts/canvas-a2ui-copy.ts && node --import tsx scripts/copy-hook-metadata.ts && node --import tsx scripts/copy-export-html-templates.ts && node --import tsx scripts/write-build-info.ts && node --import tsx scripts/write-cli-startup-metadata.ts && node --import tsx scripts/write-cli-compat.ts",
@@ -1093,17 +1093,17 @@
    "start": "node scripts/run-node.mjs",
    "test": "node scripts/test-projects.mjs",
    "test:all": "pnpm lint && pnpm build && pnpm test && pnpm test:e2e && pnpm test:live && pnpm test:docker:all",
-    "test:auth:compat": "vitest run --config vitest.gateway.config.ts src/gateway/server.auth.compat-baseline.test.ts src/gateway/client.test.ts src/gateway/reconnect-gating.test.ts src/gateway/protocol/connect-error-details.test.ts",
+    "test:auth:compat": "node scripts/run-vitest.mjs run --config vitest.gateway.config.ts src/gateway/server.auth.compat-baseline.test.ts src/gateway/client.test.ts src/gateway/reconnect-gating.test.ts src/gateway/protocol/connect-error-details.test.ts",
    "test:build:singleton": "node scripts/test-built-plugin-singleton.mjs",
-    "test:bundled": "vitest run --config vitest.bundled.config.ts",
-    "test:changed": "vitest run --config vitest.config.ts --changed origin/main",
-    "test:changed:max": "OPENCLAW_VITEST_MAX_WORKERS=8 vitest run --config vitest.config.ts --changed origin/main",
-    "test:channels": "vitest run --config vitest.channels.config.ts",
+    "test:bundled": "node scripts/run-vitest.mjs run --config vitest.bundled.config.ts",
+    "test:changed": "node scripts/run-vitest.mjs run --config vitest.config.ts --changed origin/main",
+    "test:changed:max": "OPENCLAW_VITEST_MAX_WORKERS=8 node scripts/run-vitest.mjs run --config vitest.config.ts --changed origin/main",
+    "test:channels": "node scripts/run-vitest.mjs run --config vitest.channels.config.ts",
    "test:contracts": "pnpm test:contracts:channels && pnpm test:contracts:plugins",
-    "test:contracts:channels": "pnpm exec vitest run --config vitest.contracts.config.ts --maxWorkers=1 src/channels/plugins/contracts",
-    "test:contracts:plugins": "pnpm exec vitest run --config vitest.contracts.config.ts --maxWorkers=1 src/plugins/contracts",
-    "test:coverage": "vitest run --config vitest.unit.config.ts --coverage",
-    "test:coverage:changed": "vitest run --config vitest.unit.config.ts --coverage --changed origin/main",
+    "test:contracts:channels": "node scripts/run-vitest.mjs run --config vitest.contracts.config.ts --maxWorkers=1 src/channels/plugins/contracts",
+    "test:contracts:plugins": "node scripts/run-vitest.mjs run --config vitest.contracts.config.ts --maxWorkers=1 src/plugins/contracts",
+    "test:coverage": "node scripts/run-vitest.mjs run --config vitest.unit.config.ts --coverage",
+    "test:coverage:changed": "node scripts/run-vitest.mjs run --config vitest.unit.config.ts --coverage --changed origin/main",
    "test:docker:all": "pnpm test:docker:live-build && OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:live-models && OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:live-gateway && pnpm test:docker:openwebui && pnpm test:docker:onboard && pnpm test:docker:gateway-network && pnpm test:docker:mcp-channels && pnpm test:docker:qr && pnpm test:docker:doctor-switch && pnpm test:docker:plugins && pnpm test:docker:cleanup",
    "test:docker:cleanup": "bash scripts/test-cleanup-docker.sh",
    "test:docker:doctor-switch": "bash scripts/e2e/doctor-install-switch-docker.sh",
@@ -1118,15 +1118,15 @@
    "test:docker:openwebui": "bash scripts/e2e/openwebui-docker.sh",
    "test:docker:plugins": "bash scripts/e2e/plugins-docker.sh",
    "test:docker:qr": "bash scripts/e2e/qr-import-docker.sh",
-    "test:e2e": "vitest run --config vitest.e2e.config.ts",
-    "test:e2e:openshell": "OPENCLAW_E2E_OPENSHELL=1 vitest run --config vitest.e2e.config.ts test/openshell-sandbox.e2e.test.ts",
+    "test:e2e": "node scripts/run-vitest.mjs run --config vitest.e2e.config.ts",
+    "test:e2e:openshell": "OPENCLAW_E2E_OPENSHELL=1 node scripts/run-vitest.mjs run --config vitest.e2e.config.ts test/openshell-sandbox.e2e.test.ts",
    "test:extension": "node scripts/test-extension.mjs",
-    "test:extensions": "vitest run --config vitest.extensions.config.ts",
+    "test:extensions": "node scripts/run-vitest.mjs run --config vitest.extensions.config.ts",
    "test:extensions:batch": "node scripts/test-extension-batch.mjs",
    "test:extensions:memory": "node scripts/profile-extension-memory.mjs",
-    "test:fast": "vitest run --config vitest.unit.config.ts",
+    "test:fast": "node scripts/run-vitest.mjs run --config vitest.unit.config.ts",
    "test:force": "node --import tsx scripts/test-force.ts",
-    "test:gateway": "vitest run --config vitest.gateway.config.ts --pool=forks",
+    "test:gateway": "node scripts/run-vitest.mjs run --config vitest.gateway.config.ts --pool=forks",
    "test:gateway:watch-regression": "node scripts/check-gateway-watch-regression.mjs",
    "test:install:e2e": "bash scripts/test-install-sh-e2e-docker.sh",
    "test:install:e2e:anthropic": "OPENCLAW_E2E_MODELS=anthropic bash scripts/test-install-sh-e2e-docker.sh",
@@ -1142,11 +1142,11 @@
    "test:parallels:windows": "bash scripts/e2e/parallels-windows-smoke.sh",
    "test:perf:budget": "node scripts/test-perf-budget.mjs",
    "test:perf:hotspots": "node scripts/test-hotspots.mjs",
-    "test:perf:imports": "OPENCLAW_VITEST_IMPORT_DURATIONS=1 OPENCLAW_VITEST_PRINT_IMPORT_BREAKDOWN=1 vitest run --config vitest.config.ts",
-    "test:perf:imports:changed": "OPENCLAW_VITEST_IMPORT_DURATIONS=1 OPENCLAW_VITEST_PRINT_IMPORT_BREAKDOWN=1 vitest run --config vitest.config.ts --changed origin/main",
+    "test:perf:imports": "OPENCLAW_VITEST_IMPORT_DURATIONS=1 OPENCLAW_VITEST_PRINT_IMPORT_BREAKDOWN=1 node scripts/run-vitest.mjs run --config vitest.config.ts",
+    "test:perf:imports:changed": "OPENCLAW_VITEST_IMPORT_DURATIONS=1 OPENCLAW_VITEST_PRINT_IMPORT_BREAKDOWN=1 node scripts/run-vitest.mjs run --config vitest.config.ts --changed origin/main",
    "test:perf:profile:main": "node scripts/run-vitest-profile.mjs main",
    "test:perf:profile:runner": "node scripts/run-vitest-profile.mjs runner",
-    "test:sectriage": "pnpm exec vitest run --config vitest.gateway.config.ts && vitest run --config vitest.unit.config.ts --exclude src/daemon/launchd.integration.test.ts --exclude src/process/exec.test.ts",
+    "test:sectriage": "node scripts/run-vitest.mjs run --config vitest.gateway.config.ts && node scripts/run-vitest.mjs run --config vitest.unit.config.ts --exclude src/daemon/launchd.integration.test.ts --exclude src/process/exec.test.ts",
    "test:serial": "OPENCLAW_VITEST_MAX_WORKERS=1 node scripts/test-projects.mjs",
    "test:startup:bench": "node --import tsx scripts/bench-cli-startup.ts",
    "test:startup:bench:check": "node scripts/test-cli-startup-bench-budget.mjs",
--- a/scripts/run-vitest.mjs
+++ b/scripts/run-vitest.mjs
@@ -0,0 +1,26 @@
+import { spawnPnpmRunner } from "./pnpm-runner.mjs";
+
+const forwardedArgs = process.argv.slice(2);
+
+if (forwardedArgs.length === 0) {
+  console.error("usage: node scripts/run-vitest.mjs <vitest args...>");
+  process.exit(1);
+}
+
+const child = spawnPnpmRunner({
+  pnpmArgs: ["exec", "vitest", ...forwardedArgs],
+  env: process.env,
+});
+
+child.on("exit", (code, signal) => {
+  if (signal) {
+    process.kill(process.pid, signal);
+    return;
+  }
+  process.exit(code ?? 1);
+});
+
+child.on("error", (error) => {
+  console.error(error);
+  process.exit(1);
+});
--- a/vitest.config.ts
+++ b/vitest.config.ts
@@ -2,10 +2,11 @@ import { defineConfig } from "vitest/config";
 import {
  resolveDefaultVitestPool,
  resolveLocalVitestMaxWorkers,
+  resolveLocalVitestScheduling,
  sharedVitestConfig,
 } from "./vitest.shared.config.ts";

-export { resolveDefaultVitestPool, resolveLocalVitestMaxWorkers };
+export { resolveDefaultVitestPool, resolveLocalVitestMaxWorkers, resolveLocalVitestScheduling };

 export const rootVitestProjects = [
  "vitest.unit.config.ts",
--- a/vitest.shared.config.ts
+++ b/vitest.shared.config.ts
@@ -7,6 +7,11 @@ import {
  BUNDLED_PLUGIN_TEST_GLOB,
 } from "./vitest.bundled-plugin-paths.ts";
 import { loadVitestExperimentalConfig } from "./vitest.performance-config.ts";
+import {
+  detectVitestProcessStats,
+  shouldPrintVitestThrottle,
+  type VitestProcessStats,
+} from "./vitest.system-load.ts";

 const clamp = (value: number, min: number, max: number) => Math.max(min, Math.min(max, value));

@@ -15,6 +20,11 @@ function parsePositiveInt(value: string | undefined): number | null {
  return Number.isFinite(parsed) && parsed > 0 ? parsed : null;
 }

+function isSystemThrottleDisabled(env: Record<string, string | undefined>): boolean {
+  const normalized = env.OPENCLAW_VITEST_DISABLE_SYSTEM_THROTTLE?.trim().toLowerCase();
+  return normalized === "1" || normalized === "true";
+}
+
 type VitestHostInfo = {
  cpuCount?: number;
  loadAverage1m?: number;
@@ -23,6 +33,12 @@ type VitestHostInfo = {

 export type OpenClawVitestPool = "forks";

+export type LocalVitestScheduling = {
+  maxWorkers: number;
+  fileParallelism: boolean;
+  throttledBySystem: boolean;
+};
+
 export const jsdomOptimizedDeps = {
  optimizer: {
    web: {
@@ -44,10 +60,26 @@ function detectVitestHostInfo(): Required<VitestHostInfo> {
 export function resolveLocalVitestMaxWorkers(
  env: Record<string, string | undefined> = process.env,
  system: VitestHostInfo = detectVitestHostInfo(),
+  pool: OpenClawVitestPool = resolveDefaultVitestPool(env),
+  processStats: VitestProcessStats = detectVitestProcessStats(env),
 ): number {
+  return resolveLocalVitestScheduling(env, system, pool, processStats).maxWorkers;
+}
+
+export function resolveLocalVitestScheduling(
+  env: Record<string, string | undefined> = process.env,
+  system: VitestHostInfo = detectVitestHostInfo(),
+  pool: OpenClawVitestPool = resolveDefaultVitestPool(env),
+  processStats: VitestProcessStats = detectVitestProcessStats(env),
+): LocalVitestScheduling {
  const override = parsePositiveInt(env.OPENCLAW_VITEST_MAX_WORKERS ?? env.OPENCLAW_TEST_WORKERS);
  if (override !== null) {
-    return clamp(override, 1, 16);
+    const maxWorkers = clamp(override, 1, 16);
+    return {
+      maxWorkers,
+      fileParallelism: maxWorkers > 1,
+      throttledBySystem: false,
+    };
  }

  const cpuCount = Math.max(1, system.cpuCount ?? 1);
@@ -76,7 +108,50 @@ export function resolveLocalVitestMaxWorkers(
    inferred = Math.max(1, inferred - 1);
  }

-  return clamp(inferred, 1, 16);
+  inferred = clamp(inferred, 1, 16);
+
+  if (isSystemThrottleDisabled(env)) {
+    return {
+      maxWorkers: inferred,
+      fileParallelism: true,
+      throttledBySystem: false,
+    };
+  }
+
+  const highSystemContention =
+    loadRatio >= 1 ||
+    processStats.otherVitestWorkerCount >= 2 ||
+    processStats.otherVitestCpuPercent >= 150 ||
+    processStats.otherVitestRootCount >= 2;
+
+  if (highSystemContention) {
+    return {
+      maxWorkers: 1,
+      fileParallelism: false,
+      throttledBySystem: true,
+    };
+  }
+
+  const moderateSystemContention =
+    loadRatio >= 0.75 ||
+    processStats.otherVitestWorkerCount >= 1 ||
+    processStats.otherVitestCpuPercent >= 75 ||
+    processStats.otherVitestRootCount >= 1;
+
+  if (moderateSystemContention) {
+    const maxWorkers = Math.min(inferred, 2);
+    return {
+      maxWorkers,
+      fileParallelism: true,
+      throttledBySystem: maxWorkers < inferred,
+    };
+  }
+
+  return {
+    maxWorkers: inferred,
+    fileParallelism: true,
+    throttledBySystem: false,
+  };
 }

 export function resolveDefaultVitestPool(
@@ -93,9 +168,21 @@ const repoRoot = path.dirname(fileURLToPath(import.meta.url));
 const isCI = process.env.CI === "true" || process.env.GITHUB_ACTIONS === "true";
 const isWindows = process.platform === "win32";
 const defaultPool = resolveDefaultVitestPool();
-const localWorkers = resolveLocalVitestMaxWorkers(process.env, detectVitestHostInfo());
+const localScheduling = resolveLocalVitestScheduling(
+  process.env,
+  detectVitestHostInfo(),
+  defaultPool,
+);
 const ciWorkers = isWindows ? 2 : 3;

+if (!isCI && localScheduling.throttledBySystem && shouldPrintVitestThrottle(process.env)) {
+  console.error(
+    `[vitest] throttling local workers to ${localScheduling.maxWorkers}${
+      localScheduling.fileParallelism ? "" : " with file parallelism disabled"
+    } because the host already looks busy.`,
+  );
+}
+
 export const sharedVitestConfig = {
  resolve: {
    alias: [
@@ -119,7 +206,8 @@ export const sharedVitestConfig = {
    unstubEnvs: true,
    unstubGlobals: true,
    pool: defaultPool,
-    maxWorkers: isCI ? ciWorkers : localWorkers,
+    maxWorkers: isCI ? ciWorkers : localScheduling.maxWorkers,
+    fileParallelism: isCI ? true : localScheduling.fileParallelism,
    forceRerunTriggers: [
      "package.json",
      "pnpm-lock.yaml",
--- a/vitest.system-load.ts
+++ b/vitest.system-load.ts
@@ -0,0 +1,122 @@
+import { spawnSync } from "node:child_process";
+
+type EnvMap = Record<string, string | undefined>;
+
+export type VitestProcessStats = {
+  otherVitestRootCount: number;
+  otherVitestWorkerCount: number;
+  otherVitestCpuPercent: number;
+};
+
+type PsResult = {
+  status: number | null;
+  stdout: string;
+};
+
+type DetectVitestProcessStatsOptions = {
+  platform?: NodeJS.Platform;
+  selfPid?: number;
+  runPs?: () => PsResult;
+};
+
+const EMPTY_VITEST_PROCESS_STATS: VitestProcessStats = {
+  otherVitestRootCount: 0,
+  otherVitestWorkerCount: 0,
+  otherVitestCpuPercent: 0,
+};
+
+const BOOLEAN_TRUE_VALUES = new Set(["1", "true"]);
+
+function isExplicitlyEnabled(value: string | undefined): boolean {
+  const normalized = value?.trim().toLowerCase();
+  return normalized ? BOOLEAN_TRUE_VALUES.has(normalized) : false;
+}
+
+function isVitestWorkerArgs(args: string): boolean {
+  return args.includes("/vitest/dist/workers/") || args.includes("\\vitest\\dist\\workers\\");
+}
+
+function isVitestRootArgs(args: string): boolean {
+  return (
+    args.includes("node_modules/.bin/vitest") ||
+    /\bvitest(?:\.(?:m?js|cmd|exe))?\b/u.test(args) ||
+    args.includes("scripts/test-projects.mjs") ||
+    args.includes("scripts/run-vitest.mjs")
+  );
+}
+
+function normalizeCpu(rawCpu: string): number {
+  const parsed = Number.parseFloat(rawCpu);
+  return Number.isFinite(parsed) && parsed > 0 ? parsed : 0;
+}
+
+export function parseVitestProcessStats(
+  psOutput: string,
+  selfPid: number = process.pid,
+): VitestProcessStats {
+  const stats = { ...EMPTY_VITEST_PROCESS_STATS };
+
+  for (const line of psOutput.split("\n")) {
+    const trimmed = line.trim();
+    if (trimmed.length === 0) {
+      continue;
+    }
+
+    const match = /^(\d+)\s+([0-9.]+)\s+(.*)$/u.exec(trimmed);
+    if (!match) {
+      continue;
+    }
+
+    const [, rawPid, rawCpu, args] = match;
+    const pid = Number.parseInt(rawPid, 10);
+    if (!Number.isFinite(pid) || pid === selfPid) {
+      continue;
+    }
+
+    if (!isVitestWorkerArgs(args) && !isVitestRootArgs(args)) {
+      continue;
+    }
+
+    stats.otherVitestCpuPercent += normalizeCpu(rawCpu);
+    if (isVitestWorkerArgs(args)) {
+      stats.otherVitestWorkerCount += 1;
+    } else {
+      stats.otherVitestRootCount += 1;
+    }
+  }
+
+  stats.otherVitestCpuPercent = Number.parseFloat(stats.otherVitestCpuPercent.toFixed(1));
+  return stats;
+}
+
+export function detectVitestProcessStats(
+  env: EnvMap = process.env,
+  options: DetectVitestProcessStatsOptions = {},
+): VitestProcessStats {
+  const platform = options.platform ?? process.platform;
+  if (platform === "win32") {
+    return { ...EMPTY_VITEST_PROCESS_STATS };
+  }
+
+  if (isExplicitlyEnabled(env.OPENCLAW_VITEST_DISABLE_SYSTEM_THROTTLE)) {
+    return { ...EMPTY_VITEST_PROCESS_STATS };
+  }
+
+  const result =
+    options.runPs?.() ??
+    spawnSync("ps", ["-xao", "pid=,pcpu=,args="], {
+      encoding: "utf8",
+      stdio: ["ignore", "pipe", "ignore"],
+    });
+
+  if (result.status === 0 && typeof result.stdout === "string" && result.stdout.length > 0) {
+    return parseVitestProcessStats(result.stdout, options.selfPid ?? process.pid);
+  }
+
+  return { ...EMPTY_VITEST_PROCESS_STATS };
+}
+
+export function shouldPrintVitestThrottle(env: EnvMap = process.env): boolean {
+  const normalized = env.OPENCLAW_VITEST_PRINT_SYSTEM_THROTTLE?.trim().toLowerCase();
+  return normalized ? BOOLEAN_TRUE_VALUES.has(normalized) : false;
+}