perf: automate vitest thread candidate scans

package.json

@@ -699,6 +699,7 @@
     "test:parallels:macos": "bash scripts/e2e/parallels-macos-smoke.sh",
     "test:parallels:windows": "bash scripts/e2e/parallels-windows-smoke.sh",
     "test:perf:budget": "node scripts/test-perf-budget.mjs",
+    "test:perf:find-thread-candidates": "node scripts/test-find-thread-candidates.mjs",
     "test:perf:hotspots": "node scripts/test-hotspots.mjs",
     "test:perf:update-memory-hotspots": "node scripts/test-update-memory-hotspots.mjs",
     "test:perf:update-timings": "node scripts/test-update-timings.mjs",

scripts/test-find-thread-candidates.mjs

@@ -0,0 +1,242 @@
+import { spawnSync } from "node:child_process";
+import path from "node:path";
+import { pathToFileURL } from "node:url";
+import { loadTestRunnerBehavior, loadUnitTimingManifest } from "./test-runner-manifest.mjs";
+
+function readEnvNumber(name) {
+  const raw = process.env[name]?.trim();
+  if (!raw) {
+    return null;
+  }
+  const parsed = Number.parseFloat(raw);
+  return Number.isFinite(parsed) ? parsed : null;
+}
+
+export function parseArgs(argv) {
+  const args = {
+    config: "vitest.unit.config.ts",
+    limit: Number.isFinite(readEnvNumber("OPENCLAW_TEST_THREAD_CANDIDATE_LIMIT"))
+      ? Math.max(1, Math.floor(readEnvNumber("OPENCLAW_TEST_THREAD_CANDIDATE_LIMIT")))
+      : 20,
+    minDurationMs: readEnvNumber("OPENCLAW_TEST_THREAD_CANDIDATE_MIN_DURATION_MS") ?? 250,
+    minGainMs: readEnvNumber("OPENCLAW_TEST_THREAD_CANDIDATE_MIN_GAIN_MS") ?? 100,
+    minGainPct: readEnvNumber("OPENCLAW_TEST_THREAD_CANDIDATE_MIN_GAIN_PCT") ?? 10,
+    json: false,
+    files: [],
+  };
+  for (let i = 0; i < argv.length; i += 1) {
+    const arg = argv[i];
+    if (arg === "--") {
+      continue;
+    }
+    if (arg === "--config") {
+      args.config = argv[i + 1] ?? args.config;
+      i += 1;
+      continue;
+    }
+    if (arg === "--limit") {
+      const parsed = Number.parseInt(argv[i + 1] ?? "", 10);
+      if (Number.isFinite(parsed) && parsed > 0) {
+        args.limit = parsed;
+      }
+      i += 1;
+      continue;
+    }
+    if (arg === "--min-duration-ms") {
+      const parsed = Number.parseFloat(argv[i + 1] ?? "");
+      if (Number.isFinite(parsed) && parsed > 0) {
+        args.minDurationMs = parsed;
+      }
+      i += 1;
+      continue;
+    }
+    if (arg === "--min-gain-ms") {
+      const parsed = Number.parseFloat(argv[i + 1] ?? "");
+      if (Number.isFinite(parsed) && parsed > 0) {
+        args.minGainMs = parsed;
+      }
+      i += 1;
+      continue;
+    }
+    if (arg === "--min-gain-pct") {
+      const parsed = Number.parseFloat(argv[i + 1] ?? "");
+      if (Number.isFinite(parsed) && parsed > 0) {
+        args.minGainPct = parsed;
+      }
+      i += 1;
+      continue;
+    }
+    if (arg === "--json") {
+      args.json = true;
+      continue;
+    }
+    if (arg.startsWith("-")) {
+      throw new Error(`Unknown option: ${arg}`);
+    }
+    args.files.push(arg);
+  }
+  return args;
+}
+
+export function getExistingThreadCandidateExclusions(behavior) {
+  return new Set([
+    ...(behavior.unit?.isolated ?? []).map((entry) => entry.file),
+    ...(behavior.unit?.singletonIsolated ?? []).map((entry) => entry.file),
+    ...(behavior.unit?.threadSingleton ?? []).map((entry) => entry.file),
+    ...(behavior.unit?.vmForkSingleton ?? []).map((entry) => entry.file),
+  ]);
+}
+
+export function selectThreadCandidateFiles({
+  files,
+  timings,
+  exclude = new Set(),
+  limit,
+  minDurationMs,
+  includeUnknownDuration = false,
+}) {
+  return files
+    .map((file) => ({
+      file,
+      durationMs: timings.files[file]?.durationMs ?? null,
+    }))
+    .filter((entry) => !exclude.has(entry.file))
+    .filter((entry) =>
+      entry.durationMs === null ? includeUnknownDuration : entry.durationMs >= minDurationMs,
+    )
+    .toSorted((a, b) => b.durationMs - a.durationMs)
+    .slice(0, limit)
+    .map((entry) => entry.file);
+}
+
+export function summarizeThreadBenchmark({ file, forks, threads, minGainMs, minGainPct }) {
+  const forkOk = forks.exitCode === 0;
+  const threadOk = threads.exitCode === 0;
+  const gainMs = forks.elapsedMs - threads.elapsedMs;
+  const gainPct = forks.elapsedMs > 0 ? (gainMs / forks.elapsedMs) * 100 : 0;
+  const recommended =
+    forkOk &&
+    threadOk &&
+    gainMs >= minGainMs &&
+    gainPct >= minGainPct &&
+    threads.elapsedMs < forks.elapsedMs;
+  return {
+    file,
+    forks,
+    threads,
+    gainMs,
+    gainPct,
+    recommended,
+  };
+}
+
+function formatMs(ms) {
+  return `${ms.toFixed(0)}ms`;
+}
+
+function benchmarkFile({ config, file, pool }) {
+  const startedAt = process.hrtime.bigint();
+  const run = spawnSync("pnpm", ["vitest", "run", "--config", config, `--pool=${pool}`, file], {
+    encoding: "utf8",
+    env: process.env,
+    maxBuffer: 20 * 1024 * 1024,
+  });
+  const elapsedMs = Number(process.hrtime.bigint() - startedAt) / 1_000_000;
+  return {
+    pool,
+    exitCode: run.status ?? 1,
+    elapsedMs,
+    stderr: run.stderr ?? "",
+    stdout: run.stdout ?? "",
+  };
+}
+
+function buildOutput(results) {
+  return results.map((result) => ({
+    file: result.file,
+    forksMs: Math.round(result.forks.elapsedMs),
+    threadsMs: Math.round(result.threads.elapsedMs),
+    gainMs: Math.round(result.gainMs),
+    gainPct: Number(result.gainPct.toFixed(1)),
+    forksExitCode: result.forks.exitCode,
+    threadsExitCode: result.threads.exitCode,
+    recommended: result.recommended,
+  }));
+}
+
+async function main() {
+  const opts = parseArgs(process.argv.slice(2));
+  const behavior = loadTestRunnerBehavior();
+  const timings = loadUnitTimingManifest();
+  const exclude = getExistingThreadCandidateExclusions(behavior);
+  const inputFiles = opts.files.length > 0 ? opts.files : Object.keys(timings.files);
+  const candidates = selectThreadCandidateFiles({
+    files: inputFiles,
+    timings,
+    exclude,
+    limit: opts.limit,
+    minDurationMs: opts.minDurationMs,
+    includeUnknownDuration: opts.files.length > 0,
+  });
+
+  const results = [];
+  for (const file of candidates) {
+    const forks = benchmarkFile({ config: opts.config, file, pool: "forks" });
+    const threads = benchmarkFile({ config: opts.config, file, pool: "threads" });
+    results.push(
+      summarizeThreadBenchmark({
+        file,
+        forks,
+        threads,
+        minGainMs: opts.minGainMs,
+        minGainPct: opts.minGainPct,
+      }),
+    );
+  }
+
+  if (opts.json) {
+    console.log(JSON.stringify(buildOutput(results), null, 2));
+    return;
+  }
+
+  console.log(
+    `[test-find-thread-candidates] tested=${String(results.length)} minGain=${formatMs(
+      opts.minGainMs,
+    )} minGainPct=${String(opts.minGainPct)}%`,
+  );
+  for (const result of results) {
+    const status = result.recommended
+      ? "recommend"
+      : result.forks.exitCode !== 0
+        ? "forks-failed"
+        : result.threads.exitCode !== 0
+          ? "threads-failed"
+          : "skip";
+    console.log(
+      `${status.padEnd(14, " ")} ${result.file} forks=${formatMs(result.forks.elapsedMs)} threads=${formatMs(
+        result.threads.elapsedMs,
+      )} gain=${formatMs(result.gainMs)} (${result.gainPct.toFixed(1)}%)`,
+    );
+    if (result.threads.exitCode !== 0) {
+      const firstErrorLine =
+        result.threads.stderr
+          .split(/\r?\n/u)
+          .find(
+            (line) => line.includes("Error") || line.includes("TypeError") || line.includes("FAIL"),
+          ) ?? "threads failed";
+      console.log(`  ${firstErrorLine}`);
+    }
+  }
+}
+
+const isMain =
+  process.argv[1] && pathToFileURL(path.resolve(process.argv[1])).href === import.meta.url;
+
+if (isMain) {
+  try {
+    await main();
+  } catch (error) {
+    console.error(error instanceof Error ? error.message : String(error));
+    process.exit(1);
+  }
+}

test/fixtures/test-parallel.behavior.json

@@ -313,6 +313,10 @@
         "file": "src/channels/plugins/actions/actions.test.ts",
         "reason": "Terminates cleanly under threads, but not process forks on this host."
       },
+      {
+        "file": "test/extension-plugin-sdk-boundary.test.ts",
+        "reason": "Measured ~12% faster under threads than forks on this host while keeping the file green."
+      },
       {
         "file": "src/hooks/install.test.ts",
         "reason": "Measured ~14% faster under threads than forks on this host while keeping the file green."
@@ -329,10 +333,30 @@
         "file": "src/plugin-sdk/subpaths.test.ts",
         "reason": "Measured ~23% faster under threads than forks on this host while keeping the file green."
       },
+      {
+        "file": "src/plugin-sdk/channel-import-guardrails.test.ts",
+        "reason": "Measured ~30% faster under threads than forks on this host while keeping the file green."
+      },
+      {
+        "file": "src/plugins/contracts/wizard.contract.test.ts",
+        "reason": "Measured ~9% faster under threads than forks on this host while keeping the file green."
+      },
       {
         "file": "src/plugins/install.test.ts",
         "reason": "Measured ~18% faster under threads than forks on this host while keeping the file green."
       },
+      {
+        "file": "src/plugins/interactive.test.ts",
+        "reason": "Measured ~9% faster under threads than forks on this host while keeping the file green."
+      },
+      {
+        "file": "src/config/config.plugin-validation.test.ts",
+        "reason": "Measured ~11% faster under threads than forks on this host while keeping the file green."
+      },
+      {
+        "file": "src/config/schema.help.quality.test.ts",
+        "reason": "Measured ~13% faster under threads than forks on this host while keeping the file green."
+      },
       {
         "file": "src/cron/isolated-agent.uses-last-non-empty-agent-text-as.test.ts",
         "reason": "Measured ~15% faster under threads than forks on this host while keeping the file green."
@@ -369,22 +393,66 @@
         "file": "src/infra/provider-usage.test.ts",
         "reason": "Measured ~17% faster under threads than forks on this host while keeping the file green."
       },
+      {
+        "file": "src/infra/provider-usage.auth.normalizes-keys.test.ts",
+        "reason": "Measured ~12% faster under threads than forks on this host while keeping the file green."
+      },
+      {
+        "file": "src/infra/provider-usage.auth.plugin.test.ts",
+        "reason": "Measured ~13% faster under threads than forks on this host while keeping the file green."
+      },
+      {
+        "file": "src/infra/outbound/targets.test.ts",
+        "reason": "Measured ~14% faster under threads than forks on this host while keeping the file green."
+      },
+      {
+        "file": "src/infra/outbound/agent-delivery.test.ts",
+        "reason": "Measured ~17% faster under threads than forks on this host while keeping the file green."
+      },
       {
         "file": "src/infra/fs-pinned-write-helper.test.ts",
         "reason": "Measured ~13% faster under threads than forks on this host while keeping the file green."
       },
+      {
+        "file": "src/infra/fs-safe.test.ts",
+        "reason": "Measured ~17% faster under threads than forks on this host while keeping the file green."
+      },
+      {
+        "file": "src/infra/archive-staging.test.ts",
+        "reason": "Measured ~12% faster under threads than forks on this host while keeping the file green."
+      },
       {
         "file": "src/acp/server.startup.test.ts",
         "reason": "Measured ~11% faster under threads than forks on this host while keeping the file green."
       },
+      {
+        "file": "src/acp/client.test.ts",
+        "reason": "Measured ~18% faster under threads than forks on this host while keeping the file green."
+      },
       {
         "file": "src/daemon/schtasks.startup-fallback.test.ts",
         "reason": "Measured ~15% faster under threads than forks on this host while keeping the file green."
       },
+      {
+        "file": "src/media-understanding/resolve.test.ts",
+        "reason": "Measured ~9% faster under threads than forks on this host while keeping the file green."
+      },
+      {
+        "file": "src/media-understanding/runner.video.test.ts",
+        "reason": "Measured ~25% faster under threads than forks on this host while keeping the file green."
+      },
       {
         "file": "src/secrets/audit.test.ts",
         "reason": "Measured ~14% faster under threads than forks on this host while keeping the file green."
       },
+      {
+        "file": "src/secrets/runtime-web-tools.test.ts",
+        "reason": "Measured ~13% faster under threads than forks on this host while keeping the file green."
+      },
+      {
+        "file": "src/entry.version-fast-path.test.ts",
+        "reason": "Measured ~13% faster under threads than forks on this host while keeping the file green."
+      },
       {
         "file": "src/security/audit.test.ts",
         "reason": "Measured ~40% faster under threads than forks on this host while keeping the file green."
@@ -393,6 +461,10 @@
         "file": "ui/src/ui/views/chat.test.ts",
         "reason": "Measured ~25% faster under threads than forks on this host while keeping the file green."
       },
+      {
+        "file": "src/cli/program/preaction.test.ts",
+        "reason": "Measured ~21% faster under threads than forks on this host while keeping the file green."
+      },
       {
         "file": "src/tts/tts.test.ts",
         "reason": "Terminates cleanly under threads, but not process forks on this host."

test/scripts/test-find-thread-candidates.test.ts

@@ -0,0 +1,116 @@
+import { describe, expect, it } from "vitest";
+import {
+  getExistingThreadCandidateExclusions,
+  parseArgs,
+  selectThreadCandidateFiles,
+  summarizeThreadBenchmark,
+} from "../../scripts/test-find-thread-candidates.mjs";
+
+describe("scripts/test-find-thread-candidates parseArgs", () => {
+  it("parses explicit thresholds and positional files", () => {
+    expect(
+      parseArgs([
+        "--limit",
+        "4",
+        "--min-duration-ms",
+        "600",
+        "--min-gain-ms",
+        "120",
+        "--min-gain-pct",
+        "15",
+        "--json",
+        "src/a.test.ts",
+      ]),
+    ).toEqual({
+      config: "vitest.unit.config.ts",
+      limit: 4,
+      minDurationMs: 600,
+      minGainMs: 120,
+      minGainPct: 15,
+      json: true,
+      files: ["src/a.test.ts"],
+    });
+  });
+});
+
+describe("scripts/test-find-thread-candidates exclusions", () => {
+  it("collects already-pinned files across behavior buckets", () => {
+    expect(
+      getExistingThreadCandidateExclusions({
+        unit: {
+          isolated: [{ file: "src/a.test.ts" }],
+          singletonIsolated: [{ file: "src/b.test.ts" }],
+          threadSingleton: [{ file: "src/c.test.ts" }],
+          vmForkSingleton: [{ file: "src/d.test.ts" }],
+        },
+      }),
+    ).toEqual(new Set(["src/a.test.ts", "src/b.test.ts", "src/c.test.ts", "src/d.test.ts"]));
+  });
+});
+
+describe("scripts/test-find-thread-candidates selection", () => {
+  it("keeps only known, unpinned files above the duration floor", () => {
+    expect(
+      selectThreadCandidateFiles({
+        files: ["src/a.test.ts", "src/b.test.ts", "src/c.test.ts", "src/d.test.ts"],
+        timings: {
+          files: {
+            "src/a.test.ts": { durationMs: 1100 },
+            "src/b.test.ts": { durationMs: 700 },
+            "src/c.test.ts": { durationMs: 300 },
+          },
+        },
+        exclude: new Set(["src/b.test.ts"]),
+        limit: 10,
+        minDurationMs: 500,
+      }),
+    ).toEqual(["src/a.test.ts"]);
+  });
+
+  it("allows explicit unknown-duration files when requested", () => {
+    expect(
+      selectThreadCandidateFiles({
+        files: ["src/a.test.ts", "src/b.test.ts"],
+        timings: {
+          files: {
+            "src/a.test.ts": { durationMs: 700 },
+          },
+        },
+        exclude: new Set(),
+        limit: 10,
+        minDurationMs: 500,
+        includeUnknownDuration: true,
+      }),
+    ).toEqual(["src/a.test.ts", "src/b.test.ts"]);
+  });
+});
+
+describe("scripts/test-find-thread-candidates summarizeThreadBenchmark", () => {
+  it("recommends clear thread wins", () => {
+    expect(
+      summarizeThreadBenchmark({
+        file: "src/a.test.ts",
+        forks: { exitCode: 0, elapsedMs: 1000, stderr: "", stdout: "" },
+        threads: { exitCode: 0, elapsedMs: 780, stderr: "", stdout: "" },
+        minGainMs: 100,
+        minGainPct: 10,
+      }),
+    ).toMatchObject({
+      file: "src/a.test.ts",
+      gainMs: 220,
+      recommended: true,
+    });
+  });
+
+  it("rejects thread failures even when the measured wall time is lower", () => {
+    expect(
+      summarizeThreadBenchmark({
+        file: "src/b.test.ts",
+        forks: { exitCode: 0, elapsedMs: 1000, stderr: "", stdout: "" },
+        threads: { exitCode: 1, elapsedMs: 400, stderr: "TypeError", stdout: "" },
+        minGainMs: 100,
+        minGainPct: 10,
+      }).recommended,
+    ).toBe(false);
+  });
+});