diff --git a/package.json b/package.json index 4fc460f401b..713b027b4e4 100644 --- a/package.json +++ b/package.json @@ -699,6 +699,7 @@ "test:parallels:macos": "bash scripts/e2e/parallels-macos-smoke.sh", "test:parallels:windows": "bash scripts/e2e/parallels-windows-smoke.sh", "test:perf:budget": "node scripts/test-perf-budget.mjs", + "test:perf:find-thread-candidates": "node scripts/test-find-thread-candidates.mjs", "test:perf:hotspots": "node scripts/test-hotspots.mjs", "test:perf:update-memory-hotspots": "node scripts/test-update-memory-hotspots.mjs", "test:perf:update-timings": "node scripts/test-update-timings.mjs", diff --git a/scripts/test-find-thread-candidates.mjs b/scripts/test-find-thread-candidates.mjs new file mode 100644 index 00000000000..b674317455d --- /dev/null +++ b/scripts/test-find-thread-candidates.mjs @@ -0,0 +1,242 @@ +import { spawnSync } from "node:child_process"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { loadTestRunnerBehavior, loadUnitTimingManifest } from "./test-runner-manifest.mjs"; + +function readEnvNumber(name) { + const raw = process.env[name]?.trim(); + if (!raw) { + return null; + } + const parsed = Number.parseFloat(raw); + return Number.isFinite(parsed) ? parsed : null; +} + +export function parseArgs(argv) { + const args = { + config: "vitest.unit.config.ts", + limit: Number.isFinite(readEnvNumber("OPENCLAW_TEST_THREAD_CANDIDATE_LIMIT")) + ? Math.max(1, Math.floor(readEnvNumber("OPENCLAW_TEST_THREAD_CANDIDATE_LIMIT"))) + : 20, + minDurationMs: readEnvNumber("OPENCLAW_TEST_THREAD_CANDIDATE_MIN_DURATION_MS") ?? 250, + minGainMs: readEnvNumber("OPENCLAW_TEST_THREAD_CANDIDATE_MIN_GAIN_MS") ?? 100, + minGainPct: readEnvNumber("OPENCLAW_TEST_THREAD_CANDIDATE_MIN_GAIN_PCT") ?? 10, + json: false, + files: [], + }; + for (let i = 0; i < argv.length; i += 1) { + const arg = argv[i]; + if (arg === "--") { + continue; + } + if (arg === "--config") { + args.config = argv[i + 1] ?? args.config; + i += 1; + continue; + } + if (arg === "--limit") { + const parsed = Number.parseInt(argv[i + 1] ?? "", 10); + if (Number.isFinite(parsed) && parsed > 0) { + args.limit = parsed; + } + i += 1; + continue; + } + if (arg === "--min-duration-ms") { + const parsed = Number.parseFloat(argv[i + 1] ?? ""); + if (Number.isFinite(parsed) && parsed > 0) { + args.minDurationMs = parsed; + } + i += 1; + continue; + } + if (arg === "--min-gain-ms") { + const parsed = Number.parseFloat(argv[i + 1] ?? ""); + if (Number.isFinite(parsed) && parsed > 0) { + args.minGainMs = parsed; + } + i += 1; + continue; + } + if (arg === "--min-gain-pct") { + const parsed = Number.parseFloat(argv[i + 1] ?? ""); + if (Number.isFinite(parsed) && parsed > 0) { + args.minGainPct = parsed; + } + i += 1; + continue; + } + if (arg === "--json") { + args.json = true; + continue; + } + if (arg.startsWith("-")) { + throw new Error(`Unknown option: ${arg}`); + } + args.files.push(arg); + } + return args; +} + +export function getExistingThreadCandidateExclusions(behavior) { + return new Set([ + ...(behavior.unit?.isolated ?? []).map((entry) => entry.file), + ...(behavior.unit?.singletonIsolated ?? []).map((entry) => entry.file), + ...(behavior.unit?.threadSingleton ?? []).map((entry) => entry.file), + ...(behavior.unit?.vmForkSingleton ?? []).map((entry) => entry.file), + ]); +} + +export function selectThreadCandidateFiles({ + files, + timings, + exclude = new Set(), + limit, + minDurationMs, + includeUnknownDuration = false, +}) { + return files + .map((file) => ({ + file, + durationMs: timings.files[file]?.durationMs ?? null, + })) + .filter((entry) => !exclude.has(entry.file)) + .filter((entry) => + entry.durationMs === null ? includeUnknownDuration : entry.durationMs >= minDurationMs, + ) + .toSorted((a, b) => b.durationMs - a.durationMs) + .slice(0, limit) + .map((entry) => entry.file); +} + +export function summarizeThreadBenchmark({ file, forks, threads, minGainMs, minGainPct }) { + const forkOk = forks.exitCode === 0; + const threadOk = threads.exitCode === 0; + const gainMs = forks.elapsedMs - threads.elapsedMs; + const gainPct = forks.elapsedMs > 0 ? (gainMs / forks.elapsedMs) * 100 : 0; + const recommended = + forkOk && + threadOk && + gainMs >= minGainMs && + gainPct >= minGainPct && + threads.elapsedMs < forks.elapsedMs; + return { + file, + forks, + threads, + gainMs, + gainPct, + recommended, + }; +} + +function formatMs(ms) { + return `${ms.toFixed(0)}ms`; +} + +function benchmarkFile({ config, file, pool }) { + const startedAt = process.hrtime.bigint(); + const run = spawnSync("pnpm", ["vitest", "run", "--config", config, `--pool=${pool}`, file], { + encoding: "utf8", + env: process.env, + maxBuffer: 20 * 1024 * 1024, + }); + const elapsedMs = Number(process.hrtime.bigint() - startedAt) / 1_000_000; + return { + pool, + exitCode: run.status ?? 1, + elapsedMs, + stderr: run.stderr ?? "", + stdout: run.stdout ?? "", + }; +} + +function buildOutput(results) { + return results.map((result) => ({ + file: result.file, + forksMs: Math.round(result.forks.elapsedMs), + threadsMs: Math.round(result.threads.elapsedMs), + gainMs: Math.round(result.gainMs), + gainPct: Number(result.gainPct.toFixed(1)), + forksExitCode: result.forks.exitCode, + threadsExitCode: result.threads.exitCode, + recommended: result.recommended, + })); +} + +async function main() { + const opts = parseArgs(process.argv.slice(2)); + const behavior = loadTestRunnerBehavior(); + const timings = loadUnitTimingManifest(); + const exclude = getExistingThreadCandidateExclusions(behavior); + const inputFiles = opts.files.length > 0 ? opts.files : Object.keys(timings.files); + const candidates = selectThreadCandidateFiles({ + files: inputFiles, + timings, + exclude, + limit: opts.limit, + minDurationMs: opts.minDurationMs, + includeUnknownDuration: opts.files.length > 0, + }); + + const results = []; + for (const file of candidates) { + const forks = benchmarkFile({ config: opts.config, file, pool: "forks" }); + const threads = benchmarkFile({ config: opts.config, file, pool: "threads" }); + results.push( + summarizeThreadBenchmark({ + file, + forks, + threads, + minGainMs: opts.minGainMs, + minGainPct: opts.minGainPct, + }), + ); + } + + if (opts.json) { + console.log(JSON.stringify(buildOutput(results), null, 2)); + return; + } + + console.log( + `[test-find-thread-candidates] tested=${String(results.length)} minGain=${formatMs( + opts.minGainMs, + )} minGainPct=${String(opts.minGainPct)}%`, + ); + for (const result of results) { + const status = result.recommended + ? "recommend" + : result.forks.exitCode !== 0 + ? "forks-failed" + : result.threads.exitCode !== 0 + ? "threads-failed" + : "skip"; + console.log( + `${status.padEnd(14, " ")} ${result.file} forks=${formatMs(result.forks.elapsedMs)} threads=${formatMs( + result.threads.elapsedMs, + )} gain=${formatMs(result.gainMs)} (${result.gainPct.toFixed(1)}%)`, + ); + if (result.threads.exitCode !== 0) { + const firstErrorLine = + result.threads.stderr + .split(/\r?\n/u) + .find( + (line) => line.includes("Error") || line.includes("TypeError") || line.includes("FAIL"), + ) ?? "threads failed"; + console.log(` ${firstErrorLine}`); + } + } +} + +const isMain = + process.argv[1] && pathToFileURL(path.resolve(process.argv[1])).href === import.meta.url; + +if (isMain) { + try { + await main(); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); + } +} diff --git a/test/fixtures/test-parallel.behavior.json b/test/fixtures/test-parallel.behavior.json index da454dd3870..dafa5001cb8 100644 --- a/test/fixtures/test-parallel.behavior.json +++ b/test/fixtures/test-parallel.behavior.json @@ -313,6 +313,10 @@ "file": "src/channels/plugins/actions/actions.test.ts", "reason": "Terminates cleanly under threads, but not process forks on this host." }, + { + "file": "test/extension-plugin-sdk-boundary.test.ts", + "reason": "Measured ~12% faster under threads than forks on this host while keeping the file green." + }, { "file": "src/hooks/install.test.ts", "reason": "Measured ~14% faster under threads than forks on this host while keeping the file green." @@ -329,10 +333,30 @@ "file": "src/plugin-sdk/subpaths.test.ts", "reason": "Measured ~23% faster under threads than forks on this host while keeping the file green." }, + { + "file": "src/plugin-sdk/channel-import-guardrails.test.ts", + "reason": "Measured ~30% faster under threads than forks on this host while keeping the file green." + }, + { + "file": "src/plugins/contracts/wizard.contract.test.ts", + "reason": "Measured ~9% faster under threads than forks on this host while keeping the file green." + }, { "file": "src/plugins/install.test.ts", "reason": "Measured ~18% faster under threads than forks on this host while keeping the file green." }, + { + "file": "src/plugins/interactive.test.ts", + "reason": "Measured ~9% faster under threads than forks on this host while keeping the file green." + }, + { + "file": "src/config/config.plugin-validation.test.ts", + "reason": "Measured ~11% faster under threads than forks on this host while keeping the file green." + }, + { + "file": "src/config/schema.help.quality.test.ts", + "reason": "Measured ~13% faster under threads than forks on this host while keeping the file green." + }, { "file": "src/cron/isolated-agent.uses-last-non-empty-agent-text-as.test.ts", "reason": "Measured ~15% faster under threads than forks on this host while keeping the file green." @@ -369,22 +393,66 @@ "file": "src/infra/provider-usage.test.ts", "reason": "Measured ~17% faster under threads than forks on this host while keeping the file green." }, + { + "file": "src/infra/provider-usage.auth.normalizes-keys.test.ts", + "reason": "Measured ~12% faster under threads than forks on this host while keeping the file green." + }, + { + "file": "src/infra/provider-usage.auth.plugin.test.ts", + "reason": "Measured ~13% faster under threads than forks on this host while keeping the file green." + }, + { + "file": "src/infra/outbound/targets.test.ts", + "reason": "Measured ~14% faster under threads than forks on this host while keeping the file green." + }, + { + "file": "src/infra/outbound/agent-delivery.test.ts", + "reason": "Measured ~17% faster under threads than forks on this host while keeping the file green." + }, { "file": "src/infra/fs-pinned-write-helper.test.ts", "reason": "Measured ~13% faster under threads than forks on this host while keeping the file green." }, + { + "file": "src/infra/fs-safe.test.ts", + "reason": "Measured ~17% faster under threads than forks on this host while keeping the file green." + }, + { + "file": "src/infra/archive-staging.test.ts", + "reason": "Measured ~12% faster under threads than forks on this host while keeping the file green." + }, { "file": "src/acp/server.startup.test.ts", "reason": "Measured ~11% faster under threads than forks on this host while keeping the file green." }, + { + "file": "src/acp/client.test.ts", + "reason": "Measured ~18% faster under threads than forks on this host while keeping the file green." + }, { "file": "src/daemon/schtasks.startup-fallback.test.ts", "reason": "Measured ~15% faster under threads than forks on this host while keeping the file green." }, + { + "file": "src/media-understanding/resolve.test.ts", + "reason": "Measured ~9% faster under threads than forks on this host while keeping the file green." + }, + { + "file": "src/media-understanding/runner.video.test.ts", + "reason": "Measured ~25% faster under threads than forks on this host while keeping the file green." + }, { "file": "src/secrets/audit.test.ts", "reason": "Measured ~14% faster under threads than forks on this host while keeping the file green." }, + { + "file": "src/secrets/runtime-web-tools.test.ts", + "reason": "Measured ~13% faster under threads than forks on this host while keeping the file green." + }, + { + "file": "src/entry.version-fast-path.test.ts", + "reason": "Measured ~13% faster under threads than forks on this host while keeping the file green." + }, { "file": "src/security/audit.test.ts", "reason": "Measured ~40% faster under threads than forks on this host while keeping the file green." @@ -393,6 +461,10 @@ "file": "ui/src/ui/views/chat.test.ts", "reason": "Measured ~25% faster under threads than forks on this host while keeping the file green." }, + { + "file": "src/cli/program/preaction.test.ts", + "reason": "Measured ~21% faster under threads than forks on this host while keeping the file green." + }, { "file": "src/tts/tts.test.ts", "reason": "Terminates cleanly under threads, but not process forks on this host." diff --git a/test/scripts/test-find-thread-candidates.test.ts b/test/scripts/test-find-thread-candidates.test.ts new file mode 100644 index 00000000000..9f79cd1c4a7 --- /dev/null +++ b/test/scripts/test-find-thread-candidates.test.ts @@ -0,0 +1,116 @@ +import { describe, expect, it } from "vitest"; +import { + getExistingThreadCandidateExclusions, + parseArgs, + selectThreadCandidateFiles, + summarizeThreadBenchmark, +} from "../../scripts/test-find-thread-candidates.mjs"; + +describe("scripts/test-find-thread-candidates parseArgs", () => { + it("parses explicit thresholds and positional files", () => { + expect( + parseArgs([ + "--limit", + "4", + "--min-duration-ms", + "600", + "--min-gain-ms", + "120", + "--min-gain-pct", + "15", + "--json", + "src/a.test.ts", + ]), + ).toEqual({ + config: "vitest.unit.config.ts", + limit: 4, + minDurationMs: 600, + minGainMs: 120, + minGainPct: 15, + json: true, + files: ["src/a.test.ts"], + }); + }); +}); + +describe("scripts/test-find-thread-candidates exclusions", () => { + it("collects already-pinned files across behavior buckets", () => { + expect( + getExistingThreadCandidateExclusions({ + unit: { + isolated: [{ file: "src/a.test.ts" }], + singletonIsolated: [{ file: "src/b.test.ts" }], + threadSingleton: [{ file: "src/c.test.ts" }], + vmForkSingleton: [{ file: "src/d.test.ts" }], + }, + }), + ).toEqual(new Set(["src/a.test.ts", "src/b.test.ts", "src/c.test.ts", "src/d.test.ts"])); + }); +}); + +describe("scripts/test-find-thread-candidates selection", () => { + it("keeps only known, unpinned files above the duration floor", () => { + expect( + selectThreadCandidateFiles({ + files: ["src/a.test.ts", "src/b.test.ts", "src/c.test.ts", "src/d.test.ts"], + timings: { + files: { + "src/a.test.ts": { durationMs: 1100 }, + "src/b.test.ts": { durationMs: 700 }, + "src/c.test.ts": { durationMs: 300 }, + }, + }, + exclude: new Set(["src/b.test.ts"]), + limit: 10, + minDurationMs: 500, + }), + ).toEqual(["src/a.test.ts"]); + }); + + it("allows explicit unknown-duration files when requested", () => { + expect( + selectThreadCandidateFiles({ + files: ["src/a.test.ts", "src/b.test.ts"], + timings: { + files: { + "src/a.test.ts": { durationMs: 700 }, + }, + }, + exclude: new Set(), + limit: 10, + minDurationMs: 500, + includeUnknownDuration: true, + }), + ).toEqual(["src/a.test.ts", "src/b.test.ts"]); + }); +}); + +describe("scripts/test-find-thread-candidates summarizeThreadBenchmark", () => { + it("recommends clear thread wins", () => { + expect( + summarizeThreadBenchmark({ + file: "src/a.test.ts", + forks: { exitCode: 0, elapsedMs: 1000, stderr: "", stdout: "" }, + threads: { exitCode: 0, elapsedMs: 780, stderr: "", stdout: "" }, + minGainMs: 100, + minGainPct: 10, + }), + ).toMatchObject({ + file: "src/a.test.ts", + gainMs: 220, + recommended: true, + }); + }); + + it("rejects thread failures even when the measured wall time is lower", () => { + expect( + summarizeThreadBenchmark({ + file: "src/b.test.ts", + forks: { exitCode: 0, elapsedMs: 1000, stderr: "", stdout: "" }, + threads: { exitCode: 1, elapsedMs: 400, stderr: "TypeError", stdout: "" }, + minGainMs: 100, + minGainPct: 10, + }).recommended, + ).toBe(false); + }); +});