From 8e444ac5a600138eaea403d71a644eeabbd05d0d Mon Sep 17 00:00:00 2001 From: Gustavo Madeira Santana Date: Thu, 16 Apr 2026 23:40:54 -0400 Subject: [PATCH] Tests: add grouped performance report benchmark --- package.json | 2 + scripts/lib/test-group-report.mjs | 486 +++++++++++++++++++++++++ scripts/test-group-report.mjs | 362 ++++++++++++++++++ test/scripts/test-group-report.test.ts | 203 +++++++++++ 4 files changed, 1053 insertions(+) create mode 100644 scripts/lib/test-group-report.mjs create mode 100644 scripts/test-group-report.mjs create mode 100644 test/scripts/test-group-report.test.ts diff --git a/package.json b/package.json index a30418562d6..1aa971c2358 100644 --- a/package.json +++ b/package.json @@ -1346,6 +1346,8 @@ "test:parallels:windows": "bash scripts/e2e/parallels-windows-smoke.sh", "test:perf:budget": "node scripts/test-perf-budget.mjs", "test:perf:changed:bench": "node scripts/bench-test-changed.mjs", + "test:perf:groups": "node scripts/test-group-report.mjs", + "test:perf:groups:compare": "node scripts/test-group-report.mjs --compare", "test:perf:hotspots": "node scripts/test-hotspots.mjs", "test:perf:imports": "OPENCLAW_VITEST_IMPORT_DURATIONS=1 OPENCLAW_VITEST_PRINT_IMPORT_BREAKDOWN=1 node scripts/test-projects.mjs", "test:perf:imports:changed": "OPENCLAW_VITEST_IMPORT_DURATIONS=1 OPENCLAW_VITEST_PRINT_IMPORT_BREAKDOWN=1 node scripts/test-projects.mjs --changed origin/main", diff --git a/scripts/lib/test-group-report.mjs b/scripts/lib/test-group-report.mjs new file mode 100644 index 00000000000..c7f7bc84097 --- /dev/null +++ b/scripts/lib/test-group-report.mjs @@ -0,0 +1,486 @@ +import path from "node:path"; +import { collectVitestFileDurations, normalizeTrackedRepoPath } from "../test-report-utils.mjs"; +import { formatMs } from "./vitest-report-cli-utils.mjs"; + +export function formatBytesAsMb(valueBytes) { + return valueBytes === null || valueBytes === undefined + ? "n/a" + : `${(valueBytes / 1024 / 1024).toFixed(1)}MB`; +} + +export function formatSignedMs(value, digits = 1) { + return `${value > 0 ? "+" : ""}${formatMs(value, digits)}`; +} + +export function formatSignedBytesAsMb(valueBytes) { + return valueBytes === null || valueBytes === undefined + ? "n/a" + : `${valueBytes > 0 ? "+" : ""}${formatBytesAsMb(valueBytes)}`; +} + +export function normalizeConfigLabel(config) { + return config.replace(/^test\/vitest\/vitest\./u, "").replace(/\.config\.ts$/u, ""); +} + +export function resolveTestArea(file) { + const normalized = normalizeTrackedRepoPath(file); + const parts = normalized.split("/"); + if (parts[0] === "extensions" && parts[1]) { + return `extensions/${parts[1]}`; + } + if (parts[0] === "src" && parts[1]) { + return `src/${parts[1]}`; + } + if (parts[0] === "packages" && parts[1]) { + return `packages/${parts[1]}`; + } + if (parts[0] === "apps" && parts[1]) { + return `apps/${parts[1]}`; + } + if (parts[0] === "ui") { + return parts[3] ? `ui/${parts[3]}` : "ui"; + } + if (parts[0] === "test" && parts[1]) { + return `test/${parts[1]}`; + } + return parts[0] || normalized; +} + +export function resolveTestFolder(file, depth = 2) { + const normalized = normalizeTrackedRepoPath(file); + const dir = path.posix.dirname(normalized); + if (dir === ".") { + return normalized; + } + return dir.split("/").slice(0, Math.max(1, depth)).join("/"); +} + +export function resolveGroupKey(file, mode = "area") { + if (mode === "folder") { + return resolveTestFolder(file, 3); + } + if (mode === "top") { + return normalizeTrackedRepoPath(file).split("/")[0] || file; + } + return resolveTestArea(file); +} + +function createCounter(key) { + return { + key, + durationMs: 0, + fileCount: 0, + testCount: 0, + configs: new Set(), + }; +} + +function addFileEntry(target, entry, config) { + target.durationMs += entry.durationMs; + target.fileCount += 1; + target.testCount += entry.testCount; + target.configs.add(config); +} + +function finalizeCounter(counter) { + return { + key: counter.key, + durationMs: counter.durationMs, + fileCount: counter.fileCount, + testCount: counter.testCount, + configs: [...counter.configs].toSorted((left, right) => left.localeCompare(right)), + }; +} + +export function buildGroupedTestReport(params) { + const byGroup = new Map(); + const byConfig = new Map(); + const files = []; + + for (const input of params.reports) { + const config = normalizeConfigLabel(input.config); + const fileEntries = collectVitestFileDurations(input.report, normalizeTrackedRepoPath); + const configCounter = byConfig.get(config) ?? createCounter(config); + byConfig.set(config, configCounter); + + for (const entry of fileEntries) { + const groupKey = resolveGroupKey(entry.file, params.groupBy); + const groupCounter = byGroup.get(groupKey) ?? createCounter(groupKey); + byGroup.set(groupKey, groupCounter); + addFileEntry(groupCounter, entry, config); + addFileEntry(configCounter, entry, config); + files.push({ ...entry, config, group: groupKey }); + } + } + + const sortByDuration = (left, right) => + right.durationMs - left.durationMs || left.key.localeCompare(right.key); + const sortFilesByDuration = (left, right) => + right.durationMs - left.durationMs || left.file.localeCompare(right.file); + + const groups = [...byGroup.values()].map(finalizeCounter).toSorted(sortByDuration); + const configs = [...byConfig.values()].map(finalizeCounter).toSorted(sortByDuration); + const topFiles = files.toSorted(sortFilesByDuration); + const totals = groups.reduce( + (acc, group) => ({ + durationMs: acc.durationMs + group.durationMs, + fileCount: acc.fileCount + group.fileCount, + testCount: acc.testCount + group.testCount, + }), + { durationMs: 0, fileCount: 0, testCount: 0 }, + ); + + return { + generatedAt: new Date().toISOString(), + groupBy: params.groupBy, + totals, + groups, + configs, + topFiles, + }; +} + +function percentDelta(beforeValue, afterValue) { + if (beforeValue === 0) { + return afterValue === 0 ? 0 : null; + } + return ((afterValue - beforeValue) / beforeValue) * 100; +} + +function formatPercent(value) { + if (value === null || value === undefined) { + return "new"; + } + return `${value > 0 ? "+" : ""}${value.toFixed(1)}%`; +} + +function normalizeCounter(item) { + return { + durationMs: item?.durationMs ?? 0, + fileCount: item?.fileCount ?? 0, + testCount: item?.testCount ?? 0, + }; +} + +function compareStatus(beforeItem, afterItem) { + if (beforeItem && afterItem) { + return "changed"; + } + return beforeItem ? "removed" : "added"; +} + +function compareCounters(beforeItems = [], afterItems = []) { + const beforeByKey = new Map(beforeItems.map((item) => [item.key, item])); + const afterByKey = new Map(afterItems.map((item) => [item.key, item])); + const keys = new Set([...beforeByKey.keys(), ...afterByKey.keys()]); + + return [...keys] + .map((key) => { + const beforeItem = beforeByKey.get(key); + const afterItem = afterByKey.get(key); + const before = normalizeCounter(beforeItem); + const after = normalizeCounter(afterItem); + return { + key, + status: compareStatus(beforeItem, afterItem), + before, + after, + delta: { + durationMs: after.durationMs - before.durationMs, + fileCount: after.fileCount - before.fileCount, + testCount: after.testCount - before.testCount, + }, + percent: { + durationMs: percentDelta(before.durationMs, after.durationMs), + }, + }; + }) + .toSorted( + (left, right) => + Math.abs(right.delta.durationMs) - Math.abs(left.delta.durationMs) || + left.key.localeCompare(right.key), + ); +} + +function normalizeFileCounter(item) { + return { + durationMs: item?.durationMs ?? 0, + testCount: item?.testCount ?? 0, + }; +} + +function fileKey(item) { + return `${item.config}\0${item.file}`; +} + +function compareFiles(beforeFiles = [], afterFiles = []) { + const beforeByKey = new Map(beforeFiles.map((item) => [fileKey(item), item])); + const afterByKey = new Map(afterFiles.map((item) => [fileKey(item), item])); + const keys = new Set([...beforeByKey.keys(), ...afterByKey.keys()]); + + return [...keys] + .map((key) => { + const beforeItem = beforeByKey.get(key); + const afterItem = afterByKey.get(key); + const before = normalizeFileCounter(beforeItem); + const after = normalizeFileCounter(afterItem); + const source = afterItem ?? beforeItem; + return { + key, + config: source.config, + file: source.file, + group: source.group, + status: compareStatus(beforeItem, afterItem), + before, + after, + delta: { + durationMs: after.durationMs - before.durationMs, + testCount: after.testCount - before.testCount, + }, + percent: { + durationMs: percentDelta(before.durationMs, after.durationMs), + }, + }; + }) + .toSorted( + (left, right) => + Math.abs(right.delta.durationMs) - Math.abs(left.delta.durationMs) || + left.file.localeCompare(right.file) || + left.config.localeCompare(right.config), + ); +} + +function runKey(run) { + return normalizeConfigLabel(run.config); +} + +function compareOptionalNumber(beforeValue, afterValue) { + if (typeof beforeValue !== "number" || typeof afterValue !== "number") { + return null; + } + return afterValue - beforeValue; +} + +function normalizeRun(run) { + return run + ? { + elapsedMs: typeof run.elapsedMs === "number" ? run.elapsedMs : null, + maxRssBytes: typeof run.maxRssBytes === "number" ? run.maxRssBytes : null, + status: typeof run.status === "number" ? run.status : null, + } + : { + elapsedMs: null, + maxRssBytes: null, + status: null, + }; +} + +function compareRuns(beforeRuns = [], afterRuns = []) { + const beforeByKey = new Map(beforeRuns.map((run) => [runKey(run), run])); + const afterByKey = new Map(afterRuns.map((run) => [runKey(run), run])); + const keys = new Set([...beforeByKey.keys(), ...afterByKey.keys()]); + + return [...keys] + .map((key) => { + const beforeRun = beforeByKey.get(key); + const afterRun = afterByKey.get(key); + const before = normalizeRun(beforeRun); + const after = normalizeRun(afterRun); + return { + key, + status: compareStatus(beforeRun, afterRun), + before, + after, + delta: { + elapsedMs: compareOptionalNumber(before.elapsedMs, after.elapsedMs), + maxRssBytes: compareOptionalNumber(before.maxRssBytes, after.maxRssBytes), + }, + }; + }) + .toSorted((left, right) => { + const leftMagnitude = Math.abs(left.delta.elapsedMs ?? left.delta.maxRssBytes ?? 0); + const rightMagnitude = Math.abs(right.delta.elapsedMs ?? right.delta.maxRssBytes ?? 0); + return rightMagnitude - leftMagnitude || left.key.localeCompare(right.key); + }); +} + +export function buildGroupedTestComparison(params) { + const before = params.before; + const after = params.after; + const beforeTotals = normalizeCounter(before.totals); + const afterTotals = normalizeCounter(after.totals); + const warnings = []; + + if (before.groupBy !== after.groupBy) { + warnings.push(`groupBy differs: before=${before.groupBy} after=${after.groupBy}`); + } + + return { + generatedAt: new Date().toISOString(), + command: "test-group-report:compare", + groupBy: after.groupBy ?? before.groupBy, + warnings, + totals: { + before: beforeTotals, + after: afterTotals, + delta: { + durationMs: afterTotals.durationMs - beforeTotals.durationMs, + fileCount: afterTotals.fileCount - beforeTotals.fileCount, + testCount: afterTotals.testCount - beforeTotals.testCount, + }, + percent: { + durationMs: percentDelta(beforeTotals.durationMs, afterTotals.durationMs), + }, + }, + groups: compareCounters(before.groups, after.groups), + configs: compareCounters(before.configs, after.configs), + files: compareFiles(before.topFiles, after.topFiles), + runs: compareRuns(before.runs, after.runs), + inputs: { + before: params.beforePath ?? null, + after: params.afterPath ?? null, + }, + }; +} + +function formatCountDelta(value) { + return `${value > 0 ? "+" : ""}${value}`; +} + +function formatOptionalMs(value) { + return typeof value === "number" ? formatMs(value) : "n/a"; +} + +function formatOptionalSignedMs(value) { + return typeof value === "number" ? formatSignedMs(value) : "n/a"; +} + +function formatOptionalBytes(value) { + return typeof value === "number" ? formatBytesAsMb(value) : "n/a"; +} + +function formatOptionalSignedBytes(value) { + return typeof value === "number" ? formatSignedBytesAsMb(value) : "n/a"; +} + +function pushChangeRows(lines, entries, options) { + const selected = entries.slice(0, options.limit); + if (selected.length === 0) { + lines.push(" (none)"); + return; + } + + for (const [index, entry] of selected.entries()) { + lines.push( + `${String(index + 1).padStart(2, " ")}. ${formatSignedMs(entry.delta.durationMs).padStart(11, " ")} (${formatPercent(entry.percent.durationMs).padStart(7, " ")}) | before=${formatMs(entry.before.durationMs).padStart(10, " ")} after=${formatMs(entry.after.durationMs).padStart(10, " ")} | files=${formatCountDelta(entry.delta.fileCount ?? 0).padStart(4, " ")} tests=${formatCountDelta(entry.delta.testCount ?? 0).padStart(5, " ")} | ${entry.key}`, + ); + } +} + +function pushFileChangeRows(lines, entries, options) { + const selected = entries.slice(0, options.limit); + if (selected.length === 0) { + lines.push(" (none)"); + return; + } + + for (const [index, entry] of selected.entries()) { + lines.push( + `${String(index + 1).padStart(2, " ")}. ${formatSignedMs(entry.delta.durationMs).padStart(11, " ")} (${formatPercent(entry.percent.durationMs).padStart(7, " ")}) | before=${formatMs(entry.before.durationMs).padStart(10, " ")} after=${formatMs(entry.after.durationMs).padStart(10, " ")} | tests=${formatCountDelta(entry.delta.testCount).padStart(4, " ")} | ${entry.config} | ${entry.file}`, + ); + } +} + +export function renderGroupedTestComparison(comparison, options = {}) { + const limit = options.limit ?? 25; + const topFiles = options.topFiles ?? 25; + const groupRegressions = comparison.groups.filter((entry) => entry.delta.durationMs > 0); + const groupGains = comparison.groups.filter((entry) => entry.delta.durationMs < 0); + const fileRegressions = comparison.files.filter((entry) => entry.delta.durationMs > 0); + const fileGains = comparison.files.filter((entry) => entry.delta.durationMs < 0); + const addedFiles = comparison.files.filter((entry) => entry.status === "added").length; + const removedFiles = comparison.files.filter((entry) => entry.status === "removed").length; + const lines = [ + `[test-group-report:compare] groupBy=${comparison.groupBy} file-sum=${formatMs(comparison.totals.before.durationMs)} -> ${formatMs(comparison.totals.after.durationMs)} (${formatSignedMs(comparison.totals.delta.durationMs)}, ${formatPercent(comparison.totals.percent.durationMs)}) files=${comparison.totals.before.fileCount}->${comparison.totals.after.fileCount} (${formatCountDelta(comparison.totals.delta.fileCount)}) tests=${comparison.totals.before.testCount}->${comparison.totals.after.testCount} (${formatCountDelta(comparison.totals.delta.testCount)}) addedFiles=${addedFiles} removedFiles=${removedFiles}`, + ]; + + for (const warning of comparison.warnings) { + lines.push(`[test-group-report:compare] warning: ${warning}`); + } + + lines.push( + "", + `Top group regressions (${Math.min(limit, groupRegressions.length)} of ${groupRegressions.length})`, + ); + pushChangeRows(lines, groupRegressions, { limit }); + + lines.push("", `Top group gains (${Math.min(limit, groupGains.length)} of ${groupGains.length})`); + pushChangeRows(lines, groupGains, { limit }); + + lines.push( + "", + `Config duration deltas (${Math.min(limit, comparison.configs.length)} of ${comparison.configs.length})`, + ); + pushChangeRows(lines, comparison.configs, { limit }); + + if (comparison.runs.length > 0) { + lines.push( + "", + `Config wall/RSS deltas (${Math.min(limit, comparison.runs.length)} of ${comparison.runs.length})`, + ); + for (const [index, run] of comparison.runs.slice(0, limit).entries()) { + lines.push( + `${String(index + 1).padStart(2, " ")}. wall=${formatOptionalSignedMs(run.delta.elapsedMs).padStart(11, " ")} before=${formatOptionalMs(run.before.elapsedMs).padStart(10, " ")} after=${formatOptionalMs(run.after.elapsedMs).padStart(10, " ")} | rss=${formatOptionalSignedBytes(run.delta.maxRssBytes).padStart(10, " ")} before=${formatOptionalBytes(run.before.maxRssBytes).padStart(9, " ")} after=${formatOptionalBytes(run.after.maxRssBytes).padStart(9, " ")} | status=${run.before.status ?? "n/a"}->${run.after.status ?? "n/a"} | ${run.key}`, + ); + } + } + + lines.push( + "", + `Top file regressions (${Math.min(topFiles, fileRegressions.length)} of ${fileRegressions.length})`, + ); + pushFileChangeRows(lines, fileRegressions, { limit: topFiles }); + + lines.push("", `Top file gains (${Math.min(topFiles, fileGains.length)} of ${fileGains.length})`); + pushFileChangeRows(lines, fileGains, { limit: topFiles }); + + return lines.join("\n"); +} + +export function renderGroupedTestReport(report, options = {}) { + const limit = options.limit ?? 25; + const topFiles = options.topFiles ?? 25; + const lines = [ + `[test-group-report] groupBy=${report.groupBy} files=${report.totals.fileCount} tests=${report.totals.testCount} file-sum=${formatMs(report.totals.durationMs)}`, + "", + `Top groups (${Math.min(limit, report.groups.length)} of ${report.groups.length})`, + ]; + + for (const [index, group] of report.groups.slice(0, limit).entries()) { + lines.push( + `${String(index + 1).padStart(2, " ")}. ${formatMs(group.durationMs).padStart(10, " ")} | files=${String(group.fileCount).padStart(4, " ")} | tests=${String(group.testCount).padStart(5, " ")} | ${group.key}`, + ); + } + + lines.push( + "", + `Top configs (${Math.min(limit, report.configs.length)} of ${report.configs.length})`, + ); + for (const [index, config] of report.configs.slice(0, limit).entries()) { + lines.push( + `${String(index + 1).padStart(2, " ")}. ${formatMs(config.durationMs).padStart(10, " ")} | files=${String(config.fileCount).padStart(4, " ")} | tests=${String(config.testCount).padStart(5, " ")} | ${config.key}`, + ); + } + + lines.push( + "", + `Top files (${Math.min(topFiles, report.topFiles.length)} of ${report.topFiles.length})`, + ); + for (const [index, file] of report.topFiles.slice(0, topFiles).entries()) { + lines.push( + `${String(index + 1).padStart(2, " ")}. ${formatMs(file.durationMs).padStart(10, " ")} | tests=${String(file.testCount).padStart(4, " ")} | ${file.config} | ${file.file}`, + ); + } + + return lines.join("\n"); +} diff --git a/scripts/test-group-report.mjs b/scripts/test-group-report.mjs new file mode 100644 index 00000000000..ee9161d068b --- /dev/null +++ b/scripts/test-group-report.mjs @@ -0,0 +1,362 @@ +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { + buildGroupedTestComparison, + buildGroupedTestReport, + formatBytesAsMb, + normalizeConfigLabel, + renderGroupedTestComparison, + renderGroupedTestReport, +} from "./lib/test-group-report.mjs"; +import { formatMs } from "./lib/vitest-report-cli-utils.mjs"; +import { resolveVitestNodeArgs } from "./run-vitest.mjs"; +import { buildFullSuiteVitestRunPlans } from "./test-projects.test-support.mjs"; + +const DEFAULT_OUTPUT = ".artifacts/test-perf/group-report.json"; +const DEFAULT_COMPARE_OUTPUT = ".artifacts/test-perf/group-report-compare.json"; + +function usage() { + return [ + "Usage: node scripts/test-group-report.mjs [options] [-- ]", + "", + "Build a grouped Vitest duration report from one or more JSON reports.", + "", + "Options:", + " --config Vitest config to run (repeatable)", + " --compare ", + " Compare two grouped report JSON files", + " --report Existing Vitest JSON report to read (repeatable)", + " --full-suite Run every full-suite leaf Vitest config serially", + " --group-by area | folder | top (default: area)", + " --output JSON report path (default: .artifacts/test-perf/group-report.json)", + " --limit Number of groups/configs to print (default: 25)", + " --top-files Number of files to print (default: 25)", + " --allow-failures Write a report even when a Vitest run exits non-zero", + " --no-rss Skip macOS max RSS measurement", + " --help Show this help", + "", + "Examples:", + " pnpm test:perf:groups --config test/vitest/vitest.unit-fast.config.ts", + " pnpm test:perf:groups --full-suite --allow-failures", + " pnpm test:perf:groups:compare .artifacts/test-perf/baseline-before.json .artifacts/test-perf/after-first-fix.json", + ].join("\n"); +} + +function parsePositiveInt(value, fallback) { + const parsed = Number.parseInt(value ?? "", 10); + return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback; +} + +export function parseTestGroupReportArgs(argv) { + const args = { + allowFailures: false, + compare: null, + configs: [], + fullSuite: false, + groupBy: "area", + limit: 25, + output: null, + reports: [], + rss: process.platform === "darwin", + topFiles: 25, + vitestArgs: [], + }; + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg === "--") { + args.vitestArgs = argv.slice(index + 1); + break; + } + if (arg === "--help") { + args.help = true; + continue; + } + if (arg === "--allow-failures") { + args.allowFailures = true; + continue; + } + if (arg === "--full-suite") { + args.fullSuite = true; + continue; + } + if (arg === "--no-rss") { + args.rss = false; + continue; + } + if (arg === "--config") { + args.configs.push(argv[index + 1] ?? ""); + index += 1; + continue; + } + if (arg === "--compare") { + args.compare = { + before: argv[index + 1] ?? "", + after: argv[index + 2] ?? "", + }; + index += 2; + continue; + } + if (arg === "--report") { + args.reports.push(argv[index + 1] ?? ""); + index += 1; + continue; + } + if (arg === "--group-by") { + args.groupBy = argv[index + 1] ?? args.groupBy; + index += 1; + continue; + } + if (arg === "--output") { + args.output = argv[index + 1] ?? args.output; + index += 1; + continue; + } + if (arg === "--limit") { + args.limit = parsePositiveInt(argv[index + 1], args.limit); + index += 1; + continue; + } + if (arg === "--top-files") { + args.topFiles = parsePositiveInt(argv[index + 1], args.topFiles); + index += 1; + continue; + } + throw new Error(`Unknown option: ${arg}`); + } + + if (!["area", "folder", "top"].includes(args.groupBy)) { + throw new Error(`Unsupported --group-by value: ${args.groupBy}`); + } + if (args.compare && (!args.compare.before || !args.compare.after)) { + throw new Error("--compare requires before and after report paths"); + } + if ( + args.compare && + (args.configs.length > 0 || + args.fullSuite || + args.reports.length > 0 || + args.vitestArgs.length > 0) + ) { + throw new Error("--compare cannot be combined with test run or report input options"); + } + + return args; +} + +function sanitizePathSegment(value) { + return ( + value + .replace(/[^A-Za-z0-9._-]+/gu, "-") + .replace(/^-+|-+$/gu, "") + .slice(0, 180) || "report" + ); +} + +function parseMaxRssBytes(output) { + const match = output.match(/(\d+)\s+maximum resident set size/u); + return match ? Number.parseInt(match[1], 10) : null; +} + +function runVitestJsonReport(params) { + fs.mkdirSync(path.dirname(params.reportPath), { recursive: true }); + fs.mkdirSync(path.dirname(params.logPath), { recursive: true }); + const command = [ + process.execPath, + "scripts/run-vitest.mjs", + "run", + "--config", + params.config, + "--reporter=json", + "--outputFile", + params.reportPath, + ...params.vitestArgs, + ]; + const startedAt = process.hrtime.bigint(); + const result = spawnSync( + params.rss ? "/usr/bin/time" : command[0], + params.rss ? ["-l", ...command] : command.slice(1), + { + cwd: process.cwd(), + encoding: "utf8", + env: { + ...process.env, + NODE_OPTIONS: [ + process.env.NODE_OPTIONS?.trim(), + ...resolveVitestNodeArgs(process.env).filter((arg) => arg !== "--no-maglev"), + ] + .filter(Boolean) + .join(" "), + }, + maxBuffer: 1024 * 1024 * 64, + }, + ); + const elapsedMs = Number.parseFloat(String(process.hrtime.bigint() - startedAt)) / 1_000_000; + const output = `${result.stdout ?? ""}${result.stderr ?? ""}`; + fs.writeFileSync(params.logPath, output, "utf8"); + return { + config: params.config, + elapsedMs, + logPath: params.logPath, + maxRssBytes: params.rss ? parseMaxRssBytes(output) : null, + reportPath: params.reportPath, + status: result.status ?? 1, + }; +} + +function readReportInput(entry) { + return { + config: entry.config, + report: JSON.parse(fs.readFileSync(entry.reportPath, "utf8")), + reportPath: entry.reportPath, + run: entry.run ?? null, + }; +} + +function readGroupedReport(reportPath) { + return JSON.parse(fs.readFileSync(reportPath, "utf8")); +} + +function resolveConfigs(args) { + if (args.reports.length > 0) { + return []; + } + if (args.fullSuite) { + return buildFullSuiteVitestRunPlans([], process.cwd()).map((plan) => plan.config); + } + return args.configs.length > 0 ? args.configs : ["test/vitest/vitest.unit.config.ts"]; +} + +function printRunLine(run) { + const label = normalizeConfigLabel(run.config); + console.log( + `[test-group-report] ${label} status=${run.status} wall=${formatMs(run.elapsedMs)} rss=${formatBytesAsMb(run.maxRssBytes)} report=${run.reportPath}`, + ); +} + +async function main() { + const args = parseTestGroupReportArgs(process.argv.slice(2)); + if (args.help) { + console.log(usage()); + return; + } + + const output = path.resolve( + args.output ?? (args.compare ? DEFAULT_COMPARE_OUTPUT : DEFAULT_OUTPUT), + ); + + if (args.compare) { + const beforePath = path.resolve(args.compare.before); + const afterPath = path.resolve(args.compare.after); + const comparison = buildGroupedTestComparison({ + before: readGroupedReport(beforePath), + after: readGroupedReport(afterPath), + beforePath, + afterPath, + }); + + fs.mkdirSync(path.dirname(output), { recursive: true }); + fs.writeFileSync(output, `${JSON.stringify(comparison, null, 2)}\n`, "utf8"); + console.log( + renderGroupedTestComparison(comparison, { limit: args.limit, topFiles: args.topFiles }), + ); + console.log(`[test-group-report:compare] wrote ${path.relative(process.cwd(), output)}`); + return; + } + + const reportDir = path.join(path.dirname(output), "vitest-json"); + const logDir = path.join(path.dirname(output), "logs"); + const runEntries = []; + const configs = resolveConfigs(args); + let failed = false; + let exitCode = 0; + + for (const reportPath of args.reports) { + runEntries.push({ + config: path.basename(reportPath).replace(/\.json$/u, ""), + reportPath: path.resolve(reportPath), + }); + } + + for (const config of configs) { + const slug = sanitizePathSegment(normalizeConfigLabel(config)); + const run = runVitestJsonReport({ + config, + logPath: path.join(logDir, `${slug}.log`), + reportPath: path.join(reportDir, `${slug}.json`), + rss: args.rss, + vitestArgs: args.vitestArgs, + }); + printRunLine(run); + if (run.status !== 0) { + failed = true; + if (!fs.existsSync(run.reportPath)) { + console.error( + `[test-group-report] missing JSON report for failed config; see ${run.logPath}`, + ); + if (!args.allowFailures) { + exitCode = run.status; + break; + } + continue; + } + console.error( + `[test-group-report] config failed; keeping partial report from ${run.reportPath}`, + ); + if (!args.allowFailures) { + exitCode = run.status; + break; + } + } + runEntries.push({ config, reportPath: run.reportPath, run }); + } + + if (exitCode !== 0) { + process.exit(exitCode); + } + + const reportInputs = runEntries + .filter((entry) => fs.existsSync(entry.reportPath)) + .map(readReportInput); + const report = buildGroupedTestReport({ + groupBy: args.groupBy, + reports: reportInputs, + }); + const envelope = { + ...report, + command: "test-group-report", + failed, + runs: reportInputs.map((entry) => entry.run).filter(Boolean), + system: { + node: process.version, + platform: process.platform, + arch: process.arch, + cpuCount: os.availableParallelism?.() ?? os.cpus().length, + totalMemoryBytes: os.totalmem(), + }, + }; + + fs.mkdirSync(path.dirname(output), { recursive: true }); + fs.writeFileSync(output, `${JSON.stringify(envelope, null, 2)}\n`, "utf8"); + console.log(renderGroupedTestReport(report, { limit: args.limit, topFiles: args.topFiles })); + console.log(`[test-group-report] wrote ${path.relative(process.cwd(), output)}`); + + if (failed && !args.allowFailures) { + process.exit(1); + } +} + +const isMain = + typeof process.argv[1] === "string" && + process.argv[1].length > 0 && + import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href; + +if (isMain) { + main().catch((error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); + }); +} diff --git a/test/scripts/test-group-report.test.ts b/test/scripts/test-group-report.test.ts new file mode 100644 index 00000000000..9a5f1088097 --- /dev/null +++ b/test/scripts/test-group-report.test.ts @@ -0,0 +1,203 @@ +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { + buildGroupedTestComparison, + buildGroupedTestReport, + renderGroupedTestComparison, + resolveGroupKey, + resolveTestArea, +} from "../../scripts/lib/test-group-report.mjs"; +import { parseTestGroupReportArgs } from "../../scripts/test-group-report.mjs"; + +describe("scripts/test-group-report grouping", () => { + it("groups repo files by stable product area", () => { + expect(resolveTestArea("extensions/discord/src/send.test.ts")).toBe("extensions/discord"); + expect(resolveTestArea("src/commands/agent.test.ts")).toBe("src/commands"); + expect(resolveTestArea("packages/plugin-sdk/src/index.test.ts")).toBe("packages/plugin-sdk"); + expect(resolveTestArea("ui/src/ui/views/chat.test.ts")).toBe("ui/views"); + expect(resolveTestArea("test/scripts/test-group-report.test.ts")).toBe("test/scripts"); + }); + + it("supports folder and top-level grouping modes", () => { + expect(resolveGroupKey("src/commands/agent.test.ts", "folder")).toBe("src/commands"); + expect(resolveGroupKey("extensions/browser/src/browser/pw.test.ts", "folder")).toBe( + "extensions/browser/src", + ); + expect(resolveGroupKey("extensions/browser/src/browser/pw.test.ts", "top")).toBe("extensions"); + }); +}); + +describe("scripts/test-group-report aggregation", () => { + it("aggregates file durations by group and config", () => { + const report = buildGroupedTestReport({ + groupBy: "area", + reports: [ + { + config: "test/vitest/vitest.commands.config.ts", + report: { + testResults: [ + { + name: path.join(process.cwd(), "src", "commands", "agent.test.ts"), + startTime: 100, + endTime: 700, + assertionResults: [{}, {}], + }, + { + name: path.join(process.cwd(), "extensions", "discord", "src", "send.test.ts"), + startTime: 200, + endTime: 450, + assertionResults: [{}], + }, + ], + }, + }, + ], + }); + + expect(report.totals).toEqual({ durationMs: 850, fileCount: 2, testCount: 3 }); + expect(report.groups.map((group) => [group.key, group.durationMs])).toEqual([ + ["src/commands", 600], + ["extensions/discord", 250], + ]); + expect(report.configs).toMatchObject([ + { + key: "commands", + durationMs: 850, + fileCount: 2, + testCount: 3, + }, + ]); + }); +}); + +describe("scripts/test-group-report comparison", () => { + it("compares grouped reports by group, file, config, and run metrics", () => { + const comparison = buildGroupedTestComparison({ + beforePath: "before.json", + afterPath: "after.json", + before: { + groupBy: "area", + totals: { durationMs: 1000, fileCount: 2, testCount: 4 }, + groups: [ + { key: "src/commands", durationMs: 700, fileCount: 1, testCount: 2 }, + { key: "extensions/discord", durationMs: 300, fileCount: 1, testCount: 2 }, + ], + configs: [{ key: "commands", durationMs: 1000, fileCount: 2, testCount: 4 }], + topFiles: [ + { + config: "commands", + file: "src/commands/agent.test.ts", + group: "src/commands", + durationMs: 700, + testCount: 2, + }, + { + config: "commands", + file: "extensions/discord/src/send.test.ts", + group: "extensions/discord", + durationMs: 300, + testCount: 2, + }, + ], + runs: [ + { + config: "test/vitest/vitest.commands.config.ts", + elapsedMs: 2000, + maxRssBytes: 1024 * 1024 * 100, + status: 0, + }, + ], + }, + after: { + groupBy: "area", + totals: { durationMs: 900, fileCount: 2, testCount: 5 }, + groups: [{ key: "src/commands", durationMs: 900, fileCount: 2, testCount: 5 }], + configs: [{ key: "commands", durationMs: 900, fileCount: 2, testCount: 5 }], + topFiles: [ + { + config: "commands", + file: "src/commands/agent.test.ts", + group: "src/commands", + durationMs: 800, + testCount: 3, + }, + { + config: "commands", + file: "src/commands/new.test.ts", + group: "src/commands", + durationMs: 100, + testCount: 2, + }, + ], + runs: [ + { + config: "test/vitest/vitest.commands.config.ts", + elapsedMs: 1800, + maxRssBytes: 1024 * 1024 * 80, + status: 0, + }, + ], + }, + }); + + expect(comparison.totals.delta).toEqual({ durationMs: -100, fileCount: 0, testCount: 1 }); + expect(comparison.groups.find((group) => group.key === "src/commands")).toMatchObject({ + delta: { durationMs: 200, fileCount: 1, testCount: 3 }, + }); + expect( + comparison.files.find((file) => file.file === "extensions/discord/src/send.test.ts"), + ).toMatchObject({ + status: "removed", + delta: { durationMs: -300, testCount: -2 }, + }); + expect(comparison.runs[0]).toMatchObject({ + key: "commands", + delta: { elapsedMs: -200, maxRssBytes: -1024 * 1024 * 20 }, + }); + + expect(renderGroupedTestComparison(comparison, { limit: 2, topFiles: 2 })).toContain( + "Top group regressions", + ); + }); +}); + +describe("scripts/test-group-report arg parsing", () => { + it("parses repeatable config and passthrough args", () => { + expect( + parseTestGroupReportArgs([ + "--config", + "a.ts", + "--config", + "b.ts", + "--group-by", + "folder", + "--allow-failures", + "--", + "--maxWorkers=1", + ]), + ).toMatchObject({ + allowFailures: true, + configs: ["a.ts", "b.ts"], + groupBy: "folder", + vitestArgs: ["--maxWorkers=1"], + }); + }); + + it("parses compare mode", () => { + expect( + parseTestGroupReportArgs([ + "--compare", + "before.json", + "after.json", + "--limit", + "5", + "--top-files", + "3", + ]), + ).toMatchObject({ + compare: { before: "before.json", after: "after.json" }, + limit: 5, + topFiles: 3, + }); + }); +});