mirror of
https://github.com/openclaw/openclaw.git
synced 2026-05-06 05:50:43 +00:00
Tests: add grouped performance report benchmark
This commit is contained in:
Gustavo Madeira Santana
@@ -1346,6 +1346,8 @@
|
||||
"test:parallels:windows": "bash scripts/e2e/parallels-windows-smoke.sh",
|
||||
"test:perf:budget": "node scripts/test-perf-budget.mjs",
|
||||
"test:perf:changed:bench": "node scripts/bench-test-changed.mjs",
|
||||
"test:perf:groups": "node scripts/test-group-report.mjs",
|
||||
"test:perf:groups:compare": "node scripts/test-group-report.mjs --compare",
|
||||
"test:perf:hotspots": "node scripts/test-hotspots.mjs",
|
||||
"test:perf:imports": "OPENCLAW_VITEST_IMPORT_DURATIONS=1 OPENCLAW_VITEST_PRINT_IMPORT_BREAKDOWN=1 node scripts/test-projects.mjs",
|
||||
"test:perf:imports:changed": "OPENCLAW_VITEST_IMPORT_DURATIONS=1 OPENCLAW_VITEST_PRINT_IMPORT_BREAKDOWN=1 node scripts/test-projects.mjs --changed origin/main",
|
||||
|
||||
486
scripts/lib/test-group-report.mjs
Normal file
486
scripts/lib/test-group-report.mjs
Normal file
@@ -0,0 +1,486 @@
|
||||
import path from "node:path";
|
||||
import { collectVitestFileDurations, normalizeTrackedRepoPath } from "../test-report-utils.mjs";
|
||||
import { formatMs } from "./vitest-report-cli-utils.mjs";
|
||||
|
||||
export function formatBytesAsMb(valueBytes) {
|
||||
return valueBytes === null || valueBytes === undefined
|
||||
? "n/a"
|
||||
: `${(valueBytes / 1024 / 1024).toFixed(1)}MB`;
|
||||
}
|
||||
|
||||
export function formatSignedMs(value, digits = 1) {
|
||||
return `${value > 0 ? "+" : ""}${formatMs(value, digits)}`;
|
||||
}
|
||||
|
||||
export function formatSignedBytesAsMb(valueBytes) {
|
||||
return valueBytes === null || valueBytes === undefined
|
||||
? "n/a"
|
||||
: `${valueBytes > 0 ? "+" : ""}${formatBytesAsMb(valueBytes)}`;
|
||||
}
|
||||
|
||||
export function normalizeConfigLabel(config) {
|
||||
return config.replace(/^test\/vitest\/vitest\./u, "").replace(/\.config\.ts$/u, "");
|
||||
}
|
||||
|
||||
export function resolveTestArea(file) {
|
||||
const normalized = normalizeTrackedRepoPath(file);
|
||||
const parts = normalized.split("/");
|
||||
if (parts[0] === "extensions" && parts[1]) {
|
||||
return `extensions/${parts[1]}`;
|
||||
}
|
||||
if (parts[0] === "src" && parts[1]) {
|
||||
return `src/${parts[1]}`;
|
||||
}
|
||||
if (parts[0] === "packages" && parts[1]) {
|
||||
return `packages/${parts[1]}`;
|
||||
}
|
||||
if (parts[0] === "apps" && parts[1]) {
|
||||
return `apps/${parts[1]}`;
|
||||
}
|
||||
if (parts[0] === "ui") {
|
||||
return parts[3] ? `ui/${parts[3]}` : "ui";
|
||||
}
|
||||
if (parts[0] === "test" && parts[1]) {
|
||||
return `test/${parts[1]}`;
|
||||
}
|
||||
return parts[0] || normalized;
|
||||
}
|
||||
|
||||
export function resolveTestFolder(file, depth = 2) {
|
||||
const normalized = normalizeTrackedRepoPath(file);
|
||||
const dir = path.posix.dirname(normalized);
|
||||
if (dir === ".") {
|
||||
return normalized;
|
||||
}
|
||||
return dir.split("/").slice(0, Math.max(1, depth)).join("/");
|
||||
}
|
||||
|
||||
export function resolveGroupKey(file, mode = "area") {
|
||||
if (mode === "folder") {
|
||||
return resolveTestFolder(file, 3);
|
||||
}
|
||||
if (mode === "top") {
|
||||
return normalizeTrackedRepoPath(file).split("/")[0] || file;
|
||||
}
|
||||
return resolveTestArea(file);
|
||||
}
|
||||
|
||||
function createCounter(key) {
|
||||
return {
|
||||
key,
|
||||
durationMs: 0,
|
||||
fileCount: 0,
|
||||
testCount: 0,
|
||||
configs: new Set(),
|
||||
};
|
||||
}
|
||||
|
||||
function addFileEntry(target, entry, config) {
|
||||
target.durationMs += entry.durationMs;
|
||||
target.fileCount += 1;
|
||||
target.testCount += entry.testCount;
|
||||
target.configs.add(config);
|
||||
}
|
||||
|
||||
function finalizeCounter(counter) {
|
||||
return {
|
||||
key: counter.key,
|
||||
durationMs: counter.durationMs,
|
||||
fileCount: counter.fileCount,
|
||||
testCount: counter.testCount,
|
||||
configs: [...counter.configs].toSorted((left, right) => left.localeCompare(right)),
|
||||
};
|
||||
}
|
||||
|
||||
export function buildGroupedTestReport(params) {
|
||||
const byGroup = new Map();
|
||||
const byConfig = new Map();
|
||||
const files = [];
|
||||
|
||||
for (const input of params.reports) {
|
||||
const config = normalizeConfigLabel(input.config);
|
||||
const fileEntries = collectVitestFileDurations(input.report, normalizeTrackedRepoPath);
|
||||
const configCounter = byConfig.get(config) ?? createCounter(config);
|
||||
byConfig.set(config, configCounter);
|
||||
|
||||
for (const entry of fileEntries) {
|
||||
const groupKey = resolveGroupKey(entry.file, params.groupBy);
|
||||
const groupCounter = byGroup.get(groupKey) ?? createCounter(groupKey);
|
||||
byGroup.set(groupKey, groupCounter);
|
||||
addFileEntry(groupCounter, entry, config);
|
||||
addFileEntry(configCounter, entry, config);
|
||||
files.push({ ...entry, config, group: groupKey });
|
||||
}
|
||||
}
|
||||
|
||||
const sortByDuration = (left, right) =>
|
||||
right.durationMs - left.durationMs || left.key.localeCompare(right.key);
|
||||
const sortFilesByDuration = (left, right) =>
|
||||
right.durationMs - left.durationMs || left.file.localeCompare(right.file);
|
||||
|
||||
const groups = [...byGroup.values()].map(finalizeCounter).toSorted(sortByDuration);
|
||||
const configs = [...byConfig.values()].map(finalizeCounter).toSorted(sortByDuration);
|
||||
const topFiles = files.toSorted(sortFilesByDuration);
|
||||
const totals = groups.reduce(
|
||||
(acc, group) => ({
|
||||
durationMs: acc.durationMs + group.durationMs,
|
||||
fileCount: acc.fileCount + group.fileCount,
|
||||
testCount: acc.testCount + group.testCount,
|
||||
}),
|
||||
{ durationMs: 0, fileCount: 0, testCount: 0 },
|
||||
);
|
||||
|
||||
return {
|
||||
generatedAt: new Date().toISOString(),
|
||||
groupBy: params.groupBy,
|
||||
totals,
|
||||
groups,
|
||||
configs,
|
||||
topFiles,
|
||||
};
|
||||
}
|
||||
|
||||
function percentDelta(beforeValue, afterValue) {
|
||||
if (beforeValue === 0) {
|
||||
return afterValue === 0 ? 0 : null;
|
||||
}
|
||||
return ((afterValue - beforeValue) / beforeValue) * 100;
|
||||
}
|
||||
|
||||
function formatPercent(value) {
|
||||
if (value === null || value === undefined) {
|
||||
return "new";
|
||||
}
|
||||
return `${value > 0 ? "+" : ""}${value.toFixed(1)}%`;
|
||||
}
|
||||
|
||||
function normalizeCounter(item) {
|
||||
return {
|
||||
durationMs: item?.durationMs ?? 0,
|
||||
fileCount: item?.fileCount ?? 0,
|
||||
testCount: item?.testCount ?? 0,
|
||||
};
|
||||
}
|
||||
|
||||
function compareStatus(beforeItem, afterItem) {
|
||||
if (beforeItem && afterItem) {
|
||||
return "changed";
|
||||
}
|
||||
return beforeItem ? "removed" : "added";
|
||||
}
|
||||
|
||||
function compareCounters(beforeItems = [], afterItems = []) {
|
||||
const beforeByKey = new Map(beforeItems.map((item) => [item.key, item]));
|
||||
const afterByKey = new Map(afterItems.map((item) => [item.key, item]));
|
||||
const keys = new Set([...beforeByKey.keys(), ...afterByKey.keys()]);
|
||||
|
||||
return [...keys]
|
||||
.map((key) => {
|
||||
const beforeItem = beforeByKey.get(key);
|
||||
const afterItem = afterByKey.get(key);
|
||||
const before = normalizeCounter(beforeItem);
|
||||
const after = normalizeCounter(afterItem);
|
||||
return {
|
||||
key,
|
||||
status: compareStatus(beforeItem, afterItem),
|
||||
before,
|
||||
after,
|
||||
delta: {
|
||||
durationMs: after.durationMs - before.durationMs,
|
||||
fileCount: after.fileCount - before.fileCount,
|
||||
testCount: after.testCount - before.testCount,
|
||||
},
|
||||
percent: {
|
||||
durationMs: percentDelta(before.durationMs, after.durationMs),
|
||||
},
|
||||
};
|
||||
})
|
||||
.toSorted(
|
||||
(left, right) =>
|
||||
Math.abs(right.delta.durationMs) - Math.abs(left.delta.durationMs) ||
|
||||
left.key.localeCompare(right.key),
|
||||
);
|
||||
}
|
||||
|
||||
function normalizeFileCounter(item) {
|
||||
return {
|
||||
durationMs: item?.durationMs ?? 0,
|
||||
testCount: item?.testCount ?? 0,
|
||||
};
|
||||
}
|
||||
|
||||
function fileKey(item) {
|
||||
return `${item.config}\0${item.file}`;
|
||||
}
|
||||
|
||||
function compareFiles(beforeFiles = [], afterFiles = []) {
|
||||
const beforeByKey = new Map(beforeFiles.map((item) => [fileKey(item), item]));
|
||||
const afterByKey = new Map(afterFiles.map((item) => [fileKey(item), item]));
|
||||
const keys = new Set([...beforeByKey.keys(), ...afterByKey.keys()]);
|
||||
|
||||
return [...keys]
|
||||
.map((key) => {
|
||||
const beforeItem = beforeByKey.get(key);
|
||||
const afterItem = afterByKey.get(key);
|
||||
const before = normalizeFileCounter(beforeItem);
|
||||
const after = normalizeFileCounter(afterItem);
|
||||
const source = afterItem ?? beforeItem;
|
||||
return {
|
||||
key,
|
||||
config: source.config,
|
||||
file: source.file,
|
||||
group: source.group,
|
||||
status: compareStatus(beforeItem, afterItem),
|
||||
before,
|
||||
after,
|
||||
delta: {
|
||||
durationMs: after.durationMs - before.durationMs,
|
||||
testCount: after.testCount - before.testCount,
|
||||
},
|
||||
percent: {
|
||||
durationMs: percentDelta(before.durationMs, after.durationMs),
|
||||
},
|
||||
};
|
||||
})
|
||||
.toSorted(
|
||||
(left, right) =>
|
||||
Math.abs(right.delta.durationMs) - Math.abs(left.delta.durationMs) ||
|
||||
left.file.localeCompare(right.file) ||
|
||||
left.config.localeCompare(right.config),
|
||||
);
|
||||
}
|
||||
|
||||
function runKey(run) {
|
||||
return normalizeConfigLabel(run.config);
|
||||
}
|
||||
|
||||
function compareOptionalNumber(beforeValue, afterValue) {
|
||||
if (typeof beforeValue !== "number" || typeof afterValue !== "number") {
|
||||
return null;
|
||||
}
|
||||
return afterValue - beforeValue;
|
||||
}
|
||||
|
||||
function normalizeRun(run) {
|
||||
return run
|
||||
? {
|
||||
elapsedMs: typeof run.elapsedMs === "number" ? run.elapsedMs : null,
|
||||
maxRssBytes: typeof run.maxRssBytes === "number" ? run.maxRssBytes : null,
|
||||
status: typeof run.status === "number" ? run.status : null,
|
||||
}
|
||||
: {
|
||||
elapsedMs: null,
|
||||
maxRssBytes: null,
|
||||
status: null,
|
||||
};
|
||||
}
|
||||
|
||||
function compareRuns(beforeRuns = [], afterRuns = []) {
|
||||
const beforeByKey = new Map(beforeRuns.map((run) => [runKey(run), run]));
|
||||
const afterByKey = new Map(afterRuns.map((run) => [runKey(run), run]));
|
||||
const keys = new Set([...beforeByKey.keys(), ...afterByKey.keys()]);
|
||||
|
||||
return [...keys]
|
||||
.map((key) => {
|
||||
const beforeRun = beforeByKey.get(key);
|
||||
const afterRun = afterByKey.get(key);
|
||||
const before = normalizeRun(beforeRun);
|
||||
const after = normalizeRun(afterRun);
|
||||
return {
|
||||
key,
|
||||
status: compareStatus(beforeRun, afterRun),
|
||||
before,
|
||||
after,
|
||||
delta: {
|
||||
elapsedMs: compareOptionalNumber(before.elapsedMs, after.elapsedMs),
|
||||
maxRssBytes: compareOptionalNumber(before.maxRssBytes, after.maxRssBytes),
|
||||
},
|
||||
};
|
||||
})
|
||||
.toSorted((left, right) => {
|
||||
const leftMagnitude = Math.abs(left.delta.elapsedMs ?? left.delta.maxRssBytes ?? 0);
|
||||
const rightMagnitude = Math.abs(right.delta.elapsedMs ?? right.delta.maxRssBytes ?? 0);
|
||||
return rightMagnitude - leftMagnitude || left.key.localeCompare(right.key);
|
||||
});
|
||||
}
|
||||
|
||||
export function buildGroupedTestComparison(params) {
|
||||
const before = params.before;
|
||||
const after = params.after;
|
||||
const beforeTotals = normalizeCounter(before.totals);
|
||||
const afterTotals = normalizeCounter(after.totals);
|
||||
const warnings = [];
|
||||
|
||||
if (before.groupBy !== after.groupBy) {
|
||||
warnings.push(`groupBy differs: before=${before.groupBy} after=${after.groupBy}`);
|
||||
}
|
||||
|
||||
return {
|
||||
generatedAt: new Date().toISOString(),
|
||||
command: "test-group-report:compare",
|
||||
groupBy: after.groupBy ?? before.groupBy,
|
||||
warnings,
|
||||
totals: {
|
||||
before: beforeTotals,
|
||||
after: afterTotals,
|
||||
delta: {
|
||||
durationMs: afterTotals.durationMs - beforeTotals.durationMs,
|
||||
fileCount: afterTotals.fileCount - beforeTotals.fileCount,
|
||||
testCount: afterTotals.testCount - beforeTotals.testCount,
|
||||
},
|
||||
percent: {
|
||||
durationMs: percentDelta(beforeTotals.durationMs, afterTotals.durationMs),
|
||||
},
|
||||
},
|
||||
groups: compareCounters(before.groups, after.groups),
|
||||
configs: compareCounters(before.configs, after.configs),
|
||||
files: compareFiles(before.topFiles, after.topFiles),
|
||||
runs: compareRuns(before.runs, after.runs),
|
||||
inputs: {
|
||||
before: params.beforePath ?? null,
|
||||
after: params.afterPath ?? null,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function formatCountDelta(value) {
|
||||
return `${value > 0 ? "+" : ""}${value}`;
|
||||
}
|
||||
|
||||
function formatOptionalMs(value) {
|
||||
return typeof value === "number" ? formatMs(value) : "n/a";
|
||||
}
|
||||
|
||||
function formatOptionalSignedMs(value) {
|
||||
return typeof value === "number" ? formatSignedMs(value) : "n/a";
|
||||
}
|
||||
|
||||
function formatOptionalBytes(value) {
|
||||
return typeof value === "number" ? formatBytesAsMb(value) : "n/a";
|
||||
}
|
||||
|
||||
function formatOptionalSignedBytes(value) {
|
||||
return typeof value === "number" ? formatSignedBytesAsMb(value) : "n/a";
|
||||
}
|
||||
|
||||
function pushChangeRows(lines, entries, options) {
|
||||
const selected = entries.slice(0, options.limit);
|
||||
if (selected.length === 0) {
|
||||
lines.push(" (none)");
|
||||
return;
|
||||
}
|
||||
|
||||
for (const [index, entry] of selected.entries()) {
|
||||
lines.push(
|
||||
`${String(index + 1).padStart(2, " ")}. ${formatSignedMs(entry.delta.durationMs).padStart(11, " ")} (${formatPercent(entry.percent.durationMs).padStart(7, " ")}) | before=${formatMs(entry.before.durationMs).padStart(10, " ")} after=${formatMs(entry.after.durationMs).padStart(10, " ")} | files=${formatCountDelta(entry.delta.fileCount ?? 0).padStart(4, " ")} tests=${formatCountDelta(entry.delta.testCount ?? 0).padStart(5, " ")} | ${entry.key}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function pushFileChangeRows(lines, entries, options) {
|
||||
const selected = entries.slice(0, options.limit);
|
||||
if (selected.length === 0) {
|
||||
lines.push(" (none)");
|
||||
return;
|
||||
}
|
||||
|
||||
for (const [index, entry] of selected.entries()) {
|
||||
lines.push(
|
||||
`${String(index + 1).padStart(2, " ")}. ${formatSignedMs(entry.delta.durationMs).padStart(11, " ")} (${formatPercent(entry.percent.durationMs).padStart(7, " ")}) | before=${formatMs(entry.before.durationMs).padStart(10, " ")} after=${formatMs(entry.after.durationMs).padStart(10, " ")} | tests=${formatCountDelta(entry.delta.testCount).padStart(4, " ")} | ${entry.config} | ${entry.file}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export function renderGroupedTestComparison(comparison, options = {}) {
|
||||
const limit = options.limit ?? 25;
|
||||
const topFiles = options.topFiles ?? 25;
|
||||
const groupRegressions = comparison.groups.filter((entry) => entry.delta.durationMs > 0);
|
||||
const groupGains = comparison.groups.filter((entry) => entry.delta.durationMs < 0);
|
||||
const fileRegressions = comparison.files.filter((entry) => entry.delta.durationMs > 0);
|
||||
const fileGains = comparison.files.filter((entry) => entry.delta.durationMs < 0);
|
||||
const addedFiles = comparison.files.filter((entry) => entry.status === "added").length;
|
||||
const removedFiles = comparison.files.filter((entry) => entry.status === "removed").length;
|
||||
const lines = [
|
||||
`[test-group-report:compare] groupBy=${comparison.groupBy} file-sum=${formatMs(comparison.totals.before.durationMs)} -> ${formatMs(comparison.totals.after.durationMs)} (${formatSignedMs(comparison.totals.delta.durationMs)}, ${formatPercent(comparison.totals.percent.durationMs)}) files=${comparison.totals.before.fileCount}->${comparison.totals.after.fileCount} (${formatCountDelta(comparison.totals.delta.fileCount)}) tests=${comparison.totals.before.testCount}->${comparison.totals.after.testCount} (${formatCountDelta(comparison.totals.delta.testCount)}) addedFiles=${addedFiles} removedFiles=${removedFiles}`,
|
||||
];
|
||||
|
||||
for (const warning of comparison.warnings) {
|
||||
lines.push(`[test-group-report:compare] warning: ${warning}`);
|
||||
}
|
||||
|
||||
lines.push(
|
||||
"",
|
||||
`Top group regressions (${Math.min(limit, groupRegressions.length)} of ${groupRegressions.length})`,
|
||||
);
|
||||
pushChangeRows(lines, groupRegressions, { limit });
|
||||
|
||||
lines.push("", `Top group gains (${Math.min(limit, groupGains.length)} of ${groupGains.length})`);
|
||||
pushChangeRows(lines, groupGains, { limit });
|
||||
|
||||
lines.push(
|
||||
"",
|
||||
`Config duration deltas (${Math.min(limit, comparison.configs.length)} of ${comparison.configs.length})`,
|
||||
);
|
||||
pushChangeRows(lines, comparison.configs, { limit });
|
||||
|
||||
if (comparison.runs.length > 0) {
|
||||
lines.push(
|
||||
"",
|
||||
`Config wall/RSS deltas (${Math.min(limit, comparison.runs.length)} of ${comparison.runs.length})`,
|
||||
);
|
||||
for (const [index, run] of comparison.runs.slice(0, limit).entries()) {
|
||||
lines.push(
|
||||
`${String(index + 1).padStart(2, " ")}. wall=${formatOptionalSignedMs(run.delta.elapsedMs).padStart(11, " ")} before=${formatOptionalMs(run.before.elapsedMs).padStart(10, " ")} after=${formatOptionalMs(run.after.elapsedMs).padStart(10, " ")} | rss=${formatOptionalSignedBytes(run.delta.maxRssBytes).padStart(10, " ")} before=${formatOptionalBytes(run.before.maxRssBytes).padStart(9, " ")} after=${formatOptionalBytes(run.after.maxRssBytes).padStart(9, " ")} | status=${run.before.status ?? "n/a"}->${run.after.status ?? "n/a"} | ${run.key}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
lines.push(
|
||||
"",
|
||||
`Top file regressions (${Math.min(topFiles, fileRegressions.length)} of ${fileRegressions.length})`,
|
||||
);
|
||||
pushFileChangeRows(lines, fileRegressions, { limit: topFiles });
|
||||
|
||||
lines.push("", `Top file gains (${Math.min(topFiles, fileGains.length)} of ${fileGains.length})`);
|
||||
pushFileChangeRows(lines, fileGains, { limit: topFiles });
|
||||
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
export function renderGroupedTestReport(report, options = {}) {
|
||||
const limit = options.limit ?? 25;
|
||||
const topFiles = options.topFiles ?? 25;
|
||||
const lines = [
|
||||
`[test-group-report] groupBy=${report.groupBy} files=${report.totals.fileCount} tests=${report.totals.testCount} file-sum=${formatMs(report.totals.durationMs)}`,
|
||||
"",
|
||||
`Top groups (${Math.min(limit, report.groups.length)} of ${report.groups.length})`,
|
||||
];
|
||||
|
||||
for (const [index, group] of report.groups.slice(0, limit).entries()) {
|
||||
lines.push(
|
||||
`${String(index + 1).padStart(2, " ")}. ${formatMs(group.durationMs).padStart(10, " ")} | files=${String(group.fileCount).padStart(4, " ")} | tests=${String(group.testCount).padStart(5, " ")} | ${group.key}`,
|
||||
);
|
||||
}
|
||||
|
||||
lines.push(
|
||||
"",
|
||||
`Top configs (${Math.min(limit, report.configs.length)} of ${report.configs.length})`,
|
||||
);
|
||||
for (const [index, config] of report.configs.slice(0, limit).entries()) {
|
||||
lines.push(
|
||||
`${String(index + 1).padStart(2, " ")}. ${formatMs(config.durationMs).padStart(10, " ")} | files=${String(config.fileCount).padStart(4, " ")} | tests=${String(config.testCount).padStart(5, " ")} | ${config.key}`,
|
||||
);
|
||||
}
|
||||
|
||||
lines.push(
|
||||
"",
|
||||
`Top files (${Math.min(topFiles, report.topFiles.length)} of ${report.topFiles.length})`,
|
||||
);
|
||||
for (const [index, file] of report.topFiles.slice(0, topFiles).entries()) {
|
||||
lines.push(
|
||||
`${String(index + 1).padStart(2, " ")}. ${formatMs(file.durationMs).padStart(10, " ")} | tests=${String(file.testCount).padStart(4, " ")} | ${file.config} | ${file.file}`,
|
||||
);
|
||||
}
|
||||
|
||||
return lines.join("\n");
|
||||
}
|
||||
362
scripts/test-group-report.mjs
Normal file
362
scripts/test-group-report.mjs
Normal file
@@ -0,0 +1,362 @@
|
||||
import { spawnSync } from "node:child_process";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import {
|
||||
buildGroupedTestComparison,
|
||||
buildGroupedTestReport,
|
||||
formatBytesAsMb,
|
||||
normalizeConfigLabel,
|
||||
renderGroupedTestComparison,
|
||||
renderGroupedTestReport,
|
||||
} from "./lib/test-group-report.mjs";
|
||||
import { formatMs } from "./lib/vitest-report-cli-utils.mjs";
|
||||
import { resolveVitestNodeArgs } from "./run-vitest.mjs";
|
||||
import { buildFullSuiteVitestRunPlans } from "./test-projects.test-support.mjs";
|
||||
|
||||
const DEFAULT_OUTPUT = ".artifacts/test-perf/group-report.json";
|
||||
const DEFAULT_COMPARE_OUTPUT = ".artifacts/test-perf/group-report-compare.json";
|
||||
|
||||
function usage() {
|
||||
return [
|
||||
"Usage: node scripts/test-group-report.mjs [options] [-- <vitest args>]",
|
||||
"",
|
||||
"Build a grouped Vitest duration report from one or more JSON reports.",
|
||||
"",
|
||||
"Options:",
|
||||
" --config <path> Vitest config to run (repeatable)",
|
||||
" --compare <before> <after>",
|
||||
" Compare two grouped report JSON files",
|
||||
" --report <path> Existing Vitest JSON report to read (repeatable)",
|
||||
" --full-suite Run every full-suite leaf Vitest config serially",
|
||||
" --group-by <mode> area | folder | top (default: area)",
|
||||
" --output <path> JSON report path (default: .artifacts/test-perf/group-report.json)",
|
||||
" --limit <count> Number of groups/configs to print (default: 25)",
|
||||
" --top-files <count> Number of files to print (default: 25)",
|
||||
" --allow-failures Write a report even when a Vitest run exits non-zero",
|
||||
" --no-rss Skip macOS max RSS measurement",
|
||||
" --help Show this help",
|
||||
"",
|
||||
"Examples:",
|
||||
" pnpm test:perf:groups --config test/vitest/vitest.unit-fast.config.ts",
|
||||
" pnpm test:perf:groups --full-suite --allow-failures",
|
||||
" pnpm test:perf:groups:compare .artifacts/test-perf/baseline-before.json .artifacts/test-perf/after-first-fix.json",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function parsePositiveInt(value, fallback) {
|
||||
const parsed = Number.parseInt(value ?? "", 10);
|
||||
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
|
||||
}
|
||||
|
||||
export function parseTestGroupReportArgs(argv) {
|
||||
const args = {
|
||||
allowFailures: false,
|
||||
compare: null,
|
||||
configs: [],
|
||||
fullSuite: false,
|
||||
groupBy: "area",
|
||||
limit: 25,
|
||||
output: null,
|
||||
reports: [],
|
||||
rss: process.platform === "darwin",
|
||||
topFiles: 25,
|
||||
vitestArgs: [],
|
||||
};
|
||||
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index];
|
||||
if (arg === "--") {
|
||||
args.vitestArgs = argv.slice(index + 1);
|
||||
break;
|
||||
}
|
||||
if (arg === "--help") {
|
||||
args.help = true;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--allow-failures") {
|
||||
args.allowFailures = true;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--full-suite") {
|
||||
args.fullSuite = true;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--no-rss") {
|
||||
args.rss = false;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--config") {
|
||||
args.configs.push(argv[index + 1] ?? "");
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--compare") {
|
||||
args.compare = {
|
||||
before: argv[index + 1] ?? "",
|
||||
after: argv[index + 2] ?? "",
|
||||
};
|
||||
index += 2;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--report") {
|
||||
args.reports.push(argv[index + 1] ?? "");
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--group-by") {
|
||||
args.groupBy = argv[index + 1] ?? args.groupBy;
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--output") {
|
||||
args.output = argv[index + 1] ?? args.output;
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--limit") {
|
||||
args.limit = parsePositiveInt(argv[index + 1], args.limit);
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--top-files") {
|
||||
args.topFiles = parsePositiveInt(argv[index + 1], args.topFiles);
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
throw new Error(`Unknown option: ${arg}`);
|
||||
}
|
||||
|
||||
if (!["area", "folder", "top"].includes(args.groupBy)) {
|
||||
throw new Error(`Unsupported --group-by value: ${args.groupBy}`);
|
||||
}
|
||||
if (args.compare && (!args.compare.before || !args.compare.after)) {
|
||||
throw new Error("--compare requires before and after report paths");
|
||||
}
|
||||
if (
|
||||
args.compare &&
|
||||
(args.configs.length > 0 ||
|
||||
args.fullSuite ||
|
||||
args.reports.length > 0 ||
|
||||
args.vitestArgs.length > 0)
|
||||
) {
|
||||
throw new Error("--compare cannot be combined with test run or report input options");
|
||||
}
|
||||
|
||||
return args;
|
||||
}
|
||||
|
||||
function sanitizePathSegment(value) {
|
||||
return (
|
||||
value
|
||||
.replace(/[^A-Za-z0-9._-]+/gu, "-")
|
||||
.replace(/^-+|-+$/gu, "")
|
||||
.slice(0, 180) || "report"
|
||||
);
|
||||
}
|
||||
|
||||
function parseMaxRssBytes(output) {
|
||||
const match = output.match(/(\d+)\s+maximum resident set size/u);
|
||||
return match ? Number.parseInt(match[1], 10) : null;
|
||||
}
|
||||
|
||||
function runVitestJsonReport(params) {
|
||||
fs.mkdirSync(path.dirname(params.reportPath), { recursive: true });
|
||||
fs.mkdirSync(path.dirname(params.logPath), { recursive: true });
|
||||
const command = [
|
||||
process.execPath,
|
||||
"scripts/run-vitest.mjs",
|
||||
"run",
|
||||
"--config",
|
||||
params.config,
|
||||
"--reporter=json",
|
||||
"--outputFile",
|
||||
params.reportPath,
|
||||
...params.vitestArgs,
|
||||
];
|
||||
const startedAt = process.hrtime.bigint();
|
||||
const result = spawnSync(
|
||||
params.rss ? "/usr/bin/time" : command[0],
|
||||
params.rss ? ["-l", ...command] : command.slice(1),
|
||||
{
|
||||
cwd: process.cwd(),
|
||||
encoding: "utf8",
|
||||
env: {
|
||||
...process.env,
|
||||
NODE_OPTIONS: [
|
||||
process.env.NODE_OPTIONS?.trim(),
|
||||
...resolveVitestNodeArgs(process.env).filter((arg) => arg !== "--no-maglev"),
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join(" "),
|
||||
},
|
||||
maxBuffer: 1024 * 1024 * 64,
|
||||
},
|
||||
);
|
||||
const elapsedMs = Number.parseFloat(String(process.hrtime.bigint() - startedAt)) / 1_000_000;
|
||||
const output = `${result.stdout ?? ""}${result.stderr ?? ""}`;
|
||||
fs.writeFileSync(params.logPath, output, "utf8");
|
||||
return {
|
||||
config: params.config,
|
||||
elapsedMs,
|
||||
logPath: params.logPath,
|
||||
maxRssBytes: params.rss ? parseMaxRssBytes(output) : null,
|
||||
reportPath: params.reportPath,
|
||||
status: result.status ?? 1,
|
||||
};
|
||||
}
|
||||
|
||||
function readReportInput(entry) {
|
||||
return {
|
||||
config: entry.config,
|
||||
report: JSON.parse(fs.readFileSync(entry.reportPath, "utf8")),
|
||||
reportPath: entry.reportPath,
|
||||
run: entry.run ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
function readGroupedReport(reportPath) {
|
||||
return JSON.parse(fs.readFileSync(reportPath, "utf8"));
|
||||
}
|
||||
|
||||
function resolveConfigs(args) {
|
||||
if (args.reports.length > 0) {
|
||||
return [];
|
||||
}
|
||||
if (args.fullSuite) {
|
||||
return buildFullSuiteVitestRunPlans([], process.cwd()).map((plan) => plan.config);
|
||||
}
|
||||
return args.configs.length > 0 ? args.configs : ["test/vitest/vitest.unit.config.ts"];
|
||||
}
|
||||
|
||||
function printRunLine(run) {
|
||||
const label = normalizeConfigLabel(run.config);
|
||||
console.log(
|
||||
`[test-group-report] ${label} status=${run.status} wall=${formatMs(run.elapsedMs)} rss=${formatBytesAsMb(run.maxRssBytes)} report=${run.reportPath}`,
|
||||
);
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const args = parseTestGroupReportArgs(process.argv.slice(2));
|
||||
if (args.help) {
|
||||
console.log(usage());
|
||||
return;
|
||||
}
|
||||
|
||||
const output = path.resolve(
|
||||
args.output ?? (args.compare ? DEFAULT_COMPARE_OUTPUT : DEFAULT_OUTPUT),
|
||||
);
|
||||
|
||||
if (args.compare) {
|
||||
const beforePath = path.resolve(args.compare.before);
|
||||
const afterPath = path.resolve(args.compare.after);
|
||||
const comparison = buildGroupedTestComparison({
|
||||
before: readGroupedReport(beforePath),
|
||||
after: readGroupedReport(afterPath),
|
||||
beforePath,
|
||||
afterPath,
|
||||
});
|
||||
|
||||
fs.mkdirSync(path.dirname(output), { recursive: true });
|
||||
fs.writeFileSync(output, `${JSON.stringify(comparison, null, 2)}\n`, "utf8");
|
||||
console.log(
|
||||
renderGroupedTestComparison(comparison, { limit: args.limit, topFiles: args.topFiles }),
|
||||
);
|
||||
console.log(`[test-group-report:compare] wrote ${path.relative(process.cwd(), output)}`);
|
||||
return;
|
||||
}
|
||||
|
||||
const reportDir = path.join(path.dirname(output), "vitest-json");
|
||||
const logDir = path.join(path.dirname(output), "logs");
|
||||
const runEntries = [];
|
||||
const configs = resolveConfigs(args);
|
||||
let failed = false;
|
||||
let exitCode = 0;
|
||||
|
||||
for (const reportPath of args.reports) {
|
||||
runEntries.push({
|
||||
config: path.basename(reportPath).replace(/\.json$/u, ""),
|
||||
reportPath: path.resolve(reportPath),
|
||||
});
|
||||
}
|
||||
|
||||
for (const config of configs) {
|
||||
const slug = sanitizePathSegment(normalizeConfigLabel(config));
|
||||
const run = runVitestJsonReport({
|
||||
config,
|
||||
logPath: path.join(logDir, `${slug}.log`),
|
||||
reportPath: path.join(reportDir, `${slug}.json`),
|
||||
rss: args.rss,
|
||||
vitestArgs: args.vitestArgs,
|
||||
});
|
||||
printRunLine(run);
|
||||
if (run.status !== 0) {
|
||||
failed = true;
|
||||
if (!fs.existsSync(run.reportPath)) {
|
||||
console.error(
|
||||
`[test-group-report] missing JSON report for failed config; see ${run.logPath}`,
|
||||
);
|
||||
if (!args.allowFailures) {
|
||||
exitCode = run.status;
|
||||
break;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
console.error(
|
||||
`[test-group-report] config failed; keeping partial report from ${run.reportPath}`,
|
||||
);
|
||||
if (!args.allowFailures) {
|
||||
exitCode = run.status;
|
||||
break;
|
||||
}
|
||||
}
|
||||
runEntries.push({ config, reportPath: run.reportPath, run });
|
||||
}
|
||||
|
||||
if (exitCode !== 0) {
|
||||
process.exit(exitCode);
|
||||
}
|
||||
|
||||
const reportInputs = runEntries
|
||||
.filter((entry) => fs.existsSync(entry.reportPath))
|
||||
.map(readReportInput);
|
||||
const report = buildGroupedTestReport({
|
||||
groupBy: args.groupBy,
|
||||
reports: reportInputs,
|
||||
});
|
||||
const envelope = {
|
||||
...report,
|
||||
command: "test-group-report",
|
||||
failed,
|
||||
runs: reportInputs.map((entry) => entry.run).filter(Boolean),
|
||||
system: {
|
||||
node: process.version,
|
||||
platform: process.platform,
|
||||
arch: process.arch,
|
||||
cpuCount: os.availableParallelism?.() ?? os.cpus().length,
|
||||
totalMemoryBytes: os.totalmem(),
|
||||
},
|
||||
};
|
||||
|
||||
fs.mkdirSync(path.dirname(output), { recursive: true });
|
||||
fs.writeFileSync(output, `${JSON.stringify(envelope, null, 2)}\n`, "utf8");
|
||||
console.log(renderGroupedTestReport(report, { limit: args.limit, topFiles: args.topFiles }));
|
||||
console.log(`[test-group-report] wrote ${path.relative(process.cwd(), output)}`);
|
||||
|
||||
if (failed && !args.allowFailures) {
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
const isMain =
|
||||
typeof process.argv[1] === "string" &&
|
||||
process.argv[1].length > 0 &&
|
||||
import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href;
|
||||
|
||||
if (isMain) {
|
||||
main().catch((error) => {
|
||||
console.error(error instanceof Error ? error.message : String(error));
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
203
test/scripts/test-group-report.test.ts
Normal file
203
test/scripts/test-group-report.test.ts
Normal file
@@ -0,0 +1,203 @@
|
||||
import path from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
buildGroupedTestComparison,
|
||||
buildGroupedTestReport,
|
||||
renderGroupedTestComparison,
|
||||
resolveGroupKey,
|
||||
resolveTestArea,
|
||||
} from "../../scripts/lib/test-group-report.mjs";
|
||||
import { parseTestGroupReportArgs } from "../../scripts/test-group-report.mjs";
|
||||
|
||||
describe("scripts/test-group-report grouping", () => {
|
||||
it("groups repo files by stable product area", () => {
|
||||
expect(resolveTestArea("extensions/discord/src/send.test.ts")).toBe("extensions/discord");
|
||||
expect(resolveTestArea("src/commands/agent.test.ts")).toBe("src/commands");
|
||||
expect(resolveTestArea("packages/plugin-sdk/src/index.test.ts")).toBe("packages/plugin-sdk");
|
||||
expect(resolveTestArea("ui/src/ui/views/chat.test.ts")).toBe("ui/views");
|
||||
expect(resolveTestArea("test/scripts/test-group-report.test.ts")).toBe("test/scripts");
|
||||
});
|
||||
|
||||
it("supports folder and top-level grouping modes", () => {
|
||||
expect(resolveGroupKey("src/commands/agent.test.ts", "folder")).toBe("src/commands");
|
||||
expect(resolveGroupKey("extensions/browser/src/browser/pw.test.ts", "folder")).toBe(
|
||||
"extensions/browser/src",
|
||||
);
|
||||
expect(resolveGroupKey("extensions/browser/src/browser/pw.test.ts", "top")).toBe("extensions");
|
||||
});
|
||||
});
|
||||
|
||||
describe("scripts/test-group-report aggregation", () => {
|
||||
it("aggregates file durations by group and config", () => {
|
||||
const report = buildGroupedTestReport({
|
||||
groupBy: "area",
|
||||
reports: [
|
||||
{
|
||||
config: "test/vitest/vitest.commands.config.ts",
|
||||
report: {
|
||||
testResults: [
|
||||
{
|
||||
name: path.join(process.cwd(), "src", "commands", "agent.test.ts"),
|
||||
startTime: 100,
|
||||
endTime: 700,
|
||||
assertionResults: [{}, {}],
|
||||
},
|
||||
{
|
||||
name: path.join(process.cwd(), "extensions", "discord", "src", "send.test.ts"),
|
||||
startTime: 200,
|
||||
endTime: 450,
|
||||
assertionResults: [{}],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(report.totals).toEqual({ durationMs: 850, fileCount: 2, testCount: 3 });
|
||||
expect(report.groups.map((group) => [group.key, group.durationMs])).toEqual([
|
||||
["src/commands", 600],
|
||||
["extensions/discord", 250],
|
||||
]);
|
||||
expect(report.configs).toMatchObject([
|
||||
{
|
||||
key: "commands",
|
||||
durationMs: 850,
|
||||
fileCount: 2,
|
||||
testCount: 3,
|
||||
},
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("scripts/test-group-report comparison", () => {
|
||||
it("compares grouped reports by group, file, config, and run metrics", () => {
|
||||
const comparison = buildGroupedTestComparison({
|
||||
beforePath: "before.json",
|
||||
afterPath: "after.json",
|
||||
before: {
|
||||
groupBy: "area",
|
||||
totals: { durationMs: 1000, fileCount: 2, testCount: 4 },
|
||||
groups: [
|
||||
{ key: "src/commands", durationMs: 700, fileCount: 1, testCount: 2 },
|
||||
{ key: "extensions/discord", durationMs: 300, fileCount: 1, testCount: 2 },
|
||||
],
|
||||
configs: [{ key: "commands", durationMs: 1000, fileCount: 2, testCount: 4 }],
|
||||
topFiles: [
|
||||
{
|
||||
config: "commands",
|
||||
file: "src/commands/agent.test.ts",
|
||||
group: "src/commands",
|
||||
durationMs: 700,
|
||||
testCount: 2,
|
||||
},
|
||||
{
|
||||
config: "commands",
|
||||
file: "extensions/discord/src/send.test.ts",
|
||||
group: "extensions/discord",
|
||||
durationMs: 300,
|
||||
testCount: 2,
|
||||
},
|
||||
],
|
||||
runs: [
|
||||
{
|
||||
config: "test/vitest/vitest.commands.config.ts",
|
||||
elapsedMs: 2000,
|
||||
maxRssBytes: 1024 * 1024 * 100,
|
||||
status: 0,
|
||||
},
|
||||
],
|
||||
},
|
||||
after: {
|
||||
groupBy: "area",
|
||||
totals: { durationMs: 900, fileCount: 2, testCount: 5 },
|
||||
groups: [{ key: "src/commands", durationMs: 900, fileCount: 2, testCount: 5 }],
|
||||
configs: [{ key: "commands", durationMs: 900, fileCount: 2, testCount: 5 }],
|
||||
topFiles: [
|
||||
{
|
||||
config: "commands",
|
||||
file: "src/commands/agent.test.ts",
|
||||
group: "src/commands",
|
||||
durationMs: 800,
|
||||
testCount: 3,
|
||||
},
|
||||
{
|
||||
config: "commands",
|
||||
file: "src/commands/new.test.ts",
|
||||
group: "src/commands",
|
||||
durationMs: 100,
|
||||
testCount: 2,
|
||||
},
|
||||
],
|
||||
runs: [
|
||||
{
|
||||
config: "test/vitest/vitest.commands.config.ts",
|
||||
elapsedMs: 1800,
|
||||
maxRssBytes: 1024 * 1024 * 80,
|
||||
status: 0,
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
expect(comparison.totals.delta).toEqual({ durationMs: -100, fileCount: 0, testCount: 1 });
|
||||
expect(comparison.groups.find((group) => group.key === "src/commands")).toMatchObject({
|
||||
delta: { durationMs: 200, fileCount: 1, testCount: 3 },
|
||||
});
|
||||
expect(
|
||||
comparison.files.find((file) => file.file === "extensions/discord/src/send.test.ts"),
|
||||
).toMatchObject({
|
||||
status: "removed",
|
||||
delta: { durationMs: -300, testCount: -2 },
|
||||
});
|
||||
expect(comparison.runs[0]).toMatchObject({
|
||||
key: "commands",
|
||||
delta: { elapsedMs: -200, maxRssBytes: -1024 * 1024 * 20 },
|
||||
});
|
||||
|
||||
expect(renderGroupedTestComparison(comparison, { limit: 2, topFiles: 2 })).toContain(
|
||||
"Top group regressions",
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("scripts/test-group-report arg parsing", () => {
|
||||
it("parses repeatable config and passthrough args", () => {
|
||||
expect(
|
||||
parseTestGroupReportArgs([
|
||||
"--config",
|
||||
"a.ts",
|
||||
"--config",
|
||||
"b.ts",
|
||||
"--group-by",
|
||||
"folder",
|
||||
"--allow-failures",
|
||||
"--",
|
||||
"--maxWorkers=1",
|
||||
]),
|
||||
).toMatchObject({
|
||||
allowFailures: true,
|
||||
configs: ["a.ts", "b.ts"],
|
||||
groupBy: "folder",
|
||||
vitestArgs: ["--maxWorkers=1"],
|
||||
});
|
||||
});
|
||||
|
||||
it("parses compare mode", () => {
|
||||
expect(
|
||||
parseTestGroupReportArgs([
|
||||
"--compare",
|
||||
"before.json",
|
||||
"after.json",
|
||||
"--limit",
|
||||
"5",
|
||||
"--top-files",
|
||||
"3",
|
||||
]),
|
||||
).toMatchObject({
|
||||
compare: { before: "before.json", after: "after.json" },
|
||||
limit: 5,
|
||||
topFiles: 3,
|
||||
});
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user