test(perf): compare saved CLI startup benchmarks (#94812)

Summary:
- Adds saved CLI startup benchmark report comparison flags to `scripts/bench-cli-startup.ts`, plus JSON output coverage and changed-target routing expectations for the new test-helper importer.
- PR surface: Tests +77, Other +109. Total +186 across 4 files.
- Reproducibility: not applicable. as a feature/tooling PR. The prior PR defects were source-proven in review comments and the current head addresses them; I did not run local tests because this review was read-only.

Automerge notes:
- Ran the ClawSweeper repair loop before final review.
- Included post-review commit in the final squash: test(perf): compare saved CLI startup benchmarks

Validation:
- ClawSweeper review passed for head 1afa110f1b.
- Required merge gates passed before the squash merge.

Prepared head SHA: 1afa110f1b
Review: https://github.com/openclaw/openclaw/pull/94812#issuecomment-4748785428

Co-authored-by: clawsweeper <274271284+clawsweeper[bot]@users.noreply.github.com>
Co-authored-by: Felix Isaac Lim <38658663+FelixIsaac@users.noreply.github.com>
This commit is contained in:
clawsweeper[bot]
2026-06-19 09:37:47 +00:00
committed by GitHub
parent 5b3d652c05
commit 2e0dfda462
4 changed files with 201 additions and 15 deletions

View File

@@ -1,6 +1,9 @@
// Bench Cli Startup tests cover bench cli startup script behavior.
import { existsSync, readFileSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import { describe, expect, it } from "vitest";
import { testing } from "../../scripts/bench-cli-startup.ts";
import { createTempDirTracker } from "../helpers/temp-dir.js";
function withEnv<T>(env: Record<string, string | undefined>, callback: () => T): T {
const previous = new Map<string, string | undefined>();
@@ -26,6 +29,72 @@ function withEnv<T>(env: Record<string, string | undefined>, callback: () => T):
}
describe("bench-cli-startup", () => {
it("writes compare-mode JSON output and creates parent directories", () => {
const tempDirs = createTempDirTracker();
const tmpDir = tempDirs.make("openclaw-cli-startup-compare-output-");
try {
const baselinePath = join(tmpDir, "baseline.json");
const candidatePath = join(tmpDir, "candidate.json");
const outputPath = join(tmpDir, "nested", "comparison.json");
const makeReport = (durationAvg: number, maxRssAvg: number) => ({
primary: {
entry: "openclaw.mjs",
cases: [
{
id: "version",
name: "--version",
args: ["--version"],
contract: null,
samples: [],
summary: {
sampleCount: 1,
durationMs: {
avg: durationAvg,
p50: durationAvg,
p95: durationAvg,
min: durationAvg,
max: durationAvg,
},
firstOutputMs: null,
maxRssMb: {
avg: maxRssAvg,
p50: maxRssAvg,
p95: maxRssAvg,
min: maxRssAvg,
max: maxRssAvg,
},
exitSummary: "code:0x1",
},
},
],
},
});
writeFileSync(baselinePath, JSON.stringify(makeReport(100, 50)), "utf8");
writeFileSync(candidatePath, JSON.stringify(makeReport(125, 60)), "utf8");
const { comparison } = testing.readBenchmarkComparison(baselinePath, candidatePath);
testing.writeJsonOutput(outputPath, comparison);
expect(existsSync(outputPath)).toBe(true);
expect(JSON.parse(readFileSync(outputPath, "utf8"))).toEqual({
baseline: baselinePath,
candidate: candidatePath,
deltas: [
{
id: "version",
name: "--version",
durationAvgDeltaMs: 25,
durationAvgDeltaPct: 25,
maxRssAvgDeltaMb: 10,
maxRssAvgDeltaPct: 20,
},
],
});
} finally {
tempDirs.cleanup();
}
});
it("fails reports with no measured samples", () => {
expect(
testing.collectFailedSamples({

View File

@@ -1918,6 +1918,8 @@ describe("scripts/test-projects changed-target routing", () => {
"test/helpers/temp-dir.ts": "export const tempDir = 'x';\n",
"test/helpers/temp-dir.test.ts":
"import { tempDir } from './temp-dir.js';\nvoid tempDir;\n",
"test/scripts/bench-cli-startup.test.ts":
"import { tempDir } from '../helpers/temp-dir.js';\nvoid tempDir;\n",
"src/foo.test.ts":
"import { tempDir } from '../test/helpers/temp-dir.js';\nvoid tempDir;\n",
},
@@ -1926,7 +1928,11 @@ describe("scripts/test-projects changed-target routing", () => {
},
);
expect(targets).toEqual(["test/helpers/temp-dir.test.ts", "src/foo.test.ts"]);
expect(targets).toEqual([
"test/helpers/temp-dir.test.ts",
"src/foo.test.ts",
"test/scripts/bench-cli-startup.test.ts",
]);
});
it("keeps the broad changed run available for shared test helpers", () => {