mirror of
https://github.com/openclaw/openclaw.git
synced 2026-06-28 13:13:39 +00:00
test(perf): compare saved CLI startup benchmarks (#94812)
Summary: - Adds saved CLI startup benchmark report comparison flags to `scripts/bench-cli-startup.ts`, plus JSON output coverage and changed-target routing expectations for the new test-helper importer. - PR surface: Tests +77, Other +109. Total +186 across 4 files. - Reproducibility: not applicable. as a feature/tooling PR. The prior PR defects were source-proven in review comments and the current head addresses them; I did not run local tests because this review was read-only. Automerge notes: - Ran the ClawSweeper repair loop before final review. - Included post-review commit in the final squash: test(perf): compare saved CLI startup benchmarks Validation: - ClawSweeper review passed for head1afa110f1b. - Required merge gates passed before the squash merge. Prepared head SHA:1afa110f1bReview: https://github.com/openclaw/openclaw/pull/94812#issuecomment-4748785428 Co-authored-by: clawsweeper <274271284+clawsweeper[bot]@users.noreply.github.com> Co-authored-by: Felix Isaac Lim <38658663+FelixIsaac@users.noreply.github.com>
This commit is contained in:
@@ -1,6 +1,9 @@
|
||||
// Bench Cli Startup tests cover bench cli startup script behavior.
|
||||
import { existsSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { testing } from "../../scripts/bench-cli-startup.ts";
|
||||
import { createTempDirTracker } from "../helpers/temp-dir.js";
|
||||
|
||||
function withEnv<T>(env: Record<string, string | undefined>, callback: () => T): T {
|
||||
const previous = new Map<string, string | undefined>();
|
||||
@@ -26,6 +29,72 @@ function withEnv<T>(env: Record<string, string | undefined>, callback: () => T):
|
||||
}
|
||||
|
||||
describe("bench-cli-startup", () => {
|
||||
it("writes compare-mode JSON output and creates parent directories", () => {
|
||||
const tempDirs = createTempDirTracker();
|
||||
const tmpDir = tempDirs.make("openclaw-cli-startup-compare-output-");
|
||||
try {
|
||||
const baselinePath = join(tmpDir, "baseline.json");
|
||||
const candidatePath = join(tmpDir, "candidate.json");
|
||||
const outputPath = join(tmpDir, "nested", "comparison.json");
|
||||
const makeReport = (durationAvg: number, maxRssAvg: number) => ({
|
||||
primary: {
|
||||
entry: "openclaw.mjs",
|
||||
cases: [
|
||||
{
|
||||
id: "version",
|
||||
name: "--version",
|
||||
args: ["--version"],
|
||||
contract: null,
|
||||
samples: [],
|
||||
summary: {
|
||||
sampleCount: 1,
|
||||
durationMs: {
|
||||
avg: durationAvg,
|
||||
p50: durationAvg,
|
||||
p95: durationAvg,
|
||||
min: durationAvg,
|
||||
max: durationAvg,
|
||||
},
|
||||
firstOutputMs: null,
|
||||
maxRssMb: {
|
||||
avg: maxRssAvg,
|
||||
p50: maxRssAvg,
|
||||
p95: maxRssAvg,
|
||||
min: maxRssAvg,
|
||||
max: maxRssAvg,
|
||||
},
|
||||
exitSummary: "code:0x1",
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
writeFileSync(baselinePath, JSON.stringify(makeReport(100, 50)), "utf8");
|
||||
writeFileSync(candidatePath, JSON.stringify(makeReport(125, 60)), "utf8");
|
||||
|
||||
const { comparison } = testing.readBenchmarkComparison(baselinePath, candidatePath);
|
||||
testing.writeJsonOutput(outputPath, comparison);
|
||||
expect(existsSync(outputPath)).toBe(true);
|
||||
expect(JSON.parse(readFileSync(outputPath, "utf8"))).toEqual({
|
||||
baseline: baselinePath,
|
||||
candidate: candidatePath,
|
||||
deltas: [
|
||||
{
|
||||
id: "version",
|
||||
name: "--version",
|
||||
durationAvgDeltaMs: 25,
|
||||
durationAvgDeltaPct: 25,
|
||||
maxRssAvgDeltaMb: 10,
|
||||
maxRssAvgDeltaPct: 20,
|
||||
},
|
||||
],
|
||||
});
|
||||
} finally {
|
||||
tempDirs.cleanup();
|
||||
}
|
||||
});
|
||||
|
||||
it("fails reports with no measured samples", () => {
|
||||
expect(
|
||||
testing.collectFailedSamples({
|
||||
|
||||
@@ -1918,6 +1918,8 @@ describe("scripts/test-projects changed-target routing", () => {
|
||||
"test/helpers/temp-dir.ts": "export const tempDir = 'x';\n",
|
||||
"test/helpers/temp-dir.test.ts":
|
||||
"import { tempDir } from './temp-dir.js';\nvoid tempDir;\n",
|
||||
"test/scripts/bench-cli-startup.test.ts":
|
||||
"import { tempDir } from '../helpers/temp-dir.js';\nvoid tempDir;\n",
|
||||
"src/foo.test.ts":
|
||||
"import { tempDir } from '../test/helpers/temp-dir.js';\nvoid tempDir;\n",
|
||||
},
|
||||
@@ -1926,7 +1928,11 @@ describe("scripts/test-projects changed-target routing", () => {
|
||||
},
|
||||
);
|
||||
|
||||
expect(targets).toEqual(["test/helpers/temp-dir.test.ts", "src/foo.test.ts"]);
|
||||
expect(targets).toEqual([
|
||||
"test/helpers/temp-dir.test.ts",
|
||||
"src/foo.test.ts",
|
||||
"test/scripts/bench-cli-startup.test.ts",
|
||||
]);
|
||||
});
|
||||
|
||||
it("keeps the broad changed run available for shared test helpers", () => {
|
||||
|
||||
Reference in New Issue
Block a user