vultr/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-05-24 21:33:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

test(qa): trace gateway rss in suite summaries

Browse Source
This commit is contained in:
Vincent Koc
2026-05-18 00:31:27 +08:00
parent fc16df30dd
commit 20ec5cdc42
4 changed files with 121 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
extensions/qa-lab/src/suite-summary.ts
View File

@@ -23,6 +23,13 @@ export type QaSuiteSummaryJson = {
gatewayProcessRssStartBytes?: number | null;
gatewayProcessRssEndBytes?: number | null;
gatewayProcessRssDeltaBytes?: number | null;
gatewayProcessRssPeakBytes?: number | null;
gatewayProcessRssPeakDeltaBytes?: number | null;
gatewayProcessRssSamples?: Array<{
label: string;
at: string;
gatewayProcessRssBytes: number;
}>;
};
run: {
startedAt: string;

28
extensions/qa-lab/src/suite.summary-json.test.ts
View File

@@ -156,6 +156,20 @@ describe("buildQaSuiteSummaryJson", () => {
gatewayProcessRssStartBytes: 100_000_000,
gatewayProcessRssEndBytes: 125_000_000,
gatewayProcessRssDeltaBytes: 25_000_000,
gatewayProcessRssPeakBytes: 140_000_000,
gatewayProcessRssPeakDeltaBytes: 40_000_000,
gatewayProcessRssSamples: [
{
label: "suite-start",
at: "2026-04-22T12:00:00.000Z",
gatewayProcessRssBytes: 100_000_000,
},
{
label: "scenario:canary:finish",
at: "2026-04-22T12:00:10.000Z",
gatewayProcessRssBytes: 140_000_000,
},
],
},
});
expect(json.metrics).toEqual({
@@ -165,6 +179,20 @@ describe("buildQaSuiteSummaryJson", () => {
gatewayProcessRssStartBytes: 100_000_000,
gatewayProcessRssEndBytes: 125_000_000,
gatewayProcessRssDeltaBytes: 25_000_000,
gatewayProcessRssPeakBytes: 140_000_000,
gatewayProcessRssPeakDeltaBytes: 40_000_000,
gatewayProcessRssSamples: [
{
label: "suite-start",
at: "2026-04-22T12:00:00.000Z",
gatewayProcessRssBytes: 100_000_000,
},
{
label: "scenario:canary:finish",
at: "2026-04-22T12:00:10.000Z",
gatewayProcessRssBytes: 140_000_000,
},
],
});
});
});

46
extensions/qa-lab/src/suite.test.ts
View File

@@ -137,6 +137,52 @@ describe("qa suite", () => {
expect(qaSuiteProgressTesting.sanitizeQaSuiteProgressValue("\u0000\u0001")).toBe("<empty>");
});
it("records gateway RSS peak and trace samples", () => {
expect(
qaSuiteProgressTesting.buildQaSuiteRuntimeMetrics({
startedAt: new Date("2026-04-22T12:00:00.000Z"),
finishedAt: new Date("2026-04-22T12:00:12.000Z"),
gatewayProcessCpuStartMs: 1_000,
gatewayProcessCpuEndMs: 4_000,
gatewayProcessRssStartBytes: 100_000_000,
gatewayProcessRssEndBytes: 125_000_000,
gatewayProcessRssSamples: [
{
label: "suite-start",
at: "2026-04-22T12:00:00.000Z",
gatewayProcessRssBytes: 100_000_000,
},
{
label: "scenario:canary:finish",
at: "2026-04-22T12:00:10.000Z",
gatewayProcessRssBytes: 140_000_000,
},
],
}),
).toEqual({
wallMs: 12_000,
gatewayProcessCpuMs: 3_000,
gatewayCpuCoreRatio: 0.25,
gatewayProcessRssStartBytes: 100_000_000,
gatewayProcessRssEndBytes: 125_000_000,
gatewayProcessRssDeltaBytes: 25_000_000,
gatewayProcessRssPeakBytes: 140_000_000,
gatewayProcessRssPeakDeltaBytes: 40_000_000,
gatewayProcessRssSamples: [
{
label: "suite-start",
at: "2026-04-22T12:00:00.000Z",
gatewayProcessRssBytes: 100_000_000,
},
{
label: "scenario:canary:finish",
at: "2026-04-22T12:00:10.000Z",
gatewayProcessRssBytes: 140_000_000,
},
],
});
});
it("builds a codex mock runtime env patch that stays on the QA mock provider", () => {
expect(
qaSuiteProgressTesting.buildQaRuntimeEnvPatch({

42
extensions/qa-lab/src/suite.ts
View File

@@ -450,6 +450,10 @@ export type QaSuiteSummaryJsonParams = {
*/
export type { QaSuiteSummaryJson } from "./suite-summary.js";
type QaSuiteGatewayRssSample = NonNullable<
NonNullable<QaSuiteSummaryJson["metrics"]>["gatewayProcessRssSamples"]
>[number];
/**
* Pure-ish JSON builder for qa-suite-summary.json. Exported so the GPT-5.5
* parity gate (agentic-parity-report.ts, #64441) and any future parity
@@ -762,8 +766,16 @@ function buildQaSuiteRuntimeMetrics(params: {
gatewayProcessCpuEndMs: number | null;
gatewayProcessRssStartBytes: number | null;
gatewayProcessRssEndBytes: number | null;
gatewayProcessRssSamples?: QaSuiteGatewayRssSample[];
}): QaSuiteSummaryJson["metrics"] {
const wallMs = Math.max(1, params.finishedAt.getTime() - params.startedAt.getTime());
const gatewayProcessRssSamples = params.gatewayProcessRssSamples ?? [];
const gatewayProcessRssPeakBytes =
gatewayProcessRssSamples.length > 0
? Math.max(...gatewayProcessRssSamples.map((sample) => sample.gatewayProcessRssBytes))
: params.gatewayProcessRssStartBytes === null || params.gatewayProcessRssEndBytes === null
? null
: Math.max(params.gatewayProcessRssStartBytes, params.gatewayProcessRssEndBytes);
const rssMetrics =
params.gatewayProcessRssStartBytes === null || params.gatewayProcessRssEndBytes === null
? {}
@@ -772,6 +784,16 @@ function buildQaSuiteRuntimeMetrics(params: {
gatewayProcessRssEndBytes: params.gatewayProcessRssEndBytes,
gatewayProcessRssDeltaBytes:
params.gatewayProcessRssEndBytes - params.gatewayProcessRssStartBytes,
...(gatewayProcessRssPeakBytes === null
? {}
: {
gatewayProcessRssPeakBytes,
gatewayProcessRssPeakDeltaBytes:
gatewayProcessRssPeakBytes - params.gatewayProcessRssStartBytes,
}),
...(gatewayProcessRssSamples.length === 0
? {}
: { gatewayProcessRssSamples }),
};
if (params.gatewayProcessCpuStartMs === null || params.gatewayProcessCpuEndMs === null) {
return { wallMs, ...rssMetrics };
@@ -1196,14 +1218,27 @@ export async function runQaSuite(params?: QaSuiteRunParams): Promise<QaSuiteResu
scenarios: liveScenarioOutcomes,
});
const gatewayProcessRssSamples: QaSuiteGatewayRssSample[] = [];
const sampleGatewayProcessRss = (label: string) => {
const gatewayProcessRssBytes = gateway.getProcessRssBytes?.() ?? null;
if (gatewayProcessRssBytes !== null) {
gatewayProcessRssSamples.push({
label,
at: new Date().toISOString(),
gatewayProcessRssBytes,
});
}
return gatewayProcessRssBytes;
};
const gatewayProcessCpuStartMs = gateway.getProcessCpuMs?.() ?? null;
const gatewayProcessRssStartBytes = gateway.getProcessRssBytes?.() ?? null;
const gatewayProcessRssStartBytes = sampleGatewayProcessRss("suite-start");
for (const [index, scenario] of selectedCatalogScenarios.entries()) {
const scenarioIdForLog = sanitizeQaSuiteProgressValue(scenario.id);
writeQaSuiteProgress(
progressEnabled,
`scenario start (${index + 1}/${selectedCatalogScenarios.length}): ${scenarioIdForLog}`,
);
sampleGatewayProcessRss(`scenario:${scenario.id}:start`);
liveScenarioOutcomes[index] = {
id: scenario.id,
name: scenario.title,
@@ -1218,6 +1253,7 @@ export async function runQaSuite(params?: QaSuiteRunParams): Promise<QaSuiteResu
});
const result = await runScenarioDefinition(env, scenario);
sampleGatewayProcessRss(`scenario:${scenario.id}:finish`);
scenarios.push(result);
writeQaSuiteProgress(
progressEnabled,
@@ -1260,7 +1296,8 @@ export async function runQaSuite(params?: QaSuiteRunParams): Promise<QaSuiteResu
gatewayProcessCpuStartMs,
gatewayProcessCpuEndMs: gateway.getProcessCpuMs?.() ?? null,
gatewayProcessRssStartBytes,
gatewayProcessRssEndBytes: gateway.getProcessRssBytes?.() ?? null,
gatewayProcessRssEndBytes: sampleGatewayProcessRss("suite-finish"),
gatewayProcessRssSamples,
});
const failedCount = scenarios.filter((scenario) => scenario.status === "fail").length;
if (scenarios.some((scenario) => scenario.status === "fail")) {
@@ -1336,6 +1373,7 @@ export async function runQaSuite(params?: QaSuiteRunParams): Promise<QaSuiteResu
}
export const qaSuiteProgressTesting = {
buildQaSuiteRuntimeMetrics,
buildQaRuntimeEnvPatch,
parseQaSuiteBooleanEnv,
remapModelRefForForcedRuntime,