diff --git a/extensions/qa-lab/src/suite-summary.ts b/extensions/qa-lab/src/suite-summary.ts index 8a4845d2d39..78d073772fb 100644 --- a/extensions/qa-lab/src/suite-summary.ts +++ b/extensions/qa-lab/src/suite-summary.ts @@ -23,6 +23,13 @@ export type QaSuiteSummaryJson = { gatewayProcessRssStartBytes?: number | null; gatewayProcessRssEndBytes?: number | null; gatewayProcessRssDeltaBytes?: number | null; + gatewayProcessRssPeakBytes?: number | null; + gatewayProcessRssPeakDeltaBytes?: number | null; + gatewayProcessRssSamples?: Array<{ + label: string; + at: string; + gatewayProcessRssBytes: number; + }>; }; run: { startedAt: string; diff --git a/extensions/qa-lab/src/suite.summary-json.test.ts b/extensions/qa-lab/src/suite.summary-json.test.ts index dead9540c7d..d731af797f2 100644 --- a/extensions/qa-lab/src/suite.summary-json.test.ts +++ b/extensions/qa-lab/src/suite.summary-json.test.ts @@ -156,6 +156,20 @@ describe("buildQaSuiteSummaryJson", () => { gatewayProcessRssStartBytes: 100_000_000, gatewayProcessRssEndBytes: 125_000_000, gatewayProcessRssDeltaBytes: 25_000_000, + gatewayProcessRssPeakBytes: 140_000_000, + gatewayProcessRssPeakDeltaBytes: 40_000_000, + gatewayProcessRssSamples: [ + { + label: "suite-start", + at: "2026-04-22T12:00:00.000Z", + gatewayProcessRssBytes: 100_000_000, + }, + { + label: "scenario:canary:finish", + at: "2026-04-22T12:00:10.000Z", + gatewayProcessRssBytes: 140_000_000, + }, + ], }, }); expect(json.metrics).toEqual({ @@ -165,6 +179,20 @@ describe("buildQaSuiteSummaryJson", () => { gatewayProcessRssStartBytes: 100_000_000, gatewayProcessRssEndBytes: 125_000_000, gatewayProcessRssDeltaBytes: 25_000_000, + gatewayProcessRssPeakBytes: 140_000_000, + gatewayProcessRssPeakDeltaBytes: 40_000_000, + gatewayProcessRssSamples: [ + { + label: "suite-start", + at: "2026-04-22T12:00:00.000Z", + gatewayProcessRssBytes: 100_000_000, + }, + { + label: "scenario:canary:finish", + at: "2026-04-22T12:00:10.000Z", + gatewayProcessRssBytes: 140_000_000, + }, + ], }); }); }); diff --git a/extensions/qa-lab/src/suite.test.ts b/extensions/qa-lab/src/suite.test.ts index f332b4c5c82..11260f3953f 100644 --- a/extensions/qa-lab/src/suite.test.ts +++ b/extensions/qa-lab/src/suite.test.ts @@ -137,6 +137,52 @@ describe("qa suite", () => { expect(qaSuiteProgressTesting.sanitizeQaSuiteProgressValue("\u0000\u0001")).toBe(""); }); + it("records gateway RSS peak and trace samples", () => { + expect( + qaSuiteProgressTesting.buildQaSuiteRuntimeMetrics({ + startedAt: new Date("2026-04-22T12:00:00.000Z"), + finishedAt: new Date("2026-04-22T12:00:12.000Z"), + gatewayProcessCpuStartMs: 1_000, + gatewayProcessCpuEndMs: 4_000, + gatewayProcessRssStartBytes: 100_000_000, + gatewayProcessRssEndBytes: 125_000_000, + gatewayProcessRssSamples: [ + { + label: "suite-start", + at: "2026-04-22T12:00:00.000Z", + gatewayProcessRssBytes: 100_000_000, + }, + { + label: "scenario:canary:finish", + at: "2026-04-22T12:00:10.000Z", + gatewayProcessRssBytes: 140_000_000, + }, + ], + }), + ).toEqual({ + wallMs: 12_000, + gatewayProcessCpuMs: 3_000, + gatewayCpuCoreRatio: 0.25, + gatewayProcessRssStartBytes: 100_000_000, + gatewayProcessRssEndBytes: 125_000_000, + gatewayProcessRssDeltaBytes: 25_000_000, + gatewayProcessRssPeakBytes: 140_000_000, + gatewayProcessRssPeakDeltaBytes: 40_000_000, + gatewayProcessRssSamples: [ + { + label: "suite-start", + at: "2026-04-22T12:00:00.000Z", + gatewayProcessRssBytes: 100_000_000, + }, + { + label: "scenario:canary:finish", + at: "2026-04-22T12:00:10.000Z", + gatewayProcessRssBytes: 140_000_000, + }, + ], + }); + }); + it("builds a codex mock runtime env patch that stays on the QA mock provider", () => { expect( qaSuiteProgressTesting.buildQaRuntimeEnvPatch({ diff --git a/extensions/qa-lab/src/suite.ts b/extensions/qa-lab/src/suite.ts index 890f4226331..0d75ecb6d94 100644 --- a/extensions/qa-lab/src/suite.ts +++ b/extensions/qa-lab/src/suite.ts @@ -450,6 +450,10 @@ export type QaSuiteSummaryJsonParams = { */ export type { QaSuiteSummaryJson } from "./suite-summary.js"; +type QaSuiteGatewayRssSample = NonNullable< + NonNullable["gatewayProcessRssSamples"] +>[number]; + /** * Pure-ish JSON builder for qa-suite-summary.json. Exported so the GPT-5.5 * parity gate (agentic-parity-report.ts, #64441) and any future parity @@ -762,8 +766,16 @@ function buildQaSuiteRuntimeMetrics(params: { gatewayProcessCpuEndMs: number | null; gatewayProcessRssStartBytes: number | null; gatewayProcessRssEndBytes: number | null; + gatewayProcessRssSamples?: QaSuiteGatewayRssSample[]; }): QaSuiteSummaryJson["metrics"] { const wallMs = Math.max(1, params.finishedAt.getTime() - params.startedAt.getTime()); + const gatewayProcessRssSamples = params.gatewayProcessRssSamples ?? []; + const gatewayProcessRssPeakBytes = + gatewayProcessRssSamples.length > 0 + ? Math.max(...gatewayProcessRssSamples.map((sample) => sample.gatewayProcessRssBytes)) + : params.gatewayProcessRssStartBytes === null || params.gatewayProcessRssEndBytes === null + ? null + : Math.max(params.gatewayProcessRssStartBytes, params.gatewayProcessRssEndBytes); const rssMetrics = params.gatewayProcessRssStartBytes === null || params.gatewayProcessRssEndBytes === null ? {} @@ -772,6 +784,16 @@ function buildQaSuiteRuntimeMetrics(params: { gatewayProcessRssEndBytes: params.gatewayProcessRssEndBytes, gatewayProcessRssDeltaBytes: params.gatewayProcessRssEndBytes - params.gatewayProcessRssStartBytes, + ...(gatewayProcessRssPeakBytes === null + ? {} + : { + gatewayProcessRssPeakBytes, + gatewayProcessRssPeakDeltaBytes: + gatewayProcessRssPeakBytes - params.gatewayProcessRssStartBytes, + }), + ...(gatewayProcessRssSamples.length === 0 + ? {} + : { gatewayProcessRssSamples }), }; if (params.gatewayProcessCpuStartMs === null || params.gatewayProcessCpuEndMs === null) { return { wallMs, ...rssMetrics }; @@ -1196,14 +1218,27 @@ export async function runQaSuite(params?: QaSuiteRunParams): Promise { + const gatewayProcessRssBytes = gateway.getProcessRssBytes?.() ?? null; + if (gatewayProcessRssBytes !== null) { + gatewayProcessRssSamples.push({ + label, + at: new Date().toISOString(), + gatewayProcessRssBytes, + }); + } + return gatewayProcessRssBytes; + }; const gatewayProcessCpuStartMs = gateway.getProcessCpuMs?.() ?? null; - const gatewayProcessRssStartBytes = gateway.getProcessRssBytes?.() ?? null; + const gatewayProcessRssStartBytes = sampleGatewayProcessRss("suite-start"); for (const [index, scenario] of selectedCatalogScenarios.entries()) { const scenarioIdForLog = sanitizeQaSuiteProgressValue(scenario.id); writeQaSuiteProgress( progressEnabled, `scenario start (${index + 1}/${selectedCatalogScenarios.length}): ${scenarioIdForLog}`, ); + sampleGatewayProcessRss(`scenario:${scenario.id}:start`); liveScenarioOutcomes[index] = { id: scenario.id, name: scenario.title, @@ -1218,6 +1253,7 @@ export async function runQaSuite(params?: QaSuiteRunParams): Promise scenario.status === "fail").length; if (scenarios.some((scenario) => scenario.status === "fail")) { @@ -1336,6 +1373,7 @@ export async function runQaSuite(params?: QaSuiteRunParams): Promise