From d034e9698a75504da2cb491233ff0c92f973a847 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 7 Jun 2026 02:59:19 +0200 Subject: [PATCH] fix(qa): trust parity scenario rows for metrics --- .../qa-lab/src/agentic-parity-report.test.ts | 21 +++++++++++++++++++ .../qa-lab/src/agentic-parity-report.ts | 8 +++---- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/extensions/qa-lab/src/agentic-parity-report.test.ts b/extensions/qa-lab/src/agentic-parity-report.test.ts index 331a0c2ed014..064614ce53a3 100644 --- a/extensions/qa-lab/src/agentic-parity-report.test.ts +++ b/extensions/qa-lab/src/agentic-parity-report.test.ts @@ -135,6 +135,27 @@ describe("qa agentic parity report", () => { }); }); + it("uses scenario rows rather than stale summary counts for parity metrics", () => { + const summary: QaParitySuiteSummary = { + counts: { + total: 2, + passed: 2, + failed: 0, + }, + scenarios: [ + { name: "Approval turn tool followthrough", status: "pass" }, + { name: "Compaction retry after mutating tool", status: "fail" }, + ], + }; + + const metrics = computeQaAgenticParityMetrics(summary); + + expect(metrics.totalScenarios).toBe(2); + expect(metrics.passedScenarios).toBe(1); + expect(metrics.failedScenarios).toBe(1); + expect(metrics.completionRate).toBe(0.5); + }); + it("keeps non-tool scenarios out of the valid-tool-call metric", () => { const summary: QaParitySuiteSummary = { scenarios: [ diff --git a/extensions/qa-lab/src/agentic-parity-report.ts b/extensions/qa-lab/src/agentic-parity-report.ts index a9b0c42dff4e..d67337a6491a 100644 --- a/extensions/qa-lab/src/agentic-parity-report.ts +++ b/extensions/qa-lab/src/agentic-parity-report.ts @@ -177,11 +177,9 @@ export function computeQaAgenticParityMetrics( const toolBackedTitleSet: ReadonlySet = new Set( QA_AGENTIC_PARITY_TOOL_BACKED_SCENARIO_TITLES, ); - const totalScenarios = summary.counts?.total ?? scenarios.length; - const passedScenarios = - summary.counts?.passed ?? scenarios.filter((scenario) => scenario.status === "pass").length; - const failedScenarios = - summary.counts?.failed ?? scenarios.filter((scenario) => scenario.status === "fail").length; + const totalScenarios = scenarios.length; + const passedScenarios = scenarios.filter((scenario) => scenario.status === "pass").length; + const failedScenarios = scenarios.filter((scenario) => scenario.status === "fail").length; const unintendedStopCount = scenarios.filter( (scenario) => scenario.status !== "pass" && scenarioHasPattern(scenario, UNINTENDED_STOP_PATTERNS),