vultr/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-07-13 11:26:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(qa): trust parity scenario rows for metrics

Browse Source
This commit is contained in:
Vincent Koc
2026-06-07 02:59:19 +02:00
parent 51848de462
commit d034e9698a
2 changed files with 24 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
extensions/qa-lab/src/agentic-parity-report.test.ts
View File

@@ -135,6 +135,27 @@ describe("qa agentic parity report", () => {
});
});
it("uses scenario rows rather than stale summary counts for parity metrics", () => {
const summary: QaParitySuiteSummary = {
counts: {
total: 2,
passed: 2,
failed: 0,
},
scenarios: [
{ name: "Approval turn tool followthrough", status: "pass" },
{ name: "Compaction retry after mutating tool", status: "fail" },
],
};
const metrics = computeQaAgenticParityMetrics(summary);
expect(metrics.totalScenarios).toBe(2);
expect(metrics.passedScenarios).toBe(1);
expect(metrics.failedScenarios).toBe(1);
expect(metrics.completionRate).toBe(0.5);
});
it("keeps non-tool scenarios out of the valid-tool-call metric", () => {
const summary: QaParitySuiteSummary = {
scenarios: [

8
extensions/qa-lab/src/agentic-parity-report.ts
View File

@@ -177,11 +177,9 @@ export function computeQaAgenticParityMetrics(
const toolBackedTitleSet: ReadonlySet<string> = new Set(
QA_AGENTIC_PARITY_TOOL_BACKED_SCENARIO_TITLES,
);
const totalScenarios = summary.counts?.total ?? scenarios.length;
const passedScenarios =
summary.counts?.passed ?? scenarios.filter((scenario) => scenario.status === "pass").length;
const failedScenarios =
summary.counts?.failed ?? scenarios.filter((scenario) => scenario.status === "fail").length;
const totalScenarios = scenarios.length;
const passedScenarios = scenarios.filter((scenario) => scenario.status === "pass").length;
const failedScenarios = scenarios.filter((scenario) => scenario.status === "fail").length;
const unintendedStopCount = scenarios.filter(
(scenario) =>
scenario.status !== "pass" && scenarioHasPattern(scenario, UNINTENDED_STOP_PATTERNS),