diff --git a/extensions/qa-lab/src/suite-launch.runtime.test.ts b/extensions/qa-lab/src/suite-launch.runtime.test.ts index 8bd9e050578f..0cff5074dcfe 100644 --- a/extensions/qa-lab/src/suite-launch.runtime.test.ts +++ b/extensions/qa-lab/src/suite-launch.runtime.test.ts @@ -28,23 +28,19 @@ async function makeTempRepo(prefix: string) { return repoRoot; } -async function writeEvidence(pathLocal: string) { - await fs.mkdir(path.dirname(pathLocal), { recursive: true }); - await fs.writeFile( - pathLocal, - `${JSON.stringify( - { - kind: "openclaw.qa.evidence-summary", - schemaVersion: 2, - generatedAt: "2026-06-14T00:00:00.000Z", - evidenceMode: "full", - entries: [], - }, - null, - 2, - )}\n`, - "utf8", - ); +async function writeEvidence(pathLocal: string, writeFile = true) { + const evidence = { + kind: "openclaw.qa.evidence-summary", + schemaVersion: 2, + generatedAt: "2026-06-14T00:00:00.000Z", + evidenceMode: "full", + entries: [], + }; + if (writeFile) { + await fs.mkdir(path.dirname(pathLocal), { recursive: true }); + await fs.writeFile(pathLocal, `${JSON.stringify(evidence, null, 2)}\n`, "utf8"); + } + return evidence; } describe("qa suite runtime launcher", () => { @@ -52,12 +48,17 @@ describe("qa suite runtime launcher", () => { runQaFlowSuite.mockReset(); runQaTestFileScenarios.mockReset(); runQaFlowSuite.mockImplementation( - async (params: { outputDir?: string; scenarioIds?: string[] } | undefined) => { + async ( + params: + | { outputDir?: string; scenarioIds?: string[]; writeEvidenceFile?: boolean } + | undefined, + ) => { const outputDir = params?.outputDir ?? "/tmp/qa-flow"; const evidencePath = path.join(outputDir, "qa-evidence.json"); - await writeEvidence(evidencePath); + const evidence = await writeEvidence(evidencePath, params?.writeEvidenceFile); const scenarioIds = params?.scenarioIds ?? ["channel-chat-baseline"]; return { + evidence, outputDir, evidencePath, reportPath: path.join(outputDir, "qa-suite-report.md"), @@ -76,14 +77,16 @@ describe("qa suite runtime launcher", () => { async (params: { outputDir: string; scenarios: Array<{ id: string; execution: { kind: "script" | "vitest" | "playwright" } }>; + writeEvidenceFile?: boolean; }) => { const [scenario] = params.scenarios; if (!scenario) { throw new Error("expected scenario"); } const evidencePath = path.join(params.outputDir, "qa-evidence.json"); - await writeEvidence(evidencePath); + const evidence = await writeEvidence(evidencePath, params.writeEvidenceFile); return { + evidence, outputDir: params.outputDir, executionKind: scenario.execution.kind, evidencePath, @@ -247,15 +250,27 @@ describe("qa suite runtime launcher", () => { expect.objectContaining({ outputDir: path.join(outputDir, "flow"), scenarioIds: ["channel-chat-baseline"], + writeEvidenceFile: false, }), ); expect(runQaTestFileScenarios).toHaveBeenCalledWith( expect.objectContaining({ outputDir: path.join(outputDir, "playwright"), + writeEvidenceFile: false, }), ); await expect(fs.access(path.join(outputDir, "qa-suite-summary.json"))).resolves.toBeUndefined(); await expect(fs.access(path.join(outputDir, "qa-evidence.json"))).resolves.toBeUndefined(); + await expect(fs.access(path.join(outputDir, "flow", "qa-evidence.json"))).rejects.toMatchObject( + { + code: "ENOENT", + }, + ); + await expect( + fs.access(path.join(outputDir, "playwright", "qa-evidence.json")), + ).rejects.toMatchObject({ + code: "ENOENT", + }); const summary = JSON.parse( await fs.readFile(path.join(outputDir, "qa-suite-summary.json"), "utf8"), ) as { diff --git a/extensions/qa-lab/src/suite-launch.runtime.ts b/extensions/qa-lab/src/suite-launch.runtime.ts index a719f22a9349..e759a0f62c3a 100644 --- a/extensions/qa-lab/src/suite-launch.runtime.ts +++ b/extensions/qa-lab/src/suite-launch.runtime.ts @@ -175,6 +175,7 @@ async function runQaTestFileSuiteFromRuntime(params: { providerMode, primaryModel, scenarios: params.scenarios, + writeEvidenceFile: runParams?.writeEvidenceFile, }); } @@ -292,6 +293,13 @@ async function readQaSuiteEvidenceSummary(evidencePath: string) { return validateQaEvidenceSummaryJson(JSON.parse(await fs.readFile(evidencePath, "utf8"))); } +async function resolveQaSuiteResultEvidenceSummary(result: { + evidence?: QaEvidenceSummaryJson; + evidencePath: string; +}) { + return result.evidence ?? (await readQaSuiteEvidenceSummary(result.evidencePath)); +} + function mergeQaEvidenceSummaries(params: { evidenceSummaries: readonly QaEvidenceSummaryJson[]; generatedAt: string; @@ -489,6 +497,7 @@ async function runUnifiedQaSuite(params: { flowPartitions.length === 1 ? suitePartitionOutputDir(outputDir, "flow") : flowSuitePartitionOutputDir(outputDir, partition.kind), + writeEvidenceFile: false, providerMode, primaryModel, alternateModel, @@ -512,7 +521,7 @@ async function runUnifiedQaSuite(params: { } } return { - evidenceSummaries: [await readQaSuiteEvidenceSummary(result.evidencePath)], + evidenceSummaries: [await resolveQaSuiteResultEvidenceSummary(result)], scenarioResults, }; }, @@ -530,13 +539,14 @@ async function runUnifiedQaSuite(params: { runParams: { ...params.runParams, outputDir: suitePartitionOutputDir(outputDir, kind), + writeEvidenceFile: false, providerMode, primaryModel, scenarioIds: testFileScenarios.map((scenario) => scenario.id), }, scenarios: testFileScenarios, }); - testFileEvidenceSummaries.push(await readQaSuiteEvidenceSummary(result.evidencePath)); + testFileEvidenceSummaries.push(await resolveQaSuiteResultEvidenceSummary(result)); testFileScenarioResults.push( ...result.results.map((scenarioResult) => ({ scenarioId: scenarioResult.scenario.id, diff --git a/extensions/qa-lab/src/suite.test.ts b/extensions/qa-lab/src/suite.test.ts index 317e6f41c1e0..98daf8e202c6 100644 --- a/extensions/qa-lab/src/suite.test.ts +++ b/extensions/qa-lab/src/suite.test.ts @@ -274,6 +274,36 @@ describe("qa suite", () => { } }); + it("can return evidence without writing duplicate child evidence files", async () => { + const outputDir = await tempDirs.makeTempDir("qa-suite-artifacts-memory-evidence-"); + try { + const artifacts = await qaSuiteProgressTesting.writeQaSuiteArtifacts({ + outputDir, + startedAt: new Date("2026-04-11T00:00:00.000Z"), + finishedAt: new Date("2026-04-11T00:01:00.000Z"), + scenarios: [{ name: "Baseline", status: "pass", steps: [] }], + scenarioDefinitions: [makeQaSuiteTestScenario("baseline")], + transport: { + id: "qa-channel", + createReportNotes: () => [], + } as unknown as QaTransportAdapter, + providerMode: "mock-openai", + primaryModel: "mock-openai/gpt-5.5", + alternateModel: "mock-openai/gpt-5.5-alt", + fastMode: true, + concurrency: 1, + writeEvidenceFile: false, + }); + + expect(artifacts.evidence?.kind).toBe(QA_EVIDENCE_SUMMARY_KIND); + await expect(fs.access(artifacts.evidencePath)).rejects.toMatchObject({ code: "ENOENT" }); + await expect(fs.access(artifacts.reportPath)).resolves.toBeUndefined(); + await expect(fs.access(artifacts.summaryPath)).resolves.toBeUndefined(); + } finally { + await fs.rm(outputDir, { recursive: true, force: true }); + } + }); + it("writes Crabline channel-driver smoke artifacts when selected", async () => { const outputDir = await tempDirs.makeTempDir("qa-suite-crabline-"); try { @@ -433,6 +463,7 @@ describe("qa suite", () => { enabledPluginIds: ["acpx"], transportReadyTimeoutMs: 180_000, forcedRuntime: "codex", + writeEvidenceFile: false, }, }), ).toMatchObject({ @@ -445,6 +476,7 @@ describe("qa suite", () => { enabledPluginIds: ["acpx"], transportReadyTimeoutMs: 180_000, forcedRuntime: "codex", + writeEvidenceFile: false, }); }); diff --git a/extensions/qa-lab/src/suite.ts b/extensions/qa-lab/src/suite.ts index 854a7d4c6da6..f5befef08823 100644 --- a/extensions/qa-lab/src/suite.ts +++ b/extensions/qa-lab/src/suite.ts @@ -18,7 +18,11 @@ import { } from "openclaw/plugin-sdk/qa-runtime"; import { fetchWithSsrFGuard } from "openclaw/plugin-sdk/ssrf-runtime"; import { assertQaSuiteArtifactWritten } from "./artifact-assertion.js"; -import { buildQaSuiteEvidenceSummary, QA_EVIDENCE_FILENAME } from "./evidence-summary.js"; +import { + buildQaSuiteEvidenceSummary, + QA_EVIDENCE_FILENAME, + type QaEvidenceSummaryJson, +} from "./evidence-summary.js"; import { startQaGatewayChild, type QaCliBackendAuthMode } from "./gateway-child.js"; import type { QaLabLatestReport, @@ -154,6 +158,9 @@ export type QaSuiteRunParams = { forcedRuntime?: RuntimeId; runtimePair?: [RuntimeId, RuntimeId]; captureRuntimeParityCell?: boolean; + // Unified suite partitions consume child evidence in memory; only the + // parent should write the aggregate qa-evidence.json artifact. + writeEvidenceFile?: boolean; }; function shouldLogQaSuiteProgress(env: NodeJS.ProcessEnv = process.env) { @@ -280,6 +287,7 @@ function liveTurnTimeoutMs( } export type QaSuiteResult = { + evidence?: QaEvidenceSummaryJson; outputDir: string; evidencePath: string; reportPath: string; @@ -470,6 +478,7 @@ function buildQaIsolatedScenarioWorkerParams(params: { transportReadyTimeoutMs: params.input?.transportReadyTimeoutMs, workerStartStaggerMs: params.input?.workerStartStaggerMs, forcedRuntime: params.input?.forcedRuntime, + writeEvidenceFile: params.input?.writeEvidenceFile, }; } @@ -814,27 +823,29 @@ async function runQaRuntimeParitySuite(params: { ); const finishedAt = new Date(); - const { evidencePath, report, reportPath, summaryPath } = await writeQaSuiteArtifacts({ - outputDir: params.outputDir, - startedAt: params.startedAt, - finishedAt, - scenarios, - scenarioDefinitions: params.selectedScenarios, - evidenceMode: params.evidenceMode, - transport, - providerMode: params.providerMode, - primaryModel: params.primaryModel, - alternateModel: params.alternateModel, - fastMode: params.fastMode, - concurrency: params.concurrency, - channelDriver: params.channelDriver, - channelDriverSelection: params.channelDriverSelection, - scenarioIds: - params.scenarioIds && params.scenarioIds.length > 0 - ? params.selectedScenarios.map((scenario) => scenario.id) - : undefined, - runtimePair: params.runtimePair, - }); + const { evidence, evidencePath, report, reportPath, summaryPath } = await writeQaSuiteArtifacts( + { + outputDir: params.outputDir, + startedAt: params.startedAt, + finishedAt, + scenarios, + scenarioDefinitions: params.selectedScenarios, + evidenceMode: params.evidenceMode, + transport, + providerMode: params.providerMode, + primaryModel: params.primaryModel, + alternateModel: params.alternateModel, + fastMode: params.fastMode, + concurrency: params.concurrency, + channelDriver: params.channelDriver, + channelDriverSelection: params.channelDriverSelection, + scenarioIds: + params.scenarioIds && params.scenarioIds.length > 0 + ? params.selectedScenarios.map((scenario) => scenario.id) + : undefined, + runtimePair: params.runtimePair, + }, + ); lab.setLatestReport({ outputPath: reportPath, markdown: report, @@ -849,6 +860,7 @@ async function runQaRuntimeParitySuite(params: { }); return { outputDir: params.outputDir, + evidence, evidencePath, reportPath, summaryPath, @@ -887,6 +899,7 @@ async function writeQaSuiteArtifacts(params: { isolatedWorkers?: boolean; scenarioIds?: readonly string[]; runtimePair?: [RuntimeId, RuntimeId]; + writeEvidenceFile?: boolean; }) { const reportPath = path.join(params.outputDir, "qa-suite-report.md"); const summaryPath = path.join(params.outputDir, "qa-suite-summary.json"); @@ -976,8 +989,9 @@ async function writeQaSuiteArtifacts(params: { "utf8", ); } + const writeEvidenceFile = params.writeEvidenceFile ?? true; await fs.writeFile(reportPath, report, "utf8"); - if (evidence) { + if (evidence && writeEvidenceFile) { await fs.writeFile(evidencePath, `${JSON.stringify(evidence, null, 2)}\n`, "utf8"); } await fs.writeFile( @@ -987,10 +1001,10 @@ async function writeQaSuiteArtifacts(params: { ); await assertQaSuiteArtifactWritten("report", reportPath); await assertQaSuiteArtifactWritten("summary", summaryPath); - if (evidence) { + if (evidence && writeEvidenceFile) { await assertQaSuiteArtifactWritten("evidence", evidencePath); } - return { evidencePath, report, reportPath, summaryPath }; + return { evidence, evidencePath, report, reportPath, summaryPath }; } function buildQaSuiteRuntimeMetrics(params: { @@ -1268,6 +1282,7 @@ export async function runQaFlowSuite(params?: QaSuiteRunParams): Promise 0 ? selectedScenarios.map((scenario) => scenario.id) @@ -1402,33 +1417,35 @@ export async function runQaFlowSuite(params?: QaSuiteRunParams): Promise 0 - ? selectedScenarios.map((scenario) => scenario.id) - : undefined, - }); + const { evidence, evidencePath, report, reportPath, summaryPath } = + await writeQaSuiteArtifacts({ + outputDir, + startedAt, + finishedAt, + scenarios, + scenarioDefinitions: selectedScenarios, + evidenceMode: params?.evidenceMode, + transport, + providerMode, + primaryModel, + alternateModel, + fastMode, + concurrency, + channelDriver: params?.channelDriver, + channelDriverSelection: params?.channelDriverSelection, + isolatedWorkers: true, + writeEvidenceFile: params?.writeEvidenceFile, + // When the caller supplied an explicit non-empty --scenario filter, + // record the executed (post-selectQaFlowSuiteScenarios-normalized) ids + // so the summary matches what actually ran. When the caller passed + // nothing or an empty array ("no filter, full lane catalog"), + // preserve the unfiltered = null semantic so the summary stays + // distinguishable from an explicit all-scenarios selection. + scenarioIds: + params?.scenarioIds && params.scenarioIds.length > 0 + ? selectedScenarios.map((scenario) => scenario.id) + : undefined, + }); lab.setLatestReport({ outputPath: reportPath, markdown: report, @@ -1440,6 +1457,7 @@ export async function runQaFlowSuite(params?: QaSuiteRunParams): Promise 0 - ? selectedScenarios.map((scenario) => scenario.id) - : undefined, - }); + const { evidence, evidencePath, report, reportPath, summaryPath } = await writeQaSuiteArtifacts( + { + outputDir, + startedAt, + finishedAt, + scenarios, + metrics, + scenarioDefinitions: selectedScenarios, + evidenceMode: params?.evidenceMode, + transport, + providerMode, + primaryModel, + alternateModel, + fastMode, + concurrency, + channelDriver: params?.channelDriver, + channelDriverSelection: params?.channelDriverSelection, + isolatedWorkers: false, + writeEvidenceFile: params?.writeEvidenceFile, + // Same "filtered → executed list, unfiltered → null" convention as + // the concurrent-path writeQaSuiteArtifacts call above. + scenarioIds: + params?.scenarioIds && params.scenarioIds.length > 0 + ? selectedScenarios.map((scenario) => scenario.id) + : undefined, + }, + ); const latestReport = { outputPath: reportPath, markdown: report, @@ -1708,6 +1729,7 @@ export async function runQaFlowSuite(params?: QaSuiteRunParams): Promise { }); }); + it("can return aggregate evidence without writing a duplicate evidence file", async () => { + const repoRoot = await makeTempRepo("qa-playwright-memory-evidence-"); + const result = await runQaTestFileScenarios({ + repoRoot, + outputDir: path.join(repoRoot, ".artifacts", "qa-e2e", "scenario-playwright"), + providerMode: "mock-openai", + primaryModel: "mock-openai/gpt-5.5", + scenarios: [makeTestFileScenario("playwright", "ui/src/ui/e2e/chat-flow.e2e.test.ts")], + writeEvidenceFile: false, + runCommand: async () => ({ + exitCode: 0, + stdout: "pass\n", + stderr: "", + }), + }); + + expect(result.evidence.entries).toHaveLength(1); + await expect(fs.access(result.evidencePath)).rejects.toMatchObject({ code: "ENOENT" }); + }); + it("runs Vitest scenarios with the declared test path and writes Vitest evidence", async () => { const repoRoot = await makeTempRepo("qa-vitest-scenario-"); const commands: QaScenarioCommandExecution[] = []; diff --git a/extensions/qa-lab/src/test-file-scenario-runner.ts b/extensions/qa-lab/src/test-file-scenario-runner.ts index 5fcde0a43d78..6232432a4628 100644 --- a/extensions/qa-lab/src/test-file-scenario-runner.ts +++ b/extensions/qa-lab/src/test-file-scenario-runner.ts @@ -42,6 +42,7 @@ export type QaTestFileScenarioRunParams = { repoRoot: string; runCommand?: QaScenarioCommandRunner; scenarios: readonly QaSeedScenarioWithSource[]; + writeEvidenceFile?: boolean; }; export type QaScenarioCommandExecution = { @@ -80,6 +81,7 @@ type QaTestFileScenarioResult = { }; export type QaTestFileScenarioRunResult = { + evidence: QaEvidenceSummaryJson; evidencePath: string; executionKind: QaTestFileExecutionKind; outputDir: string; @@ -746,10 +748,13 @@ function buildScenarioArtifactPaths(params: { async function writeTestFileEvidenceFile(params: { evidence: unknown; outputDir: string; + writeEvidenceFile?: boolean; }): Promise> { const evidencePath = path.join(params.outputDir, QA_EVIDENCE_FILENAME); - await fs.writeFile(evidencePath, `${JSON.stringify(params.evidence, null, 2)}\n`, "utf8"); - await assertQaSuiteArtifactWritten("evidence", evidencePath); + if (params.writeEvidenceFile ?? true) { + await fs.writeFile(evidencePath, `${JSON.stringify(params.evidence, null, 2)}\n`, "utf8"); + await assertQaSuiteArtifactWritten("evidence", evidencePath); + } return { evidencePath }; } @@ -802,9 +807,11 @@ export async function runQaTestFileScenarios( const paths = await writeTestFileEvidenceFile({ evidence, outputDir: params.outputDir, + writeEvidenceFile: params.writeEvidenceFile, }); return { ...paths, + evidence, executionKind: kind, outputDir: params.outputDir, results,