fix(qa-lab): avoid duplicate child evidence files (#96030)

2026-07-13 21:36:04 +00:00 · 2026-06-23 01:18:24 -07:00
parent 3d8d45fb0d
commit 32dc664b4b
6 changed files with 206 additions and 100 deletions
--- a/extensions/qa-lab/src/suite-launch.runtime.test.ts
+++ b/extensions/qa-lab/src/suite-launch.runtime.test.ts
@@ -28,23 +28,19 @@ async function makeTempRepo(prefix: string) {
  return repoRoot;
 }

-async function writeEvidence(pathLocal: string) {
-  await fs.mkdir(path.dirname(pathLocal), { recursive: true });
-  await fs.writeFile(
-    pathLocal,
-    `${JSON.stringify(
-      {
-        kind: "openclaw.qa.evidence-summary",
-        schemaVersion: 2,
-        generatedAt: "2026-06-14T00:00:00.000Z",
-        evidenceMode: "full",
-        entries: [],
-      },
-      null,
-      2,
-    )}\n`,
-    "utf8",
-  );
+async function writeEvidence(pathLocal: string, writeFile = true) {
+  const evidence = {
+    kind: "openclaw.qa.evidence-summary",
+    schemaVersion: 2,
+    generatedAt: "2026-06-14T00:00:00.000Z",
+    evidenceMode: "full",
+    entries: [],
+  };
+  if (writeFile) {
+    await fs.mkdir(path.dirname(pathLocal), { recursive: true });
+    await fs.writeFile(pathLocal, `${JSON.stringify(evidence, null, 2)}\n`, "utf8");
+  }
+  return evidence;
 }

 describe("qa suite runtime launcher", () => {
@@ -52,12 +48,17 @@ describe("qa suite runtime launcher", () => {
    runQaFlowSuite.mockReset();
    runQaTestFileScenarios.mockReset();
    runQaFlowSuite.mockImplementation(
-      async (params: { outputDir?: string; scenarioIds?: string[] } | undefined) => {
+      async (
+        params:
+          | { outputDir?: string; scenarioIds?: string[]; writeEvidenceFile?: boolean }
+          | undefined,
+      ) => {
        const outputDir = params?.outputDir ?? "/tmp/qa-flow";
        const evidencePath = path.join(outputDir, "qa-evidence.json");
-        await writeEvidence(evidencePath);
+        const evidence = await writeEvidence(evidencePath, params?.writeEvidenceFile);
        const scenarioIds = params?.scenarioIds ?? ["channel-chat-baseline"];
        return {
+          evidence,
          outputDir,
          evidencePath,
          reportPath: path.join(outputDir, "qa-suite-report.md"),
@@ -76,14 +77,16 @@ describe("qa suite runtime launcher", () => {
      async (params: {
        outputDir: string;
        scenarios: Array<{ id: string; execution: { kind: "script" | "vitest" | "playwright" } }>;
+        writeEvidenceFile?: boolean;
      }) => {
        const [scenario] = params.scenarios;
        if (!scenario) {
          throw new Error("expected scenario");
        }
        const evidencePath = path.join(params.outputDir, "qa-evidence.json");
-        await writeEvidence(evidencePath);
+        const evidence = await writeEvidence(evidencePath, params.writeEvidenceFile);
        return {
+          evidence,
          outputDir: params.outputDir,
          executionKind: scenario.execution.kind,
          evidencePath,
@@ -247,15 +250,27 @@ describe("qa suite runtime launcher", () => {
      expect.objectContaining({
        outputDir: path.join(outputDir, "flow"),
        scenarioIds: ["channel-chat-baseline"],
+        writeEvidenceFile: false,
      }),
    );
    expect(runQaTestFileScenarios).toHaveBeenCalledWith(
      expect.objectContaining({
        outputDir: path.join(outputDir, "playwright"),
+        writeEvidenceFile: false,
      }),
    );
    await expect(fs.access(path.join(outputDir, "qa-suite-summary.json"))).resolves.toBeUndefined();
    await expect(fs.access(path.join(outputDir, "qa-evidence.json"))).resolves.toBeUndefined();
+    await expect(fs.access(path.join(outputDir, "flow", "qa-evidence.json"))).rejects.toMatchObject(
+      {
+        code: "ENOENT",
+      },
+    );
+    await expect(
+      fs.access(path.join(outputDir, "playwright", "qa-evidence.json")),
+    ).rejects.toMatchObject({
+      code: "ENOENT",
+    });
    const summary = JSON.parse(
      await fs.readFile(path.join(outputDir, "qa-suite-summary.json"), "utf8"),
    ) as {
--- a/extensions/qa-lab/src/suite-launch.runtime.ts
+++ b/extensions/qa-lab/src/suite-launch.runtime.ts
@@ -175,6 +175,7 @@ async function runQaTestFileSuiteFromRuntime(params: {
    providerMode,
    primaryModel,
    scenarios: params.scenarios,
+    writeEvidenceFile: runParams?.writeEvidenceFile,
  });
 }

@@ -292,6 +293,13 @@ async function readQaSuiteEvidenceSummary(evidencePath: string) {
  return validateQaEvidenceSummaryJson(JSON.parse(await fs.readFile(evidencePath, "utf8")));
 }

+async function resolveQaSuiteResultEvidenceSummary(result: {
+  evidence?: QaEvidenceSummaryJson;
+  evidencePath: string;
+}) {
+  return result.evidence ?? (await readQaSuiteEvidenceSummary(result.evidencePath));
+}
+
 function mergeQaEvidenceSummaries(params: {
  evidenceSummaries: readonly QaEvidenceSummaryJson[];
  generatedAt: string;
@@ -489,6 +497,7 @@ async function runUnifiedQaSuite(params: {
              flowPartitions.length === 1
                ? suitePartitionOutputDir(outputDir, "flow")
                : flowSuitePartitionOutputDir(outputDir, partition.kind),
+            writeEvidenceFile: false,
            providerMode,
            primaryModel,
            alternateModel,
@@ -512,7 +521,7 @@ async function runUnifiedQaSuite(params: {
            }
          }
          return {
-            evidenceSummaries: [await readQaSuiteEvidenceSummary(result.evidencePath)],
+            evidenceSummaries: [await resolveQaSuiteResultEvidenceSummary(result)],
            scenarioResults,
          };
        },
@@ -530,13 +539,14 @@ async function runUnifiedQaSuite(params: {
            runParams: {
              ...params.runParams,
              outputDir: suitePartitionOutputDir(outputDir, kind),
+              writeEvidenceFile: false,
              providerMode,
              primaryModel,
              scenarioIds: testFileScenarios.map((scenario) => scenario.id),
            },
            scenarios: testFileScenarios,
          });
-          testFileEvidenceSummaries.push(await readQaSuiteEvidenceSummary(result.evidencePath));
+          testFileEvidenceSummaries.push(await resolveQaSuiteResultEvidenceSummary(result));
          testFileScenarioResults.push(
            ...result.results.map((scenarioResult) => ({
              scenarioId: scenarioResult.scenario.id,
--- a/extensions/qa-lab/src/suite.test.ts
+++ b/extensions/qa-lab/src/suite.test.ts
@@ -274,6 +274,36 @@ describe("qa suite", () => {
    }
  });

+  it("can return evidence without writing duplicate child evidence files", async () => {
+    const outputDir = await tempDirs.makeTempDir("qa-suite-artifacts-memory-evidence-");
+    try {
+      const artifacts = await qaSuiteProgressTesting.writeQaSuiteArtifacts({
+        outputDir,
+        startedAt: new Date("2026-04-11T00:00:00.000Z"),
+        finishedAt: new Date("2026-04-11T00:01:00.000Z"),
+        scenarios: [{ name: "Baseline", status: "pass", steps: [] }],
+        scenarioDefinitions: [makeQaSuiteTestScenario("baseline")],
+        transport: {
+          id: "qa-channel",
+          createReportNotes: () => [],
+        } as unknown as QaTransportAdapter,
+        providerMode: "mock-openai",
+        primaryModel: "mock-openai/gpt-5.5",
+        alternateModel: "mock-openai/gpt-5.5-alt",
+        fastMode: true,
+        concurrency: 1,
+        writeEvidenceFile: false,
+      });
+
+      expect(artifacts.evidence?.kind).toBe(QA_EVIDENCE_SUMMARY_KIND);
+      await expect(fs.access(artifacts.evidencePath)).rejects.toMatchObject({ code: "ENOENT" });
+      await expect(fs.access(artifacts.reportPath)).resolves.toBeUndefined();
+      await expect(fs.access(artifacts.summaryPath)).resolves.toBeUndefined();
+    } finally {
+      await fs.rm(outputDir, { recursive: true, force: true });
+    }
+  });
+
  it("writes Crabline channel-driver smoke artifacts when selected", async () => {
    const outputDir = await tempDirs.makeTempDir("qa-suite-crabline-");
    try {
@@ -433,6 +463,7 @@ describe("qa suite", () => {
          enabledPluginIds: ["acpx"],
          transportReadyTimeoutMs: 180_000,
          forcedRuntime: "codex",
+          writeEvidenceFile: false,
        },
      }),
    ).toMatchObject({
@@ -445,6 +476,7 @@ describe("qa suite", () => {
      enabledPluginIds: ["acpx"],
      transportReadyTimeoutMs: 180_000,
      forcedRuntime: "codex",
+      writeEvidenceFile: false,
    });
  });

--- a/extensions/qa-lab/src/suite.ts
+++ b/extensions/qa-lab/src/suite.ts
@@ -18,7 +18,11 @@ import {
 } from "openclaw/plugin-sdk/qa-runtime";
 import { fetchWithSsrFGuard } from "openclaw/plugin-sdk/ssrf-runtime";
 import { assertQaSuiteArtifactWritten } from "./artifact-assertion.js";
-import { buildQaSuiteEvidenceSummary, QA_EVIDENCE_FILENAME } from "./evidence-summary.js";
+import {
+  buildQaSuiteEvidenceSummary,
+  QA_EVIDENCE_FILENAME,
+  type QaEvidenceSummaryJson,
+} from "./evidence-summary.js";
 import { startQaGatewayChild, type QaCliBackendAuthMode } from "./gateway-child.js";
 import type {
  QaLabLatestReport,
@@ -154,6 +158,9 @@ export type QaSuiteRunParams = {
  forcedRuntime?: RuntimeId;
  runtimePair?: [RuntimeId, RuntimeId];
  captureRuntimeParityCell?: boolean;
+  // Unified suite partitions consume child evidence in memory; only the
+  // parent should write the aggregate qa-evidence.json artifact.
+  writeEvidenceFile?: boolean;
 };

 function shouldLogQaSuiteProgress(env: NodeJS.ProcessEnv = process.env) {
@@ -280,6 +287,7 @@ function liveTurnTimeoutMs(
 }

 export type QaSuiteResult = {
+  evidence?: QaEvidenceSummaryJson;
  outputDir: string;
  evidencePath: string;
  reportPath: string;
@@ -470,6 +478,7 @@ function buildQaIsolatedScenarioWorkerParams(params: {
    transportReadyTimeoutMs: params.input?.transportReadyTimeoutMs,
    workerStartStaggerMs: params.input?.workerStartStaggerMs,
    forcedRuntime: params.input?.forcedRuntime,
+    writeEvidenceFile: params.input?.writeEvidenceFile,
  };
 }

@@ -814,27 +823,29 @@ async function runQaRuntimeParitySuite(params: {
    );

    const finishedAt = new Date();
-    const { evidencePath, report, reportPath, summaryPath } = await writeQaSuiteArtifacts({
-      outputDir: params.outputDir,
-      startedAt: params.startedAt,
-      finishedAt,
-      scenarios,
-      scenarioDefinitions: params.selectedScenarios,
-      evidenceMode: params.evidenceMode,
-      transport,
-      providerMode: params.providerMode,
-      primaryModel: params.primaryModel,
-      alternateModel: params.alternateModel,
-      fastMode: params.fastMode,
-      concurrency: params.concurrency,
-      channelDriver: params.channelDriver,
-      channelDriverSelection: params.channelDriverSelection,
-      scenarioIds:
-        params.scenarioIds && params.scenarioIds.length > 0
-          ? params.selectedScenarios.map((scenario) => scenario.id)
-          : undefined,
-      runtimePair: params.runtimePair,
-    });
+    const { evidence, evidencePath, report, reportPath, summaryPath } = await writeQaSuiteArtifacts(
+      {
+        outputDir: params.outputDir,
+        startedAt: params.startedAt,
+        finishedAt,
+        scenarios,
+        scenarioDefinitions: params.selectedScenarios,
+        evidenceMode: params.evidenceMode,
+        transport,
+        providerMode: params.providerMode,
+        primaryModel: params.primaryModel,
+        alternateModel: params.alternateModel,
+        fastMode: params.fastMode,
+        concurrency: params.concurrency,
+        channelDriver: params.channelDriver,
+        channelDriverSelection: params.channelDriverSelection,
+        scenarioIds:
+          params.scenarioIds && params.scenarioIds.length > 0
+            ? params.selectedScenarios.map((scenario) => scenario.id)
+            : undefined,
+        runtimePair: params.runtimePair,
+      },
+    );
    lab.setLatestReport({
      outputPath: reportPath,
      markdown: report,
@@ -849,6 +860,7 @@ async function runQaRuntimeParitySuite(params: {
    });
    return {
      outputDir: params.outputDir,
+      evidence,
      evidencePath,
      reportPath,
      summaryPath,
@@ -887,6 +899,7 @@ async function writeQaSuiteArtifacts(params: {
  isolatedWorkers?: boolean;
  scenarioIds?: readonly string[];
  runtimePair?: [RuntimeId, RuntimeId];
+  writeEvidenceFile?: boolean;
 }) {
  const reportPath = path.join(params.outputDir, "qa-suite-report.md");
  const summaryPath = path.join(params.outputDir, "qa-suite-summary.json");
@@ -976,8 +989,9 @@ async function writeQaSuiteArtifacts(params: {
      "utf8",
    );
  }
+  const writeEvidenceFile = params.writeEvidenceFile ?? true;
  await fs.writeFile(reportPath, report, "utf8");
-  if (evidence) {
+  if (evidence && writeEvidenceFile) {
    await fs.writeFile(evidencePath, `${JSON.stringify(evidence, null, 2)}\n`, "utf8");
  }
  await fs.writeFile(
@@ -987,10 +1001,10 @@ async function writeQaSuiteArtifacts(params: {
  );
  await assertQaSuiteArtifactWritten("report", reportPath);
  await assertQaSuiteArtifactWritten("summary", summaryPath);
-  if (evidence) {
+  if (evidence && writeEvidenceFile) {
    await assertQaSuiteArtifactWritten("evidence", evidencePath);
  }
-  return { evidencePath, report, reportPath, summaryPath };
+  return { evidence, evidencePath, report, reportPath, summaryPath };
 }

 function buildQaSuiteRuntimeMetrics(params: {
@@ -1268,6 +1282,7 @@ export async function runQaFlowSuite(params?: QaSuiteRunParams): Promise<QaSuite
            channelDriver: params?.channelDriver,
            channelDriverSelection: params?.channelDriverSelection,
            isolatedWorkers: true,
+            writeEvidenceFile: params?.writeEvidenceFile,
            scenarioIds:
              params?.scenarioIds && params.scenarioIds.length > 0
                ? selectedScenarios.map((scenario) => scenario.id)
@@ -1402,33 +1417,35 @@ export async function runQaFlowSuite(params?: QaSuiteRunParams): Promise<QaSuite
        finishedAt: finishedAt.toISOString(),
        scenarios: [...liveScenarioOutcomes],
      });
-      const { evidencePath, report, reportPath, summaryPath } = await writeQaSuiteArtifacts({
-        outputDir,
-        startedAt,
-        finishedAt,
-        scenarios,
-        scenarioDefinitions: selectedScenarios,
-        evidenceMode: params?.evidenceMode,
-        transport,
-        providerMode,
-        primaryModel,
-        alternateModel,
-        fastMode,
-        concurrency,
-        channelDriver: params?.channelDriver,
-        channelDriverSelection: params?.channelDriverSelection,
-        isolatedWorkers: true,
-        // When the caller supplied an explicit non-empty --scenario filter,
-        // record the executed (post-selectQaFlowSuiteScenarios-normalized) ids
-        // so the summary matches what actually ran. When the caller passed
-        // nothing or an empty array ("no filter, full lane catalog"),
-        // preserve the unfiltered = null semantic so the summary stays
-        // distinguishable from an explicit all-scenarios selection.
-        scenarioIds:
-          params?.scenarioIds && params.scenarioIds.length > 0
-            ? selectedScenarios.map((scenario) => scenario.id)
-            : undefined,
-      });
+      const { evidence, evidencePath, report, reportPath, summaryPath } =
+        await writeQaSuiteArtifacts({
+          outputDir,
+          startedAt,
+          finishedAt,
+          scenarios,
+          scenarioDefinitions: selectedScenarios,
+          evidenceMode: params?.evidenceMode,
+          transport,
+          providerMode,
+          primaryModel,
+          alternateModel,
+          fastMode,
+          concurrency,
+          channelDriver: params?.channelDriver,
+          channelDriverSelection: params?.channelDriverSelection,
+          isolatedWorkers: true,
+          writeEvidenceFile: params?.writeEvidenceFile,
+          // When the caller supplied an explicit non-empty --scenario filter,
+          // record the executed (post-selectQaFlowSuiteScenarios-normalized) ids
+          // so the summary matches what actually ran. When the caller passed
+          // nothing or an empty array ("no filter, full lane catalog"),
+          // preserve the unfiltered = null semantic so the summary stays
+          // distinguishable from an explicit all-scenarios selection.
+          scenarioIds:
+            params?.scenarioIds && params.scenarioIds.length > 0
+              ? selectedScenarios.map((scenario) => scenario.id)
+              : undefined,
+        });
      lab.setLatestReport({
        outputPath: reportPath,
        markdown: report,
@@ -1440,6 +1457,7 @@ export async function runQaFlowSuite(params?: QaSuiteRunParams): Promise<QaSuite
      );
      return {
        outputDir,
+        evidence,
        evidencePath,
        reportPath,
        summaryPath,
@@ -1671,30 +1689,33 @@ export async function runQaFlowSuite(params?: QaSuiteRunParams): Promise<QaSuite
      finishedAt: finishedAt.toISOString(),
      scenarios: [...liveScenarioOutcomes],
    });
-    const { evidencePath, report, reportPath, summaryPath } = await writeQaSuiteArtifacts({
-      outputDir,
-      startedAt,
-      finishedAt,
-      scenarios,
-      metrics,
-      scenarioDefinitions: selectedScenarios,
-      evidenceMode: params?.evidenceMode,
-      transport,
-      providerMode,
-      primaryModel,
-      alternateModel,
-      fastMode,
-      concurrency,
-      channelDriver: params?.channelDriver,
-      channelDriverSelection: params?.channelDriverSelection,
-      isolatedWorkers: false,
-      // Same "filtered → executed list, unfiltered → null" convention as
-      // the concurrent-path writeQaSuiteArtifacts call above.
-      scenarioIds:
-        params?.scenarioIds && params.scenarioIds.length > 0
-          ? selectedScenarios.map((scenario) => scenario.id)
-          : undefined,
-    });
+    const { evidence, evidencePath, report, reportPath, summaryPath } = await writeQaSuiteArtifacts(
+      {
+        outputDir,
+        startedAt,
+        finishedAt,
+        scenarios,
+        metrics,
+        scenarioDefinitions: selectedScenarios,
+        evidenceMode: params?.evidenceMode,
+        transport,
+        providerMode,
+        primaryModel,
+        alternateModel,
+        fastMode,
+        concurrency,
+        channelDriver: params?.channelDriver,
+        channelDriverSelection: params?.channelDriverSelection,
+        isolatedWorkers: false,
+        writeEvidenceFile: params?.writeEvidenceFile,
+        // Same "filtered → executed list, unfiltered → null" convention as
+        // the concurrent-path writeQaSuiteArtifacts call above.
+        scenarioIds:
+          params?.scenarioIds && params.scenarioIds.length > 0
+            ? selectedScenarios.map((scenario) => scenario.id)
+            : undefined,
+      },
+    );
    const latestReport = {
      outputPath: reportPath,
      markdown: report,
@@ -1708,6 +1729,7 @@ export async function runQaFlowSuite(params?: QaSuiteRunParams): Promise<QaSuite

    return {
      outputDir,
+      evidence,
      evidencePath,
      reportPath,
      summaryPath,
--- a/extensions/qa-lab/src/test-file-scenario-runner.test.ts
+++ b/extensions/qa-lab/src/test-file-scenario-runner.test.ts
@@ -241,6 +241,26 @@ describe("qa test file scenario runner", () => {
    });
  });

+  it("can return aggregate evidence without writing a duplicate evidence file", async () => {
+    const repoRoot = await makeTempRepo("qa-playwright-memory-evidence-");
+    const result = await runQaTestFileScenarios({
+      repoRoot,
+      outputDir: path.join(repoRoot, ".artifacts", "qa-e2e", "scenario-playwright"),
+      providerMode: "mock-openai",
+      primaryModel: "mock-openai/gpt-5.5",
+      scenarios: [makeTestFileScenario("playwright", "ui/src/ui/e2e/chat-flow.e2e.test.ts")],
+      writeEvidenceFile: false,
+      runCommand: async () => ({
+        exitCode: 0,
+        stdout: "pass\n",
+        stderr: "",
+      }),
+    });
+
+    expect(result.evidence.entries).toHaveLength(1);
+    await expect(fs.access(result.evidencePath)).rejects.toMatchObject({ code: "ENOENT" });
+  });
+
  it("runs Vitest scenarios with the declared test path and writes Vitest evidence", async () => {
    const repoRoot = await makeTempRepo("qa-vitest-scenario-");
    const commands: QaScenarioCommandExecution[] = [];
--- a/extensions/qa-lab/src/test-file-scenario-runner.ts
+++ b/extensions/qa-lab/src/test-file-scenario-runner.ts
@@ -42,6 +42,7 @@ export type QaTestFileScenarioRunParams = {
  repoRoot: string;
  runCommand?: QaScenarioCommandRunner;
  scenarios: readonly QaSeedScenarioWithSource[];
+  writeEvidenceFile?: boolean;
 };

 export type QaScenarioCommandExecution = {
@@ -80,6 +81,7 @@ type QaTestFileScenarioResult = {
 };

 export type QaTestFileScenarioRunResult = {
+  evidence: QaEvidenceSummaryJson;
  evidencePath: string;
  executionKind: QaTestFileExecutionKind;
  outputDir: string;
@@ -746,10 +748,13 @@ function buildScenarioArtifactPaths(params: {
 async function writeTestFileEvidenceFile(params: {
  evidence: unknown;
  outputDir: string;
+  writeEvidenceFile?: boolean;
 }): Promise<Pick<QaTestFileScenarioRunResult, "evidencePath">> {
  const evidencePath = path.join(params.outputDir, QA_EVIDENCE_FILENAME);
-  await fs.writeFile(evidencePath, `${JSON.stringify(params.evidence, null, 2)}\n`, "utf8");
-  await assertQaSuiteArtifactWritten("evidence", evidencePath);
+  if (params.writeEvidenceFile ?? true) {
+    await fs.writeFile(evidencePath, `${JSON.stringify(params.evidence, null, 2)}\n`, "utf8");
+    await assertQaSuiteArtifactWritten("evidence", evidencePath);
+  }
  return { evidencePath };
 }

@@ -802,9 +807,11 @@ export async function runQaTestFileScenarios(
  const paths = await writeTestFileEvidenceFile({
    evidence,
    outputDir: params.outputDir,
+    writeEvidenceFile: params.writeEvidenceFile,
  });
  return {
    ...paths,
+    evidence,
    executionKind: kind,
    outputDir: params.outputDir,
    results,