From 8924e27bb6ea18e63bc3487793e4789a2cd815fa Mon Sep 17 00:00:00 2001 From: Dallin Romney Date: Thu, 16 Jul 2026 13:51:18 -0700 Subject: [PATCH] refactor(qa): keep synthetic UX evidence out of scenario catalog (#109315) * refactor(qa): keep UX evidence producer out of catalog * test(qa): preserve UX producer runner integration --- .../qa-lab/src/evidence-gallery.test.ts | 43 ++++--- .../qa-lab/src/scenario-catalog.test.ts | 9 -- .../src/test-file-scenario-runner.test.ts | 112 ++++++++---------- extensions/qa-lab/web/src/ui-render.test.ts | 16 +-- .../ui/ux-matrix-evidence-dashboard.yaml | 35 ------ scripts/qa/ux-matrix-evidence-producer.ts | 50 +++----- .../qa-ux-matrix-evidence-producer.test.ts | 37 ++++++ 7 files changed, 135 insertions(+), 167 deletions(-) delete mode 100644 qa/scenarios/ui/ux-matrix-evidence-dashboard.yaml diff --git a/extensions/qa-lab/src/evidence-gallery.test.ts b/extensions/qa-lab/src/evidence-gallery.test.ts index 4a1309a21054..6999f6c5ebcf 100644 --- a/extensions/qa-lab/src/evidence-gallery.test.ts +++ b/extensions/qa-lab/src/evidence-gallery.test.ts @@ -314,17 +314,17 @@ describe("evidence gallery", () => { suiteDir, "script", ...producerRootLeakSegments(repoRoot), - "ux-matrix-evidence-dashboard", + "ux-matrix-producer", "run-1", ); const expectedWebScreenshotNeedle = process.platform === "win32" ? ".artifacts/qa-e2e/suite/script/nested" - : ".artifacts/qa-e2e/suite/script/nested/ux-matrix-evidence-dashboard/run-1/surfaces/web-ui/stages/first-run/screenshot.png"; + : ".artifacts/qa-e2e/suite/script/nested/ux-matrix-producer/run-1/surfaces/web-ui/stages/first-run/screenshot.png"; const expectedCliLogNeedle = process.platform === "win32" ? ".artifacts/qa-e2e/suite/script/nested" - : ".artifacts/qa-e2e/suite/script/nested/ux-matrix-evidence-dashboard/run-1/surfaces/cli/stages/error-state/logs.txt"; + : ".artifacts/qa-e2e/suite/script/nested/ux-matrix-producer/run-1/surfaces/cli/stages/error-state/logs.txt"; await fs.mkdir(path.join(runDir, "surfaces", "web-ui", "stages", "first-run"), { recursive: true, }); @@ -365,10 +365,10 @@ describe("evidence gallery", () => { cells: [ null, { - coverageIds: [`${repoRoot}/ui.control`], + coverageIds: [], runner: { availability: "local", - command: `${repoRoot}/openclaw.mjs qa suite --scenario ux-matrix-evidence-dashboard`, + command: `node --import tsx ${repoRoot}/scripts/qa/ux-matrix-evidence-producer.ts --artifact-base ${runDir}`, lane: "web-ui-playwright", workflow: `${repoRoot}/.github/workflows/ux-matrix-qa.yml#ux-matrix-local`, }, @@ -377,10 +377,11 @@ describe("evidence gallery", () => { surface: "web-ui", }, { - coverageIds: ["cli.entrypoint"], + coverageIds: [], runner: { availability: "local", - command: "pnpm openclaw qa suite --scenario ux-matrix-evidence-dashboard", + command: + "node --import tsx scripts/qa/ux-matrix-evidence-producer.ts --artifact-base .artifacts/qa-e2e/ux-matrix", lane: "cli-status", workflow: ".github/workflows/ux-matrix-qa.yml#ux-matrix-local", }, @@ -399,7 +400,11 @@ describe("evidence gallery", () => { }, }); await fs.writeFile(path.join(runDir, "scorecard.md"), "# UX Matrix\n\n- pass: 1\n", "utf8"); - await fs.writeFile(path.join(runDir, "commands.txt"), "node ux matrix\n", "utf8"); + await fs.writeFile( + path.join(runDir, "commands.txt"), + "node --import tsx scripts/qa/ux-matrix-evidence-producer.ts --artifact-base .artifacts/qa-e2e/ux-matrix\n", + "utf8", + ); await fs.mkdir(path.join(runDir, "preflight"), { recursive: true }); await fs.writeFile(path.join(runDir, "preflight", "memory.txt"), "memory ok\n", "utf8"); await fs.writeFile( @@ -419,9 +424,9 @@ describe("evidence gallery", () => { kind: "ux-matrix-cell", id: "ux-matrix.web-ui.first-run", title: `UX Matrix: web-ui / first-run at ${repoRoot}`, - source: { path: "scripts/ux-matrix/dashboard.ts" }, + source: { path: "scripts/qa/ux-matrix-evidence-producer.ts" }, }, - coverage: [{ id: "ui.control", role: "primary" }], + coverage: [], execution: { runner: "ux-matrix-dashboard", environment: { @@ -458,9 +463,9 @@ describe("evidence gallery", () => { kind: "ux-matrix-cell", id: "qa-lab.wrapper-cli-error", title: "UX Matrix: cli / error-state", - source: { path: "scripts/ux-matrix/dashboard.ts" }, + source: { path: "scripts/qa/ux-matrix-evidence-producer.ts" }, }, - coverage: [{ id: "cli.status-snapshots", role: "primary" }], + coverage: [], execution: { runner: "ux-matrix-dashboard", environment: { @@ -530,10 +535,11 @@ describe("evidence gallery", () => { { artifactKinds: ["screenshot"], artifactPaths: [expect.stringContaining(expectedWebScreenshotNeedle)], - coverageIds: ["/ui.control"], + coverageIds: [], runner: { availability: "local", - command: "/openclaw.mjs qa suite --scenario ux-matrix-evidence-dashboard", + command: + "node --import tsx /scripts/qa/ux-matrix-evidence-producer.ts --artifact-base /.artifacts/qa-e2e/suite/script/nested/ux-matrix-producer/run-1", lane: "web-ui-playwright", workflow: "/.github/workflows/ux-matrix-qa.yml#ux-matrix-local", }, @@ -546,10 +552,11 @@ describe("evidence gallery", () => { { artifactKinds: [], artifactPaths: [], - coverageIds: ["cli.entrypoint"], + coverageIds: [], runner: { availability: "local", - command: "pnpm openclaw qa suite --scenario ux-matrix-evidence-dashboard", + command: + "node --import tsx scripts/qa/ux-matrix-evidence-producer.ts --artifact-base .artifacts/qa-e2e/ux-matrix", lane: "cli-status", workflow: ".github/workflows/ux-matrix-qa.yml#ux-matrix-local", }, @@ -576,7 +583,9 @@ describe("evidence gallery", () => { expect(decodeURIComponent(model.producerContext?.scorecard?.href ?? "")).not.toContain( repoRoot, ); - expect(model.producerContext?.commands?.preview).toBe("node ux matrix\n"); + expect(model.producerContext?.commands?.preview).toBe( + "node --import tsx scripts/qa/ux-matrix-evidence-producer.ts --artifact-base .artifacts/qa-e2e/ux-matrix\n", + ); expect(model.producerContext?.commands?.path).toContain("commands.txt"); expect(decodeURIComponent(model.producerContext?.commands?.href ?? "")).not.toContain(repoRoot); expect(model.producerContext?.manifest?.preview).toContain('"runId": "run-1"'); diff --git a/extensions/qa-lab/src/scenario-catalog.test.ts b/extensions/qa-lab/src/scenario-catalog.test.ts index 6e609d94d112..b672a8fab05a 100644 --- a/extensions/qa-lab/src/scenario-catalog.test.ts +++ b/extensions/qa-lab/src/scenario-catalog.test.ts @@ -231,7 +231,6 @@ describe("qa scenario catalog", () => { it("loads native test execution scenarios from YAML", () => { const scenario = readQaScenarioById("control-ui-chat-flow-playwright"); - const uxMatrix = readQaScenarioById("ux-matrix-evidence-dashboard"); const otelSmoke = readQaScenarioById("qa-otel-smoke"); expect(scenario.execution.kind).toBe("playwright"); @@ -244,14 +243,6 @@ describe("qa scenario catalog", () => { ); expect(scenario.execution.flow).toBeUndefined(); expect(scenario.coverage?.primary).toContain("ui.control"); - expect(uxMatrix.execution.kind).toBe("script"); - if (uxMatrix.execution.kind !== "script") { - throw new Error(`expected script scenario, got ${uxMatrix.execution.kind}`); - } - expect(uxMatrix.execution.path).toBe("scripts/qa/ux-matrix-evidence-producer.ts"); - expect(uxMatrix.execution.args).toStrictEqual(["--artifact-base", "${outputDir}"]); - expect(uxMatrix.execution.config).toBeUndefined(); - expect(uxMatrix.coverage?.primary).toContain("qa.artifact-safety"); expect(otelSmoke.execution.kind).toBe("script"); if (otelSmoke.execution.kind !== "script") { throw new Error(`expected script scenario, got ${otelSmoke.execution.kind}`); diff --git a/extensions/qa-lab/src/test-file-scenario-runner.test.ts b/extensions/qa-lab/src/test-file-scenario-runner.test.ts index f72b9fc76ddd..a41b7be1f458 100644 --- a/extensions/qa-lab/src/test-file-scenario-runner.test.ts +++ b/extensions/qa-lab/src/test-file-scenario-runner.test.ts @@ -4,7 +4,7 @@ import path from "node:path"; import { setTimeout as sleep } from "node:timers/promises"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { validateQaEvidenceSummaryJson } from "./evidence-summary.js"; -import { readQaScenarioById, type QaSeedScenarioWithSource } from "./scenario-catalog.js"; +import type { QaSeedScenarioWithSource } from "./scenario-catalog.js"; import { createTempDirHarness } from "./temp-dir.test-helper.js"; import { qaTestFileScenarioRunnerTesting, @@ -1081,72 +1081,58 @@ describe("qa test file scenario runner", () => { expect(artifactPath?.includes("..")).toBe(false); }); - describe("UX Matrix scenario composition", () => { - let outputDir: string; - let result: Awaited>; - let evidence: ReturnType; + it("imports the standalone UX Matrix producer as coverage-free infrastructure", async () => { + const outputDir = await fs.mkdtemp(path.join(os.tmpdir(), "qa-ux-matrix-producer-")); + tempRoots.push(outputDir); + // The runner accepts scenario-shaped execution input, but this test fixture is not cataloged + // and deliberately carries no product taxonomy coverage. + const infrastructureFixture: QaSeedScenarioWithSource = { + id: "scenario-script", + title: "UX Matrix producer infrastructure fixture", + surface: "qa-lab", + objective: "Exercise the standalone UX Matrix evidence producer through the script runner.", + successCriteria: ["The runner imports the producer's structured evidence bundle."], + codeRefs: ["scripts/qa/ux-matrix-evidence-producer.ts"], + sourcePath: "test/scripts/qa-ux-matrix-evidence-producer.test.ts", + execution: { + kind: "script", + path: "scripts/qa/ux-matrix-evidence-producer.ts", + allowBlockedEvidence: true, + args: ["--artifact-base", "${outputDir}", "--skip-visual-proof"], + }, + }; - beforeAll(async () => { - outputDir = await fs.mkdtemp(path.join(os.tmpdir(), "qa-ux-matrix-script-")); - tempRoots.push(outputDir); - const scenario = readQaScenarioById("ux-matrix-evidence-dashboard"); - - expect(scenario.execution.kind).toBe("script"); - result = await runQaTestFileScenarios({ - repoRoot: process.cwd(), - outputDir, - providerMode: "mock-openai", - primaryModel: "mock-openai/gpt-5.6-luna", - scenarios: [scenario], - env: { - OPENCLAW_QA_REF: "scenario-ref", - } as NodeJS.ProcessEnv, - }); - evidence = validateQaEvidenceSummaryJson( - JSON.parse(await fs.readFile(result.evidencePath, "utf8")), - ); + const result = await runQaTestFileScenarios({ + repoRoot: process.cwd(), + outputDir, + providerMode: "mock-openai", + primaryModel: "mock-openai/gpt-5.6-luna", + scenarios: [infrastructureFixture], + env: { OPENCLAW_QA_REF: "infrastructure-fixture" } as NodeJS.ProcessEnv, }); + const evidence = validateQaEvidenceSummaryJson( + JSON.parse(await fs.readFile(result.evidencePath, "utf8")), + ); - it("runs the checked-in producer and imports its evidence bundle", () => { - expect(result.executionKind).toBe("script"); - expect(result.results[0]?.producerEvidence?.entries).toHaveLength(3); - expect(evidence.entries.map((entry) => entry.test.id)).toEqual([ - "ux-matrix.qa-lab.producer-artifact-fixture", - "ux-matrix.control-ui.screenshot-artifact", - "ux-matrix.cli.entrypoint-help", - ]); - expect( - evidence.entries.flatMap((entry) => entry.coverage.map((coverage) => coverage.id)), - ).toEqual( - expect.arrayContaining([ - "qa.artifact-safety", - "tools.evidence", - "workspace.artifacts", - "ui.control", - "gateway.control-ui-hosting", - "cli.entrypoint", - "cli.status-snapshots", - ]), - ); - const artifactKinds = evidence.entries.flatMap( + expect(result.executionKind).toBe("script"); + expect(result.results[0]).toMatchObject({ status: "pass" }); + expect(result.results[0]?.producerEvidence?.entries).toHaveLength(3); + expect(evidence.entries.map((entry) => entry.test.id)).toEqual([ + "ux-matrix.qa-lab.producer-artifact-fixture", + "ux-matrix.control-ui.screenshot-artifact", + "ux-matrix.cli.entrypoint-help", + ]); + expect(evidence.entries.every((entry) => entry.coverage.length === 0)).toBe(true); + expect( + evidence.entries.flatMap( (entry) => entry.execution?.artifacts.map((artifact) => artifact.kind) ?? [], - ); - expect(artifactKinds).toEqual(expect.arrayContaining(["html", "log"])); - const fixtureEntry = evidence.entries.find( - (entry) => entry.test.id === "ux-matrix.qa-lab.producer-artifact-fixture", - ); - expect(fixtureEntry?.execution?.artifacts.map((artifact) => artifact.path)).toContain( - path.join( - outputDir, - "ux-matrix-evidence-dashboard", - "surfaces", - "qa-lab", - "stages", - "producer-artifact-fixture", - "producer-artifact-fixture.html", - ), - ); - }); + ), + ).toEqual(expect.arrayContaining(["html", "log"])); + expect( + evidence.entries + .flatMap((entry) => entry.execution?.artifacts.map((artifact) => artifact.path) ?? []) + .some((artifactPath) => artifactPath.includes(path.join(outputDir, "scenario-script"))), + ).toBe(true); }); }); /* oxlint-disable max-lines -- TODO: split this grandfathered oversized file. */ diff --git a/extensions/qa-lab/web/src/ui-render.test.ts b/extensions/qa-lab/web/src/ui-render.test.ts index 82c25af02ce6..f5266c760f7b 100644 --- a/extensions/qa-lab/web/src/ui-render.test.ts +++ b/extensions/qa-lab/web/src/ui-render.test.ts @@ -252,7 +252,7 @@ describe("QA Lab UI evidence render", () => { source: "ux-matrix:web-ui:first-run", }, ], - coverage: [{ id: "ui.control", role: "primary" }], + coverage: [], failureReason: null, id: "ux-matrix.web-ui.first-run", kind: "ux-matrix-cell", @@ -279,10 +279,11 @@ describe("QA Lab UI evidence render", () => { { artifactKinds: ["screenshot"], artifactPaths: ["screenshot.png"], - coverageIds: ["ui.control"], + coverageIds: [], runner: { availability: "local", - command: "pnpm openclaw qa suite --scenario ux-matrix-evidence-dashboard", + command: + "node --import tsx scripts/qa/ux-matrix-evidence-producer.ts --artifact-base .artifacts/qa-e2e/ux-matrix", lane: "web-ui-playwright", workflow: ".github/workflows/ux-matrix-qa.yml#ux-matrix-local", }, @@ -295,10 +296,11 @@ describe("QA Lab UI evidence render", () => { { artifactKinds: [], artifactPaths: [], - coverageIds: ["cli.entrypoint"], + coverageIds: [], runner: { availability: "local", - command: "pnpm openclaw qa suite --scenario ux-matrix-evidence-dashboard", + command: + "node --import tsx scripts/qa/ux-matrix-evidence-producer.ts --artifact-base .artifacts/qa-e2e/ux-matrix", lane: "cli-status", workflow: ".github/workflows/ux-matrix-qa.yml#ux-matrix-local", }, @@ -316,7 +318,7 @@ describe("QA Lab UI evidence render", () => { }, preflight: { adbDevices: null, memory: null }, releaseLedger: null, - rootPath: ".artifacts/qa-e2e/suite/script/ux-matrix-evidence-dashboard/run-1", + rootPath: ".artifacts/qa-e2e/suite/script/ux-matrix-producer/run-1", scorecard: null, }, profile: null, @@ -329,7 +331,7 @@ describe("QA Lab UI evidence render", () => { expect(html).toContain('data-evidence-entry-id="ux-matrix.web-ui.first-run"'); expect(html).toContain("evidence-matrix-cell-proof-gap"); expect(html).toContain("not executed in this run"); - expect(html).toContain("Coverage: cli.entrypoint"); + expect(html).not.toContain("Coverage:"); expect(html).toContain("Runner: cli-status"); expect(html).toContain("Open media artifact"); expect(html).toContain("Open video artifact"); diff --git a/qa/scenarios/ui/ux-matrix-evidence-dashboard.yaml b/qa/scenarios/ui/ux-matrix-evidence-dashboard.yaml deleted file mode 100644 index d095a5f32451..000000000000 --- a/qa/scenarios/ui/ux-matrix-evidence-dashboard.yaml +++ /dev/null @@ -1,35 +0,0 @@ -title: UX Matrix evidence dashboard - -scenario: - id: ux-matrix-evidence-dashboard - surface: control-ui - category: browser-control-ui-and-webchat.browser-ui - coverage: - primary: - - qa.artifact-safety - secondary: - - ui.control - - gateway.control-ui-hosting - - cli.entrypoint - - cli.status-snapshots - - tools.evidence - - workspace.artifacts - objective: Produce UX Matrix evidence artifacts through the QA Lab script producer contract. - successCriteria: - - The script producer writes manifest.json, matrix.json, release-ledger.json, scorecard.md, latest-run.json, and qa-evidence.json. - - Evidence entries declare coverage IDs and artifact paths for generated logs, screenshot proof, and video-style proof. - - Missing local browser or CLI prerequisites are represented as blocked evidence with logs instead of fake artifacts. - docsRefs: - - docs/concepts/qa-e2e-automation.md - - docs/web/control-ui.md - codeRefs: - - scripts/qa/ux-matrix-evidence-producer.ts - - extensions/qa-lab/src/test-file-scenario-runner.ts - execution: - kind: script - path: scripts/qa/ux-matrix-evidence-producer.ts - summary: Script-backed UX Matrix producer that emits QA Lab evidence artifacts. - allowBlockedEvidence: true - args: - - --artifact-base - - ${outputDir} diff --git a/scripts/qa/ux-matrix-evidence-producer.ts b/scripts/qa/ux-matrix-evidence-producer.ts index 5241464d9f45..5fe9f3d80412 100644 --- a/scripts/qa/ux-matrix-evidence-producer.ts +++ b/scripts/qa/ux-matrix-evidence-producer.ts @@ -1,4 +1,4 @@ -// Produces a QA Lab UX Matrix evidence bundle through the script scenario contract. +// Produces standalone QA Lab UX Matrix fixture artifacts for gallery and evidence tests. import { execFile } from "node:child_process"; import fs from "node:fs/promises"; import os from "node:os"; @@ -16,16 +16,13 @@ import { type QaEvidenceSummaryEntry, type QaEvidenceSummaryJson, } from "../../extensions/qa-lab/src/evidence-summary.js"; -import { readQaScenarioById } from "../../extensions/qa-lab/src/scenario-catalog.js"; import { ensurePlaywrightChromium, resolveSystemChromiumExecutablePath, } from "../ensure-playwright-chromium.mjs"; const execFileAsync = promisify(execFile); -const SCENARIO_ID = "ux-matrix-evidence-dashboard"; const SOURCE_PATH = "scripts/qa/ux-matrix-evidence-producer.ts"; -const SUITE_COMMAND = `pnpm openclaw qa suite --scenario ${SCENARIO_ID}`; type MatrixCell = { artifacts: Array<{ kind: string; path: string }>; @@ -227,21 +224,7 @@ function buildExecution(params: { }; } -function readCanonicalScenarioCoverage(): QaEvidenceSummaryEntry["coverage"] { - // Matrix cells are artifacts of one catalog scenario, not independent scenarios. - // Evidence therefore uses that scenario's mapping instead of cell-local coverage IDs. - const coverage = readQaScenarioById(SCENARIO_ID).coverage; - return [ - ...(coverage?.primary ?? []).map((id) => ({ id, role: "primary" as const })), - ...(coverage?.secondary ?? []).map((id) => ({ id, role: "secondary" as const })), - ]; -} - -function buildEvidenceEntry( - cell: MatrixCell, - repoRoot: string, - coverage: QaEvidenceSummaryEntry["coverage"], -): QaEvidenceSummaryEntry { +function buildEvidenceEntry(cell: MatrixCell, repoRoot: string): QaEvidenceSummaryEntry { const source = `ux-matrix:${cell.surface}:${cell.stage}`; return { test: { @@ -250,7 +233,8 @@ function buildEvidenceEntry( title: cell.title, source: { path: SOURCE_PATH }, }, - coverage, + // These entries prove the evidence/gallery infrastructure, not product taxonomy behavior. + coverage: [], refs: [ { kind: "code", path: SOURCE_PATH }, { kind: "docs", path: "docs/concepts/qa-e2e-automation.md" }, @@ -282,13 +266,12 @@ function buildEvidenceSummary(params: { generatedAt: string; repoRoot: string; }): QaEvidenceSummaryJson { - const coverage = readCanonicalScenarioCoverage(); return validateQaEvidenceSummaryJson({ kind: QA_EVIDENCE_SUMMARY_KIND, schemaVersion: QA_EVIDENCE_SUMMARY_SCHEMA_VERSION, generatedAt: params.generatedAt, evidenceMode: "full", - entries: params.cells.map((cell) => buildEvidenceEntry(cell, params.repoRoot, coverage)), + entries: params.cells.map((cell) => buildEvidenceEntry(cell, params.repoRoot)), }); } @@ -500,9 +483,9 @@ async function writeProducerArtifactFixtureHtml(params: {

UX Matrix Producer Artifact Fixture

-

Script-produced ${escapeHtml(QA_EVIDENCE_FILENAME)} for ${escapeHtml( - SCENARIO_ID, - )}; this fixture is not the QA Lab Evidence Archive UI.

+

Standalone script-produced ${escapeHtml( + QA_EVIDENCE_FILENAME, + )}; this fixture is not the QA Lab Evidence Archive UI or product-behavior proof.

UX Matrix entries

    ${entryRows}
@@ -612,7 +595,6 @@ async function writeProducerMetadata(params: { cells: readonly MatrixCell[]; repoRoot: string; }) { - const coverageIds = readCanonicalScenarioCoverage().map((coverage) => coverage.id); const counts = params.cells.reduce>((acc, cell) => { acc[cell.status] = (acc[cell.status] ?? 0) + 1; return acc; @@ -620,13 +602,12 @@ async function writeProducerMetadata(params: { await writeJson(path.join(params.artifactBase, "manifest.json"), { kind: "openclaw.qa.ux-matrix", run: { - scenarioId: SCENARIO_ID, status: counts.fail ? "fail" : counts.blocked ? "blocked" : "pass", }, }); await writeJson(path.join(params.artifactBase, "matrix.json"), { cells: params.cells.map((cell) => ({ - coverageIds, + coverageIds: [], stage: cell.stage, status: cell.status, surface: cell.surface, @@ -635,7 +616,7 @@ async function writeProducerMetadata(params: { }); await writeJson(path.join(params.artifactBase, "release-ledger.json"), { entries: params.cells.map((cell) => ({ - coverageIds, + coverageIds: [], stage: cell.stage, status: cell.status, surface: cell.surface, @@ -644,13 +625,10 @@ async function writeProducerMetadata(params: { }); await writeText( path.join(params.artifactBase, "commands.txt"), - [ - `${SUITE_COMMAND} --output-dir ${toRepoRelativePath(params.repoRoot, params.artifactBase)}`, - `node --import tsx ${SOURCE_PATH} --artifact-base ${toRepoRelativePath( - params.repoRoot, - params.artifactBase, - )}`, - ].join("\n") + "\n", + `node --import tsx ${SOURCE_PATH} --artifact-base ${toRepoRelativePath( + params.repoRoot, + params.artifactBase, + )}\n`, ); await writeText( path.join(params.artifactBase, "scorecard.md"), diff --git a/test/scripts/qa-ux-matrix-evidence-producer.test.ts b/test/scripts/qa-ux-matrix-evidence-producer.test.ts index f770870ff17e..349c57db0d2f 100644 --- a/test/scripts/qa-ux-matrix-evidence-producer.test.ts +++ b/test/scripts/qa-ux-matrix-evidence-producer.test.ts @@ -139,6 +139,43 @@ describe("QA UX Matrix evidence producer CLI", () => { expect(cliLog).not.toContain(fakeRepoRoot); expect(`${evidence}\n${cliLog}`).toContain(""); expect(visualLog).toBe("blocked: --skip-visual-proof was set\n"); + + const evidenceJson = JSON.parse(evidence) as { + entries: Array<{ + coverage: unknown[]; + execution?: { artifacts: Array<{ kind: string }> }; + test: { id: string }; + }>; + }; + const matrix = JSON.parse( + fs.readFileSync(path.join(artifactBase, "matrix.json"), "utf8"), + ) as { cells: Array<{ coverageIds: unknown[] }> }; + const releaseLedger = JSON.parse( + fs.readFileSync(path.join(artifactBase, "release-ledger.json"), "utf8"), + ) as { entries: Array<{ coverageIds: unknown[] }> }; + const manifest = JSON.parse( + fs.readFileSync(path.join(artifactBase, "manifest.json"), "utf8"), + ) as { run: Record }; + const commands = fs.readFileSync(path.join(artifactBase, "commands.txt"), "utf8"); + + expect(evidenceJson.entries.map((entry) => entry.test.id)).toEqual([ + "ux-matrix.qa-lab.producer-artifact-fixture", + "ux-matrix.control-ui.screenshot-artifact", + "ux-matrix.cli.entrypoint-help", + ]); + expect(evidenceJson.entries.every((entry) => entry.coverage.length === 0)).toBe(true); + expect(matrix.cells.every((cell) => cell.coverageIds.length === 0)).toBe(true); + expect(releaseLedger.entries.every((entry) => entry.coverageIds.length === 0)).toBe(true); + expect(manifest.run).not.toHaveProperty("scenarioId"); + expect(commands).toContain( + "node --import tsx scripts/qa/ux-matrix-evidence-producer.ts --artifact-base", + ); + expect(commands).not.toContain("qa suite --scenario"); + expect( + evidenceJson.entries.flatMap( + (entry) => entry.execution?.artifacts.map((artifact) => artifact.kind) ?? [], + ), + ).toEqual(expect.arrayContaining(["html", "log"])); } finally { fs.rmSync(artifactBase, { recursive: true, force: true }); fs.rmSync(fakeRepoRoot, { recursive: true, force: true });