diff --git a/extensions/qa-matrix/src/runners/contract/runtime.test.ts b/extensions/qa-matrix/src/runners/contract/runtime.test.ts index 5a88a2eb42a1..f98cde6cf779 100644 --- a/extensions/qa-matrix/src/runners/contract/runtime.test.ts +++ b/extensions/qa-matrix/src/runners/contract/runtime.test.ts @@ -1,4 +1,5 @@ // Qa Matrix tests cover runtime plugin behavior. +import path from "node:path"; import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; import { MAX_TIMER_TIMEOUT_MS } from "openclaw/plugin-sdk/number-runtime"; import { renderQaMarkdownReport } from "openclaw/plugin-sdk/qa-runtime"; @@ -77,6 +78,19 @@ function buildMatrixQaSummaryInput( } describe("matrix live qa runtime", () => { + it("uses unique default artifact directories", () => { + const repoRoot = "/repo"; + const firstOutputDir = liveTesting.resolveMatrixQaOutputDir({ repoRoot }); + const secondOutputDir = liveTesting.resolveMatrixQaOutputDir({ repoRoot }); + + expect(path.dirname(firstOutputDir)).toBe(path.join(repoRoot, ".artifacts", "qa-e2e")); + expect(path.basename(firstOutputDir)).toMatch(/^matrix-[a-z0-9]+-[a-f0-9]{8}$/u); + expect(secondOutputDir).not.toBe(firstOutputDir); + expect( + liveTesting.resolveMatrixQaOutputDir({ outputDir: ".artifacts/custom", repoRoot }), + ).toBe(".artifacts/custom"); + }); + it("prints Matrix QA progress by default for non-interactive runs", () => { const previous = process.env.OPENCLAW_QA_MATRIX_PROGRESS; delete process.env.OPENCLAW_QA_MATRIX_PROGRESS; diff --git a/extensions/qa-matrix/src/runners/contract/runtime.ts b/extensions/qa-matrix/src/runners/contract/runtime.ts index 367b04d12706..74caf221d334 100644 --- a/extensions/qa-matrix/src/runners/contract/runtime.ts +++ b/extensions/qa-matrix/src/runners/contract/runtime.ts @@ -29,6 +29,7 @@ import { } from "../../substrate/config.js"; import type { MatrixQaObservedEvent } from "../../substrate/events.js"; import { startMatrixQaHarness } from "../../substrate/harness.runtime.js"; +import { createLiveTransportQaRunId } from "../../shared/live-transport-artifacts.js"; import { resolveMatrixQaModels, type ResolvedMatrixQaModels } from "./model-selection.js"; import type { MatrixQaSyncStreams } from "./scenario-runtime-shared.js"; import { @@ -623,6 +624,13 @@ function isMatrixQaStaleConfigPatchError(error: unknown) { return formatErrorMessage(error).toLowerCase().includes("config changed since last load"); } +function resolveMatrixQaOutputDir(params: { outputDir?: string; repoRoot: string }) { + return ( + params.outputDir ?? + path.join(params.repoRoot, ".artifacts", "qa-e2e", `matrix-${createLiveTransportQaRunId()}`) + ); +} + async function startMatrixQaLiveLaneGateway(params: { repoRoot: string; transport: { @@ -657,9 +665,7 @@ export async function runMatrixQaLive(params: { alternateModel?: string; }): Promise { const repoRoot = path.resolve(params.repoRoot ?? process.cwd()); - const outputDir = - params.outputDir ?? - path.join(repoRoot, ".artifacts", "qa-e2e", `matrix-${Date.now().toString(36)}`); + const outputDir = resolveMatrixQaOutputDir({ outputDir: params.outputDir, repoRoot }); await fs.mkdir(outputDir, { recursive: true }); const defaultModels = resolveMatrixQaModels({ @@ -1276,6 +1282,7 @@ export const testing = { isMatrixAccountReady, patchMatrixQaGatewayConfig, remainingMatrixQaRunMs, + resolveMatrixQaOutputDir, resolveMatrixQaCanaryTimeoutMs, resolveMatrixQaModels, shouldWriteMatrixQaProgress, diff --git a/extensions/qa-matrix/src/shared/live-transport-artifacts.ts b/extensions/qa-matrix/src/shared/live-transport-artifacts.ts new file mode 100644 index 000000000000..27499821755c --- /dev/null +++ b/extensions/qa-matrix/src/shared/live-transport-artifacts.ts @@ -0,0 +1,6 @@ +// Qa Matrix plugin module implements live transport artifact behavior. +import { randomUUID } from "node:crypto"; + +export function createLiveTransportQaRunId() { + return `${Date.now().toString(36)}-${randomUUID().slice(0, 8)}`; +} diff --git a/extensions/qa-matrix/src/shared/live-transport-cli.runtime.test.ts b/extensions/qa-matrix/src/shared/live-transport-cli.runtime.test.ts index 887a46a0c0be..0caae15ea424 100644 --- a/extensions/qa-matrix/src/shared/live-transport-cli.runtime.test.ts +++ b/extensions/qa-matrix/src/shared/live-transport-cli.runtime.test.ts @@ -4,6 +4,7 @@ import os from "node:os"; import path from "node:path"; import { startLiveTransportQaOutputTee } from "openclaw/plugin-sdk/qa-runtime"; import { afterEach, describe, expect, it } from "vitest"; +import { resolveLiveTransportQaRunOptions } from "./live-transport-cli.runtime.js"; const tmpDirs: string[] = []; @@ -12,6 +13,22 @@ describe("live transport CLI runtime", () => { await Promise.all(tmpDirs.splice(0).map((dir) => rm(dir, { recursive: true, force: true }))); }); + it("uses unique default output dirs for CLI runs", () => { + const repoRoot = "/repo"; + const firstOutputDir = resolveLiveTransportQaRunOptions({ repoRoot }).outputDir; + const secondOutputDir = resolveLiveTransportQaRunOptions({ repoRoot }).outputDir; + + expect(path.dirname(firstOutputDir)).toBe(path.join(repoRoot, ".artifacts", "qa-e2e")); + expect(path.basename(firstOutputDir)).toMatch(/^matrix-[a-z0-9]+-[a-f0-9]{8}$/u); + expect(secondOutputDir).not.toBe(firstOutputDir); + expect( + resolveLiveTransportQaRunOptions({ + repoRoot, + outputDir: ".artifacts/custom", + }).outputDir, + ).toBe(path.join(repoRoot, ".artifacts/custom")); + }); + it("tees stdout and stderr into an output artifact", async () => { const outputDir = await mkdtemp(path.join(os.tmpdir(), "matrix-qa-output-")); tmpDirs.push(outputDir); diff --git a/extensions/qa-matrix/src/shared/live-transport-cli.runtime.ts b/extensions/qa-matrix/src/shared/live-transport-cli.runtime.ts index ee2c1e52611f..0fef6c72d6bd 100644 --- a/extensions/qa-matrix/src/shared/live-transport-cli.runtime.ts +++ b/extensions/qa-matrix/src/shared/live-transport-cli.runtime.ts @@ -3,6 +3,7 @@ import path from "node:path"; import { resolveRepoRelativeOutputDir } from "../cli-paths.js"; import type { QaProviderMode } from "../run-config.js"; import { normalizeQaProviderMode } from "../run-config.js"; +import { createLiveTransportQaRunId } from "./live-transport-artifacts.js"; import type { LiveTransportQaCommandOptions } from "./live-transport-cli.js"; export function resolveLiveTransportQaRunOptions( @@ -15,7 +16,7 @@ export function resolveLiveTransportQaRunOptions( const repoRoot = path.resolve(opts.repoRoot ?? process.cwd()); const outputDir = resolveRepoRelativeOutputDir(repoRoot, opts.outputDir) ?? - path.join(repoRoot, ".artifacts", "qa-e2e", `matrix-${Date.now().toString(36)}`); + path.join(repoRoot, ".artifacts", "qa-e2e", `matrix-${createLiveTransportQaRunId()}`); return { repoRoot, outputDir,