fix(qa): avoid matrix qa artifact collisions

This commit is contained in:
Vincent Koc
2026-06-23 11:32:08 +02:00
parent d37300f357
commit 932b58b94b
5 changed files with 49 additions and 4 deletions

View File

@@ -1,4 +1,5 @@
// Qa Matrix tests cover runtime plugin behavior.
import path from "node:path";
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
import { MAX_TIMER_TIMEOUT_MS } from "openclaw/plugin-sdk/number-runtime";
import { renderQaMarkdownReport } from "openclaw/plugin-sdk/qa-runtime";
@@ -77,6 +78,19 @@ function buildMatrixQaSummaryInput(
}
describe("matrix live qa runtime", () => {
it("uses unique default artifact directories", () => {
const repoRoot = "/repo";
const firstOutputDir = liveTesting.resolveMatrixQaOutputDir({ repoRoot });
const secondOutputDir = liveTesting.resolveMatrixQaOutputDir({ repoRoot });
expect(path.dirname(firstOutputDir)).toBe(path.join(repoRoot, ".artifacts", "qa-e2e"));
expect(path.basename(firstOutputDir)).toMatch(/^matrix-[a-z0-9]+-[a-f0-9]{8}$/u);
expect(secondOutputDir).not.toBe(firstOutputDir);
expect(
liveTesting.resolveMatrixQaOutputDir({ outputDir: ".artifacts/custom", repoRoot }),
).toBe(".artifacts/custom");
});
it("prints Matrix QA progress by default for non-interactive runs", () => {
const previous = process.env.OPENCLAW_QA_MATRIX_PROGRESS;
delete process.env.OPENCLAW_QA_MATRIX_PROGRESS;

View File

@@ -29,6 +29,7 @@ import {
} from "../../substrate/config.js";
import type { MatrixQaObservedEvent } from "../../substrate/events.js";
import { startMatrixQaHarness } from "../../substrate/harness.runtime.js";
import { createLiveTransportQaRunId } from "../../shared/live-transport-artifacts.js";
import { resolveMatrixQaModels, type ResolvedMatrixQaModels } from "./model-selection.js";
import type { MatrixQaSyncStreams } from "./scenario-runtime-shared.js";
import {
@@ -623,6 +624,13 @@ function isMatrixQaStaleConfigPatchError(error: unknown) {
return formatErrorMessage(error).toLowerCase().includes("config changed since last load");
}
function resolveMatrixQaOutputDir(params: { outputDir?: string; repoRoot: string }) {
return (
params.outputDir ??
path.join(params.repoRoot, ".artifacts", "qa-e2e", `matrix-${createLiveTransportQaRunId()}`)
);
}
async function startMatrixQaLiveLaneGateway(params: {
repoRoot: string;
transport: {
@@ -657,9 +665,7 @@ export async function runMatrixQaLive(params: {
alternateModel?: string;
}): Promise<MatrixQaRunResult> {
const repoRoot = path.resolve(params.repoRoot ?? process.cwd());
const outputDir =
params.outputDir ??
path.join(repoRoot, ".artifacts", "qa-e2e", `matrix-${Date.now().toString(36)}`);
const outputDir = resolveMatrixQaOutputDir({ outputDir: params.outputDir, repoRoot });
await fs.mkdir(outputDir, { recursive: true });
const defaultModels = resolveMatrixQaModels({
@@ -1276,6 +1282,7 @@ export const testing = {
isMatrixAccountReady,
patchMatrixQaGatewayConfig,
remainingMatrixQaRunMs,
resolveMatrixQaOutputDir,
resolveMatrixQaCanaryTimeoutMs,
resolveMatrixQaModels,
shouldWriteMatrixQaProgress,

View File

@@ -0,0 +1,6 @@
// Qa Matrix plugin module implements live transport artifact behavior.
import { randomUUID } from "node:crypto";
export function createLiveTransportQaRunId() {
return `${Date.now().toString(36)}-${randomUUID().slice(0, 8)}`;
}

View File

@@ -4,6 +4,7 @@ import os from "node:os";
import path from "node:path";
import { startLiveTransportQaOutputTee } from "openclaw/plugin-sdk/qa-runtime";
import { afterEach, describe, expect, it } from "vitest";
import { resolveLiveTransportQaRunOptions } from "./live-transport-cli.runtime.js";
const tmpDirs: string[] = [];
@@ -12,6 +13,22 @@ describe("live transport CLI runtime", () => {
await Promise.all(tmpDirs.splice(0).map((dir) => rm(dir, { recursive: true, force: true })));
});
it("uses unique default output dirs for CLI runs", () => {
const repoRoot = "/repo";
const firstOutputDir = resolveLiveTransportQaRunOptions({ repoRoot }).outputDir;
const secondOutputDir = resolveLiveTransportQaRunOptions({ repoRoot }).outputDir;
expect(path.dirname(firstOutputDir)).toBe(path.join(repoRoot, ".artifacts", "qa-e2e"));
expect(path.basename(firstOutputDir)).toMatch(/^matrix-[a-z0-9]+-[a-f0-9]{8}$/u);
expect(secondOutputDir).not.toBe(firstOutputDir);
expect(
resolveLiveTransportQaRunOptions({
repoRoot,
outputDir: ".artifacts/custom",
}).outputDir,
).toBe(path.join(repoRoot, ".artifacts/custom"));
});
it("tees stdout and stderr into an output artifact", async () => {
const outputDir = await mkdtemp(path.join(os.tmpdir(), "matrix-qa-output-"));
tmpDirs.push(outputDir);

View File

@@ -3,6 +3,7 @@ import path from "node:path";
import { resolveRepoRelativeOutputDir } from "../cli-paths.js";
import type { QaProviderMode } from "../run-config.js";
import { normalizeQaProviderMode } from "../run-config.js";
import { createLiveTransportQaRunId } from "./live-transport-artifacts.js";
import type { LiveTransportQaCommandOptions } from "./live-transport-cli.js";
export function resolveLiveTransportQaRunOptions(
@@ -15,7 +16,7 @@ export function resolveLiveTransportQaRunOptions(
const repoRoot = path.resolve(opts.repoRoot ?? process.cwd());
const outputDir =
resolveRepoRelativeOutputDir(repoRoot, opts.outputDir) ??
path.join(repoRoot, ".artifacts", "qa-e2e", `matrix-${Date.now().toString(36)}`);
path.join(repoRoot, ".artifacts", "qa-e2e", `matrix-${createLiveTransportQaRunId()}`);
return {
repoRoot,
outputDir,