mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-06 19:03:40 +00:00
647 lines
21 KiB
TypeScript
647 lines
21 KiB
TypeScript
// Qa Lab plugin module implements suite launch behavior.
|
|
import fs from "node:fs/promises";
|
|
import path from "node:path";
|
|
import { renderQaMarkdownReport, type QaReportScenario } from "openclaw/plugin-sdk/qa-runtime";
|
|
import { toRepoRelativePath } from "./cli-paths.js";
|
|
import {
|
|
QA_EVIDENCE_FILENAME,
|
|
QA_EVIDENCE_SUMMARY_KIND,
|
|
QA_EVIDENCE_SUMMARY_SCHEMA_VERSION,
|
|
validateQaEvidenceSummaryJson,
|
|
type QaEvidenceSummaryJson,
|
|
} from "./evidence-summary.js";
|
|
import { isQaFastModeEnabled } from "./model-selection.js";
|
|
import { DEFAULT_QA_PROVIDER_MODE } from "./providers/index.js";
|
|
import {
|
|
defaultQaSuiteConcurrencyForTransport,
|
|
normalizeQaTransportId,
|
|
} from "./qa-transport-registry.js";
|
|
import { defaultQaModelForMode, normalizeQaProviderMode } from "./run-config.js";
|
|
import {
|
|
readQaBootstrapScenarioCatalog,
|
|
type QaSeedScenarioWithSource,
|
|
} from "./scenario-catalog.js";
|
|
import {
|
|
normalizeQaSuiteConcurrency,
|
|
resolveQaSuiteOutputDir,
|
|
resolveQaSuiteWorkerStartStaggerMs,
|
|
scenarioRequiresIsolatedQaSuiteWorker,
|
|
} from "./suite-planning.js";
|
|
import {
|
|
buildQaSuiteSummaryJson,
|
|
type QaSuiteResult,
|
|
type QaSuiteRunParams,
|
|
type QaSuiteScenarioResult,
|
|
type QaSuiteSummaryJson,
|
|
} from "./suite.js";
|
|
import {
|
|
isQaTestFileScenario,
|
|
runQaTestFileScenarios,
|
|
type QaTestFileExecutionKind,
|
|
type QaTestFileScenario,
|
|
type QaTestFileScenarioRunResult,
|
|
} from "./test-file-scenario-runner.js";
|
|
|
|
export type QaSuiteRuntimeResult =
|
|
| {
|
|
executionKind: "flow";
|
|
result: QaSuiteResult;
|
|
}
|
|
| {
|
|
executionKind: "suite";
|
|
result: QaUnifiedSuiteResult;
|
|
};
|
|
|
|
export type QaUnifiedSuiteResult = {
|
|
evidencePath: string;
|
|
outputDir: string;
|
|
report: string;
|
|
reportPath: string;
|
|
scenarios: QaSuiteScenarioResult[];
|
|
summaryPath: string;
|
|
};
|
|
|
|
type QaSuiteExecutionPlan =
|
|
| {
|
|
kind: "flow";
|
|
}
|
|
| {
|
|
kind: "unified";
|
|
scenarios: QaSeedScenarioWithSource[];
|
|
flowScenarios: QaSeedScenarioWithSource[];
|
|
testFileScenariosByKind: Map<QaTestFileExecutionKind, QaTestFileScenario[]>;
|
|
};
|
|
|
|
const MAX_SHARED_FLOW_PARTITIONS = 4;
|
|
const MAX_ISOLATED_FLOW_CONCURRENCY = 8;
|
|
const ISOLATED_FLOW_WORKER_START_STAGGER_MS = 1_500;
|
|
|
|
type QaUnifiedPartitionResult = {
|
|
evidenceSummaries: QaEvidenceSummaryJson[];
|
|
scenarioResults: Array<{
|
|
result: QaSuiteScenarioResult;
|
|
scenarioId: string;
|
|
}>;
|
|
};
|
|
|
|
type QaUnifiedPartitionTask = {
|
|
run: () => Promise<QaUnifiedPartitionResult>;
|
|
weight: number;
|
|
};
|
|
|
|
async function loadQaLabServerRuntime() {
|
|
const { startQaLabServer } = await import("./lab-server.js");
|
|
return startQaLabServer;
|
|
}
|
|
|
|
async function loadQaFlowSuiteRuntime() {
|
|
const [{ runQaFlowSuite }, startLab] = await Promise.all([
|
|
import("./suite.js"),
|
|
loadQaLabServerRuntime(),
|
|
]);
|
|
return async (params: QaSuiteRunParams | undefined) =>
|
|
await runQaFlowSuite({
|
|
...params,
|
|
startLab: params?.startLab ?? startLab,
|
|
});
|
|
}
|
|
|
|
function resolveRequestedScenarios(params: {
|
|
scenarioIds: readonly string[];
|
|
scenarios: ReturnType<typeof readQaBootstrapScenarioCatalog>["scenarios"];
|
|
}) {
|
|
const scenarioById = new Map(params.scenarios.map((scenario) => [scenario.id, scenario]));
|
|
return params.scenarioIds.map((scenarioId) => {
|
|
const scenario = scenarioById.get(scenarioId);
|
|
if (!scenario) {
|
|
throw new Error(`unknown QA scenario id(s): ${scenarioId}`);
|
|
}
|
|
return scenario;
|
|
});
|
|
}
|
|
|
|
function resolveSuiteExecutionPlan(params: QaSuiteRunParams | undefined): QaSuiteExecutionPlan {
|
|
const scenarioIds = params?.scenarioIds ?? [];
|
|
if (scenarioIds.length === 0) {
|
|
return { kind: "flow" };
|
|
}
|
|
const selectedScenarios = resolveRequestedScenarios({
|
|
scenarioIds,
|
|
scenarios: readQaBootstrapScenarioCatalog().scenarios,
|
|
});
|
|
const flowScenarios = selectedScenarios.filter((scenario) => !isQaTestFileScenario(scenario));
|
|
const testFileScenariosByKind = new Map<QaTestFileExecutionKind, QaTestFileScenario[]>();
|
|
for (const scenario of selectedScenarios) {
|
|
if (!isQaTestFileScenario(scenario)) {
|
|
continue;
|
|
}
|
|
const scenarios = testFileScenariosByKind.get(scenario.execution.kind) ?? [];
|
|
scenarios.push(scenario);
|
|
testFileScenariosByKind.set(scenario.execution.kind, scenarios);
|
|
}
|
|
if (testFileScenariosByKind.size === 0) {
|
|
return { kind: "flow" };
|
|
}
|
|
return {
|
|
kind: "unified",
|
|
scenarios: selectedScenarios,
|
|
flowScenarios,
|
|
testFileScenariosByKind,
|
|
};
|
|
}
|
|
|
|
async function runQaTestFileSuiteFromRuntime(params: {
|
|
runParams: QaSuiteRunParams | undefined;
|
|
scenarios: readonly QaTestFileScenario[];
|
|
}): Promise<QaTestFileScenarioRunResult> {
|
|
const runParams = params.runParams;
|
|
if (runParams?.runtimePair) {
|
|
throw new Error("--runtime-pair requires execution.kind: flow scenarios.");
|
|
}
|
|
if (runParams?.forcedRuntime) {
|
|
throw new Error("forced runtime execution requires execution.kind: flow scenarios.");
|
|
}
|
|
if (runParams?.captureRuntimeParityCell) {
|
|
throw new Error("runtime parity capture requires execution.kind: flow scenarios.");
|
|
}
|
|
const repoRoot = path.resolve(runParams?.repoRoot ?? process.cwd());
|
|
const outputDir = await resolveQaSuiteOutputDir(repoRoot, runParams?.outputDir);
|
|
const providerMode = normalizeQaProviderMode(runParams?.providerMode ?? DEFAULT_QA_PROVIDER_MODE);
|
|
const primaryModel = runParams?.primaryModel?.trim() || defaultQaModelForMode(providerMode);
|
|
return await runQaTestFileScenarios({
|
|
evidenceMode: runParams?.evidenceMode,
|
|
repoRoot,
|
|
outputDir,
|
|
providerMode,
|
|
primaryModel,
|
|
scenarios: params.scenarios,
|
|
writeEvidenceFile: runParams?.writeEvidenceFile,
|
|
});
|
|
}
|
|
|
|
function rejectFlowOnlySuiteOptionsForUnifiedRun(runParams: QaSuiteRunParams | undefined) {
|
|
if (runParams?.runtimePair) {
|
|
throw new Error("--runtime-pair requires execution.kind: flow scenarios.");
|
|
}
|
|
if (runParams?.forcedRuntime) {
|
|
throw new Error("forced runtime execution requires execution.kind: flow scenarios.");
|
|
}
|
|
if (runParams?.captureRuntimeParityCell) {
|
|
throw new Error("runtime parity capture requires execution.kind: flow scenarios.");
|
|
}
|
|
}
|
|
|
|
function suitePartitionOutputDir(outputDir: string, kind: "flow" | QaTestFileExecutionKind) {
|
|
return path.join(outputDir, kind);
|
|
}
|
|
|
|
function flowSuitePartitionOutputDir(outputDir: string, partition: string) {
|
|
return path.join(suitePartitionOutputDir(outputDir, "flow"), partition);
|
|
}
|
|
|
|
function partitionSharedFlowScenarios(
|
|
scenarios: readonly QaSeedScenarioWithSource[],
|
|
concurrency: number,
|
|
) {
|
|
const partitionCount = Math.min(
|
|
Math.max(1, Math.floor(concurrency)),
|
|
MAX_SHARED_FLOW_PARTITIONS,
|
|
scenarios.length,
|
|
);
|
|
const partitions = Array.from({ length: partitionCount }, (): QaSeedScenarioWithSource[] => []);
|
|
for (const [index, scenario] of scenarios.entries()) {
|
|
const partition = partitions[index % partitionCount];
|
|
if (!partition) {
|
|
throw new Error("failed to partition shared QA flow scenarios");
|
|
}
|
|
partition.push(scenario);
|
|
}
|
|
return partitions.filter((partition) => partition.length > 0);
|
|
}
|
|
|
|
async function runWeightedUnifiedPartitionTasks(
|
|
tasks: readonly QaUnifiedPartitionTask[],
|
|
maxWeight: number,
|
|
) {
|
|
if (tasks.length === 0) {
|
|
return [];
|
|
}
|
|
const limit = Math.max(1, Math.floor(maxWeight));
|
|
const results: QaUnifiedPartitionResult[] = [];
|
|
let activeWeight = 0;
|
|
let settled = 0;
|
|
let nextIndex = 0;
|
|
return await new Promise<QaUnifiedPartitionResult[]>((resolve, reject) => {
|
|
let firstError: Error | undefined;
|
|
let finished = false;
|
|
const finishIfSettled = () => {
|
|
if (finished || activeWeight > 0) {
|
|
return;
|
|
}
|
|
finished = true;
|
|
if (firstError) {
|
|
reject(firstError);
|
|
return;
|
|
}
|
|
resolve(results);
|
|
};
|
|
const launch = () => {
|
|
if (firstError) {
|
|
finishIfSettled();
|
|
return;
|
|
}
|
|
while (nextIndex < tasks.length) {
|
|
const task = tasks[nextIndex];
|
|
if (!task) {
|
|
return;
|
|
}
|
|
const taskWeight = Math.max(1, Math.min(limit, Math.floor(task.weight)));
|
|
if (activeWeight > 0 && activeWeight + taskWeight > limit) {
|
|
return;
|
|
}
|
|
const index = nextIndex;
|
|
nextIndex += 1;
|
|
activeWeight += taskWeight;
|
|
task.run().then(
|
|
(result) => {
|
|
results[index] = result;
|
|
activeWeight -= taskWeight;
|
|
settled += 1;
|
|
if (settled === tasks.length) {
|
|
finishIfSettled();
|
|
return;
|
|
}
|
|
launch();
|
|
},
|
|
(error: unknown) => {
|
|
firstError = error instanceof Error ? error : new Error(String(error));
|
|
activeWeight -= taskWeight;
|
|
settled += 1;
|
|
finishIfSettled();
|
|
},
|
|
);
|
|
}
|
|
if (settled === tasks.length) {
|
|
finishIfSettled();
|
|
}
|
|
};
|
|
launch();
|
|
});
|
|
}
|
|
|
|
async function readQaSuiteEvidenceSummary(evidencePath: string) {
|
|
return validateQaEvidenceSummaryJson(JSON.parse(await fs.readFile(evidencePath, "utf8")));
|
|
}
|
|
|
|
async function resolveQaSuiteResultEvidenceSummary(result: {
|
|
evidence?: QaEvidenceSummaryJson;
|
|
evidencePath: string;
|
|
}) {
|
|
return result.evidence ?? (await readQaSuiteEvidenceSummary(result.evidencePath));
|
|
}
|
|
|
|
function mergeQaEvidenceSummaries(params: {
|
|
evidenceSummaries: readonly QaEvidenceSummaryJson[];
|
|
generatedAt: string;
|
|
}) {
|
|
const profiles = [
|
|
...new Set(
|
|
params.evidenceSummaries
|
|
.map((summary) => summary.profile?.trim())
|
|
.filter((profile): profile is string => Boolean(profile)),
|
|
),
|
|
];
|
|
return validateQaEvidenceSummaryJson({
|
|
kind: QA_EVIDENCE_SUMMARY_KIND,
|
|
schemaVersion: QA_EVIDENCE_SUMMARY_SCHEMA_VERSION,
|
|
generatedAt: params.generatedAt,
|
|
evidenceMode:
|
|
params.evidenceSummaries.length > 0 &&
|
|
params.evidenceSummaries.every((summary) => summary.evidenceMode === "slim")
|
|
? "slim"
|
|
: "full",
|
|
entries: params.evidenceSummaries.flatMap((summary) => summary.entries),
|
|
profile: profiles.length === 1 ? profiles[0] : undefined,
|
|
});
|
|
}
|
|
|
|
function testFileScenarioResultToSuiteScenario(
|
|
result: QaTestFileScenarioRunResult["results"][number],
|
|
repoRoot: string,
|
|
): QaSuiteScenarioResult {
|
|
const suiteStatus = result.status === "pass" ? "pass" : "fail";
|
|
const stepStatus = result.status === "skipped" ? "skip" : suiteStatus;
|
|
const logPath = toRepoRelativePath(repoRoot, result.logPath);
|
|
const details = [
|
|
`execution.kind=${result.scenario.execution.kind}`,
|
|
`execution.path=${result.scenario.execution.path}`,
|
|
`log=${logPath}`,
|
|
...(result.failureMessage ? [`failure=${result.failureMessage}`] : []),
|
|
].join("\n");
|
|
return {
|
|
name: result.scenario.title,
|
|
status: suiteStatus,
|
|
details,
|
|
steps: [
|
|
{
|
|
name: `Run ${result.scenario.execution.kind} test file`,
|
|
status: stepStatus,
|
|
details,
|
|
},
|
|
],
|
|
};
|
|
}
|
|
|
|
function renderUnifiedQaSuiteReport(params: {
|
|
finishedAt: Date;
|
|
scenarios: readonly QaSuiteScenarioResult[];
|
|
startedAt: Date;
|
|
}) {
|
|
return renderQaMarkdownReport({
|
|
title: "OpenClaw QA Scenario Suite",
|
|
startedAt: params.startedAt,
|
|
finishedAt: params.finishedAt,
|
|
checks: [],
|
|
scenarios: params.scenarios.map((scenario) => ({
|
|
name: scenario.name,
|
|
status: scenario.status,
|
|
details: scenario.details,
|
|
steps: scenario.steps,
|
|
})) satisfies QaReportScenario[],
|
|
});
|
|
}
|
|
|
|
async function writeUnifiedQaSuiteArtifacts(params: {
|
|
alternateModel: string;
|
|
concurrency: number;
|
|
evidence: QaEvidenceSummaryJson;
|
|
fastMode: boolean;
|
|
finishedAt: Date;
|
|
outputDir: string;
|
|
primaryModel: string;
|
|
providerMode: ReturnType<typeof normalizeQaProviderMode>;
|
|
scenarioIds: readonly string[];
|
|
scenarios: readonly QaSuiteScenarioResult[];
|
|
startedAt: Date;
|
|
}) {
|
|
await fs.mkdir(params.outputDir, { recursive: true });
|
|
const evidencePath = path.join(params.outputDir, QA_EVIDENCE_FILENAME);
|
|
const reportPath = path.join(params.outputDir, "qa-suite-report.md");
|
|
const summaryPath = path.join(params.outputDir, "qa-suite-summary.json");
|
|
const report = renderUnifiedQaSuiteReport({
|
|
finishedAt: params.finishedAt,
|
|
scenarios: params.scenarios,
|
|
startedAt: params.startedAt,
|
|
});
|
|
const summary = buildQaSuiteSummaryJson({
|
|
alternateModel: params.alternateModel,
|
|
concurrency: params.concurrency,
|
|
evidence: params.evidence,
|
|
fastMode: params.fastMode,
|
|
finishedAt: params.finishedAt,
|
|
primaryModel: params.primaryModel,
|
|
providerMode: params.providerMode,
|
|
scenarioIds: params.scenarioIds,
|
|
scenarios: [...params.scenarios],
|
|
startedAt: params.startedAt,
|
|
}) satisfies QaSuiteSummaryJson;
|
|
await fs.writeFile(evidencePath, `${JSON.stringify(params.evidence, null, 2)}\n`, "utf8");
|
|
await fs.writeFile(reportPath, report, "utf8");
|
|
await fs.writeFile(summaryPath, `${JSON.stringify(summary, null, 2)}\n`, "utf8");
|
|
return {
|
|
evidencePath,
|
|
outputDir: params.outputDir,
|
|
report,
|
|
reportPath,
|
|
scenarios: [...params.scenarios],
|
|
summaryPath,
|
|
} satisfies QaUnifiedSuiteResult;
|
|
}
|
|
|
|
async function runUnifiedQaSuite(params: {
|
|
plan: Extract<QaSuiteExecutionPlan, { kind: "unified" }>;
|
|
runParams: QaSuiteRunParams | undefined;
|
|
}): Promise<QaUnifiedSuiteResult> {
|
|
rejectFlowOnlySuiteOptionsForUnifiedRun(params.runParams);
|
|
const startedAt = new Date();
|
|
const repoRoot = path.resolve(params.runParams?.repoRoot ?? process.cwd());
|
|
const outputDir = await resolveQaSuiteOutputDir(repoRoot, params.runParams?.outputDir);
|
|
const providerMode = normalizeQaProviderMode(
|
|
params.runParams?.providerMode ?? DEFAULT_QA_PROVIDER_MODE,
|
|
);
|
|
const primaryModel =
|
|
params.runParams?.primaryModel?.trim() || defaultQaModelForMode(providerMode);
|
|
const alternateModel =
|
|
params.runParams?.alternateModel?.trim() || defaultQaModelForMode(providerMode, true);
|
|
const fastMode =
|
|
typeof params.runParams?.fastMode === "boolean"
|
|
? params.runParams.fastMode
|
|
: isQaFastModeEnabled({ primaryModel, alternateModel });
|
|
const transportId = normalizeQaTransportId(params.runParams?.transportId);
|
|
const defaultConcurrency = params.runParams?.channelDriverSelection
|
|
? 1
|
|
: defaultQaSuiteConcurrencyForTransport(transportId);
|
|
const concurrency = normalizeQaSuiteConcurrency(
|
|
params.runParams?.concurrency,
|
|
params.plan.scenarios.length,
|
|
defaultConcurrency,
|
|
);
|
|
const evidenceSummaries: QaEvidenceSummaryJson[] = [];
|
|
const scenarioResultsById = new Map<string, QaSuiteScenarioResult>();
|
|
const sharedFlowPartitionTasks: QaUnifiedPartitionTask[] = [];
|
|
const isolatedFlowPartitionTasks: QaUnifiedPartitionTask[] = [];
|
|
const testFilePartitionTasks: QaUnifiedPartitionTask[] = [];
|
|
if (params.plan.flowScenarios.length > 0) {
|
|
const sharedFlowScenarios = params.plan.flowScenarios.filter(
|
|
(scenario) => !scenarioRequiresIsolatedQaSuiteWorker(scenario),
|
|
);
|
|
const isolatedFlowScenarios = params.plan.flowScenarios.filter(
|
|
scenarioRequiresIsolatedQaSuiteWorker,
|
|
);
|
|
const sharedFlowPartitions = partitionSharedFlowScenarios(sharedFlowScenarios, concurrency);
|
|
const isolatedFlowConcurrency = Math.min(
|
|
concurrency,
|
|
MAX_ISOLATED_FLOW_CONCURRENCY,
|
|
isolatedFlowScenarios.length,
|
|
);
|
|
const isolatedFlowPartitions =
|
|
isolatedFlowConcurrency === 1 && isolatedFlowScenarios.length > 1
|
|
? isolatedFlowScenarios.map((scenario, index) => ({
|
|
kind: `isolated-${index + 1}`,
|
|
scenarios: [scenario],
|
|
concurrency: 1,
|
|
}))
|
|
: [
|
|
{
|
|
kind: "isolated",
|
|
scenarios: isolatedFlowScenarios,
|
|
concurrency: isolatedFlowConcurrency,
|
|
},
|
|
];
|
|
const flowPartitions = [
|
|
...sharedFlowPartitions.map((scenarios, index) => ({
|
|
kind: sharedFlowPartitions.length === 1 ? "shared" : `shared-${index + 1}`,
|
|
scenarios,
|
|
concurrency: 1,
|
|
})),
|
|
...isolatedFlowPartitions,
|
|
].filter((partition) => partition.scenarios.length > 0);
|
|
const runFlowSuite = await loadQaFlowSuiteRuntime();
|
|
for (const partition of flowPartitions) {
|
|
const isolatedPartition =
|
|
partition.kind === "isolated" || partition.kind.startsWith("isolated-");
|
|
const task = {
|
|
weight: partition.concurrency,
|
|
run: async () => {
|
|
const result = await runFlowSuite({
|
|
...params.runParams,
|
|
outputDir:
|
|
flowPartitions.length === 1
|
|
? suitePartitionOutputDir(outputDir, "flow")
|
|
: flowSuitePartitionOutputDir(outputDir, partition.kind),
|
|
writeEvidenceFile: false,
|
|
providerMode,
|
|
primaryModel,
|
|
alternateModel,
|
|
fastMode,
|
|
concurrency: partition.concurrency,
|
|
workerStartStaggerMs: isolatedPartition
|
|
? (params.runParams?.workerStartStaggerMs ??
|
|
resolveQaSuiteWorkerStartStaggerMs(
|
|
partition.concurrency,
|
|
process.env,
|
|
ISOLATED_FLOW_WORKER_START_STAGGER_MS,
|
|
))
|
|
: params.runParams?.workerStartStaggerMs,
|
|
scenarioIds: partition.scenarios.map((scenario) => scenario.id),
|
|
});
|
|
const scenarioResults: QaUnifiedPartitionResult["scenarioResults"] = [];
|
|
for (const [index, scenario] of partition.scenarios.entries()) {
|
|
const scenarioResult = result.scenarios[index];
|
|
if (scenarioResult) {
|
|
scenarioResults.push({ scenarioId: scenario.id, result: scenarioResult });
|
|
}
|
|
}
|
|
return {
|
|
evidenceSummaries: [await resolveQaSuiteResultEvidenceSummary(result)],
|
|
scenarioResults,
|
|
};
|
|
},
|
|
} satisfies QaUnifiedPartitionTask;
|
|
if (isolatedPartition) {
|
|
isolatedFlowPartitionTasks.push(task);
|
|
} else {
|
|
sharedFlowPartitionTasks.push(task);
|
|
}
|
|
}
|
|
}
|
|
if (params.plan.testFileScenariosByKind.size > 0) {
|
|
testFilePartitionTasks.push({
|
|
weight: 1,
|
|
run: async () => {
|
|
const testFileEvidenceSummaries: QaEvidenceSummaryJson[] = [];
|
|
const testFileScenarioResults: QaUnifiedPartitionResult["scenarioResults"] = [];
|
|
for (const [kind, testFileScenarios] of params.plan.testFileScenariosByKind) {
|
|
const result = await runQaTestFileSuiteFromRuntime({
|
|
runParams: {
|
|
...params.runParams,
|
|
outputDir: suitePartitionOutputDir(outputDir, kind),
|
|
writeEvidenceFile: false,
|
|
providerMode,
|
|
primaryModel,
|
|
scenarioIds: testFileScenarios.map((scenario) => scenario.id),
|
|
},
|
|
scenarios: testFileScenarios,
|
|
});
|
|
testFileEvidenceSummaries.push(await resolveQaSuiteResultEvidenceSummary(result));
|
|
testFileScenarioResults.push(
|
|
...result.results.map((scenarioResult) => ({
|
|
scenarioId: scenarioResult.scenario.id,
|
|
result: testFileScenarioResultToSuiteScenario(scenarioResult, repoRoot),
|
|
})),
|
|
);
|
|
}
|
|
return {
|
|
evidenceSummaries: testFileEvidenceSummaries,
|
|
scenarioResults: testFileScenarioResults,
|
|
};
|
|
},
|
|
});
|
|
}
|
|
const partitionTasks = [
|
|
...sharedFlowPartitionTasks,
|
|
...testFilePartitionTasks,
|
|
...isolatedFlowPartitionTasks,
|
|
];
|
|
const partitionResults = await runWeightedUnifiedPartitionTasks(partitionTasks, concurrency);
|
|
for (const partitionResult of partitionResults) {
|
|
for (const scenarioResult of partitionResult.scenarioResults) {
|
|
scenarioResultsById.set(scenarioResult.scenarioId, scenarioResult.result);
|
|
}
|
|
evidenceSummaries.push(...partitionResult.evidenceSummaries);
|
|
}
|
|
const finishedAt = new Date();
|
|
const evidence = mergeQaEvidenceSummaries({
|
|
evidenceSummaries,
|
|
generatedAt: finishedAt.toISOString(),
|
|
});
|
|
const scenarios = params.plan.scenarios.map((scenario) => {
|
|
const result = scenarioResultsById.get(scenario.id);
|
|
if (result) {
|
|
return result;
|
|
}
|
|
return {
|
|
name: scenario.title,
|
|
status: "fail",
|
|
details: "suite partition returned no scenario result",
|
|
steps: [
|
|
{
|
|
name: "suite partition",
|
|
status: "fail",
|
|
details: "suite partition returned no scenario result",
|
|
},
|
|
],
|
|
} satisfies QaSuiteScenarioResult;
|
|
});
|
|
return await writeUnifiedQaSuiteArtifacts({
|
|
alternateModel,
|
|
concurrency,
|
|
evidence,
|
|
fastMode,
|
|
finishedAt,
|
|
outputDir,
|
|
primaryModel,
|
|
providerMode,
|
|
scenarioIds: params.plan.scenarios.map((scenario) => scenario.id),
|
|
scenarios,
|
|
startedAt,
|
|
});
|
|
}
|
|
|
|
export async function runQaSuite(...args: [QaSuiteRunParams?]): Promise<QaSuiteRuntimeResult> {
|
|
const runParams = args[0];
|
|
const plan = resolveSuiteExecutionPlan(runParams);
|
|
if (plan.kind === "unified") {
|
|
const result = await runUnifiedQaSuite({
|
|
runParams,
|
|
plan,
|
|
});
|
|
return {
|
|
executionKind: "suite",
|
|
result,
|
|
};
|
|
}
|
|
return {
|
|
executionKind: "flow",
|
|
result: await runQaFlowSuiteFromRuntime(...args),
|
|
};
|
|
}
|
|
|
|
export async function runQaFlowSuiteFromRuntime(
|
|
...args: [QaSuiteRunParams?]
|
|
): Promise<QaSuiteResult> {
|
|
return await (
|
|
await loadQaFlowSuiteRuntime()
|
|
)(args[0]);
|
|
}
|