diff --git a/config/max-lines-baseline.txt b/config/max-lines-baseline.txt index f48fa8fef3aa..b6d779c762e1 100644 --- a/config/max-lines-baseline.txt +++ b/config/max-lines-baseline.txt @@ -235,7 +235,6 @@ extensions/qa-lab/src/runtime-tool-fixture.ts extensions/qa-lab/src/scorecard-taxonomy.ts extensions/qa-lab/src/suite-launch.runtime.test.ts extensions/qa-lab/src/suite-launch.runtime.ts -extensions/qa-lab/src/suite.ts extensions/qa-lab/src/test-file-scenario-runner.test.ts extensions/qa-lab/web/src/app.ts extensions/qqbot/src/engine/gateway/outbound-dispatch.test.ts diff --git a/extensions/qa-lab/src/suite-artifacts.ts b/extensions/qa-lab/src/suite-artifacts.ts new file mode 100644 index 000000000000..f63c0705d18f --- /dev/null +++ b/extensions/qa-lab/src/suite-artifacts.ts @@ -0,0 +1,285 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import type { OpenClawCrablineChannelDriverSelection } from "@openclaw/crabline"; +import { assertQaSuiteArtifactWritten } from "./artifact-assertion.js"; +import { + hasQaCrablineArtifactPath, + resolveQaCrablineChannelDriverArtifactPaths, + type QaSuiteChannelDriverSelection, +} from "./crabline-artifacts.js"; +import { buildQaSuiteEvidenceSummary, QA_EVIDENCE_FILENAME } from "./evidence-summary.js"; +import type { QaProviderMode } from "./model-selection.js"; +import type { QaTransportAdapter } from "./qa-transport.js"; +import { renderQaMarkdownReport, type QaReportScenario } from "./report.js"; +import type { RuntimeId } from "./runtime-parity.js"; +import type { QaSeedScenarioWithSource } from "./scenario-catalog.js"; +import type { QaScorecardChannelDriver, QaScorecardEvidenceMode } from "./scorecard-taxonomy.js"; +import { splitModelRef } from "./suite-planning.js"; +import { countQaSuiteFailedScenarios, type QaSuiteSummaryJson } from "./suite-summary.js"; +import { createQaSuiteReportNotes } from "./suite-support.js"; +import type { QaSuiteScenarioResult } from "./suite-types.js"; + +type QaCrablineRuntime = typeof import("@openclaw/crabline"); +type QaCrablineChannelDriverSmokeResult = Awaited< + ReturnType +>; + +export type QaSuiteSummaryJsonParams = { + scenarios: QaSuiteScenarioResult[]; + startedAt: Date; + finishedAt: Date; + metrics?: QaSuiteSummaryJson["metrics"]; + evidence?: QaSuiteSummaryJson["evidence"]; + providerMode: QaProviderMode; + primaryModel: string; + alternateModel: string; + fastMode: boolean; + concurrency: number; + channelDriver?: QaScorecardChannelDriver | null; + channelDriverSelection?: QaSuiteChannelDriverSelection | null; + scenarioIds?: readonly string[]; + runtimePair?: [RuntimeId, RuntimeId]; +}; + +/** + * Strongly-typed shape of `qa-suite-summary.json`. The GPT-5.6 Luna parity gate + * (agentic-parity-report.ts, #64441) and any future parity wrapper can + * import this type instead of re-declaring the shape, so changes to the + * summary schema propagate through to every consumer at type-check time. + */ +export type QaSuiteGatewayRssSample = NonNullable< + NonNullable["gatewayProcessRssSamples"] +>[number]; + +export type QaSuiteGatewayHeapSnapshot = NonNullable< + NonNullable["gatewayHeapSnapshots"] +>[number]; + +/** + * Pure-ish JSON builder for qa-suite-summary.json. Exported so the GPT-5.6 Luna + * parity gate (agentic-parity-report.ts, #64441) and any future parity + * runner can assert-and-trust the provider/model that produced a given + * summary instead of blindly accepting the caller's candidateLabel / + * baselineLabel. Without the `run` block, a maintainer who swaps candidate + * and baseline summary paths could silently produce a mislabeled verdict. + * + * `scenarioIds` is only recorded when the caller passed a non-empty array + * (an explicit scenario selection). A missing or empty array means "no + * filter, full lane-selected catalog", which the summary encodes as `null` + * so parity/report tooling doesn't mistake a full run for an explicit + * empty selection. + */ +export function buildQaSuiteSummaryJson(params: QaSuiteSummaryJsonParams): QaSuiteSummaryJson { + const primarySplit = splitModelRef(params.primaryModel); + const alternateSplit = splitModelRef(params.alternateModel); + return { + scenarios: params.scenarios, + counts: { + total: params.scenarios.length, + passed: params.scenarios.filter((scenario) => scenario.status === "pass").length, + failed: countQaSuiteFailedScenarios(params.scenarios), + skipped: params.scenarios.filter((scenario) => scenario.status === "skip").length, + }, + ...(params.metrics ? { metrics: params.metrics } : {}), + ...(params.evidence ? { evidence: params.evidence } : {}), + run: { + startedAt: params.startedAt.toISOString(), + finishedAt: params.finishedAt.toISOString(), + providerMode: params.providerMode, + primaryModel: params.primaryModel, + primaryProvider: primarySplit?.provider ?? null, + primaryModelName: primarySplit?.model ?? null, + alternateModel: params.alternateModel, + alternateProvider: alternateSplit?.provider ?? null, + alternateModelName: alternateSplit?.model ?? null, + fastMode: params.fastMode, + concurrency: params.concurrency, + channelDriver: params.channelDriver ?? params.channelDriverSelection?.channelDriver ?? null, + channel: params.channelDriverSelection?.channel ?? null, + channelCapabilityMatrixPath: params.channelDriverSelection?.capabilityMatrixPath ?? null, + channelDriverSmokePath: params.channelDriverSelection?.smokeArtifactPath ?? null, + scenarioIds: + params.scenarioIds && params.scenarioIds.length > 0 ? [...params.scenarioIds] : null, + runtimePair: params.runtimePair ?? null, + }, + }; +} + +export async function writeQaSuiteArtifacts(params: { + repoRoot?: string; + outputDir: string; + startedAt: Date; + finishedAt: Date; + scenarios: QaSuiteScenarioResult[]; + scenarioDefinitions?: readonly QaSeedScenarioWithSource[]; + evidenceMode?: QaScorecardEvidenceMode; + metrics?: QaSuiteSummaryJson["metrics"]; + transport: QaTransportAdapter; + // Reuse the canonical QaProviderMode union instead of re-declaring it + // inline. Loop 6 already unified `QaSuiteSummaryJsonParams.providerMode` + // on this type; keeping the writer in sync prevents drift when model- + // selection.ts adds a new provider mode. + providerMode: QaProviderMode; + primaryModel: string; + alternateModel: string; + fastMode: boolean; + concurrency: number; + channelDriver?: QaScorecardChannelDriver | null; + channelDriverSelection?: OpenClawCrablineChannelDriverSelection | null; + isolatedWorkers?: boolean; + scenarioIds?: readonly string[]; + runtimePair?: [RuntimeId, RuntimeId]; + writeEvidenceFile?: boolean; + runCrablineChannelDriverSmoke?: ( + params: Parameters[0], + ) => Promise; +}) { + const reportPath = path.join(params.outputDir, "qa-suite-report.md"); + const summaryPath = path.join(params.outputDir, "qa-suite-summary.json"); + const evidencePath = path.join(params.outputDir, QA_EVIDENCE_FILENAME); + const crablineChannelDriverSelection = params.channelDriverSelection; + // Non-Crabline package acceptance mounts this source without plugin-local + // dependencies. Keep the owner runtime outside every unrelated live path. + const crablineRuntime = crablineChannelDriverSelection + ? await import("@openclaw/crabline") + : undefined; + let crablineChannelDriverSmoke: QaCrablineChannelDriverSmokeResult | undefined; + if (crablineChannelDriverSelection) { + const runCrablineChannelDriverSmoke = + params.runCrablineChannelDriverSmoke ?? + crablineRuntime?.runOpenClawCrablineChannelDriverSmoke; + if (!runCrablineChannelDriverSmoke) { + throw new Error("Crabline runtime did not provide its channel-driver smoke helper."); + } + crablineChannelDriverSmoke = await runCrablineChannelDriverSmoke({ + outputDir: params.outputDir, + selection: crablineChannelDriverSelection, + }); + } + const crablineChannelDriverArtifactPaths = resolveQaCrablineChannelDriverArtifactPaths({ + result: crablineChannelDriverSmoke, + selection: crablineChannelDriverSelection, + }); + const effectiveChannelDriverSelection: QaSuiteChannelDriverSelection | null | undefined = + crablineChannelDriverSelection && crablineChannelDriverArtifactPaths + ? { + ...crablineChannelDriverSelection, + ...crablineChannelDriverArtifactPaths, + } + : crablineChannelDriverSelection; + const report = renderQaMarkdownReport({ + title: "OpenClaw QA Scenario Suite", + startedAt: params.startedAt, + finishedAt: params.finishedAt, + checks: [], + scenarios: params.scenarios.map((scenario) => ({ + name: scenario.name, + status: scenario.status, + details: scenario.details, + steps: scenario.steps, + })) satisfies QaReportScenario[], + notes: createQaSuiteReportNotes({ + ...params, + channelDriverSelection: effectiveChannelDriverSelection, + createCrablineChannelReportNotes: crablineRuntime?.createOpenClawCrablineChannelReportNotes, + }), + }); + const evidence = + params.scenarioDefinitions && params.scenarioDefinitions.length > 0 + ? buildQaSuiteEvidenceSummary({ + artifactPaths: [ + { kind: "summary", path: path.basename(summaryPath) }, + { kind: "report", path: path.basename(reportPath) }, + ...(effectiveChannelDriverSelection + ? [ + { + kind: "channel-capability-matrix", + path: effectiveChannelDriverSelection.capabilityMatrixPath, + }, + { + kind: "channel-driver-smoke", + path: effectiveChannelDriverSelection.smokeArtifactPath, + }, + ] + : []), + ], + evidenceMode: params.evidenceMode, + channelId: params.channelDriverSelection?.channel ?? params.transport.id, + channelDriver: params.channelDriver ?? params.channelDriverSelection?.channelDriver, + env: process.env, + generatedAt: params.finishedAt.toISOString(), + primaryModel: params.primaryModel, + providerMode: params.providerMode, + repoRoot: params.repoRoot, + scenarioDefinitions: params.scenarioDefinitions, + scenarioResults: params.scenarios, + }) + : undefined; + if ( + crablineChannelDriverSelection && + crablineChannelDriverSmoke && + !hasQaCrablineArtifactPath(crablineChannelDriverSmoke.capabilityMatrixPath) + ) { + await fs.writeFile( + path.join(params.outputDir, crablineChannelDriverSelection.capabilityMatrixPath), + `${JSON.stringify( + { + version: 1, + source: "openclaw/crabline", + channelDriver: crablineChannelDriverSelection.channelDriver, + selectedChannel: crablineChannelDriverSelection.channel, + manifestPath: crablineChannelDriverSmoke.manifestPath, + report: crablineChannelDriverSmoke.capabilityReport, + }, + null, + 2, + )}\n`, + "utf8", + ); + } + if ( + crablineChannelDriverSelection && + crablineChannelDriverSmoke && + !hasQaCrablineArtifactPath(crablineChannelDriverSmoke.smokeArtifactPath) + ) { + await fs.writeFile( + path.join(params.outputDir, crablineChannelDriverSelection.smokeArtifactPath), + `${JSON.stringify( + { + version: 1, + source: "openclaw/crabline", + channelDriver: crablineChannelDriverSelection.channelDriver, + selectedChannel: crablineChannelDriverSelection.channel, + manifestPath: crablineChannelDriverSmoke.manifestPath, + smoke: crablineChannelDriverSmoke.smoke, + }, + null, + 2, + )}\n`, + "utf8", + ); + } + const writeEvidenceFile = params.writeEvidenceFile ?? true; + await fs.writeFile(reportPath, report, "utf8"); + if (evidence && writeEvidenceFile) { + await fs.writeFile(evidencePath, `${JSON.stringify(evidence, null, 2)}\n`, "utf8"); + } + await fs.writeFile( + summaryPath, + `${JSON.stringify( + buildQaSuiteSummaryJson({ + ...params, + channelDriverSelection: effectiveChannelDriverSelection, + }), + null, + 2, + )}\n`, + "utf8", + ); + await assertQaSuiteArtifactWritten("report", reportPath); + await assertQaSuiteArtifactWritten("summary", summaryPath); + if (evidence && writeEvidenceFile) { + await assertQaSuiteArtifactWritten("evidence", evidencePath); + } + return { evidence, evidencePath, report, reportPath, summaryPath }; +} diff --git a/extensions/qa-lab/src/suite-run-isolated.ts b/extensions/qa-lab/src/suite-run-isolated.ts new file mode 100644 index 000000000000..d68ef941b3b7 --- /dev/null +++ b/extensions/qa-lab/src/suite-run-isolated.ts @@ -0,0 +1,332 @@ +import path from "node:path"; +import { disposeRegisteredAgentHarnesses } from "openclaw/plugin-sdk/agent-harness"; +import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime"; +import type { QaLabLatestReport, QaLabScenarioOutcome } from "./lab-server.types.js"; +import { sanitizeQaProgressValue as sanitizeQaSuiteProgressValue } from "./progress-format.js"; +import { writeQaSuiteArtifacts } from "./suite-artifacts.js"; +import { mapQaSuiteWithConcurrency, resolveQaSuiteWorkerStartStaggerMs } from "./suite-planning.js"; +import { buildQaIsolatedScenarioWorkerParams } from "./suite-support.js"; +import type { + QaSuiteResolvedRunContext, + QaSuiteResult, + QaSuiteRunner, + QaSuiteRunParams, + QaSuiteScenarioResult, +} from "./suite-types.js"; +import { + createQaSuiteTransportAdapter, + requireQaSuiteStartLab, + runQaSuiteCleanupSteps, + throwQaSuiteCleanupErrors, + writeQaSuiteProgress, +} from "./suite.js"; + +export async function runQaFlowSuiteIsolated( + params: QaSuiteRunParams | undefined, + context: QaSuiteResolvedRunContext, + runQaFlowSuite: QaSuiteRunner, +): Promise { + const { + startedAt, + repoRoot, + outputDir, + transportId, + selectedScenarios, + providerMode, + primaryModel, + alternateModel, + fastMode, + concurrency, + progressEnabled, + } = context; + const ownsLab = !params?.lab; + const startLab = requireQaSuiteStartLab(params?.startLab); + const lab = + params?.lab ?? + (await startLab({ + repoRoot, + host: "127.0.0.1", + port: 0, + embeddedGateway: "disabled", + })); + const transportFactoryResult = await createQaSuiteTransportAdapter({ + adapterFactories: params?.adapterFactories, + channelDriver: params?.channelDriver, + channelId: params?.channelId, + channelDriverSelection: params?.channelDriverSelection, + adapterOptions: { + ...params?.adapterOptions, + scenarioIds: selectedScenarios.map((scenario) => scenario.id), + }, + cleanupOnFailure: ownsLab ? () => lab.stop() : undefined, + outputDir, + state: lab.state, + transportId, + }); + const transport = transportFactoryResult.adapter; + const liveScenarioOutcomes: QaLabScenarioOutcome[] = selectedScenarios.map((scenario) => ({ + id: scenario.id, + name: scenario.title, + status: "pending", + })); + const updateScenarioRun = () => + lab.setScenarioRun({ + kind: "suite", + status: "running", + startedAt: startedAt.toISOString(), + scenarios: [...liveScenarioOutcomes], + }); + const completedScenarioResults: Array = Array.from({ + length: selectedScenarios.length, + }); + let artifactWriteQueue = Promise.resolve(); + const writePartialArtifacts = () => { + const partialScenarios = completedScenarioResults.filter( + (scenario): scenario is QaSuiteScenarioResult => scenario !== undefined, + ); + const completedScenarioDefinitions = completedScenarioResults.flatMap((scenario, index) => + scenario === undefined || selectedScenarios[index] === undefined + ? [] + : [selectedScenarios[index]], + ); + if (partialScenarios.length === 0) { + return; + } + artifactWriteQueue = artifactWriteQueue + .then(async () => { + const partialFinishedAt = new Date(); + const { report, reportPath } = await writeQaSuiteArtifacts({ + repoRoot, + outputDir, + startedAt, + finishedAt: partialFinishedAt, + scenarios: partialScenarios, + scenarioDefinitions: completedScenarioDefinitions, + evidenceMode: params?.evidenceMode, + transport, + providerMode, + primaryModel, + alternateModel, + fastMode, + concurrency, + channelDriver: params?.channelDriver, + channelDriverSelection: params?.channelDriverSelection, + isolatedWorkers: true, + writeEvidenceFile: params?.writeEvidenceFile, + scenarioIds: + params?.scenarioIds && params.scenarioIds.length > 0 + ? selectedScenarios.map((scenario) => scenario.id) + : undefined, + }); + lab.setLatestReport({ + outputPath: reportPath, + markdown: report, + generatedAt: partialFinishedAt.toISOString(), + } satisfies QaLabLatestReport); + }) + .catch((error: unknown) => { + writeQaSuiteProgress( + progressEnabled, + `partial artifact write failed: ${sanitizeQaSuiteProgressValue(formatErrorMessage(error))}`, + ); + }); + }; + + let isolatedRunFailed = false; + let isolatedRunError: unknown; + let parentTransportCleaned = false; + try { + if (params?.channelDriver === "live") { + // The parent only renders aggregate artifacts. Release its live credentials + // before child workers acquire the same exclusive transport lease. + parentTransportCleaned = true; + await transportFactoryResult.cleanupWithoutGateway(); + } + updateScenarioRun(); + const workerStartStaggerMs = + params?.workerStartStaggerMs ?? resolveQaSuiteWorkerStartStaggerMs(concurrency); + writeQaSuiteProgress(progressEnabled, `scenario start stagger=${workerStartStaggerMs}ms`); + const scenarios: QaSuiteScenarioResult[] = await mapQaSuiteWithConcurrency( + selectedScenarios, + concurrency, + async (scenario, index): Promise => { + const scenarioIdForLog = sanitizeQaSuiteProgressValue(scenario.id); + writeQaSuiteProgress( + progressEnabled, + `scenario start (${index + 1}/${selectedScenarios.length}): ${scenarioIdForLog}`, + ); + liveScenarioOutcomes[index] = { + id: scenario.id, + name: scenario.title, + status: "running", + startedAt: new Date().toISOString(), + }; + updateScenarioRun(); + try { + const scenarioOutputDir = path.join(outputDir, "scenarios", scenario.id); + const result: QaSuiteResult = await runQaFlowSuite( + buildQaIsolatedScenarioWorkerParams({ + repoRoot, + outputDir: scenarioOutputDir, + providerMode, + transportId, + channelDriver: params?.channelDriver, + channelDriverSelection: params?.channelDriverSelection, + primaryModel, + alternateModel, + fastMode, + startLab, + scenario, + input: params, + }), + ); + const scenarioResult: QaSuiteScenarioResult = + result.scenarios[0] ?? + ({ + name: scenario.title, + status: "fail", + details: "isolated scenario run returned no scenario result", + steps: [ + { + name: "isolated scenario worker", + status: "fail", + details: "isolated scenario run returned no scenario result", + }, + ], + } satisfies QaSuiteScenarioResult); + liveScenarioOutcomes[index] = { + id: scenario.id, + name: scenario.title, + status: scenarioResult.status, + details: scenarioResult.details, + steps: scenarioResult.steps, + startedAt: liveScenarioOutcomes[index]?.startedAt, + finishedAt: new Date().toISOString(), + }; + updateScenarioRun(); + writeQaSuiteProgress( + progressEnabled, + `scenario ${scenarioResult.status} (${index + 1}/${selectedScenarios.length}): ${scenarioIdForLog}`, + ); + completedScenarioResults[index] = scenarioResult; + writePartialArtifacts(); + return scenarioResult; + } catch (error) { + const details = formatErrorMessage(error); + const scenarioResult = { + name: scenario.title, + status: "fail", + details, + steps: [ + { + name: "isolated scenario worker", + status: "fail", + details, + }, + ], + } satisfies QaSuiteScenarioResult; + liveScenarioOutcomes[index] = { + id: scenario.id, + name: scenario.title, + status: "fail", + details, + steps: scenarioResult.steps, + startedAt: liveScenarioOutcomes[index]?.startedAt, + finishedAt: new Date().toISOString(), + }; + updateScenarioRun(); + writeQaSuiteProgress( + progressEnabled, + `scenario fail (${index + 1}/${selectedScenarios.length}): ${scenarioIdForLog}`, + ); + completedScenarioResults[index] = scenarioResult; + writePartialArtifacts(); + return scenarioResult; + } + }, + { + startStaggerMs: workerStartStaggerMs, + shouldStop: (result) => params?.failFast === true && result.status === "fail", + }, + ); + await artifactWriteQueue; + const finishedAt = new Date(); + const failedCount = scenarios.filter((scenario) => scenario.status === "fail").length; + const skippedCount = scenarios.filter((scenario) => scenario.status === "skip").length; + lab.setScenarioRun({ + kind: "suite", + status: "completed", + startedAt: startedAt.toISOString(), + finishedAt: finishedAt.toISOString(), + scenarios: [...liveScenarioOutcomes], + }); + const { evidence, evidencePath, report, reportPath, summaryPath } = await writeQaSuiteArtifacts( + { + repoRoot, + outputDir, + startedAt, + finishedAt, + scenarios, + scenarioDefinitions: selectedScenarios, + evidenceMode: params?.evidenceMode, + transport, + providerMode, + primaryModel, + alternateModel, + fastMode, + concurrency, + channelDriver: params?.channelDriver, + channelDriverSelection: params?.channelDriverSelection, + isolatedWorkers: true, + writeEvidenceFile: params?.writeEvidenceFile, + // When the caller supplied an explicit non-empty --scenario filter, + // record the executed (post-selectQaFlowSuiteScenarios-normalized) ids + // so the summary matches what actually ran. When the caller passed + // nothing or an empty array ("no filter, full lane catalog"), + // preserve the unfiltered = null semantic so the summary stays + // distinguishable from an explicit all-scenarios selection. + scenarioIds: + params?.scenarioIds && params.scenarioIds.length > 0 + ? selectedScenarios.map((scenario) => scenario.id) + : undefined, + }, + ); + lab.setLatestReport({ + outputPath: reportPath, + markdown: report, + generatedAt: finishedAt.toISOString(), + } satisfies QaLabLatestReport); + writeQaSuiteProgress( + progressEnabled, + `run complete: passed=${scenarios.length - failedCount - skippedCount} failed=${failedCount} skipped=${skippedCount} total=${scenarios.length}`, + ); + return { + outputDir, + evidence, + evidencePath, + reportPath, + summaryPath, + report, + scenarios, + watchUrl: lab.baseUrl, + } satisfies QaSuiteResult; + } catch (error) { + isolatedRunFailed = true; + isolatedRunError = error; + throw error; + } finally { + const cleanupSteps: Array<() => Promise> = [ + ...(!parentTransportCleaned ? [() => transportFactoryResult.cleanupWithoutGateway()] : []), + () => disposeRegisteredAgentHarnesses(), + ]; + if (ownsLab) { + cleanupSteps.push(() => lab.stop()); + } + const cleanupErrors = await runQaSuiteCleanupSteps(cleanupSteps); + throwQaSuiteCleanupErrors({ + cleanupErrors, + runFailed: isolatedRunFailed, + runError: isolatedRunError, + }); + } +} diff --git a/extensions/qa-lab/src/suite-run-standard.ts b/extensions/qa-lab/src/suite-run-standard.ts new file mode 100644 index 000000000000..4e6bfb1c5d08 --- /dev/null +++ b/extensions/qa-lab/src/suite-run-standard.ts @@ -0,0 +1,427 @@ +import path from "node:path"; +import { setTimeout as sleep } from "node:timers/promises"; +import { disposeRegisteredAgentHarnesses } from "openclaw/plugin-sdk/agent-harness"; +import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; +import { startQaGatewayChild } from "./gateway-child.js"; +import type { QaLabLatestReport, QaLabScenarioOutcome } from "./lab-server.types.js"; +import { sanitizeQaProgressValue as sanitizeQaSuiteProgressValue } from "./progress-format.js"; +import { startQaProviderServer } from "./providers/server-runtime.js"; +import { captureRuntimeParityCell } from "./runtime-parity.js"; +import { + type QaSuiteGatewayHeapSnapshot, + type QaSuiteGatewayRssSample, + writeQaSuiteArtifacts, +} from "./suite-artifacts.js"; +import { applyQaMergePatch, collectQaSuiteTransportPolicy } from "./suite-planning.js"; +import { runQaSuiteRoundTripProbe } from "./suite-round-trip.js"; +import { waitForGatewayHealthy, waitForTransportReady } from "./suite-runtime-gateway.js"; +import { + buildQaGatewayHeapCheckpointRuntimeEnvPatch, + buildQaRuntimeEnvPatch, + mergeQaRuntimeEnvPatches, + runQaScenarioWithFlakeRetry, +} from "./suite-support.js"; +import type { + QaSuiteEnvironment, + QaSuiteResolvedRunContext, + QaSuiteResult, + QaSuiteRunParams, + QaSuiteScenarioRunner, + QaSuiteScenarioResult, +} from "./suite-types.js"; +import { + createQaSuiteTransportAdapter, + buildQaSuiteRuntimeMetrics, + captureGatewayHeapSnapshotCheckpoint, + requireQaSuiteStartLab, + resolveQaSuiteTransportReadyTimeoutMs, + runQaFlowSuiteCleanupPlan, + throwQaSuiteCleanupErrors, + waitForQaLabReadyOrStopOwned, + writeQaSuiteProgress, +} from "./suite.js"; +import { closeQaWebSessions } from "./web-runtime.js"; + +export async function runQaFlowSuiteStandard( + params: QaSuiteRunParams | undefined, + context: QaSuiteResolvedRunContext, + runScenarioDefinition: QaSuiteScenarioRunner, +): Promise { + const { + startedAt, + repoRoot, + outputDir, + transportId, + selectedScenarios, + providerMode, + primaryModel, + alternateModel, + fastMode, + enabledPluginIds, + gatewayConfigPatch, + gatewayRuntimeOptions, + concurrency, + progressEnabled, + gatewayHeapCheckpointsEnabled, + } = context; + const ownsLab = !params?.lab; + const startLab = params?.startLab; + writeQaSuiteProgress(progressEnabled, "lab start"); + const lab = + params?.lab ?? + (await requireQaSuiteStartLab(startLab)({ + repoRoot, + host: "127.0.0.1", + port: 0, + embeddedGateway: "disabled", + })); + writeQaSuiteProgress(progressEnabled, `lab ready: ${sanitizeQaSuiteProgressValue(lab.baseUrl)}`); + await waitForQaLabReadyOrStopOwned({ lab, ownsLab }); + const transportFactoryResult = await createQaSuiteTransportAdapter({ + adapterFactories: params?.adapterFactories, + channelDriver: params?.channelDriver, + channelId: params?.channelId, + channelDriverSelection: params?.channelDriverSelection, + adapterOptions: { + ...params?.adapterOptions, + scenarioIds: selectedScenarios.map((scenario) => scenario.id), + }, + cleanupOnFailure: ownsLab ? () => lab.stop() : undefined, + outputDir, + transportPolicy: collectQaSuiteTransportPolicy(selectedScenarios), + state: lab.state, + transportId, + }); + const transport = transportFactoryResult.adapter; + let mock: Awaited> | undefined; + let gateway: Awaited> | undefined; + let env: QaSuiteEnvironment | undefined; + let preserveGatewayRuntimeDir: string | undefined; + let runFailed = false; + let runError: unknown; + try { + writeQaSuiteProgress(progressEnabled, `provider start: ${providerMode}`); + const activeMock = await startQaProviderServer(providerMode, { + modelRefs: [primaryModel, alternateModel], + }); + mock = activeMock; + writeQaSuiteProgress( + progressEnabled, + `provider ready: ${sanitizeQaSuiteProgressValue(activeMock?.baseUrl ?? "live")}`, + ); + writeQaSuiteProgress(progressEnabled, "gateway start"); + const activeGateway = await startQaGatewayChild({ + repoRoot, + command: params?.sutOpenClawCommand, + providerBaseUrl: activeMock ? `${activeMock.baseUrl}/v1` : undefined, + transport, + transportBaseUrl: lab.listenUrl, + controlUiAllowedOrigins: [lab.listenUrl], + providerMode, + primaryModel, + alternateModel, + fastMode, + thinkingDefault: params?.thinkingDefault, + forcedRuntime: params?.forcedRuntime, + claudeCliAuthMode: params?.claudeCliAuthMode, + controlUiEnabled: params?.controlUiEnabled ?? true, + enabledPluginIds, + forwardHostHome: gatewayRuntimeOptions?.forwardHostHome, + mutateConfig: gatewayConfigPatch + ? (cfg) => applyQaMergePatch(cfg, gatewayConfigPatch) as OpenClawConfig + : undefined, + runtimeEnvPatch: mergeQaRuntimeEnvPatches( + buildQaRuntimeEnvPatch({ + providerMode, + forcedRuntime: params?.forcedRuntime, + mockBaseUrl: activeMock?.baseUrl, + }), + transport.createRuntimeEnvPatch?.(), + buildQaGatewayHeapCheckpointRuntimeEnvPatch(), + ), + }); + gateway = activeGateway; + writeQaSuiteProgress( + progressEnabled, + `gateway ready: ${sanitizeQaSuiteProgressValue(activeGateway.baseUrl)}`, + ); + lab.setControlUi({ + controlUiProxyTarget: activeGateway.baseUrl, + controlUiProxyToken: activeGateway.token, + }); + const activeEnv: QaSuiteEnvironment = { + lab, + mock: activeMock, + gateway: activeGateway, + outputDir, + // YAML scenarios should see the full staged gateway config, not just + // the transport fragment. Routing/session/plugin assertions depend on it. + cfg: activeGateway.cfg, + transport, + repoRoot, + providerMode, + primaryModel, + alternateModel, + webSessionIds: new Set(), + }; + env = activeEnv; + + const transportReadyTimeoutMs = resolveQaSuiteTransportReadyTimeoutMs( + params?.transportReadyTimeoutMs, + ); + // The gateway child already waits for /readyz before returning, but the + // selected transport can still be finishing account startup. Pay that + // readiness cost once here so the first scenario does not race bootstrap. + await waitForTransportReady(activeEnv, transportReadyTimeoutMs).catch(async () => { + await waitForGatewayHealthy(activeEnv, transportReadyTimeoutMs); + await waitForTransportReady(activeEnv, transportReadyTimeoutMs); + }); + await sleep(1_000); + const scenarios: QaSuiteScenarioResult[] = []; + const liveScenarioOutcomes: QaLabScenarioOutcome[] = selectedScenarios.map((scenario) => ({ + id: scenario.id, + name: scenario.title, + status: "pending", + })); + + lab.setScenarioRun({ + kind: "suite", + status: "running", + startedAt: startedAt.toISOString(), + scenarios: liveScenarioOutcomes, + }); + + const gatewayProcessRssSamples: QaSuiteGatewayRssSample[] = []; + const sampleGatewayProcessRss = (label: string) => { + const gatewayProcessRssBytes = activeGateway.getProcessRssBytes?.() ?? null; + if (gatewayProcessRssBytes !== null) { + gatewayProcessRssSamples.push({ + label, + at: new Date().toISOString(), + gatewayProcessRssBytes, + }); + } + return gatewayProcessRssBytes; + }; + const gatewayProcessCpuStartMs = activeGateway.getProcessCpuMs?.() ?? null; + const gatewayProcessRssStartBytes = sampleGatewayProcessRss("suite-start"); + const gatewayHeapSnapshots: QaSuiteGatewayHeapSnapshot[] = []; + const captureGatewayHeapCheckpoint = async (label: string) => { + if (!gatewayHeapCheckpointsEnabled) { + return; + } + const snapshot = await captureGatewayHeapSnapshotCheckpoint({ + gateway: activeGateway, + outputDir, + label, + }); + if (snapshot) { + gatewayHeapSnapshots.push(snapshot); + } + }; + await captureGatewayHeapCheckpoint("suite-start"); + for (const [index, scenario] of selectedScenarios.entries()) { + const scenarioIdForLog = sanitizeQaSuiteProgressValue(scenario.id); + writeQaSuiteProgress( + progressEnabled, + `scenario start (${index + 1}/${selectedScenarios.length}): ${scenarioIdForLog}`, + ); + sampleGatewayProcessRss(`scenario:${scenario.id}:start`); + liveScenarioOutcomes[index] = { + id: scenario.id, + name: scenario.title, + status: "running", + startedAt: new Date().toISOString(), + }; + lab.setScenarioRun({ + kind: "suite", + status: "running", + startedAt: startedAt.toISOString(), + scenarios: [...liveScenarioOutcomes], + }); + + const runSelectedScenario = () => runScenarioDefinition(activeEnv, scenario); + const scenarioRetryCount = + scenario.execution.kind === "flow" ? scenario.execution.retryCount : undefined; + let result: QaSuiteScenarioResult = + scenarioRetryCount === 0 + ? await runSelectedScenario() + : await runQaScenarioWithFlakeRetry(runSelectedScenario, () => + writeQaSuiteProgress( + progressEnabled, + `scenario retry (${index + 1}/${selectedScenarios.length}): ${scenarioIdForLog}`, + ), + ); + if (result.status === "pass" && params?.roundTripProbe?.scenarioId === scenario.id) { + const probeResult = await runQaSuiteRoundTripProbe({ + probe: params.roundTripProbe, + transport, + }); + const probePassed = probeResult.passed >= params.roundTripProbe.count; + result = { + ...result, + status: probePassed ? "pass" : "fail", + details: [result.details, probeResult.details].filter(Boolean).join(" | "), + timing: probeResult.timing, + steps: [ + ...result.steps, + { + name: "Round-trip samples", + status: probePassed ? "pass" : "fail", + details: probeResult.details, + }, + ], + }; + } + sampleGatewayProcessRss(`scenario:${scenario.id}:finish`); + scenarios.push(result); + writeQaSuiteProgress( + progressEnabled, + `scenario ${result.status} (${index + 1}/${selectedScenarios.length}): ${scenarioIdForLog}`, + ); + liveScenarioOutcomes[index] = { + id: scenario.id, + name: scenario.title, + status: result.status, + details: result.details, + steps: result.steps, + startedAt: liveScenarioOutcomes[index]?.startedAt, + finishedAt: new Date().toISOString(), + }; + lab.setScenarioRun({ + kind: "suite", + status: "running", + startedAt: startedAt.toISOString(), + scenarios: [...liveScenarioOutcomes], + }); + if (params?.failFast === true && result.status === "fail") { + break; + } + } + + const runtimeParityScenario = scenarios[0]; + const runtimeParityCell = + params?.captureRuntimeParityCell && + params.forcedRuntime && + selectedScenarios.length === 1 && + runtimeParityScenario + ? await captureRuntimeParityCell({ + runtime: params.forcedRuntime, + gateway: activeGateway, + scenarioResult: runtimeParityScenario, + wallClockMs: Math.max(1, Date.now() - startedAt.getTime()), + mockBaseUrl: activeMock?.baseUrl, + }) + : undefined; + const finishedAt = new Date(); + await captureGatewayHeapCheckpoint("suite-finish"); + const metrics = buildQaSuiteRuntimeMetrics({ + startedAt, + finishedAt, + gatewayProcessCpuStartMs, + gatewayProcessCpuEndMs: activeGateway.getProcessCpuMs?.() ?? null, + gatewayProcessRssStartBytes, + gatewayProcessRssEndBytes: sampleGatewayProcessRss("suite-finish"), + gatewayProcessRssSamples, + gatewayHeapSnapshots, + }); + const failedCount = scenarios.filter((scenario) => scenario.status === "fail").length; + const skippedCount = scenarios.filter((scenario) => scenario.status === "skip").length; + if ( + scenarios.some((scenario) => scenario.status === "fail") || + gatewayRuntimeOptions?.preserveDebugArtifacts === true + ) { + preserveGatewayRuntimeDir = path.join(outputDir, "artifacts", "gateway-runtime"); + } + lab.setScenarioRun({ + kind: "suite", + status: "completed", + startedAt: startedAt.toISOString(), + finishedAt: finishedAt.toISOString(), + scenarios: [...liveScenarioOutcomes], + }); + const { evidence, evidencePath, report, reportPath, summaryPath } = await writeQaSuiteArtifacts( + { + repoRoot, + outputDir, + startedAt, + finishedAt, + scenarios, + metrics, + scenarioDefinitions: selectedScenarios, + evidenceMode: params?.evidenceMode, + transport, + providerMode, + primaryModel, + alternateModel, + fastMode, + concurrency, + channelDriver: params?.channelDriver, + channelDriverSelection: params?.channelDriverSelection, + isolatedWorkers: false, + writeEvidenceFile: params?.writeEvidenceFile, + // Same "filtered → executed list, unfiltered → null" convention as + // the concurrent-path writeQaSuiteArtifacts call above. + scenarioIds: + params?.scenarioIds && params.scenarioIds.length > 0 + ? selectedScenarios.map((scenario) => scenario.id) + : undefined, + }, + ); + const latestReport = { + outputPath: reportPath, + markdown: report, + generatedAt: finishedAt.toISOString(), + } satisfies QaLabLatestReport; + lab.setLatestReport(latestReport); + writeQaSuiteProgress( + progressEnabled, + `run complete: passed=${scenarios.length - failedCount - skippedCount} failed=${failedCount} skipped=${skippedCount} total=${scenarios.length}`, + ); + + return { + outputDir, + evidence, + evidencePath, + reportPath, + summaryPath, + report, + scenarios, + watchUrl: lab.baseUrl, + ...(runtimeParityCell ? { runtimeParityCell } : {}), + } satisfies QaSuiteResult; + } catch (error) { + runFailed = true; + runError = error; + preserveGatewayRuntimeDir = path.join(outputDir, "artifacts", "gateway-runtime"); + throw error; + } finally { + const activeEnv = env; + const keepTemp = process.env.OPENCLAW_QA_KEEP_TEMP === "1" || false; + const activeGateway = gateway; + const activeMock = mock; + const cleanupErrors = await runQaFlowSuiteCleanupPlan({ + closeWebSessions: activeEnv ? () => closeQaWebSessions(activeEnv.webSessionIds) : undefined, + cleanupTransportBeforeGatewayStop: () => transportFactoryResult.cleanupBeforeGatewayStop(), + cleanupTransportAfterGatewayStop: () => transportFactoryResult.cleanupAfterGatewayStop(), + stopGateway: activeGateway + ? () => + activeGateway.stop({ + keepTemp, + preserveToDir: keepTemp ? undefined : preserveGatewayRuntimeDir, + }) + : undefined, + disposeAgentHarnesses: () => disposeRegisteredAgentHarnesses(), + stopProvider: activeMock ? () => activeMock.stop() : undefined, + finishLab: ownsLab + ? () => lab.stop() + : async () => { + lab.setControlUi({ + controlUiUrl: null, + controlUiProxyTarget: null, + }); + }, + }); + throwQaSuiteCleanupErrors({ cleanupErrors, runFailed, runError }); + } +} diff --git a/extensions/qa-lab/src/suite-run.runtime.ts b/extensions/qa-lab/src/suite-run.runtime.ts new file mode 100644 index 000000000000..bde4ed9dfdd2 --- /dev/null +++ b/extensions/qa-lab/src/suite-run.runtime.ts @@ -0,0 +1,157 @@ +import path from "node:path"; +import { + defaultQaSuiteConcurrencyForTransport, + normalizeQaTransportId, +} from "./qa-transport-registry.js"; +import { readQaBootstrapScenarioCatalog } from "./scenario-catalog.js"; +import { + resolveRequestedQaSuiteModels, + resolveSelectedQaSuiteModels, +} from "./suite-model-selection.js"; +import { + collectQaSuiteGatewayConfigPatch, + collectQaSuiteGatewayRuntimeOptions, + collectQaSuitePluginIds, + normalizeQaSuiteConcurrency, + resolveQaSuiteOutputDir, + selectQaFlowSuiteScenarios, +} from "./suite-planning.js"; +import { runQaFlowSuiteIsolated } from "./suite-run-isolated.js"; +import { runQaFlowSuiteStandard } from "./suite-run-standard.js"; +import { runQaRuntimeParitySuite } from "./suite-runtime-parity-runner.js"; +import { shouldCaptureGatewayHeapCheckpoints } from "./suite-support.js"; +import type { QaSuiteResolvedRunContext, QaSuiteResult, QaSuiteRunParams } from "./suite-types.js"; +import { + formatQaSuiteRunStartProgress, + runQaSuiteScenarioDefinitionForRuntime, + shouldLogQaSuiteProgress, + shouldRunQaSuiteWithIsolatedScenarioWorkers, + writeQaSuiteProgress, +} from "./suite.js"; + +export async function runQaFlowSuiteFromRuntime(params?: QaSuiteRunParams): Promise { + const startedAt = new Date(); + const repoRoot = path.resolve(params?.repoRoot ?? process.cwd()); + const requestedModels = resolveRequestedQaSuiteModels(params ?? {}); + const transportId = normalizeQaTransportId(params?.transportId); + const outputDir = await resolveQaSuiteOutputDir(repoRoot, params?.outputDir); + const catalog = readQaBootstrapScenarioCatalog(); + const channelDriver = params?.channelDriver ?? params?.channelDriverSelection?.channelDriver; + const selectedScenarios = selectQaFlowSuiteScenarios({ + scenarios: catalog.scenarios, + scenarioIds: params?.scenarioIds, + providerMode: requestedModels.providerMode, + primaryModel: requestedModels.primaryModel, + channelDriver, + channel: params?.channelId ?? params?.channelDriverSelection?.channel, + claudeCliAuthMode: params?.claudeCliAuthMode, + }); + const { alternateModel, fastMode, primaryModel, providerMode } = resolveSelectedQaSuiteModels({ + alternateModelExplicit: params?.alternateModel !== undefined, + fastMode: params?.fastMode, + primaryModelExplicit: params?.primaryModel !== undefined, + requested: requestedModels, + scenarios: selectedScenarios, + }); + if ( + params?.roundTripProbe && + !selectedScenarios.some((scenario) => scenario.id === params.roundTripProbe?.scenarioId) + ) { + throw new Error( + `QA round-trip probe scenario is not selected: ${params.roundTripProbe.scenarioId}`, + ); + } + if (params?.roundTripProbe && params.runtimePair) { + throw new Error("QA round-trip probes are not supported with runtime-pair runs."); + } + const enabledPluginIds = [ + ...new Set([ + ...collectQaSuitePluginIds(selectedScenarios), + ...(params?.enabledPluginIds ?? []).map((pluginId) => pluginId.trim()).filter(Boolean), + ...(params?.forcedRuntime && params.forcedRuntime !== "openclaw" + ? [params.forcedRuntime] + : []), + ]), + ]; + const gatewayConfigPatch = collectQaSuiteGatewayConfigPatch( + selectedScenarios, + params?.adapterOptions?.sutAccountId?.trim() || + (channelDriver === "crabline" ? "default" : "sut"), + ); + const gatewayRuntimeOptions = collectQaSuiteGatewayRuntimeOptions(selectedScenarios); + const concurrency = params?.failFast + ? 1 + : normalizeQaSuiteConcurrency( + params?.concurrency, + selectedScenarios.length, + params?.channelDriverSelection ? 1 : defaultQaSuiteConcurrencyForTransport(transportId), + ); + const progressEnabled = shouldLogQaSuiteProgress(); + const context: QaSuiteResolvedRunContext = { + startedAt, + repoRoot, + outputDir, + transportId, + selectedScenarios, + providerMode, + primaryModel, + alternateModel, + fastMode, + channelDriver, + enabledPluginIds, + gatewayConfigPatch, + gatewayRuntimeOptions, + concurrency, + progressEnabled, + gatewayHeapCheckpointsEnabled: shouldCaptureGatewayHeapCheckpoints(), + }; + writeQaSuiteProgress( + progressEnabled, + formatQaSuiteRunStartProgress({ + selectedScenarioCount: selectedScenarios.length, + concurrency, + transportId, + channelDriver: params?.channelDriver, + channelDriverSelection: params?.channelDriverSelection, + }), + ); + const useIsolatedScenarioWorkers = shouldRunQaSuiteWithIsolatedScenarioWorkers({ + scenarios: selectedScenarios, + concurrency, + lab: params?.lab, + startLab: params?.startLab, + }); + if (params?.runtimePair) { + return await runQaRuntimeParitySuite({ + runQaFlowSuite: runQaFlowSuiteFromRuntime, + adapterFactories: params.adapterFactories, + channelId: params.channelId, + adapterOptions: params.adapterOptions, + evidenceMode: params.evidenceMode, + repoRoot, + outputDir, + startedAt, + providerMode, + transportId, + channelDriverSelection: params.channelDriverSelection, + channelDriver: params.channelDriver, + primaryModel, + alternateModel, + fastMode, + thinkingDefault: params.thinkingDefault, + claudeCliAuthMode: params.claudeCliAuthMode, + enabledPluginIds: params.enabledPluginIds, + concurrency, + selectedScenarios, + startLab: params.startLab, + lab: params.lab, + progressEnabled, + scenarioIds: params.scenarioIds, + runtimePair: params.runtimePair, + writeEvidenceFile: params.writeEvidenceFile, + }); + } + return useIsolatedScenarioWorkers + ? await runQaFlowSuiteIsolated(params, context, runQaFlowSuiteFromRuntime) + : await runQaFlowSuiteStandard(params, context, runQaSuiteScenarioDefinitionForRuntime); +} diff --git a/extensions/qa-lab/src/suite-runtime-parity-runner.ts b/extensions/qa-lab/src/suite-runtime-parity-runner.ts new file mode 100644 index 000000000000..d7ba77669a47 --- /dev/null +++ b/extensions/qa-lab/src/suite-runtime-parity-runner.ts @@ -0,0 +1,346 @@ +import path from "node:path"; +import type { OpenClawCrablineChannelDriverSelection } from "@openclaw/crabline"; +import type { QaCliBackendAuthMode } from "./gateway-child.js"; +import type { + QaLabLatestReport, + QaLabScenarioOutcome, + QaLabServerHandle, +} from "./lab-server.types.js"; +import type { QaProviderMode } from "./model-selection.js"; +import { sanitizeQaProgressValue as sanitizeQaSuiteProgressValue } from "./progress-format.js"; +import type { QaThinkingLevel } from "./qa-gateway-config.js"; +import type { QaTransportAdapterFactory, QaTransportId } from "./qa-transport-registry.js"; +import { + isRuntimeParityResultPass, + runRuntimeParityScenario, + type RuntimeId, + type RuntimeParityCell, + type RuntimeParityResult, +} from "./runtime-parity.js"; +import { readQaBootstrapScenarioCatalog } from "./scenario-catalog.js"; +import type { QaScorecardChannelDriver, QaScorecardEvidenceMode } from "./scorecard-taxonomy.js"; +import { writeQaSuiteArtifacts } from "./suite-artifacts.js"; +import { + collectQaSuiteTransportPolicy, + mapQaSuiteWithConcurrency, + resolveQaSuiteWorkerStartStaggerMs, + scenarioRequiresControlUi, +} from "./suite-planning.js"; +import { remapModelRefForForcedRuntime } from "./suite-support.js"; +import type { + QaSuiteRunParams, + QaSuiteRunner, + QaSuiteScenarioResult, + QaSuiteStartLabFn, + QaSuiteResult, +} from "./suite-types.js"; +import { + createQaSuiteTransportAdapter, + requireQaSuiteStartLab, + writeQaSuiteProgress, +} from "./suite.js"; + +function isRuntimeParityPass(result: RuntimeParityResult) { + return isRuntimeParityResultPass(result); +} + +function formatRuntimeParityCellDetails(cell: RuntimeParityCell) { + const errors = [cell.transportErrorClass, cell.runtimeErrorClass].filter(Boolean).join(", "); + const sentinels = cell.sentinelFindings?.map((finding) => finding.kind).join(", "); + return [ + `runtime=${cell.runtime}`, + `wallMs=${cell.wallClockMs}`, + `toolCalls=${cell.toolCalls.length}`, + `finalChars=${cell.finalText.length}`, + `tokens=${cell.usage.totalTokens}`, + ...(errors ? [`errors=${errors}`] : []), + ...(sentinels ? [`sentinels=${sentinels}`] : []), + ].join(" "); +} + +function buildRuntimeParityScenarioResult(params: { + scenarioName: string; + result: RuntimeParityResult; +}): QaSuiteScenarioResult { + const driftStepStatus = isRuntimeParityPass(params.result) ? "pass" : "fail"; + const openclawCell = params.result.cells.openclaw; + return { + name: params.scenarioName, + status: driftStepStatus, + details: params.result.driftDetails ?? `runtime drift classified as ${params.result.drift}`, + steps: [ + { + name: openclawCell.runtime, + status: + openclawCell.runtimeErrorClass || openclawCell.transportErrorClass ? "fail" : "pass", + details: formatRuntimeParityCellDetails(openclawCell), + }, + { + name: params.result.cells.codex.runtime, + status: + params.result.cells.codex.runtimeErrorClass || + params.result.cells.codex.transportErrorClass + ? "fail" + : "pass", + details: formatRuntimeParityCellDetails(params.result.cells.codex), + }, + { + name: "runtime drift", + status: driftStepStatus, + details: params.result.driftDetails ?? params.result.drift, + }, + ], + runtimeParity: params.result, + }; +} + +export async function runQaRuntimeParitySuite(params: { + runQaFlowSuite: QaSuiteRunner; + adapterOptions?: QaSuiteRunParams["adapterOptions"]; + adapterFactories?: readonly QaTransportAdapterFactory[]; + channelId?: string; + evidenceMode?: QaScorecardEvidenceMode; + repoRoot: string; + outputDir: string; + startedAt: Date; + providerMode: QaProviderMode; + transportId: QaTransportId; + primaryModel: string; + alternateModel: string; + fastMode: boolean; + thinkingDefault?: QaThinkingLevel; + claudeCliAuthMode?: QaCliBackendAuthMode; + enabledPluginIds?: string[]; + channelDriver?: QaScorecardChannelDriver | null; + channelDriverSelection?: OpenClawCrablineChannelDriverSelection | null; + concurrency: number; + selectedScenarios: ReturnType["scenarios"]; + startLab?: QaSuiteStartLabFn; + lab?: QaLabServerHandle; + progressEnabled: boolean; + scenarioIds?: readonly string[]; + runtimePair: [RuntimeId, RuntimeId]; + writeEvidenceFile?: boolean; +}) { + const ownsLab = !params.lab; + const startLab = requireQaSuiteStartLab(params.startLab); + const lab = + params.lab ?? + (await startLab({ + repoRoot: params.repoRoot, + host: "127.0.0.1", + port: 0, + embeddedGateway: "disabled", + })); + const transportFactoryResult = await createQaSuiteTransportAdapter({ + adapterFactories: params.adapterFactories, + channelDriver: params.channelDriver, + channelId: params.channelId, + channelDriverSelection: params.channelDriverSelection, + adapterOptions: params.adapterOptions, + cleanupOnFailure: ownsLab ? () => lab.stop() : undefined, + outputDir: params.outputDir, + transportPolicy: collectQaSuiteTransportPolicy(params.selectedScenarios), + state: lab.state, + transportId: params.transportId, + }); + const transport = transportFactoryResult.adapter; + const liveScenarioOutcomes: QaLabScenarioOutcome[] = params.selectedScenarios.map((scenario) => ({ + id: scenario.id, + name: scenario.title, + status: "pending", + })); + lab.setScenarioRun({ + kind: "suite", + status: "running", + startedAt: params.startedAt.toISOString(), + scenarios: [...liveScenarioOutcomes], + }); + + try { + const scenarios = await mapQaSuiteWithConcurrency( + params.selectedScenarios, + params.concurrency, + async (scenario, index): Promise => { + const scenarioIdForLog = sanitizeQaSuiteProgressValue(scenario.id); + writeQaSuiteProgress( + params.progressEnabled, + `runtime pair start (${index + 1}/${params.selectedScenarios.length}): ${scenarioIdForLog}`, + ); + liveScenarioOutcomes[index] = { + id: scenario.id, + name: scenario.title, + status: "running", + startedAt: new Date().toISOString(), + }; + lab.setScenarioRun({ + kind: "suite", + status: "running", + startedAt: params.startedAt.toISOString(), + scenarios: [...liveScenarioOutcomes], + }); + + const parity = await runRuntimeParityScenario({ + scenarioId: scenario.id, + runtimeParityUsage: scenario.runtimeParityUsage, + runCell: async (runtime) => { + const cellOutputDir = path.join( + params.outputDir, + "runtime-cells", + scenario.id, + runtime, + ); + const cellStartedAt = Date.now(); + const cellResult = await params.runQaFlowSuite({ + adapterFactories: params.adapterFactories, + channelId: params.channelId, + adapterOptions: params.adapterOptions, + repoRoot: params.repoRoot, + outputDir: cellOutputDir, + providerMode: params.providerMode, + transportId: params.transportId, + channelDriver: params.channelDriver ?? undefined, + channelDriverSelection: params.channelDriverSelection, + primaryModel: remapModelRefForForcedRuntime({ + modelRef: params.primaryModel, + providerMode: params.providerMode, + forcedRuntime: runtime, + }), + alternateModel: remapModelRefForForcedRuntime({ + modelRef: params.alternateModel, + providerMode: params.providerMode, + forcedRuntime: runtime, + }), + fastMode: params.fastMode, + thinkingDefault: params.thinkingDefault, + claudeCliAuthMode: params.claudeCliAuthMode, + scenarioIds: [scenario.id], + concurrency: 1, + enabledPluginIds: params.enabledPluginIds, + startLab, + controlUiEnabled: scenarioRequiresControlUi(scenario), + forcedRuntime: runtime, + captureRuntimeParityCell: true, + writeEvidenceFile: params.writeEvidenceFile, + }); + const scenarioResult = + cellResult.scenarios[0] ?? + ({ + name: scenario.title, + status: "fail", + details: "runtime parity cell returned no scenario result", + steps: [ + { + name: "runtime parity cell", + status: "fail", + details: "runtime parity cell returned no scenario result", + }, + ], + } satisfies QaSuiteScenarioResult); + const fallbackCell = { + runtime, + transcriptBytes: "", + toolCalls: [], + finalText: "", + usage: { + inputTokens: 0, + outputTokens: 0, + totalTokens: 0, + }, + wallClockMs: Math.max(1, Date.now() - cellStartedAt), + runtimeErrorClass: "capture-missing", + bootStateLines: [], + } satisfies RuntimeParityCell; + return { + scenarioStatus: scenarioResult.status === "pass" ? "pass" : "fail", + scenarioDetails: scenarioResult.details, + cell: cellResult.runtimeParityCell ?? fallbackCell, + }; + }, + }); + + const result = buildRuntimeParityScenarioResult({ + scenarioName: scenario.title, + result: parity, + }); + liveScenarioOutcomes[index] = { + id: scenario.id, + name: scenario.title, + status: result.status, + details: result.details, + steps: result.steps, + startedAt: liveScenarioOutcomes[index]?.startedAt, + finishedAt: new Date().toISOString(), + }; + lab.setScenarioRun({ + kind: "suite", + status: "running", + startedAt: params.startedAt.toISOString(), + scenarios: [...liveScenarioOutcomes], + }); + writeQaSuiteProgress( + params.progressEnabled, + `runtime pair ${result.status} (${index + 1}/${params.selectedScenarios.length}): ${scenarioIdForLog}`, + ); + return result; + }, + { + startStaggerMs: resolveQaSuiteWorkerStartStaggerMs(params.concurrency), + }, + ); + + const finishedAt = new Date(); + const { evidence, evidencePath, report, reportPath, summaryPath } = await writeQaSuiteArtifacts( + { + repoRoot: params.repoRoot, + outputDir: params.outputDir, + startedAt: params.startedAt, + finishedAt, + scenarios, + scenarioDefinitions: params.selectedScenarios, + evidenceMode: params.evidenceMode, + transport, + providerMode: params.providerMode, + primaryModel: params.primaryModel, + alternateModel: params.alternateModel, + fastMode: params.fastMode, + concurrency: params.concurrency, + channelDriver: params.channelDriver, + channelDriverSelection: params.channelDriverSelection, + scenarioIds: + params.scenarioIds && params.scenarioIds.length > 0 + ? params.selectedScenarios.map((scenario) => scenario.id) + : undefined, + runtimePair: params.runtimePair, + writeEvidenceFile: params.writeEvidenceFile, + }, + ); + lab.setLatestReport({ + outputPath: reportPath, + markdown: report, + generatedAt: finishedAt.toISOString(), + } satisfies QaLabLatestReport); + lab.setScenarioRun({ + kind: "suite", + status: "completed", + startedAt: params.startedAt.toISOString(), + finishedAt: finishedAt.toISOString(), + scenarios: [...liveScenarioOutcomes], + }); + return { + outputDir: params.outputDir, + evidence, + evidencePath, + reportPath, + summaryPath, + report, + scenarios, + watchUrl: lab.baseUrl, + } satisfies QaSuiteResult; + } finally { + await transportFactoryResult.cleanupWithoutGateway(); + if (ownsLab) { + await lab.stop(); + } + } +} diff --git a/extensions/qa-lab/src/suite-support.ts b/extensions/qa-lab/src/suite-support.ts new file mode 100644 index 000000000000..5156074f9d5d --- /dev/null +++ b/extensions/qa-lab/src/suite-support.ts @@ -0,0 +1,183 @@ +import type { OpenClawCrablineChannelDriverSelection } from "@openclaw/crabline"; +import type { QaSuiteChannelDriverSelection } from "./crabline-artifacts.js"; +import type { QaProviderMode } from "./model-selection.js"; +import { parseQaProgressBooleanEnv as parseQaSuiteBooleanEnv } from "./progress-format.js"; +import type { QaTransportId } from "./qa-transport-registry.js"; +import type { QaTransportAdapter } from "./qa-transport.js"; +import type { RuntimeId } from "./runtime-parity.js"; +import { readQaBootstrapScenarioCatalog } from "./scenario-catalog.js"; +import type { QaScorecardChannelDriver } from "./scorecard-taxonomy.js"; +import { scenarioRequiresControlUi, splitModelRef } from "./suite-planning.js"; +import type { QaSuiteRunParams, QaSuiteScenarioResult, QaSuiteStartLabFn } from "./suite-types.js"; + +type QaCrablineRuntime = typeof import("@openclaw/crabline"); + +/** + * One bounded retry for live-model flake: flow scenarios time out under model + * latency spikes, so a first failure gets a single rerun. A retry pass keeps + * the first attempt visible in details; a retry failure keeps the original + * diagnostics so deterministic regressions still fail the suite. + */ +export async function runQaScenarioWithFlakeRetry( + run: () => Promise, + onRetry?: () => void, +): Promise { + const first = await run(); + if (first.status !== "fail") { + return first; + } + onRetry?.(); + const second = await run(); + if (second.status !== "pass") { + return first; + } + return { + ...second, + details: [second.details, `passed on retry; first attempt: ${first.details ?? "failed"}`] + .filter(Boolean) + .join(" | "), + }; +} + +export function createQaSuiteReportNotes(params: { + transport: QaTransportAdapter; + channelDriverSelection?: QaSuiteChannelDriverSelection | null; + providerMode: QaProviderMode; + primaryModel: string; + alternateModel: string; + fastMode: boolean; + concurrency: number; + isolatedWorkers?: boolean; + createCrablineChannelReportNotes?: QaCrablineRuntime["createOpenClawCrablineChannelReportNotes"]; +}) { + return [ + ...params.transport.createReportNotes(params), + // Crabline reports completed generation paths through this filename-narrowed selection. + ...(params.createCrablineChannelReportNotes?.( + params.channelDriverSelection as OpenClawCrablineChannelDriverSelection | null | undefined, + ) ?? []), + ]; +} + +export function buildQaIsolatedScenarioWorkerParams(params: { + repoRoot: string; + outputDir: string; + providerMode: QaProviderMode; + transportId: QaTransportId; + channelDriver?: QaScorecardChannelDriver; + channelDriverSelection?: OpenClawCrablineChannelDriverSelection | null; + primaryModel: string; + alternateModel: string; + fastMode: boolean; + scenario: ReturnType["scenarios"][number]; + input?: QaSuiteRunParams; + startLab: QaSuiteStartLabFn; +}): QaSuiteRunParams { + return { + adapterFactories: params.input?.adapterFactories, + adapterOptions: params.input?.adapterOptions, + channelId: params.input?.channelId, + repoRoot: params.repoRoot, + sutOpenClawCommand: params.input?.sutOpenClawCommand, + outputDir: params.outputDir, + providerMode: params.providerMode, + transportId: params.transportId, + channelDriver: params.channelDriver, + channelDriverSelection: params.channelDriverSelection, + primaryModel: params.primaryModel, + alternateModel: params.alternateModel, + fastMode: params.fastMode, + thinkingDefault: params.input?.thinkingDefault, + claudeCliAuthMode: params.input?.claudeCliAuthMode, + scenarioIds: [params.scenario.id], + enabledPluginIds: params.input?.enabledPluginIds, + concurrency: 1, + startLab: params.startLab, + controlUiEnabled: scenarioRequiresControlUi(params.scenario), + transportReadyTimeoutMs: params.input?.transportReadyTimeoutMs, + workerStartStaggerMs: params.input?.workerStartStaggerMs, + forcedRuntime: params.input?.forcedRuntime, + roundTripProbe: + params.input?.roundTripProbe?.scenarioId === params.scenario.id + ? params.input.roundTripProbe + : undefined, + writeEvidenceFile: params.input?.writeEvidenceFile, + }; +} + +export function remapModelRefForForcedRuntime(params: { + modelRef: string; + providerMode: QaProviderMode; + forcedRuntime?: RuntimeId; +}) { + if (params.forcedRuntime !== "codex" || params.providerMode !== "mock-openai") { + return params.modelRef; + } + const split = splitModelRef(params.modelRef); + if (!split || split.provider !== "mock-openai") { + return params.modelRef; + } + return `openai/${split.model}`; +} + +export function buildQaRuntimeEnvPatch(params: { + providerMode: QaProviderMode; + forcedRuntime?: RuntimeId; + mockBaseUrl?: string; +}): NodeJS.ProcessEnv | undefined { + const patch: NodeJS.ProcessEnv = {}; + if (params.forcedRuntime) { + patch.OPENCLAW_BUILD_PRIVATE_QA = "1"; + patch.OPENCLAW_QA_FORCE_RUNTIME = params.forcedRuntime; + } + if (params.forcedRuntime !== "codex" || params.providerMode !== "mock-openai") { + return Object.keys(patch).length > 0 ? patch : undefined; + } + let mockBaseUrl = params.mockBaseUrl?.trim(); + while (mockBaseUrl?.endsWith("/")) { + mockBaseUrl = mockBaseUrl.slice(0, -1); + } + if (!mockBaseUrl) { + return Object.keys(patch).length > 0 ? patch : undefined; + } + // The forced codex lane uses the Codex app-server's native OpenAI provider + // path, so pin the managed app-server to the QA mock endpoint instead of + // leaking to the maintainer's real OpenAI config. + patch.OPENCLAW_CODEX_APP_SERVER_ARGS = `app-server -c openai_base_url=${mockBaseUrl}/v1 --listen stdio://`; + patch.OPENAI_API_KEY = "qa-mock-openai-key"; + patch.CODEX_API_KEY = "qa-mock-openai-key"; + return patch; +} + +export function appendNodeOption(raw: string | undefined, option: string) { + const parts = (raw ?? "").split(/\s+/u).filter(Boolean); + return parts.includes(option) ? parts.join(" ") : [...parts, option].join(" "); +} + +export function shouldCaptureGatewayHeapCheckpoints(env: NodeJS.ProcessEnv = process.env) { + return parseQaSuiteBooleanEnv(env.OPENCLAW_QA_GATEWAY_HEAP_CHECKPOINTS) === true; +} + +export function buildQaGatewayHeapCheckpointRuntimeEnvPatch( + env: NodeJS.ProcessEnv = process.env, +): NodeJS.ProcessEnv | undefined { + if (!shouldCaptureGatewayHeapCheckpoints(env)) { + return undefined; + } + return { + NODE_OPTIONS: appendNodeOption(env.NODE_OPTIONS, "--heapsnapshot-signal=SIGUSR2"), + }; +} + +export function mergeQaRuntimeEnvPatches( + ...patches: Array +): NodeJS.ProcessEnv | undefined { + const merged: NodeJS.ProcessEnv = {}; + for (const patch of patches) { + if (!patch) { + continue; + } + Object.assign(merged, patch); + } + return Object.keys(merged).length > 0 ? merged : undefined; +} diff --git a/extensions/qa-lab/src/suite-types.ts b/extensions/qa-lab/src/suite-types.ts new file mode 100644 index 000000000000..c0627dea653e --- /dev/null +++ b/extensions/qa-lab/src/suite-types.ts @@ -0,0 +1,112 @@ +import type { OpenClawCrablineChannelDriverSelection } from "@openclaw/crabline"; +import type { QaEvidenceTiming, QaEvidenceSummaryJson } from "./evidence-summary.js"; +import type { QaCliBackendAuthMode, QaGatewayChildCommand } from "./gateway-child.js"; +import type { QaLabServerHandle, QaLabServerStartParams } from "./lab-server.types.js"; +import type { QaProviderMode } from "./model-selection.js"; +import type { QaThinkingLevel } from "./qa-gateway-config.js"; +import type { + QaTransportAdapterFactory, + QaTransportFactoryContext, + QaTransportId, +} from "./qa-transport-registry.js"; +import type { QaReportCheck } from "./report.js"; +import type { RuntimeId, RuntimeParityCell, RuntimeParityResult } from "./runtime-parity.js"; +import type { QaScorecardChannelDriver, QaScorecardEvidenceMode } from "./scorecard-taxonomy.js"; +import type { QaSuiteRoundTripProbe } from "./suite-round-trip.js"; +import type { QaSuiteRuntimeEnv } from "./suite-runtime-types.js"; + +export type QaSuiteScenarioResult = { + name: string; + status: "pass" | "fail" | "skip"; + steps: QaReportCheck[]; + details?: string; + timing?: QaEvidenceTiming; + runtimeParity?: RuntimeParityResult; +}; + +export type QaSuiteEnvironment = { + lab: QaLabServerHandle; + webSessionIds: Set; +} & QaSuiteRuntimeEnv; + +export type QaSuiteStartLabFn = (params?: QaLabServerStartParams) => Promise; + +export type QaSuiteRunParams = { + adapterOptions?: QaTransportFactoryContext["adapterOptions"]; + adapterFactories?: readonly QaTransportAdapterFactory[]; + channelId?: string; + evidenceMode?: QaScorecardEvidenceMode; + repoRoot?: string; + sutOpenClawCommand?: QaGatewayChildCommand; + outputDir?: string; + providerMode?: QaProviderMode; + transportId?: QaTransportId; + channelDriver?: QaScorecardChannelDriver; + channelDriverSelection?: OpenClawCrablineChannelDriverSelection | null; + primaryModel?: string; + alternateModel?: string; + fastMode?: boolean; + failFast?: boolean; + thinkingDefault?: QaThinkingLevel; + claudeCliAuthMode?: QaCliBackendAuthMode; + scenarioIds?: string[]; + lab?: QaLabServerHandle; + startLab?: QaSuiteStartLabFn; + concurrency?: number; + enabledPluginIds?: string[]; + controlUiEnabled?: boolean; + transportReadyTimeoutMs?: number; + workerStartStaggerMs?: number; + forcedRuntime?: RuntimeId; + runtimePair?: [RuntimeId, RuntimeId]; + captureRuntimeParityCell?: boolean; + roundTripProbe?: QaSuiteRoundTripProbe; + // Unified suite partitions consume child evidence in memory; only the + // parent should write the aggregate qa-evidence.json artifact. + writeEvidenceFile?: boolean; +}; + +export type QaSuiteResult = { + evidence?: QaEvidenceSummaryJson; + outputDir: string; + evidencePath: string; + reportPath: string; + summaryPath: string; + report: string; + scenarios: QaSuiteScenarioResult[]; + watchUrl: string; + runtimeParityCell?: RuntimeParityCell; +}; + +export type QaSuiteRunner = (params?: QaSuiteRunParams) => Promise; +export type QaSuiteScenarioRunner = ( + env: QaSuiteEnvironment, + scenario: ReturnType< + typeof import("./scenario-catalog.js").readQaBootstrapScenarioCatalog + >["scenarios"][number], +) => Promise; + +export type QaSuiteResolvedRunContext = { + startedAt: Date; + repoRoot: string; + outputDir: string; + transportId: QaTransportId; + selectedScenarios: ReturnType< + typeof import("./scenario-catalog.js").readQaBootstrapScenarioCatalog + >["scenarios"]; + providerMode: QaProviderMode; + primaryModel: string; + alternateModel: string; + fastMode: boolean; + channelDriver?: QaScorecardChannelDriver; + enabledPluginIds: string[]; + gatewayConfigPatch: ReturnType< + typeof import("./suite-planning.js").collectQaSuiteGatewayConfigPatch + >; + gatewayRuntimeOptions: ReturnType< + typeof import("./suite-planning.js").collectQaSuiteGatewayRuntimeOptions + >; + concurrency: number; + progressEnabled: boolean; + gatewayHeapCheckpointsEnabled: boolean; +}; diff --git a/extensions/qa-lab/src/suite.ts b/extensions/qa-lab/src/suite.ts index de6adad05496..3c3f965f7169 100644 --- a/extensions/qa-lab/src/suite.ts +++ b/extensions/qa-lab/src/suite.ts @@ -1,102 +1,60 @@ -// Qa Lab plugin module implements suite behavior. +// QA Lab plugin module implements suite behavior. import fs from "node:fs/promises"; import path from "node:path"; import { setTimeout as sleep } from "node:timers/promises"; import type { OpenClawCrablineChannelDriverSelection } from "@openclaw/crabline"; -import { disposeRegisteredAgentHarnesses } from "openclaw/plugin-sdk/agent-harness"; -import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts"; import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime"; import { parseStrictPositiveInteger } from "openclaw/plugin-sdk/number-runtime"; import { fetchWithSsrFGuard } from "openclaw/plugin-sdk/ssrf-runtime"; -import { assertQaSuiteArtifactWritten } from "./artifact-assertion.js"; -import { - hasQaCrablineArtifactPath, - resolveQaCrablineChannelDriverArtifactPaths, - type QaSuiteChannelDriverSelection, -} from "./crabline-artifacts.js"; -import { - buildQaSuiteEvidenceSummary, - QA_EVIDENCE_FILENAME, - type QaEvidenceTiming, - type QaEvidenceSummaryJson, -} from "./evidence-summary.js"; -import { - startQaGatewayChild, - type QaCliBackendAuthMode, - type QaGatewayChildCommand, -} from "./gateway-child.js"; +import type { QaEvidenceTiming, QaEvidenceSummaryJson } from "./evidence-summary.js"; +import type { QaCliBackendAuthMode, QaGatewayChildCommand } from "./gateway-child.js"; +import { startQaGatewayChild } from "./gateway-child.js"; import { discardIgnoredResponseBody } from "./ignored-response-body.js"; -import type { - QaLabLatestReport, - QaLabScenarioOutcome, - QaLabServerHandle, - QaLabServerStartParams, -} from "./lab-server.types.js"; +import type { QaLabServerHandle, QaLabServerStartParams } from "./lab-server.types.js"; import { resolveQaLiveTurnTimeoutMs } from "./live-timeout.js"; import type { QaProviderMode } from "./model-selection.js"; import { parseQaProgressBooleanEnv as parseQaSuiteBooleanEnv, sanitizeQaProgressValue as sanitizeQaSuiteProgressValue, } from "./progress-format.js"; -import { startQaProviderServer } from "./providers/server-runtime.js"; import type { QaThinkingLevel } from "./qa-gateway-config.js"; import { createQaTransportAdapter, - defaultQaSuiteConcurrencyForTransport, - normalizeQaTransportId, type QaTransportAdapterFactory, type QaTransportFactoryContext, type QaTransportId, } from "./qa-transport-registry.js"; -import type { QaTransportAdapter } from "./qa-transport.js"; -import { renderQaMarkdownReport, type QaReportCheck, type QaReportScenario } from "./report.js"; -import { - captureRuntimeParityCell, - isRuntimeParityResultPass, - runRuntimeParityScenario, - type RuntimeId, - type RuntimeParityCell, - type RuntimeParityResult, -} from "./runtime-parity.js"; -import { - readQaBootstrapScenarioCatalog, - type QaSeedScenarioWithSource, -} from "./scenario-catalog.js"; +import type { QaReportCheck } from "./report.js"; +import type { RuntimeId, RuntimeParityCell, RuntimeParityResult } from "./runtime-parity.js"; +import { readQaBootstrapScenarioCatalog } from "./scenario-catalog.js"; import type { QaScorecardChannelDriver, QaScorecardEvidenceMode } from "./scorecard-taxonomy.js"; import { - resolveRequestedQaSuiteModels, - resolveSelectedQaSuiteModels, -} from "./suite-model-selection.js"; + type QaSuiteGatewayHeapSnapshot, + type QaSuiteGatewayRssSample, + writeQaSuiteArtifacts, +} from "./suite-artifacts.js"; import { - applyQaMergePatch, - collectQaSuiteGatewayConfigPatch, - collectQaSuiteGatewayRuntimeOptions, - collectQaSuiteTransportPolicy, - collectQaSuitePluginIds, - mapQaSuiteWithConcurrency, - normalizeQaSuiteConcurrency, - resolveQaSuiteWorkerStartStaggerMs, - resolveQaSuiteOutputDir, scenarioRequiresControlUi, - selectQaFlowSuiteScenarios, shouldUseIsolatedQaSuiteScenarioWorkers, splitModelRef, } from "./suite-planning.js"; -import { runQaSuiteRoundTripProbe, type QaSuiteRoundTripProbe } from "./suite-round-trip.js"; +import type { QaSuiteRoundTripProbe } from "./suite-round-trip.js"; import { createQaSuiteScenarioStepRunner, runQaSuiteScenarioDefinition, runQaSuiteScenarioSteps, } from "./suite-runtime-flow.js"; -import { waitForGatewayHealthy, waitForTransportReady } from "./suite-runtime-gateway.js"; import type { QaSuiteRuntimeEnv } from "./suite-runtime-types.js"; -import { countQaSuiteFailedScenarios, type QaSuiteSummaryJson } from "./suite-summary.js"; -import { closeQaWebSessions } from "./web-runtime.js"; +import type { QaSuiteSummaryJson } from "./suite-summary.js"; +import { + appendNodeOption, + buildQaGatewayHeapCheckpointRuntimeEnvPatch, + buildQaIsolatedScenarioWorkerParams, + buildQaRuntimeEnvPatch, + mergeQaRuntimeEnvPatches, + remapModelRefForForcedRuntime, +} from "./suite-support.js"; -type QaCrablineRuntime = typeof import("@openclaw/crabline"); -type QaCrablineChannelDriverSmokeResult = Awaited< - ReturnType ->; function resolveQaSuiteControlUiEnabled(params: { explicit?: boolean; scenarios: ReturnType["scenarios"]; @@ -122,7 +80,7 @@ type QaSuiteEnvironment = { export type QaSuiteStartLabFn = (params?: QaLabServerStartParams) => Promise; -async function createQaSuiteTransportAdapter(params: { +export async function createQaSuiteTransportAdapter(params: { adapterOptions?: QaSuiteRunParams["adapterOptions"]; adapterFactories?: readonly QaTransportAdapterFactory[]; channelDriver?: QaScorecardChannelDriver | null; @@ -204,7 +162,7 @@ export type QaSuiteRunParams = { writeEvidenceFile?: boolean; }; -function shouldLogQaSuiteProgress(env: NodeJS.ProcessEnv = process.env) { +export function shouldLogQaSuiteProgress(env: NodeJS.ProcessEnv = process.env) { const override = parseQaSuiteBooleanEnv(env.OPENCLAW_QA_SUITE_PROGRESS); if (override !== undefined) { return override; @@ -212,7 +170,7 @@ function shouldLogQaSuiteProgress(env: NodeJS.ProcessEnv = process.env) { return parseQaSuiteBooleanEnv(env.CI) === true; } -function resolveQaSuiteTransportReadyTimeoutMs( +export function resolveQaSuiteTransportReadyTimeoutMs( explicitTimeoutMs?: number, env: NodeJS.ProcessEnv = process.env, ) { @@ -234,14 +192,14 @@ function resolveQaSuiteTransportReadyTimeoutMs( return parsed; } -function writeQaSuiteProgress(enabled: boolean, message: string) { +export function writeQaSuiteProgress(enabled: boolean, message: string) { if (!enabled) { return; } process.stderr.write(`[qa-suite] ${message}\n`); } -function formatQaSuiteRunStartProgress(params: { +export function formatQaSuiteRunStartProgress(params: { selectedScenarioCount: number; concurrency: number; transportId: QaTransportId; @@ -294,7 +252,7 @@ async function waitForQaLabReady(baseUrl: string, timeoutMs = 10_000) { throw new Error(`timed out after ${timeoutMs}ms waiting for qa-lab ready`); } -async function waitForQaLabReadyOrStopOwned(params: { +export async function waitForQaLabReadyOrStopOwned(params: { lab: Pick; ownsLab: boolean; timeoutMs?: number; @@ -309,7 +267,7 @@ async function waitForQaLabReadyOrStopOwned(params: { } } -async function runQaSuiteCleanupSteps(steps: ReadonlyArray<() => Promise>) { +export async function runQaSuiteCleanupSteps(steps: ReadonlyArray<() => Promise>) { const errors: unknown[] = []; for (const step of steps) { try { @@ -321,7 +279,7 @@ async function runQaSuiteCleanupSteps(steps: ReadonlyArray<() => Promise>) return errors; } -async function runQaFlowSuiteCleanupPlan(params: { +export async function runQaFlowSuiteCleanupPlan(params: { closeWebSessions?: () => Promise; cleanupTransportBeforeGatewayStop: () => Promise; cleanupTransportAfterGatewayStop: () => Promise; @@ -355,7 +313,7 @@ async function runQaFlowSuiteCleanupPlan(params: { return errors; } -function throwQaSuiteCleanupErrors(params: { +export function throwQaSuiteCleanupErrors(params: { cleanupErrors: unknown[]; runFailed: boolean; runError: unknown; @@ -373,7 +331,7 @@ function throwQaSuiteCleanupErrors(params: { ); } -function requireQaSuiteStartLab(startLab: QaSuiteStartLabFn | undefined): QaSuiteStartLabFn { +export function requireQaSuiteStartLab(startLab: QaSuiteStartLabFn | undefined): QaSuiteStartLabFn { if (startLab) { return startLab; } @@ -382,7 +340,7 @@ function requireQaSuiteStartLab(startLab: QaSuiteStartLabFn | undefined): QaSuit ); } -function shouldRunQaSuiteWithIsolatedScenarioWorkers(params: { +export function shouldRunQaSuiteWithIsolatedScenarioWorkers(params: { scenarios: ReturnType["scenarios"]; concurrency: number; lab?: QaLabServerHandle; @@ -425,34 +383,7 @@ export type QaSuiteResult = { runtimeParityCell?: RuntimeParityCell; }; -/** - * One bounded retry for live-model flake: flow scenarios time out under model - * latency spikes, so a first failure gets a single rerun. A retry pass keeps - * the first attempt visible in details; a retry failure keeps the original - * diagnostics so deterministic regressions still fail the suite. - */ -async function runQaScenarioWithFlakeRetry( - run: () => Promise, - onRetry?: () => void, -): Promise { - const first = await run(); - if (first.status !== "fail") { - return first; - } - onRetry?.(); - const second = await run(); - if (second.status !== "pass") { - return first; - } - return { - ...second, - details: [second.details, `passed on retry; first attempt: ${first.details ?? "failed"}`] - .filter(Boolean) - .join(" | "), - }; -} - -async function runScenarioDefinition( +export async function runQaSuiteScenarioDefinitionForRuntime( env: QaSuiteEnvironment, scenario: ReturnType["scenarios"][number], ) { @@ -472,714 +403,9 @@ async function runScenarioDefinition( }); } -function isRuntimeParityPass(result: RuntimeParityResult) { - return isRuntimeParityResultPass(result); -} - -function formatRuntimeParityCellDetails(cell: RuntimeParityCell) { - const errors = [cell.transportErrorClass, cell.runtimeErrorClass].filter(Boolean).join(", "); - const sentinels = cell.sentinelFindings?.map((finding) => finding.kind).join(", "); - return [ - `runtime=${cell.runtime}`, - `wallMs=${cell.wallClockMs}`, - `toolCalls=${cell.toolCalls.length}`, - `finalChars=${cell.finalText.length}`, - `tokens=${cell.usage.totalTokens}`, - ...(errors ? [`errors=${errors}`] : []), - ...(sentinels ? [`sentinels=${sentinels}`] : []), - ].join(" "); -} - -function buildRuntimeParityScenarioResult(params: { - scenarioName: string; - result: RuntimeParityResult; -}): QaSuiteScenarioResult { - const driftStepStatus = isRuntimeParityPass(params.result) ? "pass" : "fail"; - const openclawCell = params.result.cells.openclaw; - return { - name: params.scenarioName, - status: driftStepStatus, - details: params.result.driftDetails ?? `runtime drift classified as ${params.result.drift}`, - steps: [ - { - name: openclawCell.runtime, - status: - openclawCell.runtimeErrorClass || openclawCell.transportErrorClass ? "fail" : "pass", - details: formatRuntimeParityCellDetails(openclawCell), - }, - { - name: params.result.cells.codex.runtime, - status: - params.result.cells.codex.runtimeErrorClass || - params.result.cells.codex.transportErrorClass - ? "fail" - : "pass", - details: formatRuntimeParityCellDetails(params.result.cells.codex), - }, - { - name: "runtime drift", - status: driftStepStatus, - details: params.result.driftDetails ?? params.result.drift, - }, - ], - runtimeParity: params.result, - }; -} - -function createQaSuiteReportNotes(params: { - transport: QaTransportAdapter; - channelDriverSelection?: QaSuiteChannelDriverSelection | null; - providerMode: QaProviderMode; - primaryModel: string; - alternateModel: string; - fastMode: boolean; - concurrency: number; - isolatedWorkers?: boolean; - createCrablineChannelReportNotes?: QaCrablineRuntime["createOpenClawCrablineChannelReportNotes"]; -}) { - return [ - ...params.transport.createReportNotes(params), - // Crabline reports completed generation paths through this filename-narrowed selection. - ...(params.createCrablineChannelReportNotes?.( - params.channelDriverSelection as OpenClawCrablineChannelDriverSelection | null | undefined, - ) ?? []), - ]; -} - -function buildQaIsolatedScenarioWorkerParams(params: { - repoRoot: string; - outputDir: string; - providerMode: QaProviderMode; - transportId: QaTransportId; - channelDriver?: QaScorecardChannelDriver; - channelDriverSelection?: OpenClawCrablineChannelDriverSelection | null; - primaryModel: string; - alternateModel: string; - fastMode: boolean; - scenario: ReturnType["scenarios"][number]; - input?: QaSuiteRunParams; - startLab: QaSuiteStartLabFn; -}): QaSuiteRunParams { - return { - adapterFactories: params.input?.adapterFactories, - adapterOptions: params.input?.adapterOptions, - channelId: params.input?.channelId, - repoRoot: params.repoRoot, - sutOpenClawCommand: params.input?.sutOpenClawCommand, - outputDir: params.outputDir, - providerMode: params.providerMode, - transportId: params.transportId, - channelDriver: params.channelDriver, - channelDriverSelection: params.channelDriverSelection, - primaryModel: params.primaryModel, - alternateModel: params.alternateModel, - fastMode: params.fastMode, - thinkingDefault: params.input?.thinkingDefault, - claudeCliAuthMode: params.input?.claudeCliAuthMode, - scenarioIds: [params.scenario.id], - enabledPluginIds: params.input?.enabledPluginIds, - concurrency: 1, - startLab: params.startLab, - controlUiEnabled: scenarioRequiresControlUi(params.scenario), - transportReadyTimeoutMs: params.input?.transportReadyTimeoutMs, - workerStartStaggerMs: params.input?.workerStartStaggerMs, - forcedRuntime: params.input?.forcedRuntime, - roundTripProbe: - params.input?.roundTripProbe?.scenarioId === params.scenario.id - ? params.input.roundTripProbe - : undefined, - writeEvidenceFile: params.input?.writeEvidenceFile, - }; -} - -function remapModelRefForForcedRuntime(params: { - modelRef: string; - providerMode: QaProviderMode; - forcedRuntime?: RuntimeId; -}) { - if (params.forcedRuntime !== "codex" || params.providerMode !== "mock-openai") { - return params.modelRef; - } - const split = splitModelRef(params.modelRef); - if (!split || split.provider !== "mock-openai") { - return params.modelRef; - } - return `openai/${split.model}`; -} - -function buildQaRuntimeEnvPatch(params: { - providerMode: QaProviderMode; - forcedRuntime?: RuntimeId; - mockBaseUrl?: string; -}): NodeJS.ProcessEnv | undefined { - const patch: NodeJS.ProcessEnv = {}; - if (params.forcedRuntime) { - patch.OPENCLAW_BUILD_PRIVATE_QA = "1"; - patch.OPENCLAW_QA_FORCE_RUNTIME = params.forcedRuntime; - } - if (params.forcedRuntime !== "codex" || params.providerMode !== "mock-openai") { - return Object.keys(patch).length > 0 ? patch : undefined; - } - const mockBaseUrl = params.mockBaseUrl?.trim().replace(/\/+$/u, ""); - if (!mockBaseUrl) { - return Object.keys(patch).length > 0 ? patch : undefined; - } - // The forced codex lane uses the Codex app-server's native OpenAI provider - // path, so pin the managed app-server to the QA mock endpoint instead of - // leaking to the maintainer's real OpenAI config. - patch.OPENCLAW_CODEX_APP_SERVER_ARGS = `app-server -c openai_base_url=${mockBaseUrl}/v1 --listen stdio://`; - patch.OPENAI_API_KEY = "qa-mock-openai-key"; - patch.CODEX_API_KEY = "qa-mock-openai-key"; - return patch; -} - -function appendNodeOption(raw: string | undefined, option: string) { - const parts = (raw ?? "").split(/\s+/u).filter(Boolean); - return parts.includes(option) ? parts.join(" ") : [...parts, option].join(" "); -} - -function shouldCaptureGatewayHeapCheckpoints(env: NodeJS.ProcessEnv = process.env) { - return parseQaSuiteBooleanEnv(env.OPENCLAW_QA_GATEWAY_HEAP_CHECKPOINTS) === true; -} - -function buildQaGatewayHeapCheckpointRuntimeEnvPatch( - env: NodeJS.ProcessEnv = process.env, -): NodeJS.ProcessEnv | undefined { - if (!shouldCaptureGatewayHeapCheckpoints(env)) { - return undefined; - } - return { - NODE_OPTIONS: appendNodeOption(env.NODE_OPTIONS, "--heapsnapshot-signal=SIGUSR2"), - }; -} - -function mergeQaRuntimeEnvPatches( - ...patches: Array -): NodeJS.ProcessEnv | undefined { - const merged: NodeJS.ProcessEnv = {}; - for (const patch of patches) { - if (!patch) { - continue; - } - Object.assign(merged, patch); - } - return Object.keys(merged).length > 0 ? merged : undefined; -} - -export type QaSuiteSummaryJsonParams = { - scenarios: QaSuiteScenarioResult[]; - startedAt: Date; - finishedAt: Date; - metrics?: QaSuiteSummaryJson["metrics"]; - evidence?: QaSuiteSummaryJson["evidence"]; - providerMode: QaProviderMode; - primaryModel: string; - alternateModel: string; - fastMode: boolean; - concurrency: number; - channelDriver?: QaScorecardChannelDriver | null; - channelDriverSelection?: QaSuiteChannelDriverSelection | null; - scenarioIds?: readonly string[]; - runtimePair?: [RuntimeId, RuntimeId]; -}; - -/** - * Strongly-typed shape of `qa-suite-summary.json`. The GPT-5.6 Luna parity gate - * (agentic-parity-report.ts, #64441) and any future parity wrapper can - * import this type instead of re-declaring the shape, so changes to the - * summary schema propagate through to every consumer at type-check time. - */ -export type { QaSuiteSummaryJson } from "./suite-summary.js"; - -type QaSuiteGatewayRssSample = NonNullable< - NonNullable["gatewayProcessRssSamples"] ->[number]; - type QaGatewayHandle = Awaited>; -type QaSuiteGatewayHeapSnapshot = NonNullable< - NonNullable["gatewayHeapSnapshots"] ->[number]; -/** - * Pure-ish JSON builder for qa-suite-summary.json. Exported so the GPT-5.6 Luna - * parity gate (agentic-parity-report.ts, #64441) and any future parity - * runner can assert-and-trust the provider/model that produced a given - * summary instead of blindly accepting the caller's candidateLabel / - * baselineLabel. Without the `run` block, a maintainer who swaps candidate - * and baseline summary paths could silently produce a mislabeled verdict. - * - * `scenarioIds` is only recorded when the caller passed a non-empty array - * (an explicit scenario selection). A missing or empty array means "no - * filter, full lane-selected catalog", which the summary encodes as `null` - * so parity/report tooling doesn't mistake a full run for an explicit - * empty selection. - */ -export function buildQaSuiteSummaryJson(params: QaSuiteSummaryJsonParams): QaSuiteSummaryJson { - const primarySplit = splitModelRef(params.primaryModel); - const alternateSplit = splitModelRef(params.alternateModel); - return { - scenarios: params.scenarios, - counts: { - total: params.scenarios.length, - passed: params.scenarios.filter((scenario) => scenario.status === "pass").length, - failed: countQaSuiteFailedScenarios(params.scenarios), - skipped: params.scenarios.filter((scenario) => scenario.status === "skip").length, - }, - ...(params.metrics ? { metrics: params.metrics } : {}), - ...(params.evidence ? { evidence: params.evidence } : {}), - run: { - startedAt: params.startedAt.toISOString(), - finishedAt: params.finishedAt.toISOString(), - providerMode: params.providerMode, - primaryModel: params.primaryModel, - primaryProvider: primarySplit?.provider ?? null, - primaryModelName: primarySplit?.model ?? null, - alternateModel: params.alternateModel, - alternateProvider: alternateSplit?.provider ?? null, - alternateModelName: alternateSplit?.model ?? null, - fastMode: params.fastMode, - concurrency: params.concurrency, - channelDriver: params.channelDriver ?? params.channelDriverSelection?.channelDriver ?? null, - channel: params.channelDriverSelection?.channel ?? null, - channelCapabilityMatrixPath: params.channelDriverSelection?.capabilityMatrixPath ?? null, - channelDriverSmokePath: params.channelDriverSelection?.smokeArtifactPath ?? null, - scenarioIds: - params.scenarioIds && params.scenarioIds.length > 0 ? [...params.scenarioIds] : null, - runtimePair: params.runtimePair ?? null, - }, - }; -} - -async function runQaRuntimeParitySuite(params: { - adapterOptions?: QaSuiteRunParams["adapterOptions"]; - adapterFactories?: readonly QaTransportAdapterFactory[]; - channelId?: string; - evidenceMode?: QaScorecardEvidenceMode; - repoRoot: string; - outputDir: string; - startedAt: Date; - providerMode: QaProviderMode; - transportId: QaTransportId; - primaryModel: string; - alternateModel: string; - fastMode: boolean; - thinkingDefault?: QaThinkingLevel; - claudeCliAuthMode?: QaCliBackendAuthMode; - enabledPluginIds?: string[]; - channelDriver?: QaScorecardChannelDriver | null; - channelDriverSelection?: OpenClawCrablineChannelDriverSelection | null; - concurrency: number; - selectedScenarios: ReturnType["scenarios"]; - startLab?: QaSuiteStartLabFn; - lab?: QaLabServerHandle; - progressEnabled: boolean; - scenarioIds?: readonly string[]; - runtimePair: [RuntimeId, RuntimeId]; - writeEvidenceFile?: boolean; -}) { - const ownsLab = !params.lab; - const startLab = requireQaSuiteStartLab(params.startLab); - const lab = - params.lab ?? - (await startLab({ - repoRoot: params.repoRoot, - host: "127.0.0.1", - port: 0, - embeddedGateway: "disabled", - })); - const transportFactoryResult = await createQaSuiteTransportAdapter({ - adapterFactories: params.adapterFactories, - channelDriver: params.channelDriver, - channelId: params.channelId, - channelDriverSelection: params.channelDriverSelection, - adapterOptions: params.adapterOptions, - cleanupOnFailure: ownsLab ? () => lab.stop() : undefined, - outputDir: params.outputDir, - transportPolicy: collectQaSuiteTransportPolicy(params.selectedScenarios), - state: lab.state, - transportId: params.transportId, - }); - const transport = transportFactoryResult.adapter; - const liveScenarioOutcomes: QaLabScenarioOutcome[] = params.selectedScenarios.map((scenario) => ({ - id: scenario.id, - name: scenario.title, - status: "pending", - })); - lab.setScenarioRun({ - kind: "suite", - status: "running", - startedAt: params.startedAt.toISOString(), - scenarios: [...liveScenarioOutcomes], - }); - - try { - const scenarios = await mapQaSuiteWithConcurrency( - params.selectedScenarios, - params.concurrency, - async (scenario, index): Promise => { - const scenarioIdForLog = sanitizeQaSuiteProgressValue(scenario.id); - writeQaSuiteProgress( - params.progressEnabled, - `runtime pair start (${index + 1}/${params.selectedScenarios.length}): ${scenarioIdForLog}`, - ); - liveScenarioOutcomes[index] = { - id: scenario.id, - name: scenario.title, - status: "running", - startedAt: new Date().toISOString(), - }; - lab.setScenarioRun({ - kind: "suite", - status: "running", - startedAt: params.startedAt.toISOString(), - scenarios: [...liveScenarioOutcomes], - }); - - const parity = await runRuntimeParityScenario({ - scenarioId: scenario.id, - runtimeParityUsage: scenario.runtimeParityUsage, - runCell: async (runtime) => { - const cellOutputDir = path.join( - params.outputDir, - "runtime-cells", - scenario.id, - runtime, - ); - const cellStartedAt = Date.now(); - const cellResult = await runQaFlowSuite({ - adapterFactories: params.adapterFactories, - channelId: params.channelId, - adapterOptions: params.adapterOptions, - repoRoot: params.repoRoot, - outputDir: cellOutputDir, - providerMode: params.providerMode, - transportId: params.transportId, - channelDriver: params.channelDriver ?? undefined, - channelDriverSelection: params.channelDriverSelection, - primaryModel: remapModelRefForForcedRuntime({ - modelRef: params.primaryModel, - providerMode: params.providerMode, - forcedRuntime: runtime, - }), - alternateModel: remapModelRefForForcedRuntime({ - modelRef: params.alternateModel, - providerMode: params.providerMode, - forcedRuntime: runtime, - }), - fastMode: params.fastMode, - thinkingDefault: params.thinkingDefault, - claudeCliAuthMode: params.claudeCliAuthMode, - scenarioIds: [scenario.id], - concurrency: 1, - enabledPluginIds: params.enabledPluginIds, - startLab, - controlUiEnabled: scenarioRequiresControlUi(scenario), - forcedRuntime: runtime, - captureRuntimeParityCell: true, - writeEvidenceFile: params.writeEvidenceFile, - }); - const scenarioResult = - cellResult.scenarios[0] ?? - ({ - name: scenario.title, - status: "fail", - details: "runtime parity cell returned no scenario result", - steps: [ - { - name: "runtime parity cell", - status: "fail", - details: "runtime parity cell returned no scenario result", - }, - ], - } satisfies QaSuiteScenarioResult); - const fallbackCell = { - runtime, - transcriptBytes: "", - toolCalls: [], - finalText: "", - usage: { - inputTokens: 0, - outputTokens: 0, - totalTokens: 0, - }, - wallClockMs: Math.max(1, Date.now() - cellStartedAt), - runtimeErrorClass: "capture-missing", - bootStateLines: [], - } satisfies RuntimeParityCell; - return { - scenarioStatus: scenarioResult.status === "pass" ? "pass" : "fail", - scenarioDetails: scenarioResult.details, - cell: cellResult.runtimeParityCell ?? fallbackCell, - }; - }, - }); - - const result = buildRuntimeParityScenarioResult({ - scenarioName: scenario.title, - result: parity, - }); - liveScenarioOutcomes[index] = { - id: scenario.id, - name: scenario.title, - status: result.status, - details: result.details, - steps: result.steps, - startedAt: liveScenarioOutcomes[index]?.startedAt, - finishedAt: new Date().toISOString(), - }; - lab.setScenarioRun({ - kind: "suite", - status: "running", - startedAt: params.startedAt.toISOString(), - scenarios: [...liveScenarioOutcomes], - }); - writeQaSuiteProgress( - params.progressEnabled, - `runtime pair ${result.status} (${index + 1}/${params.selectedScenarios.length}): ${scenarioIdForLog}`, - ); - return result; - }, - { - startStaggerMs: resolveQaSuiteWorkerStartStaggerMs(params.concurrency), - }, - ); - - const finishedAt = new Date(); - const { evidence, evidencePath, report, reportPath, summaryPath } = await writeQaSuiteArtifacts( - { - repoRoot: params.repoRoot, - outputDir: params.outputDir, - startedAt: params.startedAt, - finishedAt, - scenarios, - scenarioDefinitions: params.selectedScenarios, - evidenceMode: params.evidenceMode, - transport, - providerMode: params.providerMode, - primaryModel: params.primaryModel, - alternateModel: params.alternateModel, - fastMode: params.fastMode, - concurrency: params.concurrency, - channelDriver: params.channelDriver, - channelDriverSelection: params.channelDriverSelection, - scenarioIds: - params.scenarioIds && params.scenarioIds.length > 0 - ? params.selectedScenarios.map((scenario) => scenario.id) - : undefined, - runtimePair: params.runtimePair, - writeEvidenceFile: params.writeEvidenceFile, - }, - ); - lab.setLatestReport({ - outputPath: reportPath, - markdown: report, - generatedAt: finishedAt.toISOString(), - } satisfies QaLabLatestReport); - lab.setScenarioRun({ - kind: "suite", - status: "completed", - startedAt: params.startedAt.toISOString(), - finishedAt: finishedAt.toISOString(), - scenarios: [...liveScenarioOutcomes], - }); - return { - outputDir: params.outputDir, - evidence, - evidencePath, - reportPath, - summaryPath, - report, - scenarios, - watchUrl: lab.baseUrl, - } satisfies QaSuiteResult; - } finally { - await transportFactoryResult.cleanupWithoutGateway(); - if (ownsLab) { - await lab.stop(); - } - } -} - -async function writeQaSuiteArtifacts(params: { - repoRoot?: string; - outputDir: string; - startedAt: Date; - finishedAt: Date; - scenarios: QaSuiteScenarioResult[]; - scenarioDefinitions?: readonly QaSeedScenarioWithSource[]; - evidenceMode?: QaScorecardEvidenceMode; - metrics?: QaSuiteSummaryJson["metrics"]; - transport: QaTransportAdapter; - // Reuse the canonical QaProviderMode union instead of re-declaring it - // inline. Loop 6 already unified `QaSuiteSummaryJsonParams.providerMode` - // on this type; keeping the writer in sync prevents drift when model- - // selection.ts adds a new provider mode. - providerMode: QaProviderMode; - primaryModel: string; - alternateModel: string; - fastMode: boolean; - concurrency: number; - channelDriver?: QaScorecardChannelDriver | null; - channelDriverSelection?: OpenClawCrablineChannelDriverSelection | null; - isolatedWorkers?: boolean; - scenarioIds?: readonly string[]; - runtimePair?: [RuntimeId, RuntimeId]; - writeEvidenceFile?: boolean; - runCrablineChannelDriverSmoke?: ( - params: Parameters[0], - ) => Promise; -}) { - const reportPath = path.join(params.outputDir, "qa-suite-report.md"); - const summaryPath = path.join(params.outputDir, "qa-suite-summary.json"); - const evidencePath = path.join(params.outputDir, QA_EVIDENCE_FILENAME); - const crablineChannelDriverSelection = params.channelDriverSelection; - // Non-Crabline package acceptance mounts this source without plugin-local - // dependencies. Keep the owner runtime outside every unrelated live path. - const crablineRuntime = crablineChannelDriverSelection - ? await import("@openclaw/crabline") - : undefined; - let crablineChannelDriverSmoke: QaCrablineChannelDriverSmokeResult | undefined; - if (crablineChannelDriverSelection) { - const runCrablineChannelDriverSmoke = - params.runCrablineChannelDriverSmoke ?? - crablineRuntime?.runOpenClawCrablineChannelDriverSmoke; - if (!runCrablineChannelDriverSmoke) { - throw new Error("Crabline runtime did not provide its channel-driver smoke helper."); - } - crablineChannelDriverSmoke = await runCrablineChannelDriverSmoke({ - outputDir: params.outputDir, - selection: crablineChannelDriverSelection, - }); - } - const crablineChannelDriverArtifactPaths = resolveQaCrablineChannelDriverArtifactPaths({ - result: crablineChannelDriverSmoke, - selection: crablineChannelDriverSelection, - }); - const effectiveChannelDriverSelection: QaSuiteChannelDriverSelection | null | undefined = - crablineChannelDriverSelection && crablineChannelDriverArtifactPaths - ? { - ...crablineChannelDriverSelection, - ...crablineChannelDriverArtifactPaths, - } - : crablineChannelDriverSelection; - const report = renderQaMarkdownReport({ - title: "OpenClaw QA Scenario Suite", - startedAt: params.startedAt, - finishedAt: params.finishedAt, - checks: [], - scenarios: params.scenarios.map((scenario) => ({ - name: scenario.name, - status: scenario.status, - details: scenario.details, - steps: scenario.steps, - })) satisfies QaReportScenario[], - notes: createQaSuiteReportNotes({ - ...params, - channelDriverSelection: effectiveChannelDriverSelection, - createCrablineChannelReportNotes: crablineRuntime?.createOpenClawCrablineChannelReportNotes, - }), - }); - const evidence = - params.scenarioDefinitions && params.scenarioDefinitions.length > 0 - ? buildQaSuiteEvidenceSummary({ - artifactPaths: [ - { kind: "summary", path: path.basename(summaryPath) }, - { kind: "report", path: path.basename(reportPath) }, - ...(effectiveChannelDriverSelection - ? [ - { - kind: "channel-capability-matrix", - path: effectiveChannelDriverSelection.capabilityMatrixPath, - }, - { - kind: "channel-driver-smoke", - path: effectiveChannelDriverSelection.smokeArtifactPath, - }, - ] - : []), - ], - evidenceMode: params.evidenceMode, - channelId: params.channelDriverSelection?.channel ?? params.transport.id, - channelDriver: params.channelDriver ?? params.channelDriverSelection?.channelDriver, - env: process.env, - generatedAt: params.finishedAt.toISOString(), - primaryModel: params.primaryModel, - providerMode: params.providerMode, - repoRoot: params.repoRoot, - scenarioDefinitions: params.scenarioDefinitions, - scenarioResults: params.scenarios, - }) - : undefined; - if ( - crablineChannelDriverSelection && - crablineChannelDriverSmoke && - !hasQaCrablineArtifactPath(crablineChannelDriverSmoke.capabilityMatrixPath) - ) { - await fs.writeFile( - path.join(params.outputDir, crablineChannelDriverSelection.capabilityMatrixPath), - `${JSON.stringify( - { - version: 1, - source: "openclaw/crabline", - channelDriver: crablineChannelDriverSelection.channelDriver, - selectedChannel: crablineChannelDriverSelection.channel, - manifestPath: crablineChannelDriverSmoke.manifestPath, - report: crablineChannelDriverSmoke.capabilityReport, - }, - null, - 2, - )}\n`, - "utf8", - ); - } - if ( - crablineChannelDriverSelection && - crablineChannelDriverSmoke && - !hasQaCrablineArtifactPath(crablineChannelDriverSmoke.smokeArtifactPath) - ) { - await fs.writeFile( - path.join(params.outputDir, crablineChannelDriverSelection.smokeArtifactPath), - `${JSON.stringify( - { - version: 1, - source: "openclaw/crabline", - channelDriver: crablineChannelDriverSelection.channelDriver, - selectedChannel: crablineChannelDriverSelection.channel, - manifestPath: crablineChannelDriverSmoke.manifestPath, - smoke: crablineChannelDriverSmoke.smoke, - }, - null, - 2, - )}\n`, - "utf8", - ); - } - const writeEvidenceFile = params.writeEvidenceFile ?? true; - await fs.writeFile(reportPath, report, "utf8"); - if (evidence && writeEvidenceFile) { - await fs.writeFile(evidencePath, `${JSON.stringify(evidence, null, 2)}\n`, "utf8"); - } - await fs.writeFile( - summaryPath, - `${JSON.stringify( - buildQaSuiteSummaryJson({ - ...params, - channelDriverSelection: effectiveChannelDriverSelection, - }), - null, - 2, - )}\n`, - "utf8", - ); - await assertQaSuiteArtifactWritten("report", reportPath); - await assertQaSuiteArtifactWritten("summary", summaryPath); - if (evidence && writeEvidenceFile) { - await assertQaSuiteArtifactWritten("evidence", evidencePath); - } - return { evidence, evidencePath, report, reportPath, summaryPath }; -} - -function buildQaSuiteRuntimeMetrics(params: { +export function buildQaSuiteRuntimeMetrics(params: { startedAt: Date; finishedAt: Date; gatewayProcessCpuStartMs: number | null; @@ -1267,7 +493,7 @@ async function waitForStableFileSize(pathName: string) { return stats.size; } -async function captureGatewayHeapSnapshotCheckpoint(params: { +export async function captureGatewayHeapSnapshotCheckpoint(params: { gateway: QaGatewayHandle; outputDir: string; label: string; @@ -1308,764 +534,13 @@ async function captureGatewayHeapSnapshotCheckpoint(params: { }; } +export { buildQaSuiteSummaryJson } from "./suite-artifacts.js"; +export type { QaSuiteSummaryJsonParams } from "./suite-artifacts.js"; +export type { QaSuiteSummaryJson } from "./suite-summary.js"; + export async function runQaFlowSuite(params?: QaSuiteRunParams): Promise { - const startedAt = new Date(); - const repoRoot = path.resolve(params?.repoRoot ?? process.cwd()); - const requestedModels = resolveRequestedQaSuiteModels(params ?? {}); - const transportId = normalizeQaTransportId(params?.transportId); - const outputDir = await resolveQaSuiteOutputDir(repoRoot, params?.outputDir); - const catalog = readQaBootstrapScenarioCatalog(); - const channelDriver = params?.channelDriver ?? params?.channelDriverSelection?.channelDriver; - const selectedScenarios = selectQaFlowSuiteScenarios({ - scenarios: catalog.scenarios, - scenarioIds: params?.scenarioIds, - providerMode: requestedModels.providerMode, - primaryModel: requestedModels.primaryModel, - channelDriver, - channel: params?.channelId ?? params?.channelDriverSelection?.channel, - claudeCliAuthMode: params?.claudeCliAuthMode, - }); - const { alternateModel, fastMode, primaryModel, providerMode } = resolveSelectedQaSuiteModels({ - alternateModelExplicit: params?.alternateModel !== undefined, - fastMode: params?.fastMode, - primaryModelExplicit: params?.primaryModel !== undefined, - requested: requestedModels, - scenarios: selectedScenarios, - }); - if ( - params?.roundTripProbe && - !selectedScenarios.some((scenario) => scenario.id === params.roundTripProbe?.scenarioId) - ) { - throw new Error( - `QA round-trip probe scenario is not selected: ${params.roundTripProbe.scenarioId}`, - ); - } - if (params?.roundTripProbe && params.runtimePair) { - throw new Error("QA round-trip probes are not supported with runtime-pair runs."); - } - const enabledPluginIds = [ - ...new Set([ - ...collectQaSuitePluginIds(selectedScenarios), - ...(params?.enabledPluginIds ?? []).map((pluginId) => pluginId.trim()).filter(Boolean), - ...(params?.forcedRuntime && params.forcedRuntime !== "openclaw" - ? [params.forcedRuntime] - : []), - ]), - ]; - const gatewayConfigPatch = collectQaSuiteGatewayConfigPatch( - selectedScenarios, - params?.adapterOptions?.sutAccountId?.trim() || - (channelDriver === "crabline" ? "default" : "sut"), - ); - const gatewayRuntimeOptions = collectQaSuiteGatewayRuntimeOptions(selectedScenarios); - const concurrency = params?.failFast - ? 1 - : normalizeQaSuiteConcurrency( - params?.concurrency, - selectedScenarios.length, - params?.channelDriverSelection ? 1 : defaultQaSuiteConcurrencyForTransport(transportId), - ); - const progressEnabled = shouldLogQaSuiteProgress(); - const gatewayHeapCheckpointsEnabled = shouldCaptureGatewayHeapCheckpoints(); - writeQaSuiteProgress( - progressEnabled, - formatQaSuiteRunStartProgress({ - selectedScenarioCount: selectedScenarios.length, - concurrency, - transportId, - channelDriver: params?.channelDriver, - channelDriverSelection: params?.channelDriverSelection, - }), - ); - const useIsolatedScenarioWorkers = shouldRunQaSuiteWithIsolatedScenarioWorkers({ - scenarios: selectedScenarios, - concurrency, - lab: params?.lab, - startLab: params?.startLab, - }); - - if (params?.runtimePair) { - return await runQaRuntimeParitySuite({ - adapterFactories: params.adapterFactories, - channelId: params.channelId, - adapterOptions: params.adapterOptions, - evidenceMode: params.evidenceMode, - repoRoot, - outputDir, - startedAt, - providerMode, - transportId, - channelDriverSelection: params?.channelDriverSelection, - channelDriver: params?.channelDriver, - primaryModel, - alternateModel, - fastMode, - thinkingDefault: params.thinkingDefault, - claudeCliAuthMode: params.claudeCliAuthMode, - enabledPluginIds: params.enabledPluginIds, - concurrency, - selectedScenarios, - startLab: params.startLab, - lab: params.lab, - progressEnabled, - scenarioIds: params.scenarioIds, - runtimePair: params.runtimePair, - writeEvidenceFile: params.writeEvidenceFile, - }); - } - - if (useIsolatedScenarioWorkers) { - const ownsLab = !params?.lab; - const startLab = requireQaSuiteStartLab(params?.startLab); - const lab = - params?.lab ?? - (await startLab({ - repoRoot, - host: "127.0.0.1", - port: 0, - embeddedGateway: "disabled", - })); - const transportFactoryResult = await createQaSuiteTransportAdapter({ - adapterFactories: params?.adapterFactories, - channelDriver: params?.channelDriver, - channelId: params?.channelId, - channelDriverSelection: params?.channelDriverSelection, - adapterOptions: { - ...params?.adapterOptions, - scenarioIds: selectedScenarios.map((scenario) => scenario.id), - }, - cleanupOnFailure: ownsLab ? () => lab.stop() : undefined, - outputDir, - state: lab.state, - transportId, - }); - const transport = transportFactoryResult.adapter; - const liveScenarioOutcomes: QaLabScenarioOutcome[] = selectedScenarios.map((scenario) => ({ - id: scenario.id, - name: scenario.title, - status: "pending", - })); - const updateScenarioRun = () => - lab.setScenarioRun({ - kind: "suite", - status: "running", - startedAt: startedAt.toISOString(), - scenarios: [...liveScenarioOutcomes], - }); - const completedScenarioResults: Array = Array.from({ - length: selectedScenarios.length, - }); - let artifactWriteQueue = Promise.resolve(); - const writePartialArtifacts = () => { - const partialScenarios = completedScenarioResults.filter( - (scenario): scenario is QaSuiteScenarioResult => scenario !== undefined, - ); - const completedScenarioDefinitions = completedScenarioResults.flatMap((scenario, index) => - scenario === undefined || selectedScenarios[index] === undefined - ? [] - : [selectedScenarios[index]], - ); - if (partialScenarios.length === 0) { - return; - } - artifactWriteQueue = artifactWriteQueue - .then(async () => { - const partialFinishedAt = new Date(); - const { report, reportPath } = await writeQaSuiteArtifacts({ - repoRoot, - outputDir, - startedAt, - finishedAt: partialFinishedAt, - scenarios: partialScenarios, - scenarioDefinitions: completedScenarioDefinitions, - evidenceMode: params?.evidenceMode, - transport, - providerMode, - primaryModel, - alternateModel, - fastMode, - concurrency, - channelDriver: params?.channelDriver, - channelDriverSelection: params?.channelDriverSelection, - isolatedWorkers: true, - writeEvidenceFile: params?.writeEvidenceFile, - scenarioIds: - params?.scenarioIds && params.scenarioIds.length > 0 - ? selectedScenarios.map((scenario) => scenario.id) - : undefined, - }); - lab.setLatestReport({ - outputPath: reportPath, - markdown: report, - generatedAt: partialFinishedAt.toISOString(), - } satisfies QaLabLatestReport); - }) - .catch((error: unknown) => { - writeQaSuiteProgress( - progressEnabled, - `partial artifact write failed: ${sanitizeQaSuiteProgressValue(formatErrorMessage(error))}`, - ); - }); - }; - - let isolatedRunFailed = false; - let isolatedRunError: unknown; - let parentTransportCleaned = false; - try { - if (params?.channelDriver === "live") { - // The parent only renders aggregate artifacts. Release its live credentials - // before child workers acquire the same exclusive transport lease. - parentTransportCleaned = true; - await transportFactoryResult.cleanupWithoutGateway(); - } - updateScenarioRun(); - const workerStartStaggerMs = - params?.workerStartStaggerMs ?? resolveQaSuiteWorkerStartStaggerMs(concurrency); - writeQaSuiteProgress(progressEnabled, `scenario start stagger=${workerStartStaggerMs}ms`); - const scenarios: QaSuiteScenarioResult[] = await mapQaSuiteWithConcurrency( - selectedScenarios, - concurrency, - async (scenario, index): Promise => { - const scenarioIdForLog = sanitizeQaSuiteProgressValue(scenario.id); - writeQaSuiteProgress( - progressEnabled, - `scenario start (${index + 1}/${selectedScenarios.length}): ${scenarioIdForLog}`, - ); - liveScenarioOutcomes[index] = { - id: scenario.id, - name: scenario.title, - status: "running", - startedAt: new Date().toISOString(), - }; - updateScenarioRun(); - try { - const scenarioOutputDir = path.join(outputDir, "scenarios", scenario.id); - const result: QaSuiteResult = await runQaFlowSuite( - buildQaIsolatedScenarioWorkerParams({ - repoRoot, - outputDir: scenarioOutputDir, - providerMode, - transportId, - channelDriver: params?.channelDriver, - channelDriverSelection: params?.channelDriverSelection, - primaryModel, - alternateModel, - fastMode, - startLab, - scenario, - input: params, - }), - ); - const scenarioResult: QaSuiteScenarioResult = - result.scenarios[0] ?? - ({ - name: scenario.title, - status: "fail", - details: "isolated scenario run returned no scenario result", - steps: [ - { - name: "isolated scenario worker", - status: "fail", - details: "isolated scenario run returned no scenario result", - }, - ], - } satisfies QaSuiteScenarioResult); - liveScenarioOutcomes[index] = { - id: scenario.id, - name: scenario.title, - status: scenarioResult.status, - details: scenarioResult.details, - steps: scenarioResult.steps, - startedAt: liveScenarioOutcomes[index]?.startedAt, - finishedAt: new Date().toISOString(), - }; - updateScenarioRun(); - writeQaSuiteProgress( - progressEnabled, - `scenario ${scenarioResult.status} (${index + 1}/${selectedScenarios.length}): ${scenarioIdForLog}`, - ); - completedScenarioResults[index] = scenarioResult; - writePartialArtifacts(); - return scenarioResult; - } catch (error) { - const details = formatErrorMessage(error); - const scenarioResult = { - name: scenario.title, - status: "fail", - details, - steps: [ - { - name: "isolated scenario worker", - status: "fail", - details, - }, - ], - } satisfies QaSuiteScenarioResult; - liveScenarioOutcomes[index] = { - id: scenario.id, - name: scenario.title, - status: "fail", - details, - steps: scenarioResult.steps, - startedAt: liveScenarioOutcomes[index]?.startedAt, - finishedAt: new Date().toISOString(), - }; - updateScenarioRun(); - writeQaSuiteProgress( - progressEnabled, - `scenario fail (${index + 1}/${selectedScenarios.length}): ${scenarioIdForLog}`, - ); - completedScenarioResults[index] = scenarioResult; - writePartialArtifacts(); - return scenarioResult; - } - }, - { - startStaggerMs: workerStartStaggerMs, - shouldStop: (result) => params?.failFast === true && result.status === "fail", - }, - ); - await artifactWriteQueue; - const finishedAt = new Date(); - const failedCount = scenarios.filter((scenario) => scenario.status === "fail").length; - const skippedCount = scenarios.filter((scenario) => scenario.status === "skip").length; - lab.setScenarioRun({ - kind: "suite", - status: "completed", - startedAt: startedAt.toISOString(), - finishedAt: finishedAt.toISOString(), - scenarios: [...liveScenarioOutcomes], - }); - const { evidence, evidencePath, report, reportPath, summaryPath } = - await writeQaSuiteArtifacts({ - repoRoot, - outputDir, - startedAt, - finishedAt, - scenarios, - scenarioDefinitions: selectedScenarios, - evidenceMode: params?.evidenceMode, - transport, - providerMode, - primaryModel, - alternateModel, - fastMode, - concurrency, - channelDriver: params?.channelDriver, - channelDriverSelection: params?.channelDriverSelection, - isolatedWorkers: true, - writeEvidenceFile: params?.writeEvidenceFile, - // When the caller supplied an explicit non-empty --scenario filter, - // record the executed (post-selectQaFlowSuiteScenarios-normalized) ids - // so the summary matches what actually ran. When the caller passed - // nothing or an empty array ("no filter, full lane catalog"), - // preserve the unfiltered = null semantic so the summary stays - // distinguishable from an explicit all-scenarios selection. - scenarioIds: - params?.scenarioIds && params.scenarioIds.length > 0 - ? selectedScenarios.map((scenario) => scenario.id) - : undefined, - }); - lab.setLatestReport({ - outputPath: reportPath, - markdown: report, - generatedAt: finishedAt.toISOString(), - } satisfies QaLabLatestReport); - writeQaSuiteProgress( - progressEnabled, - `run complete: passed=${scenarios.length - failedCount - skippedCount} failed=${failedCount} skipped=${skippedCount} total=${scenarios.length}`, - ); - return { - outputDir, - evidence, - evidencePath, - reportPath, - summaryPath, - report, - scenarios, - watchUrl: lab.baseUrl, - } satisfies QaSuiteResult; - } catch (error) { - isolatedRunFailed = true; - isolatedRunError = error; - throw error; - } finally { - const cleanupSteps: Array<() => Promise> = [ - ...(!parentTransportCleaned ? [() => transportFactoryResult.cleanupWithoutGateway()] : []), - () => disposeRegisteredAgentHarnesses(), - ]; - if (ownsLab) { - cleanupSteps.push(() => lab.stop()); - } - const cleanupErrors = await runQaSuiteCleanupSteps(cleanupSteps); - throwQaSuiteCleanupErrors({ - cleanupErrors, - runFailed: isolatedRunFailed, - runError: isolatedRunError, - }); - } - } - - const ownsLab = !params?.lab; - const startLab = params?.startLab; - writeQaSuiteProgress(progressEnabled, "lab start"); - const lab = - params?.lab ?? - (await requireQaSuiteStartLab(startLab)({ - repoRoot, - host: "127.0.0.1", - port: 0, - embeddedGateway: "disabled", - })); - writeQaSuiteProgress(progressEnabled, `lab ready: ${sanitizeQaSuiteProgressValue(lab.baseUrl)}`); - await waitForQaLabReadyOrStopOwned({ lab, ownsLab }); - const transportFactoryResult = await createQaSuiteTransportAdapter({ - adapterFactories: params?.adapterFactories, - channelDriver: params?.channelDriver, - channelId: params?.channelId, - channelDriverSelection: params?.channelDriverSelection, - adapterOptions: { - ...params?.adapterOptions, - scenarioIds: selectedScenarios.map((scenario) => scenario.id), - }, - cleanupOnFailure: ownsLab ? () => lab.stop() : undefined, - outputDir, - transportPolicy: collectQaSuiteTransportPolicy(selectedScenarios), - state: lab.state, - transportId, - }); - const transport = transportFactoryResult.adapter; - let mock: Awaited> | undefined; - let gateway: Awaited> | undefined; - let env: QaSuiteEnvironment | undefined; - let preserveGatewayRuntimeDir: string | undefined; - let runFailed = false; - let runError: unknown; - try { - writeQaSuiteProgress(progressEnabled, `provider start: ${providerMode}`); - const activeMock = await startQaProviderServer(providerMode, { - modelRefs: [primaryModel, alternateModel], - }); - mock = activeMock; - writeQaSuiteProgress( - progressEnabled, - `provider ready: ${sanitizeQaSuiteProgressValue(activeMock?.baseUrl ?? "live")}`, - ); - writeQaSuiteProgress(progressEnabled, "gateway start"); - const activeGateway = await startQaGatewayChild({ - repoRoot, - command: params?.sutOpenClawCommand, - providerBaseUrl: activeMock ? `${activeMock.baseUrl}/v1` : undefined, - transport, - transportBaseUrl: lab.listenUrl, - controlUiAllowedOrigins: [lab.listenUrl], - providerMode, - primaryModel, - alternateModel, - fastMode, - thinkingDefault: params?.thinkingDefault, - forcedRuntime: params?.forcedRuntime, - claudeCliAuthMode: params?.claudeCliAuthMode, - controlUiEnabled: params?.controlUiEnabled ?? true, - enabledPluginIds, - forwardHostHome: gatewayRuntimeOptions?.forwardHostHome, - mutateConfig: gatewayConfigPatch - ? (cfg) => applyQaMergePatch(cfg, gatewayConfigPatch) as OpenClawConfig - : undefined, - runtimeEnvPatch: mergeQaRuntimeEnvPatches( - buildQaRuntimeEnvPatch({ - providerMode, - forcedRuntime: params?.forcedRuntime, - mockBaseUrl: activeMock?.baseUrl, - }), - transport.createRuntimeEnvPatch?.(), - buildQaGatewayHeapCheckpointRuntimeEnvPatch(), - ), - }); - gateway = activeGateway; - writeQaSuiteProgress( - progressEnabled, - `gateway ready: ${sanitizeQaSuiteProgressValue(activeGateway.baseUrl)}`, - ); - lab.setControlUi({ - controlUiProxyTarget: activeGateway.baseUrl, - controlUiProxyToken: activeGateway.token, - }); - const activeEnv: QaSuiteEnvironment = { - lab, - mock: activeMock, - gateway: activeGateway, - outputDir, - // YAML scenarios should see the full staged gateway config, not just - // the transport fragment. Routing/session/plugin assertions depend on it. - cfg: activeGateway.cfg, - transport, - repoRoot, - providerMode, - primaryModel, - alternateModel, - webSessionIds: new Set(), - }; - env = activeEnv; - - const transportReadyTimeoutMs = resolveQaSuiteTransportReadyTimeoutMs( - params?.transportReadyTimeoutMs, - ); - // The gateway child already waits for /readyz before returning, but the - // selected transport can still be finishing account startup. Pay that - // readiness cost once here so the first scenario does not race bootstrap. - await waitForTransportReady(activeEnv, transportReadyTimeoutMs).catch(async () => { - await waitForGatewayHealthy(activeEnv, transportReadyTimeoutMs); - await waitForTransportReady(activeEnv, transportReadyTimeoutMs); - }); - await sleep(1_000); - const scenarios: QaSuiteScenarioResult[] = []; - const liveScenarioOutcomes: QaLabScenarioOutcome[] = selectedScenarios.map((scenario) => ({ - id: scenario.id, - name: scenario.title, - status: "pending", - })); - - lab.setScenarioRun({ - kind: "suite", - status: "running", - startedAt: startedAt.toISOString(), - scenarios: liveScenarioOutcomes, - }); - - const gatewayProcessRssSamples: QaSuiteGatewayRssSample[] = []; - const sampleGatewayProcessRss = (label: string) => { - const gatewayProcessRssBytes = activeGateway.getProcessRssBytes?.() ?? null; - if (gatewayProcessRssBytes !== null) { - gatewayProcessRssSamples.push({ - label, - at: new Date().toISOString(), - gatewayProcessRssBytes, - }); - } - return gatewayProcessRssBytes; - }; - const gatewayProcessCpuStartMs = activeGateway.getProcessCpuMs?.() ?? null; - const gatewayProcessRssStartBytes = sampleGatewayProcessRss("suite-start"); - const gatewayHeapSnapshots: QaSuiteGatewayHeapSnapshot[] = []; - const captureGatewayHeapCheckpoint = async (label: string) => { - if (!gatewayHeapCheckpointsEnabled) { - return; - } - const snapshot = await captureGatewayHeapSnapshotCheckpoint({ - gateway: activeGateway, - outputDir, - label, - }); - if (snapshot) { - gatewayHeapSnapshots.push(snapshot); - } - }; - await captureGatewayHeapCheckpoint("suite-start"); - for (const [index, scenario] of selectedScenarios.entries()) { - const scenarioIdForLog = sanitizeQaSuiteProgressValue(scenario.id); - writeQaSuiteProgress( - progressEnabled, - `scenario start (${index + 1}/${selectedScenarios.length}): ${scenarioIdForLog}`, - ); - sampleGatewayProcessRss(`scenario:${scenario.id}:start`); - liveScenarioOutcomes[index] = { - id: scenario.id, - name: scenario.title, - status: "running", - startedAt: new Date().toISOString(), - }; - lab.setScenarioRun({ - kind: "suite", - status: "running", - startedAt: startedAt.toISOString(), - scenarios: [...liveScenarioOutcomes], - }); - - const runSelectedScenario = () => runScenarioDefinition(activeEnv, scenario); - const scenarioRetryCount = - scenario.execution.kind === "flow" ? scenario.execution.retryCount : undefined; - let result: QaSuiteScenarioResult = - scenarioRetryCount === 0 - ? await runSelectedScenario() - : await runQaScenarioWithFlakeRetry(runSelectedScenario, () => - writeQaSuiteProgress( - progressEnabled, - `scenario retry (${index + 1}/${selectedScenarios.length}): ${scenarioIdForLog}`, - ), - ); - if (result.status === "pass" && params?.roundTripProbe?.scenarioId === scenario.id) { - const probeResult = await runQaSuiteRoundTripProbe({ - probe: params.roundTripProbe, - transport, - }); - const probePassed = probeResult.passed >= params.roundTripProbe.count; - result = { - ...result, - status: probePassed ? "pass" : "fail", - details: [result.details, probeResult.details].filter(Boolean).join(" | "), - timing: probeResult.timing, - steps: [ - ...result.steps, - { - name: "Round-trip samples", - status: probePassed ? "pass" : "fail", - details: probeResult.details, - }, - ], - }; - } - sampleGatewayProcessRss(`scenario:${scenario.id}:finish`); - scenarios.push(result); - writeQaSuiteProgress( - progressEnabled, - `scenario ${result.status} (${index + 1}/${selectedScenarios.length}): ${scenarioIdForLog}`, - ); - liveScenarioOutcomes[index] = { - id: scenario.id, - name: scenario.title, - status: result.status, - details: result.details, - steps: result.steps, - startedAt: liveScenarioOutcomes[index]?.startedAt, - finishedAt: new Date().toISOString(), - }; - lab.setScenarioRun({ - kind: "suite", - status: "running", - startedAt: startedAt.toISOString(), - scenarios: [...liveScenarioOutcomes], - }); - if (params?.failFast === true && result.status === "fail") { - break; - } - } - - const runtimeParityScenario = scenarios[0]; - const runtimeParityCell = - params?.captureRuntimeParityCell && - params.forcedRuntime && - selectedScenarios.length === 1 && - runtimeParityScenario - ? await captureRuntimeParityCell({ - runtime: params.forcedRuntime, - gateway: activeGateway, - scenarioResult: runtimeParityScenario, - wallClockMs: Math.max(1, Date.now() - startedAt.getTime()), - mockBaseUrl: activeMock?.baseUrl, - }) - : undefined; - const finishedAt = new Date(); - await captureGatewayHeapCheckpoint("suite-finish"); - const metrics = buildQaSuiteRuntimeMetrics({ - startedAt, - finishedAt, - gatewayProcessCpuStartMs, - gatewayProcessCpuEndMs: activeGateway.getProcessCpuMs?.() ?? null, - gatewayProcessRssStartBytes, - gatewayProcessRssEndBytes: sampleGatewayProcessRss("suite-finish"), - gatewayProcessRssSamples, - gatewayHeapSnapshots, - }); - const failedCount = scenarios.filter((scenario) => scenario.status === "fail").length; - const skippedCount = scenarios.filter((scenario) => scenario.status === "skip").length; - if ( - scenarios.some((scenario) => scenario.status === "fail") || - gatewayRuntimeOptions?.preserveDebugArtifacts === true - ) { - preserveGatewayRuntimeDir = path.join(outputDir, "artifacts", "gateway-runtime"); - } - lab.setScenarioRun({ - kind: "suite", - status: "completed", - startedAt: startedAt.toISOString(), - finishedAt: finishedAt.toISOString(), - scenarios: [...liveScenarioOutcomes], - }); - const { evidence, evidencePath, report, reportPath, summaryPath } = await writeQaSuiteArtifacts( - { - repoRoot, - outputDir, - startedAt, - finishedAt, - scenarios, - metrics, - scenarioDefinitions: selectedScenarios, - evidenceMode: params?.evidenceMode, - transport, - providerMode, - primaryModel, - alternateModel, - fastMode, - concurrency, - channelDriver: params?.channelDriver, - channelDriverSelection: params?.channelDriverSelection, - isolatedWorkers: false, - writeEvidenceFile: params?.writeEvidenceFile, - // Same "filtered → executed list, unfiltered → null" convention as - // the concurrent-path writeQaSuiteArtifacts call above. - scenarioIds: - params?.scenarioIds && params.scenarioIds.length > 0 - ? selectedScenarios.map((scenario) => scenario.id) - : undefined, - }, - ); - const latestReport = { - outputPath: reportPath, - markdown: report, - generatedAt: finishedAt.toISOString(), - } satisfies QaLabLatestReport; - lab.setLatestReport(latestReport); - writeQaSuiteProgress( - progressEnabled, - `run complete: passed=${scenarios.length - failedCount - skippedCount} failed=${failedCount} skipped=${skippedCount} total=${scenarios.length}`, - ); - - return { - outputDir, - evidence, - evidencePath, - reportPath, - summaryPath, - report, - scenarios, - watchUrl: lab.baseUrl, - ...(runtimeParityCell ? { runtimeParityCell } : {}), - } satisfies QaSuiteResult; - } catch (error) { - runFailed = true; - runError = error; - preserveGatewayRuntimeDir = path.join(outputDir, "artifacts", "gateway-runtime"); - throw error; - } finally { - const activeEnv = env; - const keepTemp = process.env.OPENCLAW_QA_KEEP_TEMP === "1" || false; - const activeGateway = gateway; - const activeMock = mock; - const cleanupErrors = await runQaFlowSuiteCleanupPlan({ - closeWebSessions: activeEnv ? () => closeQaWebSessions(activeEnv.webSessionIds) : undefined, - cleanupTransportBeforeGatewayStop: () => transportFactoryResult.cleanupBeforeGatewayStop(), - cleanupTransportAfterGatewayStop: () => transportFactoryResult.cleanupAfterGatewayStop(), - stopGateway: activeGateway - ? () => - activeGateway.stop({ - keepTemp, - preserveToDir: keepTemp ? undefined : preserveGatewayRuntimeDir, - }) - : undefined, - disposeAgentHarnesses: () => disposeRegisteredAgentHarnesses(), - stopProvider: activeMock ? () => activeMock.stop() : undefined, - finishLab: ownsLab - ? () => lab.stop() - : async () => { - lab.setControlUi({ - controlUiUrl: null, - controlUiProxyTarget: null, - }); - }, - }); - throwQaSuiteCleanupErrors({ cleanupErrors, runFailed, runError }); - } + const { runQaFlowSuiteFromRuntime } = await import("./suite-run.runtime.js"); + return await runQaFlowSuiteFromRuntime(params); } export const qaSuiteProgressTesting = { @@ -2092,4 +567,3 @@ export const qaSuiteProgressTesting = { waitForQaLabReadyOrStopOwned, writeQaSuiteArtifacts, }; -/* oxlint-disable max-lines -- TODO: split this grandfathered oversized file. */