mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-06 18:43:38 +00:00
677c8ff846
* improve: enforce canonical QA scenario ownership * docs: refresh QA documentation map * docs: condense QA ownership guidance * docs: remove QA migration guidance * refactor: simplify QA scenario ownership checks * docs: align QA coverage authoring guidance
1656 lines
56 KiB
TypeScript
1656 lines
56 KiB
TypeScript
// Qa Lab plugin module implements cli behavior.
|
|
import fs from "node:fs/promises";
|
|
import path from "node:path";
|
|
import {
|
|
OPENCLAW_CRABLINE_DEFAULT_CHANNEL,
|
|
resolveOpenClawCrablineChannelDriverSelection,
|
|
} from "@openclaw/crabline";
|
|
import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
|
|
import { parseStrictPositiveInteger } from "openclaw/plugin-sdk/number-runtime";
|
|
import { uniqueStrings } from "openclaw/plugin-sdk/string-coerce-runtime";
|
|
import {
|
|
buildQaAgenticParityComparison,
|
|
buildQaRuntimeParityReport,
|
|
renderQaAgenticParityMarkdownReport,
|
|
renderQaRuntimeParityMarkdownReport,
|
|
type QaParitySuiteSummary,
|
|
type QaRuntimeParitySuiteSummary,
|
|
} from "./agentic-parity-report.js";
|
|
import { resolveQaParityPackScenarioIds } from "./agentic-parity.js";
|
|
import { createQaArtifactRunId } from "./artifact-run-id.js";
|
|
import { runQaCharacterEval, type QaCharacterModelOptions } from "./character-eval.js";
|
|
import { resolveRepoRelativeOutputDir } from "./cli-paths.js";
|
|
import {
|
|
buildQaConfidenceReport,
|
|
readQaConfidenceManifestFile,
|
|
renderQaConfidenceMarkdownReport,
|
|
writeQaConfidenceSelfTestArtifacts,
|
|
} from "./confidence-report.js";
|
|
import {
|
|
buildQaCoverageInventory,
|
|
findQaScenarioMatches,
|
|
renderQaCoverageMarkdownReport,
|
|
renderQaScenarioMatchesMarkdownReport,
|
|
} from "./coverage-report.js";
|
|
import { buildQaDockerHarnessImage, writeQaDockerHarnessFiles } from "./docker-harness.js";
|
|
import { runQaDockerUp } from "./docker-up.runtime.js";
|
|
import { QaSuiteArtifactError, QaSuiteInfraError } from "./errors.js";
|
|
import type { QaCliBackendAuthMode } from "./gateway-child.js";
|
|
import {
|
|
createMockJsonlReplayCellRunner,
|
|
renderJsonlReplayMarkdownReport,
|
|
runJsonlReplay,
|
|
type JsonlReplayInput,
|
|
} from "./jsonl-replay.js";
|
|
import { startQaLabServer } from "./lab-server.js";
|
|
import { runQaManualLane } from "./manual-lane.runtime.js";
|
|
import { loadNonYamlScenarioRefs } from "./live-transports/shared/live-transport-scenarios.js";
|
|
import { runQaMultipass } from "./multipass.runtime.js";
|
|
import { DEFAULT_QA_LIVE_PROVIDER_MODE, getQaProvider } from "./providers/index.js";
|
|
import {
|
|
QA_FRONTIER_PARITY_BASELINE_LABEL,
|
|
QA_FRONTIER_PARITY_CANDIDATE_LABEL,
|
|
} from "./providers/live-frontier/parity.js";
|
|
import { startQaProviderServer } from "./providers/server-runtime.js";
|
|
import {
|
|
addQaCredentialSet,
|
|
diagnoseQaCredentialBroker,
|
|
listQaCredentialSets,
|
|
QaCredentialAdminError,
|
|
removeQaCredentialSet,
|
|
type QaCredentialRecord,
|
|
} from "./qa-credentials-admin.runtime.js";
|
|
import { normalizeQaThinkingLevel, type QaThinkingLevel } from "./qa-gateway-config.js";
|
|
import { normalizeQaTransportId, type QaTransportId } from "./qa-transport-registry.js";
|
|
import {
|
|
defaultQaModelForMode,
|
|
normalizeQaProviderMode,
|
|
type QaProviderMode,
|
|
type QaProviderModeInput,
|
|
} from "./run-config.js";
|
|
import type { RuntimeId } from "./runtime-parity.js";
|
|
import {
|
|
QA_RUNTIME_PARITY_TIERS,
|
|
readQaScenarioPack,
|
|
type QaRuntimeParityTier,
|
|
} from "./scenario-catalog.js";
|
|
import { resolveQaScenarioPackScenarioIds } from "./scenario-packs.js";
|
|
import { attachQaProfileScorecardEvidenceToFile } from "./scorecard-evidence.js";
|
|
import {
|
|
qaScorecardChannelDriverSchema,
|
|
readQaScorecardTaxonomyReport,
|
|
type QaScorecardCategoryCoverageReport,
|
|
type QaScorecardChannelDriver,
|
|
type QaScorecardEvidenceMode,
|
|
} from "./scorecard-taxonomy.js";
|
|
import { isQaSelfCheckSuccessful } from "./self-check.js";
|
|
import { runQaFlowSuiteFromRuntime, runQaSuite } from "./suite-launch.runtime.js";
|
|
import { resolveQaSuiteScenarioChannel, scenarioMatchesQaProviderLane } from "./suite-planning.js";
|
|
import { readQaSuiteFailedOrSkippedScenarioCountFromFile } from "./suite-summary.js";
|
|
import {
|
|
buildTokenEfficiencyReport,
|
|
renderTokenEfficiencyMarkdownReport,
|
|
type TokenEfficiencySuiteSummary,
|
|
} from "./token-efficiency-report.js";
|
|
import {
|
|
buildQaToolCoverageReport,
|
|
renderQaToolCoverageMarkdownReport,
|
|
type QaToolCoverageSuiteSummary,
|
|
} from "./tool-coverage-report.js";
|
|
|
|
const QA_SUITE_INFRA_RETRY_LIMIT = 1;
|
|
const QA_CREDENTIAL_PAYLOAD_MAX_BYTES_ENV = "OPENCLAW_QA_CREDENTIAL_PAYLOAD_MAX_BYTES";
|
|
const DEFAULT_QA_CREDENTIAL_PAYLOAD_MAX_BYTES = 64 * 1024 * 1024;
|
|
const QA_SUITE_INFRA_RETRY_NETWORK_ERROR_CODES = new Set([
|
|
"ECONNRESET",
|
|
"ECONNREFUSED",
|
|
"EPIPE",
|
|
"ETIMEDOUT",
|
|
"UND_ERR_SOCKET",
|
|
]);
|
|
|
|
type InterruptibleServer = {
|
|
baseUrl: string;
|
|
stop(): Promise<void>;
|
|
};
|
|
|
|
export type QaLabSelfCheckCommandOptions = {
|
|
repoRoot?: string;
|
|
output?: string;
|
|
};
|
|
|
|
type QaScenarioProviderCommandOptions = {
|
|
transportId?: string;
|
|
providerMode?: QaProviderModeInput;
|
|
primaryModel?: string;
|
|
alternateModel?: string;
|
|
fastMode?: boolean;
|
|
};
|
|
|
|
type QaScenarioRunCommandOptions = QaScenarioProviderCommandOptions & {
|
|
evidenceMode?: QaScorecardEvidenceMode;
|
|
repoRoot?: string;
|
|
outputDir?: string;
|
|
concurrency?: number;
|
|
allowFailures?: boolean;
|
|
};
|
|
|
|
export type QaProfileCommandOptions = QaScenarioRunCommandOptions & {
|
|
profile: string;
|
|
surface?: string;
|
|
category?: string;
|
|
scenarioIds?: string[];
|
|
};
|
|
|
|
export type QaSuiteCommandOptions = QaScenarioRunCommandOptions & {
|
|
channelDriver?: string;
|
|
channel?: string;
|
|
runner?: string;
|
|
thinking?: string;
|
|
cliAuthMode?: string;
|
|
parityPack?: string;
|
|
pack?: string;
|
|
scenarioIds?: string[];
|
|
enabledPluginIds?: string[];
|
|
image?: string;
|
|
cpus?: number;
|
|
memory?: string;
|
|
disk?: string;
|
|
preflight?: boolean;
|
|
runtimePair?: string;
|
|
runtimeParityTier?: string[];
|
|
};
|
|
|
|
function normalizeQaSuiteChannelDriver(
|
|
input?: string | null,
|
|
): QaScorecardChannelDriver | undefined {
|
|
const normalized = input?.trim().toLowerCase();
|
|
if (!normalized) {
|
|
return undefined;
|
|
}
|
|
const parsed = qaScorecardChannelDriverSchema.safeParse(normalized);
|
|
if (parsed.success) {
|
|
return parsed.data;
|
|
}
|
|
throw new Error(`--channel-driver must be one of qa-channel, crabline, or live, got "${input}".`);
|
|
}
|
|
|
|
function resolveQaManualLaneModels(opts: {
|
|
providerMode: QaProviderMode;
|
|
primaryModel?: string;
|
|
alternateModel?: string;
|
|
}) {
|
|
const primaryModel = opts.primaryModel?.trim() || defaultQaModelForMode(opts.providerMode);
|
|
const alternateModel = opts.alternateModel?.trim();
|
|
return {
|
|
primaryModel,
|
|
alternateModel:
|
|
alternateModel && alternateModel.length > 0
|
|
? alternateModel
|
|
: opts.primaryModel?.trim()
|
|
? primaryModel
|
|
: defaultQaModelForMode(opts.providerMode, true),
|
|
};
|
|
}
|
|
|
|
function parseQaThinkingLevel(
|
|
label: string,
|
|
value: string | undefined,
|
|
): QaThinkingLevel | undefined {
|
|
if (value === undefined) {
|
|
return undefined;
|
|
}
|
|
const normalized = normalizeQaThinkingLevel(value);
|
|
if (!normalized) {
|
|
throw new Error(
|
|
`${label} must be one of off, minimal, low, medium, high, xhigh, adaptive, max`,
|
|
);
|
|
}
|
|
return normalized;
|
|
}
|
|
|
|
function parseQaModelThinkingOverrides(entries: readonly string[] | undefined) {
|
|
const overrides: Record<string, QaThinkingLevel> = {};
|
|
for (const entry of entries ?? []) {
|
|
const separatorIndex = entry.lastIndexOf("=");
|
|
if (separatorIndex <= 0 || separatorIndex === entry.length - 1) {
|
|
throw new Error(`--model-thinking must use provider/model=level, got "${entry}"`);
|
|
}
|
|
const model = entry.slice(0, separatorIndex).trim();
|
|
const level = parseQaThinkingLevel("--model-thinking", entry.slice(separatorIndex + 1).trim());
|
|
if (!model || !level) {
|
|
throw new Error(`--model-thinking must use provider/model=level, got "${entry}"`);
|
|
}
|
|
overrides[model] = level;
|
|
}
|
|
return Object.keys(overrides).length > 0 ? overrides : undefined;
|
|
}
|
|
|
|
function parseQaBooleanModelOption(label: string, value: string) {
|
|
switch (value.trim().toLowerCase()) {
|
|
case "1":
|
|
case "on":
|
|
case "true":
|
|
case "yes":
|
|
return true;
|
|
case "0":
|
|
case "false":
|
|
case "no":
|
|
case "off":
|
|
return false;
|
|
default:
|
|
throw new Error(`${label} fast must be one of true, false, on, off, yes, no, 1, 0`);
|
|
}
|
|
}
|
|
|
|
function parseQaPositiveIntegerOption(label: string, value: number | undefined) {
|
|
if (value === undefined) {
|
|
return undefined;
|
|
}
|
|
if (!Number.isSafeInteger(value) || value < 1) {
|
|
throw new Error(`${label} must be a positive integer`);
|
|
}
|
|
return value;
|
|
}
|
|
|
|
function normalizeQaOptionalModelRef(input: string | undefined) {
|
|
const model = input?.trim();
|
|
return model && model.length > 0 ? model : undefined;
|
|
}
|
|
|
|
function normalizeQaRuntimeId(value: string): RuntimeId | undefined {
|
|
if (value === "openclaw" || value === "pi") {
|
|
return "openclaw";
|
|
}
|
|
if (value === "codex") {
|
|
return "codex";
|
|
}
|
|
return undefined;
|
|
}
|
|
|
|
function parseQaRuntimePair(value: string | undefined): [RuntimeId, RuntimeId] | undefined {
|
|
if (!value?.trim()) {
|
|
return undefined;
|
|
}
|
|
const runtimes = value
|
|
.split(",")
|
|
.map((part) => part.trim().toLowerCase())
|
|
.filter(Boolean)
|
|
.map(normalizeQaRuntimeId);
|
|
if (runtimes.length !== 2) {
|
|
throw new Error('--runtime-pair must use exactly two runtimes, e.g. "openclaw,codex".');
|
|
}
|
|
const [left, right] = runtimes;
|
|
if (!left || !right) {
|
|
throw new Error('--runtime-pair only supports "openclaw" and "codex".');
|
|
}
|
|
if (left === right) {
|
|
throw new Error("--runtime-pair must compare two different runtimes.");
|
|
}
|
|
return ["openclaw", "codex"];
|
|
}
|
|
|
|
function parseQaRuntimeParityTierFilters(input: string[] | undefined): QaRuntimeParityTier[] {
|
|
const rawValues = [
|
|
...new Set(
|
|
(input ?? [])
|
|
.flatMap((value) => value.split(","))
|
|
.map((value) => value.trim().toLowerCase())
|
|
.filter(Boolean),
|
|
),
|
|
];
|
|
const validTiers = new Set<string>(QA_RUNTIME_PARITY_TIERS);
|
|
for (const value of rawValues) {
|
|
if (!validTiers.has(value)) {
|
|
throw new Error(
|
|
`--runtime-parity-tier must be one of ${QA_RUNTIME_PARITY_TIERS.join(", ")}, got "${value}".`,
|
|
);
|
|
}
|
|
}
|
|
return rawValues as QaRuntimeParityTier[];
|
|
}
|
|
|
|
function resolveQaRuntimeParityTierScenarioIds(params: {
|
|
scenarioIds: string[];
|
|
runtimeParityTiers: readonly QaRuntimeParityTier[];
|
|
}): string[] {
|
|
if (params.runtimeParityTiers.length === 0) {
|
|
return params.scenarioIds;
|
|
}
|
|
const tierSet = new Set(params.runtimeParityTiers);
|
|
const matchingScenarioIds = readQaScenarioPack()
|
|
.scenarios.filter(
|
|
(scenario) => scenario.runtimeParityTier && tierSet.has(scenario.runtimeParityTier),
|
|
)
|
|
.map((scenario) => scenario.id);
|
|
if (matchingScenarioIds.length === 0) {
|
|
throw new Error(
|
|
`--runtime-parity-tier matched no scenarios for ${params.runtimeParityTiers.join(", ")}.`,
|
|
);
|
|
}
|
|
return uniqueStrings([...params.scenarioIds, ...matchingScenarioIds]);
|
|
}
|
|
|
|
function rejectNonFlowScenarioIdsForMultipass(scenarioIds: readonly string[]) {
|
|
if (scenarioIds.length === 0) {
|
|
return;
|
|
}
|
|
const scenarioById = new Map(
|
|
readQaScenarioPack().scenarios.map((scenario) => [scenario.id, scenario]),
|
|
);
|
|
const nonFlowScenarios = scenarioIds.flatMap((scenarioId) => {
|
|
const scenario = scenarioById.get(scenarioId);
|
|
return scenario && scenario.execution.kind !== "flow"
|
|
? [`${scenario.id} (${scenario.execution.kind})`]
|
|
: [];
|
|
});
|
|
if (nonFlowScenarios.length > 0) {
|
|
throw new Error(
|
|
`--runner multipass requires execution.kind: flow scenarios; unsupported scenario(s): ${nonFlowScenarios.join(", ")}`,
|
|
);
|
|
}
|
|
}
|
|
|
|
function isQaSuiteInfraRetryableError(error: unknown) {
|
|
if (error instanceof QaSuiteArtifactError || error instanceof QaSuiteInfraError) {
|
|
return true;
|
|
}
|
|
return hasQaSuiteRetryableNetworkCode(error);
|
|
}
|
|
|
|
function hasQaSuiteRetryableNetworkCode(error: unknown) {
|
|
let current: unknown = error;
|
|
for (let depth = 0; depth < 4 && current; depth += 1) {
|
|
if (typeof current !== "object") {
|
|
return false;
|
|
}
|
|
const record = current as { cause?: unknown; code?: unknown };
|
|
if (
|
|
typeof record.code === "string" &&
|
|
QA_SUITE_INFRA_RETRY_NETWORK_ERROR_CODES.has(record.code.toUpperCase())
|
|
) {
|
|
return true;
|
|
}
|
|
current = record.cause;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
async function runQaSuiteWithInfraRetry<Result>(
|
|
run: () => Promise<Result>,
|
|
maxRetries = QA_SUITE_INFRA_RETRY_LIMIT,
|
|
) {
|
|
for (let attempt = 0; attempt <= maxRetries; attempt += 1) {
|
|
try {
|
|
return await run();
|
|
} catch (error) {
|
|
const retryable = isQaSuiteInfraRetryableError(error);
|
|
if (!retryable || attempt >= maxRetries) {
|
|
throw error;
|
|
}
|
|
process.stderr.write(
|
|
`[qa-suite] infra retry ${attempt + 1}/${maxRetries}: ${formatErrorMessage(error)}\n`,
|
|
);
|
|
}
|
|
}
|
|
throw new Error("unreachable qa suite retry state");
|
|
}
|
|
|
|
async function runQaParityPreflight(params: {
|
|
repoRoot: string;
|
|
transportId: QaTransportId;
|
|
providerMode: QaProviderMode;
|
|
primaryModel?: string;
|
|
alternateModel?: string;
|
|
allowFailures?: boolean;
|
|
}) {
|
|
const outputDir = path.join(
|
|
params.repoRoot,
|
|
".artifacts",
|
|
"qa-e2e",
|
|
"preflight",
|
|
`suite-${createQaArtifactRunId()}`,
|
|
);
|
|
const result = await runQaSuiteWithInfraRetry(() =>
|
|
runQaFlowSuiteFromRuntime({
|
|
repoRoot: params.repoRoot,
|
|
outputDir,
|
|
transportId: params.transportId,
|
|
providerMode: params.providerMode,
|
|
primaryModel: params.primaryModel,
|
|
alternateModel: params.alternateModel,
|
|
scenarioIds: ["approval-turn-tool-followthrough"],
|
|
concurrency: 1,
|
|
}),
|
|
);
|
|
process.stdout.write(`QA parity preflight watch: ${result.watchUrl}\n`);
|
|
process.stdout.write(`QA parity preflight report: ${result.reportPath}\n`);
|
|
process.stdout.write(`QA parity preflight summary: ${result.summaryPath}\n`);
|
|
const blockingScenarioCount = await readQaSuiteFailedOrSkippedScenarioCountFromFile(
|
|
result.summaryPath,
|
|
);
|
|
if (blockingScenarioCount > 0) {
|
|
if (params.allowFailures === true) {
|
|
return;
|
|
}
|
|
throw new Error(
|
|
`QA parity preflight failed with ${blockingScenarioCount} failing or skipped scenario${blockingScenarioCount === 1 ? "" : "s"}.`,
|
|
);
|
|
}
|
|
}
|
|
|
|
function parseQaCliBackendAuthMode(value: string | undefined): QaCliBackendAuthMode | undefined {
|
|
const normalized = value?.trim().toLowerCase();
|
|
if (!normalized) {
|
|
return undefined;
|
|
}
|
|
if (normalized === "auto" || normalized === "api-key" || normalized === "subscription") {
|
|
return normalized;
|
|
}
|
|
throw new Error("--cli-auth-mode must be one of auto, api-key, subscription");
|
|
}
|
|
|
|
function parseQaCredentialListStatus(value: string | undefined) {
|
|
if (value === undefined) {
|
|
return undefined;
|
|
}
|
|
const normalized = value.trim().toLowerCase();
|
|
if (normalized === "active" || normalized === "disabled" || normalized === "all") {
|
|
return normalized;
|
|
}
|
|
throw new Error('--status must be one of "active", "disabled", or "all".');
|
|
}
|
|
|
|
function normalizeQaCredentialAdminError(error: unknown) {
|
|
if (error instanceof QaCredentialAdminError) {
|
|
return {
|
|
code: error.code,
|
|
message: error.message,
|
|
};
|
|
}
|
|
return {
|
|
code: "UNEXPECTED_ERROR",
|
|
message: formatErrorMessage(error),
|
|
};
|
|
}
|
|
|
|
function writeQaCredentialCommandErrorJson(action: string, error: unknown) {
|
|
const normalized = normalizeQaCredentialAdminError(error);
|
|
process.stdout.write(
|
|
`${JSON.stringify(
|
|
{
|
|
status: "error",
|
|
action,
|
|
code: normalized.code,
|
|
message: normalized.message,
|
|
},
|
|
null,
|
|
2,
|
|
)}\n`,
|
|
);
|
|
}
|
|
|
|
function parseQaModelSpecs(label: string, entries: readonly string[] | undefined) {
|
|
const models: string[] = [];
|
|
const optionsByModel: Record<string, QaCharacterModelOptions> = {};
|
|
|
|
for (const entry of entries ?? []) {
|
|
const parts = entry.split(",").map((part) => part.trim());
|
|
const model = parts[0];
|
|
if (!model) {
|
|
throw new Error(`${label} must start with provider/model, got "${entry}"`);
|
|
}
|
|
models.push(model);
|
|
const options: QaCharacterModelOptions = {};
|
|
for (const part of parts.slice(1)) {
|
|
if (!part) {
|
|
throw new Error(`${label} option cannot be empty in "${entry}"`);
|
|
}
|
|
if (part === "fast") {
|
|
options.fastMode = true;
|
|
continue;
|
|
}
|
|
if (part === "no-fast") {
|
|
options.fastMode = false;
|
|
continue;
|
|
}
|
|
const separatorIndex = part.indexOf("=");
|
|
if (separatorIndex <= 0 || separatorIndex === part.length - 1) {
|
|
throw new Error(
|
|
`${label} options must be thinking=<level>, fast, no-fast, or fast=<boolean>, got "${part}"`,
|
|
);
|
|
}
|
|
const key = part.slice(0, separatorIndex).trim();
|
|
const value = part.slice(separatorIndex + 1).trim();
|
|
switch (key) {
|
|
case "thinking": {
|
|
const thinkingDefault = parseQaThinkingLevel(`${label} thinking`, value);
|
|
if (!thinkingDefault) {
|
|
throw new Error(
|
|
`${label} thinking must be one of off, minimal, low, medium, high, xhigh, adaptive, max`,
|
|
);
|
|
}
|
|
options.thinkingDefault = thinkingDefault;
|
|
break;
|
|
}
|
|
case "fast":
|
|
options.fastMode = parseQaBooleanModelOption(label, value);
|
|
break;
|
|
default:
|
|
throw new Error(`${label} does not support option "${key}" in "${entry}"`);
|
|
}
|
|
}
|
|
if (Object.keys(options).length > 0) {
|
|
optionsByModel[model] = { ...optionsByModel[model], ...options };
|
|
}
|
|
}
|
|
|
|
return {
|
|
models,
|
|
optionsByModel: Object.keys(optionsByModel).length > 0 ? optionsByModel : undefined,
|
|
};
|
|
}
|
|
|
|
async function runInterruptibleServer(label: string, server: InterruptibleServer) {
|
|
process.stdout.write(`${label}: ${server.baseUrl}\n`);
|
|
process.stdout.write("Press Ctrl+C to stop.\n");
|
|
|
|
const shutdown = async () => {
|
|
process.off("SIGINT", onSignal);
|
|
process.off("SIGTERM", onSignal);
|
|
await server.stop();
|
|
process.exit(0);
|
|
};
|
|
|
|
const onSignal = () => {
|
|
void shutdown();
|
|
};
|
|
|
|
process.on("SIGINT", onSignal);
|
|
process.on("SIGTERM", onSignal);
|
|
await new Promise(() => {});
|
|
}
|
|
|
|
function resolveQaCredentialPayloadFileMaxBytes(env: NodeJS.ProcessEnv = process.env) {
|
|
const raw = env[QA_CREDENTIAL_PAYLOAD_MAX_BYTES_ENV]?.trim();
|
|
if (!raw) {
|
|
return DEFAULT_QA_CREDENTIAL_PAYLOAD_MAX_BYTES;
|
|
}
|
|
const parsed = parseStrictPositiveInteger(raw);
|
|
if (parsed === undefined) {
|
|
throw new Error(`${QA_CREDENTIAL_PAYLOAD_MAX_BYTES_ENV} must be a positive integer.`);
|
|
}
|
|
return parsed;
|
|
}
|
|
|
|
async function readQaCredentialPayloadFile(filePath: string) {
|
|
const maxBytes = resolveQaCredentialPayloadFileMaxBytes();
|
|
const stat = await fs.stat(filePath);
|
|
if (!stat.isFile()) {
|
|
throw new Error("Payload file must be a regular JSON file.");
|
|
}
|
|
if (stat.size > maxBytes) {
|
|
throw new Error(
|
|
`Payload file exceeds ${QA_CREDENTIAL_PAYLOAD_MAX_BYTES_ENV} (${maxBytes} bytes).`,
|
|
);
|
|
}
|
|
const text = await fs.readFile(filePath, "utf8");
|
|
let payload: unknown;
|
|
try {
|
|
payload = JSON.parse(text) as unknown;
|
|
} catch (error) {
|
|
throw new Error(`Payload file must contain valid JSON: ${formatErrorMessage(error)}`, {
|
|
cause: error,
|
|
});
|
|
}
|
|
if (!payload || typeof payload !== "object" || Array.isArray(payload)) {
|
|
throw new Error("Payload file JSON must be an object.");
|
|
}
|
|
return payload as Record<string, unknown>;
|
|
}
|
|
|
|
function formatQaCredentialLeaseState(credential: QaCredentialRecord) {
|
|
if (!credential.lease) {
|
|
return "no";
|
|
}
|
|
return `yes(${credential.lease.actorRole}:${credential.lease.ownerId})`;
|
|
}
|
|
|
|
function printQaCredentialListTable(credentials: QaCredentialRecord[]) {
|
|
if (credentials.length === 0) {
|
|
process.stdout.write("No credentials matched.\n");
|
|
return;
|
|
}
|
|
const rows = credentials.map((credential) => ({
|
|
credentialId: credential.credentialId,
|
|
fingerprint: credential.credentialFingerprint ?? "",
|
|
kind: credential.kind,
|
|
status: credential.status,
|
|
leased: formatQaCredentialLeaseState(credential),
|
|
note: credential.note ?? "",
|
|
}));
|
|
const idWidth = Math.max("credentialId".length, ...rows.map((row) => row.credentialId.length));
|
|
const fingerprintWidth = Math.max(
|
|
"fingerprint".length,
|
|
...rows.map((row) => row.fingerprint.length),
|
|
);
|
|
const kindWidth = Math.max("kind".length, ...rows.map((row) => row.kind.length));
|
|
const statusWidth = Math.max("status".length, ...rows.map((row) => row.status.length));
|
|
const leaseWidth = Math.max("leased".length, ...rows.map((row) => row.leased.length));
|
|
process.stdout.write(
|
|
`${"credentialId".padEnd(idWidth)} ${"fingerprint".padEnd(fingerprintWidth)} ${"kind".padEnd(kindWidth)} ${"status".padEnd(statusWidth)} ${"leased".padEnd(leaseWidth)} note\n`,
|
|
);
|
|
for (const row of rows) {
|
|
process.stdout.write(
|
|
`${row.credentialId.padEnd(idWidth)} ${row.fingerprint.padEnd(fingerprintWidth)} ${row.kind.padEnd(kindWidth)} ${row.status.padEnd(statusWidth)} ${row.leased.padEnd(leaseWidth)} ${row.note}\n`,
|
|
);
|
|
}
|
|
}
|
|
|
|
function printQaCredentialDoctorTable(
|
|
result: Awaited<ReturnType<typeof diagnoseQaCredentialBroker>>,
|
|
) {
|
|
process.stdout.write(`QA credentials doctor: ${result.status}\n`);
|
|
const nameWidth = Math.max("check".length, ...result.checks.map((check) => check.name.length));
|
|
for (const check of result.checks) {
|
|
process.stdout.write(
|
|
`${check.name.padEnd(nameWidth)} ${check.status.padEnd(4)} ${check.details ?? ""}\n`,
|
|
);
|
|
}
|
|
}
|
|
|
|
export async function runQaLabSelfCheckCommand(opts: QaLabSelfCheckCommandOptions) {
|
|
const repoRoot = path.resolve(opts.repoRoot ?? process.cwd());
|
|
const server = await startQaLabServer({
|
|
repoRoot,
|
|
outputPath: opts.output ? path.resolve(repoRoot, opts.output) : undefined,
|
|
});
|
|
try {
|
|
const result = await server.runSelfCheck();
|
|
process.stdout.write(`QA self-check report: ${result.outputPath}\n`);
|
|
if (!isQaSelfCheckSuccessful(result)) {
|
|
throw new Error(`QA self-check failed. See ${result.outputPath}.`);
|
|
}
|
|
} finally {
|
|
await server.stop();
|
|
}
|
|
}
|
|
|
|
export async function runQaProfileCommand(opts: QaProfileCommandOptions) {
|
|
const repoRoot = path.resolve(opts.repoRoot ?? process.cwd());
|
|
const scenarioPack = readQaScenarioPack();
|
|
const scorecardReport = readQaScorecardTaxonomyReport(scenarioPack.scenarios);
|
|
const profile = normalizeQaRunProfile(
|
|
opts.profile,
|
|
scorecardReport.profiles.map((entry) => entry.id),
|
|
);
|
|
const profileReport = scorecardReport.profiles.find((entry) => entry.id === profile);
|
|
if (!profileReport) {
|
|
throw new Error(`taxonomy.yaml does not define QA run profile ${profile}.`);
|
|
}
|
|
const categories = scorecardReport.categories.filter((category) =>
|
|
qaScorecardCategoryMatchesRunProfile(category, {
|
|
profile,
|
|
surface: opts.surface,
|
|
category: opts.category,
|
|
}),
|
|
);
|
|
if (categories.length === 0) {
|
|
throw new Error(formatQaRunProfileNoMatchMessage(opts));
|
|
}
|
|
|
|
const scenarioBySourcePath = new Map(
|
|
scenarioPack.scenarios.map((scenario) => [scenario.sourcePath, scenario] as const),
|
|
);
|
|
const taxonomyScenariosForCategories = uniqueStrings(
|
|
categories.flatMap((category) => category.scenarioRefs),
|
|
)
|
|
.map((scenarioRef) => scenarioBySourcePath.get(scenarioRef))
|
|
.filter((scenario): scenario is NonNullable<typeof scenario> => scenario !== undefined);
|
|
const requestedScenarioIds = uniqueStrings(
|
|
(opts.scenarioIds ?? []).map((scenarioId) => scenarioId.trim()).filter(Boolean),
|
|
);
|
|
const taxonomyScenarios =
|
|
requestedScenarioIds.length === 0
|
|
? taxonomyScenariosForCategories
|
|
: taxonomyScenariosForCategories.filter((scenario) =>
|
|
requestedScenarioIds.includes(scenario.id),
|
|
);
|
|
if (requestedScenarioIds.length > 0 && taxonomyScenarios.length === 0) {
|
|
throw new Error(
|
|
`qa run did not find taxonomy scenarios for ${formatQaRunProfileFilterList(opts)} --scenario ${requestedScenarioIds.join(",")}.`,
|
|
);
|
|
}
|
|
const matchedScenarioIds = new Set(taxonomyScenarios.map((scenario) => scenario.id));
|
|
const missingScenarioIds = requestedScenarioIds.filter(
|
|
(scenarioId) => !matchedScenarioIds.has(scenarioId),
|
|
);
|
|
if (missingScenarioIds.length > 0) {
|
|
throw new Error(
|
|
`qa run did not find taxonomy scenarios for ${formatQaRunProfileFilterList(opts)} --scenario ${missingScenarioIds.join(",")}.`,
|
|
);
|
|
}
|
|
const providerMode = opts.providerMode ?? defaultQaRunProfileProviderMode(profile);
|
|
const normalizedProviderMode = normalizeQaProviderMode(providerMode);
|
|
const primaryModel = opts.primaryModel?.trim() || defaultQaModelForMode(normalizedProviderMode);
|
|
const scenarios = taxonomyScenarios.filter((scenario) =>
|
|
scenarioMatchesQaProviderLane({
|
|
scenario,
|
|
providerMode: normalizedProviderMode,
|
|
primaryModel,
|
|
channelDriver: profileReport.channelDriver,
|
|
}),
|
|
);
|
|
if (scenarios.length === 0) {
|
|
throw new Error(
|
|
`qa run --qa-profile ${profile} did not resolve any executable QA scenarios for provider mode ${normalizedProviderMode}.`,
|
|
);
|
|
}
|
|
|
|
process.stdout.write(
|
|
`QA run profile: ${profile}; categories: ${categories.length}; scenarios: ${scenarios.length}\n`,
|
|
);
|
|
let evidencePath: string | undefined;
|
|
await withTemporaryQaProfileEnv(profile, async () => {
|
|
const suiteResult = await runQaSuiteCommand({
|
|
repoRoot,
|
|
outputDir: opts.outputDir,
|
|
evidenceMode: opts.evidenceMode,
|
|
transportId: opts.transportId,
|
|
providerMode,
|
|
primaryModel: opts.primaryModel,
|
|
alternateModel: opts.alternateModel,
|
|
fastMode: opts.fastMode,
|
|
scenarioIds: scenarios.map((scenario) => scenario.id),
|
|
concurrency: opts.concurrency,
|
|
allowFailures: opts.allowFailures,
|
|
channelDriver: profileReport.channelDriver,
|
|
});
|
|
evidencePath =
|
|
suiteResult && "evidencePath" in suiteResult ? suiteResult.evidencePath : undefined;
|
|
});
|
|
if (!evidencePath) {
|
|
throw new Error("qa run --qa-profile did not produce qa-evidence.json.");
|
|
}
|
|
await attachQaProfileScorecardEvidenceToFile({
|
|
evidencePath,
|
|
evidenceMode: opts.evidenceMode,
|
|
profile,
|
|
filters: {
|
|
surface: opts.surface,
|
|
category: opts.category,
|
|
},
|
|
categories,
|
|
});
|
|
process.stdout.write(`QA profile scorecard: ${evidencePath}\n`);
|
|
}
|
|
|
|
function selectQaScenarioDefinitionsForChannelResolution(params: {
|
|
scenarioIds: string[];
|
|
providerMode: QaProviderMode;
|
|
primaryModel: string;
|
|
channelDriver?: QaScorecardChannelDriver | null;
|
|
claudeCliAuthMode?: QaCliBackendAuthMode;
|
|
}) {
|
|
const scenarios = readQaScenarioPack().scenarios;
|
|
if (params.scenarioIds.length > 0) {
|
|
const scenarioById = new Map(scenarios.map((scenario) => [scenario.id, scenario]));
|
|
return params.scenarioIds.flatMap((scenarioId) => {
|
|
const scenario = scenarioById.get(scenarioId);
|
|
return scenario ? [scenario] : [];
|
|
});
|
|
}
|
|
return scenarios.filter((scenario) =>
|
|
scenarioMatchesQaProviderLane({
|
|
scenario,
|
|
providerMode: params.providerMode,
|
|
primaryModel: params.primaryModel,
|
|
channelDriver: params.channelDriver,
|
|
claudeCliAuthMode: params.claudeCliAuthMode,
|
|
}),
|
|
);
|
|
}
|
|
|
|
function normalizeQaRunProfile(value: string, profileIds: readonly string[]) {
|
|
if (profileIds.length === 0) {
|
|
throw new Error("taxonomy.yaml does not define QA run profiles.");
|
|
}
|
|
const normalized = value.trim();
|
|
if (profileIds.includes(normalized)) {
|
|
return normalized;
|
|
}
|
|
throw new Error(`--qa-profile must be one of ${profileIds.join(", ")}, got "${value}".`);
|
|
}
|
|
|
|
function defaultQaRunProfileProviderMode(profile: string): QaProviderModeInput {
|
|
return profile === "smoke-ci" ? "mock-openai" : DEFAULT_QA_LIVE_PROVIDER_MODE;
|
|
}
|
|
|
|
function qaScorecardCategoryMatchesRunProfile(
|
|
category: QaScorecardCategoryCoverageReport,
|
|
opts: { profile: string; surface?: string; category?: string },
|
|
): boolean {
|
|
if (!category.profiles.includes(opts.profile)) {
|
|
return false;
|
|
}
|
|
if (opts.surface?.trim()) {
|
|
const surface = opts.surface.trim();
|
|
if (category.taxonomySurfaceId !== surface && !category.id.startsWith(`${surface}.`)) {
|
|
return false;
|
|
}
|
|
}
|
|
if (opts.category?.trim() && category.id !== opts.category.trim()) {
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
function formatQaRunProfileNoMatchMessage(
|
|
opts: Pick<QaProfileCommandOptions, "profile" | "surface" | "category">,
|
|
) {
|
|
return `qa run did not find taxonomy categories for ${formatQaRunProfileFilterList(opts)}.`;
|
|
}
|
|
|
|
function formatQaRunProfileFilterList(
|
|
opts: Pick<QaProfileCommandOptions, "profile" | "surface" | "category">,
|
|
) {
|
|
const filters = [
|
|
`--qa-profile ${opts.profile}`,
|
|
opts.surface?.trim() ? `--surface ${opts.surface.trim()}` : null,
|
|
opts.category?.trim() ? `--category ${opts.category.trim()}` : null,
|
|
].filter((filter): filter is string => filter !== null);
|
|
return filters.join(" ");
|
|
}
|
|
|
|
async function withTemporaryQaProfileEnv<T>(profile: string, run: () => Promise<T>): Promise<T> {
|
|
const previousProfile = process.env.OPENCLAW_QA_PROFILE;
|
|
process.env.OPENCLAW_QA_PROFILE = profile;
|
|
try {
|
|
return await run();
|
|
} finally {
|
|
if (previousProfile === undefined) {
|
|
delete process.env.OPENCLAW_QA_PROFILE;
|
|
} else {
|
|
process.env.OPENCLAW_QA_PROFILE = previousProfile;
|
|
}
|
|
}
|
|
}
|
|
|
|
export async function runQaSuiteCommand(opts: QaSuiteCommandOptions) {
|
|
const repoRoot = path.resolve(opts.repoRoot ?? process.cwd());
|
|
const transportId = normalizeQaTransportId(opts.transportId);
|
|
const runner = (opts.runner ?? "host").trim().toLowerCase();
|
|
const explicitScenarioIds = resolveQaScenarioPackScenarioIds({
|
|
pack: opts.pack,
|
|
scenarioIds: resolveQaParityPackScenarioIds({
|
|
parityPack: opts.parityPack,
|
|
scenarioIds: opts.scenarioIds,
|
|
}),
|
|
});
|
|
const runtimeParityTiers = parseQaRuntimeParityTierFilters(opts.runtimeParityTier);
|
|
const scenarioIds = resolveQaRuntimeParityTierScenarioIds({
|
|
scenarioIds: explicitScenarioIds,
|
|
runtimeParityTiers,
|
|
});
|
|
const allowFailures = opts.allowFailures === true;
|
|
const providerMode = normalizeQaProviderMode(opts.providerMode);
|
|
const runtimePair = parseQaRuntimePair(opts.runtimePair);
|
|
const claudeCliAuthMode = parseQaCliBackendAuthMode(opts.cliAuthMode);
|
|
const primaryModel = normalizeQaOptionalModelRef(opts.primaryModel);
|
|
const alternateModel = normalizeQaOptionalModelRef(opts.alternateModel);
|
|
const channelDriver = normalizeQaSuiteChannelDriver(opts.channelDriver);
|
|
if (opts.channel?.trim() && channelDriver !== "crabline") {
|
|
throw new Error(
|
|
"--channel override is currently only supported with --channel-driver crabline.",
|
|
);
|
|
}
|
|
if (runner !== "host" && runner !== "multipass") {
|
|
throw new Error(`--runner must be one of host or multipass, got "${opts.runner}".`);
|
|
}
|
|
if (opts.preflight === true && runner !== "host") {
|
|
throw new Error("--preflight requires --runner host.");
|
|
}
|
|
const channelDriverSelection =
|
|
channelDriver === "crabline"
|
|
? resolveOpenClawCrablineChannelDriverSelection({
|
|
channel: resolveQaSuiteScenarioChannel({
|
|
defaultChannel: OPENCLAW_CRABLINE_DEFAULT_CHANNEL,
|
|
explicitChannel: opts.channel,
|
|
scenarios: selectQaScenarioDefinitionsForChannelResolution({
|
|
scenarioIds,
|
|
providerMode,
|
|
primaryModel: primaryModel ?? defaultQaModelForMode(providerMode),
|
|
channelDriver,
|
|
claudeCliAuthMode,
|
|
}),
|
|
}),
|
|
})
|
|
: undefined;
|
|
if (
|
|
runner === "host" &&
|
|
(opts.image !== undefined ||
|
|
opts.cpus !== undefined ||
|
|
opts.memory !== undefined ||
|
|
opts.disk !== undefined)
|
|
) {
|
|
throw new Error("--image, --cpus, --memory, and --disk require --runner multipass.");
|
|
}
|
|
if (runner === "multipass" && opts.cliAuthMode !== undefined) {
|
|
throw new Error("--cli-auth-mode requires --runner host.");
|
|
}
|
|
if (runner === "multipass") {
|
|
rejectNonFlowScenarioIdsForMultipass(scenarioIds);
|
|
const thinkingDefault = parseQaThinkingLevel("--thinking", opts.thinking);
|
|
const result = await runQaMultipass({
|
|
repoRoot,
|
|
outputDir: resolveRepoRelativeOutputDir(repoRoot, opts.outputDir),
|
|
transportId,
|
|
...(opts.providerMode !== undefined ? { providerMode } : {}),
|
|
primaryModel,
|
|
alternateModel,
|
|
fastMode: opts.fastMode,
|
|
...(thinkingDefault ? { thinkingDefault } : {}),
|
|
allowFailures: true,
|
|
scenarioIds,
|
|
...(opts.concurrency !== undefined
|
|
? { concurrency: parseQaPositiveIntegerOption("--concurrency", opts.concurrency) }
|
|
: {}),
|
|
...(runtimePair ? { runtimePair } : {}),
|
|
...(channelDriverSelection ? { channelDriverSelection } : {}),
|
|
...(opts.enabledPluginIds !== undefined ? { enabledPluginIds: opts.enabledPluginIds } : {}),
|
|
image: opts.image,
|
|
cpus: parseQaPositiveIntegerOption("--cpus", opts.cpus),
|
|
memory: opts.memory,
|
|
disk: opts.disk,
|
|
});
|
|
process.stdout.write(`QA Multipass dir: ${result.outputDir}\n`);
|
|
process.stdout.write(`QA Multipass report: ${result.reportPath}\n`);
|
|
process.stdout.write(`QA Multipass summary: ${result.summaryPath}\n`);
|
|
process.stdout.write(`QA Multipass host log: ${result.hostLogPath}\n`);
|
|
process.stdout.write(`QA Multipass bootstrap log: ${result.bootstrapLogPath}\n`);
|
|
if (!allowFailures) {
|
|
const blockingScenarioCount = await readQaSuiteFailedOrSkippedScenarioCountFromFile(
|
|
result.summaryPath,
|
|
);
|
|
if (blockingScenarioCount > 0) {
|
|
process.exitCode = 1;
|
|
}
|
|
}
|
|
return result;
|
|
}
|
|
if (opts.preflight === true) {
|
|
await runQaParityPreflight({
|
|
repoRoot,
|
|
transportId,
|
|
providerMode,
|
|
primaryModel,
|
|
alternateModel,
|
|
allowFailures,
|
|
});
|
|
return undefined;
|
|
}
|
|
const thinkingDefault = parseQaThinkingLevel("--thinking", opts.thinking);
|
|
const runtimeResult = await runQaSuiteWithInfraRetry(() =>
|
|
runQaSuite({
|
|
repoRoot,
|
|
outputDir: resolveRepoRelativeOutputDir(repoRoot, opts.outputDir),
|
|
evidenceMode: opts.evidenceMode,
|
|
transportId,
|
|
channelDriver,
|
|
channelDriverSelection,
|
|
...(opts.providerMode !== undefined ? { providerMode } : {}),
|
|
primaryModel,
|
|
alternateModel,
|
|
fastMode: opts.fastMode,
|
|
...(thinkingDefault ? { thinkingDefault } : {}),
|
|
...(claudeCliAuthMode ? { claudeCliAuthMode } : {}),
|
|
scenarioIds,
|
|
...(opts.enabledPluginIds !== undefined ? { enabledPluginIds: opts.enabledPluginIds } : {}),
|
|
...(opts.concurrency !== undefined
|
|
? { concurrency: parseQaPositiveIntegerOption("--concurrency", opts.concurrency) }
|
|
: {}),
|
|
...(runtimePair ? { runtimePair } : {}),
|
|
}),
|
|
);
|
|
switch (runtimeResult.executionKind) {
|
|
case "suite": {
|
|
const result = runtimeResult.result;
|
|
process.stdout.write(`QA suite report: ${result.reportPath}\n`);
|
|
process.stdout.write(`QA suite evidence: ${result.evidencePath}\n`);
|
|
process.stdout.write(`QA suite summary: ${result.summaryPath}\n`);
|
|
if (!allowFailures && result.scenarios.some((scenario) => scenario.status !== "pass")) {
|
|
process.exitCode = 1;
|
|
}
|
|
return result;
|
|
}
|
|
case "flow": {
|
|
const result = runtimeResult.result;
|
|
process.stdout.write(`QA suite watch: ${result.watchUrl}\n`);
|
|
process.stdout.write(`QA suite report: ${result.reportPath}\n`);
|
|
process.stdout.write(`QA suite evidence: ${result.evidencePath}\n`);
|
|
process.stdout.write(`QA suite summary: ${result.summaryPath}\n`);
|
|
const blockingScenarioCount = await readQaSuiteFailedOrSkippedScenarioCountFromFile(
|
|
result.summaryPath,
|
|
);
|
|
if (!allowFailures && blockingScenarioCount > 0) {
|
|
process.exitCode = 1;
|
|
}
|
|
return result;
|
|
}
|
|
}
|
|
return undefined;
|
|
}
|
|
|
|
export async function runQaParityReportCommand(opts: {
|
|
repoRoot?: string;
|
|
candidateSummary?: string;
|
|
baselineSummary?: string;
|
|
candidateLabel?: string;
|
|
baselineLabel?: string;
|
|
outputDir?: string;
|
|
runtimeAxis?: boolean;
|
|
summary?: string;
|
|
tokenEfficiency?: boolean;
|
|
}) {
|
|
const repoRoot = path.resolve(opts.repoRoot ?? process.cwd());
|
|
if (opts.tokenEfficiency === true && opts.runtimeAxis !== true) {
|
|
throw new Error("--token-efficiency requires --runtime-axis.");
|
|
}
|
|
const outputDir =
|
|
resolveRepoRelativeOutputDir(repoRoot, opts.outputDir) ??
|
|
path.join(repoRoot, ".artifacts", "qa-e2e", `parity-${createQaArtifactRunId()}`);
|
|
await fs.mkdir(outputDir, { recursive: true });
|
|
|
|
if (opts.runtimeAxis === true) {
|
|
if (!opts.summary?.trim()) {
|
|
throw new Error("--runtime-axis requires --summary.");
|
|
}
|
|
const summaryPath = path.resolve(repoRoot, opts.summary);
|
|
const summary = JSON.parse(
|
|
await fs.readFile(summaryPath, "utf8"),
|
|
) as QaRuntimeParitySuiteSummary;
|
|
const reportPayload = buildQaRuntimeParityReport({ summary });
|
|
const report = renderQaRuntimeParityMarkdownReport(reportPayload);
|
|
const reportPath = path.join(outputDir, "qa-runtime-parity-report.md");
|
|
const runtimeSummaryPath = path.join(outputDir, "qa-runtime-parity-summary.json");
|
|
await fs.writeFile(reportPath, report, "utf8");
|
|
await fs.writeFile(runtimeSummaryPath, `${JSON.stringify(reportPayload, null, 2)}\n`, "utf8");
|
|
|
|
process.stdout.write(`QA runtime parity report: ${reportPath}\n`);
|
|
process.stdout.write(`QA runtime parity summary: ${runtimeSummaryPath}\n`);
|
|
process.stdout.write(`QA runtime parity verdict: ${reportPayload.pass ? "pass" : "fail"}\n`);
|
|
|
|
let tokenEfficiencyPass = true;
|
|
if (opts.tokenEfficiency === true) {
|
|
const tokenPayload = buildTokenEfficiencyReport({
|
|
summary: summary as TokenEfficiencySuiteSummary,
|
|
});
|
|
tokenEfficiencyPass = tokenPayload.pass;
|
|
const tokenReport = renderTokenEfficiencyMarkdownReport(tokenPayload);
|
|
const tokenReportPath = path.join(outputDir, "qa-runtime-token-efficiency-report.md");
|
|
const tokenSummaryPath = path.join(outputDir, "qa-runtime-token-efficiency-summary.json");
|
|
await fs.writeFile(tokenReportPath, tokenReport, "utf8");
|
|
await fs.writeFile(tokenSummaryPath, `${JSON.stringify(tokenPayload, null, 2)}\n`, "utf8");
|
|
process.stdout.write(`QA runtime token efficiency report: ${tokenReportPath}\n`);
|
|
process.stdout.write(`QA runtime token efficiency summary: ${tokenSummaryPath}\n`);
|
|
process.stdout.write(
|
|
`QA runtime token efficiency verdict: ${tokenPayload.status === "skipped" ? "skipped" : tokenPayload.pass ? "pass" : "fail"}\n`,
|
|
);
|
|
}
|
|
|
|
if (!reportPayload.pass || !tokenEfficiencyPass) {
|
|
process.exitCode = 1;
|
|
}
|
|
return;
|
|
}
|
|
|
|
if (!opts.candidateSummary?.trim() || !opts.baselineSummary?.trim()) {
|
|
throw new Error(
|
|
"--candidate-summary and --baseline-summary are required unless --runtime-axis is set.",
|
|
);
|
|
}
|
|
const candidateSummaryPath = path.resolve(repoRoot, opts.candidateSummary);
|
|
const baselineSummaryPath = path.resolve(repoRoot, opts.baselineSummary);
|
|
const candidateSummary = JSON.parse(
|
|
await fs.readFile(candidateSummaryPath, "utf8"),
|
|
) as QaParitySuiteSummary;
|
|
const baselineSummary = JSON.parse(
|
|
await fs.readFile(baselineSummaryPath, "utf8"),
|
|
) as QaParitySuiteSummary;
|
|
|
|
const comparison = buildQaAgenticParityComparison({
|
|
candidateLabel: opts.candidateLabel?.trim() || QA_FRONTIER_PARITY_CANDIDATE_LABEL,
|
|
baselineLabel: opts.baselineLabel?.trim() || QA_FRONTIER_PARITY_BASELINE_LABEL,
|
|
candidateSummary,
|
|
baselineSummary,
|
|
});
|
|
const report = renderQaAgenticParityMarkdownReport(comparison);
|
|
const reportPath = path.join(outputDir, "qa-agentic-parity-report.md");
|
|
const summaryPath = path.join(outputDir, "qa-agentic-parity-summary.json");
|
|
await fs.writeFile(reportPath, report, "utf8");
|
|
await fs.writeFile(summaryPath, `${JSON.stringify(comparison, null, 2)}\n`, "utf8");
|
|
|
|
process.stdout.write(`QA parity report: ${reportPath}\n`);
|
|
process.stdout.write(`QA parity summary: ${summaryPath}\n`);
|
|
process.stdout.write(`QA parity verdict: ${comparison.pass ? "pass" : "fail"}\n`);
|
|
if (!comparison.pass) {
|
|
process.exitCode = 1;
|
|
}
|
|
}
|
|
|
|
export async function runQaConfidenceReportCommand(opts: {
|
|
repoRoot?: string;
|
|
manifest: string;
|
|
artifactRoot?: string;
|
|
outputDir?: string;
|
|
strictZeroUnknowns?: boolean;
|
|
strictGlobalPass?: boolean;
|
|
}) {
|
|
const repoRoot = path.resolve(opts.repoRoot ?? process.cwd());
|
|
const manifestPath = path.resolve(repoRoot, opts.manifest);
|
|
const artifactRoot = path.resolve(repoRoot, opts.artifactRoot ?? ".");
|
|
const outputDir =
|
|
resolveRepoRelativeOutputDir(repoRoot, opts.outputDir) ??
|
|
path.join(repoRoot, ".artifacts", "qa-e2e", `confidence-${createQaArtifactRunId()}`);
|
|
await fs.mkdir(outputDir, { recursive: true });
|
|
const manifest = await readQaConfidenceManifestFile(manifestPath);
|
|
const reportPayload = await buildQaConfidenceReport({
|
|
manifest,
|
|
artifactRoot,
|
|
strictZeroUnknowns: opts.strictZeroUnknowns === true,
|
|
strictGlobalPass: opts.strictGlobalPass === true,
|
|
});
|
|
const report = renderQaConfidenceMarkdownReport(reportPayload);
|
|
const reportPath = path.join(outputDir, "qa-confidence-report.md");
|
|
const summaryPath = path.join(outputDir, "qa-confidence-summary.json");
|
|
await fs.writeFile(reportPath, report, "utf8");
|
|
await fs.writeFile(summaryPath, `${JSON.stringify(reportPayload, null, 2)}\n`, "utf8");
|
|
process.stdout.write(`QA confidence report: ${reportPath}\n`);
|
|
process.stdout.write(`QA confidence summary: ${summaryPath}\n`);
|
|
process.stdout.write(`QA confidence verdict: ${reportPayload.pass ? "pass" : "fail"}\n`);
|
|
if (!reportPayload.pass) {
|
|
process.exitCode = 1;
|
|
}
|
|
}
|
|
|
|
export async function runQaConfidenceSelfTestCommand(opts: {
|
|
repoRoot?: string;
|
|
outputDir?: string;
|
|
}) {
|
|
const repoRoot = path.resolve(opts.repoRoot ?? process.cwd());
|
|
const outputDir =
|
|
resolveRepoRelativeOutputDir(repoRoot, opts.outputDir) ??
|
|
path.join(repoRoot, ".artifacts", "qa-e2e", `confidence-self-test-${createQaArtifactRunId()}`);
|
|
const result = await writeQaConfidenceSelfTestArtifacts({ outputDir });
|
|
process.stdout.write(`QA confidence self-test report: ${result.reportPath}\n`);
|
|
process.stdout.write(`QA confidence self-test summary: ${result.summaryPath}\n`);
|
|
process.stdout.write(
|
|
`QA confidence self-test verdict: ${result.summary.pass ? "pass" : "fail"}\n`,
|
|
);
|
|
if (!result.summary.pass) {
|
|
process.exitCode = 1;
|
|
}
|
|
}
|
|
|
|
export async function runQaCoverageReportCommand(opts: {
|
|
repoRoot?: string;
|
|
output?: string;
|
|
json?: boolean;
|
|
tools?: boolean;
|
|
summary?: string;
|
|
match?: string[];
|
|
}) {
|
|
const repoRoot = path.resolve(opts.repoRoot ?? process.cwd());
|
|
const outputPath = opts.output ? path.resolve(repoRoot, opts.output) : undefined;
|
|
const scenarios = readQaScenarioPack().scenarios;
|
|
let body: string;
|
|
let outputLabel = "QA coverage report";
|
|
if (opts.tools === true) {
|
|
if (opts.match && opts.match.length > 0) {
|
|
throw new Error("--match cannot be combined with --tools.");
|
|
}
|
|
const summary = opts.summary?.trim()
|
|
? (JSON.parse(
|
|
await fs.readFile(path.resolve(repoRoot, opts.summary), "utf8"),
|
|
) as QaToolCoverageSuiteSummary)
|
|
: undefined;
|
|
const report = buildQaToolCoverageReport({ scenarios, summary });
|
|
body = opts.json
|
|
? `${JSON.stringify(report, null, 2)}\n`
|
|
: renderQaToolCoverageMarkdownReport(report);
|
|
outputLabel = "QA tool coverage report";
|
|
if (summary && !report.pass) {
|
|
process.exitCode = 1;
|
|
}
|
|
} else {
|
|
if (opts.summary?.trim()) {
|
|
throw new Error("--summary requires --tools.");
|
|
}
|
|
const query = opts.match?.join(" ").trim();
|
|
if (query) {
|
|
const matches = findQaScenarioMatches(scenarios, query);
|
|
body = opts.json
|
|
? `${JSON.stringify({ query, matches }, null, 2)}\n`
|
|
: renderQaScenarioMatchesMarkdownReport({ query, matches });
|
|
outputLabel = "QA scenario match report";
|
|
} else {
|
|
const inventory = buildQaCoverageInventory(scenarios, {
|
|
nonYamlScenarios: await loadNonYamlScenarioRefs(),
|
|
});
|
|
body = opts.json
|
|
? `${JSON.stringify(inventory, null, 2)}\n`
|
|
: renderQaCoverageMarkdownReport(inventory);
|
|
}
|
|
}
|
|
|
|
if (outputPath) {
|
|
await fs.mkdir(path.dirname(outputPath), { recursive: true });
|
|
await fs.writeFile(outputPath, body, "utf8");
|
|
process.stdout.write(`${outputLabel}: ${outputPath}\n`);
|
|
return;
|
|
}
|
|
|
|
process.stdout.write(body);
|
|
}
|
|
|
|
export async function runQaJsonlReplayCommand(opts: {
|
|
repoRoot?: string;
|
|
transcripts?: string;
|
|
outputDir?: string;
|
|
runtimePair?: string;
|
|
providerMode?: QaProviderModeInput;
|
|
}) {
|
|
const repoRoot = path.resolve(opts.repoRoot ?? process.cwd());
|
|
const runtimePair = parseQaRuntimePair(opts.runtimePair) ?? ["openclaw", "codex"];
|
|
if (runtimePair[0] !== "openclaw" || runtimePair[1] !== "codex") {
|
|
throw new Error('--runtime-pair for jsonl-replay must be "openclaw,codex".');
|
|
}
|
|
const providerMode = normalizeQaProviderMode(opts.providerMode ?? "mock-openai");
|
|
if (providerMode !== "mock-openai") {
|
|
throw new Error("qa jsonl-replay currently supports mock-openai curated fixtures only.");
|
|
}
|
|
const transcriptDir = path.resolve(repoRoot, opts.transcripts ?? "qa/scenarios/jsonl-replay");
|
|
const outputDir =
|
|
resolveRepoRelativeOutputDir(repoRoot, opts.outputDir) ??
|
|
path.join(repoRoot, ".artifacts", "qa-e2e", `jsonl-replay-${createQaArtifactRunId()}`);
|
|
await fs.mkdir(outputDir, { recursive: true });
|
|
const result = await runJsonlReplay(
|
|
{
|
|
directory: transcriptDir,
|
|
runtimePair: runtimePair as JsonlReplayInput["runtimePair"],
|
|
providerMode,
|
|
},
|
|
{ runCell: createMockJsonlReplayCellRunner() },
|
|
);
|
|
const reportPayload = {
|
|
generatedAt: new Date().toISOString(),
|
|
providerMode,
|
|
runtimePair: runtimePair as JsonlReplayInput["runtimePair"],
|
|
transcripts: result.transcripts,
|
|
};
|
|
const report = renderJsonlReplayMarkdownReport(reportPayload);
|
|
const reportPath = path.join(outputDir, "qa-jsonl-replay-report.md");
|
|
const summaryPath = path.join(outputDir, "qa-jsonl-replay-summary.json");
|
|
await fs.writeFile(reportPath, report, "utf8");
|
|
await fs.writeFile(summaryPath, `${JSON.stringify(result, null, 2)}\n`, "utf8");
|
|
process.stdout.write(`QA JSONL replay report: ${reportPath}\n`);
|
|
process.stdout.write(`QA JSONL replay summary: ${summaryPath}\n`);
|
|
}
|
|
|
|
export async function runQaCharacterEvalCommand(opts: {
|
|
repoRoot?: string;
|
|
outputDir?: string;
|
|
model?: string[];
|
|
scenario?: string;
|
|
fast?: boolean;
|
|
thinking?: string;
|
|
modelThinking?: string[];
|
|
judgeModel?: string[];
|
|
judgeTimeoutMs?: number;
|
|
blindJudgeModels?: boolean;
|
|
concurrency?: number;
|
|
judgeConcurrency?: number;
|
|
}) {
|
|
const repoRoot = path.resolve(opts.repoRoot ?? process.cwd());
|
|
const candidates = parseQaModelSpecs("--model", opts.model);
|
|
const judges = parseQaModelSpecs("--judge-model", opts.judgeModel);
|
|
const result = await runQaCharacterEval({
|
|
repoRoot,
|
|
outputDir: resolveRepoRelativeOutputDir(repoRoot, opts.outputDir),
|
|
models: candidates.models,
|
|
scenarioId: opts.scenario,
|
|
candidateFastMode: opts.fast,
|
|
candidateThinkingDefault: parseQaThinkingLevel("--thinking", opts.thinking),
|
|
candidateThinkingByModel: parseQaModelThinkingOverrides(opts.modelThinking),
|
|
candidateModelOptions: candidates.optionsByModel,
|
|
judgeModels: judges.models.length > 0 ? judges.models : undefined,
|
|
judgeModelOptions: judges.optionsByModel,
|
|
judgeTimeoutMs: opts.judgeTimeoutMs,
|
|
judgeBlindModels: opts.blindJudgeModels === true ? true : undefined,
|
|
candidateConcurrency: parseQaPositiveIntegerOption("--concurrency", opts.concurrency),
|
|
judgeConcurrency: parseQaPositiveIntegerOption("--judge-concurrency", opts.judgeConcurrency),
|
|
progress: (message) => process.stderr.write(`${message}\n`),
|
|
});
|
|
process.stdout.write(`QA character eval report: ${result.reportPath}\n`);
|
|
process.stdout.write(`QA character eval summary: ${result.summaryPath}\n`);
|
|
}
|
|
|
|
export async function runQaManualLaneCommand(opts: {
|
|
repoRoot?: string;
|
|
transportId?: string;
|
|
providerMode?: QaProviderModeInput;
|
|
primaryModel?: string;
|
|
alternateModel?: string;
|
|
fastMode?: boolean;
|
|
message: string;
|
|
timeoutMs?: number;
|
|
}) {
|
|
const repoRoot = path.resolve(opts.repoRoot ?? process.cwd());
|
|
const transportId = normalizeQaTransportId(opts.transportId);
|
|
const providerMode: QaProviderMode =
|
|
opts.providerMode === undefined
|
|
? DEFAULT_QA_LIVE_PROVIDER_MODE
|
|
: normalizeQaProviderMode(opts.providerMode);
|
|
const models = resolveQaManualLaneModels({
|
|
providerMode,
|
|
primaryModel: opts.primaryModel,
|
|
alternateModel: opts.alternateModel,
|
|
});
|
|
const result = await runQaManualLane({
|
|
repoRoot,
|
|
transportId,
|
|
providerMode,
|
|
primaryModel: models.primaryModel,
|
|
alternateModel: models.alternateModel,
|
|
fastMode: opts.fastMode,
|
|
message: opts.message,
|
|
timeoutMs: opts.timeoutMs,
|
|
});
|
|
process.stdout.write(JSON.stringify(result, null, 2));
|
|
process.stdout.write("\n");
|
|
}
|
|
|
|
export async function runQaCredentialsAddCommand(opts: {
|
|
actorId?: string;
|
|
endpointPrefix?: string;
|
|
json?: boolean;
|
|
kind: string;
|
|
note?: string;
|
|
payloadFile: string;
|
|
repoRoot?: string;
|
|
siteUrl?: string;
|
|
}) {
|
|
const repoRoot = path.resolve(opts.repoRoot ?? process.cwd());
|
|
try {
|
|
const payloadPath = path.resolve(repoRoot, opts.payloadFile);
|
|
const payload = await readQaCredentialPayloadFile(payloadPath);
|
|
const result = await addQaCredentialSet({
|
|
kind: opts.kind,
|
|
payload,
|
|
note: opts.note,
|
|
actorId: opts.actorId,
|
|
siteUrl: opts.siteUrl,
|
|
endpointPrefix: opts.endpointPrefix,
|
|
});
|
|
if (opts.json) {
|
|
process.stdout.write(
|
|
`${JSON.stringify({ status: "ok", action: "add", credential: result.credential }, null, 2)}\n`,
|
|
);
|
|
return;
|
|
}
|
|
process.stdout.write(`QA credential added: ${result.credential.credentialId}\n`);
|
|
process.stdout.write(`Kind: ${result.credential.kind}\n`);
|
|
process.stdout.write(`Status: ${result.credential.status}\n`);
|
|
if (result.credential.note) {
|
|
process.stdout.write(`Note: ${result.credential.note}\n`);
|
|
}
|
|
} catch (error) {
|
|
if (opts.json) {
|
|
writeQaCredentialCommandErrorJson("add", error);
|
|
process.exitCode = 1;
|
|
return;
|
|
}
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
export async function runQaCredentialsRemoveCommand(opts: {
|
|
actorId?: string;
|
|
credentialId: string;
|
|
endpointPrefix?: string;
|
|
json?: boolean;
|
|
siteUrl?: string;
|
|
}) {
|
|
try {
|
|
const result = await removeQaCredentialSet({
|
|
credentialId: opts.credentialId,
|
|
actorId: opts.actorId,
|
|
siteUrl: opts.siteUrl,
|
|
endpointPrefix: opts.endpointPrefix,
|
|
});
|
|
if (opts.json) {
|
|
process.stdout.write(
|
|
`${JSON.stringify(
|
|
{
|
|
status: "ok",
|
|
action: "remove",
|
|
changed: result.changed,
|
|
credential: result.credential,
|
|
},
|
|
null,
|
|
2,
|
|
)}\n`,
|
|
);
|
|
return;
|
|
}
|
|
process.stdout.write(
|
|
result.changed
|
|
? `QA credential removed (disabled): ${result.credential.credentialId}\n`
|
|
: `QA credential already disabled: ${result.credential.credentialId}\n`,
|
|
);
|
|
} catch (error) {
|
|
if (opts.json) {
|
|
writeQaCredentialCommandErrorJson("remove", error);
|
|
process.exitCode = 1;
|
|
return;
|
|
}
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
export async function runQaCredentialsListCommand(opts: {
|
|
actorId?: string;
|
|
endpointPrefix?: string;
|
|
json?: boolean;
|
|
kind?: string;
|
|
limit?: number;
|
|
showSecrets?: boolean;
|
|
siteUrl?: string;
|
|
status?: string;
|
|
}) {
|
|
try {
|
|
const result = await listQaCredentialSets({
|
|
actorId: opts.actorId,
|
|
siteUrl: opts.siteUrl,
|
|
endpointPrefix: opts.endpointPrefix,
|
|
kind: opts.kind?.trim(),
|
|
status: parseQaCredentialListStatus(opts.status),
|
|
includePayload: opts.showSecrets,
|
|
limit: parseQaPositiveIntegerOption("--limit", opts.limit),
|
|
});
|
|
if (opts.json) {
|
|
process.stdout.write(
|
|
`${JSON.stringify(
|
|
{
|
|
status: "ok",
|
|
action: "list",
|
|
count: result.credentials.length,
|
|
credentials: result.credentials,
|
|
},
|
|
null,
|
|
2,
|
|
)}\n`,
|
|
);
|
|
return;
|
|
}
|
|
printQaCredentialListTable(result.credentials);
|
|
if (opts.showSecrets && result.credentials.length > 0) {
|
|
process.stdout.write("\nPayloads:\n");
|
|
for (const credential of result.credentials) {
|
|
process.stdout.write(
|
|
`${credential.credentialId}: ${JSON.stringify(credential.payload ?? null)}\n`,
|
|
);
|
|
}
|
|
}
|
|
} catch (error) {
|
|
if (opts.json) {
|
|
writeQaCredentialCommandErrorJson("list", error);
|
|
process.exitCode = 1;
|
|
return;
|
|
}
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
export async function runQaCredentialsDoctorCommand(opts: {
|
|
actorId?: string;
|
|
endpointPrefix?: string;
|
|
json?: boolean;
|
|
siteUrl?: string;
|
|
}) {
|
|
const result = await diagnoseQaCredentialBroker({
|
|
actorId: opts.actorId,
|
|
endpointPrefix: opts.endpointPrefix,
|
|
siteUrl: opts.siteUrl,
|
|
});
|
|
if (opts.json) {
|
|
process.stdout.write(`${JSON.stringify(result, null, 2)}\n`);
|
|
} else {
|
|
printQaCredentialDoctorTable(result);
|
|
}
|
|
if (result.status === "fail") {
|
|
process.exitCode = 1;
|
|
}
|
|
}
|
|
|
|
export async function runQaLabUiCommand(opts: {
|
|
repoRoot?: string;
|
|
host?: string;
|
|
port?: number;
|
|
advertiseHost?: string;
|
|
advertisePort?: number;
|
|
controlUiUrl?: string;
|
|
controlUiProxyTarget?: string;
|
|
uiDistDir?: string;
|
|
autoKickoffTarget?: string;
|
|
embeddedGateway?: string;
|
|
sendKickoffOnStart?: boolean;
|
|
}) {
|
|
const repoRoot = path.resolve(opts.repoRoot ?? process.cwd());
|
|
const server = await startQaLabServer({
|
|
repoRoot,
|
|
host: opts.host,
|
|
port: Number.isFinite(opts.port) ? opts.port : undefined,
|
|
advertiseHost: opts.advertiseHost,
|
|
advertisePort: Number.isFinite(opts.advertisePort) ? opts.advertisePort : undefined,
|
|
controlUiUrl: opts.controlUiUrl,
|
|
controlUiProxyToken: process.env.OPENCLAW_QA_CONTROL_UI_PROXY_TOKEN,
|
|
controlUiProxyTarget: opts.controlUiProxyTarget,
|
|
uiDistDir: opts.uiDistDir,
|
|
autoKickoffTarget: opts.autoKickoffTarget,
|
|
embeddedGateway: opts.embeddedGateway,
|
|
sendKickoffOnStart: opts.sendKickoffOnStart,
|
|
});
|
|
await runInterruptibleServer("QA Lab UI", server);
|
|
}
|
|
|
|
export async function runQaDockerScaffoldCommand(opts: {
|
|
repoRoot?: string;
|
|
outputDir: string;
|
|
gatewayPort?: number;
|
|
qaLabPort?: number;
|
|
providerBaseUrl?: string;
|
|
image?: string;
|
|
usePrebuiltImage?: boolean;
|
|
bindUiDist?: boolean;
|
|
}) {
|
|
const repoRoot = path.resolve(opts.repoRoot ?? process.cwd());
|
|
const outputDir = resolveRepoRelativeOutputDir(repoRoot, opts.outputDir);
|
|
if (!outputDir) {
|
|
throw new Error("--output-dir is required.");
|
|
}
|
|
const result = await writeQaDockerHarnessFiles({
|
|
outputDir,
|
|
repoRoot,
|
|
gatewayPort: Number.isFinite(opts.gatewayPort) ? opts.gatewayPort : undefined,
|
|
qaLabPort: Number.isFinite(opts.qaLabPort) ? opts.qaLabPort : undefined,
|
|
providerBaseUrl: opts.providerBaseUrl,
|
|
imageName: opts.image,
|
|
usePrebuiltImage: opts.usePrebuiltImage,
|
|
bindUiDist: opts.bindUiDist,
|
|
});
|
|
process.stdout.write(`QA docker scaffold: ${result.outputDir}\n`);
|
|
}
|
|
|
|
export async function runQaDockerBuildImageCommand(opts: { repoRoot?: string; image?: string }) {
|
|
const repoRoot = path.resolve(opts.repoRoot ?? process.cwd());
|
|
const result = await buildQaDockerHarnessImage({
|
|
repoRoot,
|
|
imageName: opts.image,
|
|
});
|
|
process.stdout.write(`QA docker image: ${result.imageName}\n`);
|
|
}
|
|
|
|
export async function runQaDockerUpCommand(opts: {
|
|
repoRoot?: string;
|
|
outputDir?: string;
|
|
gatewayPort?: number;
|
|
qaLabPort?: number;
|
|
providerBaseUrl?: string;
|
|
image?: string;
|
|
usePrebuiltImage?: boolean;
|
|
bindUiDist?: boolean;
|
|
skipUiBuild?: boolean;
|
|
}) {
|
|
const repoRoot = path.resolve(opts.repoRoot ?? process.cwd());
|
|
const result = await runQaDockerUp({
|
|
repoRoot,
|
|
outputDir: resolveRepoRelativeOutputDir(repoRoot, opts.outputDir),
|
|
gatewayPort: Number.isFinite(opts.gatewayPort) ? opts.gatewayPort : undefined,
|
|
qaLabPort: Number.isFinite(opts.qaLabPort) ? opts.qaLabPort : undefined,
|
|
providerBaseUrl: opts.providerBaseUrl,
|
|
image: opts.image,
|
|
usePrebuiltImage: opts.usePrebuiltImage,
|
|
bindUiDist: opts.bindUiDist,
|
|
skipUiBuild: opts.skipUiBuild,
|
|
});
|
|
process.stdout.write(`QA docker dir: ${result.outputDir}\n`);
|
|
process.stdout.write(`QA Lab UI: ${result.qaLabUrl}\n`);
|
|
process.stdout.write(`Gateway UI: ${result.gatewayUrl}\n`);
|
|
process.stdout.write(`Stop: ${result.stopCommand}\n`);
|
|
}
|
|
|
|
export async function runQaProviderServerCommand(
|
|
providerMode: QaProviderMode,
|
|
opts: { host?: string; port?: number },
|
|
) {
|
|
const provider = getQaProvider(providerMode);
|
|
const standaloneCommand = provider.standaloneCommand;
|
|
if (!standaloneCommand) {
|
|
throw new Error(`QA provider "${providerMode}" does not expose a standalone server command.`);
|
|
}
|
|
const server = await startQaProviderServer(providerMode, {
|
|
host: opts.host,
|
|
port: Number.isFinite(opts.port) ? opts.port : undefined,
|
|
});
|
|
if (!server) {
|
|
throw new Error(`QA provider "${providerMode}" does not expose a standalone server command.`);
|
|
}
|
|
await runInterruptibleServer(standaloneCommand.serverLabel, server);
|
|
}
|
|
|
|
export const testing = {
|
|
resolveRepoRelativeOutputDir,
|
|
};
|
|
export { testing as __testing };
|