fix: normalize QA model refs for parity gates

This commit is contained in:
Peter Steinberger
2026-04-28 23:01:54 +01:00
parent 5ef6e82685
commit cc7a209982
10 changed files with 106 additions and 16 deletions

View File

@@ -62,7 +62,7 @@ env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
NODE_VERSION: "24.x"
PNPM_VERSION: "10.33.0"
OPENCLAW_CI_OPENAI_MODEL: ${{ vars.OPENCLAW_CI_OPENAI_MODEL }}
OPENCLAW_CI_OPENAI_MODEL: ${{ vars.OPENCLAW_CI_OPENAI_MODEL || 'openai/gpt-5.5' }}
jobs:
resolve_target:

View File

@@ -42,7 +42,7 @@ jobs:
# followthrough gate that expects a fast post-approval read within a 30s
# agent.wait timeout.
QA_PARITY_CONCURRENCY: "1"
OPENCLAW_CI_OPENAI_MODEL: ${{ vars.OPENCLAW_CI_OPENAI_MODEL }}
OPENCLAW_CI_OPENAI_MODEL: ${{ vars.OPENCLAW_CI_OPENAI_MODEL || 'openai/gpt-5.5' }}
OPENCLAW_QA_TRANSPORT_READY_TIMEOUT_MS: "180000"
OPENAI_API_KEY: ""
ANTHROPIC_API_KEY: ""

View File

@@ -44,7 +44,7 @@ env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
NODE_VERSION: "24.x"
PNPM_VERSION: "10.33.0"
OPENCLAW_CI_OPENAI_MODEL: ${{ vars.OPENCLAW_CI_OPENAI_MODEL }}
OPENCLAW_CI_OPENAI_MODEL: ${{ vars.OPENCLAW_CI_OPENAI_MODEL || 'openai/gpt-5.5' }}
OPENCLAW_BUILD_PRIVATE_QA: "1"
OPENCLAW_ENABLE_PRIVATE_QA_CLI: "1"

View File

@@ -206,6 +206,27 @@ describe("qa cli runtime", () => {
});
});
it("drops blank suite model refs so provider defaults apply", async () => {
await runQaSuiteCommand({
repoRoot: "/tmp/openclaw-repo",
providerMode: "mock-openai",
primaryModel: " ",
alternateModel: "",
scenarioIds: ["thread-memory-isolation"],
});
expect(runQaSuiteFromRuntime).toHaveBeenCalledWith({
repoRoot: path.resolve("/tmp/openclaw-repo"),
outputDir: undefined,
transportId: "qa-channel",
providerMode: "mock-openai",
primaryModel: undefined,
alternateModel: undefined,
fastMode: undefined,
scenarioIds: ["thread-memory-isolation"],
});
});
it("resolves telegram qa repo-root-relative paths before dispatching", async () => {
await runQaTelegramCommand({
repoRoot: "/tmp/openclaw-repo",

View File

@@ -127,6 +127,11 @@ function parseQaPositiveIntegerOption(label: string, value: number | undefined)
return Math.floor(value);
}
function normalizeQaOptionalModelRef(input: string | undefined) {
const model = input?.trim();
return model && model.length > 0 ? model : undefined;
}
async function readQaFailedScenarioCountFromSummary(summaryPath: string) {
let summaryText: string;
try {
@@ -488,6 +493,8 @@ export async function runQaSuiteCommand(opts: {
}
const providerMode = normalizeQaProviderMode(opts.providerMode);
const claudeCliAuthMode = parseQaCliBackendAuthMode(opts.cliAuthMode);
const primaryModel = normalizeQaOptionalModelRef(opts.primaryModel);
const alternateModel = normalizeQaOptionalModelRef(opts.alternateModel);
if (opts.preflight === true && runner !== "host") {
throw new Error("--preflight requires --runner host.");
}
@@ -510,8 +517,8 @@ export async function runQaSuiteCommand(opts: {
outputDir: resolveRepoRelativeOutputDir(repoRoot, opts.outputDir),
transportId,
providerMode,
primaryModel: opts.primaryModel,
alternateModel: opts.alternateModel,
primaryModel,
alternateModel,
fastMode: opts.fastMode,
...(thinkingDefault ? { thinkingDefault } : {}),
allowFailures: true,
@@ -542,8 +549,8 @@ export async function runQaSuiteCommand(opts: {
repoRoot,
transportId,
providerMode,
primaryModel: opts.primaryModel,
alternateModel: opts.alternateModel,
primaryModel,
alternateModel,
allowFailures,
});
return;
@@ -554,8 +561,8 @@ export async function runQaSuiteCommand(opts: {
outputDir: resolveRepoRelativeOutputDir(repoRoot, opts.outputDir),
transportId,
providerMode,
primaryModel: opts.primaryModel,
alternateModel: opts.alternateModel,
primaryModel,
alternateModel,
fastMode: opts.fastMode,
...(thinkingDefault ? { thinkingDefault } : {}),
...(claudeCliAuthMode ? { claudeCliAuthMode } : {}),

View File

@@ -0,0 +1,20 @@
import { describe, expect, it } from "vitest";
import { resolveLiveTransportQaRunOptions } from "./live-transport-cli.runtime.js";
describe("resolveLiveTransportQaRunOptions", () => {
it("drops blank model refs so live transports can use provider defaults", () => {
expect(
resolveLiveTransportQaRunOptions({
repoRoot: "/tmp/openclaw-repo",
providerMode: "live-frontier",
primaryModel: " ",
alternateModel: "",
}),
).toMatchObject({
repoRoot: "/tmp/openclaw-repo",
providerMode: "live-frontier",
primaryModel: undefined,
alternateModel: undefined,
});
});
});

View File

@@ -5,6 +5,11 @@ import type { QaProviderMode } from "../../run-config.js";
import { normalizeQaProviderMode } from "../../run-config.js";
import type { LiveTransportQaCommandOptions } from "./live-transport-cli.js";
function normalizeLiveTransportModelRef(input: string | undefined) {
const model = input?.trim();
return model && model.length > 0 ? model : undefined;
}
export function resolveLiveTransportQaRunOptions(
opts: LiveTransportQaCommandOptions,
): LiveTransportQaCommandOptions & {
@@ -21,8 +26,8 @@ export function resolveLiveTransportQaRunOptions(
opts.providerMode === undefined
? DEFAULT_QA_LIVE_PROVIDER_MODE
: normalizeQaProviderMode(opts.providerMode),
primaryModel: opts.primaryModel,
alternateModel: opts.alternateModel,
primaryModel: normalizeLiveTransportModelRef(opts.primaryModel),
alternateModel: normalizeLiveTransportModelRef(opts.alternateModel),
fastMode: opts.fastMode,
allowFailures: opts.allowFailures,
scenarioIds: opts.scenarioIds,

View File

@@ -109,6 +109,22 @@ describe("buildQaGatewayConfig", () => {
expect(cfg.plugins?.allow).toEqual(["acpx", "memory-core"]);
});
it("falls back to provider defaults for blank model refs", () => {
const cfg = buildQaGatewayConfig({
bind: "loopback",
gatewayPort: 18789,
gatewayToken: "token",
providerBaseUrl: "http://127.0.0.1:44080/v1",
workspaceDir: "/tmp/qa-workspace",
providerMode: "mock-openai",
primaryModel: " ",
alternateModel: "",
});
expect(getPrimaryModel(cfg.agents?.defaults?.model)).toBe("mock-openai/gpt-5.5");
expect(cfg.agents?.defaults?.models).toHaveProperty("mock-openai/gpt-5.5-alt");
});
it("can wire AIMock as a separate mock provider lane", () => {
const cfg = buildQaGatewayConfig({
bind: "loopback",

View File

@@ -29,6 +29,11 @@ export function mergeQaControlUiAllowedOrigins(extraOrigins?: string[]) {
return [...new Set([...DEFAULT_QA_CONTROL_UI_ALLOWED_ORIGINS, ...normalizedExtra])];
}
function normalizeQaGatewayModelRef(input: string | undefined, fallback: string) {
const model = input?.trim();
return model && model.length > 0 ? model : fallback;
}
export function buildQaGatewayConfig(params: {
bind: "loopback" | "lan";
gatewayPort: number;
@@ -53,9 +58,14 @@ export function buildQaGatewayConfig(params: {
const providerBaseUrl = params.providerBaseUrl ?? "http://127.0.0.1:44080/v1";
const providerMode = normalizeQaProviderMode(params.providerMode ?? DEFAULT_QA_PROVIDER_MODE);
const provider = getQaProvider(providerMode);
const primaryModel = params.primaryModel ?? defaultQaModelForMode(providerMode);
const alternateModel =
params.alternateModel ?? defaultQaModelForMode(providerMode, { alternate: true });
const primaryModel = normalizeQaGatewayModelRef(
params.primaryModel,
defaultQaModelForMode(providerMode),
);
const alternateModel = normalizeQaGatewayModelRef(
params.alternateModel,
defaultQaModelForMode(providerMode, { alternate: true }),
);
const modelProviderIds = [primaryModel, alternateModel]
.map((ref) => splitQaModelRef(ref)?.provider)
.filter((provider): provider is string => Boolean(provider));

View File

@@ -273,6 +273,11 @@ function createQaSuiteReportNotes(params: {
return params.transport.createReportNotes(params);
}
function normalizeQaSuiteModelRef(input: string | undefined, fallback: string) {
const model = input?.trim();
return model && model.length > 0 ? model : fallback;
}
export type QaSuiteSummaryJsonParams = {
scenarios: QaSuiteScenarioResult[];
startedAt: Date;
@@ -407,8 +412,14 @@ export async function runQaSuite(params?: QaSuiteRunParams): Promise<QaSuiteResu
params?.providerMode ?? DEFAULT_QA_LIVE_PROVIDER_MODE,
);
const transportId = normalizeQaTransportId(params?.transportId);
const primaryModel = params?.primaryModel ?? defaultQaModelForMode(providerMode);
const alternateModel = params?.alternateModel ?? defaultQaModelForMode(providerMode, true);
const primaryModel = normalizeQaSuiteModelRef(
params?.primaryModel,
defaultQaModelForMode(providerMode),
);
const alternateModel = normalizeQaSuiteModelRef(
params?.alternateModel,
defaultQaModelForMode(providerMode, true),
);
const fastMode =
typeof params?.fastMode === "boolean"
? params.fastMode