From cc7a209982500239fc2a35ebd1508d5e0af8bff2 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Tue, 28 Apr 2026 23:01:54 +0100 Subject: [PATCH] fix: normalize QA model refs for parity gates --- .github/workflows/openclaw-release-checks.yml | 2 +- .github/workflows/parity-gate.yml | 2 +- .../workflows/qa-live-transports-convex.yml | 2 +- extensions/qa-lab/src/cli.runtime.test.ts | 21 +++++++++++++++++++ extensions/qa-lab/src/cli.runtime.ts | 19 +++++++++++------ .../shared/live-transport-cli.runtime.test.ts | 20 ++++++++++++++++++ .../shared/live-transport-cli.runtime.ts | 9 ++++++-- .../qa-lab/src/qa-gateway-config.test.ts | 16 ++++++++++++++ extensions/qa-lab/src/qa-gateway-config.ts | 16 +++++++++++--- extensions/qa-lab/src/suite.ts | 15 +++++++++++-- 10 files changed, 106 insertions(+), 16 deletions(-) create mode 100644 extensions/qa-lab/src/live-transports/shared/live-transport-cli.runtime.test.ts diff --git a/.github/workflows/openclaw-release-checks.yml b/.github/workflows/openclaw-release-checks.yml index defe77c1be8..968b0ca8fcc 100644 --- a/.github/workflows/openclaw-release-checks.yml +++ b/.github/workflows/openclaw-release-checks.yml @@ -62,7 +62,7 @@ env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" NODE_VERSION: "24.x" PNPM_VERSION: "10.33.0" - OPENCLAW_CI_OPENAI_MODEL: ${{ vars.OPENCLAW_CI_OPENAI_MODEL }} + OPENCLAW_CI_OPENAI_MODEL: ${{ vars.OPENCLAW_CI_OPENAI_MODEL || 'openai/gpt-5.5' }} jobs: resolve_target: diff --git a/.github/workflows/parity-gate.yml b/.github/workflows/parity-gate.yml index 1792f71ba8b..f7f5a446a75 100644 --- a/.github/workflows/parity-gate.yml +++ b/.github/workflows/parity-gate.yml @@ -42,7 +42,7 @@ jobs: # followthrough gate that expects a fast post-approval read within a 30s # agent.wait timeout. QA_PARITY_CONCURRENCY: "1" - OPENCLAW_CI_OPENAI_MODEL: ${{ vars.OPENCLAW_CI_OPENAI_MODEL }} + OPENCLAW_CI_OPENAI_MODEL: ${{ vars.OPENCLAW_CI_OPENAI_MODEL || 'openai/gpt-5.5' }} OPENCLAW_QA_TRANSPORT_READY_TIMEOUT_MS: "180000" OPENAI_API_KEY: "" ANTHROPIC_API_KEY: "" diff --git a/.github/workflows/qa-live-transports-convex.yml b/.github/workflows/qa-live-transports-convex.yml index b206f1b8aa5..7651055203d 100644 --- a/.github/workflows/qa-live-transports-convex.yml +++ b/.github/workflows/qa-live-transports-convex.yml @@ -44,7 +44,7 @@ env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" NODE_VERSION: "24.x" PNPM_VERSION: "10.33.0" - OPENCLAW_CI_OPENAI_MODEL: ${{ vars.OPENCLAW_CI_OPENAI_MODEL }} + OPENCLAW_CI_OPENAI_MODEL: ${{ vars.OPENCLAW_CI_OPENAI_MODEL || 'openai/gpt-5.5' }} OPENCLAW_BUILD_PRIVATE_QA: "1" OPENCLAW_ENABLE_PRIVATE_QA_CLI: "1" diff --git a/extensions/qa-lab/src/cli.runtime.test.ts b/extensions/qa-lab/src/cli.runtime.test.ts index cd850c25b37..4753a7ed824 100644 --- a/extensions/qa-lab/src/cli.runtime.test.ts +++ b/extensions/qa-lab/src/cli.runtime.test.ts @@ -206,6 +206,27 @@ describe("qa cli runtime", () => { }); }); + it("drops blank suite model refs so provider defaults apply", async () => { + await runQaSuiteCommand({ + repoRoot: "/tmp/openclaw-repo", + providerMode: "mock-openai", + primaryModel: " ", + alternateModel: "", + scenarioIds: ["thread-memory-isolation"], + }); + + expect(runQaSuiteFromRuntime).toHaveBeenCalledWith({ + repoRoot: path.resolve("/tmp/openclaw-repo"), + outputDir: undefined, + transportId: "qa-channel", + providerMode: "mock-openai", + primaryModel: undefined, + alternateModel: undefined, + fastMode: undefined, + scenarioIds: ["thread-memory-isolation"], + }); + }); + it("resolves telegram qa repo-root-relative paths before dispatching", async () => { await runQaTelegramCommand({ repoRoot: "/tmp/openclaw-repo", diff --git a/extensions/qa-lab/src/cli.runtime.ts b/extensions/qa-lab/src/cli.runtime.ts index 8ae748d25a0..9d695e1239f 100644 --- a/extensions/qa-lab/src/cli.runtime.ts +++ b/extensions/qa-lab/src/cli.runtime.ts @@ -127,6 +127,11 @@ function parseQaPositiveIntegerOption(label: string, value: number | undefined) return Math.floor(value); } +function normalizeQaOptionalModelRef(input: string | undefined) { + const model = input?.trim(); + return model && model.length > 0 ? model : undefined; +} + async function readQaFailedScenarioCountFromSummary(summaryPath: string) { let summaryText: string; try { @@ -488,6 +493,8 @@ export async function runQaSuiteCommand(opts: { } const providerMode = normalizeQaProviderMode(opts.providerMode); const claudeCliAuthMode = parseQaCliBackendAuthMode(opts.cliAuthMode); + const primaryModel = normalizeQaOptionalModelRef(opts.primaryModel); + const alternateModel = normalizeQaOptionalModelRef(opts.alternateModel); if (opts.preflight === true && runner !== "host") { throw new Error("--preflight requires --runner host."); } @@ -510,8 +517,8 @@ export async function runQaSuiteCommand(opts: { outputDir: resolveRepoRelativeOutputDir(repoRoot, opts.outputDir), transportId, providerMode, - primaryModel: opts.primaryModel, - alternateModel: opts.alternateModel, + primaryModel, + alternateModel, fastMode: opts.fastMode, ...(thinkingDefault ? { thinkingDefault } : {}), allowFailures: true, @@ -542,8 +549,8 @@ export async function runQaSuiteCommand(opts: { repoRoot, transportId, providerMode, - primaryModel: opts.primaryModel, - alternateModel: opts.alternateModel, + primaryModel, + alternateModel, allowFailures, }); return; @@ -554,8 +561,8 @@ export async function runQaSuiteCommand(opts: { outputDir: resolveRepoRelativeOutputDir(repoRoot, opts.outputDir), transportId, providerMode, - primaryModel: opts.primaryModel, - alternateModel: opts.alternateModel, + primaryModel, + alternateModel, fastMode: opts.fastMode, ...(thinkingDefault ? { thinkingDefault } : {}), ...(claudeCliAuthMode ? { claudeCliAuthMode } : {}), diff --git a/extensions/qa-lab/src/live-transports/shared/live-transport-cli.runtime.test.ts b/extensions/qa-lab/src/live-transports/shared/live-transport-cli.runtime.test.ts new file mode 100644 index 00000000000..a699eee4a4e --- /dev/null +++ b/extensions/qa-lab/src/live-transports/shared/live-transport-cli.runtime.test.ts @@ -0,0 +1,20 @@ +import { describe, expect, it } from "vitest"; +import { resolveLiveTransportQaRunOptions } from "./live-transport-cli.runtime.js"; + +describe("resolveLiveTransportQaRunOptions", () => { + it("drops blank model refs so live transports can use provider defaults", () => { + expect( + resolveLiveTransportQaRunOptions({ + repoRoot: "/tmp/openclaw-repo", + providerMode: "live-frontier", + primaryModel: " ", + alternateModel: "", + }), + ).toMatchObject({ + repoRoot: "/tmp/openclaw-repo", + providerMode: "live-frontier", + primaryModel: undefined, + alternateModel: undefined, + }); + }); +}); diff --git a/extensions/qa-lab/src/live-transports/shared/live-transport-cli.runtime.ts b/extensions/qa-lab/src/live-transports/shared/live-transport-cli.runtime.ts index 5ab58e4e7b9..ed8eed64e73 100644 --- a/extensions/qa-lab/src/live-transports/shared/live-transport-cli.runtime.ts +++ b/extensions/qa-lab/src/live-transports/shared/live-transport-cli.runtime.ts @@ -5,6 +5,11 @@ import type { QaProviderMode } from "../../run-config.js"; import { normalizeQaProviderMode } from "../../run-config.js"; import type { LiveTransportQaCommandOptions } from "./live-transport-cli.js"; +function normalizeLiveTransportModelRef(input: string | undefined) { + const model = input?.trim(); + return model && model.length > 0 ? model : undefined; +} + export function resolveLiveTransportQaRunOptions( opts: LiveTransportQaCommandOptions, ): LiveTransportQaCommandOptions & { @@ -21,8 +26,8 @@ export function resolveLiveTransportQaRunOptions( opts.providerMode === undefined ? DEFAULT_QA_LIVE_PROVIDER_MODE : normalizeQaProviderMode(opts.providerMode), - primaryModel: opts.primaryModel, - alternateModel: opts.alternateModel, + primaryModel: normalizeLiveTransportModelRef(opts.primaryModel), + alternateModel: normalizeLiveTransportModelRef(opts.alternateModel), fastMode: opts.fastMode, allowFailures: opts.allowFailures, scenarioIds: opts.scenarioIds, diff --git a/extensions/qa-lab/src/qa-gateway-config.test.ts b/extensions/qa-lab/src/qa-gateway-config.test.ts index 48975eff2e4..415cff13878 100644 --- a/extensions/qa-lab/src/qa-gateway-config.test.ts +++ b/extensions/qa-lab/src/qa-gateway-config.test.ts @@ -109,6 +109,22 @@ describe("buildQaGatewayConfig", () => { expect(cfg.plugins?.allow).toEqual(["acpx", "memory-core"]); }); + it("falls back to provider defaults for blank model refs", () => { + const cfg = buildQaGatewayConfig({ + bind: "loopback", + gatewayPort: 18789, + gatewayToken: "token", + providerBaseUrl: "http://127.0.0.1:44080/v1", + workspaceDir: "/tmp/qa-workspace", + providerMode: "mock-openai", + primaryModel: " ", + alternateModel: "", + }); + + expect(getPrimaryModel(cfg.agents?.defaults?.model)).toBe("mock-openai/gpt-5.5"); + expect(cfg.agents?.defaults?.models).toHaveProperty("mock-openai/gpt-5.5-alt"); + }); + it("can wire AIMock as a separate mock provider lane", () => { const cfg = buildQaGatewayConfig({ bind: "loopback", diff --git a/extensions/qa-lab/src/qa-gateway-config.ts b/extensions/qa-lab/src/qa-gateway-config.ts index 66f639c8fa4..c7b7183ff5a 100644 --- a/extensions/qa-lab/src/qa-gateway-config.ts +++ b/extensions/qa-lab/src/qa-gateway-config.ts @@ -29,6 +29,11 @@ export function mergeQaControlUiAllowedOrigins(extraOrigins?: string[]) { return [...new Set([...DEFAULT_QA_CONTROL_UI_ALLOWED_ORIGINS, ...normalizedExtra])]; } +function normalizeQaGatewayModelRef(input: string | undefined, fallback: string) { + const model = input?.trim(); + return model && model.length > 0 ? model : fallback; +} + export function buildQaGatewayConfig(params: { bind: "loopback" | "lan"; gatewayPort: number; @@ -53,9 +58,14 @@ export function buildQaGatewayConfig(params: { const providerBaseUrl = params.providerBaseUrl ?? "http://127.0.0.1:44080/v1"; const providerMode = normalizeQaProviderMode(params.providerMode ?? DEFAULT_QA_PROVIDER_MODE); const provider = getQaProvider(providerMode); - const primaryModel = params.primaryModel ?? defaultQaModelForMode(providerMode); - const alternateModel = - params.alternateModel ?? defaultQaModelForMode(providerMode, { alternate: true }); + const primaryModel = normalizeQaGatewayModelRef( + params.primaryModel, + defaultQaModelForMode(providerMode), + ); + const alternateModel = normalizeQaGatewayModelRef( + params.alternateModel, + defaultQaModelForMode(providerMode, { alternate: true }), + ); const modelProviderIds = [primaryModel, alternateModel] .map((ref) => splitQaModelRef(ref)?.provider) .filter((provider): provider is string => Boolean(provider)); diff --git a/extensions/qa-lab/src/suite.ts b/extensions/qa-lab/src/suite.ts index b319bb42f49..bab656bcea5 100644 --- a/extensions/qa-lab/src/suite.ts +++ b/extensions/qa-lab/src/suite.ts @@ -273,6 +273,11 @@ function createQaSuiteReportNotes(params: { return params.transport.createReportNotes(params); } +function normalizeQaSuiteModelRef(input: string | undefined, fallback: string) { + const model = input?.trim(); + return model && model.length > 0 ? model : fallback; +} + export type QaSuiteSummaryJsonParams = { scenarios: QaSuiteScenarioResult[]; startedAt: Date; @@ -407,8 +412,14 @@ export async function runQaSuite(params?: QaSuiteRunParams): Promise