From 5f418609aac9ae93fbb4f78e573aa0377bba6e1a Mon Sep 17 00:00:00 2001 From: Gustavo Madeira Santana Date: Tue, 14 Apr 2026 13:08:48 -0400 Subject: [PATCH] QA: reuse shared runtime model defaults Route Matrix QA model selection through the qa-lab runtime surface so the live lane keeps the same preferred-model behavior as the rest of the QA host, including the Codex OAuth fallback. This removes the duplicated default table from qa-matrix, adds a narrow helper around the shared runtime seam, and locks the behavior with a focused test. --- extensions/qa-lab/src/runtime-api.ts | 1 + extensions/qa-matrix/src/run-config.ts | 22 -------- .../runners/contract/model-selection.test.ts | 50 +++++++++++++++++++ .../src/runners/contract/model-selection.ts | 26 ++++++++++ .../qa-matrix/src/runners/contract/runtime.ts | 16 +++--- src/plugin-sdk/qa-lab-runtime.ts | 7 +++ 6 files changed, 92 insertions(+), 30 deletions(-) create mode 100644 extensions/qa-matrix/src/runners/contract/model-selection.test.ts create mode 100644 extensions/qa-matrix/src/runners/contract/model-selection.ts diff --git a/extensions/qa-lab/src/runtime-api.ts b/extensions/qa-lab/src/runtime-api.ts index da7390ce5df..602832e034c 100644 --- a/extensions/qa-lab/src/runtime-api.ts +++ b/extensions/qa-lab/src/runtime-api.ts @@ -3,6 +3,7 @@ export type { OpenClawConfig } from "openclaw/plugin-sdk/config-runtime"; export { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry"; export { callGatewayFromCli } from "openclaw/plugin-sdk/browser-node-runtime"; export type { PluginRuntime } from "openclaw/plugin-sdk/runtime-store"; +export { defaultQaRuntimeModelForMode } from "./model-selection.runtime.js"; export { buildQaTarget, createQaBusThread, diff --git a/extensions/qa-matrix/src/run-config.ts b/extensions/qa-matrix/src/run-config.ts index 1a05f3793aa..36ea7f59efc 100644 --- a/extensions/qa-matrix/src/run-config.ts +++ b/extensions/qa-matrix/src/run-config.ts @@ -1,31 +1,9 @@ export type QaProviderMode = "mock-openai" | "live-frontier"; export type QaProviderModeInput = QaProviderMode | "live-openai"; -const DEFAULT_QA_MODELS = { - "live-frontier": { - primary: "openai/gpt-5.4", - alternate: "anthropic/claude-sonnet-4-6", - }, - "mock-openai": { - primary: "mock-openai/gpt-5.4", - alternate: "mock-openai/gpt-5.4-alt", - }, -} as const satisfies Record< - QaProviderMode, - { - primary: string; - alternate: string; - } ->; - export function normalizeQaProviderMode(input: unknown): QaProviderMode { if (input === "mock-openai") { return "mock-openai"; } return "live-frontier"; } - -export function defaultQaModelForMode(mode: QaProviderMode, alternate = false) { - const preset = DEFAULT_QA_MODELS[normalizeQaProviderMode(mode)]; - return alternate ? preset.alternate : preset.primary; -} diff --git a/extensions/qa-matrix/src/runners/contract/model-selection.test.ts b/extensions/qa-matrix/src/runners/contract/model-selection.test.ts new file mode 100644 index 00000000000..5a49128416b --- /dev/null +++ b/extensions/qa-matrix/src/runners/contract/model-selection.test.ts @@ -0,0 +1,50 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const loadQaLabRuntimeModule = vi.hoisted(() => vi.fn()); +const defaultQaRuntimeModelForMode = vi.hoisted(() => vi.fn()); + +vi.mock("openclaw/plugin-sdk/qa-lab-runtime", () => ({ + loadQaLabRuntimeModule, +})); + +describe("matrix qa model selection", () => { + beforeEach(() => { + defaultQaRuntimeModelForMode.mockReset().mockImplementation((mode, options) => + options?.alternate ? `${mode}:alt` : `${mode}:primary`, + ); + loadQaLabRuntimeModule.mockReset().mockReturnValue({ + defaultQaRuntimeModelForMode, + }); + }); + + it("delegates default model selection through qa-lab runtime defaults", async () => { + const { resolveMatrixQaModels } = await import("./model-selection.js"); + + expect(resolveMatrixQaModels({ providerMode: "live-openai" })).toEqual({ + providerMode: "live-frontier", + primaryModel: "live-frontier:primary", + alternateModel: "live-frontier:alt", + }); + expect(defaultQaRuntimeModelForMode).toHaveBeenNthCalledWith(1, "live-frontier"); + expect(defaultQaRuntimeModelForMode).toHaveBeenNthCalledWith(2, "live-frontier", { + alternate: true, + }); + }); + + it("preserves explicit model overrides", async () => { + const { resolveMatrixQaModels } = await import("./model-selection.js"); + + expect( + resolveMatrixQaModels({ + providerMode: "mock-openai", + primaryModel: "custom-primary", + alternateModel: "custom-alt", + }), + ).toEqual({ + providerMode: "mock-openai", + primaryModel: "custom-primary", + alternateModel: "custom-alt", + }); + expect(defaultQaRuntimeModelForMode).not.toHaveBeenCalled(); + }); +}); diff --git a/extensions/qa-matrix/src/runners/contract/model-selection.ts b/extensions/qa-matrix/src/runners/contract/model-selection.ts new file mode 100644 index 00000000000..1f3d4652be8 --- /dev/null +++ b/extensions/qa-matrix/src/runners/contract/model-selection.ts @@ -0,0 +1,26 @@ +import { loadQaLabRuntimeModule } from "openclaw/plugin-sdk/qa-lab-runtime"; +import { normalizeQaProviderMode, type QaProviderModeInput } from "../../run-config.js"; + +export type ResolvedMatrixQaModels = { + providerMode: ReturnType; + primaryModel: string; + alternateModel: string; +}; + +export function resolveMatrixQaModels(params: { + providerMode?: QaProviderModeInput; + primaryModel?: string; + alternateModel?: string; +}): ResolvedMatrixQaModels { + const providerMode = normalizeQaProviderMode(params.providerMode ?? "live-frontier"); + const qaLabRuntime = loadQaLabRuntimeModule(); + return { + providerMode, + primaryModel: + params.primaryModel?.trim() || + qaLabRuntime.defaultQaRuntimeModelForMode(providerMode), + alternateModel: + params.alternateModel?.trim() || + qaLabRuntime.defaultQaRuntimeModelForMode(providerMode, { alternate: true }), + }; +} diff --git a/extensions/qa-matrix/src/runners/contract/runtime.ts b/extensions/qa-matrix/src/runners/contract/runtime.ts index 87904fd6f1e..12a58f3ff28 100644 --- a/extensions/qa-matrix/src/runners/contract/runtime.ts +++ b/extensions/qa-matrix/src/runners/contract/runtime.ts @@ -7,11 +7,7 @@ import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime"; import { loadQaLabRuntimeModule } from "openclaw/plugin-sdk/qa-lab-runtime"; import type { QaReportCheck } from "../../report.js"; import { renderQaMarkdownReport } from "../../report.js"; -import { - defaultQaModelForMode, - normalizeQaProviderMode, - type QaProviderModeInput, -} from "../../run-config.js"; +import { type QaProviderModeInput } from "../../run-config.js"; import { appendLiveLaneIssue, buildLiveLaneArtifactsError, @@ -31,6 +27,7 @@ import { type MatrixQaCanaryArtifact, type MatrixQaScenarioArtifacts, } from "./scenarios.js"; +import { resolveMatrixQaModels } from "./model-selection.js"; type MatrixQaGatewayChild = { call( @@ -308,9 +305,11 @@ export async function runMatrixQaLive(params: { path.join(repoRoot, ".artifacts", "qa-e2e", `matrix-${Date.now().toString(36)}`); await fs.mkdir(outputDir, { recursive: true }); - const providerMode = normalizeQaProviderMode(params.providerMode ?? "live-frontier"); - const primaryModel = params.primaryModel?.trim() || defaultQaModelForMode(providerMode); - const alternateModel = params.alternateModel?.trim() || defaultQaModelForMode(providerMode, true); + const { providerMode, primaryModel, alternateModel } = resolveMatrixQaModels({ + providerMode: params.providerMode, + primaryModel: params.primaryModel, + alternateModel: params.alternateModel, + }); const sutAccountId = params.sutAccountId?.trim() || "sut"; const scenarios = findMatrixQaScenarios(params.scenarioIds); const observedEvents: MatrixQaObservedEvent[] = []; @@ -592,5 +591,6 @@ export const __testing = { buildMatrixQaConfig, buildObservedEventsArtifact, isMatrixAccountReady, + resolveMatrixQaModels, waitForMatrixChannelReady, }; diff --git a/src/plugin-sdk/qa-lab-runtime.ts b/src/plugin-sdk/qa-lab-runtime.ts index 93dc7c51834..6b003f3e1ca 100644 --- a/src/plugin-sdk/qa-lab-runtime.ts +++ b/src/plugin-sdk/qa-lab-runtime.ts @@ -1,6 +1,13 @@ import { loadBundledPluginPublicSurfaceModuleSync } from "./facade-runtime.js"; type QaLabRuntimeSurface = { + defaultQaRuntimeModelForMode: ( + mode: string, + options?: { + alternate?: boolean; + preferredLiveModel?: string; + }, + ) => string; startQaLiveLaneGateway: (...args: unknown[]) => Promise; };