QA: reuse shared runtime model defaults

Route Matrix QA model selection through the qa-lab runtime surface so the live
lane keeps the same preferred-model behavior as the rest of the QA host,
including the Codex OAuth fallback.

This removes the duplicated default table from qa-matrix, adds a narrow helper
around the shared runtime seam, and locks the behavior with a focused test.
This commit is contained in:
Gustavo Madeira Santana
2026-04-14 13:08:48 -04:00
parent 2a20f8261f
commit 5f418609aa
6 changed files with 92 additions and 30 deletions

View File

@@ -3,6 +3,7 @@ export type { OpenClawConfig } from "openclaw/plugin-sdk/config-runtime";
export { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";
export { callGatewayFromCli } from "openclaw/plugin-sdk/browser-node-runtime";
export type { PluginRuntime } from "openclaw/plugin-sdk/runtime-store";
export { defaultQaRuntimeModelForMode } from "./model-selection.runtime.js";
export {
buildQaTarget,
createQaBusThread,

View File

@@ -1,31 +1,9 @@
export type QaProviderMode = "mock-openai" | "live-frontier";
export type QaProviderModeInput = QaProviderMode | "live-openai";
const DEFAULT_QA_MODELS = {
"live-frontier": {
primary: "openai/gpt-5.4",
alternate: "anthropic/claude-sonnet-4-6",
},
"mock-openai": {
primary: "mock-openai/gpt-5.4",
alternate: "mock-openai/gpt-5.4-alt",
},
} as const satisfies Record<
QaProviderMode,
{
primary: string;
alternate: string;
}
>;
export function normalizeQaProviderMode(input: unknown): QaProviderMode {
if (input === "mock-openai") {
return "mock-openai";
}
return "live-frontier";
}
export function defaultQaModelForMode(mode: QaProviderMode, alternate = false) {
const preset = DEFAULT_QA_MODELS[normalizeQaProviderMode(mode)];
return alternate ? preset.alternate : preset.primary;
}

View File

@@ -0,0 +1,50 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
const loadQaLabRuntimeModule = vi.hoisted(() => vi.fn());
const defaultQaRuntimeModelForMode = vi.hoisted(() => vi.fn());
vi.mock("openclaw/plugin-sdk/qa-lab-runtime", () => ({
loadQaLabRuntimeModule,
}));
describe("matrix qa model selection", () => {
beforeEach(() => {
defaultQaRuntimeModelForMode.mockReset().mockImplementation((mode, options) =>
options?.alternate ? `${mode}:alt` : `${mode}:primary`,
);
loadQaLabRuntimeModule.mockReset().mockReturnValue({
defaultQaRuntimeModelForMode,
});
});
it("delegates default model selection through qa-lab runtime defaults", async () => {
const { resolveMatrixQaModels } = await import("./model-selection.js");
expect(resolveMatrixQaModels({ providerMode: "live-openai" })).toEqual({
providerMode: "live-frontier",
primaryModel: "live-frontier:primary",
alternateModel: "live-frontier:alt",
});
expect(defaultQaRuntimeModelForMode).toHaveBeenNthCalledWith(1, "live-frontier");
expect(defaultQaRuntimeModelForMode).toHaveBeenNthCalledWith(2, "live-frontier", {
alternate: true,
});
});
it("preserves explicit model overrides", async () => {
const { resolveMatrixQaModels } = await import("./model-selection.js");
expect(
resolveMatrixQaModels({
providerMode: "mock-openai",
primaryModel: "custom-primary",
alternateModel: "custom-alt",
}),
).toEqual({
providerMode: "mock-openai",
primaryModel: "custom-primary",
alternateModel: "custom-alt",
});
expect(defaultQaRuntimeModelForMode).not.toHaveBeenCalled();
});
});

View File

@@ -0,0 +1,26 @@
import { loadQaLabRuntimeModule } from "openclaw/plugin-sdk/qa-lab-runtime";
import { normalizeQaProviderMode, type QaProviderModeInput } from "../../run-config.js";
export type ResolvedMatrixQaModels = {
providerMode: ReturnType<typeof normalizeQaProviderMode>;
primaryModel: string;
alternateModel: string;
};
export function resolveMatrixQaModels(params: {
providerMode?: QaProviderModeInput;
primaryModel?: string;
alternateModel?: string;
}): ResolvedMatrixQaModels {
const providerMode = normalizeQaProviderMode(params.providerMode ?? "live-frontier");
const qaLabRuntime = loadQaLabRuntimeModule();
return {
providerMode,
primaryModel:
params.primaryModel?.trim() ||
qaLabRuntime.defaultQaRuntimeModelForMode(providerMode),
alternateModel:
params.alternateModel?.trim() ||
qaLabRuntime.defaultQaRuntimeModelForMode(providerMode, { alternate: true }),
};
}

View File

@@ -7,11 +7,7 @@ import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
import { loadQaLabRuntimeModule } from "openclaw/plugin-sdk/qa-lab-runtime";
import type { QaReportCheck } from "../../report.js";
import { renderQaMarkdownReport } from "../../report.js";
import {
defaultQaModelForMode,
normalizeQaProviderMode,
type QaProviderModeInput,
} from "../../run-config.js";
import { type QaProviderModeInput } from "../../run-config.js";
import {
appendLiveLaneIssue,
buildLiveLaneArtifactsError,
@@ -31,6 +27,7 @@ import {
type MatrixQaCanaryArtifact,
type MatrixQaScenarioArtifacts,
} from "./scenarios.js";
import { resolveMatrixQaModels } from "./model-selection.js";
type MatrixQaGatewayChild = {
call(
@@ -308,9 +305,11 @@ export async function runMatrixQaLive(params: {
path.join(repoRoot, ".artifacts", "qa-e2e", `matrix-${Date.now().toString(36)}`);
await fs.mkdir(outputDir, { recursive: true });
const providerMode = normalizeQaProviderMode(params.providerMode ?? "live-frontier");
const primaryModel = params.primaryModel?.trim() || defaultQaModelForMode(providerMode);
const alternateModel = params.alternateModel?.trim() || defaultQaModelForMode(providerMode, true);
const { providerMode, primaryModel, alternateModel } = resolveMatrixQaModels({
providerMode: params.providerMode,
primaryModel: params.primaryModel,
alternateModel: params.alternateModel,
});
const sutAccountId = params.sutAccountId?.trim() || "sut";
const scenarios = findMatrixQaScenarios(params.scenarioIds);
const observedEvents: MatrixQaObservedEvent[] = [];
@@ -592,5 +591,6 @@ export const __testing = {
buildMatrixQaConfig,
buildObservedEventsArtifact,
isMatrixAccountReady,
resolveMatrixQaModels,
waitForMatrixChannelReady,
};

View File

@@ -1,6 +1,13 @@
import { loadBundledPluginPublicSurfaceModuleSync } from "./facade-runtime.js";
type QaLabRuntimeSurface = {
defaultQaRuntimeModelForMode: (
mode: string,
options?: {
alternate?: boolean;
preferredLiveModel?: string;
},
) => string;
startQaLiveLaneGateway: (...args: unknown[]) => Promise<unknown>;
};