mirror of
https://github.com/openclaw/openclaw.git
synced 2026-05-06 08:30:42 +00:00
QA: reuse shared runtime model defaults
Route Matrix QA model selection through the qa-lab runtime surface so the live lane keeps the same preferred-model behavior as the rest of the QA host, including the Codex OAuth fallback. This removes the duplicated default table from qa-matrix, adds a narrow helper around the shared runtime seam, and locks the behavior with a focused test.
This commit is contained in:
@@ -3,6 +3,7 @@ export type { OpenClawConfig } from "openclaw/plugin-sdk/config-runtime";
|
||||
export { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";
|
||||
export { callGatewayFromCli } from "openclaw/plugin-sdk/browser-node-runtime";
|
||||
export type { PluginRuntime } from "openclaw/plugin-sdk/runtime-store";
|
||||
export { defaultQaRuntimeModelForMode } from "./model-selection.runtime.js";
|
||||
export {
|
||||
buildQaTarget,
|
||||
createQaBusThread,
|
||||
|
||||
@@ -1,31 +1,9 @@
|
||||
export type QaProviderMode = "mock-openai" | "live-frontier";
|
||||
export type QaProviderModeInput = QaProviderMode | "live-openai";
|
||||
|
||||
const DEFAULT_QA_MODELS = {
|
||||
"live-frontier": {
|
||||
primary: "openai/gpt-5.4",
|
||||
alternate: "anthropic/claude-sonnet-4-6",
|
||||
},
|
||||
"mock-openai": {
|
||||
primary: "mock-openai/gpt-5.4",
|
||||
alternate: "mock-openai/gpt-5.4-alt",
|
||||
},
|
||||
} as const satisfies Record<
|
||||
QaProviderMode,
|
||||
{
|
||||
primary: string;
|
||||
alternate: string;
|
||||
}
|
||||
>;
|
||||
|
||||
export function normalizeQaProviderMode(input: unknown): QaProviderMode {
|
||||
if (input === "mock-openai") {
|
||||
return "mock-openai";
|
||||
}
|
||||
return "live-frontier";
|
||||
}
|
||||
|
||||
export function defaultQaModelForMode(mode: QaProviderMode, alternate = false) {
|
||||
const preset = DEFAULT_QA_MODELS[normalizeQaProviderMode(mode)];
|
||||
return alternate ? preset.alternate : preset.primary;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const loadQaLabRuntimeModule = vi.hoisted(() => vi.fn());
|
||||
const defaultQaRuntimeModelForMode = vi.hoisted(() => vi.fn());
|
||||
|
||||
vi.mock("openclaw/plugin-sdk/qa-lab-runtime", () => ({
|
||||
loadQaLabRuntimeModule,
|
||||
}));
|
||||
|
||||
describe("matrix qa model selection", () => {
|
||||
beforeEach(() => {
|
||||
defaultQaRuntimeModelForMode.mockReset().mockImplementation((mode, options) =>
|
||||
options?.alternate ? `${mode}:alt` : `${mode}:primary`,
|
||||
);
|
||||
loadQaLabRuntimeModule.mockReset().mockReturnValue({
|
||||
defaultQaRuntimeModelForMode,
|
||||
});
|
||||
});
|
||||
|
||||
it("delegates default model selection through qa-lab runtime defaults", async () => {
|
||||
const { resolveMatrixQaModels } = await import("./model-selection.js");
|
||||
|
||||
expect(resolveMatrixQaModels({ providerMode: "live-openai" })).toEqual({
|
||||
providerMode: "live-frontier",
|
||||
primaryModel: "live-frontier:primary",
|
||||
alternateModel: "live-frontier:alt",
|
||||
});
|
||||
expect(defaultQaRuntimeModelForMode).toHaveBeenNthCalledWith(1, "live-frontier");
|
||||
expect(defaultQaRuntimeModelForMode).toHaveBeenNthCalledWith(2, "live-frontier", {
|
||||
alternate: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("preserves explicit model overrides", async () => {
|
||||
const { resolveMatrixQaModels } = await import("./model-selection.js");
|
||||
|
||||
expect(
|
||||
resolveMatrixQaModels({
|
||||
providerMode: "mock-openai",
|
||||
primaryModel: "custom-primary",
|
||||
alternateModel: "custom-alt",
|
||||
}),
|
||||
).toEqual({
|
||||
providerMode: "mock-openai",
|
||||
primaryModel: "custom-primary",
|
||||
alternateModel: "custom-alt",
|
||||
});
|
||||
expect(defaultQaRuntimeModelForMode).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
26
extensions/qa-matrix/src/runners/contract/model-selection.ts
Normal file
26
extensions/qa-matrix/src/runners/contract/model-selection.ts
Normal file
@@ -0,0 +1,26 @@
|
||||
import { loadQaLabRuntimeModule } from "openclaw/plugin-sdk/qa-lab-runtime";
|
||||
import { normalizeQaProviderMode, type QaProviderModeInput } from "../../run-config.js";
|
||||
|
||||
export type ResolvedMatrixQaModels = {
|
||||
providerMode: ReturnType<typeof normalizeQaProviderMode>;
|
||||
primaryModel: string;
|
||||
alternateModel: string;
|
||||
};
|
||||
|
||||
export function resolveMatrixQaModels(params: {
|
||||
providerMode?: QaProviderModeInput;
|
||||
primaryModel?: string;
|
||||
alternateModel?: string;
|
||||
}): ResolvedMatrixQaModels {
|
||||
const providerMode = normalizeQaProviderMode(params.providerMode ?? "live-frontier");
|
||||
const qaLabRuntime = loadQaLabRuntimeModule();
|
||||
return {
|
||||
providerMode,
|
||||
primaryModel:
|
||||
params.primaryModel?.trim() ||
|
||||
qaLabRuntime.defaultQaRuntimeModelForMode(providerMode),
|
||||
alternateModel:
|
||||
params.alternateModel?.trim() ||
|
||||
qaLabRuntime.defaultQaRuntimeModelForMode(providerMode, { alternate: true }),
|
||||
};
|
||||
}
|
||||
@@ -7,11 +7,7 @@ import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
|
||||
import { loadQaLabRuntimeModule } from "openclaw/plugin-sdk/qa-lab-runtime";
|
||||
import type { QaReportCheck } from "../../report.js";
|
||||
import { renderQaMarkdownReport } from "../../report.js";
|
||||
import {
|
||||
defaultQaModelForMode,
|
||||
normalizeQaProviderMode,
|
||||
type QaProviderModeInput,
|
||||
} from "../../run-config.js";
|
||||
import { type QaProviderModeInput } from "../../run-config.js";
|
||||
import {
|
||||
appendLiveLaneIssue,
|
||||
buildLiveLaneArtifactsError,
|
||||
@@ -31,6 +27,7 @@ import {
|
||||
type MatrixQaCanaryArtifact,
|
||||
type MatrixQaScenarioArtifacts,
|
||||
} from "./scenarios.js";
|
||||
import { resolveMatrixQaModels } from "./model-selection.js";
|
||||
|
||||
type MatrixQaGatewayChild = {
|
||||
call(
|
||||
@@ -308,9 +305,11 @@ export async function runMatrixQaLive(params: {
|
||||
path.join(repoRoot, ".artifacts", "qa-e2e", `matrix-${Date.now().toString(36)}`);
|
||||
await fs.mkdir(outputDir, { recursive: true });
|
||||
|
||||
const providerMode = normalizeQaProviderMode(params.providerMode ?? "live-frontier");
|
||||
const primaryModel = params.primaryModel?.trim() || defaultQaModelForMode(providerMode);
|
||||
const alternateModel = params.alternateModel?.trim() || defaultQaModelForMode(providerMode, true);
|
||||
const { providerMode, primaryModel, alternateModel } = resolveMatrixQaModels({
|
||||
providerMode: params.providerMode,
|
||||
primaryModel: params.primaryModel,
|
||||
alternateModel: params.alternateModel,
|
||||
});
|
||||
const sutAccountId = params.sutAccountId?.trim() || "sut";
|
||||
const scenarios = findMatrixQaScenarios(params.scenarioIds);
|
||||
const observedEvents: MatrixQaObservedEvent[] = [];
|
||||
@@ -592,5 +591,6 @@ export const __testing = {
|
||||
buildMatrixQaConfig,
|
||||
buildObservedEventsArtifact,
|
||||
isMatrixAccountReady,
|
||||
resolveMatrixQaModels,
|
||||
waitForMatrixChannelReady,
|
||||
};
|
||||
|
||||
@@ -1,6 +1,13 @@
|
||||
import { loadBundledPluginPublicSurfaceModuleSync } from "./facade-runtime.js";
|
||||
|
||||
type QaLabRuntimeSurface = {
|
||||
defaultQaRuntimeModelForMode: (
|
||||
mode: string,
|
||||
options?: {
|
||||
alternate?: boolean;
|
||||
preferredLiveModel?: string;
|
||||
},
|
||||
) => string;
|
||||
startQaLiveLaneGateway: (...args: unknown[]) => Promise<unknown>;
|
||||
};
|
||||
|
||||
|
||||
Reference in New Issue
Block a user