mirror of
https://github.com/openclaw/openclaw.git
synced 2026-05-06 06:20:43 +00:00
fix: normalize QA model refs for parity gates
This commit is contained in:
@@ -62,7 +62,7 @@ env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
NODE_VERSION: "24.x"
|
||||
PNPM_VERSION: "10.33.0"
|
||||
OPENCLAW_CI_OPENAI_MODEL: ${{ vars.OPENCLAW_CI_OPENAI_MODEL }}
|
||||
OPENCLAW_CI_OPENAI_MODEL: ${{ vars.OPENCLAW_CI_OPENAI_MODEL || 'openai/gpt-5.5' }}
|
||||
|
||||
jobs:
|
||||
resolve_target:
|
||||
|
||||
2
.github/workflows/parity-gate.yml
vendored
2
.github/workflows/parity-gate.yml
vendored
@@ -42,7 +42,7 @@ jobs:
|
||||
# followthrough gate that expects a fast post-approval read within a 30s
|
||||
# agent.wait timeout.
|
||||
QA_PARITY_CONCURRENCY: "1"
|
||||
OPENCLAW_CI_OPENAI_MODEL: ${{ vars.OPENCLAW_CI_OPENAI_MODEL }}
|
||||
OPENCLAW_CI_OPENAI_MODEL: ${{ vars.OPENCLAW_CI_OPENAI_MODEL || 'openai/gpt-5.5' }}
|
||||
OPENCLAW_QA_TRANSPORT_READY_TIMEOUT_MS: "180000"
|
||||
OPENAI_API_KEY: ""
|
||||
ANTHROPIC_API_KEY: ""
|
||||
|
||||
@@ -44,7 +44,7 @@ env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
NODE_VERSION: "24.x"
|
||||
PNPM_VERSION: "10.33.0"
|
||||
OPENCLAW_CI_OPENAI_MODEL: ${{ vars.OPENCLAW_CI_OPENAI_MODEL }}
|
||||
OPENCLAW_CI_OPENAI_MODEL: ${{ vars.OPENCLAW_CI_OPENAI_MODEL || 'openai/gpt-5.5' }}
|
||||
OPENCLAW_BUILD_PRIVATE_QA: "1"
|
||||
OPENCLAW_ENABLE_PRIVATE_QA_CLI: "1"
|
||||
|
||||
|
||||
@@ -206,6 +206,27 @@ describe("qa cli runtime", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("drops blank suite model refs so provider defaults apply", async () => {
|
||||
await runQaSuiteCommand({
|
||||
repoRoot: "/tmp/openclaw-repo",
|
||||
providerMode: "mock-openai",
|
||||
primaryModel: " ",
|
||||
alternateModel: "",
|
||||
scenarioIds: ["thread-memory-isolation"],
|
||||
});
|
||||
|
||||
expect(runQaSuiteFromRuntime).toHaveBeenCalledWith({
|
||||
repoRoot: path.resolve("/tmp/openclaw-repo"),
|
||||
outputDir: undefined,
|
||||
transportId: "qa-channel",
|
||||
providerMode: "mock-openai",
|
||||
primaryModel: undefined,
|
||||
alternateModel: undefined,
|
||||
fastMode: undefined,
|
||||
scenarioIds: ["thread-memory-isolation"],
|
||||
});
|
||||
});
|
||||
|
||||
it("resolves telegram qa repo-root-relative paths before dispatching", async () => {
|
||||
await runQaTelegramCommand({
|
||||
repoRoot: "/tmp/openclaw-repo",
|
||||
|
||||
@@ -127,6 +127,11 @@ function parseQaPositiveIntegerOption(label: string, value: number | undefined)
|
||||
return Math.floor(value);
|
||||
}
|
||||
|
||||
function normalizeQaOptionalModelRef(input: string | undefined) {
|
||||
const model = input?.trim();
|
||||
return model && model.length > 0 ? model : undefined;
|
||||
}
|
||||
|
||||
async function readQaFailedScenarioCountFromSummary(summaryPath: string) {
|
||||
let summaryText: string;
|
||||
try {
|
||||
@@ -488,6 +493,8 @@ export async function runQaSuiteCommand(opts: {
|
||||
}
|
||||
const providerMode = normalizeQaProviderMode(opts.providerMode);
|
||||
const claudeCliAuthMode = parseQaCliBackendAuthMode(opts.cliAuthMode);
|
||||
const primaryModel = normalizeQaOptionalModelRef(opts.primaryModel);
|
||||
const alternateModel = normalizeQaOptionalModelRef(opts.alternateModel);
|
||||
if (opts.preflight === true && runner !== "host") {
|
||||
throw new Error("--preflight requires --runner host.");
|
||||
}
|
||||
@@ -510,8 +517,8 @@ export async function runQaSuiteCommand(opts: {
|
||||
outputDir: resolveRepoRelativeOutputDir(repoRoot, opts.outputDir),
|
||||
transportId,
|
||||
providerMode,
|
||||
primaryModel: opts.primaryModel,
|
||||
alternateModel: opts.alternateModel,
|
||||
primaryModel,
|
||||
alternateModel,
|
||||
fastMode: opts.fastMode,
|
||||
...(thinkingDefault ? { thinkingDefault } : {}),
|
||||
allowFailures: true,
|
||||
@@ -542,8 +549,8 @@ export async function runQaSuiteCommand(opts: {
|
||||
repoRoot,
|
||||
transportId,
|
||||
providerMode,
|
||||
primaryModel: opts.primaryModel,
|
||||
alternateModel: opts.alternateModel,
|
||||
primaryModel,
|
||||
alternateModel,
|
||||
allowFailures,
|
||||
});
|
||||
return;
|
||||
@@ -554,8 +561,8 @@ export async function runQaSuiteCommand(opts: {
|
||||
outputDir: resolveRepoRelativeOutputDir(repoRoot, opts.outputDir),
|
||||
transportId,
|
||||
providerMode,
|
||||
primaryModel: opts.primaryModel,
|
||||
alternateModel: opts.alternateModel,
|
||||
primaryModel,
|
||||
alternateModel,
|
||||
fastMode: opts.fastMode,
|
||||
...(thinkingDefault ? { thinkingDefault } : {}),
|
||||
...(claudeCliAuthMode ? { claudeCliAuthMode } : {}),
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { resolveLiveTransportQaRunOptions } from "./live-transport-cli.runtime.js";
|
||||
|
||||
describe("resolveLiveTransportQaRunOptions", () => {
|
||||
it("drops blank model refs so live transports can use provider defaults", () => {
|
||||
expect(
|
||||
resolveLiveTransportQaRunOptions({
|
||||
repoRoot: "/tmp/openclaw-repo",
|
||||
providerMode: "live-frontier",
|
||||
primaryModel: " ",
|
||||
alternateModel: "",
|
||||
}),
|
||||
).toMatchObject({
|
||||
repoRoot: "/tmp/openclaw-repo",
|
||||
providerMode: "live-frontier",
|
||||
primaryModel: undefined,
|
||||
alternateModel: undefined,
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -5,6 +5,11 @@ import type { QaProviderMode } from "../../run-config.js";
|
||||
import { normalizeQaProviderMode } from "../../run-config.js";
|
||||
import type { LiveTransportQaCommandOptions } from "./live-transport-cli.js";
|
||||
|
||||
function normalizeLiveTransportModelRef(input: string | undefined) {
|
||||
const model = input?.trim();
|
||||
return model && model.length > 0 ? model : undefined;
|
||||
}
|
||||
|
||||
export function resolveLiveTransportQaRunOptions(
|
||||
opts: LiveTransportQaCommandOptions,
|
||||
): LiveTransportQaCommandOptions & {
|
||||
@@ -21,8 +26,8 @@ export function resolveLiveTransportQaRunOptions(
|
||||
opts.providerMode === undefined
|
||||
? DEFAULT_QA_LIVE_PROVIDER_MODE
|
||||
: normalizeQaProviderMode(opts.providerMode),
|
||||
primaryModel: opts.primaryModel,
|
||||
alternateModel: opts.alternateModel,
|
||||
primaryModel: normalizeLiveTransportModelRef(opts.primaryModel),
|
||||
alternateModel: normalizeLiveTransportModelRef(opts.alternateModel),
|
||||
fastMode: opts.fastMode,
|
||||
allowFailures: opts.allowFailures,
|
||||
scenarioIds: opts.scenarioIds,
|
||||
|
||||
@@ -109,6 +109,22 @@ describe("buildQaGatewayConfig", () => {
|
||||
expect(cfg.plugins?.allow).toEqual(["acpx", "memory-core"]);
|
||||
});
|
||||
|
||||
it("falls back to provider defaults for blank model refs", () => {
|
||||
const cfg = buildQaGatewayConfig({
|
||||
bind: "loopback",
|
||||
gatewayPort: 18789,
|
||||
gatewayToken: "token",
|
||||
providerBaseUrl: "http://127.0.0.1:44080/v1",
|
||||
workspaceDir: "/tmp/qa-workspace",
|
||||
providerMode: "mock-openai",
|
||||
primaryModel: " ",
|
||||
alternateModel: "",
|
||||
});
|
||||
|
||||
expect(getPrimaryModel(cfg.agents?.defaults?.model)).toBe("mock-openai/gpt-5.5");
|
||||
expect(cfg.agents?.defaults?.models).toHaveProperty("mock-openai/gpt-5.5-alt");
|
||||
});
|
||||
|
||||
it("can wire AIMock as a separate mock provider lane", () => {
|
||||
const cfg = buildQaGatewayConfig({
|
||||
bind: "loopback",
|
||||
|
||||
@@ -29,6 +29,11 @@ export function mergeQaControlUiAllowedOrigins(extraOrigins?: string[]) {
|
||||
return [...new Set([...DEFAULT_QA_CONTROL_UI_ALLOWED_ORIGINS, ...normalizedExtra])];
|
||||
}
|
||||
|
||||
function normalizeQaGatewayModelRef(input: string | undefined, fallback: string) {
|
||||
const model = input?.trim();
|
||||
return model && model.length > 0 ? model : fallback;
|
||||
}
|
||||
|
||||
export function buildQaGatewayConfig(params: {
|
||||
bind: "loopback" | "lan";
|
||||
gatewayPort: number;
|
||||
@@ -53,9 +58,14 @@ export function buildQaGatewayConfig(params: {
|
||||
const providerBaseUrl = params.providerBaseUrl ?? "http://127.0.0.1:44080/v1";
|
||||
const providerMode = normalizeQaProviderMode(params.providerMode ?? DEFAULT_QA_PROVIDER_MODE);
|
||||
const provider = getQaProvider(providerMode);
|
||||
const primaryModel = params.primaryModel ?? defaultQaModelForMode(providerMode);
|
||||
const alternateModel =
|
||||
params.alternateModel ?? defaultQaModelForMode(providerMode, { alternate: true });
|
||||
const primaryModel = normalizeQaGatewayModelRef(
|
||||
params.primaryModel,
|
||||
defaultQaModelForMode(providerMode),
|
||||
);
|
||||
const alternateModel = normalizeQaGatewayModelRef(
|
||||
params.alternateModel,
|
||||
defaultQaModelForMode(providerMode, { alternate: true }),
|
||||
);
|
||||
const modelProviderIds = [primaryModel, alternateModel]
|
||||
.map((ref) => splitQaModelRef(ref)?.provider)
|
||||
.filter((provider): provider is string => Boolean(provider));
|
||||
|
||||
@@ -273,6 +273,11 @@ function createQaSuiteReportNotes(params: {
|
||||
return params.transport.createReportNotes(params);
|
||||
}
|
||||
|
||||
function normalizeQaSuiteModelRef(input: string | undefined, fallback: string) {
|
||||
const model = input?.trim();
|
||||
return model && model.length > 0 ? model : fallback;
|
||||
}
|
||||
|
||||
export type QaSuiteSummaryJsonParams = {
|
||||
scenarios: QaSuiteScenarioResult[];
|
||||
startedAt: Date;
|
||||
@@ -407,8 +412,14 @@ export async function runQaSuite(params?: QaSuiteRunParams): Promise<QaSuiteResu
|
||||
params?.providerMode ?? DEFAULT_QA_LIVE_PROVIDER_MODE,
|
||||
);
|
||||
const transportId = normalizeQaTransportId(params?.transportId);
|
||||
const primaryModel = params?.primaryModel ?? defaultQaModelForMode(providerMode);
|
||||
const alternateModel = params?.alternateModel ?? defaultQaModelForMode(providerMode, true);
|
||||
const primaryModel = normalizeQaSuiteModelRef(
|
||||
params?.primaryModel,
|
||||
defaultQaModelForMode(providerMode),
|
||||
);
|
||||
const alternateModel = normalizeQaSuiteModelRef(
|
||||
params?.alternateModel,
|
||||
defaultQaModelForMode(providerMode, true),
|
||||
);
|
||||
const fastMode =
|
||||
typeof params?.fastMode === "boolean"
|
||||
? params.fastMode
|
||||
|
||||
Reference in New Issue
Block a user