QA Lab: add provider registry

Move mock and live provider behavior behind provider-owned definitions so suite, manual, Matrix, and transport lanes share defaults, auth staging, model config, and standalone server startup.

Add AIMock as a first-class local provider mode while keeping mock-openai as the scenario-aware deterministic lane.
This commit is contained in:
Gustavo Madeira Santana
2026-04-16 21:23:23 -04:00
parent 4acab55db8
commit bb7e9823a8
60 changed files with 1964 additions and 998 deletions

View File

@@ -151,6 +151,22 @@ The baseline list should stay broad enough to cover:
- repo-reading and docs-reading
- one small build task such as Lobster Invaders
## Provider mock lanes
`qa suite` has two local provider mock lanes:
- `mock-openai` is the scenario-aware OpenClaw mock. It remains the default
deterministic mock lane for repo-backed QA and parity gates.
- `aimock` starts an AIMock-backed provider server for experimental protocol,
fixture, record/replay, and chaos coverage. It is additive and does not
replace the `mock-openai` scenario dispatcher.
Provider-lane implementation lives under `extensions/qa-lab/src/providers/`.
Each provider owns its defaults, local server startup, gateway model config,
auth-profile staging needs, and live/mock capability flags. Shared suite and
gateway code should route through the provider registry instead of branching on
provider names.
## Transport adapters
`qa-lab` owns a generic transport seam for markdown QA scenarios.

View File

@@ -52,6 +52,10 @@ These commands sit beside the main test suites when you need QA-lab realism:
gateway workers, up to 64 workers or the selected scenario count. Use
`--concurrency <count>` to tune the worker count, or `--concurrency 1` for
the older serial lane.
- Supports provider modes `live-frontier`, `mock-openai`, and `aimock`.
`aimock` starts a local AIMock-backed provider server for experimental
fixture and protocol-mock coverage without replacing the scenario-aware
`mock-openai` lane.
- `pnpm openclaw qa suite --runner multipass`
- Runs the same QA suite inside a disposable Multipass Linux VM.
- Keeps the same scenario-selection behavior as `qa suite` on the host.
@@ -65,6 +69,9 @@ These commands sit beside the main test suites when you need QA-lab realism:
`.artifacts/qa-e2e/...`.
- `pnpm qa:lab:up`
- Starts the Docker-backed QA site for operator-style QA work.
- `pnpm openclaw qa aimock`
- Starts only the local AIMock provider server for direct protocol smoke
testing.
- `pnpm openclaw qa matrix`
- Runs the Matrix live QA lane against a disposable Docker-backed Tuwunel homeserver.
- This QA host is repo/dev-only today. Packaged OpenClaw installs do not ship

View File

@@ -6,7 +6,6 @@ export * from "./src/cli.js";
export * from "./src/harness-runtime.js";
export * from "./src/lab-server.js";
export * from "./src/docker-harness.js";
export * from "./src/mock-openai-server.js";
export * from "./src/qa-agent-bootstrap.js";
export * from "./src/qa-agent-workspace.js";
export * from "./src/qa-gateway-config.js";

View File

@@ -3,38 +3,28 @@ import path from "node:path";
import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
import { runQaManualLane } from "./manual-lane.runtime.js";
import { isQaFastModeModelRef, type QaProviderMode } from "./model-selection.js";
import {
QA_FRONTIER_CHARACTER_EVAL_MODELS,
QA_FRONTIER_CHARACTER_JUDGE_MODEL_OPTIONS,
QA_FRONTIER_CHARACTER_JUDGE_MODELS,
QA_FRONTIER_CHARACTER_THINKING_BY_MODEL,
} from "./providers/live-frontier/character-eval.js";
import { type QaThinkingLevel } from "./qa-gateway-config.js";
import { extractQaVisibleReplyLeakText } from "./reply-failure.js";
import { runQaSuiteFromRuntime } from "./suite-launch.runtime.js";
import type { QaSuiteResult } from "./suite.js";
const DEFAULT_CHARACTER_SCENARIO_ID = "character-vibes-gollum";
const DEFAULT_CHARACTER_EVAL_MODELS = Object.freeze([
"openai/gpt-5.4",
"openai/gpt-5.2",
"openai/gpt-5",
"anthropic/claude-opus-4-6",
"anthropic/claude-sonnet-4-6",
"zai/glm-5.1",
"moonshot/kimi-k2.5",
"google/gemini-3.1-pro-preview",
]);
const DEFAULT_CHARACTER_EVAL_MODELS = QA_FRONTIER_CHARACTER_EVAL_MODELS;
const DEFAULT_CHARACTER_THINKING: QaThinkingLevel = "high";
const DEFAULT_CHARACTER_EVAL_CONCURRENCY = 16;
const DEFAULT_CHARACTER_THINKING_BY_MODEL: Readonly<Record<string, QaThinkingLevel>> =
Object.freeze({
"openai/gpt-5.4": "xhigh",
"openai/gpt-5.2": "xhigh",
"openai/gpt-5": "xhigh",
});
const DEFAULT_JUDGE_MODELS = Object.freeze(["openai/gpt-5.4", "anthropic/claude-opus-4-6"]);
QA_FRONTIER_CHARACTER_THINKING_BY_MODEL;
const DEFAULT_JUDGE_MODELS = QA_FRONTIER_CHARACTER_JUDGE_MODELS;
const DEFAULT_JUDGE_THINKING: QaThinkingLevel = "xhigh";
const DEFAULT_JUDGE_TIMEOUT_MS = 300_000;
const DEFAULT_JUDGE_MODEL_OPTIONS: Readonly<Record<string, QaCharacterModelOptions>> =
Object.freeze({
"openai/gpt-5.4": { thinkingDefault: "xhigh" },
"anthropic/claude-opus-4-6": { thinkingDefault: "high" },
});
QA_FRONTIER_CHARACTER_JUDGE_MODEL_OPTIONS;
type QaCharacterRunStatus = "pass" | "fail";

View File

@@ -24,14 +24,8 @@ const {
writeQaDockerHarnessFiles: vi.fn(),
buildQaDockerHarnessImage: vi.fn(),
runQaDockerUp: vi.fn(),
defaultQaRuntimeModelForMode: vi.fn<(mode: string, options?: { alternate?: boolean }) => string>(
(mode, options) =>
mode === "live-frontier"
? "openai/gpt-5.4"
: options?.alternate
? "mock-openai/gpt-5.4-alt"
: "mock-openai/gpt-5.4",
),
defaultQaRuntimeModelForMode:
vi.fn<(mode: string, options?: { alternate?: boolean }) => string>(),
}));
vi.mock("./manual-lane.runtime.js", () => ({
@@ -83,6 +77,8 @@ import {
runQaSuiteCommand,
} from "./cli.runtime.js";
import { runQaTelegramCommand } from "./live-transports/telegram/cli.runtime.js";
import { defaultQaModelForMode as defaultQaProviderModelForMode } from "./model-selection.js";
import type { QaProviderModeInput } from "./run-config.js";
describe("qa cli runtime", () => {
let stdoutWrite: ReturnType<typeof vi.spyOn>;
@@ -100,11 +96,7 @@ describe("qa cli runtime", () => {
runQaDockerUp.mockReset();
defaultQaRuntimeModelForMode.mockImplementation(
(mode: string, options?: { alternate?: boolean }) =>
mode === "live-frontier"
? "openai/gpt-5.4"
: options?.alternate
? "mock-openai/gpt-5.4-alt"
: "mock-openai/gpt-5.4",
defaultQaProviderModelForMode(mode as QaProviderModeInput, options),
);
runQaSuiteFromRuntime.mockResolvedValue({
watchUrl: "http://127.0.0.1:43124",
@@ -234,22 +226,6 @@ describe("qa cli runtime", () => {
);
});
it("normalizes legacy live-openai suite runs onto the frontier provider mode", async () => {
await runQaSuiteCommand({
repoRoot: "/tmp/openclaw-repo",
providerMode: "live-openai",
scenarioIds: ["approval-turn-tool-followthrough"],
});
expect(runQaSuiteFromRuntime).toHaveBeenCalledWith(
expect.objectContaining({
repoRoot: path.resolve("/tmp/openclaw-repo"),
transportId: "qa-channel",
providerMode: "live-frontier",
}),
);
});
it("passes host suite concurrency through", async () => {
await runQaSuiteCommand({
repoRoot: "/tmp/openclaw-repo",
@@ -602,6 +578,25 @@ describe("qa cli runtime", () => {
});
});
it("defaults manual aimock runs onto the aimock model lane", async () => {
await runQaManualLaneCommand({
repoRoot: "/tmp/openclaw-repo",
providerMode: "aimock",
message: "read qa kickoff and reply short",
});
expect(runQaManualLane).toHaveBeenCalledWith({
repoRoot: path.resolve("/tmp/openclaw-repo"),
transportId: "qa-channel",
providerMode: "aimock",
primaryModel: "aimock/gpt-5.4",
alternateModel: "aimock/gpt-5.4-alt",
fastMode: undefined,
message: "read qa kickoff and reply short",
timeoutMs: undefined,
});
});
it("defaults manual frontier runs onto the frontier model lane", async () => {
await runQaManualLaneCommand({
repoRoot: "/tmp/openclaw-repo",
@@ -640,31 +635,11 @@ describe("qa cli runtime", () => {
});
});
it("normalizes legacy live-openai manual runs onto the frontier provider mode", async () => {
await runQaManualLaneCommand({
repoRoot: "/tmp/openclaw-repo",
providerMode: "live-openai",
message: "read qa kickoff and reply short",
});
expect(runQaManualLane).toHaveBeenCalledWith(
expect.objectContaining({
repoRoot: path.resolve("/tmp/openclaw-repo"),
transportId: "qa-channel",
providerMode: "live-frontier",
primaryModel: "openai/gpt-5.4",
alternateModel: "openai/gpt-5.4",
}),
);
});
it("defaults manual frontier runs onto Codex OAuth when the runtime resolver prefers it", async () => {
defaultQaRuntimeModelForMode.mockImplementation((mode, options) =>
mode === "live-frontier"
? "openai-codex/gpt-5.4"
: options?.alternate
? "mock-openai/gpt-5.4-alt"
: "mock-openai/gpt-5.4",
: defaultQaProviderModelForMode(mode as QaProviderModeInput, options),
);
await runQaManualLaneCommand({

View File

@@ -14,8 +14,13 @@ import { runQaDockerUp } from "./docker-up.runtime.js";
import type { QaCliBackendAuthMode } from "./gateway-child.js";
import { startQaLabServer } from "./lab-server.js";
import { runQaManualLane } from "./manual-lane.runtime.js";
import { startQaMockOpenAiServer } from "./mock-openai-server.js";
import { runQaMultipass } from "./multipass.runtime.js";
import { DEFAULT_QA_LIVE_PROVIDER_MODE, getQaProvider } from "./providers/index.js";
import {
QA_FRONTIER_PARITY_BASELINE_LABEL,
QA_FRONTIER_PARITY_CANDIDATE_LABEL,
} from "./providers/live-frontier/parity.js";
import { startQaProviderServer } from "./providers/server-runtime.js";
import {
addQaCredentialSet,
listQaCredentialSets,
@@ -419,8 +424,8 @@ export async function runQaParityReportCommand(opts: {
) as QaParitySuiteSummary;
const comparison = buildQaAgenticParityComparison({
candidateLabel: opts.candidateLabel?.trim() || "openai/gpt-5.4",
baselineLabel: opts.baselineLabel?.trim() || "anthropic/claude-opus-4-6",
candidateLabel: opts.candidateLabel?.trim() || QA_FRONTIER_PARITY_CANDIDATE_LABEL,
baselineLabel: opts.baselineLabel?.trim() || QA_FRONTIER_PARITY_BASELINE_LABEL,
candidateSummary,
baselineSummary,
});
@@ -488,7 +493,9 @@ export async function runQaManualLaneCommand(opts: {
const repoRoot = path.resolve(opts.repoRoot ?? process.cwd());
const transportId = normalizeQaTransportId(opts.transportId);
const providerMode: QaProviderMode =
opts.providerMode === undefined ? "live-frontier" : normalizeQaProviderMode(opts.providerMode);
opts.providerMode === undefined
? DEFAULT_QA_LIVE_PROVIDER_MODE
: normalizeQaProviderMode(opts.providerMode);
const models = resolveQaManualLaneModels({
providerMode,
primaryModel: opts.primaryModel,
@@ -748,12 +755,23 @@ export async function runQaDockerUpCommand(opts: {
process.stdout.write(`Stop: ${result.stopCommand}\n`);
}
export async function runQaMockOpenAiCommand(opts: { host?: string; port?: number }) {
const server = await startQaMockOpenAiServer({
export async function runQaProviderServerCommand(
providerMode: QaProviderMode,
opts: { host?: string; port?: number },
) {
const provider = getQaProvider(providerMode);
const standaloneCommand = provider.standaloneCommand;
if (!standaloneCommand) {
throw new Error(`QA provider "${providerMode}" does not expose a standalone server command.`);
}
const server = await startQaProviderServer(providerMode, {
host: opts.host,
port: Number.isFinite(opts.port) ? opts.port : undefined,
});
await runInterruptibleServer("QA mock OpenAI", server);
if (!server) {
throw new Error(`QA provider "${providerMode}" does not expose a standalone server command.`);
}
await runInterruptibleServer(standaloneCommand.serverLabel, server);
}
export const __testing = {

View File

@@ -44,11 +44,13 @@ const {
runQaCredentialsAddCommand,
runQaCredentialsListCommand,
runQaCredentialsRemoveCommand,
runQaProviderServerCommand,
runQaTelegramCommand,
} = vi.hoisted(() => ({
runQaCredentialsAddCommand: vi.fn(),
runQaCredentialsListCommand: vi.fn(),
runQaCredentialsRemoveCommand: vi.fn(),
runQaProviderServerCommand: vi.fn(),
runQaTelegramCommand: vi.fn(),
}));
@@ -70,6 +72,7 @@ vi.mock("./cli.runtime.js", () => ({
runQaCredentialsAddCommand,
runQaCredentialsListCommand,
runQaCredentialsRemoveCommand,
runQaProviderServerCommand,
}));
import { registerQaLabCli } from "./cli.js";
@@ -82,6 +85,7 @@ describe("qa cli registration", () => {
runQaCredentialsAddCommand.mockReset();
runQaCredentialsListCommand.mockReset();
runQaCredentialsRemoveCommand.mockReset();
runQaProviderServerCommand.mockReset();
runQaTelegramCommand.mockReset();
listQaRunnerCliContributions
.mockReset()
@@ -116,6 +120,20 @@ describe("qa cli registration", () => {
);
});
it("registers standalone provider server commands from the provider registry", async () => {
const qa = program.commands.find((command) => command.name() === "qa");
expect(qa?.commands.map((command) => command.name())).toEqual(
expect.arrayContaining(["mock-openai", "aimock"]),
);
await program.parseAsync(["node", "openclaw", "qa", "aimock", "--port", "44080"]);
expect(runQaProviderServerCommand).toHaveBeenCalledWith("aimock", {
host: "127.0.0.1",
port: 44080,
});
});
it("shows an enable hint when a discovered runner plugin is installed but blocked", async () => {
listQaRunnerCliContributions.mockReset().mockReturnValue([createBlockedQaRunnerContribution()]);
const blockedProgram = new Command();

View File

@@ -1,7 +1,16 @@
import type { Command } from "commander";
import { collectString } from "./cli-options.js";
import { listLiveTransportQaCliRegistrations } from "./live-transports/cli.js";
import type { QaProviderModeInput } from "./run-config.js";
import {
DEFAULT_QA_LIVE_PROVIDER_MODE,
formatQaProviderModeHelp,
listQaStandaloneProviderCommands,
} from "./providers/index.js";
import {
QA_FRONTIER_PARITY_BASELINE_LABEL,
QA_FRONTIER_PARITY_CANDIDATE_LABEL,
} from "./providers/live-frontier/parity.js";
import type { QaProviderMode, QaProviderModeInput } from "./run-config.js";
import { hasQaScenarioPack } from "./scenario-catalog.js";
type QaLabCliRuntime = typeof import("./cli.runtime.js");
@@ -174,9 +183,12 @@ async function runQaDockerUp(opts: {
await runtime.runQaDockerUpCommand(opts);
}
async function runQaMockOpenAi(opts: { host?: string; port?: number }) {
async function runQaProviderServer(
providerMode: QaProviderMode,
opts: { host?: string; port?: number },
) {
const runtime = await loadQaLabCliRuntime();
await runtime.runQaMockOpenAiCommand(opts);
await runtime.runQaProviderServerCommand(providerMode, opts);
}
export function isQaLabCliAvailable(): boolean {
@@ -208,11 +220,7 @@ export function registerQaLabCli(program: Command) {
.option("--output-dir <path>", "Suite artifact directory")
.option("--runner <kind>", "Execution runner: host or multipass", "host")
.option("--transport <id>", "QA transport id", "qa-channel")
.option(
"--provider-mode <mode>",
"Provider mode: mock-openai or live-frontier (legacy live-openai still works)",
"live-frontier",
)
.option("--provider-mode <mode>", formatQaProviderModeHelp(), DEFAULT_QA_LIVE_PROVIDER_MODE)
.option("--model <ref>", "Primary provider/model ref")
.option("--alt-model <ref>", "Alternate provider/model ref")
.option(
@@ -274,8 +282,12 @@ export function registerQaLabCli(program: Command) {
.requiredOption("--candidate-summary <path>", "Candidate qa-suite-summary.json path")
.requiredOption("--baseline-summary <path>", "Baseline qa-suite-summary.json path")
.option("--repo-root <path>", "Repository root to target when running from a neutral cwd")
.option("--candidate-label <label>", "Candidate display label", "openai/gpt-5.4")
.option("--baseline-label <label>", "Baseline display label", "anthropic/claude-opus-4-6")
.option(
"--candidate-label <label>",
"Candidate display label",
QA_FRONTIER_PARITY_CANDIDATE_LABEL,
)
.option("--baseline-label <label>", "Baseline display label", QA_FRONTIER_PARITY_BASELINE_LABEL)
.option("--output-dir <path>", "Artifact directory for the parity report")
.action(
async (opts: {
@@ -355,11 +367,7 @@ export function registerQaLabCli(program: Command) {
.requiredOption("--message <text>", "Prompt to send to the QA agent")
.option("--repo-root <path>", "Repository root to target when running from a neutral cwd")
.option("--transport <id>", "QA transport id", "qa-channel")
.option(
"--provider-mode <mode>",
"Provider mode: mock-openai or live-frontier (legacy live-openai still works)",
"live-frontier",
)
.option("--provider-mode <mode>", formatQaProviderModeHelp(), DEFAULT_QA_LIVE_PROVIDER_MODE)
.option("--model <ref>", "Primary provider/model ref (defaults by provider mode)")
.option("--alt-model <ref>", "Alternate provider/model ref")
.option("--fast", "Enable provider fast mode where supported", false)
@@ -574,13 +582,15 @@ export function registerQaLabCli(program: Command) {
},
);
qa.command("mock-openai")
.description("Run the local mock OpenAI Responses API server for QA")
.option("--host <host>", "Bind host", "127.0.0.1")
.option("--port <port>", "Bind port", (value: string) => Number(value))
.action(async (opts: { host?: string; port?: number }) => {
await runQaMockOpenAi(opts);
});
for (const providerCommand of listQaStandaloneProviderCommands()) {
qa.command(providerCommand.name)
.description(providerCommand.description)
.option("--host <host>", "Bind host", "127.0.0.1")
.option("--port <port>", "Bind port", (value: string) => Number(value))
.action(async (opts: { host?: string; port?: number }) => {
await runQaProviderServer(providerCommand.providerMode, opts);
});
}
for (const lane of listLiveTransportQaCliRegistrations()) {
assertNoQaSubcommandCollision(qa, lane.commandName);

View File

@@ -8,11 +8,6 @@ import path from "node:path";
import { setTimeout as sleep } from "node:timers/promises";
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-runtime";
import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
import {
applyAuthProfileConfig,
upsertAuthProfile,
validateAnthropicSetupToken,
} from "openclaw/plugin-sdk/provider-auth";
import type { ModelProviderConfig } from "openclaw/plugin-sdk/provider-model-shared";
import { fetchWithSsrFGuard } from "openclaw/plugin-sdk/ssrf-runtime";
import { resolvePreferredOpenClawTmpDir } from "openclaw/plugin-sdk/temp-path";
@@ -25,72 +20,27 @@ import {
import { assertRepoBoundPath, ensureRepoBoundDirectory } from "./cli-paths.js";
import { formatQaGatewayLogsForError, redactQaGatewayDebugText } from "./gateway-log-redaction.js";
import { startQaGatewayRpcClient } from "./gateway-rpc-client.js";
import { splitQaModelRef } from "./model-selection.js";
import { splitQaModelRef, type QaProviderMode } from "./model-selection.js";
import { resolveQaNodeExecPath } from "./node-exec.js";
import {
normalizeQaProviderModeEnv,
QA_LIVE_PROVIDER_CONFIG_PATH_ENV,
resolveQaLiveCliAuthEnv,
resolveQaLiveProviderConfigPath,
type QaCliBackendAuthMode,
} from "./providers/env.js";
import { DEFAULT_QA_PROVIDER_MODE, getQaProvider } from "./providers/index.js";
import {
QA_LIVE_ANTHROPIC_SETUP_TOKEN_ENV,
QA_LIVE_SETUP_TOKEN_VALUE_ENV,
stageQaLiveAnthropicSetupToken,
} from "./providers/live-frontier/auth.js";
import { stageQaMockAuthProfiles } from "./providers/shared/mock-auth.js";
import { seedQaAgentWorkspace } from "./qa-agent-workspace.js";
import { buildQaGatewayConfig, type QaThinkingLevel } from "./qa-gateway-config.js";
import type { QaTransportAdapter } from "./qa-transport.js";
const QA_LIVE_ENV_ALIASES = Object.freeze([
{
liveVar: "OPENCLAW_LIVE_OPENAI_KEY",
providerVar: "OPENAI_API_KEY",
},
{
liveVar: "OPENCLAW_LIVE_ANTHROPIC_KEY",
providerVar: "ANTHROPIC_API_KEY",
},
{
liveVar: "OPENCLAW_LIVE_GEMINI_KEY",
providerVar: "GEMINI_API_KEY",
},
]);
const QA_MOCK_BLOCKED_ENV_VARS = Object.freeze([
"ANTHROPIC_API_KEY",
"ANTHROPIC_OAUTH_TOKEN",
"AWS_ACCESS_KEY_ID",
"AWS_BEARER_TOKEN_BEDROCK",
"AWS_REGION",
"AWS_SECRET_ACCESS_KEY",
"AWS_SESSION_TOKEN",
"GEMINI_API_KEY",
"GEMINI_API_KEYS",
"GOOGLE_API_KEY",
"MISTRAL_API_KEY",
"OPENAI_API_KEY",
"OPENAI_API_KEYS",
"OPENAI_BASE_URL",
"CODEX_HOME",
"OPENCLAW_LIVE_ANTHROPIC_KEY",
"OPENCLAW_LIVE_ANTHROPIC_KEYS",
"OPENCLAW_LIVE_GEMINI_KEY",
"OPENCLAW_LIVE_OPENAI_KEY",
"VOYAGE_API_KEY",
]);
const QA_MOCK_BLOCKED_ENV_KEY_PATTERNS = Object.freeze([
/^DISCORD_/i,
/^TELEGRAM_/i,
/^SLACK_/i,
/^MATRIX_/i,
/^SIGNAL_/i,
/^WHATSAPP_/i,
/^IMESSAGE_/i,
/^ZALO/i,
/^TWILIO_/i,
/^PLIVO_/i,
/^NGROK_/i,
]);
const QA_LIVE_PROVIDER_CONFIG_PATH_ENV = "OPENCLAW_QA_LIVE_PROVIDER_CONFIG_PATH";
const QA_LIVE_ANTHROPIC_SETUP_TOKEN_ENV = "OPENCLAW_QA_LIVE_ANTHROPIC_SETUP_TOKEN";
const QA_LIVE_SETUP_TOKEN_VALUE_ENV = "OPENCLAW_LIVE_SETUP_TOKEN_VALUE";
const QA_LIVE_ANTHROPIC_SETUP_TOKEN_PROFILE_ENV = "OPENCLAW_QA_LIVE_ANTHROPIC_SETUP_TOKEN_PROFILE";
const QA_LIVE_ANTHROPIC_SETUP_TOKEN_PROFILE_ID = "anthropic:qa-setup-token";
const QA_LIVE_CLI_BACKEND_PRESERVE_ENV = "OPENCLAW_LIVE_CLI_BACKEND_PRESERVE_ENV";
const QA_LIVE_CLI_BACKEND_AUTH_MODE_ENV = "OPENCLAW_LIVE_CLI_BACKEND_AUTH_MODE";
export type QaCliBackendAuthMode = "auto" | "api-key" | "subscription";
export type { QaCliBackendAuthMode } from "./providers/env.js";
const QA_GATEWAY_CHILD_STARTUP_MAX_ATTEMPTS = 5;
async function getFreePort() {
return await new Promise<number>((resolve, reject) => {
@@ -199,108 +149,8 @@ function appendQaGatewayTempRoot(details: string, tempRoot: string) {
: `${details}\nQA gateway temp root preserved at ${tempRoot}`;
}
export function normalizeQaProviderModeEnv(
env: NodeJS.ProcessEnv,
providerMode?: "mock-openai" | "live-frontier",
) {
if (providerMode === "mock-openai") {
for (const key of QA_MOCK_BLOCKED_ENV_VARS) {
delete env[key];
}
for (const key of Object.keys(env)) {
if (QA_MOCK_BLOCKED_ENV_KEY_PATTERNS.some((pattern) => pattern.test(key))) {
delete env[key];
}
}
return env;
}
if (providerMode === "live-frontier") {
for (const { liveVar, providerVar } of QA_LIVE_ENV_ALIASES) {
const liveValue = env[liveVar]?.trim();
if (!liveValue || env[providerVar]?.trim()) {
continue;
}
env[providerVar] = liveValue;
}
}
return env;
}
export function resolveQaGatewayChildProviderMode(
providerMode?: "mock-openai" | "live-frontier",
): "mock-openai" | "live-frontier" {
return providerMode ?? "mock-openai";
}
function resolveQaLiveCliAuthEnv(
baseEnv: NodeJS.ProcessEnv,
opts?: {
forwardHostHomeForClaudeCli?: boolean;
claudeCliAuthMode?: QaCliBackendAuthMode;
},
) {
const parsePreservedCliEnv = () => {
const raw = baseEnv[QA_LIVE_CLI_BACKEND_PRESERVE_ENV]?.trim();
if (raw?.startsWith("[")) {
try {
const parsed = JSON.parse(raw) as unknown;
return Array.isArray(parsed)
? parsed.filter((entry): entry is string => typeof entry === "string")
: [];
} catch {
return [];
}
}
return (raw ?? "").split(/[,\s]+/).filter((entry) => entry.length > 0);
};
const renderPreservedCliEnv = (values: string[]) => JSON.stringify([...new Set(values)]);
const authMode = opts?.claudeCliAuthMode ?? "auto";
const hasAnthropicKey = Boolean(
baseEnv.ANTHROPIC_API_KEY?.trim() || baseEnv.OPENCLAW_LIVE_ANTHROPIC_KEY?.trim(),
);
if (opts?.forwardHostHomeForClaudeCli && authMode === "api-key" && !hasAnthropicKey) {
throw new Error(
"Claude CLI API-key QA mode requires ANTHROPIC_API_KEY or OPENCLAW_LIVE_ANTHROPIC_KEY",
);
}
const preserveEnvValues = (() => {
if (!opts?.forwardHostHomeForClaudeCli) {
return undefined;
}
const values = parsePreservedCliEnv().filter((entry) => entry !== "ANTHROPIC_API_KEY");
if (authMode === "api-key" || (authMode === "auto" && hasAnthropicKey)) {
values.push("ANTHROPIC_API_KEY");
}
return renderPreservedCliEnv(values);
})();
const claudeCliEnv = opts?.forwardHostHomeForClaudeCli
? {
[QA_LIVE_CLI_BACKEND_AUTH_MODE_ENV]: authMode,
...(preserveEnvValues ? { [QA_LIVE_CLI_BACKEND_PRESERVE_ENV]: preserveEnvValues } : {}),
}
: {};
const configuredCodexHome = baseEnv.CODEX_HOME?.trim();
if (configuredCodexHome) {
return {
CODEX_HOME: configuredCodexHome,
...claudeCliEnv,
...(opts?.forwardHostHomeForClaudeCli && baseEnv.HOME?.trim()
? { HOME: baseEnv.HOME.trim() }
: {}),
};
}
const hostHome = baseEnv.HOME?.trim();
if (!hostHome) {
return {};
}
const codexHome = path.join(hostHome, ".codex");
return {
...(existsSync(codexHome) ? { CODEX_HOME: codexHome } : {}),
...claudeCliEnv,
...(opts?.forwardHostHomeForClaudeCli ? { HOME: hostHome } : {}),
};
export function resolveQaGatewayChildProviderMode(providerMode?: QaProviderMode): QaProviderMode {
return providerMode ?? DEFAULT_QA_PROVIDER_MODE;
}
export function buildQaRuntimeEnv(params: {
@@ -314,19 +164,20 @@ export function buildQaRuntimeEnv(params: {
xdgCacheHome: string;
bundledPluginsDir?: string;
compatibilityHostVersion?: string;
providerMode?: "mock-openai" | "live-frontier";
providerMode?: QaProviderMode;
baseEnv?: NodeJS.ProcessEnv;
forwardHostHomeForClaudeCli?: boolean;
claudeCliAuthMode?: QaCliBackendAuthMode;
}) {
const baseEnv = params.baseEnv ?? process.env;
const provider = params.providerMode ? getQaProvider(params.providerMode) : null;
const forwardedHostHome = params.forwardHostHome
? baseEnv.HOME?.trim() || os.homedir()
: undefined;
const env: NodeJS.ProcessEnv = {
...baseEnv,
HOME: forwardedHostHome ?? params.homeDir,
...(params.providerMode === "live-frontier"
...(provider?.appliesLiveEnvAliases
? resolveQaLiveCliAuthEnv(baseEnv, {
forwardHostHomeForClaudeCli: params.forwardHostHomeForClaudeCli,
claudeCliAuthMode: params.claudeCliAuthMode,
@@ -360,119 +211,6 @@ export function buildQaRuntimeEnv(params: {
return normalizedEnv;
}
function resolveQaLiveAnthropicSetupToken(env: NodeJS.ProcessEnv = process.env) {
const token = (
env[QA_LIVE_ANTHROPIC_SETUP_TOKEN_ENV]?.trim() ||
env[QA_LIVE_SETUP_TOKEN_VALUE_ENV]?.trim() ||
""
).replaceAll(/\s+/g, "");
if (!token) {
return null;
}
const tokenError = validateAnthropicSetupToken(token);
if (tokenError) {
throw new Error(`Invalid QA Anthropic setup-token: ${tokenError}`);
}
const profileId =
env[QA_LIVE_ANTHROPIC_SETUP_TOKEN_PROFILE_ENV]?.trim() ||
QA_LIVE_ANTHROPIC_SETUP_TOKEN_PROFILE_ID;
return { token, profileId };
}
export async function stageQaLiveAnthropicSetupToken(params: {
cfg: OpenClawConfig;
stateDir: string;
env?: NodeJS.ProcessEnv;
}): Promise<OpenClawConfig> {
const resolved = resolveQaLiveAnthropicSetupToken(params.env);
if (!resolved) {
return params.cfg;
}
const agentDir = path.join(params.stateDir, "agents", "main", "agent");
await fs.mkdir(agentDir, { recursive: true });
upsertAuthProfile({
profileId: resolved.profileId,
credential: {
type: "token",
provider: "anthropic",
token: resolved.token,
},
agentDir,
});
return applyAuthProfileConfig(params.cfg, {
profileId: resolved.profileId,
provider: "anthropic",
mode: "token",
displayName: "QA setup-token",
});
}
/** Providers the mock-openai harness stages placeholder credentials for. */
export const QA_MOCK_AUTH_PROVIDERS = Object.freeze(["openai", "anthropic"] as const);
/** Agent IDs the mock-openai harness stages credentials under. */
export const QA_MOCK_AUTH_AGENT_IDS = Object.freeze(["main", "qa"] as const);
export function buildQaMockProfileId(provider: string): string {
return `qa-mock-${provider}`;
}
/**
* In mock-openai mode the qa suite runs against the embedded mock server
* instead of a real provider API. The mock does not validate credentials, but
* the agent auth layer still needs a matching `api_key` auth profile in
* `auth-profiles.json` before it will route the request through
* `providerBaseUrl`. Without this staging step, every scenario fails with
* `FailoverError: No API key found for provider "openai"` before the mock
* server ever sees a request.
*
* Stages a placeholder `api_key` profile per provider in each of the agent
* dirs the qa suite uses (`main` for the runtime config, `qa` for scenario
* runs) and returns a config with matching `auth.profiles` entries so the
* runtime accepts the profile on the first lookup.
*
* The placeholder value `qa-mock-not-a-real-key` is intentionally not
* shaped like a real API key (no `sk-` prefix that would trip secret
* scanners). It only needs to be non-empty to pass the credential
* serializer; anything beyond that is ignored by the mock.
*/
export async function stageQaMockAuthProfiles(params: {
cfg: OpenClawConfig;
stateDir: string;
agentIds?: readonly string[];
providers?: readonly string[];
}): Promise<OpenClawConfig> {
const agentIds = [...new Set(params.agentIds ?? QA_MOCK_AUTH_AGENT_IDS)];
const providers = [...new Set(params.providers ?? QA_MOCK_AUTH_PROVIDERS)];
let next = params.cfg;
for (const agentId of agentIds) {
const agentDir = path.join(params.stateDir, "agents", agentId, "agent");
await fs.mkdir(agentDir, { recursive: true });
for (const provider of providers) {
const profileId = buildQaMockProfileId(provider);
upsertAuthProfile({
profileId,
credential: {
type: "api_key",
provider,
key: "qa-mock-not-a-real-key",
displayName: `QA mock ${provider} credential`,
},
agentDir,
});
}
}
for (const provider of providers) {
next = applyAuthProfileConfig(next, {
profileId: buildQaMockProfileId(provider),
provider,
mode: "api_key",
displayName: `QA mock ${provider} credential`,
});
}
return next;
}
function isRetryableGatewayCallError(details: string): boolean {
return (
details.includes("handshake timeout") ||
@@ -519,7 +257,6 @@ export const __testing = {
redactQaGatewayDebugText,
readQaLiveProviderConfigOverrides,
resolveQaGatewayChildProviderMode,
resolveQaLiveAnthropicSetupToken,
stageQaLiveAnthropicSetupToken,
stageQaMockAuthProfiles,
resolveQaLiveCliAuthEnv,
@@ -578,24 +315,6 @@ async function stopQaGatewayChildProcessTree(
await waitForQaGatewayChildExit(child, opts?.forceTimeoutMs ?? 2_000);
}
function resolveQaUserPath(value: string, env: NodeJS.ProcessEnv = process.env) {
if (value === "~") {
return env.HOME ?? os.homedir();
}
if (value.startsWith("~/")) {
return path.join(env.HOME ?? os.homedir(), value.slice(2));
}
return path.resolve(value);
}
function resolveQaLiveProviderConfigPath(env: NodeJS.ProcessEnv = process.env) {
const explicit =
env[QA_LIVE_PROVIDER_CONFIG_PATH_ENV]?.trim() || env.OPENCLAW_CONFIG_PATH?.trim();
return explicit
? { path: resolveQaUserPath(explicit, env), explicit: true }
: { path: path.join(os.homedir(), ".openclaw", "openclaw.json"), explicit: false };
}
function isRecord(value: unknown): value is Record<string, unknown> {
return Boolean(value) && typeof value === "object" && !Array.isArray(value);
}
@@ -705,7 +424,7 @@ export async function startQaGatewayChild(params: {
transport: Pick<QaTransportAdapter, "requiredPluginIds" | "createGatewayConfig">;
transportBaseUrl: string;
controlUiAllowedOrigins?: string[];
providerMode?: "mock-openai" | "live-frontier";
providerMode?: QaProviderMode;
primaryModel?: string;
alternateModel?: string;
fastMode?: boolean;
@@ -741,14 +460,14 @@ export async function startQaGatewayChild(params: {
fs.mkdir(xdgCacheHome, { recursive: true }),
]);
const providerMode = resolveQaGatewayChildProviderMode(params.providerMode);
const liveProviderIds =
providerMode === "live-frontier"
? [params.primaryModel, params.alternateModel]
.map((modelRef) =>
typeof modelRef === "string" ? splitQaModelRef(modelRef)?.provider : undefined,
)
.filter((providerId): providerId is string => Boolean(providerId))
: [];
const resolvedProvider = getQaProvider(providerMode);
const liveProviderIds = resolvedProvider.usesModelProviderPlugins
? [params.primaryModel, params.alternateModel]
.map((modelRef) =>
typeof modelRef === "string" ? splitQaModelRef(modelRef)?.provider : undefined,
)
.filter((providerId): providerId is string => Boolean(providerId))
: [];
const liveProviderConfigs = await readQaLiveProviderConfigOverrides({
providerIds: liveProviderIds,
});
@@ -794,10 +513,12 @@ export async function startQaGatewayChild(params: {
cfg,
stateDir,
});
if (providerMode === "mock-openai") {
const mockAuthProviders = getQaProvider(providerMode).mockAuthProviders;
if (mockAuthProviders && mockAuthProviders.length > 0) {
cfg = await stageQaMockAuthProfiles({
cfg,
stateDir,
providers: mockAuthProviders,
});
}
return params.mutateConfig ? params.mutateConfig(cfg) : cfg;

View File

@@ -1,23 +1,8 @@
import { QA_PROVIDER_SECRET_ENV_VARS } from "./providers/env.js";
const QA_GATEWAY_DEBUG_SECRET_ENV_VARS = Object.freeze([
"ANTHROPIC_API_KEY",
"ANTHROPIC_OAUTH_TOKEN",
"AWS_ACCESS_KEY_ID",
"AWS_BEARER_TOKEN_BEDROCK",
"AWS_SECRET_ACCESS_KEY",
"AWS_SESSION_TOKEN",
"ANTHROPIC_API_KEYS",
"GEMINI_API_KEY",
"GEMINI_API_KEYS",
"GOOGLE_API_KEY",
"MISTRAL_API_KEY",
"OPENAI_API_KEY",
"OPENAI_API_KEYS",
...QA_PROVIDER_SECRET_ENV_VARS,
"OPENCLAW_GATEWAY_TOKEN",
"OPENCLAW_LIVE_ANTHROPIC_KEY",
"OPENCLAW_LIVE_ANTHROPIC_KEYS",
"OPENCLAW_LIVE_GEMINI_KEY",
"OPENCLAW_LIVE_OPENAI_KEY",
"VOYAGE_API_KEY",
]);
export function redactQaGatewayDebugText(text: string) {

View File

@@ -1,41 +1,21 @@
import type { QaProviderMode } from "./model-selection.js";
import { getQaProvider } from "./providers/index.js";
type QaLiveTimeoutProfile = {
providerMode: "mock-openai" | "live-frontier";
providerMode: QaProviderMode;
primaryModel: string;
alternateModel: string;
};
function isAnthropicModel(modelRef: string) {
return modelRef.startsWith("anthropic/");
}
function isOpenAiModel(modelRef: string) {
return modelRef.startsWith("openai/");
}
function isGptFiveModel(modelRef: string) {
return isOpenAiModel(modelRef) && modelRef.slice("openai/".length).startsWith("gpt-5");
}
function isClaudeOpusModel(modelRef: string) {
return isAnthropicModel(modelRef) && modelRef.includes("claude-opus");
}
export function resolveQaLiveTurnTimeoutMs(
profile: QaLiveTimeoutProfile,
fallbackMs: number,
modelRef = profile.primaryModel,
) {
if (profile.providerMode === "mock-openai") {
return fallbackMs;
}
if (isClaudeOpusModel(modelRef)) {
return Math.max(fallbackMs, 240_000);
}
if (isAnthropicModel(modelRef)) {
return Math.max(fallbackMs, 180_000);
}
if (isGptFiveModel(modelRef)) {
return Math.max(fallbackMs, 360_000);
}
return Math.max(fallbackMs, 120_000);
return getQaProvider(profile.providerMode).resolveTurnTimeoutMs({
primaryModel: profile.primaryModel,
alternateModel: profile.alternateModel,
modelRef,
fallbackMs,
});
}

View File

@@ -1,16 +1,16 @@
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
const { startQaGatewayChild, startQaMockOpenAiServer } = vi.hoisted(() => ({
const { startQaGatewayChild, startQaProviderServer } = vi.hoisted(() => ({
startQaGatewayChild: vi.fn(),
startQaMockOpenAiServer: vi.fn(),
startQaProviderServer: vi.fn(),
}));
vi.mock("../../gateway-child.js", () => ({
startQaGatewayChild,
}));
vi.mock("../../mock-openai-server.js", () => ({
startQaMockOpenAiServer,
vi.mock("../../providers/server-runtime.js", () => ({
startQaProviderServer,
}));
import { startQaLiveLaneGateway } from "./live-gateway.runtime.js";
@@ -48,17 +48,21 @@ describe("startQaLiveLaneGateway", () => {
gatewayCall.mockReset();
mockStop.mockReset();
startQaGatewayChild.mockReset();
startQaMockOpenAiServer.mockReset();
startQaProviderServer.mockReset();
startQaGatewayChild.mockResolvedValue({
call: gatewayCall,
cfg: {},
stop: gatewayStop,
});
startQaMockOpenAiServer.mockResolvedValue({
baseUrl: "http://127.0.0.1:44080",
stop: mockStop,
});
startQaProviderServer.mockImplementation(async (providerMode: string) =>
providerMode === "mock-openai"
? {
baseUrl: "http://127.0.0.1:44080",
stop: mockStop,
}
: null,
);
});
afterEach(() => {
@@ -76,10 +80,7 @@ describe("startQaLiveLaneGateway", () => {
controlUiEnabled: false,
});
expect(startQaMockOpenAiServer).toHaveBeenCalledWith({
host: "127.0.0.1",
port: 0,
});
expect(startQaProviderServer).toHaveBeenCalledWith("mock-openai");
expect(startQaGatewayChild).toHaveBeenCalledWith(
expect.objectContaining({
transportBaseUrl: "http://127.0.0.1:43123",
@@ -104,7 +105,7 @@ describe("startQaLiveLaneGateway", () => {
controlUiEnabled: false,
});
expect(startQaMockOpenAiServer).not.toHaveBeenCalled();
expect(startQaProviderServer).toHaveBeenCalledWith("live-frontier");
expect(startQaGatewayChild).toHaveBeenCalledWith(
expect.objectContaining({
transportBaseUrl: "http://127.0.0.1:43123",

View File

@@ -1,12 +1,13 @@
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-runtime";
import { startQaGatewayChild, type QaCliBackendAuthMode } from "../../gateway-child.js";
import { startQaMockOpenAiServer } from "../../mock-openai-server.js";
import type { QaProviderMode } from "../../model-selection.js";
import { startQaProviderServer } from "../../providers/server-runtime.js";
import type { QaThinkingLevel } from "../../qa-gateway-config.js";
import { appendLiveLaneIssue } from "./live-lane-helpers.js";
async function stopQaLiveLaneResources(resources: {
gateway: Awaited<ReturnType<typeof startQaGatewayChild>>;
mock: Awaited<ReturnType<typeof startQaMockOpenAiServer>> | null;
mock: { baseUrl: string; stop(): Promise<void> } | null;
}) {
const errors: string[] = [];
try {
@@ -36,7 +37,7 @@ export async function startQaLiveLaneGateway(params: {
};
transportBaseUrl: string;
controlUiAllowedOrigins?: string[];
providerMode: "mock-openai" | "live-frontier";
providerMode: QaProviderMode;
primaryModel: string;
alternateModel: string;
fastMode?: boolean;
@@ -45,13 +46,7 @@ export async function startQaLiveLaneGateway(params: {
controlUiEnabled?: boolean;
mutateConfig?: (cfg: OpenClawConfig) => OpenClawConfig;
}) {
const mock =
params.providerMode === "mock-openai"
? await startQaMockOpenAiServer({
host: "127.0.0.1",
port: 0,
})
: null;
const mock = await startQaProviderServer(params.providerMode);
try {
const gateway = await startQaGatewayChild({
repoRoot: params.repoRoot,

View File

@@ -1,5 +1,6 @@
import path from "node:path";
import { resolveRepoRelativeOutputDir } from "../../cli-paths.js";
import { DEFAULT_QA_LIVE_PROVIDER_MODE } from "../../providers/index.js";
import type { QaProviderMode } from "../../run-config.js";
import { normalizeQaProviderMode } from "../../run-config.js";
import type { LiveTransportQaCommandOptions } from "./live-transport-cli.js";
@@ -18,7 +19,7 @@ export function resolveLiveTransportQaRunOptions(
),
providerMode:
opts.providerMode === undefined
? "live-frontier"
? DEFAULT_QA_LIVE_PROVIDER_MODE
: normalizeQaProviderMode(opts.providerMode),
primaryModel: opts.primaryModel,
alternateModel: opts.alternateModel,

View File

@@ -1,5 +1,6 @@
import type { Command } from "commander";
import { collectString } from "../../cli-options.js";
import { DEFAULT_QA_LIVE_PROVIDER_MODE, formatQaProviderModeHelp } from "../../providers/index.js";
import type { QaProviderModeInput } from "../../run-config.js";
export type LiveTransportQaCommandOptions = {
@@ -78,11 +79,7 @@ export function registerLiveTransportQaCli(params: {
.description(params.description)
.option("--repo-root <path>", "Repository root to target when running from a neutral cwd")
.option("--output-dir <path>", params.outputDirHelp)
.option(
"--provider-mode <mode>",
"Provider mode: mock-openai or live-frontier (legacy live-openai still works)",
"live-frontier",
)
.option("--provider-mode <mode>", formatQaProviderModeHelp(), DEFAULT_QA_LIVE_PROVIDER_MODE)
.option("--model <ref>", "Primary provider/model ref")
.option("--alt-model <ref>", "Alternate provider/model ref")
.option("--scenario <id>", params.scenarioHelp, collectString, [])

View File

@@ -6,6 +6,7 @@ import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
import { fetchWithSsrFGuard } from "openclaw/plugin-sdk/ssrf-runtime";
import { z } from "zod";
import { startQaGatewayChild } from "../../gateway-child.js";
import { DEFAULT_QA_LIVE_PROVIDER_MODE } from "../../providers/index.js";
import {
defaultQaModelForMode,
normalizeQaProviderMode,
@@ -858,7 +859,9 @@ export async function runTelegramQaLive(params: {
};
const runtimeEnv = credentialLease.payload;
const providerMode = normalizeQaProviderMode(params.providerMode ?? "live-frontier");
const providerMode = normalizeQaProviderMode(
params.providerMode ?? DEFAULT_QA_LIVE_PROVIDER_MODE,
);
const primaryModel = params.primaryModel?.trim() || defaultQaModelForMode(providerMode);
const alternateModel = params.alternateModel?.trim() || defaultQaModelForMode(providerMode, true);
const sutAccountId = params.sutAccountId?.trim() || "sut";

View File

@@ -1,9 +1,9 @@
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
const { startQaLabServer, startQaGatewayChild, startQaMockOpenAiServer } = vi.hoisted(() => ({
const { startQaLabServer, startQaGatewayChild, startQaProviderServer } = vi.hoisted(() => ({
startQaLabServer: vi.fn(),
startQaGatewayChild: vi.fn(),
startQaMockOpenAiServer: vi.fn(),
startQaProviderServer: vi.fn(),
}));
vi.mock("./lab-server.js", () => ({
@@ -14,8 +14,8 @@ vi.mock("./gateway-child.js", () => ({
startQaGatewayChild,
}));
vi.mock("./mock-openai-server.js", () => ({
startQaMockOpenAiServer,
vi.mock("./providers/server-runtime.js", () => ({
startQaProviderServer,
}));
import { runQaManualLane } from "./manual-lane.runtime.js";
@@ -31,7 +31,7 @@ describe("runQaManualLane", () => {
labStop.mockReset();
startQaLabServer.mockReset();
startQaGatewayChild.mockReset();
startQaMockOpenAiServer.mockReset();
startQaProviderServer.mockReset();
startQaLabServer.mockResolvedValue({
listenUrl: "http://127.0.0.1:43124",
@@ -64,10 +64,14 @@ describe("runQaManualLane", () => {
stop: gatewayStop,
});
startQaMockOpenAiServer.mockResolvedValue({
baseUrl: "http://127.0.0.1:44080",
stop: mockStop,
});
startQaProviderServer.mockImplementation(async (providerMode: string) =>
providerMode === "mock-openai"
? {
baseUrl: "http://127.0.0.1:44080",
stop: mockStop,
}
: null,
);
});
afterEach(() => {
@@ -84,10 +88,7 @@ describe("runQaManualLane", () => {
timeoutMs: 5_000,
});
expect(startQaMockOpenAiServer).toHaveBeenCalledWith({
host: "127.0.0.1",
port: 0,
});
expect(startQaProviderServer).toHaveBeenCalledWith("mock-openai");
expect(startQaGatewayChild).toHaveBeenCalledWith(
expect.objectContaining({
repoRoot: "/tmp/openclaw-repo",
@@ -115,7 +116,7 @@ describe("runQaManualLane", () => {
timeoutMs: 5_000,
});
expect(startQaMockOpenAiServer).not.toHaveBeenCalled();
expect(startQaProviderServer).toHaveBeenCalledWith("live-frontier");
expect(startQaLabServer).toHaveBeenCalledWith({
repoRoot: "/tmp/openclaw-repo",
embeddedGateway: "disabled",

View File

@@ -4,14 +4,15 @@ import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
import { startQaGatewayChild } from "./gateway-child.js";
import { startQaLabServer } from "./lab-server.js";
import { resolveQaLiveTurnTimeoutMs } from "./live-timeout.js";
import { startQaMockOpenAiServer } from "./mock-openai-server.js";
import type { QaProviderMode } from "./model-selection.js";
import { startQaProviderServer } from "./providers/server-runtime.js";
import type { QaThinkingLevel } from "./qa-gateway-config.js";
import { createQaTransportAdapter, type QaTransportId } from "./qa-transport-registry.js";
type QaManualLaneParams = {
repoRoot: string;
transportId?: QaTransportId;
providerMode: "mock-openai" | "live-frontier";
providerMode: QaProviderMode;
primaryModel: string;
alternateModel: string;
fastMode?: boolean;
@@ -21,7 +22,7 @@ type QaManualLaneParams = {
};
function resolveManualLaneTimeoutMs(params: {
providerMode: "mock-openai" | "live-frontier";
providerMode: QaProviderMode;
primaryModel: string;
alternateModel: string;
timeoutMs?: number;
@@ -54,13 +55,7 @@ export async function runQaManualLane(params: QaManualLaneParams) {
id: params.transportId ?? "qa-channel",
state: lab.state,
});
const mock =
params.providerMode === "mock-openai"
? await startQaMockOpenAiServer({
host: "127.0.0.1",
port: 0,
})
: null;
const mock = await startQaProviderServer(params.providerMode);
const gateway = await startQaGatewayChild({
repoRoot: params.repoRoot,
providerBaseUrl: mock ? `${mock.baseUrl}/v1` : undefined,

View File

@@ -2,14 +2,18 @@ import { spawn } from "node:child_process";
import fs from "node:fs/promises";
import path from "node:path";
import { resolvePreferredOpenClawTmpDir } from "openclaw/plugin-sdk/temp-path";
import { resolveQaNodeExecPath } from "./node-exec.js";
import {
isPreferredQaLiveFrontierCatalogModel,
QA_FRONTIER_CATALOG_ALTERNATE_MODEL,
QA_FRONTIER_CATALOG_PRIMARY_MODEL,
QA_FRONTIER_PROVIDER_IDS,
} from "./providers/live-frontier/catalog.js";
import {
createQaChannelGatewayConfig,
QA_CHANNEL_REQUIRED_PLUGIN_IDS,
} from "./qa-channel-transport.js";
import { buildQaGatewayConfig } from "./qa-gateway-config.js";
import { resolveQaNodeExecPath } from "./node-exec.js";
const QA_FRONTIER_PROVIDER_IDS = ["anthropic", "google", "openai"] as const;
type ModelRow = {
key: string;
@@ -48,7 +52,7 @@ export function selectQaRunnerModelOptions(rows: ModelRow[]): QaRunnerModelOptio
name: row.name,
provider: parsed?.provider ?? "unknown",
input: row.input,
preferred: row.key === "openai/gpt-5.4",
preferred: isPreferredQaLiveFrontierCatalogModel(row.key),
} satisfies QaRunnerModelOption;
});
@@ -110,8 +114,8 @@ export async function loadQaRunnerModelOptions(params: { repoRoot: string; signa
gatewayToken: "qa-model-catalog",
workspaceDir,
providerMode: "live-frontier",
primaryModel: "openai/gpt-5.4",
alternateModel: "anthropic/claude-sonnet-4-6",
primaryModel: QA_FRONTIER_CATALOG_PRIMARY_MODEL,
alternateModel: QA_FRONTIER_CATALOG_ALTERNATE_MODEL,
enabledProviderIds: [...QA_FRONTIER_PROVIDER_IDS],
imageGenerationModel: null,
controlUiEnabled: false,
@@ -128,24 +132,20 @@ export async function loadQaRunnerModelOptions(params: { repoRoot: string; signa
await new Promise<void>((resolve, reject) => {
let aborted = params.signal?.aborted === true;
let forceKillTimer: NodeJS.Timeout | undefined;
const child = spawn(
nodeExecPath,
["dist/index.js", "models", "list", "--all", "--json"],
{
cwd: params.repoRoot,
env: {
...process.env,
HOME: homeDir,
OPENCLAW_HOME: homeDir,
OPENCLAW_CONFIG_PATH: configPath,
OPENCLAW_STATE_DIR: stateDir,
OPENCLAW_OAUTH_DIR: path.join(stateDir, "credentials"),
OPENCLAW_CODEX_DISCOVERY_LIVE: "0",
},
detached: process.platform !== "win32",
stdio: ["ignore", "pipe", "pipe"],
const child = spawn(nodeExecPath, ["dist/index.js", "models", "list", "--all", "--json"], {
cwd: params.repoRoot,
env: {
...process.env,
HOME: homeDir,
OPENCLAW_HOME: homeDir,
OPENCLAW_CONFIG_PATH: configPath,
OPENCLAW_STATE_DIR: stateDir,
OPENCLAW_OAUTH_DIR: path.join(stateDir, "credentials"),
OPENCLAW_CODEX_DISCOVERY_LIVE: "0",
},
);
detached: process.platform !== "win32",
stdio: ["ignore", "pipe", "pipe"],
});
const cleanup = () => {
params.signal?.removeEventListener("abort", abortCatalogLoad);
if (forceKillTimer) {

View File

@@ -56,7 +56,7 @@ describe("qa model selection runtime", () => {
expect(defaultQaRuntimeModelForMode("live-frontier")).toBe("openai/gpt-5.4");
});
it("leaves mock-openai defaults unchanged", () => {
it("leaves mock defaults unchanged", () => {
listProfilesForProvider.mockImplementation((_store: unknown, provider: string) =>
provider === "openai-codex" ? ["openai-codex:user@example.com"] : [],
);
@@ -65,5 +65,7 @@ describe("qa model selection runtime", () => {
expect(defaultQaRuntimeModelForMode("mock-openai", { alternate: true })).toBe(
"mock-openai/gpt-5.4-alt",
);
expect(defaultQaRuntimeModelForMode("aimock")).toBe("aimock/gpt-5.4");
expect(defaultQaRuntimeModelForMode("aimock", { alternate: true })).toBe("aimock/gpt-5.4-alt");
});
});

View File

@@ -1,30 +1,13 @@
import {
listProfilesForProvider,
loadAuthProfileStoreForRuntime,
} from "openclaw/plugin-sdk/agent-runtime";
import { resolveEnvApiKey } from "openclaw/plugin-sdk/provider-auth";
import { defaultQaModelForMode, type QaProviderModeInput } from "./model-selection.js";
const QA_CODEX_OAUTH_LIVE_MODEL = "openai-codex/gpt-5.4";
defaultQaModelForMode,
normalizeQaProviderMode,
type QaProviderModeInput,
} from "./model-selection.js";
import { DEFAULT_QA_LIVE_PROVIDER_MODE } from "./providers/index.js";
import { resolveQaLiveFrontierPreferredModel } from "./providers/live-frontier/model-selection.runtime.js";
export function resolveQaPreferredLiveModel() {
if (resolveEnvApiKey("openai")?.apiKey) {
return undefined;
}
try {
const store = loadAuthProfileStoreForRuntime(undefined, {
readOnly: true,
allowKeychainPrompt: false,
});
if (listProfilesForProvider(store, "openai").length > 0) {
return undefined;
}
return listProfilesForProvider(store, "openai-codex").length > 0
? QA_CODEX_OAUTH_LIVE_MODEL
: undefined;
} catch {
return undefined;
}
return resolveQaLiveFrontierPreferredModel();
}
export function defaultQaRuntimeModelForMode(
@@ -34,8 +17,13 @@ export function defaultQaRuntimeModelForMode(
preferredLiveModel?: string;
},
) {
const preferredLiveModel =
options?.preferredLiveModel ??
(normalizeQaProviderMode(mode) === DEFAULT_QA_LIVE_PROVIDER_MODE
? resolveQaPreferredLiveModel()
: undefined);
return defaultQaModelForMode(mode, {
...options,
preferredLiveModel: options?.preferredLiveModel ?? resolveQaPreferredLiveModel(),
preferredLiveModel,
});
}

View File

@@ -1,14 +1,18 @@
export type QaProviderMode = "mock-openai" | "live-frontier";
export type QaProviderModeInput = QaProviderMode | "live-openai";
import {
DEFAULT_QA_LIVE_PROVIDER_MODE,
getQaProvider,
type QaProviderMode,
type QaProviderModeInput,
} from "./providers/index.js";
export type { QaProviderMode, QaProviderModeInput } from "./providers/index.js";
export type QaModelSelection = {
primaryModel: string;
alternateModel: string;
};
export function normalizeQaProviderMode(mode: QaProviderModeInput): QaProviderMode {
return mode === "live-openai" ? "live-frontier" : mode;
}
export { normalizeQaProviderMode } from "./providers/index.js";
export function defaultQaModelForMode(
mode: QaProviderModeInput,
@@ -17,10 +21,7 @@ export function defaultQaModelForMode(
preferredLiveModel?: string;
},
) {
if (normalizeQaProviderMode(mode) === "live-frontier") {
return options?.preferredLiveModel ?? "openai/gpt-5.4";
}
return options?.alternate ? "mock-openai/gpt-5.4-alt" : "mock-openai/gpt-5.4";
return getQaProvider(mode).defaultModel(options);
}
export function splitQaModelRef(ref: string) {
@@ -35,7 +36,7 @@ export function splitQaModelRef(ref: string) {
}
export function isQaFastModeModelRef(ref: string) {
return splitQaModelRef(ref)?.provider === "openai";
return getQaProvider(DEFAULT_QA_LIVE_PROVIDER_MODE).usesFastModeByDefault(ref);
}
export function isQaFastModeEnabled(selection: QaModelSelection) {

View File

@@ -2,9 +2,11 @@ import { execFile } from "node:child_process";
import { randomUUID } from "node:crypto";
import fs from "node:fs";
import { access, appendFile, mkdir, writeFile } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { resolvePreferredOpenClawTmpDir } from "openclaw/plugin-sdk/temp-path";
import type { QaProviderMode } from "./model-selection.js";
import { resolveQaForwardedLiveEnv, resolveQaLiveProviderConfigPath } from "./providers/env.js";
import { DEFAULT_QA_LIVE_PROVIDER_MODE, getQaProvider } from "./providers/index.js";
const MULTIPASS_MOUNTED_REPO_PATH = "/workspace/openclaw-host";
const MULTIPASS_GUEST_REPO_PATH = "/workspace/openclaw";
@@ -30,50 +32,6 @@ const MULTIPASS_REPO_SYNC_EXCLUDES = [
const MULTIPASS_EXEC_MAX_BUFFER = 64 * 1024 * 1024;
const MULTIPASS_GUEST_RUN_TIMEOUT_MS = 60 * 60 * 1000;
const QA_LIVE_ENV_ALIASES = Object.freeze([
{
liveVar: "OPENCLAW_LIVE_OPENAI_KEY",
providerVar: "OPENAI_API_KEY",
},
{
liveVar: "OPENCLAW_LIVE_ANTHROPIC_KEY",
providerVar: "ANTHROPIC_API_KEY",
},
{
liveVar: "OPENCLAW_LIVE_GEMINI_KEY",
providerVar: "GEMINI_API_KEY",
},
]);
const QA_LIVE_ALLOWED_ENV_VARS = Object.freeze([
"ANTHROPIC_API_KEY",
"ANTHROPIC_OAUTH_TOKEN",
"AWS_ACCESS_KEY_ID",
"AWS_BEARER_TOKEN_BEDROCK",
"AWS_REGION",
"AWS_SECRET_ACCESS_KEY",
"AWS_SESSION_TOKEN",
"GEMINI_API_KEY",
"GEMINI_API_KEYS",
"GOOGLE_API_KEY",
"MISTRAL_API_KEY",
"OPENAI_API_KEY",
"OPENAI_API_KEYS",
"OPENAI_BASE_URL",
"OPENCLAW_LIVE_ANTHROPIC_KEY",
"OPENCLAW_LIVE_ANTHROPIC_KEYS",
"OPENCLAW_LIVE_GEMINI_KEY",
"OPENCLAW_LIVE_OPENAI_KEY",
"OPENCLAW_QA_LIVE_PROVIDER_CONFIG_PATH",
"OPENCLAW_CONFIG_PATH",
"VOYAGE_API_KEY",
]);
const QA_LIVE_ALLOWED_ENV_PATTERNS = Object.freeze([
/^[A-Z0-9_]+_API_KEYS$/u,
/^[A-Z0-9_]+_API_KEY_[0-9]+$/u,
/^OPENCLAW_LIVE_[A-Z0-9_]+_KEYS$/u,
]);
export const qaMultipassDefaultResources = {
image: "lts",
cpus: 2,
@@ -109,7 +67,7 @@ export type QaMultipassPlan = {
disk: string;
pnpmVersion: string;
transportId: string;
providerMode: "mock-openai" | "live-frontier";
providerMode: QaProviderMode;
primaryModel?: string;
alternateModel?: string;
fastMode?: boolean;
@@ -256,63 +214,6 @@ function resolveMultipassInstallHint() {
return "https://multipass.run/install";
}
function resolveUserPath(value: string, env: NodeJS.ProcessEnv = process.env) {
if (value === "~") {
return env.HOME ?? os.homedir();
}
if (value.startsWith("~/")) {
return path.join(env.HOME ?? os.homedir(), value.slice(2));
}
return path.resolve(value);
}
function resolveLiveProviderConfigPath(env: NodeJS.ProcessEnv = process.env) {
const explicit =
env.OPENCLAW_QA_LIVE_PROVIDER_CONFIG_PATH?.trim() || env.OPENCLAW_CONFIG_PATH?.trim();
return explicit
? { path: resolveUserPath(explicit, env), explicit: true }
: { path: path.join(os.homedir(), ".openclaw", "openclaw.json"), explicit: false };
}
function resolveQaLiveCliAuthEnv(baseEnv: NodeJS.ProcessEnv) {
const configuredCodexHome = baseEnv.CODEX_HOME?.trim();
if (configuredCodexHome) {
const codexHome = resolveUserPath(configuredCodexHome, baseEnv);
return fs.existsSync(codexHome) ? { CODEX_HOME: codexHome } : {};
}
const hostHome = baseEnv.HOME?.trim();
const effectiveHome = hostHome || os.homedir();
const codexHome = path.join(effectiveHome, ".codex");
return fs.existsSync(codexHome) ? { CODEX_HOME: codexHome } : {};
}
function resolveForwardedLiveEnv(baseEnv: NodeJS.ProcessEnv = process.env) {
const forwarded: Record<string, string> = {};
for (const [key, rawValue] of Object.entries(baseEnv)) {
if (
!QA_LIVE_ALLOWED_ENV_VARS.includes(key) &&
!QA_LIVE_ALLOWED_ENV_PATTERNS.some((pattern) => pattern.test(key))
) {
continue;
}
const value = rawValue?.trim();
if (value) {
forwarded[key] = value;
}
}
for (const { liveVar, providerVar } of QA_LIVE_ENV_ALIASES) {
const liveValue = forwarded[liveVar]?.trim();
if (liveValue && !forwarded[providerVar]?.trim()) {
forwarded[providerVar] = liveValue;
}
}
const liveCliAuth = resolveQaLiveCliAuthEnv(baseEnv);
if (liveCliAuth.CODEX_HOME) {
forwarded.CODEX_HOME = liveCliAuth.CODEX_HOME;
}
return forwarded;
}
function createQaMultipassOutputDir(repoRoot: string) {
return path.join(repoRoot, ".artifacts", "qa-e2e", `multipass-${createOutputStamp()}`);
}
@@ -332,7 +233,7 @@ export function createQaMultipassPlan(params: {
repoRoot: string;
outputDir?: string;
transportId?: string;
providerMode?: "mock-openai" | "live-frontier";
providerMode?: QaProviderMode;
primaryModel?: string;
alternateModel?: string;
fastMode?: boolean;
@@ -346,11 +247,13 @@ export function createQaMultipassPlan(params: {
const outputDir = params.outputDir ?? createQaMultipassOutputDir(params.repoRoot);
const scenarioIds = [...new Set(params.scenarioIds ?? [])];
const transportId = params.transportId?.trim() || "qa-channel";
const providerMode = params.providerMode ?? "live-frontier";
const forwardedEnv = providerMode === "live-frontier" ? resolveForwardedLiveEnv() : {};
const providerMode = params.providerMode ?? DEFAULT_QA_LIVE_PROVIDER_MODE;
const provider = getQaProvider(providerMode);
const forwardedEnv = provider.appliesLiveEnvAliases ? resolveQaForwardedLiveEnv() : {};
const hostCodexHomePath = forwardedEnv.CODEX_HOME;
const liveProviderConfig =
providerMode === "live-frontier" ? resolveLiveProviderConfigPath() : undefined;
const liveProviderConfig = provider.usesModelProviderPlugins
? resolveQaLiveProviderConfigPath()
: undefined;
const hostLiveProviderConfigPath =
liveProviderConfig && fs.existsSync(liveProviderConfig.path)
? liveProviderConfig.path
@@ -637,7 +540,7 @@ export async function runQaMultipass(params: {
repoRoot: string;
outputDir?: string;
transportId?: string;
providerMode?: "mock-openai" | "live-frontier";
providerMode?: QaProviderMode;
primaryModel?: string;
alternateModel?: string;
fastMode?: boolean;

View File

@@ -0,0 +1,21 @@
# QA Provider Lanes
QA provider lanes are registered in `index.ts` and implemented in one folder per
provider. Shared provider contracts and mock-provider helpers live in `shared/`.
Mock lanes should use `shared/mock-provider-definition.ts` unless they need a
custom shape.
Each provider definition owns:
- its accepted `providerMode`
- default primary, alternate, and image-generation model refs
- gateway `models.providers` config, when the lane needs config injection
- model runtime params such as SSE transport, fast mode, or thinking defaults
- optional local server startup for mock lanes
- optional placeholder auth profile providers for mock lanes
- whether the lane uses real provider plugins and live environment aliases
Shared suite code should import only `providers/index.ts` and ask the selected
provider for behavior. Do not add provider-name branches to suite, gateway,
manual-lane, or live-transport runtime code unless the registry contract is
missing a needed capability.

View File

@@ -0,0 +1,9 @@
import { createMockQaProviderDefinition } from "../shared/mock-provider-definition.js";
export const aimockProviderDefinition = createMockQaProviderDefinition({
mode: "aimock",
commandName: "aimock",
commandDescription: "Run the local AIMock provider server for QA",
serverLabel: "QA AIMock",
mockAuthProviders: ["aimock", "openai", "anthropic"],
});

View File

@@ -0,0 +1,82 @@
import { describe, expect, it } from "vitest";
import { startQaAimockServer } from "./server.js";
function makeResponsesInput(text: string) {
return {
role: "user",
content: [
{
type: "input_text",
text,
},
],
};
}
describe("qa aimock server", () => {
it("serves OpenAI Responses text replies and debug request snapshots", async () => {
const server = await startQaAimockServer({
host: "127.0.0.1",
port: 0,
});
try {
const response = await fetch(`${server.baseUrl}/v1/responses`, {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({
model: "aimock/gpt-5.4",
stream: false,
input: [makeResponsesInput("hello aimock")],
}),
});
expect(response.status).toBe(200);
expect(await response.json()).toMatchObject({
status: "completed",
model: "aimock/gpt-5.4",
});
const debug = await fetch(`${server.baseUrl}/debug/last-request`);
expect(debug.status).toBe(200);
expect(await debug.json()).toMatchObject({
prompt: "hello aimock",
allInputText: "hello aimock",
model: "aimock/gpt-5.4",
providerVariant: "openai",
});
} finally {
await server.stop();
}
});
it("records the request list for scenario assertions", async () => {
const server = await startQaAimockServer({
host: "127.0.0.1",
port: 0,
});
try {
const response = await fetch(`${server.baseUrl}/v1/responses`, {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({
model: "aimock/gpt-5.4",
stream: false,
input: [makeResponsesInput("@openclaw explain the QA lab")],
}),
});
expect(response.status).toBe(200);
expect(await response.json()).toMatchObject({
status: "completed",
});
const debug = await fetch(`${server.baseUrl}/debug/requests`);
expect(debug.status).toBe(200);
expect(await debug.json()).toEqual([
expect.objectContaining({
prompt: "@openclaw explain the QA lab",
}),
]);
} finally {
await server.stop();
}
});
});

View File

@@ -0,0 +1,203 @@
import type { IncomingMessage, ServerResponse } from "node:http";
import {
LLMock,
type ChatCompletionRequest,
type JournalEntry,
type Mountable,
} from "@copilotkit/aimock";
type AimockRequestSnapshot = {
raw: string;
body: Record<string, unknown>;
prompt: string;
allInputText: string;
toolOutput: string;
model: string;
providerVariant: "openai" | "anthropic" | "unknown";
imageInputCount: number;
plannedToolName?: string;
};
function writeJson(res: ServerResponse, status: number, body: unknown) {
const text = JSON.stringify(body);
res.writeHead(status, {
"content-type": "application/json; charset=utf-8",
"content-length": Buffer.byteLength(text),
"cache-control": "no-store",
});
res.end(text);
}
function stringifyContent(content: unknown): string {
if (typeof content === "string") {
return content;
}
if (Array.isArray(content)) {
return content
.map((part) => stringifyContent(part))
.filter(Boolean)
.join("\n");
}
if (content && typeof content === "object") {
const record = content as Record<string, unknown>;
if (typeof record.text === "string") {
return record.text;
}
if (typeof record.content === "string") {
return record.content;
}
if (typeof record.output === "string") {
return record.output;
}
}
return "";
}
function requestMessages(body: ChatCompletionRequest | null | undefined) {
return Array.isArray(body?.messages) ? body.messages : [];
}
function extractLastUserText(body: ChatCompletionRequest | null | undefined) {
const messages = requestMessages(body);
for (let index = messages.length - 1; index >= 0; index -= 1) {
const message = messages[index];
if (message?.role === "user") {
return stringifyContent(message.content);
}
}
return "";
}
function extractAllInputText(body: ChatCompletionRequest | null | undefined) {
return requestMessages(body)
.map((message) => stringifyContent(message.content))
.filter(Boolean)
.join("\n");
}
function extractToolOutput(body: ChatCompletionRequest | null | undefined) {
const messages = requestMessages(body);
for (let index = messages.length - 1; index >= 0; index -= 1) {
const message = messages[index];
if (message?.role === "tool") {
return stringifyContent(message.content);
}
}
return "";
}
function countImageInputs(value: unknown): number {
if (Array.isArray(value)) {
return value.reduce((sum, entry) => sum + countImageInputs(entry), 0);
}
if (!value || typeof value !== "object") {
return 0;
}
const record = value as Record<string, unknown>;
const type = typeof record.type === "string" ? record.type : "";
const imageLikeType =
type === "input_image" || type === "image" || type === "image_url" || type === "media";
const nested =
countImageInputs(record.content) +
countImageInputs(record.image_url) +
countImageInputs(record.source);
return (imageLikeType ? 1 : 0) + nested;
}
function resolveProviderVariant(model: string): AimockRequestSnapshot["providerVariant"] {
const normalized = model.trim().toLowerCase();
const provider = /^([^/:]+)[/:]/.exec(normalized)?.[1] ?? normalized;
if (provider === "openai" || provider === "aimock" || provider === "openai-codex") {
return "openai";
}
if (provider === "anthropic" || provider === "claude-cli") {
return "anthropic";
}
if (/^(?:gpt-|o1-|openai-)/.test(normalized)) {
return "openai";
}
if (/^(?:claude-|anthropic-)/.test(normalized)) {
return "anthropic";
}
return "unknown";
}
function extractPlannedToolName(entry: JournalEntry) {
const response = entry.response.fixture?.response as
| { toolCalls?: Array<{ name?: unknown }> }
| undefined;
const name = response?.toolCalls?.[0]?.name;
return typeof name === "string" && name.length > 0 ? name : undefined;
}
function toRequestSnapshot(entry: JournalEntry): AimockRequestSnapshot {
const body = entry.body ?? null;
const model = typeof body?.model === "string" ? body.model : "";
return {
raw: JSON.stringify(body ?? {}),
body: (body ?? {}) as Record<string, unknown>,
prompt: extractLastUserText(body),
allInputText: extractAllInputText(body),
toolOutput: extractToolOutput(body),
model,
providerVariant: resolveProviderVariant(model),
imageInputCount: countImageInputs(requestMessages(body)),
plannedToolName: extractPlannedToolName(entry),
};
}
function createDebugMount(mock: LLMock): Mountable {
return {
async handleRequest(_req: IncomingMessage, res: ServerResponse, pathname: string) {
const entries = mock.getRequests();
if (pathname === "/last-request") {
const lastEntry = entries.at(-1);
writeJson(
res,
200,
lastEntry ? toRequestSnapshot(lastEntry) : { ok: false, error: "no request recorded" },
);
return true;
}
if (pathname === "/requests") {
writeJson(
res,
200,
entries.map((entry) => toRequestSnapshot(entry)),
);
return true;
}
if (pathname === "/image-generations") {
writeJson(
res,
200,
entries
.filter((entry) => entry.path === "/v1/images/generations")
.map((entry) => entry.body ?? {}),
);
return true;
}
return false;
},
};
}
export async function startQaAimockServer(params?: { host?: string; port?: number }) {
const mock = new LLMock({
host: params?.host ?? "127.0.0.1",
port: params?.port ?? 0,
strict: false,
logLevel: "silent",
});
mock.mount("/debug", createDebugMount(mock));
mock.onMessage(/.*/, { content: "AIMOCK_QA_OK" });
await mock.start();
return {
baseUrl: mock.baseUrl,
async stop() {
await mock.stop();
},
};
}

View File

@@ -0,0 +1,229 @@
import { existsSync } from "node:fs";
import os from "node:os";
import path from "node:path";
import type { QaProviderMode } from "./index.js";
import { getQaProvider } from "./index.js";
const QA_LIVE_ENV_ALIASES = Object.freeze([
{
liveVar: "OPENCLAW_LIVE_OPENAI_KEY",
providerVar: "OPENAI_API_KEY",
},
{
liveVar: "OPENCLAW_LIVE_ANTHROPIC_KEY",
providerVar: "ANTHROPIC_API_KEY",
},
{
liveVar: "OPENCLAW_LIVE_GEMINI_KEY",
providerVar: "GEMINI_API_KEY",
},
]);
export const QA_LIVE_PROVIDER_CONFIG_PATH_ENV = "OPENCLAW_QA_LIVE_PROVIDER_CONFIG_PATH";
export const QA_LIVE_CLI_BACKEND_PRESERVE_ENV = "OPENCLAW_LIVE_CLI_BACKEND_PRESERVE_ENV";
export const QA_LIVE_CLI_BACKEND_AUTH_MODE_ENV = "OPENCLAW_LIVE_CLI_BACKEND_AUTH_MODE";
export type QaCliBackendAuthMode = "auto" | "api-key" | "subscription";
export const QA_PROVIDER_SECRET_ENV_VARS = Object.freeze([
"ANTHROPIC_API_KEY",
"ANTHROPIC_OAUTH_TOKEN",
"AWS_ACCESS_KEY_ID",
"AWS_BEARER_TOKEN_BEDROCK",
"AWS_SECRET_ACCESS_KEY",
"AWS_SESSION_TOKEN",
"ANTHROPIC_API_KEYS",
"GEMINI_API_KEY",
"GEMINI_API_KEYS",
"GOOGLE_API_KEY",
"MISTRAL_API_KEY",
"OPENAI_API_KEY",
"OPENAI_API_KEYS",
"OPENCLAW_LIVE_ANTHROPIC_KEY",
"OPENCLAW_LIVE_ANTHROPIC_KEYS",
"OPENCLAW_LIVE_GEMINI_KEY",
"OPENCLAW_LIVE_OPENAI_KEY",
"VOYAGE_API_KEY",
]);
const QA_MOCK_BLOCKED_ENV_VARS = Object.freeze([
...QA_PROVIDER_SECRET_ENV_VARS,
"AWS_REGION",
"OPENAI_BASE_URL",
"CODEX_HOME",
]);
const QA_MOCK_BLOCKED_ENV_KEY_PATTERNS = Object.freeze([
/^DISCORD_/i,
/^TELEGRAM_/i,
/^SLACK_/i,
/^MATRIX_/i,
/^SIGNAL_/i,
/^WHATSAPP_/i,
/^IMESSAGE_/i,
/^ZALO/i,
/^TWILIO_/i,
/^PLIVO_/i,
/^NGROK_/i,
]);
const QA_LIVE_ALLOWED_ENV_VARS = Object.freeze([
...QA_PROVIDER_SECRET_ENV_VARS,
"AWS_REGION",
"OPENAI_BASE_URL",
QA_LIVE_PROVIDER_CONFIG_PATH_ENV,
"OPENCLAW_CONFIG_PATH",
]);
const QA_LIVE_ALLOWED_ENV_PATTERNS = Object.freeze([
/^[A-Z0-9_]+_API_KEYS$/u,
/^[A-Z0-9_]+_API_KEY_[0-9]+$/u,
/^OPENCLAW_LIVE_[A-Z0-9_]+_KEYS$/u,
]);
function resolveUserPath(value: string, env: NodeJS.ProcessEnv = process.env) {
if (value === "~") {
return env.HOME ?? os.homedir();
}
if (value.startsWith("~/")) {
return path.join(env.HOME ?? os.homedir(), value.slice(2));
}
return path.resolve(value);
}
function applyLiveProviderEnvAliases(env: NodeJS.ProcessEnv | Record<string, string>) {
for (const { liveVar, providerVar } of QA_LIVE_ENV_ALIASES) {
const liveValue = env[liveVar]?.trim();
if (!liveValue || env[providerVar]?.trim()) {
continue;
}
env[providerVar] = liveValue;
}
}
function parsePreservedCliEnv(baseEnv: NodeJS.ProcessEnv) {
const raw = baseEnv[QA_LIVE_CLI_BACKEND_PRESERVE_ENV]?.trim();
if (raw?.startsWith("[")) {
try {
const parsed = JSON.parse(raw) as unknown;
return Array.isArray(parsed)
? parsed.filter((entry): entry is string => typeof entry === "string")
: [];
} catch {
return [];
}
}
return (raw ?? "").split(/[,\s]+/).filter((entry) => entry.length > 0);
}
function renderPreservedCliEnv(values: string[]) {
return JSON.stringify([...new Set(values)]);
}
export function normalizeQaProviderModeEnv(env: NodeJS.ProcessEnv, providerMode?: QaProviderMode) {
const provider = providerMode ? getQaProvider(providerMode) : null;
if (provider?.scrubsLiveProviderEnv) {
for (const key of QA_MOCK_BLOCKED_ENV_VARS) {
delete env[key];
}
for (const key of Object.keys(env)) {
if (QA_MOCK_BLOCKED_ENV_KEY_PATTERNS.some((pattern) => pattern.test(key))) {
delete env[key];
}
}
return env;
}
if (provider?.appliesLiveEnvAliases) {
applyLiveProviderEnvAliases(env);
}
return env;
}
export function resolveQaLiveCliAuthEnv(
baseEnv: NodeJS.ProcessEnv,
opts?: {
forwardHostHomeForClaudeCli?: boolean;
claudeCliAuthMode?: QaCliBackendAuthMode;
},
) {
const authMode = opts?.claudeCliAuthMode ?? "auto";
const hasAnthropicKey = Boolean(
baseEnv.ANTHROPIC_API_KEY?.trim() || baseEnv.OPENCLAW_LIVE_ANTHROPIC_KEY?.trim(),
);
if (opts?.forwardHostHomeForClaudeCli && authMode === "api-key" && !hasAnthropicKey) {
throw new Error(
"Claude CLI API-key QA mode requires ANTHROPIC_API_KEY or OPENCLAW_LIVE_ANTHROPIC_KEY",
);
}
const preserveEnvValues = (() => {
if (!opts?.forwardHostHomeForClaudeCli) {
return undefined;
}
const values = parsePreservedCliEnv(baseEnv).filter((entry) => entry !== "ANTHROPIC_API_KEY");
if (authMode === "api-key" || (authMode === "auto" && hasAnthropicKey)) {
values.push("ANTHROPIC_API_KEY");
}
return renderPreservedCliEnv(values);
})();
const claudeCliEnv = opts?.forwardHostHomeForClaudeCli
? {
[QA_LIVE_CLI_BACKEND_AUTH_MODE_ENV]: authMode,
...(preserveEnvValues ? { [QA_LIVE_CLI_BACKEND_PRESERVE_ENV]: preserveEnvValues } : {}),
}
: {};
const configuredCodexHome = baseEnv.CODEX_HOME?.trim();
if (configuredCodexHome) {
return {
CODEX_HOME: configuredCodexHome,
...claudeCliEnv,
...(opts?.forwardHostHomeForClaudeCli && baseEnv.HOME?.trim()
? { HOME: baseEnv.HOME.trim() }
: {}),
};
}
const hostHome = baseEnv.HOME?.trim();
if (!hostHome) {
return {};
}
const codexHome = path.join(hostHome, ".codex");
return {
...(existsSync(codexHome) ? { CODEX_HOME: codexHome } : {}),
...claudeCliEnv,
...(opts?.forwardHostHomeForClaudeCli ? { HOME: hostHome } : {}),
};
}
export function resolveQaLiveProviderConfigPath(env: NodeJS.ProcessEnv = process.env) {
const explicit =
env[QA_LIVE_PROVIDER_CONFIG_PATH_ENV]?.trim() || env.OPENCLAW_CONFIG_PATH?.trim();
return explicit
? { path: resolveUserPath(explicit, env), explicit: true }
: { path: path.join(os.homedir(), ".openclaw", "openclaw.json"), explicit: false };
}
export function resolveQaForwardedLiveEnv(baseEnv: NodeJS.ProcessEnv = process.env) {
const forwarded: Record<string, string> = {};
for (const [key, rawValue] of Object.entries(baseEnv)) {
if (
!QA_LIVE_ALLOWED_ENV_VARS.includes(key) &&
!QA_LIVE_ALLOWED_ENV_PATTERNS.some((pattern) => pattern.test(key))
) {
continue;
}
const value = rawValue?.trim();
if (value) {
forwarded[key] = value;
}
}
applyLiveProviderEnvAliases(forwarded);
const configuredCodexHome = baseEnv.CODEX_HOME?.trim();
const codexHome = configuredCodexHome
? resolveUserPath(configuredCodexHome, baseEnv)
: path.join(baseEnv.HOME?.trim() || os.homedir(), ".codex");
if (existsSync(codexHome)) {
forwarded.CODEX_HOME = codexHome;
}
return forwarded;
}

View File

@@ -0,0 +1,46 @@
import { describe, expect, it } from "vitest";
import { buildQaImageGenerationConfigPatch } from "./image-generation.js";
describe("QA provider image generation config", () => {
it("uses the selected mock provider for mock-openai image generation", () => {
const patch = buildQaImageGenerationConfigPatch({
providerMode: "mock-openai",
providerBaseUrl: "http://127.0.0.1:44080/v1",
requiredPluginIds: ["qa-channel"],
});
expect(patch.plugins.allow).toEqual(["memory-core", "qa-channel"]);
expect(patch.agents.defaults.imageGenerationModel.primary).toBe("mock-openai/gpt-image-1");
expect(patch.models?.providers["mock-openai"]?.baseUrl).toBe("http://127.0.0.1:44080/v1");
});
it("uses the selected mock provider for AIMock image generation", () => {
const patch = buildQaImageGenerationConfigPatch({
providerMode: "aimock",
providerBaseUrl: "http://127.0.0.1:45080/v1",
requiredPluginIds: [],
});
expect(patch.agents.defaults.imageGenerationModel.primary).toBe("aimock/gpt-image-1");
expect(patch.models?.providers.aimock?.baseUrl).toBe("http://127.0.0.1:45080/v1");
expect(patch.models?.providers["mock-openai"]).toBeUndefined();
});
it("enables the live image provider plugin without replacing live model config", () => {
const patch = buildQaImageGenerationConfigPatch({
providerMode: "live-frontier",
requiredPluginIds: ["qa-channel"],
});
expect(patch.plugins).toEqual({
allow: ["memory-core", "openai", "qa-channel"],
entries: {
openai: {
enabled: true,
},
},
});
expect(patch.agents.defaults.imageGenerationModel.primary).toBe("openai/gpt-image-1");
expect(patch).not.toHaveProperty("models");
});
});

View File

@@ -0,0 +1,73 @@
import type { QaProviderMode } from "./index.js";
import { getQaProvider } from "./index.js";
type QaImageGenerationPatchInput = {
providerMode: QaProviderMode;
providerBaseUrl?: string;
requiredPluginIds: readonly string[];
};
function splitModelProviderId(modelRef: string) {
const slash = modelRef.indexOf("/");
return slash > 0 ? modelRef.slice(0, slash) : null;
}
function uniqueNonEmpty(values: readonly (string | null | undefined)[]) {
return [
...new Set(values.map((value) => value?.trim()).filter((value): value is string => !!value)),
];
}
export function buildQaImageGenerationConfigPatch(input: QaImageGenerationPatchInput) {
const provider = getQaProvider(input.providerMode);
const imageModelRef = provider.defaultImageGenerationModel({
modelProviderIds: provider.defaultImageGenerationProviderIds,
});
if (!imageModelRef) {
throw new Error(
`QA provider "${input.providerMode}" does not expose an image generation model`,
);
}
const imageProviderId = splitModelProviderId(imageModelRef);
const modelPatch = (() => {
if (provider.kind !== "mock") {
return null;
}
if (!input.providerBaseUrl) {
throw new Error(`QA provider "${input.providerMode}" requires a mock provider URL`);
}
return provider.buildGatewayModels({
providerBaseUrl: input.providerBaseUrl,
});
})();
const providerPluginIds = provider.usesModelProviderPlugins ? [imageProviderId] : [];
const enabledPluginIds = uniqueNonEmpty(providerPluginIds);
return {
plugins: {
allow: uniqueNonEmpty(["memory-core", ...enabledPluginIds, ...input.requiredPluginIds]),
...(enabledPluginIds.length > 0
? {
entries: Object.fromEntries(
enabledPluginIds.map((pluginId) => [pluginId, { enabled: true }]),
),
}
: {}),
},
...(modelPatch
? {
models: {
mode: modelPatch.mode,
providers: modelPatch.providers,
},
}
: {}),
agents: {
defaults: {
imageGenerationModel: {
primary: imageModelRef,
},
},
},
};
}

View File

@@ -0,0 +1,62 @@
import { aimockProviderDefinition } from "./aimock/index.js";
import { liveFrontierProviderDefinition } from "./live-frontier/index.js";
import { mockOpenAiProviderDefinition } from "./mock-openai/index.js";
import type { QaProviderDefinition, QaProviderMode, QaProviderModeInput } from "./shared/types.js";
export type {
QaMockProviderServer,
QaProviderDefinition,
QaProviderMode,
QaProviderModeInput,
} from "./shared/types.js";
const PROVIDERS = [
mockOpenAiProviderDefinition,
aimockProviderDefinition,
liveFrontierProviderDefinition,
] as const satisfies readonly QaProviderDefinition[];
export const DEFAULT_QA_PROVIDER_MODE = "mock-openai" satisfies QaProviderMode;
export const DEFAULT_QA_LIVE_PROVIDER_MODE = "live-frontier" satisfies QaProviderMode;
const PROVIDERS_BY_INPUT = new Map<QaProviderModeInput, QaProviderDefinition>();
for (const provider of PROVIDERS) {
PROVIDERS_BY_INPUT.set(provider.mode, provider);
}
export function isQaProviderModeInput(input: unknown): input is QaProviderModeInput {
return typeof input === "string" && PROVIDERS_BY_INPUT.has(input as QaProviderModeInput);
}
export function normalizeQaProviderMode(input: QaProviderModeInput): QaProviderMode {
return getQaProvider(input).mode;
}
export function getQaProvider(input: QaProviderModeInput): QaProviderDefinition {
const provider = PROVIDERS_BY_INPUT.get(input);
if (!provider) {
throw new Error(`unknown QA provider mode: ${input}`);
}
return provider;
}
export function listQaProviderModes() {
return PROVIDERS.map((provider) => provider.mode);
}
export function formatQaProviderModeHelp() {
return `Provider mode: ${listQaProviderModes().join(", ")}`;
}
export function listQaStandaloneProviderCommands() {
return PROVIDERS.flatMap((provider) =>
provider.standaloneCommand
? [
{
providerMode: provider.mode,
...provider.standaloneCommand,
},
]
: [],
);
}

View File

@@ -0,0 +1,60 @@
import fs from "node:fs/promises";
import path from "node:path";
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-runtime";
import {
applyAuthProfileConfig,
upsertAuthProfile,
validateAnthropicSetupToken,
} from "openclaw/plugin-sdk/provider-auth";
export const QA_LIVE_ANTHROPIC_SETUP_TOKEN_ENV = "OPENCLAW_QA_LIVE_ANTHROPIC_SETUP_TOKEN";
export const QA_LIVE_SETUP_TOKEN_VALUE_ENV = "OPENCLAW_LIVE_SETUP_TOKEN_VALUE";
const QA_LIVE_ANTHROPIC_SETUP_TOKEN_PROFILE_ENV = "OPENCLAW_QA_LIVE_ANTHROPIC_SETUP_TOKEN_PROFILE";
const QA_LIVE_ANTHROPIC_SETUP_TOKEN_PROFILE_ID = "anthropic:qa-setup-token";
function resolveQaLiveAnthropicSetupToken(env: NodeJS.ProcessEnv = process.env) {
const token = (
env[QA_LIVE_ANTHROPIC_SETUP_TOKEN_ENV]?.trim() ||
env[QA_LIVE_SETUP_TOKEN_VALUE_ENV]?.trim() ||
""
).replaceAll(/\s+/g, "");
if (!token) {
return null;
}
const tokenError = validateAnthropicSetupToken(token);
if (tokenError) {
throw new Error(`Invalid QA Anthropic setup-token: ${tokenError}`);
}
const profileId =
env[QA_LIVE_ANTHROPIC_SETUP_TOKEN_PROFILE_ENV]?.trim() ||
QA_LIVE_ANTHROPIC_SETUP_TOKEN_PROFILE_ID;
return { token, profileId };
}
export async function stageQaLiveAnthropicSetupToken(params: {
cfg: OpenClawConfig;
stateDir: string;
env?: NodeJS.ProcessEnv;
}): Promise<OpenClawConfig> {
const resolved = resolveQaLiveAnthropicSetupToken(params.env);
if (!resolved) {
return params.cfg;
}
const agentDir = path.join(params.stateDir, "agents", "main", "agent");
await fs.mkdir(agentDir, { recursive: true });
upsertAuthProfile({
profileId: resolved.profileId,
credential: {
type: "token",
provider: "anthropic",
token: resolved.token,
},
agentDir,
});
return applyAuthProfileConfig(params.cfg, {
profileId: resolved.profileId,
provider: "anthropic",
mode: "token",
displayName: "QA setup-token",
});
}

View File

@@ -0,0 +1,7 @@
export const QA_FRONTIER_PROVIDER_IDS = ["anthropic", "google", "openai"] as const;
export const QA_FRONTIER_CATALOG_PRIMARY_MODEL = "openai/gpt-5.4";
export const QA_FRONTIER_CATALOG_ALTERNATE_MODEL = "anthropic/claude-sonnet-4-6";
export function isPreferredQaLiveFrontierCatalogModel(modelRef: string) {
return modelRef === QA_FRONTIER_CATALOG_PRIMARY_MODEL;
}

View File

@@ -0,0 +1,36 @@
import type { QaThinkingLevel } from "../../qa-thinking.js";
type QaFrontierCharacterModelOptions = {
thinkingDefault?: QaThinkingLevel;
fastMode?: boolean;
};
export const QA_FRONTIER_CHARACTER_EVAL_MODELS = Object.freeze([
"openai/gpt-5.4",
"openai/gpt-5.2",
"openai/gpt-5",
"anthropic/claude-opus-4-6",
"anthropic/claude-sonnet-4-6",
"zai/glm-5.1",
"moonshot/kimi-k2.5",
"google/gemini-3.1-pro-preview",
]);
export const QA_FRONTIER_CHARACTER_THINKING_BY_MODEL: Readonly<Record<string, QaThinkingLevel>> =
Object.freeze({
"openai/gpt-5.4": "xhigh",
"openai/gpt-5.2": "xhigh",
"openai/gpt-5": "xhigh",
});
export const QA_FRONTIER_CHARACTER_JUDGE_MODELS = Object.freeze([
"openai/gpt-5.4",
"anthropic/claude-opus-4-6",
]);
export const QA_FRONTIER_CHARACTER_JUDGE_MODEL_OPTIONS: Readonly<
Record<string, QaFrontierCharacterModelOptions>
> = Object.freeze({
"openai/gpt-5.4": { thinkingDefault: "xhigh" },
"anthropic/claude-opus-4-6": { thinkingDefault: "high" },
});

View File

@@ -0,0 +1,61 @@
import type { QaProviderDefinition } from "../shared/types.js";
function isOpenAiModel(modelRef: string) {
return modelRef.startsWith("openai/");
}
function isAnthropicModel(modelRef: string) {
return modelRef.startsWith("anthropic/");
}
function isQaFastModeModelRef(modelRef: string) {
return isOpenAiModel(modelRef);
}
function isGptFiveModel(modelRef: string) {
return isOpenAiModel(modelRef) && modelRef.slice("openai/".length).startsWith("gpt-5");
}
function isClaudeOpusModel(modelRef: string) {
return isAnthropicModel(modelRef) && modelRef.includes("claude-opus");
}
export const liveFrontierProviderDefinition: QaProviderDefinition = {
mode: "live-frontier",
kind: "live",
defaultModel: (options) => options?.preferredLiveModel ?? "openai/gpt-5.4",
defaultImageGenerationProviderIds: ["openai"],
defaultImageGenerationModel: ({ modelProviderIds }) =>
modelProviderIds.includes("openai") ? "openai/gpt-image-1" : null,
usesFastModeByDefault: isQaFastModeModelRef,
resolveModelParams: ({ modelRef, fastMode, thinkingDefault }) => ({
transport: "sse",
openaiWsWarmup: false,
...(fastMode === true || isQaFastModeModelRef(modelRef) ? { fastMode: true } : {}),
...(thinkingDefault ? { thinking: thinkingDefault } : {}),
}),
resolveTurnTimeoutMs: ({ fallbackMs, modelRef }) => {
if (isClaudeOpusModel(modelRef)) {
return Math.max(fallbackMs, 240_000);
}
if (isAnthropicModel(modelRef)) {
return Math.max(fallbackMs, 180_000);
}
if (isGptFiveModel(modelRef)) {
return Math.max(fallbackMs, 360_000);
}
return Math.max(fallbackMs, 120_000);
},
buildGatewayModels: ({ liveProviderConfigs }) => {
const providers = liveProviderConfigs ?? {};
return Object.keys(providers).length > 0
? {
mode: "merge",
providers,
}
: null;
},
usesModelProviderPlugins: true,
scrubsLiveProviderEnv: false,
appliesLiveEnvAliases: true,
};

View File

@@ -0,0 +1,27 @@
import {
listProfilesForProvider,
loadAuthProfileStoreForRuntime,
} from "openclaw/plugin-sdk/agent-runtime";
import { resolveEnvApiKey } from "openclaw/plugin-sdk/provider-auth";
const QA_CODEX_OAUTH_LIVE_MODEL = "openai-codex/gpt-5.4";
export function resolveQaLiveFrontierPreferredModel() {
if (resolveEnvApiKey("openai")?.apiKey) {
return undefined;
}
try {
const store = loadAuthProfileStoreForRuntime(undefined, {
readOnly: true,
allowKeychainPrompt: false,
});
if (listProfilesForProvider(store, "openai").length > 0) {
return undefined;
}
return listProfilesForProvider(store, "openai-codex").length > 0
? QA_CODEX_OAUTH_LIVE_MODEL
: undefined;
} catch {
return undefined;
}
}

View File

@@ -0,0 +1,2 @@
export const QA_FRONTIER_PARITY_CANDIDATE_LABEL = "openai/gpt-5.4";
export const QA_FRONTIER_PARITY_BASELINE_LABEL = "anthropic/claude-opus-4-6";

View File

@@ -0,0 +1,9 @@
import { createMockQaProviderDefinition } from "../shared/mock-provider-definition.js";
export const mockOpenAiProviderDefinition = createMockQaProviderDefinition({
mode: "mock-openai",
commandName: "mock-openai",
commandDescription: "Run the local mock OpenAI Responses API server for QA",
serverLabel: "QA mock OpenAI",
mockAuthProviders: ["openai", "anthropic"],
});

View File

@@ -1,5 +1,5 @@
import { afterEach, describe, expect, it } from "vitest";
import { resolveProviderVariant, startQaMockOpenAiServer } from "./mock-openai-server.js";
import { resolveProviderVariant, startQaMockOpenAiServer } from "./server.js";
const cleanups: Array<() => Promise<void>> = [];
const QA_IMAGE_PNG_BASE64 =
@@ -63,6 +63,23 @@ function makeUserInput(text: string) {
};
}
const SESSIONS_SPAWN_TOOL = { type: "function", name: "sessions_spawn" } as const;
const THREAD_SUBAGENT_CHILD_ERROR_TOKEN = "QA_SUBAGENT_CHILD_ERROR";
const THREAD_SUBAGENT_TOOL_ERROR =
"thread=true requested but thread delivery is unavailable in this test harness.";
function threadSubagentTask(token: string) {
return `Reply exactly \`${token}\`. This is the marker.`;
}
function explicitSessionsSpawnPrompt(token: string) {
return [
"Use sessions_spawn for this QA check.",
`task="${threadSubagentTask(token)}"`,
"label=qa-thread-subagent thread=true mode=session runTimeoutSeconds=30",
].join(" ");
}
describe("qa mock openai server", () => {
it("serves health and streamed responses", async () => {
const server = await startQaMockOpenAiServer({
@@ -109,14 +126,14 @@ describe("qa mock openai server", () => {
},
body: JSON.stringify({
stream: true,
input: [makeUserInput("Quiet streaming QA check: reply exactly `MATRIX_QA_STREAMING_OK`.")],
input: [makeUserInput("Quiet streaming QA check: reply exactly `QA_STREAMING_OK`.")],
}),
});
expect(quietResponse.status).toBe(200);
const quietBody = await quietResponse.text();
expect(quietBody).toContain('"type":"response.output_text.delta"');
expect(quietBody).toContain('"phase":"final_answer"');
expect(quietBody).toContain("MATRIX_QA_STREAMING_OK");
expect(quietBody).toContain("QA_STREAMING_OK");
const blockResponse = await fetch(`${server.baseUrl}/v1/responses`, {
method: "POST",
@@ -548,6 +565,7 @@ describe("qa mock openai server", () => {
},
body: JSON.stringify({
stream: true,
tools: [SESSIONS_SPAWN_TOOL],
input: [
{
role: "user",
@@ -568,6 +586,67 @@ describe("qa mock openai server", () => {
expect(body).toContain('\\"thread\\":false');
});
it("emits explicitly requested sessions_spawn tool calls", async () => {
const server = await startMockServer();
const body = await expectResponsesText(server, {
stream: true,
tools: [SESSIONS_SPAWN_TOOL],
input: [
{
role: "user",
content: [
{
type: "input_text",
text: explicitSessionsSpawnPrompt("QA_SUBAGENT_CHILD_FIXED"),
},
],
},
],
});
expect(body).toContain('"name":"sessions_spawn"');
expect(body).toContain('\\"label\\":\\"qa-thread-subagent\\"');
expect(body).toContain('\\"thread\\":true');
expect(body).toContain('\\"mode\\":\\"session\\"');
expect(body).toContain("QA_SUBAGENT_CHILD_FIXED");
});
it("surfaces sessions_spawn tool errors instead of echoing child-task markers", async () => {
const server = await startMockServer();
const body = await expectResponsesJson<{
output?: Array<{ content?: Array<{ text?: string }> }>;
}>(server, {
stream: false,
tools: [SESSIONS_SPAWN_TOOL],
input: [
makeUserInput(explicitSessionsSpawnPrompt(THREAD_SUBAGENT_CHILD_ERROR_TOKEN)),
{
type: "function_call",
name: "sessions_spawn",
arguments: JSON.stringify({
task: threadSubagentTask(THREAD_SUBAGENT_CHILD_ERROR_TOKEN),
label: "qa-thread-subagent",
thread: true,
mode: "session",
runTimeoutSeconds: 30,
}),
},
{
type: "function_call_output",
output: JSON.stringify({
status: "error",
error: THREAD_SUBAGENT_TOOL_ERROR,
}),
},
],
});
const text = body.output?.[0]?.content?.[0]?.text ?? "";
expect(text).toContain(THREAD_SUBAGENT_TOOL_ERROR);
expect(text).not.toContain(THREAD_SUBAGENT_CHILD_ERROR_TOKEN);
});
it("plans memory tools and serves mock image generations", async () => {
const server = await startQaMockOpenAiServer({
host: "127.0.0.1",
@@ -850,6 +929,7 @@ describe("qa mock openai server", () => {
},
body: JSON.stringify({
stream: true,
tools: [SESSIONS_SPAWN_TOOL],
input: [
{
role: "user",
@@ -875,6 +955,7 @@ describe("qa mock openai server", () => {
},
body: JSON.stringify({
stream: true,
tools: [SESSIONS_SPAWN_TOOL],
input: [
{
role: "user",
@@ -905,6 +986,7 @@ describe("qa mock openai server", () => {
},
body: JSON.stringify({
stream: false,
tools: [SESSIONS_SPAWN_TOOL],
input: [
{
role: "user",
@@ -961,6 +1043,7 @@ describe("qa mock openai server", () => {
headers: { "content-type": "application/json" },
body: JSON.stringify({
stream: true,
tools: [SESSIONS_SPAWN_TOOL],
input: [{ role: "user", content: [{ type: "input_text", text: prompt }] }],
}),
});
@@ -972,6 +1055,7 @@ describe("qa mock openai server", () => {
headers: { "content-type": "application/json" },
body: JSON.stringify({
stream: true,
tools: [SESSIONS_SPAWN_TOOL],
input: [{ role: "user", content: [{ type: "input_text", text: prompt }] }],
}),
});
@@ -1430,6 +1514,60 @@ describe("qa mock openai server", () => {
});
});
it("preserves Anthropic /v1/messages declared tools for explicit sessions_spawn prompts", async () => {
const server = await startMockServer();
const response = await fetch(`${server.baseUrl}/v1/messages`, {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({
model: "claude-opus-4-6",
max_tokens: 256,
tools: [
{
name: "sessions_spawn",
input_schema: { type: "object", properties: {} },
},
],
messages: [
{
role: "user",
content: [
{
type: "text",
text: explicitSessionsSpawnPrompt("QA_SUBAGENT_CHILD_ANTHROPIC"),
},
],
},
],
}),
});
expect(response.status).toBe(200);
const body = (await response.json()) as {
stop_reason: string;
content: Array<Record<string, unknown>>;
};
expect(body.stop_reason).toBe("tool_use");
const toolUseBlock = body.content.find((block) => block.type === "tool_use") as
| { name: string; input: Record<string, unknown> }
| undefined;
expect(toolUseBlock?.name).toBe("sessions_spawn");
expect(toolUseBlock?.input).toMatchObject({
task: threadSubagentTask("QA_SUBAGENT_CHILD_ANTHROPIC"),
label: "qa-thread-subagent",
thread: true,
mode: "session",
runTimeoutSeconds: 30,
});
const debugResponse = await fetch(`${server.baseUrl}/debug/last-request`);
expect(debugResponse.status).toBe(200);
expect(await debugResponse.json()).toMatchObject({
model: "claude-opus-4-6",
plannedToolName: "sessions_spawn",
});
});
it("dispatches Anthropic /v1/messages tool_result follow-ups through the shared scenario logic", async () => {
// This verifies the Anthropic adapter correctly feeds tool_result
// content blocks into the shared scenario dispatcher so downstream

View File

@@ -1,6 +1,6 @@
import { createServer, type IncomingMessage, type ServerResponse } from "node:http";
import { setTimeout as sleep } from "node:timers/promises";
import { closeQaHttpServer } from "./bus-server.js";
import { closeQaHttpServer } from "../../bus-server.js";
type ResponsesInputItem = Record<string, unknown>;
@@ -142,8 +142,8 @@ const QA_REASONING_ONLY_RECOVERY_PROMPT_RE = /reasoning-only continuation qa che
const QA_REASONING_ONLY_SIDE_EFFECT_PROMPT_RE = /reasoning-only after write safety check/i;
const QA_EMPTY_RESPONSE_RECOVERY_PROMPT_RE = /empty response continuation qa check/i;
const QA_EMPTY_RESPONSE_EXHAUSTION_PROMPT_RE = /empty response exhaustion qa check/i;
const QA_QUIET_STREAMING_PROMPT_RE = /(?:matrix\s+)?quiet streaming qa check/i;
const QA_BLOCK_STREAMING_PROMPT_RE = /(?:matrix\s+)?block streaming qa check/i;
const QA_QUIET_STREAMING_PROMPT_RE = /quiet streaming qa check/i;
const QA_BLOCK_STREAMING_PROMPT_RE = /block streaming qa check/i;
const QA_REASONING_ONLY_RETRY_NEEDLE =
"recorded reasoning but did not produce a user-visible answer";
const QA_EMPTY_RESPONSE_RETRY_NEEDLE =
@@ -538,6 +538,77 @@ function extractLabeledMarkerDirective(text: string, label: string) {
);
}
function extractQuotedToolArg(text: string, name: string) {
const escapedName = name.replaceAll(/[.*+?^${}()|[\]\\]/g, "\\$&");
return extractLastCapture(text, new RegExp(`\\b${escapedName}\\s*=\\s*"([^"]+)"`, "i"));
}
function extractBareToolArg(text: string, name: string) {
const escapedName = name.replaceAll(/[.*+?^${}()|[\]\\]/g, "\\$&");
return extractLastCapture(text, new RegExp(`\\b${escapedName}\\s*=\\s*([^\\s\\\`.,;:!?]+)`, "i"));
}
function hasDeclaredTool(body: Record<string, unknown>, name: string) {
const tools = Array.isArray(body.tools) ? body.tools : [];
return tools.some((tool) => {
if (!tool || typeof tool !== "object") {
return false;
}
const record = tool as Record<string, unknown>;
if (record.name === name) {
return true;
}
const nested = record.function;
return Boolean(
nested && typeof nested === "object" && (nested as { name?: unknown }).name === name,
);
});
}
function buildExplicitSessionsSpawnArgs(text: string): Record<string, unknown> | null {
if (!/\bsessions_spawn\b/i.test(text)) {
return null;
}
const task = extractQuotedToolArg(text, "task");
if (!task) {
return null;
}
const label = extractQuotedToolArg(text, "label") ?? extractBareToolArg(text, "label");
const mode = extractBareToolArg(text, "mode")?.toLowerCase();
const runTimeoutSecondsRaw = extractBareToolArg(text, "runTimeoutSeconds");
const runTimeoutSeconds =
runTimeoutSecondsRaw && /^\d+$/.test(runTimeoutSecondsRaw)
? Number(runTimeoutSecondsRaw)
: undefined;
return {
task,
...(label ? { label } : {}),
...(extractBareToolArg(text, "thread")?.toLowerCase() === "true" ? { thread: true } : {}),
...(mode === "session" || mode === "run" ? { mode } : {}),
...(runTimeoutSeconds !== undefined ? { runTimeoutSeconds } : {}),
};
}
function extractToolErrorForNamedCall(params: {
allInputText: string;
input: ResponsesInputItem[];
name: string;
toolJson: Record<string, unknown> | null;
}) {
const error = typeof params.toolJson?.error === "string" ? params.toolJson.error.trim() : "";
if (!error) {
return undefined;
}
const namedFunctionCall = params.input.some(
(item) => item.type === "function_call" && item.name === params.name,
);
const namedPromptReference = new RegExp(`\\b${params.name}\\b`, "i").test(params.allInputText);
if (namedFunctionCall || namedPromptReference) {
return error;
}
return undefined;
}
function isHeartbeatPrompt(text: string) {
const trimmed = text.trim();
if (!trimmed || /remember this fact/i.test(trimmed)) {
@@ -573,10 +644,19 @@ function buildAssistantText(
const imageInputCount = countImageInputs(input);
const activeMemorySummary = extractActiveMemorySummary(allInputText);
const snackPreference = extractSnackPreference(activeMemorySummary ?? memorySnippet);
const sessionsSpawnError = extractToolErrorForNamedCall({
allInputText,
input,
name: "sessions_spawn",
toolJson,
});
if (/what was the qa canary code/i.test(prompt) && rememberedFact) {
return `Protocol note: the QA canary code was ${rememberedFact}.`;
}
if (sessionsSpawnError) {
return `Protocol note: sessions_spawn failed: ${sessionsSpawnError}`;
}
if (/remember this fact/i.test(prompt) && exactReplyDirective) {
return exactReplyDirective;
}
@@ -867,6 +947,7 @@ async function buildResponsesPayload(
const isBaselineUnmentionedChannelChatter = /\bno bot ping here\b/i.test(prompt);
const hasReasoningOnlyRetryInstruction = allInputText.includes(QA_REASONING_ONLY_RETRY_NEEDLE);
const hasEmptyResponseRetryInstruction = allInputText.includes(QA_EMPTY_RESPONSE_RETRY_NEEDLE);
const canCallSessionsSpawn = hasDeclaredTool(body, "sessions_spawn");
if (/remember this fact/i.test(prompt)) {
return buildAssistantEvents(buildAssistantText(input, body, scenarioState));
}
@@ -1110,7 +1191,7 @@ async function buildResponsesPayload(
size: "1024x1024",
});
}
if (/subagent fanout synthesis check/i.test(prompt)) {
if (canCallSessionsSpawn && /subagent fanout synthesis check/i.test(prompt)) {
if (!toolOutput && scenarioState.subagentFanoutPhase === 0) {
scenarioState.subagentFanoutPhase = 1;
return buildToolCallEventsWithArgs("sessions_spawn", {
@@ -1128,6 +1209,10 @@ async function buildResponsesPayload(
});
}
}
const explicitSessionsSpawnArgs = buildExplicitSessionsSpawnArgs(allInputText);
if (canCallSessionsSpawn && explicitSessionsSpawnArgs && !toolOutput) {
return buildToolCallEventsWithArgs("sessions_spawn", explicitSessionsSpawnArgs);
}
if (/tool continuity check/i.test(prompt) && !toolOutput) {
return buildToolCallEventsWithArgs("read", { path: "QA_KICKOFF_TASK.md" });
}
@@ -1155,7 +1240,11 @@ async function buildResponsesPayload(
});
}
}
if ((/\bdelegate\b/i.test(prompt) || /subagent handoff/i.test(prompt)) && !toolOutput) {
if (
canCallSessionsSpawn &&
(/\bdelegate\b/i.test(prompt) || /subagent handoff/i.test(prompt)) &&
!toolOutput
) {
return buildToolCallEventsWithArgs("sessions_spawn", {
task: "Inspect the QA workspace and return one concise protocol note.",
label: "qa-sidecar",
@@ -1547,12 +1636,13 @@ async function buildMessagesPayload(
const normalizedModel =
typeof body.model === "string" && body.model.trim() !== "" ? body.model : "claude-opus-4-6";
// Dispatch through the same scenario logic the /v1/responses route uses.
// The mock dispatcher only reads `body.input`, `body.model`, and
// `body.stream`, so a synthetic shim body is sufficient.
// Preserve declared tools so route-specific adapters mirror what the
// real provider request made available to the model.
const dispatchBody: Record<string, unknown> = {
input,
model: normalizedModel,
stream: false,
...(Array.isArray(body.tools) ? { tools: body.tools } : {}),
};
const events = await buildResponsesPayload(dispatchBody, scenarioState);
const extracted = extractFinalAssistantOutputFromEvents(events);

View File

@@ -0,0 +1,36 @@
import { getQaProvider, type QaMockProviderServer, type QaProviderModeInput } from "./index.js";
type QaProviderServerParams = {
host: string;
port: number;
};
async function startMockOpenAiProviderServer(params: QaProviderServerParams) {
const { startQaMockOpenAiServer } = await import("./mock-openai/server.js");
return await startQaMockOpenAiServer(params);
}
async function startAimockProviderServer(params: QaProviderServerParams) {
const { startQaAimockServer } = await import("./aimock/server.js");
return await startQaAimockServer(params);
}
export async function startQaProviderServer(
input: QaProviderModeInput,
params?: { host?: string; port?: number },
): Promise<QaMockProviderServer | null> {
const provider = getQaProvider(input);
const serverParams = {
host: params?.host ?? "127.0.0.1",
port: params?.port ?? 0,
};
switch (provider.mode) {
case "mock-openai":
return await startMockOpenAiProviderServer(serverParams);
case "aimock":
return await startAimockProviderServer(serverParams);
case "live-frontier":
default:
return null;
}
}

View File

@@ -0,0 +1,70 @@
import fs from "node:fs/promises";
import path from "node:path";
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-runtime";
import { applyAuthProfileConfig, upsertAuthProfile } from "openclaw/plugin-sdk/provider-auth";
/** Providers the mock harness stages placeholder credentials for by default. */
export const QA_MOCK_AUTH_PROVIDERS = Object.freeze(["openai", "anthropic"] as const);
/** Agent IDs the mock harness stages credentials under. */
export const QA_MOCK_AUTH_AGENT_IDS = Object.freeze(["main", "qa"] as const);
export function buildQaMockProfileId(provider: string): string {
return `qa-mock-${provider}`;
}
/**
* In mock provider modes the qa suite runs against an embedded mock server
* instead of a real provider API. The mock does not validate credentials, but
* the agent auth layer still needs a matching `api_key` auth profile in
* `auth-profiles.json` before it will route the request through
* `providerBaseUrl`. Without this staging step, every scenario fails with
* `FailoverError: No API key found for provider "openai"` before the mock
* server ever sees a request.
*
* Stages a placeholder `api_key` profile per provider in each of the agent
* dirs the qa suite uses (`main` for the runtime config, `qa` for scenario
* runs) and returns a config with matching `auth.profiles` entries so the
* runtime accepts the profile on the first lookup.
*
* The placeholder value `qa-mock-not-a-real-key` is intentionally not
* shaped like a real API key (no `sk-` prefix that would trip secret
* scanners). It only needs to be non-empty to pass the credential
* serializer; anything beyond that is ignored by the mock.
*/
export async function stageQaMockAuthProfiles(params: {
cfg: OpenClawConfig;
stateDir: string;
agentIds?: readonly string[];
providers?: readonly string[];
}): Promise<OpenClawConfig> {
const agentIds = [...new Set(params.agentIds ?? QA_MOCK_AUTH_AGENT_IDS)];
const providers = [...new Set(params.providers ?? QA_MOCK_AUTH_PROVIDERS)];
let next = params.cfg;
for (const agentId of agentIds) {
const agentDir = path.join(params.stateDir, "agents", agentId, "agent");
await fs.mkdir(agentDir, { recursive: true });
for (const provider of providers) {
const profileId = buildQaMockProfileId(provider);
upsertAuthProfile({
profileId,
credential: {
type: "api_key",
provider,
key: "qa-mock-not-a-real-key",
displayName: `QA mock ${provider} credential`,
},
agentDir,
});
}
}
for (const provider of providers) {
next = applyAuthProfileConfig(next, {
profileId: buildQaMockProfileId(provider),
provider,
mode: "api_key",
displayName: `QA mock ${provider} credential`,
});
}
return next;
}

View File

@@ -0,0 +1,104 @@
import type { ModelProviderConfig } from "openclaw/plugin-sdk/provider-model-shared";
const ZERO_COST = Object.freeze({
input: 0,
output: 0,
cacheRead: 0,
cacheWrite: 0,
});
function cloneProvider(provider: ModelProviderConfig): ModelProviderConfig {
return {
...provider,
models: provider.models.map((model) => ({ ...model })),
};
}
export function trimTrailingApiV1(baseUrl: string) {
return baseUrl.replace(/\/v1\/?$/i, "");
}
export function createMockOpenAiResponsesProvider(baseUrl: string): ModelProviderConfig {
return {
baseUrl,
apiKey: "test",
api: "openai-responses",
request: {
allowPrivateNetwork: true,
},
models: [
{
id: "gpt-5.4",
name: "gpt-5.4",
api: "openai-responses",
reasoning: false,
input: ["text", "image"],
cost: ZERO_COST,
contextWindow: 128_000,
maxTokens: 4096,
},
{
id: "gpt-5.4-alt",
name: "gpt-5.4-alt",
api: "openai-responses",
reasoning: false,
input: ["text", "image"],
cost: ZERO_COST,
contextWindow: 128_000,
maxTokens: 4096,
},
{
id: "gpt-image-1",
name: "gpt-image-1",
api: "openai-responses",
reasoning: false,
input: ["text"],
cost: ZERO_COST,
contextWindow: 128_000,
maxTokens: 4096,
},
],
};
}
export function createMockAnthropicMessagesProvider(baseUrl: string): ModelProviderConfig {
return {
baseUrl: trimTrailingApiV1(baseUrl),
apiKey: "test",
api: "anthropic-messages",
request: {
allowPrivateNetwork: true,
},
models: [
{
id: "claude-opus-4-6",
name: "claude-opus-4-6",
api: "anthropic-messages",
reasoning: false,
input: ["text", "image"],
cost: ZERO_COST,
contextWindow: 200_000,
maxTokens: 4096,
},
{
id: "claude-sonnet-4-6",
name: "claude-sonnet-4-6",
api: "anthropic-messages",
reasoning: false,
input: ["text", "image"],
cost: ZERO_COST,
contextWindow: 200_000,
maxTokens: 4096,
},
],
};
}
export function createMockProviderMap(primaryProviderId: string, providerBaseUrl: string) {
const primaryProvider = createMockOpenAiResponsesProvider(providerBaseUrl);
return {
[primaryProviderId]: primaryProvider,
openai: cloneProvider(primaryProvider),
anthropic: createMockAnthropicMessagesProvider(providerBaseUrl),
};
}

View File

@@ -0,0 +1,45 @@
import { createMockProviderMap } from "./mock-model-config.js";
import type { QaProviderDefinition, QaProviderMode } from "./types.js";
export type MockQaProviderDefinitionParams = {
mode: Extract<QaProviderMode, "aimock" | "mock-openai">;
commandName: string;
commandDescription: string;
serverLabel: string;
mockAuthProviders: readonly string[];
};
function mockModelRef(providerId: string, alternate?: boolean) {
return `${providerId}/${alternate ? "gpt-5.4-alt" : "gpt-5.4"}`;
}
export function createMockQaProviderDefinition(
params: MockQaProviderDefinitionParams,
): QaProviderDefinition {
return {
mode: params.mode,
kind: "mock",
standaloneCommand: {
name: params.commandName,
description: params.commandDescription,
serverLabel: params.serverLabel,
},
defaultModel: (options) => mockModelRef(params.mode, options?.alternate),
defaultImageGenerationProviderIds: [],
defaultImageGenerationModel: () => `${params.mode}/gpt-image-1`,
usesFastModeByDefault: () => false,
resolveModelParams: () => ({
transport: "sse",
openaiWsWarmup: false,
}),
resolveTurnTimeoutMs: ({ fallbackMs }) => fallbackMs,
buildGatewayModels: ({ providerBaseUrl }) => ({
mode: "replace",
providers: createMockProviderMap(params.mode, providerBaseUrl),
}),
mockAuthProviders: params.mockAuthProviders,
usesModelProviderPlugins: false,
scrubsLiveProviderEnv: true,
appliesLiveEnvAliases: false,
};
}

View File

@@ -0,0 +1,56 @@
import type { ModelProviderConfig } from "openclaw/plugin-sdk/provider-model-shared";
import type { QaThinkingLevel } from "../../qa-thinking.js";
export type QaProviderMode = "mock-openai" | "aimock" | "live-frontier";
export type QaProviderModeInput = QaProviderMode;
export type QaMockProviderServer = {
baseUrl: string;
stop(): Promise<void>;
};
export type QaProviderModelParamsInput = {
modelRef: string;
fastMode?: boolean;
thinkingDefault?: QaThinkingLevel;
};
export type QaProviderGatewayModelsInput = {
providerBaseUrl: string;
liveProviderConfigs?: Record<string, ModelProviderConfig>;
};
export type QaProviderDefaultImageInput = {
modelProviderIds: readonly string[];
};
export type QaProviderTurnTimeoutInput = {
primaryModel: string;
alternateModel: string;
modelRef: string;
fallbackMs: number;
};
export type QaProviderDefinition = {
mode: QaProviderMode;
kind: "mock" | "live";
standaloneCommand?: {
name: string;
description: string;
serverLabel: string;
};
defaultModel(options?: { alternate?: boolean; preferredLiveModel?: string }): string;
defaultImageGenerationProviderIds: readonly string[];
defaultImageGenerationModel(input: QaProviderDefaultImageInput): string | null;
usesFastModeByDefault(modelRef: string): boolean;
resolveModelParams(input: QaProviderModelParamsInput): Record<string, unknown>;
resolveTurnTimeoutMs(input: QaProviderTurnTimeoutInput): number;
buildGatewayModels(input: QaProviderGatewayModelsInput): {
mode: "replace" | "merge";
providers: Record<string, ModelProviderConfig>;
} | null;
mockAuthProviders?: readonly string[];
usesModelProviderPlugins: boolean;
scrubsLiveProviderEnv: boolean;
appliesLiveEnvAliases: boolean;
};

View File

@@ -1,5 +1,6 @@
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-runtime";
import type { QaBusState } from "./bus-state.js";
import { getQaProvider } from "./providers/index.js";
import { QaStateBackedTransportAdapter, waitForQaTransportCondition } from "./qa-transport.js";
import type {
QaTransportActionName,
@@ -71,9 +72,10 @@ export function createQaChannelGatewayConfig(params: {
}
function createQaChannelReportNotes(params: QaTransportReportParams) {
const provider = getQaProvider(params.providerMode);
return [
params.providerMode === "mock-openai"
? "Runs against qa-channel + qa-lab bus + real gateway child + mock OpenAI provider."
provider.kind === "mock"
? `Runs against qa-channel + qa-lab bus + real gateway child + ${params.providerMode} provider.`
: `Runs against qa-channel + qa-lab bus + real gateway child + live frontier models (${params.primaryModel}, ${params.alternateModel})${params.fastMode ? " with fast mode enabled" : ""}.`,
params.concurrency > 1
? `Scenarios run in isolated gateway workers with concurrency ${params.concurrency}.`

View File

@@ -96,6 +96,29 @@ describe("buildQaGatewayConfig", () => {
expect(cfg.plugins?.allow).toEqual(["memory-core"]);
});
it("can wire AIMock as a separate mock provider lane", () => {
const cfg = buildQaGatewayConfig({
bind: "loopback",
gatewayPort: 18789,
gatewayToken: "token",
providerBaseUrl: "http://127.0.0.1:45080/v1",
workspaceDir: "/tmp/qa-workspace",
providerMode: "aimock",
primaryModel: "aimock/gpt-5.4",
alternateModel: "aimock/gpt-5.4-alt",
});
expect(getPrimaryModel(cfg.agents?.defaults?.model)).toBe("aimock/gpt-5.4");
expect(cfg.agents?.defaults?.imageGenerationModel).toEqual({
primary: "aimock/gpt-image-1",
});
expect(cfg.models?.providers?.aimock?.baseUrl).toBe("http://127.0.0.1:45080/v1");
expect(cfg.models?.providers?.aimock?.api).toBe("openai-responses");
expect(cfg.models?.providers?.openai?.baseUrl).toBe("http://127.0.0.1:45080/v1");
expect(cfg.models?.providers?.anthropic?.baseUrl).toBe("http://127.0.0.1:45080");
expect(cfg.models?.providers?.["mock-openai"]).toBeUndefined();
});
it("can omit qa-channel for live transport gateway children", () => {
const cfg = buildQaGatewayConfig({
bind: "loopback",

View File

@@ -2,13 +2,17 @@ import type { OpenClawConfig } from "openclaw/plugin-sdk/config-runtime";
import type { ModelProviderConfig } from "openclaw/plugin-sdk/provider-model-shared";
import {
defaultQaModelForMode,
isQaFastModeModelRef,
normalizeQaProviderMode,
splitQaModelRef,
type QaProviderMode,
} from "./model-selection.js";
import { getQaProvider } from "./providers/index.js";
import { DEFAULT_QA_PROVIDER_MODE } from "./providers/index.js";
import { normalizeQaThinkingLevel, type QaThinkingLevel } from "./qa-thinking.js";
import type { QaTransportGatewayConfig } from "./qa-transport.js";
export { normalizeQaThinkingLevel, type QaThinkingLevel } from "./qa-thinking.js";
export const DEFAULT_QA_CONTROL_UI_ALLOWED_ORIGINS = Object.freeze([
"http://127.0.0.1:18789",
"http://localhost:18789",
@@ -16,39 +20,6 @@ export const DEFAULT_QA_CONTROL_UI_ALLOWED_ORIGINS = Object.freeze([
"http://localhost:43124",
]);
export type QaThinkingLevel = "off" | "minimal" | "low" | "medium" | "high" | "xhigh" | "adaptive";
export function normalizeQaThinkingLevel(input: unknown): QaThinkingLevel | undefined {
const value = typeof input === "string" ? input.trim().toLowerCase() : "";
const collapsed = value.replace(/[\s_-]+/g, "");
if (collapsed === "off") {
return "off";
}
if (collapsed === "minimal" || collapsed === "min") {
return "minimal";
}
if (collapsed === "low") {
return "low";
}
if (collapsed === "medium" || collapsed === "med") {
return "medium";
}
if (collapsed === "high") {
return "high";
}
if (collapsed === "xhigh" || collapsed === "extrahigh") {
return "xhigh";
}
if (collapsed === "adaptive" || collapsed === "auto") {
return "adaptive";
}
return undefined;
}
function trimTrailingApiV1(baseUrl: string) {
return baseUrl.replace(/\/v1\/?$/i, "");
}
export function mergeQaControlUiAllowedOrigins(extraOrigins?: string[]) {
const normalizedExtra = (extraOrigins ?? [])
.map((origin) => origin.trim())
@@ -65,7 +36,7 @@ export function buildQaGatewayConfig(params: {
controlUiRoot?: string;
controlUiAllowedOrigins?: string[];
controlUiEnabled?: boolean;
providerMode?: QaProviderMode | "live-openai";
providerMode?: QaProviderMode;
primaryModel?: string;
alternateModel?: string;
imageGenerationModel?: string | null;
@@ -77,108 +48,9 @@ export function buildQaGatewayConfig(params: {
fastMode?: boolean;
thinkingDefault?: QaThinkingLevel;
}): OpenClawConfig {
const mockProviderBaseUrl = params.providerBaseUrl ?? "http://127.0.0.1:44080/v1";
const mockAnthropicBaseUrl = trimTrailingApiV1(mockProviderBaseUrl);
const mockOpenAiProvider: ModelProviderConfig = {
baseUrl: mockProviderBaseUrl,
apiKey: "test",
api: "openai-responses",
request: {
allowPrivateNetwork: true,
},
models: [
{
id: "gpt-5.4",
name: "gpt-5.4",
api: "openai-responses",
reasoning: false,
input: ["text", "image"],
cost: {
input: 0,
output: 0,
cacheRead: 0,
cacheWrite: 0,
},
contextWindow: 128_000,
maxTokens: 4096,
},
{
id: "gpt-5.4-alt",
name: "gpt-5.4-alt",
api: "openai-responses",
reasoning: false,
input: ["text", "image"],
cost: {
input: 0,
output: 0,
cacheRead: 0,
cacheWrite: 0,
},
contextWindow: 128_000,
maxTokens: 4096,
},
{
id: "gpt-image-1",
name: "gpt-image-1",
api: "openai-responses",
reasoning: false,
input: ["text"],
cost: {
input: 0,
output: 0,
cacheRead: 0,
cacheWrite: 0,
},
contextWindow: 128_000,
maxTokens: 4096,
},
],
};
const mockNamedOpenAiProvider: ModelProviderConfig = {
...mockOpenAiProvider,
models: mockOpenAiProvider.models.map((model) => ({ ...model })),
};
const mockAnthropicProvider: ModelProviderConfig = {
baseUrl: mockAnthropicBaseUrl,
apiKey: "test",
api: "anthropic-messages",
request: {
allowPrivateNetwork: true,
},
models: [
{
id: "claude-opus-4-6",
name: "claude-opus-4-6",
api: "anthropic-messages",
reasoning: false,
input: ["text", "image"],
cost: {
input: 0,
output: 0,
cacheRead: 0,
cacheWrite: 0,
},
contextWindow: 200_000,
maxTokens: 4096,
},
{
id: "claude-sonnet-4-6",
name: "claude-sonnet-4-6",
api: "anthropic-messages",
reasoning: false,
input: ["text", "image"],
cost: {
input: 0,
output: 0,
cacheRead: 0,
cacheWrite: 0,
},
contextWindow: 200_000,
maxTokens: 4096,
},
],
};
const providerMode = normalizeQaProviderMode(params.providerMode ?? "mock-openai");
const providerBaseUrl = params.providerBaseUrl ?? "http://127.0.0.1:44080/v1";
const providerMode = normalizeQaProviderMode(params.providerMode ?? DEFAULT_QA_PROVIDER_MODE);
const provider = getQaProvider(providerMode);
const primaryModel = params.primaryModel ?? defaultQaModelForMode(providerMode);
const alternateModel =
params.alternateModel ?? defaultQaModelForMode(providerMode, { alternate: true });
@@ -188,39 +60,33 @@ export function buildQaGatewayConfig(params: {
const imageGenerationModelRef =
params.imageGenerationModel !== undefined
? params.imageGenerationModel
: providerMode === "mock-openai"
? "mock-openai/gpt-image-1"
: modelProviderIds.includes("openai")
? "openai/gpt-image-1"
: null;
const selectedProviderIds =
providerMode === "live-frontier"
? [
...new Set(
[...(params.enabledProviderIds ?? []), ...modelProviderIds, imageGenerationModelRef]
.map((value) =>
typeof value === "string" ? (splitQaModelRef(value)?.provider ?? value) : null,
)
.filter((provider): provider is string => Boolean(provider)),
),
]
: [];
const selectedPluginIds =
providerMode === "live-frontier"
? [
...new Set(
(params.enabledPluginIds?.length ?? 0) > 0
? params.enabledPluginIds
: selectedProviderIds,
),
]
: [
...new Set(
(params.enabledPluginIds ?? [])
.map((pluginId) => pluginId.trim())
.filter((pluginId) => pluginId.length > 0),
),
];
: provider.defaultImageGenerationModel({ modelProviderIds });
const selectedProviderIds = provider.usesModelProviderPlugins
? [
...new Set(
[...(params.enabledProviderIds ?? []), ...modelProviderIds, imageGenerationModelRef]
.map((value) =>
typeof value === "string" ? (splitQaModelRef(value)?.provider ?? value) : null,
)
.filter((provider): provider is string => Boolean(provider)),
),
]
: [];
const selectedPluginIds = provider.usesModelProviderPlugins
? [
...new Set(
(params.enabledPluginIds?.length ?? 0) > 0
? params.enabledPluginIds
: selectedProviderIds,
),
]
: [
...new Set(
(params.enabledPluginIds ?? [])
.map((pluginId) => pluginId.trim())
.filter((pluginId) => pluginId.length > 0),
),
];
const transportPluginIds = [...new Set(params.transportPluginIds ?? [])]
.map((pluginId) => pluginId.trim())
.filter((pluginId) => pluginId.length > 0);
@@ -231,22 +97,17 @@ export function buildQaGatewayConfig(params: {
transportPluginIds.map((pluginId) => [pluginId, { enabled: true }]),
);
const allowedPlugins = [...new Set(["memory-core", ...selectedPluginIds, ...transportPluginIds])];
const liveModelParams =
providerMode === "live-frontier"
? (modelRef: string) => ({
transport: "sse",
openaiWsWarmup: false,
...(params.fastMode === true || isQaFastModeModelRef(modelRef) ? { fastMode: true } : {}),
...(params.thinkingDefault ? { thinking: params.thinkingDefault } : {}),
})
: (_modelRef: string) => ({
transport: "sse",
openaiWsWarmup: false,
});
const resolveModelParams = (modelRef: string) =>
provider.resolveModelParams({
modelRef,
fastMode: params.fastMode,
thinkingDefault: params.thinkingDefault,
});
const allowedOrigins = mergeQaControlUiAllowedOrigins(params.controlUiAllowedOrigins);
const liveProviderConfigs =
providerMode === "live-frontier" ? (params.liveProviderConfigs ?? {}) : {};
const hasLiveProviderConfigs = Object.keys(liveProviderConfigs).length > 0;
const gatewayModels = provider.buildGatewayModels({
providerBaseUrl,
liveProviderConfigs: params.liveProviderConfigs,
});
return {
plugins: {
@@ -286,10 +147,10 @@ export function buildQaGatewayConfig(params: {
},
models: {
[primaryModel]: {
params: liveModelParams(primaryModel),
params: resolveModelParams(primaryModel),
},
[alternateModel]: {
params: liveModelParams(alternateModel),
params: resolveModelParams(alternateModel),
},
},
subagents: {
@@ -319,25 +180,14 @@ export function buildQaGatewayConfig(params: {
memory: {
backend: "builtin",
},
...(providerMode === "mock-openai"
...(gatewayModels
? {
models: {
mode: "replace",
providers: {
"mock-openai": mockOpenAiProvider,
openai: mockNamedOpenAiProvider,
anthropic: mockAnthropicProvider,
},
mode: gatewayModels.mode,
providers: gatewayModels.providers,
},
}
: hasLiveProviderConfigs
? {
models: {
mode: "merge",
providers: liveProviderConfigs,
},
}
: {}),
: {}),
gateway: {
mode: "local",
bind: params.bind,

View File

@@ -0,0 +1,28 @@
export type QaThinkingLevel = "off" | "minimal" | "low" | "medium" | "high" | "xhigh" | "adaptive";
export function normalizeQaThinkingLevel(input: unknown): QaThinkingLevel | undefined {
const value = typeof input === "string" ? input.trim().toLowerCase() : "";
const collapsed = value.replace(/[\s_-]+/g, "");
if (collapsed === "off") {
return "off";
}
if (collapsed === "minimal" || collapsed === "min") {
return "minimal";
}
if (collapsed === "low") {
return "low";
}
if (collapsed === "medium" || collapsed === "med") {
return "medium";
}
if (collapsed === "high") {
return "high";
}
if (collapsed === "xhigh" || collapsed === "extrahigh") {
return "xhigh";
}
if (collapsed === "adaptive" || collapsed === "auto") {
return "adaptive";
}
return undefined;
}

View File

@@ -1,5 +1,6 @@
import { setTimeout as sleep } from "node:timers/promises";
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-runtime";
import type { QaProviderMode } from "./model-selection.js";
import { extractQaFailureReplyText } from "./reply-failure.js";
import type {
QaBusInboundMessageInput,
@@ -24,7 +25,7 @@ export type QaTransportGatewayClient = {
export type QaTransportActionName = "delete" | "edit" | "react" | "thread-create";
export type QaTransportReportParams = {
providerMode: "mock-openai" | "live-frontier";
providerMode: QaProviderMode;
primaryModel: string;
alternateModel: string;
fastMode: boolean;

View File

@@ -1,24 +1,20 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
const { defaultQaRuntimeModelForMode } = vi.hoisted(() => ({
defaultQaRuntimeModelForMode: vi.fn<(mode: string, options?: { alternate?: boolean }) => string>(
(mode, options) =>
mode === "live-frontier"
? "openai/gpt-5.4"
: options?.alternate
? "mock-openai/gpt-5.4-alt"
: "mock-openai/gpt-5.4",
),
defaultQaRuntimeModelForMode:
vi.fn<(mode: string, options?: { alternate?: boolean }) => string>(),
}));
vi.mock("./model-selection.runtime.js", () => ({
defaultQaRuntimeModelForMode,
}));
import { defaultQaModelForMode as defaultQaProviderModelForMode } from "./model-selection.js";
import {
createDefaultQaRunSelection,
createIdleQaRunnerSnapshot,
createQaRunOutputDir,
normalizeQaRunSelection,
type QaProviderModeInput,
} from "./run-config.js";
const scenarios = [
@@ -42,11 +38,7 @@ describe("qa run config", () => {
beforeEach(() => {
defaultQaRuntimeModelForMode.mockImplementation(
(mode: string, options?: { alternate?: boolean }) =>
mode === "live-frontier"
? "openai/gpt-5.4"
: options?.alternate
? "mock-openai/gpt-5.4-alt"
: "mock-openai/gpt-5.4",
defaultQaProviderModelForMode(mode as QaProviderModeInput, options),
);
});
@@ -64,7 +56,7 @@ describe("qa run config", () => {
expect(
normalizeQaRunSelection(
{
providerMode: "live-openai",
providerMode: "live-frontier",
primaryModel: "openai/gpt-5.4",
alternateModel: "",
fastMode: false,
@@ -81,6 +73,17 @@ describe("qa run config", () => {
});
});
it("rejects removed provider compatibility names", () => {
expect(() =>
normalizeQaRunSelection(
{
providerMode: "live-openai",
},
scenarios,
),
).toThrow("unknown QA provider mode: live-openai");
});
it("falls back to all scenarios when selection would otherwise be empty", () => {
const snapshot = createIdleQaRunnerSnapshot(scenarios);
expect(snapshot.status).toBe("idle");
@@ -95,6 +98,26 @@ describe("qa run config", () => {
).toEqual(["dm-chat-baseline", "thread-lifecycle"]);
});
it("normalizes aimock selections", () => {
expect(
normalizeQaRunSelection(
{
providerMode: "aimock",
primaryModel: "",
alternateModel: "",
scenarioIds: ["dm-chat-baseline"],
},
scenarios,
),
).toEqual({
providerMode: "aimock",
primaryModel: "aimock/gpt-5.4",
alternateModel: "aimock/gpt-5.4-alt",
fastMode: false,
scenarioIds: ["dm-chat-baseline"],
});
});
it("anchors generated run output dirs under the provided repo root", () => {
const outputDir = createQaRunOutputDir("/tmp/openclaw-repo");
expect(outputDir.startsWith("/tmp/openclaw-repo/.artifacts/qa-e2e/lab-")).toBe(true);
@@ -104,9 +127,7 @@ describe("qa run config", () => {
defaultQaRuntimeModelForMode.mockImplementation((mode, options) =>
mode === "live-frontier"
? "openai-codex/gpt-5.4"
: options?.alternate
? "mock-openai/gpt-5.4-alt"
: "mock-openai/gpt-5.4",
: defaultQaProviderModelForMode(mode as QaProviderModeInput, options),
);
expect(createDefaultQaRunSelection(scenarios)).toEqual({

View File

@@ -1,13 +1,16 @@
import path from "node:path";
import { defaultQaRuntimeModelForMode } from "./model-selection.runtime.js";
import {
DEFAULT_QA_LIVE_PROVIDER_MODE,
getQaProvider,
isQaProviderModeInput,
normalizeQaProviderMode as normalizeQaProviderModeInput,
type QaProviderMode,
} from "./model-selection.js";
import { defaultQaRuntimeModelForMode } from "./model-selection.runtime.js";
} from "./providers/index.js";
import type { QaSeedScenario } from "./scenario-catalog.js";
export type { QaProviderMode } from "./model-selection.js";
export type QaProviderModeInput = QaProviderMode | "live-openai";
export type { QaProviderModeInput } from "./providers/index.js";
export type QaLabRunSelection = {
providerMode: QaProviderMode;
@@ -38,7 +41,7 @@ export function defaultQaModelForMode(mode: QaProviderMode, alternate = false) {
}
export function createDefaultQaRunSelection(scenarios: QaSeedScenario[]): QaLabRunSelection {
const providerMode: QaProviderMode = "live-frontier";
const providerMode: QaProviderMode = DEFAULT_QA_LIVE_PROVIDER_MODE;
return {
providerMode,
primaryModel: defaultQaModelForMode(providerMode),
@@ -49,11 +52,14 @@ export function createDefaultQaRunSelection(scenarios: QaSeedScenario[]): QaLabR
}
export function normalizeQaProviderMode(input: unknown): QaProviderMode {
return normalizeQaProviderModeInput(
input === "mock-openai" || input === "live-frontier" || input === "live-openai"
? input
: "live-frontier",
);
if (input === undefined || input === null || input === "") {
return DEFAULT_QA_LIVE_PROVIDER_MODE;
}
if (isQaProviderModeInput(input)) {
return normalizeQaProviderModeInput(input);
}
const details = typeof input === "string" ? `: ${input}` : "";
throw new Error(`unknown QA provider mode${details}`);
}
function normalizeModel(input: unknown, fallback: string) {
@@ -87,7 +93,7 @@ export function normalizeQaRunSelection(
payload.alternateModel,
defaultQaModelForMode(providerMode, true),
),
fastMode: providerMode === "live-frontier" || payload.fastMode === true,
fastMode: getQaProvider(providerMode).kind === "live" || payload.fastMode === true,
scenarioIds: normalizeScenarioIds(payload.scenarioIds, scenarios),
};
}

View File

@@ -2,6 +2,8 @@ import path from "node:path";
import { normalizeLowercaseStringOrEmpty } from "openclaw/plugin-sdk/text-runtime";
import { ensureRepoBoundDirectory, resolveRepoRelativeOutputDir } from "./cli-paths.js";
import type { QaCliBackendAuthMode } from "./gateway-child.js";
import type { QaProviderMode } from "./model-selection.js";
import { getQaProvider } from "./providers/index.js";
import type { QaTransportId } from "./qa-transport-registry.js";
import { readQaBootstrapScenarioCatalog } from "./scenario-catalog.js";
@@ -28,10 +30,10 @@ function normalizeQaConfigString(value: unknown): string | undefined {
function scenarioMatchesLiveLane(params: {
scenario: QaSeedScenario;
primaryModel: string;
providerMode: "mock-openai" | "live-frontier";
providerMode: QaProviderMode;
claudeCliAuthMode?: QaCliBackendAuthMode;
}) {
if (params.providerMode !== "live-frontier") {
if (getQaProvider(params.providerMode).kind !== "live") {
return true;
}
const selected = splitModelRef(params.primaryModel);
@@ -54,7 +56,7 @@ function scenarioMatchesLiveLane(params: {
function selectQaSuiteScenarios(params: {
scenarios: ReturnType<typeof readQaBootstrapScenarioCatalog>["scenarios"];
scenarioIds?: string[];
providerMode: "mock-openai" | "live-frontier";
providerMode: QaProviderMode;
primaryModel: string;
claudeCliAuthMode?: QaCliBackendAuthMode;
}) {

View File

@@ -1,7 +1,8 @@
import { resolveQaLiveTurnTimeoutMs } from "./live-timeout.js";
import type { QaProviderMode } from "./model-selection.js";
type QaLiveTimeoutEnv = {
providerMode: "mock-openai" | "live-frontier";
providerMode: QaProviderMode;
primaryModel: string;
alternateModel: string;
};

View File

@@ -91,7 +91,7 @@ describe("qa suite runtime agent media helpers", () => {
).resolves.toBe(mediaPath);
});
it("applies mock image generation config with transport-required plugins", async () => {
it("applies provider image generation config with transport-required plugins", async () => {
await ensureImageGenerationConfigured({
providerMode: "mock-openai",
mock: { baseUrl: "http://127.0.0.1:9999" },
@@ -102,7 +102,7 @@ describe("qa suite runtime agent media helpers", () => {
expect.objectContaining({
patch: expect.objectContaining({
plugins: expect.objectContaining({
allow: expect.arrayContaining(["memory-core", "openai", "qa-channel", "browser"]),
allow: expect.arrayContaining(["memory-core", "qa-channel", "browser"]),
}),
}),
}),

View File

@@ -1,5 +1,6 @@
import fs from "node:fs/promises";
import path from "node:path";
import { buildQaImageGenerationConfigPatch } from "./providers/image-generation.js";
import {
fetchJson,
patchConfig,
@@ -70,63 +71,13 @@ async function resolveGeneratedImagePath(params: {
}
async function ensureImageGenerationConfigured(env: QaSuiteRuntimeEnv) {
const imageModelRef = "openai/gpt-image-1";
await patchConfig({
env,
patch:
env.providerMode === "mock-openai"
? {
plugins: {
allow: [...new Set(["memory-core", "openai", ...env.transport.requiredPluginIds])],
entries: {
openai: {
enabled: true,
},
},
},
models: {
providers: {
openai: {
baseUrl: `${env.mock?.baseUrl}/v1`,
apiKey: "test",
api: "openai-responses",
models: [
{
id: "gpt-image-1",
name: "gpt-image-1",
api: "openai-responses",
reasoning: false,
input: ["text"],
cost: {
input: 0,
output: 0,
cacheRead: 0,
cacheWrite: 0,
},
contextWindow: 128_000,
maxTokens: 4096,
},
],
},
},
},
agents: {
defaults: {
imageGenerationModel: {
primary: imageModelRef,
},
},
},
}
: {
agents: {
defaults: {
imageGenerationModel: {
primary: imageModelRef,
},
},
},
},
patch: buildQaImageGenerationConfigPatch({
providerMode: env.providerMode,
providerBaseUrl: env.mock ? `${env.mock.baseUrl}/v1` : undefined,
requiredPluginIds: env.transport.requiredPluginIds,
}),
});
await waitForGatewayHealthy(env);
await waitForTransportReady(env, 60_000);

View File

@@ -1,4 +1,5 @@
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-runtime";
import type { QaProviderMode } from "./model-selection.js";
import type { QaTransportActionName, QaTransportAdapter } from "./qa-transport.js";
export type QaRuntimeGatewayClient = {
@@ -21,7 +22,7 @@ export type QaSuiteRuntimeEnv = {
gateway: QaRuntimeGatewayClient;
transport: QaRuntimeTransport;
repoRoot: string;
providerMode: "mock-openai" | "live-frontier";
providerMode: QaProviderMode;
primaryModel: string;
alternateModel: string;
mock: {

View File

@@ -12,12 +12,13 @@ import type {
QaLabServerStartParams,
} from "./lab-server.types.js";
import { resolveQaLiveTurnTimeoutMs } from "./live-timeout.js";
import { startQaMockOpenAiServer } from "./mock-openai-server.js";
import {
isQaFastModeEnabled,
normalizeQaProviderMode,
type QaProviderMode,
} from "./model-selection.js";
import { DEFAULT_QA_LIVE_PROVIDER_MODE } from "./providers/index.js";
import { startQaProviderServer } from "./providers/server-runtime.js";
import type { QaThinkingLevel } from "./qa-gateway-config.js";
import {
createQaTransportAdapter,
@@ -69,7 +70,7 @@ export type QaSuiteStartLabFn = (params?: QaLabServerStartParams) => Promise<QaL
export type QaSuiteRunParams = {
repoRoot?: string;
outputDir?: string;
providerMode?: QaProviderMode | "live-openai";
providerMode?: QaProviderMode;
transportId?: QaTransportId;
primaryModel?: string;
alternateModel?: string;
@@ -195,7 +196,7 @@ async function runScenarioDefinition(
function createQaSuiteReportNotes(params: {
transport: QaTransportAdapter;
providerMode: "mock-openai" | "live-frontier";
providerMode: QaProviderMode;
primaryModel: string;
alternateModel: string;
fastMode: boolean;
@@ -331,7 +332,9 @@ async function writeQaSuiteArtifacts(params: {
export async function runQaSuite(params?: QaSuiteRunParams): Promise<QaSuiteResult> {
const startedAt = new Date();
const repoRoot = path.resolve(params?.repoRoot ?? process.cwd());
const providerMode = normalizeQaProviderMode(params?.providerMode ?? "live-frontier");
const providerMode = normalizeQaProviderMode(
params?.providerMode ?? DEFAULT_QA_LIVE_PROVIDER_MODE,
);
const transportId = normalizeQaTransportId(params?.transportId);
const primaryModel = params?.primaryModel ?? defaultQaModelForMode(providerMode);
const alternateModel = params?.alternateModel ?? defaultQaModelForMode(providerMode, true);
@@ -538,13 +541,7 @@ export async function runQaSuite(params?: QaSuiteRunParams): Promise<QaSuiteResu
id: transportId,
state: lab.state,
});
const mock =
providerMode === "mock-openai"
? await startQaMockOpenAiServer({
host: "127.0.0.1",
port: 0,
})
: null;
const mock = await startQaProviderServer(providerMode);
const gateway = await startQaGatewayChild({
repoRoot,
providerBaseUrl: mock ? `${mock.baseUrl}/v1` : undefined,

View File

@@ -1451,6 +1451,7 @@
"zod": "^4.3.6"
},
"devDependencies": {
"@copilotkit/aimock": "1.13.0",
"@grammyjs/types": "^3.26.0",
"@lit-labs/signals": "^0.2.0",
"@lit/context": "^1.1.6",

10
pnpm-lock.yaml generated
View File

@@ -255,6 +255,9 @@ importers:
specifier: ^4.3.6
version: 4.3.6
devDependencies:
'@copilotkit/aimock':
specifier: 1.13.0
version: 1.13.0
'@grammyjs/types':
specifier: ^3.26.0
version: 3.26.0
@@ -1734,6 +1737,11 @@ packages:
resolution: {integrity: sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ==}
engines: {node: '>=0.1.90'}
'@copilotkit/aimock@1.13.0':
resolution: {integrity: sha512-wcNQdicxSuaHf4MLtoC9bouKeYGcV9aBoIVJbeLtrBUhqz5LUAmo/U/S1WT69kyAEuM0uO1XyD3Jz+V+H9jsUw==}
engines: {node: '>=20.15.0'}
hasBin: true
'@create-markdown/preview@2.0.3':
resolution: {integrity: sha512-Vrp8DyuiouryZ3E4NQ7tBgoYQdoekd0+DzN64mZ48QYCw3V+MCb/H2q10SW8KC8XPr931XOMDvKX4I83qpQh3g==}
engines: {node: '>=20.0.0'}
@@ -8749,6 +8757,8 @@ snapshots:
'@colors/colors@1.5.0':
optional: true
'@copilotkit/aimock@1.13.0': {}
'@create-markdown/preview@2.0.3(shiki@3.23.0)':
optionalDependencies:
shiki: 3.23.0