mirror of
https://github.com/openclaw/openclaw.git
synced 2026-05-06 11:20:43 +00:00
QA Lab: add provider registry
Move mock and live provider behavior behind provider-owned definitions so suite, manual, Matrix, and transport lanes share defaults, auth staging, model config, and standalone server startup. Add AIMock as a first-class local provider mode while keeping mock-openai as the scenario-aware deterministic lane.
This commit is contained in:
@@ -151,6 +151,22 @@ The baseline list should stay broad enough to cover:
|
||||
- repo-reading and docs-reading
|
||||
- one small build task such as Lobster Invaders
|
||||
|
||||
## Provider mock lanes
|
||||
|
||||
`qa suite` has two local provider mock lanes:
|
||||
|
||||
- `mock-openai` is the scenario-aware OpenClaw mock. It remains the default
|
||||
deterministic mock lane for repo-backed QA and parity gates.
|
||||
- `aimock` starts an AIMock-backed provider server for experimental protocol,
|
||||
fixture, record/replay, and chaos coverage. It is additive and does not
|
||||
replace the `mock-openai` scenario dispatcher.
|
||||
|
||||
Provider-lane implementation lives under `extensions/qa-lab/src/providers/`.
|
||||
Each provider owns its defaults, local server startup, gateway model config,
|
||||
auth-profile staging needs, and live/mock capability flags. Shared suite and
|
||||
gateway code should route through the provider registry instead of branching on
|
||||
provider names.
|
||||
|
||||
## Transport adapters
|
||||
|
||||
`qa-lab` owns a generic transport seam for markdown QA scenarios.
|
||||
|
||||
@@ -52,6 +52,10 @@ These commands sit beside the main test suites when you need QA-lab realism:
|
||||
gateway workers, up to 64 workers or the selected scenario count. Use
|
||||
`--concurrency <count>` to tune the worker count, or `--concurrency 1` for
|
||||
the older serial lane.
|
||||
- Supports provider modes `live-frontier`, `mock-openai`, and `aimock`.
|
||||
`aimock` starts a local AIMock-backed provider server for experimental
|
||||
fixture and protocol-mock coverage without replacing the scenario-aware
|
||||
`mock-openai` lane.
|
||||
- `pnpm openclaw qa suite --runner multipass`
|
||||
- Runs the same QA suite inside a disposable Multipass Linux VM.
|
||||
- Keeps the same scenario-selection behavior as `qa suite` on the host.
|
||||
@@ -65,6 +69,9 @@ These commands sit beside the main test suites when you need QA-lab realism:
|
||||
`.artifacts/qa-e2e/...`.
|
||||
- `pnpm qa:lab:up`
|
||||
- Starts the Docker-backed QA site for operator-style QA work.
|
||||
- `pnpm openclaw qa aimock`
|
||||
- Starts only the local AIMock provider server for direct protocol smoke
|
||||
testing.
|
||||
- `pnpm openclaw qa matrix`
|
||||
- Runs the Matrix live QA lane against a disposable Docker-backed Tuwunel homeserver.
|
||||
- This QA host is repo/dev-only today. Packaged OpenClaw installs do not ship
|
||||
|
||||
@@ -6,7 +6,6 @@ export * from "./src/cli.js";
|
||||
export * from "./src/harness-runtime.js";
|
||||
export * from "./src/lab-server.js";
|
||||
export * from "./src/docker-harness.js";
|
||||
export * from "./src/mock-openai-server.js";
|
||||
export * from "./src/qa-agent-bootstrap.js";
|
||||
export * from "./src/qa-agent-workspace.js";
|
||||
export * from "./src/qa-gateway-config.js";
|
||||
|
||||
@@ -3,38 +3,28 @@ import path from "node:path";
|
||||
import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
|
||||
import { runQaManualLane } from "./manual-lane.runtime.js";
|
||||
import { isQaFastModeModelRef, type QaProviderMode } from "./model-selection.js";
|
||||
import {
|
||||
QA_FRONTIER_CHARACTER_EVAL_MODELS,
|
||||
QA_FRONTIER_CHARACTER_JUDGE_MODEL_OPTIONS,
|
||||
QA_FRONTIER_CHARACTER_JUDGE_MODELS,
|
||||
QA_FRONTIER_CHARACTER_THINKING_BY_MODEL,
|
||||
} from "./providers/live-frontier/character-eval.js";
|
||||
import { type QaThinkingLevel } from "./qa-gateway-config.js";
|
||||
import { extractQaVisibleReplyLeakText } from "./reply-failure.js";
|
||||
import { runQaSuiteFromRuntime } from "./suite-launch.runtime.js";
|
||||
import type { QaSuiteResult } from "./suite.js";
|
||||
|
||||
const DEFAULT_CHARACTER_SCENARIO_ID = "character-vibes-gollum";
|
||||
const DEFAULT_CHARACTER_EVAL_MODELS = Object.freeze([
|
||||
"openai/gpt-5.4",
|
||||
"openai/gpt-5.2",
|
||||
"openai/gpt-5",
|
||||
"anthropic/claude-opus-4-6",
|
||||
"anthropic/claude-sonnet-4-6",
|
||||
"zai/glm-5.1",
|
||||
"moonshot/kimi-k2.5",
|
||||
"google/gemini-3.1-pro-preview",
|
||||
]);
|
||||
const DEFAULT_CHARACTER_EVAL_MODELS = QA_FRONTIER_CHARACTER_EVAL_MODELS;
|
||||
const DEFAULT_CHARACTER_THINKING: QaThinkingLevel = "high";
|
||||
const DEFAULT_CHARACTER_EVAL_CONCURRENCY = 16;
|
||||
const DEFAULT_CHARACTER_THINKING_BY_MODEL: Readonly<Record<string, QaThinkingLevel>> =
|
||||
Object.freeze({
|
||||
"openai/gpt-5.4": "xhigh",
|
||||
"openai/gpt-5.2": "xhigh",
|
||||
"openai/gpt-5": "xhigh",
|
||||
});
|
||||
const DEFAULT_JUDGE_MODELS = Object.freeze(["openai/gpt-5.4", "anthropic/claude-opus-4-6"]);
|
||||
QA_FRONTIER_CHARACTER_THINKING_BY_MODEL;
|
||||
const DEFAULT_JUDGE_MODELS = QA_FRONTIER_CHARACTER_JUDGE_MODELS;
|
||||
const DEFAULT_JUDGE_THINKING: QaThinkingLevel = "xhigh";
|
||||
const DEFAULT_JUDGE_TIMEOUT_MS = 300_000;
|
||||
const DEFAULT_JUDGE_MODEL_OPTIONS: Readonly<Record<string, QaCharacterModelOptions>> =
|
||||
Object.freeze({
|
||||
"openai/gpt-5.4": { thinkingDefault: "xhigh" },
|
||||
"anthropic/claude-opus-4-6": { thinkingDefault: "high" },
|
||||
});
|
||||
QA_FRONTIER_CHARACTER_JUDGE_MODEL_OPTIONS;
|
||||
|
||||
type QaCharacterRunStatus = "pass" | "fail";
|
||||
|
||||
|
||||
@@ -24,14 +24,8 @@ const {
|
||||
writeQaDockerHarnessFiles: vi.fn(),
|
||||
buildQaDockerHarnessImage: vi.fn(),
|
||||
runQaDockerUp: vi.fn(),
|
||||
defaultQaRuntimeModelForMode: vi.fn<(mode: string, options?: { alternate?: boolean }) => string>(
|
||||
(mode, options) =>
|
||||
mode === "live-frontier"
|
||||
? "openai/gpt-5.4"
|
||||
: options?.alternate
|
||||
? "mock-openai/gpt-5.4-alt"
|
||||
: "mock-openai/gpt-5.4",
|
||||
),
|
||||
defaultQaRuntimeModelForMode:
|
||||
vi.fn<(mode: string, options?: { alternate?: boolean }) => string>(),
|
||||
}));
|
||||
|
||||
vi.mock("./manual-lane.runtime.js", () => ({
|
||||
@@ -83,6 +77,8 @@ import {
|
||||
runQaSuiteCommand,
|
||||
} from "./cli.runtime.js";
|
||||
import { runQaTelegramCommand } from "./live-transports/telegram/cli.runtime.js";
|
||||
import { defaultQaModelForMode as defaultQaProviderModelForMode } from "./model-selection.js";
|
||||
import type { QaProviderModeInput } from "./run-config.js";
|
||||
|
||||
describe("qa cli runtime", () => {
|
||||
let stdoutWrite: ReturnType<typeof vi.spyOn>;
|
||||
@@ -100,11 +96,7 @@ describe("qa cli runtime", () => {
|
||||
runQaDockerUp.mockReset();
|
||||
defaultQaRuntimeModelForMode.mockImplementation(
|
||||
(mode: string, options?: { alternate?: boolean }) =>
|
||||
mode === "live-frontier"
|
||||
? "openai/gpt-5.4"
|
||||
: options?.alternate
|
||||
? "mock-openai/gpt-5.4-alt"
|
||||
: "mock-openai/gpt-5.4",
|
||||
defaultQaProviderModelForMode(mode as QaProviderModeInput, options),
|
||||
);
|
||||
runQaSuiteFromRuntime.mockResolvedValue({
|
||||
watchUrl: "http://127.0.0.1:43124",
|
||||
@@ -234,22 +226,6 @@ describe("qa cli runtime", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("normalizes legacy live-openai suite runs onto the frontier provider mode", async () => {
|
||||
await runQaSuiteCommand({
|
||||
repoRoot: "/tmp/openclaw-repo",
|
||||
providerMode: "live-openai",
|
||||
scenarioIds: ["approval-turn-tool-followthrough"],
|
||||
});
|
||||
|
||||
expect(runQaSuiteFromRuntime).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
repoRoot: path.resolve("/tmp/openclaw-repo"),
|
||||
transportId: "qa-channel",
|
||||
providerMode: "live-frontier",
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("passes host suite concurrency through", async () => {
|
||||
await runQaSuiteCommand({
|
||||
repoRoot: "/tmp/openclaw-repo",
|
||||
@@ -602,6 +578,25 @@ describe("qa cli runtime", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("defaults manual aimock runs onto the aimock model lane", async () => {
|
||||
await runQaManualLaneCommand({
|
||||
repoRoot: "/tmp/openclaw-repo",
|
||||
providerMode: "aimock",
|
||||
message: "read qa kickoff and reply short",
|
||||
});
|
||||
|
||||
expect(runQaManualLane).toHaveBeenCalledWith({
|
||||
repoRoot: path.resolve("/tmp/openclaw-repo"),
|
||||
transportId: "qa-channel",
|
||||
providerMode: "aimock",
|
||||
primaryModel: "aimock/gpt-5.4",
|
||||
alternateModel: "aimock/gpt-5.4-alt",
|
||||
fastMode: undefined,
|
||||
message: "read qa kickoff and reply short",
|
||||
timeoutMs: undefined,
|
||||
});
|
||||
});
|
||||
|
||||
it("defaults manual frontier runs onto the frontier model lane", async () => {
|
||||
await runQaManualLaneCommand({
|
||||
repoRoot: "/tmp/openclaw-repo",
|
||||
@@ -640,31 +635,11 @@ describe("qa cli runtime", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("normalizes legacy live-openai manual runs onto the frontier provider mode", async () => {
|
||||
await runQaManualLaneCommand({
|
||||
repoRoot: "/tmp/openclaw-repo",
|
||||
providerMode: "live-openai",
|
||||
message: "read qa kickoff and reply short",
|
||||
});
|
||||
|
||||
expect(runQaManualLane).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
repoRoot: path.resolve("/tmp/openclaw-repo"),
|
||||
transportId: "qa-channel",
|
||||
providerMode: "live-frontier",
|
||||
primaryModel: "openai/gpt-5.4",
|
||||
alternateModel: "openai/gpt-5.4",
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("defaults manual frontier runs onto Codex OAuth when the runtime resolver prefers it", async () => {
|
||||
defaultQaRuntimeModelForMode.mockImplementation((mode, options) =>
|
||||
mode === "live-frontier"
|
||||
? "openai-codex/gpt-5.4"
|
||||
: options?.alternate
|
||||
? "mock-openai/gpt-5.4-alt"
|
||||
: "mock-openai/gpt-5.4",
|
||||
: defaultQaProviderModelForMode(mode as QaProviderModeInput, options),
|
||||
);
|
||||
|
||||
await runQaManualLaneCommand({
|
||||
|
||||
@@ -14,8 +14,13 @@ import { runQaDockerUp } from "./docker-up.runtime.js";
|
||||
import type { QaCliBackendAuthMode } from "./gateway-child.js";
|
||||
import { startQaLabServer } from "./lab-server.js";
|
||||
import { runQaManualLane } from "./manual-lane.runtime.js";
|
||||
import { startQaMockOpenAiServer } from "./mock-openai-server.js";
|
||||
import { runQaMultipass } from "./multipass.runtime.js";
|
||||
import { DEFAULT_QA_LIVE_PROVIDER_MODE, getQaProvider } from "./providers/index.js";
|
||||
import {
|
||||
QA_FRONTIER_PARITY_BASELINE_LABEL,
|
||||
QA_FRONTIER_PARITY_CANDIDATE_LABEL,
|
||||
} from "./providers/live-frontier/parity.js";
|
||||
import { startQaProviderServer } from "./providers/server-runtime.js";
|
||||
import {
|
||||
addQaCredentialSet,
|
||||
listQaCredentialSets,
|
||||
@@ -419,8 +424,8 @@ export async function runQaParityReportCommand(opts: {
|
||||
) as QaParitySuiteSummary;
|
||||
|
||||
const comparison = buildQaAgenticParityComparison({
|
||||
candidateLabel: opts.candidateLabel?.trim() || "openai/gpt-5.4",
|
||||
baselineLabel: opts.baselineLabel?.trim() || "anthropic/claude-opus-4-6",
|
||||
candidateLabel: opts.candidateLabel?.trim() || QA_FRONTIER_PARITY_CANDIDATE_LABEL,
|
||||
baselineLabel: opts.baselineLabel?.trim() || QA_FRONTIER_PARITY_BASELINE_LABEL,
|
||||
candidateSummary,
|
||||
baselineSummary,
|
||||
});
|
||||
@@ -488,7 +493,9 @@ export async function runQaManualLaneCommand(opts: {
|
||||
const repoRoot = path.resolve(opts.repoRoot ?? process.cwd());
|
||||
const transportId = normalizeQaTransportId(opts.transportId);
|
||||
const providerMode: QaProviderMode =
|
||||
opts.providerMode === undefined ? "live-frontier" : normalizeQaProviderMode(opts.providerMode);
|
||||
opts.providerMode === undefined
|
||||
? DEFAULT_QA_LIVE_PROVIDER_MODE
|
||||
: normalizeQaProviderMode(opts.providerMode);
|
||||
const models = resolveQaManualLaneModels({
|
||||
providerMode,
|
||||
primaryModel: opts.primaryModel,
|
||||
@@ -748,12 +755,23 @@ export async function runQaDockerUpCommand(opts: {
|
||||
process.stdout.write(`Stop: ${result.stopCommand}\n`);
|
||||
}
|
||||
|
||||
export async function runQaMockOpenAiCommand(opts: { host?: string; port?: number }) {
|
||||
const server = await startQaMockOpenAiServer({
|
||||
export async function runQaProviderServerCommand(
|
||||
providerMode: QaProviderMode,
|
||||
opts: { host?: string; port?: number },
|
||||
) {
|
||||
const provider = getQaProvider(providerMode);
|
||||
const standaloneCommand = provider.standaloneCommand;
|
||||
if (!standaloneCommand) {
|
||||
throw new Error(`QA provider "${providerMode}" does not expose a standalone server command.`);
|
||||
}
|
||||
const server = await startQaProviderServer(providerMode, {
|
||||
host: opts.host,
|
||||
port: Number.isFinite(opts.port) ? opts.port : undefined,
|
||||
});
|
||||
await runInterruptibleServer("QA mock OpenAI", server);
|
||||
if (!server) {
|
||||
throw new Error(`QA provider "${providerMode}" does not expose a standalone server command.`);
|
||||
}
|
||||
await runInterruptibleServer(standaloneCommand.serverLabel, server);
|
||||
}
|
||||
|
||||
export const __testing = {
|
||||
|
||||
@@ -44,11 +44,13 @@ const {
|
||||
runQaCredentialsAddCommand,
|
||||
runQaCredentialsListCommand,
|
||||
runQaCredentialsRemoveCommand,
|
||||
runQaProviderServerCommand,
|
||||
runQaTelegramCommand,
|
||||
} = vi.hoisted(() => ({
|
||||
runQaCredentialsAddCommand: vi.fn(),
|
||||
runQaCredentialsListCommand: vi.fn(),
|
||||
runQaCredentialsRemoveCommand: vi.fn(),
|
||||
runQaProviderServerCommand: vi.fn(),
|
||||
runQaTelegramCommand: vi.fn(),
|
||||
}));
|
||||
|
||||
@@ -70,6 +72,7 @@ vi.mock("./cli.runtime.js", () => ({
|
||||
runQaCredentialsAddCommand,
|
||||
runQaCredentialsListCommand,
|
||||
runQaCredentialsRemoveCommand,
|
||||
runQaProviderServerCommand,
|
||||
}));
|
||||
|
||||
import { registerQaLabCli } from "./cli.js";
|
||||
@@ -82,6 +85,7 @@ describe("qa cli registration", () => {
|
||||
runQaCredentialsAddCommand.mockReset();
|
||||
runQaCredentialsListCommand.mockReset();
|
||||
runQaCredentialsRemoveCommand.mockReset();
|
||||
runQaProviderServerCommand.mockReset();
|
||||
runQaTelegramCommand.mockReset();
|
||||
listQaRunnerCliContributions
|
||||
.mockReset()
|
||||
@@ -116,6 +120,20 @@ describe("qa cli registration", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("registers standalone provider server commands from the provider registry", async () => {
|
||||
const qa = program.commands.find((command) => command.name() === "qa");
|
||||
expect(qa?.commands.map((command) => command.name())).toEqual(
|
||||
expect.arrayContaining(["mock-openai", "aimock"]),
|
||||
);
|
||||
|
||||
await program.parseAsync(["node", "openclaw", "qa", "aimock", "--port", "44080"]);
|
||||
|
||||
expect(runQaProviderServerCommand).toHaveBeenCalledWith("aimock", {
|
||||
host: "127.0.0.1",
|
||||
port: 44080,
|
||||
});
|
||||
});
|
||||
|
||||
it("shows an enable hint when a discovered runner plugin is installed but blocked", async () => {
|
||||
listQaRunnerCliContributions.mockReset().mockReturnValue([createBlockedQaRunnerContribution()]);
|
||||
const blockedProgram = new Command();
|
||||
|
||||
@@ -1,7 +1,16 @@
|
||||
import type { Command } from "commander";
|
||||
import { collectString } from "./cli-options.js";
|
||||
import { listLiveTransportQaCliRegistrations } from "./live-transports/cli.js";
|
||||
import type { QaProviderModeInput } from "./run-config.js";
|
||||
import {
|
||||
DEFAULT_QA_LIVE_PROVIDER_MODE,
|
||||
formatQaProviderModeHelp,
|
||||
listQaStandaloneProviderCommands,
|
||||
} from "./providers/index.js";
|
||||
import {
|
||||
QA_FRONTIER_PARITY_BASELINE_LABEL,
|
||||
QA_FRONTIER_PARITY_CANDIDATE_LABEL,
|
||||
} from "./providers/live-frontier/parity.js";
|
||||
import type { QaProviderMode, QaProviderModeInput } from "./run-config.js";
|
||||
import { hasQaScenarioPack } from "./scenario-catalog.js";
|
||||
|
||||
type QaLabCliRuntime = typeof import("./cli.runtime.js");
|
||||
@@ -174,9 +183,12 @@ async function runQaDockerUp(opts: {
|
||||
await runtime.runQaDockerUpCommand(opts);
|
||||
}
|
||||
|
||||
async function runQaMockOpenAi(opts: { host?: string; port?: number }) {
|
||||
async function runQaProviderServer(
|
||||
providerMode: QaProviderMode,
|
||||
opts: { host?: string; port?: number },
|
||||
) {
|
||||
const runtime = await loadQaLabCliRuntime();
|
||||
await runtime.runQaMockOpenAiCommand(opts);
|
||||
await runtime.runQaProviderServerCommand(providerMode, opts);
|
||||
}
|
||||
|
||||
export function isQaLabCliAvailable(): boolean {
|
||||
@@ -208,11 +220,7 @@ export function registerQaLabCli(program: Command) {
|
||||
.option("--output-dir <path>", "Suite artifact directory")
|
||||
.option("--runner <kind>", "Execution runner: host or multipass", "host")
|
||||
.option("--transport <id>", "QA transport id", "qa-channel")
|
||||
.option(
|
||||
"--provider-mode <mode>",
|
||||
"Provider mode: mock-openai or live-frontier (legacy live-openai still works)",
|
||||
"live-frontier",
|
||||
)
|
||||
.option("--provider-mode <mode>", formatQaProviderModeHelp(), DEFAULT_QA_LIVE_PROVIDER_MODE)
|
||||
.option("--model <ref>", "Primary provider/model ref")
|
||||
.option("--alt-model <ref>", "Alternate provider/model ref")
|
||||
.option(
|
||||
@@ -274,8 +282,12 @@ export function registerQaLabCli(program: Command) {
|
||||
.requiredOption("--candidate-summary <path>", "Candidate qa-suite-summary.json path")
|
||||
.requiredOption("--baseline-summary <path>", "Baseline qa-suite-summary.json path")
|
||||
.option("--repo-root <path>", "Repository root to target when running from a neutral cwd")
|
||||
.option("--candidate-label <label>", "Candidate display label", "openai/gpt-5.4")
|
||||
.option("--baseline-label <label>", "Baseline display label", "anthropic/claude-opus-4-6")
|
||||
.option(
|
||||
"--candidate-label <label>",
|
||||
"Candidate display label",
|
||||
QA_FRONTIER_PARITY_CANDIDATE_LABEL,
|
||||
)
|
||||
.option("--baseline-label <label>", "Baseline display label", QA_FRONTIER_PARITY_BASELINE_LABEL)
|
||||
.option("--output-dir <path>", "Artifact directory for the parity report")
|
||||
.action(
|
||||
async (opts: {
|
||||
@@ -355,11 +367,7 @@ export function registerQaLabCli(program: Command) {
|
||||
.requiredOption("--message <text>", "Prompt to send to the QA agent")
|
||||
.option("--repo-root <path>", "Repository root to target when running from a neutral cwd")
|
||||
.option("--transport <id>", "QA transport id", "qa-channel")
|
||||
.option(
|
||||
"--provider-mode <mode>",
|
||||
"Provider mode: mock-openai or live-frontier (legacy live-openai still works)",
|
||||
"live-frontier",
|
||||
)
|
||||
.option("--provider-mode <mode>", formatQaProviderModeHelp(), DEFAULT_QA_LIVE_PROVIDER_MODE)
|
||||
.option("--model <ref>", "Primary provider/model ref (defaults by provider mode)")
|
||||
.option("--alt-model <ref>", "Alternate provider/model ref")
|
||||
.option("--fast", "Enable provider fast mode where supported", false)
|
||||
@@ -574,13 +582,15 @@ export function registerQaLabCli(program: Command) {
|
||||
},
|
||||
);
|
||||
|
||||
qa.command("mock-openai")
|
||||
.description("Run the local mock OpenAI Responses API server for QA")
|
||||
.option("--host <host>", "Bind host", "127.0.0.1")
|
||||
.option("--port <port>", "Bind port", (value: string) => Number(value))
|
||||
.action(async (opts: { host?: string; port?: number }) => {
|
||||
await runQaMockOpenAi(opts);
|
||||
});
|
||||
for (const providerCommand of listQaStandaloneProviderCommands()) {
|
||||
qa.command(providerCommand.name)
|
||||
.description(providerCommand.description)
|
||||
.option("--host <host>", "Bind host", "127.0.0.1")
|
||||
.option("--port <port>", "Bind port", (value: string) => Number(value))
|
||||
.action(async (opts: { host?: string; port?: number }) => {
|
||||
await runQaProviderServer(providerCommand.providerMode, opts);
|
||||
});
|
||||
}
|
||||
|
||||
for (const lane of listLiveTransportQaCliRegistrations()) {
|
||||
assertNoQaSubcommandCollision(qa, lane.commandName);
|
||||
|
||||
@@ -8,11 +8,6 @@ import path from "node:path";
|
||||
import { setTimeout as sleep } from "node:timers/promises";
|
||||
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-runtime";
|
||||
import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
|
||||
import {
|
||||
applyAuthProfileConfig,
|
||||
upsertAuthProfile,
|
||||
validateAnthropicSetupToken,
|
||||
} from "openclaw/plugin-sdk/provider-auth";
|
||||
import type { ModelProviderConfig } from "openclaw/plugin-sdk/provider-model-shared";
|
||||
import { fetchWithSsrFGuard } from "openclaw/plugin-sdk/ssrf-runtime";
|
||||
import { resolvePreferredOpenClawTmpDir } from "openclaw/plugin-sdk/temp-path";
|
||||
@@ -25,72 +20,27 @@ import {
|
||||
import { assertRepoBoundPath, ensureRepoBoundDirectory } from "./cli-paths.js";
|
||||
import { formatQaGatewayLogsForError, redactQaGatewayDebugText } from "./gateway-log-redaction.js";
|
||||
import { startQaGatewayRpcClient } from "./gateway-rpc-client.js";
|
||||
import { splitQaModelRef } from "./model-selection.js";
|
||||
import { splitQaModelRef, type QaProviderMode } from "./model-selection.js";
|
||||
import { resolveQaNodeExecPath } from "./node-exec.js";
|
||||
import {
|
||||
normalizeQaProviderModeEnv,
|
||||
QA_LIVE_PROVIDER_CONFIG_PATH_ENV,
|
||||
resolveQaLiveCliAuthEnv,
|
||||
resolveQaLiveProviderConfigPath,
|
||||
type QaCliBackendAuthMode,
|
||||
} from "./providers/env.js";
|
||||
import { DEFAULT_QA_PROVIDER_MODE, getQaProvider } from "./providers/index.js";
|
||||
import {
|
||||
QA_LIVE_ANTHROPIC_SETUP_TOKEN_ENV,
|
||||
QA_LIVE_SETUP_TOKEN_VALUE_ENV,
|
||||
stageQaLiveAnthropicSetupToken,
|
||||
} from "./providers/live-frontier/auth.js";
|
||||
import { stageQaMockAuthProfiles } from "./providers/shared/mock-auth.js";
|
||||
import { seedQaAgentWorkspace } from "./qa-agent-workspace.js";
|
||||
import { buildQaGatewayConfig, type QaThinkingLevel } from "./qa-gateway-config.js";
|
||||
import type { QaTransportAdapter } from "./qa-transport.js";
|
||||
|
||||
const QA_LIVE_ENV_ALIASES = Object.freeze([
|
||||
{
|
||||
liveVar: "OPENCLAW_LIVE_OPENAI_KEY",
|
||||
providerVar: "OPENAI_API_KEY",
|
||||
},
|
||||
{
|
||||
liveVar: "OPENCLAW_LIVE_ANTHROPIC_KEY",
|
||||
providerVar: "ANTHROPIC_API_KEY",
|
||||
},
|
||||
{
|
||||
liveVar: "OPENCLAW_LIVE_GEMINI_KEY",
|
||||
providerVar: "GEMINI_API_KEY",
|
||||
},
|
||||
]);
|
||||
|
||||
const QA_MOCK_BLOCKED_ENV_VARS = Object.freeze([
|
||||
"ANTHROPIC_API_KEY",
|
||||
"ANTHROPIC_OAUTH_TOKEN",
|
||||
"AWS_ACCESS_KEY_ID",
|
||||
"AWS_BEARER_TOKEN_BEDROCK",
|
||||
"AWS_REGION",
|
||||
"AWS_SECRET_ACCESS_KEY",
|
||||
"AWS_SESSION_TOKEN",
|
||||
"GEMINI_API_KEY",
|
||||
"GEMINI_API_KEYS",
|
||||
"GOOGLE_API_KEY",
|
||||
"MISTRAL_API_KEY",
|
||||
"OPENAI_API_KEY",
|
||||
"OPENAI_API_KEYS",
|
||||
"OPENAI_BASE_URL",
|
||||
"CODEX_HOME",
|
||||
"OPENCLAW_LIVE_ANTHROPIC_KEY",
|
||||
"OPENCLAW_LIVE_ANTHROPIC_KEYS",
|
||||
"OPENCLAW_LIVE_GEMINI_KEY",
|
||||
"OPENCLAW_LIVE_OPENAI_KEY",
|
||||
"VOYAGE_API_KEY",
|
||||
]);
|
||||
|
||||
const QA_MOCK_BLOCKED_ENV_KEY_PATTERNS = Object.freeze([
|
||||
/^DISCORD_/i,
|
||||
/^TELEGRAM_/i,
|
||||
/^SLACK_/i,
|
||||
/^MATRIX_/i,
|
||||
/^SIGNAL_/i,
|
||||
/^WHATSAPP_/i,
|
||||
/^IMESSAGE_/i,
|
||||
/^ZALO/i,
|
||||
/^TWILIO_/i,
|
||||
/^PLIVO_/i,
|
||||
/^NGROK_/i,
|
||||
]);
|
||||
|
||||
const QA_LIVE_PROVIDER_CONFIG_PATH_ENV = "OPENCLAW_QA_LIVE_PROVIDER_CONFIG_PATH";
|
||||
const QA_LIVE_ANTHROPIC_SETUP_TOKEN_ENV = "OPENCLAW_QA_LIVE_ANTHROPIC_SETUP_TOKEN";
|
||||
const QA_LIVE_SETUP_TOKEN_VALUE_ENV = "OPENCLAW_LIVE_SETUP_TOKEN_VALUE";
|
||||
const QA_LIVE_ANTHROPIC_SETUP_TOKEN_PROFILE_ENV = "OPENCLAW_QA_LIVE_ANTHROPIC_SETUP_TOKEN_PROFILE";
|
||||
const QA_LIVE_ANTHROPIC_SETUP_TOKEN_PROFILE_ID = "anthropic:qa-setup-token";
|
||||
const QA_LIVE_CLI_BACKEND_PRESERVE_ENV = "OPENCLAW_LIVE_CLI_BACKEND_PRESERVE_ENV";
|
||||
const QA_LIVE_CLI_BACKEND_AUTH_MODE_ENV = "OPENCLAW_LIVE_CLI_BACKEND_AUTH_MODE";
|
||||
export type QaCliBackendAuthMode = "auto" | "api-key" | "subscription";
|
||||
export type { QaCliBackendAuthMode } from "./providers/env.js";
|
||||
const QA_GATEWAY_CHILD_STARTUP_MAX_ATTEMPTS = 5;
|
||||
async function getFreePort() {
|
||||
return await new Promise<number>((resolve, reject) => {
|
||||
@@ -199,108 +149,8 @@ function appendQaGatewayTempRoot(details: string, tempRoot: string) {
|
||||
: `${details}\nQA gateway temp root preserved at ${tempRoot}`;
|
||||
}
|
||||
|
||||
export function normalizeQaProviderModeEnv(
|
||||
env: NodeJS.ProcessEnv,
|
||||
providerMode?: "mock-openai" | "live-frontier",
|
||||
) {
|
||||
if (providerMode === "mock-openai") {
|
||||
for (const key of QA_MOCK_BLOCKED_ENV_VARS) {
|
||||
delete env[key];
|
||||
}
|
||||
for (const key of Object.keys(env)) {
|
||||
if (QA_MOCK_BLOCKED_ENV_KEY_PATTERNS.some((pattern) => pattern.test(key))) {
|
||||
delete env[key];
|
||||
}
|
||||
}
|
||||
return env;
|
||||
}
|
||||
|
||||
if (providerMode === "live-frontier") {
|
||||
for (const { liveVar, providerVar } of QA_LIVE_ENV_ALIASES) {
|
||||
const liveValue = env[liveVar]?.trim();
|
||||
if (!liveValue || env[providerVar]?.trim()) {
|
||||
continue;
|
||||
}
|
||||
env[providerVar] = liveValue;
|
||||
}
|
||||
}
|
||||
|
||||
return env;
|
||||
}
|
||||
|
||||
export function resolveQaGatewayChildProviderMode(
|
||||
providerMode?: "mock-openai" | "live-frontier",
|
||||
): "mock-openai" | "live-frontier" {
|
||||
return providerMode ?? "mock-openai";
|
||||
}
|
||||
|
||||
function resolveQaLiveCliAuthEnv(
|
||||
baseEnv: NodeJS.ProcessEnv,
|
||||
opts?: {
|
||||
forwardHostHomeForClaudeCli?: boolean;
|
||||
claudeCliAuthMode?: QaCliBackendAuthMode;
|
||||
},
|
||||
) {
|
||||
const parsePreservedCliEnv = () => {
|
||||
const raw = baseEnv[QA_LIVE_CLI_BACKEND_PRESERVE_ENV]?.trim();
|
||||
if (raw?.startsWith("[")) {
|
||||
try {
|
||||
const parsed = JSON.parse(raw) as unknown;
|
||||
return Array.isArray(parsed)
|
||||
? parsed.filter((entry): entry is string => typeof entry === "string")
|
||||
: [];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
return (raw ?? "").split(/[,\s]+/).filter((entry) => entry.length > 0);
|
||||
};
|
||||
const renderPreservedCliEnv = (values: string[]) => JSON.stringify([...new Set(values)]);
|
||||
const authMode = opts?.claudeCliAuthMode ?? "auto";
|
||||
const hasAnthropicKey = Boolean(
|
||||
baseEnv.ANTHROPIC_API_KEY?.trim() || baseEnv.OPENCLAW_LIVE_ANTHROPIC_KEY?.trim(),
|
||||
);
|
||||
if (opts?.forwardHostHomeForClaudeCli && authMode === "api-key" && !hasAnthropicKey) {
|
||||
throw new Error(
|
||||
"Claude CLI API-key QA mode requires ANTHROPIC_API_KEY or OPENCLAW_LIVE_ANTHROPIC_KEY",
|
||||
);
|
||||
}
|
||||
const preserveEnvValues = (() => {
|
||||
if (!opts?.forwardHostHomeForClaudeCli) {
|
||||
return undefined;
|
||||
}
|
||||
const values = parsePreservedCliEnv().filter((entry) => entry !== "ANTHROPIC_API_KEY");
|
||||
if (authMode === "api-key" || (authMode === "auto" && hasAnthropicKey)) {
|
||||
values.push("ANTHROPIC_API_KEY");
|
||||
}
|
||||
return renderPreservedCliEnv(values);
|
||||
})();
|
||||
const claudeCliEnv = opts?.forwardHostHomeForClaudeCli
|
||||
? {
|
||||
[QA_LIVE_CLI_BACKEND_AUTH_MODE_ENV]: authMode,
|
||||
...(preserveEnvValues ? { [QA_LIVE_CLI_BACKEND_PRESERVE_ENV]: preserveEnvValues } : {}),
|
||||
}
|
||||
: {};
|
||||
const configuredCodexHome = baseEnv.CODEX_HOME?.trim();
|
||||
if (configuredCodexHome) {
|
||||
return {
|
||||
CODEX_HOME: configuredCodexHome,
|
||||
...claudeCliEnv,
|
||||
...(opts?.forwardHostHomeForClaudeCli && baseEnv.HOME?.trim()
|
||||
? { HOME: baseEnv.HOME.trim() }
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
const hostHome = baseEnv.HOME?.trim();
|
||||
if (!hostHome) {
|
||||
return {};
|
||||
}
|
||||
const codexHome = path.join(hostHome, ".codex");
|
||||
return {
|
||||
...(existsSync(codexHome) ? { CODEX_HOME: codexHome } : {}),
|
||||
...claudeCliEnv,
|
||||
...(opts?.forwardHostHomeForClaudeCli ? { HOME: hostHome } : {}),
|
||||
};
|
||||
export function resolveQaGatewayChildProviderMode(providerMode?: QaProviderMode): QaProviderMode {
|
||||
return providerMode ?? DEFAULT_QA_PROVIDER_MODE;
|
||||
}
|
||||
|
||||
export function buildQaRuntimeEnv(params: {
|
||||
@@ -314,19 +164,20 @@ export function buildQaRuntimeEnv(params: {
|
||||
xdgCacheHome: string;
|
||||
bundledPluginsDir?: string;
|
||||
compatibilityHostVersion?: string;
|
||||
providerMode?: "mock-openai" | "live-frontier";
|
||||
providerMode?: QaProviderMode;
|
||||
baseEnv?: NodeJS.ProcessEnv;
|
||||
forwardHostHomeForClaudeCli?: boolean;
|
||||
claudeCliAuthMode?: QaCliBackendAuthMode;
|
||||
}) {
|
||||
const baseEnv = params.baseEnv ?? process.env;
|
||||
const provider = params.providerMode ? getQaProvider(params.providerMode) : null;
|
||||
const forwardedHostHome = params.forwardHostHome
|
||||
? baseEnv.HOME?.trim() || os.homedir()
|
||||
: undefined;
|
||||
const env: NodeJS.ProcessEnv = {
|
||||
...baseEnv,
|
||||
HOME: forwardedHostHome ?? params.homeDir,
|
||||
...(params.providerMode === "live-frontier"
|
||||
...(provider?.appliesLiveEnvAliases
|
||||
? resolveQaLiveCliAuthEnv(baseEnv, {
|
||||
forwardHostHomeForClaudeCli: params.forwardHostHomeForClaudeCli,
|
||||
claudeCliAuthMode: params.claudeCliAuthMode,
|
||||
@@ -360,119 +211,6 @@ export function buildQaRuntimeEnv(params: {
|
||||
return normalizedEnv;
|
||||
}
|
||||
|
||||
function resolveQaLiveAnthropicSetupToken(env: NodeJS.ProcessEnv = process.env) {
|
||||
const token = (
|
||||
env[QA_LIVE_ANTHROPIC_SETUP_TOKEN_ENV]?.trim() ||
|
||||
env[QA_LIVE_SETUP_TOKEN_VALUE_ENV]?.trim() ||
|
||||
""
|
||||
).replaceAll(/\s+/g, "");
|
||||
if (!token) {
|
||||
return null;
|
||||
}
|
||||
const tokenError = validateAnthropicSetupToken(token);
|
||||
if (tokenError) {
|
||||
throw new Error(`Invalid QA Anthropic setup-token: ${tokenError}`);
|
||||
}
|
||||
const profileId =
|
||||
env[QA_LIVE_ANTHROPIC_SETUP_TOKEN_PROFILE_ENV]?.trim() ||
|
||||
QA_LIVE_ANTHROPIC_SETUP_TOKEN_PROFILE_ID;
|
||||
return { token, profileId };
|
||||
}
|
||||
|
||||
export async function stageQaLiveAnthropicSetupToken(params: {
|
||||
cfg: OpenClawConfig;
|
||||
stateDir: string;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
}): Promise<OpenClawConfig> {
|
||||
const resolved = resolveQaLiveAnthropicSetupToken(params.env);
|
||||
if (!resolved) {
|
||||
return params.cfg;
|
||||
}
|
||||
const agentDir = path.join(params.stateDir, "agents", "main", "agent");
|
||||
await fs.mkdir(agentDir, { recursive: true });
|
||||
upsertAuthProfile({
|
||||
profileId: resolved.profileId,
|
||||
credential: {
|
||||
type: "token",
|
||||
provider: "anthropic",
|
||||
token: resolved.token,
|
||||
},
|
||||
agentDir,
|
||||
});
|
||||
return applyAuthProfileConfig(params.cfg, {
|
||||
profileId: resolved.profileId,
|
||||
provider: "anthropic",
|
||||
mode: "token",
|
||||
displayName: "QA setup-token",
|
||||
});
|
||||
}
|
||||
|
||||
/** Providers the mock-openai harness stages placeholder credentials for. */
|
||||
export const QA_MOCK_AUTH_PROVIDERS = Object.freeze(["openai", "anthropic"] as const);
|
||||
|
||||
/** Agent IDs the mock-openai harness stages credentials under. */
|
||||
export const QA_MOCK_AUTH_AGENT_IDS = Object.freeze(["main", "qa"] as const);
|
||||
|
||||
export function buildQaMockProfileId(provider: string): string {
|
||||
return `qa-mock-${provider}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* In mock-openai mode the qa suite runs against the embedded mock server
|
||||
* instead of a real provider API. The mock does not validate credentials, but
|
||||
* the agent auth layer still needs a matching `api_key` auth profile in
|
||||
* `auth-profiles.json` before it will route the request through
|
||||
* `providerBaseUrl`. Without this staging step, every scenario fails with
|
||||
* `FailoverError: No API key found for provider "openai"` before the mock
|
||||
* server ever sees a request.
|
||||
*
|
||||
* Stages a placeholder `api_key` profile per provider in each of the agent
|
||||
* dirs the qa suite uses (`main` for the runtime config, `qa` for scenario
|
||||
* runs) and returns a config with matching `auth.profiles` entries so the
|
||||
* runtime accepts the profile on the first lookup.
|
||||
*
|
||||
* The placeholder value `qa-mock-not-a-real-key` is intentionally not
|
||||
* shaped like a real API key (no `sk-` prefix that would trip secret
|
||||
* scanners). It only needs to be non-empty to pass the credential
|
||||
* serializer; anything beyond that is ignored by the mock.
|
||||
*/
|
||||
export async function stageQaMockAuthProfiles(params: {
|
||||
cfg: OpenClawConfig;
|
||||
stateDir: string;
|
||||
agentIds?: readonly string[];
|
||||
providers?: readonly string[];
|
||||
}): Promise<OpenClawConfig> {
|
||||
const agentIds = [...new Set(params.agentIds ?? QA_MOCK_AUTH_AGENT_IDS)];
|
||||
const providers = [...new Set(params.providers ?? QA_MOCK_AUTH_PROVIDERS)];
|
||||
let next = params.cfg;
|
||||
for (const agentId of agentIds) {
|
||||
const agentDir = path.join(params.stateDir, "agents", agentId, "agent");
|
||||
await fs.mkdir(agentDir, { recursive: true });
|
||||
for (const provider of providers) {
|
||||
const profileId = buildQaMockProfileId(provider);
|
||||
upsertAuthProfile({
|
||||
profileId,
|
||||
credential: {
|
||||
type: "api_key",
|
||||
provider,
|
||||
key: "qa-mock-not-a-real-key",
|
||||
displayName: `QA mock ${provider} credential`,
|
||||
},
|
||||
agentDir,
|
||||
});
|
||||
}
|
||||
}
|
||||
for (const provider of providers) {
|
||||
next = applyAuthProfileConfig(next, {
|
||||
profileId: buildQaMockProfileId(provider),
|
||||
provider,
|
||||
mode: "api_key",
|
||||
displayName: `QA mock ${provider} credential`,
|
||||
});
|
||||
}
|
||||
return next;
|
||||
}
|
||||
|
||||
function isRetryableGatewayCallError(details: string): boolean {
|
||||
return (
|
||||
details.includes("handshake timeout") ||
|
||||
@@ -519,7 +257,6 @@ export const __testing = {
|
||||
redactQaGatewayDebugText,
|
||||
readQaLiveProviderConfigOverrides,
|
||||
resolveQaGatewayChildProviderMode,
|
||||
resolveQaLiveAnthropicSetupToken,
|
||||
stageQaLiveAnthropicSetupToken,
|
||||
stageQaMockAuthProfiles,
|
||||
resolveQaLiveCliAuthEnv,
|
||||
@@ -578,24 +315,6 @@ async function stopQaGatewayChildProcessTree(
|
||||
await waitForQaGatewayChildExit(child, opts?.forceTimeoutMs ?? 2_000);
|
||||
}
|
||||
|
||||
function resolveQaUserPath(value: string, env: NodeJS.ProcessEnv = process.env) {
|
||||
if (value === "~") {
|
||||
return env.HOME ?? os.homedir();
|
||||
}
|
||||
if (value.startsWith("~/")) {
|
||||
return path.join(env.HOME ?? os.homedir(), value.slice(2));
|
||||
}
|
||||
return path.resolve(value);
|
||||
}
|
||||
|
||||
function resolveQaLiveProviderConfigPath(env: NodeJS.ProcessEnv = process.env) {
|
||||
const explicit =
|
||||
env[QA_LIVE_PROVIDER_CONFIG_PATH_ENV]?.trim() || env.OPENCLAW_CONFIG_PATH?.trim();
|
||||
return explicit
|
||||
? { path: resolveQaUserPath(explicit, env), explicit: true }
|
||||
: { path: path.join(os.homedir(), ".openclaw", "openclaw.json"), explicit: false };
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return Boolean(value) && typeof value === "object" && !Array.isArray(value);
|
||||
}
|
||||
@@ -705,7 +424,7 @@ export async function startQaGatewayChild(params: {
|
||||
transport: Pick<QaTransportAdapter, "requiredPluginIds" | "createGatewayConfig">;
|
||||
transportBaseUrl: string;
|
||||
controlUiAllowedOrigins?: string[];
|
||||
providerMode?: "mock-openai" | "live-frontier";
|
||||
providerMode?: QaProviderMode;
|
||||
primaryModel?: string;
|
||||
alternateModel?: string;
|
||||
fastMode?: boolean;
|
||||
@@ -741,14 +460,14 @@ export async function startQaGatewayChild(params: {
|
||||
fs.mkdir(xdgCacheHome, { recursive: true }),
|
||||
]);
|
||||
const providerMode = resolveQaGatewayChildProviderMode(params.providerMode);
|
||||
const liveProviderIds =
|
||||
providerMode === "live-frontier"
|
||||
? [params.primaryModel, params.alternateModel]
|
||||
.map((modelRef) =>
|
||||
typeof modelRef === "string" ? splitQaModelRef(modelRef)?.provider : undefined,
|
||||
)
|
||||
.filter((providerId): providerId is string => Boolean(providerId))
|
||||
: [];
|
||||
const resolvedProvider = getQaProvider(providerMode);
|
||||
const liveProviderIds = resolvedProvider.usesModelProviderPlugins
|
||||
? [params.primaryModel, params.alternateModel]
|
||||
.map((modelRef) =>
|
||||
typeof modelRef === "string" ? splitQaModelRef(modelRef)?.provider : undefined,
|
||||
)
|
||||
.filter((providerId): providerId is string => Boolean(providerId))
|
||||
: [];
|
||||
const liveProviderConfigs = await readQaLiveProviderConfigOverrides({
|
||||
providerIds: liveProviderIds,
|
||||
});
|
||||
@@ -794,10 +513,12 @@ export async function startQaGatewayChild(params: {
|
||||
cfg,
|
||||
stateDir,
|
||||
});
|
||||
if (providerMode === "mock-openai") {
|
||||
const mockAuthProviders = getQaProvider(providerMode).mockAuthProviders;
|
||||
if (mockAuthProviders && mockAuthProviders.length > 0) {
|
||||
cfg = await stageQaMockAuthProfiles({
|
||||
cfg,
|
||||
stateDir,
|
||||
providers: mockAuthProviders,
|
||||
});
|
||||
}
|
||||
return params.mutateConfig ? params.mutateConfig(cfg) : cfg;
|
||||
|
||||
@@ -1,23 +1,8 @@
|
||||
import { QA_PROVIDER_SECRET_ENV_VARS } from "./providers/env.js";
|
||||
|
||||
const QA_GATEWAY_DEBUG_SECRET_ENV_VARS = Object.freeze([
|
||||
"ANTHROPIC_API_KEY",
|
||||
"ANTHROPIC_OAUTH_TOKEN",
|
||||
"AWS_ACCESS_KEY_ID",
|
||||
"AWS_BEARER_TOKEN_BEDROCK",
|
||||
"AWS_SECRET_ACCESS_KEY",
|
||||
"AWS_SESSION_TOKEN",
|
||||
"ANTHROPIC_API_KEYS",
|
||||
"GEMINI_API_KEY",
|
||||
"GEMINI_API_KEYS",
|
||||
"GOOGLE_API_KEY",
|
||||
"MISTRAL_API_KEY",
|
||||
"OPENAI_API_KEY",
|
||||
"OPENAI_API_KEYS",
|
||||
...QA_PROVIDER_SECRET_ENV_VARS,
|
||||
"OPENCLAW_GATEWAY_TOKEN",
|
||||
"OPENCLAW_LIVE_ANTHROPIC_KEY",
|
||||
"OPENCLAW_LIVE_ANTHROPIC_KEYS",
|
||||
"OPENCLAW_LIVE_GEMINI_KEY",
|
||||
"OPENCLAW_LIVE_OPENAI_KEY",
|
||||
"VOYAGE_API_KEY",
|
||||
]);
|
||||
|
||||
export function redactQaGatewayDebugText(text: string) {
|
||||
|
||||
@@ -1,41 +1,21 @@
|
||||
import type { QaProviderMode } from "./model-selection.js";
|
||||
import { getQaProvider } from "./providers/index.js";
|
||||
|
||||
type QaLiveTimeoutProfile = {
|
||||
providerMode: "mock-openai" | "live-frontier";
|
||||
providerMode: QaProviderMode;
|
||||
primaryModel: string;
|
||||
alternateModel: string;
|
||||
};
|
||||
|
||||
function isAnthropicModel(modelRef: string) {
|
||||
return modelRef.startsWith("anthropic/");
|
||||
}
|
||||
|
||||
function isOpenAiModel(modelRef: string) {
|
||||
return modelRef.startsWith("openai/");
|
||||
}
|
||||
|
||||
function isGptFiveModel(modelRef: string) {
|
||||
return isOpenAiModel(modelRef) && modelRef.slice("openai/".length).startsWith("gpt-5");
|
||||
}
|
||||
|
||||
function isClaudeOpusModel(modelRef: string) {
|
||||
return isAnthropicModel(modelRef) && modelRef.includes("claude-opus");
|
||||
}
|
||||
|
||||
export function resolveQaLiveTurnTimeoutMs(
|
||||
profile: QaLiveTimeoutProfile,
|
||||
fallbackMs: number,
|
||||
modelRef = profile.primaryModel,
|
||||
) {
|
||||
if (profile.providerMode === "mock-openai") {
|
||||
return fallbackMs;
|
||||
}
|
||||
if (isClaudeOpusModel(modelRef)) {
|
||||
return Math.max(fallbackMs, 240_000);
|
||||
}
|
||||
if (isAnthropicModel(modelRef)) {
|
||||
return Math.max(fallbackMs, 180_000);
|
||||
}
|
||||
if (isGptFiveModel(modelRef)) {
|
||||
return Math.max(fallbackMs, 360_000);
|
||||
}
|
||||
return Math.max(fallbackMs, 120_000);
|
||||
return getQaProvider(profile.providerMode).resolveTurnTimeoutMs({
|
||||
primaryModel: profile.primaryModel,
|
||||
alternateModel: profile.alternateModel,
|
||||
modelRef,
|
||||
fallbackMs,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1,16 +1,16 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const { startQaGatewayChild, startQaMockOpenAiServer } = vi.hoisted(() => ({
|
||||
const { startQaGatewayChild, startQaProviderServer } = vi.hoisted(() => ({
|
||||
startQaGatewayChild: vi.fn(),
|
||||
startQaMockOpenAiServer: vi.fn(),
|
||||
startQaProviderServer: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("../../gateway-child.js", () => ({
|
||||
startQaGatewayChild,
|
||||
}));
|
||||
|
||||
vi.mock("../../mock-openai-server.js", () => ({
|
||||
startQaMockOpenAiServer,
|
||||
vi.mock("../../providers/server-runtime.js", () => ({
|
||||
startQaProviderServer,
|
||||
}));
|
||||
|
||||
import { startQaLiveLaneGateway } from "./live-gateway.runtime.js";
|
||||
@@ -48,17 +48,21 @@ describe("startQaLiveLaneGateway", () => {
|
||||
gatewayCall.mockReset();
|
||||
mockStop.mockReset();
|
||||
startQaGatewayChild.mockReset();
|
||||
startQaMockOpenAiServer.mockReset();
|
||||
startQaProviderServer.mockReset();
|
||||
|
||||
startQaGatewayChild.mockResolvedValue({
|
||||
call: gatewayCall,
|
||||
cfg: {},
|
||||
stop: gatewayStop,
|
||||
});
|
||||
startQaMockOpenAiServer.mockResolvedValue({
|
||||
baseUrl: "http://127.0.0.1:44080",
|
||||
stop: mockStop,
|
||||
});
|
||||
startQaProviderServer.mockImplementation(async (providerMode: string) =>
|
||||
providerMode === "mock-openai"
|
||||
? {
|
||||
baseUrl: "http://127.0.0.1:44080",
|
||||
stop: mockStop,
|
||||
}
|
||||
: null,
|
||||
);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
@@ -76,10 +80,7 @@ describe("startQaLiveLaneGateway", () => {
|
||||
controlUiEnabled: false,
|
||||
});
|
||||
|
||||
expect(startQaMockOpenAiServer).toHaveBeenCalledWith({
|
||||
host: "127.0.0.1",
|
||||
port: 0,
|
||||
});
|
||||
expect(startQaProviderServer).toHaveBeenCalledWith("mock-openai");
|
||||
expect(startQaGatewayChild).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
transportBaseUrl: "http://127.0.0.1:43123",
|
||||
@@ -104,7 +105,7 @@ describe("startQaLiveLaneGateway", () => {
|
||||
controlUiEnabled: false,
|
||||
});
|
||||
|
||||
expect(startQaMockOpenAiServer).not.toHaveBeenCalled();
|
||||
expect(startQaProviderServer).toHaveBeenCalledWith("live-frontier");
|
||||
expect(startQaGatewayChild).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
transportBaseUrl: "http://127.0.0.1:43123",
|
||||
|
||||
@@ -1,12 +1,13 @@
|
||||
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-runtime";
|
||||
import { startQaGatewayChild, type QaCliBackendAuthMode } from "../../gateway-child.js";
|
||||
import { startQaMockOpenAiServer } from "../../mock-openai-server.js";
|
||||
import type { QaProviderMode } from "../../model-selection.js";
|
||||
import { startQaProviderServer } from "../../providers/server-runtime.js";
|
||||
import type { QaThinkingLevel } from "../../qa-gateway-config.js";
|
||||
import { appendLiveLaneIssue } from "./live-lane-helpers.js";
|
||||
|
||||
async function stopQaLiveLaneResources(resources: {
|
||||
gateway: Awaited<ReturnType<typeof startQaGatewayChild>>;
|
||||
mock: Awaited<ReturnType<typeof startQaMockOpenAiServer>> | null;
|
||||
mock: { baseUrl: string; stop(): Promise<void> } | null;
|
||||
}) {
|
||||
const errors: string[] = [];
|
||||
try {
|
||||
@@ -36,7 +37,7 @@ export async function startQaLiveLaneGateway(params: {
|
||||
};
|
||||
transportBaseUrl: string;
|
||||
controlUiAllowedOrigins?: string[];
|
||||
providerMode: "mock-openai" | "live-frontier";
|
||||
providerMode: QaProviderMode;
|
||||
primaryModel: string;
|
||||
alternateModel: string;
|
||||
fastMode?: boolean;
|
||||
@@ -45,13 +46,7 @@ export async function startQaLiveLaneGateway(params: {
|
||||
controlUiEnabled?: boolean;
|
||||
mutateConfig?: (cfg: OpenClawConfig) => OpenClawConfig;
|
||||
}) {
|
||||
const mock =
|
||||
params.providerMode === "mock-openai"
|
||||
? await startQaMockOpenAiServer({
|
||||
host: "127.0.0.1",
|
||||
port: 0,
|
||||
})
|
||||
: null;
|
||||
const mock = await startQaProviderServer(params.providerMode);
|
||||
try {
|
||||
const gateway = await startQaGatewayChild({
|
||||
repoRoot: params.repoRoot,
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import path from "node:path";
|
||||
import { resolveRepoRelativeOutputDir } from "../../cli-paths.js";
|
||||
import { DEFAULT_QA_LIVE_PROVIDER_MODE } from "../../providers/index.js";
|
||||
import type { QaProviderMode } from "../../run-config.js";
|
||||
import { normalizeQaProviderMode } from "../../run-config.js";
|
||||
import type { LiveTransportQaCommandOptions } from "./live-transport-cli.js";
|
||||
@@ -18,7 +19,7 @@ export function resolveLiveTransportQaRunOptions(
|
||||
),
|
||||
providerMode:
|
||||
opts.providerMode === undefined
|
||||
? "live-frontier"
|
||||
? DEFAULT_QA_LIVE_PROVIDER_MODE
|
||||
: normalizeQaProviderMode(opts.providerMode),
|
||||
primaryModel: opts.primaryModel,
|
||||
alternateModel: opts.alternateModel,
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import type { Command } from "commander";
|
||||
import { collectString } from "../../cli-options.js";
|
||||
import { DEFAULT_QA_LIVE_PROVIDER_MODE, formatQaProviderModeHelp } from "../../providers/index.js";
|
||||
import type { QaProviderModeInput } from "../../run-config.js";
|
||||
|
||||
export type LiveTransportQaCommandOptions = {
|
||||
@@ -78,11 +79,7 @@ export function registerLiveTransportQaCli(params: {
|
||||
.description(params.description)
|
||||
.option("--repo-root <path>", "Repository root to target when running from a neutral cwd")
|
||||
.option("--output-dir <path>", params.outputDirHelp)
|
||||
.option(
|
||||
"--provider-mode <mode>",
|
||||
"Provider mode: mock-openai or live-frontier (legacy live-openai still works)",
|
||||
"live-frontier",
|
||||
)
|
||||
.option("--provider-mode <mode>", formatQaProviderModeHelp(), DEFAULT_QA_LIVE_PROVIDER_MODE)
|
||||
.option("--model <ref>", "Primary provider/model ref")
|
||||
.option("--alt-model <ref>", "Alternate provider/model ref")
|
||||
.option("--scenario <id>", params.scenarioHelp, collectString, [])
|
||||
|
||||
@@ -6,6 +6,7 @@ import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
|
||||
import { fetchWithSsrFGuard } from "openclaw/plugin-sdk/ssrf-runtime";
|
||||
import { z } from "zod";
|
||||
import { startQaGatewayChild } from "../../gateway-child.js";
|
||||
import { DEFAULT_QA_LIVE_PROVIDER_MODE } from "../../providers/index.js";
|
||||
import {
|
||||
defaultQaModelForMode,
|
||||
normalizeQaProviderMode,
|
||||
@@ -858,7 +859,9 @@ export async function runTelegramQaLive(params: {
|
||||
};
|
||||
|
||||
const runtimeEnv = credentialLease.payload;
|
||||
const providerMode = normalizeQaProviderMode(params.providerMode ?? "live-frontier");
|
||||
const providerMode = normalizeQaProviderMode(
|
||||
params.providerMode ?? DEFAULT_QA_LIVE_PROVIDER_MODE,
|
||||
);
|
||||
const primaryModel = params.primaryModel?.trim() || defaultQaModelForMode(providerMode);
|
||||
const alternateModel = params.alternateModel?.trim() || defaultQaModelForMode(providerMode, true);
|
||||
const sutAccountId = params.sutAccountId?.trim() || "sut";
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const { startQaLabServer, startQaGatewayChild, startQaMockOpenAiServer } = vi.hoisted(() => ({
|
||||
const { startQaLabServer, startQaGatewayChild, startQaProviderServer } = vi.hoisted(() => ({
|
||||
startQaLabServer: vi.fn(),
|
||||
startQaGatewayChild: vi.fn(),
|
||||
startQaMockOpenAiServer: vi.fn(),
|
||||
startQaProviderServer: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("./lab-server.js", () => ({
|
||||
@@ -14,8 +14,8 @@ vi.mock("./gateway-child.js", () => ({
|
||||
startQaGatewayChild,
|
||||
}));
|
||||
|
||||
vi.mock("./mock-openai-server.js", () => ({
|
||||
startQaMockOpenAiServer,
|
||||
vi.mock("./providers/server-runtime.js", () => ({
|
||||
startQaProviderServer,
|
||||
}));
|
||||
|
||||
import { runQaManualLane } from "./manual-lane.runtime.js";
|
||||
@@ -31,7 +31,7 @@ describe("runQaManualLane", () => {
|
||||
labStop.mockReset();
|
||||
startQaLabServer.mockReset();
|
||||
startQaGatewayChild.mockReset();
|
||||
startQaMockOpenAiServer.mockReset();
|
||||
startQaProviderServer.mockReset();
|
||||
|
||||
startQaLabServer.mockResolvedValue({
|
||||
listenUrl: "http://127.0.0.1:43124",
|
||||
@@ -64,10 +64,14 @@ describe("runQaManualLane", () => {
|
||||
stop: gatewayStop,
|
||||
});
|
||||
|
||||
startQaMockOpenAiServer.mockResolvedValue({
|
||||
baseUrl: "http://127.0.0.1:44080",
|
||||
stop: mockStop,
|
||||
});
|
||||
startQaProviderServer.mockImplementation(async (providerMode: string) =>
|
||||
providerMode === "mock-openai"
|
||||
? {
|
||||
baseUrl: "http://127.0.0.1:44080",
|
||||
stop: mockStop,
|
||||
}
|
||||
: null,
|
||||
);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
@@ -84,10 +88,7 @@ describe("runQaManualLane", () => {
|
||||
timeoutMs: 5_000,
|
||||
});
|
||||
|
||||
expect(startQaMockOpenAiServer).toHaveBeenCalledWith({
|
||||
host: "127.0.0.1",
|
||||
port: 0,
|
||||
});
|
||||
expect(startQaProviderServer).toHaveBeenCalledWith("mock-openai");
|
||||
expect(startQaGatewayChild).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
repoRoot: "/tmp/openclaw-repo",
|
||||
@@ -115,7 +116,7 @@ describe("runQaManualLane", () => {
|
||||
timeoutMs: 5_000,
|
||||
});
|
||||
|
||||
expect(startQaMockOpenAiServer).not.toHaveBeenCalled();
|
||||
expect(startQaProviderServer).toHaveBeenCalledWith("live-frontier");
|
||||
expect(startQaLabServer).toHaveBeenCalledWith({
|
||||
repoRoot: "/tmp/openclaw-repo",
|
||||
embeddedGateway: "disabled",
|
||||
|
||||
@@ -4,14 +4,15 @@ import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
|
||||
import { startQaGatewayChild } from "./gateway-child.js";
|
||||
import { startQaLabServer } from "./lab-server.js";
|
||||
import { resolveQaLiveTurnTimeoutMs } from "./live-timeout.js";
|
||||
import { startQaMockOpenAiServer } from "./mock-openai-server.js";
|
||||
import type { QaProviderMode } from "./model-selection.js";
|
||||
import { startQaProviderServer } from "./providers/server-runtime.js";
|
||||
import type { QaThinkingLevel } from "./qa-gateway-config.js";
|
||||
import { createQaTransportAdapter, type QaTransportId } from "./qa-transport-registry.js";
|
||||
|
||||
type QaManualLaneParams = {
|
||||
repoRoot: string;
|
||||
transportId?: QaTransportId;
|
||||
providerMode: "mock-openai" | "live-frontier";
|
||||
providerMode: QaProviderMode;
|
||||
primaryModel: string;
|
||||
alternateModel: string;
|
||||
fastMode?: boolean;
|
||||
@@ -21,7 +22,7 @@ type QaManualLaneParams = {
|
||||
};
|
||||
|
||||
function resolveManualLaneTimeoutMs(params: {
|
||||
providerMode: "mock-openai" | "live-frontier";
|
||||
providerMode: QaProviderMode;
|
||||
primaryModel: string;
|
||||
alternateModel: string;
|
||||
timeoutMs?: number;
|
||||
@@ -54,13 +55,7 @@ export async function runQaManualLane(params: QaManualLaneParams) {
|
||||
id: params.transportId ?? "qa-channel",
|
||||
state: lab.state,
|
||||
});
|
||||
const mock =
|
||||
params.providerMode === "mock-openai"
|
||||
? await startQaMockOpenAiServer({
|
||||
host: "127.0.0.1",
|
||||
port: 0,
|
||||
})
|
||||
: null;
|
||||
const mock = await startQaProviderServer(params.providerMode);
|
||||
const gateway = await startQaGatewayChild({
|
||||
repoRoot: params.repoRoot,
|
||||
providerBaseUrl: mock ? `${mock.baseUrl}/v1` : undefined,
|
||||
|
||||
@@ -2,14 +2,18 @@ import { spawn } from "node:child_process";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { resolvePreferredOpenClawTmpDir } from "openclaw/plugin-sdk/temp-path";
|
||||
import { resolveQaNodeExecPath } from "./node-exec.js";
|
||||
import {
|
||||
isPreferredQaLiveFrontierCatalogModel,
|
||||
QA_FRONTIER_CATALOG_ALTERNATE_MODEL,
|
||||
QA_FRONTIER_CATALOG_PRIMARY_MODEL,
|
||||
QA_FRONTIER_PROVIDER_IDS,
|
||||
} from "./providers/live-frontier/catalog.js";
|
||||
import {
|
||||
createQaChannelGatewayConfig,
|
||||
QA_CHANNEL_REQUIRED_PLUGIN_IDS,
|
||||
} from "./qa-channel-transport.js";
|
||||
import { buildQaGatewayConfig } from "./qa-gateway-config.js";
|
||||
import { resolveQaNodeExecPath } from "./node-exec.js";
|
||||
|
||||
const QA_FRONTIER_PROVIDER_IDS = ["anthropic", "google", "openai"] as const;
|
||||
|
||||
type ModelRow = {
|
||||
key: string;
|
||||
@@ -48,7 +52,7 @@ export function selectQaRunnerModelOptions(rows: ModelRow[]): QaRunnerModelOptio
|
||||
name: row.name,
|
||||
provider: parsed?.provider ?? "unknown",
|
||||
input: row.input,
|
||||
preferred: row.key === "openai/gpt-5.4",
|
||||
preferred: isPreferredQaLiveFrontierCatalogModel(row.key),
|
||||
} satisfies QaRunnerModelOption;
|
||||
});
|
||||
|
||||
@@ -110,8 +114,8 @@ export async function loadQaRunnerModelOptions(params: { repoRoot: string; signa
|
||||
gatewayToken: "qa-model-catalog",
|
||||
workspaceDir,
|
||||
providerMode: "live-frontier",
|
||||
primaryModel: "openai/gpt-5.4",
|
||||
alternateModel: "anthropic/claude-sonnet-4-6",
|
||||
primaryModel: QA_FRONTIER_CATALOG_PRIMARY_MODEL,
|
||||
alternateModel: QA_FRONTIER_CATALOG_ALTERNATE_MODEL,
|
||||
enabledProviderIds: [...QA_FRONTIER_PROVIDER_IDS],
|
||||
imageGenerationModel: null,
|
||||
controlUiEnabled: false,
|
||||
@@ -128,24 +132,20 @@ export async function loadQaRunnerModelOptions(params: { repoRoot: string; signa
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
let aborted = params.signal?.aborted === true;
|
||||
let forceKillTimer: NodeJS.Timeout | undefined;
|
||||
const child = spawn(
|
||||
nodeExecPath,
|
||||
["dist/index.js", "models", "list", "--all", "--json"],
|
||||
{
|
||||
cwd: params.repoRoot,
|
||||
env: {
|
||||
...process.env,
|
||||
HOME: homeDir,
|
||||
OPENCLAW_HOME: homeDir,
|
||||
OPENCLAW_CONFIG_PATH: configPath,
|
||||
OPENCLAW_STATE_DIR: stateDir,
|
||||
OPENCLAW_OAUTH_DIR: path.join(stateDir, "credentials"),
|
||||
OPENCLAW_CODEX_DISCOVERY_LIVE: "0",
|
||||
},
|
||||
detached: process.platform !== "win32",
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
const child = spawn(nodeExecPath, ["dist/index.js", "models", "list", "--all", "--json"], {
|
||||
cwd: params.repoRoot,
|
||||
env: {
|
||||
...process.env,
|
||||
HOME: homeDir,
|
||||
OPENCLAW_HOME: homeDir,
|
||||
OPENCLAW_CONFIG_PATH: configPath,
|
||||
OPENCLAW_STATE_DIR: stateDir,
|
||||
OPENCLAW_OAUTH_DIR: path.join(stateDir, "credentials"),
|
||||
OPENCLAW_CODEX_DISCOVERY_LIVE: "0",
|
||||
},
|
||||
);
|
||||
detached: process.platform !== "win32",
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
});
|
||||
const cleanup = () => {
|
||||
params.signal?.removeEventListener("abort", abortCatalogLoad);
|
||||
if (forceKillTimer) {
|
||||
|
||||
@@ -56,7 +56,7 @@ describe("qa model selection runtime", () => {
|
||||
expect(defaultQaRuntimeModelForMode("live-frontier")).toBe("openai/gpt-5.4");
|
||||
});
|
||||
|
||||
it("leaves mock-openai defaults unchanged", () => {
|
||||
it("leaves mock defaults unchanged", () => {
|
||||
listProfilesForProvider.mockImplementation((_store: unknown, provider: string) =>
|
||||
provider === "openai-codex" ? ["openai-codex:user@example.com"] : [],
|
||||
);
|
||||
@@ -65,5 +65,7 @@ describe("qa model selection runtime", () => {
|
||||
expect(defaultQaRuntimeModelForMode("mock-openai", { alternate: true })).toBe(
|
||||
"mock-openai/gpt-5.4-alt",
|
||||
);
|
||||
expect(defaultQaRuntimeModelForMode("aimock")).toBe("aimock/gpt-5.4");
|
||||
expect(defaultQaRuntimeModelForMode("aimock", { alternate: true })).toBe("aimock/gpt-5.4-alt");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,30 +1,13 @@
|
||||
import {
|
||||
listProfilesForProvider,
|
||||
loadAuthProfileStoreForRuntime,
|
||||
} from "openclaw/plugin-sdk/agent-runtime";
|
||||
import { resolveEnvApiKey } from "openclaw/plugin-sdk/provider-auth";
|
||||
import { defaultQaModelForMode, type QaProviderModeInput } from "./model-selection.js";
|
||||
|
||||
const QA_CODEX_OAUTH_LIVE_MODEL = "openai-codex/gpt-5.4";
|
||||
defaultQaModelForMode,
|
||||
normalizeQaProviderMode,
|
||||
type QaProviderModeInput,
|
||||
} from "./model-selection.js";
|
||||
import { DEFAULT_QA_LIVE_PROVIDER_MODE } from "./providers/index.js";
|
||||
import { resolveQaLiveFrontierPreferredModel } from "./providers/live-frontier/model-selection.runtime.js";
|
||||
|
||||
export function resolveQaPreferredLiveModel() {
|
||||
if (resolveEnvApiKey("openai")?.apiKey) {
|
||||
return undefined;
|
||||
}
|
||||
try {
|
||||
const store = loadAuthProfileStoreForRuntime(undefined, {
|
||||
readOnly: true,
|
||||
allowKeychainPrompt: false,
|
||||
});
|
||||
if (listProfilesForProvider(store, "openai").length > 0) {
|
||||
return undefined;
|
||||
}
|
||||
return listProfilesForProvider(store, "openai-codex").length > 0
|
||||
? QA_CODEX_OAUTH_LIVE_MODEL
|
||||
: undefined;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
return resolveQaLiveFrontierPreferredModel();
|
||||
}
|
||||
|
||||
export function defaultQaRuntimeModelForMode(
|
||||
@@ -34,8 +17,13 @@ export function defaultQaRuntimeModelForMode(
|
||||
preferredLiveModel?: string;
|
||||
},
|
||||
) {
|
||||
const preferredLiveModel =
|
||||
options?.preferredLiveModel ??
|
||||
(normalizeQaProviderMode(mode) === DEFAULT_QA_LIVE_PROVIDER_MODE
|
||||
? resolveQaPreferredLiveModel()
|
||||
: undefined);
|
||||
return defaultQaModelForMode(mode, {
|
||||
...options,
|
||||
preferredLiveModel: options?.preferredLiveModel ?? resolveQaPreferredLiveModel(),
|
||||
preferredLiveModel,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1,14 +1,18 @@
|
||||
export type QaProviderMode = "mock-openai" | "live-frontier";
|
||||
export type QaProviderModeInput = QaProviderMode | "live-openai";
|
||||
import {
|
||||
DEFAULT_QA_LIVE_PROVIDER_MODE,
|
||||
getQaProvider,
|
||||
type QaProviderMode,
|
||||
type QaProviderModeInput,
|
||||
} from "./providers/index.js";
|
||||
|
||||
export type { QaProviderMode, QaProviderModeInput } from "./providers/index.js";
|
||||
|
||||
export type QaModelSelection = {
|
||||
primaryModel: string;
|
||||
alternateModel: string;
|
||||
};
|
||||
|
||||
export function normalizeQaProviderMode(mode: QaProviderModeInput): QaProviderMode {
|
||||
return mode === "live-openai" ? "live-frontier" : mode;
|
||||
}
|
||||
export { normalizeQaProviderMode } from "./providers/index.js";
|
||||
|
||||
export function defaultQaModelForMode(
|
||||
mode: QaProviderModeInput,
|
||||
@@ -17,10 +21,7 @@ export function defaultQaModelForMode(
|
||||
preferredLiveModel?: string;
|
||||
},
|
||||
) {
|
||||
if (normalizeQaProviderMode(mode) === "live-frontier") {
|
||||
return options?.preferredLiveModel ?? "openai/gpt-5.4";
|
||||
}
|
||||
return options?.alternate ? "mock-openai/gpt-5.4-alt" : "mock-openai/gpt-5.4";
|
||||
return getQaProvider(mode).defaultModel(options);
|
||||
}
|
||||
|
||||
export function splitQaModelRef(ref: string) {
|
||||
@@ -35,7 +36,7 @@ export function splitQaModelRef(ref: string) {
|
||||
}
|
||||
|
||||
export function isQaFastModeModelRef(ref: string) {
|
||||
return splitQaModelRef(ref)?.provider === "openai";
|
||||
return getQaProvider(DEFAULT_QA_LIVE_PROVIDER_MODE).usesFastModeByDefault(ref);
|
||||
}
|
||||
|
||||
export function isQaFastModeEnabled(selection: QaModelSelection) {
|
||||
|
||||
@@ -2,9 +2,11 @@ import { execFile } from "node:child_process";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import fs from "node:fs";
|
||||
import { access, appendFile, mkdir, writeFile } from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { resolvePreferredOpenClawTmpDir } from "openclaw/plugin-sdk/temp-path";
|
||||
import type { QaProviderMode } from "./model-selection.js";
|
||||
import { resolveQaForwardedLiveEnv, resolveQaLiveProviderConfigPath } from "./providers/env.js";
|
||||
import { DEFAULT_QA_LIVE_PROVIDER_MODE, getQaProvider } from "./providers/index.js";
|
||||
|
||||
const MULTIPASS_MOUNTED_REPO_PATH = "/workspace/openclaw-host";
|
||||
const MULTIPASS_GUEST_REPO_PATH = "/workspace/openclaw";
|
||||
@@ -30,50 +32,6 @@ const MULTIPASS_REPO_SYNC_EXCLUDES = [
|
||||
const MULTIPASS_EXEC_MAX_BUFFER = 64 * 1024 * 1024;
|
||||
const MULTIPASS_GUEST_RUN_TIMEOUT_MS = 60 * 60 * 1000;
|
||||
|
||||
const QA_LIVE_ENV_ALIASES = Object.freeze([
|
||||
{
|
||||
liveVar: "OPENCLAW_LIVE_OPENAI_KEY",
|
||||
providerVar: "OPENAI_API_KEY",
|
||||
},
|
||||
{
|
||||
liveVar: "OPENCLAW_LIVE_ANTHROPIC_KEY",
|
||||
providerVar: "ANTHROPIC_API_KEY",
|
||||
},
|
||||
{
|
||||
liveVar: "OPENCLAW_LIVE_GEMINI_KEY",
|
||||
providerVar: "GEMINI_API_KEY",
|
||||
},
|
||||
]);
|
||||
|
||||
const QA_LIVE_ALLOWED_ENV_VARS = Object.freeze([
|
||||
"ANTHROPIC_API_KEY",
|
||||
"ANTHROPIC_OAUTH_TOKEN",
|
||||
"AWS_ACCESS_KEY_ID",
|
||||
"AWS_BEARER_TOKEN_BEDROCK",
|
||||
"AWS_REGION",
|
||||
"AWS_SECRET_ACCESS_KEY",
|
||||
"AWS_SESSION_TOKEN",
|
||||
"GEMINI_API_KEY",
|
||||
"GEMINI_API_KEYS",
|
||||
"GOOGLE_API_KEY",
|
||||
"MISTRAL_API_KEY",
|
||||
"OPENAI_API_KEY",
|
||||
"OPENAI_API_KEYS",
|
||||
"OPENAI_BASE_URL",
|
||||
"OPENCLAW_LIVE_ANTHROPIC_KEY",
|
||||
"OPENCLAW_LIVE_ANTHROPIC_KEYS",
|
||||
"OPENCLAW_LIVE_GEMINI_KEY",
|
||||
"OPENCLAW_LIVE_OPENAI_KEY",
|
||||
"OPENCLAW_QA_LIVE_PROVIDER_CONFIG_PATH",
|
||||
"OPENCLAW_CONFIG_PATH",
|
||||
"VOYAGE_API_KEY",
|
||||
]);
|
||||
const QA_LIVE_ALLOWED_ENV_PATTERNS = Object.freeze([
|
||||
/^[A-Z0-9_]+_API_KEYS$/u,
|
||||
/^[A-Z0-9_]+_API_KEY_[0-9]+$/u,
|
||||
/^OPENCLAW_LIVE_[A-Z0-9_]+_KEYS$/u,
|
||||
]);
|
||||
|
||||
export const qaMultipassDefaultResources = {
|
||||
image: "lts",
|
||||
cpus: 2,
|
||||
@@ -109,7 +67,7 @@ export type QaMultipassPlan = {
|
||||
disk: string;
|
||||
pnpmVersion: string;
|
||||
transportId: string;
|
||||
providerMode: "mock-openai" | "live-frontier";
|
||||
providerMode: QaProviderMode;
|
||||
primaryModel?: string;
|
||||
alternateModel?: string;
|
||||
fastMode?: boolean;
|
||||
@@ -256,63 +214,6 @@ function resolveMultipassInstallHint() {
|
||||
return "https://multipass.run/install";
|
||||
}
|
||||
|
||||
function resolveUserPath(value: string, env: NodeJS.ProcessEnv = process.env) {
|
||||
if (value === "~") {
|
||||
return env.HOME ?? os.homedir();
|
||||
}
|
||||
if (value.startsWith("~/")) {
|
||||
return path.join(env.HOME ?? os.homedir(), value.slice(2));
|
||||
}
|
||||
return path.resolve(value);
|
||||
}
|
||||
|
||||
function resolveLiveProviderConfigPath(env: NodeJS.ProcessEnv = process.env) {
|
||||
const explicit =
|
||||
env.OPENCLAW_QA_LIVE_PROVIDER_CONFIG_PATH?.trim() || env.OPENCLAW_CONFIG_PATH?.trim();
|
||||
return explicit
|
||||
? { path: resolveUserPath(explicit, env), explicit: true }
|
||||
: { path: path.join(os.homedir(), ".openclaw", "openclaw.json"), explicit: false };
|
||||
}
|
||||
|
||||
function resolveQaLiveCliAuthEnv(baseEnv: NodeJS.ProcessEnv) {
|
||||
const configuredCodexHome = baseEnv.CODEX_HOME?.trim();
|
||||
if (configuredCodexHome) {
|
||||
const codexHome = resolveUserPath(configuredCodexHome, baseEnv);
|
||||
return fs.existsSync(codexHome) ? { CODEX_HOME: codexHome } : {};
|
||||
}
|
||||
const hostHome = baseEnv.HOME?.trim();
|
||||
const effectiveHome = hostHome || os.homedir();
|
||||
const codexHome = path.join(effectiveHome, ".codex");
|
||||
return fs.existsSync(codexHome) ? { CODEX_HOME: codexHome } : {};
|
||||
}
|
||||
|
||||
function resolveForwardedLiveEnv(baseEnv: NodeJS.ProcessEnv = process.env) {
|
||||
const forwarded: Record<string, string> = {};
|
||||
for (const [key, rawValue] of Object.entries(baseEnv)) {
|
||||
if (
|
||||
!QA_LIVE_ALLOWED_ENV_VARS.includes(key) &&
|
||||
!QA_LIVE_ALLOWED_ENV_PATTERNS.some((pattern) => pattern.test(key))
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
const value = rawValue?.trim();
|
||||
if (value) {
|
||||
forwarded[key] = value;
|
||||
}
|
||||
}
|
||||
for (const { liveVar, providerVar } of QA_LIVE_ENV_ALIASES) {
|
||||
const liveValue = forwarded[liveVar]?.trim();
|
||||
if (liveValue && !forwarded[providerVar]?.trim()) {
|
||||
forwarded[providerVar] = liveValue;
|
||||
}
|
||||
}
|
||||
const liveCliAuth = resolveQaLiveCliAuthEnv(baseEnv);
|
||||
if (liveCliAuth.CODEX_HOME) {
|
||||
forwarded.CODEX_HOME = liveCliAuth.CODEX_HOME;
|
||||
}
|
||||
return forwarded;
|
||||
}
|
||||
|
||||
function createQaMultipassOutputDir(repoRoot: string) {
|
||||
return path.join(repoRoot, ".artifacts", "qa-e2e", `multipass-${createOutputStamp()}`);
|
||||
}
|
||||
@@ -332,7 +233,7 @@ export function createQaMultipassPlan(params: {
|
||||
repoRoot: string;
|
||||
outputDir?: string;
|
||||
transportId?: string;
|
||||
providerMode?: "mock-openai" | "live-frontier";
|
||||
providerMode?: QaProviderMode;
|
||||
primaryModel?: string;
|
||||
alternateModel?: string;
|
||||
fastMode?: boolean;
|
||||
@@ -346,11 +247,13 @@ export function createQaMultipassPlan(params: {
|
||||
const outputDir = params.outputDir ?? createQaMultipassOutputDir(params.repoRoot);
|
||||
const scenarioIds = [...new Set(params.scenarioIds ?? [])];
|
||||
const transportId = params.transportId?.trim() || "qa-channel";
|
||||
const providerMode = params.providerMode ?? "live-frontier";
|
||||
const forwardedEnv = providerMode === "live-frontier" ? resolveForwardedLiveEnv() : {};
|
||||
const providerMode = params.providerMode ?? DEFAULT_QA_LIVE_PROVIDER_MODE;
|
||||
const provider = getQaProvider(providerMode);
|
||||
const forwardedEnv = provider.appliesLiveEnvAliases ? resolveQaForwardedLiveEnv() : {};
|
||||
const hostCodexHomePath = forwardedEnv.CODEX_HOME;
|
||||
const liveProviderConfig =
|
||||
providerMode === "live-frontier" ? resolveLiveProviderConfigPath() : undefined;
|
||||
const liveProviderConfig = provider.usesModelProviderPlugins
|
||||
? resolveQaLiveProviderConfigPath()
|
||||
: undefined;
|
||||
const hostLiveProviderConfigPath =
|
||||
liveProviderConfig && fs.existsSync(liveProviderConfig.path)
|
||||
? liveProviderConfig.path
|
||||
@@ -637,7 +540,7 @@ export async function runQaMultipass(params: {
|
||||
repoRoot: string;
|
||||
outputDir?: string;
|
||||
transportId?: string;
|
||||
providerMode?: "mock-openai" | "live-frontier";
|
||||
providerMode?: QaProviderMode;
|
||||
primaryModel?: string;
|
||||
alternateModel?: string;
|
||||
fastMode?: boolean;
|
||||
|
||||
21
extensions/qa-lab/src/providers/README.md
Normal file
21
extensions/qa-lab/src/providers/README.md
Normal file
@@ -0,0 +1,21 @@
|
||||
# QA Provider Lanes
|
||||
|
||||
QA provider lanes are registered in `index.ts` and implemented in one folder per
|
||||
provider. Shared provider contracts and mock-provider helpers live in `shared/`.
|
||||
Mock lanes should use `shared/mock-provider-definition.ts` unless they need a
|
||||
custom shape.
|
||||
|
||||
Each provider definition owns:
|
||||
|
||||
- its accepted `providerMode`
|
||||
- default primary, alternate, and image-generation model refs
|
||||
- gateway `models.providers` config, when the lane needs config injection
|
||||
- model runtime params such as SSE transport, fast mode, or thinking defaults
|
||||
- optional local server startup for mock lanes
|
||||
- optional placeholder auth profile providers for mock lanes
|
||||
- whether the lane uses real provider plugins and live environment aliases
|
||||
|
||||
Shared suite code should import only `providers/index.ts` and ask the selected
|
||||
provider for behavior. Do not add provider-name branches to suite, gateway,
|
||||
manual-lane, or live-transport runtime code unless the registry contract is
|
||||
missing a needed capability.
|
||||
9
extensions/qa-lab/src/providers/aimock/index.ts
Normal file
9
extensions/qa-lab/src/providers/aimock/index.ts
Normal file
@@ -0,0 +1,9 @@
|
||||
import { createMockQaProviderDefinition } from "../shared/mock-provider-definition.js";
|
||||
|
||||
export const aimockProviderDefinition = createMockQaProviderDefinition({
|
||||
mode: "aimock",
|
||||
commandName: "aimock",
|
||||
commandDescription: "Run the local AIMock provider server for QA",
|
||||
serverLabel: "QA AIMock",
|
||||
mockAuthProviders: ["aimock", "openai", "anthropic"],
|
||||
});
|
||||
82
extensions/qa-lab/src/providers/aimock/server.test.ts
Normal file
82
extensions/qa-lab/src/providers/aimock/server.test.ts
Normal file
@@ -0,0 +1,82 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { startQaAimockServer } from "./server.js";
|
||||
|
||||
function makeResponsesInput(text: string) {
|
||||
return {
|
||||
role: "user",
|
||||
content: [
|
||||
{
|
||||
type: "input_text",
|
||||
text,
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
describe("qa aimock server", () => {
|
||||
it("serves OpenAI Responses text replies and debug request snapshots", async () => {
|
||||
const server = await startQaAimockServer({
|
||||
host: "127.0.0.1",
|
||||
port: 0,
|
||||
});
|
||||
try {
|
||||
const response = await fetch(`${server.baseUrl}/v1/responses`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
model: "aimock/gpt-5.4",
|
||||
stream: false,
|
||||
input: [makeResponsesInput("hello aimock")],
|
||||
}),
|
||||
});
|
||||
expect(response.status).toBe(200);
|
||||
expect(await response.json()).toMatchObject({
|
||||
status: "completed",
|
||||
model: "aimock/gpt-5.4",
|
||||
});
|
||||
|
||||
const debug = await fetch(`${server.baseUrl}/debug/last-request`);
|
||||
expect(debug.status).toBe(200);
|
||||
expect(await debug.json()).toMatchObject({
|
||||
prompt: "hello aimock",
|
||||
allInputText: "hello aimock",
|
||||
model: "aimock/gpt-5.4",
|
||||
providerVariant: "openai",
|
||||
});
|
||||
} finally {
|
||||
await server.stop();
|
||||
}
|
||||
});
|
||||
|
||||
it("records the request list for scenario assertions", async () => {
|
||||
const server = await startQaAimockServer({
|
||||
host: "127.0.0.1",
|
||||
port: 0,
|
||||
});
|
||||
try {
|
||||
const response = await fetch(`${server.baseUrl}/v1/responses`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
model: "aimock/gpt-5.4",
|
||||
stream: false,
|
||||
input: [makeResponsesInput("@openclaw explain the QA lab")],
|
||||
}),
|
||||
});
|
||||
expect(response.status).toBe(200);
|
||||
expect(await response.json()).toMatchObject({
|
||||
status: "completed",
|
||||
});
|
||||
|
||||
const debug = await fetch(`${server.baseUrl}/debug/requests`);
|
||||
expect(debug.status).toBe(200);
|
||||
expect(await debug.json()).toEqual([
|
||||
expect.objectContaining({
|
||||
prompt: "@openclaw explain the QA lab",
|
||||
}),
|
||||
]);
|
||||
} finally {
|
||||
await server.stop();
|
||||
}
|
||||
});
|
||||
});
|
||||
203
extensions/qa-lab/src/providers/aimock/server.ts
Normal file
203
extensions/qa-lab/src/providers/aimock/server.ts
Normal file
@@ -0,0 +1,203 @@
|
||||
import type { IncomingMessage, ServerResponse } from "node:http";
|
||||
import {
|
||||
LLMock,
|
||||
type ChatCompletionRequest,
|
||||
type JournalEntry,
|
||||
type Mountable,
|
||||
} from "@copilotkit/aimock";
|
||||
|
||||
type AimockRequestSnapshot = {
|
||||
raw: string;
|
||||
body: Record<string, unknown>;
|
||||
prompt: string;
|
||||
allInputText: string;
|
||||
toolOutput: string;
|
||||
model: string;
|
||||
providerVariant: "openai" | "anthropic" | "unknown";
|
||||
imageInputCount: number;
|
||||
plannedToolName?: string;
|
||||
};
|
||||
|
||||
function writeJson(res: ServerResponse, status: number, body: unknown) {
|
||||
const text = JSON.stringify(body);
|
||||
res.writeHead(status, {
|
||||
"content-type": "application/json; charset=utf-8",
|
||||
"content-length": Buffer.byteLength(text),
|
||||
"cache-control": "no-store",
|
||||
});
|
||||
res.end(text);
|
||||
}
|
||||
|
||||
function stringifyContent(content: unknown): string {
|
||||
if (typeof content === "string") {
|
||||
return content;
|
||||
}
|
||||
if (Array.isArray(content)) {
|
||||
return content
|
||||
.map((part) => stringifyContent(part))
|
||||
.filter(Boolean)
|
||||
.join("\n");
|
||||
}
|
||||
if (content && typeof content === "object") {
|
||||
const record = content as Record<string, unknown>;
|
||||
if (typeof record.text === "string") {
|
||||
return record.text;
|
||||
}
|
||||
if (typeof record.content === "string") {
|
||||
return record.content;
|
||||
}
|
||||
if (typeof record.output === "string") {
|
||||
return record.output;
|
||||
}
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
function requestMessages(body: ChatCompletionRequest | null | undefined) {
|
||||
return Array.isArray(body?.messages) ? body.messages : [];
|
||||
}
|
||||
|
||||
function extractLastUserText(body: ChatCompletionRequest | null | undefined) {
|
||||
const messages = requestMessages(body);
|
||||
for (let index = messages.length - 1; index >= 0; index -= 1) {
|
||||
const message = messages[index];
|
||||
if (message?.role === "user") {
|
||||
return stringifyContent(message.content);
|
||||
}
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
function extractAllInputText(body: ChatCompletionRequest | null | undefined) {
|
||||
return requestMessages(body)
|
||||
.map((message) => stringifyContent(message.content))
|
||||
.filter(Boolean)
|
||||
.join("\n");
|
||||
}
|
||||
|
||||
function extractToolOutput(body: ChatCompletionRequest | null | undefined) {
|
||||
const messages = requestMessages(body);
|
||||
for (let index = messages.length - 1; index >= 0; index -= 1) {
|
||||
const message = messages[index];
|
||||
if (message?.role === "tool") {
|
||||
return stringifyContent(message.content);
|
||||
}
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
function countImageInputs(value: unknown): number {
|
||||
if (Array.isArray(value)) {
|
||||
return value.reduce((sum, entry) => sum + countImageInputs(entry), 0);
|
||||
}
|
||||
if (!value || typeof value !== "object") {
|
||||
return 0;
|
||||
}
|
||||
const record = value as Record<string, unknown>;
|
||||
const type = typeof record.type === "string" ? record.type : "";
|
||||
const imageLikeType =
|
||||
type === "input_image" || type === "image" || type === "image_url" || type === "media";
|
||||
const nested =
|
||||
countImageInputs(record.content) +
|
||||
countImageInputs(record.image_url) +
|
||||
countImageInputs(record.source);
|
||||
return (imageLikeType ? 1 : 0) + nested;
|
||||
}
|
||||
|
||||
function resolveProviderVariant(model: string): AimockRequestSnapshot["providerVariant"] {
|
||||
const normalized = model.trim().toLowerCase();
|
||||
const provider = /^([^/:]+)[/:]/.exec(normalized)?.[1] ?? normalized;
|
||||
if (provider === "openai" || provider === "aimock" || provider === "openai-codex") {
|
||||
return "openai";
|
||||
}
|
||||
if (provider === "anthropic" || provider === "claude-cli") {
|
||||
return "anthropic";
|
||||
}
|
||||
if (/^(?:gpt-|o1-|openai-)/.test(normalized)) {
|
||||
return "openai";
|
||||
}
|
||||
if (/^(?:claude-|anthropic-)/.test(normalized)) {
|
||||
return "anthropic";
|
||||
}
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
function extractPlannedToolName(entry: JournalEntry) {
|
||||
const response = entry.response.fixture?.response as
|
||||
| { toolCalls?: Array<{ name?: unknown }> }
|
||||
| undefined;
|
||||
const name = response?.toolCalls?.[0]?.name;
|
||||
return typeof name === "string" && name.length > 0 ? name : undefined;
|
||||
}
|
||||
|
||||
function toRequestSnapshot(entry: JournalEntry): AimockRequestSnapshot {
|
||||
const body = entry.body ?? null;
|
||||
const model = typeof body?.model === "string" ? body.model : "";
|
||||
return {
|
||||
raw: JSON.stringify(body ?? {}),
|
||||
body: (body ?? {}) as Record<string, unknown>,
|
||||
prompt: extractLastUserText(body),
|
||||
allInputText: extractAllInputText(body),
|
||||
toolOutput: extractToolOutput(body),
|
||||
model,
|
||||
providerVariant: resolveProviderVariant(model),
|
||||
imageInputCount: countImageInputs(requestMessages(body)),
|
||||
plannedToolName: extractPlannedToolName(entry),
|
||||
};
|
||||
}
|
||||
|
||||
function createDebugMount(mock: LLMock): Mountable {
|
||||
return {
|
||||
async handleRequest(_req: IncomingMessage, res: ServerResponse, pathname: string) {
|
||||
const entries = mock.getRequests();
|
||||
if (pathname === "/last-request") {
|
||||
const lastEntry = entries.at(-1);
|
||||
writeJson(
|
||||
res,
|
||||
200,
|
||||
lastEntry ? toRequestSnapshot(lastEntry) : { ok: false, error: "no request recorded" },
|
||||
);
|
||||
return true;
|
||||
}
|
||||
if (pathname === "/requests") {
|
||||
writeJson(
|
||||
res,
|
||||
200,
|
||||
entries.map((entry) => toRequestSnapshot(entry)),
|
||||
);
|
||||
return true;
|
||||
}
|
||||
if (pathname === "/image-generations") {
|
||||
writeJson(
|
||||
res,
|
||||
200,
|
||||
entries
|
||||
.filter((entry) => entry.path === "/v1/images/generations")
|
||||
.map((entry) => entry.body ?? {}),
|
||||
);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export async function startQaAimockServer(params?: { host?: string; port?: number }) {
|
||||
const mock = new LLMock({
|
||||
host: params?.host ?? "127.0.0.1",
|
||||
port: params?.port ?? 0,
|
||||
strict: false,
|
||||
logLevel: "silent",
|
||||
});
|
||||
|
||||
mock.mount("/debug", createDebugMount(mock));
|
||||
mock.onMessage(/.*/, { content: "AIMOCK_QA_OK" });
|
||||
|
||||
await mock.start();
|
||||
return {
|
||||
baseUrl: mock.baseUrl,
|
||||
async stop() {
|
||||
await mock.stop();
|
||||
},
|
||||
};
|
||||
}
|
||||
229
extensions/qa-lab/src/providers/env.ts
Normal file
229
extensions/qa-lab/src/providers/env.ts
Normal file
@@ -0,0 +1,229 @@
|
||||
import { existsSync } from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import type { QaProviderMode } from "./index.js";
|
||||
import { getQaProvider } from "./index.js";
|
||||
|
||||
const QA_LIVE_ENV_ALIASES = Object.freeze([
|
||||
{
|
||||
liveVar: "OPENCLAW_LIVE_OPENAI_KEY",
|
||||
providerVar: "OPENAI_API_KEY",
|
||||
},
|
||||
{
|
||||
liveVar: "OPENCLAW_LIVE_ANTHROPIC_KEY",
|
||||
providerVar: "ANTHROPIC_API_KEY",
|
||||
},
|
||||
{
|
||||
liveVar: "OPENCLAW_LIVE_GEMINI_KEY",
|
||||
providerVar: "GEMINI_API_KEY",
|
||||
},
|
||||
]);
|
||||
|
||||
export const QA_LIVE_PROVIDER_CONFIG_PATH_ENV = "OPENCLAW_QA_LIVE_PROVIDER_CONFIG_PATH";
|
||||
export const QA_LIVE_CLI_BACKEND_PRESERVE_ENV = "OPENCLAW_LIVE_CLI_BACKEND_PRESERVE_ENV";
|
||||
export const QA_LIVE_CLI_BACKEND_AUTH_MODE_ENV = "OPENCLAW_LIVE_CLI_BACKEND_AUTH_MODE";
|
||||
export type QaCliBackendAuthMode = "auto" | "api-key" | "subscription";
|
||||
|
||||
export const QA_PROVIDER_SECRET_ENV_VARS = Object.freeze([
|
||||
"ANTHROPIC_API_KEY",
|
||||
"ANTHROPIC_OAUTH_TOKEN",
|
||||
"AWS_ACCESS_KEY_ID",
|
||||
"AWS_BEARER_TOKEN_BEDROCK",
|
||||
"AWS_SECRET_ACCESS_KEY",
|
||||
"AWS_SESSION_TOKEN",
|
||||
"ANTHROPIC_API_KEYS",
|
||||
"GEMINI_API_KEY",
|
||||
"GEMINI_API_KEYS",
|
||||
"GOOGLE_API_KEY",
|
||||
"MISTRAL_API_KEY",
|
||||
"OPENAI_API_KEY",
|
||||
"OPENAI_API_KEYS",
|
||||
"OPENCLAW_LIVE_ANTHROPIC_KEY",
|
||||
"OPENCLAW_LIVE_ANTHROPIC_KEYS",
|
||||
"OPENCLAW_LIVE_GEMINI_KEY",
|
||||
"OPENCLAW_LIVE_OPENAI_KEY",
|
||||
"VOYAGE_API_KEY",
|
||||
]);
|
||||
|
||||
const QA_MOCK_BLOCKED_ENV_VARS = Object.freeze([
|
||||
...QA_PROVIDER_SECRET_ENV_VARS,
|
||||
"AWS_REGION",
|
||||
"OPENAI_BASE_URL",
|
||||
"CODEX_HOME",
|
||||
]);
|
||||
|
||||
const QA_MOCK_BLOCKED_ENV_KEY_PATTERNS = Object.freeze([
|
||||
/^DISCORD_/i,
|
||||
/^TELEGRAM_/i,
|
||||
/^SLACK_/i,
|
||||
/^MATRIX_/i,
|
||||
/^SIGNAL_/i,
|
||||
/^WHATSAPP_/i,
|
||||
/^IMESSAGE_/i,
|
||||
/^ZALO/i,
|
||||
/^TWILIO_/i,
|
||||
/^PLIVO_/i,
|
||||
/^NGROK_/i,
|
||||
]);
|
||||
|
||||
const QA_LIVE_ALLOWED_ENV_VARS = Object.freeze([
|
||||
...QA_PROVIDER_SECRET_ENV_VARS,
|
||||
"AWS_REGION",
|
||||
"OPENAI_BASE_URL",
|
||||
QA_LIVE_PROVIDER_CONFIG_PATH_ENV,
|
||||
"OPENCLAW_CONFIG_PATH",
|
||||
]);
|
||||
|
||||
const QA_LIVE_ALLOWED_ENV_PATTERNS = Object.freeze([
|
||||
/^[A-Z0-9_]+_API_KEYS$/u,
|
||||
/^[A-Z0-9_]+_API_KEY_[0-9]+$/u,
|
||||
/^OPENCLAW_LIVE_[A-Z0-9_]+_KEYS$/u,
|
||||
]);
|
||||
|
||||
function resolveUserPath(value: string, env: NodeJS.ProcessEnv = process.env) {
|
||||
if (value === "~") {
|
||||
return env.HOME ?? os.homedir();
|
||||
}
|
||||
if (value.startsWith("~/")) {
|
||||
return path.join(env.HOME ?? os.homedir(), value.slice(2));
|
||||
}
|
||||
return path.resolve(value);
|
||||
}
|
||||
|
||||
function applyLiveProviderEnvAliases(env: NodeJS.ProcessEnv | Record<string, string>) {
|
||||
for (const { liveVar, providerVar } of QA_LIVE_ENV_ALIASES) {
|
||||
const liveValue = env[liveVar]?.trim();
|
||||
if (!liveValue || env[providerVar]?.trim()) {
|
||||
continue;
|
||||
}
|
||||
env[providerVar] = liveValue;
|
||||
}
|
||||
}
|
||||
|
||||
function parsePreservedCliEnv(baseEnv: NodeJS.ProcessEnv) {
|
||||
const raw = baseEnv[QA_LIVE_CLI_BACKEND_PRESERVE_ENV]?.trim();
|
||||
if (raw?.startsWith("[")) {
|
||||
try {
|
||||
const parsed = JSON.parse(raw) as unknown;
|
||||
return Array.isArray(parsed)
|
||||
? parsed.filter((entry): entry is string => typeof entry === "string")
|
||||
: [];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
return (raw ?? "").split(/[,\s]+/).filter((entry) => entry.length > 0);
|
||||
}
|
||||
|
||||
function renderPreservedCliEnv(values: string[]) {
|
||||
return JSON.stringify([...new Set(values)]);
|
||||
}
|
||||
|
||||
export function normalizeQaProviderModeEnv(env: NodeJS.ProcessEnv, providerMode?: QaProviderMode) {
|
||||
const provider = providerMode ? getQaProvider(providerMode) : null;
|
||||
if (provider?.scrubsLiveProviderEnv) {
|
||||
for (const key of QA_MOCK_BLOCKED_ENV_VARS) {
|
||||
delete env[key];
|
||||
}
|
||||
for (const key of Object.keys(env)) {
|
||||
if (QA_MOCK_BLOCKED_ENV_KEY_PATTERNS.some((pattern) => pattern.test(key))) {
|
||||
delete env[key];
|
||||
}
|
||||
}
|
||||
return env;
|
||||
}
|
||||
|
||||
if (provider?.appliesLiveEnvAliases) {
|
||||
applyLiveProviderEnvAliases(env);
|
||||
}
|
||||
|
||||
return env;
|
||||
}
|
||||
|
||||
export function resolveQaLiveCliAuthEnv(
|
||||
baseEnv: NodeJS.ProcessEnv,
|
||||
opts?: {
|
||||
forwardHostHomeForClaudeCli?: boolean;
|
||||
claudeCliAuthMode?: QaCliBackendAuthMode;
|
||||
},
|
||||
) {
|
||||
const authMode = opts?.claudeCliAuthMode ?? "auto";
|
||||
const hasAnthropicKey = Boolean(
|
||||
baseEnv.ANTHROPIC_API_KEY?.trim() || baseEnv.OPENCLAW_LIVE_ANTHROPIC_KEY?.trim(),
|
||||
);
|
||||
if (opts?.forwardHostHomeForClaudeCli && authMode === "api-key" && !hasAnthropicKey) {
|
||||
throw new Error(
|
||||
"Claude CLI API-key QA mode requires ANTHROPIC_API_KEY or OPENCLAW_LIVE_ANTHROPIC_KEY",
|
||||
);
|
||||
}
|
||||
const preserveEnvValues = (() => {
|
||||
if (!opts?.forwardHostHomeForClaudeCli) {
|
||||
return undefined;
|
||||
}
|
||||
const values = parsePreservedCliEnv(baseEnv).filter((entry) => entry !== "ANTHROPIC_API_KEY");
|
||||
if (authMode === "api-key" || (authMode === "auto" && hasAnthropicKey)) {
|
||||
values.push("ANTHROPIC_API_KEY");
|
||||
}
|
||||
return renderPreservedCliEnv(values);
|
||||
})();
|
||||
const claudeCliEnv = opts?.forwardHostHomeForClaudeCli
|
||||
? {
|
||||
[QA_LIVE_CLI_BACKEND_AUTH_MODE_ENV]: authMode,
|
||||
...(preserveEnvValues ? { [QA_LIVE_CLI_BACKEND_PRESERVE_ENV]: preserveEnvValues } : {}),
|
||||
}
|
||||
: {};
|
||||
const configuredCodexHome = baseEnv.CODEX_HOME?.trim();
|
||||
if (configuredCodexHome) {
|
||||
return {
|
||||
CODEX_HOME: configuredCodexHome,
|
||||
...claudeCliEnv,
|
||||
...(opts?.forwardHostHomeForClaudeCli && baseEnv.HOME?.trim()
|
||||
? { HOME: baseEnv.HOME.trim() }
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
const hostHome = baseEnv.HOME?.trim();
|
||||
if (!hostHome) {
|
||||
return {};
|
||||
}
|
||||
const codexHome = path.join(hostHome, ".codex");
|
||||
return {
|
||||
...(existsSync(codexHome) ? { CODEX_HOME: codexHome } : {}),
|
||||
...claudeCliEnv,
|
||||
...(opts?.forwardHostHomeForClaudeCli ? { HOME: hostHome } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
export function resolveQaLiveProviderConfigPath(env: NodeJS.ProcessEnv = process.env) {
|
||||
const explicit =
|
||||
env[QA_LIVE_PROVIDER_CONFIG_PATH_ENV]?.trim() || env.OPENCLAW_CONFIG_PATH?.trim();
|
||||
return explicit
|
||||
? { path: resolveUserPath(explicit, env), explicit: true }
|
||||
: { path: path.join(os.homedir(), ".openclaw", "openclaw.json"), explicit: false };
|
||||
}
|
||||
|
||||
export function resolveQaForwardedLiveEnv(baseEnv: NodeJS.ProcessEnv = process.env) {
|
||||
const forwarded: Record<string, string> = {};
|
||||
for (const [key, rawValue] of Object.entries(baseEnv)) {
|
||||
if (
|
||||
!QA_LIVE_ALLOWED_ENV_VARS.includes(key) &&
|
||||
!QA_LIVE_ALLOWED_ENV_PATTERNS.some((pattern) => pattern.test(key))
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
const value = rawValue?.trim();
|
||||
if (value) {
|
||||
forwarded[key] = value;
|
||||
}
|
||||
}
|
||||
applyLiveProviderEnvAliases(forwarded);
|
||||
|
||||
const configuredCodexHome = baseEnv.CODEX_HOME?.trim();
|
||||
const codexHome = configuredCodexHome
|
||||
? resolveUserPath(configuredCodexHome, baseEnv)
|
||||
: path.join(baseEnv.HOME?.trim() || os.homedir(), ".codex");
|
||||
if (existsSync(codexHome)) {
|
||||
forwarded.CODEX_HOME = codexHome;
|
||||
}
|
||||
return forwarded;
|
||||
}
|
||||
46
extensions/qa-lab/src/providers/image-generation.test.ts
Normal file
46
extensions/qa-lab/src/providers/image-generation.test.ts
Normal file
@@ -0,0 +1,46 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { buildQaImageGenerationConfigPatch } from "./image-generation.js";
|
||||
|
||||
describe("QA provider image generation config", () => {
|
||||
it("uses the selected mock provider for mock-openai image generation", () => {
|
||||
const patch = buildQaImageGenerationConfigPatch({
|
||||
providerMode: "mock-openai",
|
||||
providerBaseUrl: "http://127.0.0.1:44080/v1",
|
||||
requiredPluginIds: ["qa-channel"],
|
||||
});
|
||||
|
||||
expect(patch.plugins.allow).toEqual(["memory-core", "qa-channel"]);
|
||||
expect(patch.agents.defaults.imageGenerationModel.primary).toBe("mock-openai/gpt-image-1");
|
||||
expect(patch.models?.providers["mock-openai"]?.baseUrl).toBe("http://127.0.0.1:44080/v1");
|
||||
});
|
||||
|
||||
it("uses the selected mock provider for AIMock image generation", () => {
|
||||
const patch = buildQaImageGenerationConfigPatch({
|
||||
providerMode: "aimock",
|
||||
providerBaseUrl: "http://127.0.0.1:45080/v1",
|
||||
requiredPluginIds: [],
|
||||
});
|
||||
|
||||
expect(patch.agents.defaults.imageGenerationModel.primary).toBe("aimock/gpt-image-1");
|
||||
expect(patch.models?.providers.aimock?.baseUrl).toBe("http://127.0.0.1:45080/v1");
|
||||
expect(patch.models?.providers["mock-openai"]).toBeUndefined();
|
||||
});
|
||||
|
||||
it("enables the live image provider plugin without replacing live model config", () => {
|
||||
const patch = buildQaImageGenerationConfigPatch({
|
||||
providerMode: "live-frontier",
|
||||
requiredPluginIds: ["qa-channel"],
|
||||
});
|
||||
|
||||
expect(patch.plugins).toEqual({
|
||||
allow: ["memory-core", "openai", "qa-channel"],
|
||||
entries: {
|
||||
openai: {
|
||||
enabled: true,
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(patch.agents.defaults.imageGenerationModel.primary).toBe("openai/gpt-image-1");
|
||||
expect(patch).not.toHaveProperty("models");
|
||||
});
|
||||
});
|
||||
73
extensions/qa-lab/src/providers/image-generation.ts
Normal file
73
extensions/qa-lab/src/providers/image-generation.ts
Normal file
@@ -0,0 +1,73 @@
|
||||
import type { QaProviderMode } from "./index.js";
|
||||
import { getQaProvider } from "./index.js";
|
||||
|
||||
type QaImageGenerationPatchInput = {
|
||||
providerMode: QaProviderMode;
|
||||
providerBaseUrl?: string;
|
||||
requiredPluginIds: readonly string[];
|
||||
};
|
||||
|
||||
function splitModelProviderId(modelRef: string) {
|
||||
const slash = modelRef.indexOf("/");
|
||||
return slash > 0 ? modelRef.slice(0, slash) : null;
|
||||
}
|
||||
|
||||
function uniqueNonEmpty(values: readonly (string | null | undefined)[]) {
|
||||
return [
|
||||
...new Set(values.map((value) => value?.trim()).filter((value): value is string => !!value)),
|
||||
];
|
||||
}
|
||||
|
||||
export function buildQaImageGenerationConfigPatch(input: QaImageGenerationPatchInput) {
|
||||
const provider = getQaProvider(input.providerMode);
|
||||
const imageModelRef = provider.defaultImageGenerationModel({
|
||||
modelProviderIds: provider.defaultImageGenerationProviderIds,
|
||||
});
|
||||
if (!imageModelRef) {
|
||||
throw new Error(
|
||||
`QA provider "${input.providerMode}" does not expose an image generation model`,
|
||||
);
|
||||
}
|
||||
const imageProviderId = splitModelProviderId(imageModelRef);
|
||||
const modelPatch = (() => {
|
||||
if (provider.kind !== "mock") {
|
||||
return null;
|
||||
}
|
||||
if (!input.providerBaseUrl) {
|
||||
throw new Error(`QA provider "${input.providerMode}" requires a mock provider URL`);
|
||||
}
|
||||
return provider.buildGatewayModels({
|
||||
providerBaseUrl: input.providerBaseUrl,
|
||||
});
|
||||
})();
|
||||
const providerPluginIds = provider.usesModelProviderPlugins ? [imageProviderId] : [];
|
||||
const enabledPluginIds = uniqueNonEmpty(providerPluginIds);
|
||||
|
||||
return {
|
||||
plugins: {
|
||||
allow: uniqueNonEmpty(["memory-core", ...enabledPluginIds, ...input.requiredPluginIds]),
|
||||
...(enabledPluginIds.length > 0
|
||||
? {
|
||||
entries: Object.fromEntries(
|
||||
enabledPluginIds.map((pluginId) => [pluginId, { enabled: true }]),
|
||||
),
|
||||
}
|
||||
: {}),
|
||||
},
|
||||
...(modelPatch
|
||||
? {
|
||||
models: {
|
||||
mode: modelPatch.mode,
|
||||
providers: modelPatch.providers,
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
agents: {
|
||||
defaults: {
|
||||
imageGenerationModel: {
|
||||
primary: imageModelRef,
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
62
extensions/qa-lab/src/providers/index.ts
Normal file
62
extensions/qa-lab/src/providers/index.ts
Normal file
@@ -0,0 +1,62 @@
|
||||
import { aimockProviderDefinition } from "./aimock/index.js";
|
||||
import { liveFrontierProviderDefinition } from "./live-frontier/index.js";
|
||||
import { mockOpenAiProviderDefinition } from "./mock-openai/index.js";
|
||||
import type { QaProviderDefinition, QaProviderMode, QaProviderModeInput } from "./shared/types.js";
|
||||
|
||||
export type {
|
||||
QaMockProviderServer,
|
||||
QaProviderDefinition,
|
||||
QaProviderMode,
|
||||
QaProviderModeInput,
|
||||
} from "./shared/types.js";
|
||||
|
||||
const PROVIDERS = [
|
||||
mockOpenAiProviderDefinition,
|
||||
aimockProviderDefinition,
|
||||
liveFrontierProviderDefinition,
|
||||
] as const satisfies readonly QaProviderDefinition[];
|
||||
|
||||
export const DEFAULT_QA_PROVIDER_MODE = "mock-openai" satisfies QaProviderMode;
|
||||
export const DEFAULT_QA_LIVE_PROVIDER_MODE = "live-frontier" satisfies QaProviderMode;
|
||||
|
||||
const PROVIDERS_BY_INPUT = new Map<QaProviderModeInput, QaProviderDefinition>();
|
||||
for (const provider of PROVIDERS) {
|
||||
PROVIDERS_BY_INPUT.set(provider.mode, provider);
|
||||
}
|
||||
|
||||
export function isQaProviderModeInput(input: unknown): input is QaProviderModeInput {
|
||||
return typeof input === "string" && PROVIDERS_BY_INPUT.has(input as QaProviderModeInput);
|
||||
}
|
||||
|
||||
export function normalizeQaProviderMode(input: QaProviderModeInput): QaProviderMode {
|
||||
return getQaProvider(input).mode;
|
||||
}
|
||||
|
||||
export function getQaProvider(input: QaProviderModeInput): QaProviderDefinition {
|
||||
const provider = PROVIDERS_BY_INPUT.get(input);
|
||||
if (!provider) {
|
||||
throw new Error(`unknown QA provider mode: ${input}`);
|
||||
}
|
||||
return provider;
|
||||
}
|
||||
|
||||
export function listQaProviderModes() {
|
||||
return PROVIDERS.map((provider) => provider.mode);
|
||||
}
|
||||
|
||||
export function formatQaProviderModeHelp() {
|
||||
return `Provider mode: ${listQaProviderModes().join(", ")}`;
|
||||
}
|
||||
|
||||
export function listQaStandaloneProviderCommands() {
|
||||
return PROVIDERS.flatMap((provider) =>
|
||||
provider.standaloneCommand
|
||||
? [
|
||||
{
|
||||
providerMode: provider.mode,
|
||||
...provider.standaloneCommand,
|
||||
},
|
||||
]
|
||||
: [],
|
||||
);
|
||||
}
|
||||
60
extensions/qa-lab/src/providers/live-frontier/auth.ts
Normal file
60
extensions/qa-lab/src/providers/live-frontier/auth.ts
Normal file
@@ -0,0 +1,60 @@
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-runtime";
|
||||
import {
|
||||
applyAuthProfileConfig,
|
||||
upsertAuthProfile,
|
||||
validateAnthropicSetupToken,
|
||||
} from "openclaw/plugin-sdk/provider-auth";
|
||||
|
||||
export const QA_LIVE_ANTHROPIC_SETUP_TOKEN_ENV = "OPENCLAW_QA_LIVE_ANTHROPIC_SETUP_TOKEN";
|
||||
export const QA_LIVE_SETUP_TOKEN_VALUE_ENV = "OPENCLAW_LIVE_SETUP_TOKEN_VALUE";
|
||||
const QA_LIVE_ANTHROPIC_SETUP_TOKEN_PROFILE_ENV = "OPENCLAW_QA_LIVE_ANTHROPIC_SETUP_TOKEN_PROFILE";
|
||||
const QA_LIVE_ANTHROPIC_SETUP_TOKEN_PROFILE_ID = "anthropic:qa-setup-token";
|
||||
|
||||
function resolveQaLiveAnthropicSetupToken(env: NodeJS.ProcessEnv = process.env) {
|
||||
const token = (
|
||||
env[QA_LIVE_ANTHROPIC_SETUP_TOKEN_ENV]?.trim() ||
|
||||
env[QA_LIVE_SETUP_TOKEN_VALUE_ENV]?.trim() ||
|
||||
""
|
||||
).replaceAll(/\s+/g, "");
|
||||
if (!token) {
|
||||
return null;
|
||||
}
|
||||
const tokenError = validateAnthropicSetupToken(token);
|
||||
if (tokenError) {
|
||||
throw new Error(`Invalid QA Anthropic setup-token: ${tokenError}`);
|
||||
}
|
||||
const profileId =
|
||||
env[QA_LIVE_ANTHROPIC_SETUP_TOKEN_PROFILE_ENV]?.trim() ||
|
||||
QA_LIVE_ANTHROPIC_SETUP_TOKEN_PROFILE_ID;
|
||||
return { token, profileId };
|
||||
}
|
||||
|
||||
export async function stageQaLiveAnthropicSetupToken(params: {
|
||||
cfg: OpenClawConfig;
|
||||
stateDir: string;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
}): Promise<OpenClawConfig> {
|
||||
const resolved = resolveQaLiveAnthropicSetupToken(params.env);
|
||||
if (!resolved) {
|
||||
return params.cfg;
|
||||
}
|
||||
const agentDir = path.join(params.stateDir, "agents", "main", "agent");
|
||||
await fs.mkdir(agentDir, { recursive: true });
|
||||
upsertAuthProfile({
|
||||
profileId: resolved.profileId,
|
||||
credential: {
|
||||
type: "token",
|
||||
provider: "anthropic",
|
||||
token: resolved.token,
|
||||
},
|
||||
agentDir,
|
||||
});
|
||||
return applyAuthProfileConfig(params.cfg, {
|
||||
profileId: resolved.profileId,
|
||||
provider: "anthropic",
|
||||
mode: "token",
|
||||
displayName: "QA setup-token",
|
||||
});
|
||||
}
|
||||
7
extensions/qa-lab/src/providers/live-frontier/catalog.ts
Normal file
7
extensions/qa-lab/src/providers/live-frontier/catalog.ts
Normal file
@@ -0,0 +1,7 @@
|
||||
export const QA_FRONTIER_PROVIDER_IDS = ["anthropic", "google", "openai"] as const;
|
||||
export const QA_FRONTIER_CATALOG_PRIMARY_MODEL = "openai/gpt-5.4";
|
||||
export const QA_FRONTIER_CATALOG_ALTERNATE_MODEL = "anthropic/claude-sonnet-4-6";
|
||||
|
||||
export function isPreferredQaLiveFrontierCatalogModel(modelRef: string) {
|
||||
return modelRef === QA_FRONTIER_CATALOG_PRIMARY_MODEL;
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
import type { QaThinkingLevel } from "../../qa-thinking.js";
|
||||
|
||||
type QaFrontierCharacterModelOptions = {
|
||||
thinkingDefault?: QaThinkingLevel;
|
||||
fastMode?: boolean;
|
||||
};
|
||||
|
||||
export const QA_FRONTIER_CHARACTER_EVAL_MODELS = Object.freeze([
|
||||
"openai/gpt-5.4",
|
||||
"openai/gpt-5.2",
|
||||
"openai/gpt-5",
|
||||
"anthropic/claude-opus-4-6",
|
||||
"anthropic/claude-sonnet-4-6",
|
||||
"zai/glm-5.1",
|
||||
"moonshot/kimi-k2.5",
|
||||
"google/gemini-3.1-pro-preview",
|
||||
]);
|
||||
|
||||
export const QA_FRONTIER_CHARACTER_THINKING_BY_MODEL: Readonly<Record<string, QaThinkingLevel>> =
|
||||
Object.freeze({
|
||||
"openai/gpt-5.4": "xhigh",
|
||||
"openai/gpt-5.2": "xhigh",
|
||||
"openai/gpt-5": "xhigh",
|
||||
});
|
||||
|
||||
export const QA_FRONTIER_CHARACTER_JUDGE_MODELS = Object.freeze([
|
||||
"openai/gpt-5.4",
|
||||
"anthropic/claude-opus-4-6",
|
||||
]);
|
||||
|
||||
export const QA_FRONTIER_CHARACTER_JUDGE_MODEL_OPTIONS: Readonly<
|
||||
Record<string, QaFrontierCharacterModelOptions>
|
||||
> = Object.freeze({
|
||||
"openai/gpt-5.4": { thinkingDefault: "xhigh" },
|
||||
"anthropic/claude-opus-4-6": { thinkingDefault: "high" },
|
||||
});
|
||||
61
extensions/qa-lab/src/providers/live-frontier/index.ts
Normal file
61
extensions/qa-lab/src/providers/live-frontier/index.ts
Normal file
@@ -0,0 +1,61 @@
|
||||
import type { QaProviderDefinition } from "../shared/types.js";
|
||||
|
||||
function isOpenAiModel(modelRef: string) {
|
||||
return modelRef.startsWith("openai/");
|
||||
}
|
||||
|
||||
function isAnthropicModel(modelRef: string) {
|
||||
return modelRef.startsWith("anthropic/");
|
||||
}
|
||||
|
||||
function isQaFastModeModelRef(modelRef: string) {
|
||||
return isOpenAiModel(modelRef);
|
||||
}
|
||||
|
||||
function isGptFiveModel(modelRef: string) {
|
||||
return isOpenAiModel(modelRef) && modelRef.slice("openai/".length).startsWith("gpt-5");
|
||||
}
|
||||
|
||||
function isClaudeOpusModel(modelRef: string) {
|
||||
return isAnthropicModel(modelRef) && modelRef.includes("claude-opus");
|
||||
}
|
||||
|
||||
export const liveFrontierProviderDefinition: QaProviderDefinition = {
|
||||
mode: "live-frontier",
|
||||
kind: "live",
|
||||
defaultModel: (options) => options?.preferredLiveModel ?? "openai/gpt-5.4",
|
||||
defaultImageGenerationProviderIds: ["openai"],
|
||||
defaultImageGenerationModel: ({ modelProviderIds }) =>
|
||||
modelProviderIds.includes("openai") ? "openai/gpt-image-1" : null,
|
||||
usesFastModeByDefault: isQaFastModeModelRef,
|
||||
resolveModelParams: ({ modelRef, fastMode, thinkingDefault }) => ({
|
||||
transport: "sse",
|
||||
openaiWsWarmup: false,
|
||||
...(fastMode === true || isQaFastModeModelRef(modelRef) ? { fastMode: true } : {}),
|
||||
...(thinkingDefault ? { thinking: thinkingDefault } : {}),
|
||||
}),
|
||||
resolveTurnTimeoutMs: ({ fallbackMs, modelRef }) => {
|
||||
if (isClaudeOpusModel(modelRef)) {
|
||||
return Math.max(fallbackMs, 240_000);
|
||||
}
|
||||
if (isAnthropicModel(modelRef)) {
|
||||
return Math.max(fallbackMs, 180_000);
|
||||
}
|
||||
if (isGptFiveModel(modelRef)) {
|
||||
return Math.max(fallbackMs, 360_000);
|
||||
}
|
||||
return Math.max(fallbackMs, 120_000);
|
||||
},
|
||||
buildGatewayModels: ({ liveProviderConfigs }) => {
|
||||
const providers = liveProviderConfigs ?? {};
|
||||
return Object.keys(providers).length > 0
|
||||
? {
|
||||
mode: "merge",
|
||||
providers,
|
||||
}
|
||||
: null;
|
||||
},
|
||||
usesModelProviderPlugins: true,
|
||||
scrubsLiveProviderEnv: false,
|
||||
appliesLiveEnvAliases: true,
|
||||
};
|
||||
@@ -0,0 +1,27 @@
|
||||
import {
|
||||
listProfilesForProvider,
|
||||
loadAuthProfileStoreForRuntime,
|
||||
} from "openclaw/plugin-sdk/agent-runtime";
|
||||
import { resolveEnvApiKey } from "openclaw/plugin-sdk/provider-auth";
|
||||
|
||||
const QA_CODEX_OAUTH_LIVE_MODEL = "openai-codex/gpt-5.4";
|
||||
|
||||
export function resolveQaLiveFrontierPreferredModel() {
|
||||
if (resolveEnvApiKey("openai")?.apiKey) {
|
||||
return undefined;
|
||||
}
|
||||
try {
|
||||
const store = loadAuthProfileStoreForRuntime(undefined, {
|
||||
readOnly: true,
|
||||
allowKeychainPrompt: false,
|
||||
});
|
||||
if (listProfilesForProvider(store, "openai").length > 0) {
|
||||
return undefined;
|
||||
}
|
||||
return listProfilesForProvider(store, "openai-codex").length > 0
|
||||
? QA_CODEX_OAUTH_LIVE_MODEL
|
||||
: undefined;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
2
extensions/qa-lab/src/providers/live-frontier/parity.ts
Normal file
2
extensions/qa-lab/src/providers/live-frontier/parity.ts
Normal file
@@ -0,0 +1,2 @@
|
||||
export const QA_FRONTIER_PARITY_CANDIDATE_LABEL = "openai/gpt-5.4";
|
||||
export const QA_FRONTIER_PARITY_BASELINE_LABEL = "anthropic/claude-opus-4-6";
|
||||
9
extensions/qa-lab/src/providers/mock-openai/index.ts
Normal file
9
extensions/qa-lab/src/providers/mock-openai/index.ts
Normal file
@@ -0,0 +1,9 @@
|
||||
import { createMockQaProviderDefinition } from "../shared/mock-provider-definition.js";
|
||||
|
||||
export const mockOpenAiProviderDefinition = createMockQaProviderDefinition({
|
||||
mode: "mock-openai",
|
||||
commandName: "mock-openai",
|
||||
commandDescription: "Run the local mock OpenAI Responses API server for QA",
|
||||
serverLabel: "QA mock OpenAI",
|
||||
mockAuthProviders: ["openai", "anthropic"],
|
||||
});
|
||||
@@ -1,5 +1,5 @@
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { resolveProviderVariant, startQaMockOpenAiServer } from "./mock-openai-server.js";
|
||||
import { resolveProviderVariant, startQaMockOpenAiServer } from "./server.js";
|
||||
|
||||
const cleanups: Array<() => Promise<void>> = [];
|
||||
const QA_IMAGE_PNG_BASE64 =
|
||||
@@ -63,6 +63,23 @@ function makeUserInput(text: string) {
|
||||
};
|
||||
}
|
||||
|
||||
const SESSIONS_SPAWN_TOOL = { type: "function", name: "sessions_spawn" } as const;
|
||||
const THREAD_SUBAGENT_CHILD_ERROR_TOKEN = "QA_SUBAGENT_CHILD_ERROR";
|
||||
const THREAD_SUBAGENT_TOOL_ERROR =
|
||||
"thread=true requested but thread delivery is unavailable in this test harness.";
|
||||
|
||||
function threadSubagentTask(token: string) {
|
||||
return `Reply exactly \`${token}\`. This is the marker.`;
|
||||
}
|
||||
|
||||
function explicitSessionsSpawnPrompt(token: string) {
|
||||
return [
|
||||
"Use sessions_spawn for this QA check.",
|
||||
`task="${threadSubagentTask(token)}"`,
|
||||
"label=qa-thread-subagent thread=true mode=session runTimeoutSeconds=30",
|
||||
].join(" ");
|
||||
}
|
||||
|
||||
describe("qa mock openai server", () => {
|
||||
it("serves health and streamed responses", async () => {
|
||||
const server = await startQaMockOpenAiServer({
|
||||
@@ -109,14 +126,14 @@ describe("qa mock openai server", () => {
|
||||
},
|
||||
body: JSON.stringify({
|
||||
stream: true,
|
||||
input: [makeUserInput("Quiet streaming QA check: reply exactly `MATRIX_QA_STREAMING_OK`.")],
|
||||
input: [makeUserInput("Quiet streaming QA check: reply exactly `QA_STREAMING_OK`.")],
|
||||
}),
|
||||
});
|
||||
expect(quietResponse.status).toBe(200);
|
||||
const quietBody = await quietResponse.text();
|
||||
expect(quietBody).toContain('"type":"response.output_text.delta"');
|
||||
expect(quietBody).toContain('"phase":"final_answer"');
|
||||
expect(quietBody).toContain("MATRIX_QA_STREAMING_OK");
|
||||
expect(quietBody).toContain("QA_STREAMING_OK");
|
||||
|
||||
const blockResponse = await fetch(`${server.baseUrl}/v1/responses`, {
|
||||
method: "POST",
|
||||
@@ -548,6 +565,7 @@ describe("qa mock openai server", () => {
|
||||
},
|
||||
body: JSON.stringify({
|
||||
stream: true,
|
||||
tools: [SESSIONS_SPAWN_TOOL],
|
||||
input: [
|
||||
{
|
||||
role: "user",
|
||||
@@ -568,6 +586,67 @@ describe("qa mock openai server", () => {
|
||||
expect(body).toContain('\\"thread\\":false');
|
||||
});
|
||||
|
||||
it("emits explicitly requested sessions_spawn tool calls", async () => {
|
||||
const server = await startMockServer();
|
||||
|
||||
const body = await expectResponsesText(server, {
|
||||
stream: true,
|
||||
tools: [SESSIONS_SPAWN_TOOL],
|
||||
input: [
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{
|
||||
type: "input_text",
|
||||
text: explicitSessionsSpawnPrompt("QA_SUBAGENT_CHILD_FIXED"),
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(body).toContain('"name":"sessions_spawn"');
|
||||
expect(body).toContain('\\"label\\":\\"qa-thread-subagent\\"');
|
||||
expect(body).toContain('\\"thread\\":true');
|
||||
expect(body).toContain('\\"mode\\":\\"session\\"');
|
||||
expect(body).toContain("QA_SUBAGENT_CHILD_FIXED");
|
||||
});
|
||||
|
||||
it("surfaces sessions_spawn tool errors instead of echoing child-task markers", async () => {
|
||||
const server = await startMockServer();
|
||||
|
||||
const body = await expectResponsesJson<{
|
||||
output?: Array<{ content?: Array<{ text?: string }> }>;
|
||||
}>(server, {
|
||||
stream: false,
|
||||
tools: [SESSIONS_SPAWN_TOOL],
|
||||
input: [
|
||||
makeUserInput(explicitSessionsSpawnPrompt(THREAD_SUBAGENT_CHILD_ERROR_TOKEN)),
|
||||
{
|
||||
type: "function_call",
|
||||
name: "sessions_spawn",
|
||||
arguments: JSON.stringify({
|
||||
task: threadSubagentTask(THREAD_SUBAGENT_CHILD_ERROR_TOKEN),
|
||||
label: "qa-thread-subagent",
|
||||
thread: true,
|
||||
mode: "session",
|
||||
runTimeoutSeconds: 30,
|
||||
}),
|
||||
},
|
||||
{
|
||||
type: "function_call_output",
|
||||
output: JSON.stringify({
|
||||
status: "error",
|
||||
error: THREAD_SUBAGENT_TOOL_ERROR,
|
||||
}),
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
const text = body.output?.[0]?.content?.[0]?.text ?? "";
|
||||
expect(text).toContain(THREAD_SUBAGENT_TOOL_ERROR);
|
||||
expect(text).not.toContain(THREAD_SUBAGENT_CHILD_ERROR_TOKEN);
|
||||
});
|
||||
|
||||
it("plans memory tools and serves mock image generations", async () => {
|
||||
const server = await startQaMockOpenAiServer({
|
||||
host: "127.0.0.1",
|
||||
@@ -850,6 +929,7 @@ describe("qa mock openai server", () => {
|
||||
},
|
||||
body: JSON.stringify({
|
||||
stream: true,
|
||||
tools: [SESSIONS_SPAWN_TOOL],
|
||||
input: [
|
||||
{
|
||||
role: "user",
|
||||
@@ -875,6 +955,7 @@ describe("qa mock openai server", () => {
|
||||
},
|
||||
body: JSON.stringify({
|
||||
stream: true,
|
||||
tools: [SESSIONS_SPAWN_TOOL],
|
||||
input: [
|
||||
{
|
||||
role: "user",
|
||||
@@ -905,6 +986,7 @@ describe("qa mock openai server", () => {
|
||||
},
|
||||
body: JSON.stringify({
|
||||
stream: false,
|
||||
tools: [SESSIONS_SPAWN_TOOL],
|
||||
input: [
|
||||
{
|
||||
role: "user",
|
||||
@@ -961,6 +1043,7 @@ describe("qa mock openai server", () => {
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
stream: true,
|
||||
tools: [SESSIONS_SPAWN_TOOL],
|
||||
input: [{ role: "user", content: [{ type: "input_text", text: prompt }] }],
|
||||
}),
|
||||
});
|
||||
@@ -972,6 +1055,7 @@ describe("qa mock openai server", () => {
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
stream: true,
|
||||
tools: [SESSIONS_SPAWN_TOOL],
|
||||
input: [{ role: "user", content: [{ type: "input_text", text: prompt }] }],
|
||||
}),
|
||||
});
|
||||
@@ -1430,6 +1514,60 @@ describe("qa mock openai server", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("preserves Anthropic /v1/messages declared tools for explicit sessions_spawn prompts", async () => {
|
||||
const server = await startMockServer();
|
||||
|
||||
const response = await fetch(`${server.baseUrl}/v1/messages`, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
model: "claude-opus-4-6",
|
||||
max_tokens: 256,
|
||||
tools: [
|
||||
{
|
||||
name: "sessions_spawn",
|
||||
input_schema: { type: "object", properties: {} },
|
||||
},
|
||||
],
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{
|
||||
type: "text",
|
||||
text: explicitSessionsSpawnPrompt("QA_SUBAGENT_CHILD_ANTHROPIC"),
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
}),
|
||||
});
|
||||
expect(response.status).toBe(200);
|
||||
const body = (await response.json()) as {
|
||||
stop_reason: string;
|
||||
content: Array<Record<string, unknown>>;
|
||||
};
|
||||
expect(body.stop_reason).toBe("tool_use");
|
||||
const toolUseBlock = body.content.find((block) => block.type === "tool_use") as
|
||||
| { name: string; input: Record<string, unknown> }
|
||||
| undefined;
|
||||
expect(toolUseBlock?.name).toBe("sessions_spawn");
|
||||
expect(toolUseBlock?.input).toMatchObject({
|
||||
task: threadSubagentTask("QA_SUBAGENT_CHILD_ANTHROPIC"),
|
||||
label: "qa-thread-subagent",
|
||||
thread: true,
|
||||
mode: "session",
|
||||
runTimeoutSeconds: 30,
|
||||
});
|
||||
|
||||
const debugResponse = await fetch(`${server.baseUrl}/debug/last-request`);
|
||||
expect(debugResponse.status).toBe(200);
|
||||
expect(await debugResponse.json()).toMatchObject({
|
||||
model: "claude-opus-4-6",
|
||||
plannedToolName: "sessions_spawn",
|
||||
});
|
||||
});
|
||||
|
||||
it("dispatches Anthropic /v1/messages tool_result follow-ups through the shared scenario logic", async () => {
|
||||
// This verifies the Anthropic adapter correctly feeds tool_result
|
||||
// content blocks into the shared scenario dispatcher so downstream
|
||||
@@ -1,6 +1,6 @@
|
||||
import { createServer, type IncomingMessage, type ServerResponse } from "node:http";
|
||||
import { setTimeout as sleep } from "node:timers/promises";
|
||||
import { closeQaHttpServer } from "./bus-server.js";
|
||||
import { closeQaHttpServer } from "../../bus-server.js";
|
||||
|
||||
type ResponsesInputItem = Record<string, unknown>;
|
||||
|
||||
@@ -142,8 +142,8 @@ const QA_REASONING_ONLY_RECOVERY_PROMPT_RE = /reasoning-only continuation qa che
|
||||
const QA_REASONING_ONLY_SIDE_EFFECT_PROMPT_RE = /reasoning-only after write safety check/i;
|
||||
const QA_EMPTY_RESPONSE_RECOVERY_PROMPT_RE = /empty response continuation qa check/i;
|
||||
const QA_EMPTY_RESPONSE_EXHAUSTION_PROMPT_RE = /empty response exhaustion qa check/i;
|
||||
const QA_QUIET_STREAMING_PROMPT_RE = /(?:matrix\s+)?quiet streaming qa check/i;
|
||||
const QA_BLOCK_STREAMING_PROMPT_RE = /(?:matrix\s+)?block streaming qa check/i;
|
||||
const QA_QUIET_STREAMING_PROMPT_RE = /quiet streaming qa check/i;
|
||||
const QA_BLOCK_STREAMING_PROMPT_RE = /block streaming qa check/i;
|
||||
const QA_REASONING_ONLY_RETRY_NEEDLE =
|
||||
"recorded reasoning but did not produce a user-visible answer";
|
||||
const QA_EMPTY_RESPONSE_RETRY_NEEDLE =
|
||||
@@ -538,6 +538,77 @@ function extractLabeledMarkerDirective(text: string, label: string) {
|
||||
);
|
||||
}
|
||||
|
||||
function extractQuotedToolArg(text: string, name: string) {
|
||||
const escapedName = name.replaceAll(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||
return extractLastCapture(text, new RegExp(`\\b${escapedName}\\s*=\\s*"([^"]+)"`, "i"));
|
||||
}
|
||||
|
||||
function extractBareToolArg(text: string, name: string) {
|
||||
const escapedName = name.replaceAll(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||
return extractLastCapture(text, new RegExp(`\\b${escapedName}\\s*=\\s*([^\\s\\\`.,;:!?]+)`, "i"));
|
||||
}
|
||||
|
||||
function hasDeclaredTool(body: Record<string, unknown>, name: string) {
|
||||
const tools = Array.isArray(body.tools) ? body.tools : [];
|
||||
return tools.some((tool) => {
|
||||
if (!tool || typeof tool !== "object") {
|
||||
return false;
|
||||
}
|
||||
const record = tool as Record<string, unknown>;
|
||||
if (record.name === name) {
|
||||
return true;
|
||||
}
|
||||
const nested = record.function;
|
||||
return Boolean(
|
||||
nested && typeof nested === "object" && (nested as { name?: unknown }).name === name,
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
function buildExplicitSessionsSpawnArgs(text: string): Record<string, unknown> | null {
|
||||
if (!/\bsessions_spawn\b/i.test(text)) {
|
||||
return null;
|
||||
}
|
||||
const task = extractQuotedToolArg(text, "task");
|
||||
if (!task) {
|
||||
return null;
|
||||
}
|
||||
const label = extractQuotedToolArg(text, "label") ?? extractBareToolArg(text, "label");
|
||||
const mode = extractBareToolArg(text, "mode")?.toLowerCase();
|
||||
const runTimeoutSecondsRaw = extractBareToolArg(text, "runTimeoutSeconds");
|
||||
const runTimeoutSeconds =
|
||||
runTimeoutSecondsRaw && /^\d+$/.test(runTimeoutSecondsRaw)
|
||||
? Number(runTimeoutSecondsRaw)
|
||||
: undefined;
|
||||
return {
|
||||
task,
|
||||
...(label ? { label } : {}),
|
||||
...(extractBareToolArg(text, "thread")?.toLowerCase() === "true" ? { thread: true } : {}),
|
||||
...(mode === "session" || mode === "run" ? { mode } : {}),
|
||||
...(runTimeoutSeconds !== undefined ? { runTimeoutSeconds } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
function extractToolErrorForNamedCall(params: {
|
||||
allInputText: string;
|
||||
input: ResponsesInputItem[];
|
||||
name: string;
|
||||
toolJson: Record<string, unknown> | null;
|
||||
}) {
|
||||
const error = typeof params.toolJson?.error === "string" ? params.toolJson.error.trim() : "";
|
||||
if (!error) {
|
||||
return undefined;
|
||||
}
|
||||
const namedFunctionCall = params.input.some(
|
||||
(item) => item.type === "function_call" && item.name === params.name,
|
||||
);
|
||||
const namedPromptReference = new RegExp(`\\b${params.name}\\b`, "i").test(params.allInputText);
|
||||
if (namedFunctionCall || namedPromptReference) {
|
||||
return error;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function isHeartbeatPrompt(text: string) {
|
||||
const trimmed = text.trim();
|
||||
if (!trimmed || /remember this fact/i.test(trimmed)) {
|
||||
@@ -573,10 +644,19 @@ function buildAssistantText(
|
||||
const imageInputCount = countImageInputs(input);
|
||||
const activeMemorySummary = extractActiveMemorySummary(allInputText);
|
||||
const snackPreference = extractSnackPreference(activeMemorySummary ?? memorySnippet);
|
||||
const sessionsSpawnError = extractToolErrorForNamedCall({
|
||||
allInputText,
|
||||
input,
|
||||
name: "sessions_spawn",
|
||||
toolJson,
|
||||
});
|
||||
|
||||
if (/what was the qa canary code/i.test(prompt) && rememberedFact) {
|
||||
return `Protocol note: the QA canary code was ${rememberedFact}.`;
|
||||
}
|
||||
if (sessionsSpawnError) {
|
||||
return `Protocol note: sessions_spawn failed: ${sessionsSpawnError}`;
|
||||
}
|
||||
if (/remember this fact/i.test(prompt) && exactReplyDirective) {
|
||||
return exactReplyDirective;
|
||||
}
|
||||
@@ -867,6 +947,7 @@ async function buildResponsesPayload(
|
||||
const isBaselineUnmentionedChannelChatter = /\bno bot ping here\b/i.test(prompt);
|
||||
const hasReasoningOnlyRetryInstruction = allInputText.includes(QA_REASONING_ONLY_RETRY_NEEDLE);
|
||||
const hasEmptyResponseRetryInstruction = allInputText.includes(QA_EMPTY_RESPONSE_RETRY_NEEDLE);
|
||||
const canCallSessionsSpawn = hasDeclaredTool(body, "sessions_spawn");
|
||||
if (/remember this fact/i.test(prompt)) {
|
||||
return buildAssistantEvents(buildAssistantText(input, body, scenarioState));
|
||||
}
|
||||
@@ -1110,7 +1191,7 @@ async function buildResponsesPayload(
|
||||
size: "1024x1024",
|
||||
});
|
||||
}
|
||||
if (/subagent fanout synthesis check/i.test(prompt)) {
|
||||
if (canCallSessionsSpawn && /subagent fanout synthesis check/i.test(prompt)) {
|
||||
if (!toolOutput && scenarioState.subagentFanoutPhase === 0) {
|
||||
scenarioState.subagentFanoutPhase = 1;
|
||||
return buildToolCallEventsWithArgs("sessions_spawn", {
|
||||
@@ -1128,6 +1209,10 @@ async function buildResponsesPayload(
|
||||
});
|
||||
}
|
||||
}
|
||||
const explicitSessionsSpawnArgs = buildExplicitSessionsSpawnArgs(allInputText);
|
||||
if (canCallSessionsSpawn && explicitSessionsSpawnArgs && !toolOutput) {
|
||||
return buildToolCallEventsWithArgs("sessions_spawn", explicitSessionsSpawnArgs);
|
||||
}
|
||||
if (/tool continuity check/i.test(prompt) && !toolOutput) {
|
||||
return buildToolCallEventsWithArgs("read", { path: "QA_KICKOFF_TASK.md" });
|
||||
}
|
||||
@@ -1155,7 +1240,11 @@ async function buildResponsesPayload(
|
||||
});
|
||||
}
|
||||
}
|
||||
if ((/\bdelegate\b/i.test(prompt) || /subagent handoff/i.test(prompt)) && !toolOutput) {
|
||||
if (
|
||||
canCallSessionsSpawn &&
|
||||
(/\bdelegate\b/i.test(prompt) || /subagent handoff/i.test(prompt)) &&
|
||||
!toolOutput
|
||||
) {
|
||||
return buildToolCallEventsWithArgs("sessions_spawn", {
|
||||
task: "Inspect the QA workspace and return one concise protocol note.",
|
||||
label: "qa-sidecar",
|
||||
@@ -1547,12 +1636,13 @@ async function buildMessagesPayload(
|
||||
const normalizedModel =
|
||||
typeof body.model === "string" && body.model.trim() !== "" ? body.model : "claude-opus-4-6";
|
||||
// Dispatch through the same scenario logic the /v1/responses route uses.
|
||||
// The mock dispatcher only reads `body.input`, `body.model`, and
|
||||
// `body.stream`, so a synthetic shim body is sufficient.
|
||||
// Preserve declared tools so route-specific adapters mirror what the
|
||||
// real provider request made available to the model.
|
||||
const dispatchBody: Record<string, unknown> = {
|
||||
input,
|
||||
model: normalizedModel,
|
||||
stream: false,
|
||||
...(Array.isArray(body.tools) ? { tools: body.tools } : {}),
|
||||
};
|
||||
const events = await buildResponsesPayload(dispatchBody, scenarioState);
|
||||
const extracted = extractFinalAssistantOutputFromEvents(events);
|
||||
36
extensions/qa-lab/src/providers/server-runtime.ts
Normal file
36
extensions/qa-lab/src/providers/server-runtime.ts
Normal file
@@ -0,0 +1,36 @@
|
||||
import { getQaProvider, type QaMockProviderServer, type QaProviderModeInput } from "./index.js";
|
||||
|
||||
type QaProviderServerParams = {
|
||||
host: string;
|
||||
port: number;
|
||||
};
|
||||
|
||||
async function startMockOpenAiProviderServer(params: QaProviderServerParams) {
|
||||
const { startQaMockOpenAiServer } = await import("./mock-openai/server.js");
|
||||
return await startQaMockOpenAiServer(params);
|
||||
}
|
||||
|
||||
async function startAimockProviderServer(params: QaProviderServerParams) {
|
||||
const { startQaAimockServer } = await import("./aimock/server.js");
|
||||
return await startQaAimockServer(params);
|
||||
}
|
||||
|
||||
export async function startQaProviderServer(
|
||||
input: QaProviderModeInput,
|
||||
params?: { host?: string; port?: number },
|
||||
): Promise<QaMockProviderServer | null> {
|
||||
const provider = getQaProvider(input);
|
||||
const serverParams = {
|
||||
host: params?.host ?? "127.0.0.1",
|
||||
port: params?.port ?? 0,
|
||||
};
|
||||
switch (provider.mode) {
|
||||
case "mock-openai":
|
||||
return await startMockOpenAiProviderServer(serverParams);
|
||||
case "aimock":
|
||||
return await startAimockProviderServer(serverParams);
|
||||
case "live-frontier":
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
}
|
||||
70
extensions/qa-lab/src/providers/shared/mock-auth.ts
Normal file
70
extensions/qa-lab/src/providers/shared/mock-auth.ts
Normal file
@@ -0,0 +1,70 @@
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-runtime";
|
||||
import { applyAuthProfileConfig, upsertAuthProfile } from "openclaw/plugin-sdk/provider-auth";
|
||||
|
||||
/** Providers the mock harness stages placeholder credentials for by default. */
|
||||
export const QA_MOCK_AUTH_PROVIDERS = Object.freeze(["openai", "anthropic"] as const);
|
||||
|
||||
/** Agent IDs the mock harness stages credentials under. */
|
||||
export const QA_MOCK_AUTH_AGENT_IDS = Object.freeze(["main", "qa"] as const);
|
||||
|
||||
export function buildQaMockProfileId(provider: string): string {
|
||||
return `qa-mock-${provider}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* In mock provider modes the qa suite runs against an embedded mock server
|
||||
* instead of a real provider API. The mock does not validate credentials, but
|
||||
* the agent auth layer still needs a matching `api_key` auth profile in
|
||||
* `auth-profiles.json` before it will route the request through
|
||||
* `providerBaseUrl`. Without this staging step, every scenario fails with
|
||||
* `FailoverError: No API key found for provider "openai"` before the mock
|
||||
* server ever sees a request.
|
||||
*
|
||||
* Stages a placeholder `api_key` profile per provider in each of the agent
|
||||
* dirs the qa suite uses (`main` for the runtime config, `qa` for scenario
|
||||
* runs) and returns a config with matching `auth.profiles` entries so the
|
||||
* runtime accepts the profile on the first lookup.
|
||||
*
|
||||
* The placeholder value `qa-mock-not-a-real-key` is intentionally not
|
||||
* shaped like a real API key (no `sk-` prefix that would trip secret
|
||||
* scanners). It only needs to be non-empty to pass the credential
|
||||
* serializer; anything beyond that is ignored by the mock.
|
||||
*/
|
||||
export async function stageQaMockAuthProfiles(params: {
|
||||
cfg: OpenClawConfig;
|
||||
stateDir: string;
|
||||
agentIds?: readonly string[];
|
||||
providers?: readonly string[];
|
||||
}): Promise<OpenClawConfig> {
|
||||
const agentIds = [...new Set(params.agentIds ?? QA_MOCK_AUTH_AGENT_IDS)];
|
||||
const providers = [...new Set(params.providers ?? QA_MOCK_AUTH_PROVIDERS)];
|
||||
let next = params.cfg;
|
||||
for (const agentId of agentIds) {
|
||||
const agentDir = path.join(params.stateDir, "agents", agentId, "agent");
|
||||
await fs.mkdir(agentDir, { recursive: true });
|
||||
for (const provider of providers) {
|
||||
const profileId = buildQaMockProfileId(provider);
|
||||
upsertAuthProfile({
|
||||
profileId,
|
||||
credential: {
|
||||
type: "api_key",
|
||||
provider,
|
||||
key: "qa-mock-not-a-real-key",
|
||||
displayName: `QA mock ${provider} credential`,
|
||||
},
|
||||
agentDir,
|
||||
});
|
||||
}
|
||||
}
|
||||
for (const provider of providers) {
|
||||
next = applyAuthProfileConfig(next, {
|
||||
profileId: buildQaMockProfileId(provider),
|
||||
provider,
|
||||
mode: "api_key",
|
||||
displayName: `QA mock ${provider} credential`,
|
||||
});
|
||||
}
|
||||
return next;
|
||||
}
|
||||
104
extensions/qa-lab/src/providers/shared/mock-model-config.ts
Normal file
104
extensions/qa-lab/src/providers/shared/mock-model-config.ts
Normal file
@@ -0,0 +1,104 @@
|
||||
import type { ModelProviderConfig } from "openclaw/plugin-sdk/provider-model-shared";
|
||||
|
||||
const ZERO_COST = Object.freeze({
|
||||
input: 0,
|
||||
output: 0,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
});
|
||||
|
||||
function cloneProvider(provider: ModelProviderConfig): ModelProviderConfig {
|
||||
return {
|
||||
...provider,
|
||||
models: provider.models.map((model) => ({ ...model })),
|
||||
};
|
||||
}
|
||||
|
||||
export function trimTrailingApiV1(baseUrl: string) {
|
||||
return baseUrl.replace(/\/v1\/?$/i, "");
|
||||
}
|
||||
|
||||
export function createMockOpenAiResponsesProvider(baseUrl: string): ModelProviderConfig {
|
||||
return {
|
||||
baseUrl,
|
||||
apiKey: "test",
|
||||
api: "openai-responses",
|
||||
request: {
|
||||
allowPrivateNetwork: true,
|
||||
},
|
||||
models: [
|
||||
{
|
||||
id: "gpt-5.4",
|
||||
name: "gpt-5.4",
|
||||
api: "openai-responses",
|
||||
reasoning: false,
|
||||
input: ["text", "image"],
|
||||
cost: ZERO_COST,
|
||||
contextWindow: 128_000,
|
||||
maxTokens: 4096,
|
||||
},
|
||||
{
|
||||
id: "gpt-5.4-alt",
|
||||
name: "gpt-5.4-alt",
|
||||
api: "openai-responses",
|
||||
reasoning: false,
|
||||
input: ["text", "image"],
|
||||
cost: ZERO_COST,
|
||||
contextWindow: 128_000,
|
||||
maxTokens: 4096,
|
||||
},
|
||||
{
|
||||
id: "gpt-image-1",
|
||||
name: "gpt-image-1",
|
||||
api: "openai-responses",
|
||||
reasoning: false,
|
||||
input: ["text"],
|
||||
cost: ZERO_COST,
|
||||
contextWindow: 128_000,
|
||||
maxTokens: 4096,
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
export function createMockAnthropicMessagesProvider(baseUrl: string): ModelProviderConfig {
|
||||
return {
|
||||
baseUrl: trimTrailingApiV1(baseUrl),
|
||||
apiKey: "test",
|
||||
api: "anthropic-messages",
|
||||
request: {
|
||||
allowPrivateNetwork: true,
|
||||
},
|
||||
models: [
|
||||
{
|
||||
id: "claude-opus-4-6",
|
||||
name: "claude-opus-4-6",
|
||||
api: "anthropic-messages",
|
||||
reasoning: false,
|
||||
input: ["text", "image"],
|
||||
cost: ZERO_COST,
|
||||
contextWindow: 200_000,
|
||||
maxTokens: 4096,
|
||||
},
|
||||
{
|
||||
id: "claude-sonnet-4-6",
|
||||
name: "claude-sonnet-4-6",
|
||||
api: "anthropic-messages",
|
||||
reasoning: false,
|
||||
input: ["text", "image"],
|
||||
cost: ZERO_COST,
|
||||
contextWindow: 200_000,
|
||||
maxTokens: 4096,
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
export function createMockProviderMap(primaryProviderId: string, providerBaseUrl: string) {
|
||||
const primaryProvider = createMockOpenAiResponsesProvider(providerBaseUrl);
|
||||
return {
|
||||
[primaryProviderId]: primaryProvider,
|
||||
openai: cloneProvider(primaryProvider),
|
||||
anthropic: createMockAnthropicMessagesProvider(providerBaseUrl),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
import { createMockProviderMap } from "./mock-model-config.js";
|
||||
import type { QaProviderDefinition, QaProviderMode } from "./types.js";
|
||||
|
||||
export type MockQaProviderDefinitionParams = {
|
||||
mode: Extract<QaProviderMode, "aimock" | "mock-openai">;
|
||||
commandName: string;
|
||||
commandDescription: string;
|
||||
serverLabel: string;
|
||||
mockAuthProviders: readonly string[];
|
||||
};
|
||||
|
||||
function mockModelRef(providerId: string, alternate?: boolean) {
|
||||
return `${providerId}/${alternate ? "gpt-5.4-alt" : "gpt-5.4"}`;
|
||||
}
|
||||
|
||||
export function createMockQaProviderDefinition(
|
||||
params: MockQaProviderDefinitionParams,
|
||||
): QaProviderDefinition {
|
||||
return {
|
||||
mode: params.mode,
|
||||
kind: "mock",
|
||||
standaloneCommand: {
|
||||
name: params.commandName,
|
||||
description: params.commandDescription,
|
||||
serverLabel: params.serverLabel,
|
||||
},
|
||||
defaultModel: (options) => mockModelRef(params.mode, options?.alternate),
|
||||
defaultImageGenerationProviderIds: [],
|
||||
defaultImageGenerationModel: () => `${params.mode}/gpt-image-1`,
|
||||
usesFastModeByDefault: () => false,
|
||||
resolveModelParams: () => ({
|
||||
transport: "sse",
|
||||
openaiWsWarmup: false,
|
||||
}),
|
||||
resolveTurnTimeoutMs: ({ fallbackMs }) => fallbackMs,
|
||||
buildGatewayModels: ({ providerBaseUrl }) => ({
|
||||
mode: "replace",
|
||||
providers: createMockProviderMap(params.mode, providerBaseUrl),
|
||||
}),
|
||||
mockAuthProviders: params.mockAuthProviders,
|
||||
usesModelProviderPlugins: false,
|
||||
scrubsLiveProviderEnv: true,
|
||||
appliesLiveEnvAliases: false,
|
||||
};
|
||||
}
|
||||
56
extensions/qa-lab/src/providers/shared/types.ts
Normal file
56
extensions/qa-lab/src/providers/shared/types.ts
Normal file
@@ -0,0 +1,56 @@
|
||||
import type { ModelProviderConfig } from "openclaw/plugin-sdk/provider-model-shared";
|
||||
import type { QaThinkingLevel } from "../../qa-thinking.js";
|
||||
|
||||
export type QaProviderMode = "mock-openai" | "aimock" | "live-frontier";
|
||||
export type QaProviderModeInput = QaProviderMode;
|
||||
|
||||
export type QaMockProviderServer = {
|
||||
baseUrl: string;
|
||||
stop(): Promise<void>;
|
||||
};
|
||||
|
||||
export type QaProviderModelParamsInput = {
|
||||
modelRef: string;
|
||||
fastMode?: boolean;
|
||||
thinkingDefault?: QaThinkingLevel;
|
||||
};
|
||||
|
||||
export type QaProviderGatewayModelsInput = {
|
||||
providerBaseUrl: string;
|
||||
liveProviderConfigs?: Record<string, ModelProviderConfig>;
|
||||
};
|
||||
|
||||
export type QaProviderDefaultImageInput = {
|
||||
modelProviderIds: readonly string[];
|
||||
};
|
||||
|
||||
export type QaProviderTurnTimeoutInput = {
|
||||
primaryModel: string;
|
||||
alternateModel: string;
|
||||
modelRef: string;
|
||||
fallbackMs: number;
|
||||
};
|
||||
|
||||
export type QaProviderDefinition = {
|
||||
mode: QaProviderMode;
|
||||
kind: "mock" | "live";
|
||||
standaloneCommand?: {
|
||||
name: string;
|
||||
description: string;
|
||||
serverLabel: string;
|
||||
};
|
||||
defaultModel(options?: { alternate?: boolean; preferredLiveModel?: string }): string;
|
||||
defaultImageGenerationProviderIds: readonly string[];
|
||||
defaultImageGenerationModel(input: QaProviderDefaultImageInput): string | null;
|
||||
usesFastModeByDefault(modelRef: string): boolean;
|
||||
resolveModelParams(input: QaProviderModelParamsInput): Record<string, unknown>;
|
||||
resolveTurnTimeoutMs(input: QaProviderTurnTimeoutInput): number;
|
||||
buildGatewayModels(input: QaProviderGatewayModelsInput): {
|
||||
mode: "replace" | "merge";
|
||||
providers: Record<string, ModelProviderConfig>;
|
||||
} | null;
|
||||
mockAuthProviders?: readonly string[];
|
||||
usesModelProviderPlugins: boolean;
|
||||
scrubsLiveProviderEnv: boolean;
|
||||
appliesLiveEnvAliases: boolean;
|
||||
};
|
||||
@@ -1,5 +1,6 @@
|
||||
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-runtime";
|
||||
import type { QaBusState } from "./bus-state.js";
|
||||
import { getQaProvider } from "./providers/index.js";
|
||||
import { QaStateBackedTransportAdapter, waitForQaTransportCondition } from "./qa-transport.js";
|
||||
import type {
|
||||
QaTransportActionName,
|
||||
@@ -71,9 +72,10 @@ export function createQaChannelGatewayConfig(params: {
|
||||
}
|
||||
|
||||
function createQaChannelReportNotes(params: QaTransportReportParams) {
|
||||
const provider = getQaProvider(params.providerMode);
|
||||
return [
|
||||
params.providerMode === "mock-openai"
|
||||
? "Runs against qa-channel + qa-lab bus + real gateway child + mock OpenAI provider."
|
||||
provider.kind === "mock"
|
||||
? `Runs against qa-channel + qa-lab bus + real gateway child + ${params.providerMode} provider.`
|
||||
: `Runs against qa-channel + qa-lab bus + real gateway child + live frontier models (${params.primaryModel}, ${params.alternateModel})${params.fastMode ? " with fast mode enabled" : ""}.`,
|
||||
params.concurrency > 1
|
||||
? `Scenarios run in isolated gateway workers with concurrency ${params.concurrency}.`
|
||||
|
||||
@@ -96,6 +96,29 @@ describe("buildQaGatewayConfig", () => {
|
||||
expect(cfg.plugins?.allow).toEqual(["memory-core"]);
|
||||
});
|
||||
|
||||
it("can wire AIMock as a separate mock provider lane", () => {
|
||||
const cfg = buildQaGatewayConfig({
|
||||
bind: "loopback",
|
||||
gatewayPort: 18789,
|
||||
gatewayToken: "token",
|
||||
providerBaseUrl: "http://127.0.0.1:45080/v1",
|
||||
workspaceDir: "/tmp/qa-workspace",
|
||||
providerMode: "aimock",
|
||||
primaryModel: "aimock/gpt-5.4",
|
||||
alternateModel: "aimock/gpt-5.4-alt",
|
||||
});
|
||||
|
||||
expect(getPrimaryModel(cfg.agents?.defaults?.model)).toBe("aimock/gpt-5.4");
|
||||
expect(cfg.agents?.defaults?.imageGenerationModel).toEqual({
|
||||
primary: "aimock/gpt-image-1",
|
||||
});
|
||||
expect(cfg.models?.providers?.aimock?.baseUrl).toBe("http://127.0.0.1:45080/v1");
|
||||
expect(cfg.models?.providers?.aimock?.api).toBe("openai-responses");
|
||||
expect(cfg.models?.providers?.openai?.baseUrl).toBe("http://127.0.0.1:45080/v1");
|
||||
expect(cfg.models?.providers?.anthropic?.baseUrl).toBe("http://127.0.0.1:45080");
|
||||
expect(cfg.models?.providers?.["mock-openai"]).toBeUndefined();
|
||||
});
|
||||
|
||||
it("can omit qa-channel for live transport gateway children", () => {
|
||||
const cfg = buildQaGatewayConfig({
|
||||
bind: "loopback",
|
||||
|
||||
@@ -2,13 +2,17 @@ import type { OpenClawConfig } from "openclaw/plugin-sdk/config-runtime";
|
||||
import type { ModelProviderConfig } from "openclaw/plugin-sdk/provider-model-shared";
|
||||
import {
|
||||
defaultQaModelForMode,
|
||||
isQaFastModeModelRef,
|
||||
normalizeQaProviderMode,
|
||||
splitQaModelRef,
|
||||
type QaProviderMode,
|
||||
} from "./model-selection.js";
|
||||
import { getQaProvider } from "./providers/index.js";
|
||||
import { DEFAULT_QA_PROVIDER_MODE } from "./providers/index.js";
|
||||
import { normalizeQaThinkingLevel, type QaThinkingLevel } from "./qa-thinking.js";
|
||||
import type { QaTransportGatewayConfig } from "./qa-transport.js";
|
||||
|
||||
export { normalizeQaThinkingLevel, type QaThinkingLevel } from "./qa-thinking.js";
|
||||
|
||||
export const DEFAULT_QA_CONTROL_UI_ALLOWED_ORIGINS = Object.freeze([
|
||||
"http://127.0.0.1:18789",
|
||||
"http://localhost:18789",
|
||||
@@ -16,39 +20,6 @@ export const DEFAULT_QA_CONTROL_UI_ALLOWED_ORIGINS = Object.freeze([
|
||||
"http://localhost:43124",
|
||||
]);
|
||||
|
||||
export type QaThinkingLevel = "off" | "minimal" | "low" | "medium" | "high" | "xhigh" | "adaptive";
|
||||
|
||||
export function normalizeQaThinkingLevel(input: unknown): QaThinkingLevel | undefined {
|
||||
const value = typeof input === "string" ? input.trim().toLowerCase() : "";
|
||||
const collapsed = value.replace(/[\s_-]+/g, "");
|
||||
if (collapsed === "off") {
|
||||
return "off";
|
||||
}
|
||||
if (collapsed === "minimal" || collapsed === "min") {
|
||||
return "minimal";
|
||||
}
|
||||
if (collapsed === "low") {
|
||||
return "low";
|
||||
}
|
||||
if (collapsed === "medium" || collapsed === "med") {
|
||||
return "medium";
|
||||
}
|
||||
if (collapsed === "high") {
|
||||
return "high";
|
||||
}
|
||||
if (collapsed === "xhigh" || collapsed === "extrahigh") {
|
||||
return "xhigh";
|
||||
}
|
||||
if (collapsed === "adaptive" || collapsed === "auto") {
|
||||
return "adaptive";
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function trimTrailingApiV1(baseUrl: string) {
|
||||
return baseUrl.replace(/\/v1\/?$/i, "");
|
||||
}
|
||||
|
||||
export function mergeQaControlUiAllowedOrigins(extraOrigins?: string[]) {
|
||||
const normalizedExtra = (extraOrigins ?? [])
|
||||
.map((origin) => origin.trim())
|
||||
@@ -65,7 +36,7 @@ export function buildQaGatewayConfig(params: {
|
||||
controlUiRoot?: string;
|
||||
controlUiAllowedOrigins?: string[];
|
||||
controlUiEnabled?: boolean;
|
||||
providerMode?: QaProviderMode | "live-openai";
|
||||
providerMode?: QaProviderMode;
|
||||
primaryModel?: string;
|
||||
alternateModel?: string;
|
||||
imageGenerationModel?: string | null;
|
||||
@@ -77,108 +48,9 @@ export function buildQaGatewayConfig(params: {
|
||||
fastMode?: boolean;
|
||||
thinkingDefault?: QaThinkingLevel;
|
||||
}): OpenClawConfig {
|
||||
const mockProviderBaseUrl = params.providerBaseUrl ?? "http://127.0.0.1:44080/v1";
|
||||
const mockAnthropicBaseUrl = trimTrailingApiV1(mockProviderBaseUrl);
|
||||
const mockOpenAiProvider: ModelProviderConfig = {
|
||||
baseUrl: mockProviderBaseUrl,
|
||||
apiKey: "test",
|
||||
api: "openai-responses",
|
||||
request: {
|
||||
allowPrivateNetwork: true,
|
||||
},
|
||||
models: [
|
||||
{
|
||||
id: "gpt-5.4",
|
||||
name: "gpt-5.4",
|
||||
api: "openai-responses",
|
||||
reasoning: false,
|
||||
input: ["text", "image"],
|
||||
cost: {
|
||||
input: 0,
|
||||
output: 0,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
},
|
||||
contextWindow: 128_000,
|
||||
maxTokens: 4096,
|
||||
},
|
||||
{
|
||||
id: "gpt-5.4-alt",
|
||||
name: "gpt-5.4-alt",
|
||||
api: "openai-responses",
|
||||
reasoning: false,
|
||||
input: ["text", "image"],
|
||||
cost: {
|
||||
input: 0,
|
||||
output: 0,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
},
|
||||
contextWindow: 128_000,
|
||||
maxTokens: 4096,
|
||||
},
|
||||
{
|
||||
id: "gpt-image-1",
|
||||
name: "gpt-image-1",
|
||||
api: "openai-responses",
|
||||
reasoning: false,
|
||||
input: ["text"],
|
||||
cost: {
|
||||
input: 0,
|
||||
output: 0,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
},
|
||||
contextWindow: 128_000,
|
||||
maxTokens: 4096,
|
||||
},
|
||||
],
|
||||
};
|
||||
const mockNamedOpenAiProvider: ModelProviderConfig = {
|
||||
...mockOpenAiProvider,
|
||||
models: mockOpenAiProvider.models.map((model) => ({ ...model })),
|
||||
};
|
||||
const mockAnthropicProvider: ModelProviderConfig = {
|
||||
baseUrl: mockAnthropicBaseUrl,
|
||||
apiKey: "test",
|
||||
api: "anthropic-messages",
|
||||
request: {
|
||||
allowPrivateNetwork: true,
|
||||
},
|
||||
models: [
|
||||
{
|
||||
id: "claude-opus-4-6",
|
||||
name: "claude-opus-4-6",
|
||||
api: "anthropic-messages",
|
||||
reasoning: false,
|
||||
input: ["text", "image"],
|
||||
cost: {
|
||||
input: 0,
|
||||
output: 0,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
},
|
||||
contextWindow: 200_000,
|
||||
maxTokens: 4096,
|
||||
},
|
||||
{
|
||||
id: "claude-sonnet-4-6",
|
||||
name: "claude-sonnet-4-6",
|
||||
api: "anthropic-messages",
|
||||
reasoning: false,
|
||||
input: ["text", "image"],
|
||||
cost: {
|
||||
input: 0,
|
||||
output: 0,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
},
|
||||
contextWindow: 200_000,
|
||||
maxTokens: 4096,
|
||||
},
|
||||
],
|
||||
};
|
||||
const providerMode = normalizeQaProviderMode(params.providerMode ?? "mock-openai");
|
||||
const providerBaseUrl = params.providerBaseUrl ?? "http://127.0.0.1:44080/v1";
|
||||
const providerMode = normalizeQaProviderMode(params.providerMode ?? DEFAULT_QA_PROVIDER_MODE);
|
||||
const provider = getQaProvider(providerMode);
|
||||
const primaryModel = params.primaryModel ?? defaultQaModelForMode(providerMode);
|
||||
const alternateModel =
|
||||
params.alternateModel ?? defaultQaModelForMode(providerMode, { alternate: true });
|
||||
@@ -188,39 +60,33 @@ export function buildQaGatewayConfig(params: {
|
||||
const imageGenerationModelRef =
|
||||
params.imageGenerationModel !== undefined
|
||||
? params.imageGenerationModel
|
||||
: providerMode === "mock-openai"
|
||||
? "mock-openai/gpt-image-1"
|
||||
: modelProviderIds.includes("openai")
|
||||
? "openai/gpt-image-1"
|
||||
: null;
|
||||
const selectedProviderIds =
|
||||
providerMode === "live-frontier"
|
||||
? [
|
||||
...new Set(
|
||||
[...(params.enabledProviderIds ?? []), ...modelProviderIds, imageGenerationModelRef]
|
||||
.map((value) =>
|
||||
typeof value === "string" ? (splitQaModelRef(value)?.provider ?? value) : null,
|
||||
)
|
||||
.filter((provider): provider is string => Boolean(provider)),
|
||||
),
|
||||
]
|
||||
: [];
|
||||
const selectedPluginIds =
|
||||
providerMode === "live-frontier"
|
||||
? [
|
||||
...new Set(
|
||||
(params.enabledPluginIds?.length ?? 0) > 0
|
||||
? params.enabledPluginIds
|
||||
: selectedProviderIds,
|
||||
),
|
||||
]
|
||||
: [
|
||||
...new Set(
|
||||
(params.enabledPluginIds ?? [])
|
||||
.map((pluginId) => pluginId.trim())
|
||||
.filter((pluginId) => pluginId.length > 0),
|
||||
),
|
||||
];
|
||||
: provider.defaultImageGenerationModel({ modelProviderIds });
|
||||
const selectedProviderIds = provider.usesModelProviderPlugins
|
||||
? [
|
||||
...new Set(
|
||||
[...(params.enabledProviderIds ?? []), ...modelProviderIds, imageGenerationModelRef]
|
||||
.map((value) =>
|
||||
typeof value === "string" ? (splitQaModelRef(value)?.provider ?? value) : null,
|
||||
)
|
||||
.filter((provider): provider is string => Boolean(provider)),
|
||||
),
|
||||
]
|
||||
: [];
|
||||
const selectedPluginIds = provider.usesModelProviderPlugins
|
||||
? [
|
||||
...new Set(
|
||||
(params.enabledPluginIds?.length ?? 0) > 0
|
||||
? params.enabledPluginIds
|
||||
: selectedProviderIds,
|
||||
),
|
||||
]
|
||||
: [
|
||||
...new Set(
|
||||
(params.enabledPluginIds ?? [])
|
||||
.map((pluginId) => pluginId.trim())
|
||||
.filter((pluginId) => pluginId.length > 0),
|
||||
),
|
||||
];
|
||||
const transportPluginIds = [...new Set(params.transportPluginIds ?? [])]
|
||||
.map((pluginId) => pluginId.trim())
|
||||
.filter((pluginId) => pluginId.length > 0);
|
||||
@@ -231,22 +97,17 @@ export function buildQaGatewayConfig(params: {
|
||||
transportPluginIds.map((pluginId) => [pluginId, { enabled: true }]),
|
||||
);
|
||||
const allowedPlugins = [...new Set(["memory-core", ...selectedPluginIds, ...transportPluginIds])];
|
||||
const liveModelParams =
|
||||
providerMode === "live-frontier"
|
||||
? (modelRef: string) => ({
|
||||
transport: "sse",
|
||||
openaiWsWarmup: false,
|
||||
...(params.fastMode === true || isQaFastModeModelRef(modelRef) ? { fastMode: true } : {}),
|
||||
...(params.thinkingDefault ? { thinking: params.thinkingDefault } : {}),
|
||||
})
|
||||
: (_modelRef: string) => ({
|
||||
transport: "sse",
|
||||
openaiWsWarmup: false,
|
||||
});
|
||||
const resolveModelParams = (modelRef: string) =>
|
||||
provider.resolveModelParams({
|
||||
modelRef,
|
||||
fastMode: params.fastMode,
|
||||
thinkingDefault: params.thinkingDefault,
|
||||
});
|
||||
const allowedOrigins = mergeQaControlUiAllowedOrigins(params.controlUiAllowedOrigins);
|
||||
const liveProviderConfigs =
|
||||
providerMode === "live-frontier" ? (params.liveProviderConfigs ?? {}) : {};
|
||||
const hasLiveProviderConfigs = Object.keys(liveProviderConfigs).length > 0;
|
||||
const gatewayModels = provider.buildGatewayModels({
|
||||
providerBaseUrl,
|
||||
liveProviderConfigs: params.liveProviderConfigs,
|
||||
});
|
||||
|
||||
return {
|
||||
plugins: {
|
||||
@@ -286,10 +147,10 @@ export function buildQaGatewayConfig(params: {
|
||||
},
|
||||
models: {
|
||||
[primaryModel]: {
|
||||
params: liveModelParams(primaryModel),
|
||||
params: resolveModelParams(primaryModel),
|
||||
},
|
||||
[alternateModel]: {
|
||||
params: liveModelParams(alternateModel),
|
||||
params: resolveModelParams(alternateModel),
|
||||
},
|
||||
},
|
||||
subagents: {
|
||||
@@ -319,25 +180,14 @@ export function buildQaGatewayConfig(params: {
|
||||
memory: {
|
||||
backend: "builtin",
|
||||
},
|
||||
...(providerMode === "mock-openai"
|
||||
...(gatewayModels
|
||||
? {
|
||||
models: {
|
||||
mode: "replace",
|
||||
providers: {
|
||||
"mock-openai": mockOpenAiProvider,
|
||||
openai: mockNamedOpenAiProvider,
|
||||
anthropic: mockAnthropicProvider,
|
||||
},
|
||||
mode: gatewayModels.mode,
|
||||
providers: gatewayModels.providers,
|
||||
},
|
||||
}
|
||||
: hasLiveProviderConfigs
|
||||
? {
|
||||
models: {
|
||||
mode: "merge",
|
||||
providers: liveProviderConfigs,
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
: {}),
|
||||
gateway: {
|
||||
mode: "local",
|
||||
bind: params.bind,
|
||||
|
||||
28
extensions/qa-lab/src/qa-thinking.ts
Normal file
28
extensions/qa-lab/src/qa-thinking.ts
Normal file
@@ -0,0 +1,28 @@
|
||||
export type QaThinkingLevel = "off" | "minimal" | "low" | "medium" | "high" | "xhigh" | "adaptive";
|
||||
|
||||
export function normalizeQaThinkingLevel(input: unknown): QaThinkingLevel | undefined {
|
||||
const value = typeof input === "string" ? input.trim().toLowerCase() : "";
|
||||
const collapsed = value.replace(/[\s_-]+/g, "");
|
||||
if (collapsed === "off") {
|
||||
return "off";
|
||||
}
|
||||
if (collapsed === "minimal" || collapsed === "min") {
|
||||
return "minimal";
|
||||
}
|
||||
if (collapsed === "low") {
|
||||
return "low";
|
||||
}
|
||||
if (collapsed === "medium" || collapsed === "med") {
|
||||
return "medium";
|
||||
}
|
||||
if (collapsed === "high") {
|
||||
return "high";
|
||||
}
|
||||
if (collapsed === "xhigh" || collapsed === "extrahigh") {
|
||||
return "xhigh";
|
||||
}
|
||||
if (collapsed === "adaptive" || collapsed === "auto") {
|
||||
return "adaptive";
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
import { setTimeout as sleep } from "node:timers/promises";
|
||||
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-runtime";
|
||||
import type { QaProviderMode } from "./model-selection.js";
|
||||
import { extractQaFailureReplyText } from "./reply-failure.js";
|
||||
import type {
|
||||
QaBusInboundMessageInput,
|
||||
@@ -24,7 +25,7 @@ export type QaTransportGatewayClient = {
|
||||
export type QaTransportActionName = "delete" | "edit" | "react" | "thread-create";
|
||||
|
||||
export type QaTransportReportParams = {
|
||||
providerMode: "mock-openai" | "live-frontier";
|
||||
providerMode: QaProviderMode;
|
||||
primaryModel: string;
|
||||
alternateModel: string;
|
||||
fastMode: boolean;
|
||||
|
||||
@@ -1,24 +1,20 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const { defaultQaRuntimeModelForMode } = vi.hoisted(() => ({
|
||||
defaultQaRuntimeModelForMode: vi.fn<(mode: string, options?: { alternate?: boolean }) => string>(
|
||||
(mode, options) =>
|
||||
mode === "live-frontier"
|
||||
? "openai/gpt-5.4"
|
||||
: options?.alternate
|
||||
? "mock-openai/gpt-5.4-alt"
|
||||
: "mock-openai/gpt-5.4",
|
||||
),
|
||||
defaultQaRuntimeModelForMode:
|
||||
vi.fn<(mode: string, options?: { alternate?: boolean }) => string>(),
|
||||
}));
|
||||
|
||||
vi.mock("./model-selection.runtime.js", () => ({
|
||||
defaultQaRuntimeModelForMode,
|
||||
}));
|
||||
import { defaultQaModelForMode as defaultQaProviderModelForMode } from "./model-selection.js";
|
||||
import {
|
||||
createDefaultQaRunSelection,
|
||||
createIdleQaRunnerSnapshot,
|
||||
createQaRunOutputDir,
|
||||
normalizeQaRunSelection,
|
||||
type QaProviderModeInput,
|
||||
} from "./run-config.js";
|
||||
|
||||
const scenarios = [
|
||||
@@ -42,11 +38,7 @@ describe("qa run config", () => {
|
||||
beforeEach(() => {
|
||||
defaultQaRuntimeModelForMode.mockImplementation(
|
||||
(mode: string, options?: { alternate?: boolean }) =>
|
||||
mode === "live-frontier"
|
||||
? "openai/gpt-5.4"
|
||||
: options?.alternate
|
||||
? "mock-openai/gpt-5.4-alt"
|
||||
: "mock-openai/gpt-5.4",
|
||||
defaultQaProviderModelForMode(mode as QaProviderModeInput, options),
|
||||
);
|
||||
});
|
||||
|
||||
@@ -64,7 +56,7 @@ describe("qa run config", () => {
|
||||
expect(
|
||||
normalizeQaRunSelection(
|
||||
{
|
||||
providerMode: "live-openai",
|
||||
providerMode: "live-frontier",
|
||||
primaryModel: "openai/gpt-5.4",
|
||||
alternateModel: "",
|
||||
fastMode: false,
|
||||
@@ -81,6 +73,17 @@ describe("qa run config", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects removed provider compatibility names", () => {
|
||||
expect(() =>
|
||||
normalizeQaRunSelection(
|
||||
{
|
||||
providerMode: "live-openai",
|
||||
},
|
||||
scenarios,
|
||||
),
|
||||
).toThrow("unknown QA provider mode: live-openai");
|
||||
});
|
||||
|
||||
it("falls back to all scenarios when selection would otherwise be empty", () => {
|
||||
const snapshot = createIdleQaRunnerSnapshot(scenarios);
|
||||
expect(snapshot.status).toBe("idle");
|
||||
@@ -95,6 +98,26 @@ describe("qa run config", () => {
|
||||
).toEqual(["dm-chat-baseline", "thread-lifecycle"]);
|
||||
});
|
||||
|
||||
it("normalizes aimock selections", () => {
|
||||
expect(
|
||||
normalizeQaRunSelection(
|
||||
{
|
||||
providerMode: "aimock",
|
||||
primaryModel: "",
|
||||
alternateModel: "",
|
||||
scenarioIds: ["dm-chat-baseline"],
|
||||
},
|
||||
scenarios,
|
||||
),
|
||||
).toEqual({
|
||||
providerMode: "aimock",
|
||||
primaryModel: "aimock/gpt-5.4",
|
||||
alternateModel: "aimock/gpt-5.4-alt",
|
||||
fastMode: false,
|
||||
scenarioIds: ["dm-chat-baseline"],
|
||||
});
|
||||
});
|
||||
|
||||
it("anchors generated run output dirs under the provided repo root", () => {
|
||||
const outputDir = createQaRunOutputDir("/tmp/openclaw-repo");
|
||||
expect(outputDir.startsWith("/tmp/openclaw-repo/.artifacts/qa-e2e/lab-")).toBe(true);
|
||||
@@ -104,9 +127,7 @@ describe("qa run config", () => {
|
||||
defaultQaRuntimeModelForMode.mockImplementation((mode, options) =>
|
||||
mode === "live-frontier"
|
||||
? "openai-codex/gpt-5.4"
|
||||
: options?.alternate
|
||||
? "mock-openai/gpt-5.4-alt"
|
||||
: "mock-openai/gpt-5.4",
|
||||
: defaultQaProviderModelForMode(mode as QaProviderModeInput, options),
|
||||
);
|
||||
|
||||
expect(createDefaultQaRunSelection(scenarios)).toEqual({
|
||||
|
||||
@@ -1,13 +1,16 @@
|
||||
import path from "node:path";
|
||||
import { defaultQaRuntimeModelForMode } from "./model-selection.runtime.js";
|
||||
import {
|
||||
DEFAULT_QA_LIVE_PROVIDER_MODE,
|
||||
getQaProvider,
|
||||
isQaProviderModeInput,
|
||||
normalizeQaProviderMode as normalizeQaProviderModeInput,
|
||||
type QaProviderMode,
|
||||
} from "./model-selection.js";
|
||||
import { defaultQaRuntimeModelForMode } from "./model-selection.runtime.js";
|
||||
} from "./providers/index.js";
|
||||
import type { QaSeedScenario } from "./scenario-catalog.js";
|
||||
|
||||
export type { QaProviderMode } from "./model-selection.js";
|
||||
export type QaProviderModeInput = QaProviderMode | "live-openai";
|
||||
export type { QaProviderModeInput } from "./providers/index.js";
|
||||
|
||||
export type QaLabRunSelection = {
|
||||
providerMode: QaProviderMode;
|
||||
@@ -38,7 +41,7 @@ export function defaultQaModelForMode(mode: QaProviderMode, alternate = false) {
|
||||
}
|
||||
|
||||
export function createDefaultQaRunSelection(scenarios: QaSeedScenario[]): QaLabRunSelection {
|
||||
const providerMode: QaProviderMode = "live-frontier";
|
||||
const providerMode: QaProviderMode = DEFAULT_QA_LIVE_PROVIDER_MODE;
|
||||
return {
|
||||
providerMode,
|
||||
primaryModel: defaultQaModelForMode(providerMode),
|
||||
@@ -49,11 +52,14 @@ export function createDefaultQaRunSelection(scenarios: QaSeedScenario[]): QaLabR
|
||||
}
|
||||
|
||||
export function normalizeQaProviderMode(input: unknown): QaProviderMode {
|
||||
return normalizeQaProviderModeInput(
|
||||
input === "mock-openai" || input === "live-frontier" || input === "live-openai"
|
||||
? input
|
||||
: "live-frontier",
|
||||
);
|
||||
if (input === undefined || input === null || input === "") {
|
||||
return DEFAULT_QA_LIVE_PROVIDER_MODE;
|
||||
}
|
||||
if (isQaProviderModeInput(input)) {
|
||||
return normalizeQaProviderModeInput(input);
|
||||
}
|
||||
const details = typeof input === "string" ? `: ${input}` : "";
|
||||
throw new Error(`unknown QA provider mode${details}`);
|
||||
}
|
||||
|
||||
function normalizeModel(input: unknown, fallback: string) {
|
||||
@@ -87,7 +93,7 @@ export function normalizeQaRunSelection(
|
||||
payload.alternateModel,
|
||||
defaultQaModelForMode(providerMode, true),
|
||||
),
|
||||
fastMode: providerMode === "live-frontier" || payload.fastMode === true,
|
||||
fastMode: getQaProvider(providerMode).kind === "live" || payload.fastMode === true,
|
||||
scenarioIds: normalizeScenarioIds(payload.scenarioIds, scenarios),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -2,6 +2,8 @@ import path from "node:path";
|
||||
import { normalizeLowercaseStringOrEmpty } from "openclaw/plugin-sdk/text-runtime";
|
||||
import { ensureRepoBoundDirectory, resolveRepoRelativeOutputDir } from "./cli-paths.js";
|
||||
import type { QaCliBackendAuthMode } from "./gateway-child.js";
|
||||
import type { QaProviderMode } from "./model-selection.js";
|
||||
import { getQaProvider } from "./providers/index.js";
|
||||
import type { QaTransportId } from "./qa-transport-registry.js";
|
||||
import { readQaBootstrapScenarioCatalog } from "./scenario-catalog.js";
|
||||
|
||||
@@ -28,10 +30,10 @@ function normalizeQaConfigString(value: unknown): string | undefined {
|
||||
function scenarioMatchesLiveLane(params: {
|
||||
scenario: QaSeedScenario;
|
||||
primaryModel: string;
|
||||
providerMode: "mock-openai" | "live-frontier";
|
||||
providerMode: QaProviderMode;
|
||||
claudeCliAuthMode?: QaCliBackendAuthMode;
|
||||
}) {
|
||||
if (params.providerMode !== "live-frontier") {
|
||||
if (getQaProvider(params.providerMode).kind !== "live") {
|
||||
return true;
|
||||
}
|
||||
const selected = splitModelRef(params.primaryModel);
|
||||
@@ -54,7 +56,7 @@ function scenarioMatchesLiveLane(params: {
|
||||
function selectQaSuiteScenarios(params: {
|
||||
scenarios: ReturnType<typeof readQaBootstrapScenarioCatalog>["scenarios"];
|
||||
scenarioIds?: string[];
|
||||
providerMode: "mock-openai" | "live-frontier";
|
||||
providerMode: QaProviderMode;
|
||||
primaryModel: string;
|
||||
claudeCliAuthMode?: QaCliBackendAuthMode;
|
||||
}) {
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
import { resolveQaLiveTurnTimeoutMs } from "./live-timeout.js";
|
||||
import type { QaProviderMode } from "./model-selection.js";
|
||||
|
||||
type QaLiveTimeoutEnv = {
|
||||
providerMode: "mock-openai" | "live-frontier";
|
||||
providerMode: QaProviderMode;
|
||||
primaryModel: string;
|
||||
alternateModel: string;
|
||||
};
|
||||
|
||||
@@ -91,7 +91,7 @@ describe("qa suite runtime agent media helpers", () => {
|
||||
).resolves.toBe(mediaPath);
|
||||
});
|
||||
|
||||
it("applies mock image generation config with transport-required plugins", async () => {
|
||||
it("applies provider image generation config with transport-required plugins", async () => {
|
||||
await ensureImageGenerationConfigured({
|
||||
providerMode: "mock-openai",
|
||||
mock: { baseUrl: "http://127.0.0.1:9999" },
|
||||
@@ -102,7 +102,7 @@ describe("qa suite runtime agent media helpers", () => {
|
||||
expect.objectContaining({
|
||||
patch: expect.objectContaining({
|
||||
plugins: expect.objectContaining({
|
||||
allow: expect.arrayContaining(["memory-core", "openai", "qa-channel", "browser"]),
|
||||
allow: expect.arrayContaining(["memory-core", "qa-channel", "browser"]),
|
||||
}),
|
||||
}),
|
||||
}),
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { buildQaImageGenerationConfigPatch } from "./providers/image-generation.js";
|
||||
import {
|
||||
fetchJson,
|
||||
patchConfig,
|
||||
@@ -70,63 +71,13 @@ async function resolveGeneratedImagePath(params: {
|
||||
}
|
||||
|
||||
async function ensureImageGenerationConfigured(env: QaSuiteRuntimeEnv) {
|
||||
const imageModelRef = "openai/gpt-image-1";
|
||||
await patchConfig({
|
||||
env,
|
||||
patch:
|
||||
env.providerMode === "mock-openai"
|
||||
? {
|
||||
plugins: {
|
||||
allow: [...new Set(["memory-core", "openai", ...env.transport.requiredPluginIds])],
|
||||
entries: {
|
||||
openai: {
|
||||
enabled: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
models: {
|
||||
providers: {
|
||||
openai: {
|
||||
baseUrl: `${env.mock?.baseUrl}/v1`,
|
||||
apiKey: "test",
|
||||
api: "openai-responses",
|
||||
models: [
|
||||
{
|
||||
id: "gpt-image-1",
|
||||
name: "gpt-image-1",
|
||||
api: "openai-responses",
|
||||
reasoning: false,
|
||||
input: ["text"],
|
||||
cost: {
|
||||
input: 0,
|
||||
output: 0,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
},
|
||||
contextWindow: 128_000,
|
||||
maxTokens: 4096,
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
agents: {
|
||||
defaults: {
|
||||
imageGenerationModel: {
|
||||
primary: imageModelRef,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
: {
|
||||
agents: {
|
||||
defaults: {
|
||||
imageGenerationModel: {
|
||||
primary: imageModelRef,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
patch: buildQaImageGenerationConfigPatch({
|
||||
providerMode: env.providerMode,
|
||||
providerBaseUrl: env.mock ? `${env.mock.baseUrl}/v1` : undefined,
|
||||
requiredPluginIds: env.transport.requiredPluginIds,
|
||||
}),
|
||||
});
|
||||
await waitForGatewayHealthy(env);
|
||||
await waitForTransportReady(env, 60_000);
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-runtime";
|
||||
import type { QaProviderMode } from "./model-selection.js";
|
||||
import type { QaTransportActionName, QaTransportAdapter } from "./qa-transport.js";
|
||||
|
||||
export type QaRuntimeGatewayClient = {
|
||||
@@ -21,7 +22,7 @@ export type QaSuiteRuntimeEnv = {
|
||||
gateway: QaRuntimeGatewayClient;
|
||||
transport: QaRuntimeTransport;
|
||||
repoRoot: string;
|
||||
providerMode: "mock-openai" | "live-frontier";
|
||||
providerMode: QaProviderMode;
|
||||
primaryModel: string;
|
||||
alternateModel: string;
|
||||
mock: {
|
||||
|
||||
@@ -12,12 +12,13 @@ import type {
|
||||
QaLabServerStartParams,
|
||||
} from "./lab-server.types.js";
|
||||
import { resolveQaLiveTurnTimeoutMs } from "./live-timeout.js";
|
||||
import { startQaMockOpenAiServer } from "./mock-openai-server.js";
|
||||
import {
|
||||
isQaFastModeEnabled,
|
||||
normalizeQaProviderMode,
|
||||
type QaProviderMode,
|
||||
} from "./model-selection.js";
|
||||
import { DEFAULT_QA_LIVE_PROVIDER_MODE } from "./providers/index.js";
|
||||
import { startQaProviderServer } from "./providers/server-runtime.js";
|
||||
import type { QaThinkingLevel } from "./qa-gateway-config.js";
|
||||
import {
|
||||
createQaTransportAdapter,
|
||||
@@ -69,7 +70,7 @@ export type QaSuiteStartLabFn = (params?: QaLabServerStartParams) => Promise<QaL
|
||||
export type QaSuiteRunParams = {
|
||||
repoRoot?: string;
|
||||
outputDir?: string;
|
||||
providerMode?: QaProviderMode | "live-openai";
|
||||
providerMode?: QaProviderMode;
|
||||
transportId?: QaTransportId;
|
||||
primaryModel?: string;
|
||||
alternateModel?: string;
|
||||
@@ -195,7 +196,7 @@ async function runScenarioDefinition(
|
||||
|
||||
function createQaSuiteReportNotes(params: {
|
||||
transport: QaTransportAdapter;
|
||||
providerMode: "mock-openai" | "live-frontier";
|
||||
providerMode: QaProviderMode;
|
||||
primaryModel: string;
|
||||
alternateModel: string;
|
||||
fastMode: boolean;
|
||||
@@ -331,7 +332,9 @@ async function writeQaSuiteArtifacts(params: {
|
||||
export async function runQaSuite(params?: QaSuiteRunParams): Promise<QaSuiteResult> {
|
||||
const startedAt = new Date();
|
||||
const repoRoot = path.resolve(params?.repoRoot ?? process.cwd());
|
||||
const providerMode = normalizeQaProviderMode(params?.providerMode ?? "live-frontier");
|
||||
const providerMode = normalizeQaProviderMode(
|
||||
params?.providerMode ?? DEFAULT_QA_LIVE_PROVIDER_MODE,
|
||||
);
|
||||
const transportId = normalizeQaTransportId(params?.transportId);
|
||||
const primaryModel = params?.primaryModel ?? defaultQaModelForMode(providerMode);
|
||||
const alternateModel = params?.alternateModel ?? defaultQaModelForMode(providerMode, true);
|
||||
@@ -538,13 +541,7 @@ export async function runQaSuite(params?: QaSuiteRunParams): Promise<QaSuiteResu
|
||||
id: transportId,
|
||||
state: lab.state,
|
||||
});
|
||||
const mock =
|
||||
providerMode === "mock-openai"
|
||||
? await startQaMockOpenAiServer({
|
||||
host: "127.0.0.1",
|
||||
port: 0,
|
||||
})
|
||||
: null;
|
||||
const mock = await startQaProviderServer(providerMode);
|
||||
const gateway = await startQaGatewayChild({
|
||||
repoRoot,
|
||||
providerBaseUrl: mock ? `${mock.baseUrl}/v1` : undefined,
|
||||
|
||||
@@ -1451,6 +1451,7 @@
|
||||
"zod": "^4.3.6"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@copilotkit/aimock": "1.13.0",
|
||||
"@grammyjs/types": "^3.26.0",
|
||||
"@lit-labs/signals": "^0.2.0",
|
||||
"@lit/context": "^1.1.6",
|
||||
|
||||
10
pnpm-lock.yaml
generated
10
pnpm-lock.yaml
generated
@@ -255,6 +255,9 @@ importers:
|
||||
specifier: ^4.3.6
|
||||
version: 4.3.6
|
||||
devDependencies:
|
||||
'@copilotkit/aimock':
|
||||
specifier: 1.13.0
|
||||
version: 1.13.0
|
||||
'@grammyjs/types':
|
||||
specifier: ^3.26.0
|
||||
version: 3.26.0
|
||||
@@ -1734,6 +1737,11 @@ packages:
|
||||
resolution: {integrity: sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ==}
|
||||
engines: {node: '>=0.1.90'}
|
||||
|
||||
'@copilotkit/aimock@1.13.0':
|
||||
resolution: {integrity: sha512-wcNQdicxSuaHf4MLtoC9bouKeYGcV9aBoIVJbeLtrBUhqz5LUAmo/U/S1WT69kyAEuM0uO1XyD3Jz+V+H9jsUw==}
|
||||
engines: {node: '>=20.15.0'}
|
||||
hasBin: true
|
||||
|
||||
'@create-markdown/preview@2.0.3':
|
||||
resolution: {integrity: sha512-Vrp8DyuiouryZ3E4NQ7tBgoYQdoekd0+DzN64mZ48QYCw3V+MCb/H2q10SW8KC8XPr931XOMDvKX4I83qpQh3g==}
|
||||
engines: {node: '>=20.0.0'}
|
||||
@@ -8749,6 +8757,8 @@ snapshots:
|
||||
'@colors/colors@1.5.0':
|
||||
optional: true
|
||||
|
||||
'@copilotkit/aimock@1.13.0': {}
|
||||
|
||||
'@create-markdown/preview@2.0.3(shiki@3.23.0)':
|
||||
optionalDependencies:
|
||||
shiki: 3.23.0
|
||||
|
||||
Reference in New Issue
Block a user