mirror of
https://github.com/openclaw/openclaw.git
synced 2026-04-12 09:41:11 +00:00
feat: add multipass runner to qa suite
This commit is contained in:
Shakker
@@ -5,6 +5,7 @@ const {
|
||||
runQaManualLane,
|
||||
runQaSuite,
|
||||
runQaCharacterEval,
|
||||
runQaMultipass,
|
||||
startQaLabServer,
|
||||
writeQaDockerHarnessFiles,
|
||||
buildQaDockerHarnessImage,
|
||||
@@ -13,6 +14,7 @@ const {
|
||||
runQaManualLane: vi.fn(),
|
||||
runQaSuite: vi.fn(),
|
||||
runQaCharacterEval: vi.fn(),
|
||||
runQaMultipass: vi.fn(),
|
||||
startQaLabServer: vi.fn(),
|
||||
writeQaDockerHarnessFiles: vi.fn(),
|
||||
buildQaDockerHarnessImage: vi.fn(),
|
||||
@@ -31,6 +33,10 @@ vi.mock("./character-eval.js", () => ({
|
||||
runQaCharacterEval,
|
||||
}));
|
||||
|
||||
vi.mock("./multipass.runtime.js", () => ({
|
||||
runQaMultipass,
|
||||
}));
|
||||
|
||||
vi.mock("./lab-server.js", () => ({
|
||||
startQaLabServer,
|
||||
}));
|
||||
@@ -62,6 +68,7 @@ describe("qa cli runtime", () => {
|
||||
runQaSuite.mockReset();
|
||||
runQaCharacterEval.mockReset();
|
||||
runQaManualLane.mockReset();
|
||||
runQaMultipass.mockReset();
|
||||
startQaLabServer.mockReset();
|
||||
writeQaDockerHarnessFiles.mockReset();
|
||||
buildQaDockerHarnessImage.mockReset();
|
||||
@@ -81,6 +88,16 @@ describe("qa cli runtime", () => {
|
||||
reply: "done",
|
||||
watchUrl: "http://127.0.0.1:43124",
|
||||
});
|
||||
runQaMultipass.mockResolvedValue({
|
||||
outputDir: "/tmp/multipass",
|
||||
reportPath: "/tmp/multipass/qa-suite-report.md",
|
||||
summaryPath: "/tmp/multipass/qa-suite-summary.json",
|
||||
hostLogPath: "/tmp/multipass/multipass-host.log",
|
||||
bootstrapLogPath: "/tmp/multipass/multipass-guest-bootstrap.log",
|
||||
guestScriptPath: "/tmp/multipass/multipass-guest-run.sh",
|
||||
vmName: "openclaw-qa-test",
|
||||
scenarioIds: ["channel-chat-baseline"],
|
||||
});
|
||||
startQaLabServer.mockResolvedValue({
|
||||
baseUrl: "http://127.0.0.1:58000",
|
||||
runSelfCheck: vi.fn().mockResolvedValue({
|
||||
@@ -267,6 +284,68 @@ describe("qa cli runtime", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("routes suite runs through multipass when the runner is selected", async () => {
|
||||
await runQaSuiteCommand({
|
||||
repoRoot: "/tmp/openclaw-repo",
|
||||
outputDir: ".artifacts/qa-multipass",
|
||||
runner: "multipass",
|
||||
providerMode: "mock-openai",
|
||||
scenarioIds: ["channel-chat-baseline"],
|
||||
image: "lts",
|
||||
cpus: 2,
|
||||
memory: "4G",
|
||||
disk: "24G",
|
||||
});
|
||||
|
||||
expect(runQaMultipass).toHaveBeenCalledWith({
|
||||
repoRoot: path.resolve("/tmp/openclaw-repo"),
|
||||
outputDir: path.resolve("/tmp/openclaw-repo", ".artifacts/qa-multipass"),
|
||||
providerMode: "mock-openai",
|
||||
primaryModel: undefined,
|
||||
alternateModel: undefined,
|
||||
fastMode: undefined,
|
||||
scenarioIds: ["channel-chat-baseline"],
|
||||
image: "lts",
|
||||
cpus: 2,
|
||||
memory: "4G",
|
||||
disk: "24G",
|
||||
});
|
||||
expect(runQaSuite).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("passes live suite selection through to the multipass runner", async () => {
|
||||
await runQaSuiteCommand({
|
||||
repoRoot: "/tmp/openclaw-repo",
|
||||
runner: "multipass",
|
||||
providerMode: "live-frontier",
|
||||
primaryModel: "openai/gpt-5.4",
|
||||
alternateModel: "openai/gpt-5.4",
|
||||
fastMode: true,
|
||||
scenarioIds: ["channel-chat-baseline"],
|
||||
});
|
||||
|
||||
expect(runQaMultipass).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
repoRoot: path.resolve("/tmp/openclaw-repo"),
|
||||
providerMode: "live-frontier",
|
||||
primaryModel: "openai/gpt-5.4",
|
||||
alternateModel: "openai/gpt-5.4",
|
||||
fastMode: true,
|
||||
scenarioIds: ["channel-chat-baseline"],
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects multipass-only suite flags on the host runner", async () => {
|
||||
await expect(
|
||||
runQaSuiteCommand({
|
||||
repoRoot: "/tmp/openclaw-repo",
|
||||
runner: "host",
|
||||
image: "lts",
|
||||
}),
|
||||
).rejects.toThrow("--image, --cpus, --memory, and --disk require --runner multipass.");
|
||||
});
|
||||
|
||||
it("defaults manual mock runs onto the mock-openai model lane", async () => {
|
||||
await runQaManualLaneCommand({
|
||||
repoRoot: "/tmp/openclaw-repo",
|
||||
|
||||
@@ -5,6 +5,7 @@ import { runQaDockerUp } from "./docker-up.runtime.js";
|
||||
import { startQaLabServer } from "./lab-server.js";
|
||||
import { runQaManualLane } from "./manual-lane.runtime.js";
|
||||
import { startQaMockOpenAiServer } from "./mock-openai-server.js";
|
||||
import { runQaMultipass } from "./multipass.runtime.js";
|
||||
import { normalizeQaThinkingLevel, type QaThinkingLevel } from "./qa-gateway-config.js";
|
||||
import {
|
||||
defaultQaModelForMode,
|
||||
@@ -193,14 +194,53 @@ export async function runQaLabSelfCheckCommand(opts: { repoRoot?: string; output
|
||||
export async function runQaSuiteCommand(opts: {
|
||||
repoRoot?: string;
|
||||
outputDir?: string;
|
||||
runner?: string;
|
||||
providerMode?: QaProviderModeInput;
|
||||
primaryModel?: string;
|
||||
alternateModel?: string;
|
||||
fastMode?: boolean;
|
||||
scenarioIds?: string[];
|
||||
image?: string;
|
||||
cpus?: number;
|
||||
memory?: string;
|
||||
disk?: string;
|
||||
}) {
|
||||
const repoRoot = path.resolve(opts.repoRoot ?? process.cwd());
|
||||
const runner = (opts.runner ?? "host").trim().toLowerCase();
|
||||
if (runner !== "host" && runner !== "multipass") {
|
||||
throw new Error(`--runner must be one of host or multipass, got "${opts.runner}".`);
|
||||
}
|
||||
const providerMode = normalizeQaProviderMode(opts.providerMode);
|
||||
if (
|
||||
runner === "host" &&
|
||||
(opts.image !== undefined ||
|
||||
opts.cpus !== undefined ||
|
||||
opts.memory !== undefined ||
|
||||
opts.disk !== undefined)
|
||||
) {
|
||||
throw new Error("--image, --cpus, --memory, and --disk require --runner multipass.");
|
||||
}
|
||||
if (runner === "multipass") {
|
||||
const result = await runQaMultipass({
|
||||
repoRoot,
|
||||
outputDir: opts.outputDir ? path.resolve(repoRoot, opts.outputDir) : undefined,
|
||||
providerMode,
|
||||
primaryModel: opts.primaryModel,
|
||||
alternateModel: opts.alternateModel,
|
||||
fastMode: opts.fastMode,
|
||||
scenarioIds: opts.scenarioIds,
|
||||
image: opts.image,
|
||||
cpus: parseQaPositiveIntegerOption("--cpus", opts.cpus),
|
||||
memory: opts.memory,
|
||||
disk: opts.disk,
|
||||
});
|
||||
process.stdout.write(`QA Multipass dir: ${result.outputDir}\n`);
|
||||
process.stdout.write(`QA Multipass report: ${result.reportPath}\n`);
|
||||
process.stdout.write(`QA Multipass summary: ${result.summaryPath}\n`);
|
||||
process.stdout.write(`QA Multipass host log: ${result.hostLogPath}\n`);
|
||||
process.stdout.write(`QA Multipass bootstrap log: ${result.bootstrapLogPath}\n`);
|
||||
return;
|
||||
}
|
||||
const result = await runQaSuite({
|
||||
repoRoot,
|
||||
outputDir: opts.outputDir ? path.resolve(repoRoot, opts.outputDir) : undefined,
|
||||
|
||||
@@ -23,6 +23,11 @@ async function runQaSuite(opts: {
|
||||
alternateModel?: string;
|
||||
fastMode?: boolean;
|
||||
scenarioIds?: string[];
|
||||
runner?: string;
|
||||
image?: string;
|
||||
cpus?: number;
|
||||
memory?: string;
|
||||
disk?: string;
|
||||
}) {
|
||||
const runtime = await loadQaLabCliRuntime();
|
||||
await runtime.runQaSuiteCommand(opts);
|
||||
@@ -138,6 +143,7 @@ export function registerQaLabCli(program: Command) {
|
||||
.description("Run repo-backed QA scenarios against the QA gateway lane")
|
||||
.option("--repo-root <path>", "Repository root to target when running from a neutral cwd")
|
||||
.option("--output-dir <path>", "Suite artifact directory")
|
||||
.option("--runner <kind>", "Execution runner: host or multipass", "host")
|
||||
.option(
|
||||
"--provider-mode <mode>",
|
||||
"Provider mode: mock-openai or live-frontier (legacy live-openai still works)",
|
||||
@@ -147,24 +153,38 @@ export function registerQaLabCli(program: Command) {
|
||||
.option("--alt-model <ref>", "Alternate provider/model ref")
|
||||
.option("--scenario <id>", "Run only the named QA scenario (repeatable)", collectString, [])
|
||||
.option("--fast", "Enable provider fast mode where supported", false)
|
||||
.option("--image <alias>", "Multipass image alias")
|
||||
.option("--cpus <count>", "Multipass vCPU count", (value: string) => Number(value))
|
||||
.option("--memory <size>", "Multipass memory size")
|
||||
.option("--disk <size>", "Multipass disk size")
|
||||
.action(
|
||||
async (opts: {
|
||||
repoRoot?: string;
|
||||
outputDir?: string;
|
||||
runner?: string;
|
||||
providerMode?: QaProviderModeInput;
|
||||
model?: string;
|
||||
altModel?: string;
|
||||
scenario?: string[];
|
||||
fast?: boolean;
|
||||
image?: string;
|
||||
cpus?: number;
|
||||
memory?: string;
|
||||
disk?: string;
|
||||
}) => {
|
||||
await runQaSuite({
|
||||
repoRoot: opts.repoRoot,
|
||||
outputDir: opts.outputDir,
|
||||
runner: opts.runner,
|
||||
providerMode: opts.providerMode,
|
||||
primaryModel: opts.model,
|
||||
alternateModel: opts.altModel,
|
||||
fastMode: opts.fast,
|
||||
scenarioIds: opts.scenario,
|
||||
image: opts.image,
|
||||
cpus: opts.cpus,
|
||||
memory: opts.memory,
|
||||
disk: opts.disk,
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
72
extensions/qa-lab/src/multipass.runtime.test.ts
Normal file
72
extensions/qa-lab/src/multipass.runtime.test.ts
Normal file
@@ -0,0 +1,72 @@
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { createQaMultipassPlan, renderQaMultipassGuestScript } from "./multipass.runtime.js";
|
||||
|
||||
describe("qa multipass runtime", () => {
|
||||
afterEach(() => {
|
||||
vi.unstubAllEnvs();
|
||||
});
|
||||
|
||||
it("reuses suite scenario semantics and resolves mounted artifact paths", () => {
|
||||
const repoRoot = process.cwd();
|
||||
const outputDir = path.join(repoRoot, ".artifacts", "qa-e2e", "multipass-test");
|
||||
const plan = createQaMultipassPlan({
|
||||
repoRoot,
|
||||
outputDir,
|
||||
});
|
||||
|
||||
expect(plan.outputDir).toBe(outputDir);
|
||||
expect(plan.scenarioIds).toEqual([]);
|
||||
expect(plan.qaCommand).not.toContain("--scenario");
|
||||
expect(plan.guestOutputDir).toBe("/workspace/openclaw-host/.artifacts/qa-e2e/multipass-test");
|
||||
expect(plan.reportPath).toBe(path.join(outputDir, "qa-suite-report.md"));
|
||||
expect(plan.summaryPath).toBe(path.join(outputDir, "qa-suite-summary.json"));
|
||||
});
|
||||
|
||||
it("renders a guest script that runs the mock qa suite with explicit scenarios", () => {
|
||||
const plan = createQaMultipassPlan({
|
||||
repoRoot: process.cwd(),
|
||||
outputDir: path.join(process.cwd(), ".artifacts", "qa-e2e", "multipass-test"),
|
||||
scenarioIds: ["channel-chat-baseline", "thread-follow-up"],
|
||||
});
|
||||
|
||||
const script = renderQaMultipassGuestScript(plan);
|
||||
|
||||
expect(script).toContain("pnpm install --frozen-lockfile");
|
||||
expect(script).toContain("pnpm build");
|
||||
expect(script).toContain("'pnpm' 'openclaw' 'qa' 'suite' '--provider-mode' 'mock-openai'");
|
||||
expect(script).toContain("'--scenario' 'channel-chat-baseline'");
|
||||
expect(script).toContain("'--scenario' 'thread-follow-up'");
|
||||
expect(script).toContain("/workspace/openclaw-host/.artifacts/qa-e2e/multipass-test");
|
||||
});
|
||||
|
||||
it("carries live suite flags and forwarded auth env into the guest command", () => {
|
||||
vi.stubEnv("OPENAI_API_KEY", "test-openai-key");
|
||||
const plan = createQaMultipassPlan({
|
||||
repoRoot: process.cwd(),
|
||||
outputDir: path.join(process.cwd(), ".artifacts", "qa-e2e", "multipass-live-test"),
|
||||
providerMode: "live-frontier",
|
||||
primaryModel: "openai/gpt-5.4",
|
||||
alternateModel: "openai/gpt-5.4",
|
||||
fastMode: true,
|
||||
scenarioIds: ["channel-chat-baseline"],
|
||||
});
|
||||
|
||||
const script = renderQaMultipassGuestScript(plan);
|
||||
|
||||
expect(plan.qaCommand).toEqual(
|
||||
expect.arrayContaining([
|
||||
"--provider-mode",
|
||||
"live-frontier",
|
||||
"--model",
|
||||
"openai/gpt-5.4",
|
||||
"--alt-model",
|
||||
"openai/gpt-5.4",
|
||||
"--fast",
|
||||
]),
|
||||
);
|
||||
expect(plan.forwardedEnv.OPENAI_API_KEY).toBe("test-openai-key");
|
||||
expect(script).toContain("OPENAI_API_KEY='test-openai-key'");
|
||||
expect(script).toContain("'pnpm' 'openclaw' 'qa' 'suite' '--provider-mode' 'live-frontier'");
|
||||
});
|
||||
});
|
||||
643
extensions/qa-lab/src/multipass.runtime.ts
Normal file
643
extensions/qa-lab/src/multipass.runtime.ts
Normal file
@@ -0,0 +1,643 @@
|
||||
import { execFile } from "node:child_process";
|
||||
import fs from "node:fs";
|
||||
import { access, appendFile, mkdir, writeFile } from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
const MULTIPASS_MOUNTED_REPO_PATH = "/workspace/openclaw-host";
|
||||
const MULTIPASS_GUEST_REPO_PATH = "/workspace/openclaw";
|
||||
const MULTIPASS_GUEST_CODEX_HOME_PATH = "/workspace/openclaw-codex-home";
|
||||
const MULTIPASS_GUEST_PACKAGES = [
|
||||
"build-essential",
|
||||
"ca-certificates",
|
||||
"curl",
|
||||
"pkg-config",
|
||||
"python3",
|
||||
"rsync",
|
||||
"xz-utils",
|
||||
] as const;
|
||||
const MULTIPASS_REPO_SYNC_EXCLUDES = [
|
||||
".git",
|
||||
"node_modules",
|
||||
".artifacts",
|
||||
".tmp",
|
||||
".turbo",
|
||||
"coverage",
|
||||
"*.heapsnapshot",
|
||||
] as const;
|
||||
|
||||
const QA_LIVE_ENV_ALIASES = Object.freeze([
|
||||
{
|
||||
liveVar: "OPENCLAW_LIVE_OPENAI_KEY",
|
||||
providerVar: "OPENAI_API_KEY",
|
||||
},
|
||||
{
|
||||
liveVar: "OPENCLAW_LIVE_ANTHROPIC_KEY",
|
||||
providerVar: "ANTHROPIC_API_KEY",
|
||||
},
|
||||
{
|
||||
liveVar: "OPENCLAW_LIVE_GEMINI_KEY",
|
||||
providerVar: "GEMINI_API_KEY",
|
||||
},
|
||||
]);
|
||||
|
||||
const QA_LIVE_ALLOWED_ENV_VARS = Object.freeze([
|
||||
"ANTHROPIC_API_KEY",
|
||||
"ANTHROPIC_OAUTH_TOKEN",
|
||||
"AWS_ACCESS_KEY_ID",
|
||||
"AWS_BEARER_TOKEN_BEDROCK",
|
||||
"AWS_REGION",
|
||||
"AWS_SECRET_ACCESS_KEY",
|
||||
"AWS_SESSION_TOKEN",
|
||||
"GEMINI_API_KEY",
|
||||
"GEMINI_API_KEYS",
|
||||
"GOOGLE_API_KEY",
|
||||
"MISTRAL_API_KEY",
|
||||
"OPENAI_API_KEY",
|
||||
"OPENAI_API_KEYS",
|
||||
"OPENAI_BASE_URL",
|
||||
"OPENCLAW_LIVE_ANTHROPIC_KEY",
|
||||
"OPENCLAW_LIVE_ANTHROPIC_KEYS",
|
||||
"OPENCLAW_LIVE_GEMINI_KEY",
|
||||
"OPENCLAW_LIVE_OPENAI_KEY",
|
||||
"OPENCLAW_QA_LIVE_PROVIDER_CONFIG_PATH",
|
||||
"OPENCLAW_CONFIG_PATH",
|
||||
"VOYAGE_API_KEY",
|
||||
]);
|
||||
|
||||
export const qaMultipassDefaultResources = {
|
||||
image: "lts",
|
||||
cpus: 2,
|
||||
memory: "4G",
|
||||
disk: "24G",
|
||||
} as const;
|
||||
|
||||
type ExecResult = {
|
||||
stdout: string;
|
||||
stderr: string;
|
||||
};
|
||||
|
||||
export type QaMultipassPlan = {
|
||||
repoRoot: string;
|
||||
outputDir: string;
|
||||
reportPath: string;
|
||||
summaryPath: string;
|
||||
hostLogPath: string;
|
||||
hostBootstrapLogPath: string;
|
||||
hostGuestScriptPath: string;
|
||||
vmName: string;
|
||||
image: string;
|
||||
cpus: number;
|
||||
memory: string;
|
||||
disk: string;
|
||||
pnpmVersion: string;
|
||||
providerMode: "mock-openai" | "live-frontier";
|
||||
primaryModel?: string;
|
||||
alternateModel?: string;
|
||||
fastMode?: boolean;
|
||||
scenarioIds: string[];
|
||||
forwardedEnv: Record<string, string>;
|
||||
hostCodexHomePath?: string;
|
||||
guestCodexHomePath?: string;
|
||||
hostLiveProviderConfigPath?: string;
|
||||
guestLiveProviderConfigPath?: string;
|
||||
guestMountedRepoPath: string;
|
||||
guestRepoPath: string;
|
||||
guestOutputDir: string;
|
||||
guestScriptPath: string;
|
||||
guestBootstrapLogPath: string;
|
||||
qaCommand: string[];
|
||||
};
|
||||
|
||||
export type QaMultipassRunResult = {
|
||||
outputDir: string;
|
||||
reportPath: string;
|
||||
summaryPath: string;
|
||||
hostLogPath: string;
|
||||
bootstrapLogPath: string;
|
||||
guestScriptPath: string;
|
||||
vmName: string;
|
||||
scenarioIds: string[];
|
||||
};
|
||||
|
||||
function shellQuote(value: string) {
|
||||
return `'${value.replaceAll("'", `'"'"'`)}'`;
|
||||
}
|
||||
|
||||
function createOutputStamp() {
|
||||
return new Date().toISOString().replaceAll(":", "").replaceAll(".", "").replace("T", "-");
|
||||
}
|
||||
|
||||
function createVmSuffix() {
|
||||
return `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`;
|
||||
}
|
||||
|
||||
function sleep(ms: number) {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
function execFileAsync(file: string, args: string[]) {
|
||||
return new Promise<ExecResult>((resolve, reject) => {
|
||||
execFile(file, args, { encoding: "utf8" }, (error, stdout, stderr) => {
|
||||
if (error) {
|
||||
const message = stderr.trim() || stdout.trim() || error.message;
|
||||
reject(new Error(message));
|
||||
return;
|
||||
}
|
||||
resolve({ stdout, stderr });
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function resolvePnpmVersion(repoRoot: string) {
|
||||
const packageJsonPath = path.join(repoRoot, "package.json");
|
||||
const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, "utf8")) as {
|
||||
packageManager?: string;
|
||||
};
|
||||
const packageManager = packageJson.packageManager ?? "";
|
||||
const match = /^pnpm@(.+)$/.exec(packageManager);
|
||||
if (!match?.[1]) {
|
||||
throw new Error(`unable to resolve pnpm version from packageManager in ${packageJsonPath}`);
|
||||
}
|
||||
return match[1];
|
||||
}
|
||||
|
||||
function resolveMultipassInstallHint() {
|
||||
if (process.platform === "darwin") {
|
||||
return "brew install --cask multipass";
|
||||
}
|
||||
if (process.platform === "win32") {
|
||||
return "winget install Canonical.Multipass";
|
||||
}
|
||||
if (process.platform === "linux") {
|
||||
return "sudo snap install multipass";
|
||||
}
|
||||
return "https://multipass.run/install";
|
||||
}
|
||||
|
||||
function resolveUserPath(value: string, env: NodeJS.ProcessEnv = process.env) {
|
||||
if (value === "~") {
|
||||
return env.HOME ?? os.homedir();
|
||||
}
|
||||
if (value.startsWith("~/")) {
|
||||
return path.join(env.HOME ?? os.homedir(), value.slice(2));
|
||||
}
|
||||
return path.resolve(value);
|
||||
}
|
||||
|
||||
function resolveLiveProviderConfigPath(env: NodeJS.ProcessEnv = process.env) {
|
||||
const explicit =
|
||||
env.OPENCLAW_QA_LIVE_PROVIDER_CONFIG_PATH?.trim() || env.OPENCLAW_CONFIG_PATH?.trim();
|
||||
return explicit
|
||||
? { path: resolveUserPath(explicit, env), explicit: true }
|
||||
: { path: path.join(os.homedir(), ".openclaw", "openclaw.json"), explicit: false };
|
||||
}
|
||||
|
||||
function resolveQaLiveCliAuthEnv(baseEnv: NodeJS.ProcessEnv) {
|
||||
const configuredCodexHome = baseEnv.CODEX_HOME?.trim();
|
||||
if (configuredCodexHome) {
|
||||
return { CODEX_HOME: resolveUserPath(configuredCodexHome, baseEnv) };
|
||||
}
|
||||
const hostHome = baseEnv.HOME?.trim();
|
||||
if (!hostHome) {
|
||||
return {};
|
||||
}
|
||||
const codexHome = path.join(hostHome, ".codex");
|
||||
return fs.existsSync(codexHome) ? { CODEX_HOME: codexHome } : {};
|
||||
}
|
||||
|
||||
function resolveForwardedLiveEnv(baseEnv: NodeJS.ProcessEnv = process.env) {
|
||||
const forwarded: Record<string, string> = {};
|
||||
for (const key of QA_LIVE_ALLOWED_ENV_VARS) {
|
||||
const value = baseEnv[key]?.trim();
|
||||
if (value) {
|
||||
forwarded[key] = value;
|
||||
}
|
||||
}
|
||||
for (const { liveVar, providerVar } of QA_LIVE_ENV_ALIASES) {
|
||||
const liveValue = forwarded[liveVar]?.trim();
|
||||
if (liveValue && !forwarded[providerVar]?.trim()) {
|
||||
forwarded[providerVar] = liveValue;
|
||||
}
|
||||
}
|
||||
const liveCliAuth = resolveQaLiveCliAuthEnv(baseEnv);
|
||||
if (liveCliAuth.CODEX_HOME) {
|
||||
forwarded.CODEX_HOME = liveCliAuth.CODEX_HOME;
|
||||
}
|
||||
return forwarded;
|
||||
}
|
||||
|
||||
function createQaMultipassOutputDir(repoRoot: string) {
|
||||
return path.join(repoRoot, ".artifacts", "qa-e2e", `multipass-${createOutputStamp()}`);
|
||||
}
|
||||
|
||||
function resolveGuestMountedPath(repoRoot: string, hostPath: string) {
|
||||
const relativePath = path.relative(repoRoot, hostPath);
|
||||
if (relativePath.startsWith("..") || path.isAbsolute(relativePath) || relativePath.length === 0) {
|
||||
throw new Error(`unable to resolve Multipass mounted path for ${hostPath}`);
|
||||
}
|
||||
return path.posix.join(MULTIPASS_MOUNTED_REPO_PATH, ...relativePath.split(path.sep));
|
||||
}
|
||||
|
||||
function appendScenarioArgs(command: string[], scenarioIds: string[]) {
|
||||
for (const scenarioId of scenarioIds) {
|
||||
command.push("--scenario", scenarioId);
|
||||
}
|
||||
return command;
|
||||
}
|
||||
|
||||
export function createQaMultipassPlan(params: {
|
||||
repoRoot: string;
|
||||
outputDir?: string;
|
||||
providerMode?: "mock-openai" | "live-frontier";
|
||||
primaryModel?: string;
|
||||
alternateModel?: string;
|
||||
fastMode?: boolean;
|
||||
scenarioIds?: string[];
|
||||
image?: string;
|
||||
cpus?: number;
|
||||
memory?: string;
|
||||
disk?: string;
|
||||
}) {
|
||||
const outputDir = params.outputDir ?? createQaMultipassOutputDir(params.repoRoot);
|
||||
const scenarioIds = [...new Set(params.scenarioIds ?? [])];
|
||||
const providerMode = params.providerMode ?? "mock-openai";
|
||||
const forwardedEnv = providerMode === "live-frontier" ? resolveForwardedLiveEnv() : {};
|
||||
const hostCodexHomePath = forwardedEnv.CODEX_HOME;
|
||||
const liveProviderConfig =
|
||||
providerMode === "live-frontier" ? resolveLiveProviderConfigPath() : undefined;
|
||||
const hostLiveProviderConfigPath =
|
||||
liveProviderConfig && fs.existsSync(liveProviderConfig.path)
|
||||
? liveProviderConfig.path
|
||||
: undefined;
|
||||
const vmName = `openclaw-qa-${createVmSuffix()}`;
|
||||
const guestOutputDir = resolveGuestMountedPath(params.repoRoot, outputDir);
|
||||
const qaCommand = appendScenarioArgs(
|
||||
[
|
||||
"pnpm",
|
||||
"openclaw",
|
||||
"qa",
|
||||
"suite",
|
||||
"--provider-mode",
|
||||
providerMode,
|
||||
"--output-dir",
|
||||
guestOutputDir,
|
||||
...(params.primaryModel ? ["--model", params.primaryModel] : []),
|
||||
...(params.alternateModel ? ["--alt-model", params.alternateModel] : []),
|
||||
...(params.fastMode ? ["--fast"] : []),
|
||||
],
|
||||
scenarioIds,
|
||||
);
|
||||
|
||||
return {
|
||||
repoRoot: params.repoRoot,
|
||||
outputDir,
|
||||
reportPath: path.join(outputDir, "qa-suite-report.md"),
|
||||
summaryPath: path.join(outputDir, "qa-suite-summary.json"),
|
||||
hostLogPath: path.join(outputDir, "multipass-host.log"),
|
||||
hostBootstrapLogPath: path.join(outputDir, "multipass-guest-bootstrap.log"),
|
||||
hostGuestScriptPath: path.join(outputDir, "multipass-guest-run.sh"),
|
||||
vmName,
|
||||
image: params.image ?? qaMultipassDefaultResources.image,
|
||||
cpus: params.cpus ?? qaMultipassDefaultResources.cpus,
|
||||
memory: params.memory ?? qaMultipassDefaultResources.memory,
|
||||
disk: params.disk ?? qaMultipassDefaultResources.disk,
|
||||
pnpmVersion: resolvePnpmVersion(params.repoRoot),
|
||||
providerMode,
|
||||
primaryModel: params.primaryModel,
|
||||
alternateModel: params.alternateModel,
|
||||
fastMode: params.fastMode,
|
||||
scenarioIds,
|
||||
forwardedEnv,
|
||||
hostCodexHomePath,
|
||||
guestCodexHomePath: hostCodexHomePath ? MULTIPASS_GUEST_CODEX_HOME_PATH : undefined,
|
||||
hostLiveProviderConfigPath,
|
||||
guestLiveProviderConfigPath: hostLiveProviderConfigPath
|
||||
? `/tmp/${vmName}-live-provider-config.json`
|
||||
: undefined,
|
||||
guestMountedRepoPath: MULTIPASS_MOUNTED_REPO_PATH,
|
||||
guestRepoPath: MULTIPASS_GUEST_REPO_PATH,
|
||||
guestOutputDir,
|
||||
guestScriptPath: `/tmp/${vmName}-qa-suite.sh`,
|
||||
guestBootstrapLogPath: `/tmp/${vmName}-bootstrap.log`,
|
||||
qaCommand,
|
||||
} satisfies QaMultipassPlan;
|
||||
}
|
||||
|
||||
export function renderQaMultipassGuestScript(plan: QaMultipassPlan) {
|
||||
const rsyncCommand = [
|
||||
"rsync -a --delete",
|
||||
...MULTIPASS_REPO_SYNC_EXCLUDES.flatMap((value) => ["--exclude", shellQuote(value)]),
|
||||
shellQuote(`${plan.guestMountedRepoPath}/`),
|
||||
shellQuote(`${plan.guestRepoPath}/`),
|
||||
].join(" ");
|
||||
const qaCommand = [
|
||||
...Object.entries(plan.forwardedEnv)
|
||||
.filter(
|
||||
([key]) =>
|
||||
key !== "CODEX_HOME" &&
|
||||
key !== "OPENCLAW_CONFIG_PATH" &&
|
||||
key !== "OPENCLAW_QA_LIVE_PROVIDER_CONFIG_PATH",
|
||||
)
|
||||
.map(([key, value]) => `${key}=${shellQuote(value)}`),
|
||||
...(plan.guestCodexHomePath ? [`CODEX_HOME=${shellQuote(plan.guestCodexHomePath)}`] : []),
|
||||
...(plan.guestLiveProviderConfigPath
|
||||
? [
|
||||
`OPENCLAW_CONFIG_PATH=${shellQuote(plan.guestLiveProviderConfigPath)}`,
|
||||
`OPENCLAW_QA_LIVE_PROVIDER_CONFIG_PATH=${shellQuote(plan.guestLiveProviderConfigPath)}`,
|
||||
]
|
||||
: []),
|
||||
plan.qaCommand.map(shellQuote).join(" "),
|
||||
].join(" ");
|
||||
|
||||
const lines = [
|
||||
"#!/usr/bin/env bash",
|
||||
"set -euo pipefail",
|
||||
"trap 'status=$?; echo \"guest failure: ${BASH_COMMAND} (exit ${status})\" >&2; exit ${status}' ERR",
|
||||
"",
|
||||
"export DEBIAN_FRONTEND=noninteractive",
|
||||
`BOOTSTRAP_LOG=${shellQuote(plan.guestBootstrapLogPath)}`,
|
||||
': > "$BOOTSTRAP_LOG"',
|
||||
"",
|
||||
"ensure_guest_packages() {",
|
||||
' sudo -E apt-get update >>"$BOOTSTRAP_LOG" 2>&1',
|
||||
" sudo -E apt-get install -y \\",
|
||||
...MULTIPASS_GUEST_PACKAGES.map((value, index) =>
|
||||
index === MULTIPASS_GUEST_PACKAGES.length - 1
|
||||
? ` ${value} >>"$BOOTSTRAP_LOG" 2>&1`
|
||||
: ` ${value} \\`,
|
||||
),
|
||||
"}",
|
||||
"",
|
||||
"ensure_node() {",
|
||||
" if command -v node >/dev/null; then",
|
||||
" local node_major",
|
||||
' node_major="$(node -p \'process.versions.node.split(".")[0]\' 2>/dev/null || echo 0)"',
|
||||
' if [ "${node_major}" -ge 22 ]; then',
|
||||
" return 0",
|
||||
" fi",
|
||||
" fi",
|
||||
" local node_arch",
|
||||
' case "$(uname -m)" in',
|
||||
' x86_64) node_arch="x64" ;;',
|
||||
' aarch64|arm64) node_arch="arm64" ;;',
|
||||
' *) echo "unsupported guest architecture for node bootstrap: $(uname -m)" >&2; return 1 ;;',
|
||||
" esac",
|
||||
" local node_tmp_dir tarball_name extract_dir base_url",
|
||||
' node_tmp_dir="$(mktemp -d)"',
|
||||
" trap 'rm -rf \"${node_tmp_dir}\"' RETURN",
|
||||
' base_url="https://nodejs.org/dist/latest-v22.x"',
|
||||
' curl -fsSL "${base_url}/SHASUMS256.txt" -o "${node_tmp_dir}/SHASUMS256.txt" >>"$BOOTSTRAP_LOG" 2>&1',
|
||||
' tarball_name="$(awk \'/linux-\'"${node_arch}"\'\\.tar\\.xz$/ { print $2; exit }\' "${node_tmp_dir}/SHASUMS256.txt")"',
|
||||
' [ -n "${tarball_name}" ] || { echo "unable to resolve node tarball for ${node_arch}" >&2; return 1; }',
|
||||
' curl -fsSL "${base_url}/${tarball_name}" -o "${node_tmp_dir}/${tarball_name}" >>"$BOOTSTRAP_LOG" 2>&1',
|
||||
' (cd "${node_tmp_dir}" && grep " ${tarball_name}$" SHASUMS256.txt | sha256sum -c -) >>"$BOOTSTRAP_LOG" 2>&1',
|
||||
' extract_dir="${tarball_name%.tar.xz}"',
|
||||
' sudo mkdir -p /usr/local/lib/nodejs >>"$BOOTSTRAP_LOG" 2>&1',
|
||||
' sudo rm -rf "/usr/local/lib/nodejs/${extract_dir}" >>"$BOOTSTRAP_LOG" 2>&1',
|
||||
' sudo tar -xJf "${node_tmp_dir}/${tarball_name}" -C /usr/local/lib/nodejs >>"$BOOTSTRAP_LOG" 2>&1',
|
||||
' sudo ln -sf "/usr/local/lib/nodejs/${extract_dir}/bin/node" /usr/local/bin/node >>"$BOOTSTRAP_LOG" 2>&1',
|
||||
' sudo ln -sf "/usr/local/lib/nodejs/${extract_dir}/bin/npm" /usr/local/bin/npm >>"$BOOTSTRAP_LOG" 2>&1',
|
||||
' sudo ln -sf "/usr/local/lib/nodejs/${extract_dir}/bin/npx" /usr/local/bin/npx >>"$BOOTSTRAP_LOG" 2>&1',
|
||||
' sudo ln -sf "/usr/local/lib/nodejs/${extract_dir}/bin/corepack" /usr/local/bin/corepack >>"$BOOTSTRAP_LOG" 2>&1',
|
||||
"}",
|
||||
"",
|
||||
"ensure_pnpm() {",
|
||||
' sudo env PATH="/usr/local/bin:/usr/bin:/bin" corepack enable >>"$BOOTSTRAP_LOG" 2>&1',
|
||||
` sudo env PATH="/usr/local/bin:/usr/bin:/bin" corepack prepare pnpm@${plan.pnpmVersion} --activate >>"$BOOTSTRAP_LOG" 2>&1`,
|
||||
"}",
|
||||
"",
|
||||
'command -v sudo >/dev/null || { echo "missing sudo in guest" >&2; exit 1; }',
|
||||
"ensure_guest_packages",
|
||||
"ensure_node",
|
||||
"ensure_pnpm",
|
||||
'command -v node >/dev/null || { echo "missing node after guest bootstrap" >&2; exit 1; }',
|
||||
'command -v pnpm >/dev/null || { echo "missing pnpm after guest bootstrap" >&2; exit 1; }',
|
||||
'command -v rsync >/dev/null || { echo "missing rsync after guest bootstrap" >&2; exit 1; }',
|
||||
"",
|
||||
`mkdir -p ${shellQuote(path.posix.dirname(plan.guestRepoPath))}`,
|
||||
`rm -rf ${shellQuote(plan.guestRepoPath)}`,
|
||||
`mkdir -p ${shellQuote(plan.guestRepoPath)}`,
|
||||
`mkdir -p ${shellQuote(plan.guestOutputDir)}`,
|
||||
rsyncCommand,
|
||||
`cd ${shellQuote(plan.guestRepoPath)}`,
|
||||
'pnpm install --frozen-lockfile >>"$BOOTSTRAP_LOG" 2>&1',
|
||||
'pnpm build >>"$BOOTSTRAP_LOG" 2>&1',
|
||||
qaCommand,
|
||||
"",
|
||||
];
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
async function appendMultipassLog(logPath: string, message: string) {
|
||||
await appendFile(logPath, message, "utf8");
|
||||
}
|
||||
|
||||
async function runMultipassCommand(logPath: string, args: string[]) {
|
||||
await appendMultipassLog(logPath, `$ ${["multipass", ...args].join(" ")}\n`);
|
||||
const result = await execFileAsync("multipass", args);
|
||||
if (result.stdout.trim()) {
|
||||
await appendMultipassLog(logPath, `${result.stdout.trim()}\n`);
|
||||
}
|
||||
if (result.stderr.trim()) {
|
||||
await appendMultipassLog(logPath, `${result.stderr.trim()}\n`);
|
||||
}
|
||||
await appendMultipassLog(logPath, "\n");
|
||||
return result;
|
||||
}
|
||||
|
||||
async function waitForGuestReady(logPath: string, vmName: string) {
|
||||
let lastError: unknown;
|
||||
for (let attempt = 1; attempt <= 12; attempt += 1) {
|
||||
try {
|
||||
await runMultipassCommand(logPath, ["exec", vmName, "--", "bash", "-lc", "echo guest-ready"]);
|
||||
return;
|
||||
} catch (error) {
|
||||
lastError = error;
|
||||
await appendMultipassLog(
|
||||
logPath,
|
||||
`guest-ready retry ${attempt}/12: ${error instanceof Error ? error.message : String(error)}\n\n`,
|
||||
);
|
||||
if (attempt < 12) {
|
||||
await sleep(2_000);
|
||||
}
|
||||
}
|
||||
}
|
||||
throw lastError instanceof Error ? lastError : new Error(String(lastError));
|
||||
}
|
||||
|
||||
async function mountRepo(logPath: string, repoRoot: string, vmName: string) {
|
||||
let lastError: unknown;
|
||||
for (let attempt = 1; attempt <= 5; attempt += 1) {
|
||||
try {
|
||||
await runMultipassCommand(logPath, [
|
||||
"mount",
|
||||
repoRoot,
|
||||
`${vmName}:${MULTIPASS_MOUNTED_REPO_PATH}`,
|
||||
]);
|
||||
return;
|
||||
} catch (error) {
|
||||
lastError = error;
|
||||
await appendMultipassLog(
|
||||
logPath,
|
||||
`mount retry ${attempt}/5: ${error instanceof Error ? error.message : String(error)}\n\n`,
|
||||
);
|
||||
if (attempt < 5) {
|
||||
await sleep(2_000);
|
||||
}
|
||||
}
|
||||
}
|
||||
throw lastError instanceof Error ? lastError : new Error(String(lastError));
|
||||
}
|
||||
|
||||
async function mountCodexHome(logPath: string, hostCodexHomePath: string, vmName: string) {
|
||||
let lastError: unknown;
|
||||
for (let attempt = 1; attempt <= 5; attempt += 1) {
|
||||
try {
|
||||
await runMultipassCommand(logPath, [
|
||||
"mount",
|
||||
hostCodexHomePath,
|
||||
`${vmName}:${MULTIPASS_GUEST_CODEX_HOME_PATH}`,
|
||||
]);
|
||||
return;
|
||||
} catch (error) {
|
||||
lastError = error;
|
||||
await appendMultipassLog(
|
||||
logPath,
|
||||
`codex-home mount retry ${attempt}/5: ${error instanceof Error ? error.message : String(error)}\n\n`,
|
||||
);
|
||||
if (attempt < 5) {
|
||||
await sleep(2_000);
|
||||
}
|
||||
}
|
||||
}
|
||||
throw lastError instanceof Error ? lastError : new Error(String(lastError));
|
||||
}
|
||||
|
||||
async function transferLiveProviderConfig(plan: QaMultipassPlan) {
|
||||
if (!plan.hostLiveProviderConfigPath || !plan.guestLiveProviderConfigPath) {
|
||||
return;
|
||||
}
|
||||
await runMultipassCommand(plan.hostLogPath, [
|
||||
"transfer",
|
||||
plan.hostLiveProviderConfigPath,
|
||||
`${plan.vmName}:${plan.guestLiveProviderConfigPath}`,
|
||||
]);
|
||||
}
|
||||
|
||||
async function tryCopyGuestBootstrapLog(plan: QaMultipassPlan) {
|
||||
try {
|
||||
await runMultipassCommand(plan.hostLogPath, [
|
||||
"transfer",
|
||||
`${plan.vmName}:${plan.guestBootstrapLogPath}`,
|
||||
plan.hostBootstrapLogPath,
|
||||
]);
|
||||
} catch (error) {
|
||||
await appendMultipassLog(
|
||||
plan.hostLogPath,
|
||||
`bootstrap log transfer skipped: ${error instanceof Error ? error.message : String(error)}\n\n`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export async function runQaMultipass(params: {
|
||||
repoRoot: string;
|
||||
outputDir?: string;
|
||||
providerMode?: "mock-openai" | "live-frontier";
|
||||
primaryModel?: string;
|
||||
alternateModel?: string;
|
||||
fastMode?: boolean;
|
||||
scenarioIds?: string[];
|
||||
image?: string;
|
||||
cpus?: number;
|
||||
memory?: string;
|
||||
disk?: string;
|
||||
}) {
|
||||
const plan = createQaMultipassPlan(params);
|
||||
await mkdir(plan.outputDir, { recursive: true });
|
||||
await writeFile(
|
||||
plan.hostLogPath,
|
||||
`# OpenClaw QA Multipass host log\nvmName=${plan.vmName}\noutputDir=${plan.outputDir}\n\n`,
|
||||
"utf8",
|
||||
);
|
||||
await writeFile(plan.hostGuestScriptPath, renderQaMultipassGuestScript(plan), "utf8");
|
||||
|
||||
try {
|
||||
await execFileAsync("multipass", ["version"]);
|
||||
} catch {
|
||||
throw new Error(
|
||||
`Multipass is not installed on this host. Install it with '${resolveMultipassInstallHint()}', then rerun 'pnpm openclaw qa suite --runner multipass'.`,
|
||||
);
|
||||
}
|
||||
|
||||
let launched = false;
|
||||
try {
|
||||
await runMultipassCommand(plan.hostLogPath, [
|
||||
"launch",
|
||||
"--name",
|
||||
plan.vmName,
|
||||
"--cpus",
|
||||
String(plan.cpus),
|
||||
"--memory",
|
||||
plan.memory,
|
||||
"--disk",
|
||||
plan.disk,
|
||||
plan.image,
|
||||
]);
|
||||
launched = true;
|
||||
await waitForGuestReady(plan.hostLogPath, plan.vmName);
|
||||
await mountRepo(plan.hostLogPath, plan.repoRoot, plan.vmName);
|
||||
if (plan.hostCodexHomePath) {
|
||||
await mountCodexHome(plan.hostLogPath, plan.hostCodexHomePath, plan.vmName);
|
||||
}
|
||||
await transferLiveProviderConfig(plan);
|
||||
await runMultipassCommand(plan.hostLogPath, [
|
||||
"transfer",
|
||||
plan.hostGuestScriptPath,
|
||||
`${plan.vmName}:${plan.guestScriptPath}`,
|
||||
]);
|
||||
await runMultipassCommand(plan.hostLogPath, [
|
||||
"exec",
|
||||
plan.vmName,
|
||||
"--",
|
||||
"chmod",
|
||||
"+x",
|
||||
plan.guestScriptPath,
|
||||
]);
|
||||
await runMultipassCommand(plan.hostLogPath, ["exec", plan.vmName, "--", plan.guestScriptPath]);
|
||||
await tryCopyGuestBootstrapLog(plan);
|
||||
} catch (error) {
|
||||
if (launched) {
|
||||
await tryCopyGuestBootstrapLog(plan);
|
||||
}
|
||||
throw new Error(
|
||||
`QA Multipass run failed: ${error instanceof Error ? error.message : String(error)}. See ${plan.hostLogPath}.`,
|
||||
{ cause: error },
|
||||
);
|
||||
} finally {
|
||||
if (launched) {
|
||||
try {
|
||||
await runMultipassCommand(plan.hostLogPath, ["delete", "--purge", plan.vmName]);
|
||||
} catch (error) {
|
||||
await appendMultipassLog(
|
||||
plan.hostLogPath,
|
||||
`cleanup error: ${error instanceof Error ? error.message : String(error)}\n\n`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
await access(plan.reportPath);
|
||||
await access(plan.summaryPath);
|
||||
|
||||
return {
|
||||
outputDir: plan.outputDir,
|
||||
reportPath: plan.reportPath,
|
||||
summaryPath: plan.summaryPath,
|
||||
hostLogPath: plan.hostLogPath,
|
||||
bootstrapLogPath: plan.hostBootstrapLogPath,
|
||||
guestScriptPath: plan.hostGuestScriptPath,
|
||||
vmName: plan.vmName,
|
||||
scenarioIds: plan.scenarioIds,
|
||||
} satisfies QaMultipassRunResult;
|
||||
}
|
||||
Reference in New Issue
Block a user