diff --git a/extensions/qa-lab/src/cli.runtime.test.ts b/extensions/qa-lab/src/cli.runtime.test.ts index 8bccb9735a0..c1c199e11f6 100644 --- a/extensions/qa-lab/src/cli.runtime.test.ts +++ b/extensions/qa-lab/src/cli.runtime.test.ts @@ -5,6 +5,7 @@ const { runQaManualLane, runQaSuite, runQaCharacterEval, + runQaMultipass, startQaLabServer, writeQaDockerHarnessFiles, buildQaDockerHarnessImage, @@ -13,6 +14,7 @@ const { runQaManualLane: vi.fn(), runQaSuite: vi.fn(), runQaCharacterEval: vi.fn(), + runQaMultipass: vi.fn(), startQaLabServer: vi.fn(), writeQaDockerHarnessFiles: vi.fn(), buildQaDockerHarnessImage: vi.fn(), @@ -31,6 +33,10 @@ vi.mock("./character-eval.js", () => ({ runQaCharacterEval, })); +vi.mock("./multipass.runtime.js", () => ({ + runQaMultipass, +})); + vi.mock("./lab-server.js", () => ({ startQaLabServer, })); @@ -62,6 +68,7 @@ describe("qa cli runtime", () => { runQaSuite.mockReset(); runQaCharacterEval.mockReset(); runQaManualLane.mockReset(); + runQaMultipass.mockReset(); startQaLabServer.mockReset(); writeQaDockerHarnessFiles.mockReset(); buildQaDockerHarnessImage.mockReset(); @@ -81,6 +88,16 @@ describe("qa cli runtime", () => { reply: "done", watchUrl: "http://127.0.0.1:43124", }); + runQaMultipass.mockResolvedValue({ + outputDir: "/tmp/multipass", + reportPath: "/tmp/multipass/qa-suite-report.md", + summaryPath: "/tmp/multipass/qa-suite-summary.json", + hostLogPath: "/tmp/multipass/multipass-host.log", + bootstrapLogPath: "/tmp/multipass/multipass-guest-bootstrap.log", + guestScriptPath: "/tmp/multipass/multipass-guest-run.sh", + vmName: "openclaw-qa-test", + scenarioIds: ["channel-chat-baseline"], + }); startQaLabServer.mockResolvedValue({ baseUrl: "http://127.0.0.1:58000", runSelfCheck: vi.fn().mockResolvedValue({ @@ -267,6 +284,68 @@ describe("qa cli runtime", () => { }); }); + it("routes suite runs through multipass when the runner is selected", async () => { + await runQaSuiteCommand({ + repoRoot: "/tmp/openclaw-repo", + outputDir: ".artifacts/qa-multipass", + runner: "multipass", + providerMode: "mock-openai", + scenarioIds: ["channel-chat-baseline"], + image: "lts", + cpus: 2, + memory: "4G", + disk: "24G", + }); + + expect(runQaMultipass).toHaveBeenCalledWith({ + repoRoot: path.resolve("/tmp/openclaw-repo"), + outputDir: path.resolve("/tmp/openclaw-repo", ".artifacts/qa-multipass"), + providerMode: "mock-openai", + primaryModel: undefined, + alternateModel: undefined, + fastMode: undefined, + scenarioIds: ["channel-chat-baseline"], + image: "lts", + cpus: 2, + memory: "4G", + disk: "24G", + }); + expect(runQaSuite).not.toHaveBeenCalled(); + }); + + it("passes live suite selection through to the multipass runner", async () => { + await runQaSuiteCommand({ + repoRoot: "/tmp/openclaw-repo", + runner: "multipass", + providerMode: "live-frontier", + primaryModel: "openai/gpt-5.4", + alternateModel: "openai/gpt-5.4", + fastMode: true, + scenarioIds: ["channel-chat-baseline"], + }); + + expect(runQaMultipass).toHaveBeenCalledWith( + expect.objectContaining({ + repoRoot: path.resolve("/tmp/openclaw-repo"), + providerMode: "live-frontier", + primaryModel: "openai/gpt-5.4", + alternateModel: "openai/gpt-5.4", + fastMode: true, + scenarioIds: ["channel-chat-baseline"], + }), + ); + }); + + it("rejects multipass-only suite flags on the host runner", async () => { + await expect( + runQaSuiteCommand({ + repoRoot: "/tmp/openclaw-repo", + runner: "host", + image: "lts", + }), + ).rejects.toThrow("--image, --cpus, --memory, and --disk require --runner multipass."); + }); + it("defaults manual mock runs onto the mock-openai model lane", async () => { await runQaManualLaneCommand({ repoRoot: "/tmp/openclaw-repo", diff --git a/extensions/qa-lab/src/cli.runtime.ts b/extensions/qa-lab/src/cli.runtime.ts index a32f407bb97..a6b628d7518 100644 --- a/extensions/qa-lab/src/cli.runtime.ts +++ b/extensions/qa-lab/src/cli.runtime.ts @@ -5,6 +5,7 @@ import { runQaDockerUp } from "./docker-up.runtime.js"; import { startQaLabServer } from "./lab-server.js"; import { runQaManualLane } from "./manual-lane.runtime.js"; import { startQaMockOpenAiServer } from "./mock-openai-server.js"; +import { runQaMultipass } from "./multipass.runtime.js"; import { normalizeQaThinkingLevel, type QaThinkingLevel } from "./qa-gateway-config.js"; import { defaultQaModelForMode, @@ -193,14 +194,53 @@ export async function runQaLabSelfCheckCommand(opts: { repoRoot?: string; output export async function runQaSuiteCommand(opts: { repoRoot?: string; outputDir?: string; + runner?: string; providerMode?: QaProviderModeInput; primaryModel?: string; alternateModel?: string; fastMode?: boolean; scenarioIds?: string[]; + image?: string; + cpus?: number; + memory?: string; + disk?: string; }) { const repoRoot = path.resolve(opts.repoRoot ?? process.cwd()); + const runner = (opts.runner ?? "host").trim().toLowerCase(); + if (runner !== "host" && runner !== "multipass") { + throw new Error(`--runner must be one of host or multipass, got "${opts.runner}".`); + } const providerMode = normalizeQaProviderMode(opts.providerMode); + if ( + runner === "host" && + (opts.image !== undefined || + opts.cpus !== undefined || + opts.memory !== undefined || + opts.disk !== undefined) + ) { + throw new Error("--image, --cpus, --memory, and --disk require --runner multipass."); + } + if (runner === "multipass") { + const result = await runQaMultipass({ + repoRoot, + outputDir: opts.outputDir ? path.resolve(repoRoot, opts.outputDir) : undefined, + providerMode, + primaryModel: opts.primaryModel, + alternateModel: opts.alternateModel, + fastMode: opts.fastMode, + scenarioIds: opts.scenarioIds, + image: opts.image, + cpus: parseQaPositiveIntegerOption("--cpus", opts.cpus), + memory: opts.memory, + disk: opts.disk, + }); + process.stdout.write(`QA Multipass dir: ${result.outputDir}\n`); + process.stdout.write(`QA Multipass report: ${result.reportPath}\n`); + process.stdout.write(`QA Multipass summary: ${result.summaryPath}\n`); + process.stdout.write(`QA Multipass host log: ${result.hostLogPath}\n`); + process.stdout.write(`QA Multipass bootstrap log: ${result.bootstrapLogPath}\n`); + return; + } const result = await runQaSuite({ repoRoot, outputDir: opts.outputDir ? path.resolve(repoRoot, opts.outputDir) : undefined, diff --git a/extensions/qa-lab/src/cli.ts b/extensions/qa-lab/src/cli.ts index a83f15bb767..3e2ba35508e 100644 --- a/extensions/qa-lab/src/cli.ts +++ b/extensions/qa-lab/src/cli.ts @@ -23,6 +23,11 @@ async function runQaSuite(opts: { alternateModel?: string; fastMode?: boolean; scenarioIds?: string[]; + runner?: string; + image?: string; + cpus?: number; + memory?: string; + disk?: string; }) { const runtime = await loadQaLabCliRuntime(); await runtime.runQaSuiteCommand(opts); @@ -138,6 +143,7 @@ export function registerQaLabCli(program: Command) { .description("Run repo-backed QA scenarios against the QA gateway lane") .option("--repo-root ", "Repository root to target when running from a neutral cwd") .option("--output-dir ", "Suite artifact directory") + .option("--runner ", "Execution runner: host or multipass", "host") .option( "--provider-mode ", "Provider mode: mock-openai or live-frontier (legacy live-openai still works)", @@ -147,24 +153,38 @@ export function registerQaLabCli(program: Command) { .option("--alt-model ", "Alternate provider/model ref") .option("--scenario ", "Run only the named QA scenario (repeatable)", collectString, []) .option("--fast", "Enable provider fast mode where supported", false) + .option("--image ", "Multipass image alias") + .option("--cpus ", "Multipass vCPU count", (value: string) => Number(value)) + .option("--memory ", "Multipass memory size") + .option("--disk ", "Multipass disk size") .action( async (opts: { repoRoot?: string; outputDir?: string; + runner?: string; providerMode?: QaProviderModeInput; model?: string; altModel?: string; scenario?: string[]; fast?: boolean; + image?: string; + cpus?: number; + memory?: string; + disk?: string; }) => { await runQaSuite({ repoRoot: opts.repoRoot, outputDir: opts.outputDir, + runner: opts.runner, providerMode: opts.providerMode, primaryModel: opts.model, alternateModel: opts.altModel, fastMode: opts.fast, scenarioIds: opts.scenario, + image: opts.image, + cpus: opts.cpus, + memory: opts.memory, + disk: opts.disk, }); }, ); diff --git a/extensions/qa-lab/src/multipass.runtime.test.ts b/extensions/qa-lab/src/multipass.runtime.test.ts new file mode 100644 index 00000000000..c2e0b9cd1d3 --- /dev/null +++ b/extensions/qa-lab/src/multipass.runtime.test.ts @@ -0,0 +1,72 @@ +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { createQaMultipassPlan, renderQaMultipassGuestScript } from "./multipass.runtime.js"; + +describe("qa multipass runtime", () => { + afterEach(() => { + vi.unstubAllEnvs(); + }); + + it("reuses suite scenario semantics and resolves mounted artifact paths", () => { + const repoRoot = process.cwd(); + const outputDir = path.join(repoRoot, ".artifacts", "qa-e2e", "multipass-test"); + const plan = createQaMultipassPlan({ + repoRoot, + outputDir, + }); + + expect(plan.outputDir).toBe(outputDir); + expect(plan.scenarioIds).toEqual([]); + expect(plan.qaCommand).not.toContain("--scenario"); + expect(plan.guestOutputDir).toBe("/workspace/openclaw-host/.artifacts/qa-e2e/multipass-test"); + expect(plan.reportPath).toBe(path.join(outputDir, "qa-suite-report.md")); + expect(plan.summaryPath).toBe(path.join(outputDir, "qa-suite-summary.json")); + }); + + it("renders a guest script that runs the mock qa suite with explicit scenarios", () => { + const plan = createQaMultipassPlan({ + repoRoot: process.cwd(), + outputDir: path.join(process.cwd(), ".artifacts", "qa-e2e", "multipass-test"), + scenarioIds: ["channel-chat-baseline", "thread-follow-up"], + }); + + const script = renderQaMultipassGuestScript(plan); + + expect(script).toContain("pnpm install --frozen-lockfile"); + expect(script).toContain("pnpm build"); + expect(script).toContain("'pnpm' 'openclaw' 'qa' 'suite' '--provider-mode' 'mock-openai'"); + expect(script).toContain("'--scenario' 'channel-chat-baseline'"); + expect(script).toContain("'--scenario' 'thread-follow-up'"); + expect(script).toContain("/workspace/openclaw-host/.artifacts/qa-e2e/multipass-test"); + }); + + it("carries live suite flags and forwarded auth env into the guest command", () => { + vi.stubEnv("OPENAI_API_KEY", "test-openai-key"); + const plan = createQaMultipassPlan({ + repoRoot: process.cwd(), + outputDir: path.join(process.cwd(), ".artifacts", "qa-e2e", "multipass-live-test"), + providerMode: "live-frontier", + primaryModel: "openai/gpt-5.4", + alternateModel: "openai/gpt-5.4", + fastMode: true, + scenarioIds: ["channel-chat-baseline"], + }); + + const script = renderQaMultipassGuestScript(plan); + + expect(plan.qaCommand).toEqual( + expect.arrayContaining([ + "--provider-mode", + "live-frontier", + "--model", + "openai/gpt-5.4", + "--alt-model", + "openai/gpt-5.4", + "--fast", + ]), + ); + expect(plan.forwardedEnv.OPENAI_API_KEY).toBe("test-openai-key"); + expect(script).toContain("OPENAI_API_KEY='test-openai-key'"); + expect(script).toContain("'pnpm' 'openclaw' 'qa' 'suite' '--provider-mode' 'live-frontier'"); + }); +}); diff --git a/extensions/qa-lab/src/multipass.runtime.ts b/extensions/qa-lab/src/multipass.runtime.ts new file mode 100644 index 00000000000..0e356702dbb --- /dev/null +++ b/extensions/qa-lab/src/multipass.runtime.ts @@ -0,0 +1,643 @@ +import { execFile } from "node:child_process"; +import fs from "node:fs"; +import { access, appendFile, mkdir, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; + +const MULTIPASS_MOUNTED_REPO_PATH = "/workspace/openclaw-host"; +const MULTIPASS_GUEST_REPO_PATH = "/workspace/openclaw"; +const MULTIPASS_GUEST_CODEX_HOME_PATH = "/workspace/openclaw-codex-home"; +const MULTIPASS_GUEST_PACKAGES = [ + "build-essential", + "ca-certificates", + "curl", + "pkg-config", + "python3", + "rsync", + "xz-utils", +] as const; +const MULTIPASS_REPO_SYNC_EXCLUDES = [ + ".git", + "node_modules", + ".artifacts", + ".tmp", + ".turbo", + "coverage", + "*.heapsnapshot", +] as const; + +const QA_LIVE_ENV_ALIASES = Object.freeze([ + { + liveVar: "OPENCLAW_LIVE_OPENAI_KEY", + providerVar: "OPENAI_API_KEY", + }, + { + liveVar: "OPENCLAW_LIVE_ANTHROPIC_KEY", + providerVar: "ANTHROPIC_API_KEY", + }, + { + liveVar: "OPENCLAW_LIVE_GEMINI_KEY", + providerVar: "GEMINI_API_KEY", + }, +]); + +const QA_LIVE_ALLOWED_ENV_VARS = Object.freeze([ + "ANTHROPIC_API_KEY", + "ANTHROPIC_OAUTH_TOKEN", + "AWS_ACCESS_KEY_ID", + "AWS_BEARER_TOKEN_BEDROCK", + "AWS_REGION", + "AWS_SECRET_ACCESS_KEY", + "AWS_SESSION_TOKEN", + "GEMINI_API_KEY", + "GEMINI_API_KEYS", + "GOOGLE_API_KEY", + "MISTRAL_API_KEY", + "OPENAI_API_KEY", + "OPENAI_API_KEYS", + "OPENAI_BASE_URL", + "OPENCLAW_LIVE_ANTHROPIC_KEY", + "OPENCLAW_LIVE_ANTHROPIC_KEYS", + "OPENCLAW_LIVE_GEMINI_KEY", + "OPENCLAW_LIVE_OPENAI_KEY", + "OPENCLAW_QA_LIVE_PROVIDER_CONFIG_PATH", + "OPENCLAW_CONFIG_PATH", + "VOYAGE_API_KEY", +]); + +export const qaMultipassDefaultResources = { + image: "lts", + cpus: 2, + memory: "4G", + disk: "24G", +} as const; + +type ExecResult = { + stdout: string; + stderr: string; +}; + +export type QaMultipassPlan = { + repoRoot: string; + outputDir: string; + reportPath: string; + summaryPath: string; + hostLogPath: string; + hostBootstrapLogPath: string; + hostGuestScriptPath: string; + vmName: string; + image: string; + cpus: number; + memory: string; + disk: string; + pnpmVersion: string; + providerMode: "mock-openai" | "live-frontier"; + primaryModel?: string; + alternateModel?: string; + fastMode?: boolean; + scenarioIds: string[]; + forwardedEnv: Record; + hostCodexHomePath?: string; + guestCodexHomePath?: string; + hostLiveProviderConfigPath?: string; + guestLiveProviderConfigPath?: string; + guestMountedRepoPath: string; + guestRepoPath: string; + guestOutputDir: string; + guestScriptPath: string; + guestBootstrapLogPath: string; + qaCommand: string[]; +}; + +export type QaMultipassRunResult = { + outputDir: string; + reportPath: string; + summaryPath: string; + hostLogPath: string; + bootstrapLogPath: string; + guestScriptPath: string; + vmName: string; + scenarioIds: string[]; +}; + +function shellQuote(value: string) { + return `'${value.replaceAll("'", `'"'"'`)}'`; +} + +function createOutputStamp() { + return new Date().toISOString().replaceAll(":", "").replaceAll(".", "").replace("T", "-"); +} + +function createVmSuffix() { + return `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`; +} + +function sleep(ms: number) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function execFileAsync(file: string, args: string[]) { + return new Promise((resolve, reject) => { + execFile(file, args, { encoding: "utf8" }, (error, stdout, stderr) => { + if (error) { + const message = stderr.trim() || stdout.trim() || error.message; + reject(new Error(message)); + return; + } + resolve({ stdout, stderr }); + }); + }); +} + +function resolvePnpmVersion(repoRoot: string) { + const packageJsonPath = path.join(repoRoot, "package.json"); + const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, "utf8")) as { + packageManager?: string; + }; + const packageManager = packageJson.packageManager ?? ""; + const match = /^pnpm@(.+)$/.exec(packageManager); + if (!match?.[1]) { + throw new Error(`unable to resolve pnpm version from packageManager in ${packageJsonPath}`); + } + return match[1]; +} + +function resolveMultipassInstallHint() { + if (process.platform === "darwin") { + return "brew install --cask multipass"; + } + if (process.platform === "win32") { + return "winget install Canonical.Multipass"; + } + if (process.platform === "linux") { + return "sudo snap install multipass"; + } + return "https://multipass.run/install"; +} + +function resolveUserPath(value: string, env: NodeJS.ProcessEnv = process.env) { + if (value === "~") { + return env.HOME ?? os.homedir(); + } + if (value.startsWith("~/")) { + return path.join(env.HOME ?? os.homedir(), value.slice(2)); + } + return path.resolve(value); +} + +function resolveLiveProviderConfigPath(env: NodeJS.ProcessEnv = process.env) { + const explicit = + env.OPENCLAW_QA_LIVE_PROVIDER_CONFIG_PATH?.trim() || env.OPENCLAW_CONFIG_PATH?.trim(); + return explicit + ? { path: resolveUserPath(explicit, env), explicit: true } + : { path: path.join(os.homedir(), ".openclaw", "openclaw.json"), explicit: false }; +} + +function resolveQaLiveCliAuthEnv(baseEnv: NodeJS.ProcessEnv) { + const configuredCodexHome = baseEnv.CODEX_HOME?.trim(); + if (configuredCodexHome) { + return { CODEX_HOME: resolveUserPath(configuredCodexHome, baseEnv) }; + } + const hostHome = baseEnv.HOME?.trim(); + if (!hostHome) { + return {}; + } + const codexHome = path.join(hostHome, ".codex"); + return fs.existsSync(codexHome) ? { CODEX_HOME: codexHome } : {}; +} + +function resolveForwardedLiveEnv(baseEnv: NodeJS.ProcessEnv = process.env) { + const forwarded: Record = {}; + for (const key of QA_LIVE_ALLOWED_ENV_VARS) { + const value = baseEnv[key]?.trim(); + if (value) { + forwarded[key] = value; + } + } + for (const { liveVar, providerVar } of QA_LIVE_ENV_ALIASES) { + const liveValue = forwarded[liveVar]?.trim(); + if (liveValue && !forwarded[providerVar]?.trim()) { + forwarded[providerVar] = liveValue; + } + } + const liveCliAuth = resolveQaLiveCliAuthEnv(baseEnv); + if (liveCliAuth.CODEX_HOME) { + forwarded.CODEX_HOME = liveCliAuth.CODEX_HOME; + } + return forwarded; +} + +function createQaMultipassOutputDir(repoRoot: string) { + return path.join(repoRoot, ".artifacts", "qa-e2e", `multipass-${createOutputStamp()}`); +} + +function resolveGuestMountedPath(repoRoot: string, hostPath: string) { + const relativePath = path.relative(repoRoot, hostPath); + if (relativePath.startsWith("..") || path.isAbsolute(relativePath) || relativePath.length === 0) { + throw new Error(`unable to resolve Multipass mounted path for ${hostPath}`); + } + return path.posix.join(MULTIPASS_MOUNTED_REPO_PATH, ...relativePath.split(path.sep)); +} + +function appendScenarioArgs(command: string[], scenarioIds: string[]) { + for (const scenarioId of scenarioIds) { + command.push("--scenario", scenarioId); + } + return command; +} + +export function createQaMultipassPlan(params: { + repoRoot: string; + outputDir?: string; + providerMode?: "mock-openai" | "live-frontier"; + primaryModel?: string; + alternateModel?: string; + fastMode?: boolean; + scenarioIds?: string[]; + image?: string; + cpus?: number; + memory?: string; + disk?: string; +}) { + const outputDir = params.outputDir ?? createQaMultipassOutputDir(params.repoRoot); + const scenarioIds = [...new Set(params.scenarioIds ?? [])]; + const providerMode = params.providerMode ?? "mock-openai"; + const forwardedEnv = providerMode === "live-frontier" ? resolveForwardedLiveEnv() : {}; + const hostCodexHomePath = forwardedEnv.CODEX_HOME; + const liveProviderConfig = + providerMode === "live-frontier" ? resolveLiveProviderConfigPath() : undefined; + const hostLiveProviderConfigPath = + liveProviderConfig && fs.existsSync(liveProviderConfig.path) + ? liveProviderConfig.path + : undefined; + const vmName = `openclaw-qa-${createVmSuffix()}`; + const guestOutputDir = resolveGuestMountedPath(params.repoRoot, outputDir); + const qaCommand = appendScenarioArgs( + [ + "pnpm", + "openclaw", + "qa", + "suite", + "--provider-mode", + providerMode, + "--output-dir", + guestOutputDir, + ...(params.primaryModel ? ["--model", params.primaryModel] : []), + ...(params.alternateModel ? ["--alt-model", params.alternateModel] : []), + ...(params.fastMode ? ["--fast"] : []), + ], + scenarioIds, + ); + + return { + repoRoot: params.repoRoot, + outputDir, + reportPath: path.join(outputDir, "qa-suite-report.md"), + summaryPath: path.join(outputDir, "qa-suite-summary.json"), + hostLogPath: path.join(outputDir, "multipass-host.log"), + hostBootstrapLogPath: path.join(outputDir, "multipass-guest-bootstrap.log"), + hostGuestScriptPath: path.join(outputDir, "multipass-guest-run.sh"), + vmName, + image: params.image ?? qaMultipassDefaultResources.image, + cpus: params.cpus ?? qaMultipassDefaultResources.cpus, + memory: params.memory ?? qaMultipassDefaultResources.memory, + disk: params.disk ?? qaMultipassDefaultResources.disk, + pnpmVersion: resolvePnpmVersion(params.repoRoot), + providerMode, + primaryModel: params.primaryModel, + alternateModel: params.alternateModel, + fastMode: params.fastMode, + scenarioIds, + forwardedEnv, + hostCodexHomePath, + guestCodexHomePath: hostCodexHomePath ? MULTIPASS_GUEST_CODEX_HOME_PATH : undefined, + hostLiveProviderConfigPath, + guestLiveProviderConfigPath: hostLiveProviderConfigPath + ? `/tmp/${vmName}-live-provider-config.json` + : undefined, + guestMountedRepoPath: MULTIPASS_MOUNTED_REPO_PATH, + guestRepoPath: MULTIPASS_GUEST_REPO_PATH, + guestOutputDir, + guestScriptPath: `/tmp/${vmName}-qa-suite.sh`, + guestBootstrapLogPath: `/tmp/${vmName}-bootstrap.log`, + qaCommand, + } satisfies QaMultipassPlan; +} + +export function renderQaMultipassGuestScript(plan: QaMultipassPlan) { + const rsyncCommand = [ + "rsync -a --delete", + ...MULTIPASS_REPO_SYNC_EXCLUDES.flatMap((value) => ["--exclude", shellQuote(value)]), + shellQuote(`${plan.guestMountedRepoPath}/`), + shellQuote(`${plan.guestRepoPath}/`), + ].join(" "); + const qaCommand = [ + ...Object.entries(plan.forwardedEnv) + .filter( + ([key]) => + key !== "CODEX_HOME" && + key !== "OPENCLAW_CONFIG_PATH" && + key !== "OPENCLAW_QA_LIVE_PROVIDER_CONFIG_PATH", + ) + .map(([key, value]) => `${key}=${shellQuote(value)}`), + ...(plan.guestCodexHomePath ? [`CODEX_HOME=${shellQuote(plan.guestCodexHomePath)}`] : []), + ...(plan.guestLiveProviderConfigPath + ? [ + `OPENCLAW_CONFIG_PATH=${shellQuote(plan.guestLiveProviderConfigPath)}`, + `OPENCLAW_QA_LIVE_PROVIDER_CONFIG_PATH=${shellQuote(plan.guestLiveProviderConfigPath)}`, + ] + : []), + plan.qaCommand.map(shellQuote).join(" "), + ].join(" "); + + const lines = [ + "#!/usr/bin/env bash", + "set -euo pipefail", + "trap 'status=$?; echo \"guest failure: ${BASH_COMMAND} (exit ${status})\" >&2; exit ${status}' ERR", + "", + "export DEBIAN_FRONTEND=noninteractive", + `BOOTSTRAP_LOG=${shellQuote(plan.guestBootstrapLogPath)}`, + ': > "$BOOTSTRAP_LOG"', + "", + "ensure_guest_packages() {", + ' sudo -E apt-get update >>"$BOOTSTRAP_LOG" 2>&1', + " sudo -E apt-get install -y \\", + ...MULTIPASS_GUEST_PACKAGES.map((value, index) => + index === MULTIPASS_GUEST_PACKAGES.length - 1 + ? ` ${value} >>"$BOOTSTRAP_LOG" 2>&1` + : ` ${value} \\`, + ), + "}", + "", + "ensure_node() {", + " if command -v node >/dev/null; then", + " local node_major", + ' node_major="$(node -p \'process.versions.node.split(".")[0]\' 2>/dev/null || echo 0)"', + ' if [ "${node_major}" -ge 22 ]; then', + " return 0", + " fi", + " fi", + " local node_arch", + ' case "$(uname -m)" in', + ' x86_64) node_arch="x64" ;;', + ' aarch64|arm64) node_arch="arm64" ;;', + ' *) echo "unsupported guest architecture for node bootstrap: $(uname -m)" >&2; return 1 ;;', + " esac", + " local node_tmp_dir tarball_name extract_dir base_url", + ' node_tmp_dir="$(mktemp -d)"', + " trap 'rm -rf \"${node_tmp_dir}\"' RETURN", + ' base_url="https://nodejs.org/dist/latest-v22.x"', + ' curl -fsSL "${base_url}/SHASUMS256.txt" -o "${node_tmp_dir}/SHASUMS256.txt" >>"$BOOTSTRAP_LOG" 2>&1', + ' tarball_name="$(awk \'/linux-\'"${node_arch}"\'\\.tar\\.xz$/ { print $2; exit }\' "${node_tmp_dir}/SHASUMS256.txt")"', + ' [ -n "${tarball_name}" ] || { echo "unable to resolve node tarball for ${node_arch}" >&2; return 1; }', + ' curl -fsSL "${base_url}/${tarball_name}" -o "${node_tmp_dir}/${tarball_name}" >>"$BOOTSTRAP_LOG" 2>&1', + ' (cd "${node_tmp_dir}" && grep " ${tarball_name}$" SHASUMS256.txt | sha256sum -c -) >>"$BOOTSTRAP_LOG" 2>&1', + ' extract_dir="${tarball_name%.tar.xz}"', + ' sudo mkdir -p /usr/local/lib/nodejs >>"$BOOTSTRAP_LOG" 2>&1', + ' sudo rm -rf "/usr/local/lib/nodejs/${extract_dir}" >>"$BOOTSTRAP_LOG" 2>&1', + ' sudo tar -xJf "${node_tmp_dir}/${tarball_name}" -C /usr/local/lib/nodejs >>"$BOOTSTRAP_LOG" 2>&1', + ' sudo ln -sf "/usr/local/lib/nodejs/${extract_dir}/bin/node" /usr/local/bin/node >>"$BOOTSTRAP_LOG" 2>&1', + ' sudo ln -sf "/usr/local/lib/nodejs/${extract_dir}/bin/npm" /usr/local/bin/npm >>"$BOOTSTRAP_LOG" 2>&1', + ' sudo ln -sf "/usr/local/lib/nodejs/${extract_dir}/bin/npx" /usr/local/bin/npx >>"$BOOTSTRAP_LOG" 2>&1', + ' sudo ln -sf "/usr/local/lib/nodejs/${extract_dir}/bin/corepack" /usr/local/bin/corepack >>"$BOOTSTRAP_LOG" 2>&1', + "}", + "", + "ensure_pnpm() {", + ' sudo env PATH="/usr/local/bin:/usr/bin:/bin" corepack enable >>"$BOOTSTRAP_LOG" 2>&1', + ` sudo env PATH="/usr/local/bin:/usr/bin:/bin" corepack prepare pnpm@${plan.pnpmVersion} --activate >>"$BOOTSTRAP_LOG" 2>&1`, + "}", + "", + 'command -v sudo >/dev/null || { echo "missing sudo in guest" >&2; exit 1; }', + "ensure_guest_packages", + "ensure_node", + "ensure_pnpm", + 'command -v node >/dev/null || { echo "missing node after guest bootstrap" >&2; exit 1; }', + 'command -v pnpm >/dev/null || { echo "missing pnpm after guest bootstrap" >&2; exit 1; }', + 'command -v rsync >/dev/null || { echo "missing rsync after guest bootstrap" >&2; exit 1; }', + "", + `mkdir -p ${shellQuote(path.posix.dirname(plan.guestRepoPath))}`, + `rm -rf ${shellQuote(plan.guestRepoPath)}`, + `mkdir -p ${shellQuote(plan.guestRepoPath)}`, + `mkdir -p ${shellQuote(plan.guestOutputDir)}`, + rsyncCommand, + `cd ${shellQuote(plan.guestRepoPath)}`, + 'pnpm install --frozen-lockfile >>"$BOOTSTRAP_LOG" 2>&1', + 'pnpm build >>"$BOOTSTRAP_LOG" 2>&1', + qaCommand, + "", + ]; + return lines.join("\n"); +} + +async function appendMultipassLog(logPath: string, message: string) { + await appendFile(logPath, message, "utf8"); +} + +async function runMultipassCommand(logPath: string, args: string[]) { + await appendMultipassLog(logPath, `$ ${["multipass", ...args].join(" ")}\n`); + const result = await execFileAsync("multipass", args); + if (result.stdout.trim()) { + await appendMultipassLog(logPath, `${result.stdout.trim()}\n`); + } + if (result.stderr.trim()) { + await appendMultipassLog(logPath, `${result.stderr.trim()}\n`); + } + await appendMultipassLog(logPath, "\n"); + return result; +} + +async function waitForGuestReady(logPath: string, vmName: string) { + let lastError: unknown; + for (let attempt = 1; attempt <= 12; attempt += 1) { + try { + await runMultipassCommand(logPath, ["exec", vmName, "--", "bash", "-lc", "echo guest-ready"]); + return; + } catch (error) { + lastError = error; + await appendMultipassLog( + logPath, + `guest-ready retry ${attempt}/12: ${error instanceof Error ? error.message : String(error)}\n\n`, + ); + if (attempt < 12) { + await sleep(2_000); + } + } + } + throw lastError instanceof Error ? lastError : new Error(String(lastError)); +} + +async function mountRepo(logPath: string, repoRoot: string, vmName: string) { + let lastError: unknown; + for (let attempt = 1; attempt <= 5; attempt += 1) { + try { + await runMultipassCommand(logPath, [ + "mount", + repoRoot, + `${vmName}:${MULTIPASS_MOUNTED_REPO_PATH}`, + ]); + return; + } catch (error) { + lastError = error; + await appendMultipassLog( + logPath, + `mount retry ${attempt}/5: ${error instanceof Error ? error.message : String(error)}\n\n`, + ); + if (attempt < 5) { + await sleep(2_000); + } + } + } + throw lastError instanceof Error ? lastError : new Error(String(lastError)); +} + +async function mountCodexHome(logPath: string, hostCodexHomePath: string, vmName: string) { + let lastError: unknown; + for (let attempt = 1; attempt <= 5; attempt += 1) { + try { + await runMultipassCommand(logPath, [ + "mount", + hostCodexHomePath, + `${vmName}:${MULTIPASS_GUEST_CODEX_HOME_PATH}`, + ]); + return; + } catch (error) { + lastError = error; + await appendMultipassLog( + logPath, + `codex-home mount retry ${attempt}/5: ${error instanceof Error ? error.message : String(error)}\n\n`, + ); + if (attempt < 5) { + await sleep(2_000); + } + } + } + throw lastError instanceof Error ? lastError : new Error(String(lastError)); +} + +async function transferLiveProviderConfig(plan: QaMultipassPlan) { + if (!plan.hostLiveProviderConfigPath || !plan.guestLiveProviderConfigPath) { + return; + } + await runMultipassCommand(plan.hostLogPath, [ + "transfer", + plan.hostLiveProviderConfigPath, + `${plan.vmName}:${plan.guestLiveProviderConfigPath}`, + ]); +} + +async function tryCopyGuestBootstrapLog(plan: QaMultipassPlan) { + try { + await runMultipassCommand(plan.hostLogPath, [ + "transfer", + `${plan.vmName}:${plan.guestBootstrapLogPath}`, + plan.hostBootstrapLogPath, + ]); + } catch (error) { + await appendMultipassLog( + plan.hostLogPath, + `bootstrap log transfer skipped: ${error instanceof Error ? error.message : String(error)}\n\n`, + ); + } +} + +export async function runQaMultipass(params: { + repoRoot: string; + outputDir?: string; + providerMode?: "mock-openai" | "live-frontier"; + primaryModel?: string; + alternateModel?: string; + fastMode?: boolean; + scenarioIds?: string[]; + image?: string; + cpus?: number; + memory?: string; + disk?: string; +}) { + const plan = createQaMultipassPlan(params); + await mkdir(plan.outputDir, { recursive: true }); + await writeFile( + plan.hostLogPath, + `# OpenClaw QA Multipass host log\nvmName=${plan.vmName}\noutputDir=${plan.outputDir}\n\n`, + "utf8", + ); + await writeFile(plan.hostGuestScriptPath, renderQaMultipassGuestScript(plan), "utf8"); + + try { + await execFileAsync("multipass", ["version"]); + } catch { + throw new Error( + `Multipass is not installed on this host. Install it with '${resolveMultipassInstallHint()}', then rerun 'pnpm openclaw qa suite --runner multipass'.`, + ); + } + + let launched = false; + try { + await runMultipassCommand(plan.hostLogPath, [ + "launch", + "--name", + plan.vmName, + "--cpus", + String(plan.cpus), + "--memory", + plan.memory, + "--disk", + plan.disk, + plan.image, + ]); + launched = true; + await waitForGuestReady(plan.hostLogPath, plan.vmName); + await mountRepo(plan.hostLogPath, plan.repoRoot, plan.vmName); + if (plan.hostCodexHomePath) { + await mountCodexHome(plan.hostLogPath, plan.hostCodexHomePath, plan.vmName); + } + await transferLiveProviderConfig(plan); + await runMultipassCommand(plan.hostLogPath, [ + "transfer", + plan.hostGuestScriptPath, + `${plan.vmName}:${plan.guestScriptPath}`, + ]); + await runMultipassCommand(plan.hostLogPath, [ + "exec", + plan.vmName, + "--", + "chmod", + "+x", + plan.guestScriptPath, + ]); + await runMultipassCommand(plan.hostLogPath, ["exec", plan.vmName, "--", plan.guestScriptPath]); + await tryCopyGuestBootstrapLog(plan); + } catch (error) { + if (launched) { + await tryCopyGuestBootstrapLog(plan); + } + throw new Error( + `QA Multipass run failed: ${error instanceof Error ? error.message : String(error)}. See ${plan.hostLogPath}.`, + { cause: error }, + ); + } finally { + if (launched) { + try { + await runMultipassCommand(plan.hostLogPath, ["delete", "--purge", plan.vmName]); + } catch (error) { + await appendMultipassLog( + plan.hostLogPath, + `cleanup error: ${error instanceof Error ? error.message : String(error)}\n\n`, + ); + } + } + } + + await access(plan.reportPath); + await access(plan.summaryPath); + + return { + outputDir: plan.outputDir, + reportPath: plan.reportPath, + summaryPath: plan.summaryPath, + hostLogPath: plan.hostLogPath, + bootstrapLogPath: plan.hostBootstrapLogPath, + guestScriptPath: plan.hostGuestScriptPath, + vmName: plan.vmName, + scenarioIds: plan.scenarioIds, + } satisfies QaMultipassRunResult; +}