From 9cc802241c7fd4be55ddfe8a2c0176e898e98b7c Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 3 May 2026 20:10:38 -0700 Subject: [PATCH] fix(qa): accept testbox smoke lease ids --- CHANGELOG.md | 1 + .../desktop-browser-smoke.runtime.test.ts | 58 +++++++++++++++++ .../mantis/desktop-browser-smoke.runtime.ts | 4 +- .../slack-desktop-smoke.runtime.test.ts | 63 +++++++++++++++++++ .../src/mantis/slack-desktop-smoke.runtime.ts | 4 +- 5 files changed, 126 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c748752d0f6..64c92a26f11 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ Docs: https://docs.openclaw.ai - QA/Mantis: add `pnpm openclaw qa mantis slack-desktop-smoke` to run Slack live QA inside a Crabbox VNC desktop, open Slack Web, and capture desktop screenshots beside the Slack QA artifacts. - QA/Mantis: pass the runtime env through desktop-browser Crabbox and artifact-copy child commands, so embedded Mantis callers can provide Crabbox credentials without mutating the parent process. Thanks @vincentkoc. - QA/Mantis: return the copied Slack desktop screenshot path even when remote Slack QA fails, so the CLI still prints the failure screenshot artifact. Thanks @vincentkoc. +- QA/Mantis: accept Blacksmith Testbox `tbx_...` lease ids from desktop smoke warmup, so provider overrides do not fail before inspect/run. Thanks @vincentkoc. - QA/Slack: add a Slack live transport QA runner with canary and mention-gating coverage for the private bot-to-bot harness. Thanks @vincentkoc. - Gateway/performance: lazy-load the heavy cron runtime after the rest of Gateway startup, defer restart-sentinel refresh after readiness, and let the Gateway startup benchmark write per-run V8 CPU profiles with `--cpu-prof-dir`. - Gateway/performance: keep raw channel-config schema parsing from discovering bundled plugin runtime metadata, and add `pnpm gateway:watch --benchmark-no-force` for profiling startup without the default port cleanup. diff --git a/extensions/qa-lab/src/mantis/desktop-browser-smoke.runtime.test.ts b/extensions/qa-lab/src/mantis/desktop-browser-smoke.runtime.test.ts index d2c6623555e..c8e4315c338 100644 --- a/extensions/qa-lab/src/mantis/desktop-browser-smoke.runtime.test.ts +++ b/extensions/qa-lab/src/mantis/desktop-browser-smoke.runtime.test.ts @@ -129,6 +129,64 @@ describe("mantis desktop browser smoke runtime", () => { expect(runner).not.toHaveBeenCalled(); }); + it("accepts Blacksmith Testbox lease ids from Crabbox warmup", async () => { + const commands: { args: readonly string[]; command: string }[] = []; + const runner = vi.fn(async (command: string, args: readonly string[]) => { + commands.push({ command, args }); + if (command === "/tmp/crabbox" && args[0] === "warmup") { + return { stdout: "ready: tbx_abc-123_more\n", stderr: "" }; + } + if (command === "/tmp/crabbox" && args[0] === "inspect") { + return { + stdout: `${JSON.stringify({ + host: "203.0.113.10", + id: "tbx_abc-123_more", + provider: "blacksmith-testbox", + sshKey: "/tmp/key", + sshPort: "2222", + sshUser: "crabbox", + state: "active", + })}\n`, + stderr: "", + }; + } + if (command === "rsync") { + const outputDir = args.at(-1); + await fs.mkdir(outputDir as string, { recursive: true }); + await fs.writeFile(path.join(outputDir as string, "desktop-browser-smoke.png"), "png"); + await fs.writeFile(path.join(outputDir as string, "remote-metadata.json"), "{}\n"); + await fs.writeFile(path.join(outputDir as string, "chrome.log"), "chrome\n"); + } + return { stdout: "", stderr: "" }; + }); + + const result = await runMantisDesktopBrowserSmoke({ + commandRunner: runner, + crabboxBin: "/tmp/crabbox", + now: () => new Date("2026-05-04T12:30:00.000Z"), + outputDir: ".artifacts/qa-e2e/mantis/desktop-browser-testbox", + provider: "blacksmith-testbox", + repoRoot, + }); + + expect(result.status).toBe("pass"); + expect(commands).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + args: expect.arrayContaining(["--id", "tbx_abc-123_more"]), + command: "/tmp/crabbox", + }), + ]), + ); + const summary = JSON.parse(await fs.readFile(result.summaryPath, "utf8")) as { + crabbox: { id: string; provider: string }; + }; + expect(summary.crabbox).toMatchObject({ + id: "tbx_abc-123_more", + provider: "blacksmith-testbox", + }); + }); + it("keeps an existing lease and writes failure reports when the remote run fails", async () => { const commands: { args: readonly string[]; command: string }[] = []; const runner = vi.fn(async (command: string, args: readonly string[]) => { diff --git a/extensions/qa-lab/src/mantis/desktop-browser-smoke.runtime.ts b/extensions/qa-lab/src/mantis/desktop-browser-smoke.runtime.ts index 81c531a3c4d..2c687690631 100644 --- a/extensions/qa-lab/src/mantis/desktop-browser-smoke.runtime.ts +++ b/extensions/qa-lab/src/mantis/desktop-browser-smoke.runtime.ts @@ -170,7 +170,7 @@ async function resolveCrabboxBin(params: { } function extractLeaseId(output: string) { - return output.match(/\bcbx_[a-f0-9]+\b/u)?.[0]; + return output.match(/\b(?:cbx_[a-f0-9]+|tbx_[A-Za-z0-9_-]+)\b/u)?.[0]; } function shellQuote(value: string) { @@ -346,7 +346,7 @@ async function warmupCrabbox(params: { }); const leaseId = extractLeaseId(`${result.stdout}\n${result.stderr}`); if (!leaseId) { - throw new Error("Crabbox warmup did not print a cbx_ lease id."); + throw new Error("Crabbox warmup did not print a lease id."); } return leaseId; } diff --git a/extensions/qa-lab/src/mantis/slack-desktop-smoke.runtime.test.ts b/extensions/qa-lab/src/mantis/slack-desktop-smoke.runtime.test.ts index cdf3891be27..a91209de9e5 100644 --- a/extensions/qa-lab/src/mantis/slack-desktop-smoke.runtime.test.ts +++ b/extensions/qa-lab/src/mantis/slack-desktop-smoke.runtime.test.ts @@ -175,4 +175,67 @@ describe("mantis Slack desktop smoke runtime", () => { expect(summary.error).toContain("remote Slack QA failed"); expect(summary.artifacts.screenshotPath).toContain("slack-desktop-smoke.png"); }); + + it("accepts Blacksmith Testbox lease ids from Crabbox warmup", async () => { + const commands: { args: readonly string[]; command: string }[] = []; + const runner = vi.fn(async (command: string, args: readonly string[]) => { + commands.push({ command, args }); + if (command === "/tmp/crabbox" && args[0] === "warmup") { + return { stdout: "ready: tbx_abc-123_more\n", stderr: "" }; + } + if (command === "/tmp/crabbox" && args[0] === "inspect") { + return { + stdout: `${JSON.stringify({ + host: "203.0.113.10", + id: "tbx_abc-123_more", + provider: "blacksmith-testbox", + sshKey: "/tmp/key", + sshPort: "2222", + sshUser: "crabbox", + state: "active", + })}\n`, + stderr: "", + }; + } + if (command === "rsync") { + const outputDir = args.at(-1); + await fs.mkdir(outputDir as string, { recursive: true }); + if (String(outputDir).endsWith("slack-qa/")) { + await fs.writeFile(path.join(outputDir as string, "slack-qa-report.md"), "# Slack\n"); + } else { + await fs.writeFile(path.join(outputDir as string, "slack-desktop-smoke.png"), "png"); + await fs.writeFile(path.join(outputDir as string, "remote-metadata.json"), "{}\n"); + await fs.writeFile(path.join(outputDir as string, "chrome.log"), "chrome\n"); + await fs.writeFile(path.join(outputDir as string, "slack-desktop-command.log"), "qa\n"); + } + } + return { stdout: "", stderr: "" }; + }); + + const result = await runMantisSlackDesktopSmoke({ + commandRunner: runner, + crabboxBin: "/tmp/crabbox", + now: () => new Date("2026-05-04T13:30:00.000Z"), + outputDir: ".artifacts/qa-e2e/mantis/slack-desktop-testbox", + provider: "blacksmith-testbox", + repoRoot, + }); + + expect(result.status).toBe("pass"); + expect(commands).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + args: expect.arrayContaining(["--id", "tbx_abc-123_more"]), + command: "/tmp/crabbox", + }), + ]), + ); + const summary = JSON.parse(await fs.readFile(result.summaryPath, "utf8")) as { + crabbox: { id: string; provider: string }; + }; + expect(summary.crabbox).toMatchObject({ + id: "tbx_abc-123_more", + provider: "blacksmith-testbox", + }); + }); }); diff --git a/extensions/qa-lab/src/mantis/slack-desktop-smoke.runtime.ts b/extensions/qa-lab/src/mantis/slack-desktop-smoke.runtime.ts index b86d51bead9..6f7a1a71ec0 100644 --- a/extensions/qa-lab/src/mantis/slack-desktop-smoke.runtime.ts +++ b/extensions/qa-lab/src/mantis/slack-desktop-smoke.runtime.ts @@ -199,7 +199,7 @@ function buildCrabboxEnv(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv { } function extractLeaseId(output: string) { - return output.match(/\bcbx_[a-f0-9]+\b/u)?.[0]; + return output.match(/\b(?:cbx_[a-f0-9]+|tbx_[A-Za-z0-9_-]+)\b/u)?.[0]; } function shellQuote(value: string) { @@ -479,7 +479,7 @@ async function warmupCrabbox(params: { }); const leaseId = extractLeaseId(`${result.stdout}\n${result.stderr}`); if (!leaseId) { - throw new Error("Crabbox warmup did not print a cbx_ lease id."); + throw new Error("Crabbox warmup did not print a lease id."); } return leaseId; }