From 0283b05d702a1666dc760ea2005ebaf6cc7268bd Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Tue, 5 May 2026 10:27:23 +0100 Subject: [PATCH] fix: harden Mantis Slack desktop gateway proof --- .../slack-desktop-smoke.runtime.test.ts | 82 ++++++++++++++++++- .../src/mantis/slack-desktop-smoke.runtime.ts | 64 +++++++++++++-- 2 files changed, 140 insertions(+), 6 deletions(-) diff --git a/extensions/qa-lab/src/mantis/slack-desktop-smoke.runtime.test.ts b/extensions/qa-lab/src/mantis/slack-desktop-smoke.runtime.test.ts index a9f693815f0..ab1413c3cf9 100644 --- a/extensions/qa-lab/src/mantis/slack-desktop-smoke.runtime.test.ts +++ b/extensions/qa-lab/src/mantis/slack-desktop-smoke.runtime.test.ts @@ -208,7 +208,15 @@ describe("mantis Slack desktop smoke runtime", () => { await fs.mkdir(outputDir as string, { recursive: true }); if (!String(outputDir).endsWith("slack-qa/")) { await fs.writeFile(path.join(outputDir as string, "slack-desktop-smoke.png"), "png"); - await fs.writeFile(path.join(outputDir as string, "remote-metadata.json"), "{}\n"); + await fs.writeFile( + path.join(outputDir as string, "remote-metadata.json"), + `${JSON.stringify({ + gatewayAlive: true, + gatewayPid: "1234", + openedUrl: "https://app.slack.com/client/TLEASED/CLEASED", + qaExitCode: 0, + })}\n`, + ); await fs.writeFile(path.join(outputDir as string, "slack-desktop-command.log"), "qa\n"); } } @@ -249,10 +257,82 @@ describe("mantis Slack desktop smoke runtime", () => { const remoteScript = runCommand?.args.at(-1); expect(remoteScript).toContain("setup_gateway=1"); expect(remoteScript).toContain("openclaw gateway run"); + expect(remoteScript).toContain('"$out/openclaw-gateway.log"'); + expect(remoteScript).toContain('kill -0 "$gateway_pid"'); + expect(remoteScript).toContain('disown "$gateway_pid"'); expect(fetchMock.mock.calls.map(([url]) => describeFetchInput(url))).toEqual([ "https://example.convex.site/qa-credentials/v1/acquire", "https://example.convex.site/qa-credentials/v1/release", ]); + const summary = JSON.parse(await fs.readFile(result.summaryPath, "utf8")) as { + slackUrl: string; + }; + expect(summary.slackUrl).toBe("https://app.slack.com/client/TLEASED/CLEASED"); + }); + + it("passes gateway setup when Crabbox returns non-zero after remote metadata proves success", async () => { + const runner = vi.fn(async (command: string, args: readonly string[]) => { + if (command === "/tmp/crabbox" && args[0] === "warmup") { + return { stdout: "ready lease cbx_cafe123\n", stderr: "" }; + } + if (command === "/tmp/crabbox" && args[0] === "inspect") { + return { + stdout: `${JSON.stringify({ + host: "203.0.113.10", + id: "cbx_cafe123", + provider: "hetzner", + sshKey: "/tmp/key", + sshPort: "2222", + sshUser: "crabbox", + state: "active", + })}\n`, + stderr: "", + }; + } + if (command === "/tmp/crabbox" && args[0] === "run") { + throw new Error("remote command exited 1"); + } + if (command === "rsync") { + const outputDir = args.at(-1); + await fs.mkdir(outputDir as string, { recursive: true }); + if (!String(outputDir).endsWith("slack-qa/")) { + await fs.writeFile(path.join(outputDir as string, "slack-desktop-smoke.png"), "png"); + await fs.writeFile( + path.join(outputDir as string, "remote-metadata.json"), + `${JSON.stringify({ + gatewayAlive: true, + gatewayPid: "4321", + openedUrl: "https://app.slack.com/client/TOK/COK", + qaExitCode: 0, + })}\n`, + ); + } + } + return { stdout: "", stderr: "" }; + }); + + const result = await runMantisSlackDesktopSmoke({ + commandRunner: runner, + crabboxBin: "/tmp/crabbox", + env: { + OPENAI_API_KEY: "openai-runtime-key", + OPENCLAW_MANTIS_SLACK_APP_TOKEN: "xapp-direct", + OPENCLAW_MANTIS_SLACK_BOT_TOKEN: "xoxb-direct", + PATH: process.env.PATH, + }, + gatewaySetup: true, + now: () => new Date("2026-05-04T14:30:00.000Z"), + outputDir: ".artifacts/qa-e2e/mantis/slack-desktop-gateway-metadata", + repoRoot, + }); + + expect(result.status).toBe("pass"); + const summary = JSON.parse(await fs.readFile(result.summaryPath, "utf8")) as { + status: string; + warning?: string; + }; + expect(summary.status).toBe("pass"); + expect(summary.warning).toContain("remote command exited 1"); }); it("copies the screenshot before reporting a failed remote Slack QA run", async () => { diff --git a/extensions/qa-lab/src/mantis/slack-desktop-smoke.runtime.ts b/extensions/qa-lab/src/mantis/slack-desktop-smoke.runtime.ts index 78e285b4145..8351a0b5860 100644 --- a/extensions/qa-lab/src/mantis/slack-desktop-smoke.runtime.ts +++ b/extensions/qa-lab/src/mantis/slack-desktop-smoke.runtime.ts @@ -100,6 +100,14 @@ type MantisSlackDesktopSmokeSummary = { slackUrl?: string; startedAt: string; status: "pass" | "fail"; + warning?: string; +}; + +type SlackDesktopRemoteMetadata = { + gatewayAlive?: boolean; + gatewayPid?: string; + openedUrl?: string; + qaExitCode?: number; }; const DEFAULT_PROVIDER = "hetzner"; @@ -183,6 +191,31 @@ async function pathExists(filePath: string) { } } +async function readRemoteMetadata( + outputDir: string, +): Promise { + const metadataPath = path.join(outputDir, "remote-metadata.json"); + if (!(await pathExists(metadataPath))) { + return undefined; + } + try { + const parsed = JSON.parse(await fs.readFile(metadataPath, "utf8")) as unknown; + if (!parsed || typeof parsed !== "object") { + return undefined; + } + const candidate = parsed as Record; + return { + gatewayAlive: + typeof candidate.gatewayAlive === "boolean" ? candidate.gatewayAlive : undefined, + gatewayPid: typeof candidate.gatewayPid === "string" ? candidate.gatewayPid : undefined, + openedUrl: typeof candidate.openedUrl === "string" ? candidate.openedUrl : undefined, + qaExitCode: typeof candidate.qaExitCode === "number" ? candidate.qaExitCode : undefined, + }; + } catch { + return undefined; + } +} + async function resolveCrabboxBin(params: { env: NodeJS.ProcessEnv; explicit?: string; @@ -444,7 +477,8 @@ if [ "$setup_gateway" = "1" ]; then --window-size=1440,1000 \ --window-position=0,0 \ --class=mantis-slack-desktop-smoke \ - "$slack_url" >"$out/chrome.log" 2>&1 & + "$slack_url" "$out/chrome.log" 2>&1 & + disown "$!" >/dev/null 2>&1 || true else "$browser_bin" \ --user-data-dir="$profile" \ @@ -496,9 +530,16 @@ qa_status=0 MANTIS_SLACK_PATCH pnpm openclaw config patch --file "$out/slack.socket.patch.json5" --dry-run pnpm openclaw config patch --file "$out/slack.socket.patch.json5" - nohup pnpm openclaw gateway run --dev --allow-unconfigured --port 38973 --cli-backend-logs >"$out/openclaw-gateway.log" 2>&1 & - echo "$!" >"$out/openclaw-gateway.pid" + nohup pnpm openclaw gateway run --dev --allow-unconfigured --port 38973 --cli-backend-logs "$out/openclaw-gateway.log" 2>&1 & + gateway_pid="$!" + echo "$gateway_pid" >"$out/openclaw-gateway.pid" sleep 12 + if ! kill -0 "$gateway_pid" >/dev/null 2>&1; then + echo "OpenClaw gateway exited during startup." >&2 + wait "$gateway_pid" || true + exit 1 + fi + disown "$gateway_pid" >/dev/null 2>&1 || true else qa_args=(openclaw qa slack --repo-root . --output-dir "$out/slack-qa" --provider-mode "$provider_mode" --model "$primary_model" --alt-model "$alternate_model" --credential-source "$credential_source" --credential-role "$credential_role") if [ "$fast_mode" = "1" ]; then @@ -522,6 +563,8 @@ cat >"$out/remote-metadata.json" </dev/null 2>&1; then echo true; else echo false; fi), + "gatewayPid": "$(if [ -f "$out/openclaw-gateway.pid" ]; then cat "$out/openclaw-gateway.pid"; fi)", "gatewayPort": 38973, "qaExitCode": $qa_status, "credentialSource": "$credential_source", @@ -776,6 +819,7 @@ export async function runMantisSlackDesktopSmoke( let screenshotPath: string | undefined; let slackQaDir: string | undefined; let videoPath: string | undefined; + let remoteMetadata: SlackDesktopRemoteMetadata | undefined; try { const preparedCredentialEnv = await prepareGatewayCredentialEnv({ @@ -855,13 +899,22 @@ export async function runMantisSlackDesktopSmoke( if (!(await pathExists(videoPath))) { videoPath = undefined; } + remoteMetadata = await readRemoteMetadata(outputDir); slackQaDir = path.join(outputDir, "slack-qa"); if (!(await pathExists(screenshotPath))) { throw new Error("Slack desktop screenshot was not copied back from Crabbox."); } - if (remoteRunError) { + const gatewaySetupCompleted = + gatewaySetup && remoteMetadata?.qaExitCode === 0 && remoteMetadata.gatewayAlive === true; + if (remoteRunError && !gatewaySetupCompleted) { throw remoteRunError; } + if (gatewaySetup && !gatewaySetupCompleted) { + throw new Error("Slack desktop gateway setup did not report a live OpenClaw gateway."); + } + const ignoredRemoteRunError = remoteRunError + ? `Crabbox returned a non-zero command status after the gateway setup completed: ${formatErrorMessage(remoteRunError)}` + : undefined; summary = { artifacts: { reportPath, @@ -882,9 +935,10 @@ export async function runMantisSlackDesktopSmoke( finishedAt: new Date().toISOString(), outputDir, remoteOutputDir, - slackUrl, + slackUrl: trimToValue(remoteMetadata?.openedUrl) ?? slackUrl, startedAt: startedAt.toISOString(), status: "pass", + warning: ignoredRemoteRunError, }; return { outputDir,