From 26bc40c1a4f2736d88aa3b068c3cea25b989c914 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Tue, 5 May 2026 21:06:11 +0100 Subject: [PATCH] perf: add Mantis Slack hydrate timings --- .../workflows/mantis-slack-desktop-smoke.yml | 22 +- CHANGELOG.md | 1 + docs/concepts/qa-e2e-automation.md | 15 +- extensions/qa-lab/src/mantis/cli.ts | 8 +- .../slack-desktop-smoke.runtime.test.ts | 69 ++++++ .../src/mantis/slack-desktop-smoke.runtime.ts | 212 ++++++++++++++---- 6 files changed, 277 insertions(+), 50 deletions(-) diff --git a/.github/workflows/mantis-slack-desktop-smoke.yml b/.github/workflows/mantis-slack-desktop-smoke.yml index ce021474f45..3694cb6dcf4 100644 --- a/.github/workflows/mantis-slack-desktop-smoke.yml +++ b/.github/workflows/mantis-slack-desktop-smoke.yml @@ -34,6 +34,14 @@ on: description: Optional existing Crabbox desktop/browser lease id or slug to reuse required: false type: string + hydrate_mode: + description: Remote workspace hydrate mode + required: false + default: source + type: choice + options: + - source + - prehydrated permissions: contents: write @@ -153,6 +161,16 @@ jobs: - name: Build Mantis harness run: pnpm build + - name: Cache Mantis candidate pnpm store + uses: actions/cache@v4 + with: + path: | + ~/.local/share/pnpm/store + ~/.cache/pnpm + key: mantis-slack-pnpm-${{ runner.os }}-${{ env.NODE_VERSION }}-${{ hashFiles('pnpm-lock.yaml') }} + restore-keys: | + mantis-slack-pnpm-${{ runner.os }}-${{ env.NODE_VERSION }}- + - name: Setup Go for Crabbox CLI uses: actions/setup-go@v6 with: @@ -185,7 +203,7 @@ jobs: worktree_root=".artifacts/qa-e2e/mantis/slack-desktop-smoke-worktrees" mkdir -p "$worktree_root" git worktree add --detach "$worktree_root/candidate" "$CANDIDATE_SHA" - pnpm --dir "$worktree_root/candidate" install --frozen-lockfile + pnpm --dir "$worktree_root/candidate" install --frozen-lockfile --prefer-offline pnpm --dir "$worktree_root/candidate" build - name: Run Slack desktop scenario @@ -205,6 +223,7 @@ jobs: CRABBOX_LEASE_ID: ${{ inputs.crabbox_lease_id }} CRABBOX_PROVIDER: ${{ inputs.crabbox_provider }} KEEP_VM: ${{ inputs.keep_vm }} + HYDRATE_MODE: ${{ inputs.hydrate_mode }} SCENARIO_ID: ${{ inputs.scenario_id }} shell: bash run: | @@ -254,6 +273,7 @@ jobs: --credential-source convex \ --credential-role ci \ --provider-mode live-frontier \ + --hydrate-mode "$HYDRATE_MODE" \ --model openai/gpt-5.4 \ --alt-model openai/gpt-5.4 \ --fast \ diff --git a/CHANGELOG.md b/CHANGELOG.md index 242399626a8..2f3c33818c0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -72,6 +72,7 @@ Docs: https://docs.openclaw.ai - QA/Mantis: add `pnpm openclaw qa mantis slack-desktop-smoke` to run Slack live QA inside a Crabbox VNC desktop, open Slack Web, and capture desktop screenshots beside the Slack QA artifacts. - QA/Mantis: add visual desktop tasks with Crabbox MP4 recording, screenshot capture, and optional image-understanding assertions, and preserve video artifacts in Mantis before/after reports. - QA/Mantis: reuse Crabbox desktop/browser capture tooling and pnpm store caches during Slack desktop smoke runs, reducing per-scenario setup work before screenshots and videos are captured. +- QA/Mantis: add Slack desktop hydrate modes and per-phase timing reports so warm prehydrated VNC leases can skip source install/build while cold runs still prove the full source checkout. - QA/Mantis: pass the runtime env through desktop-browser Crabbox and artifact-copy child commands, so embedded Mantis callers can provide Crabbox credentials without mutating the parent process. Thanks @vincentkoc. - QA/Mantis: return the copied Slack desktop screenshot path even when remote Slack QA fails, so the CLI still prints the failure screenshot artifact. Thanks @vincentkoc. - QA/Mantis: accept Blacksmith Testbox `tbx_...` lease ids from desktop smoke warmup, so provider overrides do not fail before inspect/run. Thanks @vincentkoc. diff --git a/docs/concepts/qa-e2e-automation.md b/docs/concepts/qa-e2e-automation.md index da089a18baa..74e607b2bb1 100644 --- a/docs/concepts/qa-e2e-automation.md +++ b/docs/concepts/qa-e2e-automation.md @@ -136,10 +136,17 @@ copies `slack-qa/`, `slack-desktop-smoke.png`, and `slack-desktop-smoke.mp4` when video capture is available back to the Mantis artifact directory. Crabbox desktop/browser leases provide the capture tools and browser/native-build helper packages up front, so the scenario should only install fallbacks on older -leases. Reuse `--lease-id ` after logging in to Slack Web manually -through VNC; reused leases also keep Crabbox's pnpm store cache warm. With -`--gateway-setup`, Mantis leaves a persistent OpenClaw Slack gateway running -inside the VM on port `38973`; without it, the command runs the normal +leases. Mantis reports total and per-phase timings in +`mantis-slack-desktop-smoke-report.md` so slow runs show whether time went into +lease warmup, credential acquisition, remote setup, or artifact copy. Reuse +`--lease-id ` after logging in to Slack Web manually through VNC; +reused leases also keep Crabbox's pnpm store cache warm. The default +`--hydrate-mode source` verifies from a source checkout and runs install/build +inside the VM. Use `--hydrate-mode prehydrated` only when the reused remote +workspace already has `node_modules` and a built `dist/`; that mode skips the +expensive install/build step and fails closed when the workspace is not ready. +With `--gateway-setup`, Mantis leaves a persistent OpenClaw Slack gateway +running inside the VM on port `38973`; without it, the command runs the normal bot-to-bot Slack QA lane and exits after artifact capture. For an agent/CV style desktop task, run: diff --git a/extensions/qa-lab/src/mantis/cli.ts b/extensions/qa-lab/src/mantis/cli.ts index f89d3171673..27bf7528f1e 100644 --- a/extensions/qa-lab/src/mantis/cli.ts +++ b/extensions/qa-lab/src/mantis/cli.ts @@ -3,7 +3,10 @@ import { createLazyCliRuntimeLoader } from "../live-transports/shared/live-trans import type { MantisDesktopBrowserSmokeOptions } from "./desktop-browser-smoke.runtime.js"; import type { MantisDiscordSmokeOptions } from "./discord-smoke.runtime.js"; import type { MantisBeforeAfterOptions } from "./run.runtime.js"; -import type { MantisSlackDesktopSmokeOptions } from "./slack-desktop-smoke.runtime.js"; +import type { + MantisSlackDesktopHydrateMode, + MantisSlackDesktopSmokeOptions, +} from "./slack-desktop-smoke.runtime.js"; import type { MantisVisualDriverOptions, MantisVisualTaskOptions, @@ -96,6 +99,7 @@ type MantisSlackDesktopSmokeCommanderOptions = { credentialSource?: string; fast?: boolean; gatewaySetup?: boolean; + hydrateMode?: MantisSlackDesktopHydrateMode; idleTimeout?: string; keepLease?: boolean; leaseId?: string; @@ -278,6 +282,7 @@ export function registerMantisCli(qa: Command) { .option("--slack-url ", "Slack web URL to open in the visible browser") .option("--slack-channel-id ", "Slack channel id for gateway setup allowlist") .option("--provider-mode ", "QA provider mode") + .option("--hydrate-mode ", "Remote hydrate mode: source or prehydrated") .option("--model ", "Primary provider/model ref") .option("--alt-model ", "Alternate provider/model ref") .option( @@ -297,6 +302,7 @@ export function registerMantisCli(qa: Command) { credentialSource: opts.credentialSource, fastMode: opts.fast, gatewaySetup: opts.gatewaySetup, + hydrateMode: opts.hydrateMode, idleTimeout: opts.idleTimeout, keepLease: opts.keepLease, leaseId: opts.leaseId, diff --git a/extensions/qa-lab/src/mantis/slack-desktop-smoke.runtime.test.ts b/extensions/qa-lab/src/mantis/slack-desktop-smoke.runtime.test.ts index 4749a0d8bce..6020fcf30ab 100644 --- a/extensions/qa-lab/src/mantis/slack-desktop-smoke.runtime.test.ts +++ b/extensions/qa-lab/src/mantis/slack-desktop-smoke.runtime.test.ts @@ -119,6 +119,7 @@ describe("mantis Slack desktop smoke runtime", () => { )?.args; expect(runArgs).not.toContain("--no-sync"); const remoteScript = runArgs?.at(-1); + expect(remoteScript).toContain("hydrate_mode='source'"); expect(remoteScript).toContain("${BROWSER:-}"); expect(remoteScript).toContain("${CHROME_BIN:-}"); expect(remoteScript).toContain("PNPM_STORE_DIR"); @@ -146,15 +147,83 @@ describe("mantis Slack desktop smoke runtime", () => { await expect(fs.readFile(result.videoPath ?? "", "utf8")).resolves.toBe("mp4"); const summary = JSON.parse(await fs.readFile(result.summaryPath, "utf8")) as { crabbox: { id: string; vncCommand: string }; + hydrateMode: string; status: string; + timings: { phases: { name: string; status: string }[]; totalMs: number }; }; expect(summary).toMatchObject({ crabbox: { id: "cbx_abc123", vncCommand: "/tmp/crabbox vnc --provider hetzner --id cbx_abc123 --open", }, + hydrateMode: "source", status: "pass", }); + expect(summary.timings.totalMs).toBeGreaterThanOrEqual(0); + expect(summary.timings.phases.map((phase) => phase.name)).toEqual( + expect.arrayContaining([ + "crabbox.warmup", + "crabbox.inspect", + "credentials.prepare", + "crabbox.remote_run", + "artifacts.copy", + ]), + ); + }); + + it("supports prehydrated remote workspaces without installing or building inside the VM", async () => { + const commands: { args: readonly string[]; command: string }[] = []; + const runner = vi.fn(async (command: string, args: readonly string[]) => { + commands.push({ command, args }); + if (command === "/tmp/crabbox" && args[0] === "inspect") { + return { + stdout: `${JSON.stringify({ + host: "203.0.113.10", + id: "cbx_warm", + provider: "hetzner", + sshKey: "/tmp/key", + sshPort: "2222", + sshUser: "crabbox", + })}\n`, + stderr: "", + }; + } + if (command === "rsync") { + const outputDir = args.at(-1); + await fs.mkdir(outputDir as string, { recursive: true }); + if (!String(outputDir).endsWith("slack-qa/")) { + await fs.writeFile(path.join(outputDir as string, "slack-desktop-smoke.png"), "png"); + await fs.writeFile( + path.join(outputDir as string, "remote-metadata.json"), + `${JSON.stringify({ hydrateMode: "prehydrated", qaExitCode: 0 })}\n`, + ); + } + } + return { stdout: "", stderr: "" }; + }); + + const result = await runMantisSlackDesktopSmoke({ + commandRunner: runner, + crabboxBin: "/tmp/crabbox", + hydrateMode: "prehydrated", + leaseId: "cbx_warm", + outputDir: ".artifacts/qa-e2e/mantis/slack-desktop-prehydrated", + repoRoot, + }); + + expect(result.status).toBe("pass"); + const remoteScript = commands + .find((entry) => entry.command === "/tmp/crabbox" && entry.args[0] === "run") + ?.args.at(-1); + expect(remoteScript).toContain("hydrate_mode='prehydrated'"); + expect(remoteScript).toContain("hydrate-mode=prehydrated requires node_modules"); + expect(remoteScript).toContain("hydrate-mode=prehydrated requires a built dist/ directory"); + const summary = JSON.parse(await fs.readFile(result.summaryPath, "utf8")) as { + hydrateMode: string; + timings: { phases: { name: string; status: string }[] }; + }; + expect(summary.hydrateMode).toBe("prehydrated"); + expect(summary.timings.phases.map((phase) => phase.name)).not.toContain("crabbox.warmup"); }); it("leases Convex Slack credentials for gateway setup and maps them into the VM env", async () => { diff --git a/extensions/qa-lab/src/mantis/slack-desktop-smoke.runtime.ts b/extensions/qa-lab/src/mantis/slack-desktop-smoke.runtime.ts index 690e6f68ef8..6a02854cf47 100644 --- a/extensions/qa-lab/src/mantis/slack-desktop-smoke.runtime.ts +++ b/extensions/qa-lab/src/mantis/slack-desktop-smoke.runtime.ts @@ -17,6 +17,7 @@ export type MantisSlackDesktopSmokeOptions = { env?: NodeJS.ProcessEnv; fastMode?: boolean; gatewaySetup?: boolean; + hydrateMode?: MantisSlackDesktopHydrateMode; idleTimeout?: string; keepLease?: boolean; leaseId?: string; @@ -33,6 +34,8 @@ export type MantisSlackDesktopSmokeOptions = { ttl?: string; }; +export type MantisSlackDesktopHydrateMode = "prehydrated" | "source"; + export type MantisSlackDesktopSmokeResult = { outputDir: string; reportPath: string; @@ -95,17 +98,33 @@ type MantisSlackDesktopSmokeSummary = { }; error?: string; finishedAt: string; + hydrateMode: MantisSlackDesktopHydrateMode; outputDir: string; remoteOutputDir: string; slackUrl?: string; startedAt: string; status: "pass" | "fail"; + timings: MantisPhaseTimings; warning?: string; }; +type MantisPhaseTiming = { + durationMs: number; + finishedAt: string; + name: string; + startedAt: string; + status: "fail" | "pass"; +}; + +type MantisPhaseTimings = { + phases: MantisPhaseTiming[]; + totalMs: number; +}; + type SlackDesktopRemoteMetadata = { gatewayAlive?: boolean; gatewayPid?: string; + hydrateMode?: string; openedUrl?: string; qaExitCode?: number; }; @@ -119,6 +138,7 @@ const DEFAULT_CREDENTIAL_ROLE = "maintainer"; const DEFAULT_PROVIDER_MODE = "live-frontier"; const DEFAULT_MODEL = "openai/gpt-5.4"; const DEFAULT_SLACK_CHANNEL_ID = "C0AUXUC5AGN"; +const DEFAULT_HYDRATE_MODE: MantisSlackDesktopHydrateMode = "source"; const CRABBOX_BIN_ENV = "OPENCLAW_MANTIS_CRABBOX_BIN"; const CRABBOX_PROVIDER_ENV = "OPENCLAW_MANTIS_CRABBOX_PROVIDER"; const CRABBOX_CLASS_ENV = "OPENCLAW_MANTIS_CRABBOX_CLASS"; @@ -126,6 +146,7 @@ const CRABBOX_LEASE_ID_ENV = "OPENCLAW_MANTIS_CRABBOX_LEASE_ID"; const CRABBOX_KEEP_ENV = "OPENCLAW_MANTIS_KEEP_VM"; const CRABBOX_IDLE_TIMEOUT_ENV = "OPENCLAW_MANTIS_CRABBOX_IDLE_TIMEOUT"; const CRABBOX_TTL_ENV = "OPENCLAW_MANTIS_CRABBOX_TTL"; +const HYDRATE_MODE_ENV = "OPENCLAW_MANTIS_HYDRATE_MODE"; const SLACK_URL_ENV = "OPENCLAW_MANTIS_SLACK_URL"; const SLACK_CHANNEL_ID_ENV = "OPENCLAW_MANTIS_SLACK_CHANNEL_ID"; @@ -139,6 +160,52 @@ function isTruthyOptIn(value: string | undefined) { return normalized === "1" || normalized === "true" || normalized === "yes"; } +function normalizeHydrateMode( + value: string | undefined, +): MantisSlackDesktopHydrateMode | undefined { + const normalized = trimToValue(value)?.toLowerCase(); + if (!normalized) { + return undefined; + } + if (normalized === "source" || normalized === "prehydrated") { + return normalized; + } + throw new Error(`Unsupported Mantis Slack desktop hydrate mode: ${value}`); +} + +function createPhaseTimer(startedAt: Date) { + const phases: MantisPhaseTiming[] = []; + const origin = startedAt.getTime(); + function recordPhase(name: string, phaseStarted: Date, status: MantisPhaseTiming["status"]) { + const phaseFinished = new Date(); + phases.push({ + durationMs: phaseFinished.getTime() - phaseStarted.getTime(), + finishedAt: phaseFinished.toISOString(), + name, + startedAt: phaseStarted.toISOString(), + status, + }); + } + async function timePhase(name: string, run: () => Promise): Promise { + const phaseStarted = new Date(); + try { + const result = await run(); + recordPhase(name, phaseStarted, "pass"); + return result; + } catch (error) { + recordPhase(name, phaseStarted, "fail"); + throw error; + } + } + function snapshot(now = new Date()): MantisPhaseTimings { + return { + phases: [...phases], + totalMs: now.getTime() - origin, + }; + } + return { recordPhase, snapshot, timePhase }; +} + function defaultOutputDir(repoRoot: string, startedAt: Date) { const stamp = startedAt.toISOString().replace(/[:.]/gu, "-"); return path.join(repoRoot, ".artifacts", "qa-e2e", "mantis", `slack-desktop-${stamp}`); @@ -208,6 +275,7 @@ async function readRemoteMetadata( gatewayAlive: typeof candidate.gatewayAlive === "boolean" ? candidate.gatewayAlive : undefined, gatewayPid: typeof candidate.gatewayPid === "string" ? candidate.gatewayPid : undefined, + hydrateMode: typeof candidate.hydrateMode === "string" ? candidate.hydrateMode : undefined, openedUrl: typeof candidate.openedUrl === "string" ? candidate.openedUrl : undefined, qaExitCode: typeof candidate.qaExitCode === "number" ? candidate.qaExitCode : undefined, }; @@ -359,6 +427,7 @@ function renderRemoteScript(params: { credentialRole: string; credentialSource: string; fastMode: boolean; + hydrateMode: MantisSlackDesktopHydrateMode; primaryModel: string; providerMode: string; remoteOutputDir: string; @@ -375,6 +444,7 @@ function renderRemoteScript(params: { const primaryModel = shellQuote(params.primaryModel); const alternateModel = shellQuote(params.alternateModel); const fastMode = params.fastMode ? "1" : "0"; + const hydrateMode = shellQuote(params.hydrateMode); const setupGateway = params.setupGateway ? "1" : "0"; const slackChannelId = shellQuote(params.slackChannelId); const scenarioArgs = params.scenarioIds.flatMap((id) => ["--scenario", shellQuote(id)]).join(" "); @@ -387,6 +457,7 @@ provider_mode=${providerMode} primary_model=${primaryModel} alternate_model=${alternateModel} fast_mode=${fastMode} +hydrate_mode=${hydrateMode} setup_gateway=${setupGateway} slack_channel_id=${slackChannelId} rm -rf "$out" @@ -495,18 +566,32 @@ qa_status=0 { set -e echo "remote pwd: $(pwd)" - if ! command -v make >/dev/null 2>&1 || ! command -v python3 >/dev/null 2>&1; then - sudo apt-get update -y >>"$out/apt.log" 2>&1 || true - sudo DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential python3 >>"$out/apt.log" 2>&1 || true - fi sudo corepack enable || sudo npm install -g pnpm@10.33.2 - if [ -d /var/cache/crabbox ]; then - export PNPM_STORE_DIR="\${PNPM_STORE_DIR:-/var/cache/crabbox/pnpm}" - mkdir -p "$PNPM_STORE_DIR" >/dev/null 2>&1 || true - pnpm config set store-dir "$PNPM_STORE_DIR" >/dev/null 2>&1 || true + if [ "$hydrate_mode" = "source" ]; then + if ! command -v make >/dev/null 2>&1 || ! command -v python3 >/dev/null 2>&1; then + sudo apt-get update -y >>"$out/apt.log" 2>&1 || true + sudo DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential python3 >>"$out/apt.log" 2>&1 || true + fi + if [ -d /var/cache/crabbox ]; then + export PNPM_STORE_DIR="\${PNPM_STORE_DIR:-/var/cache/crabbox/pnpm}" + mkdir -p "$PNPM_STORE_DIR" >/dev/null 2>&1 || true + pnpm config set store-dir "$PNPM_STORE_DIR" >/dev/null 2>&1 || true + fi + pnpm install --frozen-lockfile --prefer-offline + pnpm build + elif [ "$hydrate_mode" = "prehydrated" ]; then + test -d node_modules || { + echo "hydrate-mode=prehydrated requires node_modules in the remote workspace." >&2 + exit 3 + } + test -d dist || { + echo "hydrate-mode=prehydrated requires a built dist/ directory in the remote workspace." >&2 + exit 3 + } + else + echo "Unsupported hydrate mode: $hydrate_mode" >&2 + exit 3 fi - pnpm install --frozen-lockfile --prefer-offline - pnpm build if [ "$setup_gateway" = "1" ]; then export OPENCLAW_HOME="$HOME/.openclaw-mantis/slack-openclaw" mkdir -p "$OPENCLAW_HOME" @@ -579,6 +664,7 @@ cat >"$out/remote-metadata.json" < `- ${phase.name}: ${Math.round(phase.durationMs / 100) / 10}s (${phase.status})`, + ), "", "## Artifacts", "", @@ -781,6 +875,7 @@ export async function runMantisSlackDesktopSmoke( ): Promise { const env = buildCrabboxEnv(opts.env ?? process.env); const startedAt = (opts.now ?? (() => new Date()))(); + const timer = createPhaseTimer(startedAt); const repoRoot = path.resolve(opts.repoRoot ?? process.cwd()); const outputDir = await ensureRepoBoundDirectory( repoRoot, @@ -806,6 +901,10 @@ export async function runMantisSlackDesktopSmoke( const primaryModel = trimToValue(opts.primaryModel) ?? DEFAULT_MODEL; const alternateModel = trimToValue(opts.alternateModel) ?? primaryModel; const fastMode = opts.fastMode ?? true; + const hydrateMode = + normalizeHydrateMode(opts.hydrateMode) ?? + normalizeHydrateMode(env[HYDRATE_MODE_ENV]) ?? + DEFAULT_HYDRATE_MODE; const gatewaySetup = opts.gatewaySetup ?? false; const scenarioIds = opts.scenarioIds ?? []; const slackChannelId = @@ -833,33 +932,44 @@ export async function runMantisSlackDesktopSmoke( try { leaseId = leaseId ?? - (await warmupCrabbox({ + (await timer.timePhase("crabbox.warmup", () => + warmupCrabbox({ + crabboxBin, + cwd: repoRoot, + env, + idleTimeout, + machineClass, + provider, + runner, + ttl, + }), + )); + if (!leaseId) { + throw new Error("Crabbox lease id was not resolved."); + } + const resolvedLeaseId = leaseId; + const inspected = await timer.timePhase("crabbox.inspect", () => + inspectCrabbox({ crabboxBin, cwd: repoRoot, env, - idleTimeout, - machineClass, + leaseId: resolvedLeaseId, provider, runner, - ttl, - })); - const inspected = await inspectCrabbox({ - crabboxBin, - cwd: repoRoot, - env, - leaseId, - provider, - runner, - }); - const preparedCredentialEnv = await prepareGatewayCredentialEnv({ - credentialRole, - credentialSource, - env, - gatewaySetup, - }); + }), + ); + const preparedCredentialEnv = await timer.timePhase("credentials.prepare", () => + prepareGatewayCredentialEnv({ + credentialRole, + credentialSource, + env, + gatewaySetup, + }), + ); credentialLease = preparedCredentialEnv.credentialLease; leaseHeartbeat = preparedCredentialEnv.leaseHeartbeat; let remoteRunError: unknown; + const remoteRunStartedAt = new Date(); await runCommand({ command: crabboxBin, args: [ @@ -867,7 +977,7 @@ export async function runMantisSlackDesktopSmoke( "--provider", provider, "--id", - leaseId, + resolvedLeaseId, "--desktop", "--browser", "--shell", @@ -877,6 +987,7 @@ export async function runMantisSlackDesktopSmoke( credentialRole, credentialSource, fastMode, + hydrateMode, primaryModel, providerMode, remoteOutputDir, @@ -890,19 +1001,27 @@ export async function runMantisSlackDesktopSmoke( env, runner, stdio: "inherit", - }).catch((error: unknown) => { - remoteRunError = error; - return { stdout: "", stderr: "" }; - }); + }).then( + () => { + timer.recordPhase("crabbox.remote_run", remoteRunStartedAt, "pass"); + }, + (error: unknown) => { + timer.recordPhase("crabbox.remote_run", remoteRunStartedAt, "fail"); + remoteRunError = error; + return { stdout: "", stderr: "" }; + }, + ); leaseHeartbeat?.throwIfFailed(); - await copyRemoteArtifacts({ - cwd: repoRoot, - env, - inspect: inspected, - outputDir, - remoteOutputDir, - runner, - }); + await timer.timePhase("artifacts.copy", () => + copyRemoteArtifacts({ + cwd: repoRoot, + env, + inspect: inspected, + outputDir, + remoteOutputDir, + runner, + }), + ); screenshotPath = path.join(outputDir, "slack-desktop-smoke.png"); videoPath = path.join(outputDir, "slack-desktop-smoke.mp4"); if (!(await pathExists(videoPath))) { @@ -932,18 +1051,20 @@ export async function runMantisSlackDesktopSmoke( crabbox: { bin: crabboxBin, createdLease, - id: leaseId, + id: resolvedLeaseId, provider, slug: inspected.slug, state: inspected.state, - vncCommand: `${crabboxBin} vnc --provider ${provider} --id ${leaseId} --open`, + vncCommand: `${crabboxBin} vnc --provider ${provider} --id ${resolvedLeaseId} --open`, }, finishedAt: new Date().toISOString(), + hydrateMode: normalizeHydrateMode(remoteMetadata?.hydrateMode) ?? hydrateMode, outputDir, remoteOutputDir, slackUrl: trimToValue(remoteMetadata?.openedUrl) ?? slackUrl, startedAt: startedAt.toISOString(), status: "pass", + timings: timer.snapshot(), }; return { outputDir, @@ -973,11 +1094,13 @@ export async function runMantisSlackDesktopSmoke( }, error: formatErrorMessage(error), finishedAt: new Date().toISOString(), + hydrateMode, outputDir, remoteOutputDir, slackUrl, startedAt: startedAt.toISOString(), status: "fail", + timings: timer.snapshot(), }; await fs.writeFile(path.join(outputDir, "error.txt"), `${summary.error}\n`, "utf8"); return { @@ -991,6 +1114,7 @@ export async function runMantisSlackDesktopSmoke( } finally { if (summary) { summary.finishedAt = new Date().toISOString(); + summary.timings = timer.snapshot(); await fs.writeFile(summaryPath, `${JSON.stringify(summary, null, 2)}\n`, "utf8"); await fs.writeFile(reportPath, renderReport(summary), "utf8"); }