import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import { loadEvidenceManifest, renderEvidenceComment, } from "../../scripts/mantis/publish-pr-evidence.mjs"; const tempDirs: string[] = []; afterEach(() => { for (const dir of tempDirs.splice(0)) { rmSync(dir, { recursive: true, force: true }); } }); function writeFixtureManifest() { const dir = mkdtempSync(path.join(tmpdir(), "mantis-evidence-test-")); tempDirs.push(dir); mkdirSync(path.join(dir, "baseline"), { recursive: true }); mkdirSync(path.join(dir, "candidate"), { recursive: true }); writeFileSync(path.join(dir, "baseline", "timeline.png"), "baseline timeline"); writeFileSync(path.join(dir, "candidate", "timeline.png"), "candidate timeline"); writeFileSync(path.join(dir, "baseline", "change.mp4"), "baseline clip"); const manifestPath = path.join(dir, "mantis-evidence.json"); writeFileSync( manifestPath, JSON.stringify({ schemaVersion: 1, id: "discord-status-reactions", title: "Mantis Discord Status Reactions QA", summary: "Mantis reran the scenario.", scenario: "discord-status-reactions-tool-only", comparison: { baseline: { expected: "queued-only", sha: "aaa", status: "fail", }, candidate: { expected: "queued -> thinking -> done", sha: "bbb", status: "pass", }, pass: true, }, artifacts: [ { alt: "Baseline timeline", kind: "timeline", label: "Baseline queued-only", lane: "baseline", path: "baseline/timeline.png", targetPath: "baseline.png", }, { alt: "Candidate timeline", kind: "timeline", label: "Candidate queued -> thinking -> done", lane: "candidate", path: "candidate/timeline.png", targetPath: "candidate.png", }, { kind: "motionClip", label: "Baseline change MP4", lane: "baseline", path: "baseline/change.mp4", targetPath: "baseline-change.mp4", }, ], }), ); return manifestPath; } describe("scripts/mantis/publish-pr-evidence", () => { it("renders a manifest-driven PR comment with inline screenshots and video links", () => { const manifest = loadEvidenceManifest(writeFixtureManifest()); const body = renderEvidenceComment({ artifactRoot: "mantis/discord/pr-1/run-1", artifactUrl: "https://github.com/openclaw/openclaw/actions/runs/1/artifacts/2", manifest, marker: "", rawBase: "https://raw.githubusercontent.com/openclaw/openclaw/qa-artifacts/mantis/discord/pr-1/run-1", requestSource: "workflow_dispatch", runUrl: "https://github.com/openclaw/openclaw/actions/runs/1", treeUrl: "https://github.com/openclaw/openclaw/tree/qa-artifacts/mantis/discord/pr-1/run-1", }); expect(body).toContain(""); expect(body).toContain("Summary: Mantis reran the scenario."); expect(body).toContain("| Baseline queued-only | Candidate queued -> thinking -> done |"); expect(body).toContain( ' { const dir = mkdtempSync(path.join(tmpdir(), "mantis-evidence-test-")); tempDirs.push(dir); writeFileSync(path.join(dir, "summary.json"), JSON.stringify({ status: "fail" })); writeFileSync(path.join(dir, "report.md"), "bootstrap failed before screenshot"); const manifestPath = path.join(dir, "mantis-evidence.json"); writeFileSync( manifestPath, JSON.stringify({ schemaVersion: 1, id: "slack-desktop-smoke", title: "Mantis Slack Desktop Smoke QA", summary: "Mantis could not finish VM setup.", scenario: "slack-openclaw-desktop-smoke", comparison: { candidate: { expected: "Slack QA and VM gateway setup pass", sha: "bbb", status: "fail", }, pass: false, }, artifacts: [ { alt: "Slack Web desktop screenshot from the Mantis VM", inline: true, kind: "desktopScreenshot", label: "Slack desktop/VNC browser", lane: "candidate", path: "slack-desktop-smoke.png", required: false, targetPath: "slack-desktop.png", }, { kind: "metadata", label: "Slack desktop summary", lane: "run", path: "summary.json", targetPath: "summary.json", }, { kind: "report", label: "Slack desktop report", lane: "run", path: "report.md", targetPath: "report.md", }, ], }), ); const manifest = loadEvidenceManifest(manifestPath); expect(manifest.artifacts.map((artifact) => artifact.targetPath)).toEqual([ "summary.json", "report.md", "mantis-evidence.json", ]); const body = renderEvidenceComment({ artifactRoot: "mantis/slack/pr-1/run-1", artifactUrl: "https://github.com/openclaw/openclaw/actions/runs/1/artifacts/2", manifest, marker: "", rawBase: "https://raw.githubusercontent.com/openclaw/openclaw/qa-artifacts/mantis/slack/pr-1/run-1", requestSource: "workflow_dispatch", runUrl: "https://github.com/openclaw/openclaw/actions/runs/1", treeUrl: "https://github.com/openclaw/openclaw/tree/qa-artifacts/mantis/slack/pr-1/run-1", }); expect(body).toContain("Summary: Mantis could not finish VM setup."); expect(body).toContain("- Overall: `false`"); expect(body).not.toContain(" { const dir = mkdtempSync(path.join(tmpdir(), "mantis-evidence-test-")); tempDirs.push(dir); const manifestPath = path.join(dir, "mantis-evidence.json"); writeFileSync( manifestPath, JSON.stringify({ artifacts: [ { kind: "metadata", path: "../outside.json", }, ], id: "bad", scenario: "bad", schemaVersion: 1, title: "Bad", }), ); expect(() => loadEvidenceManifest(manifestPath)).toThrow(/escapes manifest directory/u); }); });