From ad2d13cc678c01115ed2820380b01aa72699f66f Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Wed, 6 May 2026 01:16:47 +0100 Subject: [PATCH] fix(discord): preserve thread reply file attachments --- CHANGELOG.md | 2 + docs/concepts/mantis.md | 17 + docs/concepts/qa-e2e-automation.md | 7 + .../src/actions/handle-action.guild-admin.ts | 14 +- .../discord/src/actions/handle-action.test.ts | 32 ++ .../discord/discord-live.runtime.test.ts | 20 + .../discord/discord-live.runtime.ts | 348 +++++++++++++++++- .../qa-lab/src/mantis/run.runtime.test.ts | 82 +++++ extensions/qa-lab/src/mantis/run.runtime.ts | 70 +++- 9 files changed, 572 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1f4f95954c2..9294e482160 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,8 @@ Docs: https://docs.openclaw.ai - Gateway/diagnostics: add startup phase spans, active work labels, stale terminal bridge markers, and default sync-I/O tracing in `pnpm gateway:watch` so slow Gateway turns are easier to attribute from logs and stability diagnostics. - Plugins/loader: preserve real compiled plugin module evaluation errors on the native fast path instead of treating every thrown `.js` module as a source-transform fallback miss. Thanks @vincentkoc. - QA/Mantis: add `pnpm openclaw qa mantis slack-desktop-smoke` to run Slack live QA inside a Crabbox VNC desktop, open Slack Web, and capture desktop screenshots beside the Slack QA artifacts. +- QA/Mantis: add an opt-in Discord thread attachment before/after scenario that creates a real thread, calls `message.thread-reply` with `filePath`, and captures baseline/candidate screenshot evidence. +- Discord: preserve `filePath` and `path` attachments when replying to a thread with the message tool. - QA/Mantis: add visual desktop tasks with Crabbox MP4 recording, screenshot capture, and optional image-understanding assertions, and preserve video artifacts in Mantis before/after reports. - QA/WhatsApp: add `pnpm openclaw qa whatsapp` for live DM canary and pairing-gate coverage using two pre-linked WhatsApp Web sessions from the QA credential pool. - QA/Mantis: pass the runtime env through desktop-browser Crabbox and artifact-copy child commands, so embedded Mantis callers can provide Crabbox credentials without mutating the parent process. Thanks @vincentkoc. diff --git a/docs/concepts/mantis.md b/docs/concepts/mantis.md index 6bfd68ca49c..a247c834b36 100644 --- a/docs/concepts/mantis.md +++ b/docs/concepts/mantis.md @@ -89,6 +89,23 @@ directory, installs dependencies, builds each ref, runs the scenario with and `mantis-report.md`. For the first Discord scenario, a successful verification means baseline status is `fail` and candidate status is `pass`. +The second Discord before/after probe targets thread attachments: + +```bash +pnpm openclaw qa mantis run \ + --transport discord \ + --scenario discord-thread-reply-filepath-attachment \ + --baseline \ + --candidate \ + --output-dir .artifacts/qa-e2e/mantis/local-discord-thread-attachment +``` + +That scenario posts a parent message with the driver bot, creates a real Discord +thread, calls OpenClaw's `message.thread-reply` action with a repo-local +`filePath`, then polls the thread for the SUT reply and attachment filename. The +baseline screenshot shows the reply with no attachment; the candidate screenshot +shows the expected `mantis-thread-report.md` attachment. + The first VM/browser primitive is the desktop smoke: ```bash diff --git a/docs/concepts/qa-e2e-automation.md b/docs/concepts/qa-e2e-automation.md index 72706a59ebd..af2e3cf6f82 100644 --- a/docs/concepts/qa-e2e-automation.md +++ b/docs/concepts/qa-e2e-automation.md @@ -113,6 +113,13 @@ The full CLI reference, profile/scenario catalog, env vars, and artifact layout The scenarios cover transport behavior that unit tests cannot prove end to end: mention gating, allow-bot policies, allowlists, top-level and threaded replies, DM routing, reaction handling, inbound edit suppression, restart replay dedupe, homeserver interruption recovery, approval metadata delivery, media handling, and Matrix E2EE bootstrap/recovery/verification flows. The E2EE CLI profile also drives `openclaw matrix encryption setup` and verification commands through the same disposable homeserver before checking gateway replies. +Discord also has Mantis-only opt-in scenarios for bug reproduction. Use +`--scenario discord-status-reactions-tool-only` for the explicit status reaction +timeline, or `--scenario discord-thread-reply-filepath-attachment` to create a +real Discord thread and verify that `message.thread-reply` preserves a +`filePath` attachment. These scenarios stay out of the default live Discord lane +because they are before/after repro probes rather than broad smoke coverage. + CI uses the same command surface in `.github/workflows/qa-live-transports-convex.yml`. Scheduled and default manual runs execute the fast Matrix profile with live frontier credentials, `--fast`, and `OPENCLAW_QA_MATRIX_NO_REPLY_WINDOW_MS=3000`. Manual `matrix_profile=all` fans out into the five profile shards so the exhaustive catalog can run in parallel while keeping one artifact directory per shard. For transport-real Telegram, Discord, and Slack smoke lanes: diff --git a/extensions/discord/src/actions/handle-action.guild-admin.ts b/extensions/discord/src/actions/handle-action.guild-admin.ts index 338bc76ae47..71ad5d1287b 100644 --- a/extensions/discord/src/actions/handle-action.guild-admin.ts +++ b/extensions/discord/src/actions/handle-action.guild-admin.ts @@ -19,7 +19,13 @@ import { type Ctx = Pick< ChannelMessageActionContext, - "action" | "params" | "cfg" | "accountId" | "requesterSenderId" | "mediaLocalRoots" + | "action" + | "params" + | "cfg" + | "accountId" + | "requesterSenderId" + | "mediaLocalRoots" + | "mediaReadFile" >; export async function tryHandleDiscordMessageActionGuildAdmin(params: { @@ -365,7 +371,10 @@ export async function tryHandleDiscordMessageActionGuildAdmin(params: { const content = readStringParam(actionParams, "message", { required: true, }); - const mediaUrl = readStringParam(actionParams, "media", { trim: false }); + const mediaUrl = + readStringParam(actionParams, "media", { trim: false }) ?? + readStringParam(actionParams, "path", { trim: false }) ?? + readStringParam(actionParams, "filePath", { trim: false }); const replyTo = readStringParam(actionParams, "replyTo"); // `message.thread-reply` (tool) uses `threadId`, while the CLI historically used `to`/`channelId`. @@ -383,6 +392,7 @@ export async function tryHandleDiscordMessageActionGuildAdmin(params: { replyTo: replyTo ?? undefined, }, cfg, + { mediaLocalRoots: ctx.mediaLocalRoots, mediaReadFile: ctx.mediaReadFile }, ); } diff --git a/extensions/discord/src/actions/handle-action.test.ts b/extensions/discord/src/actions/handle-action.test.ts index 1798be6bcbe..4847659bb73 100644 --- a/extensions/discord/src/actions/handle-action.test.ts +++ b/extensions/discord/src/actions/handle-action.test.ts @@ -216,6 +216,38 @@ describe("handleDiscordMessageAction", () => { expect(handleDiscordActionMock).not.toHaveBeenCalled(); }); + it("maps thread-reply filePath to Discord threadReply with media read context", async () => { + const mediaReadFile = vi.fn(async () => Buffer.from("report")); + + await handleDiscordMessageAction({ + action: "thread-reply", + params: { + threadId: "thread-123", + message: "thread update", + filePath: "/tmp/agent-root/report.md", + }, + cfg: { + channels: { discord: { token: "tok", actions: { threads: true } } }, + } as OpenClawConfig, + mediaLocalRoots: ["/tmp/agent-root"], + mediaReadFile, + }); + + expect(handleDiscordActionMock).toHaveBeenCalledWith( + expect.objectContaining({ + action: "threadReply", + channelId: "thread-123", + content: "thread update", + mediaUrl: "/tmp/agent-root/report.md", + }), + expect.any(Object), + { + mediaLocalRoots: ["/tmp/agent-root"], + mediaReadFile, + }, + ); + }); + it("forwards top-level components on sends", async () => { const components = { blocks: [{ type: "text", text: "Pick one" }] }; diff --git a/extensions/qa-lab/src/live-transports/discord/discord-live.runtime.test.ts b/extensions/qa-lab/src/live-transports/discord/discord-live.runtime.test.ts index e1ccc2f26f6..6da7d90ddc9 100644 --- a/extensions/qa-lab/src/live-transports/discord/discord-live.runtime.test.ts +++ b/extensions/qa-lab/src/live-transports/discord/discord-live.runtime.test.ts @@ -250,6 +250,11 @@ describe("discord live qa runtime", () => { expect( __testing.findScenario(["discord-status-reactions-tool-only"]).map((scenario) => scenario.id), ).toEqual(["discord-status-reactions-tool-only"]); + expect( + __testing + .findScenario(["discord-thread-reply-filepath-attachment"]) + .map((scenario) => scenario.id), + ).toEqual(["discord-thread-reply-filepath-attachment"]); }); it("collects the status reaction sequence across timeline snapshots", () => { @@ -323,6 +328,21 @@ describe("discord live qa runtime", () => { expect(html).toContain("Seen: 👀 → 🤔"); }); + it("renders a human-readable thread attachment artifact", () => { + const html = __testing.renderDiscordThreadReplyAttachmentHtml({ + attachmentFilenames: [], + expectedAttachmentFilename: "mantis-thread-report.md", + messageContent: "Mantis thread attachment reply", + scenarioTitle: "Discord thread reply preserves filePath attachment", + status: "fail", + threadName: "mantis-thread-filepath-1234", + }); + + expect(html).toContain("Attachment missing"); + expect(html).toContain("No attachments on the SUT thread reply"); + expect(html).toContain("mantis-thread-report.md"); + }); + it("waits for the Discord account to become connected, not just running", async () => { vi.useFakeTimers(); try { diff --git a/extensions/qa-lab/src/live-transports/discord/discord-live.runtime.ts b/extensions/qa-lab/src/live-transports/discord/discord-live.runtime.ts index 3a1a31648b1..4f5b216c7f0 100644 --- a/extensions/qa-lab/src/live-transports/discord/discord-live.runtime.ts +++ b/extensions/qa-lab/src/live-transports/discord/discord-live.runtime.ts @@ -2,7 +2,7 @@ import { randomUUID } from "node:crypto"; import fs from "node:fs/promises"; import path from "node:path"; import { pathToFileURL } from "node:url"; -import { requestDiscord } from "@openclaw/discord/api.js"; +import { handleDiscordMessageAction, requestDiscord } from "@openclaw/discord/api.js"; import { DEFAULT_EMOJIS } from "openclaw/plugin-sdk/channel-feedback"; import type { OpenClawConfig } from "openclaw/plugin-sdk/config-types"; import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime"; @@ -40,6 +40,7 @@ type DiscordQaScenarioId = | "discord-canary" | "discord-mention-gating" | "discord-native-help-command-registration" + | "discord-thread-reply-filepath-attachment" | "discord-status-reactions-tool-only"; type DiscordQaScenarioRun = @@ -58,6 +59,12 @@ type DiscordQaScenarioRun = kind: "status-reactions-tool-only"; expectedSequence: string[]; input: string; + } + | { + kind: "thread-reply-filepath-attachment"; + expectedAttachmentFilename: string; + input: string; + replyContent: string; }; type DiscordQaScenarioDefinition = LiveTransportScenarioDefinition & { @@ -74,6 +81,7 @@ type DiscordMessage = { id: string; channel_id: string; guild_id?: string; + attachments?: DiscordAttachment[]; content?: string; reactions?: DiscordReaction[]; timestamp?: string; @@ -81,6 +89,19 @@ type DiscordMessage = { referenced_message?: { id?: string } | null; }; +type DiscordAttachment = { + id?: string; + filename?: string; + size?: number; + url?: string; +}; + +type DiscordThread = { + id: string; + name?: string; + parent_id?: string; +}; + type DiscordReaction = { count?: number; emoji?: { @@ -192,6 +213,21 @@ type DiscordStatusReactionTimeline = { triggerMessageId: string; }; +type DiscordThreadReplyAttachmentEvidence = { + attachmentFilenames: string[]; + expectedAttachmentFilename: string; + htmlPath?: string; + messageContent?: string; + messageId?: string; + scenarioId: DiscordQaScenarioId; + scenarioTitle: string; + screenshotPath?: string; + screenshotWarning?: string; + status: "pass" | "fail"; + threadId: string; + threadName: string; +}; + const DISCORD_QA_CAPTURE_CONTENT_ENV = "OPENCLAW_QA_DISCORD_CAPTURE_CONTENT"; const QA_REDACT_PUBLIC_METADATA_ENV = "OPENCLAW_QA_REDACT_PUBLIC_METADATA"; const DISCORD_QA_ENV_KEYS = [ @@ -260,10 +296,26 @@ const DISCORD_QA_SCENARIOS: DiscordQaScenarioDefinition[] = [ }; }, }, + { + id: "discord-thread-reply-filepath-attachment", + title: "Discord thread reply preserves filePath attachment", + timeoutMs: 45_000, + buildRun: () => { + const token = `DISCORD_QA_THREAD_FILE_${randomUUID().slice(0, 8).toUpperCase()}`; + return { + kind: "thread-reply-filepath-attachment", + input: `Mantis Discord thread attachment parent ${token}`, + replyContent: `Mantis thread attachment reply ${token}`, + expectedAttachmentFilename: "mantis-thread-report.md", + }; + }, + }, ]; const DISCORD_QA_DEFAULT_SCENARIOS = DISCORD_QA_SCENARIOS.filter( - (scenario) => scenario.id !== "discord-status-reactions-tool-only", + (scenario) => + scenario.id !== "discord-status-reactions-tool-only" && + scenario.id !== "discord-thread-reply-filepath-attachment", ); const DISCORD_QA_STANDARD_SCENARIO_IDS = collectLiveTransportStandardScenarioCoverage({ @@ -461,6 +513,52 @@ async function listChannelMessagesAfter(params: { ); } +async function createThreadFromMessage(params: { + token: string; + channelId: string; + messageId: string; + name: string; +}) { + return await requestDiscord( + `/channels/${params.channelId}/messages/${params.messageId}/threads`, + params.token, + { + body: { + name: params.name, + auto_archive_duration: 60, + }, + timeoutMs: 15_000, + }, + ); +} + +async function archiveDiscordThread(params: { token: string; threadId: string }) { + await requestDiscord(`/channels/${params.threadId}`, params.token, { + body: { + archived: true, + }, + method: "PATCH", + timeoutMs: 15_000, + }); +} + +async function joinDiscordThread(params: { token: string; threadId: string }) { + await requestDiscord(`/channels/${params.threadId}/thread-members/@me`, params.token, { + method: "PUT", + timeoutMs: 15_000, + }); +} + +async function listThreadMessages(params: { token: string; threadId: string }) { + return await requestDiscord( + `/channels/${params.threadId}/messages?limit=50`, + params.token, + { + timeoutMs: 15_000, + }, + ); +} + function reactionEmojiName(reaction: DiscordReaction) { return reaction.emoji?.name?.trim() || reaction.emoji?.id?.trim() || ""; } @@ -590,6 +688,11 @@ async function writeDiscordStatusReactionEvidence(params: { snapshots: params.timeline.snapshots, }); await fs.writeFile(htmlPath, html, { encoding: "utf8", mode: 0o600 }); + const screenshot = await writeHtmlScreenshot({ htmlPath, screenshotPath }); + return { htmlPath, ...screenshot }; +} + +async function writeHtmlScreenshot(params: { htmlPath: string; screenshotPath: string }) { try { const browser = await chromium.launch({ channel: "chrome", @@ -597,20 +700,95 @@ async function writeDiscordStatusReactionEvidence(params: { }); try { const page = await browser.newPage({ viewport: { width: 1104, height: 760 } }); - await page.goto(pathToFileURL(htmlPath).toString(), { + await page.goto(pathToFileURL(params.htmlPath).toString(), { waitUntil: "domcontentloaded", timeout: 15_000, }); - await page.screenshot({ path: screenshotPath, fullPage: true }); - return { htmlPath, screenshotPath }; + await page.screenshot({ path: params.screenshotPath, fullPage: true }); + return { screenshotPath: params.screenshotPath }; } finally { await browser.close(); } } catch (error) { - return { htmlPath, screenshotWarning: formatErrorMessage(error) }; + return { screenshotWarning: formatErrorMessage(error) }; } } +function renderDiscordThreadReplyAttachmentHtml(params: { + attachmentFilenames: readonly string[]; + expectedAttachmentFilename: string; + messageContent?: string; + scenarioTitle: string; + status: "pass" | "fail"; + threadName: string; +}) { + const hasAttachment = params.attachmentFilenames.includes(params.expectedAttachmentFilename); + const attachmentRows = + params.attachmentFilenames.length > 0 + ? params.attachmentFilenames + .map((filename) => `${escapeHtml(filename)}`) + .join("") + : 'No attachments on the SUT thread reply'; + return ` + + + + ${escapeHtml(params.scenarioTitle)} + + + +
+

${escapeHtml(params.scenarioTitle)}

+
Thread: ${escapeHtml(params.threadName)}
+
+
OpenClaw Discord SUT
+
${params.status === "pass" ? "Attachment found" : "Attachment missing"}
+
${escapeHtml(params.messageContent ?? "No SUT reply content captured")}
+
${attachmentRows}
+
Expected attachment: ${escapeHtml(params.expectedAttachmentFilename)}
+
+
+ +`; +} + +async function writeDiscordThreadReplyAttachmentEvidence(params: { + evidence: DiscordThreadReplyAttachmentEvidence; + outputDir: string; +}) { + const htmlPath = path.join(params.outputDir, `${params.evidence.scenarioId}-attachment.html`); + const screenshotPath = path.join( + params.outputDir, + `${params.evidence.scenarioId}-attachment.png`, + ); + const html = renderDiscordThreadReplyAttachmentHtml({ + attachmentFilenames: params.evidence.attachmentFilenames, + expectedAttachmentFilename: params.evidence.expectedAttachmentFilename, + messageContent: params.evidence.messageContent, + scenarioTitle: params.evidence.scenarioTitle, + status: params.evidence.status, + threadName: params.evidence.threadName, + }); + await fs.writeFile(htmlPath, html, { encoding: "utf8", mode: 0o600 }); + const screenshot = await writeHtmlScreenshot({ htmlPath, screenshotPath }); + return { htmlPath, ...screenshot }; +} + async function observeStatusReactionTimeline(params: { channelId: string; expectedSequence: string[]; @@ -730,6 +908,140 @@ async function pollChannelMessages(params: { throw new Error(`timed out after ${params.timeoutMs}ms waiting for Discord message`); } +async function pollThreadReplyMessage(params: { + token: string; + threadId: string; + replyContent: string; + sutBotId: string; + timeoutMs: number; +}) { + const startedAt = Date.now(); + while (Date.now() - startedAt < params.timeoutMs) { + const messages = await listThreadMessages({ + token: params.token, + threadId: params.threadId, + }); + const match = messages.find( + (message) => + message.author?.id === params.sutBotId && + Boolean(message.content?.includes(params.replyContent)), + ); + if (match) { + return match; + } + await new Promise((resolve) => setTimeout(resolve, 1_000)); + } + return undefined; +} + +async function runDiscordThreadReplyFilePathAttachmentScenario(params: { + cfg: OpenClawConfig; + driverBotId: string; + outputDir: string; + runtimeEnv: DiscordQaRuntimeEnv; + scenario: DiscordQaScenarioDefinition; + scenarioRun: Extract; + sutAccountId: string; + sutBotId: string; +}) { + const threadName = `mantis-thread-filepath-${randomUUID().slice(0, 8)}`; + const parent = await sendChannelMessage( + params.runtimeEnv.driverBotToken, + params.runtimeEnv.channelId, + params.scenarioRun.input, + ); + const thread = await createThreadFromMessage({ + token: params.runtimeEnv.driverBotToken, + channelId: params.runtimeEnv.channelId, + messageId: parent.id, + name: threadName, + }); + const attachmentPath = path.join(params.outputDir, params.scenarioRun.expectedAttachmentFilename); + await fs.writeFile( + attachmentPath, + [ + "# Mantis Discord Thread Attachment", + "", + `Parent message: ${parent.id}`, + `Thread: ${thread.id}`, + `Marker: ${params.scenarioRun.replyContent}`, + "", + ].join("\n"), + { encoding: "utf8", mode: 0o600 }, + ); + + try { + await joinDiscordThread({ + token: params.runtimeEnv.sutBotToken, + threadId: thread.id, + }); + await handleDiscordMessageAction({ + action: "thread-reply", + params: { + threadId: thread.id, + message: params.scenarioRun.replyContent, + filePath: attachmentPath, + }, + cfg: params.cfg, + accountId: params.sutAccountId, + requesterSenderId: params.driverBotId, + mediaLocalRoots: [params.outputDir], + mediaReadFile: async (filePath) => await fs.readFile(filePath), + }); + + const reply = await pollThreadReplyMessage({ + token: params.runtimeEnv.driverBotToken, + threadId: thread.id, + replyContent: params.scenarioRun.replyContent, + sutBotId: params.sutBotId, + timeoutMs: params.scenario.timeoutMs, + }); + const attachmentFilenames = (reply?.attachments ?? []) + .map((attachment) => attachment.filename?.trim() ?? "") + .filter(Boolean) + .toSorted(); + const status = attachmentFilenames.includes(params.scenarioRun.expectedAttachmentFilename) + ? "pass" + : "fail"; + const evidence: DiscordThreadReplyAttachmentEvidence = { + attachmentFilenames, + expectedAttachmentFilename: params.scenarioRun.expectedAttachmentFilename, + messageContent: reply?.content, + messageId: reply?.id, + scenarioId: params.scenario.id, + scenarioTitle: params.scenario.title, + status, + threadId: thread.id, + threadName, + }; + const artifactEvidence = await writeDiscordThreadReplyAttachmentEvidence({ + evidence, + outputDir: params.outputDir, + }); + return { + id: params.scenario.id, + title: params.scenario.title, + status, + details: + status === "pass" + ? `thread reply attached ${params.scenarioRun.expectedAttachmentFilename}` + : reply + ? `thread reply omitted ${params.scenarioRun.expectedAttachmentFilename}; saw ${attachmentFilenames.join(", ") || "no attachments"}` + : "thread reply was not observed", + artifactPaths: { + attachmentSource: attachmentPath, + html: artifactEvidence.htmlPath, + ...(artifactEvidence.screenshotPath ? { screenshot: artifactEvidence.screenshotPath } : {}), + }, + } satisfies DiscordQaScenarioResult; + } finally { + await archiveDiscordThread({ + token: params.runtimeEnv.driverBotToken, + threadId: thread.id, + }).catch(() => {}); + } +} + async function waitForDiscordChannelRunning( gateway: Awaited>, accountId: string, @@ -1071,6 +1383,29 @@ export async function runDiscordQaLive(params: { }); continue; } + if (scenarioRun.kind === "thread-reply-filepath-attachment") { + const result = await runDiscordThreadReplyFilePathAttachmentScenario({ + cfg: buildDiscordQaConfig( + {}, + { + guildId: runtimeEnv.guildId, + channelId: runtimeEnv.channelId, + driverBotId: driverIdentity.id, + sutAccountId, + sutBotToken: runtimeEnv.sutBotToken, + }, + ), + driverBotId: driverIdentity.id, + outputDir, + runtimeEnv, + scenario, + scenarioRun, + sutAccountId, + sutBotId: sutIdentity.id, + }); + scenarioResults.push(result); + continue; + } const sent = await sendChannelMessage( runtimeEnv.driverBotToken, runtimeEnv.channelId, @@ -1305,6 +1640,7 @@ export const __testing = { normalizeDiscordObservedMessage, parseDiscordQaCredentialPayload, renderDiscordStatusReactionHtml, + renderDiscordThreadReplyAttachmentHtml, resolveDiscordQaRuntimeEnv, waitForDiscordChannelRunning, }; diff --git a/extensions/qa-lab/src/mantis/run.runtime.test.ts b/extensions/qa-lab/src/mantis/run.runtime.test.ts index 74000d27ddf..75d9f4ae26c 100644 --- a/extensions/qa-lab/src/mantis/run.runtime.test.ts +++ b/extensions/qa-lab/src/mantis/run.runtime.test.ts @@ -103,4 +103,86 @@ describe("mantis before/after runtime", () => { fs.readFile(path.join(result.outputDir, "candidate", "candidate.mp4"), "utf8"), ).resolves.toBe("candidate video"); }); + + it("supports the Discord thread filePath attachment Mantis scenario", async () => { + const runner = vi.fn(async (command: string, args: readonly string[]) => { + if (command !== "pnpm" || !args.includes("openclaw")) { + return; + } + const repoRootArg = args[args.indexOf("--repo-root") + 1]; + const outputDirArg = args[args.indexOf("--output-dir") + 1]; + const lane = outputDirArg.endsWith("baseline") ? "baseline" : "candidate"; + const outputDir = path.join(repoRootArg, outputDirArg); + await fs.mkdir(outputDir, { recursive: true }); + const screenshotPath = path.join(outputDir, `${lane}-thread-attachment.png`); + await fs.writeFile(screenshotPath, `${lane} attachment screenshot`); + await fs.writeFile( + path.join(outputDir, "discord-qa-summary.json"), + `${JSON.stringify( + { + scenarios: [ + { + artifactPaths: { screenshot: screenshotPath }, + details: + lane === "baseline" + ? "thread reply omitted mantis-thread-report.md" + : "thread reply attached mantis-thread-report.md", + id: "discord-thread-reply-filepath-attachment", + status: lane === "baseline" ? "fail" : "pass", + }, + ], + }, + null, + 2, + )}\n`, + ); + }); + + const result = await runMantisBeforeAfter({ + baseline: "bug-sha", + candidate: "fix-sha", + commandRunner: runner, + now: () => new Date("2026-05-03T12:00:00.000Z"), + outputDir: ".artifacts/qa-e2e/mantis/thread-run", + repoRoot, + scenario: "discord-thread-reply-filepath-attachment", + skipBuild: true, + skipInstall: true, + }); + + expect(result.status).toBe("pass"); + const comparison = JSON.parse(await fs.readFile(result.comparisonPath, "utf8")) as { + baseline: { expected: string; reproduced: boolean }; + candidate: { expected: string; fixed: boolean }; + pass: boolean; + }; + expect(comparison).toMatchObject({ + baseline: { + expected: "thread reply omits filePath attachment", + reproduced: true, + }, + candidate: { + expected: "thread reply includes filePath attachment", + fixed: true, + }, + pass: true, + }); + const manifest = JSON.parse(await fs.readFile(result.manifestPath, "utf8")) as { + artifacts: { alt?: string; label: string }[]; + title: string; + }; + expect(manifest.title).toBe("Mantis Discord Thread Attachment QA"); + expect(manifest.artifacts).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + alt: "Baseline Discord thread reply without filePath attachment", + label: "Baseline missing filePath attachment", + }), + expect.objectContaining({ + alt: "Candidate Discord thread reply with filePath attachment", + label: "Candidate includes filePath attachment", + }), + ]), + ); + }); }); diff --git a/extensions/qa-lab/src/mantis/run.runtime.ts b/extensions/qa-lab/src/mantis/run.runtime.ts index 66753f0fb80..01b5422820d 100644 --- a/extensions/qa-lab/src/mantis/run.runtime.ts +++ b/extensions/qa-lab/src/mantis/run.runtime.ts @@ -55,9 +55,21 @@ type LaneResult = { videoPath?: string; }; +type MantisScenarioConfig = { + baselineExpected: string; + baselineLabel: string; + baselineScreenshotAlt: string; + candidateExpected: string; + candidateLabel: string; + candidateScreenshotAlt: string; + defaultBaselineRef: string; + id: string; + title: string; +}; + type Comparison = { baseline: { - expected: "queued-only"; + expected: string; ref: string; reproduced: boolean; screenshotPath?: string; @@ -65,7 +77,7 @@ type Comparison = { videoPath?: string; }; candidate: { - expected: "queued -> thinking -> done"; + expected: string; fixed: boolean; ref: string; screenshotPath?: string; @@ -80,12 +92,38 @@ type Comparison = { const DEFAULT_BASELINE_REF = "0bf06e953fdda290799fc9fb9244a8f67fdae593"; const DEFAULT_CANDIDATE_REF = "HEAD"; const DEFAULT_SCENARIO = "discord-status-reactions-tool-only"; +const DISCORD_THREAD_FILEPATH_ATTACHMENT_SCENARIO = "discord-thread-reply-filepath-attachment"; const DEFAULT_TRANSPORT = "discord"; const DEFAULT_PROVIDER_MODE = "live-frontier"; const DEFAULT_MODEL = "openai/gpt-5.4"; const DEFAULT_CREDENTIAL_SOURCE = "convex"; const DEFAULT_CREDENTIAL_ROLE = "ci"; +const MANTIS_SCENARIO_CONFIGS: Record = { + [DEFAULT_SCENARIO]: { + baselineExpected: "queued-only", + baselineLabel: "Baseline queued-only", + baselineScreenshotAlt: "Baseline Discord status reaction timeline", + candidateExpected: "queued -> thinking -> done", + candidateLabel: "Candidate queued -> thinking -> done", + candidateScreenshotAlt: "Candidate Discord status reaction timeline", + defaultBaselineRef: DEFAULT_BASELINE_REF, + id: DEFAULT_SCENARIO, + title: "Mantis Discord Status Reactions QA", + }, + [DISCORD_THREAD_FILEPATH_ATTACHMENT_SCENARIO]: { + baselineExpected: "thread reply omits filePath attachment", + baselineLabel: "Baseline missing filePath attachment", + baselineScreenshotAlt: "Baseline Discord thread reply without filePath attachment", + candidateExpected: "thread reply includes filePath attachment", + candidateLabel: "Candidate includes filePath attachment", + candidateScreenshotAlt: "Candidate Discord thread reply with filePath attachment", + defaultBaselineRef: "81349cdc2a9d5143fd0991ed858b739e7d96e05c", + id: DISCORD_THREAD_FILEPATH_ATTACHMENT_SCENARIO, + title: "Mantis Discord Thread Attachment QA", + }, +}; + function trimToValue(value: string | undefined) { const trimmed = value?.trim(); return trimmed && trimmed.length > 0 ? trimmed : undefined; @@ -177,9 +215,10 @@ function renderReport(params: { candidate: LaneResult; comparison: Comparison; outputDir: string; + scenarioConfig: MantisScenarioConfig; }) { const lines = [ - "# Mantis Before/After", + `# ${params.scenarioConfig.title}`, "", `Status: ${params.comparison.pass ? "pass" : "fail"}`, `Transport: ${params.comparison.transport}`, @@ -230,6 +269,7 @@ function buildEvidenceManifest(params: { candidate: LaneResult; comparison: Comparison; outputDir: string; + scenarioConfig: MantisScenarioConfig; }) { const artifacts: { alt?: string; @@ -259,9 +299,9 @@ function buildEvidenceManifest(params: { const baselineScreenshot = relativeArtifactPath(params.outputDir, params.baseline.screenshotPath); if (baselineScreenshot) { artifacts.push({ - alt: "Baseline Discord status reaction timeline", + alt: params.scenarioConfig.baselineScreenshotAlt, kind: "timeline", - label: "Baseline queued-only", + label: params.scenarioConfig.baselineLabel, lane: "baseline", path: baselineScreenshot, targetPath: "baseline.png", @@ -274,9 +314,9 @@ function buildEvidenceManifest(params: { ); if (candidateScreenshot) { artifacts.push({ - alt: "Candidate Discord status reaction timeline", + alt: params.scenarioConfig.candidateScreenshotAlt, kind: "timeline", - label: "Candidate queued -> thinking -> done", + label: params.scenarioConfig.candidateLabel, lane: "candidate", path: candidateScreenshot, targetPath: "candidate.png", @@ -314,7 +354,7 @@ function buildEvidenceManifest(params: { schemaVersion: 1, summary: "Mantis ran the before/after scenario, captured baseline and candidate evidence, and compared the expected bug reproduction against the candidate fix.", - title: "Mantis Before/After QA", + title: params.scenarioConfig.title, }; } @@ -452,10 +492,14 @@ export async function runMantisBeforeAfter( const scenario = normalizeRequiredLiteral( opts.scenario, DEFAULT_SCENARIO, - [DEFAULT_SCENARIO], + Object.keys(MANTIS_SCENARIO_CONFIGS), "--scenario", ); - const baseline = trimToValue(opts.baseline) ?? DEFAULT_BASELINE_REF; + const scenarioConfig = MANTIS_SCENARIO_CONFIGS[scenario]; + if (!scenarioConfig) { + throw new Error(`Unsupported Mantis scenario: ${scenario}`); + } + const baseline = trimToValue(opts.baseline) ?? scenarioConfig.defaultBaselineRef; const candidate = trimToValue(opts.candidate) ?? DEFAULT_CANDIDATE_REF; const runner = opts.commandRunner ?? defaultCommandRunner; const worktreeRoot = path.join(outputDir, "worktrees"); @@ -495,7 +539,7 @@ export async function runMantisBeforeAfter( }); const comparison = { baseline: { - expected: "queued-only", + expected: scenarioConfig.baselineExpected, ref: baseline, reproduced: baselineResult.status === "fail", screenshotPath: baselineResult.screenshotPath, @@ -503,7 +547,7 @@ export async function runMantisBeforeAfter( videoPath: baselineResult.videoPath, }, candidate: { - expected: "queued -> thinking -> done", + expected: scenarioConfig.candidateExpected, fixed: candidateResult.status === "pass", ref: candidate, screenshotPath: candidateResult.screenshotPath, @@ -522,6 +566,7 @@ export async function runMantisBeforeAfter( candidate: candidateResult, comparison, outputDir, + scenarioConfig, }), "utf8", ); @@ -533,6 +578,7 @@ export async function runMantisBeforeAfter( candidate: candidateResult, comparison, outputDir, + scenarioConfig, }), null, 2,