fix(discord): preserve thread reply file attachments

This commit is contained in:
Peter Steinberger
2026-05-06 01:16:47 +01:00
parent 6aaf235aee
commit ad2d13cc67
9 changed files with 572 additions and 20 deletions

View File

@@ -38,6 +38,8 @@ Docs: https://docs.openclaw.ai
- Gateway/diagnostics: add startup phase spans, active work labels, stale terminal bridge markers, and default sync-I/O tracing in `pnpm gateway:watch` so slow Gateway turns are easier to attribute from logs and stability diagnostics.
- Plugins/loader: preserve real compiled plugin module evaluation errors on the native fast path instead of treating every thrown `.js` module as a source-transform fallback miss. Thanks @vincentkoc.
- QA/Mantis: add `pnpm openclaw qa mantis slack-desktop-smoke` to run Slack live QA inside a Crabbox VNC desktop, open Slack Web, and capture desktop screenshots beside the Slack QA artifacts.
- QA/Mantis: add an opt-in Discord thread attachment before/after scenario that creates a real thread, calls `message.thread-reply` with `filePath`, and captures baseline/candidate screenshot evidence.
- Discord: preserve `filePath` and `path` attachments when replying to a thread with the message tool.
- QA/Mantis: add visual desktop tasks with Crabbox MP4 recording, screenshot capture, and optional image-understanding assertions, and preserve video artifacts in Mantis before/after reports.
- QA/WhatsApp: add `pnpm openclaw qa whatsapp` for live DM canary and pairing-gate coverage using two pre-linked WhatsApp Web sessions from the QA credential pool.
- QA/Mantis: pass the runtime env through desktop-browser Crabbox and artifact-copy child commands, so embedded Mantis callers can provide Crabbox credentials without mutating the parent process. Thanks @vincentkoc.

View File

@@ -89,6 +89,23 @@ directory, installs dependencies, builds each ref, runs the scenario with
and `mantis-report.md`. For the first Discord scenario, a successful verification
means baseline status is `fail` and candidate status is `pass`.
The second Discord before/after probe targets thread attachments:
```bash
pnpm openclaw qa mantis run \
--transport discord \
--scenario discord-thread-reply-filepath-attachment \
--baseline <bug-ref> \
--candidate <fix-ref> \
--output-dir .artifacts/qa-e2e/mantis/local-discord-thread-attachment
```
That scenario posts a parent message with the driver bot, creates a real Discord
thread, calls OpenClaw's `message.thread-reply` action with a repo-local
`filePath`, then polls the thread for the SUT reply and attachment filename. The
baseline screenshot shows the reply with no attachment; the candidate screenshot
shows the expected `mantis-thread-report.md` attachment.
The first VM/browser primitive is the desktop smoke:
```bash

View File

@@ -113,6 +113,13 @@ The full CLI reference, profile/scenario catalog, env vars, and artifact layout
The scenarios cover transport behavior that unit tests cannot prove end to end: mention gating, allow-bot policies, allowlists, top-level and threaded replies, DM routing, reaction handling, inbound edit suppression, restart replay dedupe, homeserver interruption recovery, approval metadata delivery, media handling, and Matrix E2EE bootstrap/recovery/verification flows. The E2EE CLI profile also drives `openclaw matrix encryption setup` and verification commands through the same disposable homeserver before checking gateway replies.
Discord also has Mantis-only opt-in scenarios for bug reproduction. Use
`--scenario discord-status-reactions-tool-only` for the explicit status reaction
timeline, or `--scenario discord-thread-reply-filepath-attachment` to create a
real Discord thread and verify that `message.thread-reply` preserves a
`filePath` attachment. These scenarios stay out of the default live Discord lane
because they are before/after repro probes rather than broad smoke coverage.
CI uses the same command surface in `.github/workflows/qa-live-transports-convex.yml`. Scheduled and default manual runs execute the fast Matrix profile with live frontier credentials, `--fast`, and `OPENCLAW_QA_MATRIX_NO_REPLY_WINDOW_MS=3000`. Manual `matrix_profile=all` fans out into the five profile shards so the exhaustive catalog can run in parallel while keeping one artifact directory per shard.
For transport-real Telegram, Discord, and Slack smoke lanes:

View File

@@ -19,7 +19,13 @@ import {
type Ctx = Pick<
ChannelMessageActionContext,
"action" | "params" | "cfg" | "accountId" | "requesterSenderId" | "mediaLocalRoots"
| "action"
| "params"
| "cfg"
| "accountId"
| "requesterSenderId"
| "mediaLocalRoots"
| "mediaReadFile"
>;
export async function tryHandleDiscordMessageActionGuildAdmin(params: {
@@ -365,7 +371,10 @@ export async function tryHandleDiscordMessageActionGuildAdmin(params: {
const content = readStringParam(actionParams, "message", {
required: true,
});
const mediaUrl = readStringParam(actionParams, "media", { trim: false });
const mediaUrl =
readStringParam(actionParams, "media", { trim: false }) ??
readStringParam(actionParams, "path", { trim: false }) ??
readStringParam(actionParams, "filePath", { trim: false });
const replyTo = readStringParam(actionParams, "replyTo");
// `message.thread-reply` (tool) uses `threadId`, while the CLI historically used `to`/`channelId`.
@@ -383,6 +392,7 @@ export async function tryHandleDiscordMessageActionGuildAdmin(params: {
replyTo: replyTo ?? undefined,
},
cfg,
{ mediaLocalRoots: ctx.mediaLocalRoots, mediaReadFile: ctx.mediaReadFile },
);
}

View File

@@ -216,6 +216,38 @@ describe("handleDiscordMessageAction", () => {
expect(handleDiscordActionMock).not.toHaveBeenCalled();
});
it("maps thread-reply filePath to Discord threadReply with media read context", async () => {
const mediaReadFile = vi.fn(async () => Buffer.from("report"));
await handleDiscordMessageAction({
action: "thread-reply",
params: {
threadId: "thread-123",
message: "thread update",
filePath: "/tmp/agent-root/report.md",
},
cfg: {
channels: { discord: { token: "tok", actions: { threads: true } } },
} as OpenClawConfig,
mediaLocalRoots: ["/tmp/agent-root"],
mediaReadFile,
});
expect(handleDiscordActionMock).toHaveBeenCalledWith(
expect.objectContaining({
action: "threadReply",
channelId: "thread-123",
content: "thread update",
mediaUrl: "/tmp/agent-root/report.md",
}),
expect.any(Object),
{
mediaLocalRoots: ["/tmp/agent-root"],
mediaReadFile,
},
);
});
it("forwards top-level components on sends", async () => {
const components = { blocks: [{ type: "text", text: "Pick one" }] };

View File

@@ -250,6 +250,11 @@ describe("discord live qa runtime", () => {
expect(
__testing.findScenario(["discord-status-reactions-tool-only"]).map((scenario) => scenario.id),
).toEqual(["discord-status-reactions-tool-only"]);
expect(
__testing
.findScenario(["discord-thread-reply-filepath-attachment"])
.map((scenario) => scenario.id),
).toEqual(["discord-thread-reply-filepath-attachment"]);
});
it("collects the status reaction sequence across timeline snapshots", () => {
@@ -323,6 +328,21 @@ describe("discord live qa runtime", () => {
expect(html).toContain("Seen: 👀 → 🤔");
});
it("renders a human-readable thread attachment artifact", () => {
const html = __testing.renderDiscordThreadReplyAttachmentHtml({
attachmentFilenames: [],
expectedAttachmentFilename: "mantis-thread-report.md",
messageContent: "Mantis thread attachment reply",
scenarioTitle: "Discord thread reply preserves filePath attachment",
status: "fail",
threadName: "mantis-thread-filepath-1234",
});
expect(html).toContain("Attachment missing");
expect(html).toContain("No attachments on the SUT thread reply");
expect(html).toContain("mantis-thread-report.md");
});
it("waits for the Discord account to become connected, not just running", async () => {
vi.useFakeTimers();
try {

View File

@@ -2,7 +2,7 @@ import { randomUUID } from "node:crypto";
import fs from "node:fs/promises";
import path from "node:path";
import { pathToFileURL } from "node:url";
import { requestDiscord } from "@openclaw/discord/api.js";
import { handleDiscordMessageAction, requestDiscord } from "@openclaw/discord/api.js";
import { DEFAULT_EMOJIS } from "openclaw/plugin-sdk/channel-feedback";
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-types";
import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
@@ -40,6 +40,7 @@ type DiscordQaScenarioId =
| "discord-canary"
| "discord-mention-gating"
| "discord-native-help-command-registration"
| "discord-thread-reply-filepath-attachment"
| "discord-status-reactions-tool-only";
type DiscordQaScenarioRun =
@@ -58,6 +59,12 @@ type DiscordQaScenarioRun =
kind: "status-reactions-tool-only";
expectedSequence: string[];
input: string;
}
| {
kind: "thread-reply-filepath-attachment";
expectedAttachmentFilename: string;
input: string;
replyContent: string;
};
type DiscordQaScenarioDefinition = LiveTransportScenarioDefinition<DiscordQaScenarioId> & {
@@ -74,6 +81,7 @@ type DiscordMessage = {
id: string;
channel_id: string;
guild_id?: string;
attachments?: DiscordAttachment[];
content?: string;
reactions?: DiscordReaction[];
timestamp?: string;
@@ -81,6 +89,19 @@ type DiscordMessage = {
referenced_message?: { id?: string } | null;
};
type DiscordAttachment = {
id?: string;
filename?: string;
size?: number;
url?: string;
};
type DiscordThread = {
id: string;
name?: string;
parent_id?: string;
};
type DiscordReaction = {
count?: number;
emoji?: {
@@ -192,6 +213,21 @@ type DiscordStatusReactionTimeline = {
triggerMessageId: string;
};
type DiscordThreadReplyAttachmentEvidence = {
attachmentFilenames: string[];
expectedAttachmentFilename: string;
htmlPath?: string;
messageContent?: string;
messageId?: string;
scenarioId: DiscordQaScenarioId;
scenarioTitle: string;
screenshotPath?: string;
screenshotWarning?: string;
status: "pass" | "fail";
threadId: string;
threadName: string;
};
const DISCORD_QA_CAPTURE_CONTENT_ENV = "OPENCLAW_QA_DISCORD_CAPTURE_CONTENT";
const QA_REDACT_PUBLIC_METADATA_ENV = "OPENCLAW_QA_REDACT_PUBLIC_METADATA";
const DISCORD_QA_ENV_KEYS = [
@@ -260,10 +296,26 @@ const DISCORD_QA_SCENARIOS: DiscordQaScenarioDefinition[] = [
};
},
},
{
id: "discord-thread-reply-filepath-attachment",
title: "Discord thread reply preserves filePath attachment",
timeoutMs: 45_000,
buildRun: () => {
const token = `DISCORD_QA_THREAD_FILE_${randomUUID().slice(0, 8).toUpperCase()}`;
return {
kind: "thread-reply-filepath-attachment",
input: `Mantis Discord thread attachment parent ${token}`,
replyContent: `Mantis thread attachment reply ${token}`,
expectedAttachmentFilename: "mantis-thread-report.md",
};
},
},
];
const DISCORD_QA_DEFAULT_SCENARIOS = DISCORD_QA_SCENARIOS.filter(
(scenario) => scenario.id !== "discord-status-reactions-tool-only",
(scenario) =>
scenario.id !== "discord-status-reactions-tool-only" &&
scenario.id !== "discord-thread-reply-filepath-attachment",
);
const DISCORD_QA_STANDARD_SCENARIO_IDS = collectLiveTransportStandardScenarioCoverage({
@@ -461,6 +513,52 @@ async function listChannelMessagesAfter(params: {
);
}
async function createThreadFromMessage(params: {
token: string;
channelId: string;
messageId: string;
name: string;
}) {
return await requestDiscord<DiscordThread>(
`/channels/${params.channelId}/messages/${params.messageId}/threads`,
params.token,
{
body: {
name: params.name,
auto_archive_duration: 60,
},
timeoutMs: 15_000,
},
);
}
async function archiveDiscordThread(params: { token: string; threadId: string }) {
await requestDiscord<DiscordThread>(`/channels/${params.threadId}`, params.token, {
body: {
archived: true,
},
method: "PATCH",
timeoutMs: 15_000,
});
}
async function joinDiscordThread(params: { token: string; threadId: string }) {
await requestDiscord<void>(`/channels/${params.threadId}/thread-members/@me`, params.token, {
method: "PUT",
timeoutMs: 15_000,
});
}
async function listThreadMessages(params: { token: string; threadId: string }) {
return await requestDiscord<DiscordMessage[]>(
`/channels/${params.threadId}/messages?limit=50`,
params.token,
{
timeoutMs: 15_000,
},
);
}
function reactionEmojiName(reaction: DiscordReaction) {
return reaction.emoji?.name?.trim() || reaction.emoji?.id?.trim() || "";
}
@@ -590,6 +688,11 @@ async function writeDiscordStatusReactionEvidence(params: {
snapshots: params.timeline.snapshots,
});
await fs.writeFile(htmlPath, html, { encoding: "utf8", mode: 0o600 });
const screenshot = await writeHtmlScreenshot({ htmlPath, screenshotPath });
return { htmlPath, ...screenshot };
}
async function writeHtmlScreenshot(params: { htmlPath: string; screenshotPath: string }) {
try {
const browser = await chromium.launch({
channel: "chrome",
@@ -597,20 +700,95 @@ async function writeDiscordStatusReactionEvidence(params: {
});
try {
const page = await browser.newPage({ viewport: { width: 1104, height: 760 } });
await page.goto(pathToFileURL(htmlPath).toString(), {
await page.goto(pathToFileURL(params.htmlPath).toString(), {
waitUntil: "domcontentloaded",
timeout: 15_000,
});
await page.screenshot({ path: screenshotPath, fullPage: true });
return { htmlPath, screenshotPath };
await page.screenshot({ path: params.screenshotPath, fullPage: true });
return { screenshotPath: params.screenshotPath };
} finally {
await browser.close();
}
} catch (error) {
return { htmlPath, screenshotWarning: formatErrorMessage(error) };
return { screenshotWarning: formatErrorMessage(error) };
}
}
function renderDiscordThreadReplyAttachmentHtml(params: {
attachmentFilenames: readonly string[];
expectedAttachmentFilename: string;
messageContent?: string;
scenarioTitle: string;
status: "pass" | "fail";
threadName: string;
}) {
const hasAttachment = params.attachmentFilenames.includes(params.expectedAttachmentFilename);
const attachmentRows =
params.attachmentFilenames.length > 0
? params.attachmentFilenames
.map((filename) => `<span class="attachment">${escapeHtml(filename)}</span>`)
.join("")
: '<span class="missing">No attachments on the SUT thread reply</span>';
return `<!doctype html>
<html>
<head>
<meta charset="utf-8" />
<title>${escapeHtml(params.scenarioTitle)}</title>
<style>
body { margin: 0; background: #313338; color: #f2f3f5; font-family: Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; }
main { width: 1040px; padding: 32px; }
h1 { font-size: 26px; margin: 0 0 8px; font-weight: 700; letter-spacing: 0; }
.sub { color: #b5bac1; margin-bottom: 24px; }
.message { background: #2b2d31; border-left: 4px solid ${hasAttachment ? "#23a55a" : "#da373c"}; padding: 20px; border-radius: 8px; }
.author { color: #f2f3f5; font-weight: 700; margin-bottom: 8px; }
.content { color: #dbdee1; line-height: 1.45; margin-bottom: 16px; }
.badge { display: inline-flex; align-items: center; border-radius: 16px; padding: 6px 10px; font-size: 13px; font-weight: 700; background: ${hasAttachment ? "#1f3b2d" : "#4a2527"}; border: 1px solid ${hasAttachment ? "#2d7d46" : "#a1282e"}; color: #f2f3f5; margin-bottom: 18px; }
.attachments { display: flex; flex-wrap: wrap; gap: 10px; margin-top: 10px; }
.attachment { display: inline-flex; align-items: center; gap: 8px; border: 1px solid #5865f2; background: #202136; color: #cfd4ff; border-radius: 6px; padding: 10px 12px; font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; }
.attachment::before { content: "file"; color: #b5bac1; font-family: Inter, ui-sans-serif, system-ui, sans-serif; font-size: 12px; text-transform: uppercase; }
.missing { color: #ffb4b4; border: 1px solid #a1282e; background: #3a2023; border-radius: 6px; padding: 10px 12px; }
.expected { color: #b5bac1; margin-top: 18px; font-size: 14px; }
code { color: #f2f3f5; background: #1e1f22; border-radius: 4px; padding: 2px 5px; }
</style>
</head>
<body>
<main>
<h1>${escapeHtml(params.scenarioTitle)}</h1>
<div class="sub">Thread: ${escapeHtml(params.threadName)}</div>
<section class="message">
<div class="author">OpenClaw Discord SUT</div>
<div class="badge">${params.status === "pass" ? "Attachment found" : "Attachment missing"}</div>
<div class="content">${escapeHtml(params.messageContent ?? "No SUT reply content captured")}</div>
<div class="attachments">${attachmentRows}</div>
<div class="expected">Expected attachment: <code>${escapeHtml(params.expectedAttachmentFilename)}</code></div>
</section>
</main>
</body>
</html>`;
}
async function writeDiscordThreadReplyAttachmentEvidence(params: {
evidence: DiscordThreadReplyAttachmentEvidence;
outputDir: string;
}) {
const htmlPath = path.join(params.outputDir, `${params.evidence.scenarioId}-attachment.html`);
const screenshotPath = path.join(
params.outputDir,
`${params.evidence.scenarioId}-attachment.png`,
);
const html = renderDiscordThreadReplyAttachmentHtml({
attachmentFilenames: params.evidence.attachmentFilenames,
expectedAttachmentFilename: params.evidence.expectedAttachmentFilename,
messageContent: params.evidence.messageContent,
scenarioTitle: params.evidence.scenarioTitle,
status: params.evidence.status,
threadName: params.evidence.threadName,
});
await fs.writeFile(htmlPath, html, { encoding: "utf8", mode: 0o600 });
const screenshot = await writeHtmlScreenshot({ htmlPath, screenshotPath });
return { htmlPath, ...screenshot };
}
async function observeStatusReactionTimeline(params: {
channelId: string;
expectedSequence: string[];
@@ -730,6 +908,140 @@ async function pollChannelMessages(params: {
throw new Error(`timed out after ${params.timeoutMs}ms waiting for Discord message`);
}
async function pollThreadReplyMessage(params: {
token: string;
threadId: string;
replyContent: string;
sutBotId: string;
timeoutMs: number;
}) {
const startedAt = Date.now();
while (Date.now() - startedAt < params.timeoutMs) {
const messages = await listThreadMessages({
token: params.token,
threadId: params.threadId,
});
const match = messages.find(
(message) =>
message.author?.id === params.sutBotId &&
Boolean(message.content?.includes(params.replyContent)),
);
if (match) {
return match;
}
await new Promise((resolve) => setTimeout(resolve, 1_000));
}
return undefined;
}
async function runDiscordThreadReplyFilePathAttachmentScenario(params: {
cfg: OpenClawConfig;
driverBotId: string;
outputDir: string;
runtimeEnv: DiscordQaRuntimeEnv;
scenario: DiscordQaScenarioDefinition;
scenarioRun: Extract<DiscordQaScenarioRun, { kind: "thread-reply-filepath-attachment" }>;
sutAccountId: string;
sutBotId: string;
}) {
const threadName = `mantis-thread-filepath-${randomUUID().slice(0, 8)}`;
const parent = await sendChannelMessage(
params.runtimeEnv.driverBotToken,
params.runtimeEnv.channelId,
params.scenarioRun.input,
);
const thread = await createThreadFromMessage({
token: params.runtimeEnv.driverBotToken,
channelId: params.runtimeEnv.channelId,
messageId: parent.id,
name: threadName,
});
const attachmentPath = path.join(params.outputDir, params.scenarioRun.expectedAttachmentFilename);
await fs.writeFile(
attachmentPath,
[
"# Mantis Discord Thread Attachment",
"",
`Parent message: ${parent.id}`,
`Thread: ${thread.id}`,
`Marker: ${params.scenarioRun.replyContent}`,
"",
].join("\n"),
{ encoding: "utf8", mode: 0o600 },
);
try {
await joinDiscordThread({
token: params.runtimeEnv.sutBotToken,
threadId: thread.id,
});
await handleDiscordMessageAction({
action: "thread-reply",
params: {
threadId: thread.id,
message: params.scenarioRun.replyContent,
filePath: attachmentPath,
},
cfg: params.cfg,
accountId: params.sutAccountId,
requesterSenderId: params.driverBotId,
mediaLocalRoots: [params.outputDir],
mediaReadFile: async (filePath) => await fs.readFile(filePath),
});
const reply = await pollThreadReplyMessage({
token: params.runtimeEnv.driverBotToken,
threadId: thread.id,
replyContent: params.scenarioRun.replyContent,
sutBotId: params.sutBotId,
timeoutMs: params.scenario.timeoutMs,
});
const attachmentFilenames = (reply?.attachments ?? [])
.map((attachment) => attachment.filename?.trim() ?? "")
.filter(Boolean)
.toSorted();
const status = attachmentFilenames.includes(params.scenarioRun.expectedAttachmentFilename)
? "pass"
: "fail";
const evidence: DiscordThreadReplyAttachmentEvidence = {
attachmentFilenames,
expectedAttachmentFilename: params.scenarioRun.expectedAttachmentFilename,
messageContent: reply?.content,
messageId: reply?.id,
scenarioId: params.scenario.id,
scenarioTitle: params.scenario.title,
status,
threadId: thread.id,
threadName,
};
const artifactEvidence = await writeDiscordThreadReplyAttachmentEvidence({
evidence,
outputDir: params.outputDir,
});
return {
id: params.scenario.id,
title: params.scenario.title,
status,
details:
status === "pass"
? `thread reply attached ${params.scenarioRun.expectedAttachmentFilename}`
: reply
? `thread reply omitted ${params.scenarioRun.expectedAttachmentFilename}; saw ${attachmentFilenames.join(", ") || "no attachments"}`
: "thread reply was not observed",
artifactPaths: {
attachmentSource: attachmentPath,
html: artifactEvidence.htmlPath,
...(artifactEvidence.screenshotPath ? { screenshot: artifactEvidence.screenshotPath } : {}),
},
} satisfies DiscordQaScenarioResult;
} finally {
await archiveDiscordThread({
token: params.runtimeEnv.driverBotToken,
threadId: thread.id,
}).catch(() => {});
}
}
async function waitForDiscordChannelRunning(
gateway: Awaited<ReturnType<typeof startQaGatewayChild>>,
accountId: string,
@@ -1071,6 +1383,29 @@ export async function runDiscordQaLive(params: {
});
continue;
}
if (scenarioRun.kind === "thread-reply-filepath-attachment") {
const result = await runDiscordThreadReplyFilePathAttachmentScenario({
cfg: buildDiscordQaConfig(
{},
{
guildId: runtimeEnv.guildId,
channelId: runtimeEnv.channelId,
driverBotId: driverIdentity.id,
sutAccountId,
sutBotToken: runtimeEnv.sutBotToken,
},
),
driverBotId: driverIdentity.id,
outputDir,
runtimeEnv,
scenario,
scenarioRun,
sutAccountId,
sutBotId: sutIdentity.id,
});
scenarioResults.push(result);
continue;
}
const sent = await sendChannelMessage(
runtimeEnv.driverBotToken,
runtimeEnv.channelId,
@@ -1305,6 +1640,7 @@ export const __testing = {
normalizeDiscordObservedMessage,
parseDiscordQaCredentialPayload,
renderDiscordStatusReactionHtml,
renderDiscordThreadReplyAttachmentHtml,
resolveDiscordQaRuntimeEnv,
waitForDiscordChannelRunning,
};

View File

@@ -103,4 +103,86 @@ describe("mantis before/after runtime", () => {
fs.readFile(path.join(result.outputDir, "candidate", "candidate.mp4"), "utf8"),
).resolves.toBe("candidate video");
});
it("supports the Discord thread filePath attachment Mantis scenario", async () => {
const runner = vi.fn(async (command: string, args: readonly string[]) => {
if (command !== "pnpm" || !args.includes("openclaw")) {
return;
}
const repoRootArg = args[args.indexOf("--repo-root") + 1];
const outputDirArg = args[args.indexOf("--output-dir") + 1];
const lane = outputDirArg.endsWith("baseline") ? "baseline" : "candidate";
const outputDir = path.join(repoRootArg, outputDirArg);
await fs.mkdir(outputDir, { recursive: true });
const screenshotPath = path.join(outputDir, `${lane}-thread-attachment.png`);
await fs.writeFile(screenshotPath, `${lane} attachment screenshot`);
await fs.writeFile(
path.join(outputDir, "discord-qa-summary.json"),
`${JSON.stringify(
{
scenarios: [
{
artifactPaths: { screenshot: screenshotPath },
details:
lane === "baseline"
? "thread reply omitted mantis-thread-report.md"
: "thread reply attached mantis-thread-report.md",
id: "discord-thread-reply-filepath-attachment",
status: lane === "baseline" ? "fail" : "pass",
},
],
},
null,
2,
)}\n`,
);
});
const result = await runMantisBeforeAfter({
baseline: "bug-sha",
candidate: "fix-sha",
commandRunner: runner,
now: () => new Date("2026-05-03T12:00:00.000Z"),
outputDir: ".artifacts/qa-e2e/mantis/thread-run",
repoRoot,
scenario: "discord-thread-reply-filepath-attachment",
skipBuild: true,
skipInstall: true,
});
expect(result.status).toBe("pass");
const comparison = JSON.parse(await fs.readFile(result.comparisonPath, "utf8")) as {
baseline: { expected: string; reproduced: boolean };
candidate: { expected: string; fixed: boolean };
pass: boolean;
};
expect(comparison).toMatchObject({
baseline: {
expected: "thread reply omits filePath attachment",
reproduced: true,
},
candidate: {
expected: "thread reply includes filePath attachment",
fixed: true,
},
pass: true,
});
const manifest = JSON.parse(await fs.readFile(result.manifestPath, "utf8")) as {
artifacts: { alt?: string; label: string }[];
title: string;
};
expect(manifest.title).toBe("Mantis Discord Thread Attachment QA");
expect(manifest.artifacts).toEqual(
expect.arrayContaining([
expect.objectContaining({
alt: "Baseline Discord thread reply without filePath attachment",
label: "Baseline missing filePath attachment",
}),
expect.objectContaining({
alt: "Candidate Discord thread reply with filePath attachment",
label: "Candidate includes filePath attachment",
}),
]),
);
});
});

View File

@@ -55,9 +55,21 @@ type LaneResult = {
videoPath?: string;
};
type MantisScenarioConfig = {
baselineExpected: string;
baselineLabel: string;
baselineScreenshotAlt: string;
candidateExpected: string;
candidateLabel: string;
candidateScreenshotAlt: string;
defaultBaselineRef: string;
id: string;
title: string;
};
type Comparison = {
baseline: {
expected: "queued-only";
expected: string;
ref: string;
reproduced: boolean;
screenshotPath?: string;
@@ -65,7 +77,7 @@ type Comparison = {
videoPath?: string;
};
candidate: {
expected: "queued -> thinking -> done";
expected: string;
fixed: boolean;
ref: string;
screenshotPath?: string;
@@ -80,12 +92,38 @@ type Comparison = {
const DEFAULT_BASELINE_REF = "0bf06e953fdda290799fc9fb9244a8f67fdae593";
const DEFAULT_CANDIDATE_REF = "HEAD";
const DEFAULT_SCENARIO = "discord-status-reactions-tool-only";
const DISCORD_THREAD_FILEPATH_ATTACHMENT_SCENARIO = "discord-thread-reply-filepath-attachment";
const DEFAULT_TRANSPORT = "discord";
const DEFAULT_PROVIDER_MODE = "live-frontier";
const DEFAULT_MODEL = "openai/gpt-5.4";
const DEFAULT_CREDENTIAL_SOURCE = "convex";
const DEFAULT_CREDENTIAL_ROLE = "ci";
const MANTIS_SCENARIO_CONFIGS: Record<string, MantisScenarioConfig> = {
[DEFAULT_SCENARIO]: {
baselineExpected: "queued-only",
baselineLabel: "Baseline queued-only",
baselineScreenshotAlt: "Baseline Discord status reaction timeline",
candidateExpected: "queued -> thinking -> done",
candidateLabel: "Candidate queued -> thinking -> done",
candidateScreenshotAlt: "Candidate Discord status reaction timeline",
defaultBaselineRef: DEFAULT_BASELINE_REF,
id: DEFAULT_SCENARIO,
title: "Mantis Discord Status Reactions QA",
},
[DISCORD_THREAD_FILEPATH_ATTACHMENT_SCENARIO]: {
baselineExpected: "thread reply omits filePath attachment",
baselineLabel: "Baseline missing filePath attachment",
baselineScreenshotAlt: "Baseline Discord thread reply without filePath attachment",
candidateExpected: "thread reply includes filePath attachment",
candidateLabel: "Candidate includes filePath attachment",
candidateScreenshotAlt: "Candidate Discord thread reply with filePath attachment",
defaultBaselineRef: "81349cdc2a9d5143fd0991ed858b739e7d96e05c",
id: DISCORD_THREAD_FILEPATH_ATTACHMENT_SCENARIO,
title: "Mantis Discord Thread Attachment QA",
},
};
function trimToValue(value: string | undefined) {
const trimmed = value?.trim();
return trimmed && trimmed.length > 0 ? trimmed : undefined;
@@ -177,9 +215,10 @@ function renderReport(params: {
candidate: LaneResult;
comparison: Comparison;
outputDir: string;
scenarioConfig: MantisScenarioConfig;
}) {
const lines = [
"# Mantis Before/After",
`# ${params.scenarioConfig.title}`,
"",
`Status: ${params.comparison.pass ? "pass" : "fail"}`,
`Transport: ${params.comparison.transport}`,
@@ -230,6 +269,7 @@ function buildEvidenceManifest(params: {
candidate: LaneResult;
comparison: Comparison;
outputDir: string;
scenarioConfig: MantisScenarioConfig;
}) {
const artifacts: {
alt?: string;
@@ -259,9 +299,9 @@ function buildEvidenceManifest(params: {
const baselineScreenshot = relativeArtifactPath(params.outputDir, params.baseline.screenshotPath);
if (baselineScreenshot) {
artifacts.push({
alt: "Baseline Discord status reaction timeline",
alt: params.scenarioConfig.baselineScreenshotAlt,
kind: "timeline",
label: "Baseline queued-only",
label: params.scenarioConfig.baselineLabel,
lane: "baseline",
path: baselineScreenshot,
targetPath: "baseline.png",
@@ -274,9 +314,9 @@ function buildEvidenceManifest(params: {
);
if (candidateScreenshot) {
artifacts.push({
alt: "Candidate Discord status reaction timeline",
alt: params.scenarioConfig.candidateScreenshotAlt,
kind: "timeline",
label: "Candidate queued -> thinking -> done",
label: params.scenarioConfig.candidateLabel,
lane: "candidate",
path: candidateScreenshot,
targetPath: "candidate.png",
@@ -314,7 +354,7 @@ function buildEvidenceManifest(params: {
schemaVersion: 1,
summary:
"Mantis ran the before/after scenario, captured baseline and candidate evidence, and compared the expected bug reproduction against the candidate fix.",
title: "Mantis Before/After QA",
title: params.scenarioConfig.title,
};
}
@@ -452,10 +492,14 @@ export async function runMantisBeforeAfter(
const scenario = normalizeRequiredLiteral(
opts.scenario,
DEFAULT_SCENARIO,
[DEFAULT_SCENARIO],
Object.keys(MANTIS_SCENARIO_CONFIGS),
"--scenario",
);
const baseline = trimToValue(opts.baseline) ?? DEFAULT_BASELINE_REF;
const scenarioConfig = MANTIS_SCENARIO_CONFIGS[scenario];
if (!scenarioConfig) {
throw new Error(`Unsupported Mantis scenario: ${scenario}`);
}
const baseline = trimToValue(opts.baseline) ?? scenarioConfig.defaultBaselineRef;
const candidate = trimToValue(opts.candidate) ?? DEFAULT_CANDIDATE_REF;
const runner = opts.commandRunner ?? defaultCommandRunner;
const worktreeRoot = path.join(outputDir, "worktrees");
@@ -495,7 +539,7 @@ export async function runMantisBeforeAfter(
});
const comparison = {
baseline: {
expected: "queued-only",
expected: scenarioConfig.baselineExpected,
ref: baseline,
reproduced: baselineResult.status === "fail",
screenshotPath: baselineResult.screenshotPath,
@@ -503,7 +547,7 @@ export async function runMantisBeforeAfter(
videoPath: baselineResult.videoPath,
},
candidate: {
expected: "queued -> thinking -> done",
expected: scenarioConfig.candidateExpected,
fixed: candidateResult.status === "pass",
ref: candidate,
screenshotPath: candidateResult.screenshotPath,
@@ -522,6 +566,7 @@ export async function runMantisBeforeAfter(
candidate: candidateResult,
comparison,
outputDir,
scenarioConfig,
}),
"utf8",
);
@@ -533,6 +578,7 @@ export async function runMantisBeforeAfter(
candidate: candidateResult,
comparison,
outputDir,
scenarioConfig,
}),
null,
2,