mirror of
https://github.com/openclaw/openclaw.git
synced 2026-05-06 05:30:42 +00:00
fix(discord): preserve thread reply file attachments
This commit is contained in:
@@ -38,6 +38,8 @@ Docs: https://docs.openclaw.ai
|
||||
- Gateway/diagnostics: add startup phase spans, active work labels, stale terminal bridge markers, and default sync-I/O tracing in `pnpm gateway:watch` so slow Gateway turns are easier to attribute from logs and stability diagnostics.
|
||||
- Plugins/loader: preserve real compiled plugin module evaluation errors on the native fast path instead of treating every thrown `.js` module as a source-transform fallback miss. Thanks @vincentkoc.
|
||||
- QA/Mantis: add `pnpm openclaw qa mantis slack-desktop-smoke` to run Slack live QA inside a Crabbox VNC desktop, open Slack Web, and capture desktop screenshots beside the Slack QA artifacts.
|
||||
- QA/Mantis: add an opt-in Discord thread attachment before/after scenario that creates a real thread, calls `message.thread-reply` with `filePath`, and captures baseline/candidate screenshot evidence.
|
||||
- Discord: preserve `filePath` and `path` attachments when replying to a thread with the message tool.
|
||||
- QA/Mantis: add visual desktop tasks with Crabbox MP4 recording, screenshot capture, and optional image-understanding assertions, and preserve video artifacts in Mantis before/after reports.
|
||||
- QA/WhatsApp: add `pnpm openclaw qa whatsapp` for live DM canary and pairing-gate coverage using two pre-linked WhatsApp Web sessions from the QA credential pool.
|
||||
- QA/Mantis: pass the runtime env through desktop-browser Crabbox and artifact-copy child commands, so embedded Mantis callers can provide Crabbox credentials without mutating the parent process. Thanks @vincentkoc.
|
||||
|
||||
@@ -89,6 +89,23 @@ directory, installs dependencies, builds each ref, runs the scenario with
|
||||
and `mantis-report.md`. For the first Discord scenario, a successful verification
|
||||
means baseline status is `fail` and candidate status is `pass`.
|
||||
|
||||
The second Discord before/after probe targets thread attachments:
|
||||
|
||||
```bash
|
||||
pnpm openclaw qa mantis run \
|
||||
--transport discord \
|
||||
--scenario discord-thread-reply-filepath-attachment \
|
||||
--baseline <bug-ref> \
|
||||
--candidate <fix-ref> \
|
||||
--output-dir .artifacts/qa-e2e/mantis/local-discord-thread-attachment
|
||||
```
|
||||
|
||||
That scenario posts a parent message with the driver bot, creates a real Discord
|
||||
thread, calls OpenClaw's `message.thread-reply` action with a repo-local
|
||||
`filePath`, then polls the thread for the SUT reply and attachment filename. The
|
||||
baseline screenshot shows the reply with no attachment; the candidate screenshot
|
||||
shows the expected `mantis-thread-report.md` attachment.
|
||||
|
||||
The first VM/browser primitive is the desktop smoke:
|
||||
|
||||
```bash
|
||||
|
||||
@@ -113,6 +113,13 @@ The full CLI reference, profile/scenario catalog, env vars, and artifact layout
|
||||
|
||||
The scenarios cover transport behavior that unit tests cannot prove end to end: mention gating, allow-bot policies, allowlists, top-level and threaded replies, DM routing, reaction handling, inbound edit suppression, restart replay dedupe, homeserver interruption recovery, approval metadata delivery, media handling, and Matrix E2EE bootstrap/recovery/verification flows. The E2EE CLI profile also drives `openclaw matrix encryption setup` and verification commands through the same disposable homeserver before checking gateway replies.
|
||||
|
||||
Discord also has Mantis-only opt-in scenarios for bug reproduction. Use
|
||||
`--scenario discord-status-reactions-tool-only` for the explicit status reaction
|
||||
timeline, or `--scenario discord-thread-reply-filepath-attachment` to create a
|
||||
real Discord thread and verify that `message.thread-reply` preserves a
|
||||
`filePath` attachment. These scenarios stay out of the default live Discord lane
|
||||
because they are before/after repro probes rather than broad smoke coverage.
|
||||
|
||||
CI uses the same command surface in `.github/workflows/qa-live-transports-convex.yml`. Scheduled and default manual runs execute the fast Matrix profile with live frontier credentials, `--fast`, and `OPENCLAW_QA_MATRIX_NO_REPLY_WINDOW_MS=3000`. Manual `matrix_profile=all` fans out into the five profile shards so the exhaustive catalog can run in parallel while keeping one artifact directory per shard.
|
||||
|
||||
For transport-real Telegram, Discord, and Slack smoke lanes:
|
||||
|
||||
@@ -19,7 +19,13 @@ import {
|
||||
|
||||
type Ctx = Pick<
|
||||
ChannelMessageActionContext,
|
||||
"action" | "params" | "cfg" | "accountId" | "requesterSenderId" | "mediaLocalRoots"
|
||||
| "action"
|
||||
| "params"
|
||||
| "cfg"
|
||||
| "accountId"
|
||||
| "requesterSenderId"
|
||||
| "mediaLocalRoots"
|
||||
| "mediaReadFile"
|
||||
>;
|
||||
|
||||
export async function tryHandleDiscordMessageActionGuildAdmin(params: {
|
||||
@@ -365,7 +371,10 @@ export async function tryHandleDiscordMessageActionGuildAdmin(params: {
|
||||
const content = readStringParam(actionParams, "message", {
|
||||
required: true,
|
||||
});
|
||||
const mediaUrl = readStringParam(actionParams, "media", { trim: false });
|
||||
const mediaUrl =
|
||||
readStringParam(actionParams, "media", { trim: false }) ??
|
||||
readStringParam(actionParams, "path", { trim: false }) ??
|
||||
readStringParam(actionParams, "filePath", { trim: false });
|
||||
const replyTo = readStringParam(actionParams, "replyTo");
|
||||
|
||||
// `message.thread-reply` (tool) uses `threadId`, while the CLI historically used `to`/`channelId`.
|
||||
@@ -383,6 +392,7 @@ export async function tryHandleDiscordMessageActionGuildAdmin(params: {
|
||||
replyTo: replyTo ?? undefined,
|
||||
},
|
||||
cfg,
|
||||
{ mediaLocalRoots: ctx.mediaLocalRoots, mediaReadFile: ctx.mediaReadFile },
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -216,6 +216,38 @@ describe("handleDiscordMessageAction", () => {
|
||||
expect(handleDiscordActionMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("maps thread-reply filePath to Discord threadReply with media read context", async () => {
|
||||
const mediaReadFile = vi.fn(async () => Buffer.from("report"));
|
||||
|
||||
await handleDiscordMessageAction({
|
||||
action: "thread-reply",
|
||||
params: {
|
||||
threadId: "thread-123",
|
||||
message: "thread update",
|
||||
filePath: "/tmp/agent-root/report.md",
|
||||
},
|
||||
cfg: {
|
||||
channels: { discord: { token: "tok", actions: { threads: true } } },
|
||||
} as OpenClawConfig,
|
||||
mediaLocalRoots: ["/tmp/agent-root"],
|
||||
mediaReadFile,
|
||||
});
|
||||
|
||||
expect(handleDiscordActionMock).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
action: "threadReply",
|
||||
channelId: "thread-123",
|
||||
content: "thread update",
|
||||
mediaUrl: "/tmp/agent-root/report.md",
|
||||
}),
|
||||
expect.any(Object),
|
||||
{
|
||||
mediaLocalRoots: ["/tmp/agent-root"],
|
||||
mediaReadFile,
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it("forwards top-level components on sends", async () => {
|
||||
const components = { blocks: [{ type: "text", text: "Pick one" }] };
|
||||
|
||||
|
||||
@@ -250,6 +250,11 @@ describe("discord live qa runtime", () => {
|
||||
expect(
|
||||
__testing.findScenario(["discord-status-reactions-tool-only"]).map((scenario) => scenario.id),
|
||||
).toEqual(["discord-status-reactions-tool-only"]);
|
||||
expect(
|
||||
__testing
|
||||
.findScenario(["discord-thread-reply-filepath-attachment"])
|
||||
.map((scenario) => scenario.id),
|
||||
).toEqual(["discord-thread-reply-filepath-attachment"]);
|
||||
});
|
||||
|
||||
it("collects the status reaction sequence across timeline snapshots", () => {
|
||||
@@ -323,6 +328,21 @@ describe("discord live qa runtime", () => {
|
||||
expect(html).toContain("Seen: 👀 → 🤔");
|
||||
});
|
||||
|
||||
it("renders a human-readable thread attachment artifact", () => {
|
||||
const html = __testing.renderDiscordThreadReplyAttachmentHtml({
|
||||
attachmentFilenames: [],
|
||||
expectedAttachmentFilename: "mantis-thread-report.md",
|
||||
messageContent: "Mantis thread attachment reply",
|
||||
scenarioTitle: "Discord thread reply preserves filePath attachment",
|
||||
status: "fail",
|
||||
threadName: "mantis-thread-filepath-1234",
|
||||
});
|
||||
|
||||
expect(html).toContain("Attachment missing");
|
||||
expect(html).toContain("No attachments on the SUT thread reply");
|
||||
expect(html).toContain("mantis-thread-report.md");
|
||||
});
|
||||
|
||||
it("waits for the Discord account to become connected, not just running", async () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
|
||||
@@ -2,7 +2,7 @@ import { randomUUID } from "node:crypto";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { requestDiscord } from "@openclaw/discord/api.js";
|
||||
import { handleDiscordMessageAction, requestDiscord } from "@openclaw/discord/api.js";
|
||||
import { DEFAULT_EMOJIS } from "openclaw/plugin-sdk/channel-feedback";
|
||||
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-types";
|
||||
import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
|
||||
@@ -40,6 +40,7 @@ type DiscordQaScenarioId =
|
||||
| "discord-canary"
|
||||
| "discord-mention-gating"
|
||||
| "discord-native-help-command-registration"
|
||||
| "discord-thread-reply-filepath-attachment"
|
||||
| "discord-status-reactions-tool-only";
|
||||
|
||||
type DiscordQaScenarioRun =
|
||||
@@ -58,6 +59,12 @@ type DiscordQaScenarioRun =
|
||||
kind: "status-reactions-tool-only";
|
||||
expectedSequence: string[];
|
||||
input: string;
|
||||
}
|
||||
| {
|
||||
kind: "thread-reply-filepath-attachment";
|
||||
expectedAttachmentFilename: string;
|
||||
input: string;
|
||||
replyContent: string;
|
||||
};
|
||||
|
||||
type DiscordQaScenarioDefinition = LiveTransportScenarioDefinition<DiscordQaScenarioId> & {
|
||||
@@ -74,6 +81,7 @@ type DiscordMessage = {
|
||||
id: string;
|
||||
channel_id: string;
|
||||
guild_id?: string;
|
||||
attachments?: DiscordAttachment[];
|
||||
content?: string;
|
||||
reactions?: DiscordReaction[];
|
||||
timestamp?: string;
|
||||
@@ -81,6 +89,19 @@ type DiscordMessage = {
|
||||
referenced_message?: { id?: string } | null;
|
||||
};
|
||||
|
||||
type DiscordAttachment = {
|
||||
id?: string;
|
||||
filename?: string;
|
||||
size?: number;
|
||||
url?: string;
|
||||
};
|
||||
|
||||
type DiscordThread = {
|
||||
id: string;
|
||||
name?: string;
|
||||
parent_id?: string;
|
||||
};
|
||||
|
||||
type DiscordReaction = {
|
||||
count?: number;
|
||||
emoji?: {
|
||||
@@ -192,6 +213,21 @@ type DiscordStatusReactionTimeline = {
|
||||
triggerMessageId: string;
|
||||
};
|
||||
|
||||
type DiscordThreadReplyAttachmentEvidence = {
|
||||
attachmentFilenames: string[];
|
||||
expectedAttachmentFilename: string;
|
||||
htmlPath?: string;
|
||||
messageContent?: string;
|
||||
messageId?: string;
|
||||
scenarioId: DiscordQaScenarioId;
|
||||
scenarioTitle: string;
|
||||
screenshotPath?: string;
|
||||
screenshotWarning?: string;
|
||||
status: "pass" | "fail";
|
||||
threadId: string;
|
||||
threadName: string;
|
||||
};
|
||||
|
||||
const DISCORD_QA_CAPTURE_CONTENT_ENV = "OPENCLAW_QA_DISCORD_CAPTURE_CONTENT";
|
||||
const QA_REDACT_PUBLIC_METADATA_ENV = "OPENCLAW_QA_REDACT_PUBLIC_METADATA";
|
||||
const DISCORD_QA_ENV_KEYS = [
|
||||
@@ -260,10 +296,26 @@ const DISCORD_QA_SCENARIOS: DiscordQaScenarioDefinition[] = [
|
||||
};
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "discord-thread-reply-filepath-attachment",
|
||||
title: "Discord thread reply preserves filePath attachment",
|
||||
timeoutMs: 45_000,
|
||||
buildRun: () => {
|
||||
const token = `DISCORD_QA_THREAD_FILE_${randomUUID().slice(0, 8).toUpperCase()}`;
|
||||
return {
|
||||
kind: "thread-reply-filepath-attachment",
|
||||
input: `Mantis Discord thread attachment parent ${token}`,
|
||||
replyContent: `Mantis thread attachment reply ${token}`,
|
||||
expectedAttachmentFilename: "mantis-thread-report.md",
|
||||
};
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
const DISCORD_QA_DEFAULT_SCENARIOS = DISCORD_QA_SCENARIOS.filter(
|
||||
(scenario) => scenario.id !== "discord-status-reactions-tool-only",
|
||||
(scenario) =>
|
||||
scenario.id !== "discord-status-reactions-tool-only" &&
|
||||
scenario.id !== "discord-thread-reply-filepath-attachment",
|
||||
);
|
||||
|
||||
const DISCORD_QA_STANDARD_SCENARIO_IDS = collectLiveTransportStandardScenarioCoverage({
|
||||
@@ -461,6 +513,52 @@ async function listChannelMessagesAfter(params: {
|
||||
);
|
||||
}
|
||||
|
||||
async function createThreadFromMessage(params: {
|
||||
token: string;
|
||||
channelId: string;
|
||||
messageId: string;
|
||||
name: string;
|
||||
}) {
|
||||
return await requestDiscord<DiscordThread>(
|
||||
`/channels/${params.channelId}/messages/${params.messageId}/threads`,
|
||||
params.token,
|
||||
{
|
||||
body: {
|
||||
name: params.name,
|
||||
auto_archive_duration: 60,
|
||||
},
|
||||
timeoutMs: 15_000,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
async function archiveDiscordThread(params: { token: string; threadId: string }) {
|
||||
await requestDiscord<DiscordThread>(`/channels/${params.threadId}`, params.token, {
|
||||
body: {
|
||||
archived: true,
|
||||
},
|
||||
method: "PATCH",
|
||||
timeoutMs: 15_000,
|
||||
});
|
||||
}
|
||||
|
||||
async function joinDiscordThread(params: { token: string; threadId: string }) {
|
||||
await requestDiscord<void>(`/channels/${params.threadId}/thread-members/@me`, params.token, {
|
||||
method: "PUT",
|
||||
timeoutMs: 15_000,
|
||||
});
|
||||
}
|
||||
|
||||
async function listThreadMessages(params: { token: string; threadId: string }) {
|
||||
return await requestDiscord<DiscordMessage[]>(
|
||||
`/channels/${params.threadId}/messages?limit=50`,
|
||||
params.token,
|
||||
{
|
||||
timeoutMs: 15_000,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
function reactionEmojiName(reaction: DiscordReaction) {
|
||||
return reaction.emoji?.name?.trim() || reaction.emoji?.id?.trim() || "";
|
||||
}
|
||||
@@ -590,6 +688,11 @@ async function writeDiscordStatusReactionEvidence(params: {
|
||||
snapshots: params.timeline.snapshots,
|
||||
});
|
||||
await fs.writeFile(htmlPath, html, { encoding: "utf8", mode: 0o600 });
|
||||
const screenshot = await writeHtmlScreenshot({ htmlPath, screenshotPath });
|
||||
return { htmlPath, ...screenshot };
|
||||
}
|
||||
|
||||
async function writeHtmlScreenshot(params: { htmlPath: string; screenshotPath: string }) {
|
||||
try {
|
||||
const browser = await chromium.launch({
|
||||
channel: "chrome",
|
||||
@@ -597,20 +700,95 @@ async function writeDiscordStatusReactionEvidence(params: {
|
||||
});
|
||||
try {
|
||||
const page = await browser.newPage({ viewport: { width: 1104, height: 760 } });
|
||||
await page.goto(pathToFileURL(htmlPath).toString(), {
|
||||
await page.goto(pathToFileURL(params.htmlPath).toString(), {
|
||||
waitUntil: "domcontentloaded",
|
||||
timeout: 15_000,
|
||||
});
|
||||
await page.screenshot({ path: screenshotPath, fullPage: true });
|
||||
return { htmlPath, screenshotPath };
|
||||
await page.screenshot({ path: params.screenshotPath, fullPage: true });
|
||||
return { screenshotPath: params.screenshotPath };
|
||||
} finally {
|
||||
await browser.close();
|
||||
}
|
||||
} catch (error) {
|
||||
return { htmlPath, screenshotWarning: formatErrorMessage(error) };
|
||||
return { screenshotWarning: formatErrorMessage(error) };
|
||||
}
|
||||
}
|
||||
|
||||
function renderDiscordThreadReplyAttachmentHtml(params: {
|
||||
attachmentFilenames: readonly string[];
|
||||
expectedAttachmentFilename: string;
|
||||
messageContent?: string;
|
||||
scenarioTitle: string;
|
||||
status: "pass" | "fail";
|
||||
threadName: string;
|
||||
}) {
|
||||
const hasAttachment = params.attachmentFilenames.includes(params.expectedAttachmentFilename);
|
||||
const attachmentRows =
|
||||
params.attachmentFilenames.length > 0
|
||||
? params.attachmentFilenames
|
||||
.map((filename) => `<span class="attachment">${escapeHtml(filename)}</span>`)
|
||||
.join("")
|
||||
: '<span class="missing">No attachments on the SUT thread reply</span>';
|
||||
return `<!doctype html>
|
||||
<html>
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<title>${escapeHtml(params.scenarioTitle)}</title>
|
||||
<style>
|
||||
body { margin: 0; background: #313338; color: #f2f3f5; font-family: Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; }
|
||||
main { width: 1040px; padding: 32px; }
|
||||
h1 { font-size: 26px; margin: 0 0 8px; font-weight: 700; letter-spacing: 0; }
|
||||
.sub { color: #b5bac1; margin-bottom: 24px; }
|
||||
.message { background: #2b2d31; border-left: 4px solid ${hasAttachment ? "#23a55a" : "#da373c"}; padding: 20px; border-radius: 8px; }
|
||||
.author { color: #f2f3f5; font-weight: 700; margin-bottom: 8px; }
|
||||
.content { color: #dbdee1; line-height: 1.45; margin-bottom: 16px; }
|
||||
.badge { display: inline-flex; align-items: center; border-radius: 16px; padding: 6px 10px; font-size: 13px; font-weight: 700; background: ${hasAttachment ? "#1f3b2d" : "#4a2527"}; border: 1px solid ${hasAttachment ? "#2d7d46" : "#a1282e"}; color: #f2f3f5; margin-bottom: 18px; }
|
||||
.attachments { display: flex; flex-wrap: wrap; gap: 10px; margin-top: 10px; }
|
||||
.attachment { display: inline-flex; align-items: center; gap: 8px; border: 1px solid #5865f2; background: #202136; color: #cfd4ff; border-radius: 6px; padding: 10px 12px; font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; }
|
||||
.attachment::before { content: "file"; color: #b5bac1; font-family: Inter, ui-sans-serif, system-ui, sans-serif; font-size: 12px; text-transform: uppercase; }
|
||||
.missing { color: #ffb4b4; border: 1px solid #a1282e; background: #3a2023; border-radius: 6px; padding: 10px 12px; }
|
||||
.expected { color: #b5bac1; margin-top: 18px; font-size: 14px; }
|
||||
code { color: #f2f3f5; background: #1e1f22; border-radius: 4px; padding: 2px 5px; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<main>
|
||||
<h1>${escapeHtml(params.scenarioTitle)}</h1>
|
||||
<div class="sub">Thread: ${escapeHtml(params.threadName)}</div>
|
||||
<section class="message">
|
||||
<div class="author">OpenClaw Discord SUT</div>
|
||||
<div class="badge">${params.status === "pass" ? "Attachment found" : "Attachment missing"}</div>
|
||||
<div class="content">${escapeHtml(params.messageContent ?? "No SUT reply content captured")}</div>
|
||||
<div class="attachments">${attachmentRows}</div>
|
||||
<div class="expected">Expected attachment: <code>${escapeHtml(params.expectedAttachmentFilename)}</code></div>
|
||||
</section>
|
||||
</main>
|
||||
</body>
|
||||
</html>`;
|
||||
}
|
||||
|
||||
async function writeDiscordThreadReplyAttachmentEvidence(params: {
|
||||
evidence: DiscordThreadReplyAttachmentEvidence;
|
||||
outputDir: string;
|
||||
}) {
|
||||
const htmlPath = path.join(params.outputDir, `${params.evidence.scenarioId}-attachment.html`);
|
||||
const screenshotPath = path.join(
|
||||
params.outputDir,
|
||||
`${params.evidence.scenarioId}-attachment.png`,
|
||||
);
|
||||
const html = renderDiscordThreadReplyAttachmentHtml({
|
||||
attachmentFilenames: params.evidence.attachmentFilenames,
|
||||
expectedAttachmentFilename: params.evidence.expectedAttachmentFilename,
|
||||
messageContent: params.evidence.messageContent,
|
||||
scenarioTitle: params.evidence.scenarioTitle,
|
||||
status: params.evidence.status,
|
||||
threadName: params.evidence.threadName,
|
||||
});
|
||||
await fs.writeFile(htmlPath, html, { encoding: "utf8", mode: 0o600 });
|
||||
const screenshot = await writeHtmlScreenshot({ htmlPath, screenshotPath });
|
||||
return { htmlPath, ...screenshot };
|
||||
}
|
||||
|
||||
async function observeStatusReactionTimeline(params: {
|
||||
channelId: string;
|
||||
expectedSequence: string[];
|
||||
@@ -730,6 +908,140 @@ async function pollChannelMessages(params: {
|
||||
throw new Error(`timed out after ${params.timeoutMs}ms waiting for Discord message`);
|
||||
}
|
||||
|
||||
async function pollThreadReplyMessage(params: {
|
||||
token: string;
|
||||
threadId: string;
|
||||
replyContent: string;
|
||||
sutBotId: string;
|
||||
timeoutMs: number;
|
||||
}) {
|
||||
const startedAt = Date.now();
|
||||
while (Date.now() - startedAt < params.timeoutMs) {
|
||||
const messages = await listThreadMessages({
|
||||
token: params.token,
|
||||
threadId: params.threadId,
|
||||
});
|
||||
const match = messages.find(
|
||||
(message) =>
|
||||
message.author?.id === params.sutBotId &&
|
||||
Boolean(message.content?.includes(params.replyContent)),
|
||||
);
|
||||
if (match) {
|
||||
return match;
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 1_000));
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
async function runDiscordThreadReplyFilePathAttachmentScenario(params: {
|
||||
cfg: OpenClawConfig;
|
||||
driverBotId: string;
|
||||
outputDir: string;
|
||||
runtimeEnv: DiscordQaRuntimeEnv;
|
||||
scenario: DiscordQaScenarioDefinition;
|
||||
scenarioRun: Extract<DiscordQaScenarioRun, { kind: "thread-reply-filepath-attachment" }>;
|
||||
sutAccountId: string;
|
||||
sutBotId: string;
|
||||
}) {
|
||||
const threadName = `mantis-thread-filepath-${randomUUID().slice(0, 8)}`;
|
||||
const parent = await sendChannelMessage(
|
||||
params.runtimeEnv.driverBotToken,
|
||||
params.runtimeEnv.channelId,
|
||||
params.scenarioRun.input,
|
||||
);
|
||||
const thread = await createThreadFromMessage({
|
||||
token: params.runtimeEnv.driverBotToken,
|
||||
channelId: params.runtimeEnv.channelId,
|
||||
messageId: parent.id,
|
||||
name: threadName,
|
||||
});
|
||||
const attachmentPath = path.join(params.outputDir, params.scenarioRun.expectedAttachmentFilename);
|
||||
await fs.writeFile(
|
||||
attachmentPath,
|
||||
[
|
||||
"# Mantis Discord Thread Attachment",
|
||||
"",
|
||||
`Parent message: ${parent.id}`,
|
||||
`Thread: ${thread.id}`,
|
||||
`Marker: ${params.scenarioRun.replyContent}`,
|
||||
"",
|
||||
].join("\n"),
|
||||
{ encoding: "utf8", mode: 0o600 },
|
||||
);
|
||||
|
||||
try {
|
||||
await joinDiscordThread({
|
||||
token: params.runtimeEnv.sutBotToken,
|
||||
threadId: thread.id,
|
||||
});
|
||||
await handleDiscordMessageAction({
|
||||
action: "thread-reply",
|
||||
params: {
|
||||
threadId: thread.id,
|
||||
message: params.scenarioRun.replyContent,
|
||||
filePath: attachmentPath,
|
||||
},
|
||||
cfg: params.cfg,
|
||||
accountId: params.sutAccountId,
|
||||
requesterSenderId: params.driverBotId,
|
||||
mediaLocalRoots: [params.outputDir],
|
||||
mediaReadFile: async (filePath) => await fs.readFile(filePath),
|
||||
});
|
||||
|
||||
const reply = await pollThreadReplyMessage({
|
||||
token: params.runtimeEnv.driverBotToken,
|
||||
threadId: thread.id,
|
||||
replyContent: params.scenarioRun.replyContent,
|
||||
sutBotId: params.sutBotId,
|
||||
timeoutMs: params.scenario.timeoutMs,
|
||||
});
|
||||
const attachmentFilenames = (reply?.attachments ?? [])
|
||||
.map((attachment) => attachment.filename?.trim() ?? "")
|
||||
.filter(Boolean)
|
||||
.toSorted();
|
||||
const status = attachmentFilenames.includes(params.scenarioRun.expectedAttachmentFilename)
|
||||
? "pass"
|
||||
: "fail";
|
||||
const evidence: DiscordThreadReplyAttachmentEvidence = {
|
||||
attachmentFilenames,
|
||||
expectedAttachmentFilename: params.scenarioRun.expectedAttachmentFilename,
|
||||
messageContent: reply?.content,
|
||||
messageId: reply?.id,
|
||||
scenarioId: params.scenario.id,
|
||||
scenarioTitle: params.scenario.title,
|
||||
status,
|
||||
threadId: thread.id,
|
||||
threadName,
|
||||
};
|
||||
const artifactEvidence = await writeDiscordThreadReplyAttachmentEvidence({
|
||||
evidence,
|
||||
outputDir: params.outputDir,
|
||||
});
|
||||
return {
|
||||
id: params.scenario.id,
|
||||
title: params.scenario.title,
|
||||
status,
|
||||
details:
|
||||
status === "pass"
|
||||
? `thread reply attached ${params.scenarioRun.expectedAttachmentFilename}`
|
||||
: reply
|
||||
? `thread reply omitted ${params.scenarioRun.expectedAttachmentFilename}; saw ${attachmentFilenames.join(", ") || "no attachments"}`
|
||||
: "thread reply was not observed",
|
||||
artifactPaths: {
|
||||
attachmentSource: attachmentPath,
|
||||
html: artifactEvidence.htmlPath,
|
||||
...(artifactEvidence.screenshotPath ? { screenshot: artifactEvidence.screenshotPath } : {}),
|
||||
},
|
||||
} satisfies DiscordQaScenarioResult;
|
||||
} finally {
|
||||
await archiveDiscordThread({
|
||||
token: params.runtimeEnv.driverBotToken,
|
||||
threadId: thread.id,
|
||||
}).catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
async function waitForDiscordChannelRunning(
|
||||
gateway: Awaited<ReturnType<typeof startQaGatewayChild>>,
|
||||
accountId: string,
|
||||
@@ -1071,6 +1383,29 @@ export async function runDiscordQaLive(params: {
|
||||
});
|
||||
continue;
|
||||
}
|
||||
if (scenarioRun.kind === "thread-reply-filepath-attachment") {
|
||||
const result = await runDiscordThreadReplyFilePathAttachmentScenario({
|
||||
cfg: buildDiscordQaConfig(
|
||||
{},
|
||||
{
|
||||
guildId: runtimeEnv.guildId,
|
||||
channelId: runtimeEnv.channelId,
|
||||
driverBotId: driverIdentity.id,
|
||||
sutAccountId,
|
||||
sutBotToken: runtimeEnv.sutBotToken,
|
||||
},
|
||||
),
|
||||
driverBotId: driverIdentity.id,
|
||||
outputDir,
|
||||
runtimeEnv,
|
||||
scenario,
|
||||
scenarioRun,
|
||||
sutAccountId,
|
||||
sutBotId: sutIdentity.id,
|
||||
});
|
||||
scenarioResults.push(result);
|
||||
continue;
|
||||
}
|
||||
const sent = await sendChannelMessage(
|
||||
runtimeEnv.driverBotToken,
|
||||
runtimeEnv.channelId,
|
||||
@@ -1305,6 +1640,7 @@ export const __testing = {
|
||||
normalizeDiscordObservedMessage,
|
||||
parseDiscordQaCredentialPayload,
|
||||
renderDiscordStatusReactionHtml,
|
||||
renderDiscordThreadReplyAttachmentHtml,
|
||||
resolveDiscordQaRuntimeEnv,
|
||||
waitForDiscordChannelRunning,
|
||||
};
|
||||
|
||||
@@ -103,4 +103,86 @@ describe("mantis before/after runtime", () => {
|
||||
fs.readFile(path.join(result.outputDir, "candidate", "candidate.mp4"), "utf8"),
|
||||
).resolves.toBe("candidate video");
|
||||
});
|
||||
|
||||
it("supports the Discord thread filePath attachment Mantis scenario", async () => {
|
||||
const runner = vi.fn(async (command: string, args: readonly string[]) => {
|
||||
if (command !== "pnpm" || !args.includes("openclaw")) {
|
||||
return;
|
||||
}
|
||||
const repoRootArg = args[args.indexOf("--repo-root") + 1];
|
||||
const outputDirArg = args[args.indexOf("--output-dir") + 1];
|
||||
const lane = outputDirArg.endsWith("baseline") ? "baseline" : "candidate";
|
||||
const outputDir = path.join(repoRootArg, outputDirArg);
|
||||
await fs.mkdir(outputDir, { recursive: true });
|
||||
const screenshotPath = path.join(outputDir, `${lane}-thread-attachment.png`);
|
||||
await fs.writeFile(screenshotPath, `${lane} attachment screenshot`);
|
||||
await fs.writeFile(
|
||||
path.join(outputDir, "discord-qa-summary.json"),
|
||||
`${JSON.stringify(
|
||||
{
|
||||
scenarios: [
|
||||
{
|
||||
artifactPaths: { screenshot: screenshotPath },
|
||||
details:
|
||||
lane === "baseline"
|
||||
? "thread reply omitted mantis-thread-report.md"
|
||||
: "thread reply attached mantis-thread-report.md",
|
||||
id: "discord-thread-reply-filepath-attachment",
|
||||
status: lane === "baseline" ? "fail" : "pass",
|
||||
},
|
||||
],
|
||||
},
|
||||
null,
|
||||
2,
|
||||
)}\n`,
|
||||
);
|
||||
});
|
||||
|
||||
const result = await runMantisBeforeAfter({
|
||||
baseline: "bug-sha",
|
||||
candidate: "fix-sha",
|
||||
commandRunner: runner,
|
||||
now: () => new Date("2026-05-03T12:00:00.000Z"),
|
||||
outputDir: ".artifacts/qa-e2e/mantis/thread-run",
|
||||
repoRoot,
|
||||
scenario: "discord-thread-reply-filepath-attachment",
|
||||
skipBuild: true,
|
||||
skipInstall: true,
|
||||
});
|
||||
|
||||
expect(result.status).toBe("pass");
|
||||
const comparison = JSON.parse(await fs.readFile(result.comparisonPath, "utf8")) as {
|
||||
baseline: { expected: string; reproduced: boolean };
|
||||
candidate: { expected: string; fixed: boolean };
|
||||
pass: boolean;
|
||||
};
|
||||
expect(comparison).toMatchObject({
|
||||
baseline: {
|
||||
expected: "thread reply omits filePath attachment",
|
||||
reproduced: true,
|
||||
},
|
||||
candidate: {
|
||||
expected: "thread reply includes filePath attachment",
|
||||
fixed: true,
|
||||
},
|
||||
pass: true,
|
||||
});
|
||||
const manifest = JSON.parse(await fs.readFile(result.manifestPath, "utf8")) as {
|
||||
artifacts: { alt?: string; label: string }[];
|
||||
title: string;
|
||||
};
|
||||
expect(manifest.title).toBe("Mantis Discord Thread Attachment QA");
|
||||
expect(manifest.artifacts).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
alt: "Baseline Discord thread reply without filePath attachment",
|
||||
label: "Baseline missing filePath attachment",
|
||||
}),
|
||||
expect.objectContaining({
|
||||
alt: "Candidate Discord thread reply with filePath attachment",
|
||||
label: "Candidate includes filePath attachment",
|
||||
}),
|
||||
]),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -55,9 +55,21 @@ type LaneResult = {
|
||||
videoPath?: string;
|
||||
};
|
||||
|
||||
type MantisScenarioConfig = {
|
||||
baselineExpected: string;
|
||||
baselineLabel: string;
|
||||
baselineScreenshotAlt: string;
|
||||
candidateExpected: string;
|
||||
candidateLabel: string;
|
||||
candidateScreenshotAlt: string;
|
||||
defaultBaselineRef: string;
|
||||
id: string;
|
||||
title: string;
|
||||
};
|
||||
|
||||
type Comparison = {
|
||||
baseline: {
|
||||
expected: "queued-only";
|
||||
expected: string;
|
||||
ref: string;
|
||||
reproduced: boolean;
|
||||
screenshotPath?: string;
|
||||
@@ -65,7 +77,7 @@ type Comparison = {
|
||||
videoPath?: string;
|
||||
};
|
||||
candidate: {
|
||||
expected: "queued -> thinking -> done";
|
||||
expected: string;
|
||||
fixed: boolean;
|
||||
ref: string;
|
||||
screenshotPath?: string;
|
||||
@@ -80,12 +92,38 @@ type Comparison = {
|
||||
const DEFAULT_BASELINE_REF = "0bf06e953fdda290799fc9fb9244a8f67fdae593";
|
||||
const DEFAULT_CANDIDATE_REF = "HEAD";
|
||||
const DEFAULT_SCENARIO = "discord-status-reactions-tool-only";
|
||||
const DISCORD_THREAD_FILEPATH_ATTACHMENT_SCENARIO = "discord-thread-reply-filepath-attachment";
|
||||
const DEFAULT_TRANSPORT = "discord";
|
||||
const DEFAULT_PROVIDER_MODE = "live-frontier";
|
||||
const DEFAULT_MODEL = "openai/gpt-5.4";
|
||||
const DEFAULT_CREDENTIAL_SOURCE = "convex";
|
||||
const DEFAULT_CREDENTIAL_ROLE = "ci";
|
||||
|
||||
const MANTIS_SCENARIO_CONFIGS: Record<string, MantisScenarioConfig> = {
|
||||
[DEFAULT_SCENARIO]: {
|
||||
baselineExpected: "queued-only",
|
||||
baselineLabel: "Baseline queued-only",
|
||||
baselineScreenshotAlt: "Baseline Discord status reaction timeline",
|
||||
candidateExpected: "queued -> thinking -> done",
|
||||
candidateLabel: "Candidate queued -> thinking -> done",
|
||||
candidateScreenshotAlt: "Candidate Discord status reaction timeline",
|
||||
defaultBaselineRef: DEFAULT_BASELINE_REF,
|
||||
id: DEFAULT_SCENARIO,
|
||||
title: "Mantis Discord Status Reactions QA",
|
||||
},
|
||||
[DISCORD_THREAD_FILEPATH_ATTACHMENT_SCENARIO]: {
|
||||
baselineExpected: "thread reply omits filePath attachment",
|
||||
baselineLabel: "Baseline missing filePath attachment",
|
||||
baselineScreenshotAlt: "Baseline Discord thread reply without filePath attachment",
|
||||
candidateExpected: "thread reply includes filePath attachment",
|
||||
candidateLabel: "Candidate includes filePath attachment",
|
||||
candidateScreenshotAlt: "Candidate Discord thread reply with filePath attachment",
|
||||
defaultBaselineRef: "81349cdc2a9d5143fd0991ed858b739e7d96e05c",
|
||||
id: DISCORD_THREAD_FILEPATH_ATTACHMENT_SCENARIO,
|
||||
title: "Mantis Discord Thread Attachment QA",
|
||||
},
|
||||
};
|
||||
|
||||
function trimToValue(value: string | undefined) {
|
||||
const trimmed = value?.trim();
|
||||
return trimmed && trimmed.length > 0 ? trimmed : undefined;
|
||||
@@ -177,9 +215,10 @@ function renderReport(params: {
|
||||
candidate: LaneResult;
|
||||
comparison: Comparison;
|
||||
outputDir: string;
|
||||
scenarioConfig: MantisScenarioConfig;
|
||||
}) {
|
||||
const lines = [
|
||||
"# Mantis Before/After",
|
||||
`# ${params.scenarioConfig.title}`,
|
||||
"",
|
||||
`Status: ${params.comparison.pass ? "pass" : "fail"}`,
|
||||
`Transport: ${params.comparison.transport}`,
|
||||
@@ -230,6 +269,7 @@ function buildEvidenceManifest(params: {
|
||||
candidate: LaneResult;
|
||||
comparison: Comparison;
|
||||
outputDir: string;
|
||||
scenarioConfig: MantisScenarioConfig;
|
||||
}) {
|
||||
const artifacts: {
|
||||
alt?: string;
|
||||
@@ -259,9 +299,9 @@ function buildEvidenceManifest(params: {
|
||||
const baselineScreenshot = relativeArtifactPath(params.outputDir, params.baseline.screenshotPath);
|
||||
if (baselineScreenshot) {
|
||||
artifacts.push({
|
||||
alt: "Baseline Discord status reaction timeline",
|
||||
alt: params.scenarioConfig.baselineScreenshotAlt,
|
||||
kind: "timeline",
|
||||
label: "Baseline queued-only",
|
||||
label: params.scenarioConfig.baselineLabel,
|
||||
lane: "baseline",
|
||||
path: baselineScreenshot,
|
||||
targetPath: "baseline.png",
|
||||
@@ -274,9 +314,9 @@ function buildEvidenceManifest(params: {
|
||||
);
|
||||
if (candidateScreenshot) {
|
||||
artifacts.push({
|
||||
alt: "Candidate Discord status reaction timeline",
|
||||
alt: params.scenarioConfig.candidateScreenshotAlt,
|
||||
kind: "timeline",
|
||||
label: "Candidate queued -> thinking -> done",
|
||||
label: params.scenarioConfig.candidateLabel,
|
||||
lane: "candidate",
|
||||
path: candidateScreenshot,
|
||||
targetPath: "candidate.png",
|
||||
@@ -314,7 +354,7 @@ function buildEvidenceManifest(params: {
|
||||
schemaVersion: 1,
|
||||
summary:
|
||||
"Mantis ran the before/after scenario, captured baseline and candidate evidence, and compared the expected bug reproduction against the candidate fix.",
|
||||
title: "Mantis Before/After QA",
|
||||
title: params.scenarioConfig.title,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -452,10 +492,14 @@ export async function runMantisBeforeAfter(
|
||||
const scenario = normalizeRequiredLiteral(
|
||||
opts.scenario,
|
||||
DEFAULT_SCENARIO,
|
||||
[DEFAULT_SCENARIO],
|
||||
Object.keys(MANTIS_SCENARIO_CONFIGS),
|
||||
"--scenario",
|
||||
);
|
||||
const baseline = trimToValue(opts.baseline) ?? DEFAULT_BASELINE_REF;
|
||||
const scenarioConfig = MANTIS_SCENARIO_CONFIGS[scenario];
|
||||
if (!scenarioConfig) {
|
||||
throw new Error(`Unsupported Mantis scenario: ${scenario}`);
|
||||
}
|
||||
const baseline = trimToValue(opts.baseline) ?? scenarioConfig.defaultBaselineRef;
|
||||
const candidate = trimToValue(opts.candidate) ?? DEFAULT_CANDIDATE_REF;
|
||||
const runner = opts.commandRunner ?? defaultCommandRunner;
|
||||
const worktreeRoot = path.join(outputDir, "worktrees");
|
||||
@@ -495,7 +539,7 @@ export async function runMantisBeforeAfter(
|
||||
});
|
||||
const comparison = {
|
||||
baseline: {
|
||||
expected: "queued-only",
|
||||
expected: scenarioConfig.baselineExpected,
|
||||
ref: baseline,
|
||||
reproduced: baselineResult.status === "fail",
|
||||
screenshotPath: baselineResult.screenshotPath,
|
||||
@@ -503,7 +547,7 @@ export async function runMantisBeforeAfter(
|
||||
videoPath: baselineResult.videoPath,
|
||||
},
|
||||
candidate: {
|
||||
expected: "queued -> thinking -> done",
|
||||
expected: scenarioConfig.candidateExpected,
|
||||
fixed: candidateResult.status === "pass",
|
||||
ref: candidate,
|
||||
screenshotPath: candidateResult.screenshotPath,
|
||||
@@ -522,6 +566,7 @@ export async function runMantisBeforeAfter(
|
||||
candidate: candidateResult,
|
||||
comparison,
|
||||
outputDir,
|
||||
scenarioConfig,
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
@@ -533,6 +578,7 @@ export async function runMantisBeforeAfter(
|
||||
candidate: candidateResult,
|
||||
comparison,
|
||||
outputDir,
|
||||
scenarioConfig,
|
||||
}),
|
||||
null,
|
||||
2,
|
||||
|
||||
Reference in New Issue
Block a user