feat(qa): add Mantis Discord status reaction scenario (#76747)

* feat(qa): add Mantis Discord status reaction scenario

* fix(qa): retry Discord rate limits in Mantis runs

* refactor(qa): reuse Discord API retry helper

* fix(qa): import Discord API through package surface

* fix(ci): generate Discord boundary declarations

* fix(ci): keep xai boundary overrides stable
This commit is contained in:
Peter Steinberger
2026-05-03 17:00:06 +01:00
committed by GitHub
parent 1e8de7661e
commit 77a50db9ea
15 changed files with 990 additions and 130 deletions

View File

@@ -22,6 +22,7 @@ export {
resolveDiscordMaxLinesPerMessage,
} from "./src/accounts.js";
export { tryHandleDiscordMessageActionGuildAdmin } from "./src/actions/handle-action.guild-admin.js";
export { DiscordApiError, fetchDiscord, requestDiscord } from "./src/api.js";
export { buildDiscordComponentMessage } from "./src/components.js";
type DiscordMessageActionHandler =
typeof import("./src/channel-actions.runtime.js").handleDiscordMessageAction;

View File

@@ -1,6 +1,6 @@
import { withFetchPreconnect } from "openclaw/plugin-sdk/test-env";
import { beforeEach, describe, expect, it, vi } from "vitest";
import { DiscordApiError, fetchDiscord } from "./api.js";
import { DiscordApiError, fetchDiscord, requestDiscord } from "./api.js";
import { jsonResponse } from "./test-http-helpers.js";
describe("fetchDiscord", () => {
@@ -127,4 +127,23 @@ describe("fetchDiscord", () => {
expect(result).toHaveLength(1);
expect(calls).toBe(2);
});
it("sends JSON request bodies through the shared retry helper", async () => {
let request: RequestInit | undefined;
const fetcher = withFetchPreconnect(async (_url, init) => {
request = init;
return jsonResponse({ id: "42" }, 200);
});
const result = await requestDiscord<{ id: string }>("/channels/c/messages", "test", {
body: { content: "hello" },
fetcher,
retry: { attempts: 1 },
});
expect(result).toEqual({ id: "42" });
expect(request?.method).toBe("POST");
expect(request?.body).toBe(JSON.stringify({ content: "hello" }));
expect(new Headers(request?.headers).get("content-type")).toBe("application/json");
});
});

View File

@@ -126,13 +126,45 @@ type DiscordFetchOptions = {
label?: string;
};
export async function fetchDiscord<T>(
type DiscordApiRequestOptions = DiscordFetchOptions & {
body?: unknown;
fetcher?: typeof fetch;
headers?: Record<string, string>;
method?: string;
signal?: AbortSignal;
timeoutMs?: number;
};
function normalizeDiscordRequestBody(body: unknown, headers: Headers): BodyInit | null | undefined {
if (body === undefined) {
return undefined;
}
if (
typeof body === "string" ||
body instanceof Blob ||
body instanceof FormData ||
body instanceof URLSearchParams ||
body instanceof ArrayBuffer
) {
return body;
}
headers.set("Content-Type", headers.get("Content-Type") ?? "application/json");
return JSON.stringify(body);
}
function resolveDiscordRequestSignal(options: DiscordApiRequestOptions) {
if (options.signal || typeof options.timeoutMs !== "number") {
return options.signal;
}
return AbortSignal.timeout(options.timeoutMs);
}
export async function requestDiscord<T>(
path: string,
token: string,
fetcher: typeof fetch = fetch,
options?: DiscordFetchOptions,
options?: DiscordApiRequestOptions,
): Promise<T> {
const fetchImpl = resolveFetch(fetcher);
const fetchImpl = resolveFetch(options?.fetcher ?? fetch);
if (!fetchImpl) {
throw new Error("fetch is not available");
}
@@ -140,11 +172,17 @@ export async function fetchDiscord<T>(
const retryConfig = resolveRetryConfig(DISCORD_API_RETRY_DEFAULTS, options?.retry);
return retryAsync(
async () => {
const headers = new Headers(options?.headers);
headers.set("Authorization", `Bot ${token}`);
const body = normalizeDiscordRequestBody(options?.body, headers);
const res = await fetchImpl(`${DISCORD_API_BASE}${path}`, {
headers: { Authorization: `Bot ${token}` },
method: options?.method ?? (body === undefined ? "GET" : "POST"),
headers,
body,
signal: resolveDiscordRequestSignal(options ?? {}),
});
const text = await res.text().catch(() => "");
if (!res.ok) {
const text = await res.text().catch(() => "");
const detail = formatDiscordApiErrorText(text, res);
const suffix = detail ? `: ${detail}` : "";
const retryAfter =
@@ -157,7 +195,10 @@ export async function fetchDiscord<T>(
retryAfter,
);
}
return (await res.json()) as T;
if (!text.trim()) {
return undefined as T;
}
return JSON.parse(text) as T;
},
{
...retryConfig,
@@ -167,3 +208,12 @@ export async function fetchDiscord<T>(
},
);
}
export async function fetchDiscord<T>(
path: string,
token: string,
fetcher: typeof fetch = fetch,
options?: DiscordFetchOptions,
): Promise<T> {
return await requestDiscord<T>(path, token, { ...options, fetcher, method: "GET" });
}

View File

@@ -12,6 +12,7 @@
"zod": "^4.4.1"
},
"devDependencies": {
"@openclaw/discord": "workspace:*",
"@openclaw/plugin-sdk": "workspace:*",
"openclaw": "workspace:*"
},

View File

@@ -12,6 +12,7 @@ export async function runQaDiscordCommand(opts: LiveTransportQaCommandOptions) {
report: result.reportPath,
summary: result.summaryPath,
"observed messages": result.observedMessagesPath,
...(result.reactionTimelinesPath ? { "reaction timelines": result.reactionTimelinesPath } : {}),
...(result.gatewayDebugDirPath ? { "gateway debug logs": result.gatewayDebugDirPath } : {}),
});
if (

View File

@@ -6,29 +6,8 @@ import {
} from "../shared/live-transport-scenarios.js";
import { __testing } from "./discord-live.runtime.js";
const fetchWithSsrFGuardMock = vi.hoisted(() =>
vi.fn(async (params: { url: string; init?: RequestInit; signal?: AbortSignal }) => ({
response: await fetch(params.url, {
...params.init,
signal: params.signal,
}),
release: async () => {},
})),
);
vi.mock("openclaw/plugin-sdk/ssrf-runtime", async () => {
const actual = await vi.importActual<typeof import("openclaw/plugin-sdk/ssrf-runtime")>(
"openclaw/plugin-sdk/ssrf-runtime",
);
return {
...actual,
fetchWithSsrFGuard: fetchWithSsrFGuardMock,
};
});
describe("discord live qa runtime", () => {
afterEach(() => {
fetchWithSsrFGuardMock.mockClear();
vi.restoreAllMocks();
vi.unstubAllGlobals();
});
@@ -162,6 +141,47 @@ describe("discord live qa runtime", () => {
});
});
it("injects tool-only Discord status reaction config for the Mantis scenario", () => {
const next = __testing.buildDiscordQaConfig(
{},
{
guildId: "123456789012345678",
channelId: "223456789012345678",
driverBotId: "423456789012345678",
sutAccountId: "sut",
sutBotToken: "sut-token",
},
{ statusReactionsToolOnly: true },
);
expect(next.messages).toMatchObject({
ackReaction: "👀",
ackReactionScope: "all",
groupChat: { visibleReplies: "message_tool" },
statusReactions: {
enabled: true,
timing: { debounceMs: 0 },
},
});
expect(next.channels?.discord).toMatchObject({
accounts: {
sut: {
allowBots: true,
guilds: {
"123456789012345678": {
requireMention: false,
channels: {
"223456789012345678": {
requireMention: false,
},
},
},
},
},
},
});
});
it("normalizes observed Discord messages", () => {
expect(
__testing.normalizeDiscordObservedMessage({
@@ -227,6 +247,80 @@ describe("discord live qa runtime", () => {
"discord-mention-gating",
"discord-native-help-command-registration",
]);
expect(
__testing.findScenario(["discord-status-reactions-tool-only"]).map((scenario) => scenario.id),
).toEqual(["discord-status-reactions-tool-only"]);
});
it("collects the status reaction sequence across timeline snapshots", () => {
expect(
__testing.collectSeenReactionSequence(
[
{
elapsedMs: 0,
observedAt: "2026-05-03T12:00:00.000Z",
reactions: [{ emoji: "👀", count: 1, me: true }],
},
{
elapsedMs: 250,
observedAt: "2026-05-03T12:00:00.250Z",
reactions: [
{ emoji: "👀", count: 1, me: true },
{ emoji: "🤔", count: 1, me: true },
],
},
{
elapsedMs: 500,
observedAt: "2026-05-03T12:00:00.500Z",
reactions: [{ emoji: "👍", count: 1, me: true }],
},
],
["👀", "🤔", "👍"],
),
).toEqual(["👀", "🤔", "👍"]);
});
it("normalizes reaction snapshots from Discord messages", () => {
expect(
__testing.normalizeDiscordReactionSnapshot({
startedAtMs: new Date("2026-05-03T12:00:00.000Z").getTime(),
observedAt: new Date("2026-05-03T12:00:01.000Z"),
message: {
id: "523456789012345678",
channel_id: "223456789012345678",
reactions: [
{ count: 1, emoji: { name: "🤔" }, me: true },
{ count: 2, emoji: { name: "👀" }, me: false },
],
},
}),
).toEqual({
elapsedMs: 1000,
observedAt: "2026-05-03T12:00:01.000Z",
reactions: [
{ emoji: "👀", count: 2, me: false },
{ emoji: "🤔", count: 1, me: true },
],
});
});
it("renders a human-readable status reaction timeline artifact", () => {
const html = __testing.renderDiscordStatusReactionHtml({
scenarioTitle: "Discord status reactions",
expectedSequence: ["👀", "🤔", "👍"],
seenSequence: ["👀", "🤔"],
snapshots: [
{
elapsedMs: 0,
observedAt: "2026-05-03T12:00:00.000Z",
reactions: [{ emoji: "👀", count: 1, me: true }],
},
],
});
expect(html).toContain("Discord status reactions");
expect(html).toContain("Expected: 👀 → 🤔 → 👍");
expect(html).toContain("Seen: 👀 → 🤔");
});
it("waits for the Discord account to become connected, not just running", async () => {
@@ -387,7 +481,7 @@ describe("discord live qa runtime", () => {
}
});
it("adds an abort deadline to Discord API requests", async () => {
it("uses the Discord API helper timeout for identity probes", async () => {
const controller = new AbortController();
const timeoutSpy = vi.spyOn(AbortSignal, "timeout").mockReturnValue(controller.signal);
let signal: AbortSignal | undefined;
@@ -404,22 +498,45 @@ describe("discord live qa runtime", () => {
}),
);
await expect(
__testing.callDiscordApi({
token: "token",
path: "/users/@me",
timeoutMs: 25,
}),
).resolves.toEqual({
await expect(__testing.getCurrentDiscordUser("token")).resolves.toEqual({
id: "423456789012345678",
});
expect(timeoutSpy).toHaveBeenCalledWith(25);
expect(timeoutSpy).toHaveBeenCalledWith(15_000);
expect(signal).toBe(controller.signal);
expect(signal?.aborted).toBe(false);
controller.abort();
expect(signal?.aborted).toBe(true);
});
it("retries Discord REST requests after a 429 rate limit", async () => {
vi.stubGlobal(
"fetch",
vi
.fn()
.mockResolvedValueOnce(
new Response(JSON.stringify({ message: "You are being rate limited.", retry_after: 0 }), {
status: 429,
headers: {
"content-type": "application/json",
},
}),
)
.mockResolvedValueOnce(
new Response(JSON.stringify({ id: "423456789012345678" }), {
status: 200,
headers: {
"content-type": "application/json",
},
}),
),
);
await expect(__testing.getCurrentDiscordUser("token")).resolves.toEqual({
id: "423456789012345678",
});
expect(fetch).toHaveBeenCalledTimes(2);
});
it("redacts observed message content by default in artifacts", () => {
expect(
__testing.buildObservedMessagesArtifact({

View File

@@ -1,9 +1,12 @@
import { randomUUID } from "node:crypto";
import fs from "node:fs/promises";
import path from "node:path";
import { pathToFileURL } from "node:url";
import { requestDiscord } from "@openclaw/discord/api.js";
import { DEFAULT_EMOJIS } from "openclaw/plugin-sdk/channel-feedback";
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-types";
import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
import { fetchWithSsrFGuard } from "openclaw/plugin-sdk/ssrf-runtime";
import { chromium } from "playwright-core";
import { z } from "zod";
import { startQaGatewayChild } from "../../gateway-child.js";
import { DEFAULT_QA_LIVE_PROVIDER_MODE } from "../../providers/index.js";
@@ -36,7 +39,8 @@ type DiscordQaRuntimeEnv = {
type DiscordQaScenarioId =
| "discord-canary"
| "discord-mention-gating"
| "discord-native-help-command-registration";
| "discord-native-help-command-registration"
| "discord-status-reactions-tool-only";
type DiscordQaScenarioRun =
| {
@@ -49,6 +53,11 @@ type DiscordQaScenarioRun =
| {
kind: "application-command-registration";
expectedCommandNames: string[];
}
| {
kind: "status-reactions-tool-only";
expectedSequence: string[];
input: string;
};
type DiscordQaScenarioDefinition = LiveTransportScenarioDefinition<DiscordQaScenarioId> & {
@@ -66,11 +75,21 @@ type DiscordMessage = {
channel_id: string;
guild_id?: string;
content?: string;
reactions?: DiscordReaction[];
timestamp?: string;
author?: DiscordUser;
referenced_message?: { id?: string } | null;
};
type DiscordReaction = {
count?: number;
emoji?: {
id?: string | null;
name?: string | null;
};
me?: boolean;
};
type DiscordApplicationCommand = {
id: string;
name?: string;
@@ -107,6 +126,7 @@ type DiscordObservedMessageArtifact = {
};
type DiscordQaScenarioResult = {
artifactPaths?: Record<string, string>;
id: string;
title: string;
status: "pass" | "fail";
@@ -116,6 +136,7 @@ type DiscordQaScenarioResult = {
type DiscordQaRunResult = {
outputDir: string;
reportPath: string;
reactionTimelinesPath?: string;
summaryPath: string;
observedMessagesPath: string;
gatewayDebugDirPath?: string;
@@ -123,6 +144,12 @@ type DiscordQaRunResult = {
};
type DiscordQaSummary = {
artifacts: {
observedMessagesPath: string;
reactionTimelinesPath?: string;
reportPath: string;
summaryPath: string;
};
credentials: {
credentialId?: string;
kind: string;
@@ -143,7 +170,28 @@ type DiscordQaSummary = {
scenarios: DiscordQaScenarioResult[];
};
const DISCORD_API_BASE_URL = "https://discord.com/api/v10";
type DiscordReactionSnapshot = {
elapsedMs: number;
observedAt: string;
reactions: Array<{
count: number;
emoji: string;
me: boolean;
}>;
};
type DiscordStatusReactionTimeline = {
expectedSequence: string[];
htmlPath?: string;
scenarioId: DiscordQaScenarioId;
scenarioTitle: string;
screenshotPath?: string;
screenshotWarning?: string;
seenSequence: string[];
snapshots: DiscordReactionSnapshot[];
triggerMessageId: string;
};
const DISCORD_QA_CAPTURE_CONTENT_ENV = "OPENCLAW_QA_DISCORD_CAPTURE_CONTENT";
const QA_REDACT_PUBLIC_METADATA_ENV = "OPENCLAW_QA_REDACT_PUBLIC_METADATA";
const DISCORD_QA_ENV_KEYS = [
@@ -195,8 +243,29 @@ const DISCORD_QA_SCENARIOS: DiscordQaScenarioDefinition[] = [
expectedCommandNames: ["help"],
}),
},
{
id: "discord-status-reactions-tool-only",
title: "Discord explicit status reactions run in tool-only reply mode",
timeoutMs: 75_000,
buildRun: () => {
const token = `DISCORD_QA_STATUS_${randomUUID().slice(0, 8).toUpperCase()}`;
return {
kind: "status-reactions-tool-only",
input: [
`Mantis status reaction QA marker ${token}.`,
"Think briefly, then reply with only this exact marker:",
token,
].join(" "),
expectedSequence: ["👀", DEFAULT_EMOJIS.thinking, DEFAULT_EMOJIS.done],
};
},
},
];
const DISCORD_QA_DEFAULT_SCENARIOS = DISCORD_QA_SCENARIOS.filter(
(scenario) => scenario.id !== "discord-status-reactions-tool-only",
);
const DISCORD_QA_STANDARD_SCENARIO_IDS = collectLiveTransportStandardScenarioCoverage({
scenarios: DISCORD_QA_SCENARIOS,
});
@@ -272,12 +341,41 @@ function buildDiscordQaConfig(
sutAccountId: string;
sutBotToken: string;
},
options: {
statusReactionsToolOnly?: boolean;
} = {},
): OpenClawConfig {
const pluginAllow = [...new Set([...(baseCfg.plugins?.allow ?? []), "discord"])];
const pluginEntries = {
...baseCfg.plugins?.entries,
discord: { enabled: true },
};
const requireMention = !options.statusReactionsToolOnly;
const messages = options.statusReactionsToolOnly
? {
...baseCfg.messages,
ackReaction: "👀",
ackReactionScope: "all" as const,
groupChat: {
...baseCfg.messages?.groupChat,
visibleReplies: "message_tool" as const,
},
statusReactions: {
...baseCfg.messages?.statusReactions,
enabled: true,
timing: {
...baseCfg.messages?.statusReactions?.timing,
debounceMs: 0,
},
},
}
: {
...baseCfg.messages,
groupChat: {
...baseCfg.messages?.groupChat,
visibleReplies: "automatic" as const,
},
};
return {
...baseCfg,
plugins: {
@@ -285,13 +383,7 @@ function buildDiscordQaConfig(
allow: pluginAllow,
entries: pluginEntries,
},
messages: {
...baseCfg.messages,
groupChat: {
...baseCfg.messages?.groupChat,
visibleReplies: "automatic",
},
},
messages,
channels: {
...baseCfg.channels,
discord: {
@@ -301,16 +393,16 @@ function buildDiscordQaConfig(
[params.sutAccountId]: {
enabled: true,
token: params.sutBotToken,
allowBots: "mentions",
allowBots: options.statusReactionsToolOnly ? true : "mentions",
groupPolicy: "allowlist",
guilds: {
[params.guildId]: {
requireMention: true,
requireMention,
users: [params.driverBotId],
channels: {
[params.channelId]: {
enabled: true,
requireMention: true,
requireMention,
users: [params.driverBotId],
},
},
@@ -323,70 +415,34 @@ function buildDiscordQaConfig(
};
}
async function callDiscordApi<T>(params: {
token: string;
path: string;
init?: RequestInit;
timeoutMs?: number;
}): Promise<T> {
const headers = new Headers(params.init?.headers);
headers.set("authorization", `Bot ${params.token}`);
if (params.init?.body) {
headers.set("content-type", "application/json");
}
const { response, release } = await fetchWithSsrFGuard({
url: `${DISCORD_API_BASE_URL}${params.path}`,
init: {
...params.init,
headers,
},
signal: AbortSignal.timeout(params.timeoutMs ?? 15_000),
policy: { hostnameAllowlist: ["discord.com"] },
auditContext: "qa-lab-discord-live",
});
try {
const text = await response.text();
const payload = text.trim() ? (JSON.parse(text) as unknown) : undefined;
if (!response.ok) {
const message =
typeof payload === "object" &&
payload !== null &&
typeof (payload as { message?: unknown }).message === "string"
? (payload as { message: string }).message
: text.trim();
throw new Error(
message || `Discord API ${params.path} failed with status ${response.status}`,
);
}
return payload as T;
} finally {
await release();
}
}
async function getCurrentDiscordUser(token: string) {
return await callDiscordApi<DiscordUser>({
token,
path: "/users/@me",
return await requestDiscord<DiscordUser>("/users/@me", token, {
timeoutMs: 15_000,
});
}
async function sendChannelMessage(token: string, channelId: string, content: string) {
return await callDiscordApi<DiscordMessage>({
token,
path: `/channels/${channelId}/messages`,
init: {
method: "POST",
body: JSON.stringify({
content,
allowed_mentions: {
parse: ["users"],
},
}),
return await requestDiscord<DiscordMessage>(`/channels/${channelId}/messages`, token, {
body: {
content,
allowed_mentions: {
parse: ["users"],
},
},
timeoutMs: 15_000,
});
}
async function getChannelMessage(params: { token: string; channelId: string; messageId: string }) {
return await requestDiscord<DiscordMessage>(
`/channels/${params.channelId}/messages/${params.messageId}`,
params.token,
{
timeoutMs: 15_000,
},
);
}
async function listChannelMessagesAfter(params: {
token: string;
channelId: string;
@@ -396,17 +452,215 @@ async function listChannelMessagesAfter(params: {
after: params.afterSnowflake,
limit: "50",
});
return await callDiscordApi<DiscordMessage[]>({
token: params.token,
path: `/channels/${params.channelId}/messages?${query.toString()}`,
return await requestDiscord<DiscordMessage[]>(
`/channels/${params.channelId}/messages?${query.toString()}`,
params.token,
{
timeoutMs: 15_000,
},
);
}
function reactionEmojiName(reaction: DiscordReaction) {
return reaction.emoji?.name?.trim() || reaction.emoji?.id?.trim() || "";
}
function normalizeDiscordReactionSnapshot(params: {
message: DiscordMessage;
observedAt: Date;
startedAtMs: number;
}): DiscordReactionSnapshot {
return {
elapsedMs: Math.max(0, params.observedAt.getTime() - params.startedAtMs),
observedAt: params.observedAt.toISOString(),
reactions: (params.message.reactions ?? [])
.map((reaction) => ({
emoji: reactionEmojiName(reaction),
count: Math.max(0, Math.floor(reaction.count ?? 0)),
me: reaction.me === true,
}))
.filter((reaction) => reaction.emoji.length > 0)
.toSorted((a, b) => a.emoji.localeCompare(b.emoji)),
};
}
function collectSeenReactionSequence(
snapshots: readonly DiscordReactionSnapshot[],
expectedSequence: readonly string[],
) {
const seen = new Set<string>();
const sequence: string[] = [];
for (const snapshot of snapshots) {
const snapshotEmojis = new Set(snapshot.reactions.map((reaction) => reaction.emoji));
for (const emoji of expectedSequence) {
if (snapshotEmojis.has(emoji) && !seen.has(emoji)) {
seen.add(emoji);
sequence.push(emoji);
}
}
}
return sequence;
}
function escapeHtml(value: string) {
return value
.replace(/&/gu, "&amp;")
.replace(/</gu, "&lt;")
.replace(/>/gu, "&gt;")
.replace(/"/gu, "&quot;");
}
function renderDiscordStatusReactionHtml(params: {
expectedSequence: readonly string[];
scenarioTitle: string;
seenSequence: readonly string[];
snapshots: readonly DiscordReactionSnapshot[];
}) {
const rows = params.snapshots
.map((snapshot) => {
const reactions = snapshot.reactions
.map(
(reaction) =>
`<span class="pill"><span class="emoji">${escapeHtml(reaction.emoji)}</span><span class="count">${reaction.count}</span></span>`,
)
.join("");
return `<tr><td>${snapshot.elapsedMs}ms</td><td>${escapeHtml(snapshot.observedAt)}</td><td>${reactions || '<span class="muted">none</span>'}</td></tr>`;
})
.join("\n");
return `<!doctype html>
<html>
<head>
<meta charset="utf-8" />
<title>${escapeHtml(params.scenarioTitle)}</title>
<style>
body { margin: 0; background: #313338; color: #f2f3f5; font-family: Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; }
main { width: 1040px; padding: 32px; }
h1 { font-size: 26px; margin: 0 0 8px; font-weight: 700; letter-spacing: 0; }
.sub { color: #b5bac1; margin-bottom: 24px; }
.message { background: #2b2d31; border-left: 4px solid #5865f2; padding: 20px; border-radius: 8px; margin-bottom: 24px; }
.author { color: #f2f3f5; font-weight: 700; margin-bottom: 8px; }
.content { color: #dbdee1; line-height: 1.45; }
.sequence { display: flex; gap: 12px; margin-top: 18px; align-items: center; }
.step { background: #404249; border: 1px solid #4e5058; border-radius: 18px; padding: 7px 12px; font-size: 20px; min-width: 42px; text-align: center; }
.step.seen { background: #1f3b2d; border-color: #2d7d46; }
table { width: 100%; border-collapse: collapse; background: #2b2d31; border-radius: 8px; overflow: hidden; }
th, td { text-align: left; padding: 12px 14px; border-bottom: 1px solid #404249; vertical-align: top; }
th { color: #b5bac1; font-size: 13px; text-transform: uppercase; }
.pill { display: inline-flex; align-items: center; gap: 6px; border: 1px solid #4e5058; border-radius: 14px; padding: 4px 9px; margin: 0 8px 8px 0; background: #383a40; }
.emoji { font-size: 18px; }
.count { color: #b5bac1; font-size: 13px; }
.muted { color: #949ba4; }
</style>
</head>
<body>
<main>
<h1>${escapeHtml(params.scenarioTitle)}</h1>
<div class="sub">Expected: ${params.expectedSequence.map(escapeHtml).join(" → ")} · Seen: ${params.seenSequence.map(escapeHtml).join(" → ") || "none"}</div>
<section class="message">
<div class="author">Mantis Discord QA</div>
<div class="content">Reaction timeline captured from the real Discord triggering message via REST polling.</div>
<div class="sequence">
${params.expectedSequence
.map(
(emoji) =>
`<span class="step ${params.seenSequence.includes(emoji) ? "seen" : ""}">${escapeHtml(emoji)}</span>`,
)
.join("")}
</div>
</section>
<table>
<thead><tr><th>Elapsed</th><th>Observed At</th><th>Reactions</th></tr></thead>
<tbody>${rows}</tbody>
</table>
</main>
</body>
</html>`;
}
async function writeDiscordStatusReactionEvidence(params: {
outputDir: string;
timeline: DiscordStatusReactionTimeline;
}) {
const htmlPath = path.join(params.outputDir, `${params.timeline.scenarioId}-timeline.html`);
const screenshotPath = path.join(params.outputDir, `${params.timeline.scenarioId}-timeline.png`);
const html = renderDiscordStatusReactionHtml({
expectedSequence: params.timeline.expectedSequence,
scenarioTitle: params.timeline.scenarioTitle,
seenSequence: params.timeline.seenSequence,
snapshots: params.timeline.snapshots,
});
await fs.writeFile(htmlPath, html, { encoding: "utf8", mode: 0o600 });
try {
const browser = await chromium.launch({
channel: "chrome",
headless: true,
});
try {
const page = await browser.newPage({ viewport: { width: 1104, height: 760 } });
await page.goto(pathToFileURL(htmlPath).toString(), {
waitUntil: "domcontentloaded",
timeout: 15_000,
});
await page.screenshot({ path: screenshotPath, fullPage: true });
return { htmlPath, screenshotPath };
} finally {
await browser.close();
}
} catch (error) {
return { htmlPath, screenshotWarning: formatErrorMessage(error) };
}
}
async function observeStatusReactionTimeline(params: {
channelId: string;
expectedSequence: string[];
messageId: string;
scenarioId: DiscordQaScenarioId;
scenarioTitle: string;
timeoutMs: number;
token: string;
}) {
const startedAtMs = Date.now();
const snapshots: DiscordReactionSnapshot[] = [];
let seenSequence: string[] = [];
while (Date.now() - startedAtMs < params.timeoutMs) {
const observedAt = new Date();
const message = await getChannelMessage({
token: params.token,
channelId: params.channelId,
messageId: params.messageId,
});
snapshots.push(
normalizeDiscordReactionSnapshot({
message,
observedAt,
startedAtMs,
}),
);
seenSequence = collectSeenReactionSequence(snapshots, params.expectedSequence);
if (params.expectedSequence.every((emoji) => seenSequence.includes(emoji))) {
break;
}
await new Promise((resolve) => setTimeout(resolve, 250));
}
return {
expectedSequence: params.expectedSequence,
scenarioId: params.scenarioId,
scenarioTitle: params.scenarioTitle,
seenSequence,
snapshots,
triggerMessageId: params.messageId,
} satisfies DiscordStatusReactionTimeline;
}
async function listApplicationCommands(params: { token: string; applicationId: string }) {
return await callDiscordApi<DiscordApplicationCommand[]>({
token: params.token,
path: `/applications/${params.applicationId}/commands`,
});
return await requestDiscord<DiscordApplicationCommand[]>(
`/applications/${params.applicationId}/commands`,
params.token,
{
timeoutMs: 15_000,
},
);
}
function compareDiscordSnowflakes(a: string, b: string) {
@@ -566,6 +820,11 @@ function renderDiscordQaMarkdown(params: {
lines.push("");
lines.push(`- Status: ${scenario.status}`);
lines.push(`- Details: ${scenario.details}`);
if (scenario.artifactPaths && Object.keys(scenario.artifactPaths).length > 0) {
for (const [label, artifactPath] of Object.entries(scenario.artifactPaths)) {
lines.push(`- ${label}: \`${artifactPath}\``);
}
}
lines.push("");
}
if (params.gatewayDebugDirPath) {
@@ -625,10 +884,11 @@ function buildObservedMessagesArtifact(params: {
}
function findScenario(ids?: string[]) {
const scenarios = ids && ids.length > 0 ? DISCORD_QA_SCENARIOS : DISCORD_QA_DEFAULT_SCENARIOS;
return selectLiveTransportScenarios({
ids,
laneLabel: "Discord",
scenarios: DISCORD_QA_SCENARIOS,
scenarios,
});
}
@@ -717,6 +977,14 @@ export async function runDiscordQaLive(params: {
const alternateModel = params.alternateModel?.trim() || defaultQaModelForMode(providerMode, true);
const sutAccountId = params.sutAccountId?.trim() || "sut";
const scenarios = findScenario(params.scenarioIds);
const statusReactionScenarioRequested = scenarios.some(
(scenario) => scenario.id === "discord-status-reactions-tool-only",
);
if (statusReactionScenarioRequested && scenarios.length > 1) {
throw new Error(
"discord-status-reactions-tool-only must run by itself because it changes Discord tool-only reply config.",
);
}
const credentialLease = await acquireQaCredentialLease({
kind: "discord",
@@ -732,6 +1000,7 @@ export async function runDiscordQaLive(params: {
const runtimeEnv = credentialLease.payload;
const observedMessages: DiscordObservedMessage[] = [];
const reactionTimelines: DiscordStatusReactionTimeline[] = [];
const redactPublicMetadata = isTruthyOptIn(process.env[QA_REDACT_PUBLIC_METADATA_ENV]);
const includeObservedMessageContent = isTruthyOptIn(process.env[DISCORD_QA_CAPTURE_CONTENT_ENV]);
const startedAt = new Date().toISOString();
@@ -766,13 +1035,17 @@ export async function runDiscordQaLive(params: {
fastMode: params.fastMode,
controlUiEnabled: false,
mutateConfig: (cfg) =>
buildDiscordQaConfig(cfg, {
guildId: runtimeEnv.guildId,
channelId: runtimeEnv.channelId,
driverBotId: driverIdentity.id,
sutAccountId,
sutBotToken: runtimeEnv.sutBotToken,
}),
buildDiscordQaConfig(
cfg,
{
guildId: runtimeEnv.guildId,
channelId: runtimeEnv.channelId,
driverBotId: driverIdentity.id,
sutAccountId,
sutBotToken: runtimeEnv.sutBotToken,
},
{ statusReactionsToolOnly: statusReactionScenarioRequested },
),
});
try {
await waitForDiscordChannelRunning(gatewayHarness.gateway, sutAccountId);
@@ -803,6 +1076,39 @@ export async function runDiscordQaLive(params: {
runtimeEnv.channelId,
scenarioRun.input,
);
if (scenarioRun.kind === "status-reactions-tool-only") {
const timeline = await observeStatusReactionTimeline({
token: runtimeEnv.driverBotToken,
channelId: runtimeEnv.channelId,
expectedSequence: scenarioRun.expectedSequence,
messageId: sent.id,
scenarioId: scenario.id,
scenarioTitle: scenario.title,
timeoutMs: scenario.timeoutMs,
});
const evidence = await writeDiscordStatusReactionEvidence({ outputDir, timeline });
const enrichedTimeline = { ...timeline, ...evidence };
reactionTimelines.push(enrichedTimeline);
const missing = scenarioRun.expectedSequence.filter(
(emoji) => !timeline.seenSequence.includes(emoji),
);
scenarioResults.push({
id: scenario.id,
title: scenario.title,
status: missing.length === 0 ? "pass" : "fail",
details:
missing.length === 0
? `reaction timeline matched ${timeline.seenSequence.join(" -> ")}`
: `reaction timeline missing ${missing.join(", ")}; saw ${timeline.seenSequence.join(" -> ") || "none"}`,
artifactPaths: {
...(enrichedTimeline.htmlPath ? { html: enrichedTimeline.htmlPath } : {}),
...(enrichedTimeline.screenshotPath
? { screenshot: enrichedTimeline.screenshotPath }
: {}),
},
});
continue;
}
const matched = await pollChannelMessages({
token: runtimeEnv.driverBotToken,
channelId: runtimeEnv.channelId,
@@ -885,6 +1191,14 @@ export async function runDiscordQaLive(params: {
const passedCount = scenarioResults.filter((entry) => entry.status === "pass").length;
const failedCount = scenarioResults.filter((entry) => entry.status === "fail").length;
const summary: DiscordQaSummary = {
artifacts: {
reportPath: path.join(outputDir, "discord-qa-report.md"),
summaryPath: path.join(outputDir, "discord-qa-summary.json"),
observedMessagesPath: path.join(outputDir, "discord-qa-observed-messages.json"),
...(reactionTimelines.length > 0
? { reactionTimelinesPath: path.join(outputDir, "discord-qa-reaction-timelines.json") }
: {}),
},
credentials: {
source: credentialLease.source,
kind: credentialLease.kind,
@@ -907,6 +1221,7 @@ export async function runDiscordQaLive(params: {
const reportPath = path.join(outputDir, "discord-qa-report.md");
const summaryPath = path.join(outputDir, "discord-qa-summary.json");
const observedMessagesPath = path.join(outputDir, "discord-qa-observed-messages.json");
const reactionTimelinesPath = path.join(outputDir, "discord-qa-reaction-timelines.json");
await fs.writeFile(
reportPath,
`${renderDiscordQaMarkdown({
@@ -939,10 +1254,17 @@ export async function runDiscordQaLive(params: {
)}\n`,
{ encoding: "utf8", mode: 0o600 },
);
if (reactionTimelines.length > 0) {
await fs.writeFile(reactionTimelinesPath, `${JSON.stringify(reactionTimelines, null, 2)}\n`, {
encoding: "utf8",
mode: 0o600,
});
}
const artifactPaths = {
report: reportPath,
summary: summaryPath,
observedMessages: observedMessagesPath,
...(reactionTimelines.length > 0 ? { reactionTimelines: reactionTimelinesPath } : {}),
...(preservedGatewayDebugArtifacts ? { gatewayDebug: gatewayDebugDirPath } : {}),
};
if (cleanupIssues.length > 0) {
@@ -958,6 +1280,7 @@ export async function runDiscordQaLive(params: {
return {
outputDir,
reportPath,
...(reactionTimelines.length > 0 ? { reactionTimelinesPath } : {}),
summaryPath,
observedMessagesPath,
...(preservedGatewayDebugArtifacts ? { gatewayDebugDirPath } : {}),
@@ -968,16 +1291,20 @@ export async function runDiscordQaLive(params: {
export const __testing = {
DISCORD_QA_SCENARIOS,
DISCORD_QA_STANDARD_SCENARIO_IDS,
collectSeenReactionSequence,
assertDiscordScenarioReply,
assertDiscordApplicationCommandsRegistered,
buildDiscordQaConfig,
buildObservedMessagesArtifact,
callDiscordApi,
findScenario,
getCurrentDiscordUser,
getChannelMessage,
listApplicationCommands,
matchesDiscordScenarioReply,
normalizeDiscordReactionSnapshot,
normalizeDiscordObservedMessage,
parseDiscordQaCredentialPayload,
renderDiscordStatusReactionHtml,
resolveDiscordQaRuntimeEnv,
waitForDiscordChannelRunning,
};

View File

@@ -36,6 +36,7 @@
],
"openclaw/plugin-sdk/ssrf-runtime": ["../dist/plugin-sdk/src/plugin-sdk/ssrf-runtime.d.ts"],
"@openclaw/qa-channel/api.js": ["../dist/plugin-sdk/extensions/qa-channel/api.d.ts"],
"@openclaw/discord/api.js": ["../dist/plugin-sdk/extensions/discord/api.d.ts"],
"@openclaw/*.js": ["../packages/plugin-sdk/dist/extensions/*.d.ts", "../extensions/*"],
"@openclaw/*": ["../packages/plugin-sdk/dist/extensions/*", "../extensions/*"],
"@openclaw/plugin-sdk/*": ["../dist/plugin-sdk/src/plugin-sdk/*.d.ts"]