mirror of
https://github.com/openclaw/openclaw.git
synced 2026-05-06 05:40:44 +00:00
Add Discord live QA lane (#70792)
* Add Discord live QA lane * Add Discord native command QA coverage
This commit is contained in:
Patrick Erichsen
@@ -14,6 +14,10 @@ on:
|
||||
description: Optional comma-separated Telegram scenario ids
|
||||
required: false
|
||||
type: string
|
||||
discord_scenario:
|
||||
description: Optional comma-separated Discord scenario ids
|
||||
required: false
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -346,3 +350,95 @@ jobs:
|
||||
path: ${{ steps.run_lane.outputs.output_dir }}
|
||||
retention-days: 14
|
||||
if-no-files-found: warn
|
||||
|
||||
run_live_discord:
|
||||
name: Run Discord live QA lane with Convex leases
|
||||
needs: [authorize_actor, validate_selected_ref]
|
||||
runs-on: blacksmith-32vcpu-ubuntu-2404
|
||||
timeout-minutes: 60
|
||||
environment: qa-live-shared
|
||||
steps:
|
||||
- name: Checkout selected ref
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
ref: ${{ needs.validate_selected_ref.outputs.selected_sha }}
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Setup Node environment
|
||||
uses: ./.github/actions/setup-node-env
|
||||
with:
|
||||
node-version: ${{ env.NODE_VERSION }}
|
||||
pnpm-version: ${{ env.PNPM_VERSION }}
|
||||
install-bun: "true"
|
||||
|
||||
- name: Validate required QA credential env
|
||||
env:
|
||||
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
||||
OPENCLAW_QA_CONVEX_SITE_URL: ${{ secrets.OPENCLAW_QA_CONVEX_SITE_URL }}
|
||||
OPENCLAW_QA_CONVEX_SECRET_CI: ${{ secrets.OPENCLAW_QA_CONVEX_SECRET_CI }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
require_var() {
|
||||
local key="$1"
|
||||
if [[ -z "${!key:-}" ]]; then
|
||||
echo "Missing required ${key}." >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
require_var OPENAI_API_KEY
|
||||
require_var OPENCLAW_QA_CONVEX_SITE_URL
|
||||
require_var OPENCLAW_QA_CONVEX_SECRET_CI
|
||||
|
||||
- name: Build private QA runtime
|
||||
run: pnpm build
|
||||
|
||||
- name: Run Discord live lane
|
||||
id: run_lane
|
||||
shell: bash
|
||||
env:
|
||||
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
||||
OPENCLAW_QA_CONVEX_SITE_URL: ${{ secrets.OPENCLAW_QA_CONVEX_SITE_URL }}
|
||||
OPENCLAW_QA_CONVEX_SECRET_CI: ${{ secrets.OPENCLAW_QA_CONVEX_SECRET_CI }}
|
||||
OPENCLAW_QA_REDACT_PUBLIC_METADATA: "1"
|
||||
OPENCLAW_QA_DISCORD_CAPTURE_CONTENT: "1"
|
||||
INPUT_SCENARIO: ${{ github.event_name == 'workflow_dispatch' && inputs.discord_scenario || '' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
output_dir=".artifacts/qa-e2e/discord-live-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
|
||||
scenario_args=()
|
||||
|
||||
if [[ -n "${INPUT_SCENARIO// }" ]]; then
|
||||
IFS=',' read -r -a raw_scenarios <<<"${INPUT_SCENARIO}"
|
||||
for raw in "${raw_scenarios[@]}"; do
|
||||
scenario="$(printf '%s' "${raw}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')"
|
||||
if [[ -n "${scenario}" ]]; then
|
||||
scenario_args+=(--scenario "${scenario}")
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
echo "output_dir=${output_dir}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
pnpm openclaw qa discord \
|
||||
--repo-root . \
|
||||
--output-dir "${output_dir}" \
|
||||
--provider-mode live-frontier \
|
||||
--model openai/gpt-5.4 \
|
||||
--alt-model openai/gpt-5.4 \
|
||||
--fast \
|
||||
--credential-source convex \
|
||||
--credential-role ci \
|
||||
"${scenario_args[@]}"
|
||||
|
||||
- name: Upload Discord QA artifacts
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: qa-live-discord-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: ${{ steps.run_lane.outputs.output_dir }}
|
||||
retention-days: 14
|
||||
if-no-files-found: warn
|
||||
@@ -83,16 +83,34 @@ you want artifacts without a failing exit code.
|
||||
The Telegram report and summary include per-reply RTT from the driver message
|
||||
send request to the observed SUT reply, starting with the canary.
|
||||
|
||||
For a transport-real Discord smoke lane, run:
|
||||
|
||||
```bash
|
||||
pnpm openclaw qa discord
|
||||
```
|
||||
|
||||
That lane targets one real private Discord guild channel with two bots: a
|
||||
driver bot controlled by the harness and a SUT bot started by the child
|
||||
OpenClaw gateway through the bundled Discord plugin. It requires
|
||||
`OPENCLAW_QA_DISCORD_GUILD_ID`, `OPENCLAW_QA_DISCORD_CHANNEL_ID`,
|
||||
`OPENCLAW_QA_DISCORD_DRIVER_BOT_TOKEN`, `OPENCLAW_QA_DISCORD_SUT_BOT_TOKEN`,
|
||||
and `OPENCLAW_QA_DISCORD_SUT_APPLICATION_ID` when using env credentials.
|
||||
The lane verifies channel mention handling and checks that the SUT bot has
|
||||
registered the native `/help` command with Discord.
|
||||
The command exits non-zero when any scenario fails. Use `--allow-failures` when
|
||||
you want artifacts without a failing exit code.
|
||||
|
||||
Live transport lanes now share one smaller contract instead of each inventing
|
||||
their own scenario list shape:
|
||||
|
||||
`qa-channel` remains the broad synthetic product-behavior suite and is not part
|
||||
of the live transport coverage matrix.
|
||||
|
||||
| Lane | Canary | Mention gating | Allowlist block | Top-level reply | Restart resume | Thread follow-up | Thread isolation | Reaction observation | Help command |
|
||||
| -------- | ------ | -------------- | --------------- | --------------- | -------------- | ---------------- | ---------------- | -------------------- | ------------ |
|
||||
| Matrix | x | x | x | x | x | x | x | x | |
|
||||
| Telegram | x | | | | | | | | x |
|
||||
| Lane | Canary | Mention gating | Allowlist block | Top-level reply | Restart resume | Thread follow-up | Thread isolation | Reaction observation | Help command | Native command registration |
|
||||
| -------- | ------ | -------------- | --------------- | --------------- | -------------- | ---------------- | ---------------- | -------------------- | ------------ | --------------------------- |
|
||||
| Matrix | x | x | x | x | x | x | x | x | | |
|
||||
| Telegram | x | x | | | | | | | x | |
|
||||
| Discord | x | x | | | | | | | | x |
|
||||
|
||||
This keeps `qa-channel` as the broad product-behavior suite while Matrix,
|
||||
Telegram, and future live transports share one explicit transport-contract
|
||||
|
||||
@@ -595,6 +595,52 @@ describe("preflightDiscordMessage", () => {
|
||||
expect(result).not.toBeNull();
|
||||
});
|
||||
|
||||
it("hydrates mention metadata from REST when bot mention syntax is present but mentions are missing", async () => {
|
||||
const channelId = "channel-bot-mentions-hydrated";
|
||||
const guildId = "guild-bot-mentions-hydrated";
|
||||
const botId = "123456789012345678";
|
||||
const message = createDiscordMessage({
|
||||
id: "m-bot-mentions-hydrated",
|
||||
channelId,
|
||||
content: `hi <@${botId}>`,
|
||||
author: {
|
||||
id: "relay-bot-1",
|
||||
bot: true,
|
||||
username: "Relay",
|
||||
},
|
||||
mentionedUsers: [],
|
||||
});
|
||||
const client = createGuildTextClient(channelId);
|
||||
client.rest = {
|
||||
get: vi.fn(async () => ({
|
||||
id: message.id,
|
||||
content: message.content,
|
||||
mentions: [{ id: botId, username: "OpenClaw", bot: true }],
|
||||
mention_roles: [],
|
||||
mention_everyone: false,
|
||||
})),
|
||||
} as unknown as DiscordClient["rest"];
|
||||
|
||||
const result = await preflightDiscordMessage({
|
||||
...createPreflightArgs({
|
||||
cfg: DEFAULT_PREFLIGHT_CFG,
|
||||
discordConfig: {
|
||||
allowBots: "mentions",
|
||||
} as DiscordConfig,
|
||||
data: createGuildEvent({
|
||||
channelId,
|
||||
guildId,
|
||||
author: message.author,
|
||||
message,
|
||||
}),
|
||||
client,
|
||||
}),
|
||||
botUserId: botId,
|
||||
});
|
||||
|
||||
expect(result).not.toBeNull();
|
||||
});
|
||||
|
||||
it("still drops bot control commands without a real mention when allowBots=mentions", async () => {
|
||||
const channelId = "channel-bot-command-no-mention";
|
||||
const guildId = "guild-bot-command-no-mention";
|
||||
|
||||
@@ -324,15 +324,29 @@ function mergeFetchedDiscordMessage(base: Message, fetched: APIMessage): Message
|
||||
}) as unknown as Message;
|
||||
}
|
||||
|
||||
async function hydrateDiscordMessageIfEmpty(params: {
|
||||
function shouldHydrateDiscordMessage(params: { message: Message }) {
|
||||
const currentText = resolveDiscordMessageText(params.message, {
|
||||
includeForwarded: true,
|
||||
});
|
||||
if (!currentText) {
|
||||
return true;
|
||||
}
|
||||
const hasMentionMetadata =
|
||||
(params.message.mentionedUsers?.length ?? 0) > 0 ||
|
||||
(params.message.mentionedRoles?.length ?? 0) > 0 ||
|
||||
params.message.mentionedEveryone;
|
||||
if (hasMentionMetadata) {
|
||||
return false;
|
||||
}
|
||||
return /<@!?\d+>|<@&\d+>|@everyone|@here/u.test(currentText);
|
||||
}
|
||||
|
||||
async function hydrateDiscordMessageIfNeeded(params: {
|
||||
client: DiscordMessagePreflightParams["client"];
|
||||
message: Message;
|
||||
messageChannelId: string;
|
||||
}): Promise<Message> {
|
||||
const currentText = resolveDiscordMessageText(params.message, {
|
||||
includeForwarded: true,
|
||||
});
|
||||
if (currentText) {
|
||||
if (!shouldHydrateDiscordMessage({ message: params.message })) {
|
||||
return params.message;
|
||||
}
|
||||
const rest = params.client.rest as { get?: (route: string) => Promise<unknown> } | undefined;
|
||||
@@ -346,7 +360,7 @@ async function hydrateDiscordMessageIfEmpty(params: {
|
||||
if (!fetched) {
|
||||
return params.message;
|
||||
}
|
||||
logVerbose(`discord: hydrated empty inbound payload via REST for ${params.message.id}`);
|
||||
logVerbose(`discord: hydrated inbound payload via REST for ${params.message.id}`);
|
||||
return mergeFetchedDiscordMessage(params.message, fetched);
|
||||
} catch (err) {
|
||||
logVerbose(`discord: failed to hydrate message ${params.message.id}: ${String(err)}`);
|
||||
@@ -383,7 +397,7 @@ export async function preflightDiscordMessage(
|
||||
return null;
|
||||
}
|
||||
|
||||
message = await hydrateDiscordMessageIfEmpty({
|
||||
message = await hydrateDiscordMessageIfNeeded({
|
||||
client: params.client,
|
||||
message,
|
||||
messageChannelId,
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { listQaRunnerCliContributions } from "openclaw/plugin-sdk/qa-runner-runtime";
|
||||
import { discordQaCliRegistration } from "./discord/cli.js";
|
||||
import type { LiveTransportQaCliRegistration } from "./shared/live-transport-cli.js";
|
||||
import { telegramQaCliRegistration } from "./telegram/cli.js";
|
||||
|
||||
@@ -36,6 +37,7 @@ function createQaRunnerCliRegistration(
|
||||
|
||||
export const LIVE_TRANSPORT_QA_CLI_REGISTRATIONS: readonly LiveTransportQaCliRegistration[] = [
|
||||
telegramQaCliRegistration,
|
||||
discordQaCliRegistration,
|
||||
];
|
||||
|
||||
export function listLiveTransportQaCliRegistrations(): readonly LiveTransportQaCliRegistration[] {
|
||||
|
||||
23
extensions/qa-lab/src/live-transports/discord/cli.runtime.ts
Normal file
23
extensions/qa-lab/src/live-transports/discord/cli.runtime.ts
Normal file
@@ -0,0 +1,23 @@
|
||||
import type { LiveTransportQaCommandOptions } from "../shared/live-transport-cli.js";
|
||||
import {
|
||||
printLiveTransportQaArtifacts,
|
||||
resolveLiveTransportQaRunOptions,
|
||||
} from "../shared/live-transport-cli.runtime.js";
|
||||
import { runDiscordQaLive } from "./discord-live.runtime.js";
|
||||
|
||||
export async function runQaDiscordCommand(opts: LiveTransportQaCommandOptions) {
|
||||
const runOptions = resolveLiveTransportQaRunOptions(opts);
|
||||
const result = await runDiscordQaLive(runOptions);
|
||||
printLiveTransportQaArtifacts("Discord QA", {
|
||||
report: result.reportPath,
|
||||
summary: result.summaryPath,
|
||||
"observed messages": result.observedMessagesPath,
|
||||
...(result.gatewayDebugDirPath ? { "gateway debug logs": result.gatewayDebugDirPath } : {}),
|
||||
});
|
||||
if (
|
||||
!runOptions.allowFailures &&
|
||||
result.scenarios.some((scenario) => scenario.status === "fail")
|
||||
) {
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
37
extensions/qa-lab/src/live-transports/discord/cli.ts
Normal file
37
extensions/qa-lab/src/live-transports/discord/cli.ts
Normal file
@@ -0,0 +1,37 @@
|
||||
import type { Command } from "commander";
|
||||
import {
|
||||
createLazyCliRuntimeLoader,
|
||||
createLiveTransportQaCliRegistration,
|
||||
type LiveTransportQaCliRegistration,
|
||||
type LiveTransportQaCommandOptions,
|
||||
} from "../shared/live-transport-cli.js";
|
||||
|
||||
type DiscordQaCliRuntime = typeof import("./cli.runtime.js");
|
||||
|
||||
const loadDiscordQaCliRuntime = createLazyCliRuntimeLoader<DiscordQaCliRuntime>(
|
||||
() => import("./cli.runtime.js"),
|
||||
);
|
||||
|
||||
async function runQaDiscord(opts: LiveTransportQaCommandOptions) {
|
||||
const runtime = await loadDiscordQaCliRuntime();
|
||||
await runtime.runQaDiscordCommand(opts);
|
||||
}
|
||||
|
||||
export const discordQaCliRegistration: LiveTransportQaCliRegistration =
|
||||
createLiveTransportQaCliRegistration({
|
||||
commandName: "discord",
|
||||
credentialOptions: {
|
||||
sourceDescription: "Credential source for Discord QA: env or convex (default: env)",
|
||||
roleDescription:
|
||||
"Credential role for convex auth: maintainer or ci (default: ci in CI, maintainer otherwise)",
|
||||
},
|
||||
description: "Run the Discord live QA lane against a private guild bot-to-bot harness",
|
||||
outputDirHelp: "Discord QA artifact directory",
|
||||
scenarioHelp: "Run only the named Discord QA scenario (repeatable)",
|
||||
sutAccountHelp: "Temporary Discord account id inside the QA gateway config",
|
||||
run: runQaDiscord,
|
||||
});
|
||||
|
||||
export function registerDiscordQaCli(qa: Command) {
|
||||
discordQaCliRegistration.register(qa);
|
||||
}
|
||||
@@ -0,0 +1,453 @@
|
||||
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-runtime";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
LIVE_TRANSPORT_BASELINE_STANDARD_SCENARIO_IDS,
|
||||
findMissingLiveTransportStandardScenarios,
|
||||
} from "../shared/live-transport-scenarios.js";
|
||||
import { __testing } from "./discord-live.runtime.js";
|
||||
|
||||
const fetchWithSsrFGuardMock = vi.hoisted(() =>
|
||||
vi.fn(async (params: { url: string; init?: RequestInit; signal?: AbortSignal }) => ({
|
||||
response: await fetch(params.url, {
|
||||
...params.init,
|
||||
signal: params.signal,
|
||||
}),
|
||||
release: async () => {},
|
||||
})),
|
||||
);
|
||||
|
||||
vi.mock("openclaw/plugin-sdk/ssrf-runtime", async () => {
|
||||
const actual = await vi.importActual<typeof import("openclaw/plugin-sdk/ssrf-runtime")>(
|
||||
"openclaw/plugin-sdk/ssrf-runtime",
|
||||
);
|
||||
return {
|
||||
...actual,
|
||||
fetchWithSsrFGuard: fetchWithSsrFGuardMock,
|
||||
};
|
||||
});
|
||||
|
||||
describe("discord live qa runtime", () => {
|
||||
afterEach(() => {
|
||||
fetchWithSsrFGuardMock.mockClear();
|
||||
vi.restoreAllMocks();
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
it("resolves required Discord QA env vars", () => {
|
||||
expect(
|
||||
__testing.resolveDiscordQaRuntimeEnv({
|
||||
OPENCLAW_QA_DISCORD_GUILD_ID: "123456789012345678",
|
||||
OPENCLAW_QA_DISCORD_CHANNEL_ID: "223456789012345678",
|
||||
OPENCLAW_QA_DISCORD_DRIVER_BOT_TOKEN: "driver",
|
||||
OPENCLAW_QA_DISCORD_SUT_BOT_TOKEN: "sut",
|
||||
OPENCLAW_QA_DISCORD_SUT_APPLICATION_ID: "323456789012345678",
|
||||
}),
|
||||
).toEqual({
|
||||
guildId: "123456789012345678",
|
||||
channelId: "223456789012345678",
|
||||
driverBotToken: "driver",
|
||||
sutBotToken: "sut",
|
||||
sutApplicationId: "323456789012345678",
|
||||
});
|
||||
});
|
||||
|
||||
it("fails when a required Discord QA env var is missing", () => {
|
||||
expect(() =>
|
||||
__testing.resolveDiscordQaRuntimeEnv({
|
||||
OPENCLAW_QA_DISCORD_GUILD_ID: "123456789012345678",
|
||||
OPENCLAW_QA_DISCORD_CHANNEL_ID: "223456789012345678",
|
||||
OPENCLAW_QA_DISCORD_DRIVER_BOT_TOKEN: "driver",
|
||||
OPENCLAW_QA_DISCORD_SUT_BOT_TOKEN: "sut",
|
||||
}),
|
||||
).toThrow("OPENCLAW_QA_DISCORD_SUT_APPLICATION_ID");
|
||||
});
|
||||
|
||||
it("fails when Discord IDs are not snowflakes", () => {
|
||||
expect(() =>
|
||||
__testing.resolveDiscordQaRuntimeEnv({
|
||||
OPENCLAW_QA_DISCORD_GUILD_ID: "qa-guild",
|
||||
OPENCLAW_QA_DISCORD_CHANNEL_ID: "223456789012345678",
|
||||
OPENCLAW_QA_DISCORD_DRIVER_BOT_TOKEN: "driver",
|
||||
OPENCLAW_QA_DISCORD_SUT_BOT_TOKEN: "sut",
|
||||
OPENCLAW_QA_DISCORD_SUT_APPLICATION_ID: "323456789012345678",
|
||||
}),
|
||||
).toThrow("OPENCLAW_QA_DISCORD_GUILD_ID must be a Discord snowflake.");
|
||||
});
|
||||
|
||||
it("parses Discord pooled credential payloads", () => {
|
||||
expect(
|
||||
__testing.parseDiscordQaCredentialPayload({
|
||||
guildId: "123456789012345678",
|
||||
channelId: "223456789012345678",
|
||||
driverBotToken: "driver",
|
||||
sutBotToken: "sut",
|
||||
sutApplicationId: "323456789012345678",
|
||||
}),
|
||||
).toEqual({
|
||||
guildId: "123456789012345678",
|
||||
channelId: "223456789012345678",
|
||||
driverBotToken: "driver",
|
||||
sutBotToken: "sut",
|
||||
sutApplicationId: "323456789012345678",
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects Discord pooled credential payloads with bad snowflakes", () => {
|
||||
expect(() =>
|
||||
__testing.parseDiscordQaCredentialPayload({
|
||||
guildId: "123456789012345678",
|
||||
channelId: "channel",
|
||||
driverBotToken: "driver",
|
||||
sutBotToken: "sut",
|
||||
sutApplicationId: "323456789012345678",
|
||||
}),
|
||||
).toThrow("Discord credential payload_CHANNEL_ID must be a Discord snowflake.");
|
||||
});
|
||||
|
||||
it("injects a temporary Discord account into the QA gateway config", () => {
|
||||
const baseCfg: OpenClawConfig = {
|
||||
plugins: {
|
||||
allow: ["memory-core", "qa-channel"],
|
||||
entries: {
|
||||
"memory-core": { enabled: true },
|
||||
"qa-channel": { enabled: true },
|
||||
},
|
||||
},
|
||||
channels: {
|
||||
"qa-channel": {
|
||||
enabled: true,
|
||||
baseUrl: "http://127.0.0.1:43123",
|
||||
botUserId: "openclaw",
|
||||
botDisplayName: "OpenClaw QA",
|
||||
allowFrom: ["*"],
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const next = __testing.buildDiscordQaConfig(baseCfg, {
|
||||
guildId: "123456789012345678",
|
||||
channelId: "223456789012345678",
|
||||
driverBotId: "423456789012345678",
|
||||
sutAccountId: "sut",
|
||||
sutBotToken: "sut-token",
|
||||
});
|
||||
|
||||
expect(next.plugins?.allow).toContain("discord");
|
||||
expect(next.plugins?.entries?.discord).toEqual({ enabled: true });
|
||||
expect(next.channels?.discord).toEqual({
|
||||
enabled: true,
|
||||
defaultAccount: "sut",
|
||||
accounts: {
|
||||
sut: {
|
||||
enabled: true,
|
||||
token: "sut-token",
|
||||
allowBots: "mentions",
|
||||
groupPolicy: "allowlist",
|
||||
guilds: {
|
||||
"123456789012345678": {
|
||||
requireMention: true,
|
||||
users: ["423456789012345678"],
|
||||
channels: {
|
||||
"223456789012345678": {
|
||||
enabled: true,
|
||||
requireMention: true,
|
||||
users: ["423456789012345678"],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("normalizes observed Discord messages", () => {
|
||||
expect(
|
||||
__testing.normalizeDiscordObservedMessage({
|
||||
id: "523456789012345678",
|
||||
channel_id: "223456789012345678",
|
||||
guild_id: "123456789012345678",
|
||||
content: "hello",
|
||||
timestamp: "2026-04-22T12:00:00.000Z",
|
||||
author: {
|
||||
id: "423456789012345678",
|
||||
username: "driver",
|
||||
bot: true,
|
||||
},
|
||||
referenced_message: { id: "323456789012345678" },
|
||||
}),
|
||||
).toEqual({
|
||||
messageId: "523456789012345678",
|
||||
channelId: "223456789012345678",
|
||||
guildId: "123456789012345678",
|
||||
senderId: "423456789012345678",
|
||||
senderIsBot: true,
|
||||
senderUsername: "driver",
|
||||
text: "hello",
|
||||
replyToMessageId: "323456789012345678",
|
||||
timestamp: "2026-04-22T12:00:00.000Z",
|
||||
});
|
||||
});
|
||||
|
||||
it("matches Discord scenario replies by SUT id and marker", () => {
|
||||
expect(
|
||||
__testing.matchesDiscordScenarioReply({
|
||||
channelId: "223456789012345678",
|
||||
sutBotId: "323456789012345678",
|
||||
matchText: "DISCORD_QA_ECHO_TOKEN",
|
||||
message: {
|
||||
messageId: "523456789012345678",
|
||||
channelId: "223456789012345678",
|
||||
senderId: "323456789012345678",
|
||||
senderIsBot: true,
|
||||
text: "reply DISCORD_QA_ECHO_TOKEN",
|
||||
},
|
||||
}),
|
||||
).toBe(true);
|
||||
expect(
|
||||
__testing.matchesDiscordScenarioReply({
|
||||
channelId: "223456789012345678",
|
||||
sutBotId: "323456789012345678",
|
||||
matchText: "DISCORD_QA_ECHO_TOKEN",
|
||||
message: {
|
||||
messageId: "523456789012345679",
|
||||
channelId: "223456789012345678",
|
||||
senderId: "423456789012345678",
|
||||
senderIsBot: true,
|
||||
text: "reply DISCORD_QA_ECHO_TOKEN",
|
||||
},
|
||||
}),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("includes the Discord live scenarios", () => {
|
||||
expect(__testing.findScenario().map((scenario) => scenario.id)).toEqual([
|
||||
"discord-canary",
|
||||
"discord-mention-gating",
|
||||
"discord-native-help-command-registration",
|
||||
]);
|
||||
});
|
||||
|
||||
it("waits for the Discord account to become connected, not just running", async () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
const gateway = {
|
||||
call: vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce({
|
||||
channelAccounts: {
|
||||
discord: [
|
||||
{ accountId: "sut", running: true, connected: false, restartPending: false },
|
||||
],
|
||||
},
|
||||
})
|
||||
.mockResolvedValueOnce({
|
||||
channelAccounts: {
|
||||
discord: [
|
||||
{ accountId: "sut", running: true, connected: true, restartPending: false },
|
||||
],
|
||||
},
|
||||
}),
|
||||
} as unknown as Parameters<typeof __testing.waitForDiscordChannelRunning>[0];
|
||||
|
||||
const readyPromise = __testing.waitForDiscordChannelRunning(gateway, "sut");
|
||||
await vi.advanceTimersByTimeAsync(600);
|
||||
|
||||
await expect(readyPromise).resolves.toBeUndefined();
|
||||
expect(gateway.call).toHaveBeenCalledTimes(2);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it("reports the last Discord status when connection readiness times out", async () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
const gateway = {
|
||||
call: vi.fn().mockResolvedValue({
|
||||
channelAccounts: {
|
||||
discord: [
|
||||
{
|
||||
accountId: "sut",
|
||||
running: true,
|
||||
connected: false,
|
||||
restartPending: false,
|
||||
lastError: null,
|
||||
lastDisconnect: { error: "runtime-not-ready" },
|
||||
},
|
||||
],
|
||||
},
|
||||
}),
|
||||
} as unknown as Parameters<typeof __testing.waitForDiscordChannelRunning>[0];
|
||||
|
||||
const readyPromise = __testing.waitForDiscordChannelRunning(gateway, "sut");
|
||||
const assertion = expect(readyPromise).rejects.toThrow(
|
||||
'discord account "sut" did not become connected (last status: running=true connected=false',
|
||||
);
|
||||
await vi.advanceTimersByTimeAsync(45_500);
|
||||
await assertion;
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it("fails when any requested Discord scenario id is unknown", () => {
|
||||
expect(() => __testing.findScenario(["discord-canary", "typo-scenario"])).toThrow(
|
||||
"unknown Discord QA scenario id(s): typo-scenario",
|
||||
);
|
||||
});
|
||||
|
||||
it("tracks Discord live coverage against the shared transport contract", () => {
|
||||
expect(__testing.DISCORD_QA_STANDARD_SCENARIO_IDS).toEqual(["canary", "mention-gating"]);
|
||||
expect(
|
||||
findMissingLiveTransportStandardScenarios({
|
||||
coveredStandardScenarioIds: __testing.DISCORD_QA_STANDARD_SCENARIO_IDS,
|
||||
expectedStandardScenarioIds: LIVE_TRANSPORT_BASELINE_STANDARD_SCENARIO_IDS,
|
||||
}),
|
||||
).toEqual(["allowlist-block", "top-level-reply-shape", "restart-resume"]);
|
||||
});
|
||||
|
||||
it("lists Discord application commands through the REST API", async () => {
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.fn(async (_input: string | URL | globalThis.Request, init?: RequestInit) => {
|
||||
expect(init?.headers).toBeInstanceOf(Headers);
|
||||
expect((init?.headers as Headers).get("authorization")).toBe("Bot token");
|
||||
return new Response(
|
||||
JSON.stringify([
|
||||
{ id: "623456789012345678", name: "help" },
|
||||
{ id: "623456789012345679", name: "commands" },
|
||||
]),
|
||||
{
|
||||
status: 200,
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
},
|
||||
},
|
||||
);
|
||||
}),
|
||||
);
|
||||
|
||||
await expect(
|
||||
__testing.listApplicationCommands({
|
||||
token: "token",
|
||||
applicationId: "323456789012345678",
|
||||
}),
|
||||
).resolves.toEqual([
|
||||
{ id: "623456789012345678", name: "help" },
|
||||
{ id: "623456789012345679", name: "commands" },
|
||||
]);
|
||||
});
|
||||
|
||||
it("waits for required Discord application commands to be registered", async () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce(
|
||||
new Response(JSON.stringify([{ id: "623456789012345679", name: "commands" }]), {
|
||||
status: 200,
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
},
|
||||
}),
|
||||
)
|
||||
.mockResolvedValueOnce(
|
||||
new Response(
|
||||
JSON.stringify([
|
||||
{ id: "623456789012345679", name: "commands" },
|
||||
{ id: "623456789012345678", name: "help" },
|
||||
]),
|
||||
{
|
||||
status: 200,
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
},
|
||||
},
|
||||
),
|
||||
),
|
||||
);
|
||||
|
||||
const registeredPromise = __testing.assertDiscordApplicationCommandsRegistered({
|
||||
token: "token",
|
||||
applicationId: "323456789012345678",
|
||||
expectedCommandNames: ["help"],
|
||||
timeoutMs: 5_000,
|
||||
});
|
||||
await vi.advanceTimersByTimeAsync(1_100);
|
||||
|
||||
await expect(registeredPromise).resolves.toEqual({
|
||||
commandNames: ["commands", "help"],
|
||||
});
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it("adds an abort deadline to Discord API requests", async () => {
|
||||
const controller = new AbortController();
|
||||
const timeoutSpy = vi.spyOn(AbortSignal, "timeout").mockReturnValue(controller.signal);
|
||||
let signal: AbortSignal | undefined;
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.fn(async (_input: string | URL | globalThis.Request, init?: RequestInit) => {
|
||||
signal = init?.signal as AbortSignal | undefined;
|
||||
return new Response(JSON.stringify({ id: "423456789012345678" }), {
|
||||
status: 200,
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
},
|
||||
});
|
||||
}),
|
||||
);
|
||||
|
||||
await expect(
|
||||
__testing.callDiscordApi({
|
||||
token: "token",
|
||||
path: "/users/@me",
|
||||
timeoutMs: 25,
|
||||
}),
|
||||
).resolves.toEqual({
|
||||
id: "423456789012345678",
|
||||
});
|
||||
expect(timeoutSpy).toHaveBeenCalledWith(25);
|
||||
expect(signal).toBe(controller.signal);
|
||||
expect(signal?.aborted).toBe(false);
|
||||
controller.abort();
|
||||
expect(signal?.aborted).toBe(true);
|
||||
});
|
||||
|
||||
it("redacts observed message content by default in artifacts", () => {
|
||||
expect(
|
||||
__testing.buildObservedMessagesArtifact({
|
||||
includeContent: false,
|
||||
redactMetadata: false,
|
||||
observedMessages: [
|
||||
{
|
||||
messageId: "523456789012345678",
|
||||
channelId: "223456789012345678",
|
||||
guildId: "123456789012345678",
|
||||
senderId: "323456789012345678",
|
||||
senderIsBot: true,
|
||||
senderUsername: "sut",
|
||||
text: "secret text",
|
||||
timestamp: "2026-04-22T12:00:00.000Z",
|
||||
},
|
||||
],
|
||||
}),
|
||||
).toEqual([
|
||||
{
|
||||
messageId: "523456789012345678",
|
||||
channelId: "223456789012345678",
|
||||
guildId: "123456789012345678",
|
||||
senderId: "323456789012345678",
|
||||
senderIsBot: true,
|
||||
senderUsername: "sut",
|
||||
replyToMessageId: undefined,
|
||||
timestamp: "2026-04-22T12:00:00.000Z",
|
||||
},
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,978 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-runtime";
|
||||
import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
|
||||
import { fetchWithSsrFGuard } from "openclaw/plugin-sdk/ssrf-runtime";
|
||||
import { z } from "zod";
|
||||
import { startQaGatewayChild } from "../../gateway-child.js";
|
||||
import { DEFAULT_QA_LIVE_PROVIDER_MODE } from "../../providers/index.js";
|
||||
import {
|
||||
defaultQaModelForMode,
|
||||
normalizeQaProviderMode,
|
||||
type QaProviderModeInput,
|
||||
} from "../../run-config.js";
|
||||
import {
|
||||
acquireQaCredentialLease,
|
||||
startQaCredentialLeaseHeartbeat,
|
||||
type QaCredentialRole,
|
||||
} from "../shared/credential-lease.runtime.js";
|
||||
import { startQaLiveLaneGateway } from "../shared/live-gateway.runtime.js";
|
||||
import { appendLiveLaneIssue, buildLiveLaneArtifactsError } from "../shared/live-lane-helpers.js";
|
||||
import {
|
||||
collectLiveTransportStandardScenarioCoverage,
|
||||
selectLiveTransportScenarios,
|
||||
type LiveTransportScenarioDefinition,
|
||||
} from "../shared/live-transport-scenarios.js";
|
||||
|
||||
type DiscordQaRuntimeEnv = {
|
||||
guildId: string;
|
||||
channelId: string;
|
||||
driverBotToken: string;
|
||||
sutBotToken: string;
|
||||
sutApplicationId: string;
|
||||
};
|
||||
|
||||
type DiscordQaScenarioId =
|
||||
| "discord-canary"
|
||||
| "discord-mention-gating"
|
||||
| "discord-native-help-command-registration";
|
||||
|
||||
type DiscordQaScenarioRun =
|
||||
| {
|
||||
kind: "channel-message";
|
||||
expectReply: boolean;
|
||||
input: string;
|
||||
expectedTextIncludes?: string[];
|
||||
matchText?: string;
|
||||
}
|
||||
| {
|
||||
kind: "application-command-registration";
|
||||
expectedCommandNames: string[];
|
||||
};
|
||||
|
||||
type DiscordQaScenarioDefinition = LiveTransportScenarioDefinition<DiscordQaScenarioId> & {
|
||||
buildRun: (sutApplicationId: string) => DiscordQaScenarioRun;
|
||||
};
|
||||
|
||||
type DiscordUser = {
|
||||
id: string;
|
||||
username?: string;
|
||||
bot?: boolean;
|
||||
};
|
||||
|
||||
type DiscordMessage = {
|
||||
id: string;
|
||||
channel_id: string;
|
||||
guild_id?: string;
|
||||
content?: string;
|
||||
timestamp?: string;
|
||||
author?: DiscordUser;
|
||||
referenced_message?: { id?: string } | null;
|
||||
};
|
||||
|
||||
type DiscordApplicationCommand = {
|
||||
id: string;
|
||||
name?: string;
|
||||
};
|
||||
|
||||
type DiscordObservedMessage = {
|
||||
messageId: string;
|
||||
channelId: string;
|
||||
guildId?: string;
|
||||
senderId: string;
|
||||
senderIsBot: boolean;
|
||||
senderUsername?: string;
|
||||
scenarioId?: string;
|
||||
scenarioTitle?: string;
|
||||
matchedScenario?: boolean;
|
||||
text: string;
|
||||
replyToMessageId?: string;
|
||||
timestamp?: string;
|
||||
};
|
||||
|
||||
type DiscordObservedMessageArtifact = {
|
||||
messageId?: string;
|
||||
channelId?: string;
|
||||
guildId?: string;
|
||||
senderId?: string;
|
||||
senderIsBot: boolean;
|
||||
senderUsername?: string;
|
||||
scenarioId?: string;
|
||||
scenarioTitle?: string;
|
||||
matchedScenario?: boolean;
|
||||
text?: string;
|
||||
replyToMessageId?: string;
|
||||
timestamp?: string;
|
||||
};
|
||||
|
||||
type DiscordQaScenarioResult = {
|
||||
id: string;
|
||||
title: string;
|
||||
status: "pass" | "fail";
|
||||
details: string;
|
||||
};
|
||||
|
||||
export type DiscordQaRunResult = {
|
||||
outputDir: string;
|
||||
reportPath: string;
|
||||
summaryPath: string;
|
||||
observedMessagesPath: string;
|
||||
gatewayDebugDirPath?: string;
|
||||
scenarios: DiscordQaScenarioResult[];
|
||||
};
|
||||
|
||||
type DiscordQaSummary = {
|
||||
credentials: {
|
||||
credentialId?: string;
|
||||
kind: string;
|
||||
ownerId?: string;
|
||||
role?: QaCredentialRole;
|
||||
source: "convex" | "env";
|
||||
};
|
||||
guildId: string;
|
||||
channelId: string;
|
||||
startedAt: string;
|
||||
finishedAt: string;
|
||||
cleanupIssues: string[];
|
||||
counts: {
|
||||
total: number;
|
||||
passed: number;
|
||||
failed: number;
|
||||
};
|
||||
scenarios: DiscordQaScenarioResult[];
|
||||
};
|
||||
|
||||
const DISCORD_API_BASE_URL = "https://discord.com/api/v10";
|
||||
const DISCORD_QA_CAPTURE_CONTENT_ENV = "OPENCLAW_QA_DISCORD_CAPTURE_CONTENT";
|
||||
const QA_REDACT_PUBLIC_METADATA_ENV = "OPENCLAW_QA_REDACT_PUBLIC_METADATA";
|
||||
const DISCORD_QA_ENV_KEYS = [
|
||||
"OPENCLAW_QA_DISCORD_GUILD_ID",
|
||||
"OPENCLAW_QA_DISCORD_CHANNEL_ID",
|
||||
"OPENCLAW_QA_DISCORD_DRIVER_BOT_TOKEN",
|
||||
"OPENCLAW_QA_DISCORD_SUT_BOT_TOKEN",
|
||||
"OPENCLAW_QA_DISCORD_SUT_APPLICATION_ID",
|
||||
] as const;
|
||||
|
||||
const DISCORD_QA_SCENARIOS: DiscordQaScenarioDefinition[] = [
|
||||
{
|
||||
id: "discord-canary",
|
||||
standardId: "canary",
|
||||
title: "Discord canary echo",
|
||||
timeoutMs: 45_000,
|
||||
buildRun: (sutApplicationId) => {
|
||||
const token = `DISCORD_QA_ECHO_${randomUUID().slice(0, 8).toUpperCase()}`;
|
||||
return {
|
||||
kind: "channel-message",
|
||||
expectReply: true,
|
||||
input: `<@${sutApplicationId}> reply with only this exact marker: ${token}`,
|
||||
expectedTextIncludes: [token],
|
||||
matchText: token,
|
||||
};
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "discord-mention-gating",
|
||||
standardId: "mention-gating",
|
||||
title: "Discord unmentioned message does not trigger",
|
||||
timeoutMs: 8_000,
|
||||
buildRun: () => {
|
||||
const token = `DISCORD_QA_NOMENTION_${randomUUID().slice(0, 8).toUpperCase()}`;
|
||||
return {
|
||||
kind: "channel-message",
|
||||
expectReply: false,
|
||||
input: `reply with only this exact marker: ${token}`,
|
||||
matchText: token,
|
||||
};
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "discord-native-help-command-registration",
|
||||
title: "Discord native help command is registered",
|
||||
timeoutMs: 45_000,
|
||||
buildRun: () => ({
|
||||
kind: "application-command-registration",
|
||||
expectedCommandNames: ["help"],
|
||||
}),
|
||||
},
|
||||
];
|
||||
|
||||
export const DISCORD_QA_STANDARD_SCENARIO_IDS = collectLiveTransportStandardScenarioCoverage({
|
||||
scenarios: DISCORD_QA_SCENARIOS,
|
||||
});
|
||||
|
||||
const discordQaCredentialPayloadSchema = z.object({
|
||||
guildId: z.string().trim().min(1),
|
||||
channelId: z.string().trim().min(1),
|
||||
driverBotToken: z.string().trim().min(1),
|
||||
sutBotToken: z.string().trim().min(1),
|
||||
sutApplicationId: z.string().trim().min(1),
|
||||
});
|
||||
|
||||
function isDiscordSnowflake(value: string) {
|
||||
return /^\d{17,20}$/u.test(value);
|
||||
}
|
||||
|
||||
function assertDiscordSnowflake(value: string, label: string) {
|
||||
if (!isDiscordSnowflake(value)) {
|
||||
throw new Error(`${label} must be a Discord snowflake.`);
|
||||
}
|
||||
}
|
||||
|
||||
function resolveEnvValue(env: NodeJS.ProcessEnv, key: (typeof DISCORD_QA_ENV_KEYS)[number]) {
|
||||
const value = env[key]?.trim();
|
||||
if (!value) {
|
||||
throw new Error(`Missing ${key}.`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function isTruthyOptIn(value: string | undefined) {
|
||||
const normalized = value?.trim().toLowerCase();
|
||||
return normalized === "1" || normalized === "true" || normalized === "yes";
|
||||
}
|
||||
|
||||
export function resolveDiscordQaRuntimeEnv(
|
||||
env: NodeJS.ProcessEnv = process.env,
|
||||
): DiscordQaRuntimeEnv {
|
||||
const runtimeEnv = {
|
||||
guildId: resolveEnvValue(env, "OPENCLAW_QA_DISCORD_GUILD_ID"),
|
||||
channelId: resolveEnvValue(env, "OPENCLAW_QA_DISCORD_CHANNEL_ID"),
|
||||
driverBotToken: resolveEnvValue(env, "OPENCLAW_QA_DISCORD_DRIVER_BOT_TOKEN"),
|
||||
sutBotToken: resolveEnvValue(env, "OPENCLAW_QA_DISCORD_SUT_BOT_TOKEN"),
|
||||
sutApplicationId: resolveEnvValue(env, "OPENCLAW_QA_DISCORD_SUT_APPLICATION_ID"),
|
||||
};
|
||||
validateDiscordQaRuntimeEnv(runtimeEnv, "OPENCLAW_QA_DISCORD");
|
||||
return runtimeEnv;
|
||||
}
|
||||
|
||||
function validateDiscordQaRuntimeEnv(runtimeEnv: DiscordQaRuntimeEnv, prefix: string) {
|
||||
assertDiscordSnowflake(runtimeEnv.guildId, `${prefix}_GUILD_ID`);
|
||||
assertDiscordSnowflake(runtimeEnv.channelId, `${prefix}_CHANNEL_ID`);
|
||||
assertDiscordSnowflake(runtimeEnv.sutApplicationId, `${prefix}_SUT_APPLICATION_ID`);
|
||||
}
|
||||
|
||||
function parseDiscordQaCredentialPayload(payload: unknown): DiscordQaRuntimeEnv {
|
||||
const parsed = discordQaCredentialPayloadSchema.parse(payload);
|
||||
const runtimeEnv = {
|
||||
guildId: parsed.guildId,
|
||||
channelId: parsed.channelId,
|
||||
driverBotToken: parsed.driverBotToken,
|
||||
sutBotToken: parsed.sutBotToken,
|
||||
sutApplicationId: parsed.sutApplicationId,
|
||||
};
|
||||
validateDiscordQaRuntimeEnv(runtimeEnv, "Discord credential payload");
|
||||
return runtimeEnv;
|
||||
}
|
||||
|
||||
function buildDiscordQaConfig(
|
||||
baseCfg: OpenClawConfig,
|
||||
params: {
|
||||
guildId: string;
|
||||
channelId: string;
|
||||
driverBotId: string;
|
||||
sutAccountId: string;
|
||||
sutBotToken: string;
|
||||
},
|
||||
): OpenClawConfig {
|
||||
const pluginAllow = [...new Set([...(baseCfg.plugins?.allow ?? []), "discord"])];
|
||||
const pluginEntries = {
|
||||
...baseCfg.plugins?.entries,
|
||||
discord: { enabled: true },
|
||||
};
|
||||
return {
|
||||
...baseCfg,
|
||||
plugins: {
|
||||
...baseCfg.plugins,
|
||||
allow: pluginAllow,
|
||||
entries: pluginEntries,
|
||||
},
|
||||
channels: {
|
||||
...baseCfg.channels,
|
||||
discord: {
|
||||
enabled: true,
|
||||
defaultAccount: params.sutAccountId,
|
||||
accounts: {
|
||||
[params.sutAccountId]: {
|
||||
enabled: true,
|
||||
token: params.sutBotToken,
|
||||
allowBots: "mentions",
|
||||
groupPolicy: "allowlist",
|
||||
guilds: {
|
||||
[params.guildId]: {
|
||||
requireMention: true,
|
||||
users: [params.driverBotId],
|
||||
channels: {
|
||||
[params.channelId]: {
|
||||
enabled: true,
|
||||
requireMention: true,
|
||||
users: [params.driverBotId],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function callDiscordApi<T>(params: {
|
||||
token: string;
|
||||
path: string;
|
||||
init?: RequestInit;
|
||||
timeoutMs?: number;
|
||||
}): Promise<T> {
|
||||
const headers = new Headers(params.init?.headers);
|
||||
headers.set("authorization", `Bot ${params.token}`);
|
||||
if (params.init?.body) {
|
||||
headers.set("content-type", "application/json");
|
||||
}
|
||||
const { response, release } = await fetchWithSsrFGuard({
|
||||
url: `${DISCORD_API_BASE_URL}${params.path}`,
|
||||
init: {
|
||||
...params.init,
|
||||
headers,
|
||||
},
|
||||
signal: AbortSignal.timeout(params.timeoutMs ?? 15_000),
|
||||
policy: { hostnameAllowlist: ["discord.com"] },
|
||||
auditContext: "qa-lab-discord-live",
|
||||
});
|
||||
try {
|
||||
const text = await response.text();
|
||||
const payload = text.trim() ? (JSON.parse(text) as unknown) : undefined;
|
||||
if (!response.ok) {
|
||||
const message =
|
||||
typeof payload === "object" &&
|
||||
payload !== null &&
|
||||
typeof (payload as { message?: unknown }).message === "string"
|
||||
? (payload as { message: string }).message
|
||||
: text.trim();
|
||||
throw new Error(
|
||||
message || `Discord API ${params.path} failed with status ${response.status}`,
|
||||
);
|
||||
}
|
||||
return payload as T;
|
||||
} finally {
|
||||
await release();
|
||||
}
|
||||
}
|
||||
|
||||
async function getCurrentDiscordUser(token: string) {
|
||||
return await callDiscordApi<DiscordUser>({
|
||||
token,
|
||||
path: "/users/@me",
|
||||
});
|
||||
}
|
||||
|
||||
async function sendChannelMessage(token: string, channelId: string, content: string) {
|
||||
return await callDiscordApi<DiscordMessage>({
|
||||
token,
|
||||
path: `/channels/${channelId}/messages`,
|
||||
init: {
|
||||
method: "POST",
|
||||
body: JSON.stringify({
|
||||
content,
|
||||
allowed_mentions: {
|
||||
parse: ["users"],
|
||||
},
|
||||
}),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
async function listChannelMessagesAfter(params: {
|
||||
token: string;
|
||||
channelId: string;
|
||||
afterSnowflake: string;
|
||||
}) {
|
||||
const query = new URLSearchParams({
|
||||
after: params.afterSnowflake,
|
||||
limit: "50",
|
||||
});
|
||||
return await callDiscordApi<DiscordMessage[]>({
|
||||
token: params.token,
|
||||
path: `/channels/${params.channelId}/messages?${query.toString()}`,
|
||||
});
|
||||
}
|
||||
|
||||
async function listApplicationCommands(params: { token: string; applicationId: string }) {
|
||||
return await callDiscordApi<DiscordApplicationCommand[]>({
|
||||
token: params.token,
|
||||
path: `/applications/${params.applicationId}/commands`,
|
||||
});
|
||||
}
|
||||
|
||||
function compareDiscordSnowflakes(a: string, b: string) {
|
||||
const left = BigInt(a);
|
||||
const right = BigInt(b);
|
||||
return left < right ? -1 : left > right ? 1 : 0;
|
||||
}
|
||||
|
||||
function normalizeDiscordObservedMessage(message: DiscordMessage): DiscordObservedMessage | null {
|
||||
if (!message.author?.id) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
messageId: message.id,
|
||||
channelId: message.channel_id,
|
||||
guildId: message.guild_id,
|
||||
senderId: message.author.id,
|
||||
senderIsBot: message.author.bot === true,
|
||||
senderUsername: message.author.username,
|
||||
text: message.content ?? "",
|
||||
replyToMessageId: message.referenced_message?.id,
|
||||
timestamp: message.timestamp,
|
||||
};
|
||||
}
|
||||
|
||||
async function pollChannelMessages(params: {
|
||||
token: string;
|
||||
channelId: string;
|
||||
afterSnowflake: string;
|
||||
timeoutMs: number;
|
||||
predicate: (message: DiscordObservedMessage) => boolean;
|
||||
observedMessages: DiscordObservedMessage[];
|
||||
observationScenarioId: string;
|
||||
observationScenarioTitle: string;
|
||||
}) {
|
||||
const startedAt = Date.now();
|
||||
let afterSnowflake = params.afterSnowflake;
|
||||
while (Date.now() - startedAt < params.timeoutMs) {
|
||||
const messages = await listChannelMessagesAfter({
|
||||
token: params.token,
|
||||
channelId: params.channelId,
|
||||
afterSnowflake,
|
||||
});
|
||||
const sorted = messages
|
||||
.filter((message) => isDiscordSnowflake(message.id))
|
||||
.toSorted((a, b) => compareDiscordSnowflakes(a.id, b.id));
|
||||
for (const message of sorted) {
|
||||
afterSnowflake = message.id;
|
||||
const normalized = normalizeDiscordObservedMessage(message);
|
||||
if (!normalized) {
|
||||
continue;
|
||||
}
|
||||
const matchedScenario = params.predicate(normalized);
|
||||
const observedMessage: DiscordObservedMessage = {
|
||||
...normalized,
|
||||
scenarioId: params.observationScenarioId,
|
||||
scenarioTitle: params.observationScenarioTitle,
|
||||
matchedScenario,
|
||||
};
|
||||
params.observedMessages.push(observedMessage);
|
||||
if (matchedScenario) {
|
||||
return { message: observedMessage, afterSnowflake };
|
||||
}
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 1_000));
|
||||
}
|
||||
throw new Error(`timed out after ${params.timeoutMs}ms waiting for Discord message`);
|
||||
}
|
||||
|
||||
async function waitForDiscordChannelRunning(
|
||||
gateway: Awaited<ReturnType<typeof startQaGatewayChild>>,
|
||||
accountId: string,
|
||||
) {
|
||||
const startedAt = Date.now();
|
||||
let lastStatus:
|
||||
| {
|
||||
running?: boolean;
|
||||
connected?: boolean;
|
||||
restartPending?: boolean;
|
||||
lastConnectedAt?: number;
|
||||
lastDisconnect?: unknown;
|
||||
lastError?: string;
|
||||
}
|
||||
| undefined;
|
||||
while (Date.now() - startedAt < 45_000) {
|
||||
try {
|
||||
const payload = (await gateway.call(
|
||||
"channels.status",
|
||||
{ probe: false, timeoutMs: 2_000 },
|
||||
{ timeoutMs: 5_000 },
|
||||
)) as {
|
||||
channelAccounts?: Record<
|
||||
string,
|
||||
Array<{
|
||||
accountId?: string;
|
||||
running?: boolean;
|
||||
connected?: boolean;
|
||||
restartPending?: boolean;
|
||||
lastConnectedAt?: number;
|
||||
lastDisconnect?: unknown;
|
||||
lastError?: string;
|
||||
}>
|
||||
>;
|
||||
};
|
||||
const accounts = payload.channelAccounts?.discord ?? [];
|
||||
const match = accounts.find((entry) => entry.accountId === accountId);
|
||||
lastStatus = match
|
||||
? {
|
||||
running: match.running,
|
||||
connected: match.connected,
|
||||
restartPending: match.restartPending,
|
||||
lastConnectedAt: match.lastConnectedAt,
|
||||
lastDisconnect: match.lastDisconnect,
|
||||
lastError: match.lastError,
|
||||
}
|
||||
: undefined;
|
||||
if (match?.running && match.connected === true && match.restartPending !== true) {
|
||||
return;
|
||||
}
|
||||
} catch {
|
||||
// retry
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 500));
|
||||
}
|
||||
const details = lastStatus
|
||||
? ` (last status: running=${String(lastStatus.running)} connected=${String(lastStatus.connected)} restartPending=${String(lastStatus.restartPending)} lastConnectedAt=${String(lastStatus.lastConnectedAt)} lastError=${lastStatus.lastError ?? "null"} lastDisconnect=${JSON.stringify(lastStatus.lastDisconnect)})`
|
||||
: "";
|
||||
throw new Error(`discord account "${accountId}" did not become connected${details}`);
|
||||
}
|
||||
|
||||
function renderDiscordQaMarkdown(params: {
|
||||
cleanupIssues: string[];
|
||||
credentialSource: "convex" | "env";
|
||||
redactMetadata: boolean;
|
||||
guildId: string;
|
||||
channelId: string;
|
||||
gatewayDebugDirPath?: string;
|
||||
startedAt: string;
|
||||
finishedAt: string;
|
||||
scenarios: DiscordQaScenarioResult[];
|
||||
}) {
|
||||
const lines = [
|
||||
"# Discord QA Report",
|
||||
"",
|
||||
`- Credential source: \`${params.credentialSource}\``,
|
||||
`- Guild: \`${params.guildId}\``,
|
||||
`- Channel: \`${params.channelId}\``,
|
||||
`- Metadata redaction: \`${params.redactMetadata ? "enabled" : "disabled"}\``,
|
||||
`- Started: ${params.startedAt}`,
|
||||
`- Finished: ${params.finishedAt}`,
|
||||
"",
|
||||
"## Scenarios",
|
||||
"",
|
||||
];
|
||||
for (const scenario of params.scenarios) {
|
||||
lines.push(`### ${scenario.title}`);
|
||||
lines.push("");
|
||||
lines.push(`- Status: ${scenario.status}`);
|
||||
lines.push(`- Details: ${scenario.details}`);
|
||||
lines.push("");
|
||||
}
|
||||
if (params.gatewayDebugDirPath) {
|
||||
lines.push("## Gateway Debug Logs");
|
||||
lines.push("");
|
||||
lines.push(`- Preserved at: \`${params.gatewayDebugDirPath}\``);
|
||||
lines.push("");
|
||||
}
|
||||
if (params.cleanupIssues.length > 0) {
|
||||
lines.push("## Cleanup");
|
||||
lines.push("");
|
||||
for (const issue of params.cleanupIssues) {
|
||||
lines.push(`- ${issue}`);
|
||||
}
|
||||
lines.push("");
|
||||
}
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
function buildObservedMessagesArtifact(params: {
|
||||
observedMessages: DiscordObservedMessage[];
|
||||
includeContent: boolean;
|
||||
redactMetadata: boolean;
|
||||
}) {
|
||||
return params.observedMessages.map<DiscordObservedMessageArtifact>((message) => {
|
||||
const scenarioContext = {
|
||||
...(message.scenarioId ? { scenarioId: message.scenarioId } : {}),
|
||||
...(message.scenarioTitle ? { scenarioTitle: message.scenarioTitle } : {}),
|
||||
...(typeof message.matchedScenario === "boolean"
|
||||
? { matchedScenario: message.matchedScenario }
|
||||
: {}),
|
||||
};
|
||||
const base = params.redactMetadata
|
||||
? {
|
||||
...scenarioContext,
|
||||
senderIsBot: message.senderIsBot,
|
||||
}
|
||||
: {
|
||||
...scenarioContext,
|
||||
messageId: message.messageId,
|
||||
channelId: message.channelId,
|
||||
guildId: message.guildId,
|
||||
senderId: message.senderId,
|
||||
senderIsBot: message.senderIsBot,
|
||||
senderUsername: message.senderUsername,
|
||||
replyToMessageId: message.replyToMessageId,
|
||||
timestamp: message.timestamp,
|
||||
};
|
||||
if (!params.includeContent) {
|
||||
return base;
|
||||
}
|
||||
return {
|
||||
...base,
|
||||
text: message.text,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
function findScenario(ids?: string[]) {
|
||||
return selectLiveTransportScenarios({
|
||||
ids,
|
||||
laneLabel: "Discord",
|
||||
scenarios: DISCORD_QA_SCENARIOS,
|
||||
});
|
||||
}
|
||||
|
||||
function matchesDiscordScenarioReply(params: {
|
||||
channelId: string;
|
||||
message: DiscordObservedMessage;
|
||||
matchText?: string;
|
||||
sutBotId: string;
|
||||
}) {
|
||||
return (
|
||||
params.message.channelId === params.channelId &&
|
||||
params.message.senderId === params.sutBotId &&
|
||||
Boolean(params.matchText && params.message.text.includes(params.matchText))
|
||||
);
|
||||
}
|
||||
|
||||
function assertDiscordScenarioReply(params: {
|
||||
expectedTextIncludes?: string[];
|
||||
message: DiscordObservedMessage;
|
||||
}) {
|
||||
if (!params.message.text.trim()) {
|
||||
throw new Error(`reply message ${params.message.messageId} was empty`);
|
||||
}
|
||||
for (const expected of params.expectedTextIncludes ?? []) {
|
||||
if (!params.message.text.includes(expected)) {
|
||||
throw new Error(
|
||||
`reply message ${params.message.messageId} missing expected text: ${expected}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function assertDiscordApplicationCommandsRegistered(params: {
|
||||
applicationId: string;
|
||||
expectedCommandNames: string[];
|
||||
timeoutMs: number;
|
||||
token: string;
|
||||
}) {
|
||||
const startedAt = Date.now();
|
||||
let lastNames: string[] = [];
|
||||
while (Date.now() - startedAt < params.timeoutMs) {
|
||||
const commands = await listApplicationCommands({
|
||||
token: params.token,
|
||||
applicationId: params.applicationId,
|
||||
});
|
||||
lastNames = commands
|
||||
.map((command) => command.name ?? "")
|
||||
.filter(Boolean)
|
||||
.toSorted();
|
||||
const nameSet = new Set(lastNames);
|
||||
const missing = params.expectedCommandNames.filter((name) => !nameSet.has(name));
|
||||
if (missing.length === 0) {
|
||||
return { commandNames: lastNames };
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 1_000));
|
||||
}
|
||||
throw new Error(
|
||||
`missing Discord native command(s): ${params.expectedCommandNames
|
||||
.filter((name) => !lastNames.includes(name))
|
||||
.join(", ")} (registered: ${lastNames.join(", ") || "none"})`,
|
||||
);
|
||||
}
|
||||
|
||||
export async function runDiscordQaLive(params: {
|
||||
repoRoot?: string;
|
||||
outputDir?: string;
|
||||
providerMode?: QaProviderModeInput;
|
||||
primaryModel?: string;
|
||||
alternateModel?: string;
|
||||
fastMode?: boolean;
|
||||
scenarioIds?: string[];
|
||||
sutAccountId?: string;
|
||||
credentialSource?: string;
|
||||
credentialRole?: string;
|
||||
}): Promise<DiscordQaRunResult> {
|
||||
const repoRoot = path.resolve(params.repoRoot ?? process.cwd());
|
||||
const outputDir =
|
||||
params.outputDir ??
|
||||
path.join(repoRoot, ".artifacts", "qa-e2e", `discord-${Date.now().toString(36)}`);
|
||||
await fs.mkdir(outputDir, { recursive: true });
|
||||
|
||||
const providerMode = normalizeQaProviderMode(
|
||||
params.providerMode ?? DEFAULT_QA_LIVE_PROVIDER_MODE,
|
||||
);
|
||||
const primaryModel = params.primaryModel?.trim() || defaultQaModelForMode(providerMode);
|
||||
const alternateModel = params.alternateModel?.trim() || defaultQaModelForMode(providerMode, true);
|
||||
const sutAccountId = params.sutAccountId?.trim() || "sut";
|
||||
const scenarios = findScenario(params.scenarioIds);
|
||||
|
||||
const credentialLease = await acquireQaCredentialLease({
|
||||
kind: "discord",
|
||||
source: params.credentialSource,
|
||||
role: params.credentialRole,
|
||||
resolveEnvPayload: () => resolveDiscordQaRuntimeEnv(),
|
||||
parsePayload: parseDiscordQaCredentialPayload,
|
||||
});
|
||||
const leaseHeartbeat = startQaCredentialLeaseHeartbeat(credentialLease);
|
||||
const assertLeaseHealthy = () => {
|
||||
leaseHeartbeat.throwIfFailed();
|
||||
};
|
||||
|
||||
const runtimeEnv = credentialLease.payload;
|
||||
const observedMessages: DiscordObservedMessage[] = [];
|
||||
const redactPublicMetadata = isTruthyOptIn(process.env[QA_REDACT_PUBLIC_METADATA_ENV]);
|
||||
const includeObservedMessageContent = isTruthyOptIn(process.env[DISCORD_QA_CAPTURE_CONTENT_ENV]);
|
||||
const startedAt = new Date().toISOString();
|
||||
const scenarioResults: DiscordQaScenarioResult[] = [];
|
||||
const cleanupIssues: string[] = [];
|
||||
const gatewayDebugDirPath = path.join(outputDir, "gateway-debug");
|
||||
let preservedGatewayDebugArtifacts = false;
|
||||
try {
|
||||
const [driverIdentity, sutIdentity] = await Promise.all([
|
||||
getCurrentDiscordUser(runtimeEnv.driverBotToken),
|
||||
getCurrentDiscordUser(runtimeEnv.sutBotToken),
|
||||
]);
|
||||
if (driverIdentity.id === sutIdentity.id) {
|
||||
throw new Error("Discord QA requires two distinct bots for driver and SUT.");
|
||||
}
|
||||
if (sutIdentity.id !== runtimeEnv.sutApplicationId) {
|
||||
throw new Error(
|
||||
"Discord QA SUT application id must match the SUT bot user id returned by Discord.",
|
||||
);
|
||||
}
|
||||
|
||||
const gatewayHarness = await startQaLiveLaneGateway({
|
||||
repoRoot,
|
||||
transport: {
|
||||
requiredPluginIds: [],
|
||||
createGatewayConfig: () => ({}),
|
||||
},
|
||||
transportBaseUrl: "http://127.0.0.1:0",
|
||||
providerMode,
|
||||
primaryModel,
|
||||
alternateModel,
|
||||
fastMode: params.fastMode,
|
||||
controlUiEnabled: false,
|
||||
mutateConfig: (cfg) =>
|
||||
buildDiscordQaConfig(cfg, {
|
||||
guildId: runtimeEnv.guildId,
|
||||
channelId: runtimeEnv.channelId,
|
||||
driverBotId: driverIdentity.id,
|
||||
sutAccountId,
|
||||
sutBotToken: runtimeEnv.sutBotToken,
|
||||
}),
|
||||
});
|
||||
try {
|
||||
await waitForDiscordChannelRunning(gatewayHarness.gateway, sutAccountId);
|
||||
assertLeaseHealthy();
|
||||
for (const scenario of scenarios) {
|
||||
assertLeaseHealthy();
|
||||
const scenarioRun = scenario.buildRun(runtimeEnv.sutApplicationId);
|
||||
try {
|
||||
if (scenarioRun.kind === "application-command-registration") {
|
||||
const registered = await assertDiscordApplicationCommandsRegistered({
|
||||
token: runtimeEnv.sutBotToken,
|
||||
applicationId: runtimeEnv.sutApplicationId,
|
||||
expectedCommandNames: scenarioRun.expectedCommandNames,
|
||||
timeoutMs: scenario.timeoutMs,
|
||||
});
|
||||
scenarioResults.push({
|
||||
id: scenario.id,
|
||||
title: scenario.title,
|
||||
status: "pass",
|
||||
details: redactPublicMetadata
|
||||
? "native command registered"
|
||||
: `native command registered (${registered.commandNames.join(", ")})`,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
const sent = await sendChannelMessage(
|
||||
runtimeEnv.driverBotToken,
|
||||
runtimeEnv.channelId,
|
||||
scenarioRun.input,
|
||||
);
|
||||
const matched = await pollChannelMessages({
|
||||
token: runtimeEnv.driverBotToken,
|
||||
channelId: runtimeEnv.channelId,
|
||||
afterSnowflake: sent.id,
|
||||
timeoutMs: scenario.timeoutMs,
|
||||
observedMessages,
|
||||
observationScenarioId: scenario.id,
|
||||
observationScenarioTitle: scenario.title,
|
||||
predicate: (message) =>
|
||||
matchesDiscordScenarioReply({
|
||||
channelId: runtimeEnv.channelId,
|
||||
matchText: scenarioRun.matchText,
|
||||
message,
|
||||
sutBotId: sutIdentity.id,
|
||||
}),
|
||||
});
|
||||
if (!scenarioRun.expectReply) {
|
||||
throw new Error(`unexpected reply message ${matched.message.messageId} matched`);
|
||||
}
|
||||
assertDiscordScenarioReply({
|
||||
expectedTextIncludes: scenarioRun.expectedTextIncludes,
|
||||
message: matched.message,
|
||||
});
|
||||
scenarioResults.push({
|
||||
id: scenario.id,
|
||||
title: scenario.title,
|
||||
status: "pass",
|
||||
details: redactPublicMetadata
|
||||
? "reply matched"
|
||||
: `reply message ${matched.message.messageId} matched`,
|
||||
});
|
||||
} catch (error) {
|
||||
if (scenarioRun.kind === "channel-message" && !scenarioRun.expectReply) {
|
||||
const details = formatErrorMessage(error);
|
||||
if (details === `timed out after ${scenario.timeoutMs}ms waiting for Discord message`) {
|
||||
scenarioResults.push({
|
||||
id: scenario.id,
|
||||
title: scenario.title,
|
||||
status: "pass",
|
||||
details: "no reply",
|
||||
});
|
||||
continue;
|
||||
}
|
||||
}
|
||||
scenarioResults.push({
|
||||
id: scenario.id,
|
||||
title: scenario.title,
|
||||
status: "fail",
|
||||
details: formatErrorMessage(error),
|
||||
});
|
||||
}
|
||||
assertLeaseHealthy();
|
||||
}
|
||||
} finally {
|
||||
try {
|
||||
const shouldPreserveGatewayDebugArtifacts = scenarioResults.some(
|
||||
(scenario) => scenario.status === "fail",
|
||||
);
|
||||
await gatewayHarness.stop(
|
||||
shouldPreserveGatewayDebugArtifacts ? { preserveToDir: gatewayDebugDirPath } : undefined,
|
||||
);
|
||||
preservedGatewayDebugArtifacts = shouldPreserveGatewayDebugArtifacts;
|
||||
} catch (error) {
|
||||
appendLiveLaneIssue(cleanupIssues, "live gateway cleanup", error);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
await leaseHeartbeat.stop();
|
||||
try {
|
||||
await credentialLease.release();
|
||||
} catch (error) {
|
||||
appendLiveLaneIssue(cleanupIssues, "credential lease release", error);
|
||||
}
|
||||
}
|
||||
|
||||
const finishedAt = new Date().toISOString();
|
||||
const publishedCleanupIssues = redactPublicMetadata
|
||||
? cleanupIssues.map(() => "details redacted (OPENCLAW_QA_REDACT_PUBLIC_METADATA=1)")
|
||||
: cleanupIssues;
|
||||
const passedCount = scenarioResults.filter((entry) => entry.status === "pass").length;
|
||||
const failedCount = scenarioResults.filter((entry) => entry.status === "fail").length;
|
||||
const summary: DiscordQaSummary = {
|
||||
credentials: {
|
||||
source: credentialLease.source,
|
||||
kind: credentialLease.kind,
|
||||
role: credentialLease.role,
|
||||
ownerId: redactPublicMetadata ? undefined : credentialLease.ownerId,
|
||||
credentialId: redactPublicMetadata ? undefined : credentialLease.credentialId,
|
||||
},
|
||||
guildId: redactPublicMetadata ? "<redacted>" : runtimeEnv.guildId,
|
||||
channelId: redactPublicMetadata ? "<redacted>" : runtimeEnv.channelId,
|
||||
startedAt,
|
||||
finishedAt,
|
||||
cleanupIssues: publishedCleanupIssues,
|
||||
counts: {
|
||||
total: scenarioResults.length,
|
||||
passed: passedCount,
|
||||
failed: failedCount,
|
||||
},
|
||||
scenarios: scenarioResults,
|
||||
};
|
||||
const reportPath = path.join(outputDir, "discord-qa-report.md");
|
||||
const summaryPath = path.join(outputDir, "discord-qa-summary.json");
|
||||
const observedMessagesPath = path.join(outputDir, "discord-qa-observed-messages.json");
|
||||
await fs.writeFile(
|
||||
reportPath,
|
||||
`${renderDiscordQaMarkdown({
|
||||
cleanupIssues: publishedCleanupIssues,
|
||||
credentialSource: credentialLease.source,
|
||||
redactMetadata: redactPublicMetadata,
|
||||
guildId: redactPublicMetadata ? "<redacted>" : runtimeEnv.guildId,
|
||||
channelId: redactPublicMetadata ? "<redacted>" : runtimeEnv.channelId,
|
||||
gatewayDebugDirPath: preservedGatewayDebugArtifacts ? gatewayDebugDirPath : undefined,
|
||||
startedAt,
|
||||
finishedAt,
|
||||
scenarios: scenarioResults,
|
||||
})}\n`,
|
||||
{ encoding: "utf8", mode: 0o600 },
|
||||
);
|
||||
await fs.writeFile(summaryPath, `${JSON.stringify(summary, null, 2)}\n`, {
|
||||
encoding: "utf8",
|
||||
mode: 0o600,
|
||||
});
|
||||
await fs.writeFile(
|
||||
observedMessagesPath,
|
||||
`${JSON.stringify(
|
||||
buildObservedMessagesArtifact({
|
||||
observedMessages,
|
||||
includeContent: includeObservedMessageContent,
|
||||
redactMetadata: redactPublicMetadata,
|
||||
}),
|
||||
null,
|
||||
2,
|
||||
)}\n`,
|
||||
{ encoding: "utf8", mode: 0o600 },
|
||||
);
|
||||
const artifactPaths = {
|
||||
report: reportPath,
|
||||
summary: summaryPath,
|
||||
observedMessages: observedMessagesPath,
|
||||
...(preservedGatewayDebugArtifacts ? { gatewayDebug: gatewayDebugDirPath } : {}),
|
||||
};
|
||||
if (cleanupIssues.length > 0) {
|
||||
throw new Error(
|
||||
buildLiveLaneArtifactsError({
|
||||
heading: "Discord QA cleanup failed after artifacts were written.",
|
||||
details: publishedCleanupIssues,
|
||||
artifacts: artifactPaths,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
outputDir,
|
||||
reportPath,
|
||||
summaryPath,
|
||||
observedMessagesPath,
|
||||
...(preservedGatewayDebugArtifacts ? { gatewayDebugDirPath } : {}),
|
||||
scenarios: scenarioResults,
|
||||
};
|
||||
}
|
||||
|
||||
export const __testing = {
|
||||
DISCORD_QA_SCENARIOS,
|
||||
DISCORD_QA_STANDARD_SCENARIO_IDS,
|
||||
assertDiscordScenarioReply,
|
||||
assertDiscordApplicationCommandsRegistered,
|
||||
buildDiscordQaConfig,
|
||||
buildObservedMessagesArtifact,
|
||||
callDiscordApi,
|
||||
findScenario,
|
||||
listApplicationCommands,
|
||||
matchesDiscordScenarioReply,
|
||||
normalizeDiscordObservedMessage,
|
||||
parseDiscordQaCredentialPayload,
|
||||
resolveDiscordQaRuntimeEnv,
|
||||
waitForDiscordChannelRunning,
|
||||
};
|
||||
@@ -94,6 +94,22 @@ describe("startQaLiveLaneGateway", () => {
|
||||
expect(mockStop).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("forwards gateway stop options to the child harness", async () => {
|
||||
const harness = await startQaLiveLaneGateway({
|
||||
repoRoot: "/tmp/openclaw-repo",
|
||||
transport: createStubTransport(),
|
||||
transportBaseUrl: "http://127.0.0.1:43123",
|
||||
providerMode: "mock-openai",
|
||||
primaryModel: "mock-openai/gpt-5.4",
|
||||
alternateModel: "mock-openai/gpt-5.4-alt",
|
||||
controlUiEnabled: false,
|
||||
});
|
||||
|
||||
await harness.stop({ preserveToDir: ".artifacts/qa-e2e/debug" });
|
||||
expect(gatewayStop).toHaveBeenCalledWith({ preserveToDir: ".artifacts/qa-e2e/debug" });
|
||||
expect(mockStop).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("skips mock bootstrap for live frontier runs", async () => {
|
||||
const harness = await startQaLiveLaneGateway({
|
||||
repoRoot: "/tmp/openclaw-repo",
|
||||
|
||||
@@ -5,13 +5,16 @@ import { startQaProviderServer } from "../../providers/server-runtime.js";
|
||||
import type { QaThinkingLevel } from "../../qa-gateway-config.js";
|
||||
import { appendLiveLaneIssue } from "./live-lane-helpers.js";
|
||||
|
||||
async function stopQaLiveLaneResources(resources: {
|
||||
gateway: Awaited<ReturnType<typeof startQaGatewayChild>>;
|
||||
mock: { baseUrl: string; stop(): Promise<void> } | null;
|
||||
}) {
|
||||
async function stopQaLiveLaneResources(
|
||||
resources: {
|
||||
gateway: Awaited<ReturnType<typeof startQaGatewayChild>>;
|
||||
mock: { baseUrl: string; stop(): Promise<void> } | null;
|
||||
},
|
||||
opts?: { keepTemp?: boolean; preserveToDir?: string },
|
||||
) {
|
||||
const errors: string[] = [];
|
||||
try {
|
||||
await resources.gateway.stop();
|
||||
await resources.gateway.stop(opts);
|
||||
} catch (error) {
|
||||
appendLiveLaneIssue(errors, "gateway stop failed", error);
|
||||
}
|
||||
@@ -66,8 +69,8 @@ export async function startQaLiveLaneGateway(params: {
|
||||
return {
|
||||
gateway,
|
||||
mock,
|
||||
async stop() {
|
||||
await stopQaLiveLaneResources({ gateway, mock });
|
||||
async stop(opts?: { keepTemp?: boolean; preserveToDir?: string }) {
|
||||
await stopQaLiveLaneResources({ gateway, mock }, opts);
|
||||
},
|
||||
};
|
||||
} catch (error) {
|
||||
|
||||
Reference in New Issue
Block a user