diff --git a/.github/workflows/qa-live-telegram-convex.yml b/.github/workflows/qa-live-transports-convex.yml similarity index 78% rename from .github/workflows/qa-live-telegram-convex.yml rename to .github/workflows/qa-live-transports-convex.yml index c42fa0de2ac..762f00d75f9 100644 --- a/.github/workflows/qa-live-telegram-convex.yml +++ b/.github/workflows/qa-live-transports-convex.yml @@ -14,6 +14,10 @@ on: description: Optional comma-separated Telegram scenario ids required: false type: string + discord_scenario: + description: Optional comma-separated Discord scenario ids + required: false + type: string permissions: contents: read @@ -346,3 +350,95 @@ jobs: path: ${{ steps.run_lane.outputs.output_dir }} retention-days: 14 if-no-files-found: warn + + run_live_discord: + name: Run Discord live QA lane with Convex leases + needs: [authorize_actor, validate_selected_ref] + runs-on: blacksmith-32vcpu-ubuntu-2404 + timeout-minutes: 60 + environment: qa-live-shared + steps: + - name: Checkout selected ref + uses: actions/checkout@v6 + with: + ref: ${{ needs.validate_selected_ref.outputs.selected_sha }} + fetch-depth: 1 + + - name: Setup Node environment + uses: ./.github/actions/setup-node-env + with: + node-version: ${{ env.NODE_VERSION }} + pnpm-version: ${{ env.PNPM_VERSION }} + install-bun: "true" + + - name: Validate required QA credential env + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + OPENCLAW_QA_CONVEX_SITE_URL: ${{ secrets.OPENCLAW_QA_CONVEX_SITE_URL }} + OPENCLAW_QA_CONVEX_SECRET_CI: ${{ secrets.OPENCLAW_QA_CONVEX_SECRET_CI }} + shell: bash + run: | + set -euo pipefail + + require_var() { + local key="$1" + if [[ -z "${!key:-}" ]]; then + echo "Missing required ${key}." >&2 + exit 1 + fi + } + + require_var OPENAI_API_KEY + require_var OPENCLAW_QA_CONVEX_SITE_URL + require_var OPENCLAW_QA_CONVEX_SECRET_CI + + - name: Build private QA runtime + run: pnpm build + + - name: Run Discord live lane + id: run_lane + shell: bash + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + OPENCLAW_QA_CONVEX_SITE_URL: ${{ secrets.OPENCLAW_QA_CONVEX_SITE_URL }} + OPENCLAW_QA_CONVEX_SECRET_CI: ${{ secrets.OPENCLAW_QA_CONVEX_SECRET_CI }} + OPENCLAW_QA_REDACT_PUBLIC_METADATA: "1" + OPENCLAW_QA_DISCORD_CAPTURE_CONTENT: "1" + INPUT_SCENARIO: ${{ github.event_name == 'workflow_dispatch' && inputs.discord_scenario || '' }} + run: | + set -euo pipefail + + output_dir=".artifacts/qa-e2e/discord-live-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + scenario_args=() + + if [[ -n "${INPUT_SCENARIO// }" ]]; then + IFS=',' read -r -a raw_scenarios <<<"${INPUT_SCENARIO}" + for raw in "${raw_scenarios[@]}"; do + scenario="$(printf '%s' "${raw}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + if [[ -n "${scenario}" ]]; then + scenario_args+=(--scenario "${scenario}") + fi + done + fi + + echo "output_dir=${output_dir}" >> "$GITHUB_OUTPUT" + + pnpm openclaw qa discord \ + --repo-root . \ + --output-dir "${output_dir}" \ + --provider-mode live-frontier \ + --model openai/gpt-5.4 \ + --alt-model openai/gpt-5.4 \ + --fast \ + --credential-source convex \ + --credential-role ci \ + "${scenario_args[@]}" + + - name: Upload Discord QA artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: qa-live-discord-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ steps.run_lane.outputs.output_dir }} + retention-days: 14 + if-no-files-found: warn diff --git a/docs/concepts/qa-e2e-automation.md b/docs/concepts/qa-e2e-automation.md index af0b43e2030..eb9d63adbd5 100644 --- a/docs/concepts/qa-e2e-automation.md +++ b/docs/concepts/qa-e2e-automation.md @@ -83,16 +83,34 @@ you want artifacts without a failing exit code. The Telegram report and summary include per-reply RTT from the driver message send request to the observed SUT reply, starting with the canary. +For a transport-real Discord smoke lane, run: + +```bash +pnpm openclaw qa discord +``` + +That lane targets one real private Discord guild channel with two bots: a +driver bot controlled by the harness and a SUT bot started by the child +OpenClaw gateway through the bundled Discord plugin. It requires +`OPENCLAW_QA_DISCORD_GUILD_ID`, `OPENCLAW_QA_DISCORD_CHANNEL_ID`, +`OPENCLAW_QA_DISCORD_DRIVER_BOT_TOKEN`, `OPENCLAW_QA_DISCORD_SUT_BOT_TOKEN`, +and `OPENCLAW_QA_DISCORD_SUT_APPLICATION_ID` when using env credentials. +The lane verifies channel mention handling and checks that the SUT bot has +registered the native `/help` command with Discord. +The command exits non-zero when any scenario fails. Use `--allow-failures` when +you want artifacts without a failing exit code. + Live transport lanes now share one smaller contract instead of each inventing their own scenario list shape: `qa-channel` remains the broad synthetic product-behavior suite and is not part of the live transport coverage matrix. -| Lane | Canary | Mention gating | Allowlist block | Top-level reply | Restart resume | Thread follow-up | Thread isolation | Reaction observation | Help command | -| -------- | ------ | -------------- | --------------- | --------------- | -------------- | ---------------- | ---------------- | -------------------- | ------------ | -| Matrix | x | x | x | x | x | x | x | x | | -| Telegram | x | | | | | | | | x | +| Lane | Canary | Mention gating | Allowlist block | Top-level reply | Restart resume | Thread follow-up | Thread isolation | Reaction observation | Help command | Native command registration | +| -------- | ------ | -------------- | --------------- | --------------- | -------------- | ---------------- | ---------------- | -------------------- | ------------ | --------------------------- | +| Matrix | x | x | x | x | x | x | x | x | | | +| Telegram | x | x | | | | | | | x | | +| Discord | x | x | | | | | | | | x | This keeps `qa-channel` as the broad product-behavior suite while Matrix, Telegram, and future live transports share one explicit transport-contract diff --git a/extensions/discord/src/monitor/message-handler.preflight.test.ts b/extensions/discord/src/monitor/message-handler.preflight.test.ts index e4ad1f70621..ad90ebb400a 100644 --- a/extensions/discord/src/monitor/message-handler.preflight.test.ts +++ b/extensions/discord/src/monitor/message-handler.preflight.test.ts @@ -595,6 +595,52 @@ describe("preflightDiscordMessage", () => { expect(result).not.toBeNull(); }); + it("hydrates mention metadata from REST when bot mention syntax is present but mentions are missing", async () => { + const channelId = "channel-bot-mentions-hydrated"; + const guildId = "guild-bot-mentions-hydrated"; + const botId = "123456789012345678"; + const message = createDiscordMessage({ + id: "m-bot-mentions-hydrated", + channelId, + content: `hi <@${botId}>`, + author: { + id: "relay-bot-1", + bot: true, + username: "Relay", + }, + mentionedUsers: [], + }); + const client = createGuildTextClient(channelId); + client.rest = { + get: vi.fn(async () => ({ + id: message.id, + content: message.content, + mentions: [{ id: botId, username: "OpenClaw", bot: true }], + mention_roles: [], + mention_everyone: false, + })), + } as unknown as DiscordClient["rest"]; + + const result = await preflightDiscordMessage({ + ...createPreflightArgs({ + cfg: DEFAULT_PREFLIGHT_CFG, + discordConfig: { + allowBots: "mentions", + } as DiscordConfig, + data: createGuildEvent({ + channelId, + guildId, + author: message.author, + message, + }), + client, + }), + botUserId: botId, + }); + + expect(result).not.toBeNull(); + }); + it("still drops bot control commands without a real mention when allowBots=mentions", async () => { const channelId = "channel-bot-command-no-mention"; const guildId = "guild-bot-command-no-mention"; diff --git a/extensions/discord/src/monitor/message-handler.preflight.ts b/extensions/discord/src/monitor/message-handler.preflight.ts index a94b281ba71..f8be60a3103 100644 --- a/extensions/discord/src/monitor/message-handler.preflight.ts +++ b/extensions/discord/src/monitor/message-handler.preflight.ts @@ -324,15 +324,29 @@ function mergeFetchedDiscordMessage(base: Message, fetched: APIMessage): Message }) as unknown as Message; } -async function hydrateDiscordMessageIfEmpty(params: { +function shouldHydrateDiscordMessage(params: { message: Message }) { + const currentText = resolveDiscordMessageText(params.message, { + includeForwarded: true, + }); + if (!currentText) { + return true; + } + const hasMentionMetadata = + (params.message.mentionedUsers?.length ?? 0) > 0 || + (params.message.mentionedRoles?.length ?? 0) > 0 || + params.message.mentionedEveryone; + if (hasMentionMetadata) { + return false; + } + return /<@!?\d+>|<@&\d+>|@everyone|@here/u.test(currentText); +} + +async function hydrateDiscordMessageIfNeeded(params: { client: DiscordMessagePreflightParams["client"]; message: Message; messageChannelId: string; }): Promise { - const currentText = resolveDiscordMessageText(params.message, { - includeForwarded: true, - }); - if (currentText) { + if (!shouldHydrateDiscordMessage({ message: params.message })) { return params.message; } const rest = params.client.rest as { get?: (route: string) => Promise } | undefined; @@ -346,7 +360,7 @@ async function hydrateDiscordMessageIfEmpty(params: { if (!fetched) { return params.message; } - logVerbose(`discord: hydrated empty inbound payload via REST for ${params.message.id}`); + logVerbose(`discord: hydrated inbound payload via REST for ${params.message.id}`); return mergeFetchedDiscordMessage(params.message, fetched); } catch (err) { logVerbose(`discord: failed to hydrate message ${params.message.id}: ${String(err)}`); @@ -383,7 +397,7 @@ export async function preflightDiscordMessage( return null; } - message = await hydrateDiscordMessageIfEmpty({ + message = await hydrateDiscordMessageIfNeeded({ client: params.client, message, messageChannelId, diff --git a/extensions/qa-lab/src/live-transports/cli.ts b/extensions/qa-lab/src/live-transports/cli.ts index 993ecd79ac4..3aee9f847da 100644 --- a/extensions/qa-lab/src/live-transports/cli.ts +++ b/extensions/qa-lab/src/live-transports/cli.ts @@ -1,4 +1,5 @@ import { listQaRunnerCliContributions } from "openclaw/plugin-sdk/qa-runner-runtime"; +import { discordQaCliRegistration } from "./discord/cli.js"; import type { LiveTransportQaCliRegistration } from "./shared/live-transport-cli.js"; import { telegramQaCliRegistration } from "./telegram/cli.js"; @@ -36,6 +37,7 @@ function createQaRunnerCliRegistration( export const LIVE_TRANSPORT_QA_CLI_REGISTRATIONS: readonly LiveTransportQaCliRegistration[] = [ telegramQaCliRegistration, + discordQaCliRegistration, ]; export function listLiveTransportQaCliRegistrations(): readonly LiveTransportQaCliRegistration[] { diff --git a/extensions/qa-lab/src/live-transports/discord/cli.runtime.ts b/extensions/qa-lab/src/live-transports/discord/cli.runtime.ts new file mode 100644 index 00000000000..4ebbd8d4039 --- /dev/null +++ b/extensions/qa-lab/src/live-transports/discord/cli.runtime.ts @@ -0,0 +1,23 @@ +import type { LiveTransportQaCommandOptions } from "../shared/live-transport-cli.js"; +import { + printLiveTransportQaArtifacts, + resolveLiveTransportQaRunOptions, +} from "../shared/live-transport-cli.runtime.js"; +import { runDiscordQaLive } from "./discord-live.runtime.js"; + +export async function runQaDiscordCommand(opts: LiveTransportQaCommandOptions) { + const runOptions = resolveLiveTransportQaRunOptions(opts); + const result = await runDiscordQaLive(runOptions); + printLiveTransportQaArtifacts("Discord QA", { + report: result.reportPath, + summary: result.summaryPath, + "observed messages": result.observedMessagesPath, + ...(result.gatewayDebugDirPath ? { "gateway debug logs": result.gatewayDebugDirPath } : {}), + }); + if ( + !runOptions.allowFailures && + result.scenarios.some((scenario) => scenario.status === "fail") + ) { + process.exitCode = 1; + } +} diff --git a/extensions/qa-lab/src/live-transports/discord/cli.ts b/extensions/qa-lab/src/live-transports/discord/cli.ts new file mode 100644 index 00000000000..a2e66cc7f02 --- /dev/null +++ b/extensions/qa-lab/src/live-transports/discord/cli.ts @@ -0,0 +1,37 @@ +import type { Command } from "commander"; +import { + createLazyCliRuntimeLoader, + createLiveTransportQaCliRegistration, + type LiveTransportQaCliRegistration, + type LiveTransportQaCommandOptions, +} from "../shared/live-transport-cli.js"; + +type DiscordQaCliRuntime = typeof import("./cli.runtime.js"); + +const loadDiscordQaCliRuntime = createLazyCliRuntimeLoader( + () => import("./cli.runtime.js"), +); + +async function runQaDiscord(opts: LiveTransportQaCommandOptions) { + const runtime = await loadDiscordQaCliRuntime(); + await runtime.runQaDiscordCommand(opts); +} + +export const discordQaCliRegistration: LiveTransportQaCliRegistration = + createLiveTransportQaCliRegistration({ + commandName: "discord", + credentialOptions: { + sourceDescription: "Credential source for Discord QA: env or convex (default: env)", + roleDescription: + "Credential role for convex auth: maintainer or ci (default: ci in CI, maintainer otherwise)", + }, + description: "Run the Discord live QA lane against a private guild bot-to-bot harness", + outputDirHelp: "Discord QA artifact directory", + scenarioHelp: "Run only the named Discord QA scenario (repeatable)", + sutAccountHelp: "Temporary Discord account id inside the QA gateway config", + run: runQaDiscord, + }); + +export function registerDiscordQaCli(qa: Command) { + discordQaCliRegistration.register(qa); +} diff --git a/extensions/qa-lab/src/live-transports/discord/discord-live.runtime.test.ts b/extensions/qa-lab/src/live-transports/discord/discord-live.runtime.test.ts new file mode 100644 index 00000000000..bb7790035ad --- /dev/null +++ b/extensions/qa-lab/src/live-transports/discord/discord-live.runtime.test.ts @@ -0,0 +1,453 @@ +import type { OpenClawConfig } from "openclaw/plugin-sdk/config-runtime"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + LIVE_TRANSPORT_BASELINE_STANDARD_SCENARIO_IDS, + findMissingLiveTransportStandardScenarios, +} from "../shared/live-transport-scenarios.js"; +import { __testing } from "./discord-live.runtime.js"; + +const fetchWithSsrFGuardMock = vi.hoisted(() => + vi.fn(async (params: { url: string; init?: RequestInit; signal?: AbortSignal }) => ({ + response: await fetch(params.url, { + ...params.init, + signal: params.signal, + }), + release: async () => {}, + })), +); + +vi.mock("openclaw/plugin-sdk/ssrf-runtime", async () => { + const actual = await vi.importActual( + "openclaw/plugin-sdk/ssrf-runtime", + ); + return { + ...actual, + fetchWithSsrFGuard: fetchWithSsrFGuardMock, + }; +}); + +describe("discord live qa runtime", () => { + afterEach(() => { + fetchWithSsrFGuardMock.mockClear(); + vi.restoreAllMocks(); + vi.unstubAllGlobals(); + }); + + it("resolves required Discord QA env vars", () => { + expect( + __testing.resolveDiscordQaRuntimeEnv({ + OPENCLAW_QA_DISCORD_GUILD_ID: "123456789012345678", + OPENCLAW_QA_DISCORD_CHANNEL_ID: "223456789012345678", + OPENCLAW_QA_DISCORD_DRIVER_BOT_TOKEN: "driver", + OPENCLAW_QA_DISCORD_SUT_BOT_TOKEN: "sut", + OPENCLAW_QA_DISCORD_SUT_APPLICATION_ID: "323456789012345678", + }), + ).toEqual({ + guildId: "123456789012345678", + channelId: "223456789012345678", + driverBotToken: "driver", + sutBotToken: "sut", + sutApplicationId: "323456789012345678", + }); + }); + + it("fails when a required Discord QA env var is missing", () => { + expect(() => + __testing.resolveDiscordQaRuntimeEnv({ + OPENCLAW_QA_DISCORD_GUILD_ID: "123456789012345678", + OPENCLAW_QA_DISCORD_CHANNEL_ID: "223456789012345678", + OPENCLAW_QA_DISCORD_DRIVER_BOT_TOKEN: "driver", + OPENCLAW_QA_DISCORD_SUT_BOT_TOKEN: "sut", + }), + ).toThrow("OPENCLAW_QA_DISCORD_SUT_APPLICATION_ID"); + }); + + it("fails when Discord IDs are not snowflakes", () => { + expect(() => + __testing.resolveDiscordQaRuntimeEnv({ + OPENCLAW_QA_DISCORD_GUILD_ID: "qa-guild", + OPENCLAW_QA_DISCORD_CHANNEL_ID: "223456789012345678", + OPENCLAW_QA_DISCORD_DRIVER_BOT_TOKEN: "driver", + OPENCLAW_QA_DISCORD_SUT_BOT_TOKEN: "sut", + OPENCLAW_QA_DISCORD_SUT_APPLICATION_ID: "323456789012345678", + }), + ).toThrow("OPENCLAW_QA_DISCORD_GUILD_ID must be a Discord snowflake."); + }); + + it("parses Discord pooled credential payloads", () => { + expect( + __testing.parseDiscordQaCredentialPayload({ + guildId: "123456789012345678", + channelId: "223456789012345678", + driverBotToken: "driver", + sutBotToken: "sut", + sutApplicationId: "323456789012345678", + }), + ).toEqual({ + guildId: "123456789012345678", + channelId: "223456789012345678", + driverBotToken: "driver", + sutBotToken: "sut", + sutApplicationId: "323456789012345678", + }); + }); + + it("rejects Discord pooled credential payloads with bad snowflakes", () => { + expect(() => + __testing.parseDiscordQaCredentialPayload({ + guildId: "123456789012345678", + channelId: "channel", + driverBotToken: "driver", + sutBotToken: "sut", + sutApplicationId: "323456789012345678", + }), + ).toThrow("Discord credential payload_CHANNEL_ID must be a Discord snowflake."); + }); + + it("injects a temporary Discord account into the QA gateway config", () => { + const baseCfg: OpenClawConfig = { + plugins: { + allow: ["memory-core", "qa-channel"], + entries: { + "memory-core": { enabled: true }, + "qa-channel": { enabled: true }, + }, + }, + channels: { + "qa-channel": { + enabled: true, + baseUrl: "http://127.0.0.1:43123", + botUserId: "openclaw", + botDisplayName: "OpenClaw QA", + allowFrom: ["*"], + }, + }, + }; + + const next = __testing.buildDiscordQaConfig(baseCfg, { + guildId: "123456789012345678", + channelId: "223456789012345678", + driverBotId: "423456789012345678", + sutAccountId: "sut", + sutBotToken: "sut-token", + }); + + expect(next.plugins?.allow).toContain("discord"); + expect(next.plugins?.entries?.discord).toEqual({ enabled: true }); + expect(next.channels?.discord).toEqual({ + enabled: true, + defaultAccount: "sut", + accounts: { + sut: { + enabled: true, + token: "sut-token", + allowBots: "mentions", + groupPolicy: "allowlist", + guilds: { + "123456789012345678": { + requireMention: true, + users: ["423456789012345678"], + channels: { + "223456789012345678": { + enabled: true, + requireMention: true, + users: ["423456789012345678"], + }, + }, + }, + }, + }, + }, + }); + }); + + it("normalizes observed Discord messages", () => { + expect( + __testing.normalizeDiscordObservedMessage({ + id: "523456789012345678", + channel_id: "223456789012345678", + guild_id: "123456789012345678", + content: "hello", + timestamp: "2026-04-22T12:00:00.000Z", + author: { + id: "423456789012345678", + username: "driver", + bot: true, + }, + referenced_message: { id: "323456789012345678" }, + }), + ).toEqual({ + messageId: "523456789012345678", + channelId: "223456789012345678", + guildId: "123456789012345678", + senderId: "423456789012345678", + senderIsBot: true, + senderUsername: "driver", + text: "hello", + replyToMessageId: "323456789012345678", + timestamp: "2026-04-22T12:00:00.000Z", + }); + }); + + it("matches Discord scenario replies by SUT id and marker", () => { + expect( + __testing.matchesDiscordScenarioReply({ + channelId: "223456789012345678", + sutBotId: "323456789012345678", + matchText: "DISCORD_QA_ECHO_TOKEN", + message: { + messageId: "523456789012345678", + channelId: "223456789012345678", + senderId: "323456789012345678", + senderIsBot: true, + text: "reply DISCORD_QA_ECHO_TOKEN", + }, + }), + ).toBe(true); + expect( + __testing.matchesDiscordScenarioReply({ + channelId: "223456789012345678", + sutBotId: "323456789012345678", + matchText: "DISCORD_QA_ECHO_TOKEN", + message: { + messageId: "523456789012345679", + channelId: "223456789012345678", + senderId: "423456789012345678", + senderIsBot: true, + text: "reply DISCORD_QA_ECHO_TOKEN", + }, + }), + ).toBe(false); + }); + + it("includes the Discord live scenarios", () => { + expect(__testing.findScenario().map((scenario) => scenario.id)).toEqual([ + "discord-canary", + "discord-mention-gating", + "discord-native-help-command-registration", + ]); + }); + + it("waits for the Discord account to become connected, not just running", async () => { + vi.useFakeTimers(); + try { + const gateway = { + call: vi + .fn() + .mockResolvedValueOnce({ + channelAccounts: { + discord: [ + { accountId: "sut", running: true, connected: false, restartPending: false }, + ], + }, + }) + .mockResolvedValueOnce({ + channelAccounts: { + discord: [ + { accountId: "sut", running: true, connected: true, restartPending: false }, + ], + }, + }), + } as unknown as Parameters[0]; + + const readyPromise = __testing.waitForDiscordChannelRunning(gateway, "sut"); + await vi.advanceTimersByTimeAsync(600); + + await expect(readyPromise).resolves.toBeUndefined(); + expect(gateway.call).toHaveBeenCalledTimes(2); + } finally { + vi.useRealTimers(); + } + }); + + it("reports the last Discord status when connection readiness times out", async () => { + vi.useFakeTimers(); + try { + const gateway = { + call: vi.fn().mockResolvedValue({ + channelAccounts: { + discord: [ + { + accountId: "sut", + running: true, + connected: false, + restartPending: false, + lastError: null, + lastDisconnect: { error: "runtime-not-ready" }, + }, + ], + }, + }), + } as unknown as Parameters[0]; + + const readyPromise = __testing.waitForDiscordChannelRunning(gateway, "sut"); + const assertion = expect(readyPromise).rejects.toThrow( + 'discord account "sut" did not become connected (last status: running=true connected=false', + ); + await vi.advanceTimersByTimeAsync(45_500); + await assertion; + } finally { + vi.useRealTimers(); + } + }); + + it("fails when any requested Discord scenario id is unknown", () => { + expect(() => __testing.findScenario(["discord-canary", "typo-scenario"])).toThrow( + "unknown Discord QA scenario id(s): typo-scenario", + ); + }); + + it("tracks Discord live coverage against the shared transport contract", () => { + expect(__testing.DISCORD_QA_STANDARD_SCENARIO_IDS).toEqual(["canary", "mention-gating"]); + expect( + findMissingLiveTransportStandardScenarios({ + coveredStandardScenarioIds: __testing.DISCORD_QA_STANDARD_SCENARIO_IDS, + expectedStandardScenarioIds: LIVE_TRANSPORT_BASELINE_STANDARD_SCENARIO_IDS, + }), + ).toEqual(["allowlist-block", "top-level-reply-shape", "restart-resume"]); + }); + + it("lists Discord application commands through the REST API", async () => { + vi.stubGlobal( + "fetch", + vi.fn(async (_input: string | URL | globalThis.Request, init?: RequestInit) => { + expect(init?.headers).toBeInstanceOf(Headers); + expect((init?.headers as Headers).get("authorization")).toBe("Bot token"); + return new Response( + JSON.stringify([ + { id: "623456789012345678", name: "help" }, + { id: "623456789012345679", name: "commands" }, + ]), + { + status: 200, + headers: { + "content-type": "application/json", + }, + }, + ); + }), + ); + + await expect( + __testing.listApplicationCommands({ + token: "token", + applicationId: "323456789012345678", + }), + ).resolves.toEqual([ + { id: "623456789012345678", name: "help" }, + { id: "623456789012345679", name: "commands" }, + ]); + }); + + it("waits for required Discord application commands to be registered", async () => { + vi.useFakeTimers(); + try { + vi.stubGlobal( + "fetch", + vi + .fn() + .mockResolvedValueOnce( + new Response(JSON.stringify([{ id: "623456789012345679", name: "commands" }]), { + status: 200, + headers: { + "content-type": "application/json", + }, + }), + ) + .mockResolvedValueOnce( + new Response( + JSON.stringify([ + { id: "623456789012345679", name: "commands" }, + { id: "623456789012345678", name: "help" }, + ]), + { + status: 200, + headers: { + "content-type": "application/json", + }, + }, + ), + ), + ); + + const registeredPromise = __testing.assertDiscordApplicationCommandsRegistered({ + token: "token", + applicationId: "323456789012345678", + expectedCommandNames: ["help"], + timeoutMs: 5_000, + }); + await vi.advanceTimersByTimeAsync(1_100); + + await expect(registeredPromise).resolves.toEqual({ + commandNames: ["commands", "help"], + }); + } finally { + vi.useRealTimers(); + } + }); + + it("adds an abort deadline to Discord API requests", async () => { + const controller = new AbortController(); + const timeoutSpy = vi.spyOn(AbortSignal, "timeout").mockReturnValue(controller.signal); + let signal: AbortSignal | undefined; + vi.stubGlobal( + "fetch", + vi.fn(async (_input: string | URL | globalThis.Request, init?: RequestInit) => { + signal = init?.signal as AbortSignal | undefined; + return new Response(JSON.stringify({ id: "423456789012345678" }), { + status: 200, + headers: { + "content-type": "application/json", + }, + }); + }), + ); + + await expect( + __testing.callDiscordApi({ + token: "token", + path: "/users/@me", + timeoutMs: 25, + }), + ).resolves.toEqual({ + id: "423456789012345678", + }); + expect(timeoutSpy).toHaveBeenCalledWith(25); + expect(signal).toBe(controller.signal); + expect(signal?.aborted).toBe(false); + controller.abort(); + expect(signal?.aborted).toBe(true); + }); + + it("redacts observed message content by default in artifacts", () => { + expect( + __testing.buildObservedMessagesArtifact({ + includeContent: false, + redactMetadata: false, + observedMessages: [ + { + messageId: "523456789012345678", + channelId: "223456789012345678", + guildId: "123456789012345678", + senderId: "323456789012345678", + senderIsBot: true, + senderUsername: "sut", + text: "secret text", + timestamp: "2026-04-22T12:00:00.000Z", + }, + ], + }), + ).toEqual([ + { + messageId: "523456789012345678", + channelId: "223456789012345678", + guildId: "123456789012345678", + senderId: "323456789012345678", + senderIsBot: true, + senderUsername: "sut", + replyToMessageId: undefined, + timestamp: "2026-04-22T12:00:00.000Z", + }, + ]); + }); +}); diff --git a/extensions/qa-lab/src/live-transports/discord/discord-live.runtime.ts b/extensions/qa-lab/src/live-transports/discord/discord-live.runtime.ts new file mode 100644 index 00000000000..07ccc88335d --- /dev/null +++ b/extensions/qa-lab/src/live-transports/discord/discord-live.runtime.ts @@ -0,0 +1,978 @@ +import { randomUUID } from "node:crypto"; +import fs from "node:fs/promises"; +import path from "node:path"; +import type { OpenClawConfig } from "openclaw/plugin-sdk/config-runtime"; +import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime"; +import { fetchWithSsrFGuard } from "openclaw/plugin-sdk/ssrf-runtime"; +import { z } from "zod"; +import { startQaGatewayChild } from "../../gateway-child.js"; +import { DEFAULT_QA_LIVE_PROVIDER_MODE } from "../../providers/index.js"; +import { + defaultQaModelForMode, + normalizeQaProviderMode, + type QaProviderModeInput, +} from "../../run-config.js"; +import { + acquireQaCredentialLease, + startQaCredentialLeaseHeartbeat, + type QaCredentialRole, +} from "../shared/credential-lease.runtime.js"; +import { startQaLiveLaneGateway } from "../shared/live-gateway.runtime.js"; +import { appendLiveLaneIssue, buildLiveLaneArtifactsError } from "../shared/live-lane-helpers.js"; +import { + collectLiveTransportStandardScenarioCoverage, + selectLiveTransportScenarios, + type LiveTransportScenarioDefinition, +} from "../shared/live-transport-scenarios.js"; + +type DiscordQaRuntimeEnv = { + guildId: string; + channelId: string; + driverBotToken: string; + sutBotToken: string; + sutApplicationId: string; +}; + +type DiscordQaScenarioId = + | "discord-canary" + | "discord-mention-gating" + | "discord-native-help-command-registration"; + +type DiscordQaScenarioRun = + | { + kind: "channel-message"; + expectReply: boolean; + input: string; + expectedTextIncludes?: string[]; + matchText?: string; + } + | { + kind: "application-command-registration"; + expectedCommandNames: string[]; + }; + +type DiscordQaScenarioDefinition = LiveTransportScenarioDefinition & { + buildRun: (sutApplicationId: string) => DiscordQaScenarioRun; +}; + +type DiscordUser = { + id: string; + username?: string; + bot?: boolean; +}; + +type DiscordMessage = { + id: string; + channel_id: string; + guild_id?: string; + content?: string; + timestamp?: string; + author?: DiscordUser; + referenced_message?: { id?: string } | null; +}; + +type DiscordApplicationCommand = { + id: string; + name?: string; +}; + +type DiscordObservedMessage = { + messageId: string; + channelId: string; + guildId?: string; + senderId: string; + senderIsBot: boolean; + senderUsername?: string; + scenarioId?: string; + scenarioTitle?: string; + matchedScenario?: boolean; + text: string; + replyToMessageId?: string; + timestamp?: string; +}; + +type DiscordObservedMessageArtifact = { + messageId?: string; + channelId?: string; + guildId?: string; + senderId?: string; + senderIsBot: boolean; + senderUsername?: string; + scenarioId?: string; + scenarioTitle?: string; + matchedScenario?: boolean; + text?: string; + replyToMessageId?: string; + timestamp?: string; +}; + +type DiscordQaScenarioResult = { + id: string; + title: string; + status: "pass" | "fail"; + details: string; +}; + +export type DiscordQaRunResult = { + outputDir: string; + reportPath: string; + summaryPath: string; + observedMessagesPath: string; + gatewayDebugDirPath?: string; + scenarios: DiscordQaScenarioResult[]; +}; + +type DiscordQaSummary = { + credentials: { + credentialId?: string; + kind: string; + ownerId?: string; + role?: QaCredentialRole; + source: "convex" | "env"; + }; + guildId: string; + channelId: string; + startedAt: string; + finishedAt: string; + cleanupIssues: string[]; + counts: { + total: number; + passed: number; + failed: number; + }; + scenarios: DiscordQaScenarioResult[]; +}; + +const DISCORD_API_BASE_URL = "https://discord.com/api/v10"; +const DISCORD_QA_CAPTURE_CONTENT_ENV = "OPENCLAW_QA_DISCORD_CAPTURE_CONTENT"; +const QA_REDACT_PUBLIC_METADATA_ENV = "OPENCLAW_QA_REDACT_PUBLIC_METADATA"; +const DISCORD_QA_ENV_KEYS = [ + "OPENCLAW_QA_DISCORD_GUILD_ID", + "OPENCLAW_QA_DISCORD_CHANNEL_ID", + "OPENCLAW_QA_DISCORD_DRIVER_BOT_TOKEN", + "OPENCLAW_QA_DISCORD_SUT_BOT_TOKEN", + "OPENCLAW_QA_DISCORD_SUT_APPLICATION_ID", +] as const; + +const DISCORD_QA_SCENARIOS: DiscordQaScenarioDefinition[] = [ + { + id: "discord-canary", + standardId: "canary", + title: "Discord canary echo", + timeoutMs: 45_000, + buildRun: (sutApplicationId) => { + const token = `DISCORD_QA_ECHO_${randomUUID().slice(0, 8).toUpperCase()}`; + return { + kind: "channel-message", + expectReply: true, + input: `<@${sutApplicationId}> reply with only this exact marker: ${token}`, + expectedTextIncludes: [token], + matchText: token, + }; + }, + }, + { + id: "discord-mention-gating", + standardId: "mention-gating", + title: "Discord unmentioned message does not trigger", + timeoutMs: 8_000, + buildRun: () => { + const token = `DISCORD_QA_NOMENTION_${randomUUID().slice(0, 8).toUpperCase()}`; + return { + kind: "channel-message", + expectReply: false, + input: `reply with only this exact marker: ${token}`, + matchText: token, + }; + }, + }, + { + id: "discord-native-help-command-registration", + title: "Discord native help command is registered", + timeoutMs: 45_000, + buildRun: () => ({ + kind: "application-command-registration", + expectedCommandNames: ["help"], + }), + }, +]; + +export const DISCORD_QA_STANDARD_SCENARIO_IDS = collectLiveTransportStandardScenarioCoverage({ + scenarios: DISCORD_QA_SCENARIOS, +}); + +const discordQaCredentialPayloadSchema = z.object({ + guildId: z.string().trim().min(1), + channelId: z.string().trim().min(1), + driverBotToken: z.string().trim().min(1), + sutBotToken: z.string().trim().min(1), + sutApplicationId: z.string().trim().min(1), +}); + +function isDiscordSnowflake(value: string) { + return /^\d{17,20}$/u.test(value); +} + +function assertDiscordSnowflake(value: string, label: string) { + if (!isDiscordSnowflake(value)) { + throw new Error(`${label} must be a Discord snowflake.`); + } +} + +function resolveEnvValue(env: NodeJS.ProcessEnv, key: (typeof DISCORD_QA_ENV_KEYS)[number]) { + const value = env[key]?.trim(); + if (!value) { + throw new Error(`Missing ${key}.`); + } + return value; +} + +function isTruthyOptIn(value: string | undefined) { + const normalized = value?.trim().toLowerCase(); + return normalized === "1" || normalized === "true" || normalized === "yes"; +} + +export function resolveDiscordQaRuntimeEnv( + env: NodeJS.ProcessEnv = process.env, +): DiscordQaRuntimeEnv { + const runtimeEnv = { + guildId: resolveEnvValue(env, "OPENCLAW_QA_DISCORD_GUILD_ID"), + channelId: resolveEnvValue(env, "OPENCLAW_QA_DISCORD_CHANNEL_ID"), + driverBotToken: resolveEnvValue(env, "OPENCLAW_QA_DISCORD_DRIVER_BOT_TOKEN"), + sutBotToken: resolveEnvValue(env, "OPENCLAW_QA_DISCORD_SUT_BOT_TOKEN"), + sutApplicationId: resolveEnvValue(env, "OPENCLAW_QA_DISCORD_SUT_APPLICATION_ID"), + }; + validateDiscordQaRuntimeEnv(runtimeEnv, "OPENCLAW_QA_DISCORD"); + return runtimeEnv; +} + +function validateDiscordQaRuntimeEnv(runtimeEnv: DiscordQaRuntimeEnv, prefix: string) { + assertDiscordSnowflake(runtimeEnv.guildId, `${prefix}_GUILD_ID`); + assertDiscordSnowflake(runtimeEnv.channelId, `${prefix}_CHANNEL_ID`); + assertDiscordSnowflake(runtimeEnv.sutApplicationId, `${prefix}_SUT_APPLICATION_ID`); +} + +function parseDiscordQaCredentialPayload(payload: unknown): DiscordQaRuntimeEnv { + const parsed = discordQaCredentialPayloadSchema.parse(payload); + const runtimeEnv = { + guildId: parsed.guildId, + channelId: parsed.channelId, + driverBotToken: parsed.driverBotToken, + sutBotToken: parsed.sutBotToken, + sutApplicationId: parsed.sutApplicationId, + }; + validateDiscordQaRuntimeEnv(runtimeEnv, "Discord credential payload"); + return runtimeEnv; +} + +function buildDiscordQaConfig( + baseCfg: OpenClawConfig, + params: { + guildId: string; + channelId: string; + driverBotId: string; + sutAccountId: string; + sutBotToken: string; + }, +): OpenClawConfig { + const pluginAllow = [...new Set([...(baseCfg.plugins?.allow ?? []), "discord"])]; + const pluginEntries = { + ...baseCfg.plugins?.entries, + discord: { enabled: true }, + }; + return { + ...baseCfg, + plugins: { + ...baseCfg.plugins, + allow: pluginAllow, + entries: pluginEntries, + }, + channels: { + ...baseCfg.channels, + discord: { + enabled: true, + defaultAccount: params.sutAccountId, + accounts: { + [params.sutAccountId]: { + enabled: true, + token: params.sutBotToken, + allowBots: "mentions", + groupPolicy: "allowlist", + guilds: { + [params.guildId]: { + requireMention: true, + users: [params.driverBotId], + channels: { + [params.channelId]: { + enabled: true, + requireMention: true, + users: [params.driverBotId], + }, + }, + }, + }, + }, + }, + }, + }, + }; +} + +async function callDiscordApi(params: { + token: string; + path: string; + init?: RequestInit; + timeoutMs?: number; +}): Promise { + const headers = new Headers(params.init?.headers); + headers.set("authorization", `Bot ${params.token}`); + if (params.init?.body) { + headers.set("content-type", "application/json"); + } + const { response, release } = await fetchWithSsrFGuard({ + url: `${DISCORD_API_BASE_URL}${params.path}`, + init: { + ...params.init, + headers, + }, + signal: AbortSignal.timeout(params.timeoutMs ?? 15_000), + policy: { hostnameAllowlist: ["discord.com"] }, + auditContext: "qa-lab-discord-live", + }); + try { + const text = await response.text(); + const payload = text.trim() ? (JSON.parse(text) as unknown) : undefined; + if (!response.ok) { + const message = + typeof payload === "object" && + payload !== null && + typeof (payload as { message?: unknown }).message === "string" + ? (payload as { message: string }).message + : text.trim(); + throw new Error( + message || `Discord API ${params.path} failed with status ${response.status}`, + ); + } + return payload as T; + } finally { + await release(); + } +} + +async function getCurrentDiscordUser(token: string) { + return await callDiscordApi({ + token, + path: "/users/@me", + }); +} + +async function sendChannelMessage(token: string, channelId: string, content: string) { + return await callDiscordApi({ + token, + path: `/channels/${channelId}/messages`, + init: { + method: "POST", + body: JSON.stringify({ + content, + allowed_mentions: { + parse: ["users"], + }, + }), + }, + }); +} + +async function listChannelMessagesAfter(params: { + token: string; + channelId: string; + afterSnowflake: string; +}) { + const query = new URLSearchParams({ + after: params.afterSnowflake, + limit: "50", + }); + return await callDiscordApi({ + token: params.token, + path: `/channels/${params.channelId}/messages?${query.toString()}`, + }); +} + +async function listApplicationCommands(params: { token: string; applicationId: string }) { + return await callDiscordApi({ + token: params.token, + path: `/applications/${params.applicationId}/commands`, + }); +} + +function compareDiscordSnowflakes(a: string, b: string) { + const left = BigInt(a); + const right = BigInt(b); + return left < right ? -1 : left > right ? 1 : 0; +} + +function normalizeDiscordObservedMessage(message: DiscordMessage): DiscordObservedMessage | null { + if (!message.author?.id) { + return null; + } + return { + messageId: message.id, + channelId: message.channel_id, + guildId: message.guild_id, + senderId: message.author.id, + senderIsBot: message.author.bot === true, + senderUsername: message.author.username, + text: message.content ?? "", + replyToMessageId: message.referenced_message?.id, + timestamp: message.timestamp, + }; +} + +async function pollChannelMessages(params: { + token: string; + channelId: string; + afterSnowflake: string; + timeoutMs: number; + predicate: (message: DiscordObservedMessage) => boolean; + observedMessages: DiscordObservedMessage[]; + observationScenarioId: string; + observationScenarioTitle: string; +}) { + const startedAt = Date.now(); + let afterSnowflake = params.afterSnowflake; + while (Date.now() - startedAt < params.timeoutMs) { + const messages = await listChannelMessagesAfter({ + token: params.token, + channelId: params.channelId, + afterSnowflake, + }); + const sorted = messages + .filter((message) => isDiscordSnowflake(message.id)) + .toSorted((a, b) => compareDiscordSnowflakes(a.id, b.id)); + for (const message of sorted) { + afterSnowflake = message.id; + const normalized = normalizeDiscordObservedMessage(message); + if (!normalized) { + continue; + } + const matchedScenario = params.predicate(normalized); + const observedMessage: DiscordObservedMessage = { + ...normalized, + scenarioId: params.observationScenarioId, + scenarioTitle: params.observationScenarioTitle, + matchedScenario, + }; + params.observedMessages.push(observedMessage); + if (matchedScenario) { + return { message: observedMessage, afterSnowflake }; + } + } + await new Promise((resolve) => setTimeout(resolve, 1_000)); + } + throw new Error(`timed out after ${params.timeoutMs}ms waiting for Discord message`); +} + +async function waitForDiscordChannelRunning( + gateway: Awaited>, + accountId: string, +) { + const startedAt = Date.now(); + let lastStatus: + | { + running?: boolean; + connected?: boolean; + restartPending?: boolean; + lastConnectedAt?: number; + lastDisconnect?: unknown; + lastError?: string; + } + | undefined; + while (Date.now() - startedAt < 45_000) { + try { + const payload = (await gateway.call( + "channels.status", + { probe: false, timeoutMs: 2_000 }, + { timeoutMs: 5_000 }, + )) as { + channelAccounts?: Record< + string, + Array<{ + accountId?: string; + running?: boolean; + connected?: boolean; + restartPending?: boolean; + lastConnectedAt?: number; + lastDisconnect?: unknown; + lastError?: string; + }> + >; + }; + const accounts = payload.channelAccounts?.discord ?? []; + const match = accounts.find((entry) => entry.accountId === accountId); + lastStatus = match + ? { + running: match.running, + connected: match.connected, + restartPending: match.restartPending, + lastConnectedAt: match.lastConnectedAt, + lastDisconnect: match.lastDisconnect, + lastError: match.lastError, + } + : undefined; + if (match?.running && match.connected === true && match.restartPending !== true) { + return; + } + } catch { + // retry + } + await new Promise((resolve) => setTimeout(resolve, 500)); + } + const details = lastStatus + ? ` (last status: running=${String(lastStatus.running)} connected=${String(lastStatus.connected)} restartPending=${String(lastStatus.restartPending)} lastConnectedAt=${String(lastStatus.lastConnectedAt)} lastError=${lastStatus.lastError ?? "null"} lastDisconnect=${JSON.stringify(lastStatus.lastDisconnect)})` + : ""; + throw new Error(`discord account "${accountId}" did not become connected${details}`); +} + +function renderDiscordQaMarkdown(params: { + cleanupIssues: string[]; + credentialSource: "convex" | "env"; + redactMetadata: boolean; + guildId: string; + channelId: string; + gatewayDebugDirPath?: string; + startedAt: string; + finishedAt: string; + scenarios: DiscordQaScenarioResult[]; +}) { + const lines = [ + "# Discord QA Report", + "", + `- Credential source: \`${params.credentialSource}\``, + `- Guild: \`${params.guildId}\``, + `- Channel: \`${params.channelId}\``, + `- Metadata redaction: \`${params.redactMetadata ? "enabled" : "disabled"}\``, + `- Started: ${params.startedAt}`, + `- Finished: ${params.finishedAt}`, + "", + "## Scenarios", + "", + ]; + for (const scenario of params.scenarios) { + lines.push(`### ${scenario.title}`); + lines.push(""); + lines.push(`- Status: ${scenario.status}`); + lines.push(`- Details: ${scenario.details}`); + lines.push(""); + } + if (params.gatewayDebugDirPath) { + lines.push("## Gateway Debug Logs"); + lines.push(""); + lines.push(`- Preserved at: \`${params.gatewayDebugDirPath}\``); + lines.push(""); + } + if (params.cleanupIssues.length > 0) { + lines.push("## Cleanup"); + lines.push(""); + for (const issue of params.cleanupIssues) { + lines.push(`- ${issue}`); + } + lines.push(""); + } + return lines.join("\n"); +} + +function buildObservedMessagesArtifact(params: { + observedMessages: DiscordObservedMessage[]; + includeContent: boolean; + redactMetadata: boolean; +}) { + return params.observedMessages.map((message) => { + const scenarioContext = { + ...(message.scenarioId ? { scenarioId: message.scenarioId } : {}), + ...(message.scenarioTitle ? { scenarioTitle: message.scenarioTitle } : {}), + ...(typeof message.matchedScenario === "boolean" + ? { matchedScenario: message.matchedScenario } + : {}), + }; + const base = params.redactMetadata + ? { + ...scenarioContext, + senderIsBot: message.senderIsBot, + } + : { + ...scenarioContext, + messageId: message.messageId, + channelId: message.channelId, + guildId: message.guildId, + senderId: message.senderId, + senderIsBot: message.senderIsBot, + senderUsername: message.senderUsername, + replyToMessageId: message.replyToMessageId, + timestamp: message.timestamp, + }; + if (!params.includeContent) { + return base; + } + return { + ...base, + text: message.text, + }; + }); +} + +function findScenario(ids?: string[]) { + return selectLiveTransportScenarios({ + ids, + laneLabel: "Discord", + scenarios: DISCORD_QA_SCENARIOS, + }); +} + +function matchesDiscordScenarioReply(params: { + channelId: string; + message: DiscordObservedMessage; + matchText?: string; + sutBotId: string; +}) { + return ( + params.message.channelId === params.channelId && + params.message.senderId === params.sutBotId && + Boolean(params.matchText && params.message.text.includes(params.matchText)) + ); +} + +function assertDiscordScenarioReply(params: { + expectedTextIncludes?: string[]; + message: DiscordObservedMessage; +}) { + if (!params.message.text.trim()) { + throw new Error(`reply message ${params.message.messageId} was empty`); + } + for (const expected of params.expectedTextIncludes ?? []) { + if (!params.message.text.includes(expected)) { + throw new Error( + `reply message ${params.message.messageId} missing expected text: ${expected}`, + ); + } + } +} + +async function assertDiscordApplicationCommandsRegistered(params: { + applicationId: string; + expectedCommandNames: string[]; + timeoutMs: number; + token: string; +}) { + const startedAt = Date.now(); + let lastNames: string[] = []; + while (Date.now() - startedAt < params.timeoutMs) { + const commands = await listApplicationCommands({ + token: params.token, + applicationId: params.applicationId, + }); + lastNames = commands + .map((command) => command.name ?? "") + .filter(Boolean) + .toSorted(); + const nameSet = new Set(lastNames); + const missing = params.expectedCommandNames.filter((name) => !nameSet.has(name)); + if (missing.length === 0) { + return { commandNames: lastNames }; + } + await new Promise((resolve) => setTimeout(resolve, 1_000)); + } + throw new Error( + `missing Discord native command(s): ${params.expectedCommandNames + .filter((name) => !lastNames.includes(name)) + .join(", ")} (registered: ${lastNames.join(", ") || "none"})`, + ); +} + +export async function runDiscordQaLive(params: { + repoRoot?: string; + outputDir?: string; + providerMode?: QaProviderModeInput; + primaryModel?: string; + alternateModel?: string; + fastMode?: boolean; + scenarioIds?: string[]; + sutAccountId?: string; + credentialSource?: string; + credentialRole?: string; +}): Promise { + const repoRoot = path.resolve(params.repoRoot ?? process.cwd()); + const outputDir = + params.outputDir ?? + path.join(repoRoot, ".artifacts", "qa-e2e", `discord-${Date.now().toString(36)}`); + await fs.mkdir(outputDir, { recursive: true }); + + const providerMode = normalizeQaProviderMode( + params.providerMode ?? DEFAULT_QA_LIVE_PROVIDER_MODE, + ); + const primaryModel = params.primaryModel?.trim() || defaultQaModelForMode(providerMode); + const alternateModel = params.alternateModel?.trim() || defaultQaModelForMode(providerMode, true); + const sutAccountId = params.sutAccountId?.trim() || "sut"; + const scenarios = findScenario(params.scenarioIds); + + const credentialLease = await acquireQaCredentialLease({ + kind: "discord", + source: params.credentialSource, + role: params.credentialRole, + resolveEnvPayload: () => resolveDiscordQaRuntimeEnv(), + parsePayload: parseDiscordQaCredentialPayload, + }); + const leaseHeartbeat = startQaCredentialLeaseHeartbeat(credentialLease); + const assertLeaseHealthy = () => { + leaseHeartbeat.throwIfFailed(); + }; + + const runtimeEnv = credentialLease.payload; + const observedMessages: DiscordObservedMessage[] = []; + const redactPublicMetadata = isTruthyOptIn(process.env[QA_REDACT_PUBLIC_METADATA_ENV]); + const includeObservedMessageContent = isTruthyOptIn(process.env[DISCORD_QA_CAPTURE_CONTENT_ENV]); + const startedAt = new Date().toISOString(); + const scenarioResults: DiscordQaScenarioResult[] = []; + const cleanupIssues: string[] = []; + const gatewayDebugDirPath = path.join(outputDir, "gateway-debug"); + let preservedGatewayDebugArtifacts = false; + try { + const [driverIdentity, sutIdentity] = await Promise.all([ + getCurrentDiscordUser(runtimeEnv.driverBotToken), + getCurrentDiscordUser(runtimeEnv.sutBotToken), + ]); + if (driverIdentity.id === sutIdentity.id) { + throw new Error("Discord QA requires two distinct bots for driver and SUT."); + } + if (sutIdentity.id !== runtimeEnv.sutApplicationId) { + throw new Error( + "Discord QA SUT application id must match the SUT bot user id returned by Discord.", + ); + } + + const gatewayHarness = await startQaLiveLaneGateway({ + repoRoot, + transport: { + requiredPluginIds: [], + createGatewayConfig: () => ({}), + }, + transportBaseUrl: "http://127.0.0.1:0", + providerMode, + primaryModel, + alternateModel, + fastMode: params.fastMode, + controlUiEnabled: false, + mutateConfig: (cfg) => + buildDiscordQaConfig(cfg, { + guildId: runtimeEnv.guildId, + channelId: runtimeEnv.channelId, + driverBotId: driverIdentity.id, + sutAccountId, + sutBotToken: runtimeEnv.sutBotToken, + }), + }); + try { + await waitForDiscordChannelRunning(gatewayHarness.gateway, sutAccountId); + assertLeaseHealthy(); + for (const scenario of scenarios) { + assertLeaseHealthy(); + const scenarioRun = scenario.buildRun(runtimeEnv.sutApplicationId); + try { + if (scenarioRun.kind === "application-command-registration") { + const registered = await assertDiscordApplicationCommandsRegistered({ + token: runtimeEnv.sutBotToken, + applicationId: runtimeEnv.sutApplicationId, + expectedCommandNames: scenarioRun.expectedCommandNames, + timeoutMs: scenario.timeoutMs, + }); + scenarioResults.push({ + id: scenario.id, + title: scenario.title, + status: "pass", + details: redactPublicMetadata + ? "native command registered" + : `native command registered (${registered.commandNames.join(", ")})`, + }); + continue; + } + const sent = await sendChannelMessage( + runtimeEnv.driverBotToken, + runtimeEnv.channelId, + scenarioRun.input, + ); + const matched = await pollChannelMessages({ + token: runtimeEnv.driverBotToken, + channelId: runtimeEnv.channelId, + afterSnowflake: sent.id, + timeoutMs: scenario.timeoutMs, + observedMessages, + observationScenarioId: scenario.id, + observationScenarioTitle: scenario.title, + predicate: (message) => + matchesDiscordScenarioReply({ + channelId: runtimeEnv.channelId, + matchText: scenarioRun.matchText, + message, + sutBotId: sutIdentity.id, + }), + }); + if (!scenarioRun.expectReply) { + throw new Error(`unexpected reply message ${matched.message.messageId} matched`); + } + assertDiscordScenarioReply({ + expectedTextIncludes: scenarioRun.expectedTextIncludes, + message: matched.message, + }); + scenarioResults.push({ + id: scenario.id, + title: scenario.title, + status: "pass", + details: redactPublicMetadata + ? "reply matched" + : `reply message ${matched.message.messageId} matched`, + }); + } catch (error) { + if (scenarioRun.kind === "channel-message" && !scenarioRun.expectReply) { + const details = formatErrorMessage(error); + if (details === `timed out after ${scenario.timeoutMs}ms waiting for Discord message`) { + scenarioResults.push({ + id: scenario.id, + title: scenario.title, + status: "pass", + details: "no reply", + }); + continue; + } + } + scenarioResults.push({ + id: scenario.id, + title: scenario.title, + status: "fail", + details: formatErrorMessage(error), + }); + } + assertLeaseHealthy(); + } + } finally { + try { + const shouldPreserveGatewayDebugArtifacts = scenarioResults.some( + (scenario) => scenario.status === "fail", + ); + await gatewayHarness.stop( + shouldPreserveGatewayDebugArtifacts ? { preserveToDir: gatewayDebugDirPath } : undefined, + ); + preservedGatewayDebugArtifacts = shouldPreserveGatewayDebugArtifacts; + } catch (error) { + appendLiveLaneIssue(cleanupIssues, "live gateway cleanup", error); + } + } + } finally { + await leaseHeartbeat.stop(); + try { + await credentialLease.release(); + } catch (error) { + appendLiveLaneIssue(cleanupIssues, "credential lease release", error); + } + } + + const finishedAt = new Date().toISOString(); + const publishedCleanupIssues = redactPublicMetadata + ? cleanupIssues.map(() => "details redacted (OPENCLAW_QA_REDACT_PUBLIC_METADATA=1)") + : cleanupIssues; + const passedCount = scenarioResults.filter((entry) => entry.status === "pass").length; + const failedCount = scenarioResults.filter((entry) => entry.status === "fail").length; + const summary: DiscordQaSummary = { + credentials: { + source: credentialLease.source, + kind: credentialLease.kind, + role: credentialLease.role, + ownerId: redactPublicMetadata ? undefined : credentialLease.ownerId, + credentialId: redactPublicMetadata ? undefined : credentialLease.credentialId, + }, + guildId: redactPublicMetadata ? "" : runtimeEnv.guildId, + channelId: redactPublicMetadata ? "" : runtimeEnv.channelId, + startedAt, + finishedAt, + cleanupIssues: publishedCleanupIssues, + counts: { + total: scenarioResults.length, + passed: passedCount, + failed: failedCount, + }, + scenarios: scenarioResults, + }; + const reportPath = path.join(outputDir, "discord-qa-report.md"); + const summaryPath = path.join(outputDir, "discord-qa-summary.json"); + const observedMessagesPath = path.join(outputDir, "discord-qa-observed-messages.json"); + await fs.writeFile( + reportPath, + `${renderDiscordQaMarkdown({ + cleanupIssues: publishedCleanupIssues, + credentialSource: credentialLease.source, + redactMetadata: redactPublicMetadata, + guildId: redactPublicMetadata ? "" : runtimeEnv.guildId, + channelId: redactPublicMetadata ? "" : runtimeEnv.channelId, + gatewayDebugDirPath: preservedGatewayDebugArtifacts ? gatewayDebugDirPath : undefined, + startedAt, + finishedAt, + scenarios: scenarioResults, + })}\n`, + { encoding: "utf8", mode: 0o600 }, + ); + await fs.writeFile(summaryPath, `${JSON.stringify(summary, null, 2)}\n`, { + encoding: "utf8", + mode: 0o600, + }); + await fs.writeFile( + observedMessagesPath, + `${JSON.stringify( + buildObservedMessagesArtifact({ + observedMessages, + includeContent: includeObservedMessageContent, + redactMetadata: redactPublicMetadata, + }), + null, + 2, + )}\n`, + { encoding: "utf8", mode: 0o600 }, + ); + const artifactPaths = { + report: reportPath, + summary: summaryPath, + observedMessages: observedMessagesPath, + ...(preservedGatewayDebugArtifacts ? { gatewayDebug: gatewayDebugDirPath } : {}), + }; + if (cleanupIssues.length > 0) { + throw new Error( + buildLiveLaneArtifactsError({ + heading: "Discord QA cleanup failed after artifacts were written.", + details: publishedCleanupIssues, + artifacts: artifactPaths, + }), + ); + } + + return { + outputDir, + reportPath, + summaryPath, + observedMessagesPath, + ...(preservedGatewayDebugArtifacts ? { gatewayDebugDirPath } : {}), + scenarios: scenarioResults, + }; +} + +export const __testing = { + DISCORD_QA_SCENARIOS, + DISCORD_QA_STANDARD_SCENARIO_IDS, + assertDiscordScenarioReply, + assertDiscordApplicationCommandsRegistered, + buildDiscordQaConfig, + buildObservedMessagesArtifact, + callDiscordApi, + findScenario, + listApplicationCommands, + matchesDiscordScenarioReply, + normalizeDiscordObservedMessage, + parseDiscordQaCredentialPayload, + resolveDiscordQaRuntimeEnv, + waitForDiscordChannelRunning, +}; diff --git a/extensions/qa-lab/src/live-transports/shared/live-gateway.runtime.test.ts b/extensions/qa-lab/src/live-transports/shared/live-gateway.runtime.test.ts index f552f147b4f..64e100220cd 100644 --- a/extensions/qa-lab/src/live-transports/shared/live-gateway.runtime.test.ts +++ b/extensions/qa-lab/src/live-transports/shared/live-gateway.runtime.test.ts @@ -94,6 +94,22 @@ describe("startQaLiveLaneGateway", () => { expect(mockStop).toHaveBeenCalledTimes(1); }); + it("forwards gateway stop options to the child harness", async () => { + const harness = await startQaLiveLaneGateway({ + repoRoot: "/tmp/openclaw-repo", + transport: createStubTransport(), + transportBaseUrl: "http://127.0.0.1:43123", + providerMode: "mock-openai", + primaryModel: "mock-openai/gpt-5.4", + alternateModel: "mock-openai/gpt-5.4-alt", + controlUiEnabled: false, + }); + + await harness.stop({ preserveToDir: ".artifacts/qa-e2e/debug" }); + expect(gatewayStop).toHaveBeenCalledWith({ preserveToDir: ".artifacts/qa-e2e/debug" }); + expect(mockStop).toHaveBeenCalledTimes(1); + }); + it("skips mock bootstrap for live frontier runs", async () => { const harness = await startQaLiveLaneGateway({ repoRoot: "/tmp/openclaw-repo", diff --git a/extensions/qa-lab/src/live-transports/shared/live-gateway.runtime.ts b/extensions/qa-lab/src/live-transports/shared/live-gateway.runtime.ts index 0130f94ceb7..3dd23511fda 100644 --- a/extensions/qa-lab/src/live-transports/shared/live-gateway.runtime.ts +++ b/extensions/qa-lab/src/live-transports/shared/live-gateway.runtime.ts @@ -5,13 +5,16 @@ import { startQaProviderServer } from "../../providers/server-runtime.js"; import type { QaThinkingLevel } from "../../qa-gateway-config.js"; import { appendLiveLaneIssue } from "./live-lane-helpers.js"; -async function stopQaLiveLaneResources(resources: { - gateway: Awaited>; - mock: { baseUrl: string; stop(): Promise } | null; -}) { +async function stopQaLiveLaneResources( + resources: { + gateway: Awaited>; + mock: { baseUrl: string; stop(): Promise } | null; + }, + opts?: { keepTemp?: boolean; preserveToDir?: string }, +) { const errors: string[] = []; try { - await resources.gateway.stop(); + await resources.gateway.stop(opts); } catch (error) { appendLiveLaneIssue(errors, "gateway stop failed", error); } @@ -66,8 +69,8 @@ export async function startQaLiveLaneGateway(params: { return { gateway, mock, - async stop() { - await stopQaLiveLaneResources({ gateway, mock }); + async stop(opts?: { keepTemp?: boolean; preserveToDir?: string }) { + await stopQaLiveLaneResources({ gateway, mock }, opts); }, }; } catch (error) {