diff --git a/.github/workflows/mantis-discord-smoke.yml b/.github/workflows/mantis-discord-smoke.yml new file mode 100644 index 00000000000..12ca7680141 --- /dev/null +++ b/.github/workflows/mantis-discord-smoke.yml @@ -0,0 +1,169 @@ +name: Mantis Discord Smoke + +on: + workflow_dispatch: + inputs: + ref: + description: Ref, tag, or SHA to run + required: true + default: main + type: string + post_message: + description: Post a smoke message and reaction to the configured Discord channel + required: true + default: true + type: boolean + +permissions: + contents: read + pull-requests: read + +concurrency: + group: mantis-discord-smoke-${{ inputs.ref }}-${{ github.run_attempt }} + cancel-in-progress: false + +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" + NODE_VERSION: "24.x" + PNPM_VERSION: "10.33.0" + OPENCLAW_BUILD_PRIVATE_QA: "1" + OPENCLAW_ENABLE_PRIVATE_QA_CLI: "1" + +jobs: + authorize_actor: + name: Authorize workflow actor + runs-on: blacksmith-8vcpu-ubuntu-2404 + steps: + - name: Require maintainer-level repository access + uses: actions/github-script@v8 + with: + script: | + const allowed = new Set(["admin", "maintain", "write"]); + const { owner, repo } = context.repo; + const { data } = await github.rest.repos.getCollaboratorPermissionLevel({ + owner, + repo, + username: context.actor, + }); + const permission = data.permission; + core.info(`Actor ${context.actor} permission: ${permission}`); + if (!allowed.has(permission)) { + core.setFailed( + `Workflow requires write/maintain/admin access. Actor "${context.actor}" has "${permission}".`, + ); + } + + validate_selected_ref: + name: Validate selected ref + needs: authorize_actor + runs-on: blacksmith-8vcpu-ubuntu-2404 + outputs: + selected_revision: ${{ steps.validate.outputs.selected_revision }} + trusted_reason: ${{ steps.validate.outputs.trusted_reason }} + steps: + - name: Checkout selected ref + uses: actions/checkout@v6 + with: + persist-credentials: false + ref: ${{ inputs.ref }} + fetch-depth: 0 + + - name: Validate selected ref + id: validate + env: + GH_TOKEN: ${{ github.token }} + INPUT_REF: ${{ inputs.ref }} + shell: bash + run: | + set -euo pipefail + selected_revision="$(git rev-parse HEAD)" + trusted_reason="" + + git fetch --no-tags origin +refs/heads/main:refs/remotes/origin/main + + if git merge-base --is-ancestor "$selected_revision" refs/remotes/origin/main; then + trusted_reason="main-ancestor" + elif git tag --points-at "$selected_revision" | grep -Eq '^v'; then + trusted_reason="release-tag" + elif [[ "$INPUT_REF" =~ ^release/[0-9]{4}\.[0-9]+\.[0-9]+$ ]]; then + git fetch --no-tags origin "+refs/heads/${INPUT_REF}:refs/remotes/origin/${INPUT_REF}" + release_branch_sha="$(git rev-parse "refs/remotes/origin/${INPUT_REF}")" + if [[ "$selected_revision" == "$release_branch_sha" ]]; then + trusted_reason="release-branch-head" + fi + else + pr_head_count="$( + gh api \ + -H "Accept: application/vnd.github+json" \ + "repos/${GITHUB_REPOSITORY}/commits/${selected_revision}/pulls" \ + --jq '[.[] | select(.state == "open" and .head.repo.full_name == "'"${GITHUB_REPOSITORY}"'" and .head.sha == "'"${selected_revision}"'")] | length' + )" + if [[ "$pr_head_count" != "0" ]]; then + trusted_reason="open-pr-head" + fi + fi + + if [[ -z "$trusted_reason" ]]; then + echo "Ref '${INPUT_REF}' resolved to $selected_revision, which is not trusted for this secret-bearing Mantis run." >&2 + echo "Allowed refs must be on main, point to a release tag, match a release branch head, or match an open PR head in ${GITHUB_REPOSITORY}." >&2 + exit 1 + fi + + echo "selected_revision=$selected_revision" >> "$GITHUB_OUTPUT" + echo "trusted_reason=$trusted_reason" >> "$GITHUB_OUTPUT" + { + echo "Validated ref: \`${INPUT_REF}\`" + echo "Resolved SHA: \`$selected_revision\`" + echo "Trust reason: \`$trusted_reason\`" + } >> "$GITHUB_STEP_SUMMARY" + + run_discord_smoke: + name: Run Mantis Discord smoke + needs: validate_selected_ref + runs-on: blacksmith-8vcpu-ubuntu-2404 + timeout-minutes: 20 + environment: qa-live-shared + steps: + - name: Checkout selected ref + uses: actions/checkout@v6 + with: + persist-credentials: false + ref: ${{ needs.validate_selected_ref.outputs.selected_revision }} + fetch-depth: 1 + + - name: Setup Node environment + uses: ./.github/actions/setup-node-env + with: + node-version: ${{ env.NODE_VERSION }} + pnpm-version: ${{ env.PNPM_VERSION }} + install-bun: "true" + + - name: Build private QA runtime + run: pnpm build + + - name: Run Mantis Discord smoke + shell: bash + env: + OPENCLAW_QA_DISCORD_MANTIS_BOT_TOKEN: ${{ secrets.OPENCLAW_QA_DISCORD_MANTIS_BOT_TOKEN }} + OPENCLAW_QA_DISCORD_GUILD_ID: ${{ secrets.OPENCLAW_QA_DISCORD_GUILD_ID }} + OPENCLAW_QA_DISCORD_CHANNEL_ID: ${{ secrets.OPENCLAW_QA_DISCORD_CHANNEL_ID }} + OPENCLAW_QA_REDACT_PUBLIC_METADATA: "1" + run: | + set -euo pipefail + args=() + if [[ "${{ inputs.post_message }}" != "true" ]]; then + args+=(--skip-post) + fi + pnpm openclaw qa mantis discord-smoke \ + --repo-root . \ + --output-dir .artifacts/qa-e2e/mantis/discord-smoke \ + "${args[@]}" + + - name: Upload Mantis artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: mantis-discord-smoke-${{ github.run_id }}-${{ github.run_attempt }} + path: .artifacts/qa-e2e/mantis/ + retention-days: 14 + if-no-files-found: warn diff --git a/CHANGELOG.md b/CHANGELOG.md index 1c72b3fffd7..c699c948d9c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ Docs: https://docs.openclaw.ai ### Changes - Gateway/performance: lazy-load early runtime discovery and shutdown-hook helpers, defer maintenance timers until after readiness, and trim duplicate plugin auto-enable work during Gateway startup. +- QA/Mantis: add a `pnpm openclaw qa mantis discord-smoke` runner and manual GitHub workflow that verify the Mantis Discord bot can see the configured guild/channel, post a smoke message, add a reaction, and upload artifacts. - Gateway/performance: lazy-load the heavy cron runtime after the rest of Gateway startup, defer restart-sentinel refresh after readiness, and let the Gateway startup benchmark write per-run V8 CPU profiles with `--cpu-prof-dir`. - Gateway/performance: keep raw channel-config schema parsing from discovering bundled plugin runtime metadata, and add `pnpm gateway:watch --benchmark-no-force` for profiling startup without the default port cleanup. - Plugins/onboarding: let Manual setup install optional official plugins, including ClawHub-backed diagnostics with npm fallback, and expose the external Codex plugin as a selectable provider setup choice. Thanks @vincentkoc. diff --git a/docs/.i18n/glossary.zh-CN.json b/docs/.i18n/glossary.zh-CN.json index 7f796a7b7b3..fe937d74086 100644 --- a/docs/.i18n/glossary.zh-CN.json +++ b/docs/.i18n/glossary.zh-CN.json @@ -15,6 +15,10 @@ "source": "OpenAI provider", "target": "OpenAI provider" }, + { + "source": "Mantis", + "target": "Mantis" + }, { "source": "OpenClaw App SDK", "target": "OpenClaw 应用 SDK" diff --git a/docs/concepts/mantis.md b/docs/concepts/mantis.md new file mode 100644 index 00000000000..c40a90f20d8 --- /dev/null +++ b/docs/concepts/mantis.md @@ -0,0 +1,412 @@ +--- +summary: "Mantis is the visual end-to-end verification system for reproducing OpenClaw bugs on live transports, capturing before and after evidence, and attaching artifacts to PRs." +title: "Mantis" +read_when: + - Building or running live visual QA for OpenClaw bugs + - Adding before and after verification for a pull request + - Adding Discord, Slack, WhatsApp, or other live transport scenarios + - Debugging QA runs that need screenshots, browser automation, or VNC access +--- + +Mantis is the OpenClaw end-to-end verification system for bugs that need a real +runtime, a real transport, and visible proof. It runs a scenario against a known +bad ref, captures evidence, runs the same scenario against a candidate ref, and +publishes the comparison as artifacts that a maintainer can inspect from a PR or +from a local command. + +Mantis starts with Discord because Discord gives us a high-value first lane: +real bot auth, real guild channels, reactions, threads, native commands, and a +browser UI where humans can visually confirm what the transport showed. + +## Goals + +- Reproduce a bug from a GitHub issue or PR with the same transport shape users + see. +- Capture a **before** artifact on the baseline ref before applying the fix. +- Capture an **after** artifact on the candidate ref after applying the fix. +- Use a deterministic oracle whenever possible, such as a Discord REST reaction + read or channel transcript check. +- Capture screenshots when the bug has a visible UI surface. +- Run locally from an agent-controlled CLI and remotely from GitHub. +- Preserve enough machine state for VNC rescue when login, browser automation, or + provider auth gets stuck. +- Post concise status to an operator Discord channel when the run is blocked, + needs manual VNC help, or finishes. + +## Non Goals + +- Mantis is not a replacement for unit tests. A Mantis run should usually become + a smaller regression test after the fix is understood. +- Mantis is not the normal fast CI gate. It is slower, uses live credentials, and + is reserved for bugs where the live environment matters. +- Mantis should not require a human for normal operation. Manual VNC is a rescue + path, not the happy path. +- Mantis does not store raw secrets in artifacts, logs, screenshots, Markdown + reports, or PR comments. + +## Ownership + +Mantis lives in the OpenClaw QA stack. + +- OpenClaw owns the scenario runtime, transport adapters, evidence schema, and + local CLI under `pnpm openclaw qa mantis`. +- QA Lab owns the live transport harness pieces, browser capture helpers, and + artifact writers. +- Crabbox owns warmed Linux machines when a remote VM is needed. +- GitHub Actions owns the remote workflow entrypoint and artifact retention. +- ClawSweeper owns GitHub comment routing: parsing maintainer commands, + dispatching the workflow, and posting the final PR comment. +- OpenClaw agents drive Mantis through Codex when a scenario needs agentic setup, + debugging, or stuck-state reporting. + +This boundary keeps transport knowledge in OpenClaw, machine scheduling in +Crabbox, and maintainer workflow glue in ClawSweeper. + +## Command Shape + +The first local command verifies the Discord bot, guild, channel, message send, +reaction send, and artifact path: + +```bash +pnpm openclaw qa mantis discord-smoke \ + --output-dir .artifacts/qa-e2e/mantis/discord-smoke +``` + +The later before and after runner should accept this shape: + +```bash +pnpm openclaw qa mantis run \ + --transport discord \ + --scenario discord-status-reactions-tool-only \ + --baseline origin/main \ + --candidate HEAD \ + --output-dir .artifacts/qa-e2e/mantis/local-discord-status-reactions +``` + +The GitHub smoke workflow is `Mantis Discord Smoke`. The before and after GitHub +workflow should accept equivalent inputs: + +- `transport`: `discord` for the first version. +- `scenario`: one or more scenario ids. +- `baseline_ref`: default `origin/main` or the linked issue's reported bad tag. +- `candidate_ref`: the PR head SHA. +- `machine_provider`: `aws` by default, with later `hetzner` fallback. +- `post_to_pr`: whether ClawSweeper should comment with the result. + +ClawSweeper command examples: + +```text +@clawsweeper mantis discord discord-status-reactions-tool-only +@clawsweeper verify e2e discord +``` + +The first command is explicit and scenario-focused. The second can later map a PR +or issue to recommended Mantis scenarios from labels, changed files, and +ClawSweeper review findings. + +## Run Lifecycle + +1. Acquire credentials. +2. Allocate or reuse a VM. +3. Prepare a clean checkout for the baseline ref. +4. Install dependencies and build only what the scenario needs. +5. Start a child OpenClaw Gateway with an isolated state directory. +6. Configure the live transport, provider, model, and browser profile. +7. Run the scenario and capture baseline evidence. +8. Stop the gateway and preserve logs. +9. Prepare the candidate ref in the same VM. +10. Run the same scenario and capture candidate evidence. +11. Compare the oracle results and visual evidence. +12. Write Markdown, JSON, logs, screenshots, and optional trace artifacts. +13. Upload GitHub Actions artifacts. +14. Post a concise PR or Discord status message. + +The scenario should be able to fail in two different ways: + +- **Bug reproduced**: baseline failed in the expected way. +- **Harness failure**: environment setup, credentials, Discord API, browser, or + provider failed before the bug oracle was meaningful. + +The final report must separate these cases so maintainers do not confuse a flaky +environment with product behavior. + +## Discord MVP + +The first scenario should target Discord status reactions in guild channels where +the source reply delivery mode is `message_tool_only`. + +Why it is a good Mantis seed: + +- It is visible in Discord as reactions on the triggering message. +- It has a strong REST oracle through Discord message reaction state. +- It exercises a real OpenClaw Gateway, Discord bot auth, message dispatch, + source reply delivery mode, status reaction state, and model turn lifecycle. +- It is narrow enough to keep the first implementation honest. + +Expected scenario shape: + +```yaml +id: discord-status-reactions-tool-only +transport: discord +baseline: + expect: + reproduced: true +candidate: + expect: + fixed: true +config: + messages: + ackReaction: "πŸ‘€" + ackReactionScope: "group-mentions" + groupChat: + visibleReplies: "message_tool" + statusReactions: + enabled: true + timing: + debounceMs: 0 +discord: + requireMention: true + notifyChannel: operator-notify +evidence: + rest: + messageReactions: true + browser: + screenshotMessageRow: true +``` + +Baseline evidence should show the queued acknowledgement reaction but no +lifecycle transition in tool-only mode. Candidate evidence should show lifecycle +status reactions running when `messages.statusReactions.enabled` is explicitly +true. + +## Existing QA Pieces + +Mantis should build on the existing private QA stack instead of starting from +zero: + +- `pnpm openclaw qa discord` already runs a live Discord lane with driver and + SUT bots. +- The live transport runner already writes reports and observed-message + artifacts under `.artifacts/qa-e2e/`. +- Convex credential leases already provide exclusive access to shared live + transport credentials. +- The browser control service already supports screenshots, snapshots, + headless managed profiles, and remote CDP profiles. +- QA Lab already has a debugger UI and bus for transport-shaped testing. + +The first Mantis implementation can be a thin before/after runner over these +pieces, plus one visual evidence layer. + +## Evidence Model + +Every run writes a stable artifact directory: + +```text +.artifacts/qa-e2e/mantis// + mantis-report.md + mantis-summary.json + baseline/ + summary.json + discord-message.json + screenshot-message-row.png + gateway-debug/ + candidate/ + summary.json + discord-message.json + screenshot-message-row.png + gateway-debug/ + comparison.json + run.log +``` + +`mantis-summary.json` should be the machine-readable source of truth. The +Markdown report is for PR comments and human review. + +The summary must include: + +- refs and SHAs tested +- transport and scenario id +- machine provider and machine id or lease id +- credential source without secret values +- baseline result +- candidate result +- whether the bug reproduced on baseline +- whether the candidate fixed it +- artifact paths +- sanitized setup or cleanup issues + +Screenshots are evidence, not secrets. They still need redaction discipline: +private channel names, user names, or message content may appear. For public PRs, +prefer GitHub Actions artifact links over inline images until the redaction story +is stronger. + +## Browser And VNC + +The browser lane has two modes: + +- **Headless automation**: default for CI. Chrome runs with CDP enabled, and + Playwright or OpenClaw browser control captures screenshots. +- **VNC rescue**: enabled on the same VM when login, MFA, Discord anti-automation, + or visual debugging needs a human. + +The Discord observer browser profile should be persistent enough to avoid +logging in for every run, but isolated from personal browser state. A profile +belongs to the Mantis machine pool, not to a developer laptop. + +When Mantis gets stuck, it posts a Discord status message with: + +- run id +- scenario id +- machine provider +- artifact directory +- VNC or noVNC connection instructions if available +- short blocker text + +The first private deployment can post these messages to the existing operator +channel and move to a dedicated Mantis channel later. + +## Machines + +Mantis should prefer AWS through Crabbox for the first remote implementation. +Crabbox gives us warmed machines, lease tracking, hydration, logs, results, and +cleanup. If AWS capacity is too slow or unavailable, add a Hetzner provider +behind the same machine interface. + +Minimum VM requirements: + +- Linux with a desktop-capable Chrome or Chromium install +- CDP access for browser automation +- VNC or noVNC for rescue +- Node 22 and pnpm +- OpenClaw checkout and dependency cache +- Playwright Chromium browser cache when Playwright is used +- enough CPU and memory for one OpenClaw Gateway, one browser, and one model run +- outbound access to Discord, GitHub, model providers, and the credential broker + +The VM should not keep long-lived raw secrets outside the expected credential or +browser profile stores. + +## Secrets + +Secrets live in GitHub organization or repository secrets for remote runs, and in +a local operator-controlled secret file for local runs. + +Recommended secret names: + +- `OPENCLAW_QA_DISCORD_MANTIS_BOT_TOKEN` +- `OPENCLAW_QA_DISCORD_DRIVER_BOT_TOKEN` +- `OPENCLAW_QA_DISCORD_SUT_BOT_TOKEN` +- `OPENCLAW_QA_DISCORD_GUILD_ID` +- `OPENCLAW_QA_DISCORD_CHANNEL_ID` +- `OPENCLAW_QA_DISCORD_NOTIFY_CHANNEL_ID` +- `OPENCLAW_QA_REDACT_PUBLIC_METADATA=1` for public GitHub artifact uploads +- `OPENCLAW_QA_CONVEX_SITE_URL` +- `OPENCLAW_QA_CONVEX_SECRET_CI` + +Long term, the Convex credential pool should remain the normal source for live +transport credentials. GitHub secrets bootstrap the broker and fallback lanes. + +The Mantis runner must never print: + +- Discord bot tokens +- provider API keys +- browser cookies +- auth profile contents +- VNC passwords +- raw credential payloads + +Public artifact uploads should also redact Discord target metadata such as bot, +guild, channel, and message ids. The GitHub smoke workflow enables +`OPENCLAW_QA_REDACT_PUBLIC_METADATA=1` for this reason. + +If a token is accidentally pasted into an issue, PR, chat, or log, rotate it +after the new secret has been stored. + +## GitHub Artifacts And PR Comments + +The first GitHub version should upload screenshots as Actions artifacts and link +them from the PR comment. Inline images can come later once redaction, retention, +and public/private repo behavior are settled. + +The PR comment should be short: + +```md +Mantis Discord verification: pass + +- Scenario: `discord-status-reactions-tool-only` +- Baseline: reproduced on `` +- Candidate: fixed on `` +- Evidence: +- Screenshots: baseline and candidate message-row captures in the artifact +``` + +When the run fails because the harness failed, the comment must say that instead +of implying the candidate failed. + +## Private Deployment Notes + +A private deployment may already have a Mantis Discord application. Reuse that +application instead of creating another app when it has the right bot +permissions and can be safely rotated. + +Set the initial operator notification channel through secrets or deployment +configuration. It can point at an existing maintainer or operations channel +first, then move to a dedicated Mantis channel once one exists. + +Do not put guild ids, channel ids, bot tokens, browser cookies, or VNC passwords +in this document. Store them in GitHub secrets, the credential broker, or the +operator's local secret store. + +## Adding A Scenario + +A Mantis scenario should declare: + +- id and title +- transport +- required credentials +- baseline ref policy +- candidate ref policy +- OpenClaw config patch +- setup steps +- stimulus +- expected baseline oracle +- expected candidate oracle +- visual capture targets +- timeout budget +- cleanup steps + +Scenarios should prefer small, typed oracles: + +- Discord reaction state for reaction bugs +- Discord message references for threading bugs +- Slack thread ts and reaction API state for Slack bugs +- email message ids and headers for email bugs +- browser screenshots when UI is the only reliable observable + +Vision checks should be additive. If a platform API can prove the bug, use the +API as the pass/fail oracle and keep screenshots for human confidence. + +## Provider Expansion + +After Discord, the same runner can add: + +- Slack: reactions, threads, app mentions, modals, file uploads. +- Email: Gmail auth and message threading using `gog` where connectors are not + enough. +- WhatsApp: QR login, re-identification, message delivery, media, reactions. +- Telegram: group mention gating, commands, reactions where available. +- Matrix: encrypted rooms, thread or reply relations, restart resume. + +Each transport should have one cheap smoke scenario and one or more bug-class +scenarios. Expensive visual scenarios should stay opt-in. + +## Open Questions + +- Which Discord bot should be the driver, and which should be the SUT, when the + existing Mantis bot is reused? +- Should the observer browser login use a human Discord account, a test account, + or only bot-readable REST evidence for the first phase? +- How long should GitHub retain Mantis artifacts for PRs? +- When should ClawSweeper automatically recommend Mantis instead of waiting for a + maintainer command? +- Should screenshots be redacted or cropped before upload for public PRs? diff --git a/docs/concepts/qa-e2e-automation.md b/docs/concepts/qa-e2e-automation.md index 8e8b1112223..ebcbeda90d0 100644 --- a/docs/concepts/qa-e2e-automation.md +++ b/docs/concepts/qa-e2e-automation.md @@ -21,6 +21,8 @@ Current pieces: drive a real channel inside a child QA gateway. - `qa/`: repo-backed seed assets for the kickoff task and baseline QA scenarios. +- [Mantis](/concepts/mantis): before and after live verification for bugs that + need real transports, browser screenshots, VM state, and PR evidence. ## Command surface @@ -45,6 +47,7 @@ script aliases; both forms are supported. | `qa matrix` | Live transport lane against a disposable Tuwunel homeserver. See [Matrix QA](/concepts/qa-matrix). | | `qa telegram` | Live transport lane against a real private Telegram group. | | `qa discord` | Live transport lane against a real private Discord guild channel. | +| `qa mantis` | Planned before and after verification runner for live transport bugs. See [Mantis](/concepts/mantis). | ## Operator flow diff --git a/extensions/qa-lab/src/cli.test.ts b/extensions/qa-lab/src/cli.test.ts index 0a87ba9f328..36062e0e3b0 100644 --- a/extensions/qa-lab/src/cli.test.ts +++ b/extensions/qa-lab/src/cli.test.ts @@ -48,6 +48,7 @@ const { runQaProviderServerCommand, runQaSuiteCommand, runQaTelegramCommand, + runMantisDiscordSmokeCommand, } = vi.hoisted(() => ({ runQaCredentialsAddCommand: vi.fn(), runQaCredentialsListCommand: vi.fn(), @@ -56,6 +57,7 @@ const { runQaProviderServerCommand: vi.fn(), runQaSuiteCommand: vi.fn(), runQaTelegramCommand: vi.fn(), + runMantisDiscordSmokeCommand: vi.fn(), })); const { listQaRunnerCliContributions } = vi.hoisted(() => ({ @@ -72,6 +74,10 @@ vi.mock("./live-transports/telegram/cli.runtime.js", () => ({ runQaTelegramCommand, })); +vi.mock("./mantis/cli.runtime.js", () => ({ + runMantisDiscordSmokeCommand, +})); + vi.mock("./cli.runtime.js", () => ({ runQaCredentialsAddCommand, runQaCredentialsListCommand, @@ -95,6 +101,7 @@ describe("qa cli registration", () => { runQaProviderServerCommand.mockReset(); runQaSuiteCommand.mockReset(); runQaTelegramCommand.mockReset(); + runMantisDiscordSmokeCommand.mockReset(); listQaRunnerCliContributions .mockReset() .mockReturnValue([createAvailableQaRunnerContribution()]); @@ -109,10 +116,51 @@ describe("qa cli registration", () => { const qa = program.commands.find((command) => command.name() === "qa"); expect(qa).toBeDefined(); expect(qa?.commands.map((command) => command.name())).toEqual( - expect.arrayContaining([TEST_QA_RUNNER.commandName, "telegram", "credentials", "coverage"]), + expect.arrayContaining([ + TEST_QA_RUNNER.commandName, + "telegram", + "mantis", + "credentials", + "coverage", + ]), ); }); + it("routes mantis discord-smoke flags into the mantis runtime command", async () => { + await program.parseAsync([ + "node", + "openclaw", + "qa", + "mantis", + "discord-smoke", + "--repo-root", + "/tmp/openclaw-repo", + "--output-dir", + ".artifacts/qa-e2e/mantis/discord-smoke", + "--guild-id", + "123456789012345678", + "--channel-id", + "223456789012345678", + "--token-file", + "/tmp/mantis-token", + "--message", + "hello from mantis", + "--skip-post", + ]); + + expect(runMantisDiscordSmokeCommand).toHaveBeenCalledWith({ + repoRoot: "/tmp/openclaw-repo", + outputDir: ".artifacts/qa-e2e/mantis/discord-smoke", + guildId: "123456789012345678", + channelId: "223456789012345678", + tokenEnv: undefined, + tokenFile: "/tmp/mantis-token", + tokenFileEnv: undefined, + message: "hello from mantis", + skipPost: true, + }); + }); + it("routes coverage report flags into the qa runtime command", async () => { await program.parseAsync([ "node", diff --git a/extensions/qa-lab/src/cli.ts b/extensions/qa-lab/src/cli.ts index 582cd39bf82..b0190cf2306 100644 --- a/extensions/qa-lab/src/cli.ts +++ b/extensions/qa-lab/src/cli.ts @@ -1,6 +1,7 @@ import type { Command } from "commander"; import { collectString } from "./cli-options.js"; import { listLiveTransportQaCliRegistrations } from "./live-transports/cli.js"; +import { registerMantisCli } from "./mantis/cli.js"; import { DEFAULT_QA_LIVE_PROVIDER_MODE, formatQaProviderModeHelp, @@ -225,6 +226,7 @@ export function registerQaLabCli(program: Command) { const qa = program .command("qa") .description("Run private QA automation flows and launch the QA debugger"); + registerMantisCli(qa); qa.command("run") .description("Run the bundled QA self-check and write a Markdown report") diff --git a/extensions/qa-lab/src/mantis/cli.runtime.ts b/extensions/qa-lab/src/mantis/cli.runtime.ts new file mode 100644 index 00000000000..0370d1b2c4f --- /dev/null +++ b/extensions/qa-lab/src/mantis/cli.runtime.ts @@ -0,0 +1,10 @@ +import { runMantisDiscordSmoke, type MantisDiscordSmokeOptions } from "./discord-smoke.runtime.js"; + +export async function runMantisDiscordSmokeCommand(opts: MantisDiscordSmokeOptions) { + const result = await runMantisDiscordSmoke(opts); + process.stdout.write(`Mantis Discord smoke report: ${result.reportPath}\n`); + process.stdout.write(`Mantis Discord smoke summary: ${result.summaryPath}\n`); + if (result.status === "fail") { + process.exitCode = 1; + } +} diff --git a/extensions/qa-lab/src/mantis/cli.ts b/extensions/qa-lab/src/mantis/cli.ts new file mode 100644 index 00000000000..4a2b22bb8ae --- /dev/null +++ b/extensions/qa-lab/src/mantis/cli.ts @@ -0,0 +1,58 @@ +import type { Command } from "commander"; +import { createLazyCliRuntimeLoader } from "../live-transports/shared/live-transport-cli.js"; +import type { MantisDiscordSmokeOptions } from "./discord-smoke.runtime.js"; + +type MantisCliRuntime = typeof import("./cli.runtime.js"); + +const loadMantisCliRuntime = createLazyCliRuntimeLoader( + () => import("./cli.runtime.js"), +); + +async function runDiscordSmoke(opts: MantisDiscordSmokeOptions) { + const runtime = await loadMantisCliRuntime(); + await runtime.runMantisDiscordSmokeCommand(opts); +} + +type MantisDiscordSmokeCommanderOptions = { + channelId?: string; + guildId?: string; + message?: string; + outputDir?: string; + repoRoot?: string; + skipPost?: boolean; + tokenFile?: string; + tokenFileEnv?: string; + tokenEnv?: string; +}; + +export function registerMantisCli(qa: Command) { + const mantis = qa + .command("mantis") + .description("Run Mantis before/after and live-smoke verification flows"); + + mantis + .command("discord-smoke") + .description("Verify the Mantis Discord bot can see the guild/channel, post, and react") + .option("--repo-root ", "Repository root to target when running from a neutral cwd") + .option("--output-dir ", "Mantis Discord smoke artifact directory") + .option("--guild-id ", "Override OPENCLAW_QA_DISCORD_GUILD_ID") + .option("--channel-id ", "Override OPENCLAW_QA_DISCORD_CHANNEL_ID") + .option("--token-env ", "Env var containing the Mantis Discord bot token") + .option("--token-file ", "File containing the Mantis Discord bot token") + .option("--token-file-env ", "Env var containing the Mantis Discord bot token file path") + .option("--message ", "Smoke message to post") + .option("--skip-post", "Only check Discord API visibility; do not post or react", false) + .action(async (opts: MantisDiscordSmokeCommanderOptions) => { + await runDiscordSmoke({ + channelId: opts.channelId, + guildId: opts.guildId, + message: opts.message, + outputDir: opts.outputDir, + repoRoot: opts.repoRoot, + skipPost: opts.skipPost, + tokenFile: opts.tokenFile, + tokenFileEnv: opts.tokenFileEnv, + tokenEnv: opts.tokenEnv, + }); + }); +} diff --git a/extensions/qa-lab/src/mantis/discord-smoke.runtime.test.ts b/extensions/qa-lab/src/mantis/discord-smoke.runtime.test.ts new file mode 100644 index 00000000000..11a3eac0bca --- /dev/null +++ b/extensions/qa-lab/src/mantis/discord-smoke.runtime.test.ts @@ -0,0 +1,310 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const { fetchWithSsrFGuard } = vi.hoisted(() => ({ + fetchWithSsrFGuard: vi.fn(), +})); + +vi.mock("openclaw/plugin-sdk/ssrf-runtime", () => ({ + fetchWithSsrFGuard, +})); + +import { runMantisDiscordSmoke } from "./discord-smoke.runtime.js"; + +function jsonResponse(payload: unknown, status = 200) { + return new Response(JSON.stringify(payload), { + status, + headers: { "content-type": "application/json" }, + }); +} + +function emptyResponse(status = 204) { + return new Response(null, { status }); +} + +describe("mantis discord smoke runtime", () => { + let repoRoot: string; + let tokenFile: string; + + beforeEach(async () => { + repoRoot = await fs.mkdtemp(path.join(os.tmpdir(), "mantis-discord-smoke-")); + tokenFile = path.join(repoRoot, "mantis-token"); + await fs.writeFile(tokenFile, "test-token", "utf8"); + fetchWithSsrFGuard.mockReset(); + const reactionPaths = new Set([ + "/api/v10/channels/1456744319972282449/messages/1500000000000000001/reactions/%F0%9F%91%80/@me", + "/api/v10/channels/1456744319972282449/messages/1500000000000000001/reactions/πŸ‘€/@me", + ]); + fetchWithSsrFGuard.mockImplementation( + async ({ url, init }: { url: string; init?: RequestInit }) => { + const pathname = new URL(url).pathname; + const method = init?.method ?? "GET"; + if (pathname === "/api/v10/users/@me") { + return { + response: jsonResponse({ id: "1489650053747314748", username: "Mantis" }), + release: vi.fn(), + }; + } + if (pathname === "/api/v10/guilds/1456350064065904867") { + return { + response: jsonResponse({ id: "1456350064065904867", name: "Friends" }), + release: vi.fn(), + }; + } + if (pathname === "/api/v10/guilds/1456350064065904867/channels") { + return { response: jsonResponse([{ id: "1456744319972282449" }]), release: vi.fn() }; + } + if (pathname === "/api/v10/channels/1456744319972282449" && method === "GET") { + return { + response: jsonResponse({ + guild_id: "1456350064065904867", + id: "1456744319972282449", + name: "maintainers", + type: 0, + }), + release: vi.fn(), + }; + } + if (pathname === "/api/v10/channels/1456744319972282449/messages" && method === "POST") { + return { + response: jsonResponse({ + id: "1500000000000000001", + channel_id: "1456744319972282449", + }), + release: vi.fn(), + }; + } + if (reactionPaths.has(pathname) && method === "PUT") { + return { response: emptyResponse(), release: vi.fn() }; + } + return { + response: jsonResponse({ message: `unexpected ${method} ${pathname}` }, 404), + release: vi.fn(), + }; + }, + ); + }); + + afterEach(async () => { + await fs.rm(repoRoot, { recursive: true, force: true }); + }); + + it("writes pass artifacts without leaking the bot token", async () => { + const result = await runMantisDiscordSmoke({ + repoRoot, + outputDir: ".artifacts/qa-e2e/mantis/test", + tokenFile, + env: { + OPENCLAW_QA_DISCORD_GUILD_ID: "1456350064065904867", + OPENCLAW_QA_DISCORD_CHANNEL_ID: "1456744319972282449", + }, + now: () => new Date("2026-05-03T12:00:00.000Z"), + }); + + expect(result.status).toBe("pass"); + const summary = JSON.parse(await fs.readFile(result.summaryPath, "utf8")) as { + status: string; + tokenSource: string; + message: { id: string; posted: boolean; reactionAdded: boolean }; + }; + expect(summary).toMatchObject({ + status: "pass", + tokenSource: "file", + message: { + id: "1500000000000000001", + posted: true, + reactionAdded: true, + }, + }); + expect(await fs.readFile(result.summaryPath, "utf8")).not.toContain("test-token"); + expect(await fs.readFile(result.reportPath, "utf8")).not.toContain("test-token"); + }); + + it("supports visibility-only smoke runs", async () => { + const result = await runMantisDiscordSmoke({ + repoRoot, + outputDir: ".artifacts/qa-e2e/mantis/visibility", + tokenFile, + skipPost: true, + env: { + OPENCLAW_QA_DISCORD_GUILD_ID: "1456350064065904867", + OPENCLAW_QA_DISCORD_CHANNEL_ID: "1456744319972282449", + }, + }); + + expect(result.status).toBe("pass"); + expect(fetchWithSsrFGuard).not.toHaveBeenCalledWith( + expect.objectContaining({ + init: expect.objectContaining({ method: "POST" }), + }), + ); + }); + + it("redacts Discord target metadata in public artifacts", async () => { + const result = await runMantisDiscordSmoke({ + repoRoot, + outputDir: ".artifacts/qa-e2e/mantis/redacted", + tokenFile, + redactPublicMetadata: true, + env: { + OPENCLAW_QA_DISCORD_GUILD_ID: "1456350064065904867", + OPENCLAW_QA_DISCORD_CHANNEL_ID: "1456744319972282449", + }, + }); + + expect(result.status).toBe("pass"); + const summaryText = await fs.readFile(result.summaryPath, "utf8"); + const reportText = await fs.readFile(result.reportPath, "utf8"); + expect(reportText).toContain("# Mantis Discord Smoke"); + expect(reportText).toContain("- Bot: ()"); + expect(reportText).toContain("- Guild: ()"); + expect(reportText).toContain("- Channel: # ()"); + for (const text of [summaryText, reportText]) { + expect(text).toContain(""); + expect(text).not.toContain("1489650053747314748"); + expect(text).not.toContain("1456350064065904867"); + expect(text).not.toContain("Friends"); + expect(text).not.toContain("1456744319972282449"); + expect(text).not.toContain("maintainers"); + expect(text).not.toContain("1500000000000000001"); + } + expect(summaryText).not.toContain("Mantis"); + expect(JSON.parse(summaryText)).toMatchObject({ + metadataRedaction: true, + bot: { id: "", username: "" }, + guild: { id: "", name: "" }, + channel: { id: "", name: "" }, + message: { id: "" }, + }); + }); + + it("fails before calling Discord when required ids are missing", async () => { + const result = await runMantisDiscordSmoke({ + repoRoot, + outputDir: ".artifacts/qa-e2e/mantis/missing", + tokenFile, + env: {}, + }); + + expect(result.status).toBe("fail"); + const errorText = await fs.readFile(path.join(result.outputDir, "error.txt"), "utf8"); + expect(errorText).toContain("Missing OPENCLAW_QA_DISCORD_GUILD_ID"); + }); + + it("fails when the channel is not in the configured guild", async () => { + fetchWithSsrFGuard.mockImplementation( + async ({ url, init }: { url: string; init?: RequestInit }) => { + const pathname = new URL(url).pathname; + const method = init?.method ?? "GET"; + if (pathname === "/api/v10/users/@me") { + return { + response: jsonResponse({ id: "1489650053747314748", username: "Mantis" }), + release: vi.fn(), + }; + } + if (pathname === "/api/v10/guilds/1456350064065904867") { + return { + response: jsonResponse({ id: "1456350064065904867", name: "Friends" }), + release: vi.fn(), + }; + } + if (pathname === "/api/v10/guilds/1456350064065904867/channels") { + return { response: jsonResponse([{ id: "1999999999999999999" }]), release: vi.fn() }; + } + if (pathname === "/api/v10/channels/1456744319972282449" && method === "GET") { + return { + response: jsonResponse({ + guild_id: "1999999999999999999", + id: "1456744319972282449", + name: "wrong-guild-channel", + type: 0, + }), + release: vi.fn(), + }; + } + return { + response: jsonResponse({ message: `unexpected ${method} ${pathname}` }, 404), + release: vi.fn(), + }; + }, + ); + + const result = await runMantisDiscordSmoke({ + repoRoot, + outputDir: ".artifacts/qa-e2e/mantis/wrong-guild", + tokenFile, + env: { + OPENCLAW_QA_DISCORD_GUILD_ID: "1456350064065904867", + OPENCLAW_QA_DISCORD_CHANNEL_ID: "1456744319972282449", + }, + }); + + expect(result.status).toBe("fail"); + const errorText = await fs.readFile(path.join(result.outputDir, "error.txt"), "utf8"); + expect(errorText).toContain("is not in guild"); + expect(fetchWithSsrFGuard).not.toHaveBeenCalledWith( + expect.objectContaining({ + init: expect.objectContaining({ method: "POST" }), + }), + ); + }); + + it("redacts response guild ids in mismatch failure artifacts", async () => { + fetchWithSsrFGuard.mockImplementation( + async ({ url, init }: { url: string; init?: RequestInit }) => { + const pathname = new URL(url).pathname; + const method = init?.method ?? "GET"; + if (pathname === "/api/v10/users/@me") { + return { + response: jsonResponse({ id: "1489650053747314748", username: "Mantis" }), + release: vi.fn(), + }; + } + if (pathname === "/api/v10/guilds/1456350064065904867") { + return { + response: jsonResponse({ id: "1456350064065904867", name: "Friends" }), + release: vi.fn(), + }; + } + if (pathname === "/api/v10/guilds/1456350064065904867/channels") { + return { response: jsonResponse([{ id: "1456744319972282449" }]), release: vi.fn() }; + } + if (pathname === "/api/v10/channels/1456744319972282449" && method === "GET") { + return { + response: jsonResponse({ + guild_id: "1999999999999999999", + id: "1456744319972282449", + name: "wrong-guild-channel", + type: 0, + }), + release: vi.fn(), + }; + } + return { + response: jsonResponse({ message: `unexpected ${method} ${pathname}` }, 404), + release: vi.fn(), + }; + }, + ); + + const result = await runMantisDiscordSmoke({ + repoRoot, + outputDir: ".artifacts/qa-e2e/mantis/wrong-guild-redacted", + tokenFile, + redactPublicMetadata: true, + env: { + OPENCLAW_QA_DISCORD_GUILD_ID: "1456350064065904867", + OPENCLAW_QA_DISCORD_CHANNEL_ID: "1456744319972282449", + }, + }); + + expect(result.status).toBe("fail"); + const errorText = await fs.readFile(path.join(result.outputDir, "error.txt"), "utf8"); + expect(errorText).toContain(""); + expect(errorText).not.toContain("1999999999999999999"); + expect(errorText).not.toContain("1456350064065904867"); + expect(errorText).not.toContain("1456744319972282449"); + }); +}); diff --git a/extensions/qa-lab/src/mantis/discord-smoke.runtime.ts b/extensions/qa-lab/src/mantis/discord-smoke.runtime.ts new file mode 100644 index 00000000000..fc35ebf5cca --- /dev/null +++ b/extensions/qa-lab/src/mantis/discord-smoke.runtime.ts @@ -0,0 +1,491 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime"; +import { fetchWithSsrFGuard } from "openclaw/plugin-sdk/ssrf-runtime"; +import { ensureRepoBoundDirectory, resolveRepoRelativeOutputDir } from "../cli-paths.js"; + +export type MantisDiscordSmokeOptions = { + channelId?: string; + env?: NodeJS.ProcessEnv; + guildId?: string; + message?: string; + now?: () => Date; + outputDir?: string; + redactPublicMetadata?: boolean; + repoRoot?: string; + skipPost?: boolean; + token?: string; + tokenEnv?: string; + tokenFile?: string; + tokenFileEnv?: string; +}; + +export type MantisDiscordSmokeResult = { + outputDir: string; + reportPath: string; + summaryPath: string; + status: "pass" | "fail"; +}; + +type DiscordUser = { + id: string; + username?: string; +}; + +type DiscordGuild = { + id: string; + name?: string; +}; + +type DiscordChannel = { + guild_id?: string; + id: string; + name?: string; + type?: number; +}; + +type DiscordMessage = { + id: string; + channel_id: string; +}; + +type DiscordApiCall = { + label: string; + method: string; + ok: boolean; + path: string; + status: number; +}; + +type MantisDiscordSmokeSummary = { + apiCalls: DiscordApiCall[]; + artifacts: { + reportPath: string; + summaryPath: string; + }; + bot?: { + id: string; + username?: string; + }; + channel?: { + id: string; + name?: string; + type?: number; + }; + finishedAt: string; + guild?: { + id: string; + name?: string; + }; + message?: { + id: string; + posted: boolean; + reactionAdded: boolean; + }; + metadataRedaction: boolean; + outputDir: string; + reportPath: string; + startedAt: string; + status: "pass" | "fail"; + summaryPath: string; + tokenSource: "env" | "file" | "option"; +}; + +const DISCORD_API_BASE_URL = "https://discord.com/api/v10"; +const DEFAULT_MANTIS_TOKEN_ENV = "OPENCLAW_QA_DISCORD_MANTIS_BOT_TOKEN"; +const DEFAULT_MANTIS_TOKEN_FILE_ENV = "OPENCLAW_QA_DISCORD_MANTIS_BOT_TOKEN_FILE"; +const DEFAULT_GUILD_ID_ENV = "OPENCLAW_QA_DISCORD_GUILD_ID"; +const DEFAULT_CHANNEL_ID_ENV = "OPENCLAW_QA_DISCORD_CHANNEL_ID"; +const QA_REDACT_PUBLIC_METADATA_ENV = "OPENCLAW_QA_REDACT_PUBLIC_METADATA"; + +function trimToValue(value: string | undefined) { + const trimmed = value?.trim(); + return trimmed && trimmed.length > 0 ? trimmed : undefined; +} + +function isTruthyOptIn(value: string | undefined) { + const normalized = value?.trim().toLowerCase(); + return normalized === "1" || normalized === "true" || normalized === "yes"; +} + +function assertDiscordSnowflake(value: string, label: string) { + if (!/^\d{17,20}$/u.test(value)) { + throw new Error(`${label} must be a Discord snowflake.`); + } +} + +async function readTokenFile(filePath: string) { + const token = trimToValue(await fs.readFile(filePath, "utf8")); + if (!token) { + throw new Error(`Mantis Discord token file is empty: ${filePath}`); + } + return token; +} + +async function resolveMantisDiscordToken(opts: MantisDiscordSmokeOptions) { + const env = opts.env ?? process.env; + const tokenEnv = trimToValue(opts.tokenEnv) ?? DEFAULT_MANTIS_TOKEN_ENV; + const tokenFileEnv = trimToValue(opts.tokenFileEnv) ?? DEFAULT_MANTIS_TOKEN_FILE_ENV; + const optionToken = trimToValue(opts.token); + if (optionToken) { + return { source: "option" as const, token: optionToken }; + } + const envToken = trimToValue(env[tokenEnv]); + if (envToken) { + return { source: "env" as const, token: envToken }; + } + const tokenFile = trimToValue(opts.tokenFile) ?? trimToValue(env[tokenFileEnv]); + if (tokenFile) { + return { source: "file" as const, token: await readTokenFile(tokenFile) }; + } + throw new Error( + `Missing Mantis Discord bot token. Set ${tokenEnv}, ${tokenFileEnv}, or pass --token-file.`, + ); +} + +function resolveRequiredSnowflake(params: { + env: NodeJS.ProcessEnv; + envKey: string; + label: string; + value?: string; +}) { + const resolved = trimToValue(params.value) ?? trimToValue(params.env[params.envKey]); + if (!resolved) { + throw new Error(`Missing ${params.envKey}.`); + } + assertDiscordSnowflake(resolved, params.label); + return resolved; +} + +function assertMantisDiscordChannelInGuild(params: { + channel: DiscordChannel; + guildChannels: readonly DiscordChannel[]; + guildId: string; + channelId: string; +}) { + if (!params.guildChannels.some((channel) => channel.id === params.channelId)) { + throw new Error( + `OPENCLAW_QA_DISCORD_CHANNEL_ID ${params.channelId} is not in guild ${params.guildId}.`, + ); + } + if (params.channel.guild_id && params.channel.guild_id !== params.guildId) { + throw new Error( + `OPENCLAW_QA_DISCORD_CHANNEL_ID ${params.channelId} belongs to guild ${params.channel.guild_id}, not ${params.guildId}.`, + ); + } +} + +function defaultMantisDiscordSmokeOutputDir(repoRoot: string, startedAt: Date) { + const stamp = startedAt.toISOString().replace(/[:.]/gu, "-"); + return path.join(repoRoot, ".artifacts", "qa-e2e", "mantis", `discord-smoke-${stamp}`); +} + +async function callDiscordApi(params: { + apiCalls: DiscordApiCall[]; + body?: unknown; + label: string; + method?: string; + path: string; + token: string; +}) { + const method = params.method ?? "GET"; + const headers = new Headers(); + headers.set("authorization", `Bot ${params.token}`); + let body: string | undefined; + if (params.body !== undefined) { + headers.set("content-type", "application/json"); + body = JSON.stringify(params.body); + } + const { response, release } = await fetchWithSsrFGuard({ + url: `${DISCORD_API_BASE_URL}${params.path}`, + init: { + method, + headers, + body, + }, + signal: AbortSignal.timeout(15_000), + policy: { hostnameAllowlist: ["discord.com"] }, + auditContext: "qa-lab-mantis-discord-smoke", + }); + try { + const text = await response.text(); + const payload = text.trim() ? (JSON.parse(text) as unknown) : undefined; + params.apiCalls.push({ + label: params.label, + method, + ok: response.ok, + path: params.path, + status: response.status, + }); + if (!response.ok) { + const message = + payload && + typeof payload === "object" && + typeof (payload as { message?: unknown }).message === "string" + ? (payload as { message: string }).message + : text.trim(); + throw new Error( + message || `Discord API ${params.path} failed with status ${response.status}`, + ); + } + return payload as T; + } finally { + await release(); + } +} + +function renderMantisDiscordSmokeReport(summary: MantisDiscordSmokeSummary) { + const lines = [ + "# Mantis Discord Smoke", + "", + `Status: ${summary.status}`, + `Metadata redaction: ${summary.metadataRedaction ? "enabled" : "disabled"}`, + `Started: ${summary.startedAt}`, + `Finished: ${summary.finishedAt}`, + `Output: ${summary.outputDir}`, + "", + "## Target", + "", + `- Bot: ${summary.bot?.username ?? "unknown"} (${summary.bot?.id ?? "unknown"})`, + `- Guild: ${summary.guild?.name ?? "unknown"} (${summary.guild?.id ?? "unknown"})`, + `- Channel: #${summary.channel?.name ?? "unknown"} (${summary.channel?.id ?? "unknown"})`, + "", + "## Message", + "", + summary.message?.posted + ? `- Posted message: ${summary.message.id}` + : "- Posted message: skipped", + summary.message?.reactionAdded ? "- Added reaction: yes" : "- Added reaction: no", + "", + "## Discord API Calls", + "", + "| Label | Method | Status |", + "| --- | --- | --- |", + ...summary.apiCalls.map((call) => `| ${call.label} | ${call.method} | ${call.status} |`), + "", + ]; + return `${lines.join("\n")}\n`; +} + +function addSensitiveValue(values: Set, value: string | undefined) { + const resolved = trimToValue(value); + if (resolved && resolved !== "") { + values.add(resolved); + } +} + +function redactMantisDiscordMetadata(text: string, sensitiveValues: ReadonlySet) { + let redacted = text; + const sortedValues = [...sensitiveValues].toSorted((a, b) => b.length - a.length); + for (const value of sortedValues) { + redacted = redacted.replaceAll(value, ""); + } + return redacted; +} + +function buildPublishedMantisDiscordSmokeSummary( + summary: MantisDiscordSmokeSummary, + sensitiveValues: ReadonlySet, +): MantisDiscordSmokeSummary { + if (!summary.metadataRedaction) { + return summary; + } + return { + ...summary, + apiCalls: summary.apiCalls.map((call) => ({ + ...call, + path: redactMantisDiscordMetadata(call.path, sensitiveValues), + })), + bot: summary.bot + ? { + id: "", + username: summary.bot.username ? "" : undefined, + } + : undefined, + channel: summary.channel + ? { + id: "", + name: summary.channel.name ? "" : undefined, + type: summary.channel.type, + } + : undefined, + guild: summary.guild + ? { + id: "", + name: summary.guild.name ? "" : undefined, + } + : undefined, + message: summary.message + ? { + ...summary.message, + id: summary.message.id ? "" : "", + } + : undefined, + }; +} + +async function writeMantisDiscordSmokeArtifacts( + summary: MantisDiscordSmokeSummary, + sensitiveValues: ReadonlySet, +) { + await fs.mkdir(summary.outputDir, { recursive: true }); + const publishedSummary = buildPublishedMantisDiscordSmokeSummary(summary, sensitiveValues); + const report = renderMantisDiscordSmokeReport(publishedSummary); + const summaryJson = `${JSON.stringify(publishedSummary, null, 2)}\n`; + await fs.writeFile(summary.reportPath, report, "utf8"); + await fs.writeFile(summary.summaryPath, summaryJson, "utf8"); +} + +export async function runMantisDiscordSmoke( + opts: MantisDiscordSmokeOptions = {}, +): Promise { + const env = opts.env ?? process.env; + const startedAt = (opts.now ?? (() => new Date()))(); + const redactPublicMetadata = + opts.redactPublicMetadata ?? isTruthyOptIn(env[QA_REDACT_PUBLIC_METADATA_ENV]); + const repoRoot = path.resolve(opts.repoRoot ?? process.cwd()); + const outputDir = await ensureRepoBoundDirectory( + repoRoot, + resolveRepoRelativeOutputDir(repoRoot, opts.outputDir) ?? + defaultMantisDiscordSmokeOutputDir(repoRoot, startedAt), + "Mantis Discord smoke output directory", + { mode: 0o755 }, + ); + const summaryPath = path.join(outputDir, "mantis-discord-smoke-summary.json"); + const reportPath = path.join(outputDir, "mantis-discord-smoke-report.md"); + const apiCalls: DiscordApiCall[] = []; + const sensitiveValues = new Set(); + const summary: MantisDiscordSmokeSummary = { + apiCalls, + artifacts: { + reportPath, + summaryPath, + }, + finishedAt: startedAt.toISOString(), + metadataRedaction: redactPublicMetadata, + outputDir, + reportPath, + startedAt: startedAt.toISOString(), + status: "fail", + summaryPath, + tokenSource: "env", + }; + + try { + const { source, token } = await resolveMantisDiscordToken(opts); + summary.tokenSource = source; + const guildId = resolveRequiredSnowflake({ + env, + envKey: DEFAULT_GUILD_ID_ENV, + label: DEFAULT_GUILD_ID_ENV, + value: opts.guildId, + }); + const channelId = resolveRequiredSnowflake({ + env, + envKey: DEFAULT_CHANNEL_ID_ENV, + label: DEFAULT_CHANNEL_ID_ENV, + value: opts.channelId, + }); + addSensitiveValue(sensitiveValues, guildId); + addSensitiveValue(sensitiveValues, channelId); + const bot = await callDiscordApi({ + apiCalls, + label: "current-user", + path: "/users/@me", + token, + }); + addSensitiveValue(sensitiveValues, bot.id); + addSensitiveValue(sensitiveValues, bot.username); + const guild = await callDiscordApi({ + apiCalls, + label: "guild", + path: `/guilds/${guildId}`, + token, + }); + addSensitiveValue(sensitiveValues, guild.id); + addSensitiveValue(sensitiveValues, guild.name); + const guildChannels = await callDiscordApi({ + apiCalls, + label: "guild-channels", + path: `/guilds/${guildId}/channels`, + token, + }); + for (const guildChannel of guildChannels) { + addSensitiveValue(sensitiveValues, guildChannel.id); + addSensitiveValue(sensitiveValues, guildChannel.guild_id); + addSensitiveValue(sensitiveValues, guildChannel.name); + } + const channel = await callDiscordApi({ + apiCalls, + label: "channel", + path: `/channels/${channelId}`, + token, + }); + addSensitiveValue(sensitiveValues, channel.id); + addSensitiveValue(sensitiveValues, channel.guild_id); + addSensitiveValue(sensitiveValues, channel.name); + assertMantisDiscordChannelInGuild({ + channel, + guildChannels, + guildId, + channelId, + }); + summary.bot = { id: bot.id, username: bot.username }; + summary.guild = { id: guild.id, name: guild.name }; + summary.channel = { id: channel.id, name: channel.name, type: channel.type }; + + if (opts.skipPost) { + summary.message = { id: "", posted: false, reactionAdded: false }; + } else { + const message = await callDiscordApi({ + apiCalls, + body: { + content: + trimToValue(opts.message) ?? `Mantis Discord smoke: OK (${startedAt.toISOString()})`, + }, + label: "post-message", + method: "POST", + path: `/channels/${channelId}/messages`, + token, + }); + addSensitiveValue(sensitiveValues, message.id); + await callDiscordApi({ + apiCalls, + label: "add-reaction", + method: "PUT", + path: `/channels/${channelId}/messages/${message.id}/reactions/%F0%9F%91%80/@me`, + token, + }); + summary.message = { id: message.id, posted: true, reactionAdded: true }; + } + + summary.status = "pass"; + } catch (error) { + summary.status = "fail"; + summary.message = summary.message ?? { + id: "", + posted: false, + reactionAdded: false, + }; + await fs.writeFile( + path.join(outputDir, "error.txt"), + `${ + redactPublicMetadata + ? redactMantisDiscordMetadata(formatErrorMessage(error), sensitiveValues) + : formatErrorMessage(error) + }${os.EOL}`, + "utf8", + ); + } finally { + summary.finishedAt = new Date().toISOString(); + await writeMantisDiscordSmokeArtifacts(summary, sensitiveValues); + } + + return { + outputDir, + reportPath, + summaryPath, + status: summary.status, + }; +}