mirror of
https://github.com/openclaw/openclaw.git
synced 2026-05-06 05:10:44 +00:00
feat: add Mantis Discord smoke runner (#76696)
* docs: add Mantis QA system design * feat: add Mantis Discord smoke runner * fix: harden Mantis Discord smoke * fix: redact Mantis Discord artifacts * fix: satisfy Mantis redaction lint * fix: redact Mantis mismatch failures * test: avoid promise assertions in Mantis tests
This commit is contained in:
committed by
GitHub
parent
6aa4fb7a69
commit
0bf06e953f
169
.github/workflows/mantis-discord-smoke.yml
vendored
Normal file
169
.github/workflows/mantis-discord-smoke.yml
vendored
Normal file
@@ -0,0 +1,169 @@
|
||||
name: Mantis Discord Smoke
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
ref:
|
||||
description: Ref, tag, or SHA to run
|
||||
required: true
|
||||
default: main
|
||||
type: string
|
||||
post_message:
|
||||
description: Post a smoke message and reaction to the configured Discord channel
|
||||
required: true
|
||||
default: true
|
||||
type: boolean
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
|
||||
concurrency:
|
||||
group: mantis-discord-smoke-${{ inputs.ref }}-${{ github.run_attempt }}
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
NODE_VERSION: "24.x"
|
||||
PNPM_VERSION: "10.33.0"
|
||||
OPENCLAW_BUILD_PRIVATE_QA: "1"
|
||||
OPENCLAW_ENABLE_PRIVATE_QA_CLI: "1"
|
||||
|
||||
jobs:
|
||||
authorize_actor:
|
||||
name: Authorize workflow actor
|
||||
runs-on: blacksmith-8vcpu-ubuntu-2404
|
||||
steps:
|
||||
- name: Require maintainer-level repository access
|
||||
uses: actions/github-script@v8
|
||||
with:
|
||||
script: |
|
||||
const allowed = new Set(["admin", "maintain", "write"]);
|
||||
const { owner, repo } = context.repo;
|
||||
const { data } = await github.rest.repos.getCollaboratorPermissionLevel({
|
||||
owner,
|
||||
repo,
|
||||
username: context.actor,
|
||||
});
|
||||
const permission = data.permission;
|
||||
core.info(`Actor ${context.actor} permission: ${permission}`);
|
||||
if (!allowed.has(permission)) {
|
||||
core.setFailed(
|
||||
`Workflow requires write/maintain/admin access. Actor "${context.actor}" has "${permission}".`,
|
||||
);
|
||||
}
|
||||
|
||||
validate_selected_ref:
|
||||
name: Validate selected ref
|
||||
needs: authorize_actor
|
||||
runs-on: blacksmith-8vcpu-ubuntu-2404
|
||||
outputs:
|
||||
selected_revision: ${{ steps.validate.outputs.selected_revision }}
|
||||
trusted_reason: ${{ steps.validate.outputs.trusted_reason }}
|
||||
steps:
|
||||
- name: Checkout selected ref
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
persist-credentials: false
|
||||
ref: ${{ inputs.ref }}
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Validate selected ref
|
||||
id: validate
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
INPUT_REF: ${{ inputs.ref }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
selected_revision="$(git rev-parse HEAD)"
|
||||
trusted_reason=""
|
||||
|
||||
git fetch --no-tags origin +refs/heads/main:refs/remotes/origin/main
|
||||
|
||||
if git merge-base --is-ancestor "$selected_revision" refs/remotes/origin/main; then
|
||||
trusted_reason="main-ancestor"
|
||||
elif git tag --points-at "$selected_revision" | grep -Eq '^v'; then
|
||||
trusted_reason="release-tag"
|
||||
elif [[ "$INPUT_REF" =~ ^release/[0-9]{4}\.[0-9]+\.[0-9]+$ ]]; then
|
||||
git fetch --no-tags origin "+refs/heads/${INPUT_REF}:refs/remotes/origin/${INPUT_REF}"
|
||||
release_branch_sha="$(git rev-parse "refs/remotes/origin/${INPUT_REF}")"
|
||||
if [[ "$selected_revision" == "$release_branch_sha" ]]; then
|
||||
trusted_reason="release-branch-head"
|
||||
fi
|
||||
else
|
||||
pr_head_count="$(
|
||||
gh api \
|
||||
-H "Accept: application/vnd.github+json" \
|
||||
"repos/${GITHUB_REPOSITORY}/commits/${selected_revision}/pulls" \
|
||||
--jq '[.[] | select(.state == "open" and .head.repo.full_name == "'"${GITHUB_REPOSITORY}"'" and .head.sha == "'"${selected_revision}"'")] | length'
|
||||
)"
|
||||
if [[ "$pr_head_count" != "0" ]]; then
|
||||
trusted_reason="open-pr-head"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ -z "$trusted_reason" ]]; then
|
||||
echo "Ref '${INPUT_REF}' resolved to $selected_revision, which is not trusted for this secret-bearing Mantis run." >&2
|
||||
echo "Allowed refs must be on main, point to a release tag, match a release branch head, or match an open PR head in ${GITHUB_REPOSITORY}." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "selected_revision=$selected_revision" >> "$GITHUB_OUTPUT"
|
||||
echo "trusted_reason=$trusted_reason" >> "$GITHUB_OUTPUT"
|
||||
{
|
||||
echo "Validated ref: \`${INPUT_REF}\`"
|
||||
echo "Resolved SHA: \`$selected_revision\`"
|
||||
echo "Trust reason: \`$trusted_reason\`"
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
run_discord_smoke:
|
||||
name: Run Mantis Discord smoke
|
||||
needs: validate_selected_ref
|
||||
runs-on: blacksmith-8vcpu-ubuntu-2404
|
||||
timeout-minutes: 20
|
||||
environment: qa-live-shared
|
||||
steps:
|
||||
- name: Checkout selected ref
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
persist-credentials: false
|
||||
ref: ${{ needs.validate_selected_ref.outputs.selected_revision }}
|
||||
fetch-depth: 1
|
||||
|
||||
- name: Setup Node environment
|
||||
uses: ./.github/actions/setup-node-env
|
||||
with:
|
||||
node-version: ${{ env.NODE_VERSION }}
|
||||
pnpm-version: ${{ env.PNPM_VERSION }}
|
||||
install-bun: "true"
|
||||
|
||||
- name: Build private QA runtime
|
||||
run: pnpm build
|
||||
|
||||
- name: Run Mantis Discord smoke
|
||||
shell: bash
|
||||
env:
|
||||
OPENCLAW_QA_DISCORD_MANTIS_BOT_TOKEN: ${{ secrets.OPENCLAW_QA_DISCORD_MANTIS_BOT_TOKEN }}
|
||||
OPENCLAW_QA_DISCORD_GUILD_ID: ${{ secrets.OPENCLAW_QA_DISCORD_GUILD_ID }}
|
||||
OPENCLAW_QA_DISCORD_CHANNEL_ID: ${{ secrets.OPENCLAW_QA_DISCORD_CHANNEL_ID }}
|
||||
OPENCLAW_QA_REDACT_PUBLIC_METADATA: "1"
|
||||
run: |
|
||||
set -euo pipefail
|
||||
args=()
|
||||
if [[ "${{ inputs.post_message }}" != "true" ]]; then
|
||||
args+=(--skip-post)
|
||||
fi
|
||||
pnpm openclaw qa mantis discord-smoke \
|
||||
--repo-root . \
|
||||
--output-dir .artifacts/qa-e2e/mantis/discord-smoke \
|
||||
"${args[@]}"
|
||||
|
||||
- name: Upload Mantis artifacts
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: mantis-discord-smoke-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: .artifacts/qa-e2e/mantis/
|
||||
retention-days: 14
|
||||
if-no-files-found: warn
|
||||
@@ -11,6 +11,7 @@ Docs: https://docs.openclaw.ai
|
||||
### Changes
|
||||
|
||||
- Gateway/performance: lazy-load early runtime discovery and shutdown-hook helpers, defer maintenance timers until after readiness, and trim duplicate plugin auto-enable work during Gateway startup.
|
||||
- QA/Mantis: add a `pnpm openclaw qa mantis discord-smoke` runner and manual GitHub workflow that verify the Mantis Discord bot can see the configured guild/channel, post a smoke message, add a reaction, and upload artifacts.
|
||||
- Gateway/performance: lazy-load the heavy cron runtime after the rest of Gateway startup, defer restart-sentinel refresh after readiness, and let the Gateway startup benchmark write per-run V8 CPU profiles with `--cpu-prof-dir`.
|
||||
- Gateway/performance: keep raw channel-config schema parsing from discovering bundled plugin runtime metadata, and add `pnpm gateway:watch --benchmark-no-force` for profiling startup without the default port cleanup.
|
||||
- Plugins/onboarding: let Manual setup install optional official plugins, including ClawHub-backed diagnostics with npm fallback, and expose the external Codex plugin as a selectable provider setup choice. Thanks @vincentkoc.
|
||||
|
||||
@@ -15,6 +15,10 @@
|
||||
"source": "OpenAI provider",
|
||||
"target": "OpenAI provider"
|
||||
},
|
||||
{
|
||||
"source": "Mantis",
|
||||
"target": "Mantis"
|
||||
},
|
||||
{
|
||||
"source": "OpenClaw App SDK",
|
||||
"target": "OpenClaw 应用 SDK"
|
||||
|
||||
412
docs/concepts/mantis.md
Normal file
412
docs/concepts/mantis.md
Normal file
@@ -0,0 +1,412 @@
|
||||
---
|
||||
summary: "Mantis is the visual end-to-end verification system for reproducing OpenClaw bugs on live transports, capturing before and after evidence, and attaching artifacts to PRs."
|
||||
title: "Mantis"
|
||||
read_when:
|
||||
- Building or running live visual QA for OpenClaw bugs
|
||||
- Adding before and after verification for a pull request
|
||||
- Adding Discord, Slack, WhatsApp, or other live transport scenarios
|
||||
- Debugging QA runs that need screenshots, browser automation, or VNC access
|
||||
---
|
||||
|
||||
Mantis is the OpenClaw end-to-end verification system for bugs that need a real
|
||||
runtime, a real transport, and visible proof. It runs a scenario against a known
|
||||
bad ref, captures evidence, runs the same scenario against a candidate ref, and
|
||||
publishes the comparison as artifacts that a maintainer can inspect from a PR or
|
||||
from a local command.
|
||||
|
||||
Mantis starts with Discord because Discord gives us a high-value first lane:
|
||||
real bot auth, real guild channels, reactions, threads, native commands, and a
|
||||
browser UI where humans can visually confirm what the transport showed.
|
||||
|
||||
## Goals
|
||||
|
||||
- Reproduce a bug from a GitHub issue or PR with the same transport shape users
|
||||
see.
|
||||
- Capture a **before** artifact on the baseline ref before applying the fix.
|
||||
- Capture an **after** artifact on the candidate ref after applying the fix.
|
||||
- Use a deterministic oracle whenever possible, such as a Discord REST reaction
|
||||
read or channel transcript check.
|
||||
- Capture screenshots when the bug has a visible UI surface.
|
||||
- Run locally from an agent-controlled CLI and remotely from GitHub.
|
||||
- Preserve enough machine state for VNC rescue when login, browser automation, or
|
||||
provider auth gets stuck.
|
||||
- Post concise status to an operator Discord channel when the run is blocked,
|
||||
needs manual VNC help, or finishes.
|
||||
|
||||
## Non Goals
|
||||
|
||||
- Mantis is not a replacement for unit tests. A Mantis run should usually become
|
||||
a smaller regression test after the fix is understood.
|
||||
- Mantis is not the normal fast CI gate. It is slower, uses live credentials, and
|
||||
is reserved for bugs where the live environment matters.
|
||||
- Mantis should not require a human for normal operation. Manual VNC is a rescue
|
||||
path, not the happy path.
|
||||
- Mantis does not store raw secrets in artifacts, logs, screenshots, Markdown
|
||||
reports, or PR comments.
|
||||
|
||||
## Ownership
|
||||
|
||||
Mantis lives in the OpenClaw QA stack.
|
||||
|
||||
- OpenClaw owns the scenario runtime, transport adapters, evidence schema, and
|
||||
local CLI under `pnpm openclaw qa mantis`.
|
||||
- QA Lab owns the live transport harness pieces, browser capture helpers, and
|
||||
artifact writers.
|
||||
- Crabbox owns warmed Linux machines when a remote VM is needed.
|
||||
- GitHub Actions owns the remote workflow entrypoint and artifact retention.
|
||||
- ClawSweeper owns GitHub comment routing: parsing maintainer commands,
|
||||
dispatching the workflow, and posting the final PR comment.
|
||||
- OpenClaw agents drive Mantis through Codex when a scenario needs agentic setup,
|
||||
debugging, or stuck-state reporting.
|
||||
|
||||
This boundary keeps transport knowledge in OpenClaw, machine scheduling in
|
||||
Crabbox, and maintainer workflow glue in ClawSweeper.
|
||||
|
||||
## Command Shape
|
||||
|
||||
The first local command verifies the Discord bot, guild, channel, message send,
|
||||
reaction send, and artifact path:
|
||||
|
||||
```bash
|
||||
pnpm openclaw qa mantis discord-smoke \
|
||||
--output-dir .artifacts/qa-e2e/mantis/discord-smoke
|
||||
```
|
||||
|
||||
The later before and after runner should accept this shape:
|
||||
|
||||
```bash
|
||||
pnpm openclaw qa mantis run \
|
||||
--transport discord \
|
||||
--scenario discord-status-reactions-tool-only \
|
||||
--baseline origin/main \
|
||||
--candidate HEAD \
|
||||
--output-dir .artifacts/qa-e2e/mantis/local-discord-status-reactions
|
||||
```
|
||||
|
||||
The GitHub smoke workflow is `Mantis Discord Smoke`. The before and after GitHub
|
||||
workflow should accept equivalent inputs:
|
||||
|
||||
- `transport`: `discord` for the first version.
|
||||
- `scenario`: one or more scenario ids.
|
||||
- `baseline_ref`: default `origin/main` or the linked issue's reported bad tag.
|
||||
- `candidate_ref`: the PR head SHA.
|
||||
- `machine_provider`: `aws` by default, with later `hetzner` fallback.
|
||||
- `post_to_pr`: whether ClawSweeper should comment with the result.
|
||||
|
||||
ClawSweeper command examples:
|
||||
|
||||
```text
|
||||
@clawsweeper mantis discord discord-status-reactions-tool-only
|
||||
@clawsweeper verify e2e discord
|
||||
```
|
||||
|
||||
The first command is explicit and scenario-focused. The second can later map a PR
|
||||
or issue to recommended Mantis scenarios from labels, changed files, and
|
||||
ClawSweeper review findings.
|
||||
|
||||
## Run Lifecycle
|
||||
|
||||
1. Acquire credentials.
|
||||
2. Allocate or reuse a VM.
|
||||
3. Prepare a clean checkout for the baseline ref.
|
||||
4. Install dependencies and build only what the scenario needs.
|
||||
5. Start a child OpenClaw Gateway with an isolated state directory.
|
||||
6. Configure the live transport, provider, model, and browser profile.
|
||||
7. Run the scenario and capture baseline evidence.
|
||||
8. Stop the gateway and preserve logs.
|
||||
9. Prepare the candidate ref in the same VM.
|
||||
10. Run the same scenario and capture candidate evidence.
|
||||
11. Compare the oracle results and visual evidence.
|
||||
12. Write Markdown, JSON, logs, screenshots, and optional trace artifacts.
|
||||
13. Upload GitHub Actions artifacts.
|
||||
14. Post a concise PR or Discord status message.
|
||||
|
||||
The scenario should be able to fail in two different ways:
|
||||
|
||||
- **Bug reproduced**: baseline failed in the expected way.
|
||||
- **Harness failure**: environment setup, credentials, Discord API, browser, or
|
||||
provider failed before the bug oracle was meaningful.
|
||||
|
||||
The final report must separate these cases so maintainers do not confuse a flaky
|
||||
environment with product behavior.
|
||||
|
||||
## Discord MVP
|
||||
|
||||
The first scenario should target Discord status reactions in guild channels where
|
||||
the source reply delivery mode is `message_tool_only`.
|
||||
|
||||
Why it is a good Mantis seed:
|
||||
|
||||
- It is visible in Discord as reactions on the triggering message.
|
||||
- It has a strong REST oracle through Discord message reaction state.
|
||||
- It exercises a real OpenClaw Gateway, Discord bot auth, message dispatch,
|
||||
source reply delivery mode, status reaction state, and model turn lifecycle.
|
||||
- It is narrow enough to keep the first implementation honest.
|
||||
|
||||
Expected scenario shape:
|
||||
|
||||
```yaml
|
||||
id: discord-status-reactions-tool-only
|
||||
transport: discord
|
||||
baseline:
|
||||
expect:
|
||||
reproduced: true
|
||||
candidate:
|
||||
expect:
|
||||
fixed: true
|
||||
config:
|
||||
messages:
|
||||
ackReaction: "👀"
|
||||
ackReactionScope: "group-mentions"
|
||||
groupChat:
|
||||
visibleReplies: "message_tool"
|
||||
statusReactions:
|
||||
enabled: true
|
||||
timing:
|
||||
debounceMs: 0
|
||||
discord:
|
||||
requireMention: true
|
||||
notifyChannel: operator-notify
|
||||
evidence:
|
||||
rest:
|
||||
messageReactions: true
|
||||
browser:
|
||||
screenshotMessageRow: true
|
||||
```
|
||||
|
||||
Baseline evidence should show the queued acknowledgement reaction but no
|
||||
lifecycle transition in tool-only mode. Candidate evidence should show lifecycle
|
||||
status reactions running when `messages.statusReactions.enabled` is explicitly
|
||||
true.
|
||||
|
||||
## Existing QA Pieces
|
||||
|
||||
Mantis should build on the existing private QA stack instead of starting from
|
||||
zero:
|
||||
|
||||
- `pnpm openclaw qa discord` already runs a live Discord lane with driver and
|
||||
SUT bots.
|
||||
- The live transport runner already writes reports and observed-message
|
||||
artifacts under `.artifacts/qa-e2e/`.
|
||||
- Convex credential leases already provide exclusive access to shared live
|
||||
transport credentials.
|
||||
- The browser control service already supports screenshots, snapshots,
|
||||
headless managed profiles, and remote CDP profiles.
|
||||
- QA Lab already has a debugger UI and bus for transport-shaped testing.
|
||||
|
||||
The first Mantis implementation can be a thin before/after runner over these
|
||||
pieces, plus one visual evidence layer.
|
||||
|
||||
## Evidence Model
|
||||
|
||||
Every run writes a stable artifact directory:
|
||||
|
||||
```text
|
||||
.artifacts/qa-e2e/mantis/<run-id>/
|
||||
mantis-report.md
|
||||
mantis-summary.json
|
||||
baseline/
|
||||
summary.json
|
||||
discord-message.json
|
||||
screenshot-message-row.png
|
||||
gateway-debug/
|
||||
candidate/
|
||||
summary.json
|
||||
discord-message.json
|
||||
screenshot-message-row.png
|
||||
gateway-debug/
|
||||
comparison.json
|
||||
run.log
|
||||
```
|
||||
|
||||
`mantis-summary.json` should be the machine-readable source of truth. The
|
||||
Markdown report is for PR comments and human review.
|
||||
|
||||
The summary must include:
|
||||
|
||||
- refs and SHAs tested
|
||||
- transport and scenario id
|
||||
- machine provider and machine id or lease id
|
||||
- credential source without secret values
|
||||
- baseline result
|
||||
- candidate result
|
||||
- whether the bug reproduced on baseline
|
||||
- whether the candidate fixed it
|
||||
- artifact paths
|
||||
- sanitized setup or cleanup issues
|
||||
|
||||
Screenshots are evidence, not secrets. They still need redaction discipline:
|
||||
private channel names, user names, or message content may appear. For public PRs,
|
||||
prefer GitHub Actions artifact links over inline images until the redaction story
|
||||
is stronger.
|
||||
|
||||
## Browser And VNC
|
||||
|
||||
The browser lane has two modes:
|
||||
|
||||
- **Headless automation**: default for CI. Chrome runs with CDP enabled, and
|
||||
Playwright or OpenClaw browser control captures screenshots.
|
||||
- **VNC rescue**: enabled on the same VM when login, MFA, Discord anti-automation,
|
||||
or visual debugging needs a human.
|
||||
|
||||
The Discord observer browser profile should be persistent enough to avoid
|
||||
logging in for every run, but isolated from personal browser state. A profile
|
||||
belongs to the Mantis machine pool, not to a developer laptop.
|
||||
|
||||
When Mantis gets stuck, it posts a Discord status message with:
|
||||
|
||||
- run id
|
||||
- scenario id
|
||||
- machine provider
|
||||
- artifact directory
|
||||
- VNC or noVNC connection instructions if available
|
||||
- short blocker text
|
||||
|
||||
The first private deployment can post these messages to the existing operator
|
||||
channel and move to a dedicated Mantis channel later.
|
||||
|
||||
## Machines
|
||||
|
||||
Mantis should prefer AWS through Crabbox for the first remote implementation.
|
||||
Crabbox gives us warmed machines, lease tracking, hydration, logs, results, and
|
||||
cleanup. If AWS capacity is too slow or unavailable, add a Hetzner provider
|
||||
behind the same machine interface.
|
||||
|
||||
Minimum VM requirements:
|
||||
|
||||
- Linux with a desktop-capable Chrome or Chromium install
|
||||
- CDP access for browser automation
|
||||
- VNC or noVNC for rescue
|
||||
- Node 22 and pnpm
|
||||
- OpenClaw checkout and dependency cache
|
||||
- Playwright Chromium browser cache when Playwright is used
|
||||
- enough CPU and memory for one OpenClaw Gateway, one browser, and one model run
|
||||
- outbound access to Discord, GitHub, model providers, and the credential broker
|
||||
|
||||
The VM should not keep long-lived raw secrets outside the expected credential or
|
||||
browser profile stores.
|
||||
|
||||
## Secrets
|
||||
|
||||
Secrets live in GitHub organization or repository secrets for remote runs, and in
|
||||
a local operator-controlled secret file for local runs.
|
||||
|
||||
Recommended secret names:
|
||||
|
||||
- `OPENCLAW_QA_DISCORD_MANTIS_BOT_TOKEN`
|
||||
- `OPENCLAW_QA_DISCORD_DRIVER_BOT_TOKEN`
|
||||
- `OPENCLAW_QA_DISCORD_SUT_BOT_TOKEN`
|
||||
- `OPENCLAW_QA_DISCORD_GUILD_ID`
|
||||
- `OPENCLAW_QA_DISCORD_CHANNEL_ID`
|
||||
- `OPENCLAW_QA_DISCORD_NOTIFY_CHANNEL_ID`
|
||||
- `OPENCLAW_QA_REDACT_PUBLIC_METADATA=1` for public GitHub artifact uploads
|
||||
- `OPENCLAW_QA_CONVEX_SITE_URL`
|
||||
- `OPENCLAW_QA_CONVEX_SECRET_CI`
|
||||
|
||||
Long term, the Convex credential pool should remain the normal source for live
|
||||
transport credentials. GitHub secrets bootstrap the broker and fallback lanes.
|
||||
|
||||
The Mantis runner must never print:
|
||||
|
||||
- Discord bot tokens
|
||||
- provider API keys
|
||||
- browser cookies
|
||||
- auth profile contents
|
||||
- VNC passwords
|
||||
- raw credential payloads
|
||||
|
||||
Public artifact uploads should also redact Discord target metadata such as bot,
|
||||
guild, channel, and message ids. The GitHub smoke workflow enables
|
||||
`OPENCLAW_QA_REDACT_PUBLIC_METADATA=1` for this reason.
|
||||
|
||||
If a token is accidentally pasted into an issue, PR, chat, or log, rotate it
|
||||
after the new secret has been stored.
|
||||
|
||||
## GitHub Artifacts And PR Comments
|
||||
|
||||
The first GitHub version should upload screenshots as Actions artifacts and link
|
||||
them from the PR comment. Inline images can come later once redaction, retention,
|
||||
and public/private repo behavior are settled.
|
||||
|
||||
The PR comment should be short:
|
||||
|
||||
```md
|
||||
Mantis Discord verification: pass
|
||||
|
||||
- Scenario: `discord-status-reactions-tool-only`
|
||||
- Baseline: reproduced on `<sha>`
|
||||
- Candidate: fixed on `<sha>`
|
||||
- Evidence: <artifact link>
|
||||
- Screenshots: baseline and candidate message-row captures in the artifact
|
||||
```
|
||||
|
||||
When the run fails because the harness failed, the comment must say that instead
|
||||
of implying the candidate failed.
|
||||
|
||||
## Private Deployment Notes
|
||||
|
||||
A private deployment may already have a Mantis Discord application. Reuse that
|
||||
application instead of creating another app when it has the right bot
|
||||
permissions and can be safely rotated.
|
||||
|
||||
Set the initial operator notification channel through secrets or deployment
|
||||
configuration. It can point at an existing maintainer or operations channel
|
||||
first, then move to a dedicated Mantis channel once one exists.
|
||||
|
||||
Do not put guild ids, channel ids, bot tokens, browser cookies, or VNC passwords
|
||||
in this document. Store them in GitHub secrets, the credential broker, or the
|
||||
operator's local secret store.
|
||||
|
||||
## Adding A Scenario
|
||||
|
||||
A Mantis scenario should declare:
|
||||
|
||||
- id and title
|
||||
- transport
|
||||
- required credentials
|
||||
- baseline ref policy
|
||||
- candidate ref policy
|
||||
- OpenClaw config patch
|
||||
- setup steps
|
||||
- stimulus
|
||||
- expected baseline oracle
|
||||
- expected candidate oracle
|
||||
- visual capture targets
|
||||
- timeout budget
|
||||
- cleanup steps
|
||||
|
||||
Scenarios should prefer small, typed oracles:
|
||||
|
||||
- Discord reaction state for reaction bugs
|
||||
- Discord message references for threading bugs
|
||||
- Slack thread ts and reaction API state for Slack bugs
|
||||
- email message ids and headers for email bugs
|
||||
- browser screenshots when UI is the only reliable observable
|
||||
|
||||
Vision checks should be additive. If a platform API can prove the bug, use the
|
||||
API as the pass/fail oracle and keep screenshots for human confidence.
|
||||
|
||||
## Provider Expansion
|
||||
|
||||
After Discord, the same runner can add:
|
||||
|
||||
- Slack: reactions, threads, app mentions, modals, file uploads.
|
||||
- Email: Gmail auth and message threading using `gog` where connectors are not
|
||||
enough.
|
||||
- WhatsApp: QR login, re-identification, message delivery, media, reactions.
|
||||
- Telegram: group mention gating, commands, reactions where available.
|
||||
- Matrix: encrypted rooms, thread or reply relations, restart resume.
|
||||
|
||||
Each transport should have one cheap smoke scenario and one or more bug-class
|
||||
scenarios. Expensive visual scenarios should stay opt-in.
|
||||
|
||||
## Open Questions
|
||||
|
||||
- Which Discord bot should be the driver, and which should be the SUT, when the
|
||||
existing Mantis bot is reused?
|
||||
- Should the observer browser login use a human Discord account, a test account,
|
||||
or only bot-readable REST evidence for the first phase?
|
||||
- How long should GitHub retain Mantis artifacts for PRs?
|
||||
- When should ClawSweeper automatically recommend Mantis instead of waiting for a
|
||||
maintainer command?
|
||||
- Should screenshots be redacted or cropped before upload for public PRs?
|
||||
@@ -21,6 +21,8 @@ Current pieces:
|
||||
drive a real channel inside a child QA gateway.
|
||||
- `qa/`: repo-backed seed assets for the kickoff task and baseline QA
|
||||
scenarios.
|
||||
- [Mantis](/concepts/mantis): before and after live verification for bugs that
|
||||
need real transports, browser screenshots, VM state, and PR evidence.
|
||||
|
||||
## Command surface
|
||||
|
||||
@@ -45,6 +47,7 @@ script aliases; both forms are supported.
|
||||
| `qa matrix` | Live transport lane against a disposable Tuwunel homeserver. See [Matrix QA](/concepts/qa-matrix). |
|
||||
| `qa telegram` | Live transport lane against a real private Telegram group. |
|
||||
| `qa discord` | Live transport lane against a real private Discord guild channel. |
|
||||
| `qa mantis` | Planned before and after verification runner for live transport bugs. See [Mantis](/concepts/mantis). |
|
||||
|
||||
## Operator flow
|
||||
|
||||
|
||||
@@ -48,6 +48,7 @@ const {
|
||||
runQaProviderServerCommand,
|
||||
runQaSuiteCommand,
|
||||
runQaTelegramCommand,
|
||||
runMantisDiscordSmokeCommand,
|
||||
} = vi.hoisted(() => ({
|
||||
runQaCredentialsAddCommand: vi.fn(),
|
||||
runQaCredentialsListCommand: vi.fn(),
|
||||
@@ -56,6 +57,7 @@ const {
|
||||
runQaProviderServerCommand: vi.fn(),
|
||||
runQaSuiteCommand: vi.fn(),
|
||||
runQaTelegramCommand: vi.fn(),
|
||||
runMantisDiscordSmokeCommand: vi.fn(),
|
||||
}));
|
||||
|
||||
const { listQaRunnerCliContributions } = vi.hoisted(() => ({
|
||||
@@ -72,6 +74,10 @@ vi.mock("./live-transports/telegram/cli.runtime.js", () => ({
|
||||
runQaTelegramCommand,
|
||||
}));
|
||||
|
||||
vi.mock("./mantis/cli.runtime.js", () => ({
|
||||
runMantisDiscordSmokeCommand,
|
||||
}));
|
||||
|
||||
vi.mock("./cli.runtime.js", () => ({
|
||||
runQaCredentialsAddCommand,
|
||||
runQaCredentialsListCommand,
|
||||
@@ -95,6 +101,7 @@ describe("qa cli registration", () => {
|
||||
runQaProviderServerCommand.mockReset();
|
||||
runQaSuiteCommand.mockReset();
|
||||
runQaTelegramCommand.mockReset();
|
||||
runMantisDiscordSmokeCommand.mockReset();
|
||||
listQaRunnerCliContributions
|
||||
.mockReset()
|
||||
.mockReturnValue([createAvailableQaRunnerContribution()]);
|
||||
@@ -109,10 +116,51 @@ describe("qa cli registration", () => {
|
||||
const qa = program.commands.find((command) => command.name() === "qa");
|
||||
expect(qa).toBeDefined();
|
||||
expect(qa?.commands.map((command) => command.name())).toEqual(
|
||||
expect.arrayContaining([TEST_QA_RUNNER.commandName, "telegram", "credentials", "coverage"]),
|
||||
expect.arrayContaining([
|
||||
TEST_QA_RUNNER.commandName,
|
||||
"telegram",
|
||||
"mantis",
|
||||
"credentials",
|
||||
"coverage",
|
||||
]),
|
||||
);
|
||||
});
|
||||
|
||||
it("routes mantis discord-smoke flags into the mantis runtime command", async () => {
|
||||
await program.parseAsync([
|
||||
"node",
|
||||
"openclaw",
|
||||
"qa",
|
||||
"mantis",
|
||||
"discord-smoke",
|
||||
"--repo-root",
|
||||
"/tmp/openclaw-repo",
|
||||
"--output-dir",
|
||||
".artifacts/qa-e2e/mantis/discord-smoke",
|
||||
"--guild-id",
|
||||
"123456789012345678",
|
||||
"--channel-id",
|
||||
"223456789012345678",
|
||||
"--token-file",
|
||||
"/tmp/mantis-token",
|
||||
"--message",
|
||||
"hello from mantis",
|
||||
"--skip-post",
|
||||
]);
|
||||
|
||||
expect(runMantisDiscordSmokeCommand).toHaveBeenCalledWith({
|
||||
repoRoot: "/tmp/openclaw-repo",
|
||||
outputDir: ".artifacts/qa-e2e/mantis/discord-smoke",
|
||||
guildId: "123456789012345678",
|
||||
channelId: "223456789012345678",
|
||||
tokenEnv: undefined,
|
||||
tokenFile: "/tmp/mantis-token",
|
||||
tokenFileEnv: undefined,
|
||||
message: "hello from mantis",
|
||||
skipPost: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("routes coverage report flags into the qa runtime command", async () => {
|
||||
await program.parseAsync([
|
||||
"node",
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import type { Command } from "commander";
|
||||
import { collectString } from "./cli-options.js";
|
||||
import { listLiveTransportQaCliRegistrations } from "./live-transports/cli.js";
|
||||
import { registerMantisCli } from "./mantis/cli.js";
|
||||
import {
|
||||
DEFAULT_QA_LIVE_PROVIDER_MODE,
|
||||
formatQaProviderModeHelp,
|
||||
@@ -225,6 +226,7 @@ export function registerQaLabCli(program: Command) {
|
||||
const qa = program
|
||||
.command("qa")
|
||||
.description("Run private QA automation flows and launch the QA debugger");
|
||||
registerMantisCli(qa);
|
||||
|
||||
qa.command("run")
|
||||
.description("Run the bundled QA self-check and write a Markdown report")
|
||||
|
||||
10
extensions/qa-lab/src/mantis/cli.runtime.ts
Normal file
10
extensions/qa-lab/src/mantis/cli.runtime.ts
Normal file
@@ -0,0 +1,10 @@
|
||||
import { runMantisDiscordSmoke, type MantisDiscordSmokeOptions } from "./discord-smoke.runtime.js";
|
||||
|
||||
export async function runMantisDiscordSmokeCommand(opts: MantisDiscordSmokeOptions) {
|
||||
const result = await runMantisDiscordSmoke(opts);
|
||||
process.stdout.write(`Mantis Discord smoke report: ${result.reportPath}\n`);
|
||||
process.stdout.write(`Mantis Discord smoke summary: ${result.summaryPath}\n`);
|
||||
if (result.status === "fail") {
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
58
extensions/qa-lab/src/mantis/cli.ts
Normal file
58
extensions/qa-lab/src/mantis/cli.ts
Normal file
@@ -0,0 +1,58 @@
|
||||
import type { Command } from "commander";
|
||||
import { createLazyCliRuntimeLoader } from "../live-transports/shared/live-transport-cli.js";
|
||||
import type { MantisDiscordSmokeOptions } from "./discord-smoke.runtime.js";
|
||||
|
||||
type MantisCliRuntime = typeof import("./cli.runtime.js");
|
||||
|
||||
const loadMantisCliRuntime = createLazyCliRuntimeLoader<MantisCliRuntime>(
|
||||
() => import("./cli.runtime.js"),
|
||||
);
|
||||
|
||||
async function runDiscordSmoke(opts: MantisDiscordSmokeOptions) {
|
||||
const runtime = await loadMantisCliRuntime();
|
||||
await runtime.runMantisDiscordSmokeCommand(opts);
|
||||
}
|
||||
|
||||
type MantisDiscordSmokeCommanderOptions = {
|
||||
channelId?: string;
|
||||
guildId?: string;
|
||||
message?: string;
|
||||
outputDir?: string;
|
||||
repoRoot?: string;
|
||||
skipPost?: boolean;
|
||||
tokenFile?: string;
|
||||
tokenFileEnv?: string;
|
||||
tokenEnv?: string;
|
||||
};
|
||||
|
||||
export function registerMantisCli(qa: Command) {
|
||||
const mantis = qa
|
||||
.command("mantis")
|
||||
.description("Run Mantis before/after and live-smoke verification flows");
|
||||
|
||||
mantis
|
||||
.command("discord-smoke")
|
||||
.description("Verify the Mantis Discord bot can see the guild/channel, post, and react")
|
||||
.option("--repo-root <path>", "Repository root to target when running from a neutral cwd")
|
||||
.option("--output-dir <path>", "Mantis Discord smoke artifact directory")
|
||||
.option("--guild-id <id>", "Override OPENCLAW_QA_DISCORD_GUILD_ID")
|
||||
.option("--channel-id <id>", "Override OPENCLAW_QA_DISCORD_CHANNEL_ID")
|
||||
.option("--token-env <name>", "Env var containing the Mantis Discord bot token")
|
||||
.option("--token-file <path>", "File containing the Mantis Discord bot token")
|
||||
.option("--token-file-env <name>", "Env var containing the Mantis Discord bot token file path")
|
||||
.option("--message <text>", "Smoke message to post")
|
||||
.option("--skip-post", "Only check Discord API visibility; do not post or react", false)
|
||||
.action(async (opts: MantisDiscordSmokeCommanderOptions) => {
|
||||
await runDiscordSmoke({
|
||||
channelId: opts.channelId,
|
||||
guildId: opts.guildId,
|
||||
message: opts.message,
|
||||
outputDir: opts.outputDir,
|
||||
repoRoot: opts.repoRoot,
|
||||
skipPost: opts.skipPost,
|
||||
tokenFile: opts.tokenFile,
|
||||
tokenFileEnv: opts.tokenFileEnv,
|
||||
tokenEnv: opts.tokenEnv,
|
||||
});
|
||||
});
|
||||
}
|
||||
310
extensions/qa-lab/src/mantis/discord-smoke.runtime.test.ts
Normal file
310
extensions/qa-lab/src/mantis/discord-smoke.runtime.test.ts
Normal file
@@ -0,0 +1,310 @@
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const { fetchWithSsrFGuard } = vi.hoisted(() => ({
|
||||
fetchWithSsrFGuard: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("openclaw/plugin-sdk/ssrf-runtime", () => ({
|
||||
fetchWithSsrFGuard,
|
||||
}));
|
||||
|
||||
import { runMantisDiscordSmoke } from "./discord-smoke.runtime.js";
|
||||
|
||||
function jsonResponse(payload: unknown, status = 200) {
|
||||
return new Response(JSON.stringify(payload), {
|
||||
status,
|
||||
headers: { "content-type": "application/json" },
|
||||
});
|
||||
}
|
||||
|
||||
function emptyResponse(status = 204) {
|
||||
return new Response(null, { status });
|
||||
}
|
||||
|
||||
describe("mantis discord smoke runtime", () => {
|
||||
let repoRoot: string;
|
||||
let tokenFile: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
repoRoot = await fs.mkdtemp(path.join(os.tmpdir(), "mantis-discord-smoke-"));
|
||||
tokenFile = path.join(repoRoot, "mantis-token");
|
||||
await fs.writeFile(tokenFile, "test-token", "utf8");
|
||||
fetchWithSsrFGuard.mockReset();
|
||||
const reactionPaths = new Set([
|
||||
"/api/v10/channels/1456744319972282449/messages/1500000000000000001/reactions/%F0%9F%91%80/@me",
|
||||
"/api/v10/channels/1456744319972282449/messages/1500000000000000001/reactions/👀/@me",
|
||||
]);
|
||||
fetchWithSsrFGuard.mockImplementation(
|
||||
async ({ url, init }: { url: string; init?: RequestInit }) => {
|
||||
const pathname = new URL(url).pathname;
|
||||
const method = init?.method ?? "GET";
|
||||
if (pathname === "/api/v10/users/@me") {
|
||||
return {
|
||||
response: jsonResponse({ id: "1489650053747314748", username: "Mantis" }),
|
||||
release: vi.fn(),
|
||||
};
|
||||
}
|
||||
if (pathname === "/api/v10/guilds/1456350064065904867") {
|
||||
return {
|
||||
response: jsonResponse({ id: "1456350064065904867", name: "Friends" }),
|
||||
release: vi.fn(),
|
||||
};
|
||||
}
|
||||
if (pathname === "/api/v10/guilds/1456350064065904867/channels") {
|
||||
return { response: jsonResponse([{ id: "1456744319972282449" }]), release: vi.fn() };
|
||||
}
|
||||
if (pathname === "/api/v10/channels/1456744319972282449" && method === "GET") {
|
||||
return {
|
||||
response: jsonResponse({
|
||||
guild_id: "1456350064065904867",
|
||||
id: "1456744319972282449",
|
||||
name: "maintainers",
|
||||
type: 0,
|
||||
}),
|
||||
release: vi.fn(),
|
||||
};
|
||||
}
|
||||
if (pathname === "/api/v10/channels/1456744319972282449/messages" && method === "POST") {
|
||||
return {
|
||||
response: jsonResponse({
|
||||
id: "1500000000000000001",
|
||||
channel_id: "1456744319972282449",
|
||||
}),
|
||||
release: vi.fn(),
|
||||
};
|
||||
}
|
||||
if (reactionPaths.has(pathname) && method === "PUT") {
|
||||
return { response: emptyResponse(), release: vi.fn() };
|
||||
}
|
||||
return {
|
||||
response: jsonResponse({ message: `unexpected ${method} ${pathname}` }, 404),
|
||||
release: vi.fn(),
|
||||
};
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await fs.rm(repoRoot, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("writes pass artifacts without leaking the bot token", async () => {
|
||||
const result = await runMantisDiscordSmoke({
|
||||
repoRoot,
|
||||
outputDir: ".artifacts/qa-e2e/mantis/test",
|
||||
tokenFile,
|
||||
env: {
|
||||
OPENCLAW_QA_DISCORD_GUILD_ID: "1456350064065904867",
|
||||
OPENCLAW_QA_DISCORD_CHANNEL_ID: "1456744319972282449",
|
||||
},
|
||||
now: () => new Date("2026-05-03T12:00:00.000Z"),
|
||||
});
|
||||
|
||||
expect(result.status).toBe("pass");
|
||||
const summary = JSON.parse(await fs.readFile(result.summaryPath, "utf8")) as {
|
||||
status: string;
|
||||
tokenSource: string;
|
||||
message: { id: string; posted: boolean; reactionAdded: boolean };
|
||||
};
|
||||
expect(summary).toMatchObject({
|
||||
status: "pass",
|
||||
tokenSource: "file",
|
||||
message: {
|
||||
id: "1500000000000000001",
|
||||
posted: true,
|
||||
reactionAdded: true,
|
||||
},
|
||||
});
|
||||
expect(await fs.readFile(result.summaryPath, "utf8")).not.toContain("test-token");
|
||||
expect(await fs.readFile(result.reportPath, "utf8")).not.toContain("test-token");
|
||||
});
|
||||
|
||||
it("supports visibility-only smoke runs", async () => {
|
||||
const result = await runMantisDiscordSmoke({
|
||||
repoRoot,
|
||||
outputDir: ".artifacts/qa-e2e/mantis/visibility",
|
||||
tokenFile,
|
||||
skipPost: true,
|
||||
env: {
|
||||
OPENCLAW_QA_DISCORD_GUILD_ID: "1456350064065904867",
|
||||
OPENCLAW_QA_DISCORD_CHANNEL_ID: "1456744319972282449",
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.status).toBe("pass");
|
||||
expect(fetchWithSsrFGuard).not.toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
init: expect.objectContaining({ method: "POST" }),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("redacts Discord target metadata in public artifacts", async () => {
|
||||
const result = await runMantisDiscordSmoke({
|
||||
repoRoot,
|
||||
outputDir: ".artifacts/qa-e2e/mantis/redacted",
|
||||
tokenFile,
|
||||
redactPublicMetadata: true,
|
||||
env: {
|
||||
OPENCLAW_QA_DISCORD_GUILD_ID: "1456350064065904867",
|
||||
OPENCLAW_QA_DISCORD_CHANNEL_ID: "1456744319972282449",
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.status).toBe("pass");
|
||||
const summaryText = await fs.readFile(result.summaryPath, "utf8");
|
||||
const reportText = await fs.readFile(result.reportPath, "utf8");
|
||||
expect(reportText).toContain("# Mantis Discord Smoke");
|
||||
expect(reportText).toContain("- Bot: <redacted> (<redacted>)");
|
||||
expect(reportText).toContain("- Guild: <redacted> (<redacted>)");
|
||||
expect(reportText).toContain("- Channel: #<redacted> (<redacted>)");
|
||||
for (const text of [summaryText, reportText]) {
|
||||
expect(text).toContain("<redacted>");
|
||||
expect(text).not.toContain("1489650053747314748");
|
||||
expect(text).not.toContain("1456350064065904867");
|
||||
expect(text).not.toContain("Friends");
|
||||
expect(text).not.toContain("1456744319972282449");
|
||||
expect(text).not.toContain("maintainers");
|
||||
expect(text).not.toContain("1500000000000000001");
|
||||
}
|
||||
expect(summaryText).not.toContain("Mantis");
|
||||
expect(JSON.parse(summaryText)).toMatchObject({
|
||||
metadataRedaction: true,
|
||||
bot: { id: "<redacted>", username: "<redacted>" },
|
||||
guild: { id: "<redacted>", name: "<redacted>" },
|
||||
channel: { id: "<redacted>", name: "<redacted>" },
|
||||
message: { id: "<redacted>" },
|
||||
});
|
||||
});
|
||||
|
||||
it("fails before calling Discord when required ids are missing", async () => {
|
||||
const result = await runMantisDiscordSmoke({
|
||||
repoRoot,
|
||||
outputDir: ".artifacts/qa-e2e/mantis/missing",
|
||||
tokenFile,
|
||||
env: {},
|
||||
});
|
||||
|
||||
expect(result.status).toBe("fail");
|
||||
const errorText = await fs.readFile(path.join(result.outputDir, "error.txt"), "utf8");
|
||||
expect(errorText).toContain("Missing OPENCLAW_QA_DISCORD_GUILD_ID");
|
||||
});
|
||||
|
||||
it("fails when the channel is not in the configured guild", async () => {
|
||||
fetchWithSsrFGuard.mockImplementation(
|
||||
async ({ url, init }: { url: string; init?: RequestInit }) => {
|
||||
const pathname = new URL(url).pathname;
|
||||
const method = init?.method ?? "GET";
|
||||
if (pathname === "/api/v10/users/@me") {
|
||||
return {
|
||||
response: jsonResponse({ id: "1489650053747314748", username: "Mantis" }),
|
||||
release: vi.fn(),
|
||||
};
|
||||
}
|
||||
if (pathname === "/api/v10/guilds/1456350064065904867") {
|
||||
return {
|
||||
response: jsonResponse({ id: "1456350064065904867", name: "Friends" }),
|
||||
release: vi.fn(),
|
||||
};
|
||||
}
|
||||
if (pathname === "/api/v10/guilds/1456350064065904867/channels") {
|
||||
return { response: jsonResponse([{ id: "1999999999999999999" }]), release: vi.fn() };
|
||||
}
|
||||
if (pathname === "/api/v10/channels/1456744319972282449" && method === "GET") {
|
||||
return {
|
||||
response: jsonResponse({
|
||||
guild_id: "1999999999999999999",
|
||||
id: "1456744319972282449",
|
||||
name: "wrong-guild-channel",
|
||||
type: 0,
|
||||
}),
|
||||
release: vi.fn(),
|
||||
};
|
||||
}
|
||||
return {
|
||||
response: jsonResponse({ message: `unexpected ${method} ${pathname}` }, 404),
|
||||
release: vi.fn(),
|
||||
};
|
||||
},
|
||||
);
|
||||
|
||||
const result = await runMantisDiscordSmoke({
|
||||
repoRoot,
|
||||
outputDir: ".artifacts/qa-e2e/mantis/wrong-guild",
|
||||
tokenFile,
|
||||
env: {
|
||||
OPENCLAW_QA_DISCORD_GUILD_ID: "1456350064065904867",
|
||||
OPENCLAW_QA_DISCORD_CHANNEL_ID: "1456744319972282449",
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.status).toBe("fail");
|
||||
const errorText = await fs.readFile(path.join(result.outputDir, "error.txt"), "utf8");
|
||||
expect(errorText).toContain("is not in guild");
|
||||
expect(fetchWithSsrFGuard).not.toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
init: expect.objectContaining({ method: "POST" }),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("redacts response guild ids in mismatch failure artifacts", async () => {
|
||||
fetchWithSsrFGuard.mockImplementation(
|
||||
async ({ url, init }: { url: string; init?: RequestInit }) => {
|
||||
const pathname = new URL(url).pathname;
|
||||
const method = init?.method ?? "GET";
|
||||
if (pathname === "/api/v10/users/@me") {
|
||||
return {
|
||||
response: jsonResponse({ id: "1489650053747314748", username: "Mantis" }),
|
||||
release: vi.fn(),
|
||||
};
|
||||
}
|
||||
if (pathname === "/api/v10/guilds/1456350064065904867") {
|
||||
return {
|
||||
response: jsonResponse({ id: "1456350064065904867", name: "Friends" }),
|
||||
release: vi.fn(),
|
||||
};
|
||||
}
|
||||
if (pathname === "/api/v10/guilds/1456350064065904867/channels") {
|
||||
return { response: jsonResponse([{ id: "1456744319972282449" }]), release: vi.fn() };
|
||||
}
|
||||
if (pathname === "/api/v10/channels/1456744319972282449" && method === "GET") {
|
||||
return {
|
||||
response: jsonResponse({
|
||||
guild_id: "1999999999999999999",
|
||||
id: "1456744319972282449",
|
||||
name: "wrong-guild-channel",
|
||||
type: 0,
|
||||
}),
|
||||
release: vi.fn(),
|
||||
};
|
||||
}
|
||||
return {
|
||||
response: jsonResponse({ message: `unexpected ${method} ${pathname}` }, 404),
|
||||
release: vi.fn(),
|
||||
};
|
||||
},
|
||||
);
|
||||
|
||||
const result = await runMantisDiscordSmoke({
|
||||
repoRoot,
|
||||
outputDir: ".artifacts/qa-e2e/mantis/wrong-guild-redacted",
|
||||
tokenFile,
|
||||
redactPublicMetadata: true,
|
||||
env: {
|
||||
OPENCLAW_QA_DISCORD_GUILD_ID: "1456350064065904867",
|
||||
OPENCLAW_QA_DISCORD_CHANNEL_ID: "1456744319972282449",
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.status).toBe("fail");
|
||||
const errorText = await fs.readFile(path.join(result.outputDir, "error.txt"), "utf8");
|
||||
expect(errorText).toContain("<redacted>");
|
||||
expect(errorText).not.toContain("1999999999999999999");
|
||||
expect(errorText).not.toContain("1456350064065904867");
|
||||
expect(errorText).not.toContain("1456744319972282449");
|
||||
});
|
||||
});
|
||||
491
extensions/qa-lab/src/mantis/discord-smoke.runtime.ts
Normal file
491
extensions/qa-lab/src/mantis/discord-smoke.runtime.ts
Normal file
@@ -0,0 +1,491 @@
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
|
||||
import { fetchWithSsrFGuard } from "openclaw/plugin-sdk/ssrf-runtime";
|
||||
import { ensureRepoBoundDirectory, resolveRepoRelativeOutputDir } from "../cli-paths.js";
|
||||
|
||||
export type MantisDiscordSmokeOptions = {
|
||||
channelId?: string;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
guildId?: string;
|
||||
message?: string;
|
||||
now?: () => Date;
|
||||
outputDir?: string;
|
||||
redactPublicMetadata?: boolean;
|
||||
repoRoot?: string;
|
||||
skipPost?: boolean;
|
||||
token?: string;
|
||||
tokenEnv?: string;
|
||||
tokenFile?: string;
|
||||
tokenFileEnv?: string;
|
||||
};
|
||||
|
||||
export type MantisDiscordSmokeResult = {
|
||||
outputDir: string;
|
||||
reportPath: string;
|
||||
summaryPath: string;
|
||||
status: "pass" | "fail";
|
||||
};
|
||||
|
||||
type DiscordUser = {
|
||||
id: string;
|
||||
username?: string;
|
||||
};
|
||||
|
||||
type DiscordGuild = {
|
||||
id: string;
|
||||
name?: string;
|
||||
};
|
||||
|
||||
type DiscordChannel = {
|
||||
guild_id?: string;
|
||||
id: string;
|
||||
name?: string;
|
||||
type?: number;
|
||||
};
|
||||
|
||||
type DiscordMessage = {
|
||||
id: string;
|
||||
channel_id: string;
|
||||
};
|
||||
|
||||
type DiscordApiCall = {
|
||||
label: string;
|
||||
method: string;
|
||||
ok: boolean;
|
||||
path: string;
|
||||
status: number;
|
||||
};
|
||||
|
||||
type MantisDiscordSmokeSummary = {
|
||||
apiCalls: DiscordApiCall[];
|
||||
artifacts: {
|
||||
reportPath: string;
|
||||
summaryPath: string;
|
||||
};
|
||||
bot?: {
|
||||
id: string;
|
||||
username?: string;
|
||||
};
|
||||
channel?: {
|
||||
id: string;
|
||||
name?: string;
|
||||
type?: number;
|
||||
};
|
||||
finishedAt: string;
|
||||
guild?: {
|
||||
id: string;
|
||||
name?: string;
|
||||
};
|
||||
message?: {
|
||||
id: string;
|
||||
posted: boolean;
|
||||
reactionAdded: boolean;
|
||||
};
|
||||
metadataRedaction: boolean;
|
||||
outputDir: string;
|
||||
reportPath: string;
|
||||
startedAt: string;
|
||||
status: "pass" | "fail";
|
||||
summaryPath: string;
|
||||
tokenSource: "env" | "file" | "option";
|
||||
};
|
||||
|
||||
const DISCORD_API_BASE_URL = "https://discord.com/api/v10";
|
||||
const DEFAULT_MANTIS_TOKEN_ENV = "OPENCLAW_QA_DISCORD_MANTIS_BOT_TOKEN";
|
||||
const DEFAULT_MANTIS_TOKEN_FILE_ENV = "OPENCLAW_QA_DISCORD_MANTIS_BOT_TOKEN_FILE";
|
||||
const DEFAULT_GUILD_ID_ENV = "OPENCLAW_QA_DISCORD_GUILD_ID";
|
||||
const DEFAULT_CHANNEL_ID_ENV = "OPENCLAW_QA_DISCORD_CHANNEL_ID";
|
||||
const QA_REDACT_PUBLIC_METADATA_ENV = "OPENCLAW_QA_REDACT_PUBLIC_METADATA";
|
||||
|
||||
function trimToValue(value: string | undefined) {
|
||||
const trimmed = value?.trim();
|
||||
return trimmed && trimmed.length > 0 ? trimmed : undefined;
|
||||
}
|
||||
|
||||
function isTruthyOptIn(value: string | undefined) {
|
||||
const normalized = value?.trim().toLowerCase();
|
||||
return normalized === "1" || normalized === "true" || normalized === "yes";
|
||||
}
|
||||
|
||||
function assertDiscordSnowflake(value: string, label: string) {
|
||||
if (!/^\d{17,20}$/u.test(value)) {
|
||||
throw new Error(`${label} must be a Discord snowflake.`);
|
||||
}
|
||||
}
|
||||
|
||||
async function readTokenFile(filePath: string) {
|
||||
const token = trimToValue(await fs.readFile(filePath, "utf8"));
|
||||
if (!token) {
|
||||
throw new Error(`Mantis Discord token file is empty: ${filePath}`);
|
||||
}
|
||||
return token;
|
||||
}
|
||||
|
||||
async function resolveMantisDiscordToken(opts: MantisDiscordSmokeOptions) {
|
||||
const env = opts.env ?? process.env;
|
||||
const tokenEnv = trimToValue(opts.tokenEnv) ?? DEFAULT_MANTIS_TOKEN_ENV;
|
||||
const tokenFileEnv = trimToValue(opts.tokenFileEnv) ?? DEFAULT_MANTIS_TOKEN_FILE_ENV;
|
||||
const optionToken = trimToValue(opts.token);
|
||||
if (optionToken) {
|
||||
return { source: "option" as const, token: optionToken };
|
||||
}
|
||||
const envToken = trimToValue(env[tokenEnv]);
|
||||
if (envToken) {
|
||||
return { source: "env" as const, token: envToken };
|
||||
}
|
||||
const tokenFile = trimToValue(opts.tokenFile) ?? trimToValue(env[tokenFileEnv]);
|
||||
if (tokenFile) {
|
||||
return { source: "file" as const, token: await readTokenFile(tokenFile) };
|
||||
}
|
||||
throw new Error(
|
||||
`Missing Mantis Discord bot token. Set ${tokenEnv}, ${tokenFileEnv}, or pass --token-file.`,
|
||||
);
|
||||
}
|
||||
|
||||
function resolveRequiredSnowflake(params: {
|
||||
env: NodeJS.ProcessEnv;
|
||||
envKey: string;
|
||||
label: string;
|
||||
value?: string;
|
||||
}) {
|
||||
const resolved = trimToValue(params.value) ?? trimToValue(params.env[params.envKey]);
|
||||
if (!resolved) {
|
||||
throw new Error(`Missing ${params.envKey}.`);
|
||||
}
|
||||
assertDiscordSnowflake(resolved, params.label);
|
||||
return resolved;
|
||||
}
|
||||
|
||||
function assertMantisDiscordChannelInGuild(params: {
|
||||
channel: DiscordChannel;
|
||||
guildChannels: readonly DiscordChannel[];
|
||||
guildId: string;
|
||||
channelId: string;
|
||||
}) {
|
||||
if (!params.guildChannels.some((channel) => channel.id === params.channelId)) {
|
||||
throw new Error(
|
||||
`OPENCLAW_QA_DISCORD_CHANNEL_ID ${params.channelId} is not in guild ${params.guildId}.`,
|
||||
);
|
||||
}
|
||||
if (params.channel.guild_id && params.channel.guild_id !== params.guildId) {
|
||||
throw new Error(
|
||||
`OPENCLAW_QA_DISCORD_CHANNEL_ID ${params.channelId} belongs to guild ${params.channel.guild_id}, not ${params.guildId}.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function defaultMantisDiscordSmokeOutputDir(repoRoot: string, startedAt: Date) {
|
||||
const stamp = startedAt.toISOString().replace(/[:.]/gu, "-");
|
||||
return path.join(repoRoot, ".artifacts", "qa-e2e", "mantis", `discord-smoke-${stamp}`);
|
||||
}
|
||||
|
||||
async function callDiscordApi<T>(params: {
|
||||
apiCalls: DiscordApiCall[];
|
||||
body?: unknown;
|
||||
label: string;
|
||||
method?: string;
|
||||
path: string;
|
||||
token: string;
|
||||
}) {
|
||||
const method = params.method ?? "GET";
|
||||
const headers = new Headers();
|
||||
headers.set("authorization", `Bot ${params.token}`);
|
||||
let body: string | undefined;
|
||||
if (params.body !== undefined) {
|
||||
headers.set("content-type", "application/json");
|
||||
body = JSON.stringify(params.body);
|
||||
}
|
||||
const { response, release } = await fetchWithSsrFGuard({
|
||||
url: `${DISCORD_API_BASE_URL}${params.path}`,
|
||||
init: {
|
||||
method,
|
||||
headers,
|
||||
body,
|
||||
},
|
||||
signal: AbortSignal.timeout(15_000),
|
||||
policy: { hostnameAllowlist: ["discord.com"] },
|
||||
auditContext: "qa-lab-mantis-discord-smoke",
|
||||
});
|
||||
try {
|
||||
const text = await response.text();
|
||||
const payload = text.trim() ? (JSON.parse(text) as unknown) : undefined;
|
||||
params.apiCalls.push({
|
||||
label: params.label,
|
||||
method,
|
||||
ok: response.ok,
|
||||
path: params.path,
|
||||
status: response.status,
|
||||
});
|
||||
if (!response.ok) {
|
||||
const message =
|
||||
payload &&
|
||||
typeof payload === "object" &&
|
||||
typeof (payload as { message?: unknown }).message === "string"
|
||||
? (payload as { message: string }).message
|
||||
: text.trim();
|
||||
throw new Error(
|
||||
message || `Discord API ${params.path} failed with status ${response.status}`,
|
||||
);
|
||||
}
|
||||
return payload as T;
|
||||
} finally {
|
||||
await release();
|
||||
}
|
||||
}
|
||||
|
||||
function renderMantisDiscordSmokeReport(summary: MantisDiscordSmokeSummary) {
|
||||
const lines = [
|
||||
"# Mantis Discord Smoke",
|
||||
"",
|
||||
`Status: ${summary.status}`,
|
||||
`Metadata redaction: ${summary.metadataRedaction ? "enabled" : "disabled"}`,
|
||||
`Started: ${summary.startedAt}`,
|
||||
`Finished: ${summary.finishedAt}`,
|
||||
`Output: ${summary.outputDir}`,
|
||||
"",
|
||||
"## Target",
|
||||
"",
|
||||
`- Bot: ${summary.bot?.username ?? "unknown"} (${summary.bot?.id ?? "unknown"})`,
|
||||
`- Guild: ${summary.guild?.name ?? "unknown"} (${summary.guild?.id ?? "unknown"})`,
|
||||
`- Channel: #${summary.channel?.name ?? "unknown"} (${summary.channel?.id ?? "unknown"})`,
|
||||
"",
|
||||
"## Message",
|
||||
"",
|
||||
summary.message?.posted
|
||||
? `- Posted message: ${summary.message.id}`
|
||||
: "- Posted message: skipped",
|
||||
summary.message?.reactionAdded ? "- Added reaction: yes" : "- Added reaction: no",
|
||||
"",
|
||||
"## Discord API Calls",
|
||||
"",
|
||||
"| Label | Method | Status |",
|
||||
"| --- | --- | --- |",
|
||||
...summary.apiCalls.map((call) => `| ${call.label} | ${call.method} | ${call.status} |`),
|
||||
"",
|
||||
];
|
||||
return `${lines.join("\n")}\n`;
|
||||
}
|
||||
|
||||
function addSensitiveValue(values: Set<string>, value: string | undefined) {
|
||||
const resolved = trimToValue(value);
|
||||
if (resolved && resolved !== "<redacted>") {
|
||||
values.add(resolved);
|
||||
}
|
||||
}
|
||||
|
||||
function redactMantisDiscordMetadata(text: string, sensitiveValues: ReadonlySet<string>) {
|
||||
let redacted = text;
|
||||
const sortedValues = [...sensitiveValues].toSorted((a, b) => b.length - a.length);
|
||||
for (const value of sortedValues) {
|
||||
redacted = redacted.replaceAll(value, "<redacted>");
|
||||
}
|
||||
return redacted;
|
||||
}
|
||||
|
||||
function buildPublishedMantisDiscordSmokeSummary(
|
||||
summary: MantisDiscordSmokeSummary,
|
||||
sensitiveValues: ReadonlySet<string>,
|
||||
): MantisDiscordSmokeSummary {
|
||||
if (!summary.metadataRedaction) {
|
||||
return summary;
|
||||
}
|
||||
return {
|
||||
...summary,
|
||||
apiCalls: summary.apiCalls.map((call) => ({
|
||||
...call,
|
||||
path: redactMantisDiscordMetadata(call.path, sensitiveValues),
|
||||
})),
|
||||
bot: summary.bot
|
||||
? {
|
||||
id: "<redacted>",
|
||||
username: summary.bot.username ? "<redacted>" : undefined,
|
||||
}
|
||||
: undefined,
|
||||
channel: summary.channel
|
||||
? {
|
||||
id: "<redacted>",
|
||||
name: summary.channel.name ? "<redacted>" : undefined,
|
||||
type: summary.channel.type,
|
||||
}
|
||||
: undefined,
|
||||
guild: summary.guild
|
||||
? {
|
||||
id: "<redacted>",
|
||||
name: summary.guild.name ? "<redacted>" : undefined,
|
||||
}
|
||||
: undefined,
|
||||
message: summary.message
|
||||
? {
|
||||
...summary.message,
|
||||
id: summary.message.id ? "<redacted>" : "",
|
||||
}
|
||||
: undefined,
|
||||
};
|
||||
}
|
||||
|
||||
async function writeMantisDiscordSmokeArtifacts(
|
||||
summary: MantisDiscordSmokeSummary,
|
||||
sensitiveValues: ReadonlySet<string>,
|
||||
) {
|
||||
await fs.mkdir(summary.outputDir, { recursive: true });
|
||||
const publishedSummary = buildPublishedMantisDiscordSmokeSummary(summary, sensitiveValues);
|
||||
const report = renderMantisDiscordSmokeReport(publishedSummary);
|
||||
const summaryJson = `${JSON.stringify(publishedSummary, null, 2)}\n`;
|
||||
await fs.writeFile(summary.reportPath, report, "utf8");
|
||||
await fs.writeFile(summary.summaryPath, summaryJson, "utf8");
|
||||
}
|
||||
|
||||
export async function runMantisDiscordSmoke(
|
||||
opts: MantisDiscordSmokeOptions = {},
|
||||
): Promise<MantisDiscordSmokeResult> {
|
||||
const env = opts.env ?? process.env;
|
||||
const startedAt = (opts.now ?? (() => new Date()))();
|
||||
const redactPublicMetadata =
|
||||
opts.redactPublicMetadata ?? isTruthyOptIn(env[QA_REDACT_PUBLIC_METADATA_ENV]);
|
||||
const repoRoot = path.resolve(opts.repoRoot ?? process.cwd());
|
||||
const outputDir = await ensureRepoBoundDirectory(
|
||||
repoRoot,
|
||||
resolveRepoRelativeOutputDir(repoRoot, opts.outputDir) ??
|
||||
defaultMantisDiscordSmokeOutputDir(repoRoot, startedAt),
|
||||
"Mantis Discord smoke output directory",
|
||||
{ mode: 0o755 },
|
||||
);
|
||||
const summaryPath = path.join(outputDir, "mantis-discord-smoke-summary.json");
|
||||
const reportPath = path.join(outputDir, "mantis-discord-smoke-report.md");
|
||||
const apiCalls: DiscordApiCall[] = [];
|
||||
const sensitiveValues = new Set<string>();
|
||||
const summary: MantisDiscordSmokeSummary = {
|
||||
apiCalls,
|
||||
artifacts: {
|
||||
reportPath,
|
||||
summaryPath,
|
||||
},
|
||||
finishedAt: startedAt.toISOString(),
|
||||
metadataRedaction: redactPublicMetadata,
|
||||
outputDir,
|
||||
reportPath,
|
||||
startedAt: startedAt.toISOString(),
|
||||
status: "fail",
|
||||
summaryPath,
|
||||
tokenSource: "env",
|
||||
};
|
||||
|
||||
try {
|
||||
const { source, token } = await resolveMantisDiscordToken(opts);
|
||||
summary.tokenSource = source;
|
||||
const guildId = resolveRequiredSnowflake({
|
||||
env,
|
||||
envKey: DEFAULT_GUILD_ID_ENV,
|
||||
label: DEFAULT_GUILD_ID_ENV,
|
||||
value: opts.guildId,
|
||||
});
|
||||
const channelId = resolveRequiredSnowflake({
|
||||
env,
|
||||
envKey: DEFAULT_CHANNEL_ID_ENV,
|
||||
label: DEFAULT_CHANNEL_ID_ENV,
|
||||
value: opts.channelId,
|
||||
});
|
||||
addSensitiveValue(sensitiveValues, guildId);
|
||||
addSensitiveValue(sensitiveValues, channelId);
|
||||
const bot = await callDiscordApi<DiscordUser>({
|
||||
apiCalls,
|
||||
label: "current-user",
|
||||
path: "/users/@me",
|
||||
token,
|
||||
});
|
||||
addSensitiveValue(sensitiveValues, bot.id);
|
||||
addSensitiveValue(sensitiveValues, bot.username);
|
||||
const guild = await callDiscordApi<DiscordGuild>({
|
||||
apiCalls,
|
||||
label: "guild",
|
||||
path: `/guilds/${guildId}`,
|
||||
token,
|
||||
});
|
||||
addSensitiveValue(sensitiveValues, guild.id);
|
||||
addSensitiveValue(sensitiveValues, guild.name);
|
||||
const guildChannels = await callDiscordApi<DiscordChannel[]>({
|
||||
apiCalls,
|
||||
label: "guild-channels",
|
||||
path: `/guilds/${guildId}/channels`,
|
||||
token,
|
||||
});
|
||||
for (const guildChannel of guildChannels) {
|
||||
addSensitiveValue(sensitiveValues, guildChannel.id);
|
||||
addSensitiveValue(sensitiveValues, guildChannel.guild_id);
|
||||
addSensitiveValue(sensitiveValues, guildChannel.name);
|
||||
}
|
||||
const channel = await callDiscordApi<DiscordChannel>({
|
||||
apiCalls,
|
||||
label: "channel",
|
||||
path: `/channels/${channelId}`,
|
||||
token,
|
||||
});
|
||||
addSensitiveValue(sensitiveValues, channel.id);
|
||||
addSensitiveValue(sensitiveValues, channel.guild_id);
|
||||
addSensitiveValue(sensitiveValues, channel.name);
|
||||
assertMantisDiscordChannelInGuild({
|
||||
channel,
|
||||
guildChannels,
|
||||
guildId,
|
||||
channelId,
|
||||
});
|
||||
summary.bot = { id: bot.id, username: bot.username };
|
||||
summary.guild = { id: guild.id, name: guild.name };
|
||||
summary.channel = { id: channel.id, name: channel.name, type: channel.type };
|
||||
|
||||
if (opts.skipPost) {
|
||||
summary.message = { id: "", posted: false, reactionAdded: false };
|
||||
} else {
|
||||
const message = await callDiscordApi<DiscordMessage>({
|
||||
apiCalls,
|
||||
body: {
|
||||
content:
|
||||
trimToValue(opts.message) ?? `Mantis Discord smoke: OK (${startedAt.toISOString()})`,
|
||||
},
|
||||
label: "post-message",
|
||||
method: "POST",
|
||||
path: `/channels/${channelId}/messages`,
|
||||
token,
|
||||
});
|
||||
addSensitiveValue(sensitiveValues, message.id);
|
||||
await callDiscordApi<void>({
|
||||
apiCalls,
|
||||
label: "add-reaction",
|
||||
method: "PUT",
|
||||
path: `/channels/${channelId}/messages/${message.id}/reactions/%F0%9F%91%80/@me`,
|
||||
token,
|
||||
});
|
||||
summary.message = { id: message.id, posted: true, reactionAdded: true };
|
||||
}
|
||||
|
||||
summary.status = "pass";
|
||||
} catch (error) {
|
||||
summary.status = "fail";
|
||||
summary.message = summary.message ?? {
|
||||
id: "",
|
||||
posted: false,
|
||||
reactionAdded: false,
|
||||
};
|
||||
await fs.writeFile(
|
||||
path.join(outputDir, "error.txt"),
|
||||
`${
|
||||
redactPublicMetadata
|
||||
? redactMantisDiscordMetadata(formatErrorMessage(error), sensitiveValues)
|
||||
: formatErrorMessage(error)
|
||||
}${os.EOL}`,
|
||||
"utf8",
|
||||
);
|
||||
} finally {
|
||||
summary.finishedAt = new Date().toISOString();
|
||||
await writeMantisDiscordSmokeArtifacts(summary, sensitiveValues);
|
||||
}
|
||||
|
||||
return {
|
||||
outputDir,
|
||||
reportPath,
|
||||
summaryPath,
|
||||
status: summary.status,
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user