From d8b82df5d430d41d14331f5fa4f64bb6e97ac4b6 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sun, 3 May 2026 21:23:25 +0100 Subject: [PATCH] ci(qa): trigger Mantis Discord QA from PR comments --- .../mantis-discord-status-reactions.yml | 130 ++++++++++++++++-- docs/concepts/mantis.md | 24 +++- 2 files changed, 139 insertions(+), 15 deletions(-) diff --git a/.github/workflows/mantis-discord-status-reactions.yml b/.github/workflows/mantis-discord-status-reactions.yml index 429f339e8a3..acf984e33bc 100644 --- a/.github/workflows/mantis-discord-status-reactions.yml +++ b/.github/workflows/mantis-discord-status-reactions.yml @@ -1,6 +1,8 @@ name: Mantis Discord Status Reactions on: + issue_comment: + types: [created] workflow_dispatch: inputs: baseline_ref: @@ -24,7 +26,7 @@ permissions: pull-requests: write concurrency: - group: mantis-discord-status-reactions-${{ inputs.baseline_ref }}-${{ inputs.candidate_ref }}-${{ github.run_attempt }} + group: mantis-discord-status-reactions-${{ github.event.issue.number || inputs.pr_number || inputs.candidate_ref || github.run_id }}-${{ github.run_attempt }} cancel-in-progress: false env: @@ -37,6 +39,19 @@ env: jobs: authorize_actor: name: Authorize workflow actor + if: >- + ${{ + github.event_name == 'workflow_dispatch' || + ( + github.event_name == 'issue_comment' && + github.event.issue.pull_request && + ( + contains(github.event.comment.body, '@Mantis') || + contains(github.event.comment.body, '@mantis') || + contains(github.event.comment.body, '/mantis') + ) + ) + }} runs-on: blacksmith-8vcpu-ubuntu-2404 steps: - name: Require maintainer-level repository access @@ -58,9 +73,101 @@ jobs: ); } + resolve_request: + name: Resolve Mantis request + needs: authorize_actor + runs-on: blacksmith-8vcpu-ubuntu-2404 + outputs: + baseline_ref: ${{ steps.resolve.outputs.baseline_ref }} + candidate_ref: ${{ steps.resolve.outputs.candidate_ref }} + pr_number: ${{ steps.resolve.outputs.pr_number }} + request_source: ${{ steps.resolve.outputs.request_source }} + should_run: ${{ steps.resolve.outputs.should_run }} + steps: + - name: Resolve refs and target PR + id: resolve + uses: actions/github-script@v8 + with: + script: | + const defaultBaseline = "0bf06e953fdda290799fc9fb9244a8f67fdae593"; + const eventName = context.eventName; + + function setOutput(name, value) { + core.setOutput(name, value ?? ""); + core.info(`${name}=${value ?? ""}`); + } + + if (eventName === "workflow_dispatch") { + const inputs = context.payload.inputs ?? {}; + setOutput("should_run", "true"); + setOutput("baseline_ref", inputs.baseline_ref || defaultBaseline); + setOutput("candidate_ref", inputs.candidate_ref || "main"); + setOutput("pr_number", inputs.pr_number || ""); + setOutput("request_source", "workflow_dispatch"); + return; + } + + if (eventName !== "issue_comment") { + core.setFailed(`Unsupported event: ${eventName}`); + return; + } + + const issue = context.payload.issue; + const body = context.payload.comment?.body ?? ""; + if (!issue?.pull_request) { + core.setFailed("Mantis issue_comment trigger requires a pull request comment."); + return; + } + + const normalized = body.toLowerCase(); + const requested = + (normalized.includes("@mantis") || normalized.includes("/mantis")) && + normalized.includes("discord") && + normalized.includes("status") && + normalized.includes("reaction"); + if (!requested) { + core.notice("Comment mentioned Mantis but did not request the Discord status-reactions scenario."); + setOutput("should_run", "false"); + setOutput("baseline_ref", ""); + setOutput("candidate_ref", ""); + setOutput("pr_number", ""); + setOutput("request_source", "unsupported_issue_comment"); + return; + } + + const { owner, repo } = context.repo; + const { data: pr } = await github.rest.pulls.get({ + owner, + repo, + pull_number: issue.number, + }); + + const baselineMatch = body.match(/(?:baseline|base)[\s:=]+([^\s`]+)/i); + const candidateMatch = body.match(/(?:candidate|head)[\s:=]+([^\s`]+)/i); + const baseline = baselineMatch?.[1] ?? defaultBaseline; + const rawCandidate = candidateMatch?.[1]; + const candidate = + rawCandidate && !["head", "pr", "pr-head"].includes(rawCandidate.toLowerCase()) + ? rawCandidate + : pr.head.sha; + + setOutput("should_run", "true"); + setOutput("baseline_ref", baseline); + setOutput("candidate_ref", candidate); + setOutput("pr_number", String(issue.number)); + setOutput("request_source", "issue_comment"); + + await github.rest.reactions.createForIssueComment({ + owner, + repo, + comment_id: context.payload.comment.id, + content: "eyes", + }).catch((error) => core.warning(`Could not add eyes reaction: ${error.message}`)); + validate_refs: name: Validate selected refs - needs: authorize_actor + needs: resolve_request + if: ${{ needs.resolve_request.outputs.should_run == 'true' }} runs-on: blacksmith-8vcpu-ubuntu-2404 outputs: baseline_revision: ${{ steps.validate.outputs.baseline_revision }} @@ -76,8 +183,8 @@ jobs: id: validate env: GH_TOKEN: ${{ github.token }} - BASELINE_REF: ${{ inputs.baseline_ref }} - CANDIDATE_REF: ${{ inputs.candidate_ref }} + BASELINE_REF: ${{ needs.resolve_request.outputs.baseline_ref }} + CANDIDATE_REF: ${{ needs.resolve_request.outputs.candidate_ref }} shell: bash run: | set -euo pipefail @@ -126,7 +233,8 @@ jobs: run_status_reactions: name: Run Discord status reaction before/after - needs: validate_refs + needs: [resolve_request, validate_refs] + if: ${{ needs.resolve_request.outputs.should_run == 'true' }} runs-on: blacksmith-8vcpu-ubuntu-2404 timeout-minutes: 180 environment: qa-live-shared @@ -268,7 +376,7 @@ jobs: - name: Create Mantis GitHub App token id: mantis_app_token - if: ${{ always() && inputs.pr_number != '' }} + if: ${{ always() && needs.resolve_request.outputs.pr_number != '' }} uses: actions/create-github-app-token@v3 with: app-id: ${{ secrets.MANTIS_GITHUB_APP_ID }} @@ -280,14 +388,14 @@ jobs: permission-pull-requests: write - name: Comment PR with inline QA screenshots - if: ${{ always() && inputs.pr_number != '' && steps.run_mantis.outputs.output_dir != '' }} + if: ${{ always() && needs.resolve_request.outputs.pr_number != '' && steps.run_mantis.outputs.output_dir != '' }} env: GH_TOKEN: ${{ steps.mantis_app_token.outputs.token }} - TARGET_PR: ${{ inputs.pr_number }} + TARGET_PR: ${{ needs.resolve_request.outputs.pr_number }} ARTIFACT_URL: ${{ steps.upload_artifact.outputs.artifact-url }} BASELINE_SHA: ${{ needs.validate_refs.outputs.baseline_revision }} CANDIDATE_SHA: ${{ needs.validate_refs.outputs.candidate_revision }} - MANTIS_COMMENT_BOT_LOGIN: ${{ vars.MANTIS_GITHUB_APP_BOT_LOGIN || 'openclaw-mantis-qa[bot]' }} + REQUEST_SOURCE: ${{ needs.resolve_request.outputs.request_source }} shell: bash run: | set -euo pipefail @@ -343,6 +451,7 @@ jobs: baseline_status="$(jq -r '.baseline.status' "$root/comparison.json")" candidate_status="$(jq -r '.candidate.status' "$root/comparison.json")" pass="$(jq -r '.pass' "$root/comparison.json")" + mantis_bot_login="$(gh api user --jq '.login')" comment_file="$(mktemp)" cat > "$comment_file" < @@ -351,6 +460,7 @@ jobs: Summary: Mantis reran Discord status reactions against the known queued-only baseline and the candidate ref. The baseline reproduced the bug, while the candidate showed the expected queued -> thinking -> done reaction sequence. - Scenario: \`discord-status-reactions-tool-only\` + - Trigger: \`${REQUEST_SOURCE}\` - Run: https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID} - Artifact: ${ARTIFACT_URL} - Baseline: \`${baseline_status}\` at \`${BASELINE_SHA}\` @@ -366,7 +476,7 @@ jobs: comment_id="$( gh api --paginate "repos/${GITHUB_REPOSITORY}/issues/${TARGET_PR}/comments" \ - --jq ".[] | select(.body | contains(\"\")) | select(.user.login == \"${MANTIS_COMMENT_BOT_LOGIN}\") | .id" \ + --jq ".[] | select(.body | contains(\"\")) | select(.user.login == \"${mantis_bot_login}\") | .id" \ | tail -n 1 )" diff --git a/docs/concepts/mantis.md b/docs/concepts/mantis.md index 04e9254af44..07953864447 100644 --- a/docs/concepts/mantis.md +++ b/docs/concepts/mantis.md @@ -101,6 +101,22 @@ worktrees, runs `discord-status-reactions-tool-only` against each worktree, and uploads `baseline/`, `candidate/`, `comparison.json`, and `mantis-report.md` as Actions artifacts. +You can also trigger the status-reactions run directly from a PR comment: + +```text +@Mantis discord status reactions +``` + +The comment trigger is intentionally narrow. It only runs on pull request +comments from users with write, maintain, or admin access, and it only recognizes +Discord status-reaction requests. By default it uses the known bad baseline ref +and the current PR head SHA as the candidate. Maintainers can override either +ref: + +```text +@Mantis discord status reactions baseline=origin/main candidate=HEAD +``` + ClawSweeper command examples: ```text @@ -361,11 +377,9 @@ messages, and other bulky evidence stay in the Actions artifact. Production workflows should post those comments with the Mantis GitHub App, not with `github-actions[bot]`. Store the app id and private key as `MANTIS_GITHUB_APP_ID` and `MANTIS_GITHUB_APP_PRIVATE_KEY` GitHub Actions -secrets. If the app is renamed, set `MANTIS_GITHUB_APP_BOT_LOGIN` as a GitHub -Actions variable to the new bot login, for example `openclaw-mantis[bot]`. The -workflow should update an existing Mantis-owned comment when one exists; if only -an older `github-actions[bot]` comment exists, it should create a new -Mantis-owned comment instead of rewriting the legacy bot comment. +secrets. The workflow resolves the bot login from the GitHub App token, updates +an existing Mantis-owned comment when one exists, and creates a new Mantis-owned +comment instead of rewriting older `github-actions[bot]` comments. The PR comment should be short and visual: