ci(qa): trigger Mantis Discord QA from PR comments

This commit is contained in:
Peter Steinberger
2026-05-03 21:23:25 +01:00
parent d4af125b52
commit d8b82df5d4
2 changed files with 139 additions and 15 deletions

View File

@@ -1,6 +1,8 @@
name: Mantis Discord Status Reactions name: Mantis Discord Status Reactions
on: on:
issue_comment:
types: [created]
workflow_dispatch: workflow_dispatch:
inputs: inputs:
baseline_ref: baseline_ref:
@@ -24,7 +26,7 @@ permissions:
pull-requests: write pull-requests: write
concurrency: concurrency:
group: mantis-discord-status-reactions-${{ inputs.baseline_ref }}-${{ inputs.candidate_ref }}-${{ github.run_attempt }} group: mantis-discord-status-reactions-${{ github.event.issue.number || inputs.pr_number || inputs.candidate_ref || github.run_id }}-${{ github.run_attempt }}
cancel-in-progress: false cancel-in-progress: false
env: env:
@@ -37,6 +39,19 @@ env:
jobs: jobs:
authorize_actor: authorize_actor:
name: Authorize workflow actor name: Authorize workflow actor
if: >-
${{
github.event_name == 'workflow_dispatch' ||
(
github.event_name == 'issue_comment' &&
github.event.issue.pull_request &&
(
contains(github.event.comment.body, '@Mantis') ||
contains(github.event.comment.body, '@mantis') ||
contains(github.event.comment.body, '/mantis')
)
)
}}
runs-on: blacksmith-8vcpu-ubuntu-2404 runs-on: blacksmith-8vcpu-ubuntu-2404
steps: steps:
- name: Require maintainer-level repository access - name: Require maintainer-level repository access
@@ -58,9 +73,101 @@ jobs:
); );
} }
resolve_request:
name: Resolve Mantis request
needs: authorize_actor
runs-on: blacksmith-8vcpu-ubuntu-2404
outputs:
baseline_ref: ${{ steps.resolve.outputs.baseline_ref }}
candidate_ref: ${{ steps.resolve.outputs.candidate_ref }}
pr_number: ${{ steps.resolve.outputs.pr_number }}
request_source: ${{ steps.resolve.outputs.request_source }}
should_run: ${{ steps.resolve.outputs.should_run }}
steps:
- name: Resolve refs and target PR
id: resolve
uses: actions/github-script@v8
with:
script: |
const defaultBaseline = "0bf06e953fdda290799fc9fb9244a8f67fdae593";
const eventName = context.eventName;
function setOutput(name, value) {
core.setOutput(name, value ?? "");
core.info(`${name}=${value ?? ""}`);
}
if (eventName === "workflow_dispatch") {
const inputs = context.payload.inputs ?? {};
setOutput("should_run", "true");
setOutput("baseline_ref", inputs.baseline_ref || defaultBaseline);
setOutput("candidate_ref", inputs.candidate_ref || "main");
setOutput("pr_number", inputs.pr_number || "");
setOutput("request_source", "workflow_dispatch");
return;
}
if (eventName !== "issue_comment") {
core.setFailed(`Unsupported event: ${eventName}`);
return;
}
const issue = context.payload.issue;
const body = context.payload.comment?.body ?? "";
if (!issue?.pull_request) {
core.setFailed("Mantis issue_comment trigger requires a pull request comment.");
return;
}
const normalized = body.toLowerCase();
const requested =
(normalized.includes("@mantis") || normalized.includes("/mantis")) &&
normalized.includes("discord") &&
normalized.includes("status") &&
normalized.includes("reaction");
if (!requested) {
core.notice("Comment mentioned Mantis but did not request the Discord status-reactions scenario.");
setOutput("should_run", "false");
setOutput("baseline_ref", "");
setOutput("candidate_ref", "");
setOutput("pr_number", "");
setOutput("request_source", "unsupported_issue_comment");
return;
}
const { owner, repo } = context.repo;
const { data: pr } = await github.rest.pulls.get({
owner,
repo,
pull_number: issue.number,
});
const baselineMatch = body.match(/(?:baseline|base)[\s:=]+([^\s`]+)/i);
const candidateMatch = body.match(/(?:candidate|head)[\s:=]+([^\s`]+)/i);
const baseline = baselineMatch?.[1] ?? defaultBaseline;
const rawCandidate = candidateMatch?.[1];
const candidate =
rawCandidate && !["head", "pr", "pr-head"].includes(rawCandidate.toLowerCase())
? rawCandidate
: pr.head.sha;
setOutput("should_run", "true");
setOutput("baseline_ref", baseline);
setOutput("candidate_ref", candidate);
setOutput("pr_number", String(issue.number));
setOutput("request_source", "issue_comment");
await github.rest.reactions.createForIssueComment({
owner,
repo,
comment_id: context.payload.comment.id,
content: "eyes",
}).catch((error) => core.warning(`Could not add eyes reaction: ${error.message}`));
validate_refs: validate_refs:
name: Validate selected refs name: Validate selected refs
needs: authorize_actor needs: resolve_request
if: ${{ needs.resolve_request.outputs.should_run == 'true' }}
runs-on: blacksmith-8vcpu-ubuntu-2404 runs-on: blacksmith-8vcpu-ubuntu-2404
outputs: outputs:
baseline_revision: ${{ steps.validate.outputs.baseline_revision }} baseline_revision: ${{ steps.validate.outputs.baseline_revision }}
@@ -76,8 +183,8 @@ jobs:
id: validate id: validate
env: env:
GH_TOKEN: ${{ github.token }} GH_TOKEN: ${{ github.token }}
BASELINE_REF: ${{ inputs.baseline_ref }} BASELINE_REF: ${{ needs.resolve_request.outputs.baseline_ref }}
CANDIDATE_REF: ${{ inputs.candidate_ref }} CANDIDATE_REF: ${{ needs.resolve_request.outputs.candidate_ref }}
shell: bash shell: bash
run: | run: |
set -euo pipefail set -euo pipefail
@@ -126,7 +233,8 @@ jobs:
run_status_reactions: run_status_reactions:
name: Run Discord status reaction before/after name: Run Discord status reaction before/after
needs: validate_refs needs: [resolve_request, validate_refs]
if: ${{ needs.resolve_request.outputs.should_run == 'true' }}
runs-on: blacksmith-8vcpu-ubuntu-2404 runs-on: blacksmith-8vcpu-ubuntu-2404
timeout-minutes: 180 timeout-minutes: 180
environment: qa-live-shared environment: qa-live-shared
@@ -268,7 +376,7 @@ jobs:
- name: Create Mantis GitHub App token - name: Create Mantis GitHub App token
id: mantis_app_token id: mantis_app_token
if: ${{ always() && inputs.pr_number != '' }} if: ${{ always() && needs.resolve_request.outputs.pr_number != '' }}
uses: actions/create-github-app-token@v3 uses: actions/create-github-app-token@v3
with: with:
app-id: ${{ secrets.MANTIS_GITHUB_APP_ID }} app-id: ${{ secrets.MANTIS_GITHUB_APP_ID }}
@@ -280,14 +388,14 @@ jobs:
permission-pull-requests: write permission-pull-requests: write
- name: Comment PR with inline QA screenshots - name: Comment PR with inline QA screenshots
if: ${{ always() && inputs.pr_number != '' && steps.run_mantis.outputs.output_dir != '' }} if: ${{ always() && needs.resolve_request.outputs.pr_number != '' && steps.run_mantis.outputs.output_dir != '' }}
env: env:
GH_TOKEN: ${{ steps.mantis_app_token.outputs.token }} GH_TOKEN: ${{ steps.mantis_app_token.outputs.token }}
TARGET_PR: ${{ inputs.pr_number }} TARGET_PR: ${{ needs.resolve_request.outputs.pr_number }}
ARTIFACT_URL: ${{ steps.upload_artifact.outputs.artifact-url }} ARTIFACT_URL: ${{ steps.upload_artifact.outputs.artifact-url }}
BASELINE_SHA: ${{ needs.validate_refs.outputs.baseline_revision }} BASELINE_SHA: ${{ needs.validate_refs.outputs.baseline_revision }}
CANDIDATE_SHA: ${{ needs.validate_refs.outputs.candidate_revision }} CANDIDATE_SHA: ${{ needs.validate_refs.outputs.candidate_revision }}
MANTIS_COMMENT_BOT_LOGIN: ${{ vars.MANTIS_GITHUB_APP_BOT_LOGIN || 'openclaw-mantis-qa[bot]' }} REQUEST_SOURCE: ${{ needs.resolve_request.outputs.request_source }}
shell: bash shell: bash
run: | run: |
set -euo pipefail set -euo pipefail
@@ -343,6 +451,7 @@ jobs:
baseline_status="$(jq -r '.baseline.status' "$root/comparison.json")" baseline_status="$(jq -r '.baseline.status' "$root/comparison.json")"
candidate_status="$(jq -r '.candidate.status' "$root/comparison.json")" candidate_status="$(jq -r '.candidate.status' "$root/comparison.json")"
pass="$(jq -r '.pass' "$root/comparison.json")" pass="$(jq -r '.pass' "$root/comparison.json")"
mantis_bot_login="$(gh api user --jq '.login')"
comment_file="$(mktemp)" comment_file="$(mktemp)"
cat > "$comment_file" <<EOF cat > "$comment_file" <<EOF
<!-- mantis-discord-status-reactions --> <!-- mantis-discord-status-reactions -->
@@ -351,6 +460,7 @@ jobs:
Summary: Mantis reran Discord status reactions against the known queued-only baseline and the candidate ref. The baseline reproduced the bug, while the candidate showed the expected queued -> thinking -> done reaction sequence. Summary: Mantis reran Discord status reactions against the known queued-only baseline and the candidate ref. The baseline reproduced the bug, while the candidate showed the expected queued -> thinking -> done reaction sequence.
- Scenario: \`discord-status-reactions-tool-only\` - Scenario: \`discord-status-reactions-tool-only\`
- Trigger: \`${REQUEST_SOURCE}\`
- Run: https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID} - Run: https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}
- Artifact: ${ARTIFACT_URL} - Artifact: ${ARTIFACT_URL}
- Baseline: \`${baseline_status}\` at \`${BASELINE_SHA}\` - Baseline: \`${baseline_status}\` at \`${BASELINE_SHA}\`
@@ -366,7 +476,7 @@ jobs:
comment_id="$( comment_id="$(
gh api --paginate "repos/${GITHUB_REPOSITORY}/issues/${TARGET_PR}/comments" \ gh api --paginate "repos/${GITHUB_REPOSITORY}/issues/${TARGET_PR}/comments" \
--jq ".[] | select(.body | contains(\"<!-- mantis-discord-status-reactions -->\")) | select(.user.login == \"${MANTIS_COMMENT_BOT_LOGIN}\") | .id" \ --jq ".[] | select(.body | contains(\"<!-- mantis-discord-status-reactions -->\")) | select(.user.login == \"${mantis_bot_login}\") | .id" \
| tail -n 1 | tail -n 1
)" )"

View File

@@ -101,6 +101,22 @@ worktrees, runs `discord-status-reactions-tool-only` against each worktree, and
uploads `baseline/`, `candidate/`, `comparison.json`, and `mantis-report.md` as uploads `baseline/`, `candidate/`, `comparison.json`, and `mantis-report.md` as
Actions artifacts. Actions artifacts.
You can also trigger the status-reactions run directly from a PR comment:
```text
@Mantis discord status reactions
```
The comment trigger is intentionally narrow. It only runs on pull request
comments from users with write, maintain, or admin access, and it only recognizes
Discord status-reaction requests. By default it uses the known bad baseline ref
and the current PR head SHA as the candidate. Maintainers can override either
ref:
```text
@Mantis discord status reactions baseline=origin/main candidate=HEAD
```
ClawSweeper command examples: ClawSweeper command examples:
```text ```text
@@ -361,11 +377,9 @@ messages, and other bulky evidence stay in the Actions artifact.
Production workflows should post those comments with the Mantis GitHub App, not Production workflows should post those comments with the Mantis GitHub App, not
with `github-actions[bot]`. Store the app id and private key as with `github-actions[bot]`. Store the app id and private key as
`MANTIS_GITHUB_APP_ID` and `MANTIS_GITHUB_APP_PRIVATE_KEY` GitHub Actions `MANTIS_GITHUB_APP_ID` and `MANTIS_GITHUB_APP_PRIVATE_KEY` GitHub Actions
secrets. If the app is renamed, set `MANTIS_GITHUB_APP_BOT_LOGIN` as a GitHub secrets. The workflow resolves the bot login from the GitHub App token, updates
Actions variable to the new bot login, for example `openclaw-mantis[bot]`. The an existing Mantis-owned comment when one exists, and creates a new Mantis-owned
workflow should update an existing Mantis-owned comment when one exists; if only comment instead of rewriting older `github-actions[bot]` comments.
an older `github-actions[bot]` comment exists, it should create a new
Mantis-owned comment instead of rewriting the legacy bot comment.
The PR comment should be short and visual: The PR comment should be short and visual: