mirror of
https://github.com/openclaw/openclaw.git
synced 2026-05-06 18:30:44 +00:00
ci(qa): trigger Mantis Discord QA from PR comments
This commit is contained in:
@@ -1,6 +1,8 @@
|
|||||||
name: Mantis Discord Status Reactions
|
name: Mantis Discord Status Reactions
|
||||||
|
|
||||||
on:
|
on:
|
||||||
|
issue_comment:
|
||||||
|
types: [created]
|
||||||
workflow_dispatch:
|
workflow_dispatch:
|
||||||
inputs:
|
inputs:
|
||||||
baseline_ref:
|
baseline_ref:
|
||||||
@@ -24,7 +26,7 @@ permissions:
|
|||||||
pull-requests: write
|
pull-requests: write
|
||||||
|
|
||||||
concurrency:
|
concurrency:
|
||||||
group: mantis-discord-status-reactions-${{ inputs.baseline_ref }}-${{ inputs.candidate_ref }}-${{ github.run_attempt }}
|
group: mantis-discord-status-reactions-${{ github.event.issue.number || inputs.pr_number || inputs.candidate_ref || github.run_id }}-${{ github.run_attempt }}
|
||||||
cancel-in-progress: false
|
cancel-in-progress: false
|
||||||
|
|
||||||
env:
|
env:
|
||||||
@@ -37,6 +39,19 @@ env:
|
|||||||
jobs:
|
jobs:
|
||||||
authorize_actor:
|
authorize_actor:
|
||||||
name: Authorize workflow actor
|
name: Authorize workflow actor
|
||||||
|
if: >-
|
||||||
|
${{
|
||||||
|
github.event_name == 'workflow_dispatch' ||
|
||||||
|
(
|
||||||
|
github.event_name == 'issue_comment' &&
|
||||||
|
github.event.issue.pull_request &&
|
||||||
|
(
|
||||||
|
contains(github.event.comment.body, '@Mantis') ||
|
||||||
|
contains(github.event.comment.body, '@mantis') ||
|
||||||
|
contains(github.event.comment.body, '/mantis')
|
||||||
|
)
|
||||||
|
)
|
||||||
|
}}
|
||||||
runs-on: blacksmith-8vcpu-ubuntu-2404
|
runs-on: blacksmith-8vcpu-ubuntu-2404
|
||||||
steps:
|
steps:
|
||||||
- name: Require maintainer-level repository access
|
- name: Require maintainer-level repository access
|
||||||
@@ -58,9 +73,101 @@ jobs:
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
resolve_request:
|
||||||
|
name: Resolve Mantis request
|
||||||
|
needs: authorize_actor
|
||||||
|
runs-on: blacksmith-8vcpu-ubuntu-2404
|
||||||
|
outputs:
|
||||||
|
baseline_ref: ${{ steps.resolve.outputs.baseline_ref }}
|
||||||
|
candidate_ref: ${{ steps.resolve.outputs.candidate_ref }}
|
||||||
|
pr_number: ${{ steps.resolve.outputs.pr_number }}
|
||||||
|
request_source: ${{ steps.resolve.outputs.request_source }}
|
||||||
|
should_run: ${{ steps.resolve.outputs.should_run }}
|
||||||
|
steps:
|
||||||
|
- name: Resolve refs and target PR
|
||||||
|
id: resolve
|
||||||
|
uses: actions/github-script@v8
|
||||||
|
with:
|
||||||
|
script: |
|
||||||
|
const defaultBaseline = "0bf06e953fdda290799fc9fb9244a8f67fdae593";
|
||||||
|
const eventName = context.eventName;
|
||||||
|
|
||||||
|
function setOutput(name, value) {
|
||||||
|
core.setOutput(name, value ?? "");
|
||||||
|
core.info(`${name}=${value ?? ""}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (eventName === "workflow_dispatch") {
|
||||||
|
const inputs = context.payload.inputs ?? {};
|
||||||
|
setOutput("should_run", "true");
|
||||||
|
setOutput("baseline_ref", inputs.baseline_ref || defaultBaseline);
|
||||||
|
setOutput("candidate_ref", inputs.candidate_ref || "main");
|
||||||
|
setOutput("pr_number", inputs.pr_number || "");
|
||||||
|
setOutput("request_source", "workflow_dispatch");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (eventName !== "issue_comment") {
|
||||||
|
core.setFailed(`Unsupported event: ${eventName}`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const issue = context.payload.issue;
|
||||||
|
const body = context.payload.comment?.body ?? "";
|
||||||
|
if (!issue?.pull_request) {
|
||||||
|
core.setFailed("Mantis issue_comment trigger requires a pull request comment.");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const normalized = body.toLowerCase();
|
||||||
|
const requested =
|
||||||
|
(normalized.includes("@mantis") || normalized.includes("/mantis")) &&
|
||||||
|
normalized.includes("discord") &&
|
||||||
|
normalized.includes("status") &&
|
||||||
|
normalized.includes("reaction");
|
||||||
|
if (!requested) {
|
||||||
|
core.notice("Comment mentioned Mantis but did not request the Discord status-reactions scenario.");
|
||||||
|
setOutput("should_run", "false");
|
||||||
|
setOutput("baseline_ref", "");
|
||||||
|
setOutput("candidate_ref", "");
|
||||||
|
setOutput("pr_number", "");
|
||||||
|
setOutput("request_source", "unsupported_issue_comment");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const { owner, repo } = context.repo;
|
||||||
|
const { data: pr } = await github.rest.pulls.get({
|
||||||
|
owner,
|
||||||
|
repo,
|
||||||
|
pull_number: issue.number,
|
||||||
|
});
|
||||||
|
|
||||||
|
const baselineMatch = body.match(/(?:baseline|base)[\s:=]+([^\s`]+)/i);
|
||||||
|
const candidateMatch = body.match(/(?:candidate|head)[\s:=]+([^\s`]+)/i);
|
||||||
|
const baseline = baselineMatch?.[1] ?? defaultBaseline;
|
||||||
|
const rawCandidate = candidateMatch?.[1];
|
||||||
|
const candidate =
|
||||||
|
rawCandidate && !["head", "pr", "pr-head"].includes(rawCandidate.toLowerCase())
|
||||||
|
? rawCandidate
|
||||||
|
: pr.head.sha;
|
||||||
|
|
||||||
|
setOutput("should_run", "true");
|
||||||
|
setOutput("baseline_ref", baseline);
|
||||||
|
setOutput("candidate_ref", candidate);
|
||||||
|
setOutput("pr_number", String(issue.number));
|
||||||
|
setOutput("request_source", "issue_comment");
|
||||||
|
|
||||||
|
await github.rest.reactions.createForIssueComment({
|
||||||
|
owner,
|
||||||
|
repo,
|
||||||
|
comment_id: context.payload.comment.id,
|
||||||
|
content: "eyes",
|
||||||
|
}).catch((error) => core.warning(`Could not add eyes reaction: ${error.message}`));
|
||||||
|
|
||||||
validate_refs:
|
validate_refs:
|
||||||
name: Validate selected refs
|
name: Validate selected refs
|
||||||
needs: authorize_actor
|
needs: resolve_request
|
||||||
|
if: ${{ needs.resolve_request.outputs.should_run == 'true' }}
|
||||||
runs-on: blacksmith-8vcpu-ubuntu-2404
|
runs-on: blacksmith-8vcpu-ubuntu-2404
|
||||||
outputs:
|
outputs:
|
||||||
baseline_revision: ${{ steps.validate.outputs.baseline_revision }}
|
baseline_revision: ${{ steps.validate.outputs.baseline_revision }}
|
||||||
@@ -76,8 +183,8 @@ jobs:
|
|||||||
id: validate
|
id: validate
|
||||||
env:
|
env:
|
||||||
GH_TOKEN: ${{ github.token }}
|
GH_TOKEN: ${{ github.token }}
|
||||||
BASELINE_REF: ${{ inputs.baseline_ref }}
|
BASELINE_REF: ${{ needs.resolve_request.outputs.baseline_ref }}
|
||||||
CANDIDATE_REF: ${{ inputs.candidate_ref }}
|
CANDIDATE_REF: ${{ needs.resolve_request.outputs.candidate_ref }}
|
||||||
shell: bash
|
shell: bash
|
||||||
run: |
|
run: |
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
@@ -126,7 +233,8 @@ jobs:
|
|||||||
|
|
||||||
run_status_reactions:
|
run_status_reactions:
|
||||||
name: Run Discord status reaction before/after
|
name: Run Discord status reaction before/after
|
||||||
needs: validate_refs
|
needs: [resolve_request, validate_refs]
|
||||||
|
if: ${{ needs.resolve_request.outputs.should_run == 'true' }}
|
||||||
runs-on: blacksmith-8vcpu-ubuntu-2404
|
runs-on: blacksmith-8vcpu-ubuntu-2404
|
||||||
timeout-minutes: 180
|
timeout-minutes: 180
|
||||||
environment: qa-live-shared
|
environment: qa-live-shared
|
||||||
@@ -268,7 +376,7 @@ jobs:
|
|||||||
|
|
||||||
- name: Create Mantis GitHub App token
|
- name: Create Mantis GitHub App token
|
||||||
id: mantis_app_token
|
id: mantis_app_token
|
||||||
if: ${{ always() && inputs.pr_number != '' }}
|
if: ${{ always() && needs.resolve_request.outputs.pr_number != '' }}
|
||||||
uses: actions/create-github-app-token@v3
|
uses: actions/create-github-app-token@v3
|
||||||
with:
|
with:
|
||||||
app-id: ${{ secrets.MANTIS_GITHUB_APP_ID }}
|
app-id: ${{ secrets.MANTIS_GITHUB_APP_ID }}
|
||||||
@@ -280,14 +388,14 @@ jobs:
|
|||||||
permission-pull-requests: write
|
permission-pull-requests: write
|
||||||
|
|
||||||
- name: Comment PR with inline QA screenshots
|
- name: Comment PR with inline QA screenshots
|
||||||
if: ${{ always() && inputs.pr_number != '' && steps.run_mantis.outputs.output_dir != '' }}
|
if: ${{ always() && needs.resolve_request.outputs.pr_number != '' && steps.run_mantis.outputs.output_dir != '' }}
|
||||||
env:
|
env:
|
||||||
GH_TOKEN: ${{ steps.mantis_app_token.outputs.token }}
|
GH_TOKEN: ${{ steps.mantis_app_token.outputs.token }}
|
||||||
TARGET_PR: ${{ inputs.pr_number }}
|
TARGET_PR: ${{ needs.resolve_request.outputs.pr_number }}
|
||||||
ARTIFACT_URL: ${{ steps.upload_artifact.outputs.artifact-url }}
|
ARTIFACT_URL: ${{ steps.upload_artifact.outputs.artifact-url }}
|
||||||
BASELINE_SHA: ${{ needs.validate_refs.outputs.baseline_revision }}
|
BASELINE_SHA: ${{ needs.validate_refs.outputs.baseline_revision }}
|
||||||
CANDIDATE_SHA: ${{ needs.validate_refs.outputs.candidate_revision }}
|
CANDIDATE_SHA: ${{ needs.validate_refs.outputs.candidate_revision }}
|
||||||
MANTIS_COMMENT_BOT_LOGIN: ${{ vars.MANTIS_GITHUB_APP_BOT_LOGIN || 'openclaw-mantis-qa[bot]' }}
|
REQUEST_SOURCE: ${{ needs.resolve_request.outputs.request_source }}
|
||||||
shell: bash
|
shell: bash
|
||||||
run: |
|
run: |
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
@@ -343,6 +451,7 @@ jobs:
|
|||||||
baseline_status="$(jq -r '.baseline.status' "$root/comparison.json")"
|
baseline_status="$(jq -r '.baseline.status' "$root/comparison.json")"
|
||||||
candidate_status="$(jq -r '.candidate.status' "$root/comparison.json")"
|
candidate_status="$(jq -r '.candidate.status' "$root/comparison.json")"
|
||||||
pass="$(jq -r '.pass' "$root/comparison.json")"
|
pass="$(jq -r '.pass' "$root/comparison.json")"
|
||||||
|
mantis_bot_login="$(gh api user --jq '.login')"
|
||||||
comment_file="$(mktemp)"
|
comment_file="$(mktemp)"
|
||||||
cat > "$comment_file" <<EOF
|
cat > "$comment_file" <<EOF
|
||||||
<!-- mantis-discord-status-reactions -->
|
<!-- mantis-discord-status-reactions -->
|
||||||
@@ -351,6 +460,7 @@ jobs:
|
|||||||
Summary: Mantis reran Discord status reactions against the known queued-only baseline and the candidate ref. The baseline reproduced the bug, while the candidate showed the expected queued -> thinking -> done reaction sequence.
|
Summary: Mantis reran Discord status reactions against the known queued-only baseline and the candidate ref. The baseline reproduced the bug, while the candidate showed the expected queued -> thinking -> done reaction sequence.
|
||||||
|
|
||||||
- Scenario: \`discord-status-reactions-tool-only\`
|
- Scenario: \`discord-status-reactions-tool-only\`
|
||||||
|
- Trigger: \`${REQUEST_SOURCE}\`
|
||||||
- Run: https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}
|
- Run: https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}
|
||||||
- Artifact: ${ARTIFACT_URL}
|
- Artifact: ${ARTIFACT_URL}
|
||||||
- Baseline: \`${baseline_status}\` at \`${BASELINE_SHA}\`
|
- Baseline: \`${baseline_status}\` at \`${BASELINE_SHA}\`
|
||||||
@@ -366,7 +476,7 @@ jobs:
|
|||||||
|
|
||||||
comment_id="$(
|
comment_id="$(
|
||||||
gh api --paginate "repos/${GITHUB_REPOSITORY}/issues/${TARGET_PR}/comments" \
|
gh api --paginate "repos/${GITHUB_REPOSITORY}/issues/${TARGET_PR}/comments" \
|
||||||
--jq ".[] | select(.body | contains(\"<!-- mantis-discord-status-reactions -->\")) | select(.user.login == \"${MANTIS_COMMENT_BOT_LOGIN}\") | .id" \
|
--jq ".[] | select(.body | contains(\"<!-- mantis-discord-status-reactions -->\")) | select(.user.login == \"${mantis_bot_login}\") | .id" \
|
||||||
| tail -n 1
|
| tail -n 1
|
||||||
)"
|
)"
|
||||||
|
|
||||||
|
|||||||
@@ -101,6 +101,22 @@ worktrees, runs `discord-status-reactions-tool-only` against each worktree, and
|
|||||||
uploads `baseline/`, `candidate/`, `comparison.json`, and `mantis-report.md` as
|
uploads `baseline/`, `candidate/`, `comparison.json`, and `mantis-report.md` as
|
||||||
Actions artifacts.
|
Actions artifacts.
|
||||||
|
|
||||||
|
You can also trigger the status-reactions run directly from a PR comment:
|
||||||
|
|
||||||
|
```text
|
||||||
|
@Mantis discord status reactions
|
||||||
|
```
|
||||||
|
|
||||||
|
The comment trigger is intentionally narrow. It only runs on pull request
|
||||||
|
comments from users with write, maintain, or admin access, and it only recognizes
|
||||||
|
Discord status-reaction requests. By default it uses the known bad baseline ref
|
||||||
|
and the current PR head SHA as the candidate. Maintainers can override either
|
||||||
|
ref:
|
||||||
|
|
||||||
|
```text
|
||||||
|
@Mantis discord status reactions baseline=origin/main candidate=HEAD
|
||||||
|
```
|
||||||
|
|
||||||
ClawSweeper command examples:
|
ClawSweeper command examples:
|
||||||
|
|
||||||
```text
|
```text
|
||||||
@@ -361,11 +377,9 @@ messages, and other bulky evidence stay in the Actions artifact.
|
|||||||
Production workflows should post those comments with the Mantis GitHub App, not
|
Production workflows should post those comments with the Mantis GitHub App, not
|
||||||
with `github-actions[bot]`. Store the app id and private key as
|
with `github-actions[bot]`. Store the app id and private key as
|
||||||
`MANTIS_GITHUB_APP_ID` and `MANTIS_GITHUB_APP_PRIVATE_KEY` GitHub Actions
|
`MANTIS_GITHUB_APP_ID` and `MANTIS_GITHUB_APP_PRIVATE_KEY` GitHub Actions
|
||||||
secrets. If the app is renamed, set `MANTIS_GITHUB_APP_BOT_LOGIN` as a GitHub
|
secrets. The workflow resolves the bot login from the GitHub App token, updates
|
||||||
Actions variable to the new bot login, for example `openclaw-mantis[bot]`. The
|
an existing Mantis-owned comment when one exists, and creates a new Mantis-owned
|
||||||
workflow should update an existing Mantis-owned comment when one exists; if only
|
comment instead of rewriting older `github-actions[bot]` comments.
|
||||||
an older `github-actions[bot]` comment exists, it should create a new
|
|
||||||
Mantis-owned comment instead of rewriting the legacy bot comment.
|
|
||||||
|
|
||||||
The PR comment should be short and visual:
|
The PR comment should be short and visual:
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user