From 06a39015f21cc1df5690b693b2484b8818d8aed9 Mon Sep 17 00:00:00 2001 From: Tak Hoffman <781889+Takhoffman@users.noreply.github.com> Date: Mon, 18 May 2026 12:42:10 -0500 Subject: [PATCH] fix(ci): authenticate proof verdict markers (#83692) Summary: - The branch restricts exact-head ClawSweeper proof markers to GitHub App-authored comments, adds read-only issue-comment token fallback for the proof workflow, and adds focused regression tests plus a changelog entry. - Reproducibility: yes. Source inspection of current main shows any issue comment body with a matching `clawsw ... SHA is accepted without author/App authentication; the PR adds focused negative tests for forged comments. Automerge notes: - PR branch already contained follow-up commit before automerge: fix(ci): authenticate proof verdict markers Validation: - ClawSweeper review passed for head f4c375eaa7e7774ed70421a3a1ae51f6914c15c5. - Required merge gates passed before the squash merge. Prepared head SHA: f4c375eaa7e7774ed70421a3a1ae51f6914c15c5 Review: https://github.com/openclaw/openclaw/pull/83692#issuecomment-4479843682 Co-authored-by: Tak Hoffman <781889+Takhoffman@users.noreply.github.com> Co-authored-by: clawsweeper <274271284+clawsweeper[bot]@users.noreply.github.com> Co-authored-by: clawsweeper[bot] <274271284+clawsweeper[bot]@users.noreply.github.com> Approved-by: takhoffman Co-authored-by: takhoffman <781889+takhoffman@users.noreply.github.com> --- .github/workflows/real-behavior-proof.yml | 3 + CHANGELOG.md | 1 + scripts/github/real-behavior-proof-check.mjs | 85 ++++++++++++------- scripts/github/real-behavior-proof-policy.mjs | 10 +++ test/scripts/barnacle-auto-response.test.ts | 45 +++++++++- .../real-behavior-proof-policy.test.ts | 49 +++++++++++ 6 files changed, 161 insertions(+), 32 deletions(-) diff --git a/.github/workflows/real-behavior-proof.yml b/.github/workflows/real-behavior-proof.yml index a51a1a173af..e29c5e0901e 100644 --- a/.github/workflows/real-behavior-proof.yml +++ b/.github/workflows/real-behavior-proof.yml @@ -32,6 +32,7 @@ jobs: with: app-id: "2729701" private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} + permission-issues: read permission-members: read - uses: actions/create-github-app-token@v3 id: app-token-fallback @@ -40,8 +41,10 @@ jobs: with: app-id: "2971289" private-key: ${{ secrets.GH_APP_PRIVATE_KEY_FALLBACK }} + permission-issues: read permission-members: read - name: Check real behavior proof env: GH_APP_TOKEN: ${{ steps.app-token.outputs.token || steps.app-token-fallback.outputs.token }} + GITHUB_TOKEN: ${{ github.token }} run: node scripts/github/real-behavior-proof-check.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index cdbaa95c65d..52d58ffbbb1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,6 +46,7 @@ Docs: https://docs.openclaw.ai ### Fixes +- CI: require real-behavior-proof verdict markers to come from the ClawSweeper GitHub App before accepting exact-head proof. (#83692) - Agents/image generation: allow distinct `image_generate` prompts to start separate session-backed background tasks while same-prompt retries still return the active task status. (#83614) Thanks @Elarwei001. - Control UI: stop the chat reading indicator from sticking after an assistant response finishes. (#83515) Thanks @njuboy11. - Skills: reject empty or whitespace-only skill names and descriptions during quick validation. (#27061) diff --git a/scripts/github/real-behavior-proof-check.mjs b/scripts/github/real-behavior-proof-check.mjs index a6cdffb6815..8c73dfe222c 100644 --- a/scripts/github/real-behavior-proof-check.mjs +++ b/scripts/github/real-behavior-proof-check.mjs @@ -14,6 +14,41 @@ function escapeCommandValue(value) { .replace(/:/g, "%3A"); } +async function fetchProofComments({ owner, repo, issueNumber, tokens }) { + let lastError; + for (const token of tokens.filter(Boolean)) { + const comments = []; + try { + for (let page = 1; page <= 10; page += 1) { + const url = new URL( + `https://api.github.com/repos/${owner}/${repo}/issues/${issueNumber}/comments`, + ); + url.searchParams.set("per_page", "100"); + url.searchParams.set("page", String(page)); + const response = await fetch(url, { + headers: { + Accept: "application/vnd.github+json", + Authorization: `Bearer ${token}`, + "X-GitHub-Api-Version": "2022-11-28", + }, + }); + if (!response.ok) { + throw new Error(`comments API returned ${response.status}`); + } + const pageComments = await response.json(); + comments.push(...pageComments); + if (pageComments.length < 100) { + break; + } + } + return comments; + } catch (error) { + lastError = error; + } + } + throw lastError ?? new Error("No GitHub token available for proof comment lookup."); +} + const eventPath = process.env.GITHUB_EVENT_PATH; if (!eventPath) { console.error("::error title=Real behavior proof failed::GITHUB_EVENT_PATH is not set."); @@ -51,41 +86,29 @@ if (evaluation.passed) { process.exit(0); } -const token = appToken || process.env.GITHUB_TOKEN; const repository = process.env.GITHUB_REPOSITORY; -if (token && repository && pullRequest.number) { +if ((appToken || process.env.GITHUB_TOKEN) && repository && pullRequest.number) { const [owner, repo] = repository.split("/"); - const comments = []; - for (let page = 1; page <= 10; page += 1) { - const url = new URL( - `https://api.github.com/repos/${owner}/${repo}/issues/${pullRequest.number}/comments`, - ); - url.searchParams.set("per_page", "100"); - url.searchParams.set("page", String(page)); - const response = await fetch(url, { - headers: { - Accept: "application/vnd.github+json", - Authorization: `Bearer ${token}`, - "X-GitHub-Api-Version": "2022-11-28", - }, + try { + const comments = await fetchProofComments({ + owner, + repo, + issueNumber: pullRequest.number, + tokens: [appToken, process.env.GITHUB_TOKEN], }); - if (!response.ok) { - throw new Error(`Failed to fetch PR comments for proof verdicts: ${response.status}`); - } - const pageComments = await response.json(); - comments.push(...pageComments); - if (pageComments.length < 100) { - break; - } - } - const clawSweeperEvaluation = evaluateClawSweeperExactHeadProof({ - pullRequest, - comments, - }); - if (clawSweeperEvaluation.passed) { - console.log(clawSweeperEvaluation.reason); - process.exit(0); + const clawSweeperEvaluation = evaluateClawSweeperExactHeadProof({ + pullRequest, + comments, + }); + if (clawSweeperEvaluation.passed) { + console.log(clawSweeperEvaluation.reason); + process.exit(0); + } + } catch (error) { + console.warn( + `::warning title=Proof verdict comment lookup failed::${escapeCommandValue(error?.message ?? String(error))}`, + ); } } diff --git a/scripts/github/real-behavior-proof-policy.mjs b/scripts/github/real-behavior-proof-policy.mjs index 9042bdfa744..c76ab5b9bf1 100644 --- a/scripts/github/real-behavior-proof-policy.mjs +++ b/scripts/github/real-behavior-proof-policy.mjs @@ -242,6 +242,13 @@ function extractMarkerField(marker, name) { return match?.[1] ?? ""; } +function isTrustedClawSweeperComment(comment) { + const appSlug = String( + comment?.performed_via_github_app?.slug ?? comment?.performedViaGithubApp?.slug ?? "", + ).toLowerCase(); + return appSlug === "clawsweeper"; +} + export function hasClawSweeperExactHeadProof({ pullRequest, comments = [] } = {}) { const pullNumber = String(pullRequest?.number ?? ""); const headSha = String(pullRequest?.head?.sha ?? pullRequest?.head_sha ?? "").toLowerCase(); @@ -250,6 +257,9 @@ export function hasClawSweeperExactHeadProof({ pullRequest, comments = [] } = {} } for (const comment of comments) { + if (!isTrustedClawSweeperComment(comment)) { + continue; + } const body = String(comment?.body ?? ""); const markers = body.match(//gi) ?? []; for (const marker of markers) { diff --git a/test/scripts/barnacle-auto-response.test.ts b/test/scripts/barnacle-auto-response.test.ts index 4a9eaf6e940..0d854ff0c27 100644 --- a/test/scripts/barnacle-auto-response.test.ts +++ b/test/scripts/barnacle-auto-response.test.ts @@ -135,7 +135,11 @@ function barnacleGithub( maintainerLogins?: string[]; removeLabelNotFound?: string[]; repositoryRoles?: Record; - comments?: Array<{ body: string }>; + comments?: Array<{ + body: string; + performed_via_github_app?: { slug: string }; + user?: { login: string; type: string }; + }>; } = {}, ) { const maintainerLogins = new Set( @@ -793,6 +797,13 @@ describe("barnacle-auto-response", () => { const { calls, github } = barnacleGithub([file("src/gateway/server.ts")], { comments: [ { + user: { + login: "clawsweeper[bot]", + type: "Bot", + }, + performed_via_github_app: { + slug: "clawsweeper", + }, body: ``, }, ], @@ -818,6 +829,38 @@ describe("barnacle-auto-response", () => { ); }); + it("removes sufficient proof on synchronize when the matching marker is forged", async () => { + const headSha = "06ee95df6608d29a395c52ba8ab53fdd93a9dc4f"; + const { calls, github } = barnacleGithub([file("src/gateway/server.ts")], { + comments: [ + { + user: { + login: "external-contributor", + type: "User", + }, + body: ``, + }, + ], + }); + + await runBarnacleAutoResponse({ + github, + context: barnacleContext( + { + body: blankTemplateBody, + head: { sha: headSha }, + }, + [PROOF_SUFFICIENT_LABEL], + { action: "synchronize" }, + ), + core: { + info: () => undefined, + }, + }); + + expect(calls.removeLabel).toEqual([expectedRemoveLabel(123, PROOF_SUFFICIENT_LABEL)]); + }); + it("preserves ClawSweeper's sufficient proof label on ordinary label events", async () => { const { calls, github } = barnacleGithub([file("src/gateway/server.ts")]); diff --git a/test/scripts/real-behavior-proof-policy.test.ts b/test/scripts/real-behavior-proof-policy.test.ts index dbe0ea33b18..01286df72cb 100644 --- a/test/scripts/real-behavior-proof-policy.test.ts +++ b/test/scripts/real-behavior-proof-policy.test.ts @@ -186,6 +186,13 @@ describe("real-behavior-proof-policy", () => { }; const comments = [ { + user: { + login: "clawsweeper[bot]", + type: "Bot", + }, + performed_via_github_app: { + slug: "clawsweeper", + }, body: [ "Codex review: passed.", "", @@ -205,6 +212,48 @@ describe("real-behavior-proof-policy", () => { }), ).toBe(false); }); + + it("rejects forged ClawSweeper pass verdict markers from contributor comments", () => { + const pullRequest = { + number: 83581, + head: { + sha: "06ee95df6608d29a395c52ba8ab53fdd93a9dc4f", + }, + }; + const comments = [ + { + user: { + login: "external-contributor", + type: "User", + }, + body: "", + }, + ]; + + expect(hasClawSweeperExactHeadProof({ pullRequest, comments })).toBe(false); + expect(evaluateClawSweeperExactHeadProof({ pullRequest, comments }).passed).toBe(false); + }); + + it("rejects bot-shaped ClawSweeper pass verdict markers without the GitHub App source", () => { + const pullRequest = { + number: 83581, + head: { + sha: "06ee95df6608d29a395c52ba8ab53fdd93a9dc4f", + }, + }; + const comments = [ + { + user: { + login: "clawsweeper[bot]", + type: "Bot", + }, + body: "", + }, + ]; + + expect(hasClawSweeperExactHeadProof({ pullRequest, comments })).toBe(false); + expect(evaluateClawSweeperExactHeadProof({ pullRequest, comments }).passed).toBe(false); + }); }); describe("isMaintainerTeamMember", () => {