diff --git a/.github/workflows/pr-ci-sweeper.yml b/.github/workflows/pr-ci-sweeper.yml new file mode 100644 index 000000000000..347136644e91 --- /dev/null +++ b/.github/workflows/pr-ci-sweeper.yml @@ -0,0 +1,67 @@ +name: PR CI Sweeper + +# Re-fires pull_request CI runs that GitHub dropped at PR open (merge-ref +# race: no CI run attaches, or an un-rerunnable startup_failure is created). +# Close/reopen with an app token re-fires the event; GITHUB_TOKEN events +# would not trigger workflows. + +on: + schedule: + - cron: "7 * * * *" + workflow_dispatch: + inputs: + dry_run: + description: Log intended re-fires without closing/reopening. + required: false + default: false + type: boolean + +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" + +concurrency: + group: pr-ci-sweeper + cancel-in-progress: false + +permissions: {} + +jobs: + sweep: + permissions: + contents: read + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + ref: ${{ github.sha }} + persist-credentials: false + - uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3 + id: app-token + continue-on-error: true + with: + app-id: "2729701" + private-key: ${{ secrets.GH_APP_PRIVATE_KEY }} + - uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3 + id: app-token-fallback + if: steps.app-token.outcome == 'failure' + with: + app-id: "2971289" + private-key: ${{ secrets.GH_APP_PRIVATE_KEY_FALLBACK }} + - name: Sweep dropped PR CI runs + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9 + with: + github-token: ${{ steps.app-token.outputs.token || steps.app-token-fallback.outputs.token }} + script: | + const { pathToFileURL } = require("node:url"); + const moduleUrl = pathToFileURL( + `${process.env.GITHUB_WORKSPACE}/scripts/github/pr-ci-sweeper.mjs`, + ); + const { runPrCiSweeper } = await import(moduleUrl.href); + + await runPrCiSweeper({ + github, + context, + core, + dryRun: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run || false }}, + appSlug: ${{ toJSON(steps.app-token.outputs.app-slug || steps.app-token-fallback.outputs.app-slug || '') }}, + }); diff --git a/scripts/github/pr-ci-sweeper.d.mts b/scripts/github/pr-ci-sweeper.d.mts new file mode 100644 index 000000000000..bb21f0e423ce --- /dev/null +++ b/scripts/github/pr-ci-sweeper.d.mts @@ -0,0 +1,19 @@ +export function classifyPrForSweep(params: { + pr: { + draft?: boolean; + created_at: string; + updated_at: string; + mergeable?: boolean | null; + auto_merge?: object | null; + }; + ciRuns: Array<{ conclusion: string | null }>; + botCloseCount: number; + now: number; +}): { action: "refire" | "skip"; reason: string }; +export function runPrCiSweeper(params: { + github: Record; + context: Record; + core: Pick & { setFailed: (message: string) => void }; + dryRun?: boolean; + appSlug?: string; +}): Promise>; diff --git a/scripts/github/pr-ci-sweeper.mjs b/scripts/github/pr-ci-sweeper.mjs new file mode 100644 index 000000000000..30175755386f --- /dev/null +++ b/scripts/github/pr-ci-sweeper.mjs @@ -0,0 +1,254 @@ +// Re-fires pull_request CI runs that GitHub dropped at PR open. Fresh PRs can +// race merge-ref computation: the open event's CI run either never attaches to +// the head SHA or is created as an un-rerunnable startup_failure. Closing and +// reopening the PR re-fires the event once the merge ref exists. The workflow +// authenticates with a GitHub App token because GITHUB_TOKEN-authored events +// do not trigger new workflow runs. + +const CI_WORKFLOW_FILE = "ci.yml"; +const LOOKBACK_MS = 24 * 60 * 60 * 1000; +// Give GitHub time to settle merge-ref computation and late run attachment +// before judging a head SHA as dropped. +const MIN_QUIET_MS = 10 * 60 * 1000; +// Two bot closes per PR: a head that still has no CI after two re-fires needs +// a human, not an hourly close/reopen loop. +const MAX_BOT_CLOSES = 2; +const MAX_REFIRES_PER_SWEEP = 10; +// Known sweeper identities for the close budget. The fallback app's login is +// only recognized while it is the active identity, so an auth failover can at +// worst double the budget to four re-fires — still bounded, and the +// newest-close ownership check keeps human closes authoritative regardless. +const KNOWN_SWEEPER_LOGINS = ["openclaw-barnacle[bot]"]; +const REOPEN_DELAY_MS = 5_000; + +const sleep = (ms) => + new Promise((resolve) => { + setTimeout(resolve, ms); + }); + +export function classifyPrForSweep({ pr, ciRuns, botCloseCount, now }) { + if (pr.draft) { + return { action: "skip", reason: "draft" }; + } + if (now - Date.parse(pr.created_at) > LOOKBACK_MS) { + return { action: "skip", reason: "outside-lookback" }; + } + if (now - Date.parse(pr.updated_at) < MIN_QUIET_MS) { + return { action: "skip", reason: "recently-updated" }; + } + // A conflicted PR legitimately has no merge ref; CI cannot attach until the + // author resolves, so re-firing would loop forever. + if (pr.mergeable === false) { + return { action: "skip", reason: "merge-conflict" }; + } + // Pending computation resolves by the next hourly sweep; do not close/reopen + // on an unknown merge state. + if (pr.mergeable === null || pr.mergeable === undefined) { + return { action: "skip", reason: "mergeability-pending" }; + } + // Closing a PR silently cancels enabled auto-merge and reopening does not + // restore it; leave those PRs (e.g. generated locale refreshes) to a human. + if (pr.auto_merge) { + return { action: "skip", reason: "auto-merge-enabled" }; + } + // Queued and in-progress runs have a null conclusion and count as attached. + if (ciRuns.some((run) => run.conclusion !== "startup_failure")) { + return { action: "skip", reason: "ci-attached" }; + } + if (botCloseCount >= MAX_BOT_CLOSES) { + return { action: "skip", reason: "refire-budget-exhausted" }; + } + return { + action: "refire", + reason: ciRuns.length === 0 ? "ci-run-missing" : "ci-startup-failure", + }; +} + +async function listPullRequestCiRuns({ github, owner, repo, headSha }) { + // Manual dispatches or other events against the same SHA neither prove nor + // repair the dropped pull_request run; judge only pull_request-event runs, + // filtered server-side and paginated so unrelated runs cannot crowd them out. + // Accepted tradeoff: a SHA shared by two PRs can mask one PR's dropped run + // behind the other's — a skip-only miss. Matching run.pull_requests instead + // would misclassify fork PRs, where GitHub leaves that array empty. + return await github.paginate(github.rest.actions.listWorkflowRuns, { + owner, + repo, + workflow_id: CI_WORKFLOW_FILE, + head_sha: headSha, + event: "pull_request", + per_page: 100, + }); +} + +// Our close call succeeded against a verified-open PR, so the sweeper owns the +// transition unless a newer close event by someone else is positively visible +// (a human close in the millisecond race window makes our update an eventless +// no-op). Stale or lagging event reads must therefore default to "ours". +async function someoneElseClosed({ + github, + owner, + repo, + pullNumber, + sweeperLogins, + knownCloseIds, +}) { + const events = await github.paginate(github.rest.issues.listEvents, { + owner, + repo, + issue_number: pullNumber, + per_page: 100, + }); + const newClose = events.findLast( + (event) => event.event === "closed" && !knownCloseIds.has(event.id), + ); + if (!newClose?.actor) { + return false; + } + return !(newClose.actor.type === "Bot" && sweeperLogins.has(newClose.actor.login)); +} + +async function reopenWithRetry({ github, core, owner, repo, pullNumber }) { + let lastError; + for (let attempt = 1; attempt <= 3; attempt += 1) { + try { + await github.rest.pulls.update({ owner, repo, pull_number: pullNumber, state: "open" }); + return true; + } catch (error) { + lastError = error; + await sleep(REOPEN_DELAY_MS * attempt); + } + } + // Never leave a swept PR closed silently: surface on the PR and fail the run. + await github.rest.issues + .createComment({ + owner, + repo, + issue_number: pullNumber, + body: "PR CI Sweeper closed this PR to re-fire a dropped CI run but could not reopen it. Please reopen manually.", + }) + .catch(() => undefined); + core.setFailed(`pr-ci-sweeper: failed to reopen #${pullNumber}: ${String(lastError)}`); + return false; +} + +export async function runPrCiSweeper({ github, context, core, dryRun = false, appSlug = "" }) { + const sweeperLogins = new Set(KNOWN_SWEEPER_LOGINS); + if (appSlug) { + sweeperLogins.add(`${appSlug}[bot]`); + } + const { owner, repo } = context.repo; + const now = Date.now(); + const results = []; + let refires = 0; + const openPrs = await github.paginate(github.rest.pulls.list, { + owner, + repo, + state: "open", + sort: "updated", + direction: "desc", + per_page: 100, + }); + for (const listed of openPrs) { + if (now - Date.parse(listed.updated_at) > LOOKBACK_MS) { + break; + } + if (refires >= MAX_REFIRES_PER_SWEEP) { + core.info(`pr-ci-sweeper: per-sweep re-fire cap (${MAX_REFIRES_PER_SWEEP}) reached`); + break; + } + if (listed.draft) { + continue; + } + const ciRuns = await listPullRequestCiRuns({ github, owner, repo, headSha: listed.head.sha }); + if (ciRuns.some((run) => run.conclusion !== "startup_failure")) { + continue; + } + // Candidate: fetch authoritative state (mergeable, current head) and the + // close history so a racing push or human action wins over the sweep. + const { data: pr } = await github.rest.pulls.get({ owner, repo, pull_number: listed.number }); + if (pr.state !== "open" || pr.head.sha !== listed.head.sha) { + continue; + } + const events = await github.paginate(github.rest.issues.listEvents, { + owner, + repo, + issue_number: pr.number, + per_page: 100, + }); + // Budget counts only this sweeper's own closes so unrelated bot + // automation cannot exhaust a PR's re-fire allowance. + const botCloseCount = events.filter( + (event) => + event.event === "closed" && + event.actor?.type === "Bot" && + sweeperLogins.has(event.actor.login), + ).length; + const verdict = classifyPrForSweep({ pr, ciRuns, botCloseCount, now }); + results.push({ number: pr.number, sha: pr.head.sha.slice(0, 12), ...verdict }); + if (verdict.action !== "refire") { + core.info(`pr-ci-sweeper: skip #${pr.number} (${verdict.reason})`); + continue; + } + refires += 1; + if (dryRun) { + core.info(`pr-ci-sweeper: dry-run, would re-fire #${pr.number} (${verdict.reason})`); + continue; + } + core.info(`pr-ci-sweeper: re-firing CI for #${pr.number} (${verdict.reason})`); + // Revalidate immediately before mutating: a human close or a fresh push in + // the classify gap must win over the sweep. + const { data: fresh } = await github.rest.pulls.get({ owner, repo, pull_number: pr.number }); + if ( + fresh.state !== "open" || + fresh.head.sha !== pr.head.sha || + fresh.auto_merge || + fresh.mergeable !== true + ) { + core.info(`pr-ci-sweeper: #${pr.number} changed during sweep; leaving it alone`); + continue; + } + // CI can attach late during the scan's own API calls; closing then would + // cancel a live run. Re-check the head immediately before mutating. + const latestRuns = await listPullRequestCiRuns({ + github, + owner, + repo, + headSha: fresh.head.sha, + }); + if (latestRuns.some((run) => run.conclusion !== "startup_failure")) { + core.info(`pr-ci-sweeper: #${pr.number} CI attached during sweep; leaving it alone`); + continue; + } + const knownCloseIds = new Set( + events.filter((event) => event.event === "closed").map((event) => event.id), + ); + await github.rest.pulls.update({ owner, repo, pull_number: pr.number, state: "closed" }); + await sleep(REOPEN_DELAY_MS); + // Skip the reopen only on positive evidence that someone else performed a + // newer close. Verification errors and stale event reads fail toward + // reopening: stranding our own close is the worse outcome. + let humanClosed = false; + try { + humanClosed = await someoneElseClosed({ + github, + owner, + repo, + pullNumber: pr.number, + sweeperLogins, + knownCloseIds, + }); + } catch (error) { + core.info(`pr-ci-sweeper: close-ownership check failed (${String(error)}); reopening`); + } + if (humanClosed) { + core.info(`pr-ci-sweeper: #${pr.number} was closed by someone else; not reopening`); + continue; + } + await reopenWithRetry({ github, core, owner, repo, pullNumber: pr.number }); + } + core.info( + `pr-ci-sweeper: checked ${openPrs.length} open PRs, ${results.length} candidates, ${refires} re-fire${refires === 1 ? "" : "s"}${dryRun ? " (dry-run)" : ""}`, + ); + return results; +} diff --git a/test/scripts/pr-ci-sweeper.test.ts b/test/scripts/pr-ci-sweeper.test.ts new file mode 100644 index 000000000000..5175f92c4145 --- /dev/null +++ b/test/scripts/pr-ci-sweeper.test.ts @@ -0,0 +1,241 @@ +import { describe, expect, it } from "vitest"; +import { classifyPrForSweep, runPrCiSweeper } from "../../scripts/github/pr-ci-sweeper.mjs"; + +const NOW = Date.parse("2026-07-18T12:00:00Z"); +const MINUTES = 60 * 1000; +const HOURS = 60 * MINUTES; + +function pr(overrides: Partial[0]["pr"]> = {}) { + return { + draft: false, + created_at: new Date(NOW - 2 * HOURS).toISOString(), + updated_at: new Date(NOW - 30 * MINUTES).toISOString(), + mergeable: true, + auto_merge: null, + ...overrides, + }; +} + +describe("classifyPrForSweep", () => { + const cases: Array<{ + name: string; + input: Parameters[0]; + expected: ReturnType; + }> = [ + { + name: "re-fires when no CI run attached", + input: { pr: pr(), ciRuns: [], botCloseCount: 0, now: NOW }, + expected: { action: "refire", reason: "ci-run-missing" }, + }, + { + name: "re-fires when only startup failures attached", + input: { + pr: pr(), + ciRuns: [{ conclusion: "startup_failure" }], + botCloseCount: 1, + now: NOW, + }, + expected: { action: "refire", reason: "ci-startup-failure" }, + }, + { + name: "skips drafts", + input: { pr: pr({ draft: true }), ciRuns: [], botCloseCount: 0, now: NOW }, + expected: { action: "skip", reason: "draft" }, + }, + { + name: "skips PRs outside the 24h lookback", + input: { + pr: pr({ created_at: new Date(NOW - 25 * HOURS).toISOString() }), + ciRuns: [], + botCloseCount: 0, + now: NOW, + }, + expected: { action: "skip", reason: "outside-lookback" }, + }, + { + name: "skips recently updated PRs so merge-ref computation can settle", + input: { + pr: pr({ updated_at: new Date(NOW - 5 * MINUTES).toISOString() }), + ciRuns: [], + botCloseCount: 0, + now: NOW, + }, + expected: { action: "skip", reason: "recently-updated" }, + }, + { + name: "skips merge conflicts whose merge ref legitimately cannot exist", + input: { pr: pr({ mergeable: false }), ciRuns: [], botCloseCount: 0, now: NOW }, + expected: { action: "skip", reason: "merge-conflict" }, + }, + { + name: "skips PRs with auto-merge enabled (close would cancel it)", + input: { + pr: pr({ auto_merge: { merge_method: "squash" } }), + ciRuns: [], + botCloseCount: 0, + now: NOW, + }, + expected: { action: "skip", reason: "auto-merge-enabled" }, + }, + { + name: "treats a completed run as attached", + input: { + pr: pr(), + ciRuns: [{ conclusion: "success" }], + botCloseCount: 0, + now: NOW, + }, + expected: { action: "skip", reason: "ci-attached" }, + }, + { + name: "treats a queued run (null conclusion) as attached", + input: { + pr: pr(), + ciRuns: [{ conclusion: null }, { conclusion: "startup_failure" }], + botCloseCount: 0, + now: NOW, + }, + expected: { action: "skip", reason: "ci-attached" }, + }, + { + name: "treats a failed run as attached (rerunnable, not sweepable)", + input: { + pr: pr(), + ciRuns: [{ conclusion: "failure" }], + botCloseCount: 0, + now: NOW, + }, + expected: { action: "skip", reason: "ci-attached" }, + }, + { + name: "stops after two bot closes", + input: { pr: pr(), ciRuns: [], botCloseCount: 2, now: NOW }, + expected: { action: "skip", reason: "refire-budget-exhausted" }, + }, + { + name: "skips while mergeability is still computing", + input: { pr: pr({ mergeable: null }), ciRuns: [], botCloseCount: 0, now: NOW }, + expected: { action: "skip", reason: "mergeability-pending" }, + }, + ]; + + it.each(cases)("$name", ({ input, expected }) => { + expect(classifyPrForSweep(input)).toEqual(expected); + }); +}); + +type FakeCall = { method: string; args: Record }; + +function fakeGithub(options: { + prs: Array>; + runsBySha: Record>; + events?: Array>; +}) { + const calls: FakeCall[] = []; + const record = (method: string, args: Record) => { + calls.push({ method, args }); + }; + const github = { + paginate: (endpoint: { endpointName: string }, args: Record) => { + record(endpoint.endpointName, args); + if (endpoint.endpointName === "pulls.list") { + return Promise.resolve(options.prs); + } + if (endpoint.endpointName === "actions.listWorkflowRuns") { + return Promise.resolve( + (options.runsBySha[args.head_sha as string] ?? []).map((run) => ({ + event: run.event ?? "pull_request", + conclusion: run.conclusion, + })), + ); + } + if (endpoint.endpointName === "issues.listEvents") { + return Promise.resolve(options.events ?? []); + } + throw new Error(`unexpected paginate ${endpoint.endpointName}`); + }, + rest: { + pulls: { + list: { endpointName: "pulls.list" }, + get: (args: Record) => { + record("pulls.get", args); + const match = options.prs.find((entry) => entry.number === args.pull_number); + return Promise.resolve({ data: match }); + }, + update: (args: Record) => { + record("pulls.update", args); + return Promise.resolve({}); + }, + }, + actions: { listWorkflowRuns: { endpointName: "actions.listWorkflowRuns" } }, + issues: { + listEvents: { endpointName: "issues.listEvents" }, + createComment: (args: Record) => { + record("issues.createComment", args); + return Promise.resolve({}); + }, + }, + }, + }; + return { github, calls }; +} + +const context = { repo: { owner: "openclaw", repo: "openclaw" } }; +const core = { info: () => {}, setFailed: () => {} }; + +describe("runPrCiSweeper", () => { + it("classifies a dropped-CI PR as refire in dry-run without mutating", async () => { + const dropped = { + ...pr(), + number: 7, + state: "open", + head: { sha: "a".repeat(40) }, + }; + const attached = { + ...pr(), + number: 8, + state: "open", + head: { sha: "b".repeat(40) }, + }; + const { github, calls } = fakeGithub({ + prs: [dropped, attached], + runsBySha: { + [dropped.head.sha]: [{ conclusion: "startup_failure" }], + [attached.head.sha]: [{ conclusion: "success" }], + }, + }); + const results = await runPrCiSweeper({ + github: github as never, + context: context as never, + core: core as never, + dryRun: true, + appSlug: "openclaw-barnacle", + }); + expect(results).toEqual([ + { number: 7, sha: "a".repeat(12), action: "refire", reason: "ci-startup-failure" }, + ]); + expect(calls.filter((call) => call.method === "pulls.update")).toEqual([]); + }); + + it("closes and reopens a dropped-CI PR in live mode", async () => { + const dropped = { + ...pr(), + number: 9, + state: "open", + head: { sha: "c".repeat(40) }, + }; + const { github, calls } = fakeGithub({ prs: [dropped], runsBySha: {} }); + const results = await runPrCiSweeper({ + github: github as never, + context: context as never, + core: core as never, + appSlug: "openclaw-barnacle", + }); + expect(results).toEqual([ + { number: 9, sha: "c".repeat(12), action: "refire", reason: "ci-run-missing" }, + ]); + expect( + calls.filter((call) => call.method === "pulls.update").map((call) => call.args.state), + ).toEqual(["closed", "open"]); + }); +});