fix(ci): sweeper revives cancelled required checks on auto-merge PRs (#112599)

* fix(ci): sweeper revives cancelled required checks on auto-merge PRs

* fix(ci): avoid shadowed identifier in sweeper revive lane

scripts/github/pr-ci-sweeper.d.mts

@@ -10,6 +10,19 @@ export function classifyPrForSweep(params: {
   botCloseCount: number;
   now: number;
 }): { action: "refire" | "skip"; reason: string };
+export function classifyRunForRevive(params: {
+  run: {
+    conclusion: string | null;
+    event: string;
+    run_attempt: number;
+    created_at: string;
+    head_branch?: string | null;
+    head_repository?: { full_name?: string };
+  };
+  prCreatedAt: string;
+  prHeadBranch?: string;
+  repoFullName?: string;
+}): { action: "revive" | "skip"; reason: string };
 export function runPrCiSweeper(params: {
   github: Record<string, unknown>;
   context: Record<string, unknown>;

scripts/github/pr-ci-sweeper.mjs

@@ -1,9 +1,10 @@
-// Re-fires pull_request CI runs that GitHub dropped at PR open. Fresh PRs can
-// race merge-ref computation: the open event's CI run either never attaches to
-// the head SHA or is created as an un-rerunnable startup_failure. Closing and
-// reopening the PR re-fires the event once the merge ref exists. The workflow
-// authenticates with a GitHub App token because GITHUB_TOKEN-authored events
-// do not trigger new workflow runs.
+// Repairs two GitHub Actions failure modes for fresh PRs. A merge-ref race can
+// drop pull_request CI entirely (or create an un-rerunnable startup_failure),
+// which close/reopen re-fires once the merge ref exists. Auto-merge PRs cannot
+// use that destructive repair, so a second lane finds cancelled Actions checks
+// attached to the PR head and reruns their PR-event workflows without disturbing
+// auto-merge. The workflow authenticates with a GitHub App token because
+// GITHUB_TOKEN-authored events do not trigger new workflow runs.
 
 const CI_WORKFLOW_FILE = "ci.yml";
 const LOOKBACK_MS = 24 * 60 * 60 * 1000;
@@ -14,6 +15,8 @@ const MIN_QUIET_MS = 10 * 60 * 1000;
 // a human, not an hourly close/reopen loop.
 const MAX_BOT_CLOSES = 2;
 const MAX_REFIRES_PER_SWEEP = 10;
+const MAX_REVIVES_PER_SWEEP = 10;
+const REVIVABLE_EVENTS = new Set(["pull_request", "pull_request_target"]);
 // Known sweeper identities for the close budget. The fallback app's login is
 // only recognized while it is the active identity, so an auth failover can at
 // worst double the budget to four re-fires — still bounded, and the
@@ -46,7 +49,7 @@ export function classifyPrForSweep({ pr, ciRuns, botCloseCount, now }) {
     return { action: "skip", reason: "merge-conflict" };
   }
   // Closing a PR silently cancels enabled auto-merge and reopening does not
-  // restore it; leave those PRs (e.g. generated locale refreshes) to a human.
+  // restore it; the non-destructive revive lane serves these PRs instead.
   if (pr.auto_merge) {
     return { action: "skip", reason: "auto-merge-enabled" };
   }
@@ -63,6 +66,37 @@ export function classifyPrForSweep({ pr, ciRuns, botCloseCount, now }) {
   };
 }
 
+export function classifyRunForRevive({ run, prCreatedAt, prHeadBranch, repoFullName }) {
+  if (run.conclusion !== "cancelled") {
+    return { action: "skip", reason: "not-cancelled" };
+  }
+  if (!REVIVABLE_EVENTS.has(run.event)) {
+    return { action: "skip", reason: "unsupported-event" };
+  }
+  // A head SHA can be reused by a later PR. Reruns replay the original event
+  // context, so a run created before this PR existed cannot safely be revived.
+  if (Date.parse(run.created_at) < Date.parse(prCreatedAt)) {
+    return { action: "skip", reason: "predates-pr" };
+  }
+  if (run.run_attempt >= 3) {
+    return { action: "skip", reason: "revive-budget-exhausted" };
+  }
+  // Trigger identity: even when a target run's head_sha is base-side and its
+  // pull_requests is empty (observed live), head_branch still names the
+  // triggering PR's branch. Same-repo branch names are unique, so requiring
+  // branch + repo match ties the rerun to this PR's event context; fork-headed
+  // or foreign-branch runs are refused rather than replayed on inference.
+  if (prHeadBranch !== undefined && run.head_branch !== prHeadBranch) {
+    return { action: "skip", reason: "different-head-branch" };
+  }
+  // Absent metadata fails closed: an unverifiable head repository must never
+  // default to "same repo" — that would replay fork-triggered privileged runs.
+  if (repoFullName !== undefined && run.head_repository?.full_name !== repoFullName) {
+    return { action: "skip", reason: "fork-head-repository" };
+  }
+  return { action: "revive", reason: "cancelled-pr-event-run" };
+}
+
 async function listPullRequestCiRuns({ github, owner, repo, headSha }) {
   // Manual dispatches or other events against the same SHA neither prove nor
   // repair the dropped pull_request run; judge only pull_request-event runs,
@@ -80,6 +114,42 @@ async function listPullRequestCiRuns({ github, owner, repo, headSha }) {
   });
 }
 
+async function listLatestChecksForHead({ github, owner, repo, headSha }) {
+  // Checks are the PR association GitHub's merge box and auto-merge actually
+  // wait on. pull_request_target workflow runs can have a base-side head_sha
+  // and an empty pull_requests array, so neither run field identifies the PR.
+  // filter=latest also omits cancelled checks superseded by a newer same-name
+  // check, leaving only cancelled work that still blocks this head.
+  // Accepted tradeoff: checks are commit-scoped, so a second PR sharing this
+  // exact head (a duplicate PR off the same automation branch) could have its
+  // run revived under our candidate's eligibility. GitHub exposes no trigger
+  // identity for these runs (pull_requests is empty on live target runs), and
+  // requiring one would skip the very runs this lane exists to repair; the
+  // worst case is duplicated bot activity on a same-branch sibling PR.
+  return await github.paginate(github.rest.checks.listForRef, {
+    owner,
+    repo,
+    ref: headSha,
+    filter: "latest",
+    per_page: 100,
+  });
+}
+
+function workflowRunIdForCheck(check) {
+  if (check.conclusion !== "cancelled" || check.app?.slug !== "github-actions") {
+    return undefined;
+  }
+  const match = check.details_url?.match(/\/actions\/runs\/(\d+)(?:\/|$)/);
+  return match ? Number(match[1]) : undefined;
+}
+
+function isExpectedReviveSkip(error) {
+  // Only a positively identified already-active run is an expected race; a bare
+  // 403 can also be a policy/permission denial that must surface, or the lane
+  // reports success while permanently unable to repair anything.
+  return /already (?:running|in progress)/i.test(String(error));
+}
+
 // Our close call succeeded against a verified-open PR, so the sweeper owns the
 // transition unless a newer close event by someone else is positively visible
 // (a human close in the millisecond race window makes our update an eventless
@@ -148,6 +218,7 @@ export async function runPrCiSweeper({
   const { owner, repo } = context.repo;
   const results = [];
   let refires = 0;
+  let revives = 0;
   const openPrs = await github.paginate(github.rest.pulls.list, {
     owner,
     repo,
@@ -156,6 +227,155 @@ export async function runPrCiSweeper({
     direction: "desc",
     per_page: 100,
   });
+  const seenRunIds = new Set();
+  reviveLane: for (const listed of openPrs) {
+    if (now - Date.parse(listed.updated_at) > LOOKBACK_MS) {
+      break;
+    }
+    if (
+      listed.draft ||
+      !listed.auto_merge ||
+      now - Date.parse(listed.created_at) > LOOKBACK_MS ||
+      now - Date.parse(listed.updated_at) < MIN_QUIET_MS
+    ) {
+      continue;
+    }
+    const checks = await listLatestChecksForHead({
+      github,
+      owner,
+      repo,
+      headSha: listed.head.sha,
+    });
+    // One workflow run fans out to many job checks; inspect each run id once
+    // per PR or a rejected matrix run costs one API call per job.
+    const inspectedForPr = new Set();
+    for (const check of checks) {
+      const runId = workflowRunIdForCheck(check);
+      if (runId === undefined || seenRunIds.has(runId) || inspectedForPr.has(runId)) {
+        continue;
+      }
+      inspectedForPr.add(runId);
+      const { data: run } = await github.rest.actions.getWorkflowRun({
+        owner,
+        repo,
+        run_id: runId,
+      });
+      const verdict = classifyRunForRevive({
+        run,
+        prCreatedAt: listed.created_at,
+        prHeadBranch: listed.head.ref,
+        repoFullName: `${owner}/${repo}`,
+      });
+      // Global suppression only once this run is actually handled: a
+      // PR-relative rejection (branch mismatch, predates-pr) must leave the
+      // run inspectable for the candidate that owns it.
+      if (verdict.action === "revive") {
+        seenRunIds.add(runId);
+      }
+      if (verdict.action !== "revive") {
+        if (verdict.reason === "revive-budget-exhausted") {
+          core.info(
+            `pr-ci-sweeper: skip cancelled run ${runId} for #${listed.number} (${verdict.reason})`,
+          );
+        }
+        continue;
+      }
+      if (revives >= MAX_REVIVES_PER_SWEEP) {
+        core.info(`pr-ci-sweeper: per-sweep revive cap (${MAX_REVIVES_PER_SWEEP}) reached`);
+        break reviveLane;
+      }
+      if (!dryRun) {
+        // Revalidate immediately before mutating, mirroring the re-fire lane:
+        // a fresh push, merge, close, or disarmed auto-merge in the scan gap
+        // must win — reviving an old head's run could cancel the new head's
+        // live run via workflow-level cancel-in-progress.
+        const { data: fresh } = await github.rest.pulls.get({
+          owner,
+          repo,
+          pull_number: listed.number,
+        });
+        if (
+          fresh.state !== "open" ||
+          fresh.draft ||
+          !fresh.auto_merge ||
+          fresh.head.sha !== listed.head.sha
+        ) {
+          core.info(`pr-ci-sweeper: #${listed.number} changed during sweep; skipping revive`);
+          continue;
+        }
+        // The scan can spend minutes across PRs; a manual rerun or new event
+        // may have replaced this check meanwhile. Rerunning a no-longer-latest
+        // cancelled run would put stale checks back in flight (or cancel a
+        // live replacement via workflow concurrency), so require the same
+        // check to still be the head's latest cancelled entry.
+        const currentChecks = await listLatestChecksForHead({
+          github,
+          owner,
+          repo,
+          headSha: listed.head.sha,
+        });
+        if (!currentChecks.some((current) => workflowRunIdForCheck(current) === runId)) {
+          core.info(
+            `pr-ci-sweeper: run ${runId} for #${listed.number} is no longer the latest cancelled check; skipping`,
+          );
+          continue;
+        }
+        // Revive only a quiescent head: any queued/in-progress Actions check
+        // means a replacement may be underway, and rerunning now could cancel
+        // it via workflow concurrency. Auto-merge waits for every check anyway,
+        // so deferring to the next sweep loses nothing.
+        const active = currentChecks.some(
+          (current) => current.app?.slug === "github-actions" && current.status !== "completed",
+        );
+        if (active) {
+          core.info(
+            `pr-ci-sweeper: #${listed.number} head has active checks; deferring revive of run ${runId}`,
+          );
+          continue;
+        }
+        // A concurrent rerun can advance the same run id to a fresh attempt in
+        // the scan gap; reclassify the current attempt so the budget and
+        // cancelled-state guards judge what the rerun would actually replay.
+        const { data: currentRun } = await github.rest.actions.getWorkflowRun({
+          owner,
+          repo,
+          run_id: runId,
+        });
+        const currentVerdict = classifyRunForRevive({
+          run: currentRun,
+          prCreatedAt: listed.created_at,
+          prHeadBranch: listed.head.ref,
+          repoFullName: `${owner}/${repo}`,
+        });
+        if (currentVerdict.action !== "revive") {
+          core.info(
+            `pr-ci-sweeper: run ${runId} for #${listed.number} changed during sweep (${currentVerdict.reason}); skipping`,
+          );
+          continue;
+        }
+      }
+      // Count only real (or dry-run-logged) revive attempts: stale candidates
+      // rejected by revalidation must not exhaust the sweep-wide cap.
+      revives += 1;
+      if (dryRun) {
+        core.info(
+          `pr-ci-sweeper: dry-run, would revive cancelled run ${runId} for #${listed.number}`,
+        );
+        continue;
+      }
+      try {
+        await github.rest.actions.reRunWorkflow({ owner, repo, run_id: runId });
+        core.info(`pr-ci-sweeper: revived cancelled run ${runId} for #${listed.number}`);
+      } catch (error) {
+        if (!isExpectedReviveSkip(error)) {
+          throw error;
+        }
+        core.info(
+          `pr-ci-sweeper: run ${runId} for #${listed.number} was not rerun (${String(error)}); skipping`,
+        );
+      }
+    }
+  }
   for (const listed of openPrs) {
     if (now - Date.parse(listed.updated_at) > LOOKBACK_MS) {
       break;
@@ -255,7 +475,7 @@ export async function runPrCiSweeper({
     await reopenWithRetry({ github, core, owner, repo, pullNumber: pr.number });
   }
   core.info(
-    `pr-ci-sweeper: checked ${openPrs.length} open PRs, ${results.length} candidates, ${refires} re-fire${refires === 1 ? "" : "s"}${dryRun ? " (dry-run)" : ""}`,
+    `pr-ci-sweeper: checked ${openPrs.length} open PRs, ${results.length} candidates, ${refires} re-fire${refires === 1 ? "" : "s"}, ${revives} revive${revives === 1 ? "" : "s"}${dryRun ? " (dry-run)" : ""}`,
   );
   return results;
 }