ci: pin full release validation children

.github/workflows/full-release-validation.yml

@@ -568,7 +568,7 @@ jobs:
 
   summary:
     name: Verify full validation
-    needs: [normal_ci, plugin_prerelease, release_checks, npm_telegram]
+    needs: [resolve_target, normal_ci, plugin_prerelease, release_checks, npm_telegram]
     if: always()
     runs-on: ubuntu-24.04
     timeout-minutes: 5
@@ -640,6 +640,7 @@ jobs:
           PLUGIN_PRERELEASE_RESULT: ${{ needs.plugin_prerelease.result }}
           RELEASE_CHECKS_RESULT: ${{ needs.release_checks.result }}
           NPM_TELEGRAM_RESULT: ${{ needs.npm_telegram.result }}
+          TARGET_SHA: ${{ needs.resolve_target.outputs.sha }}
         run: |
           set -euo pipefail
 
@@ -657,13 +658,19 @@ jobs:
               return 1
             fi
 
-            local run_json status conclusion url attempt
-            run_json="$(gh run view "$run_id" --json status,conclusion,url,attempt,jobs)"
+            local run_json status conclusion url attempt head_sha
+            run_json="$(gh run view "$run_id" --json status,conclusion,url,attempt,headSha,jobs)"
             status="$(jq -r '.status' <<< "$run_json")"
             conclusion="$(jq -r '.conclusion' <<< "$run_json")"
             url="$(jq -r '.url' <<< "$run_json")"
             attempt="$(jq -r '.attempt' <<< "$run_json")"
-            echo "${label}: ${status}/${conclusion} attempt ${attempt}: ${url}"
+            head_sha="$(jq -r '.headSha // ""' <<< "$run_json")"
+            echo "${label}: ${status}/${conclusion} attempt ${attempt} head ${head_sha}: ${url}"
+
+            if [[ -n "${TARGET_SHA// }" && "$head_sha" != "$TARGET_SHA" ]]; then
+              echo "::error::${label} child run used ${head_sha}, expected ${TARGET_SHA}. Dispatch Full Release Validation from a ref pinned to the target SHA, not a moving branch."
+              return 1
+            fi
 
             if [[ "$status" != "completed" || "$conclusion" != "success" ]]; then
               echo "::error::${label} child run ended with ${status}/${conclusion}: ${url}"
@@ -677,8 +684,8 @@ jobs:
               echo
               echo "### Child workflow overview"
               echo
-              echo "| Child | Result | Minutes | Run |"
-              echo "| --- | --- | ---: | --- |"
+              echo "| Child | Result | Minutes | Head SHA | Run |"
+              echo "| --- | --- | ---: | --- | --- |"
             } >> "$GITHUB_STEP_SUMMARY"
 
             append_child_row() {
@@ -692,7 +699,7 @@ jobs:
               fi
 
               local run_json row
-              run_json="$(gh run view "$run_id" --json status,conclusion,url,createdAt,updatedAt)"
+              run_json="$(gh run view "$run_id" --json status,conclusion,url,createdAt,updatedAt,headSha)"
               row="$(
                 jq -r --arg label "$label" '
                   def ts: fromdateiso8601;
@@ -703,7 +710,8 @@ jobs:
                     then (((($updated | ts) - ($created | ts)) / 60) * 10 | round / 10 | tostring)
                     else ""
                   end) as $minutes |
-                  "| `" + $label + "` | `" + ($run.status // "") + "/" + ($run.conclusion // "") + "` | " + $minutes + " | [run](" + ($run.url // "") + ") |"
+                  ($run.headSha // "") as $head |
+                  "| `" + $label + "` | `" + ($run.status // "") + "/" + ($run.conclusion // "") + "` | " + $minutes + " | `" + $head + "` | [run](" + ($run.url // "") + ") |"
                 ' <<< "$run_json"
               )"
               echo "$row" >> "$GITHUB_STEP_SUMMARY"

AGENTS.md

@@ -74,6 +74,7 @@ Telegraph style. Root rules only. Read scoped `AGENTS.md` before subtree work.
 - PR review answer must explicitly cover: what bug/behavior we are trying to fix; PR/issue URL(s) and affected endpoint/surface; whether this is the best possible fix, with high-certainty evidence from code, tests, CI, and shipped/current behavior.
 - When working on an issue or PR, always end the user-facing final answer with the full GitHub URL.
 - CI polling: exact SHA, needed fields only. Example: `gh api repos/<owner>/<repo>/actions/runs/<id> --jq '{status,conclusion,head_sha,updated_at,name,path}'`.
+- Full Release Validation exact-SHA proof: use `pnpm ci:full-release --sha <sha>`; do not dispatch `--ref main -f ref=<sha>` on moving `main`. GitHub dispatch refs cannot be raw SHAs, so the helper uses a temporary pinned branch and verifies child `headSha`.
 - Post-land wait: minimal. Exact landed SHA only. If superseded on `main`, same-branch `cancel-in-progress` cancellations are expected; stop once local touched-surface proof exists. Never wait for newer unrelated `main` unless asked.
 - Wait matrix:
   - never: `Auto response`, `Labeler`, `Docs Sync Publish Repo`, `Docs Agent`, `Test Performance Agent`, `Stale`.

docs/ci.md

@@ -134,6 +134,20 @@ See [Full release validation](/reference/full-release-validation) for the
 stage matrix, exact workflow job names, profile differences, artifacts, and
 focused rerun handles.
 
+For pinned commit proof on a fast-moving branch, use the helper instead of
+`gh workflow run ... --ref main -f ref=<sha>`:
+
+```bash
+pnpm ci:full-release --sha <full-sha>
+```
+
+GitHub workflow dispatch refs must be branches or tags, not raw commit SHAs. The
+helper pushes a temporary `release-ci/<sha>-...` branch at the target SHA,
+dispatches `Full Release Validation` from that pinned ref, verifies every child
+workflow `headSha` matches the target, and deletes the temporary branch when the
+run completes. The umbrella verifier also fails if any child workflow ran at a
+different SHA.
+
 `release_profile` controls live/provider breadth passed into release checks. The
 manual release workflows default to `stable`; use `full` only when you
 intentionally want the broad advisory provider/media matrix.

docs/reference/RELEASING.md

@@ -235,8 +235,21 @@ Validation` or from the `main`/release workflow ref so workflow logic and
 ## Release test boxes
 
 `Full Release Validation` is how operators kick off all pre-release tests from
-one entrypoint. Run it from the trusted `main` workflow ref and pass the release
-branch, tag, or full commit SHA as `ref`:
+one entrypoint. For a pinned commit proof on a fast-moving branch, use the
+helper so every child workflow runs from a temporary branch fixed at the target
+SHA:
+
+```bash
+pnpm ci:full-release --sha <full-sha>
+```
+
+The helper pushes `release-ci/<sha>-...`, dispatches `Full Release Validation`
+from that branch with `ref=<sha>`, verifies every child workflow `headSha`
+matches the target, then deletes the temporary branch. This avoids proving a
+newer `main` child run by accident.
+
+For release branch or tag validation, run it from the trusted `main` workflow
+ref and pass the release branch or tag as `ref`:
 
 ```bash
 gh workflow run full-release-validation.yml \
@@ -268,6 +281,9 @@ Child workflows are dispatched from the trusted ref that runs `Full Release
 Validation`, normally `--ref main`, even when the target `ref` points at an
 older release branch or tag. There is no separate Full Release Validation
 workflow-ref input; choose the trusted harness by choosing the workflow run ref.
+Do not use `--ref main -f ref=<sha>` for exact commit proof on moving `main`;
+raw commit SHAs cannot be workflow dispatch refs, so use
+`pnpm ci:full-release --sha <sha>` to create the pinned temporary branch.
 
 Use `release_profile` to select live/provider breadth:
 

package.json

@@ -1291,6 +1291,7 @@
     "check:timed:all-types": "node scripts/check-timed.mjs --include-test-types",
     "check:timed:architecture": "node scripts/check-timed.mjs --include-architecture",
     "check:workflows": "node scripts/check-workflows.mjs",
+    "ci:full-release": "node scripts/full-release-validation-at-sha.mjs",
     "ci:timings": "node scripts/ci-run-timings.mjs --latest-main",
     "ci:timings:recent": "node scripts/ci-run-timings.mjs --recent 10",
     "codex-app-server:protocol:check": "node --import tsx scripts/check-codex-app-server-protocol.ts",

scripts/full-release-validation-at-sha.mjs

@@ -0,0 +1,263 @@
+#!/usr/bin/env node
+import { execFileSync, spawnSync } from "node:child_process";
+
+const WORKFLOW = "full-release-validation.yml";
+const DEFAULT_INPUTS = {
+  provider: "openai",
+  mode: "both",
+  release_profile: "full",
+  rerun_group: "all",
+};
+
+function usage() {
+  console.error(`Usage: node scripts/full-release-validation-at-sha.mjs [--sha <sha>] [--branch <name>] [--keep-branch] [--dry-run] [-- -f key=value ...]
+
+Creates a temporary remote branch pinned to the target commit, dispatches Full
+Release Validation from that branch, watches the parent run, verifies all child
+workflow head SHAs match, then deletes the temporary branch by default.`);
+}
+
+function run(command, args, options = {}) {
+  if (options.dryRun) {
+    console.log(["+", command, ...args].join(" "));
+    return "";
+  }
+  return execFileSync(command, args, {
+    encoding: "utf8",
+    stdio: options.stdio ?? ["ignore", "pipe", "inherit"],
+  }).trim();
+}
+
+function runStatus(command, args, options = {}) {
+  if (options.dryRun) {
+    console.log(["+", command, ...args].join(" "));
+    return { status: 0, stdout: "" };
+  }
+  return spawnSync(command, args, {
+    encoding: "utf8",
+    stdio: options.stdio ?? ["ignore", "pipe", "inherit"],
+  });
+}
+
+function parseArgs(argv) {
+  const args = {
+    sha: "",
+    branch: "",
+    keepBranch: false,
+    dryRun: false,
+    inputs: { ...DEFAULT_INPUTS },
+  };
+
+  for (let i = 0; i < argv.length; i += 1) {
+    const arg = argv[i];
+    if (arg === "--help" || arg === "-h") {
+      usage();
+      process.exit(0);
+    }
+    if (arg === "--sha") {
+      args.sha = argv[++i] ?? "";
+      continue;
+    }
+    if (arg === "--branch") {
+      args.branch = argv[++i] ?? "";
+      continue;
+    }
+    if (arg === "--keep-branch") {
+      args.keepBranch = true;
+      continue;
+    }
+    if (arg === "--dry-run") {
+      args.dryRun = true;
+      continue;
+    }
+    if (arg === "--") {
+      for (const extra of argv.slice(i + 1)) {
+        const assignment = extra.startsWith("-f") ? extra.slice(2).trim() : extra;
+        const [key, ...valueParts] = assignment.split("=");
+        if (!key || valueParts.length === 0) {
+          throw new Error(`Unsupported extra argument after --: ${extra}`);
+        }
+        args.inputs[key] = valueParts.join("=");
+      }
+      break;
+    }
+    if (arg === "-f") {
+      const assignment = argv[++i] ?? "";
+      const [key, ...valueParts] = assignment.split("=");
+      if (!key || valueParts.length === 0) {
+        throw new Error(`Invalid -f assignment: ${assignment}`);
+      }
+      args.inputs[key] = valueParts.join("=");
+      continue;
+    }
+    if (arg.startsWith("-f") && arg.includes("=")) {
+      const assignment = arg.slice(2).trim();
+      const [key, ...valueParts] = assignment.split("=");
+      if (!key || valueParts.length === 0) {
+        throw new Error(`Invalid -f assignment: ${arg}`);
+      }
+      args.inputs[key] = valueParts.join("=");
+      continue;
+    }
+    throw new Error(`Unknown argument: ${arg}`);
+  }
+
+  return args;
+}
+
+function sanitizeBranchPart(value) {
+  return value
+    .replace(/[^A-Za-z0-9._/-]+/g, "-")
+    .replace(/\/+/g, "/")
+    .replace(/^[/.-]+|[/.-]+$/g, "")
+    .slice(0, 80);
+}
+
+function resolveSha(requestedSha) {
+  const rev = requestedSha || "HEAD";
+  return run("git", ["rev-parse", "--verify", `${rev}^{commit}`], { dryRun: false });
+}
+
+function collectRunId(dispatchOutput) {
+  const match = dispatchOutput.match(/actions\/runs\/(\d+)/);
+  return match?.[1] ?? "";
+}
+
+function findLatestRunId(branch, sha) {
+  const json = run("gh", [
+    "run",
+    "list",
+    "--workflow",
+    WORKFLOW,
+    "--branch",
+    branch,
+    "--event",
+    "workflow_dispatch",
+    "--limit",
+    "20",
+    "--json",
+    "databaseId,headSha,createdAt",
+  ]);
+  const runs = JSON.parse(json);
+  const match = runs.find((runItem) => runItem.headSha === sha);
+  return match?.databaseId ? String(match.databaseId) : "";
+}
+
+function childRunIds(parentRunId) {
+  const jobsJson = run("gh", ["run", "view", parentRunId, "--json", "jobs"]);
+  const jobs = JSON.parse(jobsJson).jobs ?? [];
+  const summaryJob = jobs.find((job) => job.name === "Verify full validation");
+  if (!summaryJob?.databaseId) {
+    return [];
+  }
+  const log = run("gh", [
+    "run",
+    "view",
+    parentRunId,
+    "--job",
+    String(summaryJob.databaseId),
+    "--log",
+  ]);
+  return [...new Set([...log.matchAll(/actions\/runs\/(\d+)/g)].map((match) => match[1]))];
+}
+
+function verifyChildHeads(parentRunId, sha) {
+  const ids = childRunIds(parentRunId);
+  if (ids.length === 0) {
+    throw new Error(
+      `Could not find child workflow run ids in parent verifier logs for ${parentRunId}.`,
+    );
+  }
+
+  let failed = false;
+  for (const id of ids) {
+    const json = run("gh", ["run", "view", id, "--json", "name,status,conclusion,headSha,url"]);
+    const child = JSON.parse(json);
+    const ok =
+      child.headSha === sha && child.status === "completed" && child.conclusion === "success";
+    console.log(
+      `${ok ? "ok" : "bad"} ${child.name} ${child.status}/${child.conclusion} ${child.headSha} ${child.url}`,
+    );
+    failed ||= !ok;
+  }
+  if (failed) {
+    throw new Error(`One or more child workflows failed or did not run at ${sha}.`);
+  }
+}
+
+function main() {
+  const args = parseArgs(process.argv.slice(2));
+  const sha = resolveSha(args.sha);
+  const shortSha = sha.slice(0, 12);
+  const branch = sanitizeBranchPart(args.branch || `release-ci/${shortSha}-${Date.now()}`);
+  const remoteBranchRef = `refs/heads/${branch}`;
+  const dispatchInputs = { ref: sha, ...args.inputs };
+
+  console.log(`Target SHA: ${sha}`);
+  console.log(`Temporary workflow ref: ${branch}`);
+
+  run("git", ["push", "origin", `${sha}:${remoteBranchRef}`], {
+    dryRun: args.dryRun,
+    stdio: "inherit",
+  });
+
+  let parentRunId = "";
+  try {
+    const dispatchArgs = ["workflow", "run", WORKFLOW, "--ref", branch];
+    for (const [key, value] of Object.entries(dispatchInputs)) {
+      dispatchArgs.push("-f", `${key}=${value}`);
+    }
+
+    const dispatchOutput = run("gh", dispatchArgs, { dryRun: args.dryRun });
+    if (dispatchOutput) {
+      console.log(dispatchOutput);
+    }
+    parentRunId = collectRunId(dispatchOutput);
+    if (!parentRunId && !args.dryRun) {
+      for (let attempt = 0; attempt < 60; attempt += 1) {
+        parentRunId = findLatestRunId(branch, sha);
+        if (parentRunId) {
+          break;
+        }
+        Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 5000);
+      }
+    }
+    if (!parentRunId) {
+      if (args.dryRun) {
+        return;
+      }
+      throw new Error("Could not determine Full Release Validation run id.");
+    }
+
+    console.log(`Parent run: https://github.com/openclaw/openclaw/actions/runs/${parentRunId}`);
+    const watch = runStatus(
+      "gh",
+      ["run", "watch", parentRunId, "--exit-status", "--interval", "30"],
+      {
+        stdio: "inherit",
+      },
+    );
+    if (watch.status !== 0) {
+      throw new Error(
+        `Full Release Validation failed: https://github.com/openclaw/openclaw/actions/runs/${parentRunId}`,
+      );
+    }
+    verifyChildHeads(parentRunId, sha);
+  } finally {
+    if (!args.keepBranch) {
+      run("git", ["push", "origin", `:${remoteBranchRef}`], {
+        dryRun: args.dryRun,
+        stdio: "inherit",
+      });
+    } else {
+      console.log(`Kept ${remoteBranchRef}`);
+    }
+  }
+}
+
+try {
+  main();
+} catch (error) {
+  console.error(error instanceof Error ? error.message : String(error));
+  process.exit(1);
+}

test/scripts/package-acceptance-workflow.test.ts

@@ -138,6 +138,18 @@ describe("package acceptance workflow", () => {
     expect(workflow).toContain("Published upgrade survivor scenarios:");
   });
 
+  it("requires full release child workflows to run at the resolved target SHA", () => {
+    const workflow = readFileSync(FULL_RELEASE_VALIDATION_WORKFLOW, "utf8");
+
+    expect(workflow).toContain("TARGET_SHA: ${{ needs.resolve_target.outputs.sha }}");
+    expect(workflow).toContain("--json status,conclusion,url,attempt,headSha,jobs");
+    expect(workflow).toContain("child run used ${head_sha}, expected ${TARGET_SHA}");
+    expect(workflow).toContain(
+      "Dispatch Full Release Validation from a ref pinned to the target SHA",
+    );
+    expect(workflow).toContain("| Child | Result | Minutes | Head SHA | Run |");
+  });
+
   it("keeps exhaustive update migration as a separate manual package gate", () => {
     const workflow = readFileSync(UPDATE_MIGRATION_WORKFLOW, "utf8");
     const packageWorkflow = readFileSync(PACKAGE_ACCEPTANCE_WORKFLOW, "utf8");