From 3ce8746b27e11f946822229094210d64bcbd38ed Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 2 May 2026 05:21:45 +0100 Subject: [PATCH] ci: pin full release validation children --- .github/workflows/full-release-validation.yml | 24 +- AGENTS.md | 1 + docs/ci.md | 14 + docs/reference/RELEASING.md | 20 +- package.json | 1 + scripts/full-release-validation-at-sha.mjs | 263 ++++++++++++++++++ .../package-acceptance-workflow.test.ts | 12 + 7 files changed, 325 insertions(+), 10 deletions(-) create mode 100755 scripts/full-release-validation-at-sha.mjs diff --git a/.github/workflows/full-release-validation.yml b/.github/workflows/full-release-validation.yml index 53337ab836f..72b2cab3bc4 100644 --- a/.github/workflows/full-release-validation.yml +++ b/.github/workflows/full-release-validation.yml @@ -568,7 +568,7 @@ jobs: summary: name: Verify full validation - needs: [normal_ci, plugin_prerelease, release_checks, npm_telegram] + needs: [resolve_target, normal_ci, plugin_prerelease, release_checks, npm_telegram] if: always() runs-on: ubuntu-24.04 timeout-minutes: 5 @@ -640,6 +640,7 @@ jobs: PLUGIN_PRERELEASE_RESULT: ${{ needs.plugin_prerelease.result }} RELEASE_CHECKS_RESULT: ${{ needs.release_checks.result }} NPM_TELEGRAM_RESULT: ${{ needs.npm_telegram.result }} + TARGET_SHA: ${{ needs.resolve_target.outputs.sha }} run: | set -euo pipefail @@ -657,13 +658,19 @@ jobs: return 1 fi - local run_json status conclusion url attempt - run_json="$(gh run view "$run_id" --json status,conclusion,url,attempt,jobs)" + local run_json status conclusion url attempt head_sha + run_json="$(gh run view "$run_id" --json status,conclusion,url,attempt,headSha,jobs)" status="$(jq -r '.status' <<< "$run_json")" conclusion="$(jq -r '.conclusion' <<< "$run_json")" url="$(jq -r '.url' <<< "$run_json")" attempt="$(jq -r '.attempt' <<< "$run_json")" - echo "${label}: ${status}/${conclusion} attempt ${attempt}: ${url}" + head_sha="$(jq -r '.headSha // ""' <<< "$run_json")" + echo "${label}: ${status}/${conclusion} attempt ${attempt} head ${head_sha}: ${url}" + + if [[ -n "${TARGET_SHA// }" && "$head_sha" != "$TARGET_SHA" ]]; then + echo "::error::${label} child run used ${head_sha}, expected ${TARGET_SHA}. Dispatch Full Release Validation from a ref pinned to the target SHA, not a moving branch." + return 1 + fi if [[ "$status" != "completed" || "$conclusion" != "success" ]]; then echo "::error::${label} child run ended with ${status}/${conclusion}: ${url}" @@ -677,8 +684,8 @@ jobs: echo echo "### Child workflow overview" echo - echo "| Child | Result | Minutes | Run |" - echo "| --- | --- | ---: | --- |" + echo "| Child | Result | Minutes | Head SHA | Run |" + echo "| --- | --- | ---: | --- | --- |" } >> "$GITHUB_STEP_SUMMARY" append_child_row() { @@ -692,7 +699,7 @@ jobs: fi local run_json row - run_json="$(gh run view "$run_id" --json status,conclusion,url,createdAt,updatedAt)" + run_json="$(gh run view "$run_id" --json status,conclusion,url,createdAt,updatedAt,headSha)" row="$( jq -r --arg label "$label" ' def ts: fromdateiso8601; @@ -703,7 +710,8 @@ jobs: then (((($updated | ts) - ($created | ts)) / 60) * 10 | round / 10 | tostring) else "" end) as $minutes | - "| `" + $label + "` | `" + ($run.status // "") + "/" + ($run.conclusion // "") + "` | " + $minutes + " | [run](" + ($run.url // "") + ") |" + ($run.headSha // "") as $head | + "| `" + $label + "` | `" + ($run.status // "") + "/" + ($run.conclusion // "") + "` | " + $minutes + " | `" + $head + "` | [run](" + ($run.url // "") + ") |" ' <<< "$run_json" )" echo "$row" >> "$GITHUB_STEP_SUMMARY" diff --git a/AGENTS.md b/AGENTS.md index c0c2289e486..68defee5b63 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -74,6 +74,7 @@ Telegraph style. Root rules only. Read scoped `AGENTS.md` before subtree work. - PR review answer must explicitly cover: what bug/behavior we are trying to fix; PR/issue URL(s) and affected endpoint/surface; whether this is the best possible fix, with high-certainty evidence from code, tests, CI, and shipped/current behavior. - When working on an issue or PR, always end the user-facing final answer with the full GitHub URL. - CI polling: exact SHA, needed fields only. Example: `gh api repos///actions/runs/ --jq '{status,conclusion,head_sha,updated_at,name,path}'`. +- Full Release Validation exact-SHA proof: use `pnpm ci:full-release --sha `; do not dispatch `--ref main -f ref=` on moving `main`. GitHub dispatch refs cannot be raw SHAs, so the helper uses a temporary pinned branch and verifies child `headSha`. - Post-land wait: minimal. Exact landed SHA only. If superseded on `main`, same-branch `cancel-in-progress` cancellations are expected; stop once local touched-surface proof exists. Never wait for newer unrelated `main` unless asked. - Wait matrix: - never: `Auto response`, `Labeler`, `Docs Sync Publish Repo`, `Docs Agent`, `Test Performance Agent`, `Stale`. diff --git a/docs/ci.md b/docs/ci.md index ac048ea1830..78562743cb9 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -134,6 +134,20 @@ See [Full release validation](/reference/full-release-validation) for the stage matrix, exact workflow job names, profile differences, artifacts, and focused rerun handles. +For pinned commit proof on a fast-moving branch, use the helper instead of +`gh workflow run ... --ref main -f ref=`: + +```bash +pnpm ci:full-release --sha +``` + +GitHub workflow dispatch refs must be branches or tags, not raw commit SHAs. The +helper pushes a temporary `release-ci/-...` branch at the target SHA, +dispatches `Full Release Validation` from that pinned ref, verifies every child +workflow `headSha` matches the target, and deletes the temporary branch when the +run completes. The umbrella verifier also fails if any child workflow ran at a +different SHA. + `release_profile` controls live/provider breadth passed into release checks. The manual release workflows default to `stable`; use `full` only when you intentionally want the broad advisory provider/media matrix. diff --git a/docs/reference/RELEASING.md b/docs/reference/RELEASING.md index 0d77c53c8ba..187da06dfd3 100644 --- a/docs/reference/RELEASING.md +++ b/docs/reference/RELEASING.md @@ -235,8 +235,21 @@ Validation` or from the `main`/release workflow ref so workflow logic and ## Release test boxes `Full Release Validation` is how operators kick off all pre-release tests from -one entrypoint. Run it from the trusted `main` workflow ref and pass the release -branch, tag, or full commit SHA as `ref`: +one entrypoint. For a pinned commit proof on a fast-moving branch, use the +helper so every child workflow runs from a temporary branch fixed at the target +SHA: + +```bash +pnpm ci:full-release --sha +``` + +The helper pushes `release-ci/-...`, dispatches `Full Release Validation` +from that branch with `ref=`, verifies every child workflow `headSha` +matches the target, then deletes the temporary branch. This avoids proving a +newer `main` child run by accident. + +For release branch or tag validation, run it from the trusted `main` workflow +ref and pass the release branch or tag as `ref`: ```bash gh workflow run full-release-validation.yml \ @@ -268,6 +281,9 @@ Child workflows are dispatched from the trusted ref that runs `Full Release Validation`, normally `--ref main`, even when the target `ref` points at an older release branch or tag. There is no separate Full Release Validation workflow-ref input; choose the trusted harness by choosing the workflow run ref. +Do not use `--ref main -f ref=` for exact commit proof on moving `main`; +raw commit SHAs cannot be workflow dispatch refs, so use +`pnpm ci:full-release --sha ` to create the pinned temporary branch. Use `release_profile` to select live/provider breadth: diff --git a/package.json b/package.json index ca6f45d677a..a4cc7427e53 100644 --- a/package.json +++ b/package.json @@ -1291,6 +1291,7 @@ "check:timed:all-types": "node scripts/check-timed.mjs --include-test-types", "check:timed:architecture": "node scripts/check-timed.mjs --include-architecture", "check:workflows": "node scripts/check-workflows.mjs", + "ci:full-release": "node scripts/full-release-validation-at-sha.mjs", "ci:timings": "node scripts/ci-run-timings.mjs --latest-main", "ci:timings:recent": "node scripts/ci-run-timings.mjs --recent 10", "codex-app-server:protocol:check": "node --import tsx scripts/check-codex-app-server-protocol.ts", diff --git a/scripts/full-release-validation-at-sha.mjs b/scripts/full-release-validation-at-sha.mjs new file mode 100755 index 00000000000..32fe2508dc3 --- /dev/null +++ b/scripts/full-release-validation-at-sha.mjs @@ -0,0 +1,263 @@ +#!/usr/bin/env node +import { execFileSync, spawnSync } from "node:child_process"; + +const WORKFLOW = "full-release-validation.yml"; +const DEFAULT_INPUTS = { + provider: "openai", + mode: "both", + release_profile: "full", + rerun_group: "all", +}; + +function usage() { + console.error(`Usage: node scripts/full-release-validation-at-sha.mjs [--sha ] [--branch ] [--keep-branch] [--dry-run] [-- -f key=value ...] + +Creates a temporary remote branch pinned to the target commit, dispatches Full +Release Validation from that branch, watches the parent run, verifies all child +workflow head SHAs match, then deletes the temporary branch by default.`); +} + +function run(command, args, options = {}) { + if (options.dryRun) { + console.log(["+", command, ...args].join(" ")); + return ""; + } + return execFileSync(command, args, { + encoding: "utf8", + stdio: options.stdio ?? ["ignore", "pipe", "inherit"], + }).trim(); +} + +function runStatus(command, args, options = {}) { + if (options.dryRun) { + console.log(["+", command, ...args].join(" ")); + return { status: 0, stdout: "" }; + } + return spawnSync(command, args, { + encoding: "utf8", + stdio: options.stdio ?? ["ignore", "pipe", "inherit"], + }); +} + +function parseArgs(argv) { + const args = { + sha: "", + branch: "", + keepBranch: false, + dryRun: false, + inputs: { ...DEFAULT_INPUTS }, + }; + + for (let i = 0; i < argv.length; i += 1) { + const arg = argv[i]; + if (arg === "--help" || arg === "-h") { + usage(); + process.exit(0); + } + if (arg === "--sha") { + args.sha = argv[++i] ?? ""; + continue; + } + if (arg === "--branch") { + args.branch = argv[++i] ?? ""; + continue; + } + if (arg === "--keep-branch") { + args.keepBranch = true; + continue; + } + if (arg === "--dry-run") { + args.dryRun = true; + continue; + } + if (arg === "--") { + for (const extra of argv.slice(i + 1)) { + const assignment = extra.startsWith("-f") ? extra.slice(2).trim() : extra; + const [key, ...valueParts] = assignment.split("="); + if (!key || valueParts.length === 0) { + throw new Error(`Unsupported extra argument after --: ${extra}`); + } + args.inputs[key] = valueParts.join("="); + } + break; + } + if (arg === "-f") { + const assignment = argv[++i] ?? ""; + const [key, ...valueParts] = assignment.split("="); + if (!key || valueParts.length === 0) { + throw new Error(`Invalid -f assignment: ${assignment}`); + } + args.inputs[key] = valueParts.join("="); + continue; + } + if (arg.startsWith("-f") && arg.includes("=")) { + const assignment = arg.slice(2).trim(); + const [key, ...valueParts] = assignment.split("="); + if (!key || valueParts.length === 0) { + throw new Error(`Invalid -f assignment: ${arg}`); + } + args.inputs[key] = valueParts.join("="); + continue; + } + throw new Error(`Unknown argument: ${arg}`); + } + + return args; +} + +function sanitizeBranchPart(value) { + return value + .replace(/[^A-Za-z0-9._/-]+/g, "-") + .replace(/\/+/g, "/") + .replace(/^[/.-]+|[/.-]+$/g, "") + .slice(0, 80); +} + +function resolveSha(requestedSha) { + const rev = requestedSha || "HEAD"; + return run("git", ["rev-parse", "--verify", `${rev}^{commit}`], { dryRun: false }); +} + +function collectRunId(dispatchOutput) { + const match = dispatchOutput.match(/actions\/runs\/(\d+)/); + return match?.[1] ?? ""; +} + +function findLatestRunId(branch, sha) { + const json = run("gh", [ + "run", + "list", + "--workflow", + WORKFLOW, + "--branch", + branch, + "--event", + "workflow_dispatch", + "--limit", + "20", + "--json", + "databaseId,headSha,createdAt", + ]); + const runs = JSON.parse(json); + const match = runs.find((runItem) => runItem.headSha === sha); + return match?.databaseId ? String(match.databaseId) : ""; +} + +function childRunIds(parentRunId) { + const jobsJson = run("gh", ["run", "view", parentRunId, "--json", "jobs"]); + const jobs = JSON.parse(jobsJson).jobs ?? []; + const summaryJob = jobs.find((job) => job.name === "Verify full validation"); + if (!summaryJob?.databaseId) { + return []; + } + const log = run("gh", [ + "run", + "view", + parentRunId, + "--job", + String(summaryJob.databaseId), + "--log", + ]); + return [...new Set([...log.matchAll(/actions\/runs\/(\d+)/g)].map((match) => match[1]))]; +} + +function verifyChildHeads(parentRunId, sha) { + const ids = childRunIds(parentRunId); + if (ids.length === 0) { + throw new Error( + `Could not find child workflow run ids in parent verifier logs for ${parentRunId}.`, + ); + } + + let failed = false; + for (const id of ids) { + const json = run("gh", ["run", "view", id, "--json", "name,status,conclusion,headSha,url"]); + const child = JSON.parse(json); + const ok = + child.headSha === sha && child.status === "completed" && child.conclusion === "success"; + console.log( + `${ok ? "ok" : "bad"} ${child.name} ${child.status}/${child.conclusion} ${child.headSha} ${child.url}`, + ); + failed ||= !ok; + } + if (failed) { + throw new Error(`One or more child workflows failed or did not run at ${sha}.`); + } +} + +function main() { + const args = parseArgs(process.argv.slice(2)); + const sha = resolveSha(args.sha); + const shortSha = sha.slice(0, 12); + const branch = sanitizeBranchPart(args.branch || `release-ci/${shortSha}-${Date.now()}`); + const remoteBranchRef = `refs/heads/${branch}`; + const dispatchInputs = { ref: sha, ...args.inputs }; + + console.log(`Target SHA: ${sha}`); + console.log(`Temporary workflow ref: ${branch}`); + + run("git", ["push", "origin", `${sha}:${remoteBranchRef}`], { + dryRun: args.dryRun, + stdio: "inherit", + }); + + let parentRunId = ""; + try { + const dispatchArgs = ["workflow", "run", WORKFLOW, "--ref", branch]; + for (const [key, value] of Object.entries(dispatchInputs)) { + dispatchArgs.push("-f", `${key}=${value}`); + } + + const dispatchOutput = run("gh", dispatchArgs, { dryRun: args.dryRun }); + if (dispatchOutput) { + console.log(dispatchOutput); + } + parentRunId = collectRunId(dispatchOutput); + if (!parentRunId && !args.dryRun) { + for (let attempt = 0; attempt < 60; attempt += 1) { + parentRunId = findLatestRunId(branch, sha); + if (parentRunId) { + break; + } + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 5000); + } + } + if (!parentRunId) { + if (args.dryRun) { + return; + } + throw new Error("Could not determine Full Release Validation run id."); + } + + console.log(`Parent run: https://github.com/openclaw/openclaw/actions/runs/${parentRunId}`); + const watch = runStatus( + "gh", + ["run", "watch", parentRunId, "--exit-status", "--interval", "30"], + { + stdio: "inherit", + }, + ); + if (watch.status !== 0) { + throw new Error( + `Full Release Validation failed: https://github.com/openclaw/openclaw/actions/runs/${parentRunId}`, + ); + } + verifyChildHeads(parentRunId, sha); + } finally { + if (!args.keepBranch) { + run("git", ["push", "origin", `:${remoteBranchRef}`], { + dryRun: args.dryRun, + stdio: "inherit", + }); + } else { + console.log(`Kept ${remoteBranchRef}`); + } + } +} + +try { + main(); +} catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); +} diff --git a/test/scripts/package-acceptance-workflow.test.ts b/test/scripts/package-acceptance-workflow.test.ts index 780e8e81431..6386eb441ff 100644 --- a/test/scripts/package-acceptance-workflow.test.ts +++ b/test/scripts/package-acceptance-workflow.test.ts @@ -138,6 +138,18 @@ describe("package acceptance workflow", () => { expect(workflow).toContain("Published upgrade survivor scenarios:"); }); + it("requires full release child workflows to run at the resolved target SHA", () => { + const workflow = readFileSync(FULL_RELEASE_VALIDATION_WORKFLOW, "utf8"); + + expect(workflow).toContain("TARGET_SHA: ${{ needs.resolve_target.outputs.sha }}"); + expect(workflow).toContain("--json status,conclusion,url,attempt,headSha,jobs"); + expect(workflow).toContain("child run used ${head_sha}, expected ${TARGET_SHA}"); + expect(workflow).toContain( + "Dispatch Full Release Validation from a ref pinned to the target SHA", + ); + expect(workflow).toContain("| Child | Result | Minutes | Head SHA | Run |"); + }); + it("keeps exhaustive update migration as a separate manual package gate", () => { const workflow = readFileSync(UPDATE_MIGRATION_WORKFLOW, "utf8"); const packageWorkflow = readFileSync(PACKAGE_ACCEPTANCE_WORKFLOW, "utf8");