From 494987fa6e7783ff9bdcce8a300d18e4f84445ed Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Thu, 9 Jul 2026 23:47:02 -0700 Subject: [PATCH] fix(ci): require an identical workflow harness for evidence reuse Codex review round three: a prior green run may have executed with different lane definitions; the resolver now compares the candidate run head SHA's .github/workflows tree against the current workflow ref before accepting its manifest. --- .github/workflows/full-release-validation.yml | 1 + .../find-reusable-release-validation.sh | 47 +++++++++++++++--- .../find-reusable-release-validation.test.ts | 48 ++++++++++++++++++- 3 files changed, 88 insertions(+), 8 deletions(-) diff --git a/.github/workflows/full-release-validation.yml b/.github/workflows/full-release-validation.yml index 9cd509903e2d..ad0628c0b53f 100644 --- a/.github/workflows/full-release-validation.yml +++ b/.github/workflows/full-release-validation.yml @@ -303,6 +303,7 @@ jobs: }')" bash workflow/scripts/github/find-reusable-release-validation.sh \ --target-sha "$TARGET_SHA" \ + --workflow-sha "$GITHUB_SHA" \ --release-profile "$RELEASE_PROFILE" \ --run-release-soak "$RUN_RELEASE_SOAK" \ --inputs-json "$inputs_json" \ diff --git a/scripts/github/find-reusable-release-validation.sh b/scripts/github/find-reusable-release-validation.sh index 0bc5bbab19dc..5d72413947ce 100755 --- a/scripts/github/find-reusable-release-validation.sh +++ b/scripts/github/find-reusable-release-validation.sh @@ -9,6 +9,7 @@ set -euo pipefail REPO="${GH_REPO:-}" WORKFLOW_FILE="full-release-validation.yml" TARGET_SHA="" +WORKFLOW_SHA="" RELEASE_PROFILE="" RUN_RELEASE_SOAK="false" INPUTS_JSON="" @@ -21,15 +22,17 @@ PREFLIGHT="${SCRIPT_DIR}/../release-preflight.mjs" usage() { cat >&2 <<'EOF' -Usage: find-reusable-release-validation.sh --target-sha --release-profile \ - --inputs-json [--run-release-soak ] [--repo ] \ - [--repo-dir ] [--workflow ] [--max-candidates ] [--github-output ] +Usage: find-reusable-release-validation.sh --target-sha --workflow-sha \ + --release-profile --inputs-json \ + [--run-release-soak ] [--repo ] [--repo-dir ] \ + [--workflow ] [--max-candidates ] [--github-output ] Scans recent successful Full Release Validation runs for a validation manifest whose targetSha differs from --target-sha only by release metadata paths, whose -recorded lane-selection inputs match --inputs-json exactly, and whose recorded -child runs are still green. Writes reuse=true plus evidence_* outputs when -found; reuse=false otherwise. +recorded lane-selection inputs match --inputs-json exactly, whose harness +(.github/workflows tree at the run's head SHA) matches --workflow-sha, and +whose recorded child runs are still green. Writes reuse=true plus evidence_* +outputs when found; reuse=false otherwise. EOF } @@ -39,6 +42,10 @@ while [[ $# -gt 0 ]]; do TARGET_SHA="${2:-}" shift 2 ;; + --workflow-sha) + WORKFLOW_SHA="${2:-}" + shift 2 + ;; --release-profile) RELEASE_PROFILE="${2:-}" shift 2 @@ -113,6 +120,10 @@ if [[ ! "$TARGET_SHA" =~ ^[0-9a-f]{40}$ ]]; then echo "Expected --target-sha to be a full lowercase commit SHA; got: ${TARGET_SHA}" >&2 exit 2 fi +if [[ ! "$WORKFLOW_SHA" =~ ^[0-9a-f]{40}$ ]]; then + echo "Expected --workflow-sha to be a full lowercase commit SHA; got: ${WORKFLOW_SHA}" >&2 + exit 2 +fi if [[ -z "$REPO" ]]; then echo "Expected --repo or GH_REPO." >&2 exit 2 @@ -134,11 +145,23 @@ if ! (cd "$REPO_DIR" && node "$PREFLIGHT" --macos-versions-only >&2); then no_reuse "target version metadata is inconsistent" fi +# Evidence must come from an identical harness: prior runs may have executed +# with different lane definitions. Compare the .github/workflows tree of the +# candidate run's head SHA against the current workflow ref's tree. +workflows_tree_for() { + gh api "repos/${REPO}/contents/.github/workflows?ref=$1" \ + --jq '[.[] | {path, sha}] | sort_by(.path) | tostring' +} +current_workflows_tree="" +if ! current_workflows_tree="$(workflows_tree_for "$WORKFLOW_SHA")" || [[ -z "$current_workflows_tree" ]]; then + no_reuse "could not resolve the current workflow tree" +fi + runs_json="" if ! runs_json="$( gh api -X GET "repos/${REPO}/actions/workflows/${WORKFLOW_FILE}/runs" \ -F status=success -F event=workflow_dispatch -F per_page="$MAX_CANDIDATES" \ - --jq '[.workflow_runs[] | {id, html_url}]' + --jq '[.workflow_runs[] | {id, html_url, head_sha}]' )"; then no_reuse "could not list prior successful validation runs" fi @@ -154,6 +177,16 @@ trap 'rm -rf "$work_dir"' EXIT for ((index = 0; index < run_count; index += 1)); do run_id="$(jq -r ".[${index}].id" <<< "$runs_json")" run_url="$(jq -r ".[${index}].html_url" <<< "$runs_json")" + run_head_sha="$(jq -r ".[${index}].head_sha // \"\"" <<< "$runs_json")" + + if [[ "$run_head_sha" != "$WORKFLOW_SHA" ]]; then + candidate_workflows_tree="" + if ! candidate_workflows_tree="$(workflows_tree_for "$run_head_sha")" \ + || [[ -z "$candidate_workflows_tree" || "$candidate_workflows_tree" != "$current_workflows_tree" ]]; then + echo "[evidence-reuse] run ${run_id}: harness workflows differ from the current workflow ref; skipping" >&2 + continue + fi + fi artifact_id="" if ! artifact_id="$( diff --git a/test/scripts/find-reusable-release-validation.test.ts b/test/scripts/find-reusable-release-validation.test.ts index 2eff1ee291c0..f74b4be830ba 100644 --- a/test/scripts/find-reusable-release-validation.test.ts +++ b/test/scripts/find-reusable-release-validation.test.ts @@ -98,11 +98,15 @@ fi exec cat "\${fixture}.json" `; +const WORKFLOW_SHA = "f".repeat(40); +const CANDIDATE_HEAD_SHA = "e".repeat(40); + interface FixtureOptions { runId?: string; manifest?: Record; compare?: { base: string; head: string; status: string; files: string[] } | undefined; childRunStates?: Record; + candidateWorkflowsTree?: { path: string; sha: string }[]; } function setUpFixtures(options: FixtureOptions): { fixtures: string; binDir: string } { @@ -121,9 +125,27 @@ function setUpFixtures(options: FixtureOptions): { fixtures: string; binDir: str "repos_openclaw_openclaw_actions_workflows_full-release-validation.yml_runs.json", ), JSON.stringify({ - workflow_runs: [{ id: Number(runId), html_url: `https://example.test/runs/${runId}` }], + workflow_runs: [ + { + id: Number(runId), + html_url: `https://example.test/runs/${runId}`, + head_sha: CANDIDATE_HEAD_SHA, + }, + ], }), ); + const workflowsTree = [{ path: ".github/workflows/full-release-validation.yml", sha: "aa11" }]; + writeFileSync( + join(fixtures, `repos_openclaw_openclaw_contents_.github_workflows_ref=${WORKFLOW_SHA}.json`), + JSON.stringify(workflowsTree), + ); + writeFileSync( + join( + fixtures, + `repos_openclaw_openclaw_contents_.github_workflows_ref=${CANDIDATE_HEAD_SHA}.json`, + ), + JSON.stringify(options.candidateWorkflowsTree ?? workflowsTree), + ); if (options.manifest) { writeFileSync( join(fixtures, `repos_openclaw_openclaw_actions_runs_${runId}_artifacts_per_page=100.json`), @@ -201,6 +223,8 @@ function runResolver(args: { SCRIPT_PATH, "--target-sha", args.targetSha, + "--workflow-sha", + WORKFLOW_SHA, "--release-profile", args.releaseProfile, "--run-release-soak", @@ -417,6 +441,28 @@ describe("scripts/github/find-reusable-release-validation.sh", () => { expect(parseOutput(result.stdout)).toMatchObject({ reuse: "false" }); }); + it("rejects evidence from a run with a different workflow harness", () => { + const { origin, priorSha } = createRepoPair(); + const clone = cloneHead(origin); + const { fixtures, binDir } = setUpFixtures({ + manifest: manifestFor(priorSha), + childRunStates: HEALTHY_CHILDREN, + candidateWorkflowsTree: [ + { path: ".github/workflows/full-release-validation.yml", sha: "bb22" }, + ], + }); + + const result = runResolver({ + repoDir: clone, + targetSha: priorSha, + releaseProfile: "stable", + fixtures, + binDir, + }); + expect(result.status).toBe(0); + expect(parseOutput(result.stdout)).toMatchObject({ reuse: "false" }); + }); + it("rejects targets whose version stamps are internally inconsistent", () => { const { origin, priorSha } = createRepoPair({ plistBuildVersion: "2026061000" }); const clone = cloneHead(origin);