diff --git a/scripts/check-release-metadata-only.mjs b/scripts/check-release-metadata-only.mjs index a027caf79c8b..92f54611e634 100644 --- a/scripts/check-release-metadata-only.mjs +++ b/scripts/check-release-metadata-only.mjs @@ -91,7 +91,9 @@ function readBeforeAfter(args, filePath) { const refs = refsFor(args); const before = readBlob(refs.before, filePath); let after = readBlob(refs.after, filePath); - if (!args.staged && existsSync(filePath)) { + // The worktree overlay covers uncommitted edits; an explicit --head SHA is + // a request for SHA-exact comparison and must not read the checkout. + if (!args.staged && args.head === "HEAD" && existsSync(filePath)) { const worktree = readBlob("WORKTREE", filePath); if (worktree !== after) { after = worktree; diff --git a/scripts/github/find-reusable-release-validation.sh b/scripts/github/find-reusable-release-validation.sh index 5d72413947ce..3c10ccee2521 100755 --- a/scripts/github/find-reusable-release-validation.sh +++ b/scripts/github/find-reusable-release-validation.sh @@ -145,17 +145,30 @@ if ! (cd "$REPO_DIR" && node "$PREFLIGHT" --macos-versions-only >&2); then no_reuse "target version metadata is inconsistent" fi -# Evidence must come from an identical harness: prior runs may have executed -# with different lane definitions. Compare the .github/workflows tree of the -# candidate run's head SHA against the current workflow ref's tree. -workflows_tree_for() { - gh api "repos/${REPO}/contents/.github/workflows?ref=$1" \ - --jq '[.[] | {path, sha}] | sort_by(.path) | tostring' +# Evidence must come from an equivalent harness: workflows and their helper +# scripts run from the workflow ref, so the tree diff between the candidate +# run's head SHA and the current workflow SHA must itself be metadata-only. +harness_matches() { + local candidate_sha="$1" + if [[ "$candidate_sha" == "$WORKFLOW_SHA" ]]; then + return 0 + fi + if ! git -C "$REPO_DIR" fetch --quiet --depth=1 origin "$candidate_sha" "$WORKFLOW_SHA"; then + return 1 + fi + local harness_paths + if ! harness_paths="$(git -C "$REPO_DIR" diff --name-only "$candidate_sha" "$WORKFLOW_SHA")"; then + return 1 + fi + if [[ -z "$harness_paths" ]]; then + return 0 + fi + local -a harness_path_list=() + while IFS= read -r harness_path; do + [[ -n "$harness_path" ]] && harness_path_list+=("$harness_path") + done <<< "$harness_paths" + (cd "$REPO_DIR" && node "$CLASSIFIER" --base "$candidate_sha" --head "$WORKFLOW_SHA" -- "${harness_path_list[@]}") } -current_workflows_tree="" -if ! current_workflows_tree="$(workflows_tree_for "$WORKFLOW_SHA")" || [[ -z "$current_workflows_tree" ]]; then - no_reuse "could not resolve the current workflow tree" -fi runs_json="" if ! runs_json="$( @@ -179,13 +192,9 @@ for ((index = 0; index < run_count; index += 1)); do run_url="$(jq -r ".[${index}].html_url" <<< "$runs_json")" run_head_sha="$(jq -r ".[${index}].head_sha // \"\"" <<< "$runs_json")" - if [[ "$run_head_sha" != "$WORKFLOW_SHA" ]]; then - candidate_workflows_tree="" - if ! candidate_workflows_tree="$(workflows_tree_for "$run_head_sha")" \ - || [[ -z "$candidate_workflows_tree" || "$candidate_workflows_tree" != "$current_workflows_tree" ]]; then - echo "[evidence-reuse] run ${run_id}: harness workflows differ from the current workflow ref; skipping" >&2 - continue - fi + if [[ ! "$run_head_sha" =~ ^[0-9a-f]{40}$ ]] || ! harness_matches "$run_head_sha"; then + echo "[evidence-reuse] run ${run_id}: harness differs from the current workflow ref beyond release metadata; skipping" >&2 + continue fi artifact_id="" diff --git a/test/scripts/find-reusable-release-validation.test.ts b/test/scripts/find-reusable-release-validation.test.ts index f74b4be830ba..dc1d9c38a493 100644 --- a/test/scripts/find-reusable-release-validation.test.ts +++ b/test/scripts/find-reusable-release-validation.test.ts @@ -98,15 +98,12 @@ fi exec cat "\${fixture}.json" `; -const WORKFLOW_SHA = "f".repeat(40); -const CANDIDATE_HEAD_SHA = "e".repeat(40); - interface FixtureOptions { runId?: string; + headSha: string; manifest?: Record; compare?: { base: string; head: string; status: string; files: string[] } | undefined; childRunStates?: Record; - candidateWorkflowsTree?: { path: string; sha: string }[]; } function setUpFixtures(options: FixtureOptions): { fixtures: string; binDir: string } { @@ -129,23 +126,11 @@ function setUpFixtures(options: FixtureOptions): { fixtures: string; binDir: str { id: Number(runId), html_url: `https://example.test/runs/${runId}`, - head_sha: CANDIDATE_HEAD_SHA, + head_sha: options.headSha, }, ], }), ); - const workflowsTree = [{ path: ".github/workflows/full-release-validation.yml", sha: "aa11" }]; - writeFileSync( - join(fixtures, `repos_openclaw_openclaw_contents_.github_workflows_ref=${WORKFLOW_SHA}.json`), - JSON.stringify(workflowsTree), - ); - writeFileSync( - join( - fixtures, - `repos_openclaw_openclaw_contents_.github_workflows_ref=${CANDIDATE_HEAD_SHA}.json`, - ), - JSON.stringify(options.candidateWorkflowsTree ?? workflowsTree), - ); if (options.manifest) { writeFileSync( join(fixtures, `repos_openclaw_openclaw_actions_runs_${runId}_artifacts_per_page=100.json`), @@ -211,6 +196,7 @@ const DEFAULT_INPUTS = { function runResolver(args: { repoDir: string; targetSha: string; + workflowSha: string; releaseProfile: string; runReleaseSoak?: string; inputs?: Record; @@ -224,7 +210,7 @@ function runResolver(args: { "--target-sha", args.targetSha, "--workflow-sha", - WORKFLOW_SHA, + args.workflowSha, "--release-profile", args.releaseProfile, "--run-release-soak", @@ -289,7 +275,10 @@ describe("scripts/github/find-reusable-release-validation.sh", () => { "docs(changelog): refresh", ); const clone = cloneHead(origin); + // The candidate ran when the branch was at priorSha; the current dispatch + // runs from the branch tip, so the harness delta equals the target delta. const { fixtures, binDir } = setUpFixtures({ + headSha: priorSha, manifest: manifestFor(priorSha), compare: { base: priorSha, head: targetSha, status: "ahead", files: ["CHANGELOG.md"] }, childRunStates: HEALTHY_CHILDREN, @@ -298,6 +287,7 @@ describe("scripts/github/find-reusable-release-validation.sh", () => { const result = runResolver({ repoDir: clone, targetSha, + workflowSha: targetSha, releaseProfile: "stable", fixtures, binDir, @@ -320,6 +310,7 @@ describe("scripts/github/find-reusable-release-validation.sh", () => { const clone = cloneHead(origin); // No compare fixture: an identical target must not hit the compare API. const { fixtures, binDir } = setUpFixtures({ + headSha: priorSha, manifest: manifestFor(priorSha, { evidenceReuse: { runId: "42" } }), childRunStates: HEALTHY_CHILDREN, }); @@ -327,6 +318,7 @@ describe("scripts/github/find-reusable-release-validation.sh", () => { const result = runResolver({ repoDir: clone, targetSha: priorSha, + workflowSha: priorSha, releaseProfile: "stable", fixtures, binDir, @@ -350,13 +342,17 @@ describe("scripts/github/find-reusable-release-validation.sh", () => { ); const clone = cloneHead(origin); const { fixtures, binDir } = setUpFixtures({ + headSha: priorSha, manifest: manifestFor(priorSha), compare: { base: priorSha, head: targetSha, status: "ahead", files: ["index.ts"] }, }); + // The candidate harness matches the pinned workflow SHA; only the target + // delta is non-metadata here. const result = runResolver({ repoDir: clone, targetSha, + workflowSha: priorSha, releaseProfile: "stable", fixtures, binDir, @@ -375,10 +371,14 @@ describe("scripts/github/find-reusable-release-validation.sh", () => { ); const clone = cloneHead(origin); - const beta = setUpFixtures({ manifest: manifestFor(priorSha, { releaseProfile: "beta" }) }); + const beta = setUpFixtures({ + headSha: priorSha, + manifest: manifestFor(priorSha, { releaseProfile: "beta" }), + }); const betaResult = runResolver({ repoDir: clone, targetSha, + workflowSha: priorSha, releaseProfile: "stable", fixtures: beta.fixtures, binDir: beta.binDir, @@ -387,12 +387,14 @@ describe("scripts/github/find-reusable-release-validation.sh", () => { expect(parseOutput(betaResult.stdout)).toMatchObject({ reuse: "false" }); const diverged = setUpFixtures({ + headSha: priorSha, manifest: manifestFor(priorSha), compare: { base: priorSha, head: targetSha, status: "diverged", files: ["CHANGELOG.md"] }, }); const divergedResult = runResolver({ repoDir: clone, targetSha, + workflowSha: priorSha, releaseProfile: "stable", fixtures: diverged.fixtures, binDir: diverged.binDir, @@ -405,6 +407,7 @@ describe("scripts/github/find-reusable-release-validation.sh", () => { const { origin, priorSha } = createRepoPair(); const clone = cloneHead(origin); const { fixtures, binDir } = setUpFixtures({ + headSha: priorSha, manifest: manifestFor(priorSha, { validationInputs: { ...DEFAULT_INPUTS, provider: "anthropic" }, }), @@ -414,6 +417,7 @@ describe("scripts/github/find-reusable-release-validation.sh", () => { const result = runResolver({ repoDir: clone, targetSha: priorSha, + workflowSha: priorSha, releaseProfile: "stable", fixtures, binDir, @@ -426,6 +430,7 @@ describe("scripts/github/find-reusable-release-validation.sh", () => { const { origin, priorSha } = createRepoPair(); const clone = cloneHead(origin); const { fixtures, binDir } = setUpFixtures({ + headSha: priorSha, manifest: manifestFor(priorSha), childRunStates: { "201": "completed/failure", "202": "completed/success" }, }); @@ -433,6 +438,7 @@ describe("scripts/github/find-reusable-release-validation.sh", () => { const result = runResolver({ repoDir: clone, targetSha: priorSha, + workflowSha: priorSha, releaseProfile: "stable", fixtures, binDir, @@ -441,20 +447,27 @@ describe("scripts/github/find-reusable-release-validation.sh", () => { expect(parseOutput(result.stdout)).toMatchObject({ reuse: "false" }); }); - it("rejects evidence from a run with a different workflow harness", () => { + it("rejects evidence whose harness differs beyond release metadata", () => { const { origin, priorSha } = createRepoPair(); + git(origin, ["checkout", "-q", "-b", "harness-drift"]); + const driftSha = commitFile( + origin, + "index.ts", + "export const value = 3;\n", + "ci: change harness logic", + ); + git(origin, ["checkout", "-q", "main"]); const clone = cloneHead(origin); const { fixtures, binDir } = setUpFixtures({ + headSha: driftSha, manifest: manifestFor(priorSha), childRunStates: HEALTHY_CHILDREN, - candidateWorkflowsTree: [ - { path: ".github/workflows/full-release-validation.yml", sha: "bb22" }, - ], }); const result = runResolver({ repoDir: clone, targetSha: priorSha, + workflowSha: priorSha, releaseProfile: "stable", fixtures, binDir, @@ -467,6 +480,7 @@ describe("scripts/github/find-reusable-release-validation.sh", () => { const { origin, priorSha } = createRepoPair({ plistBuildVersion: "2026061000" }); const clone = cloneHead(origin); const { fixtures, binDir } = setUpFixtures({ + headSha: priorSha, manifest: manifestFor(priorSha), childRunStates: HEALTHY_CHILDREN, }); @@ -474,6 +488,7 @@ describe("scripts/github/find-reusable-release-validation.sh", () => { const result = runResolver({ repoDir: clone, targetSha: priorSha, + workflowSha: priorSha, releaseProfile: "stable", fixtures, binDir, @@ -487,11 +502,12 @@ describe("scripts/github/find-reusable-release-validation.sh", () => { it("reports no reuse when no prior runs or manifests exist", () => { const { origin, priorSha } = createRepoPair(); const clone = cloneHead(origin); - const { fixtures, binDir } = setUpFixtures({}); + const { fixtures, binDir } = setUpFixtures({ headSha: priorSha }); const result = runResolver({ repoDir: clone, targetSha: priorSha, + workflowSha: priorSha, releaseProfile: "beta", fixtures, binDir,