From e98bb5942a611c17542ea0761e60235f79a2837d Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Thu, 9 Jul 2026 23:38:25 -0700 Subject: [PATCH] fix(ci): scope beta advisory to live-provider suites and validate reuse targets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex review round two: the reusable advisory input is global, so beta would also have softened hermetic repo/OpenShell E2E — add a scoped live_advisory input that only covers the live-provider suite jobs and narrow the umbrella fail-fast exclusion to those job names. Evidence reuse now also runs the macOS source-version consistency check against the target (release-preflight --macos-versions-only) so version-stamp deltas cannot reuse evidence while version surfaces disagree. --- .github/workflows/full-release-validation.yml | 15 +++++-- .../openclaw-live-and-e2e-checks-reusable.yml | 19 +++++--- .github/workflows/openclaw-release-checks.yml | 6 ++- .../find-reusable-release-validation.sh | 8 ++++ scripts/release-preflight.mjs | 28 +++++++++++- .../find-reusable-release-validation.test.ts | 43 ++++++++++++++++++- 6 files changed, 104 insertions(+), 15 deletions(-) diff --git a/.github/workflows/full-release-validation.yml b/.github/workflows/full-release-validation.yml index 83449ca716a8..9cd509903e2d 100644 --- a/.github/workflows/full-release-validation.yml +++ b/.github/workflows/full-release-validation.yml @@ -848,10 +848,19 @@ jobs: jq -s '[.[] | select(.status == "completed" and .conclusion != "success" and .conclusion != "skipped")]' )" if [[ "$workflow" == "openclaw-release-checks.yml" && "$RELEASE_PROFILE" == "beta" ]]; then - # Beta treats live-provider suites as advisory; their failures must - # not fail-fast-cancel the remaining release-check matrix. + # Beta treats live-provider suites as advisory (live_advisory in + # openclaw-live-and-e2e-checks-reusable.yml); their failures must + # not fail-fast-cancel the remaining release-check matrix. Repo + # E2E under the same caller stays blocking. failed_jobs_json="$( - jq '[.[] | select(.name | startswith("Run repo/live E2E validation / ") | not)]' <<< "$failed_jobs_json" + jq '[.[] | select( + (.name | startswith("Run repo/live E2E validation / ")) + and ((.name | contains("Docker live")) + or (.name | contains("Live media suites")) + or (.name | contains("validate_live_provider_suites")) + or (.name | contains("validate_release_live_cache")) + or (.name | contains("prepare_live_test_image"))) + | not)]' <<< "$failed_jobs_json" )" fi if jq -e 'length > 0' <<< "$failed_jobs_json" >/dev/null; then diff --git a/.github/workflows/openclaw-live-and-e2e-checks-reusable.yml b/.github/workflows/openclaw-live-and-e2e-checks-reusable.yml index 06cfbff7d277..346f2874994a 100644 --- a/.github/workflows/openclaw-live-and-e2e-checks-reusable.yml +++ b/.github/workflows/openclaw-live-and-e2e-checks-reusable.yml @@ -119,6 +119,11 @@ on: required: false default: false type: boolean + live_advisory: + description: Treat only live-provider suite failures as advisory for the caller; repo and Docker E2E stay blocking + required: false + default: false + type: boolean ref: description: Ref, tag, or SHA to validate required: true @@ -514,7 +519,7 @@ jobs: validate_release_live_cache: needs: validate_selected_ref if: inputs.include_live_suites && !inputs.live_models_only && (inputs.live_suite_filter == '' || inputs.live_suite_filter == 'live-cache') - continue-on-error: ${{ inputs.advisory }} + continue-on-error: ${{ inputs.advisory || inputs.live_advisory }} runs-on: ${{ inputs.use_github_hosted_runners && 'ubuntu-24.04' || 'blacksmith-8vcpu-ubuntu-2404' }} timeout-minutes: 20 env: @@ -1645,7 +1650,7 @@ jobs: prepare_live_test_image: needs: validate_selected_ref if: inputs.include_live_suites && (inputs.live_suite_filter == '' || startsWith(inputs.live_suite_filter, 'live-') || startsWith(inputs.live_suite_filter, 'docker-live-models')) - continue-on-error: ${{ inputs.advisory }} + continue-on-error: ${{ inputs.advisory || inputs.live_advisory }} runs-on: ${{ inputs.use_github_hosted_runners && 'ubuntu-24.04' || 'blacksmith-32vcpu-ubuntu-2404' }} timeout-minutes: 60 permissions: @@ -1721,7 +1726,7 @@ jobs: name: Docker live models (${{ matrix.provider_label }}) needs: [validate_selected_ref, prepare_live_test_image, plan_release_workflow_matrices] if: inputs.include_live_suites && inputs.live_model_providers == '' && (inputs.live_suite_filter == '' || inputs.live_suite_filter == 'docker-live-models') && needs.plan_release_workflow_matrices.outputs.live_models_count != '0' - continue-on-error: ${{ inputs.advisory }} + continue-on-error: ${{ inputs.advisory || inputs.live_advisory }} runs-on: ${{ inputs.use_github_hosted_runners && 'ubuntu-24.04' || 'blacksmith-32vcpu-ubuntu-2404' }} timeout-minutes: 45 strategy: @@ -1849,7 +1854,7 @@ jobs: name: Docker live models (selected providers) needs: [validate_selected_ref, prepare_live_test_image] if: inputs.include_live_suites && inputs.live_model_providers != '' && (inputs.live_suite_filter == '' || inputs.live_suite_filter == 'docker-live-models') - continue-on-error: ${{ inputs.advisory }} + continue-on-error: ${{ inputs.advisory || inputs.live_advisory }} runs-on: ${{ inputs.use_github_hosted_runners && 'ubuntu-24.04' || 'blacksmith-32vcpu-ubuntu-2404' }} timeout-minutes: 45 env: @@ -2024,7 +2029,7 @@ jobs: validate_live_provider_suites: needs: validate_selected_ref if: inputs.include_live_suites && !inputs.live_models_only && (inputs.live_suite_filter == '' || (startsWith(inputs.live_suite_filter, 'native-live-') && !startsWith(inputs.live_suite_filter, 'native-live-extensions-media') && inputs.live_suite_filter != 'native-live-extensions-a-k')) - continue-on-error: ${{ inputs.advisory }} + continue-on-error: ${{ inputs.advisory || inputs.live_advisory }} runs-on: ${{ inputs.use_github_hosted_runners && 'ubuntu-24.04' || 'blacksmith-8vcpu-ubuntu-2404' }} timeout-minutes: ${{ matrix.timeout_minutes }} strategy: @@ -2362,7 +2367,7 @@ jobs: name: Docker live suites (${{ matrix.label }}) needs: [validate_selected_ref, prepare_live_test_image] if: inputs.include_live_suites && !inputs.live_models_only && (inputs.live_suite_filter == '' || startsWith(inputs.live_suite_filter, 'live-')) - continue-on-error: ${{ inputs.advisory }} + continue-on-error: ${{ inputs.advisory || inputs.live_advisory }} runs-on: ${{ inputs.use_github_hosted_runners && 'ubuntu-24.04' || 'blacksmith-32vcpu-ubuntu-2404' }} timeout-minutes: ${{ matrix.timeout_minutes }} strategy: @@ -2599,7 +2604,7 @@ jobs: name: Live media suites (${{ matrix.label }}) needs: validate_selected_ref if: inputs.include_live_suites && !inputs.live_models_only && (inputs.live_suite_filter == '' || startsWith(inputs.live_suite_filter, 'native-live-extensions-media') || inputs.live_suite_filter == 'native-live-extensions-a-k') - continue-on-error: ${{ inputs.advisory }} + continue-on-error: ${{ inputs.advisory || inputs.live_advisory }} runs-on: ${{ inputs.use_github_hosted_runners && 'ubuntu-24.04' || 'blacksmith-8vcpu-ubuntu-2404' }} container: image: ghcr.io/openclaw/openclaw-live-media-runner:ubuntu-24.04 diff --git a/.github/workflows/openclaw-release-checks.yml b/.github/workflows/openclaw-release-checks.yml index b0328f3b1790..6241e7a6bae8 100644 --- a/.github/workflows/openclaw-release-checks.yml +++ b/.github/workflows/openclaw-release-checks.yml @@ -640,9 +640,11 @@ jobs: pull-requests: read uses: ./.github/workflows/openclaw-live-and-e2e-checks-reusable.yml with: + advisory: ${{ startsWith(github.ref, 'refs/heads/tideclaw/alpha/') }} # Live-provider suites depend on third-party model deployments; beta - # treats them as advisory while stable/full keep them blocking. - advisory: ${{ startsWith(github.ref, 'refs/heads/tideclaw/alpha/') || needs.resolve_target.outputs.release_profile == 'beta' }} + # treats only those as advisory while repo E2E stays blocking and + # stable/full keep everything blocking. + live_advisory: ${{ needs.resolve_target.outputs.release_profile == 'beta' }} ref: ${{ needs.resolve_target.outputs.revision }} include_repo_e2e: true include_release_path_suites: false diff --git a/scripts/github/find-reusable-release-validation.sh b/scripts/github/find-reusable-release-validation.sh index 14794d01d92c..0bc5bbab19dc 100755 --- a/scripts/github/find-reusable-release-validation.sh +++ b/scripts/github/find-reusable-release-validation.sh @@ -17,6 +17,7 @@ MAX_CANDIDATES=12 GITHUB_OUTPUT_FILE="${GITHUB_OUTPUT:-}" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" CLASSIFIER="${SCRIPT_DIR}/../check-release-metadata-only.mjs" +PREFLIGHT="${SCRIPT_DIR}/../release-preflight.mjs" usage() { cat >&2 <<'EOF' @@ -126,6 +127,13 @@ if ! expected_inputs="$(jq -Sc . <<< "$INPUTS_JSON" 2>/dev/null)" || [[ -z "$exp exit 2 fi +# A metadata-only diff can still leave the target's version stamps mutually +# inconsistent (for example package.json bumped without the macOS plist); +# validate the target state before trusting any prior evidence. +if ! (cd "$REPO_DIR" && node "$PREFLIGHT" --macos-versions-only >&2); then + no_reuse "target version metadata is inconsistent" +fi + runs_json="" if ! runs_json="$( gh api -X GET "repos/${REPO}/actions/workflows/${WORKFLOW_FILE}/runs" \ diff --git a/scripts/release-preflight.mjs b/scripts/release-preflight.mjs index 7a718e748df6..31820d015161 100644 --- a/scripts/release-preflight.mjs +++ b/scripts/release-preflight.mjs @@ -9,6 +9,20 @@ const parsedArgs = parseArgs(process.argv.slice(2)); const fix = parsedArgs.fix; const macosInfoPlistPath = "apps/macos/Sources/OpenClaw/Resources/Info.plist"; +// Release-evidence reuse validates version-stamp targets without running any +// package-manager commands; keep this mode dependency-free file reads only. +if (parsedArgs.macosVersionsOnly) { + const errors = collectMacosVersionErrors(); + if (errors.length !== 0) { + for (const error of errors) { + console.error(`[release-preflight] macOS app version metadata: ${error}`); + } + process.exit(1); + } + console.log("[release-preflight] macOS app version metadata OK"); + process.exit(0); +} + const fixCommands = [ { name: "plugin versions", args: ["plugins:sync"] }, { name: "npm shrinkwraps", args: ["deps:shrinkwrap:changed:generate"] }, @@ -184,6 +198,7 @@ function printCommandFailures(failures) { function parseArgs(argv) { let check = false; let wantsFix = false; + let macosVersionsOnly = false; for (const arg of argv) { if (arg === "--help") { printUsage(console.log); @@ -197,6 +212,10 @@ function parseArgs(argv) { wantsFix = true; continue; } + if (arg === "--macos-versions-only") { + macosVersionsOnly = true; + continue; + } console.error(`Unknown release preflight argument: ${arg}`); printUsage(console.error); process.exit(1); @@ -205,12 +224,17 @@ function parseArgs(argv) { console.error("Use either --fix or --check, not both."); process.exit(1); } - return { fix: wantsFix }; + if (macosVersionsOnly && (wantsFix || check)) { + console.error("Use --macos-versions-only without --fix or --check."); + process.exit(1); + } + return { fix: wantsFix, macosVersionsOnly }; } function printUsage(writeLine) { - writeLine("Usage: node scripts/release-preflight.mjs [--check|--fix]"); + writeLine("Usage: node scripts/release-preflight.mjs [--check|--fix|--macos-versions-only]"); writeLine(""); writeLine(" --check verify generated release artifacts without writing changes (default)"); writeLine(" --fix refresh generated release artifacts, then verify them"); + writeLine(" --macos-versions-only verify macOS source version metadata only, no commands"); } diff --git a/test/scripts/find-reusable-release-validation.test.ts b/test/scripts/find-reusable-release-validation.test.ts index 08a84540e449..2eff1ee291c0 100644 --- a/test/scripts/find-reusable-release-validation.test.ts +++ b/test/scripts/find-reusable-release-validation.test.ts @@ -28,7 +28,22 @@ function commitFile(repo: string, filePath: string, content: string, message: st return git(repo, ["rev-parse", "HEAD"]); } -function createRepoPair() { +function plistFor(shortVersion: string, buildVersion: string): string { + return [ + '', + '', + "", + " CFBundleShortVersionString", + ` ${shortVersion}`, + " CFBundleVersion", + ` ${buildVersion}`, + "", + "", + "", + ].join("\n"); +} + +function createRepoPair(options: { plistBuildVersion?: string } = {}) { const origin = tempDirs.make("evidence-reuse-origin-"); git(origin, ["init", "-q", "-b", "main"]); git(origin, ["config", "user.email", "test-user"]); @@ -39,6 +54,11 @@ function createRepoPair() { join(origin, "package.json"), `${JSON.stringify({ name: "x", version: "2026.7.1" }, null, 2)}\n`, ); + mkdirSync(join(origin, "apps/macos/Sources/OpenClaw/Resources"), { recursive: true }); + writeFileSync( + join(origin, "apps/macos/Sources/OpenClaw/Resources/Info.plist"), + plistFor("2026.7.1", options.plistBuildVersion ?? "2026070100"), + ); writeFileSync(join(origin, "CHANGELOG.md"), "# Changelog\n"); writeFileSync(join(origin, "index.ts"), "export const value = 1;\n"); git(origin, ["add", "-A"]); @@ -397,6 +417,27 @@ describe("scripts/github/find-reusable-release-validation.sh", () => { expect(parseOutput(result.stdout)).toMatchObject({ reuse: "false" }); }); + it("rejects targets whose version stamps are internally inconsistent", () => { + const { origin, priorSha } = createRepoPair({ plistBuildVersion: "2026061000" }); + const clone = cloneHead(origin); + const { fixtures, binDir } = setUpFixtures({ + manifest: manifestFor(priorSha), + childRunStates: HEALTHY_CHILDREN, + }); + + const result = runResolver({ + repoDir: clone, + targetSha: priorSha, + releaseProfile: "stable", + fixtures, + binDir, + }); + expect(result.status).toBe(0); + const output = parseOutput(result.stdout); + expect(output.reuse).toBe("false"); + expect(output.reuse_reason).toContain("version metadata"); + }); + it("reports no reuse when no prior runs or manifests exist", () => { const { origin, priorSha } = createRepoPair(); const clone = cloneHead(origin);