diff --git a/.github/workflows/full-release-validation.yml b/.github/workflows/full-release-validation.yml index f5ad531e1775..83449ca716a8 100644 --- a/.github/workflows/full-release-validation.yml +++ b/.github/workflows/full-release-validation.yml @@ -273,11 +273,39 @@ jobs: TARGET_SHA: ${{ needs.resolve_target.outputs.sha }} RELEASE_PROFILE: ${{ inputs.release_profile }} RUN_RELEASE_SOAK: ${{ inputs.run_release_soak || inputs.release_profile == 'stable' || inputs.release_profile == 'full' }} + PROVIDER: ${{ inputs.provider }} + MODE: ${{ inputs.mode }} + LIVE_SUITE_FILTER: ${{ inputs.live_suite_filter }} + CROSS_OS_SUITE_FILTER: ${{ inputs.cross_os_suite_filter }} + RELEASE_PACKAGE_SPEC: ${{ inputs.release_package_spec }} + PACKAGE_ACCEPTANCE_PACKAGE_SPEC: ${{ inputs.package_acceptance_package_spec }} + CODEX_PLUGIN_SPEC: ${{ inputs.codex_plugin_spec }} run: | + set -euo pipefail + # Lane-selection inputs must match the prior run's manifest exactly; + # a default-input run must not stand in for a focused one. + inputs_json="$(jq -nc \ + --arg provider "$PROVIDER" \ + --arg mode "$MODE" \ + --arg liveSuiteFilter "$LIVE_SUITE_FILTER" \ + --arg crossOsSuiteFilter "$CROSS_OS_SUITE_FILTER" \ + --arg releasePackageSpec "$RELEASE_PACKAGE_SPEC" \ + --arg packageAcceptancePackageSpec "$PACKAGE_ACCEPTANCE_PACKAGE_SPEC" \ + --arg codexPluginSpec "$CODEX_PLUGIN_SPEC" \ + '{ + provider: $provider, + mode: $mode, + liveSuiteFilter: $liveSuiteFilter, + crossOsSuiteFilter: $crossOsSuiteFilter, + releasePackageSpec: $releasePackageSpec, + packageAcceptancePackageSpec: $packageAcceptancePackageSpec, + codexPluginSpec: $codexPluginSpec + }')" bash workflow/scripts/github/find-reusable-release-validation.sh \ --target-sha "$TARGET_SHA" \ --release-profile "$RELEASE_PROFILE" \ --run-release-soak "$RUN_RELEASE_SOAK" \ + --inputs-json "$inputs_json" \ --repo "$GITHUB_REPOSITORY" \ --repo-dir target \ --github-output "$GITHUB_OUTPUT" @@ -1277,6 +1305,7 @@ jobs: EVIDENCE_ROOT_RUN_ID: ${{ needs.evidence_reuse.outputs.evidence_root_run_id }} EVIDENCE_RUN_URL: ${{ needs.evidence_reuse.outputs.evidence_run_url }} EVIDENCE_SHA: ${{ needs.evidence_reuse.outputs.evidence_sha }} + EVIDENCE_MANIFEST: ${{ needs.evidence_reuse.outputs.evidence_manifest }} RERUN_GROUP: ${{ inputs.rerun_group }} TARGET_SHA: ${{ needs.resolve_target.outputs.sha }} CHILD_WORKFLOW_REF: ${{ github.ref_name }} @@ -1538,12 +1567,22 @@ jobs: performance_required=0 if [[ "$RERUN_GROUP" == "all" && "$EVIDENCE_REUSE" == "true" ]]; then # Lanes were skipped because a prior green validation covers this - # target; re-verify the chain-root run so stale evidence cannot pass. + # target; re-verify the chain-root run and its recorded child runs + # so evidence that went stale after resolution cannot pass. evidence_state="$(gh_with_retry run view "$EVIDENCE_ROOT_RUN_ID" --json status,conclusion --jq '(.status // "") + "/" + (.conclusion // "")')" if [[ "$evidence_state" != "completed/success" ]]; then echo "::error::Reused evidence run ${EVIDENCE_ROOT_RUN_ID} is ${evidence_state}; evidence is no longer valid." failed=1 - else + fi + while IFS= read -r evidence_child_run_id; do + [[ -n "$evidence_child_run_id" ]] || continue + evidence_child_state="$(gh_with_retry run view "$evidence_child_run_id" --json status,conclusion --jq '(.status // "") + "/" + (.conclusion // "")')" + if [[ "$evidence_child_state" != "completed/success" ]]; then + echo "::error::Reused evidence child run ${evidence_child_run_id} is ${evidence_child_state}; evidence is no longer valid." + failed=1 + fi + done < <(jq -r '[.childRuns.normalCi // "", .childRuns.pluginPrerelease // "", .childRuns.releaseChecks // "", .childRuns.npmTelegram // "", (.childRuns.productPerformance.runId // "")] | map(select(. != "")) | .[]' <<< "$EVIDENCE_MANIFEST") + if [[ "$failed" == "0" ]]; then { echo "### Reused validation evidence" echo @@ -1644,9 +1683,12 @@ jobs: echo "Release checks were skipped by rerun group; skipping automatic release evidence update." exit 0 fi - # In reuse mode the child runs live on the chain-root validation run, + # In reuse mode the child runs live on the chain-root validation run + # (the evidence consumer scrapes dispatch logs from the given run), # so durable evidence must reference that run id, not this wrapper. + notes="Automatically requested by Full Release Validation ${GITHUB_RUN_ID_VALUE} after child workflows completed; the parent summary re-checks current child run conclusions." if [[ "$EVIDENCE_REUSE" == "true" && -n "${EVIDENCE_ROOT_RUN_ID// }" ]]; then + notes="Automatically requested by Full Release Validation ${GITHUB_RUN_ID_VALUE}, which reused green evidence from chain-root run ${EVIDENCE_ROOT_RUN_ID}; the release target differs from the validated SHA only by release metadata." GITHUB_RUN_ID_VALUE="$EVIDENCE_ROOT_RUN_ID" fi if [[ -z "${RELEASES_DISPATCH_TOKEN// }" ]]; then @@ -1679,7 +1721,7 @@ jobs: --arg release_id "$release_id" \ --arg release_ref "$TARGET_REF" \ --arg package_spec "$evidence_package_spec" \ - --arg notes "Automatically requested by Full Release Validation ${GITHUB_RUN_ID_VALUE} after child workflows completed; the parent summary re-checks current child run conclusions." \ + --arg notes "$notes" \ '{ event_type: "openclaw_full_release_validation_completed", client_payload: { @@ -1728,6 +1770,13 @@ jobs: EVIDENCE_REUSE: ${{ needs.evidence_reuse.outputs.reuse }} EVIDENCE_ROOT_RUN_ID: ${{ needs.evidence_reuse.outputs.evidence_root_run_id }} EVIDENCE_MANIFEST: ${{ needs.evidence_reuse.outputs.evidence_manifest }} + PROVIDER: ${{ inputs.provider }} + MODE: ${{ inputs.mode }} + LIVE_SUITE_FILTER: ${{ inputs.live_suite_filter }} + CROSS_OS_SUITE_FILTER: ${{ inputs.cross_os_suite_filter }} + RELEASE_PACKAGE_SPEC: ${{ inputs.release_package_spec }} + PACKAGE_ACCEPTANCE_PACKAGE_SPEC: ${{ inputs.package_acceptance_package_spec }} + CODEX_PLUGIN_SPEC: ${{ inputs.codex_plugin_spec }} run: | set -euo pipefail manifest_dir="${RUNNER_TEMP}/full-release-validation" @@ -1768,6 +1817,13 @@ jobs: --arg npmTelegramRunId "$NPM_TELEGRAM_RUN_ID" \ --arg performanceRunId "$PERFORMANCE_RUN_ID" \ --arg performanceConclusion "$PERFORMANCE_CONCLUSION" \ + --arg provider "$PROVIDER" \ + --arg mode "$MODE" \ + --arg liveSuiteFilter "$LIVE_SUITE_FILTER" \ + --arg crossOsSuiteFilter "$CROSS_OS_SUITE_FILTER" \ + --arg releasePackageSpec "$RELEASE_PACKAGE_SPEC" \ + --arg packageAcceptancePackageSpec "$PACKAGE_ACCEPTANCE_PACKAGE_SPEC" \ + --arg codexPluginSpec "$CODEX_PLUGIN_SPEC" \ '{ version: 2, workflowName: $workflowName, @@ -1779,6 +1835,15 @@ jobs: releaseProfile: $releaseProfile, rerunGroup: $rerunGroup, runReleaseSoak: $runReleaseSoak, + validationInputs: { + provider: $provider, + mode: $mode, + liveSuiteFilter: $liveSuiteFilter, + crossOsSuiteFilter: $crossOsSuiteFilter, + releasePackageSpec: $releasePackageSpec, + packageAcceptancePackageSpec: $packageAcceptancePackageSpec, + codexPluginSpec: $codexPluginSpec + }, controls: { stableSoakRequired: ($releaseProfile == "stable" or $releaseProfile == "full"), performanceBlocking: true diff --git a/scripts/github/find-reusable-release-validation.sh b/scripts/github/find-reusable-release-validation.sh index 3219cd981046..14794d01d92c 100755 --- a/scripts/github/find-reusable-release-validation.sh +++ b/scripts/github/find-reusable-release-validation.sh @@ -11,6 +11,7 @@ WORKFLOW_FILE="full-release-validation.yml" TARGET_SHA="" RELEASE_PROFILE="" RUN_RELEASE_SOAK="false" +INPUTS_JSON="" REPO_DIR="." MAX_CANDIDATES=12 GITHUB_OUTPUT_FILE="${GITHUB_OUTPUT:-}" @@ -20,12 +21,14 @@ CLASSIFIER="${SCRIPT_DIR}/../check-release-metadata-only.mjs" usage() { cat >&2 <<'EOF' Usage: find-reusable-release-validation.sh --target-sha --release-profile \ - [--run-release-soak ] [--repo ] [--repo-dir ] \ - [--workflow ] [--max-candidates ] [--github-output ] + --inputs-json [--run-release-soak ] [--repo ] \ + [--repo-dir ] [--workflow ] [--max-candidates ] [--github-output ] Scans recent successful Full Release Validation runs for a validation manifest -whose targetSha differs from --target-sha only by release metadata paths. -Writes reuse=true plus evidence_* outputs when found; reuse=false otherwise. +whose targetSha differs from --target-sha only by release metadata paths, whose +recorded lane-selection inputs match --inputs-json exactly, and whose recorded +child runs are still green. Writes reuse=true plus evidence_* outputs when +found; reuse=false otherwise. EOF } @@ -43,6 +46,10 @@ while [[ $# -gt 0 ]]; do RUN_RELEASE_SOAK="${2:-}" shift 2 ;; + --inputs-json) + INPUTS_JSON="${2:-}" + shift 2 + ;; --repo) REPO="${2:-}" shift 2 @@ -113,6 +120,11 @@ current_rank="$(profile_rank "$RELEASE_PROFILE")" if [[ "$current_rank" == "0" ]]; then no_reuse "unknown release profile ${RELEASE_PROFILE}" fi +expected_inputs="" +if ! expected_inputs="$(jq -Sc . <<< "$INPUTS_JSON" 2>/dev/null)" || [[ -z "$expected_inputs" ]]; then + echo "Expected --inputs-json to be a JSON object of lane-selection inputs." >&2 + exit 2 +fi runs_json="" if ! runs_json="$( @@ -174,6 +186,14 @@ for ((index = 0; index < run_count; index += 1)); do echo "[evidence-reuse] run ${run_id}: profile ${prior_profile} does not cover ${RELEASE_PROFILE}; skipping" >&2 continue fi + # Lane selection (provider, mode, filters, package specs) changes what the + # prior run proved; only exact-match manifests are reusable. Manifests + # written before validationInputs existed never match. + manifest_inputs="$(jq -Sc '.validationInputs // empty' "$manifest_path")" + if [[ -z "$manifest_inputs" || "$manifest_inputs" != "$expected_inputs" ]]; then + echo "[evidence-reuse] run ${run_id}: validation inputs differ from the current request; skipping" >&2 + continue + fi prior_soak="$(jq -r '.runReleaseSoak // "false"' "$manifest_path")" if [[ "$RUN_RELEASE_SOAK" == "true" && "$prior_soak" != "true" ]]; then echo "[evidence-reuse] run ${run_id}: no soak evidence; skipping" >&2 @@ -228,6 +248,27 @@ for ((index = 0; index < run_count; index += 1)); do fi fi + # Recorded child runs can be re-run to failure after the parent stays green; + # reuse only evidence whose children are still completed/success, matching + # the recheck the normal summary performs on its own children. + children_healthy=1 + while IFS= read -r child_run_id; do + [[ -n "$child_run_id" ]] || continue + if ! child_state="$(gh api "repos/${REPO}/actions/runs/${child_run_id}" --jq '(.status // "") + "/" + (.conclusion // "")')"; then + echo "[evidence-reuse] run ${run_id}: could not verify child run ${child_run_id}; skipping" >&2 + children_healthy=0 + break + fi + if [[ "$child_state" != "completed/success" ]]; then + echo "[evidence-reuse] run ${run_id}: child run ${child_run_id} is ${child_state}; skipping" >&2 + children_healthy=0 + break + fi + done < <(jq -r '[.childRuns.normalCi // "", .childRuns.pluginPrerelease // "", .childRuns.releaseChecks // "", .childRuns.npmTelegram // "", (.childRuns.productPerformance.runId // "")] | map(select(. != "")) | .[]' "$manifest_path") + if [[ "$children_healthy" != "1" ]]; then + continue + fi + # A reused run may itself be a reuse manifest; evidenceReuse.runId points at # the chain root that actually executed the lanes. evidence_root_run_id="$(jq -r '.evidenceReuse.runId // empty' "$manifest_path")" diff --git a/test/scripts/find-reusable-release-validation.test.ts b/test/scripts/find-reusable-release-validation.test.ts index 54e49c6c872a..08a84540e449 100644 --- a/test/scripts/find-reusable-release-validation.test.ts +++ b/test/scripts/find-reusable-release-validation.test.ts @@ -82,6 +82,7 @@ interface FixtureOptions { runId?: string; manifest?: Record; compare?: { base: string; head: string; status: string; files: string[] } | undefined; + childRunStates?: Record; } function setUpFixtures(options: FixtureOptions): { fixtures: string; binDir: string } { @@ -145,14 +146,32 @@ function setUpFixtures(options: FixtureOptions): { fixtures: string; binDir: str }), ); } + for (const [childRunId, state] of Object.entries(options.childRunStates ?? {})) { + const [status, conclusion] = state.split("/"); + writeFileSync( + join(fixtures, `repos_openclaw_openclaw_actions_runs_${childRunId}.json`), + JSON.stringify({ status, conclusion }), + ); + } return { fixtures, binDir }; } +const DEFAULT_INPUTS = { + provider: "openai", + mode: "both", + liveSuiteFilter: "", + crossOsSuiteFilter: "", + releasePackageSpec: "", + packageAcceptancePackageSpec: "", + codexPluginSpec: "", +}; + function runResolver(args: { repoDir: string; targetSha: string; releaseProfile: string; runReleaseSoak?: string; + inputs?: Record; fixtures: string; binDir: string; }) { @@ -166,6 +185,8 @@ function runResolver(args: { args.releaseProfile, "--run-release-soak", args.runReleaseSoak ?? "false", + "--inputs-json", + JSON.stringify(args.inputs ?? DEFAULT_INPUTS), "--repo", "openclaw/openclaw", "--repo-dir", @@ -206,11 +227,14 @@ function manifestFor(targetSha: string, overrides: Record = {}) releaseProfile: "stable", runReleaseSoak: "true", targetSha, - childRuns: { normalCi: "1" }, + validationInputs: DEFAULT_INPUTS, + childRuns: { normalCi: "201", productPerformance: { runId: "202" } }, ...overrides, }; } +const HEALTHY_CHILDREN = { "201": "completed/success", "202": "completed/success" }; + describe("scripts/github/find-reusable-release-validation.sh", () => { it("reuses evidence when the delta is release-metadata-only", () => { const { origin, priorSha } = createRepoPair(); @@ -224,6 +248,7 @@ describe("scripts/github/find-reusable-release-validation.sh", () => { const { fixtures, binDir } = setUpFixtures({ manifest: manifestFor(priorSha), compare: { base: priorSha, head: targetSha, status: "ahead", files: ["CHANGELOG.md"] }, + childRunStates: HEALTHY_CHILDREN, }); const result = runResolver({ @@ -252,6 +277,7 @@ describe("scripts/github/find-reusable-release-validation.sh", () => { // No compare fixture: an identical target must not hit the compare API. const { fixtures, binDir } = setUpFixtures({ manifest: manifestFor(priorSha, { evidenceReuse: { runId: "42" } }), + childRunStates: HEALTHY_CHILDREN, }); const result = runResolver({ @@ -331,6 +357,46 @@ describe("scripts/github/find-reusable-release-validation.sh", () => { expect(parseOutput(divergedResult.stdout)).toMatchObject({ reuse: "false" }); }); + it("rejects evidence recorded for different lane-selection inputs", () => { + const { origin, priorSha } = createRepoPair(); + const clone = cloneHead(origin); + const { fixtures, binDir } = setUpFixtures({ + manifest: manifestFor(priorSha, { + validationInputs: { ...DEFAULT_INPUTS, provider: "anthropic" }, + }), + childRunStates: HEALTHY_CHILDREN, + }); + + const result = runResolver({ + repoDir: clone, + targetSha: priorSha, + releaseProfile: "stable", + fixtures, + binDir, + }); + expect(result.status).toBe(0); + expect(parseOutput(result.stdout)).toMatchObject({ reuse: "false" }); + }); + + it("rejects evidence whose recorded child runs are no longer green", () => { + const { origin, priorSha } = createRepoPair(); + const clone = cloneHead(origin); + const { fixtures, binDir } = setUpFixtures({ + manifest: manifestFor(priorSha), + childRunStates: { "201": "completed/failure", "202": "completed/success" }, + }); + + const result = runResolver({ + repoDir: clone, + targetSha: priorSha, + releaseProfile: "stable", + fixtures, + binDir, + }); + expect(result.status).toBe(0); + expect(parseOutput(result.stdout)).toMatchObject({ reuse: "false" }); + }); + it("reports no reuse when no prior runs or manifests exist", () => { const { origin, priorSha } = createRepoPair(); const clone = cloneHead(origin); diff --git a/test/scripts/package-acceptance-workflow.test.ts b/test/scripts/package-acceptance-workflow.test.ts index 5f16e423dba7..003dc7046fa6 100644 --- a/test/scripts/package-acceptance-workflow.test.ts +++ b/test/scripts/package-acceptance-workflow.test.ts @@ -716,7 +716,7 @@ describe("package acceptance workflow", () => { const workflow = readFileSync(FULL_RELEASE_VALIDATION_WORKFLOW, "utf8"); const retryCalls = workflow.split("\n").filter((line) => line.includes("gh_with_retry ")); - expect(retryCalls).toHaveLength(29); + expect(retryCalls).toHaveLength(30); for (const call of retryCalls) { expect(call).toMatch(/gh_with_retry (api|run view)/u); }