diff --git a/.github/workflows/full-release-validation.yml b/.github/workflows/full-release-validation.yml index 6fdc5b32a59..b053dda59bb 100644 --- a/.github/workflows/full-release-validation.yml +++ b/.github/workflows/full-release-validation.yml @@ -883,6 +883,54 @@ jobs: } >> "$GITHUB_STEP_SUMMARY" } + summarize_failed_child() { + local label="$1" + local run_id="$2" + if [[ -z "${run_id// }" ]]; then + return 0 + fi + + local run_json status conclusion artifacts_json + run_json="$(gh run view "$run_id" --json status,conclusion,url,jobs)" + status="$(jq -r '.status' <<< "$run_json")" + conclusion="$(jq -r '.conclusion' <<< "$run_json")" + if [[ "$status" == "completed" && "$conclusion" == "success" ]]; then + return 0 + fi + + { + echo + echo "### Failed child detail: ${label}" + echo + jq -r ' + "- Run: " + (.url // ""), + "- Result: `" + (.status // "") + "/" + (.conclusion // "") + "`", + "", + "Failed jobs:", + (.jobs[] + | select(.conclusion != "success" and .conclusion != "skipped") + | "- `" + (.name | gsub("`"; "\\`")) + "`: `" + ((.conclusion // .status // "") | tostring) + "` " + (.url // "")) + ' <<< "$run_json" || true + echo + echo "Artifacts:" + artifacts_json="$( + gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${run_id}/artifacts?per_page=100" 2>/dev/null || true + )" + if [[ -n "${artifacts_json// }" ]]; then + jq -r ' + if ((.artifacts // []) | length) == 0 then + "- none" + else + (.artifacts[] + | "- `" + (.name | gsub("`"; "\\`")) + "` (" + ((.size_in_bytes // 0) | tostring) + " bytes)") + end + ' <<< "$artifacts_json" || echo "- unable to list artifacts" + else + echo "- unable to list artifacts" + fi + } >> "$GITHUB_STEP_SUMMARY" + } + failed=0 append_child_overview @@ -916,6 +964,13 @@ jobs: summarize_child_timing "release_checks" "$RELEASE_CHECKS_RUN_ID" summarize_child_timing "npm_telegram" "$NPM_TELEGRAM_RUN_ID" + if [[ "$failed" != "0" ]]; then + summarize_failed_child "normal_ci" "$NORMAL_CI_RUN_ID" + summarize_failed_child "plugin_prerelease" "$PLUGIN_PRERELEASE_RUN_ID" + summarize_failed_child "release_checks" "$RELEASE_CHECKS_RUN_ID" + summarize_failed_child "npm_telegram" "$NPM_TELEGRAM_RUN_ID" + fi + exit "$failed" - name: Request private evidence update diff --git a/package.json b/package.json index a83f22b1257..fc131e5cdd1 100644 --- a/package.json +++ b/package.json @@ -1557,6 +1557,7 @@ "qa:telegram-user:crabbox": "node --import tsx scripts/e2e/telegram-user-crabbox-proof.ts", "release-metadata:check": "node scripts/check-release-metadata-only.mjs", "release:beta-smoke": "node --import tsx scripts/release-beta-smoke.ts", + "release:candidate": "node scripts/release-candidate-checklist.mjs", "release:check": "pnpm release:generated:check && node --import tsx scripts/release-check.ts", "release:fast-pretag-check": "bash scripts/release-fast-pretag-check.sh", "release:generated:check": "node scripts/release-preflight.mjs --check", diff --git a/scripts/release-candidate-checklist.mjs b/scripts/release-candidate-checklist.mjs new file mode 100644 index 00000000000..72ffa03d851 --- /dev/null +++ b/scripts/release-candidate-checklist.mjs @@ -0,0 +1,542 @@ +#!/usr/bin/env node +import { spawnSync } from "node:child_process"; +import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { basename, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const DEFAULT_REPO = "openclaw/openclaw"; +const DEFAULT_PROVIDER = "openai"; +const DEFAULT_MODE = "both"; +const DEFAULT_RELEASE_PROFILE = "beta"; +const DEFAULT_NPM_DIST_TAG = "beta"; +const DEFAULT_PLUGIN_SCOPE = "all-publishable"; + +function usage() { + return `Usage: pnpm release:candidate -- --tag vYYYY.M.D-beta.N [options] + +Dispatches or consumes release validation runs, validates the prepared npm tarball, +builds plugin publish plans, writes a green evidence bundle, then prints the exact +OpenClaw Release Publish command. + +Options: + --tag Release tag to validate. + --workflow-ref Workflow branch/ref. Default: current branch. + --repo GitHub repo. Default: ${DEFAULT_REPO} + --full-release-run Reuse successful Full Release Validation run. + --npm-preflight-run Reuse successful OpenClaw NPM Release preflight run. + --skip-dispatch Require both run ids; do not dispatch workflows. + --skip-parallels Do not run local Parallels fresh/update beta smoke. + --provider Full validation provider. Default: ${DEFAULT_PROVIDER} + --mode Full validation cross-OS mode. Default: ${DEFAULT_MODE} + --release-profile Default: ${DEFAULT_RELEASE_PROFILE} + --npm-dist-tag Default: ${DEFAULT_NPM_DIST_TAG} + --plugin-publish-scope selected|all-publishable. Default: ${DEFAULT_PLUGIN_SCOPE} + --plugins Required when plugin scope is selected. + --output-dir Evidence output dir. Default: .artifacts/release-candidate/ +`; +} + +function requireValue(argv, index, flag) { + const value = argv[index]; + if (!value || value.startsWith("-")) { + throw new Error(`${flag} requires a value`); + } + return value; +} + +export function parseArgs(argv) { + const options = { + repo: DEFAULT_REPO, + provider: DEFAULT_PROVIDER, + mode: DEFAULT_MODE, + releaseProfile: DEFAULT_RELEASE_PROFILE, + npmDistTag: DEFAULT_NPM_DIST_TAG, + pluginPublishScope: DEFAULT_PLUGIN_SCOPE, + plugins: "", + skipDispatch: false, + skipParallels: false, + tag: "", + workflowRef: "", + fullReleaseRunId: "", + npmPreflightRunId: "", + outputDir: "", + }; + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + switch (arg) { + case "--": + break; + case "--tag": + options.tag = requireValue(argv, ++index, arg); + break; + case "--workflow-ref": + options.workflowRef = requireValue(argv, ++index, arg); + break; + case "--repo": + options.repo = requireValue(argv, ++index, arg); + break; + case "--full-release-run": + options.fullReleaseRunId = requireValue(argv, ++index, arg); + break; + case "--npm-preflight-run": + options.npmPreflightRunId = requireValue(argv, ++index, arg); + break; + case "--skip-dispatch": + options.skipDispatch = true; + break; + case "--skip-parallels": + options.skipParallels = true; + break; + case "--provider": + options.provider = requireValue(argv, ++index, arg); + break; + case "--mode": + options.mode = requireValue(argv, ++index, arg); + break; + case "--release-profile": + options.releaseProfile = requireValue(argv, ++index, arg); + break; + case "--npm-dist-tag": + options.npmDistTag = requireValue(argv, ++index, arg); + break; + case "--plugin-publish-scope": + options.pluginPublishScope = requireValue(argv, ++index, arg); + break; + case "--plugins": + options.plugins = requireValue(argv, ++index, arg); + break; + case "--output-dir": + options.outputDir = requireValue(argv, ++index, arg); + break; + case "-h": + case "--help": + process.stdout.write(usage()); + process.exit(0); + default: + throw new Error(`unknown option: ${arg}`); + } + } + if (!options.tag) { + throw new Error("--tag is required"); + } + if (options.skipDispatch && (!options.fullReleaseRunId || !options.npmPreflightRunId)) { + throw new Error("--skip-dispatch requires --full-release-run and --npm-preflight-run"); + } + if (options.pluginPublishScope === "selected" && !options.plugins.trim()) { + throw new Error("--plugin-publish-scope selected requires --plugins"); + } + if (options.pluginPublishScope === "all-publishable" && options.plugins.trim()) { + throw new Error("--plugins is only valid with --plugin-publish-scope selected"); + } + return options; +} + +function run(command, args, options = {}) { + const result = spawnSync(command, args, { + cwd: options.cwd, + encoding: "utf8", + stdio: options.capture ? ["ignore", "pipe", "pipe"] : "inherit", + }); + if (result.status !== 0) { + throw new Error( + `${command} ${args.join(" ")} failed with ${result.status ?? result.signal}\n${result.stderr ?? ""}`, + ); + } + return result.stdout ?? ""; +} + +function readJson(path, label) { + try { + return JSON.parse(readFileSync(path, "utf8")); + } catch (error) { + throw new Error( + `${label} is invalid JSON: ${error instanceof Error ? error.message : String(error)}`, + ); + } +} + +function currentBranch() { + return run("git", ["branch", "--show-current"], { capture: true }).trim(); +} + +function gitRevParse(ref) { + return run("git", ["rev-parse", ref], { capture: true }).trim(); +} + +function workflowRuns(repo) { + return JSON.parse( + run( + "gh", + [ + "run", + "list", + "--repo", + repo, + "--limit", + "100", + "--json", + "databaseId,workflowName,event,createdAt", + ], + { capture: true }, + ), + ); +} + +function beforeRunIds(repo, workflowName) { + return new Set( + workflowRuns(repo) + .filter((run) => run.workflowName === workflowName && run.event === "workflow_dispatch") + .map((run) => String(run.databaseId)), + ); +} + +async function wait(ms) { + await new Promise((resolve) => setTimeout(resolve, ms)); +} + +async function findNewRunId(repo, workflowName, beforeIds) { + for (let attempt = 0; attempt < 60; attempt += 1) { + const match = workflowRuns(repo) + .filter( + (run) => + run.workflowName === workflowName && + run.event === "workflow_dispatch" && + !beforeIds.has(String(run.databaseId)), + ) + .toSorted((a, b) => String(b.createdAt ?? "").localeCompare(String(a.createdAt ?? "")))[0]; + if (match?.databaseId) { + return String(match.databaseId); + } + await wait(5_000); + } + throw new Error(`could not find dispatched ${workflowName} run`); +} + +function dispatchWorkflow(repo, workflowFile, workflowRef, fields) { + const args = ["workflow", "run", workflowFile, "--repo", repo, "--ref", workflowRef]; + for (const [key, value] of Object.entries(fields)) { + args.push("-f", `${key}=${value}`); + } + run("gh", args); +} + +function runInfo(repo, runId) { + return JSON.parse( + run( + "gh", + [ + "run", + "view", + runId, + "--repo", + repo, + "--json", + "databaseId,workflowName,headBranch,headSha,event,status,conclusion,url,jobs", + ], + { capture: true }, + ), + ); +} + +function summarizeFailedRun(info) { + const failedJobs = (info.jobs ?? []).filter( + (job) => job.conclusion && job.conclusion !== "success" && job.conclusion !== "skipped", + ); + return [ + `${info.workflowName} ${info.databaseId} ended ${info.status}/${info.conclusion}: ${info.url}`, + ...failedJobs.map((job) => `- ${job.name}: ${job.conclusion} ${job.url ?? ""}`), + ].join("\n"); +} + +async function waitForSuccessfulRun(repo, runId, expected) { + for (;;) { + const info = runInfo(repo, runId); + console.log( + `${info.workflowName} ${runId}: ${info.status}${info.conclusion ? `/${info.conclusion}` : ""} ${info.url}`, + ); + if (info.status === "completed") { + if (info.conclusion !== "success") { + throw new Error(summarizeFailedRun(info)); + } + if (info.workflowName !== expected.workflowName) { + throw new Error( + `run ${runId} workflow mismatch: expected ${expected.workflowName}, got ${info.workflowName}`, + ); + } + if (info.headBranch !== expected.workflowRef) { + throw new Error( + `run ${runId} branch mismatch: expected ${expected.workflowRef}, got ${info.headBranch}`, + ); + } + return info; + } + await wait(30_000); + } +} + +function downloadArtifact(repo, runId, name, dir) { + rmSync(dir, { force: true, recursive: true }); + mkdirSync(dir, { recursive: true }); + run("gh", ["run", "download", runId, "--repo", repo, "--name", name, "--dir", dir]); +} + +function sha256(path) { + return run("shasum", ["-a", "256", path], { capture: true }).trim().split(/\s+/u)[0] ?? ""; +} + +function pluginPlanArgs(options) { + const args = ["--selection-mode", options.pluginPublishScope]; + if (options.pluginPublishScope === "selected") { + args.push("--plugins", options.plugins); + } + return args; +} + +function collectPluginPlan(script, options) { + return JSON.parse( + run("node", ["--import", "tsx", script, ...pluginPlanArgs(options)], { capture: true }), + ); +} + +async function collectPluginPlanWithRetry(script, options) { + let lastError; + for (let attempt = 1; attempt <= 3; attempt += 1) { + try { + return collectPluginPlan(script, options); + } catch (error) { + lastError = error; + if (attempt === 3) { + break; + } + console.warn( + `${script} failed on attempt ${attempt}; retrying: ${ + error instanceof Error ? error.message : String(error) + }`, + ); + await wait(5_000 * attempt); + } + } + throw lastError; +} + +function shellQuote(value) { + return `'${String(value).replace(/'/gu, "'\\''")}'`; +} + +export function buildPublishCommand(options) { + const fields = [ + ["tag", options.tag], + ["preflight_run_id", options.npmPreflightRunId], + ["full_release_validation_run_id", options.fullReleaseRunId], + ["npm_dist_tag", options.npmDistTag], + ["plugin_publish_scope", options.pluginPublishScope], + ["publish_openclaw_npm", "true"], + ["release_profile", options.releaseProfile], + ["wait_for_clawhub", "false"], + ]; + if (options.plugins.trim()) { + fields.push(["plugins", options.plugins]); + } + return [ + "gh", + "workflow", + "run", + "openclaw-release-publish.yml", + "--repo", + options.repo, + "--ref", + options.workflowRef, + ...fields.flatMap(([key, value]) => ["-f", `${key}=${value}`]), + ] + .map(shellQuote) + .join(" "); +} + +function validatePreflightManifest(manifest, params) { + if (manifest.releaseTag !== params.tag) { + throw new Error( + `npm preflight tag mismatch: expected ${params.tag}, got ${manifest.releaseTag}`, + ); + } + if (manifest.releaseSha !== params.targetSha) { + throw new Error( + `npm preflight SHA mismatch: expected ${params.targetSha}, got ${manifest.releaseSha}`, + ); + } + if (manifest.npmDistTag !== params.npmDistTag) { + throw new Error( + `npm preflight dist-tag mismatch: expected ${params.npmDistTag}, got ${manifest.npmDistTag}`, + ); + } + if (!manifest.tarballName || !manifest.tarballSha256) { + throw new Error("npm preflight manifest missing tarball metadata"); + } +} + +function validateFullManifest(manifest, params) { + if (manifest.workflowName !== "Full Release Validation") { + throw new Error(`full validation workflow mismatch: ${manifest.workflowName}`); + } + if (manifest.targetSha !== params.targetSha) { + throw new Error( + `full validation SHA mismatch: expected ${params.targetSha}, got ${manifest.targetSha}`, + ); + } + if (manifest.releaseProfile !== params.releaseProfile) { + throw new Error( + `full validation profile mismatch: expected ${params.releaseProfile}, got ${manifest.releaseProfile}`, + ); + } + if (manifest.rerunGroup !== "all") { + throw new Error(`full validation must use rerun_group=all, got ${manifest.rerunGroup}`); + } +} + +async function runParallelsIfNeeded(options) { + if (options.skipParallels) { + return { status: "skipped" }; + } + const version = options.tag.replace(/^v/u, ""); + run("pnpm", [ + "release:beta-smoke", + "--", + "--beta", + version, + "--ref", + options.workflowRef, + "--skip-telegram", + ]); + return { + status: "passed", + command: `pnpm release:beta-smoke -- --beta ${version} --ref ${options.workflowRef} --skip-telegram`, + }; +} + +async function main() { + const options = parseArgs(process.argv.slice(2)); + options.workflowRef ||= currentBranch(); + options.outputDir ||= join(".artifacts", "release-candidate", options.tag); + const targetSha = gitRevParse(`${options.tag}^{}`); + + if (!options.fullReleaseRunId && !options.skipDispatch) { + const before = beforeRunIds(options.repo, "Full Release Validation"); + dispatchWorkflow(options.repo, "full-release-validation.yml", options.workflowRef, { + ref: options.tag, + provider: options.provider, + mode: options.mode, + release_profile: options.releaseProfile, + run_release_soak: options.releaseProfile === "full" ? "true" : "false", + rerun_group: "all", + }); + options.fullReleaseRunId = await findNewRunId(options.repo, "Full Release Validation", before); + } + + if (!options.npmPreflightRunId && !options.skipDispatch) { + const before = beforeRunIds(options.repo, "OpenClaw NPM Release"); + dispatchWorkflow(options.repo, "openclaw-npm-release.yml", options.workflowRef, { + tag: options.tag, + preflight_only: "true", + npm_dist_tag: options.npmDistTag, + }); + options.npmPreflightRunId = await findNewRunId(options.repo, "OpenClaw NPM Release", before); + } + + const fullRun = await waitForSuccessfulRun(options.repo, options.fullReleaseRunId, { + workflowName: "Full Release Validation", + workflowRef: options.workflowRef, + }); + const npmRun = await waitForSuccessfulRun(options.repo, options.npmPreflightRunId, { + workflowName: "OpenClaw NPM Release", + workflowRef: options.workflowRef, + }); + if (fullRun.headSha !== targetSha || npmRun.headSha !== targetSha) { + throw new Error( + `run SHA mismatch: tag=${targetSha} full=${fullRun.headSha} npm=${npmRun.headSha}`, + ); + } + + const npmDir = join(options.outputDir, "npm-preflight"); + const fullDir = join(options.outputDir, "full-release-validation"); + downloadArtifact( + options.repo, + options.npmPreflightRunId, + `openclaw-npm-preflight-${options.tag}`, + npmDir, + ); + downloadArtifact( + options.repo, + options.fullReleaseRunId, + `full-release-validation-${options.fullReleaseRunId}`, + fullDir, + ); + + const npmManifest = readJson(join(npmDir, "preflight-manifest.json"), "npm preflight manifest"); + const fullManifest = readJson( + join(fullDir, "full-release-validation-manifest.json"), + "full validation manifest", + ); + validatePreflightManifest(npmManifest, { + tag: options.tag, + targetSha, + npmDistTag: options.npmDistTag, + }); + validateFullManifest(fullManifest, { + targetSha, + releaseProfile: options.releaseProfile, + }); + const tarballPath = join(npmDir, npmManifest.tarballName); + if (!existsSync(tarballPath)) { + throw new Error(`prepared tarball missing: ${tarballPath}`); + } + const actualTarballSha = sha256(tarballPath); + if (actualTarballSha !== npmManifest.tarballSha256) { + throw new Error( + `prepared tarball digest mismatch: expected ${npmManifest.tarballSha256}, got ${actualTarballSha}`, + ); + } + + const parallels = await runParallelsIfNeeded(options); + const pluginNpmPlan = await collectPluginPlanWithRetry( + "scripts/plugin-npm-release-plan.ts", + options, + ); + const pluginClawHubPlan = await collectPluginPlanWithRetry( + "scripts/plugin-clawhub-release-plan.ts", + options, + ); + const publishCommand = buildPublishCommand(options); + const evidence = { + version: 1, + tag: options.tag, + targetSha, + workflowRef: options.workflowRef, + npmDistTag: options.npmDistTag, + fullReleaseValidationRunId: options.fullReleaseRunId, + npmPreflightRunId: options.npmPreflightRunId, + fullReleaseValidationUrl: fullRun.url, + npmPreflightUrl: npmRun.url, + tarball: { + name: basename(tarballPath), + sha256: actualTarballSha, + path: tarballPath, + }, + parallels, + pluginNpmPlan, + pluginClawHubPlan, + publishCommand, + }; + mkdirSync(options.outputDir, { recursive: true }); + const evidencePath = join(options.outputDir, "release-candidate-evidence.json"); + writeFileSync(evidencePath, `${JSON.stringify(evidence, null, 2)}\n`); + + console.log(`release candidate evidence: ${evidencePath}`); + console.log("publish command:"); + console.log(publishCommand); +} + +if (process.argv[1] === fileURLToPath(import.meta.url)) { + await main().catch((error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); + }); +} diff --git a/test/scripts/package-acceptance-workflow.test.ts b/test/scripts/package-acceptance-workflow.test.ts index 61230207d98..a3f57536acd 100644 --- a/test/scripts/package-acceptance-workflow.test.ts +++ b/test/scripts/package-acceptance-workflow.test.ts @@ -863,6 +863,11 @@ describe("package artifact reuse", () => { expect(workflow).toContain("### Slowest jobs: ${label}"); expect(workflow).toContain("### Longest queues: ${label}"); + expect(workflow).toContain("Write release validation manifest"); + expect(workflow).toContain("Upload release validation manifest"); + expect(workflow).toContain("Failed child detail: ${label}"); + expect(workflow).toContain("actions/runs/${run_id}/artifacts?per_page=100"); + expect(workflow).toContain("full-release-validation-${{ github.run_id }}"); expect(workflow).toContain("| Job | Result | Queue minutes | Run minutes |"); expect(workflow).toContain( 'gh api --paginate "repos/${GITHUB_REPOSITORY}/actions/runs/${run_id}/jobs?per_page=100"', @@ -899,6 +904,9 @@ describe("package artifact reuse", () => { expect(packageJson.scripts?.["release:verify-beta"]).toBe( "node --import tsx scripts/release-verify-beta.ts", ); + expect(packageJson.scripts?.["release:candidate"]).toBe( + "node scripts/release-candidate-checklist.mjs", + ); expect(packageJson.scripts?.["release:fast-pretag-check"]).toBe( "bash scripts/release-fast-pretag-check.sh", ); diff --git a/test/scripts/release-candidate-checklist.test.ts b/test/scripts/release-candidate-checklist.test.ts new file mode 100644 index 00000000000..c0061ddd4ca --- /dev/null +++ b/test/scripts/release-candidate-checklist.test.ts @@ -0,0 +1,38 @@ +import { describe, expect, it } from "vitest"; +import { buildPublishCommand, parseArgs } from "../../scripts/release-candidate-checklist.mjs"; + +describe("release candidate checklist", () => { + it("requires run ids when dispatch is disabled", () => { + expect(() => parseArgs(["--tag", "v2026.5.14-beta.3", "--skip-dispatch"])).toThrow( + "--skip-dispatch requires --full-release-run and --npm-preflight-run", + ); + }); + + it("builds the gated release publish command from green evidence inputs", () => { + const options = { + ...parseArgs([ + "--tag", + "v2026.5.14-beta.3", + "--workflow-ref", + "release/2026.5.14", + "--full-release-run", + "111", + "--npm-preflight-run", + "222", + "--skip-dispatch", + ]), + workflowRef: "release/2026.5.14", + }; + + expect(buildPublishCommand(options)).toContain("'full_release_validation_run_id=111'"); + expect(buildPublishCommand(options)).toContain("'preflight_run_id=222'"); + expect(buildPublishCommand(options)).toContain("'tag=v2026.5.14-beta.3'"); + expect(buildPublishCommand(options)).toContain("'plugin_publish_scope=all-publishable'"); + }); + + it("requires explicit plugin names for selected plugin publish scope", () => { + expect(() => + parseArgs(["--tag", "v2026.5.14-beta.3", "--plugin-publish-scope", "selected"]), + ).toThrow("--plugin-publish-scope selected requires --plugins"); + }); +});