From c84b7cbffcc305aafa2b9927c6aadf3579a98e4b Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Tue, 5 May 2026 02:28:22 +0100 Subject: [PATCH] ci(release): speed up focused release reruns --- .github/workflows/full-release-validation.yml | 16 + ...nclaw-cross-os-release-checks-reusable.yml | 12 + .../openclaw-live-and-e2e-checks-reusable.yml | 13 +- .github/workflows/openclaw-release-checks.yml | 22 ++ docs/ci.md | 2 +- docs/reference/RELEASING.md | 5 +- docs/reference/full-release-validation.md | 11 + scripts/openclaw-cross-os-release-checks.ts | 334 ++++++++++++------ .../openclaw-cross-os-release-checks.test.ts | 57 +++ ...openclaw-cross-os-release-workflow.test.ts | 2 + .../package-acceptance-workflow.test.ts | 9 + 11 files changed, 378 insertions(+), 105 deletions(-) diff --git a/.github/workflows/full-release-validation.yml b/.github/workflows/full-release-validation.yml index ac58b612c8a..26ddafe696d 100644 --- a/.github/workflows/full-release-validation.yml +++ b/.github/workflows/full-release-validation.yml @@ -63,6 +63,11 @@ on: required: false default: "" type: string + cross_os_suite_filter: + description: Optional focused cross-OS suite filter, e.g. windows/packaged-upgrade or packaged-fresh + required: false + default: "" + type: string npm_telegram_package_spec: description: Optional published package spec for the package Telegram E2E lane required: false @@ -144,6 +149,7 @@ jobs: RUN_RELEASE_SOAK: ${{ inputs.run_release_soak || inputs.release_profile == 'full' }} RERUN_GROUP: ${{ inputs.rerun_group }} LIVE_SUITE_FILTER: ${{ inputs.live_suite_filter }} + CROSS_OS_SUITE_FILTER: ${{ inputs.cross_os_suite_filter }} run: | { echo "## Full release validation" @@ -156,6 +162,9 @@ jobs: if [[ -n "${LIVE_SUITE_FILTER// }" ]]; then echo "- Live suite filter: \`${LIVE_SUITE_FILTER}\`" fi + if [[ -n "${CROSS_OS_SUITE_FILTER// }" ]]; then + echo "- Cross-OS suite filter: \`${CROSS_OS_SUITE_FILTER}\`" + fi if [[ "$RERUN_GROUP" == "all" || "$RERUN_GROUP" == "ci" ]]; then echo "- Normal CI: \`CI\` with \`target_ref=${TARGET_SHA}\`" else @@ -410,6 +419,7 @@ jobs: RUN_RELEASE_SOAK: ${{ inputs.run_release_soak || inputs.release_profile == 'full' }} RERUN_GROUP: ${{ inputs.rerun_group }} LIVE_SUITE_FILTER: ${{ inputs.live_suite_filter }} + CROSS_OS_SUITE_FILTER: ${{ inputs.cross_os_suite_filter }} PACKAGE_ACCEPTANCE_PACKAGE_SPEC: ${{ inputs.package_acceptance_package_spec }} run: | set -euo pipefail @@ -496,6 +506,9 @@ jobs: if [[ -n "${LIVE_SUITE_FILTER// }" ]]; then echo "- Live suite filter: \`${LIVE_SUITE_FILTER}\`" fi + if [[ -n "${CROSS_OS_SUITE_FILTER// }" ]]; then + echo "- Cross-OS suite filter: \`${CROSS_OS_SUITE_FILTER}\`" + fi if [[ -n "${PACKAGE_ACCEPTANCE_PACKAGE_SPEC// }" ]]; then echo "- Package Acceptance package spec: \`${PACKAGE_ACCEPTANCE_PACKAGE_SPEC}\`" fi @@ -518,6 +531,9 @@ jobs: if [[ -n "${LIVE_SUITE_FILTER// }" ]]; then args+=(-f live_suite_filter="$LIVE_SUITE_FILTER") fi + if [[ -n "${CROSS_OS_SUITE_FILTER// }" ]]; then + args+=(-f cross_os_suite_filter="$CROSS_OS_SUITE_FILTER") + fi if [[ -n "${PACKAGE_ACCEPTANCE_PACKAGE_SPEC// }" ]]; then args+=(-f package_acceptance_package_spec="$PACKAGE_ACCEPTANCE_PACKAGE_SPEC") fi diff --git a/.github/workflows/openclaw-cross-os-release-checks-reusable.yml b/.github/workflows/openclaw-cross-os-release-checks-reusable.yml index 047477f201f..e87dfdf39de 100644 --- a/.github/workflows/openclaw-cross-os-release-checks-reusable.yml +++ b/.github/workflows/openclaw-cross-os-release-checks-reusable.yml @@ -31,6 +31,11 @@ on: - fresh - upgrade - both + suite_filter: + description: Optional focused cross-OS suite filter, e.g. windows/packaged-upgrade or packaged-fresh + required: false + default: "" + type: string previous_version: description: Optional baseline version for installer/dev-update and packaged upgrade required: false @@ -100,6 +105,11 @@ on: description: Which release-check lanes to run required: true type: string + suite_filter: + description: Optional focused cross-OS suite filter, e.g. windows/packaged-upgrade or packaged-fresh + required: false + default: "" + type: string previous_version: description: Optional baseline version for the upgrade lane (defaults to npm latest) required: false @@ -482,6 +492,7 @@ jobs: env: INPUT_REF: ${{ inputs.ref }} INPUT_MODE: ${{ inputs.mode }} + INPUT_SUITE_FILTER: ${{ inputs.suite_filter }} INPUT_UBUNTU_RUNNER: ${{ inputs.ubuntu_runner }} INPUT_WINDOWS_RUNNER: ${{ inputs.windows_runner }} INPUT_MACOS_RUNNER: ${{ inputs.macos_runner }} @@ -493,6 +504,7 @@ jobs: --resolve-matrix \ --ref "${INPUT_REF}" \ --mode "${INPUT_MODE}" \ + --suite-filter "${INPUT_SUITE_FILTER}" \ --ubuntu-runner "${INPUT_UBUNTU_RUNNER}" \ --windows-runner "${INPUT_WINDOWS_RUNNER}" \ --macos-runner "${INPUT_MACOS_RUNNER}")" diff --git a/.github/workflows/openclaw-live-and-e2e-checks-reusable.yml b/.github/workflows/openclaw-live-and-e2e-checks-reusable.yml index 1c8f7924619..43000d0f680 100644 --- a/.github/workflows/openclaw-live-and-e2e-checks-reusable.yml +++ b/.github/workflows/openclaw-live-and-e2e-checks-reusable.yml @@ -489,7 +489,18 @@ jobs: fi - name: Verify live prompt cache floors - run: pnpm test:live:cache + run: | + set -euo pipefail + for attempt in 1 2 3; do + echo "live-cache attempt ${attempt}/3" + if pnpm test:live:cache; then + exit 0 + fi + if [[ "$attempt" == "3" ]]; then + exit 1 + fi + sleep $((attempt * 15)) + done validate_repo_e2e: needs: validate_selected_ref diff --git a/.github/workflows/openclaw-release-checks.yml b/.github/workflows/openclaw-release-checks.yml index d1469c5e8c7..a97c52f50b3 100644 --- a/.github/workflows/openclaw-release-checks.yml +++ b/.github/workflows/openclaw-release-checks.yml @@ -63,6 +63,11 @@ on: required: false default: "" type: string + cross_os_suite_filter: + description: Optional focused cross-OS suite filter, e.g. windows/packaged-upgrade or packaged-fresh + required: false + default: "" + type: string package_acceptance_package_spec: description: Optional published package spec for Package Acceptance; blank uses the prepared release artifact required: false @@ -94,6 +99,7 @@ jobs: run_release_soak: ${{ steps.inputs.outputs.run_release_soak }} rerun_group: ${{ steps.inputs.outputs.rerun_group }} live_suite_filter: ${{ steps.inputs.outputs.live_suite_filter }} + cross_os_suite_filter: ${{ steps.inputs.outputs.cross_os_suite_filter }} qa_live_matrix_enabled: ${{ steps.inputs.outputs.qa_live_matrix_enabled }} qa_live_telegram_enabled: ${{ steps.inputs.outputs.qa_live_telegram_enabled }} qa_live_slack_enabled: ${{ steps.inputs.outputs.qa_live_slack_enabled }} @@ -215,6 +221,7 @@ jobs: RELEASE_RUN_RELEASE_SOAK_INPUT: ${{ inputs.run_release_soak }} RELEASE_RERUN_GROUP_INPUT: ${{ inputs.rerun_group }} RELEASE_LIVE_SUITE_FILTER_INPUT: ${{ inputs.live_suite_filter }} + RELEASE_CROSS_OS_SUITE_FILTER_INPUT: ${{ inputs.cross_os_suite_filter }} RELEASE_QA_SLACK_LIVE_CI_ENABLED: ${{ vars.OPENCLAW_QA_SLACK_LIVE_CI_ENABLED || 'false' }} RELEASE_PACKAGE_ACCEPTANCE_PACKAGE_SPEC_INPUT: ${{ inputs.package_acceptance_package_spec }} run: | @@ -292,6 +299,7 @@ jobs: printf 'run_release_soak=%s\n' "$run_release_soak" printf 'rerun_group=%s\n' "$RELEASE_RERUN_GROUP_INPUT" printf 'live_suite_filter=%s\n' "$RELEASE_LIVE_SUITE_FILTER_INPUT" + printf 'cross_os_suite_filter=%s\n' "$RELEASE_CROSS_OS_SUITE_FILTER_INPUT" printf 'qa_live_matrix_enabled=%s\n' "$qa_live_matrix_enabled" printf 'qa_live_telegram_enabled=%s\n' "$qa_live_telegram_enabled" printf 'qa_live_slack_enabled=%s\n' "$qa_live_slack_enabled" @@ -309,6 +317,7 @@ jobs: RUN_RELEASE_SOAK: ${{ steps.inputs.outputs.run_release_soak }} RELEASE_RERUN_GROUP: ${{ inputs.rerun_group }} RELEASE_LIVE_SUITE_FILTER: ${{ inputs.live_suite_filter }} + RELEASE_CROSS_OS_SUITE_FILTER: ${{ inputs.cross_os_suite_filter }} PACKAGE_ACCEPTANCE_PACKAGE_SPEC: ${{ inputs.package_acceptance_package_spec }} run: | { @@ -325,6 +334,9 @@ jobs: if [[ -n "${RELEASE_LIVE_SUITE_FILTER// }" ]]; then echo "- Live suite filter: \`${RELEASE_LIVE_SUITE_FILTER}\`" fi + if [[ -n "${RELEASE_CROSS_OS_SUITE_FILTER// }" ]]; then + echo "- Cross-OS suite filter: \`${RELEASE_CROSS_OS_SUITE_FILTER}\`" + fi echo "- QA live lanes: Matrix \`${{ steps.inputs.outputs.qa_live_matrix_enabled }}\`, Telegram \`${{ steps.inputs.outputs.qa_live_telegram_enabled }}\`, Slack \`${{ steps.inputs.outputs.qa_live_slack_enabled }}\`" if [[ -n "${PACKAGE_ACCEPTANCE_PACKAGE_SPEC// }" ]]; then echo "- Package Acceptance package spec: \`${PACKAGE_ACCEPTANCE_PACKAGE_SPEC}\`" @@ -430,6 +442,7 @@ jobs: ref: ${{ needs.resolve_target.outputs.revision }} provider: ${{ needs.resolve_target.outputs.provider }} mode: ${{ needs.resolve_target.outputs.mode }} + suite_filter: ${{ needs.resolve_target.outputs.cross_os_suite_filter }} candidate_artifact_name: ${{ needs.prepare_release_package.outputs.artifact_name }} candidate_file_name: openclaw-current.tgz candidate_version: ${{ needs.prepare_release_package.outputs.package_version }} @@ -603,6 +616,7 @@ jobs: name: Run QA Lab parity lane (${{ matrix.lane }}) needs: [resolve_target] if: contains(fromJSON('["all","qa","qa-parity"]'), needs.resolve_target.outputs.rerun_group) + continue-on-error: true runs-on: blacksmith-8vcpu-ubuntu-2404 timeout-minutes: 30 permissions: @@ -687,6 +701,7 @@ jobs: name: Run QA Lab parity report needs: [resolve_target, qa_lab_parity_lane_release_checks] if: contains(fromJSON('["all","qa","qa-parity"]'), needs.resolve_target.outputs.rerun_group) + continue-on-error: true runs-on: blacksmith-8vcpu-ubuntu-2404 timeout-minutes: 20 permissions: @@ -743,6 +758,7 @@ jobs: name: Run QA Lab live Matrix lane needs: [resolve_target] if: contains(fromJSON('["all","qa","qa-live"]'), needs.resolve_target.outputs.rerun_group) && needs.resolve_target.outputs.qa_live_matrix_enabled == 'true' + continue-on-error: true runs-on: blacksmith-8vcpu-ubuntu-2404 timeout-minutes: 60 permissions: @@ -820,6 +836,7 @@ jobs: name: Run QA Lab live Telegram lane needs: [resolve_target] if: contains(fromJSON('["all","qa","qa-live"]'), needs.resolve_target.outputs.rerun_group) && needs.resolve_target.outputs.qa_live_telegram_enabled == 'true' + continue-on-error: true runs-on: blacksmith-8vcpu-ubuntu-2404 timeout-minutes: 60 permissions: @@ -913,6 +930,7 @@ jobs: name: Run QA Lab live Slack lane needs: [resolve_target] if: contains(fromJSON('["all","qa","qa-live"]'), needs.resolve_target.outputs.rerun_group) && needs.resolve_target.outputs.qa_live_slack_enabled == 'true' && vars.OPENCLAW_QA_SLACK_LIVE_CI_ENABLED == 'true' + continue-on-error: true runs-on: blacksmith-8vcpu-ubuntu-2404 timeout-minutes: 60 permissions: @@ -1042,6 +1060,10 @@ jobs: name="${item%%=*}" result="${item#*=}" if [[ "$result" != "success" && "$result" != "skipped" ]]; then + if [[ "$name" == qa_* ]]; then + echo "::warning::${name} ended with ${result}; QA release-check lanes are advisory and do not block release validation." + continue + fi echo "::error::${name} ended with ${result}" failed=1 fi diff --git a/docs/ci.md b/docs/ci.md index 6c67eb6e1e0..dbdffeea439 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -199,7 +199,7 @@ Docker release-path soak; `full` forces soak on. The umbrella records the dispatched child run ids, and the final `Verify full validation` job re-checks current child run conclusions and appends slowest-job tables for each child run. If a child workflow is rerun and turns green, rerun only the parent verifier job to refresh the umbrella result and timing summary. -For recovery, both `Full Release Validation` and `OpenClaw Release Checks` accept `rerun_group`. Use `all` for a release candidate, `ci` for only the normal full CI child, `plugin-prerelease` for only the plugin prerelease child, `release-checks` for every release child, or a narrower group: `install-smoke`, `cross-os`, `live-e2e`, `package`, `qa`, `qa-parity`, `qa-live`, or `npm-telegram` on the umbrella. This keeps a failed release box rerun bounded after a focused fix. +For recovery, both `Full Release Validation` and `OpenClaw Release Checks` accept `rerun_group`. Use `all` for a release candidate, `ci` for only the normal full CI child, `plugin-prerelease` for only the plugin prerelease child, `release-checks` for every release child, or a narrower group: `install-smoke`, `cross-os`, `live-e2e`, `package`, `qa`, `qa-parity`, `qa-live`, or `npm-telegram` on the umbrella. This keeps a failed release box rerun bounded after a focused fix. For one failed cross-OS lane, combine `rerun_group=cross-os` with `cross_os_suite_filter`, for example `windows/packaged-upgrade`; long cross-OS commands emit heartbeat lines and packaged-upgrade summaries include per-phase timings. QA release-check lanes are advisory, so QA-only failures warn but do not block the release-check verifier. `OpenClaw Release Checks` uses the trusted workflow ref to resolve the selected ref once into a `release-package-under-test` tarball, then passes that artifact to cross-OS checks and Package Acceptance, plus the live/E2E release-path Docker workflow when soak coverage runs. That keeps the package bytes consistent across release boxes and avoids repacking the same candidate in multiple child jobs. diff --git a/docs/reference/RELEASING.md b/docs/reference/RELEASING.md index 3e972641ebe..e38894361c8 100644 --- a/docs/reference/RELEASING.md +++ b/docs/reference/RELEASING.md @@ -379,7 +379,10 @@ runs only the release-only plugin child, `release-checks` runs every release box, and the narrower release groups are `install-smoke`, `cross-os`, `live-e2e`, `package`, `qa`, `qa-parity`, `qa-live`, and `npm-telegram`. Focused `npm-telegram` reruns require `npm_telegram_package_spec`; full/all runs -with `release_profile=full` use the release-checks package artifact. +with `release_profile=full` use the release-checks package artifact. Focused +cross-OS reruns can add `cross_os_suite_filter=windows/packaged-upgrade` or +another OS/suite filter. QA release-check failures are advisory; a QA-only +failure does not block release validation. ### Vitest diff --git a/docs/reference/full-release-validation.md b/docs/reference/full-release-validation.md index 6baeea4b1ee..0c630011d07 100644 --- a/docs/reference/full-release-validation.md +++ b/docs/reference/full-release-validation.md @@ -158,6 +158,17 @@ Valid filter ids are defined in the reusable live/E2E workflow, including The `live-gateway-advisory-docker` handle is an aggregate rerun handle for its three provider shards, so it still fans out to all advisory Docker gateway jobs. +Use `cross_os_suite_filter` with `rerun_group=cross-os` when one cross-OS lane +failed. The filter accepts an OS id, a suite id, or an OS/suite pair, for +example `windows/packaged-upgrade`, `windows`, or `packaged-fresh`. Cross-OS +summaries include per-phase timings for packaged upgrade lanes, and long-running +commands print heartbeat lines so a stuck Windows update is visible before the +job timeout. + +QA release-check lanes are advisory. A QA-only failure is reported as a warning +and does not block the release-check verifier; rerun `rerun_group=qa`, +`qa-parity`, or `qa-live` when you need fresh QA evidence. + ## Evidence to keep Keep the `Full Release Validation` summary as the release-level index. It links diff --git a/scripts/openclaw-cross-os-release-checks.ts b/scripts/openclaw-cross-os-release-checks.ts index 866b656f890..bebb11426d7 100644 --- a/scripts/openclaw-cross-os-release-checks.ts +++ b/scripts/openclaw-cross-os-release-checks.ts @@ -33,6 +33,7 @@ const SUPPORTED_SUITES = new Set([ "packaged-upgrade", "dev-update", ]); +const SUPPORTED_OS_IDS = new Set(["ubuntu", "windows", "macos"]); export const CROSS_OS_AGENT_TURN_TIMEOUT_SECONDS = parsePositiveIntegerEnv( "OPENCLAW_CROSS_OS_AGENT_TURN_TIMEOUT_SECONDS", @@ -122,6 +123,10 @@ export const CROSS_OS_RELEASE_SMOKE_TOOLS_PROFILE = "minimal"; export const CROSS_OS_WINDOWS_PACKAGED_UPGRADE_STEP_TIMEOUT_SECONDS = 25 * 60; export const CROSS_OS_WINDOWS_PACKAGED_UPGRADE_WRAPPER_TIMEOUT_MS = (CROSS_OS_WINDOWS_PACKAGED_UPGRADE_STEP_TIMEOUT_SECONDS + 5 * 60) * 1000; +export const CROSS_OS_COMMAND_HEARTBEAT_SECONDS = parsePositiveIntegerEnv( + "OPENCLAW_CROSS_OS_COMMAND_HEARTBEAT_SECONDS", + 60, +); if (isMainModule()) { try { @@ -232,6 +237,7 @@ export function resolveRunnerMatrix(params) { const pick = (...values) => values.find((value) => typeof value === "string" && value.trim().length > 0)?.trim(); const suites = resolveRequestedSuites(params.mode, params.ref); + const suiteFilter = parseCrossOsSuiteFilter(params.suiteFilter ?? ""); const runners = [ { os_id: "ubuntu", @@ -252,19 +258,84 @@ export function resolveRunnerMatrix(params) { artifact_name: "macos", }, ]; - return { - include: runners.flatMap((runner) => - suites.map((suite) => + const include = runners.flatMap((runner) => + suites + .filter((suite) => suiteFilter.matches(runner.os_id, suite)) + .map((suite) => Object.assign({}, runner, { suite, suite_label: formatSuiteLabel(suite), lane: suite.includes(`upgrade`) || suite === `dev-update` ? `upgrade` : `fresh`, }), ), - ), + ); + if (include.length === 0) { + throw new Error( + `cross_os_suite_filter ${JSON.stringify(params.suiteFilter ?? "")} did not match any ${params.mode} suite.`, + ); + } + return { + include, }; } +export function parseCrossOsSuiteFilter(rawFilter) { + const tokens = String(rawFilter ?? "") + .split(/[, ]+/u) + .map((token) => normalizeCrossOsSuiteFilterToken(token)) + .filter(Boolean); + if (tokens.length === 0) { + return { + matches: () => true, + tokens, + }; + } + + const matchers = tokens.map((token) => { + if (SUPPORTED_SUITES.has(token)) { + return { osId: "", suite: token }; + } + if (SUPPORTED_OS_IDS.has(token)) { + return { osId: token, suite: "" }; + } + for (const separator of ["/", ":", "-"]) { + const matchedOs = [...SUPPORTED_OS_IDS].find((osId) => + token.startsWith(`${osId}${separator}`), + ); + if (!matchedOs) { + continue; + } + const suite = token.slice(matchedOs.length + separator.length); + if (!SUPPORTED_SUITES.has(suite)) { + break; + } + return { osId: matchedOs, suite }; + } + throw new Error( + `Unsupported cross_os_suite_filter token ${JSON.stringify(token)}. Use an OS id, suite id, or os/suite pair such as windows/packaged-upgrade.`, + ); + }); + + return { + matches: (osId, suite) => + matchers.some((matcher) => { + const osMatches = !matcher.osId || matcher.osId === osId; + const suiteMatches = !matcher.suite || matcher.suite === suite; + return osMatches && suiteMatches; + }), + tokens, + }; +} + +function normalizeCrossOsSuiteFilterToken(token) { + return token + .trim() + .toLowerCase() + .replace(/_/gu, "-") + .replace(/\s*[/:-]\s*/gu, (separator) => separator.trim()) + .replace(/\s+/gu, "-"); +} + export function readRunnerOverrideEnv(env = process.env) { const preferNonEmptyEnv = (primary: string | undefined, legacy: string | undefined) => { const primaryValue = primary?.trim(); @@ -319,6 +390,7 @@ async function main(argv) { ubuntuRunner: args["ubuntu-runner"], windowsRunner: args["windows-runner"], macosRunner: args["macos-runner"], + suiteFilter: args["suite-filter"], ...runnerOverrideEnv, }), )}\n`, @@ -733,76 +805,80 @@ async function runUpgradeLane(params) { const cleanup = []; try { const env = buildLaneEnv(lane, params.providerConfig, params.providerSecretValue); - logLanePhase(lane, "install-baseline"); - if (!params.baselineTgz && params.baselineSpec) { - await installPackageSpec({ + await runTimedLanePhase(lane, "install-baseline", async () => { + if (!params.baselineTgz && params.baselineSpec) { + await installPackageSpec({ + lane, + env, + packageSpec: params.baselineSpec, + logPath: join(params.logsDir, "upgrade-install-baseline.log"), + ignoreScripts: true, + }); + } else { + await installTarballPackage({ + lane, + env, + tgzPath: params.baselineTgz, + logPath: join(params.logsDir, "upgrade-install-baseline.log"), + ignoreScripts: true, + restoreBundledPluginPostinstall: false, + }); + } + }); + await runTimedLanePhase(lane, "run-baseline-bundled-plugin-postinstall", async () => { + await runBundledPluginPostinstall({ lane, env, - packageSpec: params.baselineSpec, logPath: join(params.logsDir, "upgrade-install-baseline.log"), - ignoreScripts: true, }); - } else { - await installTarballPackage({ - lane, - env, - tgzPath: params.baselineTgz, - logPath: join(params.logsDir, "upgrade-install-baseline.log"), - ignoreScripts: true, - restoreBundledPluginPostinstall: false, - }); - } - logLanePhase(lane, "run-baseline-bundled-plugin-postinstall"); - await runBundledPluginPostinstall({ - lane, - env, - logPath: join(params.logsDir, "upgrade-install-baseline.log"), }); const baseline = { version: readInstalledVersion(lane.prefixDir), }; - logLanePhase(lane, "update"); const updateEnv = buildRealUpdateEnv(env); const updateArgs = buildPackagedUpgradeUpdateArgs(params.candidateUrl); const updateLogPath = join(params.logsDir, "upgrade-update.log"); let updateResult; let usedWindowsPackagedUpgradeTimeoutFallback = false; - try { - updateResult = await runOpenClaw({ - lane, - env: updateEnv, - args: updateArgs, - logPath: updateLogPath, - timeoutMs: updateTimeoutMs(), - check: false, - }); - } catch (error) { - if (!isRecoverableWindowsPackagedUpgradeTimeoutError(error, process.platform)) { - throw error; + await runTimedLanePhase(lane, "update", async () => { + try { + updateResult = await runOpenClaw({ + lane, + env: updateEnv, + args: updateArgs, + logPath: updateLogPath, + timeoutMs: updateTimeoutMs(), + check: false, + }); + } catch (error) { + if (!isRecoverableWindowsPackagedUpgradeTimeoutError(error, process.platform)) { + throw error; + } + usedWindowsPackagedUpgradeTimeoutFallback = true; + appendFileSync( + updateLogPath, + `\n[release-checks] Windows baseline updater timed out after fetching candidate; falling back to direct candidate install: ${formatError(error)}\n`, + ); + updateResult = { + exitCode: 124, + stdout: "", + stderr: formatError(error), + }; } - usedWindowsPackagedUpgradeTimeoutFallback = true; - appendFileSync( - updateLogPath, - `\n[release-checks] Windows baseline updater timed out after fetching candidate; falling back to direct candidate install: ${formatError(error)}\n`, - ); - updateResult = { - exitCode: 124, - stdout: "", - stderr: formatError(error), - }; - } + }); const usedWindowsPackagedUpgradeFallback = usedWindowsPackagedUpgradeTimeoutFallback || isRecoverableWindowsPackagedUpgradeSwapCleanupFailure(updateResult, process.platform); if (usedWindowsPackagedUpgradeFallback) { - logLanePhase(lane, "update-fallback-install"); - await installPackageSpec({ - lane, - env, - packageSpec: params.candidateUrl, - logPath: join(params.logsDir, "upgrade-update-fallback-install.log"), + await runTimedLanePhase(lane, "update-fallback-install", async () => { + await installPackageSpec({ + lane, + env, + packageSpec: params.candidateUrl, + logPath: join(params.logsDir, "upgrade-update-fallback-install.log"), + }); }); } else { verifyPackagedUpgradeUpdateResult(updateResult, { @@ -816,69 +892,77 @@ async function runUpgradeLane(params) { usedWindowsPackagedUpgradeFallback, }) ) { - logLanePhase(lane, "update-status"); - await runOpenClaw({ - lane, - env: updateEnv, - args: ["update", "status", "--json"], - logPath: join(params.logsDir, "upgrade-update-status.log"), - timeoutMs: 2 * 60 * 1000, + await runTimedLanePhase(lane, "update-status", async () => { + await runOpenClaw({ + lane, + env: updateEnv, + args: ["update", "status", "--json"], + logPath: join(params.logsDir, "upgrade-update-status.log"), + timeoutMs: 2 * 60 * 1000, + }); }); } - logLanePhase(lane, "run-bundled-plugin-postinstall"); - await runBundledPluginPostinstall({ - lane, - env, - logPath: join(params.logsDir, "upgrade-bundled-plugin-postinstall.log"), + await runTimedLanePhase(lane, "run-bundled-plugin-postinstall", async () => { + await runBundledPluginPostinstall({ + lane, + env, + logPath: join(params.logsDir, "upgrade-bundled-plugin-postinstall.log"), + }); }); const installed = readInstalledMetadata(lane.prefixDir); verifyInstalledCandidate(installed, params.build); - logLanePhase(lane, "onboard"); - await runOnboard({ - lane, - env, - providerConfig: params.providerConfig, - logPath: join(params.logsDir, "upgrade-onboard.log"), + await runTimedLanePhase(lane, "onboard", async () => { + await runOnboard({ + lane, + env, + providerConfig: params.providerConfig, + logPath: join(params.logsDir, "upgrade-onboard.log"), + }); }); - logLanePhase(lane, "models-set"); - await runModelsSet({ - lane, - env, - providerConfig: params.providerConfig, - logPath: join(params.logsDir, "upgrade-models-set.log"), + await runTimedLanePhase(lane, "models-set", async () => { + await runModelsSet({ + lane, + env, + providerConfig: params.providerConfig, + logPath: join(params.logsDir, "upgrade-models-set.log"), + }); }); - logLanePhase(lane, "start-gateway"); - const gateway = await startGateway({ - lane, - env, - logPath: join(params.logsDir, "upgrade-gateway.log"), - }); + const gateway = await runTimedLanePhase(lane, "start-gateway", async () => + startGateway({ + lane, + env, + logPath: join(params.logsDir, "upgrade-gateway.log"), + }), + ); cleanup.push(() => stopGateway(gateway)); - logLanePhase(lane, "wait-gateway"); - await waitForGateway({ - lane, - env, - logPath: join(params.logsDir, "upgrade-gateway-status.log"), + await runTimedLanePhase(lane, "wait-gateway", async () => { + await waitForGateway({ + lane, + env, + logPath: join(params.logsDir, "upgrade-gateway-status.log"), + }); }); - logLanePhase(lane, "dashboard"); - await runDashboardSmoke({ - lane, - logPath: join(params.logsDir, "upgrade-dashboard.log"), + await runTimedLanePhase(lane, "dashboard", async () => { + await runDashboardSmoke({ + lane, + logPath: join(params.logsDir, "upgrade-dashboard.log"), + }); }); - logLanePhase(lane, "agent-turn"); - const agent = await runAgentTurn({ - lane, - env, - label: "upgrade", - logPath: join(params.logsDir, "upgrade-agent.log"), - }); + const agent = await runTimedLanePhase(lane, "agent-turn", async () => + runAgentTurn({ + lane, + env, + label: "upgrade", + logPath: join(params.logsDir, "upgrade-agent.log"), + }), + ); return { status: "pass", @@ -888,6 +972,7 @@ async function runUpgradeLane(params) { dashboardStatus: "pass", gatewayPort: lane.gatewayPort, agentOutput: trimForSummary(agent.stdout), + phaseTimings: lane.phaseTimings, }; } finally { await runCleanup(cleanup); @@ -1245,6 +1330,7 @@ function createLaneState(name) { stateDir, appDataDir, gatewayPort: 0, + phaseTimings: [], }; } @@ -3268,6 +3354,10 @@ async function runCommand(command, args, options) { let stderr = ""; let timedOut = false; let settled = false; + const startedAt = Date.now(); + let killWaitTimer = null; + let timer = null; + let heartbeatTimer = null; const clearTimers = () => { if (timer) { @@ -3276,6 +3366,9 @@ async function runCommand(command, args, options) { if (killWaitTimer) { clearTimeout(killWaitTimer); } + if (heartbeatTimer) { + clearInterval(heartbeatTimer); + } }; const finalize = (callback) => { @@ -3307,8 +3400,7 @@ async function runCommand(command, args, options) { child.kill(process.platform === "win32" ? undefined : "SIGKILL"); }; - let killWaitTimer = null; - const timer = + timer = options.timeoutMs && Number.isFinite(options.timeoutMs) ? setTimeout(() => { timedOut = true; @@ -3327,6 +3419,20 @@ async function runCommand(command, args, options) { }, 15_000); }, options.timeoutMs) : null; + heartbeatTimer = + CROSS_OS_COMMAND_HEARTBEAT_SECONDS > 0 + ? setInterval(() => { + const elapsedSeconds = Math.floor((Date.now() - startedAt) / 1000); + const message = `${new Date().toISOString()} still running after ${elapsedSeconds}s: ${command} ${args.join(" ")}\n`; + logStream.write(message); + process.stdout.write(`[release-checks] ${message}`); + }, CROSS_OS_COMMAND_HEARTBEAT_SECONDS * 1000) + : null; + heartbeatTimer?.unref?.(); + + logStream.write( + `${new Date().toISOString()} start command=${command} args=${args.join(" ")}\n`, + ); child.stdout?.on("data", (chunk) => { const text = chunk.toString(); @@ -3454,6 +3560,13 @@ function writeSummary(baseDir, summaryPayload) { result.agentOutput ? `- Agent output: \`${trimForSummary(result.agentOutput)}\`` : "", result.error ? `- Error: \`${trimForSummary(result.error)}\`` : "", ].filter(Boolean); + if (Array.isArray(result.phaseTimings) && result.phaseTimings.length > 0) { + lines.push("", "### Phase timings"); + for (const phase of result.phaseTimings) { + const suffix = phase.status === "pass" ? "" : ` (${phase.status})`; + lines.push(`- \`${phase.name}\`: ${Math.round(phase.durationMs / 1000)}s${suffix}`); + } + } writeFileSync(summaryMarkdownPath, `${lines.join("\n")}\n`, "utf8"); } @@ -3522,6 +3635,23 @@ function logLanePhase(lane, phase) { logPhase(`lane.${lane.name}`, phase); } +async function runTimedLanePhase(lane, phase, callback) { + const startedAt = Date.now(); + logLanePhase(lane, phase); + try { + const result = await callback(); + const durationMs = Date.now() - startedAt; + lane.phaseTimings.push({ name: phase, status: "pass", durationMs }); + logPhase(`lane.${lane.name}`, `${phase}: done in ${Math.round(durationMs / 1000)}s`); + return result; + } catch (error) { + const durationMs = Date.now() - startedAt; + lane.phaseTimings.push({ name: phase, status: "fail", durationMs }); + logPhase(`lane.${lane.name}`, `${phase}: failed in ${Math.round(durationMs / 1000)}s`); + throw error; + } +} + function trimForSummary(value) { const trimmed = value.trim(); if (trimmed.length <= 600) { diff --git a/test/scripts/openclaw-cross-os-release-checks.test.ts b/test/scripts/openclaw-cross-os-release-checks.test.ts index a334d09bd06..1a6daa9d612 100644 --- a/test/scripts/openclaw-cross-os-release-checks.test.ts +++ b/test/scripts/openclaw-cross-os-release-checks.test.ts @@ -36,6 +36,7 @@ import { CROSS_OS_DASHBOARD_FETCH_TIMEOUT_MS, CROSS_OS_DASHBOARD_SMOKE_TIMEOUT_MS, CROSS_OS_AGENT_TURN_TIMEOUT_SECONDS, + CROSS_OS_COMMAND_HEARTBEAT_SECONDS, isImmutableReleaseRef, isRecoverableWindowsPackagedUpgradeSwapCleanupFailure, isRecoverableWindowsPackagedUpgradeTimeoutError, @@ -43,6 +44,7 @@ import { normalizeRequestedRef, normalizeWindowsCommandShimPath, normalizeWindowsInstalledCliPath, + parseCrossOsSuiteFilter, parseArgs, packageHasScript, readInstalledVersion, @@ -97,6 +99,11 @@ describe("scripts/openclaw-cross-os-release-checks", () => { ).toBeGreaterThanOrEqual(5 * 60 * 1000); }); + it("prints command heartbeats before long release commands hit job timeouts", () => { + expect(CROSS_OS_COMMAND_HEARTBEAT_SECONDS).toBeGreaterThan(0); + expect(CROSS_OS_COMMAND_HEARTBEAT_SECONDS).toBeLessThanOrEqual(60); + }); + it("accepts OK agent output from the captured log when stdout is empty", () => { const dir = mkdtempSync(join(tmpdir(), "openclaw-cross-os-agent-output-")); try { @@ -372,6 +379,56 @@ describe("scripts/openclaw-cross-os-release-checks", () => { ); }); + it("filters the cross-OS runner matrix to a focused OS suite", () => { + const matrix = resolveRunnerMatrix({ + mode: "both", + ref: "main", + suiteFilter: "windows/packaged-upgrade", + ubuntuRunner: "", + windowsRunner: "", + macosRunner: "", + varUbuntuRunner: "", + varWindowsRunner: "", + varMacosRunner: "", + }); + + expect(matrix.include).toEqual([ + expect.objectContaining({ + os_id: "windows", + suite: "packaged-upgrade", + lane: "upgrade", + }), + ]); + }); + + it("filters the cross-OS runner matrix by suite across platforms", () => { + const matrix = resolveRunnerMatrix({ + mode: "both", + ref: "main", + suiteFilter: "packaged-fresh", + ubuntuRunner: "", + windowsRunner: "", + macosRunner: "", + varUbuntuRunner: "", + varWindowsRunner: "", + varMacosRunner: "", + }); + + expect(matrix.include).toHaveLength(3); + expect(matrix.include.map((entry) => entry.os_id).toSorted()).toEqual([ + "macos", + "ubuntu", + "windows", + ]); + expect(matrix.include.every((entry) => entry.suite === "packaged-fresh")).toBe(true); + }); + + it("rejects unsupported cross-OS suite filter tokens", () => { + expect(() => parseCrossOsSuiteFilter("windows/nope")).toThrow( + /Unsupported cross_os_suite_filter/u, + ); + }); + it("can rebuild the Windows PATH with or without current-process entries", () => { expect(buildWindowsPathBootstrapScript()).toContain("@($userPath, $machinePath, $env:Path)"); const persistedOnlyScript = buildWindowsPathBootstrapScript({ diff --git a/test/scripts/openclaw-cross-os-release-workflow.test.ts b/test/scripts/openclaw-cross-os-release-workflow.test.ts index fb5c42ee286..8d64e1e109c 100644 --- a/test/scripts/openclaw-cross-os-release-workflow.test.ts +++ b/test/scripts/openclaw-cross-os-release-workflow.test.ts @@ -10,6 +10,8 @@ describe("cross-OS release checks workflow", () => { const workflow = readFileSync(WORKFLOW_PATH, "utf8"); expect(workflow).toContain(HARNESS); + expect(workflow).toContain("suite_filter:"); + expect(workflow).toContain('--suite-filter "${INPUT_SUITE_FILTER}"'); expect(workflow).not.toContain('pnpm dlx "tsx@${TSX_VERSION}"'); }); diff --git a/test/scripts/package-acceptance-workflow.test.ts b/test/scripts/package-acceptance-workflow.test.ts index 3e1fd42f7de..9e325e78a81 100644 --- a/test/scripts/package-acceptance-workflow.test.ts +++ b/test/scripts/package-acceptance-workflow.test.ts @@ -307,6 +307,7 @@ describe("package artifact reuse", () => { expect(workflow).toContain("live_suite_filter:"); expect(workflow).toContain("validate_live_suite_filter:"); expect(workflow).toContain("LIVE_SUITE_FILTER: ${{ inputs.live_suite_filter }}"); + expect(workflow).toContain("live-cache attempt ${attempt}/3"); expect(workflow).toContain( "live_suite_filter '${LIVE_SUITE_FILTER}' does not match any runnable suite", ); @@ -548,6 +549,10 @@ describe("package artifact reuse", () => { ); expect(workflow).toContain("rerun_group:"); expect(workflow).toContain("live_suite_filter:"); + expect(workflow).toContain("cross_os_suite_filter:"); + expect(workflow).toContain( + "suite_filter: ${{ needs.resolve_target.outputs.cross_os_suite_filter }}", + ); expect(workflow).toContain( "live_suite_filter: ${{ needs.resolve_target.outputs.live_suite_filter }}", ); @@ -559,6 +564,7 @@ describe("package artifact reuse", () => { ); expect(workflow).toContain("- live-e2e"); expect(workflow).toContain("- qa-live"); + expect(workflow).toContain("QA release-check lanes are advisory"); }); it("detects Matrix fail-fast support for older release refs", () => { @@ -654,6 +660,7 @@ describe("package artifact reuse", () => { "child_rerun_group=all", '-f rerun_group="$child_rerun_group"', 'args+=(-f live_suite_filter="$LIVE_SUITE_FILTER")', + 'args+=(-f cross_os_suite_filter="$CROSS_OS_SUITE_FILTER")', "cancel-in-progress: ${{ inputs.ref == 'main' && inputs.rerun_group == 'all' }}", "gh run cancel", "NORMAL_CI_RESULT: ${{ needs.normal_ci.result }}", @@ -678,6 +685,8 @@ describe("package artifact reuse", () => { ); expectTextToIncludeAll(fullReleaseDocs, [ "pre-publish candidate", + "cross_os_suite_filter", + "QA release-check lanes are advisory", "silently skip that", "Telegram package lane", "| `npm-telegram` | Published-package Telegram E2E; requires `npm_telegram_package_spec`. |",