vultr/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-05-06 05:10:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

ci(release): speed up focused release reruns

Browse Source
This commit is contained in:
Peter Steinberger
2026-05-05 02:28:22 +01:00
parent 0131343db8
commit c84b7cbffc
11 changed files with 378 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
.github/workflows/full-release-validation.yml vendored
View File

@@ -63,6 +63,11 @@ on:
required: false
default: ""
type: string
cross_os_suite_filter:
description: Optional focused cross-OS suite filter, e.g. windows/packaged-upgrade or packaged-fresh
required: false
default: ""
type: string
npm_telegram_package_spec:
description: Optional published package spec for the package Telegram E2E lane
required: false
@@ -144,6 +149,7 @@ jobs:
RUN_RELEASE_SOAK: ${{ inputs.run_release_soak || inputs.release_profile == 'full' }}
RERUN_GROUP: ${{ inputs.rerun_group }}
LIVE_SUITE_FILTER: ${{ inputs.live_suite_filter }}
CROSS_OS_SUITE_FILTER: ${{ inputs.cross_os_suite_filter }}
run: |
{
echo "## Full release validation"
@@ -156,6 +162,9 @@ jobs:
if [[ -n "${LIVE_SUITE_FILTER// }" ]]; then
echo "- Live suite filter: \`${LIVE_SUITE_FILTER}\`"
fi
if [[ -n "${CROSS_OS_SUITE_FILTER// }" ]]; then
echo "- Cross-OS suite filter: \`${CROSS_OS_SUITE_FILTER}\`"
fi
if [[ "$RERUN_GROUP" == "all" || "$RERUN_GROUP" == "ci" ]]; then
echo "- Normal CI: \`CI\` with \`target_ref=${TARGET_SHA}\`"
else
@@ -410,6 +419,7 @@ jobs:
RUN_RELEASE_SOAK: ${{ inputs.run_release_soak || inputs.release_profile == 'full' }}
RERUN_GROUP: ${{ inputs.rerun_group }}
LIVE_SUITE_FILTER: ${{ inputs.live_suite_filter }}
CROSS_OS_SUITE_FILTER: ${{ inputs.cross_os_suite_filter }}
PACKAGE_ACCEPTANCE_PACKAGE_SPEC: ${{ inputs.package_acceptance_package_spec }}
run: |
set -euo pipefail
@@ -496,6 +506,9 @@ jobs:
if [[ -n "${LIVE_SUITE_FILTER// }" ]]; then
echo "- Live suite filter: \`${LIVE_SUITE_FILTER}\`"
fi
if [[ -n "${CROSS_OS_SUITE_FILTER// }" ]]; then
echo "- Cross-OS suite filter: \`${CROSS_OS_SUITE_FILTER}\`"
fi
if [[ -n "${PACKAGE_ACCEPTANCE_PACKAGE_SPEC// }" ]]; then
echo "- Package Acceptance package spec: \`${PACKAGE_ACCEPTANCE_PACKAGE_SPEC}\`"
fi
@@ -518,6 +531,9 @@ jobs:
if [[ -n "${LIVE_SUITE_FILTER// }" ]]; then
args+=(-f live_suite_filter="$LIVE_SUITE_FILTER")
fi
if [[ -n "${CROSS_OS_SUITE_FILTER// }" ]]; then
args+=(-f cross_os_suite_filter="$CROSS_OS_SUITE_FILTER")
fi
if [[ -n "${PACKAGE_ACCEPTANCE_PACKAGE_SPEC// }" ]]; then
args+=(-f package_acceptance_package_spec="$PACKAGE_ACCEPTANCE_PACKAGE_SPEC")
fi

12
.github/workflows/openclaw-cross-os-release-checks-reusable.yml vendored
View File

@@ -31,6 +31,11 @@ on:
- fresh
- upgrade
- both
suite_filter:
description: Optional focused cross-OS suite filter, e.g. windows/packaged-upgrade or packaged-fresh
required: false
default: ""
type: string
previous_version:
description: Optional baseline version for installer/dev-update and packaged upgrade
required: false
@@ -100,6 +105,11 @@ on:
description: Which release-check lanes to run
required: true
type: string
suite_filter:
description: Optional focused cross-OS suite filter, e.g. windows/packaged-upgrade or packaged-fresh
required: false
default: ""
type: string
previous_version:
description: Optional baseline version for the upgrade lane (defaults to npm latest)
required: false
@@ -482,6 +492,7 @@ jobs:
env:
INPUT_REF: ${{ inputs.ref }}
INPUT_MODE: ${{ inputs.mode }}
INPUT_SUITE_FILTER: ${{ inputs.suite_filter }}
INPUT_UBUNTU_RUNNER: ${{ inputs.ubuntu_runner }}
INPUT_WINDOWS_RUNNER: ${{ inputs.windows_runner }}
INPUT_MACOS_RUNNER: ${{ inputs.macos_runner }}
@@ -493,6 +504,7 @@ jobs:
--resolve-matrix \
--ref "${INPUT_REF}" \
--mode "${INPUT_MODE}" \
--suite-filter "${INPUT_SUITE_FILTER}" \
--ubuntu-runner "${INPUT_UBUNTU_RUNNER}" \
--windows-runner "${INPUT_WINDOWS_RUNNER}" \
--macos-runner "${INPUT_MACOS_RUNNER}")"

13
.github/workflows/openclaw-live-and-e2e-checks-reusable.yml vendored
View File

@@ -489,7 +489,18 @@ jobs:
fi
- name: Verify live prompt cache floors
run: pnpm test:live:cache
run: |
set -euo pipefail
for attempt in 1 2 3; do
echo "live-cache attempt ${attempt}/3"
if pnpm test:live:cache; then
exit 0
fi
if [[ "$attempt" == "3" ]]; then
exit 1
fi
sleep $((attempt * 15))
done
validate_repo_e2e:
needs: validate_selected_ref

22
.github/workflows/openclaw-release-checks.yml vendored
View File

@@ -63,6 +63,11 @@ on:
required: false
default: ""
type: string
cross_os_suite_filter:
description: Optional focused cross-OS suite filter, e.g. windows/packaged-upgrade or packaged-fresh
required: false
default: ""
type: string
package_acceptance_package_spec:
description: Optional published package spec for Package Acceptance; blank uses the prepared release artifact
required: false
@@ -94,6 +99,7 @@ jobs:
run_release_soak: ${{ steps.inputs.outputs.run_release_soak }}
rerun_group: ${{ steps.inputs.outputs.rerun_group }}
live_suite_filter: ${{ steps.inputs.outputs.live_suite_filter }}
cross_os_suite_filter: ${{ steps.inputs.outputs.cross_os_suite_filter }}
qa_live_matrix_enabled: ${{ steps.inputs.outputs.qa_live_matrix_enabled }}
qa_live_telegram_enabled: ${{ steps.inputs.outputs.qa_live_telegram_enabled }}
qa_live_slack_enabled: ${{ steps.inputs.outputs.qa_live_slack_enabled }}
@@ -215,6 +221,7 @@ jobs:
RELEASE_RUN_RELEASE_SOAK_INPUT: ${{ inputs.run_release_soak }}
RELEASE_RERUN_GROUP_INPUT: ${{ inputs.rerun_group }}
RELEASE_LIVE_SUITE_FILTER_INPUT: ${{ inputs.live_suite_filter }}
RELEASE_CROSS_OS_SUITE_FILTER_INPUT: ${{ inputs.cross_os_suite_filter }}
RELEASE_QA_SLACK_LIVE_CI_ENABLED: ${{ vars.OPENCLAW_QA_SLACK_LIVE_CI_ENABLED || 'false' }}
RELEASE_PACKAGE_ACCEPTANCE_PACKAGE_SPEC_INPUT: ${{ inputs.package_acceptance_package_spec }}
run: |
@@ -292,6 +299,7 @@ jobs:
printf 'run_release_soak=%s\n' "$run_release_soak"
printf 'rerun_group=%s\n' "$RELEASE_RERUN_GROUP_INPUT"
printf 'live_suite_filter=%s\n' "$RELEASE_LIVE_SUITE_FILTER_INPUT"
printf 'cross_os_suite_filter=%s\n' "$RELEASE_CROSS_OS_SUITE_FILTER_INPUT"
printf 'qa_live_matrix_enabled=%s\n' "$qa_live_matrix_enabled"
printf 'qa_live_telegram_enabled=%s\n' "$qa_live_telegram_enabled"
printf 'qa_live_slack_enabled=%s\n' "$qa_live_slack_enabled"
@@ -309,6 +317,7 @@ jobs:
RUN_RELEASE_SOAK: ${{ steps.inputs.outputs.run_release_soak }}
RELEASE_RERUN_GROUP: ${{ inputs.rerun_group }}
RELEASE_LIVE_SUITE_FILTER: ${{ inputs.live_suite_filter }}
RELEASE_CROSS_OS_SUITE_FILTER: ${{ inputs.cross_os_suite_filter }}
PACKAGE_ACCEPTANCE_PACKAGE_SPEC: ${{ inputs.package_acceptance_package_spec }}
run: |
{
@@ -325,6 +334,9 @@ jobs:
if [[ -n "${RELEASE_LIVE_SUITE_FILTER// }" ]]; then
echo "- Live suite filter: \`${RELEASE_LIVE_SUITE_FILTER}\`"
fi
if [[ -n "${RELEASE_CROSS_OS_SUITE_FILTER// }" ]]; then
echo "- Cross-OS suite filter: \`${RELEASE_CROSS_OS_SUITE_FILTER}\`"
fi
echo "- QA live lanes: Matrix \`${{ steps.inputs.outputs.qa_live_matrix_enabled }}\`, Telegram \`${{ steps.inputs.outputs.qa_live_telegram_enabled }}\`, Slack \`${{ steps.inputs.outputs.qa_live_slack_enabled }}\`"
if [[ -n "${PACKAGE_ACCEPTANCE_PACKAGE_SPEC// }" ]]; then
echo "- Package Acceptance package spec: \`${PACKAGE_ACCEPTANCE_PACKAGE_SPEC}\`"
@@ -430,6 +442,7 @@ jobs:
ref: ${{ needs.resolve_target.outputs.revision }}
provider: ${{ needs.resolve_target.outputs.provider }}
mode: ${{ needs.resolve_target.outputs.mode }}
suite_filter: ${{ needs.resolve_target.outputs.cross_os_suite_filter }}
candidate_artifact_name: ${{ needs.prepare_release_package.outputs.artifact_name }}
candidate_file_name: openclaw-current.tgz
candidate_version: ${{ needs.prepare_release_package.outputs.package_version }}
@@ -603,6 +616,7 @@ jobs:
name: Run QA Lab parity lane (${{ matrix.lane }})
needs: [resolve_target]
if: contains(fromJSON('["all","qa","qa-parity"]'), needs.resolve_target.outputs.rerun_group)
continue-on-error: true
runs-on: blacksmith-8vcpu-ubuntu-2404
timeout-minutes: 30
permissions:
@@ -687,6 +701,7 @@ jobs:
name: Run QA Lab parity report
needs: [resolve_target, qa_lab_parity_lane_release_checks]
if: contains(fromJSON('["all","qa","qa-parity"]'), needs.resolve_target.outputs.rerun_group)
continue-on-error: true
runs-on: blacksmith-8vcpu-ubuntu-2404
timeout-minutes: 20
permissions:
@@ -743,6 +758,7 @@ jobs:
name: Run QA Lab live Matrix lane
needs: [resolve_target]
if: contains(fromJSON('["all","qa","qa-live"]'), needs.resolve_target.outputs.rerun_group) && needs.resolve_target.outputs.qa_live_matrix_enabled == 'true'
continue-on-error: true
runs-on: blacksmith-8vcpu-ubuntu-2404
timeout-minutes: 60
permissions:
@@ -820,6 +836,7 @@ jobs:
name: Run QA Lab live Telegram lane
needs: [resolve_target]
if: contains(fromJSON('["all","qa","qa-live"]'), needs.resolve_target.outputs.rerun_group) && needs.resolve_target.outputs.qa_live_telegram_enabled == 'true'
continue-on-error: true
runs-on: blacksmith-8vcpu-ubuntu-2404
timeout-minutes: 60
permissions:
@@ -913,6 +930,7 @@ jobs:
name: Run QA Lab live Slack lane
needs: [resolve_target]
if: contains(fromJSON('["all","qa","qa-live"]'), needs.resolve_target.outputs.rerun_group) && needs.resolve_target.outputs.qa_live_slack_enabled == 'true' && vars.OPENCLAW_QA_SLACK_LIVE_CI_ENABLED == 'true'
continue-on-error: true
runs-on: blacksmith-8vcpu-ubuntu-2404
timeout-minutes: 60
permissions:
@@ -1042,6 +1060,10 @@ jobs:
name="${item%%=*}"
result="${item#*=}"
if [[ "$result" != "success" && "$result" != "skipped" ]]; then
if [[ "$name" == qa_* ]]; then
echo "::warning::${name} ended with ${result}; QA release-check lanes are advisory and do not block release validation."
continue
fi
echo "::error::${name} ended with ${result}"
failed=1
fi

2
docs/ci.md
View File

@@ -199,7 +199,7 @@ Docker release-path soak; `full` forces soak on.
The umbrella records the dispatched child run ids, and the final `Verify full validation` job re-checks current child run conclusions and appends slowest-job tables for each child run. If a child workflow is rerun and turns green, rerun only the parent verifier job to refresh the umbrella result and timing summary.
For recovery, both `Full Release Validation` and `OpenClaw Release Checks` accept `rerun_group`. Use `all` for a release candidate, `ci` for only the normal full CI child, `plugin-prerelease` for only the plugin prerelease child, `release-checks` for every release child, or a narrower group: `install-smoke`, `cross-os`, `live-e2e`, `package`, `qa`, `qa-parity`, `qa-live`, or `npm-telegram` on the umbrella. This keeps a failed release box rerun bounded after a focused fix.
For recovery, both `Full Release Validation` and `OpenClaw Release Checks` accept `rerun_group`. Use `all` for a release candidate, `ci` for only the normal full CI child, `plugin-prerelease` for only the plugin prerelease child, `release-checks` for every release child, or a narrower group: `install-smoke`, `cross-os`, `live-e2e`, `package`, `qa`, `qa-parity`, `qa-live`, or `npm-telegram` on the umbrella. This keeps a failed release box rerun bounded after a focused fix. For one failed cross-OS lane, combine `rerun_group=cross-os` with `cross_os_suite_filter`, for example `windows/packaged-upgrade`; long cross-OS commands emit heartbeat lines and packaged-upgrade summaries include per-phase timings. QA release-check lanes are advisory, so QA-only failures warn but do not block the release-check verifier.
`OpenClaw Release Checks` uses the trusted workflow ref to resolve the selected ref once into a `release-package-under-test` tarball, then passes that artifact to cross-OS checks and Package Acceptance, plus the live/E2E release-path Docker workflow when soak coverage runs. That keeps the package bytes consistent across release boxes and avoids repacking the same candidate in multiple child jobs.

5
docs/reference/RELEASING.md
View File

@@ -379,7 +379,10 @@ runs only the release-only plugin child, `release-checks` runs every release
box, and the narrower release groups are `install-smoke`, `cross-os`,
`live-e2e`, `package`, `qa`, `qa-parity`, `qa-live`, and `npm-telegram`.
Focused `npm-telegram` reruns require `npm_telegram_package_spec`; full/all runs
with `release_profile=full` use the release-checks package artifact.
with `release_profile=full` use the release-checks package artifact. Focused
cross-OS reruns can add `cross_os_suite_filter=windows/packaged-upgrade` or
another OS/suite filter. QA release-check failures are advisory; a QA-only
failure does not block release validation.
### Vitest

11
docs/reference/full-release-validation.md
View File

@@ -158,6 +158,17 @@ Valid filter ids are defined in the reusable live/E2E workflow, including
The `live-gateway-advisory-docker` handle is an aggregate rerun handle for its
three provider shards, so it still fans out to all advisory Docker gateway jobs.
Use `cross_os_suite_filter` with `rerun_group=cross-os` when one cross-OS lane
failed. The filter accepts an OS id, a suite id, or an OS/suite pair, for
example `windows/packaged-upgrade`, `windows`, or `packaged-fresh`. Cross-OS
summaries include per-phase timings for packaged upgrade lanes, and long-running
commands print heartbeat lines so a stuck Windows update is visible before the
job timeout.
QA release-check lanes are advisory. A QA-only failure is reported as a warning
and does not block the release-check verifier; rerun `rerun_group=qa`,
`qa-parity`, or `qa-live` when you need fresh QA evidence.
## Evidence to keep
Keep the `Full Release Validation` summary as the release-level index. It links

334
scripts/openclaw-cross-os-release-checks.ts
View File

@@ -33,6 +33,7 @@ const SUPPORTED_SUITES = new Set([
"packaged-upgrade",
"dev-update",
]);
const SUPPORTED_OS_IDS = new Set(["ubuntu", "windows", "macos"]);
export const CROSS_OS_AGENT_TURN_TIMEOUT_SECONDS = parsePositiveIntegerEnv(
"OPENCLAW_CROSS_OS_AGENT_TURN_TIMEOUT_SECONDS",
@@ -122,6 +123,10 @@ export const CROSS_OS_RELEASE_SMOKE_TOOLS_PROFILE = "minimal";
export const CROSS_OS_WINDOWS_PACKAGED_UPGRADE_STEP_TIMEOUT_SECONDS = 25 * 60;
export const CROSS_OS_WINDOWS_PACKAGED_UPGRADE_WRAPPER_TIMEOUT_MS =
(CROSS_OS_WINDOWS_PACKAGED_UPGRADE_STEP_TIMEOUT_SECONDS + 5 * 60) * 1000;
export const CROSS_OS_COMMAND_HEARTBEAT_SECONDS = parsePositiveIntegerEnv(
"OPENCLAW_CROSS_OS_COMMAND_HEARTBEAT_SECONDS",
60,
);
if (isMainModule()) {
try {
@@ -232,6 +237,7 @@ export function resolveRunnerMatrix(params) {
const pick = (...values) =>
values.find((value) => typeof value === "string" && value.trim().length > 0)?.trim();
const suites = resolveRequestedSuites(params.mode, params.ref);
const suiteFilter = parseCrossOsSuiteFilter(params.suiteFilter ?? "");
const runners = [
{
os_id: "ubuntu",
@@ -252,19 +258,84 @@ export function resolveRunnerMatrix(params) {
artifact_name: "macos",
},
];
return {
include: runners.flatMap((runner) =>
suites.map((suite) =>
const include = runners.flatMap((runner) =>
suites
.filter((suite) => suiteFilter.matches(runner.os_id, suite))
.map((suite) =>
Object.assign({}, runner, {
suite,
suite_label: formatSuiteLabel(suite),
lane: suite.includes(`upgrade`) || suite === `dev-update` ? `upgrade` : `fresh`,
}),
),
),
);
if (include.length === 0) {
throw new Error(
`cross_os_suite_filter ${JSON.stringify(params.suiteFilter ?? "")} did not match any ${params.mode} suite.`,
);
}
return {
include,
};
}
export function parseCrossOsSuiteFilter(rawFilter) {
const tokens = String(rawFilter ?? "")
.split(/[, ]+/u)
.map((token) => normalizeCrossOsSuiteFilterToken(token))
.filter(Boolean);
if (tokens.length === 0) {
return {
matches: () => true,
tokens,
};
}
const matchers = tokens.map((token) => {
if (SUPPORTED_SUITES.has(token)) {
return { osId: "", suite: token };
}
if (SUPPORTED_OS_IDS.has(token)) {
return { osId: token, suite: "" };
}
for (const separator of ["/", ":", "-"]) {
const matchedOs = [...SUPPORTED_OS_IDS].find((osId) =>
token.startsWith(`${osId}${separator}`),
);
if (!matchedOs) {
continue;
}
const suite = token.slice(matchedOs.length + separator.length);
if (!SUPPORTED_SUITES.has(suite)) {
break;
}
return { osId: matchedOs, suite };
}
throw new Error(
`Unsupported cross_os_suite_filter token ${JSON.stringify(token)}. Use an OS id, suite id, or os/suite pair such as windows/packaged-upgrade.`,
);
});
return {
matches: (osId, suite) =>
matchers.some((matcher) => {
const osMatches = !matcher.osId || matcher.osId === osId;
const suiteMatches = !matcher.suite || matcher.suite === suite;
return osMatches && suiteMatches;
}),
tokens,
};
}
function normalizeCrossOsSuiteFilterToken(token) {
return token
.trim()
.toLowerCase()
.replace(/_/gu, "-")
.replace(/\s*[/:-]\s*/gu, (separator) => separator.trim())
.replace(/\s+/gu, "-");
}
export function readRunnerOverrideEnv(env = process.env) {
const preferNonEmptyEnv = (primary: string | undefined, legacy: string | undefined) => {
const primaryValue = primary?.trim();
@@ -319,6 +390,7 @@ async function main(argv) {
ubuntuRunner: args["ubuntu-runner"],
windowsRunner: args["windows-runner"],
macosRunner: args["macos-runner"],
suiteFilter: args["suite-filter"],
...runnerOverrideEnv,
}),
)}\n`,
@@ -733,76 +805,80 @@ async function runUpgradeLane(params) {
const cleanup = [];
try {
const env = buildLaneEnv(lane, params.providerConfig, params.providerSecretValue);
logLanePhase(lane, "install-baseline");
if (!params.baselineTgz && params.baselineSpec) {
await installPackageSpec({
await runTimedLanePhase(lane, "install-baseline", async () => {
if (!params.baselineTgz && params.baselineSpec) {
await installPackageSpec({
lane,
env,
packageSpec: params.baselineSpec,
logPath: join(params.logsDir, "upgrade-install-baseline.log"),
ignoreScripts: true,
});
} else {
await installTarballPackage({
lane,
env,
tgzPath: params.baselineTgz,
logPath: join(params.logsDir, "upgrade-install-baseline.log"),
ignoreScripts: true,
restoreBundledPluginPostinstall: false,
});
}
});
await runTimedLanePhase(lane, "run-baseline-bundled-plugin-postinstall", async () => {
await runBundledPluginPostinstall({
lane,
env,
packageSpec: params.baselineSpec,
logPath: join(params.logsDir, "upgrade-install-baseline.log"),
ignoreScripts: true,
});
} else {
await installTarballPackage({
lane,
env,
tgzPath: params.baselineTgz,
logPath: join(params.logsDir, "upgrade-install-baseline.log"),
ignoreScripts: true,
restoreBundledPluginPostinstall: false,
});
}
logLanePhase(lane, "run-baseline-bundled-plugin-postinstall");
await runBundledPluginPostinstall({
lane,
env,
logPath: join(params.logsDir, "upgrade-install-baseline.log"),
});
const baseline = {
version: readInstalledVersion(lane.prefixDir),
};
logLanePhase(lane, "update");
const updateEnv = buildRealUpdateEnv(env);
const updateArgs = buildPackagedUpgradeUpdateArgs(params.candidateUrl);
const updateLogPath = join(params.logsDir, "upgrade-update.log");
let updateResult;
let usedWindowsPackagedUpgradeTimeoutFallback = false;
try {
updateResult = await runOpenClaw({
lane,
env: updateEnv,
args: updateArgs,
logPath: updateLogPath,
timeoutMs: updateTimeoutMs(),
check: false,
});
} catch (error) {
if (!isRecoverableWindowsPackagedUpgradeTimeoutError(error, process.platform)) {
throw error;
await runTimedLanePhase(lane, "update", async () => {
try {
updateResult = await runOpenClaw({
lane,
env: updateEnv,
args: updateArgs,
logPath: updateLogPath,
timeoutMs: updateTimeoutMs(),
check: false,
});
} catch (error) {
if (!isRecoverableWindowsPackagedUpgradeTimeoutError(error, process.platform)) {
throw error;
}
usedWindowsPackagedUpgradeTimeoutFallback = true;
appendFileSync(
updateLogPath,
`\n[release-checks] Windows baseline updater timed out after fetching candidate; falling back to direct candidate install: ${formatError(error)}\n`,
);
updateResult = {
exitCode: 124,
stdout: "",
stderr: formatError(error),
};
}
usedWindowsPackagedUpgradeTimeoutFallback = true;
appendFileSync(
updateLogPath,
`\n[release-checks] Windows baseline updater timed out after fetching candidate; falling back to direct candidate install: ${formatError(error)}\n`,
);
updateResult = {
exitCode: 124,
stdout: "",
stderr: formatError(error),
};
}
});
const usedWindowsPackagedUpgradeFallback =
usedWindowsPackagedUpgradeTimeoutFallback ||
isRecoverableWindowsPackagedUpgradeSwapCleanupFailure(updateResult, process.platform);
if (usedWindowsPackagedUpgradeFallback) {
logLanePhase(lane, "update-fallback-install");
await installPackageSpec({
lane,
env,
packageSpec: params.candidateUrl,
logPath: join(params.logsDir, "upgrade-update-fallback-install.log"),
await runTimedLanePhase(lane, "update-fallback-install", async () => {
await installPackageSpec({
lane,
env,
packageSpec: params.candidateUrl,
logPath: join(params.logsDir, "upgrade-update-fallback-install.log"),
});
});
} else {
verifyPackagedUpgradeUpdateResult(updateResult, {
@@ -816,69 +892,77 @@ async function runUpgradeLane(params) {
usedWindowsPackagedUpgradeFallback,
})
) {
logLanePhase(lane, "update-status");
await runOpenClaw({
lane,
env: updateEnv,
args: ["update", "status", "--json"],
logPath: join(params.logsDir, "upgrade-update-status.log"),
timeoutMs: 2 * 60 * 1000,
await runTimedLanePhase(lane, "update-status", async () => {
await runOpenClaw({
lane,
env: updateEnv,
args: ["update", "status", "--json"],
logPath: join(params.logsDir, "upgrade-update-status.log"),
timeoutMs: 2 * 60 * 1000,
});
});
}
logLanePhase(lane, "run-bundled-plugin-postinstall");
await runBundledPluginPostinstall({
lane,
env,
logPath: join(params.logsDir, "upgrade-bundled-plugin-postinstall.log"),
await runTimedLanePhase(lane, "run-bundled-plugin-postinstall", async () => {
await runBundledPluginPostinstall({
lane,
env,
logPath: join(params.logsDir, "upgrade-bundled-plugin-postinstall.log"),
});
});
const installed = readInstalledMetadata(lane.prefixDir);
verifyInstalledCandidate(installed, params.build);
logLanePhase(lane, "onboard");
await runOnboard({
lane,
env,
providerConfig: params.providerConfig,
logPath: join(params.logsDir, "upgrade-onboard.log"),
await runTimedLanePhase(lane, "onboard", async () => {
await runOnboard({
lane,
env,
providerConfig: params.providerConfig,
logPath: join(params.logsDir, "upgrade-onboard.log"),
});
});
logLanePhase(lane, "models-set");
await runModelsSet({
lane,
env,
providerConfig: params.providerConfig,
logPath: join(params.logsDir, "upgrade-models-set.log"),
await runTimedLanePhase(lane, "models-set", async () => {
await runModelsSet({
lane,
env,
providerConfig: params.providerConfig,
logPath: join(params.logsDir, "upgrade-models-set.log"),
});
});
logLanePhase(lane, "start-gateway");
const gateway = await startGateway({
lane,
env,
logPath: join(params.logsDir, "upgrade-gateway.log"),
});
const gateway = await runTimedLanePhase(lane, "start-gateway", async () =>
startGateway({
lane,
env,
logPath: join(params.logsDir, "upgrade-gateway.log"),
}),
);
cleanup.push(() => stopGateway(gateway));
logLanePhase(lane, "wait-gateway");
await waitForGateway({
lane,
env,
logPath: join(params.logsDir, "upgrade-gateway-status.log"),
await runTimedLanePhase(lane, "wait-gateway", async () => {
await waitForGateway({
lane,
env,
logPath: join(params.logsDir, "upgrade-gateway-status.log"),
});
});
logLanePhase(lane, "dashboard");
await runDashboardSmoke({
lane,
logPath: join(params.logsDir, "upgrade-dashboard.log"),
await runTimedLanePhase(lane, "dashboard", async () => {
await runDashboardSmoke({
lane,
logPath: join(params.logsDir, "upgrade-dashboard.log"),
});
});
logLanePhase(lane, "agent-turn");
const agent = await runAgentTurn({
lane,
env,
label: "upgrade",
logPath: join(params.logsDir, "upgrade-agent.log"),
});
const agent = await runTimedLanePhase(lane, "agent-turn", async () =>
runAgentTurn({
lane,
env,
label: "upgrade",
logPath: join(params.logsDir, "upgrade-agent.log"),
}),
);
return {
status: "pass",
@@ -888,6 +972,7 @@ async function runUpgradeLane(params) {
dashboardStatus: "pass",
gatewayPort: lane.gatewayPort,
agentOutput: trimForSummary(agent.stdout),
phaseTimings: lane.phaseTimings,
};
} finally {
await runCleanup(cleanup);
@@ -1245,6 +1330,7 @@ function createLaneState(name) {
stateDir,
appDataDir,
gatewayPort: 0,
phaseTimings: [],
};
}
@@ -3268,6 +3354,10 @@ async function runCommand(command, args, options) {
let stderr = "";
let timedOut = false;
let settled = false;
const startedAt = Date.now();
let killWaitTimer = null;
let timer = null;
let heartbeatTimer = null;
const clearTimers = () => {
if (timer) {
@@ -3276,6 +3366,9 @@ async function runCommand(command, args, options) {
if (killWaitTimer) {
clearTimeout(killWaitTimer);
}
if (heartbeatTimer) {
clearInterval(heartbeatTimer);
}
};
const finalize = (callback) => {
@@ -3307,8 +3400,7 @@ async function runCommand(command, args, options) {
child.kill(process.platform === "win32" ? undefined : "SIGKILL");
};
let killWaitTimer = null;
const timer =
timer =
options.timeoutMs && Number.isFinite(options.timeoutMs)
? setTimeout(() => {
timedOut = true;
@@ -3327,6 +3419,20 @@ async function runCommand(command, args, options) {
}, 15_000);
}, options.timeoutMs)
: null;
heartbeatTimer =
CROSS_OS_COMMAND_HEARTBEAT_SECONDS > 0
? setInterval(() => {
const elapsedSeconds = Math.floor((Date.now() - startedAt) / 1000);
const message = `${new Date().toISOString()} still running after ${elapsedSeconds}s: ${command} ${args.join(" ")}\n`;
logStream.write(message);
process.stdout.write(`[release-checks] ${message}`);
}, CROSS_OS_COMMAND_HEARTBEAT_SECONDS * 1000)
: null;
heartbeatTimer?.unref?.();
logStream.write(
`${new Date().toISOString()} start command=${command} args=${args.join(" ")}\n`,
);
child.stdout?.on("data", (chunk) => {
const text = chunk.toString();
@@ -3454,6 +3560,13 @@ function writeSummary(baseDir, summaryPayload) {
result.agentOutput ? `- Agent output: \`${trimForSummary(result.agentOutput)}\`` : "",
result.error ? `- Error: \`${trimForSummary(result.error)}\`` : "",
].filter(Boolean);
if (Array.isArray(result.phaseTimings) && result.phaseTimings.length > 0) {
lines.push("", "### Phase timings");
for (const phase of result.phaseTimings) {
const suffix = phase.status === "pass" ? "" : ` (${phase.status})`;
lines.push(`- \`${phase.name}\`: ${Math.round(phase.durationMs / 1000)}s${suffix}`);
}
}
writeFileSync(summaryMarkdownPath, `${lines.join("\n")}\n`, "utf8");
}
@@ -3522,6 +3635,23 @@ function logLanePhase(lane, phase) {
logPhase(`lane.${lane.name}`, phase);
}
async function runTimedLanePhase(lane, phase, callback) {
const startedAt = Date.now();
logLanePhase(lane, phase);
try {
const result = await callback();
const durationMs = Date.now() - startedAt;
lane.phaseTimings.push({ name: phase, status: "pass", durationMs });
logPhase(`lane.${lane.name}`, `${phase}: done in ${Math.round(durationMs / 1000)}s`);
return result;
} catch (error) {
const durationMs = Date.now() - startedAt;
lane.phaseTimings.push({ name: phase, status: "fail", durationMs });
logPhase(`lane.${lane.name}`, `${phase}: failed in ${Math.round(durationMs / 1000)}s`);
throw error;
}
}
function trimForSummary(value) {
const trimmed = value.trim();
if (trimmed.length <= 600) {

57
test/scripts/openclaw-cross-os-release-checks.test.ts
View File

@@ -36,6 +36,7 @@ import {
CROSS_OS_DASHBOARD_FETCH_TIMEOUT_MS,
CROSS_OS_DASHBOARD_SMOKE_TIMEOUT_MS,
CROSS_OS_AGENT_TURN_TIMEOUT_SECONDS,
CROSS_OS_COMMAND_HEARTBEAT_SECONDS,
isImmutableReleaseRef,
isRecoverableWindowsPackagedUpgradeSwapCleanupFailure,
isRecoverableWindowsPackagedUpgradeTimeoutError,
@@ -43,6 +44,7 @@ import {
normalizeRequestedRef,
normalizeWindowsCommandShimPath,
normalizeWindowsInstalledCliPath,
parseCrossOsSuiteFilter,
parseArgs,
packageHasScript,
readInstalledVersion,
@@ -97,6 +99,11 @@ describe("scripts/openclaw-cross-os-release-checks", () => {
).toBeGreaterThanOrEqual(5 * 60 * 1000);
});
it("prints command heartbeats before long release commands hit job timeouts", () => {
expect(CROSS_OS_COMMAND_HEARTBEAT_SECONDS).toBeGreaterThan(0);
expect(CROSS_OS_COMMAND_HEARTBEAT_SECONDS).toBeLessThanOrEqual(60);
});
it("accepts OK agent output from the captured log when stdout is empty", () => {
const dir = mkdtempSync(join(tmpdir(), "openclaw-cross-os-agent-output-"));
try {
@@ -372,6 +379,56 @@ describe("scripts/openclaw-cross-os-release-checks", () => {
);
});
it("filters the cross-OS runner matrix to a focused OS suite", () => {
const matrix = resolveRunnerMatrix({
mode: "both",
ref: "main",
suiteFilter: "windows/packaged-upgrade",
ubuntuRunner: "",
windowsRunner: "",
macosRunner: "",
varUbuntuRunner: "",
varWindowsRunner: "",
varMacosRunner: "",
});
expect(matrix.include).toEqual([
expect.objectContaining({
os_id: "windows",
suite: "packaged-upgrade",
lane: "upgrade",
}),
]);
});
it("filters the cross-OS runner matrix by suite across platforms", () => {
const matrix = resolveRunnerMatrix({
mode: "both",
ref: "main",
suiteFilter: "packaged-fresh",
ubuntuRunner: "",
windowsRunner: "",
macosRunner: "",
varUbuntuRunner: "",
varWindowsRunner: "",
varMacosRunner: "",
});
expect(matrix.include).toHaveLength(3);
expect(matrix.include.map((entry) => entry.os_id).toSorted()).toEqual([
"macos",
"ubuntu",
"windows",
]);
expect(matrix.include.every((entry) => entry.suite === "packaged-fresh")).toBe(true);
});
it("rejects unsupported cross-OS suite filter tokens", () => {
expect(() => parseCrossOsSuiteFilter("windows/nope")).toThrow(
/Unsupported cross_os_suite_filter/u,
);
});
it("can rebuild the Windows PATH with or without current-process entries", () => {
expect(buildWindowsPathBootstrapScript()).toContain("@($userPath, $machinePath, $env:Path)");
const persistedOnlyScript = buildWindowsPathBootstrapScript({

2
test/scripts/openclaw-cross-os-release-workflow.test.ts
View File

@@ -10,6 +10,8 @@ describe("cross-OS release checks workflow", () => {
const workflow = readFileSync(WORKFLOW_PATH, "utf8");
expect(workflow).toContain(HARNESS);
expect(workflow).toContain("suite_filter:");
expect(workflow).toContain('--suite-filter "${INPUT_SUITE_FILTER}"');
expect(workflow).not.toContain('pnpm dlx "tsx@${TSX_VERSION}"');
});

9
test/scripts/package-acceptance-workflow.test.ts
View File

@@ -307,6 +307,7 @@ describe("package artifact reuse", () => {
expect(workflow).toContain("live_suite_filter:");
expect(workflow).toContain("validate_live_suite_filter:");
expect(workflow).toContain("LIVE_SUITE_FILTER: ${{ inputs.live_suite_filter }}");
expect(workflow).toContain("live-cache attempt ${attempt}/3");
expect(workflow).toContain(
"live_suite_filter '${LIVE_SUITE_FILTER}' does not match any runnable suite",
);
@@ -548,6 +549,10 @@ describe("package artifact reuse", () => {
);
expect(workflow).toContain("rerun_group:");
expect(workflow).toContain("live_suite_filter:");
expect(workflow).toContain("cross_os_suite_filter:");
expect(workflow).toContain(
"suite_filter: ${{ needs.resolve_target.outputs.cross_os_suite_filter }}",
);
expect(workflow).toContain(
"live_suite_filter: ${{ needs.resolve_target.outputs.live_suite_filter }}",
);
@@ -559,6 +564,7 @@ describe("package artifact reuse", () => {
);
expect(workflow).toContain("- live-e2e");
expect(workflow).toContain("- qa-live");
expect(workflow).toContain("QA release-check lanes are advisory");
});
it("detects Matrix fail-fast support for older release refs", () => {
@@ -654,6 +660,7 @@ describe("package artifact reuse", () => {
"child_rerun_group=all",
'-f rerun_group="$child_rerun_group"',
'args+=(-f live_suite_filter="$LIVE_SUITE_FILTER")',
'args+=(-f cross_os_suite_filter="$CROSS_OS_SUITE_FILTER")',
"cancel-in-progress: ${{ inputs.ref == 'main' && inputs.rerun_group == 'all' }}",
"gh run cancel",
"NORMAL_CI_RESULT: ${{ needs.normal_ci.result }}",
@@ -678,6 +685,8 @@ describe("package artifact reuse", () => {
);
expectTextToIncludeAll(fullReleaseDocs, [
"pre-publish candidate",
"cross_os_suite_filter",
"QA release-check lanes are advisory",
"silently skip that",
"Telegram package lane",
"| `npm-telegram` | Published-package Telegram E2E; requires `npm_telegram_package_spec`. |",