From 4b4dfe429da66958d52a844d0f5e8424ebd70f50 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 2 May 2026 19:45:52 +0100 Subject: [PATCH] ci: expand release upgrade baselines --- .github/workflows/full-release-validation.yml | 18 +++++++++++++++++ .github/workflows/openclaw-release-checks.yml | 19 ++++++++++++++++-- .github/workflows/package-acceptance.yml | 4 ++-- docs/ci.md | 2 +- docs/help/testing-updates-plugins.md | 18 +++++++++++------ docs/help/testing.md | 2 +- docs/reference/RELEASING.md | 10 ++++++++-- docs/reference/full-release-validation.md | 8 +++++++- docs/reference/test.md | 2 +- .../package-acceptance-workflow.test.ts | 20 +++++++++++++++++-- 10 files changed, 85 insertions(+), 18 deletions(-) diff --git a/.github/workflows/full-release-validation.yml b/.github/workflows/full-release-validation.yml index 72b2cab3bc4..708b822ffbe 100644 --- a/.github/workflows/full-release-validation.yml +++ b/.github/workflows/full-release-validation.yml @@ -68,6 +68,11 @@ on: required: false default: "" type: string + package_acceptance_package_spec: + description: Optional published package spec for Package Acceptance; blank uses the SHA-built release artifact + required: false + default: "" + type: string npm_telegram_provider_mode: description: Provider mode for the package Telegram E2E lane required: false @@ -127,6 +132,7 @@ jobs: CHILD_WORKFLOW_REF: ${{ github.ref_name }} NPM_TELEGRAM_PACKAGE_SPEC: ${{ inputs.npm_telegram_package_spec }} EVIDENCE_PACKAGE_SPEC: ${{ inputs.evidence_package_spec }} + PACKAGE_ACCEPTANCE_PACKAGE_SPEC: ${{ inputs.package_acceptance_package_spec }} RELEASE_PROFILE: ${{ inputs.release_profile }} RERUN_GROUP: ${{ inputs.rerun_group }} LIVE_SUITE_FILTER: ${{ inputs.live_suite_filter }} @@ -166,6 +172,11 @@ jobs: if [[ -n "${EVIDENCE_PACKAGE_SPEC// }" ]]; then echo "- Private evidence package proof: \`${EVIDENCE_PACKAGE_SPEC}\`" fi + if [[ -n "${PACKAGE_ACCEPTANCE_PACKAGE_SPEC// }" ]]; then + echo "- Package Acceptance package spec: \`${PACKAGE_ACCEPTANCE_PACKAGE_SPEC}\`" + else + echo "- Package Acceptance package spec: SHA-built release artifact" + fi } >> "$GITHUB_STEP_SUMMARY" normal_ci: @@ -377,6 +388,7 @@ jobs: RELEASE_PROFILE: ${{ inputs.release_profile }} RERUN_GROUP: ${{ inputs.rerun_group }} LIVE_SUITE_FILTER: ${{ inputs.live_suite_filter }} + PACKAGE_ACCEPTANCE_PACKAGE_SPEC: ${{ inputs.package_acceptance_package_spec }} run: | set -euo pipefail @@ -455,6 +467,9 @@ jobs: if [[ -n "${LIVE_SUITE_FILTER// }" ]]; then echo "- Live suite filter: \`${LIVE_SUITE_FILTER}\`" fi + if [[ -n "${PACKAGE_ACCEPTANCE_PACKAGE_SPEC// }" ]]; then + echo "- Package Acceptance package spec: \`${PACKAGE_ACCEPTANCE_PACKAGE_SPEC}\`" + fi } >> "$GITHUB_STEP_SUMMARY" child_rerun_group="$RERUN_GROUP" @@ -473,6 +488,9 @@ jobs: if [[ -n "${LIVE_SUITE_FILTER// }" ]]; then args+=(-f live_suite_filter="$LIVE_SUITE_FILTER") fi + if [[ -n "${PACKAGE_ACCEPTANCE_PACKAGE_SPEC// }" ]]; then + args+=(-f package_acceptance_package_spec="$PACKAGE_ACCEPTANCE_PACKAGE_SPEC") + fi dispatch_and_wait openclaw-release-checks.yml "${args[@]}" diff --git a/.github/workflows/openclaw-release-checks.yml b/.github/workflows/openclaw-release-checks.yml index eb1f431eff7..a0e27e277a0 100644 --- a/.github/workflows/openclaw-release-checks.yml +++ b/.github/workflows/openclaw-release-checks.yml @@ -58,6 +58,11 @@ on: required: false default: "" type: string + package_acceptance_package_spec: + description: Optional published package spec for Package Acceptance; blank uses the prepared release artifact + required: false + default: "" + type: string concurrency: group: openclaw-release-checks-${{ inputs.expected_sha || inputs.ref }}-${{ inputs.rerun_group }} @@ -83,6 +88,7 @@ jobs: release_profile: ${{ steps.inputs.outputs.release_profile }} rerun_group: ${{ steps.inputs.outputs.rerun_group }} live_suite_filter: ${{ steps.inputs.outputs.live_suite_filter }} + package_acceptance_package_spec: ${{ steps.inputs.outputs.package_acceptance_package_spec }} steps: - name: Require main or release workflow ref for release checks env: @@ -199,6 +205,7 @@ jobs: RELEASE_PROFILE_INPUT: ${{ inputs.release_profile }} RELEASE_RERUN_GROUP_INPUT: ${{ inputs.rerun_group }} RELEASE_LIVE_SUITE_FILTER_INPUT: ${{ inputs.live_suite_filter }} + RELEASE_PACKAGE_ACCEPTANCE_PACKAGE_SPEC_INPUT: ${{ inputs.package_acceptance_package_spec }} run: | set -euo pipefail { @@ -208,6 +215,7 @@ jobs: printf 'release_profile=%s\n' "$RELEASE_PROFILE_INPUT" printf 'rerun_group=%s\n' "$RELEASE_RERUN_GROUP_INPUT" printf 'live_suite_filter=%s\n' "$RELEASE_LIVE_SUITE_FILTER_INPUT" + printf 'package_acceptance_package_spec=%s\n' "$RELEASE_PACKAGE_ACCEPTANCE_PACKAGE_SPEC_INPUT" } >> "$GITHUB_OUTPUT" - name: Summarize validated ref @@ -220,6 +228,7 @@ jobs: RELEASE_PROFILE: ${{ inputs.release_profile }} RELEASE_RERUN_GROUP: ${{ inputs.rerun_group }} RELEASE_LIVE_SUITE_FILTER: ${{ inputs.live_suite_filter }} + PACKAGE_ACCEPTANCE_PACKAGE_SPEC: ${{ inputs.package_acceptance_package_spec }} run: | { echo "## Release checks" @@ -234,6 +243,11 @@ jobs: if [[ -n "${RELEASE_LIVE_SUITE_FILTER// }" ]]; then echo "- Live suite filter: \`${RELEASE_LIVE_SUITE_FILTER}\`" fi + if [[ -n "${PACKAGE_ACCEPTANCE_PACKAGE_SPEC// }" ]]; then + echo "- Package Acceptance package spec: \`${PACKAGE_ACCEPTANCE_PACKAGE_SPEC}\`" + else + echo "- Package Acceptance package spec: prepared release artifact" + fi echo "- This run will execute cross-OS release validation, install smoke, QA Lab parity, Matrix, and Telegram lanes, and the non-Parallels Docker/live/openwebui coverage from the CI migration plan." } >> "$GITHUB_STEP_SUMMARY" @@ -439,12 +453,13 @@ jobs: uses: ./.github/workflows/package-acceptance.yml with: workflow_ref: ${{ github.ref_name }} - source: artifact + source: ${{ needs.resolve_target.outputs.package_acceptance_package_spec != '' && 'npm' || 'artifact' }} + package_spec: ${{ needs.resolve_target.outputs.package_acceptance_package_spec || 'openclaw@beta' }} artifact_name: ${{ needs.prepare_release_package.outputs.artifact_name }} package_sha256: ${{ needs.prepare_release_package.outputs.package_sha256 }} suite_profile: custom docker_lanes: doctor-switch update-channel-switch upgrade-survivor published-upgrade-survivor plugins-offline plugin-update - published_upgrade_survivor_baselines: release-history + published_upgrade_survivor_baselines: all-since-2026.4.23 published_upgrade_survivor_scenarios: reported-issues telegram_mode: mock-openai telegram_scenarios: telegram-help-command,telegram-commands-command,telegram-tools-compact-command,telegram-whoami-command,telegram-context-command,telegram-mention-gating diff --git a/.github/workflows/package-acceptance.yml b/.github/workflows/package-acceptance.yml index c94f81e8c44..5ca4d5f28c9 100644 --- a/.github/workflows/package-acceptance.yml +++ b/.github/workflows/package-acceptance.yml @@ -70,7 +70,7 @@ on: default: openclaw@latest type: string published_upgrade_survivor_baselines: - description: Optional baseline list for published-upgrade-survivor/update-migration; use release-history or all-since-2026.4.23 + description: Optional baseline list for published-upgrade-survivor/update-migration; use all-since-2026.4.23, release-history, or exact versions required: false default: "" type: string @@ -150,7 +150,7 @@ on: default: openclaw@latest type: string published_upgrade_survivor_baselines: - description: Optional baseline list for published-upgrade-survivor/update-migration; use release-history or all-since-2026.4.23 + description: Optional baseline list for published-upgrade-survivor/update-migration; use all-since-2026.4.23, release-history, or exact versions required: false default: "" type: string diff --git a/docs/ci.md b/docs/ci.md index bc84c81c7a1..dd587467bb3 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -260,7 +260,7 @@ For the dedicated update and plugin testing policy, including local commands, Docker lanes, Package Acceptance inputs, release defaults, and failure triage, see [Testing updates and plugins](/help/testing-updates-plugins). -Release checks call Package Acceptance with `source=artifact`, the prepared release package artifact, `suite_profile=custom`, `docker_lanes='doctor-switch update-channel-switch upgrade-survivor published-upgrade-survivor plugins-offline plugin-update'`, `published_upgrade_survivor_baselines=release-history`, `published_upgrade_survivor_scenarios=reported-issues`, and `telegram_mode=mock-openai`. This keeps package migration, update, stale-plugin-dependency cleanup, configured-plugin install repair, offline plugin, plugin-update, and Telegram proof on the same resolved package tarball. Cross-OS release checks still cover OS-specific onboarding, installer, and platform behavior; package/update product validation should start with Package Acceptance. The `published-upgrade-survivor` Docker lane validates one published package baseline per run. In Package Acceptance, the resolved `package-under-test` tarball is always the candidate and `published_upgrade_survivor_baseline` selects the fallback published baseline, defaulting to `openclaw@latest`; failed-lane rerun commands preserve that baseline. Set `published_upgrade_survivor_baselines=release-history` to expand the lane across a deduped history matrix: the latest six stable releases, `2026.4.23`, and the latest stable release before `2026-03-15`. Set `published_upgrade_survivor_scenarios=reported-issues` to expand the same baselines across issue-shaped fixtures for Feishu config, preserved bootstrap/persona files, configured OpenClaw plugin installs, tilde log paths, and stale legacy plugin dependency roots. The separate `Update Migration` workflow uses the `update-migration` Docker lane with `all-since-2026.4.23` and `plugin-deps-cleanup` when the question is exhaustive published update cleanup, not normal Full Release CI breadth. Local aggregate runs can pass exact package specs with `OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPECS`, keep a single lane with `OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPEC` such as `openclaw@2026.4.15`, or set `OPENCLAW_UPGRADE_SURVIVOR_SCENARIOS` for the scenario matrix. The published lane configures the baseline with a baked `openclaw config set` command recipe, records recipe steps in `summary.json`, and probes `/healthz`, `/readyz`, plus RPC status after Gateway start. The Windows packaged and installer fresh lanes also verify that an installed package can import a browser-control override from a raw absolute Windows path. The OpenAI cross-OS agent-turn smoke defaults to `OPENCLAW_CROSS_OS_OPENAI_MODEL` when set, otherwise `openai/gpt-5.4`, so the install and gateway proof stays on a GPT-5 test model while avoiding GPT-4.x defaults. +Release checks call Package Acceptance with `source=artifact`, the prepared release package artifact, `suite_profile=custom`, `docker_lanes='doctor-switch update-channel-switch upgrade-survivor published-upgrade-survivor plugins-offline plugin-update'`, `published_upgrade_survivor_baselines=all-since-2026.4.23`, `published_upgrade_survivor_scenarios=reported-issues`, and `telegram_mode=mock-openai`. This keeps package migration, update, stale-plugin-dependency cleanup, configured-plugin install repair, offline plugin, plugin-update, and Telegram proof on the same resolved package tarball. Set `package_acceptance_package_spec` on Full Release Validation or OpenClaw Release Checks to run that same matrix against a shipped npm package instead of the SHA-built artifact. Cross-OS release checks still cover OS-specific onboarding, installer, and platform behavior; package/update product validation should start with Package Acceptance. The `published-upgrade-survivor` Docker lane validates one published package baseline per run. In Package Acceptance, the resolved `package-under-test` tarball is always the candidate and `published_upgrade_survivor_baseline` selects the fallback published baseline, defaulting to `openclaw@latest`; failed-lane rerun commands preserve that baseline. Set `published_upgrade_survivor_baselines=all-since-2026.4.23` to expand Full Release CI across every stable npm release from `2026.4.23` through `latest`; `release-history` remains available for manual wider sampling with the older pre-date anchor. Set `published_upgrade_survivor_scenarios=reported-issues` to expand the same baselines across issue-shaped fixtures for Feishu config, preserved bootstrap/persona files, configured OpenClaw plugin installs, tilde log paths, and stale legacy plugin dependency roots. The separate `Update Migration` workflow uses the `update-migration` Docker lane with `all-since-2026.4.23` and `plugin-deps-cleanup` when the question is exhaustive published update cleanup, not normal Full Release CI breadth. Local aggregate runs can pass exact package specs with `OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPECS`, keep a single lane with `OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPEC` such as `openclaw@2026.4.15`, or set `OPENCLAW_UPGRADE_SURVIVOR_SCENARIOS` for the scenario matrix. The published lane configures the baseline with a baked `openclaw config set` command recipe, records recipe steps in `summary.json`, and probes `/healthz`, `/readyz`, plus RPC status after Gateway start. The Windows packaged and installer fresh lanes also verify that an installed package can import a browser-control override from a raw absolute Windows path. The OpenAI cross-OS agent-turn smoke defaults to `OPENCLAW_CROSS_OS_OPENAI_MODEL` when set, otherwise `openai/gpt-5.4`, so the install and gateway proof stays on a GPT-5 test model while avoiding GPT-4.x defaults. ### Legacy compatibility windows diff --git a/docs/help/testing-updates-plugins.md b/docs/help/testing-updates-plugins.md index e6f0f2b37b7..b2170e92e9f 100644 --- a/docs/help/testing-updates-plugins.md +++ b/docs/help/testing-updates-plugins.md @@ -154,6 +154,11 @@ Candidate sources: - `source=url`: validate an HTTPS tarball with required `package_sha256`. - `source=artifact`: reuse a tarball uploaded by another Actions run. +Full Release Validation uses `source=artifact` by default, built from the +resolved release SHA. For post-publish proof, pass +`package_acceptance_package_spec=openclaw@YYYY.M.D` so the same upgrade matrix +targets the shipped npm package instead. + Release checks call Package Acceptance with the package/update/plugin set: ```text @@ -163,7 +168,7 @@ doctor-switch update-channel-switch upgrade-survivor published-upgrade-survivor They also pass: ```text -published_upgrade_survivor_baselines=release-history +published_upgrade_survivor_baselines=all-since-2026.4.23 published_upgrade_survivor_scenarios=reported-issues telegram_mode=mock-openai ``` @@ -172,10 +177,11 @@ This keeps package migration, update channel switching, stale plugin dependency cleanup, offline plugin coverage, plugin update behavior, and Telegram package QA on the same resolved artifact. -`release-history` is a bounded release-check sample: latest six stable releases, -`2026.4.23`, and one older pre-date anchor. For exhaustive published update -migration coverage, use `all-since-2026.4.23` in the separate Update Migration -workflow instead of Full Release CI. +`all-since-2026.4.23` is the Full Release CI upgrade sample: every stable npm-published release from `2026.4.23` through `latest`. For exhaustive published +update migration coverage, use `all-since-2026.4.23` in the separate Update +Migration workflow instead of Full Release CI. `release-history` remains +available for manual wider sampling when you also want the legacy pre-date +anchor. Run a package profile manually when validating a candidate before release: @@ -186,7 +192,7 @@ gh workflow run package-acceptance.yml \ -f source=npm \ -f package_spec=openclaw@beta \ -f suite_profile=package \ - -f published_upgrade_survivor_baselines=release-history \ + -f published_upgrade_survivor_baselines=all-since-2026.4.23 \ -f published_upgrade_survivor_scenarios=reported-issues \ -f telegram_mode=mock-openai ``` diff --git a/docs/help/testing.md b/docs/help/testing.md index 07eda5b4656..4a0a8e971cf 100644 --- a/docs/help/testing.md +++ b/docs/help/testing.md @@ -627,7 +627,7 @@ The live-model Docker runners also bind-mount only the needed CLI auth homes (or - Npm tarball onboarding/channel/agent smoke: `pnpm test:docker:npm-onboard-channel-agent` installs the packed OpenClaw tarball globally in Docker, configures OpenAI via env-ref onboarding plus Telegram by default, runs doctor, and runs one mocked OpenAI agent turn. Reuse a prebuilt tarball with `OPENCLAW_CURRENT_PACKAGE_TGZ=/path/to/openclaw-*.tgz`, skip the host rebuild with `OPENCLAW_NPM_ONBOARD_HOST_BUILD=0`, or switch channel with `OPENCLAW_NPM_ONBOARD_CHANNEL=discord`. - Update channel switch smoke: `pnpm test:docker:update-channel-switch` installs the packed OpenClaw tarball globally in Docker, switches from package `stable` to git `dev`, verifies the persisted channel and plugin post-update work, then switches back to package `stable` and checks update status. - Upgrade survivor smoke: `pnpm test:docker:upgrade-survivor` installs the packed OpenClaw tarball over a dirty old-user fixture with agents, channel config, plugin allowlists, stale plugin dependency state, and existing workspace/session files. It runs package update plus non-interactive doctor without live provider or channel keys, then starts a loopback Gateway and checks config/state preservation plus startup/status budgets. -- Published upgrade survivor smoke: `pnpm test:docker:published-upgrade-survivor` installs `openclaw@latest` by default, seeds realistic existing-user files, configures that baseline with a baked command recipe, validates the resulting config, updates that published install to the candidate tarball, runs non-interactive doctor, writes `.artifacts/upgrade-survivor/summary.json`, then starts a loopback Gateway and checks configured intents, state preservation, startup, `/healthz`, `/readyz`, and RPC status budgets. Override one baseline with `OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPEC`, ask the aggregate scheduler to expand exact baselines with `OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPECS`, and expand issue-shaped fixtures with `OPENCLAW_UPGRADE_SURVIVOR_SCENARIOS` such as `reported-issues`; the reported-issues set includes `configured-plugin-installs` for automatic external OpenClaw plugin install repair. Package Acceptance exposes those as `published_upgrade_survivor_baseline`, `published_upgrade_survivor_baselines`, and `published_upgrade_survivor_scenarios`. +- Published upgrade survivor smoke: `pnpm test:docker:published-upgrade-survivor` installs `openclaw@latest` by default, seeds realistic existing-user files, configures that baseline with a baked command recipe, validates the resulting config, updates that published install to the candidate tarball, runs non-interactive doctor, writes `.artifacts/upgrade-survivor/summary.json`, then starts a loopback Gateway and checks configured intents, state preservation, startup, `/healthz`, `/readyz`, and RPC status budgets. Override one baseline with `OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPEC`, ask the aggregate scheduler to expand exact baselines with `OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPECS` such as `all-since-2026.4.23`, and expand issue-shaped fixtures with `OPENCLAW_UPGRADE_SURVIVOR_SCENARIOS` such as `reported-issues`; the reported-issues set includes `configured-plugin-installs` for automatic external OpenClaw plugin install repair. Package Acceptance exposes those as `published_upgrade_survivor_baseline`, `published_upgrade_survivor_baselines`, and `published_upgrade_survivor_scenarios`. - Session runtime context smoke: `pnpm test:docker:session-runtime-context` verifies hidden runtime context transcript persistence plus doctor repair of affected duplicated prompt-rewrite branches. - Bun global install smoke: `bash scripts/e2e/bun-global-install-smoke.sh` packs the current tree, installs it with `bun install -g` in an isolated home, and verifies `openclaw infer image providers --json` returns bundled image providers instead of hanging. Reuse a prebuilt tarball with `OPENCLAW_BUN_GLOBAL_SMOKE_PACKAGE_TGZ=/path/to/openclaw-*.tgz`, skip the host build with `OPENCLAW_BUN_GLOBAL_SMOKE_HOST_BUILD=0`, or copy `dist/` from a built Docker image with `OPENCLAW_BUN_GLOBAL_SMOKE_DIST_IMAGE=openclaw-dockerfile-smoke:local`. - Installer Docker smoke: `bash scripts/test-install-sh-docker.sh` shares one npm cache across its root, update, and direct-npm containers. Update smoke defaults to npm `latest` as the stable baseline before upgrading to the candidate tarball. Override with `OPENCLAW_INSTALL_SMOKE_UPDATE_BASELINE=2026.4.22` locally, or with the Install Smoke workflow's `update_baseline_version` input on GitHub. Non-root installer checks keep an isolated npm cache so root-owned cache entries do not mask user-local install behavior. Set `OPENCLAW_INSTALL_SMOKE_NPM_CACHE_DIR=/path/to/cache` to reuse the root/update/direct-npm cache across local reruns. diff --git a/docs/reference/RELEASING.md b/docs/reference/RELEASING.md index 2f6a97e6f77..c9f0cebec9e 100644 --- a/docs/reference/RELEASING.md +++ b/docs/reference/RELEASING.md @@ -123,6 +123,9 @@ the maintainer-only release runbook. Telegram E2E against the `release-package-under-test` artifact from release checks. Provide `npm_telegram_package_spec` after publishing when the same Telegram E2E should prove the published npm package too. Provide + `package_acceptance_package_spec` after publishing when Package Acceptance + should run its package/update matrix against the shipped npm package instead + of the SHA-built artifact. Provide `evidence_package_spec` when the private evidence report should prove that the validation matches a published npm package without forcing Telegram E2E. Example: @@ -472,11 +475,14 @@ Supported candidate sources: `OpenClaw Release Checks` runs Package Acceptance with `source=artifact`, the prepared release package artifact, `suite_profile=custom`, `docker_lanes=doctor-switch update-channel-switch upgrade-survivor published-upgrade-survivor plugins-offline plugin-update`, -`published_upgrade_survivor_baselines=release-history`, +`published_upgrade_survivor_baselines=all-since-2026.4.23`, `published_upgrade_survivor_scenarios=reported-issues`, and `telegram_mode=mock-openai`. Package Acceptance keeps migration, update, stale plugin dependency cleanup, offline plugin fixtures, plugin update, and Telegram -package QA against the same resolved tarball. It is the GitHub-native +package QA against the same resolved tarball. The upgrade matrix covers every stable npm-published baseline from `2026.4.23` through `latest`; use +Package Acceptance with `source=npm` for an already shipped candidate, or +`source=ref`/`source=artifact` for a SHA-backed local npm tarball before +publish. It is the GitHub-native replacement for most of the package/update coverage that previously required Parallels. Cross-OS release checks still matter for OS-specific onboarding, installer, and platform behavior, but package/update product validation should diff --git a/docs/reference/full-release-validation.md b/docs/reference/full-release-validation.md index d2249ef85c2..25198b2d298 100644 --- a/docs/reference/full-release-validation.md +++ b/docs/reference/full-release-validation.md @@ -27,6 +27,12 @@ Child workflows use the trusted workflow ref for the harness and the input `ref` for the candidate under test. That keeps new validation logic available when validating an older release branch or tag. +Package Acceptance normally builds the candidate tarball from the resolved +`ref`, including full-SHA runs dispatched with `pnpm ci:full-release`. After +publish, pass `package_acceptance_package_spec=openclaw@YYYY.M.D` (or +`openclaw@beta`/`openclaw@latest`) to run the same package/update matrix against +the shipped npm package instead. + ## Top-level stages | Stage | Details | @@ -57,7 +63,7 @@ or Docker-facing stages need it. | Cross-OS | **Job:** `cross_os_release_checks`
**Backing workflow:** `OpenClaw Cross-OS Release Checks (Reusable)`
**Tests:** fresh and upgrade lanes on Linux, Windows, and macOS for the selected provider and mode, using the candidate tarball plus a baseline package.
**Rerun:** `rerun_group=cross-os`. | | Repo and live E2E | **Job:** `Run repo/live E2E validation`
**Backing workflow:** `OpenClaw Live And E2E Checks (Reusable)`
**Tests:** repository E2E, live cache, OpenAI websocket streaming, native live provider and plugin shards, and Docker-backed live model/backend/gateway harnesses selected by `release_profile`.
**Rerun:** `rerun_group=live-e2e`, optionally with `live_suite_filter`. | | Docker release path | **Job:** `Run Docker release-path validation`
**Backing workflow:** `OpenClaw Live And E2E Checks (Reusable)`
**Tests:** release-path Docker chunks against the shared package artifact.
**Rerun:** `rerun_group=live-e2e`. | -| Package Acceptance | **Job:** `Run package acceptance`
**Backing workflow:** `Package Acceptance`
**Tests:** offline plugin package fixtures, plugin update, and mock-OpenAI Telegram package acceptance against the same tarball.
**Rerun:** `rerun_group=package`. | +| Package Acceptance | **Job:** `Run package acceptance`
**Backing workflow:** `Package Acceptance`
**Tests:** offline plugin package fixtures, plugin update, mock-OpenAI Telegram package acceptance, and published-upgrade survivor checks from every stable npm release at or after `2026.4.23` against the same tarball.
**Rerun:** `rerun_group=package`. | | QA parity | **Job:** `Run QA Lab parity lane` and `Run QA Lab parity report`
**Backing workflow:** direct jobs
**Tests:** candidate and baseline agentic parity packs, then the parity report.
**Rerun:** `rerun_group=qa-parity` or `rerun_group=qa`. | | QA live Matrix | **Job:** `Run QA Lab live Matrix lane`
**Backing workflow:** direct job
**Tests:** fast live Matrix QA profile in the `qa-live-shared` environment.
**Rerun:** `rerun_group=qa-live` or `rerun_group=qa`. | | QA live Telegram | **Job:** `Run QA Lab live Telegram lane`
**Backing workflow:** direct job
**Tests:** live Telegram QA with Convex CI credential leases.
**Rerun:** `rerun_group=qa-live` or `rerun_group=qa`. | diff --git a/docs/reference/test.md b/docs/reference/test.md index f0447216e4d..13d3cbb68ee 100644 --- a/docs/reference/test.md +++ b/docs/reference/test.md @@ -44,7 +44,7 @@ title: "Tests" - `pnpm test:docker:openwebui`: Starts Dockerized OpenClaw + Open WebUI, signs in through Open WebUI, checks `/api/models`, then runs a real proxied chat through `/api/chat/completions`. Requires a usable live model key (for example OpenAI in `~/.profile`), pulls an external Open WebUI image, and is not expected to be CI-stable like the normal unit/e2e suites. - `pnpm test:docker:mcp-channels`: Starts a seeded Gateway container and a second client container that spawns `openclaw mcp serve`, then verifies routed conversation discovery, transcript reads, attachment metadata, live event queue behavior, outbound send routing, and Claude-style channel + permission notifications over the real stdio bridge. The Claude notification assertion reads the raw stdio MCP frames directly so the smoke reflects what the bridge actually emits. - `pnpm test:docker:upgrade-survivor`: Installs the packed OpenClaw tarball over a dirty old-user fixture, runs package update plus non-interactive doctor without live provider or channel keys, then starts a loopback Gateway and checks that agents, channel config, plugin allowlists, workspace/session files, stale legacy plugin dependency state, startup, and RPC status survive. -- `pnpm test:docker:published-upgrade-survivor`: Installs `openclaw@latest` by default, seeds realistic existing-user files without live provider or channel keys, configures that baseline with a baked `openclaw config set` command recipe, updates that published install to the packed OpenClaw tarball, runs non-interactive doctor, writes `.artifacts/upgrade-survivor/summary.json`, then starts a loopback Gateway and checks that configured intents, workspace/session files, stale plugin config and legacy dependency state, startup, `/healthz`, `/readyz`, and RPC status survive or repair cleanly. Override one baseline with `OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPEC`, expand an exact matrix with `OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPECS`, or add scenario fixtures with `OPENCLAW_UPGRADE_SURVIVOR_SCENARIOS=reported-issues`; the reported-issues set includes `configured-plugin-installs` to verify configured external OpenClaw plugins install automatically during upgrade. Package Acceptance exposes those as `published_upgrade_survivor_baseline`, `published_upgrade_survivor_baselines`, and `published_upgrade_survivor_scenarios`. +- `pnpm test:docker:published-upgrade-survivor`: Installs `openclaw@latest` by default, seeds realistic existing-user files without live provider or channel keys, configures that baseline with a baked `openclaw config set` command recipe, updates that published install to the packed OpenClaw tarball, runs non-interactive doctor, writes `.artifacts/upgrade-survivor/summary.json`, then starts a loopback Gateway and checks that configured intents, workspace/session files, stale plugin config and legacy dependency state, startup, `/healthz`, `/readyz`, and RPC status survive or repair cleanly. Override one baseline with `OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPEC`, expand an exact matrix with `OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPECS` such as `all-since-2026.4.23`, or add scenario fixtures with `OPENCLAW_UPGRADE_SURVIVOR_SCENARIOS=reported-issues`; the reported-issues set includes `configured-plugin-installs` to verify configured external OpenClaw plugins install automatically during upgrade. Package Acceptance exposes those as `published_upgrade_survivor_baseline`, `published_upgrade_survivor_baselines`, and `published_upgrade_survivor_scenarios`. - `pnpm test:docker:update-migration`: Runs the published-upgrade survivor harness in the cleanup-heavy `plugin-deps-cleanup` scenario, starting at `openclaw@2026.4.23` by default. The separate `Update Migration` workflow expands this lane with `baselines=all-since-2026.4.23` so every stable published package from `.23` onward updates to the candidate and proves configured-plugin dependency cleanup outside Full Release CI. - `pnpm test:docker:plugins`: Runs install/update smoke for local path, `file:`, npm registry packages with hoisted dependencies, git moving refs, ClawHub fixtures, marketplace updates, and Claude-bundle enable/inspect. diff --git a/test/scripts/package-acceptance-workflow.test.ts b/test/scripts/package-acceptance-workflow.test.ts index 92b2fb1df20..01904225d67 100644 --- a/test/scripts/package-acceptance-workflow.test.ts +++ b/test/scripts/package-acceptance-workflow.test.ts @@ -92,6 +92,7 @@ describe("package acceptance workflow", () => { expect(workflow).toContain("suite_profile:"); expect(workflow).toContain("published_upgrade_survivor_baseline:"); expect(workflow).toContain("published_upgrade_survivor_baselines:"); + expect(workflow).toContain("all-since-2026.4.23"); expect(workflow).toContain("published_upgrade_survivor_scenarios:"); expect(workflow).toContain("scripts/resolve-upgrade-survivor-baselines.mjs"); expect(workflow).toContain("--history-count 6"); @@ -143,6 +144,10 @@ describe("package acceptance workflow", () => { const releaseChecksWorkflow = readFileSync(RELEASE_CHECKS_WORKFLOW, "utf8"); expect(workflow).toContain("TARGET_SHA: ${{ needs.resolve_target.outputs.sha }}"); + expect(workflow).toContain("package_acceptance_package_spec:"); + expect(workflow).toContain( + 'args+=(-f package_acceptance_package_spec="$PACKAGE_ACCEPTANCE_PACKAGE_SPEC")', + ); expect(workflow).toContain("--json status,conclusion,url,attempt,headSha,jobs"); expect(workflow).toContain("child run used ${head_sha}, expected ${TARGET_SHA}"); expect(workflow).toContain( @@ -150,6 +155,12 @@ describe("package acceptance workflow", () => { ); expect(workflow).toContain("| Child | Result | Minutes | Head SHA | Run |"); expect(releaseChecksWorkflow).toContain("refs/heads/release-ci/[0-9a-f]{12}-[0-9]+"); + expect(releaseChecksWorkflow).toContain( + "source: ${{ needs.resolve_target.outputs.package_acceptance_package_spec != '' && 'npm' || 'artifact' }}", + ); + expect(releaseChecksWorkflow).toContain( + "package_spec: ${{ needs.resolve_target.outputs.package_acceptance_package_spec || 'openclaw@beta' }}", + ); }); it("keeps exhaustive update migration as a separate manual package gate", () => { @@ -459,7 +470,12 @@ describe("package artifact reuse", () => { expect(workflow).toContain("include_release_path_suites: false"); expect(workflow).toContain("include_release_path_suites: true"); expect(workflow).toContain("uses: ./.github/workflows/package-acceptance.yml"); - expect(workflow).toContain("source: artifact"); + expect(workflow).toContain( + "source: ${{ needs.resolve_target.outputs.package_acceptance_package_spec != '' && 'npm' || 'artifact' }}", + ); + expect(workflow).toContain( + "package_spec: ${{ needs.resolve_target.outputs.package_acceptance_package_spec || 'openclaw@beta' }}", + ); expect(workflow).toContain(".artifacts/docker-e2e-package/package-candidate.json"); expect(workflow).toContain( "artifact_name: ${{ needs.prepare_release_package.outputs.artifact_name }}", @@ -471,7 +487,7 @@ describe("package artifact reuse", () => { expect(workflow).toContain( "docker_lanes: doctor-switch update-channel-switch upgrade-survivor published-upgrade-survivor plugins-offline plugin-update", ); - expect(workflow).toContain("published_upgrade_survivor_baselines: release-history"); + expect(workflow).toContain("published_upgrade_survivor_baselines: all-since-2026.4.23"); expect(workflow).toContain("published_upgrade_survivor_scenarios: reported-issues"); expect(workflow).toContain("telegram_mode: mock-openai"); expect(workflow).toContain(