From 39e3d8d31de860288dd7f408e1612bfbcb128208 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 27 Apr 2026 23:38:09 +0100 Subject: [PATCH] ci: shard release validation reruns --- .agents/skills/openclaw-testing/SKILL.md | 23 ++++++- .github/workflows/full-release-validation.yml | 65 +++++++++++++++++-- .../openclaw-live-and-e2e-checks-reusable.yml | 64 ++++++++++++++---- .github/workflows/openclaw-release-checks.yml | 26 ++++++++ docs/ci.md | 14 +++- docs/help/testing.md | 2 +- docs/reference/RELEASING.md | 15 +++-- scripts/lib/docker-e2e-scenarios.mjs | 48 +++++++++++--- src/plugins/bundled-runtime-deps.ts | 10 +-- src/plugins/bundled-runtime-root.test.ts | 10 ++- src/plugins/loader.test.ts | 10 ++- test/scripts/docker-e2e-plan.test.ts | 54 +++++++++++---- .../package-acceptance-workflow.test.ts | 12 +++- 13 files changed, 295 insertions(+), 58 deletions(-) diff --git a/.agents/skills/openclaw-testing/SKILL.md b/.agents/skills/openclaw-testing/SKILL.md index b3c964355a7..7098320cbf4 100644 --- a/.agents/skills/openclaw-testing/SKILL.md +++ b/.agents/skills/openclaw-testing/SKILL.md @@ -141,6 +141,11 @@ parent gate. If a child workflow failed but was later rerun successfully, rerun only the failed parent verifier job; do not dispatch a new full umbrella unless the release evidence is stale. +For bounded recovery after a focused fix, pass `-f rerun_group=`. +Supported umbrella groups are `all`, `ci`, `release-checks`, `install-smoke`, +`cross-os`, `live-e2e`, `package`, `qa`, `qa-parity`, `qa-live`, and +`npm-telegram`. Use the narrowest group that covers the failed box. + ### Release Evidence After release-candidate validation or before a release decision, record the @@ -192,9 +197,13 @@ gh workflow run openclaw-release-checks.yml \ --ref main \ -f ref= \ -f provider=openai \ - -f mode=both + -f mode=both \ + -f rerun_group=all ``` +Release-check rerun groups are `all`, `install-smoke`, `cross-os`, `live-e2e`, +`package`, `qa`, `qa-parity`, and `qa-live`. + ### QA Lab Matrix Profiles `pnpm openclaw qa matrix` defaults to `--profile all`. Do not assume the CLI @@ -236,7 +245,9 @@ gh workflow run openclaw-live-and-e2e-checks-reusable.yml \ Useful knobs: - `docker_lanes=''`: run selected Docker scheduler lanes against - prepared artifacts instead of the release chunk matrix. + prepared artifacts instead of the release chunk matrix. Multiple selected + lanes fan out as parallel targeted Docker jobs after one shared package/image + preparation step. - `include_live_suites=false`: skip live/provider suites when testing Docker scheduler or release packaging only. - `live_models_only=true`: run only Docker live model coverage. @@ -244,6 +255,14 @@ Useful knobs: targeted Docker live model job instead of the full provider matrix. - blank `live_model_providers`: run the full live-model provider matrix. +Release-path Docker chunks are currently `core`, `package-update-openai`, +`package-update-anthropic`, `package-update-core`, `plugins-runtime-core`, +`plugins-runtime-install-a`, `plugins-runtime-install-b`, +`bundled-channels-core`, `bundled-channels-update-a`, +`bundled-channels-update-b`, and `bundled-channels-contracts`. The aggregate +`bundled-channels` chunk remains valid for manual one-shot reruns, but release +checks use the split chunks. + When live suites are enabled, the workflow shards broad native `pnpm test:live` coverage through `scripts/test-live-shard.mjs` instead of one serial `live-all` job: diff --git a/.github/workflows/full-release-validation.yml b/.github/workflows/full-release-validation.yml index bcacd0d3ece..556b0e9cdee 100644 --- a/.github/workflows/full-release-validation.yml +++ b/.github/workflows/full-release-validation.yml @@ -26,6 +26,23 @@ on: - fresh - upgrade - both + rerun_group: + description: Validation group to run + required: false + default: all + type: choice + options: + - all + - ci + - release-checks + - install-smoke + - cross-os + - live-e2e + - package + - qa + - qa-parity + - qa-live + - npm-telegram npm_telegram_package_spec: description: Optional published package spec for the post-publish Telegram E2E lane required: false @@ -89,6 +106,7 @@ jobs: CHILD_WORKFLOW_REF: ${{ github.ref_name }} NPM_TELEGRAM_PACKAGE_SPEC: ${{ inputs.npm_telegram_package_spec }} EVIDENCE_PACKAGE_SPEC: ${{ inputs.evidence_package_spec }} + RERUN_GROUP: ${{ inputs.rerun_group }} run: | { echo "## Full release validation" @@ -96,8 +114,17 @@ jobs: echo "- Target ref: \`${TARGET_REF}\`" echo "- Target SHA: \`${TARGET_SHA}\`" echo "- Child workflow ref: \`${CHILD_WORKFLOW_REF}\`" - echo "- Normal CI: \`CI\` with \`target_ref=${TARGET_SHA}\`" - echo "- Release/live/Docker/package/QA: \`OpenClaw Release Checks\`" + echo "- Rerun group: \`${RERUN_GROUP}\`" + if [[ "$RERUN_GROUP" == "all" || "$RERUN_GROUP" == "ci" ]]; then + echo "- Normal CI: \`CI\` with \`target_ref=${TARGET_SHA}\`" + else + echo "- Normal CI: skipped by rerun group" + fi + if [[ "$RERUN_GROUP" != "ci" && "$RERUN_GROUP" != "npm-telegram" ]]; then + echo "- Release/live/Docker/package/QA: \`OpenClaw Release Checks\`" + else + echo "- Release/live/Docker/package/QA: skipped by rerun group" + fi if [[ -n "${NPM_TELEGRAM_PACKAGE_SPEC// }" ]]; then echo "- Post-publish Telegram E2E: \`${NPM_TELEGRAM_PACKAGE_SPEC}\`" else @@ -111,6 +138,7 @@ jobs: normal_ci: name: Run normal full CI needs: [resolve_target] + if: contains(fromJSON('["all","ci"]'), inputs.rerun_group) runs-on: ubuntu-24.04 timeout-minutes: 240 outputs: @@ -194,6 +222,7 @@ jobs: release_checks: name: Run release/live/Docker/QA validation needs: [resolve_target] + if: contains(fromJSON('["all","release-checks","install-smoke","cross-os","live-e2e","package","qa","qa-parity","qa-live"]'), inputs.rerun_group) runs-on: ubuntu-24.04 timeout-minutes: 720 outputs: @@ -210,6 +239,7 @@ jobs: CHILD_WORKFLOW_REF: ${{ github.ref_name }} PROVIDER: ${{ inputs.provider }} MODE: ${{ inputs.mode }} + RERUN_GROUP: ${{ inputs.rerun_group }} run: | set -euo pipefail @@ -274,17 +304,24 @@ jobs: echo "- Target SHA: \`${TARGET_SHA}\`" echo "- Provider: \`${PROVIDER}\`" echo "- Cross-OS mode: \`${MODE}\`" + echo "- Rerun group: \`${RERUN_GROUP}\`" } >> "$GITHUB_STEP_SUMMARY" + child_rerun_group="$RERUN_GROUP" + if [[ "$child_rerun_group" == "release-checks" ]]; then + child_rerun_group=all + fi + dispatch_and_wait openclaw-release-checks.yml \ -f ref="$TARGET_SHA" \ -f provider="$PROVIDER" \ - -f mode="$MODE" + -f mode="$MODE" \ + -f rerun_group="$child_rerun_group" npm_telegram: name: Run post-publish Telegram E2E needs: [resolve_target] - if: inputs.npm_telegram_package_spec != '' + if: inputs.npm_telegram_package_spec != '' && contains(fromJSON('["all","npm-telegram"]'), inputs.rerun_group) runs-on: ubuntu-24.04 timeout-minutes: 120 outputs: @@ -363,8 +400,13 @@ jobs: TARGET_REF: ${{ inputs.ref }} PACKAGE_SPEC: ${{ inputs.evidence_package_spec || inputs.npm_telegram_package_spec }} GITHUB_RUN_ID_VALUE: ${{ github.run_id }} + RELEASE_CHECKS_RESULT: ${{ needs.release_checks.result }} run: | set -euo pipefail + if [[ "$RELEASE_CHECKS_RESULT" == "skipped" ]]; then + echo "Release checks were skipped by rerun group; skipping automatic private evidence update." + exit 0 + fi if [[ -z "${RELEASE_PRIVATE_DISPATCH_TOKEN// }" ]]; then echo "OPENCLAW_RELEASES_PRIVATE_DISPATCH_TOKEN is not configured; skipping automatic private evidence update." exit 0 @@ -414,6 +456,8 @@ jobs: NORMAL_CI_RUN_ID: ${{ needs.normal_ci.outputs.run_id }} RELEASE_CHECKS_RUN_ID: ${{ needs.release_checks.outputs.run_id }} NPM_TELEGRAM_RUN_ID: ${{ needs.npm_telegram.outputs.run_id }} + NORMAL_CI_RESULT: ${{ needs.normal_ci.result }} + RELEASE_CHECKS_RESULT: ${{ needs.release_checks.result }} NPM_TELEGRAM_RESULT: ${{ needs.npm_telegram.result }} run: | set -euo pipefail @@ -448,8 +492,17 @@ jobs: failed=0 - check_child "normal_ci" "$NORMAL_CI_RUN_ID" 1 || failed=1 - check_child "release_checks" "$RELEASE_CHECKS_RUN_ID" 1 || failed=1 + if [[ "$NORMAL_CI_RESULT" == "skipped" && -z "${NORMAL_CI_RUN_ID// }" ]]; then + check_child "normal_ci" "" 0 || failed=1 + else + check_child "normal_ci" "$NORMAL_CI_RUN_ID" 1 || failed=1 + fi + + if [[ "$RELEASE_CHECKS_RESULT" == "skipped" && -z "${RELEASE_CHECKS_RUN_ID// }" ]]; then + check_child "release_checks" "" 0 || failed=1 + else + check_child "release_checks" "$RELEASE_CHECKS_RUN_ID" 1 || failed=1 + fi if [[ "$NPM_TELEGRAM_RESULT" == "skipped" && -z "${NPM_TELEGRAM_RUN_ID// }" ]]; then check_child "npm_telegram" "" 0 || failed=1 diff --git a/.github/workflows/openclaw-live-and-e2e-checks-reusable.yml b/.github/workflows/openclaw-live-and-e2e-checks-reusable.yml index 6111aa73538..3eb8ecb2c71 100644 --- a/.github/workflows/openclaw-live-and-e2e-checks-reusable.yml +++ b/.github/workflows/openclaw-live-and-e2e-checks-reusable.yml @@ -450,9 +450,18 @@ jobs: - chunk_id: plugins-runtime-install-b label: plugins/runtime install B timeout_minutes: 180 - - chunk_id: bundled-channels - label: bundled channels - timeout_minutes: 180 + - chunk_id: bundled-channels-core + label: bundled channels core + timeout_minutes: 90 + - chunk_id: bundled-channels-update-a + label: bundled channels update A + timeout_minutes: 90 + - chunk_id: bundled-channels-update-b + label: bundled channels update B + timeout_minutes: 90 + - chunk_id: bundled-channels-contracts + label: bundled channels contracts + timeout_minutes: 90 env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} OPENAI_BASE_URL: ${{ secrets.OPENAI_BASE_URL }} @@ -629,12 +638,43 @@ jobs: path: .artifacts/docker-tests/ if-no-files-found: ignore - validate_docker_lanes: - needs: [validate_selected_ref, prepare_docker_e2e_image] + plan_docker_lane_groups: + needs: validate_selected_ref if: inputs.docker_lanes != '' - name: Docker E2E targeted lanes + runs-on: ubuntu-24.04 + timeout-minutes: 5 + outputs: + groups_json: ${{ steps.plan.outputs.groups_json }} + steps: + - name: Plan targeted Docker lane groups + id: plan + shell: bash + env: + LANES: ${{ inputs.docker_lanes }} + run: | + set -euo pipefail + groups_json="$( + LANES="$LANES" node <<'NODE' + const lanes = [...new Set(String(process.env.LANES || "").split(/[,\s]+/u).map((lane) => lane.trim()).filter(Boolean))]; + if (lanes.length === 0) { + throw new Error("docker_lanes is required when planning targeted Docker lane groups."); + } + const sanitize = (lane) => lane.replace(/[^A-Za-z0-9._-]+/g, "-").replace(/^-+|-+$/g, "") || "targeted"; + process.stdout.write(JSON.stringify(lanes.map((lane) => ({ label: sanitize(lane), docker_lanes: lane })))); + NODE + )" + echo "groups_json=${groups_json}" >> "$GITHUB_OUTPUT" + + validate_docker_lanes: + needs: [validate_selected_ref, prepare_docker_e2e_image, plan_docker_lane_groups] + if: inputs.docker_lanes != '' + name: Docker E2E targeted lanes (${{ matrix.group.label }}) runs-on: blacksmith-32vcpu-ubuntu-2404 timeout-minutes: 180 + strategy: + fail-fast: false + matrix: + group: ${{ fromJson(needs.plan_docker_lane_groups.outputs.groups_json) }} env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} OPENAI_BASE_URL: ${{ secrets.OPENAI_BASE_URL }} @@ -688,7 +728,7 @@ jobs: OPENCLAW_CURRENT_PACKAGE_TGZ: .artifacts/docker-e2e-package/openclaw-current.tgz OPENCLAW_SKIP_DOCKER_BUILD: "1" INCLUDE_OPENWEBUI: ${{ inputs.include_openwebui }} - DOCKER_E2E_LANES: ${{ inputs.docker_lanes }} + DOCKER_E2E_LANES: ${{ matrix.group.docker_lanes }} steps: - name: Checkout selected ref uses: actions/checkout@v6 @@ -733,6 +773,8 @@ jobs: plan_path=".artifacts/docker-tests/targeted-plan.json" node scripts/test-docker-all.mjs --plan-json > "$plan_path" node scripts/docker-e2e.mjs github-outputs "$plan_path" >> "$GITHUB_OUTPUT" + suffix="$(printf '%s' "$LANES" | tr ',[:space:]' '-' | tr -cd 'A-Za-z0-9._-' | sed -E 's/-+/-/g; s/^-//; s/-$//')" + echo "artifact_suffix=${suffix:-targeted}" >> "$GITHUB_OUTPUT" echo "plan_json=$plan_path" >> "$GITHUB_OUTPUT" - name: Download OpenClaw Docker E2E package @@ -782,8 +824,8 @@ jobs: export OPENCLAW_DOCKER_ALL_PREFLIGHT=0 export OPENCLAW_DOCKER_ALL_FAIL_FAST=0 export OPENCLAW_DOCKER_ALL_INCLUDE_OPENWEBUI="${INCLUDE_OPENWEBUI}" - export OPENCLAW_DOCKER_ALL_LOG_DIR=".artifacts/docker-tests/targeted" - export OPENCLAW_DOCKER_ALL_TIMINGS_FILE=".artifacts/docker-tests/targeted-timings.json" + export OPENCLAW_DOCKER_ALL_LOG_DIR=".artifacts/docker-tests/targeted-${{ steps.plan.outputs.artifact_suffix }}" + export OPENCLAW_DOCKER_ALL_TIMINGS_FILE=".artifacts/docker-tests/targeted-${{ steps.plan.outputs.artifact_suffix }}-timings.json" export OPENCLAW_DOCKER_ALL_PNPM_COMMAND="$(command -v pnpm)" if [[ "${{ steps.plan.outputs.needs_live_image }}" == "1" ]]; then pnpm test:docker:live-build @@ -797,7 +839,7 @@ jobs: shell: bash run: | set -euo pipefail - summary=".artifacts/docker-tests/targeted/summary.json" + summary=".artifacts/docker-tests/targeted-${{ steps.plan.outputs.artifact_suffix }}/summary.json" if [[ ! -f "$summary" ]]; then echo "Docker targeted summary missing: \`$summary\`" >> "$GITHUB_STEP_SUMMARY" exit 0 @@ -808,7 +850,7 @@ jobs: if: always() uses: actions/upload-artifact@v7 with: - name: docker-e2e-targeted + name: docker-e2e-${{ steps.plan.outputs.artifact_suffix }} path: .artifacts/docker-tests/ if-no-files-found: ignore diff --git a/.github/workflows/openclaw-release-checks.yml b/.github/workflows/openclaw-release-checks.yml index eb75092bcf5..b09006deadc 100644 --- a/.github/workflows/openclaw-release-checks.yml +++ b/.github/workflows/openclaw-release-checks.yml @@ -25,6 +25,20 @@ on: - fresh - upgrade - both + rerun_group: + description: Release check group to run + required: false + default: all + type: choice + options: + - all + - install-smoke + - cross-os + - live-e2e + - package + - qa + - qa-parity + - qa-live concurrency: group: openclaw-release-checks-${{ inputs.ref }} @@ -47,6 +61,7 @@ jobs: sha: ${{ steps.ref.outputs.sha }} provider: ${{ steps.inputs.outputs.provider }} mode: ${{ steps.inputs.outputs.mode }} + rerun_group: ${{ steps.inputs.outputs.rerun_group }} steps: - name: Require main or release workflow ref for release checks env: @@ -105,12 +120,14 @@ jobs: RELEASE_REF_INPUT: ${{ inputs.ref }} RELEASE_PROVIDER_INPUT: ${{ inputs.provider }} RELEASE_MODE_INPUT: ${{ inputs.mode }} + RELEASE_RERUN_GROUP_INPUT: ${{ inputs.rerun_group }} run: | set -euo pipefail { printf 'ref=%s\n' "$RELEASE_REF_INPUT" printf 'provider=%s\n' "$RELEASE_PROVIDER_INPUT" printf 'mode=%s\n' "$RELEASE_MODE_INPUT" + printf 'rerun_group=%s\n' "$RELEASE_RERUN_GROUP_INPUT" } >> "$GITHUB_OUTPUT" - name: Summarize validated ref @@ -119,6 +136,7 @@ jobs: RELEASE_SHA: ${{ steps.ref.outputs.sha }} RELEASE_PROVIDER: ${{ inputs.provider }} RELEASE_MODE: ${{ inputs.mode }} + RELEASE_RERUN_GROUP: ${{ inputs.rerun_group }} run: | { echo "## Release checks" @@ -127,11 +145,13 @@ jobs: echo "- Validated SHA: \`${RELEASE_SHA}\`" echo "- Cross-OS provider: \`${RELEASE_PROVIDER}\`" echo "- Cross-OS mode: \`${RELEASE_MODE}\`" + echo "- Rerun group: \`${RELEASE_RERUN_GROUP}\`" echo "- This run will execute cross-OS release validation, install smoke, QA Lab parity, Matrix, and Telegram lanes, and the non-Parallels Docker/live/openwebui coverage from the CI migration plan." } >> "$GITHUB_STEP_SUMMARY" install_smoke_release_checks: needs: [resolve_target] + if: contains(fromJSON('["all","install-smoke"]'), needs.resolve_target.outputs.rerun_group) permissions: contents: read uses: ./.github/workflows/install-smoke.yml @@ -141,6 +161,7 @@ jobs: cross_os_release_checks: needs: [resolve_target] + if: contains(fromJSON('["all","cross-os"]'), needs.resolve_target.outputs.rerun_group) permissions: read-all uses: ./.github/workflows/openclaw-cross-os-release-checks-reusable.yml with: @@ -157,6 +178,7 @@ jobs: live_and_e2e_release_checks: needs: [resolve_target] + if: contains(fromJSON('["all","live-e2e"]'), needs.resolve_target.outputs.rerun_group) permissions: actions: read contents: read @@ -218,6 +240,7 @@ jobs: package_acceptance_release_checks: name: Run package acceptance needs: [resolve_target] + if: contains(fromJSON('["all","package"]'), needs.resolve_target.outputs.rerun_group) permissions: actions: read contents: read @@ -282,6 +305,7 @@ jobs: qa_lab_parity_release_checks: name: Run QA Lab parity gate needs: [resolve_target] + if: contains(fromJSON('["all","qa","qa-parity"]'), needs.resolve_target.outputs.rerun_group) runs-on: blacksmith-32vcpu-ubuntu-2404 timeout-minutes: 30 permissions: @@ -356,6 +380,7 @@ jobs: qa_live_matrix_release_checks: name: Run QA Lab live Matrix lane needs: [resolve_target] + if: contains(fromJSON('["all","qa","qa-live"]'), needs.resolve_target.outputs.rerun_group) runs-on: blacksmith-32vcpu-ubuntu-2404 timeout-minutes: 60 permissions: @@ -434,6 +459,7 @@ jobs: qa_live_telegram_release_checks: name: Run QA Lab live Telegram lane needs: [resolve_target] + if: contains(fromJSON('["all","qa","qa-live"]'), needs.resolve_target.outputs.rerun_group) runs-on: blacksmith-32vcpu-ubuntu-2404 timeout-minutes: 60 permissions: diff --git a/docs/ci.md b/docs/ci.md index f88105bc1ef..4a197ebad8d 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -19,6 +19,13 @@ provided. The umbrella records the dispatched child run ids, and the final child workflow is rerun and turns green, rerun only the parent verifier job to refresh the umbrella result. +For recovery, `Full Release Validation` and `OpenClaw Release Checks` both +accept `rerun_group`. Use `all` for a release candidate, `ci` for only the +normal full CI child, `release-checks` for every release child, or a narrower +release group: `install-smoke`, `cross-os`, `live-e2e`, `package`, `qa`, +`qa-parity`, `qa-live`, or `npm-telegram` on the umbrella. This keeps a failed +release box rerun bounded after a focused fix. + The release live/E2E child keeps broad native `pnpm test:live` coverage, but it runs it as named shards (`native-live-src-agents`, `native-live-src-gateway-core`, `native-live-src-gateway-backends`, @@ -61,7 +68,10 @@ The workflow has four jobs: `package_artifact_name=package-under-test`. The reusable workflow downloads that artifact, validates the tarball inventory, prepares package-digest Docker images when needed, and runs the selected Docker lanes against that - package instead of packing the workflow checkout. + package instead of packing the workflow checkout. When a profile selects + multiple targeted `docker_lanes`, the reusable workflow prepares the package + and shared images once, then fans those lanes out as parallel targeted Docker + jobs with unique artifacts. 3. `package_telegram` optionally calls `NPM Telegram Beta E2E`. It runs when `telegram_mode` is not `none` and installs the same `package-under-test` artifact when Package Acceptance resolved one; standalone Telegram dispatch @@ -293,6 +303,8 @@ CI routing-only edits, selected cheap core-test fixture edits, and narrow plugin Windows Node checks are scoped to Windows-specific process/path wrappers, npm/pnpm/UI runner helpers, package manager config, and the CI workflow surfaces that execute that lane; unrelated source, plugin, install-smoke, and test-only changes stay on the Linux Node lanes so they do not reserve a 16-vCPU Windows worker for coverage that is already exercised by the normal test shards. The separate `install-smoke` workflow reuses the same scope script through its own `preflight` job. It splits smoke coverage into `run_fast_install_smoke` and `run_full_install_smoke`. Pull requests run the fast path for Docker/package surfaces, bundled plugin package/manifest changes, and core plugin/channel/gateway/Plugin SDK surfaces that the Docker smoke jobs exercise. Source-only bundled plugin changes, test-only edits, and docs-only edits do not reserve Docker workers. The fast path builds the root Dockerfile image once, checks the CLI, runs the agents delete shared-workspace CLI smoke, runs the container gateway-network e2e, verifies a bundled extension build arg, and runs the bounded bundled-plugin Docker profile under a 240-second aggregate command timeout with each scenario's Docker run capped separately. The full path keeps QR package install and installer Docker/update coverage for nightly scheduled runs, manual dispatches, workflow-call release checks, and pull requests that truly touch installer/package/Docker surfaces. `main` pushes, including merge commits, do not force the full path; when changed-scope logic would request full coverage on a push, the workflow keeps the fast Docker smoke and leaves the full install smoke to nightly or release validation. The slow Bun global install image-provider smoke is separately gated by `run_bun_global_install_smoke`; it runs on the nightly schedule and from the release checks workflow, and manual `install-smoke` dispatches can opt into it, but pull requests and `main` pushes do not run it. QR and installer Docker tests keep their own install-focused Dockerfiles. Local `test:docker:all` prebuilds one shared live-test image, packs OpenClaw once as an npm tarball, and builds two shared `scripts/e2e/Dockerfile` images: a bare Node/Git runner for installer/update/plugin-dependency lanes and a functional image that installs the same tarball into `/app` for normal functionality lanes. Docker lane definitions live in `scripts/lib/docker-e2e-scenarios.mjs`, planner logic lives in `scripts/lib/docker-e2e-plan.mjs`, and the runner only executes the selected plan. The scheduler selects the image per lane with `OPENCLAW_DOCKER_E2E_BARE_IMAGE` and `OPENCLAW_DOCKER_E2E_FUNCTIONAL_IMAGE`, then runs lanes with `OPENCLAW_SKIP_DOCKER_BUILD=1`; tune the default main-pool slot count of 10 with `OPENCLAW_DOCKER_ALL_PARALLELISM` and the provider-sensitive tail-pool slot count of 10 with `OPENCLAW_DOCKER_ALL_TAIL_PARALLELISM`. Heavy lane caps default to `OPENCLAW_DOCKER_ALL_LIVE_LIMIT=9`, `OPENCLAW_DOCKER_ALL_NPM_LIMIT=10`, and `OPENCLAW_DOCKER_ALL_SERVICE_LIMIT=7` so npm install and multi-service lanes do not overcommit Docker while lighter lanes still fill available slots. A single lane heavier than the effective caps can still start from an empty pool, then runs alone until it releases capacity. Lane starts are staggered by 2 seconds by default to avoid local Docker daemon create storms; override with `OPENCLAW_DOCKER_ALL_START_STAGGER_MS=0` or another millisecond value. The local aggregate preflights Docker, removes stale OpenClaw E2E containers, emits active-lane status, persists lane timings for longest-first ordering, and supports `OPENCLAW_DOCKER_ALL_DRY_RUN=1` for scheduler inspection. It stops scheduling new pooled lanes after the first failure by default, and each lane has a 120-minute fallback timeout overrideable with `OPENCLAW_DOCKER_ALL_LANE_TIMEOUT_MS`; selected live/tail lanes use tighter per-lane caps. `OPENCLAW_DOCKER_ALL_LANES=` runs exact scheduler lanes, including release-only lanes such as `install-e2e` and split bundled update lanes such as `bundled-channel-update-acpx`, while skipping the cleanup smoke so agents can reproduce one failed lane. The reusable live/E2E workflow asks `scripts/test-docker-all.mjs --plan-json` which package, image kind, live image, lane, and credential coverage is required, then `scripts/docker-e2e.mjs` converts that plan into GitHub outputs and summaries. It either packs OpenClaw through `scripts/package-openclaw-for-docker.mjs`, downloads a current-run package artifact, or downloads a package artifact from `package_artifact_run_id`; validates the tarball inventory; builds and pushes package-digest-tagged bare/functional GHCR Docker E2E images through Blacksmith's Docker layer cache when the plan needs package-installed lanes; and reuses provided `docker_e2e_bare_image`/`docker_e2e_functional_image` inputs or existing package-digest images instead of rebuilding. The `Package Acceptance` workflow is the high-level package gate: it resolves a candidate from npm, a trusted `package_ref`, an HTTPS tarball plus SHA-256, or a prior workflow artifact, then passes that single `package-under-test` artifact into the reusable Docker E2E workflow. It keeps `workflow_ref` separate from `package_ref` so current acceptance logic can validate older trusted commits without checking out old workflow code. Release checks run a custom Package Acceptance delta for the target ref: bundled-channel compat, offline plugin fixtures, and Telegram package QA against the resolved tarball. The release-path Docker suite runs smaller chunked jobs with `OPENCLAW_SKIP_DOCKER_BUILD=1` so each chunk pulls only the image kind it needs and executes multiple lanes through the same weighted scheduler (`OPENCLAW_DOCKER_ALL_PROFILE=release-path`, `OPENCLAW_DOCKER_ALL_CHUNK=core|package-update-openai|package-update-anthropic|package-update-core|plugins-runtime-core|plugins-runtime-install-a|plugins-runtime-install-b|bundled-channels`). OpenWebUI is folded into `plugins-runtime-core` when full release-path coverage requests it, and keeps a standalone `openwebui` chunk only for OpenWebUI-only dispatches. The legacy aggregate chunk names `package-update`, `plugins-runtime`, and `plugins-integrations` still work for manual reruns, but the release workflow uses the split chunks so installer E2E and bundled plugin install/uninstall sweeps do not dominate the critical path. The `install-e2e` lane alias remains the aggregate manual rerun alias for both provider installer lanes. The `bundled-channels` chunk runs split `bundled-channel-*` and `bundled-channel-update-*` lanes rather than the serial all-in-one `bundled-channel-deps` lane. Each chunk uploads `.artifacts/docker-tests/` with lane logs, timings, `summary.json`, `failures.json`, phase timings, scheduler plan JSON, slow-lane tables, and per-lane rerun commands. The workflow `docker_lanes` input runs selected lanes against the prepared images instead of the chunk jobs, which keeps failed-lane debugging bounded to one targeted Docker job and prepares, downloads, or reuses the package artifact for that run; if a selected lane is a live Docker lane, the targeted job builds the live-test image locally for that rerun. Generated per-lane GitHub rerun commands include `package_artifact_run_id`, `package_artifact_name`, and prepared image inputs when those values exist, so a failed lane can reuse the exact package and images from the failed run. Use `pnpm test:docker:rerun ` to download Docker artifacts from a GitHub run and print combined/per-lane targeted rerun commands; use `pnpm test:docker:timings ` for slow-lane and phase critical-path summaries. The scheduled live/E2E workflow runs the full release-path Docker suite daily. The bundled update matrix is split by update target so repeated npm update and doctor repair passes can shard with other bundled checks. +Current release Docker chunks are `core`, `package-update-openai`, `package-update-anthropic`, `package-update-core`, `plugins-runtime-core`, `plugins-runtime-install-a`, `plugins-runtime-install-b`, `bundled-channels-core`, `bundled-channels-update-a`, `bundled-channels-update-b`, and `bundled-channels-contracts`. The aggregate `bundled-channels` chunk remains available for manual one-shot reruns, but the release workflow uses the split chunks so channel smokes, update targets, and setup/runtime contract checks can run in parallel. Targeted `docker_lanes` dispatches also split multiple selected lanes into parallel jobs after one shared package/image preparation step, and bundled-channel update lanes retry once for transient npm network failures. + Local changed-lane logic lives in `scripts/changed-lanes.mjs` and is executed by `scripts/check-changed.mjs`. That local check gate is stricter about architecture boundaries than the broad CI platform scope: core production changes run core prod and core test typecheck plus core lint/guards, core test-only changes run only core test typecheck plus core lint, extension production changes run extension prod and extension test typecheck plus extension lint, and extension test-only changes run extension test typecheck plus extension lint. Public Plugin SDK or plugin-contract changes expand to extension typecheck because extensions depend on those core contracts, but Vitest extension sweeps are explicit test work. Release metadata-only version bumps run targeted version/config/root-dependency checks. Unknown root/config changes fail safe to all check lanes. Manual CI dispatches run `checks-node-compat-node22` as release-candidate compatibility coverage. Normal pull requests and `main` pushes skip that lane and keep the matrix focused on the Node 24 test/channel lanes. diff --git a/docs/help/testing.md b/docs/help/testing.md index b08eb95683b..435cba63acd 100644 --- a/docs/help/testing.md +++ b/docs/help/testing.md @@ -607,7 +607,7 @@ The live-model Docker runners also bind-mount only the needed CLI auth homes (or Set `OPENCLAW_PLUGINS_E2E_CLAWHUB=0` to skip the live ClawHub block, or override the default package with `OPENCLAW_PLUGINS_E2E_CLAWHUB_SPEC` and `OPENCLAW_PLUGINS_E2E_CLAWHUB_ID`. - Plugin update unchanged smoke: `pnpm test:docker:plugin-update` (script: `scripts/e2e/plugin-update-unchanged-docker.sh`) - Config reload metadata smoke: `pnpm test:docker:config-reload` (script: `scripts/e2e/config-reload-source-docker.sh`) -- Bundled plugin runtime deps: `pnpm test:docker:bundled-channel-deps` builds a small Docker runner image by default, builds and packs OpenClaw once on the host, then mounts that tarball into each Linux install scenario. Reuse the image with `OPENCLAW_SKIP_DOCKER_BUILD=1`, skip the host rebuild after a fresh local build with `OPENCLAW_BUNDLED_CHANNEL_HOST_BUILD=0`, or point at an existing tarball with `OPENCLAW_CURRENT_PACKAGE_TGZ=/path/to/openclaw-*.tgz`. The full Docker aggregate and release-path `bundled-channels` chunk pre-pack this tarball once, then shard bundled channel checks into independent lanes, including separate update lanes for Telegram, Discord, Slack, Feishu, memory-lancedb, and ACPX. The release workflow also splits provider installer chunks and bundled plugin install/uninstall chunks; legacy `package-update`, `plugins-runtime`, and `plugins-integrations` chunks remain aggregate aliases for manual reruns. Use `OPENCLAW_BUNDLED_CHANNELS=telegram,slack` to narrow the channel matrix when running the bundled lane directly, or `OPENCLAW_BUNDLED_CHANNEL_UPDATE_TARGETS=telegram,acpx` to narrow the update scenario. The lane also verifies that `channels..enabled=false` and `plugins.entries..enabled=false` suppress doctor/runtime-dependency repair. +- Bundled plugin runtime deps: `pnpm test:docker:bundled-channel-deps` builds a small Docker runner image by default, builds and packs OpenClaw once on the host, then mounts that tarball into each Linux install scenario. Reuse the image with `OPENCLAW_SKIP_DOCKER_BUILD=1`, skip the host rebuild after a fresh local build with `OPENCLAW_BUNDLED_CHANNEL_HOST_BUILD=0`, or point at an existing tarball with `OPENCLAW_CURRENT_PACKAGE_TGZ=/path/to/openclaw-*.tgz`. The full Docker aggregate and release-path bundled-channel chunks pre-pack this tarball once, then shard bundled channel checks into independent lanes, including separate update lanes for Telegram, Discord, Slack, Feishu, memory-lancedb, and ACPX. Release chunks split channel smokes, update targets, and setup/runtime contracts into `bundled-channels-core`, `bundled-channels-update-a`, `bundled-channels-update-b`, and `bundled-channels-contracts`; the aggregate `bundled-channels` chunk remains available for manual reruns. The release workflow also splits provider installer chunks and bundled plugin install/uninstall chunks; legacy `package-update`, `plugins-runtime`, and `plugins-integrations` chunks remain aggregate aliases for manual reruns. Use `OPENCLAW_BUNDLED_CHANNELS=telegram,slack` to narrow the channel matrix when running the bundled lane directly, or `OPENCLAW_BUNDLED_CHANNEL_UPDATE_TARGETS=telegram,acpx` to narrow the update scenario. The lane also verifies that `channels..enabled=false` and `plugins.entries..enabled=false` suppress doctor/runtime-dependency repair. - Narrow bundled plugin runtime deps while iterating by disabling unrelated scenarios, for example: `OPENCLAW_BUNDLED_CHANNEL_SCENARIOS=0 OPENCLAW_BUNDLED_CHANNEL_UPDATE_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_ROOT_OWNED_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_SETUP_ENTRY_SCENARIO=0 pnpm test:docker:bundled-channel-deps`. diff --git a/docs/reference/RELEASING.md b/docs/reference/RELEASING.md index a6b45b14219..bc2b11976fb 100644 --- a/docs/reference/RELEASING.md +++ b/docs/reference/RELEASING.md @@ -290,6 +290,12 @@ stale. The umbrella's final verifier re-checks the recorded child workflow run ids, so after a child workflow is rerun successfully, rerun only the failed `Verify full validation` parent job. +For bounded recovery, pass `rerun_group` to the umbrella. `all` is the real +release-candidate run, `ci` runs only the normal CI child, `release-checks` runs +every release box, and the narrower release groups are `install-smoke`, +`cross-os`, `live-e2e`, `package`, `qa`, `qa-parity`, `qa-live`, and +`npm-telegram` when the standalone package Telegram lane is supplied. + ### Vitest The Vitest box is the manual `CI` child workflow. Manual CI intentionally @@ -327,11 +333,12 @@ Release Docker coverage includes: - repository E2E lanes - release-path Docker chunks: `core`, `package-update-openai`, `package-update-anthropic`, `package-update-core`, `plugins-runtime-core`, - `plugins-runtime-install-a`, `plugins-runtime-install-b`, and - `bundled-channels` + `plugins-runtime-install-a`, `plugins-runtime-install-b`, + `bundled-channels-core`, `bundled-channels-update-a`, + `bundled-channels-update-b`, and `bundled-channels-contracts` - OpenWebUI coverage inside the `plugins-runtime-core` chunk when requested -- split bundled-channel dependency lanes in their own `bundled-channels` chunk - instead of the serial all-in-one bundled-channel lane +- split bundled-channel dependency lanes across channel-smoke, update-target, + and setup/runtime contract chunks instead of one large bundled-channel job - split bundled plugin install/uninstall lanes `bundled-plugin-install-uninstall-0` through `bundled-plugin-install-uninstall-7` diff --git a/scripts/lib/docker-e2e-scenarios.mjs b/scripts/lib/docker-e2e-scenarios.mjs index f9e9bd3803e..5288fdb0014 100644 --- a/scripts/lib/docker-e2e-scenarios.mjs +++ b/scripts/lib/docker-e2e-scenarios.mjs @@ -107,20 +107,30 @@ function bundledChannelScenarioLane(name, env, options = {}) { ); } -const bundledScenarioLanes = [ - ...["telegram", "discord", "slack", "feishu", "memory-lancedb"].map((channel) => +const bundledChannelSmokeLanes = ["telegram", "discord", "slack", "feishu", "memory-lancedb"].map( + (channel) => npmLane( `bundled-channel-${channel}`, `OPENCLAW_BUNDLED_CHANNELS=${channel} ${bundledChannelLaneCommand}`, ), +); + +const bundledChannelUpdateLanes = [ + "telegram", + "discord", + "slack", + "feishu", + "memory-lancedb", + "acpx", +].map((target) => + bundledChannelScenarioLane( + `bundled-channel-update-${target}`, + `OPENCLAW_BUNDLED_CHANNEL_SCENARIOS=0 OPENCLAW_BUNDLED_CHANNEL_UPDATE_SCENARIO=1 OPENCLAW_BUNDLED_CHANNEL_UPDATE_TARGETS=${target} OPENCLAW_BUNDLED_CHANNEL_ROOT_OWNED_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_SETUP_ENTRY_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_LOAD_FAILURE_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_DISABLED_CONFIG_SCENARIO=0`, + { retryPatterns: LIVE_RETRY_PATTERNS, retries: 1, timeoutMs: BUNDLED_UPDATE_TIMEOUT_MS }, ), - ...["telegram", "discord", "slack", "feishu", "memory-lancedb", "acpx"].map((target) => - bundledChannelScenarioLane( - `bundled-channel-update-${target}`, - `OPENCLAW_BUNDLED_CHANNEL_SCENARIOS=0 OPENCLAW_BUNDLED_CHANNEL_UPDATE_SCENARIO=1 OPENCLAW_BUNDLED_CHANNEL_UPDATE_TARGETS=${target} OPENCLAW_BUNDLED_CHANNEL_ROOT_OWNED_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_SETUP_ENTRY_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_LOAD_FAILURE_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_DISABLED_CONFIG_SCENARIO=0`, - { timeoutMs: BUNDLED_UPDATE_TIMEOUT_MS }, - ), - ), +); + +const bundledChannelContractLanes = [ bundledChannelScenarioLane( "bundled-channel-root-owned", "OPENCLAW_BUNDLED_CHANNEL_SCENARIOS=0 OPENCLAW_BUNDLED_CHANNEL_UPDATE_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_ROOT_OWNED_SCENARIO=1 OPENCLAW_BUNDLED_CHANNEL_SETUP_ENTRY_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_LOAD_FAILURE_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_DISABLED_CONFIG_SCENARIO=0", @@ -139,6 +149,12 @@ const bundledScenarioLanes = [ ), ]; +const bundledScenarioLanes = [ + ...bundledChannelSmokeLanes, + ...bundledChannelUpdateLanes, + ...bundledChannelContractLanes, +]; + const bundledPluginInstallUninstallLanes = Array.from( { length: BUNDLED_PLUGIN_INSTALL_UNINSTALL_SHARDS }, (_, index) => @@ -465,7 +481,18 @@ const primaryReleasePathChunks = { "plugins-runtime-core": releasePathPluginRuntimeCoreLanes, "plugins-runtime-install-a": bundledPluginInstallUninstallLanes.slice(0, 4), "plugins-runtime-install-b": bundledPluginInstallUninstallLanes.slice(4), - "bundled-channels": releasePathBundledChannelLanes, + "bundled-channels-core": [releasePathBundledChannelLanes[0], ...bundledChannelSmokeLanes], + "bundled-channels-update-a": [ + bundledChannelUpdateLanes[0], + bundledChannelUpdateLanes[1], + bundledChannelUpdateLanes[4], + ], + "bundled-channels-update-b": [ + bundledChannelUpdateLanes[2], + bundledChannelUpdateLanes[3], + bundledChannelUpdateLanes[5], + ], + "bundled-channels-contracts": bundledChannelContractLanes, openwebui: [], }; @@ -477,6 +504,7 @@ const legacyReleasePathChunks = { ], "plugins-runtime": releasePathPluginRuntimeLanes, "plugins-integrations": [...releasePathPluginRuntimeLanes, ...releasePathBundledChannelLanes], + "bundled-channels": releasePathBundledChannelLanes, }; function openWebUILane() { diff --git a/src/plugins/bundled-runtime-deps.ts b/src/plugins/bundled-runtime-deps.ts index 1e9c8016608..e20ea282e0f 100644 --- a/src/plugins/bundled-runtime-deps.ts +++ b/src/plugins/bundled-runtime-deps.ts @@ -69,7 +69,6 @@ const BUNDLED_RUNTIME_DEPS_OWNERLESS_LOCK_STALE_MS = 30_000; const BUNDLED_RUNTIME_DEPS_INSTALL_PROGRESS_INTERVAL_MS = 5_000; const BUNDLED_RUNTIME_MIRROR_MATERIALIZED_EXTENSIONS = new Set([".cjs", ".js", ".mjs"]); const BUNDLED_EXTENSION_DIST_DIR = "extensions"; -const BUNDLED_RUNTIME_MIRROR_PLUGIN_REGION_RE = /(?:^|\n)\/\/#region extensions\/[^/\s]+(?:\/|$)/u; const MIRRORED_CORE_RUNTIME_DEP_NAMES = ["tslog"] as const; const MIRRORED_PACKAGE_RUNTIME_DEP_PLUGIN_ID = "openclaw-core"; @@ -82,14 +81,7 @@ export type BundledRuntimeDepsNpmRunner = { }; export function shouldMaterializeBundledRuntimeMirrorDistFile(sourcePath: string): boolean { - if (!BUNDLED_RUNTIME_MIRROR_MATERIALIZED_EXTENSIONS.has(path.extname(sourcePath))) { - return false; - } - try { - return BUNDLED_RUNTIME_MIRROR_PLUGIN_REGION_RE.test(fs.readFileSync(sourcePath, "utf8")); - } catch { - return false; - } + return BUNDLED_RUNTIME_MIRROR_MATERIALIZED_EXTENSIONS.has(path.extname(sourcePath)); } export function materializeBundledRuntimeMirrorDistFile( diff --git a/src/plugins/bundled-runtime-root.test.ts b/src/plugins/bundled-runtime-root.test.ts index b045f20b756..ac9c67867d4 100644 --- a/src/plugins/bundled-runtime-root.test.ts +++ b/src/plugins/bundled-runtime-root.test.ts @@ -20,7 +20,7 @@ afterEach(() => { }); describe("prepareBundledPluginRuntimeRoot", () => { - it("materializes plugin-owned root chunks in external mirrors", () => { + it("materializes root JavaScript chunks in external mirrors", () => { const packageRoot = makeTempRoot(); const stageDir = makeTempRoot(); const pluginRoot = path.join(packageRoot, "dist", "extensions", "browser"); @@ -42,6 +42,11 @@ describe("prepareBundledPluginRuntimeRoot", () => { ].join("\n"), "utf8", ); + fs.writeFileSync( + path.join(packageRoot, "dist", "shared-runtime.js"), + "export const shared = 'mirrored-without-region';\n", + "utf8", + ); fs.writeFileSync( path.join(pluginRoot, "index.js"), `import { marker } from "../../pw-ai.js"; export default { id: "browser", marker };\n`, @@ -106,6 +111,9 @@ describe("prepareBundledPluginRuntimeRoot", () => { expect(fs.existsSync(staleMirrorChunk)).toBe(true); expect(fs.lstatSync(staleMirrorChunk).isSymbolicLink()).toBe(false); expect(fs.readFileSync(staleMirrorChunk, "utf8")).toContain("playwright-core"); + expect(fs.lstatSync(path.join(installRoot, "dist", "shared-runtime.js")).isSymbolicLink()).toBe( + false, + ); }); it("does not copy staged runtime mirror dist files onto themselves", () => { diff --git a/src/plugins/loader.test.ts b/src/plugins/loader.test.ts index 3e15afb681d..5c711fd8e7c 100644 --- a/src/plugins/loader.test.ts +++ b/src/plugins/loader.test.ts @@ -1714,7 +1714,7 @@ module.exports = { expect(registry?.plugins.find((entry) => entry.id === "alpha")?.status).toBe("loaded"); }); - it("materializes plugin-owned root chunks in external runtime mirrors", () => { + it("materializes root JavaScript chunks in external runtime mirrors", () => { const packageRoot = makeTempDir(); const stageDir = makeTempDir(); const bundledDir = path.join(packageRoot, "dist", "extensions"); @@ -1736,6 +1736,11 @@ module.exports = { ].join("\n"), "utf-8", ); + fs.writeFileSync( + path.join(packageRoot, "dist", "shared-runtime.js"), + "export const shared = 'mirrored-without-region';\n", + "utf-8", + ); fs.writeFileSync( path.join(pluginRoot, "index.js"), [ @@ -1829,6 +1834,9 @@ module.exports = { expect(reloadedRegistry.plugins.find((entry) => entry.id === "browser")?.status).toBe("loaded"); expect(fs.existsSync(stagedMirrorChunk)).toBe(true); + expect( + fs.lstatSync(path.join(actualInstallRoot, "dist", "shared-runtime.js")).isSymbolicLink(), + ).toBe(false); }); it("loads bundled plugins with plugin-sdk imports from an external stage dir", () => { diff --git a/test/scripts/docker-e2e-plan.test.ts b/test/scripts/docker-e2e-plan.test.ts index 437c6f8ca6f..00c91c300f9 100644 --- a/test/scripts/docker-e2e-plan.test.ts +++ b/test/scripts/docker-e2e-plan.test.ts @@ -104,10 +104,25 @@ describe("scripts/lib/docker-e2e-plan", () => { profile: RELEASE_PATH_PROFILE, releaseChunk: "plugins-runtime-install-b", }); - const bundledChannels = planFor({ + const bundledChannelsCore = planFor({ includeOpenWebUI: true, profile: RELEASE_PATH_PROFILE, - releaseChunk: "bundled-channels", + releaseChunk: "bundled-channels-core", + }); + const bundledChannelsUpdateA = planFor({ + includeOpenWebUI: true, + profile: RELEASE_PATH_PROFILE, + releaseChunk: "bundled-channels-update-a", + }); + const bundledChannelsUpdateB = planFor({ + includeOpenWebUI: true, + profile: RELEASE_PATH_PROFILE, + releaseChunk: "bundled-channels-update-b", + }); + const bundledChannelsContracts = planFor({ + includeOpenWebUI: true, + profile: RELEASE_PATH_PROFILE, + releaseChunk: "bundled-channels-contracts", }); expect(packageInstallOpenAi.lanes.map((lane) => lane.name)).toEqual(["install-e2e-openai"]); @@ -142,15 +157,32 @@ describe("scripts/lib/docker-e2e-plan", () => { "bundled-plugin-install-uninstall-6", "bundled-plugin-install-uninstall-7", ]); - expect(bundledChannels.lanes.map((lane) => lane.name)).toEqual( - expect.arrayContaining([ - "plugin-update", - "bundled-channel-telegram", - "bundled-channel-update-acpx", - ]), - ); - expect(bundledChannels.lanes.map((lane) => lane.name)).not.toContain("plugins"); - expect(bundledChannels.lanes.map((lane) => lane.name)).not.toContain("openwebui"); + expect(bundledChannelsCore.lanes.map((lane) => lane.name)).toEqual([ + "plugin-update", + "bundled-channel-telegram", + "bundled-channel-discord", + "bundled-channel-slack", + "bundled-channel-feishu", + "bundled-channel-memory-lancedb", + ]); + expect(bundledChannelsUpdateA.lanes.map((lane) => lane.name)).toEqual([ + "bundled-channel-update-telegram", + "bundled-channel-update-discord", + "bundled-channel-update-memory-lancedb", + ]); + expect(bundledChannelsUpdateB.lanes.map((lane) => lane.name)).toEqual([ + "bundled-channel-update-slack", + "bundled-channel-update-feishu", + "bundled-channel-update-acpx", + ]); + expect(bundledChannelsContracts.lanes.map((lane) => lane.name)).toEqual([ + "bundled-channel-root-owned", + "bundled-channel-setup-entry", + "bundled-channel-load-failure", + "bundled-channel-disabled-config", + ]); + expect(bundledChannelsCore.lanes.map((lane) => lane.name)).not.toContain("plugins"); + expect(bundledChannelsUpdateA.lanes.map((lane) => lane.name)).not.toContain("openwebui"); }); it("keeps legacy release chunk names as aggregate aliases", () => { diff --git a/test/scripts/package-acceptance-workflow.test.ts b/test/scripts/package-acceptance-workflow.test.ts index a0e0785bd7b..ce8526a62e5 100644 --- a/test/scripts/package-acceptance-workflow.test.ts +++ b/test/scripts/package-acceptance-workflow.test.ts @@ -79,6 +79,10 @@ describe("package artifact reuse", () => { expect(workflow).not.toContain("uses: ./.github/actions/docker-e2e-plan"); expect(workflow).toContain("node scripts/test-docker-all.mjs --plan-json"); expect(workflow).toContain("node scripts/docker-e2e.mjs github-outputs"); + expect(workflow).toContain("plan_docker_lane_groups:"); + expect(workflow).toContain("Docker E2E targeted lanes (${{ matrix.group.label }})"); + expect(workflow).toContain("DOCKER_E2E_LANES: ${{ matrix.group.docker_lanes }}"); + expect(workflow).toContain("name: docker-e2e-${{ steps.plan.outputs.artifact_suffix }}"); }); it("uses Blacksmith Docker build caching for prepared E2E images", () => { @@ -126,7 +130,7 @@ describe("package artifact reuse", () => { expect(workflow).toContain("package_acceptance_release_checks:"); expect(workflow).toContain( - "live_and_e2e_release_checks:\n needs: [resolve_target]\n permissions:\n actions: read", + 'live_and_e2e_release_checks:\n needs: [resolve_target]\n if: contains(fromJSON(\'["all","live-e2e"]\'), needs.resolve_target.outputs.rerun_group)', ); expect(workflow).toContain("uses: ./.github/workflows/package-acceptance.yml"); expect(workflow).toContain("package_ref: ${{ needs.resolve_target.outputs.ref }}"); @@ -141,6 +145,9 @@ describe("package artifact reuse", () => { expect(workflow).toContain( "OPENCLAW_QA_CONVEX_SECRET_CI: ${{ secrets.OPENCLAW_QA_CONVEX_SECRET_CI }}", ); + expect(workflow).toContain("rerun_group:"); + expect(workflow).toContain("- live-e2e"); + expect(workflow).toContain("- qa-live"); }); it("detects Matrix fail-fast support for older release refs", () => { @@ -177,6 +184,9 @@ describe("package artifact reuse", () => { 'gh workflow run npm-telegram-beta-e2e.yml --ref "$CHILD_WORKFLOW_REF" "${args[@]}"', ); expect(workflow).toContain('-f harness_ref="$TARGET_SHA"'); + expect(workflow).toContain("child_rerun_group=all"); + expect(workflow).toContain('-f rerun_group="$child_rerun_group"'); + expect(workflow).toContain("NORMAL_CI_RESULT: ${{ needs.normal_ci.result }}"); expect(workflow).not.toContain("workflow_ref:"); expect(workflow).not.toContain("inputs.workflow_ref"); });