diff --git a/.agents/skills/openclaw-testing/SKILL.md b/.agents/skills/openclaw-testing/SKILL.md index a043a0508bf..b3c964355a7 100644 --- a/.agents/skills/openclaw-testing/SKILL.md +++ b/.agents/skills/openclaw-testing/SKILL.md @@ -236,7 +236,7 @@ gh workflow run openclaw-live-and-e2e-checks-reusable.yml \ Useful knobs: - `docker_lanes=''`: run selected Docker scheduler lanes against - prepared artifacts instead of the three release chunks. + prepared artifacts instead of the release chunk matrix. - `include_live_suites=false`: skip live/provider suites when testing Docker scheduler or release packaging only. - `live_models_only=true`: run only Docker live model coverage. @@ -249,10 +249,14 @@ coverage through `scripts/test-live-shard.mjs` instead of one serial `live-all` job: - `native-live-src-agents` -- `native-live-src-gateway` +- `native-live-src-gateway-core` +- `native-live-src-gateway-backends` - `native-live-test` - `native-live-extensions-a-k` -- `native-live-extensions-l-z` +- `native-live-extensions-l-n` +- `native-live-extensions-openai` +- `native-live-extensions-o-z` +- `native-live-extensions-media` Use `node scripts/test-live-shard.mjs --list` to see the exact files before rerunning a failed native live shard. @@ -304,19 +308,25 @@ generated inside GitHub artifacts include `package_artifact_run_id`, exact tarball and prepared images from the failed run. When the fix changes package contents, omit those reuse inputs so the workflow packs a new tarball. Live-only targeted reruns skip the E2E images and build only the live-test -image. Release-path normal mode fans out into four Docker chunk jobs: +image. Release-path normal mode fans out into smaller Docker chunk jobs: - `core` -- `package-update` -- `plugins-runtime` +- `package-update-openai` +- `package-update-anthropic` +- `package-update-core` +- `plugins-runtime-core` +- `plugins-runtime-install-a` +- `plugins-runtime-install-b` - `bundled-channels` -OpenWebUI is folded into `plugins-runtime` for full release-path coverage and -keeps a standalone `openwebui` chunk only for OpenWebUI-only dispatches. The -legacy `plugins-integrations` chunk still works as an aggregate alias for manual -reruns, but the release workflow uses the split chunks so plugin runtime checks -and bundled-channel checks can run on separate machines. The bundled-channel -runtime-dependency coverage inside `bundled-channels` +OpenWebUI is folded into `plugins-runtime-core` for full release-path coverage +and keeps a standalone `openwebui` chunk only for OpenWebUI-only dispatches. +The legacy `package-update`, `plugins-runtime`, and `plugins-integrations` +chunks still work as aggregate aliases for manual reruns, but the release +workflow uses the split chunks so provider installer checks, plugin runtime +checks, bundled plugin install/uninstall shards, and bundled-channel checks can +run on separate machines. The bundled-channel runtime-dependency coverage +inside `bundled-channels` uses the split `bundled-channel-*` and `bundled-channel-update-*` lanes rather than the serial `bundled-channel-deps` lane, so failures produce cheap targeted reruns for the exact channel/update scenario. The bundled plugin diff --git a/.github/workflows/openclaw-live-and-e2e-checks-reusable.yml b/.github/workflows/openclaw-live-and-e2e-checks-reusable.yml index b8fedfccfe4..6111aa73538 100644 --- a/.github/workflows/openclaw-live-and-e2e-checks-reusable.yml +++ b/.github/workflows/openclaw-live-and-e2e-checks-reusable.yml @@ -432,11 +432,23 @@ jobs: - chunk_id: core label: core timeout_minutes: 120 - - chunk_id: package-update - label: package/update + - chunk_id: package-update-openai + label: package/update OpenAI install timeout_minutes: 180 - - chunk_id: plugins-runtime - label: plugins/runtime + - chunk_id: package-update-anthropic + label: package/update Anthropic install + timeout_minutes: 180 + - chunk_id: package-update-core + label: package/update core + timeout_minutes: 120 + - chunk_id: plugins-runtime-core + label: plugins/runtime core + timeout_minutes: 180 + - chunk_id: plugins-runtime-install-a + label: plugins/runtime install A + timeout_minutes: 180 + - chunk_id: plugins-runtime-install-b + label: plugins/runtime install B timeout_minutes: 180 - chunk_id: bundled-channels label: bundled channels @@ -1431,9 +1443,15 @@ jobs: timeout_minutes: 90 needs_ffmpeg: false profile_env_only: false - - suite_id: native-live-src-gateway - label: Native live gateway - command: node scripts/test-live-shard.mjs native-live-src-gateway + - suite_id: native-live-src-gateway-core + label: Native live gateway core + command: node scripts/test-live-shard.mjs native-live-src-gateway-core + timeout_minutes: 90 + needs_ffmpeg: false + profile_env_only: false + - suite_id: native-live-src-gateway-backends + label: Native live gateway backends + command: node scripts/test-live-shard.mjs native-live-src-gateway-backends timeout_minutes: 90 needs_ffmpeg: false profile_env_only: false @@ -1449,9 +1467,27 @@ jobs: timeout_minutes: 90 needs_ffmpeg: true profile_env_only: false - - suite_id: native-live-extensions-l-z - label: Native live plugins L-Z - command: node scripts/test-live-shard.mjs native-live-extensions-l-z + - suite_id: native-live-extensions-l-n + label: Native live plugins L-N + command: node scripts/test-live-shard.mjs native-live-extensions-l-n + timeout_minutes: 90 + needs_ffmpeg: false + profile_env_only: false + - suite_id: native-live-extensions-openai + label: Native live OpenAI plugin + command: node scripts/test-live-shard.mjs native-live-extensions-openai + timeout_minutes: 90 + needs_ffmpeg: false + profile_env_only: false + - suite_id: native-live-extensions-o-z + label: Native live plugins O-Z + command: node scripts/test-live-shard.mjs native-live-extensions-o-z + timeout_minutes: 90 + needs_ffmpeg: false + profile_env_only: false + - suite_id: native-live-extensions-media + label: Native live media plugins + command: node scripts/test-live-shard.mjs native-live-extensions-media timeout_minutes: 90 needs_ffmpeg: true profile_env_only: false diff --git a/docs/ci.md b/docs/ci.md index 209cebae8cf..f88105bc1ef 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -20,11 +20,14 @@ child workflow is rerun and turns green, rerun only the parent verifier job to refresh the umbrella result. The release live/E2E child keeps broad native `pnpm test:live` coverage, but it -runs it as named shards (`native-live-src-agents`, `native-live-src-gateway`, -`native-live-test`, `native-live-extensions-a-k`, and -`native-live-extensions-l-z`) through `scripts/test-live-shard.mjs` instead of -one serial job. That keeps the same file coverage while making slow live -provider failures easier to rerun and diagnose. +runs it as named shards (`native-live-src-agents`, +`native-live-src-gateway-core`, `native-live-src-gateway-backends`, +`native-live-test`, `native-live-extensions-a-k`, +`native-live-extensions-l-n`, `native-live-extensions-openai`, +`native-live-extensions-o-z`, and `native-live-extensions-media`) through +`scripts/test-live-shard.mjs` instead of one serial job. That keeps the same +file coverage while making slow live provider failures easier to rerun and +diagnose. `Package Acceptance` is the side-run workflow for validating a package artifact without blocking the release workflow. It resolves one candidate from a @@ -288,7 +291,7 @@ act as if every scoped area changed. CI workflow edits validate the Node CI graph plus workflow linting, but do not force Windows, Android, or macOS native builds by themselves; those platform lanes stay scoped to platform source changes. CI routing-only edits, selected cheap core-test fixture edits, and narrow plugin contract helper/test-routing edits use a fast Node-only manifest path: preflight, security, and a single `checks-fast-core` task. That path avoids build artifacts, Node 22 compatibility, channel contracts, full core shards, bundled-plugin shards, and additional guard matrices when the changed files are limited to the routing or helper surfaces that the fast task exercises directly. Windows Node checks are scoped to Windows-specific process/path wrappers, npm/pnpm/UI runner helpers, package manager config, and the CI workflow surfaces that execute that lane; unrelated source, plugin, install-smoke, and test-only changes stay on the Linux Node lanes so they do not reserve a 16-vCPU Windows worker for coverage that is already exercised by the normal test shards. -The separate `install-smoke` workflow reuses the same scope script through its own `preflight` job. It splits smoke coverage into `run_fast_install_smoke` and `run_full_install_smoke`. Pull requests run the fast path for Docker/package surfaces, bundled plugin package/manifest changes, and core plugin/channel/gateway/Plugin SDK surfaces that the Docker smoke jobs exercise. Source-only bundled plugin changes, test-only edits, and docs-only edits do not reserve Docker workers. The fast path builds the root Dockerfile image once, checks the CLI, runs the agents delete shared-workspace CLI smoke, runs the container gateway-network e2e, verifies a bundled extension build arg, and runs the bounded bundled-plugin Docker profile under a 240-second aggregate command timeout with each scenario's Docker run capped separately. The full path keeps QR package install and installer Docker/update coverage for nightly scheduled runs, manual dispatches, workflow-call release checks, and pull requests that truly touch installer/package/Docker surfaces. `main` pushes, including merge commits, do not force the full path; when changed-scope logic would request full coverage on a push, the workflow keeps the fast Docker smoke and leaves the full install smoke to nightly or release validation. The slow Bun global install image-provider smoke is separately gated by `run_bun_global_install_smoke`; it runs on the nightly schedule and from the release checks workflow, and manual `install-smoke` dispatches can opt into it, but pull requests and `main` pushes do not run it. QR and installer Docker tests keep their own install-focused Dockerfiles. Local `test:docker:all` prebuilds one shared live-test image, packs OpenClaw once as an npm tarball, and builds two shared `scripts/e2e/Dockerfile` images: a bare Node/Git runner for installer/update/plugin-dependency lanes and a functional image that installs the same tarball into `/app` for normal functionality lanes. Docker lane definitions live in `scripts/lib/docker-e2e-scenarios.mjs`, planner logic lives in `scripts/lib/docker-e2e-plan.mjs`, and the runner only executes the selected plan. The scheduler selects the image per lane with `OPENCLAW_DOCKER_E2E_BARE_IMAGE` and `OPENCLAW_DOCKER_E2E_FUNCTIONAL_IMAGE`, then runs lanes with `OPENCLAW_SKIP_DOCKER_BUILD=1`; tune the default main-pool slot count of 10 with `OPENCLAW_DOCKER_ALL_PARALLELISM` and the provider-sensitive tail-pool slot count of 10 with `OPENCLAW_DOCKER_ALL_TAIL_PARALLELISM`. Heavy lane caps default to `OPENCLAW_DOCKER_ALL_LIVE_LIMIT=9`, `OPENCLAW_DOCKER_ALL_NPM_LIMIT=10`, and `OPENCLAW_DOCKER_ALL_SERVICE_LIMIT=7` so npm install and multi-service lanes do not overcommit Docker while lighter lanes still fill available slots. A single lane heavier than the effective caps can still start from an empty pool, then runs alone until it releases capacity. Lane starts are staggered by 2 seconds by default to avoid local Docker daemon create storms; override with `OPENCLAW_DOCKER_ALL_START_STAGGER_MS=0` or another millisecond value. The local aggregate preflights Docker, removes stale OpenClaw E2E containers, emits active-lane status, persists lane timings for longest-first ordering, and supports `OPENCLAW_DOCKER_ALL_DRY_RUN=1` for scheduler inspection. It stops scheduling new pooled lanes after the first failure by default, and each lane has a 120-minute fallback timeout overrideable with `OPENCLAW_DOCKER_ALL_LANE_TIMEOUT_MS`; selected live/tail lanes use tighter per-lane caps. `OPENCLAW_DOCKER_ALL_LANES=` runs exact scheduler lanes, including release-only lanes such as `install-e2e` and split bundled update lanes such as `bundled-channel-update-acpx`, while skipping the cleanup smoke so agents can reproduce one failed lane. The reusable live/E2E workflow asks `scripts/test-docker-all.mjs --plan-json` which package, image kind, live image, lane, and credential coverage is required, then `scripts/docker-e2e.mjs` converts that plan into GitHub outputs and summaries. It either packs OpenClaw through `scripts/package-openclaw-for-docker.mjs`, downloads a current-run package artifact, or downloads a package artifact from `package_artifact_run_id`; validates the tarball inventory; builds and pushes package-digest-tagged bare/functional GHCR Docker E2E images through Blacksmith's Docker layer cache when the plan needs package-installed lanes; and reuses provided `docker_e2e_bare_image`/`docker_e2e_functional_image` inputs or existing package-digest images instead of rebuilding. The `Package Acceptance` workflow is the high-level package gate: it resolves a candidate from npm, a trusted `package_ref`, an HTTPS tarball plus SHA-256, or a prior workflow artifact, then passes that single `package-under-test` artifact into the reusable Docker E2E workflow. It keeps `workflow_ref` separate from `package_ref` so current acceptance logic can validate older trusted commits without checking out old workflow code. Release checks run a custom Package Acceptance delta for the target ref: bundled-channel compat, offline plugin fixtures, and Telegram package QA against the resolved tarball. The release-path Docker suite runs four chunked jobs with `OPENCLAW_SKIP_DOCKER_BUILD=1` so each chunk pulls only the image kind it needs and executes multiple lanes through the same weighted scheduler (`OPENCLAW_DOCKER_ALL_PROFILE=release-path`, `OPENCLAW_DOCKER_ALL_CHUNK=core|package-update|plugins-runtime|bundled-channels`). OpenWebUI is folded into `plugins-runtime` when full release-path coverage requests it, and keeps a standalone `openwebui` chunk only for OpenWebUI-only dispatches. The `package-update` chunk splits installer E2E into `install-e2e-openai` and `install-e2e-anthropic`; `install-e2e` remains the aggregate manual rerun alias. The `bundled-channels` chunk runs split `bundled-channel-*` and `bundled-channel-update-*` lanes rather than the serial all-in-one `bundled-channel-deps` lane; `plugins-integrations` remains a legacy aggregate alias for manual reruns. Each chunk uploads `.artifacts/docker-tests/` with lane logs, timings, `summary.json`, `failures.json`, phase timings, scheduler plan JSON, slow-lane tables, and per-lane rerun commands. The workflow `docker_lanes` input runs selected lanes against the prepared images instead of the chunk jobs, which keeps failed-lane debugging bounded to one targeted Docker job and prepares, downloads, or reuses the package artifact for that run; if a selected lane is a live Docker lane, the targeted job builds the live-test image locally for that rerun. Generated per-lane GitHub rerun commands include `package_artifact_run_id`, `package_artifact_name`, and prepared image inputs when those values exist, so a failed lane can reuse the exact package and images from the failed run. Use `pnpm test:docker:rerun ` to download Docker artifacts from a GitHub run and print combined/per-lane targeted rerun commands; use `pnpm test:docker:timings ` for slow-lane and phase critical-path summaries. The scheduled live/E2E workflow runs the full release-path Docker suite daily. The bundled update matrix is split by update target so repeated npm update and doctor repair passes can shard with other bundled checks. +The separate `install-smoke` workflow reuses the same scope script through its own `preflight` job. It splits smoke coverage into `run_fast_install_smoke` and `run_full_install_smoke`. Pull requests run the fast path for Docker/package surfaces, bundled plugin package/manifest changes, and core plugin/channel/gateway/Plugin SDK surfaces that the Docker smoke jobs exercise. Source-only bundled plugin changes, test-only edits, and docs-only edits do not reserve Docker workers. The fast path builds the root Dockerfile image once, checks the CLI, runs the agents delete shared-workspace CLI smoke, runs the container gateway-network e2e, verifies a bundled extension build arg, and runs the bounded bundled-plugin Docker profile under a 240-second aggregate command timeout with each scenario's Docker run capped separately. The full path keeps QR package install and installer Docker/update coverage for nightly scheduled runs, manual dispatches, workflow-call release checks, and pull requests that truly touch installer/package/Docker surfaces. `main` pushes, including merge commits, do not force the full path; when changed-scope logic would request full coverage on a push, the workflow keeps the fast Docker smoke and leaves the full install smoke to nightly or release validation. The slow Bun global install image-provider smoke is separately gated by `run_bun_global_install_smoke`; it runs on the nightly schedule and from the release checks workflow, and manual `install-smoke` dispatches can opt into it, but pull requests and `main` pushes do not run it. QR and installer Docker tests keep their own install-focused Dockerfiles. Local `test:docker:all` prebuilds one shared live-test image, packs OpenClaw once as an npm tarball, and builds two shared `scripts/e2e/Dockerfile` images: a bare Node/Git runner for installer/update/plugin-dependency lanes and a functional image that installs the same tarball into `/app` for normal functionality lanes. Docker lane definitions live in `scripts/lib/docker-e2e-scenarios.mjs`, planner logic lives in `scripts/lib/docker-e2e-plan.mjs`, and the runner only executes the selected plan. The scheduler selects the image per lane with `OPENCLAW_DOCKER_E2E_BARE_IMAGE` and `OPENCLAW_DOCKER_E2E_FUNCTIONAL_IMAGE`, then runs lanes with `OPENCLAW_SKIP_DOCKER_BUILD=1`; tune the default main-pool slot count of 10 with `OPENCLAW_DOCKER_ALL_PARALLELISM` and the provider-sensitive tail-pool slot count of 10 with `OPENCLAW_DOCKER_ALL_TAIL_PARALLELISM`. Heavy lane caps default to `OPENCLAW_DOCKER_ALL_LIVE_LIMIT=9`, `OPENCLAW_DOCKER_ALL_NPM_LIMIT=10`, and `OPENCLAW_DOCKER_ALL_SERVICE_LIMIT=7` so npm install and multi-service lanes do not overcommit Docker while lighter lanes still fill available slots. A single lane heavier than the effective caps can still start from an empty pool, then runs alone until it releases capacity. Lane starts are staggered by 2 seconds by default to avoid local Docker daemon create storms; override with `OPENCLAW_DOCKER_ALL_START_STAGGER_MS=0` or another millisecond value. The local aggregate preflights Docker, removes stale OpenClaw E2E containers, emits active-lane status, persists lane timings for longest-first ordering, and supports `OPENCLAW_DOCKER_ALL_DRY_RUN=1` for scheduler inspection. It stops scheduling new pooled lanes after the first failure by default, and each lane has a 120-minute fallback timeout overrideable with `OPENCLAW_DOCKER_ALL_LANE_TIMEOUT_MS`; selected live/tail lanes use tighter per-lane caps. `OPENCLAW_DOCKER_ALL_LANES=` runs exact scheduler lanes, including release-only lanes such as `install-e2e` and split bundled update lanes such as `bundled-channel-update-acpx`, while skipping the cleanup smoke so agents can reproduce one failed lane. The reusable live/E2E workflow asks `scripts/test-docker-all.mjs --plan-json` which package, image kind, live image, lane, and credential coverage is required, then `scripts/docker-e2e.mjs` converts that plan into GitHub outputs and summaries. It either packs OpenClaw through `scripts/package-openclaw-for-docker.mjs`, downloads a current-run package artifact, or downloads a package artifact from `package_artifact_run_id`; validates the tarball inventory; builds and pushes package-digest-tagged bare/functional GHCR Docker E2E images through Blacksmith's Docker layer cache when the plan needs package-installed lanes; and reuses provided `docker_e2e_bare_image`/`docker_e2e_functional_image` inputs or existing package-digest images instead of rebuilding. The `Package Acceptance` workflow is the high-level package gate: it resolves a candidate from npm, a trusted `package_ref`, an HTTPS tarball plus SHA-256, or a prior workflow artifact, then passes that single `package-under-test` artifact into the reusable Docker E2E workflow. It keeps `workflow_ref` separate from `package_ref` so current acceptance logic can validate older trusted commits without checking out old workflow code. Release checks run a custom Package Acceptance delta for the target ref: bundled-channel compat, offline plugin fixtures, and Telegram package QA against the resolved tarball. The release-path Docker suite runs smaller chunked jobs with `OPENCLAW_SKIP_DOCKER_BUILD=1` so each chunk pulls only the image kind it needs and executes multiple lanes through the same weighted scheduler (`OPENCLAW_DOCKER_ALL_PROFILE=release-path`, `OPENCLAW_DOCKER_ALL_CHUNK=core|package-update-openai|package-update-anthropic|package-update-core|plugins-runtime-core|plugins-runtime-install-a|plugins-runtime-install-b|bundled-channels`). OpenWebUI is folded into `plugins-runtime-core` when full release-path coverage requests it, and keeps a standalone `openwebui` chunk only for OpenWebUI-only dispatches. The legacy aggregate chunk names `package-update`, `plugins-runtime`, and `plugins-integrations` still work for manual reruns, but the release workflow uses the split chunks so installer E2E and bundled plugin install/uninstall sweeps do not dominate the critical path. The `install-e2e` lane alias remains the aggregate manual rerun alias for both provider installer lanes. The `bundled-channels` chunk runs split `bundled-channel-*` and `bundled-channel-update-*` lanes rather than the serial all-in-one `bundled-channel-deps` lane. Each chunk uploads `.artifacts/docker-tests/` with lane logs, timings, `summary.json`, `failures.json`, phase timings, scheduler plan JSON, slow-lane tables, and per-lane rerun commands. The workflow `docker_lanes` input runs selected lanes against the prepared images instead of the chunk jobs, which keeps failed-lane debugging bounded to one targeted Docker job and prepares, downloads, or reuses the package artifact for that run; if a selected lane is a live Docker lane, the targeted job builds the live-test image locally for that rerun. Generated per-lane GitHub rerun commands include `package_artifact_run_id`, `package_artifact_name`, and prepared image inputs when those values exist, so a failed lane can reuse the exact package and images from the failed run. Use `pnpm test:docker:rerun ` to download Docker artifacts from a GitHub run and print combined/per-lane targeted rerun commands; use `pnpm test:docker:timings ` for slow-lane and phase critical-path summaries. The scheduled live/E2E workflow runs the full release-path Docker suite daily. The bundled update matrix is split by update target so repeated npm update and doctor repair passes can shard with other bundled checks. Local changed-lane logic lives in `scripts/changed-lanes.mjs` and is executed by `scripts/check-changed.mjs`. That local check gate is stricter about architecture boundaries than the broad CI platform scope: core production changes run core prod and core test typecheck plus core lint/guards, core test-only changes run only core test typecheck plus core lint, extension production changes run extension prod and extension test typecheck plus extension lint, and extension test-only changes run extension test typecheck plus extension lint. Public Plugin SDK or plugin-contract changes expand to extension typecheck because extensions depend on those core contracts, but Vitest extension sweeps are explicit test work. Release metadata-only version bumps run targeted version/config/root-dependency checks. Unknown root/config changes fail safe to all check lanes. diff --git a/docs/help/testing.md b/docs/help/testing.md index 686d1a23a8c..b08eb95683b 100644 --- a/docs/help/testing.md +++ b/docs/help/testing.md @@ -607,7 +607,7 @@ The live-model Docker runners also bind-mount only the needed CLI auth homes (or Set `OPENCLAW_PLUGINS_E2E_CLAWHUB=0` to skip the live ClawHub block, or override the default package with `OPENCLAW_PLUGINS_E2E_CLAWHUB_SPEC` and `OPENCLAW_PLUGINS_E2E_CLAWHUB_ID`. - Plugin update unchanged smoke: `pnpm test:docker:plugin-update` (script: `scripts/e2e/plugin-update-unchanged-docker.sh`) - Config reload metadata smoke: `pnpm test:docker:config-reload` (script: `scripts/e2e/config-reload-source-docker.sh`) -- Bundled plugin runtime deps: `pnpm test:docker:bundled-channel-deps` builds a small Docker runner image by default, builds and packs OpenClaw once on the host, then mounts that tarball into each Linux install scenario. Reuse the image with `OPENCLAW_SKIP_DOCKER_BUILD=1`, skip the host rebuild after a fresh local build with `OPENCLAW_BUNDLED_CHANNEL_HOST_BUILD=0`, or point at an existing tarball with `OPENCLAW_CURRENT_PACKAGE_TGZ=/path/to/openclaw-*.tgz`. The full Docker aggregate and release-path `bundled-channels` chunk pre-pack this tarball once, then shard bundled channel checks into independent lanes, including separate update lanes for Telegram, Discord, Slack, Feishu, memory-lancedb, and ACPX. The legacy `plugins-integrations` chunk remains an aggregate alias for manual reruns. Use `OPENCLAW_BUNDLED_CHANNELS=telegram,slack` to narrow the channel matrix when running the bundled lane directly, or `OPENCLAW_BUNDLED_CHANNEL_UPDATE_TARGETS=telegram,acpx` to narrow the update scenario. The lane also verifies that `channels..enabled=false` and `plugins.entries..enabled=false` suppress doctor/runtime-dependency repair. +- Bundled plugin runtime deps: `pnpm test:docker:bundled-channel-deps` builds a small Docker runner image by default, builds and packs OpenClaw once on the host, then mounts that tarball into each Linux install scenario. Reuse the image with `OPENCLAW_SKIP_DOCKER_BUILD=1`, skip the host rebuild after a fresh local build with `OPENCLAW_BUNDLED_CHANNEL_HOST_BUILD=0`, or point at an existing tarball with `OPENCLAW_CURRENT_PACKAGE_TGZ=/path/to/openclaw-*.tgz`. The full Docker aggregate and release-path `bundled-channels` chunk pre-pack this tarball once, then shard bundled channel checks into independent lanes, including separate update lanes for Telegram, Discord, Slack, Feishu, memory-lancedb, and ACPX. The release workflow also splits provider installer chunks and bundled plugin install/uninstall chunks; legacy `package-update`, `plugins-runtime`, and `plugins-integrations` chunks remain aggregate aliases for manual reruns. Use `OPENCLAW_BUNDLED_CHANNELS=telegram,slack` to narrow the channel matrix when running the bundled lane directly, or `OPENCLAW_BUNDLED_CHANNEL_UPDATE_TARGETS=telegram,acpx` to narrow the update scenario. The lane also verifies that `channels..enabled=false` and `plugins.entries..enabled=false` suppress doctor/runtime-dependency repair. - Narrow bundled plugin runtime deps while iterating by disabling unrelated scenarios, for example: `OPENCLAW_BUNDLED_CHANNEL_SCENARIOS=0 OPENCLAW_BUNDLED_CHANNEL_UPDATE_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_ROOT_OWNED_SCENARIO=0 OPENCLAW_BUNDLED_CHANNEL_SETUP_ENTRY_SCENARIO=0 pnpm test:docker:bundled-channel-deps`. diff --git a/docs/reference/RELEASING.md b/docs/reference/RELEASING.md index 871d0740a42..c1d212331d2 100644 --- a/docs/reference/RELEASING.md +++ b/docs/reference/RELEASING.md @@ -320,9 +320,11 @@ Release Docker coverage includes: - full install smoke with the slow Bun global install smoke enabled - repository E2E lanes -- release-path Docker chunks: `core`, `package-update`, `plugins-runtime`, and +- release-path Docker chunks: `core`, `package-update-openai`, + `package-update-anthropic`, `package-update-core`, `plugins-runtime-core`, + `plugins-runtime-install-a`, `plugins-runtime-install-b`, and `bundled-channels` -- OpenWebUI coverage inside the `plugins-runtime` chunk when requested +- OpenWebUI coverage inside the `plugins-runtime-core` chunk when requested - split bundled-channel dependency lanes in their own `bundled-channels` chunk instead of the serial all-in-one bundled-channel lane - split bundled plugin install/uninstall lanes diff --git a/scripts/lib/docker-e2e-scenarios.mjs b/scripts/lib/docker-e2e-scenarios.mjs index 869b2be3b0e..f9e9bd3803e 100644 --- a/scripts/lib/docker-e2e-scenarios.mjs +++ b/scripts/lib/docker-e2e-scenarios.mjs @@ -372,12 +372,73 @@ const releasePathPluginRuntimeLanes = [ ), ]; +const releasePathPluginRuntimeCoreLanes = [ + lane("plugins", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:plugins", { + resources: ["npm", "service"], + weight: 6, + }), + serviceLane( + "cron-mcp-cleanup", + "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:cron-mcp-cleanup", + { + resources: ["npm"], + weight: 3, + }, + ), + serviceLane( + "openai-web-search-minimal", + "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:openai-web-search-minimal", + { timeoutMs: 8 * 60 * 1000 }, + ), +]; + const releasePathBundledChannelLanes = [ npmLane("plugin-update", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:plugin-update"), ...bundledScenarioLanes, ]; -const releasePathChunks = { +const releasePathPackageInstallOpenAiLanes = [ + npmLane( + "install-e2e-openai", + "OPENCLAW_INSTALL_TAG=beta OPENCLAW_E2E_MODELS=openai OPENCLAW_INSTALL_E2E_IMAGE=openclaw-install-e2e-openai:local pnpm test:install:e2e", + { + resources: ["service"], + weight: 3, + }, + ), +]; + +const releasePathPackageInstallAnthropicLanes = [ + npmLane( + "install-e2e-anthropic", + "OPENCLAW_INSTALL_TAG=beta OPENCLAW_E2E_MODELS=anthropic OPENCLAW_INSTALL_E2E_IMAGE=openclaw-install-e2e-anthropic:local pnpm test:install:e2e", + { + resources: ["service"], + weight: 3, + }, + ), +]; + +const releasePathPackageUpdateCoreLanes = [ + npmLane( + "npm-onboard-channel-agent", + "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:npm-onboard-channel-agent", + { resources: ["service"], weight: 3 }, + ), + npmLane("doctor-switch", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:doctor-switch", { + weight: 3, + }), + npmLane( + "update-channel-switch", + "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:update-channel-switch", + { + timeoutMs: 30 * 60 * 1000, + weight: 3, + }, + ), +]; + +const primaryReleasePathChunks = { core: [ lane("qr", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:qr"), serviceLane("onboard", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:onboard", { @@ -398,46 +459,23 @@ const releasePathChunks = { weight: 3, }), ], - "package-update": [ - npmLane( - "install-e2e-openai", - "OPENCLAW_INSTALL_TAG=beta OPENCLAW_E2E_MODELS=openai OPENCLAW_INSTALL_E2E_IMAGE=openclaw-install-e2e-openai:local pnpm test:install:e2e", - { - resources: ["service"], - weight: 3, - }, - ), - npmLane( - "install-e2e-anthropic", - "OPENCLAW_INSTALL_TAG=beta OPENCLAW_E2E_MODELS=anthropic OPENCLAW_INSTALL_E2E_IMAGE=openclaw-install-e2e-anthropic:local pnpm test:install:e2e", - { - resources: ["service"], - weight: 3, - }, - ), - npmLane( - "npm-onboard-channel-agent", - "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:npm-onboard-channel-agent", - { resources: ["service"], weight: 3 }, - ), - npmLane("doctor-switch", "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:doctor-switch", { - weight: 3, - }), - npmLane( - "update-channel-switch", - "OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:update-channel-switch", - { - timeoutMs: 30 * 60 * 1000, - weight: 3, - }, - ), - ], - "plugins-runtime": releasePathPluginRuntimeLanes, + "package-update-openai": releasePathPackageInstallOpenAiLanes, + "package-update-anthropic": releasePathPackageInstallAnthropicLanes, + "package-update-core": releasePathPackageUpdateCoreLanes, + "plugins-runtime-core": releasePathPluginRuntimeCoreLanes, + "plugins-runtime-install-a": bundledPluginInstallUninstallLanes.slice(0, 4), + "plugins-runtime-install-b": bundledPluginInstallUninstallLanes.slice(4), "bundled-channels": releasePathBundledChannelLanes, openwebui: [], }; const legacyReleasePathChunks = { + "package-update": [ + ...releasePathPackageInstallOpenAiLanes, + ...releasePathPackageInstallAnthropicLanes, + ...releasePathPackageUpdateCoreLanes, + ], + "plugins-runtime": releasePathPluginRuntimeLanes, "plugins-integrations": [...releasePathPluginRuntimeLanes, ...releasePathBundledChannelLanes], }; @@ -449,11 +487,11 @@ function openWebUILane() { } export function releasePathChunkLanes(chunk, options = {}) { - const base = releasePathChunks[chunk] ?? legacyReleasePathChunks[chunk]; + const base = primaryReleasePathChunks[chunk] ?? legacyReleasePathChunks[chunk]; if (!base) { throw new Error( `OPENCLAW_DOCKER_ALL_CHUNK must be one of: ${[ - ...Object.keys(releasePathChunks), + ...Object.keys(primaryReleasePathChunks), ...Object.keys(legacyReleasePathChunks), ].join(", ")}. Got: ${JSON.stringify(chunk)}`, ); @@ -462,7 +500,9 @@ export function releasePathChunkLanes(chunk, options = {}) { return options.includeOpenWebUI ? [openWebUILane()] : []; } if ( - (chunk !== "plugins-runtime" && chunk !== "plugins-integrations") || + (chunk !== "plugins-runtime-core" && + chunk !== "plugins-runtime" && + chunk !== "plugins-integrations") || !options.includeOpenWebUI ) { return base; @@ -471,7 +511,7 @@ export function releasePathChunkLanes(chunk, options = {}) { } export function allReleasePathLanes(options = {}) { - return Object.keys(releasePathChunks) + return Object.keys(primaryReleasePathChunks) .filter((chunk) => chunk !== "openwebui") .flatMap((chunk) => releasePathChunkLanes(chunk, { diff --git a/scripts/test-live-shard.mjs b/scripts/test-live-shard.mjs index b58e352d573..a75c8ee6075 100644 --- a/scripts/test-live-shard.mjs +++ b/scripts/test-live-shard.mjs @@ -8,10 +8,14 @@ const LIVE_TEST_SUFFIX = ".live.test.ts"; export const LIVE_TEST_SHARDS = Object.freeze([ "native-live-src-agents", - "native-live-src-gateway", + "native-live-src-gateway-core", + "native-live-src-gateway-backends", "native-live-test", "native-live-extensions-a-k", - "native-live-extensions-l-z", + "native-live-extensions-l-n", + "native-live-extensions-openai", + "native-live-extensions-o-z", + "native-live-extensions-media", ]); function walkFiles(rootDir) { @@ -70,6 +74,25 @@ function isExtensionInRange(file, start, end) { return first >= start && first <= end; } +function isGatewayBackendLiveTest(file) { + return ( + file === "src/gateway/gateway-acp-bind.live.test.ts" || + file === "src/gateway/gateway-cli-backend.live.test.ts" || + file === "src/gateway/gateway-codex-bind.live.test.ts" || + file === "src/gateway/gateway-codex-harness.live.test.ts" + ); +} + +function isExtensionMediaLiveTest(file) { + return ( + file === "extensions/music-generation-providers.live.test.ts" || + file === "extensions/openai/openai-tts.live.test.ts" || + file === "extensions/video-generation-providers.live.test.ts" || + file === "extensions/volcengine/tts.live.test.ts" || + file === "extensions/vydra/vydra.live.test.ts" + ); +} + export function selectLiveShardFiles(shard, files = collectAllLiveTestFiles()) { switch (shard) { case "native-live-src-agents": @@ -78,10 +101,38 @@ export function selectLiveShardFiles(shard, files = collectAllLiveTestFiles()) { return files.filter( (file) => file.startsWith("src/gateway/") || file.startsWith("src/crestodian/"), ); + case "native-live-src-gateway-core": + return files.filter( + (file) => + (file.startsWith("src/gateway/") || file.startsWith("src/crestodian/")) && + !isGatewayBackendLiveTest(file), + ); + case "native-live-src-gateway-backends": + return files.filter(isGatewayBackendLiveTest); case "native-live-test": return files.filter((file) => file.startsWith("test/")); case "native-live-extensions-a-k": return files.filter((file) => isExtensionInRange(file, "a", "k")); + case "native-live-extensions-l-n": + return files.filter( + (file) => + isExtensionInRange(file, "l", "n") && + !file.startsWith("extensions/openai/") && + !isExtensionMediaLiveTest(file), + ); + case "native-live-extensions-openai": + return files.filter( + (file) => file.startsWith("extensions/openai/") && !isExtensionMediaLiveTest(file), + ); + case "native-live-extensions-o-z": + return files.filter( + (file) => + isExtensionInRange(file, "o", "z") && + !file.startsWith("extensions/openai/") && + !isExtensionMediaLiveTest(file), + ); + case "native-live-extensions-media": + return files.filter(isExtensionMediaLiveTest); case "native-live-extensions-l-z": return files.filter((file) => isExtensionInRange(file, "l", "z")); default: diff --git a/test/scripts/docker-e2e-plan.test.ts b/test/scripts/docker-e2e-plan.test.ts index 5d863589931..437c6f8ca6f 100644 --- a/test/scripts/docker-e2e-plan.test.ts +++ b/test/scripts/docker-e2e-plan.test.ts @@ -48,6 +48,10 @@ describe("scripts/lib/docker-e2e-plan", () => { expect(plan.lanes.map((lane) => lane.name)).toContain("bundled-channel-update-acpx"); expect(plan.lanes.map((lane) => lane.name)).toContain("bundled-plugin-install-uninstall-0"); expect(plan.lanes.map((lane) => lane.name)).toContain("bundled-plugin-install-uninstall-7"); + expect(plan.lanes.filter((lane) => lane.name === "install-e2e-openai")).toHaveLength(1); + expect( + plan.lanes.filter((lane) => lane.name === "bundled-plugin-install-uninstall-0"), + ).toHaveLength(1); expect(plan.lanes.map((lane) => lane.name)).not.toContain("bundled-plugin-install-uninstall"); expect(plan.lanes.map((lane) => lane.name)).not.toContain("bundled-channel-deps"); expect(plan.lanes.map((lane) => lane.name)).not.toContain("openwebui"); @@ -69,11 +73,36 @@ describe("scripts/lib/docker-e2e-plan", () => { expect(withOpenWebUI.lanes.map((lane) => lane.name)).toContain("openwebui"); }); - it("splits the old plugins/integrations release chunk across plugin and bundled-channel chunks", () => { - const pluginsRuntime = planFor({ + it("splits release-path package and plugin chunks across shorter CI jobs", () => { + const packageInstallOpenAi = planFor({ includeOpenWebUI: true, profile: RELEASE_PATH_PROFILE, - releaseChunk: "plugins-runtime", + releaseChunk: "package-update-openai", + }); + const packageInstallAnthropic = planFor({ + includeOpenWebUI: true, + profile: RELEASE_PATH_PROFILE, + releaseChunk: "package-update-anthropic", + }); + const packageUpdateCore = planFor({ + includeOpenWebUI: true, + profile: RELEASE_PATH_PROFILE, + releaseChunk: "package-update-core", + }); + const pluginsRuntimeCore = planFor({ + includeOpenWebUI: true, + profile: RELEASE_PATH_PROFILE, + releaseChunk: "plugins-runtime-core", + }); + const pluginsRuntimeInstallA = planFor({ + includeOpenWebUI: true, + profile: RELEASE_PATH_PROFILE, + releaseChunk: "plugins-runtime-install-a", + }); + const pluginsRuntimeInstallB = planFor({ + includeOpenWebUI: true, + profile: RELEASE_PATH_PROFILE, + releaseChunk: "plugins-runtime-install-b", }); const bundledChannels = planFor({ includeOpenWebUI: true, @@ -81,17 +110,38 @@ describe("scripts/lib/docker-e2e-plan", () => { releaseChunk: "bundled-channels", }); - expect(pluginsRuntime.lanes.map((lane) => lane.name)).toEqual( + expect(packageInstallOpenAi.lanes.map((lane) => lane.name)).toEqual(["install-e2e-openai"]); + expect(packageInstallAnthropic.lanes.map((lane) => lane.name)).toEqual([ + "install-e2e-anthropic", + ]); + expect(packageUpdateCore.lanes.map((lane) => lane.name)).toEqual([ + "npm-onboard-channel-agent", + "doctor-switch", + "update-channel-switch", + ]); + expect(pluginsRuntimeCore.lanes.map((lane) => lane.name)).toEqual( expect.arrayContaining([ "plugins", - "bundled-plugin-install-uninstall-0", - "bundled-plugin-install-uninstall-7", "cron-mcp-cleanup", "openai-web-search-minimal", "openwebui", ]), ); - expect(pluginsRuntime.lanes.map((lane) => lane.name)).not.toContain("bundled-channel-telegram"); + expect(pluginsRuntimeCore.lanes.map((lane) => lane.name)).not.toContain( + "bundled-plugin-install-uninstall-0", + ); + expect(pluginsRuntimeInstallA.lanes.map((lane) => lane.name)).toEqual([ + "bundled-plugin-install-uninstall-0", + "bundled-plugin-install-uninstall-1", + "bundled-plugin-install-uninstall-2", + "bundled-plugin-install-uninstall-3", + ]); + expect(pluginsRuntimeInstallB.lanes.map((lane) => lane.name)).toEqual([ + "bundled-plugin-install-uninstall-4", + "bundled-plugin-install-uninstall-5", + "bundled-plugin-install-uninstall-6", + "bundled-plugin-install-uninstall-7", + ]); expect(bundledChannels.lanes.map((lane) => lane.name)).toEqual( expect.arrayContaining([ "plugin-update", @@ -103,13 +153,38 @@ describe("scripts/lib/docker-e2e-plan", () => { expect(bundledChannels.lanes.map((lane) => lane.name)).not.toContain("openwebui"); }); - it("keeps the legacy plugins-integrations release chunk as an aggregate alias", () => { + it("keeps legacy release chunk names as aggregate aliases", () => { + const packageUpdate = planFor({ + includeOpenWebUI: true, + profile: RELEASE_PATH_PROFILE, + releaseChunk: "package-update", + }); + const pluginsRuntime = planFor({ + includeOpenWebUI: true, + profile: RELEASE_PATH_PROFILE, + releaseChunk: "plugins-runtime", + }); const legacy = planFor({ includeOpenWebUI: true, profile: RELEASE_PATH_PROFILE, releaseChunk: "plugins-integrations", }); + expect(packageUpdate.lanes.map((lane) => lane.name)).toEqual( + expect.arrayContaining([ + "install-e2e-openai", + "install-e2e-anthropic", + "update-channel-switch", + ]), + ); + expect(pluginsRuntime.lanes.map((lane) => lane.name)).toEqual( + expect.arrayContaining([ + "plugins", + "bundled-plugin-install-uninstall-0", + "bundled-plugin-install-uninstall-7", + "openwebui", + ]), + ); expect(legacy.lanes.map((lane) => lane.name)).toEqual( expect.arrayContaining([ "plugins", diff --git a/test/scripts/package-acceptance-workflow.test.ts b/test/scripts/package-acceptance-workflow.test.ts index d3703bbc1f1..a0e0785bd7b 100644 --- a/test/scripts/package-acceptance-workflow.test.ts +++ b/test/scripts/package-acceptance-workflow.test.ts @@ -97,9 +97,13 @@ describe("package artifact reuse", () => { expect(workflow).not.toContain("command: pnpm test:live\n"); expect(workflow).toContain("suite_id: native-live-src-agents"); expect(workflow).toContain("command: node scripts/test-live-shard.mjs native-live-src-agents"); - expect(workflow).toContain("suite_id: native-live-src-gateway"); + expect(workflow).toContain("suite_id: native-live-src-gateway-core"); + expect(workflow).toContain("suite_id: native-live-src-gateway-backends"); expect(workflow).toContain("suite_id: native-live-extensions-a-k"); - expect(workflow).toContain("suite_id: native-live-extensions-l-z"); + expect(workflow).toContain("suite_id: native-live-extensions-l-n"); + expect(workflow).toContain("suite_id: native-live-extensions-openai"); + expect(workflow).toContain("suite_id: native-live-extensions-o-z"); + expect(workflow).toContain("suite_id: native-live-extensions-media"); expect(workflow).toContain("if: matrix.needs_ffmpeg"); }); diff --git a/test/scripts/test-live-shard.test.ts b/test/scripts/test-live-shard.test.ts index 9e46b3985c6..2246fad2b07 100644 --- a/test/scripts/test-live-shard.test.ts +++ b/test/scripts/test-live-shard.test.ts @@ -18,21 +18,38 @@ describe("scripts/test-live-shard", () => { expect(new Set(selectedFiles).size).toBe(selectedFiles.length); }); - it("keeps media-capable extension and test harness files in their own shards", () => { + it("keeps slow gateway backend and media-capable extension files in their own shards", () => { const allFiles = collectAllLiveTestFiles(); + expect(selectLiveShardFiles("native-live-src-gateway-backends", allFiles)).toEqual( + expect.arrayContaining([ + "src/gateway/gateway-acp-bind.live.test.ts", + "src/gateway/gateway-cli-backend.live.test.ts", + "src/gateway/gateway-codex-bind.live.test.ts", + "src/gateway/gateway-codex-harness.live.test.ts", + ]), + ); + expect(selectLiveShardFiles("native-live-src-gateway-core", allFiles)).not.toEqual( + expect.arrayContaining(["src/gateway/gateway-cli-backend.live.test.ts"]), + ); expect(selectLiveShardFiles("native-live-test", allFiles)).toEqual( expect.arrayContaining([ "test/image-generation.infer-cli.live.test.ts", "test/image-generation.runtime.live.test.ts", ]), ); - expect(selectLiveShardFiles("native-live-extensions-l-z", allFiles)).toEqual( + expect(selectLiveShardFiles("native-live-extensions-media", allFiles)).toEqual( expect.arrayContaining([ + "extensions/openai/openai-tts.live.test.ts", "extensions/music-generation-providers.live.test.ts", "extensions/video-generation-providers.live.test.ts", + "extensions/volcengine/tts.live.test.ts", + "extensions/vydra/vydra.live.test.ts", ]), ); + expect(selectLiveShardFiles("native-live-extensions-openai", allFiles)).toEqual( + expect.arrayContaining(["extensions/openai/openai-provider.live.test.ts"]), + ); }); it("rejects unknown shard names", () => {