From a0fd105e5e41fafd7f537843af0dea9bd1a7336f Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Wed, 29 Apr 2026 08:20:06 +0100 Subject: [PATCH] ci: split plugin prerelease validation --- .agents/skills/openclaw-testing/SKILL.md | 15 +- .github/workflows/ci.yml | 274 +----------- .github/workflows/full-release-validation.yml | 121 +++++- .github/workflows/plugin-prerelease.yml | 406 ++++++++++++++++++ docs/ci.md | 28 +- docs/help/testing.md | 2 +- docs/reference/RELEASING.md | 14 +- test/scripts/ci-node-test-plan.test.ts | 2 +- .../plugin-prerelease-test-plan.test.ts | 116 +++-- 9 files changed, 632 insertions(+), 346 deletions(-) create mode 100644 .github/workflows/plugin-prerelease.yml diff --git a/.agents/skills/openclaw-testing/SKILL.md b/.agents/skills/openclaw-testing/SKILL.md index 4bcca0534ed..9dc12518ccf 100644 --- a/.agents/skills/openclaw-testing/SKILL.md +++ b/.agents/skills/openclaw-testing/SKILL.md @@ -111,8 +111,10 @@ rerun after a focused patch. the manual "everything before release" umbrella. It resolves a target ref, then dispatches: -- manual `CI` for the full normal CI graph, with release-only plugin prerelease - lanes enabled via `full_release_validation=true` +- manual `CI` for the full normal CI graph, with Android enabled via + `include_android=true` +- `Plugin Prerelease` for release-only plugin static checks, extension shards, + the release-only `agentic-plugins` shard, and plugin product Docker lanes - `OpenClaw Release Checks` for install smoke, cross-OS release checks, live and E2E checks, Docker release-path suites, OpenWebUI, QA Lab, fast Matrix, and Telegram release lanes @@ -145,7 +147,7 @@ turns green. Standalone manual `CI` dispatches do not run the plugin prerelease suite, the extension batch sweep, or the release-only `agentic-plugins` Vitest shard. Those -lanes are intentionally reserved for the Full Release Validation CI child so +lanes are intentionally reserved for the separate `Plugin Prerelease` child so PRs, main pushes, and ad hoc broad CI checks do not spend Docker/package time or all-plugin runtime time on release-only product coverage. @@ -160,9 +162,10 @@ only the failed parent verifier job; do not dispatch a new full umbrella unless the release evidence is stale. For bounded recovery after a focused fix, pass `-f rerun_group=`. -Supported umbrella groups are `all`, `ci`, `release-checks`, `install-smoke`, -`cross-os`, `live-e2e`, `package`, `qa`, `qa-parity`, `qa-live`, and -`npm-telegram`. Use the narrowest group that covers the failed box. +Supported umbrella groups are `all`, `ci`, `plugin-prerelease`, +`release-checks`, `install-smoke`, `cross-os`, `live-e2e`, `package`, `qa`, +`qa-parity`, `qa-live`, and `npm-telegram`. Use the narrowest group that covers +the failed box. ### Release Evidence diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3020616421c..025e7d2aa9e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,11 +8,6 @@ on: required: false default: "" type: string - full_release_validation: - description: Run release-only CI lanes. Reserved for Full Release Validation. - required: false - default: false - type: boolean include_android: description: Run Android lanes for this manual CI dispatch. required: false @@ -60,8 +55,6 @@ jobs: run_checks_fast: ${{ steps.manifest.outputs.run_checks_fast }} checks_fast_core_matrix: ${{ steps.manifest.outputs.checks_fast_core_matrix }} channel_contracts_matrix: ${{ steps.manifest.outputs.channel_contracts_matrix }} - run_checks_node_extensions: ${{ steps.manifest.outputs.run_checks_node_extensions }} - checks_node_extensions_matrix: ${{ steps.manifest.outputs.checks_node_extensions_matrix }} run_checks: ${{ steps.manifest.outputs.run_checks }} checks_matrix: ${{ steps.manifest.outputs.checks_matrix }} run_checks_node_core_nondist: ${{ steps.manifest.outputs.run_checks_node_core_nondist }} @@ -70,10 +63,6 @@ jobs: checks_node_core_dist_matrix: ${{ steps.manifest.outputs.checks_node_core_dist_matrix }} run_check: ${{ steps.manifest.outputs.run_check }} run_check_additional: ${{ steps.manifest.outputs.run_check_additional }} - run_plugin_prerelease_suite: ${{ steps.manifest.outputs.run_plugin_prerelease_suite }} - plugin_prerelease_ref: ${{ steps.manifest.outputs.plugin_prerelease_ref }} - plugin_prerelease_static_matrix: ${{ steps.manifest.outputs.plugin_prerelease_static_matrix }} - plugin_prerelease_docker_lanes: ${{ steps.manifest.outputs.plugin_prerelease_docker_lanes }} run_build_smoke: ${{ steps.manifest.outputs.run_build_smoke }} run_check_docs: ${{ steps.manifest.outputs.run_check_docs }} run_control_ui_i18n: ${{ steps.manifest.outputs.run_control_ui_i18n }} @@ -132,7 +121,7 @@ jobs: OPENCLAW_CI_DOCS_CHANGED: ${{ github.event_name == 'workflow_dispatch' && 'true' || steps.docs_scope.outputs.docs_changed }} OPENCLAW_CI_RUN_NODE: ${{ github.event_name == 'workflow_dispatch' && 'true' || steps.changed_scope.outputs.run_node || 'false' }} OPENCLAW_CI_RUN_MACOS: ${{ github.event_name == 'workflow_dispatch' && 'true' || steps.changed_scope.outputs.run_macos || 'false' }} - OPENCLAW_CI_RUN_ANDROID: ${{ github.event_name == 'workflow_dispatch' && (inputs.full_release_validation || inputs.include_android) && 'true' || steps.changed_scope.outputs.run_android || 'false' }} + OPENCLAW_CI_RUN_ANDROID: ${{ github.event_name == 'workflow_dispatch' && inputs.include_android && 'true' || steps.changed_scope.outputs.run_android || 'false' }} OPENCLAW_CI_RUN_WINDOWS: ${{ github.event_name == 'workflow_dispatch' && 'true' || steps.changed_scope.outputs.run_windows || 'false' }} OPENCLAW_CI_RUN_NODE_FAST_ONLY: ${{ github.event_name == 'workflow_dispatch' && 'false' || steps.changed_scope.outputs.run_node_fast_only || 'false' }} OPENCLAW_CI_RUN_NODE_FAST_PLUGIN_CONTRACTS: ${{ github.event_name == 'workflow_dispatch' && 'false' || steps.changed_scope.outputs.run_node_fast_plugin_contracts || 'false' }} @@ -140,10 +129,6 @@ jobs: OPENCLAW_CI_RUN_SKILLS_PYTHON: ${{ github.event_name == 'workflow_dispatch' && 'true' || steps.changed_scope.outputs.run_skills_python || 'false' }} OPENCLAW_CI_RUN_CONTROL_UI_I18N: ${{ github.event_name == 'workflow_dispatch' && 'true' || steps.changed_scope.outputs.run_control_ui_i18n || 'false' }} OPENCLAW_CI_CHECKOUT_REVISION: ${{ steps.checkout_ref.outputs.sha }} - OPENCLAW_CI_EVENT_NAME: ${{ github.event_name }} - OPENCLAW_CI_FULL_RELEASE_VALIDATION: ${{ github.event_name == 'workflow_dispatch' && inputs.full_release_validation && 'true' || 'false' }} - OPENCLAW_CI_PR_HEAD_REPOSITORY: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name || github.repository }} - OPENCLAW_CI_PR_HEAD_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || '' }} OPENCLAW_CI_REPOSITORY: ${{ github.repository }} run: | node --input-type=module <<'EOF' @@ -154,10 +139,6 @@ jobs: import { createChannelContractTestShards, } from "./scripts/lib/channel-contract-test-plan.mjs"; - import { - createExtensionTestShards, - DEFAULT_EXTENSION_TEST_SHARD_COUNT, - } from "./scripts/lib/extension-test-plan.mjs"; const parseBoolean = (value, fallback = false) => { if (value === undefined) return fallback; @@ -193,65 +174,6 @@ jobs: const runSkillsPython = parseBoolean(process.env.OPENCLAW_CI_RUN_SKILLS_PYTHON) && !docsOnly; const runControlUiI18n = parseBoolean(process.env.OPENCLAW_CI_RUN_CONTROL_UI_I18N) && !docsOnly; - const isFullReleaseValidationCiRun = - process.env.OPENCLAW_CI_EVENT_NAME === "workflow_dispatch" && - parseBoolean(process.env.OPENCLAW_CI_FULL_RELEASE_VALIDATION); - const trustedPluginPrereleaseRef = - process.env.OPENCLAW_CI_EVENT_NAME !== "pull_request" || - process.env.OPENCLAW_CI_PR_HEAD_REPOSITORY === process.env.OPENCLAW_CI_REPOSITORY; - const pluginPrereleaseRef = - process.env.OPENCLAW_CI_EVENT_NAME === "pull_request" && trustedPluginPrereleaseRef - ? process.env.OPENCLAW_CI_PR_HEAD_SHA - : process.env.OPENCLAW_CI_CHECKOUT_REVISION; - let runPluginPrereleaseSuite = - isFullReleaseValidationCiRun && runNodeFull && isCanonicalRepository; - const runReleaseOnlyPluginSuites = - isFullReleaseValidationCiRun && runNodeFull && isCanonicalRepository; - let pluginPrereleasePlan = { staticChecks: [], dockerLanes: [] }; - if (runPluginPrereleaseSuite) { - try { - const { assertPluginPrereleaseTestPlanComplete } = await import( - "./scripts/lib/plugin-prerelease-test-plan.mjs" - ); - pluginPrereleasePlan = assertPluginPrereleaseTestPlanComplete(); - } catch (error) { - const errorCode = - error && typeof error === "object" && "code" in error ? error.code : ""; - const moduleUrl = - error && typeof error === "object" && "url" in error ? String(error.url) : ""; - if ( - errorCode === "ERR_MODULE_NOT_FOUND" && - moduleUrl.endsWith("/scripts/lib/plugin-prerelease-test-plan.mjs") - ) { - console.warn( - "Plugin prerelease plan unavailable in target ref; skipping plugin prerelease suite.", - ); - runPluginPrereleaseSuite = false; - } else { - throw error; - } - } - } - const extensionTestShardCount = isCanonicalRepository - ? DEFAULT_EXTENSION_TEST_SHARD_COUNT - : Math.max(DEFAULT_EXTENSION_TEST_SHARD_COUNT, 36); - const extensionShardMatrix = createMatrix( - runReleaseOnlyPluginSuites - ? createExtensionTestShards({ - shardCount: extensionTestShardCount, - }).map((shard) => ({ - check_name: shard.checkName, - extensions_csv: shard.extensionIds.join(","), - runner: isCanonicalRepository && [0, 1, 2, 3].includes(shard.index) - ? "blacksmith-8vcpu-ubuntu-2404" - : isCanonicalRepository - ? "blacksmith-4vcpu-ubuntu-2404" - : "ubuntu-24.04", - shard_index: shard.index + 1, - task: "extensions-batch", - })) - : [], - ); const checksFastCoreTasks = []; if (runNodeFull) { checksFastCoreTasks.push( @@ -280,7 +202,7 @@ jobs: const nodeTestShards = runNodeFull ? createNodeTestShards({ - includeReleaseOnlyPluginShards: runReleaseOnlyPluginSuites, + includeReleaseOnlyPluginShards: false, }).map((shard) => ({ check_name: shard.checkName, runtime: "node", @@ -310,8 +232,6 @@ jobs: channel_contracts_matrix: createMatrix( runNodeFull ? createChannelContractTestShards() : [], ), - run_checks_node_extensions: runReleaseOnlyPluginSuites, - checks_node_extensions_matrix: extensionShardMatrix, run_checks: runNodeFull, checks_matrix: createMatrix( runNodeFull @@ -326,20 +246,6 @@ jobs: checks_node_core_dist_matrix: createMatrix(nodeTestDistShards), run_check: runNodeFull, run_check_additional: runNodeFull, - run_plugin_prerelease_suite: runPluginPrereleaseSuite, - plugin_prerelease_ref: runPluginPrereleaseSuite ? pluginPrereleaseRef : "", - plugin_prerelease_static_matrix: createMatrix( - runPluginPrereleaseSuite - ? pluginPrereleasePlan.staticChecks.map((check) => ({ - check_name: check.checkName, - command: check.command, - task: check.check, - })) - : [], - ), - plugin_prerelease_docker_lanes: runPluginPrereleaseSuite - ? pluginPrereleasePlan.dockerLanes.join(" ") - : "", run_build_smoke: runNodeFull, run_check_docs: docsChanged, run_control_ui_i18n: runControlUiI18n, @@ -995,97 +901,6 @@ jobs: - name: Run protocol check run: pnpm protocol:check - checks-node-extensions-shard: - permissions: - contents: read - name: ${{ matrix.check_name }} - needs: [preflight] - if: needs.preflight.outputs.run_checks_node_extensions == 'true' - runs-on: ${{ matrix.runner }} - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: ${{ fromJson(needs.preflight.outputs.checks_node_extensions_matrix) }} - steps: - - name: Checkout - shell: bash - env: - CHECKOUT_REPO: ${{ github.repository }} - CHECKOUT_SHA: ${{ needs.preflight.outputs.checkout_revision }} - CHECKOUT_TOKEN: ${{ github.token }} - run: | - set -euo pipefail - - workdir="$GITHUB_WORKSPACE" - auth_header="$(printf 'x-access-token:%s' "$CHECKOUT_TOKEN" | base64 | tr -d '\n')" - - reset_checkout_dir() { - mkdir -p "$workdir" - find "$workdir" -mindepth 1 -maxdepth 1 -exec rm -rf {} + - } - - checkout_attempt() { - local attempt="$1" - - reset_checkout_dir - git init "$workdir" >/dev/null - git config --global --add safe.directory "$workdir" - git -C "$workdir" remote add origin "https://github.com/${CHECKOUT_REPO}" - git -C "$workdir" config gc.auto 0 - - timeout --signal=TERM 30s git -C "$workdir" \ - -c protocol.version=2 \ - -c "http.https://github.com/.extraheader=AUTHORIZATION: basic ${auth_header}" \ - fetch --no-tags --prune --no-recurse-submodules --depth=1 origin \ - "+${CHECKOUT_SHA}:refs/remotes/origin/ci-target" || return 1 - - git -C "$workdir" checkout --force --detach "$CHECKOUT_SHA" || return 1 - test -f "$workdir/.github/actions/setup-node-env/action.yml" || return 1 - echo "checkout attempt ${attempt}/5 succeeded" - } - - for attempt in 1 2 3 4 5; do - if checkout_attempt "$attempt"; then - exit 0 - fi - echo "checkout attempt ${attempt}/5 failed" - sleep $((attempt * 5)) - done - - echo "checkout failed after 5 attempts" >&2 - exit 1 - - - name: Setup Node environment - uses: ./.github/actions/setup-node-env - with: - install-bun: "false" - - - name: Run extension shard - env: - NODE_OPTIONS: --max-old-space-size=6144 - OPENCLAW_EXTENSION_BATCH_PARALLEL: 2 - OPENCLAW_VITEST_MAX_WORKERS: 1 - OPENCLAW_EXTENSION_BATCH: ${{ matrix.extensions_csv }} - run: pnpm test:extensions:batch -- "$OPENCLAW_EXTENSION_BATCH" - - checks-node-extensions: - permissions: - contents: read - name: checks-node-extensions - needs: [preflight, checks-node-extensions-shard] - if: ${{ !cancelled() && always() && needs.preflight.outputs.run_checks_node_extensions == 'true' }} - runs-on: ubuntu-24.04 - timeout-minutes: 5 - steps: - - name: Verify extension shards - env: - SHARD_RESULT: ${{ needs.checks-node-extensions-shard.result }} - run: | - if [ "$SHARD_RESULT" != "success" ]; then - echo "Extension shard checks failed: $SHARD_RESULT" >&2 - exit 1 - fi - checks: permissions: contents: read @@ -1697,91 +1512,6 @@ jobs: exit 1 fi - plugin-prerelease-static-shard: - permissions: - contents: read - name: ${{ matrix.check_name }} - needs: [preflight] - if: needs.preflight.outputs.run_plugin_prerelease_suite == 'true' - runs-on: blacksmith-8vcpu-ubuntu-2404 - timeout-minutes: 45 - strategy: - fail-fast: false - matrix: ${{ fromJson(needs.preflight.outputs.plugin_prerelease_static_matrix) }} - steps: - - name: Checkout - uses: actions/checkout@v6 - with: - ref: ${{ needs.preflight.outputs.checkout_revision }} - fetch-depth: 1 - fetch-tags: false - persist-credentials: false - submodules: false - - - name: Setup Node environment - uses: ./.github/actions/setup-node-env - with: - install-bun: "false" - - - name: Run plugin prerelease static shard - env: - PLUGIN_PRERELEASE_COMMAND: ${{ matrix.command }} - PLUGIN_PRERELEASE_TASK: ${{ matrix.task }} - shell: bash - run: | - set -euo pipefail - echo "Running ${PLUGIN_PRERELEASE_TASK}: ${PLUGIN_PRERELEASE_COMMAND}" - bash -c "$PLUGIN_PRERELEASE_COMMAND" - - plugin-prerelease-docker-suite: - name: plugin-prerelease-docker-suite - needs: [preflight] - if: needs.preflight.outputs.run_plugin_prerelease_suite == 'true' - permissions: - actions: read - contents: read - packages: write - pull-requests: read - uses: ./.github/workflows/openclaw-live-and-e2e-checks-reusable.yml - with: - ref: ${{ needs.preflight.outputs.plugin_prerelease_ref }} - include_repo_e2e: false - include_release_path_suites: false - include_openwebui: false - docker_lanes: ${{ needs.preflight.outputs.plugin_prerelease_docker_lanes }} - include_live_suites: false - live_models_only: false - - plugin-prerelease-suite: - permissions: - contents: read - name: plugin-prerelease-suite - needs: [preflight, plugin-prerelease-static-shard, plugin-prerelease-docker-suite] - if: ${{ !cancelled() && always() && needs.preflight.outputs.run_plugin_prerelease_suite == 'true' }} - runs-on: ubuntu-24.04 - timeout-minutes: 5 - steps: - - name: Verify plugin prerelease suite - env: - DOCKER_RESULT: ${{ needs.plugin-prerelease-docker-suite.result }} - STATIC_RESULT: ${{ needs.plugin-prerelease-static-shard.result }} - shell: bash - run: | - set -euo pipefail - failed=0 - for result in \ - "plugin-prerelease-static=${STATIC_RESULT}" \ - "plugin-prerelease-docker=${DOCKER_RESULT}" - do - name="${result%%=*}" - status="${result#*=}" - if [ "$status" != "success" ]; then - echo "::error::${name} ended with ${status}" - failed=1 - fi - done - exit "$failed" - build-smoke: permissions: contents: read diff --git a/.github/workflows/full-release-validation.yml b/.github/workflows/full-release-validation.yml index 6fda26fe1fc..aa7d4add3d1 100644 --- a/.github/workflows/full-release-validation.yml +++ b/.github/workflows/full-release-validation.yml @@ -43,6 +43,7 @@ on: options: - all - ci + - plugin-prerelease - release-checks - install-smoke - cross-os @@ -131,11 +132,16 @@ jobs: echo "- Child workflow ref: \`${CHILD_WORKFLOW_REF}\`" echo "- Rerun group: \`${RERUN_GROUP}\`" if [[ "$RERUN_GROUP" == "all" || "$RERUN_GROUP" == "ci" ]]; then - echo "- Normal CI: \`CI\` with \`target_ref=${TARGET_SHA}\` and release-only lanes enabled" + echo "- Normal CI: \`CI\` with \`target_ref=${TARGET_SHA}\`" else echo "- Normal CI: skipped by rerun group" fi - if [[ "$RERUN_GROUP" != "ci" && "$RERUN_GROUP" != "npm-telegram" ]]; then + if [[ "$RERUN_GROUP" == "all" || "$RERUN_GROUP" == "plugin-prerelease" ]]; then + echo "- Plugin prerelease: \`Plugin Prerelease\` with \`target_ref=${TARGET_SHA}\`" + else + echo "- Plugin prerelease: skipped by rerun group" + fi + if [[ "$RERUN_GROUP" == "all" || "$RERUN_GROUP" == "release-checks" || "$RERUN_GROUP" == "install-smoke" || "$RERUN_GROUP" == "cross-os" || "$RERUN_GROUP" == "live-e2e" || "$RERUN_GROUP" == "package" || "$RERUN_GROUP" == "qa" || "$RERUN_GROUP" == "qa-parity" || "$RERUN_GROUP" == "qa-live" ]]; then echo "- Release/live/Docker/package/QA: \`OpenClaw Release Checks\`" else echo "- Release/live/Docker/package/QA: skipped by rerun group" @@ -263,7 +269,105 @@ jobs: } cancel_same_sha_push_ci - dispatch_and_wait ci.yml -f target_ref="$TARGET_SHA" -f full_release_validation=true + dispatch_and_wait ci.yml -f target_ref="$TARGET_SHA" -f include_android=true + + plugin_prerelease: + name: Run plugin prerelease validation + needs: [resolve_target] + if: contains(fromJSON('["all","plugin-prerelease"]'), inputs.rerun_group) + runs-on: ubuntu-24.04 + timeout-minutes: 300 + outputs: + run_id: ${{ steps.dispatch.outputs.run_id }} + url: ${{ steps.dispatch.outputs.url }} + conclusion: ${{ steps.dispatch.outputs.conclusion }} + steps: + - name: Dispatch and monitor plugin prerelease + id: dispatch + env: + GH_TOKEN: ${{ github.token }} + TARGET_REF: ${{ inputs.ref }} + TARGET_SHA: ${{ needs.resolve_target.outputs.sha }} + CHILD_WORKFLOW_REF: ${{ github.ref_name }} + run: | + set -euo pipefail + + dispatch_and_wait() { + local workflow="$1" + shift + + local before_json dispatch_output run_id status conclusion url + before_json="$(gh run list --workflow "$workflow" --event workflow_dispatch --limit 100 --json databaseId --jq '[.[].databaseId]')" + + dispatch_output="$(gh workflow run "$workflow" --ref "$CHILD_WORKFLOW_REF" "$@" 2>&1)" + printf '%s\n' "$dispatch_output" + run_id="$( + printf '%s\n' "$dispatch_output" | + sed -nE 's#.*actions/runs/([0-9]+).*#\1#p' | + tail -n 1 + )" + + if [[ -z "$run_id" ]]; then + for _ in $(seq 1 60); do + run_id="$( + BEFORE_IDS="$before_json" gh run list --workflow "$workflow" --event workflow_dispatch --limit 50 --json databaseId,createdAt \ + --jq 'map(select(.databaseId as $id | (env.BEFORE_IDS | fromjson | index($id) | not))) | sort_by(.createdAt) | reverse | .[0].databaseId // empty' + )" + if [[ -n "$run_id" ]]; then + break + fi + sleep 5 + done + fi + + if [[ -z "${run_id:-}" ]]; then + echo "Could not find dispatched run for ${workflow}." >&2 + exit 1 + fi + + echo "Dispatched ${workflow}: https://github.com/${GITHUB_REPOSITORY}/actions/runs/${run_id}" + echo "run_id=${run_id}" >> "$GITHUB_OUTPUT" + + cleanup_child_run() { + local exit_code=$? + trap - EXIT INT TERM + local child_status + child_status="$(gh run view "$run_id" --json status --jq '.status' 2>/dev/null || true)" + if [[ "$child_status" != "completed" ]]; then + echo "Cancelling child ${workflow} run ${run_id} after parent exit (${exit_code})." + gh run cancel "$run_id" || gh api -X POST "repos/${GITHUB_REPOSITORY}/actions/runs/${run_id}/force-cancel" || true + fi + return "$exit_code" + } + trap cleanup_child_run EXIT INT TERM + + while true; do + status="$(gh run view "$run_id" --json status --jq '.status')" + if [[ "$status" == "completed" ]]; then + break + fi + sleep 30 + done + trap - EXIT INT TERM + + conclusion="$(gh run view "$run_id" --json conclusion --jq '.conclusion')" + url="$(gh run view "$run_id" --json url --jq '.url')" + echo "${workflow} finished with ${conclusion}: ${url}" + echo "url=${url}" >> "$GITHUB_OUTPUT" + echo "conclusion=${conclusion}" >> "$GITHUB_OUTPUT" + if [[ "$conclusion" != "success" ]]; then + gh run view "$run_id" --json jobs --jq '.jobs[] | select(.conclusion != "success" and .conclusion != "skipped") | {name, conclusion, url}' || true + fi + } + + { + echo "### Plugin prerelease" + echo + echo "- Target ref: \`${TARGET_REF}\`" + echo "- Target SHA: \`${TARGET_SHA}\`" + } >> "$GITHUB_STEP_SUMMARY" + + dispatch_and_wait plugin-prerelease.yml -f target_ref="$TARGET_SHA" -f expected_sha="$TARGET_SHA" release_checks: name: Run release/live/Docker/QA validation @@ -467,7 +571,7 @@ jobs: summary: name: Verify full validation - needs: [normal_ci, release_checks, npm_telegram] + needs: [normal_ci, plugin_prerelease, release_checks, npm_telegram] if: always() runs-on: ubuntu-24.04 timeout-minutes: 5 @@ -532,9 +636,11 @@ jobs: env: GH_TOKEN: ${{ github.token }} NORMAL_CI_RUN_ID: ${{ needs.normal_ci.outputs.run_id }} + PLUGIN_PRERELEASE_RUN_ID: ${{ needs.plugin_prerelease.outputs.run_id }} RELEASE_CHECKS_RUN_ID: ${{ needs.release_checks.outputs.run_id }} NPM_TELEGRAM_RUN_ID: ${{ needs.npm_telegram.outputs.run_id }} NORMAL_CI_RESULT: ${{ needs.normal_ci.result }} + PLUGIN_PRERELEASE_RESULT: ${{ needs.plugin_prerelease.result }} RELEASE_CHECKS_RESULT: ${{ needs.release_checks.result }} NPM_TELEGRAM_RESULT: ${{ needs.npm_telegram.result }} run: | @@ -604,6 +710,12 @@ jobs: check_child "normal_ci" "$NORMAL_CI_RUN_ID" 1 || failed=1 fi + if [[ "$PLUGIN_PRERELEASE_RESULT" == "skipped" && -z "${PLUGIN_PRERELEASE_RUN_ID// }" ]]; then + check_child "plugin_prerelease" "" 0 || failed=1 + else + check_child "plugin_prerelease" "$PLUGIN_PRERELEASE_RUN_ID" 1 || failed=1 + fi + if [[ "$RELEASE_CHECKS_RESULT" == "skipped" && -z "${RELEASE_CHECKS_RUN_ID// }" ]]; then check_child "release_checks" "" 0 || failed=1 else @@ -617,6 +729,7 @@ jobs: fi summarize_child_timing "normal_ci" "$NORMAL_CI_RUN_ID" + summarize_child_timing "plugin_prerelease" "$PLUGIN_PRERELEASE_RUN_ID" summarize_child_timing "release_checks" "$RELEASE_CHECKS_RUN_ID" summarize_child_timing "npm_telegram" "$NPM_TELEGRAM_RUN_ID" diff --git a/.github/workflows/plugin-prerelease.yml b/.github/workflows/plugin-prerelease.yml new file mode 100644 index 00000000000..fab6ddb88b0 --- /dev/null +++ b/.github/workflows/plugin-prerelease.yml @@ -0,0 +1,406 @@ +name: Plugin Prerelease + +on: + workflow_dispatch: + inputs: + target_ref: + description: Branch, tag, or full commit SHA to validate + required: false + default: main + type: string + expected_sha: + description: Optional full commit SHA that target_ref must resolve to + required: false + default: "" + type: string + +permissions: + contents: read + +concurrency: + group: plugin-prerelease-${{ inputs.target_ref }} + cancel-in-progress: ${{ inputs.target_ref == 'main' }} + +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" + +jobs: + preflight: + name: Build plugin prerelease plan + runs-on: ubuntu-24.04 + timeout-minutes: 15 + outputs: + checkout_revision: ${{ steps.manifest.outputs.checkout_revision }} + run_plugin_prerelease_suite: ${{ steps.manifest.outputs.run_plugin_prerelease_suite }} + run_plugin_prerelease_static: ${{ steps.manifest.outputs.run_plugin_prerelease_static }} + plugin_prerelease_static_matrix: ${{ steps.manifest.outputs.plugin_prerelease_static_matrix }} + run_plugin_prerelease_node: ${{ steps.manifest.outputs.run_plugin_prerelease_node }} + plugin_prerelease_node_matrix: ${{ steps.manifest.outputs.plugin_prerelease_node_matrix }} + run_plugin_prerelease_extensions: ${{ steps.manifest.outputs.run_plugin_prerelease_extensions }} + plugin_prerelease_extension_matrix: ${{ steps.manifest.outputs.plugin_prerelease_extension_matrix }} + run_plugin_prerelease_docker: ${{ steps.manifest.outputs.run_plugin_prerelease_docker }} + plugin_prerelease_docker_lanes: ${{ steps.manifest.outputs.plugin_prerelease_docker_lanes }} + steps: + - name: Checkout target + uses: actions/checkout@v6 + with: + ref: ${{ inputs.target_ref }} + fetch-depth: 1 + fetch-tags: false + persist-credentials: false + submodules: false + + - name: Build plugin prerelease manifest + id: manifest + env: + EXPECTED_SHA: ${{ inputs.expected_sha }} + run: | + node --input-type=module <<'EOF' + import { appendFileSync } from "node:fs"; + import { execFileSync } from "node:child_process"; + + const createMatrix = (include) => ({ include }); + const outputPath = process.env.GITHUB_OUTPUT; + const checkoutRevision = execFileSync("git", ["rev-parse", "HEAD"], { + encoding: "utf8", + }).trim(); + const expectedSha = (process.env.EXPECTED_SHA ?? "").trim(); + if (expectedSha && expectedSha !== checkoutRevision) { + console.error( + `target_ref resolved to ${checkoutRevision}, expected ${expectedSha}`, + ); + process.exit(1); + } + + let pluginPrereleasePlan = { staticChecks: [], dockerLanes: [] }; + let extensionShards = []; + let nodeShards = []; + + try { + const { assertPluginPrereleaseTestPlanComplete } = await import( + "./scripts/lib/plugin-prerelease-test-plan.mjs" + ); + pluginPrereleasePlan = assertPluginPrereleaseTestPlanComplete(); + } catch (error) { + const errorCode = + error && typeof error === "object" && "code" in error ? error.code : ""; + const moduleUrl = + error && typeof error === "object" && "url" in error ? String(error.url) : ""; + if ( + errorCode === "ERR_MODULE_NOT_FOUND" && + moduleUrl.endsWith("/scripts/lib/plugin-prerelease-test-plan.mjs") + ) { + console.warn( + "Plugin prerelease plan unavailable in target ref; skipping static and Docker plugin prerelease lanes.", + ); + } else { + throw error; + } + } + + try { + const { createExtensionTestShards, DEFAULT_EXTENSION_TEST_SHARD_COUNT } = await import( + "./scripts/lib/extension-test-plan.mjs" + ); + extensionShards = createExtensionTestShards({ + shardCount: DEFAULT_EXTENSION_TEST_SHARD_COUNT, + }).map((shard) => ({ + check_name: shard.checkName, + extensions_csv: shard.extensionIds.join(","), + runner: [0, 1, 2, 3].includes(shard.index) + ? "blacksmith-8vcpu-ubuntu-2404" + : "blacksmith-4vcpu-ubuntu-2404", + shard_index: shard.index + 1, + task: "extensions-batch", + })); + } catch (error) { + const errorCode = + error && typeof error === "object" && "code" in error ? error.code : ""; + const moduleUrl = + error && typeof error === "object" && "url" in error ? String(error.url) : ""; + if ( + errorCode === "ERR_MODULE_NOT_FOUND" && + moduleUrl.endsWith("/scripts/lib/extension-test-plan.mjs") + ) { + console.warn( + "Extension test plan unavailable in target ref; skipping extension prerelease shards.", + ); + } else { + throw error; + } + } + + try { + const { createNodeTestShards } = await import("./scripts/lib/ci-node-test-plan.mjs"); + nodeShards = createNodeTestShards({ + includeReleaseOnlyPluginShards: true, + }) + .filter((shard) => shard.shardName === "agentic-plugins") + .map((shard) => ({ + check_name: shard.checkName, + runtime: "node", + task: "test-shard", + shard_name: shard.shardName, + configs: shard.configs, + includePatterns: shard.includePatterns, + runner: shard.runner, + })); + } catch (error) { + const errorCode = + error && typeof error === "object" && "code" in error ? error.code : ""; + const moduleUrl = + error && typeof error === "object" && "url" in error ? String(error.url) : ""; + if ( + errorCode === "ERR_MODULE_NOT_FOUND" && + moduleUrl.endsWith("/scripts/lib/ci-node-test-plan.mjs") + ) { + console.warn( + "Node test plan unavailable in target ref; skipping release-only plugin Node shard.", + ); + } else { + throw error; + } + } + + const staticChecks = pluginPrereleasePlan.staticChecks.map((check) => ({ + check_name: check.checkName, + command: check.command, + task: check.check, + })); + const dockerLanes = pluginPrereleasePlan.dockerLanes; + const runStatic = staticChecks.length > 0; + const runNode = nodeShards.length > 0; + const runExtensions = extensionShards.length > 0; + const runDocker = dockerLanes.length > 0; + const runSuite = runStatic || runNode || runExtensions || runDocker; + + const manifest = { + checkout_revision: checkoutRevision, + run_plugin_prerelease_suite: runSuite, + run_plugin_prerelease_static: runStatic, + plugin_prerelease_static_matrix: createMatrix(staticChecks), + run_plugin_prerelease_node: runNode, + plugin_prerelease_node_matrix: createMatrix(nodeShards), + run_plugin_prerelease_extensions: runExtensions, + plugin_prerelease_extension_matrix: createMatrix(extensionShards), + run_plugin_prerelease_docker: runDocker, + plugin_prerelease_docker_lanes: dockerLanes.join(" "), + }; + + for (const [key, value] of Object.entries(manifest)) { + appendFileSync( + outputPath, + `${key}=${typeof value === "string" ? value : JSON.stringify(value)}\n`, + "utf8", + ); + } + EOF + + plugin-prerelease-static-shard: + permissions: + contents: read + name: ${{ matrix.check_name }} + needs: [preflight] + if: needs.preflight.outputs.run_plugin_prerelease_static == 'true' + runs-on: blacksmith-8vcpu-ubuntu-2404 + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.preflight.outputs.plugin_prerelease_static_matrix) }} + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + ref: ${{ needs.preflight.outputs.checkout_revision }} + fetch-depth: 1 + fetch-tags: false + persist-credentials: false + submodules: false + + - name: Setup Node environment + uses: ./.github/actions/setup-node-env + with: + install-bun: "false" + + - name: Run plugin prerelease static shard + env: + PLUGIN_PRERELEASE_COMMAND: ${{ matrix.command }} + PLUGIN_PRERELEASE_TASK: ${{ matrix.task }} + shell: bash + run: | + set -euo pipefail + echo "Running ${PLUGIN_PRERELEASE_TASK}: ${PLUGIN_PRERELEASE_COMMAND}" + bash -c "$PLUGIN_PRERELEASE_COMMAND" + + plugin-prerelease-node-shard: + permissions: + contents: read + name: ${{ matrix.check_name }} + needs: [preflight] + if: needs.preflight.outputs.run_plugin_prerelease_node == 'true' + runs-on: ${{ matrix.runner || 'ubuntu-24.04' }} + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.preflight.outputs.plugin_prerelease_node_matrix) }} + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + ref: ${{ needs.preflight.outputs.checkout_revision }} + fetch-depth: 1 + fetch-tags: false + persist-credentials: false + submodules: false + + - name: Setup Node environment + uses: ./.github/actions/setup-node-env + with: + install-bun: "false" + + - name: Configure Node test resources + run: echo "OPENCLAW_VITEST_MAX_WORKERS=2" >> "$GITHUB_ENV" + + - name: Run release-only plugin Node shard + env: + NODE_OPTIONS: --max-old-space-size=6144 + OPENCLAW_NODE_TEST_CONFIGS_JSON: ${{ toJson(matrix.configs) }} + OPENCLAW_NODE_TEST_INCLUDE_PATTERNS_JSON: ${{ toJson(matrix.includePatterns) }} + OPENCLAW_VITEST_SHARD_NAME: ${{ matrix.shard_name }} + OPENCLAW_TEST_PROJECTS_PARALLEL: "2" + shell: bash + run: | + set -euo pipefail + node --input-type=module <<'EOF' + import { spawnSync } from "node:child_process"; + import { writeFileSync } from "node:fs"; + import { join } from "node:path"; + + const configs = JSON.parse(process.env.OPENCLAW_NODE_TEST_CONFIGS_JSON ?? "[]"); + if (!Array.isArray(configs) || configs.length === 0) { + console.error("Missing node test shard configs"); + process.exit(1); + } + const includePatterns = JSON.parse( + process.env.OPENCLAW_NODE_TEST_INCLUDE_PATTERNS_JSON ?? "null", + ); + const childEnv = { ...process.env }; + if (Array.isArray(includePatterns) && includePatterns.length > 0) { + const includeFile = join( + process.env.RUNNER_TEMP ?? ".", + `node-test-include-${process.env.GITHUB_JOB ?? "local"}-${Date.now()}.json`, + ); + writeFileSync(includeFile, JSON.stringify(includePatterns), "utf8"); + childEnv.OPENCLAW_VITEST_INCLUDE_FILE = includeFile; + } + + const result = spawnSync( + "pnpm", + ["exec", "node", "scripts/test-projects.mjs", ...configs], + { + env: childEnv, + stdio: "inherit", + }, + ); + process.exit(result.status ?? 1); + EOF + + plugin-prerelease-extension-shard: + permissions: + contents: read + name: ${{ matrix.check_name }} + needs: [preflight] + if: needs.preflight.outputs.run_plugin_prerelease_extensions == 'true' + runs-on: ${{ matrix.runner }} + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.preflight.outputs.plugin_prerelease_extension_matrix) }} + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + ref: ${{ needs.preflight.outputs.checkout_revision }} + fetch-depth: 1 + fetch-tags: false + persist-credentials: false + submodules: false + + - name: Setup Node environment + uses: ./.github/actions/setup-node-env + with: + install-bun: "false" + + - name: Run extension shard + env: + NODE_OPTIONS: --max-old-space-size=6144 + OPENCLAW_EXTENSION_BATCH_PARALLEL: 2 + OPENCLAW_VITEST_MAX_WORKERS: 1 + OPENCLAW_EXTENSION_BATCH: ${{ matrix.extensions_csv }} + run: pnpm test:extensions:batch -- "$OPENCLAW_EXTENSION_BATCH" + + plugin-prerelease-docker-suite: + name: plugin-prerelease-docker-suite + needs: [preflight] + if: needs.preflight.outputs.run_plugin_prerelease_docker == 'true' + permissions: + actions: read + contents: read + packages: write + pull-requests: read + uses: ./.github/workflows/openclaw-live-and-e2e-checks-reusable.yml + with: + ref: ${{ needs.preflight.outputs.checkout_revision }} + include_repo_e2e: false + include_release_path_suites: false + include_openwebui: false + docker_lanes: ${{ needs.preflight.outputs.plugin_prerelease_docker_lanes }} + include_live_suites: false + live_models_only: false + + plugin-prerelease-suite: + permissions: + contents: read + name: plugin-prerelease-suite + needs: + - preflight + - plugin-prerelease-static-shard + - plugin-prerelease-node-shard + - plugin-prerelease-extension-shard + - plugin-prerelease-docker-suite + if: ${{ !cancelled() && always() && needs.preflight.outputs.run_plugin_prerelease_suite == 'true' }} + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - name: Verify plugin prerelease suite + env: + RUN_STATIC: ${{ needs.preflight.outputs.run_plugin_prerelease_static }} + RUN_NODE: ${{ needs.preflight.outputs.run_plugin_prerelease_node }} + RUN_EXTENSIONS: ${{ needs.preflight.outputs.run_plugin_prerelease_extensions }} + RUN_DOCKER: ${{ needs.preflight.outputs.run_plugin_prerelease_docker }} + STATIC_RESULT: ${{ needs.plugin-prerelease-static-shard.result }} + NODE_RESULT: ${{ needs.plugin-prerelease-node-shard.result }} + EXTENSIONS_RESULT: ${{ needs.plugin-prerelease-extension-shard.result }} + DOCKER_RESULT: ${{ needs.plugin-prerelease-docker-suite.result }} + shell: bash + run: | + set -euo pipefail + failed=0 + check_required() { + local name="$1" + local required="$2" + local status="$3" + if [ "$required" != "true" ]; then + return 0 + fi + if [ "$status" != "success" ]; then + echo "::error::${name} ended with ${status}" + failed=1 + fi + } + + check_required "plugin-prerelease-static" "$RUN_STATIC" "$STATIC_RESULT" + check_required "plugin-prerelease-node" "$RUN_NODE" "$NODE_RESULT" + check_required "plugin-prerelease-extensions" "$RUN_EXTENSIONS" "$EXTENSIONS_RESULT" + check_required "plugin-prerelease-docker" "$RUN_DOCKER" "$DOCKER_RESULT" + exit "$failed" diff --git a/docs/ci.md b/docs/ci.md index acacc7e6680..913ba852f26 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -6,15 +6,16 @@ read_when: - You are debugging failing GitHub Actions checks --- -The CI runs on every push to `main` and every pull request. It uses smart scoping to skip expensive jobs when only unrelated areas changed. Manual `workflow_dispatch` runs intentionally bypass smart scoping and fan out the full normal CI graph for release candidates or broad validation, with Android lanes opt-in through `include_android` for standalone manual runs. Release-only plugin prerelease lanes stay off unless `Full Release Validation` dispatches CI with `full_release_validation=true`, which also enables Android. +The CI runs on every push to `main` and every pull request. It uses smart scoping to skip expensive jobs when only unrelated areas changed. Manual `workflow_dispatch` runs intentionally bypass smart scoping and fan out the full normal CI graph for release candidates or broad validation, with Android lanes opt-in through `include_android` for standalone manual runs. Release-only plugin prerelease lanes live in the separate `Plugin Prerelease` workflow and run only from `Full Release Validation` or an explicit manual dispatch. `Full Release Validation` is the manual umbrella workflow for "run everything before release." It accepts a branch, tag, or full commit SHA, dispatches the -manual `CI` workflow with that target, and dispatches `OpenClaw Release Checks` -for install smoke, package acceptance, Docker release-path suites, live/E2E, -OpenWebUI, QA Lab parity, Matrix, and Telegram lanes. It can also run the -post-publish `NPM Telegram Beta E2E` workflow when a published package spec is -provided. `release_profile=minimum|stable|full` controls the live/provider +manual `CI` workflow with that target, dispatches `Plugin Prerelease` for +release-only plugin/package/static/Docker proof, and dispatches +`OpenClaw Release Checks` for install smoke, package acceptance, Docker +release-path suites, live/E2E, OpenWebUI, QA Lab parity, Matrix, and Telegram +lanes. It can also run the post-publish `NPM Telegram Beta E2E` workflow when a +published package spec is provided. `release_profile=minimum|stable|full` controls the live/provider breadth passed into release checks: `minimum` keeps the fastest OpenAI/core release-critical lanes, `stable` adds the stable provider/backend set, and `full` runs the broad advisory provider/media matrix. The umbrella records the @@ -347,14 +348,12 @@ gh workflow run duplicate-after-merge.yml \ | `build-artifacts` | Build `dist/`, Control UI, built-artifact checks, and reusable downstream artifacts | Node-relevant changes | | `checks-fast-core` | Fast Linux correctness lanes such as bundled/plugin-contract/protocol checks | Node-relevant changes | | `checks-fast-contracts-channels` | Sharded channel contract checks with a stable aggregate check result | Node-relevant changes | -| `checks-node-extensions` | Full bundled-plugin test shards across the extension suite | Node-relevant changes | | `checks-node-core-test` | Core Node test shards, excluding channel, bundled, contract, and extension lanes | Node-relevant changes | | `check` | Sharded main local gate equivalent: prod types, lint, guards, test types, and strict smoke | Node-relevant changes | | `check-additional` | Architecture, boundary, extension-surface guards, package-boundary, and gateway-watch shards | Node-relevant changes | | `build-smoke` | Built-CLI smoke tests and startup-memory smoke | Node-relevant changes | | `checks` | Verifier for built-artifact channel tests | Node-relevant changes | | `checks-node-compat-node22` | Node 22 compatibility build and smoke lane | Manual CI dispatch for releases | -| `plugin-prerelease-suite` | Aggregate for plugin prerelease static checks and Docker product lanes | Full Release Validation CI child | | `check-docs` | Docs formatting, lint, and broken-link checks | Docs changed | | `skills-python` | Ruff + pytest for Python-backed skills | Python-skill-relevant changes | | `checks-windows` | Windows-specific process/path tests plus shared runtime import specifier regressions | Windows-relevant changes | @@ -368,9 +367,10 @@ non-Android scoped lane on: Linux Node shards, bundled-plugin shards, channel contracts, Node 22 compatibility, `check`, `check-additional`, build smoke, docs checks, Python skills, Windows, macOS, and Control UI i18n. Standalone manual CI dispatches run Android only with `include_android=true`; the full release -umbrella enables Android by passing `full_release_validation=true`. The plugin -prerelease suite is excluded from standalone manual CI and is enabled only when -the full release umbrella passes `full_release_validation=true`. Manual runs use a +umbrella enables Android by passing `include_android=true`. Plugin prerelease +static checks, the release-only `agentic-plugins` shard, the full extension +batch sweep, and plugin prerelease Docker lanes are excluded from CI and run in +the separate `Plugin Prerelease` workflow. Manual runs use a unique concurrency group so a release-candidate full suite is not cancelled by another push or PR run on the same ref. The optional `target_ref` input lets a trusted caller run that graph against a branch, tag, or full commit SHA while @@ -389,7 +389,7 @@ Jobs are ordered so cheap checks fail before expensive ones run: 1. `preflight` decides which lanes exist at all. The `docs-scope` and `changed-scope` logic are steps inside this job, not standalone jobs. 2. `security-scm-fast`, `security-dependency-audit`, `security-fast`, `check`, `check-additional`, `check-docs`, and `skills-python` fail quickly without waiting on the heavier artifact and platform matrix jobs. 3. `build-artifacts` overlaps with the fast Linux lanes so downstream consumers can start as soon as the shared build is ready. -4. Heavier platform and runtime lanes fan out after that: `checks-fast-core`, `checks-fast-contracts-channels`, `checks-node-extensions`, `checks-node-core-test`, `checks`, `checks-windows`, `macos-node`, `macos-swift`, and `android`. +4. Heavier platform and runtime lanes fan out after that: `checks-fast-core`, `checks-fast-contracts-channels`, `checks-node-core-test`, `checks`, `checks-windows`, `macos-node`, `macos-swift`, and `android`. Scope logic lives in `scripts/ci-changed-scope.mjs` and is covered by unit tests in `src/scripts/ci-changed-scope.test.ts`. Manual dispatch skips changed-scope detection and makes the preflight manifest @@ -422,9 +422,9 @@ copy of the PR. Stop that box and warm a fresh one instead of debugging the product test failure. For intentional large deletion PRs, set `OPENCLAW_TESTBOX_ALLOW_MASS_DELETIONS=1` for that sanity run. -Manual CI dispatches run `checks-node-compat-node22` as broad compatibility coverage. Android is opt-in for standalone manual CI through `include_android=true` and always enabled for `Full Release Validation`. `plugin-prerelease-suite` is more expensive product/package coverage, so it runs only when `Full Release Validation` dispatches CI with `full_release_validation=true`. Normal pull requests, `main` pushes, and standalone manual CI dispatches keep that suite off. +Manual CI dispatches run `checks-node-compat-node22` as broad compatibility coverage. Android is opt-in for standalone manual CI through `include_android=true` and always enabled for `Full Release Validation`. `Plugin Prerelease` is more expensive product/package coverage, so it is a separate workflow dispatched by `Full Release Validation` or by an explicit operator. Normal pull requests, `main` pushes, and standalone manual CI dispatches keep that suite off. -The slowest Node test families are split or balanced so each job stays small without over-reserving runners: channel contracts run as three weighted shards, bundled plugin tests balance across eight extension workers, small core unit lanes are paired, auto-reply runs as four balanced workers with the reply subtree split into agent-runner, dispatch, and commands/state-routing shards, and agentic gateway/plugin configs are spread across the existing source-only agentic Node jobs instead of waiting on built artifacts. Broad browser, QA, media, and miscellaneous plugin tests use their dedicated Vitest configs instead of the shared plugin catch-all. Extension shard jobs run up to two plugin config groups at a time with one Vitest worker per group and a larger Node heap so import-heavy plugin batches do not create extra CI jobs. The broad agents lane uses the shared Vitest file-parallel scheduler because it is import/scheduling dominated rather than owned by a single slow test file. `runtime-config` runs with the infra core-runtime shard to keep the shared runtime shard from owning the tail. Include-pattern shards record timing entries using the CI shard name, so `.artifacts/vitest-shard-timings.json` can distinguish a whole config from a filtered shard. `check-additional` keeps package-boundary compile/canary work together and separates runtime topology architecture from gateway watch coverage; the boundary guard shard runs its small independent guards concurrently inside one job. Gateway watch, channel tests, and the core support-boundary shard run concurrently inside `build-artifacts` after `dist/` and `dist-runtime/` are already built, keeping their old check names as lightweight verifier jobs while avoiding two extra Blacksmith workers and a second artifact-consumer queue. +The slowest Node test families are split or balanced so each job stays small without over-reserving runners: channel contracts run as three weighted shards, small core unit lanes are paired, auto-reply runs as four balanced workers with the reply subtree split into agent-runner, dispatch, and commands/state-routing shards, and agentic gateway/plugin configs are spread across the existing source-only agentic Node jobs instead of waiting on built artifacts. Broad browser, QA, media, and miscellaneous plugin tests use their dedicated Vitest configs instead of the shared plugin catch-all. `Plugin Prerelease` balances bundled plugin tests across eight extension workers; those extension shard jobs run up to two plugin config groups at a time with one Vitest worker per group and a larger Node heap so import-heavy plugin batches do not create extra CI jobs. The broad agents lane uses the shared Vitest file-parallel scheduler because it is import/scheduling dominated rather than owned by a single slow test file. `runtime-config` runs with the infra core-runtime shard to keep the shared runtime shard from owning the tail. Include-pattern shards record timing entries using the CI shard name, so `.artifacts/vitest-shard-timings.json` can distinguish a whole config from a filtered shard. `check-additional` keeps package-boundary compile/canary work together and separates runtime topology architecture from gateway watch coverage; the boundary guard shard runs its small independent guards concurrently inside one job. Gateway watch, channel tests, and the core support-boundary shard run concurrently inside `build-artifacts` after `dist/` and `dist-runtime/` are already built, keeping their old check names as lightweight verifier jobs while avoiding two extra Blacksmith workers and a second artifact-consumer queue. Android CI runs both `testPlayDebugUnitTest` and `testThirdPartyDebugUnitTest`, then builds the Play debug APK. The third-party flavor has no separate source set or manifest; its unit-test lane still compiles that flavor with the SMS/call-log BuildConfig flags, while avoiding a duplicate debug APK packaging job on every Android-relevant push. GitHub may mark superseded jobs as `cancelled` when a newer push lands on the same PR or `main` ref. Treat that as CI noise unless the newest run for the same ref is also failing. Aggregate shard checks use `!cancelled() && always()` so they still report normal shard failures but do not queue after the whole workflow has already been superseded. The automatic CI concurrency key is versioned (`CI-v7-*`) so a GitHub-side zombie in an old queue group cannot indefinitely block newer main runs. Manual full-suite runs use `CI-manual-v1-*` and do not cancel in-progress runs. diff --git a/docs/help/testing.md b/docs/help/testing.md index 4ada3e0a9a7..8dad38a71d8 100644 --- a/docs/help/testing.md +++ b/docs/help/testing.md @@ -408,7 +408,7 @@ Think of the suites as “increasing realism” (and increasing flakiness/cost): - Import-light unit tests from agents, commands, plugins, auto-reply helpers, `plugin-sdk`, and similar pure utility areas route through the `unit-fast` lane, which skips `test/setup-openclaw-runtime.ts`; stateful/runtime-heavy files stay on the existing lanes. - Selected `plugin-sdk` and `commands` helper source files also map changed-mode runs to explicit sibling tests in those light lanes, so helper edits avoid rerunning the full heavy suite for that directory. - `auto-reply` has dedicated buckets for top-level core helpers, top-level `reply.*` integration tests, and the `src/auto-reply/reply/**` subtree. CI further splits the reply subtree into agent-runner, dispatch, and commands/state-routing shards so one import-heavy bucket does not own the full Node tail. - - Normal PR/main CI intentionally skips the extension batch sweep and release-only `agentic-plugins` shard. Full Release Validation dispatches the CI child with `full_release_validation=true`, which turns those plugin/extension-heavy suites back on for release candidates. + - Normal PR/main CI intentionally skips the extension batch sweep and release-only `agentic-plugins` shard. Full Release Validation dispatches the separate `Plugin Prerelease` child workflow for those plugin/extension-heavy suites on release candidates. diff --git a/docs/reference/RELEASING.md b/docs/reference/RELEASING.md index dc1269145cf..b37c59199a6 100644 --- a/docs/reference/RELEASING.md +++ b/docs/reference/RELEASING.md @@ -216,8 +216,9 @@ Validation` or from the `main`/release workflow ref so workflow logic and before the release publish path - If the release work touched CI planning, extension timing manifests, or extension test matrices, regenerate and review the planner-owned - `checks-node-extensions` workflow matrix outputs from `.github/workflows/ci.yml` - before approval so release notes do not describe a stale CI layout + `plugin-prerelease-extension-shard` matrix outputs from + `.github/workflows/plugin-prerelease.yml` before approval so release notes do + not describe a stale CI layout - Stable macOS release readiness also includes the updater surfaces: - the GitHub release must end up with the packaged `.zip`, `.dmg`, and `.dSYM.zip` - `appcast.xml` on `main` must point at the new stable zip after publish @@ -306,10 +307,11 @@ ids, so after a child workflow is rerun successfully, rerun only the failed `Verify full validation` parent job. For bounded recovery, pass `rerun_group` to the umbrella. `all` is the real -release-candidate run, `ci` runs only the normal CI child, `release-checks` runs -every release box, and the narrower release groups are `install-smoke`, -`cross-os`, `live-e2e`, `package`, `qa`, `qa-parity`, `qa-live`, and -`npm-telegram` when the standalone package Telegram lane is supplied. +release-candidate run, `ci` runs only the normal CI child, `plugin-prerelease` +runs only the release-only plugin child, `release-checks` runs every release +box, and the narrower release groups are `install-smoke`, `cross-os`, +`live-e2e`, `package`, `qa`, `qa-parity`, `qa-live`, and `npm-telegram` when the +standalone package Telegram lane is supplied. ### Vitest diff --git a/test/scripts/ci-node-test-plan.test.ts b/test/scripts/ci-node-test-plan.test.ts index d1d2ba15b3f..756996bda8d 100644 --- a/test/scripts/ci-node-test-plan.test.ts +++ b/test/scripts/ci-node-test-plan.test.ts @@ -260,7 +260,7 @@ describe("scripts/lib/ci-node-test-plan.mjs", () => { }); }); - it("keeps plugin prerelease npm install coverage on the agentic plugin CI shard", () => { + it("keeps plugin prerelease npm install coverage on the release-only agentic plugin shard", () => { const pluginsShard = createNodeTestShards().find( (shard) => shard.shardName === "agentic-plugins", ); diff --git a/test/scripts/plugin-prerelease-test-plan.test.ts b/test/scripts/plugin-prerelease-test-plan.test.ts index f98d1ff1d6d..08792bb8eaf 100644 --- a/test/scripts/plugin-prerelease-test-plan.test.ts +++ b/test/scripts/plugin-prerelease-test-plan.test.ts @@ -17,8 +17,12 @@ function readFullReleaseValidationWorkflow() { return parse(readFileSync(".github/workflows/full-release-validation.yml", "utf8")); } +function readPluginPrereleaseWorkflow() { + return parse(readFileSync(".github/workflows/plugin-prerelease.yml", "utf8")); +} + describe("scripts/lib/plugin-prerelease-test-plan.mjs", () => { - it("covers every pre-release plugin skill surface in mega CI", () => { + it("covers every pre-release plugin skill surface in the plugin prerelease plan", () => { const plan = assertPluginPrereleaseTestPlanComplete(); expect(plan.surfaces).toEqual( @@ -107,77 +111,103 @@ describe("scripts/lib/plugin-prerelease-test-plan.mjs", () => { expect(script).toContain("scan_logs_for_unexpected_errors"); }); - it("wires the full plugin prerelease plan into the mega CI workflow", () => { + it("wires the full plugin prerelease plan into its release workflow", () => { const workflow = readCiWorkflow(); const preflight = workflow.jobs.preflight; - const extensionShard = workflow.jobs["checks-node-extensions-shard"]; - const extensionSuite = workflow.jobs["checks-node-extensions"]; - const staticShard = workflow.jobs["plugin-prerelease-static-shard"]; - const dockerSuite = workflow.jobs["plugin-prerelease-docker-suite"]; - const suite = workflow.jobs["plugin-prerelease-suite"]; + const pluginWorkflow = readPluginPrereleaseWorkflow(); + const pluginPreflight = pluginWorkflow.jobs.preflight; + const staticShard = pluginWorkflow.jobs["plugin-prerelease-static-shard"]; + const nodeShard = pluginWorkflow.jobs["plugin-prerelease-node-shard"]; + const extensionShard = pluginWorkflow.jobs["plugin-prerelease-extension-shard"]; + const dockerSuite = pluginWorkflow.jobs["plugin-prerelease-docker-suite"]; + const suite = pluginWorkflow.jobs["plugin-prerelease-suite"]; const releaseWorkflow = readFullReleaseValidationWorkflow(); const manifestScript = preflight.steps.find((step) => step.name === "Build CI manifest").run; const manifestEnv = preflight.steps.find((step) => step.name === "Build CI manifest").env; + const pluginManifestScript = pluginPreflight.steps.find( + (step) => step.name === "Build plugin prerelease manifest", + ).run; const normalCiScript = releaseWorkflow.jobs.normal_ci.steps.find( (step) => step.name === "Dispatch and monitor CI", ).run; + const pluginPrereleaseScript = releaseWorkflow.jobs.plugin_prerelease.steps.find( + (step) => step.name === "Dispatch and monitor plugin prerelease", + ).run; - expect(preflight.outputs).toMatchObject({ - plugin_prerelease_docker_lanes: - "${{ steps.manifest.outputs.plugin_prerelease_docker_lanes }}", - plugin_prerelease_ref: "${{ steps.manifest.outputs.plugin_prerelease_ref }}", - plugin_prerelease_static_matrix: - "${{ steps.manifest.outputs.plugin_prerelease_static_matrix }}", - run_plugin_prerelease_suite: "${{ steps.manifest.outputs.run_plugin_prerelease_suite }}", - run_checks_node_extensions: "${{ steps.manifest.outputs.run_checks_node_extensions }}", - }); + expect(workflow.jobs["plugin-prerelease-static-shard"]).toBeUndefined(); + expect(workflow.jobs["plugin-prerelease-docker-suite"]).toBeUndefined(); + expect(workflow.jobs["plugin-prerelease-suite"]).toBeUndefined(); + expect(workflow.jobs["checks-node-extensions-shard"]).toBeUndefined(); + expect(preflight.outputs).not.toHaveProperty("run_plugin_prerelease_suite"); + expect(preflight.outputs).not.toHaveProperty("run_checks_node_extensions"); expect(staticShard).toMatchObject({ name: "${{ matrix.check_name }}", "runs-on": "blacksmith-8vcpu-ubuntu-2404", }); - expect(workflow.on.workflow_dispatch.inputs.full_release_validation).toMatchObject({ - default: false, - type: "boolean", - }); + expect(workflow.on.workflow_dispatch.inputs.full_release_validation).toBeUndefined(); expect(workflow.on.workflow_dispatch.inputs.include_android).toMatchObject({ default: false, type: "boolean", }); expect(manifestEnv).toMatchObject({ - OPENCLAW_CI_FULL_RELEASE_VALIDATION: - "${{ github.event_name == 'workflow_dispatch' && inputs.full_release_validation && 'true' || 'false' }}", OPENCLAW_CI_RUN_ANDROID: - "${{ github.event_name == 'workflow_dispatch' && (inputs.full_release_validation || inputs.include_android) && 'true' || steps.changed_scope.outputs.run_android || 'false' }}", + "${{ github.event_name == 'workflow_dispatch' && inputs.include_android && 'true' || steps.changed_scope.outputs.run_android || 'false' }}", }); - expect(manifestScript).toContain("const isFullReleaseValidationCiRun ="); - expect(manifestScript).toContain( - "parseBoolean(process.env.OPENCLAW_CI_FULL_RELEASE_VALIDATION)", - ); - expect(manifestScript).toContain( - "let runPluginPrereleaseSuite =\n isFullReleaseValidationCiRun && runNodeFull && isCanonicalRepository;", - ); - expect(manifestScript).toContain("run_checks_node_extensions: runReleaseOnlyPluginSuites"); + expect(manifestEnv).not.toHaveProperty("OPENCLAW_CI_FULL_RELEASE_VALIDATION"); + expect(manifestScript).toContain("includeReleaseOnlyPluginShards: false"); + expect(manifestScript).not.toContain("plugin-prerelease-test-plan.mjs"); expect(normalCiScript).toContain( - 'dispatch_and_wait ci.yml -f target_ref="$TARGET_SHA" -f full_release_validation=true', + 'dispatch_and_wait ci.yml -f target_ref="$TARGET_SHA" -f include_android=true', ); - expect(manifestScript).toContain("await import("); - expect(manifestScript).toContain('"./scripts/lib/plugin-prerelease-test-plan.mjs"'); - expect(manifestScript).not.toContain('} from "./scripts/lib/plugin-prerelease-test-plan.mjs";'); - expect(manifestScript).toContain( - "Plugin prerelease plan unavailable in target ref; skipping plugin prerelease suite.", + expect(normalCiScript).not.toContain("full_release_validation=true"); + expect(pluginPrereleaseScript).toContain( + 'dispatch_and_wait plugin-prerelease.yml -f target_ref="$TARGET_SHA" -f expected_sha="$TARGET_SHA"', ); + expect(pluginManifestScript).toContain("await import("); + expect(pluginManifestScript).toContain('"./scripts/lib/plugin-prerelease-test-plan.mjs"'); + expect(pluginManifestScript).toContain('"./scripts/lib/extension-test-plan.mjs"'); + expect(pluginManifestScript).toContain('"./scripts/lib/ci-node-test-plan.mjs"'); + expect(pluginManifestScript).toContain('shard.shardName === "agentic-plugins"'); + expect(pluginManifestScript).toContain( + "Plugin prerelease plan unavailable in target ref; skipping static and Docker plugin prerelease lanes.", + ); + expect(pluginWorkflow.on.workflow_dispatch.inputs.target_ref).toMatchObject({ + default: "main", + type: "string", + }); + expect(pluginPreflight.outputs).toMatchObject({ + checkout_revision: "${{ steps.manifest.outputs.checkout_revision }}", + plugin_prerelease_docker_lanes: + "${{ steps.manifest.outputs.plugin_prerelease_docker_lanes }}", + plugin_prerelease_extension_matrix: + "${{ steps.manifest.outputs.plugin_prerelease_extension_matrix }}", + plugin_prerelease_node_matrix: "${{ steps.manifest.outputs.plugin_prerelease_node_matrix }}", + plugin_prerelease_static_matrix: + "${{ steps.manifest.outputs.plugin_prerelease_static_matrix }}", + run_plugin_prerelease_docker: "${{ steps.manifest.outputs.run_plugin_prerelease_docker }}", + run_plugin_prerelease_extensions: + "${{ steps.manifest.outputs.run_plugin_prerelease_extensions }}", + run_plugin_prerelease_node: "${{ steps.manifest.outputs.run_plugin_prerelease_node }}", + run_plugin_prerelease_static: "${{ steps.manifest.outputs.run_plugin_prerelease_static }}", + run_plugin_prerelease_suite: "${{ steps.manifest.outputs.run_plugin_prerelease_suite }}", + }); expect(staticShard.strategy.matrix).toBe( "${{ fromJson(needs.preflight.outputs.plugin_prerelease_static_matrix) }}", ); - expect(extensionShard.if).toBe("needs.preflight.outputs.run_checks_node_extensions == 'true'"); - expect(extensionSuite.if).toBe( - "${{ !cancelled() && always() && needs.preflight.outputs.run_checks_node_extensions == 'true' }}", + expect(nodeShard.strategy.matrix).toBe( + "${{ fromJson(needs.preflight.outputs.plugin_prerelease_node_matrix) }}", + ); + expect(extensionShard.if).toBe( + "needs.preflight.outputs.run_plugin_prerelease_extensions == 'true'", + ); + expect(extensionShard.strategy.matrix).toBe( + "${{ fromJson(needs.preflight.outputs.plugin_prerelease_extension_matrix) }}", ); expect( staticShard.steps.find((step) => step.name === "Run plugin prerelease static shard").run, ).toContain('bash -c "$PLUGIN_PRERELEASE_COMMAND"'); expect(dockerSuite).toMatchObject({ - if: "needs.preflight.outputs.run_plugin_prerelease_suite == 'true'", + if: "needs.preflight.outputs.run_plugin_prerelease_docker == 'true'", needs: ["preflight"], uses: "./.github/workflows/openclaw-live-and-e2e-checks-reusable.yml", with: { @@ -187,13 +217,15 @@ describe("scripts/lib/plugin-prerelease-test-plan.mjs", () => { include_release_path_suites: false, include_repo_e2e: false, live_models_only: false, - ref: "${{ needs.preflight.outputs.plugin_prerelease_ref }}", + ref: "${{ needs.preflight.outputs.checkout_revision }}", }, }); expect(dockerSuite.secrets).toBeUndefined(); expect(suite.needs).toEqual([ "preflight", "plugin-prerelease-static-shard", + "plugin-prerelease-node-shard", + "plugin-prerelease-extension-shard", "plugin-prerelease-docker-suite", ]); });