From 996c9d71e9d155b24b64ad910d51028f20a16312 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Wed, 29 Apr 2026 06:20:28 +0100 Subject: [PATCH] ci(test): reserve plugin prerelease for release validation --- .agents/skills/openclaw-testing/SKILL.md | 8 +- .github/workflows/ci.yml | 12 ++- .github/workflows/full-release-validation.yml | 4 +- docs/ci.md | 13 +-- ...re.gateway-auth.ollama.integration.test.ts | 92 ------------------- .../plugin-prerelease-test-plan.test.ts | 25 ++++- 6 files changed, 49 insertions(+), 105 deletions(-) delete mode 100644 src/commands/configure.gateway-auth.ollama.integration.test.ts diff --git a/.agents/skills/openclaw-testing/SKILL.md b/.agents/skills/openclaw-testing/SKILL.md index b345a4380e7..d0fa82451d9 100644 --- a/.agents/skills/openclaw-testing/SKILL.md +++ b/.agents/skills/openclaw-testing/SKILL.md @@ -111,7 +111,8 @@ rerun after a focused patch. the manual "everything before release" umbrella. It resolves a target ref, then dispatches: -- manual `CI` for the full normal CI graph +- manual `CI` for the full normal CI graph, with release-only plugin prerelease + lanes enabled via `full_release_validation=true` - `OpenClaw Release Checks` for install smoke, cross-OS release checks, live and E2E checks, Docker release-path suites, OpenWebUI, QA Lab, fast Matrix, and Telegram release lanes @@ -142,6 +143,11 @@ artifact reuse, and sharding instead. The parent verifier job appends slowest-job tables for child runs; rerun only that verifier after a child rerun turns green. +Standalone manual `CI` dispatches do not run the plugin prerelease suite. That +suite is intentionally reserved for the Full Release Validation CI child so PRs, +main pushes, and ad hoc broad CI checks do not spend Docker/package time on +release-only plugin product coverage. + If a full run is already active on a newer `origin/main`, prefer watching that run over dispatching a duplicate. If you accidentally dispatch a stale duplicate, cancel it and monitor the current run. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8a4dc8b182e..ddd88e80e93 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,6 +8,11 @@ on: required: false default: "" type: string + full_release_validation: + description: Run release-only CI lanes. Reserved for Full Release Validation. + required: false + default: false + type: boolean push: branches: [main] paths-ignore: @@ -130,6 +135,7 @@ jobs: OPENCLAW_CI_RUN_CONTROL_UI_I18N: ${{ github.event_name == 'workflow_dispatch' && 'true' || steps.changed_scope.outputs.run_control_ui_i18n || 'false' }} OPENCLAW_CI_CHECKOUT_REVISION: ${{ steps.checkout_ref.outputs.sha }} OPENCLAW_CI_EVENT_NAME: ${{ github.event_name }} + OPENCLAW_CI_FULL_RELEASE_VALIDATION: ${{ github.event_name == 'workflow_dispatch' && inputs.full_release_validation && 'true' || 'false' }} OPENCLAW_CI_PR_HEAD_REPOSITORY: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name || github.repository }} OPENCLAW_CI_PR_HEAD_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || '' }} OPENCLAW_CI_REPOSITORY: ${{ github.repository }} @@ -181,7 +187,9 @@ jobs: const runSkillsPython = parseBoolean(process.env.OPENCLAW_CI_RUN_SKILLS_PYTHON) && !docsOnly; const runControlUiI18n = parseBoolean(process.env.OPENCLAW_CI_RUN_CONTROL_UI_I18N) && !docsOnly; - const isMegaCiRun = process.env.OPENCLAW_CI_EVENT_NAME === "workflow_dispatch"; + const isFullReleaseValidationCiRun = + process.env.OPENCLAW_CI_EVENT_NAME === "workflow_dispatch" && + parseBoolean(process.env.OPENCLAW_CI_FULL_RELEASE_VALIDATION); const trustedPluginPrereleaseRef = process.env.OPENCLAW_CI_EVENT_NAME !== "pull_request" || process.env.OPENCLAW_CI_PR_HEAD_REPOSITORY === process.env.OPENCLAW_CI_REPOSITORY; @@ -190,7 +198,7 @@ jobs: ? process.env.OPENCLAW_CI_PR_HEAD_SHA : process.env.OPENCLAW_CI_CHECKOUT_REVISION; let runPluginPrereleaseSuite = - isMegaCiRun && runNodeFull && isCanonicalRepository; + isFullReleaseValidationCiRun && runNodeFull && isCanonicalRepository; let pluginPrereleasePlan = { staticChecks: [], dockerLanes: [] }; if (runPluginPrereleaseSuite) { try { diff --git a/.github/workflows/full-release-validation.yml b/.github/workflows/full-release-validation.yml index a97c12a2bf5..6fda26fe1fc 100644 --- a/.github/workflows/full-release-validation.yml +++ b/.github/workflows/full-release-validation.yml @@ -131,7 +131,7 @@ jobs: echo "- Child workflow ref: \`${CHILD_WORKFLOW_REF}\`" echo "- Rerun group: \`${RERUN_GROUP}\`" if [[ "$RERUN_GROUP" == "all" || "$RERUN_GROUP" == "ci" ]]; then - echo "- Normal CI: \`CI\` with \`target_ref=${TARGET_SHA}\`" + echo "- Normal CI: \`CI\` with \`target_ref=${TARGET_SHA}\` and release-only lanes enabled" else echo "- Normal CI: skipped by rerun group" fi @@ -263,7 +263,7 @@ jobs: } cancel_same_sha_push_ci - dispatch_and_wait ci.yml -f target_ref="$TARGET_SHA" + dispatch_and_wait ci.yml -f target_ref="$TARGET_SHA" -f full_release_validation=true release_checks: name: Run release/live/Docker/QA validation diff --git a/docs/ci.md b/docs/ci.md index 805427048ae..66727c2f045 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -6,7 +6,7 @@ read_when: - You are debugging failing GitHub Actions checks --- -The CI runs on every push to `main` and every pull request. It uses smart scoping to skip expensive jobs when only unrelated areas changed. Manual `workflow_dispatch` runs intentionally bypass smart scoping and fan out the full normal CI graph for release candidates or broad validation. +The CI runs on every push to `main` and every pull request. It uses smart scoping to skip expensive jobs when only unrelated areas changed. Manual `workflow_dispatch` runs intentionally bypass smart scoping and fan out the full normal CI graph for release candidates or broad validation. Release-only plugin prerelease lanes stay off unless `Full Release Validation` dispatches CI with `full_release_validation=true`. `Full Release Validation` is the manual umbrella workflow for "run everything before release." It accepts a branch, tag, or full commit SHA, dispatches the @@ -346,7 +346,7 @@ gh workflow run duplicate-after-merge.yml \ | `build-smoke` | Built-CLI smoke tests and startup-memory smoke | Node-relevant changes | | `checks` | Verifier for built-artifact channel tests | Node-relevant changes | | `checks-node-compat-node22` | Node 22 compatibility build and smoke lane | Manual CI dispatch for releases | -| `plugin-prerelease-suite` | Aggregate for plugin prerelease static checks and Docker product lanes | Manual CI dispatch for releases | +| `plugin-prerelease-suite` | Aggregate for plugin prerelease static checks and Docker product lanes | Full Release Validation CI child | | `check-docs` | Docs formatting, lint, and broken-link checks | Docs changed | | `skills-python` | Ruff + pytest for Python-backed skills | Python-skill-relevant changes | | `checks-windows` | Windows-specific process/path tests plus shared runtime import specifier regressions | Windows-relevant changes | @@ -357,9 +357,10 @@ gh workflow run duplicate-after-merge.yml \ Manual CI dispatches run the same job graph as normal CI but force every scoped lane on: Linux Node shards, bundled-plugin shards, channel contracts, -Node 22 compatibility, plugin prerelease coverage, `check`, -`check-additional`, build smoke, docs checks, Python skills, Windows, macOS, -Android, and Control UI i18n. Manual runs use a +Node 22 compatibility, `check`, `check-additional`, build smoke, docs checks, +Python skills, Windows, macOS, Android, and Control UI i18n. The plugin +prerelease suite is excluded from standalone manual CI and is enabled only when +the full release umbrella passes `full_release_validation=true`. Manual runs use a unique concurrency group so a release-candidate full suite is not cancelled by another push or PR run on the same ref. The optional `target_ref` input lets a trusted caller run that graph against a branch, tag, or full commit SHA while @@ -411,7 +412,7 @@ copy of the PR. Stop that box and warm a fresh one instead of debugging the product test failure. For intentional large deletion PRs, set `OPENCLAW_TESTBOX_ALLOW_MASS_DELETIONS=1` for that sanity run. -Manual CI dispatches run `checks-node-compat-node22` and `plugin-prerelease-suite` as release-candidate compatibility coverage. Normal pull requests and `main` pushes skip those lanes and keep the matrix focused on the Node 24 test/channel lanes. +Manual CI dispatches run `checks-node-compat-node22` as broad compatibility coverage. `plugin-prerelease-suite` is more expensive product/package coverage, so it runs only when `Full Release Validation` dispatches CI with `full_release_validation=true`. Normal pull requests, `main` pushes, and standalone manual CI dispatches keep that suite off. The slowest Node test families are split or balanced so each job stays small without over-reserving runners: channel contracts run as three weighted shards, bundled plugin tests balance across six extension workers, small core unit lanes are paired, auto-reply runs as four balanced workers with the reply subtree split into agent-runner, dispatch, and commands/state-routing shards, and agentic gateway/plugin configs are spread across the existing source-only agentic Node jobs instead of waiting on built artifacts. Broad browser, QA, media, and miscellaneous plugin tests use their dedicated Vitest configs instead of the shared plugin catch-all. Extension shard jobs run up to two plugin config groups at a time with one Vitest worker per group and a larger Node heap so import-heavy plugin batches do not create extra CI jobs. The broad agents lane uses the shared Vitest file-parallel scheduler because it is import/scheduling dominated rather than owned by a single slow test file. `runtime-config` runs with the infra core-runtime shard to keep the shared runtime shard from owning the tail. Include-pattern shards record timing entries using the CI shard name, so `.artifacts/vitest-shard-timings.json` can distinguish a whole config from a filtered shard. `check-additional` keeps package-boundary compile/canary work together and separates runtime topology architecture from gateway watch coverage; the boundary guard shard runs its small independent guards concurrently inside one job. Gateway watch, channel tests, and the core support-boundary shard run concurrently inside `build-artifacts` after `dist/` and `dist-runtime/` are already built, keeping their old check names as lightweight verifier jobs while avoiding two extra Blacksmith workers and a second artifact-consumer queue. Android CI runs both `testPlayDebugUnitTest` and `testThirdPartyDebugUnitTest`, then builds the Play debug APK. The third-party flavor has no separate source set or manifest; its unit-test lane still compiles that flavor with the SMS/call-log BuildConfig flags, while avoiding a duplicate debug APK packaging job on every Android-relevant push. diff --git a/src/commands/configure.gateway-auth.ollama.integration.test.ts b/src/commands/configure.gateway-auth.ollama.integration.test.ts deleted file mode 100644 index 8d7275be6ea..00000000000 --- a/src/commands/configure.gateway-auth.ollama.integration.test.ts +++ /dev/null @@ -1,92 +0,0 @@ -import { mkdtempSync } from "node:fs"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import type { OpenClawConfig } from "../config/config.js"; -import type { WizardPrompter } from "../wizard/prompts.js"; -import { promptAuthConfig } from "./configure.gateway-auth.js"; -import { makePrompter, makeRuntime } from "./setup/__tests__/test-utils.js"; - -describe("promptAuthConfig Ollama setup", () => { - const originalFetch = globalThis.fetch; - - beforeEach(() => { - vi.clearAllMocks(); - vi.stubEnv("HOME", mkdtempSync(join(tmpdir(), "openclaw-ollama-config-"))); - vi.stubGlobal( - "fetch", - vi.fn(async (url: string | URL | Request) => { - const href = typeof url === "string" ? url : "url" in url ? url.url : String(url); - if (href.endsWith("/api/tags")) { - return new Response( - JSON.stringify({ - models: [{ name: "kimi-k2.5:cloud" }, { name: "gpt-oss:20b-cloud" }], - }), - { status: 200, headers: { "content-type": "application/json" } }, - ); - } - throw new Error(`unexpected fetch: ${href}`); - }), - ); - }); - - afterEach(() => { - vi.unstubAllEnvs(); - vi.stubGlobal("fetch", originalFetch); - }); - - it("shows the model picker after cloud-only setup when Ollama models were already configured", async () => { - const select = vi.fn(async (params) => { - if (params.message === "Model/auth provider") { - return "ollama"; - } - if (params.message === "Ollama mode") { - return "cloud-only"; - } - if (params.message === "How do you want to provide this API key?") { - return "plaintext"; - } - throw new Error(`unexpected select: ${params.message}`); - }) as WizardPrompter["select"]; - const text = vi.fn(async (params) => { - if (params.message === "Ollama API key") { - return "test-ollama-key"; - } - throw new Error(`unexpected text: ${params.message}`); - }); - const multiselect = vi.fn(async (params) => - params.options.map((option: { value: string }) => option.value), - ); - const progress = vi.fn(() => ({ update: vi.fn(), stop: vi.fn() })); - const prompter = makePrompter({ select, text, multiselect, progress }); - const config = { - models: { - providers: { - ollama: { - api: "ollama", - baseUrl: "https://ollama.com", - models: [ - { - id: "kimi-k2.5:cloud", - name: "Kimi K2.5", - reasoning: false, - input: ["text"], - cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, - contextWindow: 128_000, - maxTokens: 8192, - }, - ], - }, - }, - }, - } as OpenClawConfig; - - const result = await promptAuthConfig(config, makeRuntime(), prompter); - - expect(multiselect).toHaveBeenCalled(); - expect( - multiselect.mock.calls[0]?.[0]?.options.map((option: { value: string }) => option.value), - ).toContain("ollama/kimi-k2.5:cloud"); - expect(result.agents?.defaults?.models).toHaveProperty("ollama/kimi-k2.5:cloud"); - }); -}); diff --git a/test/scripts/plugin-prerelease-test-plan.test.ts b/test/scripts/plugin-prerelease-test-plan.test.ts index 040b668a143..2ad64f57984 100644 --- a/test/scripts/plugin-prerelease-test-plan.test.ts +++ b/test/scripts/plugin-prerelease-test-plan.test.ts @@ -13,6 +13,10 @@ function readCiWorkflow() { return parse(readFileSync(".github/workflows/ci.yml", "utf8")); } +function readFullReleaseValidationWorkflow() { + return parse(readFileSync(".github/workflows/full-release-validation.yml", "utf8")); +} + describe("scripts/lib/plugin-prerelease-test-plan.mjs", () => { it("covers every pre-release plugin skill surface in mega CI", () => { const plan = assertPluginPrereleaseTestPlanComplete(); @@ -109,7 +113,12 @@ describe("scripts/lib/plugin-prerelease-test-plan.mjs", () => { const staticShard = workflow.jobs["plugin-prerelease-static-shard"]; const dockerSuite = workflow.jobs["plugin-prerelease-docker-suite"]; const suite = workflow.jobs["plugin-prerelease-suite"]; + const releaseWorkflow = readFullReleaseValidationWorkflow(); const manifestScript = preflight.steps.find((step) => step.name === "Build CI manifest").run; + const manifestEnv = preflight.steps.find((step) => step.name === "Build CI manifest").env; + const normalCiScript = releaseWorkflow.jobs.normal_ci.steps.find( + (step) => step.name === "Dispatch and monitor CI", + ).run; expect(preflight.outputs).toMatchObject({ plugin_prerelease_docker_lanes: @@ -123,11 +132,23 @@ describe("scripts/lib/plugin-prerelease-test-plan.mjs", () => { name: "${{ matrix.check_name }}", "runs-on": "blacksmith-8vcpu-ubuntu-2404", }); + expect(workflow.on.workflow_dispatch.inputs.full_release_validation).toMatchObject({ + default: false, + type: "boolean", + }); + expect(manifestEnv).toMatchObject({ + OPENCLAW_CI_FULL_RELEASE_VALIDATION: + "${{ github.event_name == 'workflow_dispatch' && inputs.full_release_validation && 'true' || 'false' }}", + }); + expect(manifestScript).toContain("const isFullReleaseValidationCiRun ="); expect(manifestScript).toContain( - 'const isMegaCiRun = process.env.OPENCLAW_CI_EVENT_NAME === "workflow_dispatch";', + "parseBoolean(process.env.OPENCLAW_CI_FULL_RELEASE_VALIDATION)", ); expect(manifestScript).toContain( - "let runPluginPrereleaseSuite =\n isMegaCiRun && runNodeFull && isCanonicalRepository;", + "let runPluginPrereleaseSuite =\n isFullReleaseValidationCiRun && runNodeFull && isCanonicalRepository;", + ); + expect(normalCiScript).toContain( + 'dispatch_and_wait ci.yml -f target_ref="$TARGET_SHA" -f full_release_validation=true', ); expect(manifestScript).toContain("await import("); expect(manifestScript).toContain('"./scripts/lib/plugin-prerelease-test-plan.mjs"');