From 3dc6e408b95d0fa70b782cc69615d63d99a59b96 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Wed, 29 Apr 2026 04:21:50 +0100 Subject: [PATCH] ci(release): isolate channel live qa from provider latency --- .github/workflows/openclaw-release-checks.yml | 26 +++------------ docs/ci.md | 11 ++++--- src/channels/plugins/acp-bindings.test.ts | 32 +++++-------------- .../package-acceptance-workflow.test.ts | 16 ++++++---- 4 files changed, 29 insertions(+), 56 deletions(-) diff --git a/.github/workflows/openclaw-release-checks.yml b/.github/workflows/openclaw-release-checks.yml index dd517450e3f..30bc3bad0cb 100644 --- a/.github/workflows/openclaw-release-checks.yml +++ b/.github/workflows/openclaw-release-checks.yml @@ -648,18 +648,6 @@ jobs: pnpm-version: ${{ env.PNPM_VERSION }} install-bun: "true" - - name: Validate required QA credential env - env: - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - shell: bash - run: | - set -euo pipefail - - if [[ -z "${OPENAI_API_KEY:-}" ]]; then - echo "Missing required OPENAI_API_KEY." >&2 - exit 1 - fi - - name: Build private QA runtime run: pnpm build @@ -667,9 +655,7 @@ jobs: id: run_lane shell: bash env: - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} OPENCLAW_QA_REDACT_PUBLIC_METADATA: "1" - OPENCLAW_QA_MATRIX_CANARY_TIMEOUT_MS: "90000" OPENCLAW_QA_MATRIX_NO_REPLY_WINDOW_MS: "3000" run: | set -euo pipefail @@ -680,9 +666,9 @@ jobs: matrix_args=( --repo-root . \ --output-dir "${output_dir}" \ - --provider-mode live-frontier \ + --provider-mode mock-openai \ --model "${OPENCLAW_CI_OPENAI_MODEL}" \ - --alt-model "${OPENCLAW_CI_OPENAI_MODEL}" \ + --alt-model openai/gpt-5.4-alt \ --profile fast \ --fast ) @@ -731,7 +717,6 @@ jobs: - name: Validate required QA credential env env: - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} OPENCLAW_QA_CONVEX_SITE_URL: ${{ secrets.OPENCLAW_QA_CONVEX_SITE_URL }} OPENCLAW_QA_CONVEX_SECRET_CI: ${{ secrets.OPENCLAW_QA_CONVEX_SECRET_CI }} shell: bash @@ -746,7 +731,6 @@ jobs: fi } - require_var OPENAI_API_KEY require_var OPENCLAW_QA_CONVEX_SITE_URL require_var OPENCLAW_QA_CONVEX_SECRET_CI @@ -757,12 +741,10 @@ jobs: id: run_lane shell: bash env: - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} OPENCLAW_QA_CONVEX_SITE_URL: ${{ secrets.OPENCLAW_QA_CONVEX_SITE_URL }} OPENCLAW_QA_CONVEX_SECRET_CI: ${{ secrets.OPENCLAW_QA_CONVEX_SECRET_CI }} OPENCLAW_QA_REDACT_PUBLIC_METADATA: "1" OPENCLAW_QA_TELEGRAM_CAPTURE_CONTENT: "1" - OPENCLAW_QA_TELEGRAM_CANARY_TIMEOUT_MS: "90000" run: | set -euo pipefail @@ -772,9 +754,9 @@ jobs: pnpm openclaw qa telegram \ --repo-root . \ --output-dir "${output_dir}" \ - --provider-mode live-frontier \ + --provider-mode mock-openai \ --model "${OPENCLAW_CI_OPENAI_MODEL}" \ - --alt-model "${OPENCLAW_CI_OPENAI_MODEL}" \ + --alt-model openai/gpt-5.4-alt \ --fast \ --credential-source convex \ --credential-role ci diff --git a/docs/ci.md b/docs/ci.md index ba1d723db4d..fc0c88a883f 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -214,10 +214,13 @@ builds the private QA runtime and compares the mock GPT-5.5 and Opus 4.6 agentic packs. The `QA-Lab - All Lanes` workflow runs nightly on `main` and on manual dispatch; it fans out the mock parity gate, live Matrix lane, and live Telegram and Discord lanes as parallel jobs. The live jobs use the -`qa-live-shared` environment, and Telegram/Discord use Convex leases. Matrix -uses `--profile fast` for scheduled and release gates, adding `--fail-fast` only -when the checked-out CLI supports it. The CLI default and manual workflow input -remain `all`; manual `matrix_profile=all` +`qa-live-shared` environment, and Telegram/Discord use Convex leases. Release +checks run Matrix and Telegram live transport lanes with the deterministic mock +provider so the channel contract is isolated from live model latency; provider +connectivity is covered by the separate live model, native provider, and Docker +provider suites. Matrix uses `--profile fast` for scheduled and release gates, +adding `--fail-fast` only when the checked-out CLI supports it. The CLI default +and manual workflow input remain `all`; manual `matrix_profile=all` dispatch always shards full Matrix coverage into `transport`, `media`, `e2ee-smoke`, `e2ee-deep`, and `e2ee-cli` jobs. `OpenClaw Release Checks` also runs the release-critical QA Lab lanes before release approval; its QA parity diff --git a/src/channels/plugins/acp-bindings.test.ts b/src/channels/plugins/acp-bindings.test.ts index 6f9c6da2cba..c606292cc02 100644 --- a/src/channels/plugins/acp-bindings.test.ts +++ b/src/channels/plugins/acp-bindings.test.ts @@ -7,8 +7,6 @@ const resolveAgentConfigMock = vi.hoisted(() => vi.fn()); const resolveDefaultAgentIdMock = vi.hoisted(() => vi.fn()); const resolveAgentWorkspaceDirMock = vi.hoisted(() => vi.fn()); const getChannelPluginMock = vi.hoisted(() => vi.fn()); -const getActivePluginChannelRegistryVersionMock = vi.hoisted(() => vi.fn()); -const requireActivePluginChannelRegistryMock = vi.hoisted(() => vi.fn(() => ({}))); vi.mock("../../agents/agent-scope.js", () => ({ resolveAgentConfig: resolveAgentConfigMock, @@ -20,11 +18,6 @@ vi.mock("./index.js", () => ({ getChannelPlugin: getChannelPluginMock, })); -vi.mock("../../plugins/runtime.js", () => ({ - getActivePluginChannelRegistryVersion: getActivePluginChannelRegistryVersionMock, - requireActivePluginChannelRegistry: requireActivePluginChannelRegistryMock, -})); - function createConfig(options?: { bindingAgentId?: string; accountId?: string }) { return { agents: { @@ -95,8 +88,6 @@ describe("configured binding registry", () => { resolveDefaultAgentIdMock.mockReset().mockReturnValue("main"); resolveAgentWorkspaceDirMock.mockReset().mockReturnValue("/tmp/workspace"); getChannelPluginMock.mockReset(); - getActivePluginChannelRegistryVersionMock.mockReset().mockReturnValue(1); - requireActivePluginChannelRegistryMock.mockReset().mockReturnValue({}); ensureConfiguredBindingBuiltinsRegistered(); }); @@ -144,7 +135,7 @@ describe("configured binding registry", () => { }); }); - it("primes compiled ACP bindings from the already loaded channel registry once", async () => { + it("primes compiled ACP bindings from the already loaded channel registry", async () => { const plugin = createDiscordAcpPlugin(); const cfg = createConfig({ bindingAgentId: "codex" }); getChannelPluginMock.mockReturnValue(plugin); @@ -161,7 +152,6 @@ describe("configured binding registry", () => { expect(primed).toEqual({ bindingCount: 1, channelCount: 1 }); expect(resolved?.statefulTarget.agentId).toBe("codex"); - expect(plugin.bindings?.compileConfiguredBinding).toHaveBeenCalledTimes(1); const second = bindingRegistry.resolveConfiguredBindingRecord({ cfg: cfg as never, @@ -205,10 +195,10 @@ describe("configured binding registry", () => { expect(resolved).toBeNull(); }); - it("rebuilds the compiled registry when the active plugin registry version changes", async () => { - const plugin = createDiscordAcpPlugin(); - getChannelPluginMock.mockReturnValue(plugin); - getActivePluginChannelRegistryVersionMock.mockReturnValue(10); + it("uses the current loaded channel plugin on each resolve", async () => { + const firstPlugin = createDiscordAcpPlugin(); + const secondPlugin = createDiscordAcpPlugin(); + getChannelPluginMock.mockReturnValueOnce(firstPlugin).mockReturnValueOnce(secondPlugin); const cfg = createConfig(); bindingRegistry.resolveConfiguredBindingRecord({ @@ -217,6 +207,7 @@ describe("configured binding registry", () => { accountId: "default", conversationId: "1479098716916023408", }); + bindingRegistry.resolveConfiguredBindingRecord({ cfg: cfg as never, channel: "discord", @@ -224,14 +215,7 @@ describe("configured binding registry", () => { conversationId: "1479098716916023408", }); - getActivePluginChannelRegistryVersionMock.mockReturnValue(11); - bindingRegistry.resolveConfiguredBindingRecord({ - cfg: cfg as never, - channel: "discord", - accountId: "default", - conversationId: "1479098716916023408", - }); - - expect(plugin.bindings?.compileConfiguredBinding).toHaveBeenCalledTimes(2); + expect(firstPlugin.bindings?.compileConfiguredBinding).toHaveBeenCalledTimes(1); + expect(secondPlugin.bindings?.compileConfiguredBinding).toHaveBeenCalledTimes(1); }); }); diff --git a/test/scripts/package-acceptance-workflow.test.ts b/test/scripts/package-acceptance-workflow.test.ts index 932b0e91a6e..fade18bd645 100644 --- a/test/scripts/package-acceptance-workflow.test.ts +++ b/test/scripts/package-acceptance-workflow.test.ts @@ -136,12 +136,16 @@ describe("package artifact reuse", () => { expect(workflow).toContain("suite_id: native-live-extensions-a-k"); expect(workflow).toContain("suite_id: native-live-extensions-l-n"); expect(workflow).toContain("suite_id: native-live-extensions-openai"); - expect(workflow).toContain("suite_id: native-live-extensions-o-z"); - expect(workflow).toContain("suite_id: native-live-extensions-media"); - expect(workflow).toMatch( - /suite_id: native-live-extensions-media-audio[\s\S]*?needs_ffmpeg: true/u, - ); - expect(workflow).toContain("if: matrix.needs_ffmpeg"); + expect(workflow).toContain("suite_id: native-live-extensions-o-z-other"); + expect(workflow).toContain("validate_live_media_provider_suites:"); + expect(workflow).toContain("image: ghcr.io/openclaw/openclaw-live-media-runner:ubuntu-24.04"); + expect(workflow).toContain("ffmpeg -version | head -1"); + expect(workflow).toContain("ffprobe -version | head -1"); + expect(workflow).toContain("suite_id: native-live-extensions-media-audio"); + expect(workflow).toContain("suite_id: native-live-extensions-media-music-google"); + expect(workflow).toContain("suite_id: native-live-extensions-media-music-minimax"); + expect(workflow).toContain("suite_id: native-live-extensions-media-video"); + expect(workflow).not.toContain("needs_ffmpeg: true"); }); it("runs Docker live harnesses from trusted helper scripts", () => {