From 806a0119f3cd0683c1b7797f835dc09203253948 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Wed, 29 Apr 2026 05:25:23 +0100 Subject: [PATCH] ci(release): reuse live test Docker image --- .../openclaw-live-and-e2e-checks-reusable.yml | 282 ++++++++++++++++-- docs/ci.md | 17 +- docs/help/testing.md | 12 +- 3 files changed, 279 insertions(+), 32 deletions(-) diff --git a/.github/workflows/openclaw-live-and-e2e-checks-reusable.yml b/.github/workflows/openclaw-live-and-e2e-checks-reusable.yml index 7b9c0ac53c0..eb9b001d5b5 100644 --- a/.github/workflows/openclaw-live-and-e2e-checks-reusable.yml +++ b/.github/workflows/openclaw-live-and-e2e-checks-reusable.yml @@ -1260,11 +1260,83 @@ jobs: provenance: mode=max push: true + prepare_live_test_image: + needs: validate_selected_ref + if: inputs.include_live_suites + runs-on: blacksmith-32vcpu-ubuntu-2404 + timeout-minutes: 60 + permissions: + contents: read + packages: write + outputs: + live_image: ${{ steps.image.outputs.live_image }} + env: + DOCKER_BUILD_SUMMARY: "false" + DOCKER_BUILD_RECORD_UPLOAD: "false" + steps: + - name: Checkout selected ref + uses: actions/checkout@v6 + with: + ref: ${{ needs.validate_selected_ref.outputs.selected_sha }} + fetch-depth: 1 + + - name: Resolve shared live-test image tag + id: image + shell: bash + env: + SELECTED_SHA: ${{ needs.validate_selected_ref.outputs.selected_sha }} + run: | + set -euo pipefail + repository="${GITHUB_REPOSITORY,,}" + live_image="ghcr.io/${repository}-live-test:${SELECTED_SHA}" + echo "live_image=${live_image}" >> "$GITHUB_OUTPUT" + echo "Shared live-test image: \`${live_image}\`" >> "$GITHUB_STEP_SUMMARY" + + - name: Log in to GHCR + uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ github.token }} + + - name: Check existing shared live-test image + id: image_exists + shell: bash + run: | + set -euo pipefail + if docker manifest inspect "${{ steps.image.outputs.live_image }}" >/dev/null 2>&1; then + echo "Shared live-test image already exists: ${{ steps.image.outputs.live_image }}" + echo "exists=1" >> "$GITHUB_OUTPUT" + else + echo "exists=0" >> "$GITHUB_OUTPUT" + fi + + - name: Setup Docker builder + if: steps.image_exists.outputs.exists != '1' + uses: useblacksmith/setup-docker-builder@ac083cc84672d01c60d5e8561d0a939b697de542 # v1 + with: + max-cache-size-mb: 800000 + + - name: Build and push shared live-test image + if: steps.image_exists.outputs.exists != '1' + uses: useblacksmith/build-push-action@cbd1f60d194a98cb3be5523b15134501eaf0fbf3 # v2 + with: + context: . + file: ./Dockerfile + target: build + build-args: | + OPENCLAW_EXTENSIONS=matrix + platforms: linux/amd64 + tags: ${{ steps.image.outputs.live_image }} + sbom: true + provenance: mode=max + push: true + validate_live_models_docker: name: Docker live models (${{ matrix.provider_label }}) - needs: validate_selected_ref + needs: [validate_selected_ref, prepare_live_test_image] if: inputs.include_live_suites && inputs.live_model_providers == '' - runs-on: ubuntu-24.04 + runs-on: blacksmith-32vcpu-ubuntu-2404 timeout-minutes: 75 strategy: fail-fast: false @@ -1332,6 +1404,8 @@ jobs: OPENCLAW_GEMINI_SETTINGS_JSON: ${{ secrets.OPENCLAW_GEMINI_SETTINGS_JSON }} FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }} OPENCLAW_LIVE_PROVIDERS: ${{ matrix.providers }} + OPENCLAW_LIVE_IMAGE: ${{ needs.prepare_live_test_image.outputs.live_image }} + OPENCLAW_SKIP_DOCKER_BUILD: "1" OPENCLAW_VITEST_MAX_WORKERS: "2" steps: - name: Checkout selected ref @@ -1361,6 +1435,14 @@ jobs: if: contains(matrix.profiles, inputs.release_test_profile) run: bash scripts/ci-hydrate-live-auth.sh + - name: Log in to GHCR + if: contains(matrix.profiles, inputs.release_test_profile) + uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ github.token }} + - name: Validate provider credential if: contains(matrix.profiles, inputs.release_test_profile) shell: bash @@ -1402,9 +1484,9 @@ jobs: validate_live_models_docker_targeted: name: Docker live models (selected providers) - needs: validate_selected_ref + needs: [validate_selected_ref, prepare_live_test_image] if: inputs.include_live_suites && inputs.live_model_providers != '' - runs-on: ubuntu-24.04 + runs-on: blacksmith-32vcpu-ubuntu-2404 timeout-minutes: 75 env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} @@ -1441,6 +1523,8 @@ jobs: OPENCLAW_GEMINI_SETTINGS_JSON: ${{ secrets.OPENCLAW_GEMINI_SETTINGS_JSON }} FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }} REQUESTED_LIVE_MODEL_PROVIDERS: ${{ inputs.live_model_providers }} + OPENCLAW_LIVE_IMAGE: ${{ needs.prepare_live_test_image.outputs.live_image }} + OPENCLAW_SKIP_DOCKER_BUILD: "1" OPENCLAW_VITEST_MAX_WORKERS: "2" steps: - name: Checkout selected ref @@ -1525,6 +1609,13 @@ jobs: - name: Hydrate live auth/profile inputs run: bash scripts/ci-hydrate-live-auth.sh + - name: Log in to GHCR + uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ github.token }} + - name: Validate provider credentials shell: bash run: | @@ -1688,30 +1779,6 @@ jobs: timeout_minutes: 90 profile_env_only: false profiles: full - - suite_id: live-gateway-docker - label: Docker live gateway - command: OPENCLAW_LIVE_DOCKER_REPO_ROOT="$GITHUB_WORKSPACE" bash .release-harness/scripts/test-live-gateway-models-docker.sh - timeout_minutes: 120 - profile_env_only: false - profiles: minimum stable full - - suite_id: live-cli-backend-docker - label: Docker live CLI backend - command: OPENCLAW_LIVE_DOCKER_REPO_ROOT="$GITHUB_WORKSPACE" bash .release-harness/scripts/test-live-cli-backend-docker.sh - timeout_minutes: 120 - profile_env_only: false - profiles: stable full - - suite_id: live-acp-bind-docker - label: Docker live ACP bind - command: OPENCLAW_LIVE_DOCKER_REPO_ROOT="$GITHUB_WORKSPACE" bash .release-harness/scripts/test-live-acp-bind-docker.sh - timeout_minutes: 120 - profile_env_only: false - profiles: stable full - - suite_id: live-codex-harness-docker - label: Docker live Codex harness - command: OPENCLAW_LIVE_DOCKER_REPO_ROOT="$GITHUB_WORKSPACE" bash .release-harness/scripts/test-live-codex-harness-docker.sh - timeout_minutes: 120 - profile_env_only: false - profiles: stable full env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} OPENAI_BASE_URL: ${{ secrets.OPENAI_BASE_URL }} @@ -1846,6 +1913,165 @@ jobs: if: contains(matrix.profiles, inputs.release_test_profile) run: ${{ matrix.command }} + validate_live_docker_provider_suites: + name: Docker live suites (${{ matrix.label }}) + needs: [validate_selected_ref, prepare_live_test_image] + if: inputs.include_live_suites && !inputs.live_models_only + runs-on: blacksmith-32vcpu-ubuntu-2404 + timeout-minutes: ${{ matrix.timeout_minutes }} + strategy: + fail-fast: false + matrix: + include: + - suite_id: live-gateway-docker + label: Docker live gateway + command: OPENCLAW_LIVE_DOCKER_REPO_ROOT="$GITHUB_WORKSPACE" bash .release-harness/scripts/test-live-gateway-models-docker.sh + timeout_minutes: 120 + profile_env_only: false + profiles: minimum stable full + - suite_id: live-cli-backend-docker + label: Docker live CLI backend + command: OPENCLAW_LIVE_DOCKER_REPO_ROOT="$GITHUB_WORKSPACE" bash .release-harness/scripts/test-live-cli-backend-docker.sh + timeout_minutes: 120 + profile_env_only: false + profiles: stable full + - suite_id: live-acp-bind-docker + label: Docker live ACP bind + command: OPENCLAW_LIVE_DOCKER_REPO_ROOT="$GITHUB_WORKSPACE" bash .release-harness/scripts/test-live-acp-bind-docker.sh + timeout_minutes: 120 + profile_env_only: false + profiles: stable full + - suite_id: live-codex-harness-docker + label: Docker live Codex harness + command: OPENCLAW_LIVE_DOCKER_REPO_ROOT="$GITHUB_WORKSPACE" bash .release-harness/scripts/test-live-codex-harness-docker.sh + timeout_minutes: 120 + profile_env_only: false + profiles: stable full + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + OPENAI_BASE_URL: ${{ secrets.OPENAI_BASE_URL }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + ANTHROPIC_API_TOKEN: ${{ secrets.ANTHROPIC_API_TOKEN }} + ANTHROPIC_API_KEY_OLD: ${{ secrets.ANTHROPIC_API_KEY_OLD }} + BYTEPLUS_API_KEY: ${{ secrets.BYTEPLUS_API_KEY }} + CEREBRAS_API_KEY: ${{ secrets.CEREBRAS_API_KEY }} + DEEPINFRA_API_KEY: ${{ secrets.DEEPINFRA_API_KEY }} + DASHSCOPE_API_KEY: ${{ secrets.DASHSCOPE_API_KEY }} + GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }} + KIMI_API_KEY: ${{ secrets.KIMI_API_KEY }} + MODELSTUDIO_API_KEY: ${{ secrets.MODELSTUDIO_API_KEY }} + MOONSHOT_API_KEY: ${{ secrets.MOONSHOT_API_KEY }} + MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }} + MINIMAX_API_KEY: ${{ secrets.MINIMAX_API_KEY }} + OPENCODE_API_KEY: ${{ secrets.OPENCODE_API_KEY }} + OPENCODE_ZEN_API_KEY: ${{ secrets.OPENCODE_ZEN_API_KEY }} + OPENCLAW_LIVE_BROWSER_CDP_URL: ${{ secrets.OPENCLAW_LIVE_BROWSER_CDP_URL }} + OPENCLAW_LIVE_SETUP_TOKEN: ${{ secrets.OPENCLAW_LIVE_SETUP_TOKEN }} + OPENCLAW_LIVE_SETUP_TOKEN_MODEL: ${{ secrets.OPENCLAW_LIVE_SETUP_TOKEN_MODEL }} + OPENCLAW_LIVE_SETUP_TOKEN_PROFILE: ${{ secrets.OPENCLAW_LIVE_SETUP_TOKEN_PROFILE }} + OPENCLAW_LIVE_SETUP_TOKEN_VALUE: ${{ secrets.OPENCLAW_LIVE_SETUP_TOKEN_VALUE }} + GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} + GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + QWEN_API_KEY: ${{ secrets.QWEN_API_KEY }} + FAL_KEY: ${{ secrets.FAL_KEY }} + RUNWAY_API_KEY: ${{ secrets.RUNWAY_API_KEY }} + DEEPGRAM_API_KEY: ${{ secrets.DEEPGRAM_API_KEY }} + TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }} + VYDRA_API_KEY: ${{ secrets.VYDRA_API_KEY }} + XAI_API_KEY: ${{ secrets.XAI_API_KEY }} + ZAI_API_KEY: ${{ secrets.ZAI_API_KEY }} + Z_AI_API_KEY: ${{ secrets.Z_AI_API_KEY }} + BYTEPLUS_ACCESS_KEY_ID: ${{ secrets.BYTEPLUS_ACCESS_KEY_ID }} + BYTEPLUS_SECRET_ACCESS_KEY: ${{ secrets.BYTEPLUS_SECRET_ACCESS_KEY }} + CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + OPENCLAW_CODEX_AUTH_JSON: ${{ secrets.OPENCLAW_CODEX_AUTH_JSON }} + OPENCLAW_CODEX_CONFIG_TOML: ${{ secrets.OPENCLAW_CODEX_CONFIG_TOML }} + OPENCLAW_CLAUDE_JSON: ${{ secrets.OPENCLAW_CLAUDE_JSON }} + OPENCLAW_CLAUDE_CREDENTIALS_JSON: ${{ secrets.OPENCLAW_CLAUDE_CREDENTIALS_JSON }} + OPENCLAW_CLAUDE_SETTINGS_JSON: ${{ secrets.OPENCLAW_CLAUDE_SETTINGS_JSON }} + OPENCLAW_CLAUDE_SETTINGS_LOCAL_JSON: ${{ secrets.OPENCLAW_CLAUDE_SETTINGS_LOCAL_JSON }} + OPENCLAW_GEMINI_SETTINGS_JSON: ${{ secrets.OPENCLAW_GEMINI_SETTINGS_JSON }} + FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }} + OPENCLAW_LIVE_IMAGE: ${{ needs.prepare_live_test_image.outputs.live_image }} + OPENCLAW_SKIP_DOCKER_BUILD: "1" + OPENCLAW_LIVE_VIDEO_GENERATION_SKIP_PROVIDERS: "" + OPENCLAW_LIVE_VYDRA_VIDEO: "1" + OPENCLAW_VITEST_MAX_WORKERS: "2" + steps: + - name: Checkout selected ref + if: contains(matrix.profiles, inputs.release_test_profile) + uses: actions/checkout@v6 + with: + ref: ${{ needs.validate_selected_ref.outputs.selected_sha }} + fetch-depth: 1 + + - name: Checkout trusted live shard harness + if: contains(matrix.profiles, inputs.release_test_profile) + uses: actions/checkout@v6 + with: + ref: ${{ github.sha }} + fetch-depth: 1 + path: .release-harness + + - name: Setup Node environment + if: contains(matrix.profiles, inputs.release_test_profile) + uses: ./.github/actions/setup-node-env + with: + node-version: ${{ env.NODE_VERSION }} + pnpm-version: ${{ env.PNPM_VERSION }} + install-bun: "true" + + - name: Hydrate live auth/profile inputs + if: contains(matrix.profiles, inputs.release_test_profile) + run: bash scripts/ci-hydrate-live-auth.sh + + - name: Log in to GHCR + if: contains(matrix.profiles, inputs.release_test_profile) + uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ github.token }} + + - name: Configure suite-specific env + if: contains(matrix.profiles, inputs.release_test_profile) + shell: bash + run: | + set -euo pipefail + if [[ "${{ matrix.profile_env_only }}" == "true" ]]; then + echo "OPENCLAW_DOCKER_PROFILE_ENV_ONLY=1" >> "$GITHUB_ENV" + fi + case "${{ matrix.suite_id }}" in + live-cli-backend-docker) + echo "OPENCLAW_LIVE_CLI_BACKEND_MODEL=codex-cli/gpt-5.5" >> "$GITHUB_ENV" + echo "OPENCLAW_LIVE_CLI_BACKEND_AUTH=api-key" >> "$GITHUB_ENV" + echo 'OPENCLAW_LIVE_CLI_BACKEND_ARGS=["exec","--json","--color","never","--sandbox","danger-full-access","--skip-git-repo-check"]' >> "$GITHUB_ENV" + echo 'OPENCLAW_LIVE_CLI_BACKEND_RESUME_ARGS=["exec","resume","{sessionId}","-c","sandbox_mode=\"danger-full-access\"","--skip-git-repo-check"]' >> "$GITHUB_ENV" + echo "OPENCLAW_LIVE_CLI_BACKEND_DEBUG=1" >> "$GITHUB_ENV" + echo "OPENCLAW_CLI_BACKEND_LOG_OUTPUT=1" >> "$GITHUB_ENV" + echo "OPENCLAW_TEST_CONSOLE=1" >> "$GITHUB_ENV" + echo "OPENCLAW_LIVE_CLI_BACKEND_USE_CI_SAFE_CODEX_CONFIG=1" >> "$GITHUB_ENV" + ;; + live-codex-harness-docker) + echo "OPENCLAW_LIVE_CODEX_HARNESS_AUTH=api-key" >> "$GITHUB_ENV" + echo "OPENCLAW_LIVE_CODEX_HARNESS_DEBUG=1" >> "$GITHUB_ENV" + echo "OPENCLAW_CLI_BACKEND_LOG_OUTPUT=1" >> "$GITHUB_ENV" + echo "OPENCLAW_TEST_CONSOLE=1" >> "$GITHUB_ENV" + ;; + live-acp-bind-docker) + if [[ -n "${GEMINI_API_KEY:-}" || -n "${GOOGLE_API_KEY:-}" ]]; then + echo "OPENCLAW_LIVE_ACP_BIND_AGENTS=claude,codex,gemini" >> "$GITHUB_ENV" + else + echo "OPENCLAW_LIVE_ACP_BIND_AGENTS=claude,codex" >> "$GITHUB_ENV" + fi + ;; + esac + + - name: Run ${{ matrix.label }} + if: contains(matrix.profiles, inputs.release_test_profile) + run: ${{ matrix.command }} + validate_live_media_provider_suites: name: Live media suites (${{ matrix.label }}) needs: validate_selected_ref diff --git a/docs/ci.md b/docs/ci.md index 084774f39a7..7566ead195b 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -52,6 +52,14 @@ The native live media shards run in live suites on normal Blacksmith runners, because container jobs are the wrong place to launch nested Docker tests. +Docker-backed live model/backend shards use a separate shared +`ghcr.io/openclaw/openclaw-live-test:` image per selected commit. The live +release workflow builds and pushes that image once, then the Docker live model, +gateway, CLI backend, ACP bind, and Codex harness shards run with +`OPENCLAW_SKIP_DOCKER_BUILD=1`. If those shards rebuild the full source Docker +target independently, the release run is misconfigured and will waste the wall +clock on duplicate image builds. + `OpenClaw Release Checks` uses the trusted workflow ref to resolve the selected ref once into a `release-package-under-test` tarball, then passes that artifact to both the live/E2E release-path Docker workflow and the package acceptance @@ -216,9 +224,12 @@ manual dispatch; it fans out the mock parity gate, live Matrix lane, and live Telegram and Discord lanes as parallel jobs. The live jobs use the `qa-live-shared` environment, and Telegram/Discord use Convex leases. Release checks run Matrix and Telegram live transport lanes with the deterministic mock -provider so the channel contract is isolated from live model latency; provider -connectivity is covered by the separate live model, native provider, and Docker -provider suites. Matrix uses `--profile fast` for scheduled and release gates, +provider and mock-qualified models (`mock-openai/gpt-5.5` and +`mock-openai/gpt-5.5-alt`) so the channel contract is isolated from live model +latency and normal provider-plugin startup. The live transport gateway also +disables memory search because QA parity covers memory behavior separately; +provider connectivity is covered by the separate live model, native provider, +and Docker provider suites. Matrix uses `--profile fast` for scheduled and release gates, adding `--fail-fast` only when the checked-out CLI supports it. The CLI default and manual workflow input remain `all`; manual `matrix_profile=all` dispatch always shards full Matrix coverage into `transport`, `media`, diff --git a/docs/help/testing.md b/docs/help/testing.md index 6d51dd957cd..150caf95008 100644 --- a/docs/help/testing.md +++ b/docs/help/testing.md @@ -110,7 +110,17 @@ parallel jobs. Scheduled QA and release checks pass Matrix `--profile fast` explicitly, while the Matrix CLI and manual workflow input default remain `all`; manual dispatch can shard `all` into `transport`, `media`, `e2ee-smoke`, `e2ee-deep`, and `e2ee-cli` jobs. `OpenClaw Release Checks` runs parity plus -the fast Matrix and Telegram lanes before release approval. +the fast Matrix and Telegram lanes before release approval, using +`mock-openai/gpt-5.5` for release transport checks so they stay deterministic +and avoid normal provider-plugin startup. These live transport gateways disable +memory search; memory behavior stays covered by the QA parity suites. + +Full release live media shards use +`ghcr.io/openclaw/openclaw-live-media-runner:ubuntu-24.04`, which already has +`ffmpeg` and `ffprobe`. Docker live model/backend shards use the shared +`ghcr.io/openclaw/openclaw-live-test:` image built once per selected +commit, then pull it with `OPENCLAW_SKIP_DOCKER_BUILD=1` instead of rebuilding +inside every shard. - `pnpm openclaw qa suite` - Runs repo-backed QA scenarios directly on the host.