diff --git a/.github/workflows/openclaw-release-checks.yml b/.github/workflows/openclaw-release-checks.yml index 99b4900c5ca..f1abaae0b2c 100644 --- a/.github/workflows/openclaw-release-checks.yml +++ b/.github/workflows/openclaw-release-checks.yml @@ -32,6 +32,8 @@ concurrency: env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" + NODE_VERSION: "24.x" + PNPM_VERSION: "10.33.0" jobs: resolve_target: @@ -121,7 +123,7 @@ jobs: echo "- Validated SHA: \`${RELEASE_SHA}\`" echo "- Cross-OS provider: \`${RELEASE_PROVIDER}\`" echo "- Cross-OS mode: \`${RELEASE_MODE}\`" - echo "- This run will execute cross-OS release validation plus the non-Parallels Docker/live/openwebui coverage from the CI migration plan." + echo "- This run will execute cross-OS release validation, QA Lab parity/live lanes, and the non-Parallels Docker/live/openwebui coverage from the CI migration plan." } >> "$GITHUB_STEP_SUMMARY" cross_os_release_checks: @@ -197,3 +199,161 @@ jobs: OPENCLAW_CLAUDE_SETTINGS_JSON: ${{ secrets.OPENCLAW_CLAUDE_SETTINGS_JSON }} OPENCLAW_CLAUDE_SETTINGS_LOCAL_JSON: ${{ secrets.OPENCLAW_CLAUDE_SETTINGS_LOCAL_JSON }} OPENCLAW_GEMINI_SETTINGS_JSON: ${{ secrets.OPENCLAW_GEMINI_SETTINGS_JSON }} + + qa_lab_parity_release_checks: + name: Run QA Lab parity gate + needs: [resolve_target] + runs-on: blacksmith-32vcpu-ubuntu-2404 + timeout-minutes: 30 + permissions: + contents: read + env: + QA_PARITY_CONCURRENCY: "1" + OPENCLAW_QA_TRANSPORT_READY_TIMEOUT_MS: "180000" + OPENAI_API_KEY: "" + ANTHROPIC_API_KEY: "" + OPENCLAW_LIVE_OPENAI_KEY: "" + OPENCLAW_LIVE_ANTHROPIC_KEY: "" + OPENCLAW_LIVE_GEMINI_KEY: "" + OPENCLAW_LIVE_SETUP_TOKEN_VALUE: "" + OPENCLAW_BUILD_PRIVATE_QA: "1" + OPENCLAW_ENABLE_PRIVATE_QA_CLI: "1" + steps: + - name: Checkout selected ref + uses: actions/checkout@v6 + with: + ref: ${{ needs.resolve_target.outputs.ref }} + fetch-depth: 1 + + - name: Setup Node environment + uses: ./.github/actions/setup-node-env + with: + node-version: ${{ env.NODE_VERSION }} + pnpm-version: ${{ env.PNPM_VERSION }} + install-bun: "true" + + - name: Build private QA runtime + run: pnpm build + + - name: Run GPT-5.4 lane + run: | + pnpm openclaw qa suite \ + --provider-mode mock-openai \ + --parity-pack agentic \ + --concurrency "${QA_PARITY_CONCURRENCY}" \ + --model openai/gpt-5.4 \ + --alt-model openai/gpt-5.4-alt \ + --output-dir .artifacts/qa-e2e/gpt54 + + - name: Run Opus 4.6 lane + run: | + pnpm openclaw qa suite \ + --provider-mode mock-openai \ + --parity-pack agentic \ + --concurrency "${QA_PARITY_CONCURRENCY}" \ + --model anthropic/claude-opus-4-6 \ + --alt-model anthropic/claude-sonnet-4-6 \ + --output-dir .artifacts/qa-e2e/opus46 + + - name: Generate parity report + run: | + pnpm openclaw qa parity-report \ + --repo-root . \ + --candidate-summary .artifacts/qa-e2e/gpt54/qa-suite-summary.json \ + --baseline-summary .artifacts/qa-e2e/opus46/qa-suite-summary.json \ + --candidate-label openai/gpt-5.4 \ + --baseline-label anthropic/claude-opus-4-6 \ + --output-dir .artifacts/qa-e2e/parity + + - name: Upload parity artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: release-qa-parity-${{ needs.resolve_target.outputs.sha }} + path: .artifacts/qa-e2e/ + retention-days: 14 + if-no-files-found: warn + + qa_live_telegram_release_checks: + name: Run QA Lab live Telegram lane + needs: [resolve_target] + runs-on: blacksmith-32vcpu-ubuntu-2404 + timeout-minutes: 60 + permissions: + contents: read + pull-requests: read + environment: qa-live-shared + env: + OPENCLAW_BUILD_PRIVATE_QA: "1" + OPENCLAW_ENABLE_PRIVATE_QA_CLI: "1" + steps: + - name: Checkout selected ref + uses: actions/checkout@v6 + with: + ref: ${{ needs.resolve_target.outputs.ref }} + fetch-depth: 1 + + - name: Setup Node environment + uses: ./.github/actions/setup-node-env + with: + node-version: ${{ env.NODE_VERSION }} + pnpm-version: ${{ env.PNPM_VERSION }} + install-bun: "true" + + - name: Validate required QA credential env + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + OPENCLAW_QA_CONVEX_SITE_URL: ${{ secrets.OPENCLAW_QA_CONVEX_SITE_URL }} + OPENCLAW_QA_CONVEX_SECRET_CI: ${{ secrets.OPENCLAW_QA_CONVEX_SECRET_CI }} + shell: bash + run: | + set -euo pipefail + + require_var() { + local key="$1" + if [[ -z "${!key:-}" ]]; then + echo "Missing required ${key}." >&2 + exit 1 + fi + } + + require_var OPENAI_API_KEY + require_var OPENCLAW_QA_CONVEX_SITE_URL + require_var OPENCLAW_QA_CONVEX_SECRET_CI + + - name: Build private QA runtime + run: pnpm build + + - name: Run Telegram live lane + id: run_lane + shell: bash + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + OPENCLAW_QA_CONVEX_SITE_URL: ${{ secrets.OPENCLAW_QA_CONVEX_SITE_URL }} + OPENCLAW_QA_CONVEX_SECRET_CI: ${{ secrets.OPENCLAW_QA_CONVEX_SECRET_CI }} + OPENCLAW_QA_REDACT_PUBLIC_METADATA: "1" + OPENCLAW_QA_TELEGRAM_CAPTURE_CONTENT: "1" + run: | + set -euo pipefail + + output_dir=".artifacts/qa-e2e/telegram-live-release-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + echo "output_dir=${output_dir}" >> "$GITHUB_OUTPUT" + + pnpm openclaw qa telegram \ + --repo-root . \ + --output-dir "${output_dir}" \ + --provider-mode live-frontier \ + --model openai/gpt-5.4 \ + --alt-model openai/gpt-5.4 \ + --fast \ + --credential-source convex \ + --credential-role ci + + - name: Upload Telegram QA artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: release-qa-live-telegram-${{ needs.resolve_target.outputs.sha }} + path: ${{ steps.run_lane.outputs.output_dir }} + retention-days: 14 + if-no-files-found: warn diff --git a/.github/workflows/parity-gate.yml b/.github/workflows/parity-gate.yml index cf46c10b300..8faa935e5a1 100644 --- a/.github/workflows/parity-gate.yml +++ b/.github/workflows/parity-gate.yml @@ -13,6 +13,9 @@ on: - "src/gateway/**" - "src/media/**" - ".github/workflows/parity-gate.yml" + schedule: + - cron: "17 3 * * *" + workflow_dispatch: permissions: contents: read diff --git a/.github/workflows/qa-live-telegram-convex.yml b/.github/workflows/qa-live-telegram-convex.yml index fe333e3bb8f..bf3c17670d5 100644 --- a/.github/workflows/qa-live-telegram-convex.yml +++ b/.github/workflows/qa-live-telegram-convex.yml @@ -1,6 +1,8 @@ name: QA-Lab - Live Telegram, Live Frontier on: + schedule: + - cron: "41 4 * * *" workflow_dispatch: inputs: ref: @@ -18,7 +20,7 @@ permissions: pull-requests: read concurrency: - group: qa-lab-live-telegram-live-frontier-${{ inputs.ref }} + group: qa-lab-live-telegram-live-frontier-${{ github.event_name == 'workflow_dispatch' && inputs.ref || github.sha }} cancel-in-progress: false env: @@ -37,6 +39,10 @@ jobs: uses: actions/github-script@v8 with: script: | + if (context.eventName === "schedule") { + core.info("Scheduled default-branch QA run; actor permission check is only required for manual dispatch."); + return; + } const allowed = new Set(["admin", "maintain", "write"]); const { owner, repo } = context.repo; const { data } = await github.rest.repos.getCollaboratorPermissionLevel({ @@ -63,14 +69,14 @@ jobs: - name: Checkout selected ref uses: actions/checkout@v6 with: - ref: ${{ inputs.ref }} + ref: ${{ github.event_name == 'workflow_dispatch' && inputs.ref || github.sha }} fetch-depth: 0 - name: Validate selected ref id: validate env: GH_TOKEN: ${{ github.token }} - INPUT_REF: ${{ inputs.ref }} + INPUT_REF: ${{ github.event_name == 'workflow_dispatch' && inputs.ref || github.sha }} shell: bash run: | set -euo pipefail @@ -162,7 +168,7 @@ jobs: OPENCLAW_QA_CONVEX_SECRET_CI: ${{ secrets.OPENCLAW_QA_CONVEX_SECRET_CI }} OPENCLAW_QA_REDACT_PUBLIC_METADATA: "1" OPENCLAW_QA_TELEGRAM_CAPTURE_CONTENT: "1" - INPUT_SCENARIO: ${{ inputs.scenario }} + INPUT_SCENARIO: ${{ github.event_name == 'workflow_dispatch' && inputs.scenario || '' }} run: | set -euo pipefail diff --git a/docs/ci.md b/docs/ci.md index 95b631f4c5e..3080ccebb4a 100644 --- a/docs/ci.md +++ b/docs/ci.md @@ -10,6 +10,14 @@ read_when: The CI runs on every push to `main` and every pull request. It uses smart scoping to skip expensive jobs when only unrelated areas changed. +QA Lab has two dedicated CI lanes outside the main smart-scoped workflow. The +`Parity gate` workflow runs on matching PR changes, every night on `main`, and +manual dispatch; it builds the private QA runtime and compares the mock +GPT-5.4 and Opus 4.6 agentic packs. The `QA-Lab - Live Telegram, Live Frontier` +workflow runs nightly on `main` and on manual dispatch; it uses the +`qa-live-shared` environment plus Convex leases for the live Telegram lane. +`OpenClaw Release Checks` also runs both QA Lab lanes before release approval. + ## Job Overview | Job | Purpose | When it runs | diff --git a/docs/help/testing.md b/docs/help/testing.md index ea4a164d76a..0346b485ea2 100644 --- a/docs/help/testing.md +++ b/docs/help/testing.md @@ -51,6 +51,12 @@ Tip: when you only need one failing case, prefer narrowing live tests via the al These commands sit beside the main test suites when you need QA-lab realism: +CI runs QA Lab in dedicated workflows. `Parity gate` runs on matching PRs, +nightly on `main`, and from manual dispatch with mock providers. `QA-Lab - Live +Telegram, Live Frontier` runs nightly on `main` and from manual dispatch with +Convex-managed live Telegram credentials. `OpenClaw Release Checks` runs both +lanes before release approval. + - `pnpm openclaw qa suite` - Runs repo-backed QA scenarios directly on the host. - Runs multiple selected scenarios in parallel by default with isolated diff --git a/docs/reference/RELEASING.md b/docs/reference/RELEASING.md index a311ce425f1..b4ede21830b 100644 --- a/docs/reference/RELEASING.md +++ b/docs/reference/RELEASING.md @@ -54,6 +54,9 @@ OpenClaw has three public release lanes: - Run `pnpm release:check` before every tagged release - Release checks now run in a separate manual workflow: `OpenClaw Release Checks` +- `OpenClaw Release Checks` also runs the QA Lab mock parity gate and the live + Telegram QA lane before release approval. The live lane uses the + `qa-live-shared` environment and Convex CI credential leases. - Cross-OS install and upgrade runtime validation is dispatched from the private caller workflow `openclaw/releases-private/.github/workflows/openclaw-cross-os-release-checks.yml`, @@ -165,8 +168,8 @@ When cutting a stable npm release: 2. Choose `npm_dist_tag=beta` for the normal beta-first flow, or `latest` only when you intentionally want a direct stable publish 3. Run `OpenClaw Release Checks` separately with the same tag or the - full current workflow-branch commit SHA when you want live prompt cache - coverage + full current workflow-branch commit SHA when you want live prompt cache, + QA Lab parity, and live Telegram coverage - This is separate on purpose so live coverage stays available without recoupling long-running or flaky checks to the publish workflow 4. Save the successful `preflight_run_id`