diff --git a/.github/actions/setup-node-env/action.yml b/.github/actions/setup-node-env/action.yml new file mode 100644 index 00000000000..71af716405c --- /dev/null +++ b/.github/actions/setup-node-env/action.yml @@ -0,0 +1,83 @@ +name: Setup Node environment +description: > + Checkout with submodule retry, install Node 22, pnpm, optionally Bun, + and run pnpm install. Used by CI gates and test jobs. +inputs: + node-version: + description: Node.js version to install. + required: false + default: "22.x" + pnpm-version: + description: pnpm version for corepack. + required: false + default: "10.23.0" + install-bun: + description: Whether to install Bun alongside Node. + required: false + default: "true" + frozen-lockfile: + description: Whether to use --frozen-lockfile for install. + required: false + default: "true" +runs: + using: composite + steps: + - name: Checkout submodules (retry) + shell: bash + run: | + set -euo pipefail + git submodule sync --recursive + for attempt in 1 2 3 4 5; do + if git -c protocol.version=2 submodule update --init --force --depth=1 --recursive; then + exit 0 + fi + echo "Submodule update failed (attempt $attempt/5). Retrying…" + sleep $((attempt * 10)) + done + exit 1 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: ${{ inputs.node-version }} + check-latest: true + + - name: Setup pnpm + cache store + uses: ./.github/actions/setup-pnpm-store-cache + with: + pnpm-version: ${{ inputs.pnpm-version }} + cache-key-suffix: "node22" + + - name: Setup Bun + if: inputs.install-bun == 'true' + uses: oven-sh/setup-bun@v2 + with: + bun-version: latest + + - name: Runtime versions + shell: bash + run: | + node -v + npm -v + pnpm -v + if command -v bun &>/dev/null; then bun -v; fi + + - name: Capture node path + shell: bash + run: echo "NODE_BIN=$(dirname "$(node -p "process.execPath")")" >> "$GITHUB_ENV" + + - name: Install dependencies + shell: bash + env: + CI: "true" + run: | + export PATH="$NODE_BIN:$PATH" + which node + node -v + pnpm -v + LOCKFILE_FLAG="" + if [ "${{ inputs.frozen-lockfile }}" = "true" ]; then + LOCKFILE_FLAG="--frozen-lockfile" + fi + pnpm install $LOCKFILE_FLAG --ignore-scripts=false --config.engine-strict=false --config.enable-pre-post-scripts=true || \ + pnpm install $LOCKFILE_FLAG --ignore-scripts=false --config.engine-strict=false --config.enable-pre-post-scripts=true diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9813f9a3a96..4a463de8fb4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -120,7 +120,7 @@ jobs: # Build dist once for Node-relevant changes and share it with downstream jobs. build-artifacts: - needs: [docs-scope, changed-scope, code-size, checks-lint] + needs: [docs-scope, changed-scope, code-size, check-lint] if: needs.docs-scope.outputs.docs_only != 'true' && (github.event_name == 'push' || needs.changed-scope.outputs.run_node == 'true') runs-on: blacksmith-4vcpu-ubuntu-2404 steps: @@ -129,49 +129,10 @@ jobs: with: submodules: false - - name: Checkout submodules (retry) - run: | - set -euo pipefail - git submodule sync --recursive - for attempt in 1 2 3 4 5; do - if git -c protocol.version=2 submodule update --init --force --depth=1 --recursive; then - exit 0 - fi - echo "Submodule update failed (attempt $attempt/5). Retrying…" - sleep $((attempt * 10)) - done - exit 1 - - - name: Setup Node.js - uses: actions/setup-node@v4 + - name: Setup Node environment + uses: ./.github/actions/setup-node-env with: - node-version: 22.x - check-latest: true - - - name: Setup pnpm + cache store - uses: ./.github/actions/setup-pnpm-store-cache - with: - pnpm-version: "10.23.0" - cache-key-suffix: "node22" - - - name: Runtime versions - run: | - node -v - npm -v - pnpm -v - - - name: Capture node path - run: echo "NODE_BIN=$(dirname \"$(node -p \"process.execPath\")\")" >> "$GITHUB_ENV" - - - name: Install dependencies (frozen) - env: - CI: true - run: | - export PATH="$NODE_BIN:$PATH" - which node - node -v - pnpm -v - pnpm install --frozen-lockfile --ignore-scripts=false --config.engine-strict=false --config.enable-pre-post-scripts=true + install-bun: "false" - name: Build dist run: pnpm build @@ -184,7 +145,7 @@ jobs: retention-days: 1 install-check: - needs: [docs-scope, changed-scope, code-size, checks-lint] + needs: [docs-scope, changed-scope, code-size, check-lint] if: needs.docs-scope.outputs.docs_only != 'true' && (github.event_name == 'push' || needs.changed-scope.outputs.run_node == 'true') runs-on: blacksmith-4vcpu-ubuntu-2404 steps: @@ -193,52 +154,13 @@ jobs: with: submodules: false - - name: Checkout submodules (retry) - run: | - set -euo pipefail - git submodule sync --recursive - for attempt in 1 2 3 4 5; do - if git -c protocol.version=2 submodule update --init --force --depth=1 --recursive; then - exit 0 - fi - echo "Submodule update failed (attempt $attempt/5). Retrying…" - sleep $((attempt * 10)) - done - exit 1 - - - name: Setup Node.js - uses: actions/setup-node@v4 + - name: Setup Node environment + uses: ./.github/actions/setup-node-env with: - node-version: 22.x - check-latest: true - - - name: Setup pnpm + cache store - uses: ./.github/actions/setup-pnpm-store-cache - with: - pnpm-version: "10.23.0" - cache-key-suffix: "node22" - - - name: Runtime versions - run: | - node -v - npm -v - pnpm -v - - - name: Capture node path - run: echo "NODE_BIN=$(dirname \"$(node -p \"process.execPath\")\")" >> "$GITHUB_ENV" - - - name: Install dependencies (frozen) - env: - CI: true - run: | - export PATH="$NODE_BIN:$PATH" - which node - node -v - pnpm -v - pnpm install --frozen-lockfile --ignore-scripts=false --config.engine-strict=false --config.enable-pre-post-scripts=true + install-bun: "false" checks: - needs: [docs-scope, changed-scope, code-size, checks-lint] + needs: [docs-scope, changed-scope, code-size, check-lint] if: needs.docs-scope.outputs.docs_only != 'true' && (github.event_name == 'push' || needs.changed-scope.outputs.run_node == 'true') runs-on: blacksmith-4vcpu-ubuntu-2404 strategy: @@ -263,134 +185,50 @@ jobs: with: submodules: false - - name: Checkout submodules (retry) - run: | - set -euo pipefail - git submodule sync --recursive - for attempt in 1 2 3 4 5; do - if git -c protocol.version=2 submodule update --init --force --depth=1 --recursive; then - exit 0 - fi - echo "Submodule update failed (attempt $attempt/5). Retrying…" - sleep $((attempt * 10)) - done - exit 1 - - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: 22.x - check-latest: true - - - name: Setup pnpm + cache store - uses: ./.github/actions/setup-pnpm-store-cache - with: - pnpm-version: "10.23.0" - cache-key-suffix: "node22" - - - name: Setup Bun - uses: oven-sh/setup-bun@v2 - with: - bun-version: latest - - - name: Runtime versions - run: | - node -v - npm -v - bun -v - pnpm -v - - - name: Capture node path - run: echo "NODE_BIN=$(dirname \"$(node -p \"process.execPath\")\")" >> "$GITHUB_ENV" - - - name: Install dependencies - env: - CI: true - run: | - export PATH="$NODE_BIN:$PATH" - which node - node -v - pnpm -v - pnpm install --frozen-lockfile --ignore-scripts=false --config.engine-strict=false --config.enable-pre-post-scripts=true || pnpm install --frozen-lockfile --ignore-scripts=false --config.engine-strict=false --config.enable-pre-post-scripts=true + - name: Setup Node environment + uses: ./.github/actions/setup-node-env - name: Run ${{ matrix.task }} (${{ matrix.runtime }}) run: ${{ matrix.command }} - # Lint and format always run, even on docs-only changes. - checks-lint: + # Format check — cheapest gate (~43s). Always runs, even on docs-only changes. + check-format: + name: "check: format" runs-on: blacksmith-4vcpu-ubuntu-2404 - strategy: - fail-fast: false - matrix: - include: - - task: lint - command: pnpm lint - - task: format - command: pnpm format steps: - name: Checkout uses: actions/checkout@v4 with: submodules: false - - name: Checkout submodules (retry) - run: | - set -euo pipefail - git submodule sync --recursive - for attempt in 1 2 3 4 5; do - if git -c protocol.version=2 submodule update --init --force --depth=1 --recursive; then - exit 0 - fi - echo "Submodule update failed (attempt $attempt/5). Retrying…" - sleep $((attempt * 10)) - done - exit 1 + - name: Setup Node environment + uses: ./.github/actions/setup-node-env - - name: Setup Node.js - uses: actions/setup-node@v4 + - name: Check formatting + run: pnpm format + + # Lint check — runs after format passes for cleaner output. + check-lint: + name: "check: lint" + needs: [check-format] + runs-on: blacksmith-4vcpu-ubuntu-2404 + steps: + - name: Checkout + uses: actions/checkout@v4 with: - node-version: 22.x - check-latest: true + submodules: false - - name: Setup pnpm + cache store - uses: ./.github/actions/setup-pnpm-store-cache - with: - pnpm-version: "10.23.0" - cache-key-suffix: "node22" + - name: Setup Node environment + uses: ./.github/actions/setup-node-env - - name: Setup Bun - uses: oven-sh/setup-bun@v2 - with: - bun-version: latest - - - name: Runtime versions - run: | - node -v - npm -v - bun -v - pnpm -v - - - name: Capture node path - run: echo "NODE_BIN=$(dirname \"$(node -p \"process.execPath\")\")" >> "$GITHUB_ENV" - - - name: Install dependencies - env: - CI: true - run: | - export PATH="$NODE_BIN:$PATH" - which node - node -v - pnpm -v - pnpm install --frozen-lockfile --ignore-scripts=false --config.engine-strict=false --config.enable-pre-post-scripts=true || pnpm install --frozen-lockfile --ignore-scripts=false --config.engine-strict=false --config.enable-pre-post-scripts=true - - - name: Run ${{ matrix.task }} - run: ${{ matrix.command }} + - name: Check lint + run: pnpm lint # Check for files that grew past LOC threshold in this PR (delta-only). # On push events, all steps are skipped and the job passes (no-op). # Heavy downstream jobs depend on this to fail fast on violations. code-size: - needs: [checks-lint] + needs: [check-format] runs-on: blacksmith-4vcpu-ubuntu-2404 steps: - name: Checkout @@ -444,7 +282,7 @@ jobs: fi checks-windows: - needs: [docs-scope, changed-scope, build-artifacts, code-size, checks-lint] + needs: [docs-scope, changed-scope, build-artifacts, code-size, check-lint] if: needs.docs-scope.outputs.docs_only != 'true' && (github.event_name == 'push' || needs.changed-scope.outputs.run_node == 'true') runs-on: blacksmith-4vcpu-windows-2025 env: @@ -565,7 +403,7 @@ jobs: # running 4 separate jobs per PR (as before) starved the queue. One job # per PR allows 5 PRs to run macOS checks simultaneously. macos: - needs: [docs-scope, changed-scope, code-size, checks-lint] + needs: [docs-scope, changed-scope, code-size, check-lint] if: github.event_name == 'pull_request' && needs.docs-scope.outputs.docs_only != 'true' && needs.changed-scope.outputs.run_macos == 'true' runs-on: macos-latest steps: @@ -574,50 +412,10 @@ jobs: with: submodules: false - - name: Checkout submodules (retry) - run: | - set -euo pipefail - git submodule sync --recursive - for attempt in 1 2 3 4 5; do - if git -c protocol.version=2 submodule update --init --force --depth=1 --recursive; then - exit 0 - fi - echo "Submodule update failed (attempt $attempt/5). Retrying…" - sleep $((attempt * 10)) - done - exit 1 - - # --- Node/pnpm setup (for TS tests) --- - - name: Setup Node.js - uses: actions/setup-node@v4 + - name: Setup Node environment + uses: ./.github/actions/setup-node-env with: - node-version: 22.x - check-latest: true - - - name: Setup pnpm + cache store - uses: ./.github/actions/setup-pnpm-store-cache - with: - pnpm-version: "10.23.0" - cache-key-suffix: "node22" - - - name: Runtime versions - run: | - node -v - npm -v - pnpm -v - - - name: Capture node path - run: echo "NODE_BIN=$(dirname \"$(node -p \"process.execPath\")\")" >> "$GITHUB_ENV" - - - name: Install dependencies - env: - CI: true - run: | - export PATH="$NODE_BIN:$PATH" - which node - node -v - pnpm -v - pnpm install --frozen-lockfile --ignore-scripts=false --config.engine-strict=false --config.enable-pre-post-scripts=true || pnpm install --frozen-lockfile --ignore-scripts=false --config.engine-strict=false --config.enable-pre-post-scripts=true + install-bun: "false" # --- Run all checks sequentially (fast gates first) --- - name: TS tests (macOS) @@ -843,7 +641,7 @@ jobs: PY android: - needs: [docs-scope, changed-scope, code-size, checks-lint] + needs: [docs-scope, changed-scope, code-size, check-lint] if: needs.docs-scope.outputs.docs_only != 'true' && (github.event_name == 'push' || needs.changed-scope.outputs.run_android == 'true') runs-on: blacksmith-4vcpu-ubuntu-2404 strategy: diff --git a/docs/ci.md b/docs/ci.md new file mode 100644 index 00000000000..da09a3f1b24 --- /dev/null +++ b/docs/ci.md @@ -0,0 +1,153 @@ +--- +title: CI Pipeline +description: How the OpenClaw CI pipeline works and why jobs are ordered the way they are. Latest changes: Feb 09, 2026 +--- + +# CI Pipeline + +OpenClaw uses a tiered CI pipeline that fails fast on cheap checks before +running expensive builds and tests. This saves runner minutes and reduces +GitHub API pressure. + +## Pipeline Tiers + +``` +Tier 0 — Scope detection (~12 s, free runners) + docs-scope → changed-scope + +Tier 1 — Cheapest gates (parallel, ~43 s) + check-format secrets + +Tier 2 — After format (parallel, ~2 min) + check-lint code-size + +Tier 3 — Build (~3 min) + build-artifacts install-check + +Tier 4 — Tests (~5 min) + checks (node tsgo / test / protocol, bun test) + checks-windows (lint / test / protocol) + +Tier 5 — Platform (most expensive) + macos (TS tests + Swift lint/build/test) + android (test + build) + ios (disabled) +``` + +## Dependency Graph + +``` +docs-scope ──► changed-scope ──┐ + │ +check-format ──► check-lint ──►├──► build-artifacts ──► checks-windows + ├─► code-size ──►├──► install-check + ├──► checks + ├──► macos + └──► android +secrets (independent) +``` + +## Job Details + +### Tier 0 — Scope Detection + +| Job | Runner | Purpose | +| --------------- | --------------- | ----------------------------------------------------------------------- | +| `docs-scope` | `ubuntu-latest` | Detects docs-only PRs to skip heavy jobs | +| `changed-scope` | `ubuntu-latest` | Detects which areas changed (node/macos/android) to skip unrelated jobs | + +### Tier 1 — Cheapest Gates + +| Job | Runner | Purpose | +| -------------- | ----------------- | ------------------------------------------- | +| `check-format` | Blacksmith 4 vCPU | Runs `pnpm format` — cheapest gate (~43 s) | +| `secrets` | Blacksmith 4 vCPU | Runs `detect-secrets` scan against baseline | + +### Tier 2 — After Format + +| Job | Runner | Depends on | Purpose | +| ------------ | ----------------- | -------------- | ----------------------------------------------------------- | +| `check-lint` | Blacksmith 4 vCPU | `check-format` | Runs `pnpm lint` — cleaner output after format passes | +| `code-size` | Blacksmith 4 vCPU | `check-format` | Checks LOC thresholds — accurate counts need formatted code | + +### Tier 3 — Build + +| Job | Runner | Depends on | Purpose | +| ----------------- | ----------------- | ------------------------- | ------------------------------------- | +| `build-artifacts` | Blacksmith 4 vCPU | `check-lint`, `code-size` | Builds dist and uploads artifact | +| `install-check` | Blacksmith 4 vCPU | `check-lint`, `code-size` | Verifies `pnpm install` works cleanly | + +### Tier 4+ — Tests and Platform + +| Job | Runner | Depends on | Purpose | +| ---------------- | ------------------ | -------------------------------------------- | ------------------------------------------------------ | +| `checks` | Blacksmith 4 vCPU | `check-lint`, `code-size` | TypeScript checks, tests (Node + Bun), protocol checks | +| `checks-windows` | Blacksmith Windows | `build-artifacts`, `check-lint`, `code-size` | Windows-specific lint, tests, protocol checks | +| `macos` | `macos-latest` | `check-lint`, `code-size` | TS tests + Swift lint/build/test (PR only) | +| `android` | Blacksmith 4 vCPU | `check-lint`, `code-size` | Gradle test + build | + +## Code-Size Gate + +The `code-size` job runs `scripts/analyze_code_files.py` on PRs to catch: + +1. **Threshold crossings** — files that grew past 1000 lines in the PR +2. **Already-large files growing** — files already over 1000 lines that got bigger +3. **Duplicate function regressions** — new duplicate functions introduced by the PR + +When `--strict` is set, any violation fails the job and blocks all downstream +work. On push to `main`, the code-size steps are skipped (the job passes as a +no-op) so pushes still run the full test suite. + +### Excluded Directories + +The analysis skips: `node_modules`, `dist`, `vendor`, `.git`, `coverage`, +`Swabble`, `skills`, `.pi` and other non-source directories. See the +`SKIP_DIRS` set in `scripts/analyze_code_files.py` for the full list. + +## Fail-Fast Behavior + +**Bad PR (formatting violations):** + +- `check-format` fails at ~43 s +- `check-lint`, `code-size`, and all downstream jobs never start +- Total cost: ~1 runner-minute + +**Bad PR (lint or LOC violations, good format):** + +- `check-format` passes → `check-lint` and `code-size` run in parallel +- One or both fail → all downstream jobs skipped +- Total cost: ~3 runner-minutes + +**Good PR:** + +- Critical path: `check-format` (43 s) → `check-lint` (1m 46 s) → `build-artifacts` → `checks` +- `code-size` runs in parallel with `check-lint`, adding no latency + +## Composite Action + +The `setup-node-env` composite action (`.github/actions/setup-node-env/`) +handles the shared setup boilerplate: + +- Submodule checkout with retry (5 attempts) +- Node.js 22 setup +- pnpm via corepack + store cache +- Optional Bun install +- `pnpm install` with retry + +This eliminates ~40 lines of duplicated YAML per job. + +## Push vs PR Behavior + +| Trigger | `code-size` | Downstream jobs | +| -------------- | ----------------------------- | --------------------- | +| Push to `main` | Steps skipped (job passes) | Run normally | +| Pull request | Full analysis with `--strict` | Blocked on violations | + +## Runners + +| Name | OS | vCPUs | Used by | +| ------------------------------- | ------------ | ----- | ---------------- | +| `blacksmith-4vcpu-ubuntu-2404` | Ubuntu 24.04 | 4 | Most jobs | +| `blacksmith-4vcpu-windows-2025` | Windows 2025 | 4 | `checks-windows` | +| `macos-latest` | macOS | — | `macos`, `ios` | +| `ubuntu-latest` | Ubuntu | 2 | Scope detection |