diff --git a/.crabbox.yaml b/.crabbox.yaml index 92d69061258..796736b2f69 100644 --- a/.crabbox.yaml +++ b/.crabbox.yaml @@ -18,6 +18,9 @@ capacity: - us-west-2 actions: workflow: .github/workflows/crabbox-hydrate.yml + # Default AWS hydration uses local Actions replay. Use + # `crabbox actions hydrate --github-runner --job hydrate-github` when the + # hydrate job needs GitHub secrets. job: hydrate ref: main runnerLabels: diff --git a/.github/workflows/crabbox-hydrate.yml b/.github/workflows/crabbox-hydrate.yml index 33dcc2d0b0e..c505e708168 100644 --- a/.github/workflows/crabbox-hydrate.yml +++ b/.github/workflows/crabbox-hydrate.yml @@ -41,6 +41,232 @@ env: jobs: hydrate: name: hydrate + if: ${{ inputs.crabbox_job != 'hydrate-github' }} + runs-on: [self-hosted, "${{ inputs.crabbox_runner_label }}"] + timeout-minutes: 120 + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ inputs.ref || github.ref }} + + - name: Setup Node.js + uses: actions/setup-node@v6 + with: + node-version: "24" + + - name: Setup pnpm and dependencies + shell: bash + env: + CI: "true" + run: | + set -euo pipefail + + export XDG_CACHE_HOME="${XDG_CACHE_HOME:-$RUNNER_TEMP/cache}" + export COREPACK_HOME="${COREPACK_HOME:-$XDG_CACHE_HOME/corepack}" + export PNPM_HOME="${PNPM_HOME:-$RUNNER_TEMP/pnpm-home}" + mkdir -p "$XDG_CACHE_HOME" "$COREPACK_HOME" "$PNPM_HOME" + export PATH="$PNPM_HOME:$PATH" + { + echo "XDG_CACHE_HOME=$XDG_CACHE_HOME" + echo "COREPACK_HOME=$COREPACK_HOME" + echo "PNPM_HOME=$PNPM_HOME" + } >> "$GITHUB_ENV" + + corepack enable + node_bin="$(dirname "$(node -p 'process.execPath')")" + echo "NODE_BIN=$node_bin" >> "$GITHUB_ENV" + echo "$node_bin" >> "$GITHUB_PATH" + export PATH="$node_bin:$PATH" + + node -v + npm -v + pnpm -v + + install_args=( + install + --prefer-offline + --ignore-scripts=false + --config.engine-strict=false + --config.enable-pre-post-scripts=true + --config.side-effects-cache=true + --frozen-lockfile + ) + append_pnpm_option_arg() { + local env_name="$1" + local option_name="$2" + local value="${!env_name-}" + if [ -n "$value" ]; then + install_args+=("--${option_name}=${value}") + fi + } + append_pnpm_option_arg PNPM_CONFIG_CHILD_CONCURRENCY child-concurrency + append_pnpm_option_arg PNPM_CONFIG_MODULES_DIR modules-dir + append_pnpm_option_arg PNPM_CONFIG_NETWORK_CONCURRENCY network-concurrency + append_pnpm_option_arg PNPM_CONFIG_VIRTUAL_STORE_DIR virtual-store-dir + if [ -n "${PNPM_CONFIG_MODULES_DIR:-}" ]; then + mkdir -p "$PNPM_CONFIG_MODULES_DIR" + ln -sfn . "$PNPM_CONFIG_MODULES_DIR/node_modules" + fi + pnpm "${install_args[@]}" || pnpm "${install_args[@]}" + if [ -n "${PNPM_CONFIG_MODULES_DIR:-}" ]; then + rm -rf node_modules + ln -sfn "$PNPM_CONFIG_MODULES_DIR" node_modules + ln -sfn . "$PNPM_CONFIG_MODULES_DIR/node_modules" + fi + + - name: Prepare Crabbox shell + shell: bash + run: | + set -euo pipefail + + if git rev-parse --is-inside-work-tree >/dev/null 2>&1; then + git fetch --no-tags --depth=50 origin "+refs/heads/main:refs/remotes/origin/main" + fi + + node_bin="$(dirname "$(node -p 'process.execPath')")" + sudo ln -sf "$node_bin/node" /usr/local/bin/node + sudo ln -sf "$node_bin/npm" /usr/local/bin/npm + sudo ln -sf "$node_bin/npx" /usr/local/bin/npx + sudo ln -sf "$node_bin/corepack" /usr/local/bin/corepack + sudo tee /usr/local/bin/pnpm >/dev/null <<'PNPM' + #!/usr/bin/env bash + exec /usr/local/bin/corepack pnpm "$@" + PNPM + sudo chmod 0755 /usr/local/bin/pnpm + + - name: Ensure Docker is running + shell: bash + run: | + set -euo pipefail + + if ! command -v docker >/dev/null 2>&1; then + echo "docker not found; installing fallback engine" + curl -fsSL https://get.docker.com | sudo sh + fi + + if command -v systemctl >/dev/null 2>&1; then + sudo systemctl start docker || true + elif command -v service >/dev/null 2>&1; then + sudo service docker start || true + fi + + if [ -S /var/run/docker.sock ]; then + sudo usermod -aG docker "$USER" || true + # The runner process keeps its original groups; grant this + # ephemeral runner session access without requiring a relogin. + sudo chmod 666 /var/run/docker.sock + fi + + if ! docker buildx version >/dev/null 2>&1; then + arch="$(uname -m)" + case "$arch" in + aarch64|arm64) buildx_arch=arm64 ;; + x86_64|amd64) buildx_arch=amd64 ;; + *) echo "unsupported buildx arch: $arch" >&2; exit 2 ;; + esac + buildx_version="${DOCKER_BUILDX_VERSION:-v0.15.1}" + mkdir -p "$HOME/.docker/cli-plugins" + curl -fsSL \ + "https://github.com/docker/buildx/releases/download/${buildx_version}/buildx-${buildx_version}.linux-${buildx_arch}" \ + -o "$HOME/.docker/cli-plugins/docker-buildx" + chmod 0755 "$HOME/.docker/cli-plugins/docker-buildx" + fi + + docker version + docker buildx version + docker compose version || true + + - name: Ensure SSH is available + shell: bash + run: | + set -euo pipefail + if command -v systemctl >/dev/null 2>&1; then + sudo systemctl start ssh || sudo systemctl start sshd || true + elif command -v service >/dev/null 2>&1; then + sudo service ssh start || sudo service sshd start || true + fi + + - name: Hydrate provider env helper + shell: bash + run: bash scripts/ci-hydrate-testbox-env.sh + + - name: Mark Crabbox ready + shell: bash + env: + CRABBOX_ID: ${{ inputs.crabbox_id }} + CRABBOX_JOB: ${{ inputs.crabbox_job }} + run: | + set -euo pipefail + job="${CRABBOX_JOB}" + if [ -z "$job" ]; then job=hydrate; fi + case "$CRABBOX_ID" in + ''|*[!A-Za-z0-9._-]*) + echo "Invalid crabbox_id" >&2 + exit 2 + ;; + esac + mkdir -p "$HOME/.crabbox/actions" + state="$HOME/.crabbox/actions/${CRABBOX_ID}.env" + env_file="$HOME/.crabbox/actions/${CRABBOX_ID}.env.sh" + services_file="$HOME/.crabbox/actions/${CRABBOX_ID}.services" + write_export() { + key="$1" + value="${!key-}" + if [ -n "$value" ]; then + printf 'export %s=%q\n' "$key" "$value" + fi + } + { + for key in CI GITHUB_ACTIONS GITHUB_WORKSPACE GITHUB_REPOSITORY GITHUB_RUN_ID GITHUB_RUN_NUMBER GITHUB_RUN_ATTEMPT GITHUB_REF GITHUB_REF_NAME GITHUB_SHA GITHUB_EVENT_NAME GITHUB_ACTOR RUNNER_OS RUNNER_ARCH RUNNER_TEMP RUNNER_TOOL_CACHE XDG_CACHE_HOME COREPACK_HOME PNPM_HOME PNPM_CONFIG_CHILD_CONCURRENCY PNPM_CONFIG_MODULES_DIR PNPM_CONFIG_NETWORK_CONCURRENCY PNPM_CONFIG_STORE_DIR PNPM_CONFIG_VERIFY_DEPS_BEFORE_RUN PNPM_CONFIG_VIRTUAL_STORE_DIR; do + write_export "$key" + done + } > "${env_file}.tmp" + mv "${env_file}.tmp" "$env_file" + { + echo "# Docker containers visible from the hydrated runner" + docker ps --format '{{.Names}}\t{{.Image}}\t{{.Ports}}' 2>/dev/null || true + } > "${services_file}.tmp" + mv "${services_file}.tmp" "$services_file" + tmp="${state}.tmp" + { + echo "WORKSPACE=${GITHUB_WORKSPACE}" + echo "RUN_ID=${GITHUB_RUN_ID}" + echo "JOB=${job}" + echo "ENV_FILE=${env_file}" + echo "SERVICES_FILE=${services_file}" + echo "READY_AT=$(date -u +%Y-%m-%dT%H:%M:%SZ)" + } > "$tmp" + mv "$tmp" "$state" + + - name: Keep Crabbox job alive + shell: bash + env: + CRABBOX_ID: ${{ inputs.crabbox_id }} + CRABBOX_KEEP_ALIVE_MINUTES: ${{ inputs.crabbox_keep_alive_minutes }} + run: | + set -euo pipefail + case "$CRABBOX_ID" in + ''|*[!A-Za-z0-9._-]*) + echo "Invalid crabbox_id" >&2 + exit 2 + ;; + esac + minutes="${CRABBOX_KEEP_ALIVE_MINUTES}" + case "$minutes" in + ''|*[!0-9]*) minutes=90 ;; + esac + stop="$HOME/.crabbox/actions/${CRABBOX_ID}.stop" + deadline=$(( $(date +%s) + minutes * 60 )) + while [ "$(date +%s)" -lt "$deadline" ]; do + if [ -f "$stop" ]; then + exit 0 + fi + sleep 15 + done + + hydrate-github: + name: hydrate-github + if: ${{ inputs.crabbox_job == 'hydrate-github' }} runs-on: [self-hosted, "${{ inputs.crabbox_runner_label }}"] timeout-minutes: 120 steps: @@ -161,7 +387,7 @@ jobs: run: | set -euo pipefail job="${CRABBOX_JOB}" - if [ -z "$job" ]; then job=hydrate; fi + if [ -z "$job" ]; then job=hydrate-github; fi case "$CRABBOX_ID" in ''|*[!A-Za-z0-9._-]*) echo "Invalid crabbox_id" >&2 diff --git a/CHANGELOG.md b/CHANGELOG.md index fe3ed02e04a..73e83269d61 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -59,6 +59,7 @@ Docs: https://docs.openclaw.ai - Providers/Anthropic: migrate 1M context handling to GA-capable Claude 4.x models by sizing eligible models at 1M without the retired `context-1m-2025-08-07` beta, ignoring that retired beta in older configs, and preserving OAuth-required Anthropic beta headers. (#45613) Thanks @haoyu-haoyu. - Twitch: keep stale message-handler cleanup callbacks from removing newer handler registrations for the same account, preserving inbound message delivery after reconnects. Fixes #83888. (#85425) Thanks @alkor2000. - Memory/LanceDB: expose public memory artifacts through the active memory provider bridge so memory-wiki imports durable memory files, daily notes, dream reports, and event logs without depending on memory-core internals. Fixes #83604. (#85060) Thanks @brokemac79. +- Crabbox: keep AWS hydration compatible with local Actions replay by inlining the hydrate workflow's Node/pnpm setup instead of invoking repo-local composite actions. - Docker setup: stop printing the Gateway bearer token in setup logs and printed follow-up commands. - Agents: let embedded compaction fallback retries proceed when PI-compatible candidates do not need agent harness plugin preparation. - Agents/tools: honor configured custom provider API keys when deciding whether media, image-generation, video-generation, music-generation, and PDF tools are available. (#85570) diff --git a/test/scripts/package-acceptance-workflow.test.ts b/test/scripts/package-acceptance-workflow.test.ts index 796b094c37b..8ad12e06f07 100644 --- a/test/scripts/package-acceptance-workflow.test.ts +++ b/test/scripts/package-acceptance-workflow.test.ts @@ -15,6 +15,7 @@ const QA_LIVE_TRANSPORTS_WORKFLOW = ".github/workflows/qa-live-transports-convex const UPDATE_MIGRATION_WORKFLOW = ".github/workflows/update-migration.yml"; const CI_CHECK_TESTBOX_WORKFLOW = ".github/workflows/ci-check-testbox.yml"; const CRABBOX_HYDRATE_WORKFLOW = ".github/workflows/crabbox-hydrate.yml"; +const CRABBOX_CONFIG = ".crabbox.yaml"; const SCHEDULED_LIVE_CHECKS_WORKFLOW = ".github/workflows/openclaw-scheduled-live-checks.yml"; const CI_HYDRATE_LIVE_AUTH_SCRIPT = "scripts/ci-hydrate-live-auth.sh"; const UPGRADE_SURVIVOR_RUN_SCRIPT = "scripts/e2e/lib/upgrade-survivor/run.sh"; @@ -105,6 +106,32 @@ describe("package acceptance workflow", () => { } }); + it("keeps Crabbox hydration compatible with local Actions replay", () => { + const crabboxConfig = parse(readFileSync(CRABBOX_CONFIG, "utf8")) as { + actions?: { job?: string }; + }; + const workflow = readWorkflow(CRABBOX_HYDRATE_WORKFLOW); + const hydrate = workflowJob(CRABBOX_HYDRATE_WORKFLOW, "hydrate"); + const hydrateGithub = workflowJob(CRABBOX_HYDRATE_WORKFLOW, "hydrate-github"); + + expect(crabboxConfig.actions?.job).toBe("hydrate"); + expect(hydrate.if).toBe("${{ inputs.crabbox_job != 'hydrate-github' }}"); + expect(workflowStep(hydrate, "Setup Node.js").uses).toBe("actions/setup-node@v6"); + expect(workflowStep(hydrate, "Setup Node.js").with?.["node-version"]).toBe("24"); + expect(workflowStep(hydrate, "Setup pnpm and dependencies").run).toContain("corepack enable"); + expect(workflowStep(hydrate, "Setup pnpm and dependencies").run).toContain("COREPACK_HOME"); + expect(workflowStep(hydrate, "Mark Crabbox ready").run).toContain("COREPACK_HOME"); + expect(workflowStep(hydrate, "Hydrate provider env helper").env).toBeUndefined(); + + expect(hydrateGithub.if).toBe("${{ inputs.crabbox_job == 'hydrate-github' }}"); + expect(workflowStep(hydrateGithub, "Setup Node environment").uses).toBe( + "./.github/actions/setup-node-env", + ); + expect(workflowStep(hydrateGithub, "Hydrate provider env helper").env?.FACTORY_API_KEY).toBe( + "${{ secrets.FACTORY_API_KEY }}", + ); + }); + it("resolves candidate package sources before reusing Docker E2E lanes", () => { const workflow = readFileSync(PACKAGE_ACCEPTANCE_WORKFLOW, "utf8"); @@ -639,7 +666,6 @@ describe("package artifact reuse", () => { const scheduledWorkflow = readFileSync(SCHEDULED_LIVE_CHECKS_WORKFLOW, "utf8"); const packageAcceptanceWorkflow = readFileSync(PACKAGE_ACCEPTANCE_WORKFLOW, "utf8"); const testboxWorkflow = readFileSync(CI_CHECK_TESTBOX_WORKFLOW, "utf8"); - const crabboxHydrateWorkflow = readFileSync(CRABBOX_HYDRATE_WORKFLOW, "utf8"); const dockerPlanAction = readFileSync(DOCKER_E2E_PLAN_ACTION, "utf8"); const hydrateScript = readFileSync(CI_HYDRATE_LIVE_AUTH_SCRIPT, "utf8"); @@ -665,7 +691,6 @@ describe("package artifact reuse", () => { scheduledWorkflow, packageAcceptanceWorkflow, testboxWorkflow, - crabboxHydrateWorkflow, ]) { expect(workflow).toContain("FACTORY_API_KEY: ${{ secrets.FACTORY_API_KEY }}"); }