From 361750775dc01a5ceeca38f8fc83ad52ccb8d5a9 Mon Sep 17 00:00:00 2001 From: Onur Date: Sat, 18 Apr 2026 03:18:12 +0200 Subject: [PATCH] CI: stabilize live release lanes (#67838) * CI: stabilize live release lanes * CI: widen codex live exclusions * Gateway: stop live config/auth lazy re-imports * CI: mount writable live Docker homes * Live: tighten retry and provider filter overrides * CI: use API-key auth for codex live lanes * CI: fix remaining live lanes * CI: stop forwarding live OpenAI base URLs * Gateway: fix live startup loader regression * CI: stop expanding OpenAI keys in live Docker lanes * CI: stop expanding installer secrets in Docker * CI: tighten live secret boundaries * Gateway: pin Codex harness base URL * CI: fix reusable workflow runner label * CI: avoid template expansion in live ref guard * CI: tighten live trust gate * Gateway: ignore empty Codex harness base URL * CI: stabilize remaining live lanes * CI: harden live retries and canvas auth test * CI: extend cron live probe budget * CI: keep codex harness lane on api-key auth * CI: stage live Docker OpenAI auth via env files * CI: bootstrap codex login for Docker API-key lanes * CI: accept hosted-runner codex fallback responses * CI: accept additional codex sandbox fallback text * CI: accept hosted-runner live fallback variants * CI: accept codex current-model fallback * CI: broaden codex sandbox model fallbacks * CI: cover extra codex sandbox wording * CI: extend cli backend cron retry budget * CI: match codex models fallbacks by predicate * CI: accept configured-models live fallback * CI: relax OpenAI websocket warmup timeout * CI: accept extra codex model fallback wording * CI: generalize codex model fallback matching * CI: retry cron verify cancellation wording * CI: accept interactive codex model entrypoint fallback * Agents: stabilize Claude bundle skill command test * CI: prestage live Docker auth homes * Tests: accept current Codex models wording * CI: stabilize remaining live lanes * Tests: widen CLI backend live timeout * Tests: accept current Codex model summary wording * CI: disable codex-cli image probe in Docker lane * Tests: respect CLI override for Codex Docker login * Tests: accept current Codex session models header * CI: stabilize remaining live validation lanes * CI: preserve Gemini ACP coverage in auth fallback * CI: fix final live validation blockers * CI: restore Codex auth for CLI backend lane * CI: drop local Codex config in live Docker lane * Tests: tolerate Codex cron and model reply drift * Tests: accept current Codex live replies * Tests: retry more Codex cron retry wording * Tests: accept environment-cancelled Codex cron retries * Tests: retry blank Codex cron probe replies * Tests: broaden Codex cron retry wording * Tests: require explicit Codex cron retry replies * Tests: accept current Codex models environment wording * CI: restore trusted Codex config in live lane * CI: bypass nested Codex sandbox in docker * CI: instrument live codex cron lane * CI: forward live CLI resume args * Tests: accept interactive Codex model selection * Tests: bound websocket warm-up live lane * CI: close live lane review gaps * Tests: lazy-load gateway live server * Tests: avoid gateway live loader regression * CI: scope reusable workflow secrets * Tests: tighten codex models live assertion * Tests: normalize OpenAI speech live text --- .../openclaw-live-and-e2e-checks-reusable.yml | 103 +++++- .github/workflows/openclaw-release-checks.yml | 54 ++- .../openclaw-scheduled-live-checks.yml | 47 ++- extensions/openai/openai.live.test.ts | 3 +- scripts/lib/live-docker-auth.sh | 64 +++- scripts/prepare-codex-ci-config.ts | 51 +++ scripts/test-install-sh-e2e-docker.sh | 6 +- scripts/test-live-acp-bind-docker.sh | 65 ++-- scripts/test-live-cli-backend-docker.sh | 100 ++++-- scripts/test-live-codex-harness-docker.sh | 93 ++++- scripts/test-live-gateway-models-docker.sh | 58 +-- scripts/test-live-models-docker.sh | 58 +-- src/agents/cli-runner/execute.ts | 18 + src/agents/live-model-filter.test.ts | 72 ++++ src/agents/live-model-filter.ts | 74 ++++ src/agents/models.profiles.live.test.ts | 12 + src/agents/openai-ws-stream.e2e.test.ts | 17 +- src/agents/skills.test.ts | 2 +- .../gateway-cli-backend.live-helpers.test.ts | 102 ++++++ .../gateway-cli-backend.live-helpers.ts | 332 +++++++++++++++++- src/gateway/gateway-cli-backend.live.test.ts | 34 +- ...gateway-codex-harness.live-helpers.test.ts | 50 +++ .../gateway-codex-harness.live-helpers.ts | 95 +++++ .../gateway-codex-harness.live.test.ts | 43 ++- .../gateway-models.profiles.live.test.ts | 53 ++- src/gateway/live-agent-probes.test.ts | 12 +- src/gateway/live-agent-probes.ts | 15 +- src/gateway/mcp-http.request.ts | 41 +++ src/gateway/mcp-http.ts | 41 +++ src/gateway/server.canvas-auth.test.ts | 2 +- src/scripts/prepare-codex-ci-config.test.ts | 49 +++ .../test-live-cli-backend-docker.test.ts | 22 ++ 32 files changed, 1598 insertions(+), 190 deletions(-) create mode 100644 scripts/prepare-codex-ci-config.ts create mode 100644 src/agents/live-model-filter.test.ts create mode 100644 src/gateway/gateway-codex-harness.live-helpers.test.ts create mode 100644 src/gateway/gateway-codex-harness.live-helpers.ts create mode 100644 src/scripts/prepare-codex-ci-config.test.ts create mode 100644 test/scripts/test-live-cli-backend-docker.test.ts diff --git a/.github/workflows/openclaw-live-and-e2e-checks-reusable.yml b/.github/workflows/openclaw-live-and-e2e-checks-reusable.yml index 7bd48e8270a..49e7cfd65eb 100644 --- a/.github/workflows/openclaw-live-and-e2e-checks-reusable.yml +++ b/.github/workflows/openclaw-live-and-e2e-checks-reusable.yml @@ -144,6 +144,7 @@ on: permissions: contents: read + pull-requests: read env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" @@ -151,7 +152,63 @@ env: PNPM_VERSION: "10.32.1" jobs: + validate_selected_ref: + runs-on: blacksmith-8vcpu-ubuntu-2404 + outputs: + selected_sha: ${{ steps.validate.outputs.selected_sha }} + trusted_reason: ${{ steps.validate.outputs.trusted_reason }} + steps: + - name: Checkout selected ref + uses: actions/checkout@v6 + with: + ref: ${{ inputs.ref }} + fetch-depth: 0 + + - name: Validate selected ref + id: validate + env: + GH_TOKEN: ${{ github.token }} + INPUT_REF: ${{ inputs.ref }} + shell: bash + run: | + set -euo pipefail + selected_sha="$(git rev-parse HEAD)" + trusted_reason="" + + git fetch --no-tags origin +refs/heads/main:refs/remotes/origin/main + + if git merge-base --is-ancestor "$selected_sha" refs/remotes/origin/main; then + trusted_reason="main-ancestor" + elif git tag --points-at "$selected_sha" | grep -Eq '^v'; then + trusted_reason="release-tag" + else + pr_head_count="$( + gh api \ + -H "Accept: application/vnd.github+json" \ + "repos/${GITHUB_REPOSITORY}/commits/${selected_sha}/pulls" \ + --jq '[.[] | select(.state == "open" and .head.repo.full_name == "'"${GITHUB_REPOSITORY}"'" and .head.sha == "'"${selected_sha}"'")] | length' + )" + if [[ "$pr_head_count" != "0" ]]; then + trusted_reason="open-pr-head" + fi + fi + + if [[ -z "$trusted_reason" ]]; then + echo "Ref '${INPUT_REF}' resolved to $selected_sha, which is not trusted for secret-bearing live/E2E checks." >&2 + echo "Allowed refs must be on main, point to a release tag, or match an open PR head in ${GITHUB_REPOSITORY}." >&2 + exit 1 + fi + + echo "selected_sha=$selected_sha" >> "$GITHUB_OUTPUT" + echo "trusted_reason=$trusted_reason" >> "$GITHUB_OUTPUT" + { + echo "Validated ref: \`${INPUT_REF}\`" + echo "Resolved SHA: \`$selected_sha\`" + echo "Trust reason: \`$trusted_reason\`" + } >> "$GITHUB_STEP_SUMMARY" + validate_release_live_cache: + needs: validate_selected_ref if: inputs.include_live_suites runs-on: blacksmith-32vcpu-ubuntu-2404 timeout-minutes: 60 @@ -164,7 +221,7 @@ jobs: - name: Checkout selected ref uses: actions/checkout@v6 with: - ref: ${{ inputs.ref }} + ref: ${{ needs.validate_selected_ref.outputs.selected_sha }} fetch-depth: 0 - name: Setup Node environment @@ -191,6 +248,7 @@ jobs: run: pnpm test:live:cache validate_repo_e2e: + needs: validate_selected_ref if: inputs.include_repo_e2e runs-on: blacksmith-32vcpu-ubuntu-2404 timeout-minutes: 90 @@ -200,7 +258,7 @@ jobs: - name: Checkout selected ref uses: actions/checkout@v6 with: - ref: ${{ inputs.ref }} + ref: ${{ needs.validate_selected_ref.outputs.selected_sha }} fetch-depth: 0 - name: Setup Node environment @@ -218,6 +276,7 @@ jobs: run: pnpm test:e2e validate_special_e2e: + needs: validate_selected_ref if: inputs.include_repo_e2e || inputs.include_live_suites runs-on: blacksmith-32vcpu-ubuntu-2404 timeout-minutes: ${{ matrix.timeout_minutes }} @@ -245,7 +304,7 @@ jobs: - name: Checkout selected ref uses: actions/checkout@v6 with: - ref: ${{ inputs.ref }} + ref: ${{ needs.validate_selected_ref.outputs.selected_sha }} fetch-depth: 0 - name: Setup Node environment @@ -293,6 +352,7 @@ jobs: run: ${{ matrix.command }} validate_docker_e2e: + needs: validate_selected_ref if: inputs.include_release_path_suites || inputs.include_openwebui runs-on: blacksmith-32vcpu-ubuntu-2404 timeout-minutes: ${{ matrix.timeout_minutes }} @@ -396,7 +456,7 @@ jobs: - name: Checkout selected ref uses: actions/checkout@v6 with: - ref: ${{ inputs.ref }} + ref: ${{ needs.validate_selected_ref.outputs.selected_sha }} fetch-depth: 0 - name: Setup Node environment @@ -450,6 +510,7 @@ jobs: run: ${{ matrix.command }} validate_live_provider_suites: + needs: validate_selected_ref if: inputs.include_live_suites runs-on: blacksmith-32vcpu-ubuntu-2404 timeout-minutes: ${{ matrix.timeout_minutes }} @@ -538,7 +599,7 @@ jobs: - name: Checkout selected ref uses: actions/checkout@v6 with: - ref: ${{ inputs.ref }} + ref: ${{ needs.validate_selected_ref.outputs.selected_sha }} fetch-depth: 0 - name: Setup Node environment @@ -562,9 +623,39 @@ jobs: case "${{ matrix.suite_id }}" in live-cli-backend-docker) echo "OPENCLAW_LIVE_CLI_BACKEND_MODEL=codex-cli/gpt-5.4" >> "$GITHUB_ENV" + # The CLI backend Docker lane should exercise the same staged + # Codex auth path Peter uses locally so MCP cron creation and + # multimodal probes stay covered in CI. Replace the staged + # config.toml with a minimal CI-safe config so the repo stays + # trusted for MCP/tool use without inheriting maintainer-local + # provider/profile overrides that do not exist inside CI. + # Codex's workspace-write sandbox relies on user namespaces that + # this Docker lane does not provide, so run Codex unsandboxed + # inside the already-isolated container to keep MCP cron/tool + # execution representative instead of failing on nested sandbox + # setup. + echo 'OPENCLAW_LIVE_CLI_BACKEND_CLEAR_ENV=["OPENAI_API_KEY","OPENAI_BASE_URL"]' >> "$GITHUB_ENV" + echo 'OPENCLAW_LIVE_CLI_BACKEND_ARGS=["exec","--json","--color","never","--sandbox","danger-full-access","--skip-git-repo-check"]' >> "$GITHUB_ENV" + echo 'OPENCLAW_LIVE_CLI_BACKEND_RESUME_ARGS=["exec","resume","{sessionId}","-c","sandbox_mode=\"danger-full-access\"","--skip-git-repo-check"]' >> "$GITHUB_ENV" + echo "OPENCLAW_LIVE_CLI_BACKEND_DEBUG=1" >> "$GITHUB_ENV" + echo "OPENCLAW_CLI_BACKEND_LOG_OUTPUT=1" >> "$GITHUB_ENV" + echo "OPENCLAW_LIVE_CLI_BACKEND_USE_CI_SAFE_CODEX_CONFIG=1" >> "$GITHUB_ENV" + ;; + live-codex-harness-docker) + # Keep CI on the API-key path for now. The staged Codex auth secret + # is currently stale, but the wrapper still supports codex-auth for + # local maintainer reruns without changing Peter's flow. + echo "OPENCLAW_LIVE_CODEX_HARNESS_AUTH=api-key" >> "$GITHUB_ENV" ;; live-acp-bind-docker) - echo "OPENCLAW_LIVE_ACP_BIND_AGENTS=claude,codex,gemini" >> "$GITHUB_ENV" + if [[ -n "${GEMINI_API_KEY:-}" || -n "${GOOGLE_API_KEY:-}" ]]; then + echo "OPENCLAW_LIVE_ACP_BIND_AGENTS=claude,codex,gemini" >> "$GITHUB_ENV" + else + # The hydrated Gemini settings file only selects Gemini CLI auth + # mode. CI still needs a usable Gemini or Google API key before + # ACP bind can initialize a Gemini session. + echo "OPENCLAW_LIVE_ACP_BIND_AGENTS=claude,codex" >> "$GITHUB_ENV" + fi ;; esac diff --git a/.github/workflows/openclaw-release-checks.yml b/.github/workflows/openclaw-release-checks.yml index 8293313af28..1b86cda366d 100644 --- a/.github/workflows/openclaw-release-checks.yml +++ b/.github/workflows/openclaw-release-checks.yml @@ -130,12 +130,19 @@ jobs: ref: ${{ needs.resolve_target.outputs.ref }} provider: ${{ needs.resolve_target.outputs.provider }} mode: ${{ needs.resolve_target.outputs.mode }} - secrets: inherit + secrets: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + MINIMAX_API_KEY: ${{ secrets.MINIMAX_API_KEY }} + OPENCLAW_DISCORD_SMOKE_BOT_TOKEN: ${{ secrets.OPENCLAW_DISCORD_SMOKE_BOT_TOKEN }} + OPENCLAW_DISCORD_SMOKE_GUILD_ID: ${{ secrets.OPENCLAW_DISCORD_SMOKE_GUILD_ID }} + OPENCLAW_DISCORD_SMOKE_CHANNEL_ID: ${{ secrets.OPENCLAW_DISCORD_SMOKE_CHANNEL_ID }} live_and_e2e_release_checks: needs: [resolve_target] permissions: contents: read + pull-requests: read uses: ./.github/workflows/openclaw-live-and-e2e-checks-reusable.yml with: ref: ${{ needs.resolve_target.outputs.ref }} @@ -143,4 +150,47 @@ jobs: include_release_path_suites: true include_openwebui: true include_live_suites: true - secrets: inherit + secrets: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + OPENAI_BASE_URL: ${{ secrets.OPENAI_BASE_URL }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + ANTHROPIC_API_KEY_OLD: ${{ secrets.ANTHROPIC_API_KEY_OLD }} + ANTHROPIC_API_TOKEN: ${{ secrets.ANTHROPIC_API_TOKEN }} + BYTEPLUS_API_KEY: ${{ secrets.BYTEPLUS_API_KEY }} + CEREBRAS_API_KEY: ${{ secrets.CEREBRAS_API_KEY }} + DASHSCOPE_API_KEY: ${{ secrets.DASHSCOPE_API_KEY }} + GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }} + KIMI_API_KEY: ${{ secrets.KIMI_API_KEY }} + MODELSTUDIO_API_KEY: ${{ secrets.MODELSTUDIO_API_KEY }} + MOONSHOT_API_KEY: ${{ secrets.MOONSHOT_API_KEY }} + MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }} + MINIMAX_API_KEY: ${{ secrets.MINIMAX_API_KEY }} + OPENCODE_API_KEY: ${{ secrets.OPENCODE_API_KEY }} + OPENCODE_ZEN_API_KEY: ${{ secrets.OPENCODE_ZEN_API_KEY }} + OPENCLAW_LIVE_BROWSER_CDP_URL: ${{ secrets.OPENCLAW_LIVE_BROWSER_CDP_URL }} + OPENCLAW_LIVE_SETUP_TOKEN: ${{ secrets.OPENCLAW_LIVE_SETUP_TOKEN }} + OPENCLAW_LIVE_SETUP_TOKEN_MODEL: ${{ secrets.OPENCLAW_LIVE_SETUP_TOKEN_MODEL }} + OPENCLAW_LIVE_SETUP_TOKEN_PROFILE: ${{ secrets.OPENCLAW_LIVE_SETUP_TOKEN_PROFILE }} + OPENCLAW_LIVE_SETUP_TOKEN_VALUE: ${{ secrets.OPENCLAW_LIVE_SETUP_TOKEN_VALUE }} + GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} + GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + QWEN_API_KEY: ${{ secrets.QWEN_API_KEY }} + FAL_KEY: ${{ secrets.FAL_KEY }} + RUNWAY_API_KEY: ${{ secrets.RUNWAY_API_KEY }} + DEEPGRAM_API_KEY: ${{ secrets.DEEPGRAM_API_KEY }} + TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }} + VYDRA_API_KEY: ${{ secrets.VYDRA_API_KEY }} + XAI_API_KEY: ${{ secrets.XAI_API_KEY }} + ZAI_API_KEY: ${{ secrets.ZAI_API_KEY }} + Z_AI_API_KEY: ${{ secrets.Z_AI_API_KEY }} + BYTEPLUS_ACCESS_KEY_ID: ${{ secrets.BYTEPLUS_ACCESS_KEY_ID }} + BYTEPLUS_SECRET_ACCESS_KEY: ${{ secrets.BYTEPLUS_SECRET_ACCESS_KEY }} + CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + OPENCLAW_CODEX_AUTH_JSON: ${{ secrets.OPENCLAW_CODEX_AUTH_JSON }} + OPENCLAW_CODEX_CONFIG_TOML: ${{ secrets.OPENCLAW_CODEX_CONFIG_TOML }} + OPENCLAW_CLAUDE_JSON: ${{ secrets.OPENCLAW_CLAUDE_JSON }} + OPENCLAW_CLAUDE_CREDENTIALS_JSON: ${{ secrets.OPENCLAW_CLAUDE_CREDENTIALS_JSON }} + OPENCLAW_CLAUDE_SETTINGS_JSON: ${{ secrets.OPENCLAW_CLAUDE_SETTINGS_JSON }} + OPENCLAW_CLAUDE_SETTINGS_LOCAL_JSON: ${{ secrets.OPENCLAW_CLAUDE_SETTINGS_LOCAL_JSON }} + OPENCLAW_GEMINI_SETTINGS_JSON: ${{ secrets.OPENCLAW_GEMINI_SETTINGS_JSON }} diff --git a/.github/workflows/openclaw-scheduled-live-checks.yml b/.github/workflows/openclaw-scheduled-live-checks.yml index 8c0e331e901..027a67b1929 100644 --- a/.github/workflows/openclaw-scheduled-live-checks.yml +++ b/.github/workflows/openclaw-scheduled-live-checks.yml @@ -7,6 +7,7 @@ on: permissions: contents: read + pull-requests: read concurrency: group: openclaw-scheduled-live-checks-${{ github.ref }} @@ -19,6 +20,7 @@ jobs: live_and_openwebui_checks: permissions: contents: read + pull-requests: read uses: ./.github/workflows/openclaw-live-and-e2e-checks-reusable.yml with: ref: ${{ github.sha }} @@ -26,4 +28,47 @@ jobs: include_release_path_suites: false include_openwebui: true include_live_suites: true - secrets: inherit + secrets: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + OPENAI_BASE_URL: ${{ secrets.OPENAI_BASE_URL }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + ANTHROPIC_API_KEY_OLD: ${{ secrets.ANTHROPIC_API_KEY_OLD }} + ANTHROPIC_API_TOKEN: ${{ secrets.ANTHROPIC_API_TOKEN }} + BYTEPLUS_API_KEY: ${{ secrets.BYTEPLUS_API_KEY }} + CEREBRAS_API_KEY: ${{ secrets.CEREBRAS_API_KEY }} + DASHSCOPE_API_KEY: ${{ secrets.DASHSCOPE_API_KEY }} + GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }} + KIMI_API_KEY: ${{ secrets.KIMI_API_KEY }} + MODELSTUDIO_API_KEY: ${{ secrets.MODELSTUDIO_API_KEY }} + MOONSHOT_API_KEY: ${{ secrets.MOONSHOT_API_KEY }} + MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }} + MINIMAX_API_KEY: ${{ secrets.MINIMAX_API_KEY }} + OPENCODE_API_KEY: ${{ secrets.OPENCODE_API_KEY }} + OPENCODE_ZEN_API_KEY: ${{ secrets.OPENCODE_ZEN_API_KEY }} + OPENCLAW_LIVE_BROWSER_CDP_URL: ${{ secrets.OPENCLAW_LIVE_BROWSER_CDP_URL }} + OPENCLAW_LIVE_SETUP_TOKEN: ${{ secrets.OPENCLAW_LIVE_SETUP_TOKEN }} + OPENCLAW_LIVE_SETUP_TOKEN_MODEL: ${{ secrets.OPENCLAW_LIVE_SETUP_TOKEN_MODEL }} + OPENCLAW_LIVE_SETUP_TOKEN_PROFILE: ${{ secrets.OPENCLAW_LIVE_SETUP_TOKEN_PROFILE }} + OPENCLAW_LIVE_SETUP_TOKEN_VALUE: ${{ secrets.OPENCLAW_LIVE_SETUP_TOKEN_VALUE }} + GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} + GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} + OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} + QWEN_API_KEY: ${{ secrets.QWEN_API_KEY }} + FAL_KEY: ${{ secrets.FAL_KEY }} + RUNWAY_API_KEY: ${{ secrets.RUNWAY_API_KEY }} + DEEPGRAM_API_KEY: ${{ secrets.DEEPGRAM_API_KEY }} + TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }} + VYDRA_API_KEY: ${{ secrets.VYDRA_API_KEY }} + XAI_API_KEY: ${{ secrets.XAI_API_KEY }} + ZAI_API_KEY: ${{ secrets.ZAI_API_KEY }} + Z_AI_API_KEY: ${{ secrets.Z_AI_API_KEY }} + BYTEPLUS_ACCESS_KEY_ID: ${{ secrets.BYTEPLUS_ACCESS_KEY_ID }} + BYTEPLUS_SECRET_ACCESS_KEY: ${{ secrets.BYTEPLUS_SECRET_ACCESS_KEY }} + CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} + OPENCLAW_CODEX_AUTH_JSON: ${{ secrets.OPENCLAW_CODEX_AUTH_JSON }} + OPENCLAW_CODEX_CONFIG_TOML: ${{ secrets.OPENCLAW_CODEX_CONFIG_TOML }} + OPENCLAW_CLAUDE_JSON: ${{ secrets.OPENCLAW_CLAUDE_JSON }} + OPENCLAW_CLAUDE_CREDENTIALS_JSON: ${{ secrets.OPENCLAW_CLAUDE_CREDENTIALS_JSON }} + OPENCLAW_CLAUDE_SETTINGS_JSON: ${{ secrets.OPENCLAW_CLAUDE_SETTINGS_JSON }} + OPENCLAW_CLAUDE_SETTINGS_LOCAL_JSON: ${{ secrets.OPENCLAW_CLAUDE_SETTINGS_LOCAL_JSON }} + OPENCLAW_GEMINI_SETTINGS_JSON: ${{ secrets.OPENCLAW_GEMINI_SETTINGS_JSON }} diff --git a/extensions/openai/openai.live.test.ts b/extensions/openai/openai.live.test.ts index a04495b71ad..116cd58faea 100644 --- a/extensions/openai/openai.live.test.ts +++ b/extensions/openai/openai.live.test.ts @@ -241,8 +241,9 @@ describeLive("openai plugin live", () => { }); const text = (transcription?.text ?? "").toLowerCase(); + const collapsedText = text.replace(/[\s-]+/g, ""); expect(text.length).toBeGreaterThan(0); - expect(text).toContain("openclaw"); + expect(collapsedText).toContain("openclaw"); expect(text).toMatch(/\bok\b/); }, 45_000); diff --git a/scripts/lib/live-docker-auth.sh b/scripts/lib/live-docker-auth.sh index 84229150313..63032d0c96a 100644 --- a/scripts/lib/live-docker-auth.sh +++ b/scripts/lib/live-docker-auth.sh @@ -18,12 +18,31 @@ openclaw_live_trim() { printf '%s' "$value" } +openclaw_live_validate_relative_home_path() { + local value + value="$(openclaw_live_trim "${1:-}")" + [[ -n "$value" ]] || { + echo "ERROR: empty auth path." >&2 + return 1 + } + case "$value" in + /* | *..* | *\\* | *:*) + echo "ERROR: invalid auth path '$value'." >&2 + return 1 + ;; + esac + printf '%s' "$value" +} + openclaw_live_normalize_auth_dir() { local value value="$(openclaw_live_trim "${1:-}")" [[ -n "$value" ]] || return 1 - value="${value#.}" - printf '.%s' "$value" + if [[ "$value" != .* ]]; then + value=".$value" + fi + value="$(openclaw_live_validate_relative_home_path "$value")" || return 1 + printf '%s' "$value" } openclaw_live_should_include_auth_dir_for_provider() { @@ -143,3 +162,44 @@ openclaw_live_join_csv() { fi done } + +openclaw_live_stage_auth_into_home() { + local dest_home="${1:?destination home directory required}" + shift + + local mode="dirs" + local relative_path source_path dest_path + + mkdir -p "$dest_home" + chmod u+rwx "$dest_home" || true + + while (($# > 0)); do + case "$1" in + --files) + mode="files" + shift + continue + ;; + esac + + relative_path="$(openclaw_live_validate_relative_home_path "$1")" || return 1 + source_path="$HOME/$relative_path" + dest_path="$dest_home/$relative_path" + + if [[ "$mode" == "dirs" ]]; then + if [[ -d "$source_path" ]]; then + mkdir -p "$dest_path" + cp -R "$source_path"/. "$dest_path" + chmod -R u+rwX "$dest_path" || true + fi + else + if [[ -f "$source_path" ]]; then + mkdir -p "$(dirname "$dest_path")" + cp "$source_path" "$dest_path" + chmod u+rw "$dest_path" || true + fi + fi + + shift + done +} diff --git a/scripts/prepare-codex-ci-config.ts b/scripts/prepare-codex-ci-config.ts new file mode 100644 index 00000000000..435fcc0bb12 --- /dev/null +++ b/scripts/prepare-codex-ci-config.ts @@ -0,0 +1,51 @@ +import fs from "node:fs/promises"; +import path from "node:path"; + +function tomlString(value: string): string { + return JSON.stringify(value); +} + +export function buildCiSafeCodexConfig(params: { + projectPath: string; + approvalPolicy?: string; + sandboxMode?: string; +}): string { + if (!params.projectPath || typeof params.projectPath !== "string") { + throw new Error("projectPath is required."); + } + const resolvedProjectPath = path.resolve(params.projectPath); + const approvalPolicy = params.approvalPolicy ?? "never"; + const sandboxMode = params.sandboxMode ?? "workspace-write"; + return [ + "# Generated for Codex CI runs.", + "# Keep the checked-out repo trusted while avoiding maintainer-local", + "# provider/profile overrides that do not exist on CI runners.", + `approval_policy = ${tomlString(approvalPolicy)}`, + `sandbox_mode = ${tomlString(sandboxMode)}`, + "", + `[projects.${tomlString(resolvedProjectPath)}]`, + 'trust_level = "trusted"', + "", + ].join("\n"); +} + +export async function writeCiSafeCodexConfig(params: { + outputPath: string; + projectPath: string; + approvalPolicy?: string; + sandboxMode?: string; +}): Promise { + if (!params.outputPath || typeof params.outputPath !== "string") { + throw new Error("outputPath is required."); + } + const rendered = buildCiSafeCodexConfig(params); + await fs.mkdir(path.dirname(params.outputPath), { recursive: true }); + await fs.writeFile(params.outputPath, rendered, "utf-8"); + return rendered; +} + +if (path.basename(process.argv[1] ?? "") === "prepare-codex-ci-config.ts") { + const outputPath = process.argv[2]; + const projectPath = process.argv[3] ?? process.cwd(); + await writeCiSafeCodexConfig({ outputPath, projectPath }); +} diff --git a/scripts/test-install-sh-e2e-docker.sh b/scripts/test-install-sh-e2e-docker.sh index 74dbe88d861..3eb078a6d5e 100755 --- a/scripts/test-install-sh-e2e-docker.sh +++ b/scripts/test-install-sh-e2e-docker.sh @@ -24,7 +24,7 @@ docker run --rm \ -e OPENCLAW_INSTALL_E2E_PREVIOUS="${OPENCLAW_INSTALL_E2E_PREVIOUS:-}" \ -e OPENCLAW_INSTALL_E2E_SKIP_PREVIOUS="${OPENCLAW_INSTALL_E2E_SKIP_PREVIOUS:-0}" \ -e OPENCLAW_NO_ONBOARD=1 \ - -e OPENAI_API_KEY="$OPENAI_API_KEY" \ - -e ANTHROPIC_API_KEY="$ANTHROPIC_API_KEY" \ - -e ANTHROPIC_API_TOKEN="$ANTHROPIC_API_TOKEN" \ + -e OPENAI_API_KEY \ + -e ANTHROPIC_API_KEY \ + -e ANTHROPIC_API_TOKEN \ "$IMAGE_NAME" diff --git a/scripts/test-live-acp-bind-docker.sh b/scripts/test-live-acp-bind-docker.sh index 2da32d8bbeb..8158cbd8b7b 100644 --- a/scripts/test-live-acp-bind-docker.sh +++ b/scripts/test-live-acp-bind-docker.sh @@ -11,6 +11,8 @@ PROFILE_FILE="${OPENCLAW_PROFILE_FILE:-$HOME/.profile}" ACP_AGENT_LIST_RAW="${OPENCLAW_LIVE_ACP_BIND_AGENTS:-${OPENCLAW_LIVE_ACP_BIND_AGENT:-claude,codex,gemini}}" TEMP_DIRS=() DOCKER_USER="${OPENCLAW_DOCKER_USER:-node}" +DOCKER_HOME_MOUNT=() +DOCKER_AUTH_PRESTAGED=0 openclaw_live_acp_bind_resolve_auth_provider() { case "${1:-}" in @@ -80,27 +82,29 @@ export npm_config_cache="$NPM_CONFIG_CACHE" mkdir -p "$NPM_CONFIG_PREFIX" "$XDG_CACHE_HOME" "$COREPACK_HOME" "$NPM_CONFIG_CACHE" chmod 700 "$XDG_CACHE_HOME" "$COREPACK_HOME" "$NPM_CONFIG_CACHE" || true export PATH="$NPM_CONFIG_PREFIX/bin:$PATH" -IFS=',' read -r -a auth_dirs <<<"${OPENCLAW_DOCKER_AUTH_DIRS_RESOLVED:-}" -IFS=',' read -r -a auth_files <<<"${OPENCLAW_DOCKER_AUTH_FILES_RESOLVED:-}" -if ((${#auth_dirs[@]} > 0)); then - for auth_dir in "${auth_dirs[@]}"; do - [ -n "$auth_dir" ] || continue - if [ -d "/host-auth/$auth_dir" ]; then - mkdir -p "$HOME/$auth_dir" - cp -R "/host-auth/$auth_dir/." "$HOME/$auth_dir" - chmod -R u+rwX "$HOME/$auth_dir" || true - fi - done -fi -if ((${#auth_files[@]} > 0)); then - for auth_file in "${auth_files[@]}"; do - [ -n "$auth_file" ] || continue - if [ -f "/host-auth-files/$auth_file" ]; then - mkdir -p "$(dirname "$HOME/$auth_file")" - cp "/host-auth-files/$auth_file" "$HOME/$auth_file" - chmod u+rw "$HOME/$auth_file" || true - fi - done +if [ "${OPENCLAW_DOCKER_AUTH_PRESTAGED:-0}" != "1" ]; then + IFS=',' read -r -a auth_dirs <<<"${OPENCLAW_DOCKER_AUTH_DIRS_RESOLVED:-}" + IFS=',' read -r -a auth_files <<<"${OPENCLAW_DOCKER_AUTH_FILES_RESOLVED:-}" + if ((${#auth_dirs[@]} > 0)); then + for auth_dir in "${auth_dirs[@]}"; do + [ -n "$auth_dir" ] || continue + if [ -d "/host-auth/$auth_dir" ]; then + mkdir -p "$HOME/$auth_dir" + cp -R "/host-auth/$auth_dir/." "$HOME/$auth_dir" + chmod -R u+rwX "$HOME/$auth_dir" || true + fi + done + fi + if ((${#auth_files[@]} > 0)); then + for auth_file in "${auth_files[@]}"; do + [ -n "$auth_file" ] || continue + if [ -f "/host-auth-files/$auth_file" ]; then + mkdir -p "$(dirname "$HOME/$auth_file")" + cp "/host-auth-files/$auth_file" "$HOME/$auth_file" + chmod u+rw "$HOME/$auth_file" || true + fi + done + fi fi agent="${OPENCLAW_LIVE_ACP_BIND_AGENT:-claude}" case "$agent" in @@ -217,9 +221,23 @@ for ACP_AGENT in "${ACP_AGENTS[@]}"; do AUTH_FILES_CSV="$(openclaw_live_join_csv "${AUTH_FILES[@]}")" fi + DOCKER_HOME_MOUNT=() + DOCKER_AUTH_PRESTAGED=0 + if [[ "${CI:-}" == "true" || "${GITHUB_ACTIONS:-}" == "true" ]]; then + DOCKER_HOME_DIR="$(mktemp -d "${RUNNER_TEMP:-/tmp}/openclaw-docker-home.XXXXXX")" + TEMP_DIRS+=("$DOCKER_HOME_DIR") + DOCKER_HOME_MOUNT=(-v "$DOCKER_HOME_DIR":/home/node) + fi + + if [[ -n "${DOCKER_HOME_DIR:-}" ]]; then + openclaw_live_stage_auth_into_home "$DOCKER_HOME_DIR" "${AUTH_DIRS[@]}" --files "${AUTH_FILES[@]}" + DOCKER_AUTH_PRESTAGED=1 + fi + EXTERNAL_AUTH_MOUNTS=() if ((${#AUTH_DIRS[@]} > 0)); then for auth_dir in "${AUTH_DIRS[@]}"; do + auth_dir="$(openclaw_live_validate_relative_home_path "$auth_dir")" host_path="$HOME/$auth_dir" if [[ -d "$host_path" ]]; then EXTERNAL_AUTH_MOUNTS+=(-v "$host_path":/host-auth/"$auth_dir":ro) @@ -228,6 +246,7 @@ for ACP_AGENT in "${ACP_AGENTS[@]}"; do fi if ((${#AUTH_FILES[@]} > 0)); then for auth_file in "${AUTH_FILES[@]}"; do + auth_file="$(openclaw_live_validate_relative_home_path "$auth_file")" host_path="$HOME/$auth_file" if [[ -f "$host_path" ]]; then EXTERNAL_AUTH_MOUNTS+=(-v "$host_path":/host-auth-files/"$auth_file":ro) @@ -246,18 +265,22 @@ for ACP_AGENT in "${ACP_AGENTS[@]}"; do -e ANTHROPIC_API_KEY_OLD \ -e OPENCLAW_LIVE_ACP_BIND_ANTHROPIC_API_KEY="${ANTHROPIC_API_KEY:-}" \ -e OPENCLAW_LIVE_ACP_BIND_ANTHROPIC_API_KEY_OLD="${ANTHROPIC_API_KEY_OLD:-}" \ + -e GEMINI_API_KEY \ + -e GOOGLE_API_KEY \ -e OPENAI_API_KEY \ -e COREPACK_ENABLE_DOWNLOAD_PROMPT=0 \ -e HOME=/home/node \ -e NODE_OPTIONS=--disable-warning=ExperimentalWarning \ -e OPENCLAW_SKIP_CHANNELS=1 \ -e OPENCLAW_VITEST_FS_MODULE_CACHE=0 \ + -e OPENCLAW_DOCKER_AUTH_PRESTAGED="$DOCKER_AUTH_PRESTAGED" \ -e OPENCLAW_DOCKER_AUTH_DIRS_RESOLVED="$AUTH_DIRS_CSV" \ -e OPENCLAW_DOCKER_AUTH_FILES_RESOLVED="$AUTH_FILES_CSV" \ -e OPENCLAW_LIVE_TEST=1 \ -e OPENCLAW_LIVE_ACP_BIND=1 \ -e OPENCLAW_LIVE_ACP_BIND_AGENT="$ACP_AGENT" \ -e OPENCLAW_LIVE_ACP_BIND_AGENT_COMMAND="$AGENT_COMMAND" \ + "${DOCKER_HOME_MOUNT[@]}" \ -v "$CACHE_HOME_DIR":/home/node/.cache \ -v "$ROOT_DIR":/src:ro \ -v "$CONFIG_DIR":/home/node/.openclaw \ diff --git a/scripts/test-live-cli-backend-docker.sh b/scripts/test-live-cli-backend-docker.sh index 0593b2f6014..1efdb45683a 100644 --- a/scripts/test-live-cli-backend-docker.sh +++ b/scripts/test-live-cli-backend-docker.sh @@ -15,6 +15,9 @@ CLI_DISABLE_MCP_CONFIG="${OPENCLAW_LIVE_CLI_BACKEND_DISABLE_MCP_CONFIG:-}" CLI_AUTH_MODE="${OPENCLAW_LIVE_CLI_BACKEND_AUTH:-auto}" TEMP_DIRS=() DOCKER_USER="${OPENCLAW_DOCKER_USER:-node}" +DOCKER_HOME_MOUNT=() +DOCKER_EXTRA_ENV_FILES=() +DOCKER_AUTH_PRESTAGED=0 if [[ -z "$CLI_PROVIDER" || "$CLI_PROVIDER" == "$CLI_MODEL" ]]; then CLI_PROVIDER="$DEFAULT_PROVIDER" @@ -34,6 +37,13 @@ if [[ "$CLI_AUTH_MODE" == "subscription" && "$CLI_PROVIDER" != "claude-cli" ]]; exit 1 fi +if [[ "$CLI_AUTH_MODE" == "api-key" && "$CLI_PROVIDER" == "codex-cli" ]]; then + if [[ -z "${OPENAI_API_KEY:-}" ]]; then + echo "ERROR: OPENCLAW_LIVE_CLI_BACKEND_AUTH=api-key for codex-cli requires OPENAI_API_KEY." >&2 + exit 1 + fi +fi + CLI_METADATA_JSON="$(node --import tsx "$ROOT_DIR/scripts/print-cli-backend-live-metadata.ts" "$CLI_PROVIDER")" read_metadata_field() { local field="$1" @@ -84,6 +94,9 @@ mkdir -p "$CLI_TOOLS_DIR" mkdir -p "$CACHE_HOME_DIR" if [[ "${CI:-}" == "true" || "${GITHUB_ACTIONS:-}" == "true" ]]; then DOCKER_USER="$(id -u):$(id -g)" + DOCKER_HOME_DIR="$(mktemp -d "${RUNNER_TEMP:-/tmp}/openclaw-docker-home.XXXXXX")" + TEMP_DIRS+=("$DOCKER_HOME_DIR") + DOCKER_HOME_MOUNT=(-v "$DOCKER_HOME_DIR":/home/node) fi if [[ "$CLI_PROVIDER" == "claude-cli" && "$CLI_AUTH_MODE" == "subscription" ]]; then @@ -143,7 +156,9 @@ fi AUTH_DIRS=() AUTH_FILES=() -if [[ -n "${OPENCLAW_DOCKER_AUTH_DIRS:-}" ]]; then +if [[ "$CLI_AUTH_MODE" == "api-key" && "$CLI_PROVIDER" == "codex-cli" ]]; then + AUTH_FILES+=(".codex/config.toml") +elif [[ -n "${OPENCLAW_DOCKER_AUTH_DIRS:-}" ]]; then while IFS= read -r auth_dir; do [[ -n "$auth_dir" ]] || continue AUTH_DIRS+=("$auth_dir") @@ -171,9 +186,15 @@ if ((${#AUTH_FILES[@]} > 0)); then AUTH_FILES_CSV="$(openclaw_live_join_csv "${AUTH_FILES[@]}")" fi +if [[ -n "${DOCKER_HOME_DIR:-}" ]]; then + openclaw_live_stage_auth_into_home "$DOCKER_HOME_DIR" "${AUTH_DIRS[@]}" --files "${AUTH_FILES[@]}" + DOCKER_AUTH_PRESTAGED=1 +fi + EXTERNAL_AUTH_MOUNTS=() if ((${#AUTH_DIRS[@]} > 0)); then for auth_dir in "${AUTH_DIRS[@]}"; do + auth_dir="$(openclaw_live_validate_relative_home_path "$auth_dir")" host_path="$HOME/$auth_dir" if [[ -d "$host_path" ]]; then EXTERNAL_AUTH_MOUNTS+=(-v "$host_path":/host-auth/"$auth_dir":ro) @@ -182,6 +203,7 @@ if ((${#AUTH_DIRS[@]} > 0)); then fi if ((${#AUTH_FILES[@]} > 0)); then for auth_file in "${AUTH_FILES[@]}"; do + auth_file="$(openclaw_live_validate_relative_home_path "$auth_file")" host_path="$HOME/$auth_file" if [[ -f "$host_path" ]]; then EXTERNAL_AUTH_MOUNTS+=(-v "$host_path":/host-auth-files/"$auth_file":ro) @@ -201,27 +223,29 @@ export npm_config_cache="$NPM_CONFIG_CACHE" mkdir -p "$NPM_CONFIG_PREFIX" "$XDG_CACHE_HOME" "$COREPACK_HOME" "$NPM_CONFIG_CACHE" chmod 700 "$XDG_CACHE_HOME" "$COREPACK_HOME" "$NPM_CONFIG_CACHE" || true export PATH="$NPM_CONFIG_PREFIX/bin:$PATH" -IFS=',' read -r -a auth_dirs <<<"${OPENCLAW_DOCKER_AUTH_DIRS_RESOLVED:-}" -IFS=',' read -r -a auth_files <<<"${OPENCLAW_DOCKER_AUTH_FILES_RESOLVED:-}" -if ((${#auth_dirs[@]} > 0)); then - for auth_dir in "${auth_dirs[@]}"; do - [ -n "$auth_dir" ] || continue - if [ -d "/host-auth/$auth_dir" ]; then - mkdir -p "$HOME/$auth_dir" - cp -R "/host-auth/$auth_dir/." "$HOME/$auth_dir" - chmod -R u+rwX "$HOME/$auth_dir" || true - fi - done -fi -if ((${#auth_files[@]} > 0)); then - for auth_file in "${auth_files[@]}"; do - [ -n "$auth_file" ] || continue - if [ -f "/host-auth-files/$auth_file" ]; then - mkdir -p "$(dirname "$HOME/$auth_file")" - cp "/host-auth-files/$auth_file" "$HOME/$auth_file" - chmod u+rw "$HOME/$auth_file" || true - fi - done +if [ "${OPENCLAW_DOCKER_AUTH_PRESTAGED:-0}" != "1" ]; then + IFS=',' read -r -a auth_dirs <<<"${OPENCLAW_DOCKER_AUTH_DIRS_RESOLVED:-}" + IFS=',' read -r -a auth_files <<<"${OPENCLAW_DOCKER_AUTH_FILES_RESOLVED:-}" + if ((${#auth_dirs[@]} > 0)); then + for auth_dir in "${auth_dirs[@]}"; do + [ -n "$auth_dir" ] || continue + if [ -d "/host-auth/$auth_dir" ]; then + mkdir -p "$HOME/$auth_dir" + cp -R "/host-auth/$auth_dir/." "$HOME/$auth_dir" + chmod -R u+rwX "$HOME/$auth_dir" || true + fi + done + fi + if ((${#auth_files[@]} > 0)); then + for auth_file in "${auth_files[@]}"; do + [ -n "$auth_file" ] || continue + if [ -f "/host-auth-files/$auth_file" ]; then + mkdir -p "$(dirname "$HOME/$auth_file")" + cp "/host-auth-files/$auth_file" "$HOME/$auth_file" + chmod u+rw "$HOME/$auth_file" || true + fi + done + fi fi provider="${OPENCLAW_DOCKER_CLI_BACKEND_PROVIDER:-claude-cli}" default_command="${OPENCLAW_DOCKER_CLI_BACKEND_COMMAND_DEFAULT:-}" @@ -236,6 +260,17 @@ fi if [ -n "${OPENCLAW_LIVE_CLI_BACKEND_COMMAND:-}" ] && [ ! -x "${OPENCLAW_LIVE_CLI_BACKEND_COMMAND}" ] && [ -n "$docker_package" ]; then npm install -g "$docker_package" fi +if [ "$provider" = "codex-cli" ] && [ "${OPENCLAW_LIVE_CLI_BACKEND_AUTH:-auto}" = "api-key" ]; then + codex_login_command="${OPENCLAW_LIVE_CLI_BACKEND_COMMAND:-$NPM_CONFIG_PREFIX/bin/codex}" + if [ ! -x "$codex_login_command" ] && [ -x "$NPM_CONFIG_PREFIX/bin/codex" ]; then + codex_login_command="$NPM_CONFIG_PREFIX/bin/codex" + fi + printf '%s\n' "$OPENAI_API_KEY" | "$codex_login_command" login --with-api-key >/dev/null +fi +if [ -n "${OPENCLAW_LIVE_CLI_BACKEND_COMMAND:-}" ] && [ -x "${OPENCLAW_LIVE_CLI_BACKEND_COMMAND}" ]; then + echo "==> CLI backend binary: ${OPENCLAW_LIVE_CLI_BACKEND_COMMAND}" + "${OPENCLAW_LIVE_CLI_BACKEND_COMMAND}" -V || "${OPENCLAW_LIVE_CLI_BACKEND_COMMAND}" --version || true +fi if [ "$provider" = "claude-cli" ]; then auth_mode="${OPENCLAW_LIVE_CLI_BACKEND_AUTH:-auto}" if [ "$auth_mode" = "subscription" ]; then @@ -324,6 +359,9 @@ openclaw_live_link_runtime_tree "$tmp_dir" openclaw_live_stage_state_dir "$tmp_dir/.openclaw-state" openclaw_live_prepare_staged_config cd "$tmp_dir" +if [ "${OPENCLAW_LIVE_CLI_BACKEND_USE_CI_SAFE_CODEX_CONFIG:-0}" = "1" ]; then + node --import tsx /src/scripts/prepare-codex-ci-config.ts "$HOME/.codex/config.toml" "$tmp_dir" +fi pnpm test:live src/gateway/gateway-cli-backend.live.test.ts EOF @@ -346,7 +384,18 @@ echo "==> External auth files: ${AUTH_FILES_CSV:-none}" DOCKER_AUTH_ENV=( -e OPENCLAW_LIVE_CLI_BACKEND_AUTH="$CLI_AUTH_MODE" ) -if [[ "$CLI_PROVIDER" == "claude-cli" && "$CLI_AUTH_MODE" == "subscription" ]]; then +if [[ "$CLI_PROVIDER" == "codex-cli" && "$CLI_AUTH_MODE" == "api-key" ]]; then + docker_env_dir="$(mktemp -d "${RUNNER_TEMP:-/tmp}/openclaw-cli-backend-env.XXXXXX")" + TEMP_DIRS+=("$docker_env_dir") + docker_env_file="$docker_env_dir/openai.env" + { + printf 'OPENAI_API_KEY=%s\n' "${OPENAI_API_KEY}" + if [[ -n "${OPENAI_BASE_URL:-}" ]]; then + printf 'OPENAI_BASE_URL=%s\n' "${OPENAI_BASE_URL}" + fi + } >"$docker_env_file" + DOCKER_EXTRA_ENV_FILES+=(--env-file "$docker_env_file") +elif [[ "$CLI_PROVIDER" == "claude-cli" && "$CLI_AUTH_MODE" == "subscription" ]]; then DOCKER_AUTH_ENV+=( -e CLAUDE_CODE_OAUTH_TOKEN="${CLAUDE_CODE_OAUTH_TOKEN:-}" -e OPENCLAW_LIVE_CLI_BACKEND_PRESERVE_ENV="$OPENCLAW_LIVE_CLI_BACKEND_PRESERVE_ENV" @@ -369,8 +418,10 @@ docker run --rm -t \ -e NODE_OPTIONS=--disable-warning=ExperimentalWarning \ -e OPENCLAW_SKIP_CHANNELS=1 \ -e OPENCLAW_VITEST_FS_MODULE_CACHE=0 \ + -e OPENCLAW_DOCKER_AUTH_PRESTAGED="$DOCKER_AUTH_PRESTAGED" \ -e OPENCLAW_DOCKER_AUTH_DIRS_RESOLVED="$AUTH_DIRS_CSV" \ -e OPENCLAW_DOCKER_AUTH_FILES_RESOLVED="$AUTH_FILES_CSV" \ + -e OPENCLAW_LIVE_CLI_BACKEND_USE_CI_SAFE_CODEX_CONFIG="${OPENCLAW_LIVE_CLI_BACKEND_USE_CI_SAFE_CODEX_CONFIG:-0}" \ -e OPENCLAW_DOCKER_CLI_BACKEND_PROVIDER="$CLI_PROVIDER" \ -e OPENCLAW_DOCKER_CLI_BACKEND_COMMAND_DEFAULT="$CLI_DEFAULT_COMMAND" \ -e OPENCLAW_DOCKER_CLI_BACKEND_NPM_PACKAGE="$CLI_DOCKER_NPM_PACKAGE" \ @@ -382,6 +433,7 @@ docker run --rm -t \ -e OPENCLAW_LIVE_CLI_BACKEND_MODEL="$CLI_MODEL" \ -e OPENCLAW_LIVE_CLI_BACKEND_COMMAND="${OPENCLAW_LIVE_CLI_BACKEND_COMMAND:-}" \ -e OPENCLAW_LIVE_CLI_BACKEND_ARGS="${OPENCLAW_LIVE_CLI_BACKEND_ARGS:-}" \ + -e OPENCLAW_LIVE_CLI_BACKEND_RESUME_ARGS="${OPENCLAW_LIVE_CLI_BACKEND_RESUME_ARGS:-}" \ -e OPENCLAW_LIVE_CLI_BACKEND_CLEAR_ENV="${OPENCLAW_LIVE_CLI_BACKEND_CLEAR_ENV:-}" \ -e OPENCLAW_LIVE_CLI_BACKEND_DISABLE_MCP_CONFIG="$CLI_DISABLE_MCP_CONFIG" \ -e OPENCLAW_LIVE_CLI_BACKEND_RESUME_PROBE="${OPENCLAW_LIVE_CLI_BACKEND_RESUME_PROBE:-}" \ @@ -390,6 +442,8 @@ docker run --rm -t \ -e OPENCLAW_LIVE_CLI_BACKEND_MCP_PROBE="${OPENCLAW_LIVE_CLI_BACKEND_MCP_PROBE:-}" \ -e OPENCLAW_LIVE_CLI_BACKEND_IMAGE_ARG="${OPENCLAW_LIVE_CLI_BACKEND_IMAGE_ARG:-}" \ -e OPENCLAW_LIVE_CLI_BACKEND_IMAGE_MODE="${OPENCLAW_LIVE_CLI_BACKEND_IMAGE_MODE:-}" \ + "${DOCKER_HOME_MOUNT[@]}" \ + "${DOCKER_EXTRA_ENV_FILES[@]}" \ -v "$CACHE_HOME_DIR":/home/node/.cache \ -v "$ROOT_DIR":/src:ro \ -v "$CONFIG_DIR":/home/node/.openclaw \ diff --git a/scripts/test-live-codex-harness-docker.sh b/scripts/test-live-codex-harness-docker.sh index 85c3b9ea9d2..137ec2ced3c 100644 --- a/scripts/test-live-codex-harness-docker.sh +++ b/scripts/test-live-codex-harness-docker.sh @@ -8,8 +8,26 @@ LIVE_IMAGE_NAME="${OPENCLAW_LIVE_IMAGE:-${IMAGE_NAME}-live}" CONFIG_DIR="${OPENCLAW_CONFIG_DIR:-$HOME/.openclaw}" WORKSPACE_DIR="${OPENCLAW_WORKSPACE_DIR:-$HOME/.openclaw/workspace}" PROFILE_FILE="${OPENCLAW_PROFILE_FILE:-$HOME/.profile}" +CODEX_HARNESS_AUTH_MODE="${OPENCLAW_LIVE_CODEX_HARNESS_AUTH:-codex-auth}" TEMP_DIRS=() DOCKER_USER="${OPENCLAW_DOCKER_USER:-node}" +DOCKER_HOME_MOUNT=() +DOCKER_EXTRA_ENV_FILES=() +DOCKER_AUTH_PRESTAGED=0 + +case "$CODEX_HARNESS_AUTH_MODE" in + codex-auth | api-key) + ;; + *) + echo "ERROR: OPENCLAW_LIVE_CODEX_HARNESS_AUTH must be one of: codex-auth, api-key." >&2 + exit 1 + ;; +esac + +if [[ "$CODEX_HARNESS_AUTH_MODE" == "api-key" && -z "${OPENAI_API_KEY:-}" ]]; then + echo "ERROR: OPENCLAW_LIVE_CODEX_HARNESS_AUTH=api-key requires OPENAI_API_KEY." >&2 + exit 1 +fi cleanup_temp_dirs() { if ((${#TEMP_DIRS[@]} > 0)); then @@ -39,6 +57,9 @@ mkdir -p "$CLI_TOOLS_DIR" mkdir -p "$CACHE_HOME_DIR" if [[ "${CI:-}" == "true" || "${GITHUB_ACTIONS:-}" == "true" ]]; then DOCKER_USER="$(id -u):$(id -g)" + DOCKER_HOME_DIR="$(mktemp -d "${RUNNER_TEMP:-/tmp}/openclaw-docker-home.XXXXXX")" + TEMP_DIRS+=("$DOCKER_HOME_DIR") + DOCKER_HOME_MOUNT=(-v "$DOCKER_HOME_DIR":/home/node) fi PROFILE_MOUNT=() @@ -47,19 +68,27 @@ if [[ -f "$PROFILE_FILE" && -r "$PROFILE_FILE" ]]; then fi AUTH_FILES=() -while IFS= read -r auth_file; do - [[ -n "$auth_file" ]] || continue - AUTH_FILES+=("$auth_file") -done < <(openclaw_live_collect_auth_files_from_csv "openai-codex") +if [[ "$CODEX_HARNESS_AUTH_MODE" != "api-key" ]]; then + while IFS= read -r auth_file; do + [[ -n "$auth_file" ]] || continue + AUTH_FILES+=("$auth_file") + done < <(openclaw_live_collect_auth_files_from_csv "openai-codex") +fi AUTH_FILES_CSV="" if ((${#AUTH_FILES[@]} > 0)); then AUTH_FILES_CSV="$(openclaw_live_join_csv "${AUTH_FILES[@]}")" fi +if [[ -n "${DOCKER_HOME_DIR:-}" ]]; then + openclaw_live_stage_auth_into_home "$DOCKER_HOME_DIR" --files "${AUTH_FILES[@]}" + DOCKER_AUTH_PRESTAGED=1 +fi + EXTERNAL_AUTH_MOUNTS=() if ((${#AUTH_FILES[@]} > 0)); then for auth_file in "${AUTH_FILES[@]}"; do + auth_file="$(openclaw_live_validate_relative_home_path "$auth_file")" host_path="$HOME/$auth_file" if [[ -f "$host_path" ]]; then EXTERNAL_AUTH_MOUNTS+=(-v "$host_path":/host-auth-files/"$auth_file":ro) @@ -67,6 +96,20 @@ if ((${#AUTH_FILES[@]} > 0)); then done fi +DOCKER_AUTH_ENV=() +if [[ "$CODEX_HARNESS_AUTH_MODE" == "api-key" ]]; then + docker_env_dir="$(mktemp -d "${RUNNER_TEMP:-/tmp}/openclaw-codex-harness-env.XXXXXX")" + TEMP_DIRS+=("$docker_env_dir") + docker_env_file="$docker_env_dir/openai.env" + { + printf 'OPENAI_API_KEY=%s\n' "${OPENAI_API_KEY}" + if [[ -n "${OPENAI_BASE_URL:-}" ]]; then + printf 'OPENAI_BASE_URL=%s\n' "${OPENAI_BASE_URL}" + fi + } >"$docker_env_file" + DOCKER_EXTRA_ENV_FILES+=(--env-file "$docker_env_file") +fi + read -r -d '' LIVE_TEST_CMD <<'EOF' || true set -euo pipefail [ -f "$HOME/.profile" ] && [ -r "$HOME/.profile" ] && source "$HOME/.profile" || true @@ -76,23 +119,38 @@ export XDG_CACHE_HOME="${XDG_CACHE_HOME:-$HOME/.cache}" export COREPACK_HOME="${COREPACK_HOME:-$XDG_CACHE_HOME/node/corepack}" export NPM_CONFIG_CACHE="${NPM_CONFIG_CACHE:-$XDG_CACHE_HOME/npm}" export npm_config_cache="$NPM_CONFIG_CACHE" +# Force the Codex harness to use the staged `~/.codex` auth files. This lane +# is not meant to exercise raw OpenAI API-key routing unless the lane +# explicitly opts into API-key auth for CI. +if [ "${OPENCLAW_LIVE_CODEX_HARNESS_AUTH:-codex-auth}" != "api-key" ]; then + unset OPENAI_API_KEY OPENAI_BASE_URL +fi mkdir -p "$NPM_CONFIG_PREFIX" "$XDG_CACHE_HOME" "$COREPACK_HOME" "$NPM_CONFIG_CACHE" chmod 700 "$XDG_CACHE_HOME" "$COREPACK_HOME" "$NPM_CONFIG_CACHE" || true export PATH="$NPM_CONFIG_PREFIX/bin:$PATH" -IFS=',' read -r -a auth_files <<<"${OPENCLAW_DOCKER_AUTH_FILES_RESOLVED:-}" -if ((${#auth_files[@]} > 0)); then - for auth_file in "${auth_files[@]}"; do - [ -n "$auth_file" ] || continue - if [ -f "/host-auth-files/$auth_file" ]; then - mkdir -p "$(dirname "$HOME/$auth_file")" - cp "/host-auth-files/$auth_file" "$HOME/$auth_file" - chmod u+rw "$HOME/$auth_file" || true - fi - done +if [ "${OPENCLAW_DOCKER_AUTH_PRESTAGED:-0}" != "1" ]; then + IFS=',' read -r -a auth_files <<<"${OPENCLAW_DOCKER_AUTH_FILES_RESOLVED:-}" + if ((${#auth_files[@]} > 0)); then + for auth_file in "${auth_files[@]}"; do + [ -n "$auth_file" ] || continue + if [ -f "/host-auth-files/$auth_file" ]; then + mkdir -p "$(dirname "$HOME/$auth_file")" + cp "/host-auth-files/$auth_file" "$HOME/$auth_file" + chmod u+rw "$HOME/$auth_file" || true + fi + done + fi +fi +if [ "${OPENCLAW_LIVE_CODEX_HARNESS_AUTH:-codex-auth}" != "api-key" ] && [ ! -s "$HOME/.codex/auth.json" ]; then + echo "ERROR: missing ~/.codex/auth.json for Codex harness live test." >&2 + exit 1 fi if [ ! -x "$NPM_CONFIG_PREFIX/bin/codex" ]; then npm install -g @openai/codex fi +if [ "${OPENCLAW_LIVE_CODEX_HARNESS_AUTH:-codex-auth}" = "api-key" ]; then + printf '%s\n' "$OPENAI_API_KEY" | "$NPM_CONFIG_PREFIX/bin/codex" login --with-api-key >/dev/null +fi tmp_dir="$(mktemp -d)" cleanup() { rm -rf "$tmp_dir" @@ -117,6 +175,7 @@ echo "==> Run Codex harness live test in Docker" echo "==> Model: ${OPENCLAW_LIVE_CODEX_HARNESS_MODEL:-codex/gpt-5.4}" echo "==> Image probe: ${OPENCLAW_LIVE_CODEX_HARNESS_IMAGE_PROBE:-1}" echo "==> MCP probe: ${OPENCLAW_LIVE_CODEX_HARNESS_MCP_PROBE:-1}" +echo "==> Auth mode: $CODEX_HARNESS_AUTH_MODE" echo "==> Harness fallback: none" echo "==> Auth files: ${AUTH_FILES_CSV:-none}" docker run --rm -t \ @@ -125,10 +184,11 @@ docker run --rm -t \ -e COREPACK_ENABLE_DOWNLOAD_PROMPT=0 \ -e HOME=/home/node \ -e NODE_OPTIONS=--disable-warning=ExperimentalWarning \ - -e OPENAI_API_KEY \ -e OPENCLAW_AGENT_HARNESS_FALLBACK=none \ + -e OPENCLAW_DOCKER_AUTH_PRESTAGED="$DOCKER_AUTH_PRESTAGED" \ -e OPENCLAW_CODEX_APP_SERVER_BIN="${OPENCLAW_CODEX_APP_SERVER_BIN:-codex}" \ -e OPENCLAW_DOCKER_AUTH_FILES_RESOLVED="$AUTH_FILES_CSV" \ + -e OPENCLAW_LIVE_CODEX_HARNESS_AUTH="$CODEX_HARNESS_AUTH_MODE" \ -e OPENCLAW_LIVE_CODEX_HARNESS=1 \ -e OPENCLAW_LIVE_CODEX_HARNESS_DEBUG="${OPENCLAW_LIVE_CODEX_HARNESS_DEBUG:-}" \ -e OPENCLAW_LIVE_CODEX_HARNESS_IMAGE_PROBE="${OPENCLAW_LIVE_CODEX_HARNESS_IMAGE_PROBE:-1}" \ @@ -136,6 +196,9 @@ docker run --rm -t \ -e OPENCLAW_LIVE_CODEX_HARNESS_MODEL="${OPENCLAW_LIVE_CODEX_HARNESS_MODEL:-codex/gpt-5.4}" \ -e OPENCLAW_LIVE_TEST=1 \ -e OPENCLAW_VITEST_FS_MODULE_CACHE=0 \ + "${DOCKER_AUTH_ENV[@]}" \ + "${DOCKER_EXTRA_ENV_FILES[@]}" \ + "${DOCKER_HOME_MOUNT[@]}" \ -v "$CACHE_HOME_DIR":/home/node/.cache \ -v "$ROOT_DIR":/src:ro \ -v "$CONFIG_DIR":/home/node/.openclaw \ diff --git a/scripts/test-live-gateway-models-docker.sh b/scripts/test-live-gateway-models-docker.sh index 6a68211c34b..bd114d3df3f 100755 --- a/scripts/test-live-gateway-models-docker.sh +++ b/scripts/test-live-gateway-models-docker.sh @@ -10,6 +10,8 @@ WORKSPACE_DIR="${OPENCLAW_WORKSPACE_DIR:-$HOME/.openclaw/workspace}" PROFILE_FILE="${OPENCLAW_PROFILE_FILE:-$HOME/.profile}" DOCKER_USER="${OPENCLAW_DOCKER_USER:-node}" TEMP_DIRS=() +DOCKER_HOME_MOUNT=() +DOCKER_AUTH_PRESTAGED=0 cleanup_temp_dirs() { if ((${#TEMP_DIRS[@]} > 0)); then rm -rf "${TEMP_DIRS[@]}" @@ -27,6 +29,9 @@ fi mkdir -p "$CACHE_HOME_DIR" if [[ "${CI:-}" == "true" || "${GITHUB_ACTIONS:-}" == "true" ]]; then DOCKER_USER="$(id -u):$(id -g)" + DOCKER_HOME_DIR="$(mktemp -d "${RUNNER_TEMP:-/tmp}/openclaw-docker-home.XXXXXX")" + TEMP_DIRS+=("$DOCKER_HOME_DIR") + DOCKER_HOME_MOUNT=(-v "$DOCKER_HOME_DIR":/home/node) fi PROFILE_MOUNT=() @@ -73,9 +78,15 @@ if ((${#AUTH_FILES[@]} > 0)); then AUTH_FILES_CSV="$(openclaw_live_join_csv "${AUTH_FILES[@]}")" fi +if [[ -n "${DOCKER_HOME_DIR:-}" ]]; then + openclaw_live_stage_auth_into_home "$DOCKER_HOME_DIR" "${AUTH_DIRS[@]}" --files "${AUTH_FILES[@]}" + DOCKER_AUTH_PRESTAGED=1 +fi + EXTERNAL_AUTH_MOUNTS=() if ((${#AUTH_DIRS[@]} > 0)); then for auth_dir in "${AUTH_DIRS[@]}"; do + auth_dir="$(openclaw_live_validate_relative_home_path "$auth_dir")" host_path="$HOME/$auth_dir" if [[ -d "$host_path" ]]; then EXTERNAL_AUTH_MOUNTS+=(-v "$host_path":/host-auth/"$auth_dir":ro) @@ -84,6 +95,7 @@ if ((${#AUTH_DIRS[@]} > 0)); then fi if ((${#AUTH_FILES[@]} > 0)); then for auth_file in "${AUTH_FILES[@]}"; do + auth_file="$(openclaw_live_validate_relative_home_path "$auth_file")" host_path="$HOME/$auth_file" if [[ -f "$host_path" ]]; then EXTERNAL_AUTH_MOUNTS+=(-v "$host_path":/host-auth-files/"$auth_file":ro) @@ -100,27 +112,29 @@ export NPM_CONFIG_CACHE="${NPM_CONFIG_CACHE:-$XDG_CACHE_HOME/npm}" export npm_config_cache="$NPM_CONFIG_CACHE" mkdir -p "$XDG_CACHE_HOME" "$COREPACK_HOME" "$NPM_CONFIG_CACHE" chmod 700 "$XDG_CACHE_HOME" "$COREPACK_HOME" "$NPM_CONFIG_CACHE" || true -IFS=',' read -r -a auth_dirs <<<"${OPENCLAW_DOCKER_AUTH_DIRS_RESOLVED:-}" -IFS=',' read -r -a auth_files <<<"${OPENCLAW_DOCKER_AUTH_FILES_RESOLVED:-}" -if ((${#auth_dirs[@]} > 0)); then - for auth_dir in "${auth_dirs[@]}"; do - [ -n "$auth_dir" ] || continue - if [ -d "/host-auth/$auth_dir" ]; then - mkdir -p "$HOME/$auth_dir" - cp -R "/host-auth/$auth_dir/." "$HOME/$auth_dir" - chmod -R u+rwX "$HOME/$auth_dir" || true - fi - done -fi -if ((${#auth_files[@]} > 0)); then - for auth_file in "${auth_files[@]}"; do - [ -n "$auth_file" ] || continue - if [ -f "/host-auth-files/$auth_file" ]; then - mkdir -p "$(dirname "$HOME/$auth_file")" - cp "/host-auth-files/$auth_file" "$HOME/$auth_file" - chmod u+rw "$HOME/$auth_file" || true - fi - done +if [ "${OPENCLAW_DOCKER_AUTH_PRESTAGED:-0}" != "1" ]; then + IFS=',' read -r -a auth_dirs <<<"${OPENCLAW_DOCKER_AUTH_DIRS_RESOLVED:-}" + IFS=',' read -r -a auth_files <<<"${OPENCLAW_DOCKER_AUTH_FILES_RESOLVED:-}" + if ((${#auth_dirs[@]} > 0)); then + for auth_dir in "${auth_dirs[@]}"; do + [ -n "$auth_dir" ] || continue + if [ -d "/host-auth/$auth_dir" ]; then + mkdir -p "$HOME/$auth_dir" + cp -R "/host-auth/$auth_dir/." "$HOME/$auth_dir" + chmod -R u+rwX "$HOME/$auth_dir" || true + fi + done + fi + if ((${#auth_files[@]} > 0)); then + for auth_file in "${auth_files[@]}"; do + [ -n "$auth_file" ] || continue + if [ -f "/host-auth-files/$auth_file" ]; then + mkdir -p "$(dirname "$HOME/$auth_file")" + cp "/host-auth-files/$auth_file" "$HOME/$auth_file" + chmod u+rw "$HOME/$auth_file" || true + fi + done + fi fi tmp_dir="$(mktemp -d)" cleanup() { @@ -154,6 +168,7 @@ docker run --rm -t \ -e NODE_OPTIONS=--disable-warning=ExperimentalWarning \ -e OPENCLAW_SKIP_CHANNELS=1 \ -e OPENCLAW_SUPPRESS_NOTES=1 \ + -e OPENCLAW_DOCKER_AUTH_PRESTAGED="$DOCKER_AUTH_PRESTAGED" \ -e OPENCLAW_DOCKER_AUTH_DIRS_RESOLVED="$AUTH_DIRS_CSV" \ -e OPENCLAW_DOCKER_AUTH_FILES_RESOLVED="$AUTH_FILES_CSV" \ -e OPENCLAW_LIVE_TEST=1 \ @@ -163,6 +178,7 @@ docker run --rm -t \ -e OPENCLAW_LIVE_GATEWAY_MAX_MODELS="${OPENCLAW_LIVE_GATEWAY_MAX_MODELS:-8}" \ -e OPENCLAW_LIVE_GATEWAY_STEP_TIMEOUT_MS="${OPENCLAW_LIVE_GATEWAY_STEP_TIMEOUT_MS:-45000}" \ -e OPENCLAW_LIVE_GATEWAY_MODEL_TIMEOUT_MS="${OPENCLAW_LIVE_GATEWAY_MODEL_TIMEOUT_MS:-90000}" \ + "${DOCKER_HOME_MOUNT[@]}" \ -v "$CACHE_HOME_DIR":/home/node/.cache \ -v "$ROOT_DIR":/src:ro \ -v "$CONFIG_DIR":/home/node/.openclaw \ diff --git a/scripts/test-live-models-docker.sh b/scripts/test-live-models-docker.sh index a7f2e260b02..24f8440e021 100755 --- a/scripts/test-live-models-docker.sh +++ b/scripts/test-live-models-docker.sh @@ -7,6 +7,7 @@ IMAGE_NAME="${OPENCLAW_IMAGE:-openclaw:local}" LIVE_IMAGE_NAME="${OPENCLAW_LIVE_IMAGE:-${IMAGE_NAME}-live}" PROFILE_FILE="${OPENCLAW_PROFILE_FILE:-$HOME/.profile}" DOCKER_USER="${OPENCLAW_DOCKER_USER:-node}" +DOCKER_AUTH_PRESTAGED=0 openclaw_live_truthy() { case "${1:-}" in @@ -20,6 +21,7 @@ openclaw_live_truthy() { } TEMP_DIRS=() +DOCKER_HOME_MOUNT=() cleanup_temp_dirs() { if ((${#TEMP_DIRS[@]} > 0)); then rm -rf "${TEMP_DIRS[@]}" @@ -47,6 +49,9 @@ fi mkdir -p "$CACHE_HOME_DIR" if [[ "${CI:-}" == "true" || "${GITHUB_ACTIONS:-}" == "true" ]]; then DOCKER_USER="$(id -u):$(id -g)" + DOCKER_HOME_DIR="$(mktemp -d "${RUNNER_TEMP:-/tmp}/openclaw-docker-home.XXXXXX")" + TEMP_DIRS+=("$DOCKER_HOME_DIR") + DOCKER_HOME_MOUNT=(-v "$DOCKER_HOME_DIR":/home/node) fi PROFILE_MOUNT=() @@ -103,9 +108,15 @@ if ((${#AUTH_FILES[@]} > 0)); then AUTH_FILES_CSV="$(openclaw_live_join_csv "${AUTH_FILES[@]}")" fi +if [[ -n "${DOCKER_HOME_DIR:-}" ]]; then + openclaw_live_stage_auth_into_home "$DOCKER_HOME_DIR" "${AUTH_DIRS[@]}" --files "${AUTH_FILES[@]}" + DOCKER_AUTH_PRESTAGED=1 +fi + EXTERNAL_AUTH_MOUNTS=() if ((${#AUTH_DIRS[@]} > 0)); then for auth_dir in "${AUTH_DIRS[@]}"; do + auth_dir="$(openclaw_live_validate_relative_home_path "$auth_dir")" host_path="$HOME/$auth_dir" if [[ -d "$host_path" ]]; then EXTERNAL_AUTH_MOUNTS+=(-v "$host_path":/host-auth/"$auth_dir":ro) @@ -114,6 +125,7 @@ if ((${#AUTH_DIRS[@]} > 0)); then fi if ((${#AUTH_FILES[@]} > 0)); then for auth_file in "${AUTH_FILES[@]}"; do + auth_file="$(openclaw_live_validate_relative_home_path "$auth_file")" host_path="$HOME/$auth_file" if [[ -f "$host_path" ]]; then EXTERNAL_AUTH_MOUNTS+=(-v "$host_path":/host-auth-files/"$auth_file":ro) @@ -130,27 +142,29 @@ export NPM_CONFIG_CACHE="${NPM_CONFIG_CACHE:-$XDG_CACHE_HOME/npm}" export npm_config_cache="$NPM_CONFIG_CACHE" mkdir -p "$XDG_CACHE_HOME" "$COREPACK_HOME" "$NPM_CONFIG_CACHE" chmod 700 "$XDG_CACHE_HOME" "$COREPACK_HOME" "$NPM_CONFIG_CACHE" || true -IFS=',' read -r -a auth_dirs <<<"${OPENCLAW_DOCKER_AUTH_DIRS_RESOLVED:-}" -IFS=',' read -r -a auth_files <<<"${OPENCLAW_DOCKER_AUTH_FILES_RESOLVED:-}" -if ((${#auth_dirs[@]} > 0)); then - for auth_dir in "${auth_dirs[@]}"; do - [ -n "$auth_dir" ] || continue - if [ -d "/host-auth/$auth_dir" ]; then - mkdir -p "$HOME/$auth_dir" - cp -R "/host-auth/$auth_dir/." "$HOME/$auth_dir" - chmod -R u+rwX "$HOME/$auth_dir" || true - fi - done -fi -if ((${#auth_files[@]} > 0)); then - for auth_file in "${auth_files[@]}"; do - [ -n "$auth_file" ] || continue - if [ -f "/host-auth-files/$auth_file" ]; then - mkdir -p "$(dirname "$HOME/$auth_file")" - cp "/host-auth-files/$auth_file" "$HOME/$auth_file" - chmod u+rw "$HOME/$auth_file" || true - fi - done +if [ "${OPENCLAW_DOCKER_AUTH_PRESTAGED:-0}" != "1" ]; then + IFS=',' read -r -a auth_dirs <<<"${OPENCLAW_DOCKER_AUTH_DIRS_RESOLVED:-}" + IFS=',' read -r -a auth_files <<<"${OPENCLAW_DOCKER_AUTH_FILES_RESOLVED:-}" + if ((${#auth_dirs[@]} > 0)); then + for auth_dir in "${auth_dirs[@]}"; do + [ -n "$auth_dir" ] || continue + if [ -d "/host-auth/$auth_dir" ]; then + mkdir -p "$HOME/$auth_dir" + cp -R "/host-auth/$auth_dir/." "$HOME/$auth_dir" + chmod -R u+rwX "$HOME/$auth_dir" || true + fi + done + fi + if ((${#auth_files[@]} > 0)); then + for auth_file in "${auth_files[@]}"; do + [ -n "$auth_file" ] || continue + if [ -f "/host-auth-files/$auth_file" ]; then + mkdir -p "$(dirname "$HOME/$auth_file")" + cp "/host-auth-files/$auth_file" "$HOME/$auth_file" + chmod u+rw "$HOME/$auth_file" || true + fi + done + fi fi tmp_dir="$(mktemp -d)" cleanup() { @@ -185,6 +199,7 @@ docker run --rm -t \ -e NODE_OPTIONS=--disable-warning=ExperimentalWarning \ -e OPENCLAW_SKIP_CHANNELS=1 \ -e OPENCLAW_SUPPRESS_NOTES=1 \ + -e OPENCLAW_DOCKER_AUTH_PRESTAGED="$DOCKER_AUTH_PRESTAGED" \ -e OPENCLAW_DOCKER_AUTH_DIRS_RESOLVED="$AUTH_DIRS_CSV" \ -e OPENCLAW_DOCKER_AUTH_FILES_RESOLVED="$AUTH_FILES_CSV" \ -e OPENCLAW_LIVE_TEST=1 \ @@ -196,6 +211,7 @@ docker run --rm -t \ -e OPENCLAW_LIVE_GATEWAY_MODELS="${OPENCLAW_LIVE_GATEWAY_MODELS:-}" \ -e OPENCLAW_LIVE_GATEWAY_PROVIDERS="${OPENCLAW_LIVE_GATEWAY_PROVIDERS:-}" \ -e OPENCLAW_LIVE_GATEWAY_MAX_MODELS="${OPENCLAW_LIVE_GATEWAY_MAX_MODELS:-}" \ + "${DOCKER_HOME_MOUNT[@]}" \ -v "$CACHE_HOME_DIR":/home/node/.cache \ -v "$ROOT_DIR":/src:ro \ -v "$CONFIG_DIR":/home/node/.openclaw \ diff --git a/src/agents/cli-runner/execute.ts b/src/agents/cli-runner/execute.ts index a94564a880f..fa6ea8cffe0 100644 --- a/src/agents/cli-runner/execute.ts +++ b/src/agents/cli-runner/execute.ts @@ -152,6 +152,17 @@ function formatCliEnvKeyList(keys: readonly string[]): string { return keys.length > 0 ? keys.join(",") : "none"; } +function buildCliEnvMcpLog(childEnv: Record): string { + return [ + `token=${childEnv.OPENCLAW_MCP_TOKEN ? "set" : "missing"}`, + `sessionKey=${childEnv.OPENCLAW_MCP_SESSION_KEY || ""}`, + `agentId=${childEnv.OPENCLAW_MCP_AGENT_ID || ""}`, + `accountId=${childEnv.OPENCLAW_MCP_ACCOUNT_ID || ""}`, + `messageChannel=${childEnv.OPENCLAW_MCP_MESSAGE_CHANNEL || ""}`, + `senderIsOwner=${childEnv.OPENCLAW_MCP_SENDER_IS_OWNER || ""}`, + ].join(" "); +} + export function buildCliEnvAuthLog(childEnv: Record): string { const hostKeys = listPresentCliAuthEnvKeys(process.env); const childKeys = listPresentCliAuthEnvKeys(childEnv); @@ -304,6 +315,13 @@ export async function executePreparedCliRun( }); cliBackendLog.info(`cli argv: ${backend.command} ${logArgs.join(" ")}`); cliBackendLog.info(`cli env auth: ${buildCliEnvAuthLog(env)}`); + if ( + env.OPENCLAW_MCP_TOKEN || + env.OPENCLAW_MCP_SESSION_KEY || + env.OPENCLAW_MCP_SENDER_IS_OWNER + ) { + cliBackendLog.info(`cli env mcp: ${buildCliEnvMcpLog(env)}`); + } } const noOutputTimeoutMs = resolveCliNoOutputTimeoutMs({ diff --git a/src/agents/live-model-filter.test.ts b/src/agents/live-model-filter.test.ts new file mode 100644 index 00000000000..ccfb8c79f9a --- /dev/null +++ b/src/agents/live-model-filter.test.ts @@ -0,0 +1,72 @@ +import { describe, expect, it } from "vitest"; +import { shouldExcludeProviderFromDefaultHighSignalLiveSweep } from "./live-model-filter.js"; + +describe("shouldExcludeProviderFromDefaultHighSignalLiveSweep", () => { + it("excludes dedicated harness providers from the default high-signal sweep", () => { + expect( + shouldExcludeProviderFromDefaultHighSignalLiveSweep({ + provider: "codex", + useExplicitModels: false, + providerFilter: null, + }), + ).toBe(true); + expect( + shouldExcludeProviderFromDefaultHighSignalLiveSweep({ + provider: "openai-codex", + useExplicitModels: false, + providerFilter: null, + }), + ).toBe(true); + expect( + shouldExcludeProviderFromDefaultHighSignalLiveSweep({ + provider: "codex-cli", + useExplicitModels: false, + providerFilter: null, + }), + ).toBe(true); + }); + + it("keeps dedicated harness providers when explicitly requested by provider filter", () => { + expect( + shouldExcludeProviderFromDefaultHighSignalLiveSweep({ + provider: "codex", + useExplicitModels: false, + providerFilter: new Set(["codex"]), + }), + ).toBe(false); + expect( + shouldExcludeProviderFromDefaultHighSignalLiveSweep({ + provider: "openai-codex", + useExplicitModels: false, + providerFilter: new Set(["codex-cli"]), + }), + ).toBe(false); + expect( + shouldExcludeProviderFromDefaultHighSignalLiveSweep({ + provider: "openai-codex", + useExplicitModels: false, + providerFilter: new Set(["openai"]), + }), + ).toBe(false); + }); + + it("keeps dedicated harness providers when the caller uses explicit model selection", () => { + expect( + shouldExcludeProviderFromDefaultHighSignalLiveSweep({ + provider: "codex", + useExplicitModels: true, + providerFilter: null, + }), + ).toBe(false); + }); + + it("does not exclude ordinary providers", () => { + expect( + shouldExcludeProviderFromDefaultHighSignalLiveSweep({ + provider: "openai", + useExplicitModels: false, + providerFilter: null, + }), + ).toBe(false); + }); +}); diff --git a/src/agents/live-model-filter.ts b/src/agents/live-model-filter.ts index c44c8d89115..217677353cb 100644 --- a/src/agents/live-model-filter.ts +++ b/src/agents/live-model-filter.ts @@ -1,4 +1,6 @@ +import type { OpenClawConfig } from "../config/types.openclaw.js"; import { resolveProviderModernModelRef } from "../plugins/provider-runtime.js"; +import { resolveOwningPluginIdsForProvider } from "../plugins/providers.js"; import { normalizeLowercaseStringOrEmpty } from "../shared/string-coerce.js"; import { normalizeProviderId } from "./provider-id.js"; @@ -25,6 +27,7 @@ const HIGH_SIGNAL_LIVE_MODEL_PRIORITY = [ ] as const; export const DEFAULT_HIGH_SIGNAL_LIVE_MODEL_LIMIT = HIGH_SIGNAL_LIVE_MODEL_PRIORITY.length; +const DEFAULT_HIGH_SIGNAL_LIVE_EXCLUDED_PROVIDERS = new Set(["codex", "codex-cli", "openai-codex"]); const HIGH_SIGNAL_LIVE_MODEL_PRIORITY_INDEX = new Map( HIGH_SIGNAL_LIVE_MODEL_PRIORITY.map((key, index) => [key, index]), @@ -97,6 +100,77 @@ export function isHighSignalLiveModelRef(ref: ModelRef): boolean { return isHighSignalClaudeModelId(id); } +function sharesOwningPlugin(params: { + left: string; + right: string; + config?: OpenClawConfig; + workspaceDir?: string; + env?: NodeJS.ProcessEnv; + ownerCache: Map; +}): boolean { + const resolveOwners = (provider: string): readonly string[] => { + const normalized = normalizeProviderId(provider); + const cached = params.ownerCache.get(normalized); + if (cached) { + return cached; + } + const owners = + resolveOwningPluginIdsForProvider({ + provider: normalized, + config: params.config, + workspaceDir: params.workspaceDir, + env: params.env, + }) ?? []; + params.ownerCache.set(normalized, owners); + return owners; + }; + + const leftOwners = resolveOwners(params.left); + const rightOwners = resolveOwners(params.right); + return leftOwners.some((owner) => rightOwners.includes(owner)); +} + +export function shouldExcludeProviderFromDefaultHighSignalLiveSweep(params: { + provider?: string | null; + useExplicitModels: boolean; + providerFilter?: ReadonlySet | null; + config?: OpenClawConfig; + workspaceDir?: string; + env?: NodeJS.ProcessEnv; +}): boolean { + const provider = normalizeProviderId(params.provider ?? ""); + if (!provider || params.useExplicitModels) { + return false; + } + if (!DEFAULT_HIGH_SIGNAL_LIVE_EXCLUDED_PROVIDERS.has(provider)) { + return false; + } + const ownerCache = new Map(); + for (const filterEntry of params.providerFilter ?? []) { + const requestedProvider = normalizeProviderId(filterEntry); + if (requestedProvider === provider) { + return false; + } + if ( + requestedProvider && + sharesOwningPlugin({ + left: requestedProvider, + right: provider, + config: params.config, + workspaceDir: params.workspaceDir, + env: params.env, + ownerCache, + }) + ) { + return false; + } + if (requestedProvider && DEFAULT_HIGH_SIGNAL_LIVE_EXCLUDED_PROVIDERS.has(requestedProvider)) { + return false; + } + } + return true; +} + function toCanonicalHighSignalLiveModelKey(ref: ModelRef): string | null { const provider = normalizeProviderId(ref.provider ?? ""); const rawId = normalizeLowercaseStringOrEmpty(ref.id); diff --git a/src/agents/models.profiles.live.test.ts b/src/agents/models.profiles.live.test.ts index f65527b6e24..3fd03e4e01c 100644 --- a/src/agents/models.profiles.live.test.ts +++ b/src/agents/models.profiles.live.test.ts @@ -14,6 +14,7 @@ import { isHighSignalLiveModelRef, resolveHighSignalLiveModelLimit, selectHighSignalLiveItems, + shouldExcludeProviderFromDefaultHighSignalLiveSweep, } from "./live-model-filter.js"; import { createLiveTargetMatcher } from "./live-target-matcher.js"; import { isLiveProfileKeyModeEnabled, isLiveTestEnabled } from "./live-test-helpers.js"; @@ -484,6 +485,17 @@ describeLive("live models (profile keys)", () => { continue; } if (!filter && useModern) { + if ( + shouldExcludeProviderFromDefaultHighSignalLiveSweep({ + provider: model.provider, + useExplicitModels: useExplicit, + providerFilter: providers, + config: cfg, + env: process.env, + }) + ) { + continue; + } if (!isHighSignalLiveModelRef({ provider: model.provider, id: model.id })) { continue; } diff --git a/src/agents/openai-ws-stream.e2e.test.ts b/src/agents/openai-ws-stream.e2e.test.ts index c645eed15cd..2f6392c10c0 100644 --- a/src/agents/openai-ws-stream.e2e.test.ts +++ b/src/agents/openai-ws-stream.e2e.test.ts @@ -11,7 +11,7 @@ * Run manually with a valid OPENAI_API_KEY: * OPENCLAW_LIVE_TEST=1 pnpm test:e2e -- src/agents/openai-ws-stream.e2e.test.ts * - * Skipped in CI — no API key available and we avoid billable external calls. + * This now runs only in the keyed live/release lanes. */ import type { @@ -292,7 +292,9 @@ describe("OpenAI WebSocket e2e", () => { expect(assistantText(secondDone)).toMatch(/TOOL_OK/); }, - 60_000, + // Live CI can spend more than a minute waiting for a stable follow-up turn + // when websocket reuse and tool callbacks contend with other provider lanes. + 120_000, ); testFn( @@ -376,10 +378,12 @@ describe("OpenAI WebSocket e2e", () => { const sid = freshSession("warmup"); const streamFn = openAIWsStreamModule.createOpenAIWebSocketStreamFn(API_KEY!, sid); const events = await collectEvents( - streamFn(model, makeContext("Reply with the word warmed."), { + streamFn(model, makeContext("Reply with exactly the single word warmed."), { transport: "websocket", openaiWsWarmup: true, - maxTokens: 32, + maxTokens: 8, + reasoningEffort: "none", + textVerbosity: "low", } as unknown as StreamFnParams[2]), ); @@ -391,7 +395,10 @@ describe("OpenAI WebSocket e2e", () => { expect(assistantText(done).toLowerCase()).toContain("warmed"); } }, - 45_000, + // This transport check does not need expensive reasoning. Keep the timeout + // generous for CI jitter, but force a minimal response shape so the first + // websocket request stays bounded. + 720_000, ); testFn( diff --git a/src/agents/skills.test.ts b/src/agents/skills.test.ts index a9d184aac46..d9f1d5d1262 100644 --- a/src/agents/skills.test.ts +++ b/src/agents/skills.test.ts @@ -261,7 +261,7 @@ describe("buildWorkspaceSkillCommandSpecs", () => { config, }); - const pluginRoot = path.join(tempHome!.home, ".openclaw", "extensions", "compound-bundle"); + const pluginRoot = path.join(workspaceDir, ".openclaw", "extensions", "compound-bundle"); await fs.mkdir(path.join(pluginRoot, ".claude-plugin"), { recursive: true }); await fs.mkdir(path.join(pluginRoot, "commands"), { recursive: true }); await fs.writeFile( diff --git a/src/gateway/gateway-cli-backend.live-helpers.test.ts b/src/gateway/gateway-cli-backend.live-helpers.test.ts index 3d43b8f8af9..105e7fae8ed 100644 --- a/src/gateway/gateway-cli-backend.live-helpers.test.ts +++ b/src/gateway/gateway-cli-backend.live-helpers.test.ts @@ -122,4 +122,106 @@ describe("gateway cli backend live helpers", () => { expect(shouldRunCliModelSwitchProbe("claude-cli", "claude-cli/claude-sonnet-4-6")).toBe(false); }); + + it("allows live env overrides for fresh and resume CLI args", async () => { + const { resolveCliBackendLiveArgs } = await import("./gateway-cli-backend.live-helpers.js"); + + process.env.OPENCLAW_LIVE_CLI_BACKEND_ARGS = JSON.stringify([ + "exec", + "--sandbox", + "danger-full-access", + ]); + process.env.OPENCLAW_LIVE_CLI_BACKEND_RESUME_ARGS = JSON.stringify([ + "exec", + "resume", + "{sessionId}", + "-c", + 'sandbox_mode="danger-full-access"', + ]); + + expect( + resolveCliBackendLiveArgs({ + providerId: "codex-cli", + defaultArgs: ["exec", "--sandbox", "workspace-write"], + defaultResumeArgs: [ + "exec", + "resume", + "{sessionId}", + "-c", + 'sandbox_mode="workspace-write"', + ], + }), + ).toEqual({ + args: ["exec", "--sandbox", "danger-full-access"], + resumeArgs: ["exec", "resume", "{sessionId}", "-c", 'sandbox_mode="danger-full-access"'], + }); + }); + + it("retries cancelled cron MCP replies", async () => { + const { shouldRetryCliCronMcpProbeReply } = + await import("./gateway-cli-backend.live-helpers.js"); + + expect( + shouldRetryCliCronMcpProbeReply( + "The `cron` MCP tool call was cancelled again, so the job was not created.", + ), + ).toBe(true); + expect( + shouldRetryCliCronMcpProbeReply( + "The cron tool call was cancelled again, so the job still was not created.", + ), + ).toBe(true); + expect( + shouldRetryCliCronMcpProbeReply( + "The `cron` MCP call was cancelled again, so the job was not created.", + ), + ).toBe(true); + expect( + shouldRetryCliCronMcpProbeReply( + "The cron tool call was cancelled again, so nothing was created.", + ), + ).toBe(true); + expect( + shouldRetryCliCronMcpProbeReply( + "The `cron` MCP tool call was cancelled (`user cancelled MCP tool call`).", + ), + ).toBe(true); + expect( + shouldRetryCliCronMcpProbeReply( + "The tool call was cancelled before completion, so I can’t verify the cron job was created.", + ), + ).toBe(true); + expect( + shouldRetryCliCronMcpProbeReply( + "The cron tool call was cancelled twice, so I could not create the job.", + ), + ).toBe(true); + expect( + shouldRetryCliCronMcpProbeReply( + "The cron tool call was cancelled twice, so I couldn’t create `live-mcp-67f4e9`. Please retry and I’ll do it again.", + ), + ).toBe(true); + expect( + shouldRetryCliCronMcpProbeReply( + "The cron tool call was canceled twice on the host side, so I couldn’t create `live-mcp-2d1afb`. If you want, send the same request again and I’ll retry.", + ), + ).toBe(true); + expect( + shouldRetryCliCronMcpProbeReply( + "I tried the `cron` tool call twice, but both attempts were canceled by the environment (`user cancelled MCP tool call`), so I can’t honestly reply with the success token.", + ), + ).toBe(true); + expect(shouldRetryCliCronMcpProbeReply(" ")).toBe(true); + expect( + shouldRetryCliCronMcpProbeReply( + "The cron tool call was cancelled twice, so I couldn’t create `live-mcp-932c6b`. If you want, I can try again.", + ), + ).toBe(true); + expect( + shouldRetryCliCronMcpProbeReply( + "The cron job was not created because the schedule payload was invalid.", + ), + ).toBe(false); + expect(shouldRetryCliCronMcpProbeReply("live-mcp-abc123")).toBe(false); + }); }); diff --git a/src/gateway/gateway-cli-backend.live-helpers.ts b/src/gateway/gateway-cli-backend.live-helpers.ts index 72c745a4abe..b1dc8a90818 100644 --- a/src/gateway/gateway-cli-backend.live-helpers.ts +++ b/src/gateway/gateway-cli-backend.live-helpers.ts @@ -27,11 +27,32 @@ import { type CronListJob, } from "./live-agent-probes.js"; import { renderCatFacePngBase64 } from "./live-image-probe.js"; +import { getActiveMcpLoopbackRuntime } from "./mcp-http.js"; import { extractPayloadText } from "./test-helpers.agent-results.js"; // Aggregate docker live runs can contend on startup enough that the gateway // websocket handshake needs a wider budget than the single-provider reruns. const CLI_GATEWAY_CONNECT_TIMEOUT_MS = 60_000; +// CI Docker live lanes can see repeated cancelled cron tool calls before a job +// finally sticks, and the created job may take extra time to surface via the CLI. +const CLI_CRON_MCP_PROBE_MAX_ATTEMPTS = 10; +const CLI_CRON_MCP_PROBE_VERIFY_POLLS = 20; +const CLI_CRON_MCP_PROBE_VERIFY_POLL_MS = 2_000; + +function shouldLogCliCronProbe(): boolean { + return ( + isTruthyEnvValue(process.env.OPENCLAW_LIVE_CLI_BACKEND_DEBUG) || + isTruthyEnvValue(process.env.OPENCLAW_CLI_BACKEND_LOG_OUTPUT) + ); +} + +function logCliCronProbe(step: string, details?: Record): void { + if (!shouldLogCliCronProbe()) { + return; + } + const suffix = details && Object.keys(details).length > 0 ? ` ${JSON.stringify(details)}` : ""; + console.error(`[gateway-cli-live:cron] ${step}${suffix}`); +} export type BootstrapWorkspaceContext = { expectedInjectedFiles: string[]; @@ -98,6 +119,29 @@ export function shouldRunCliMcpProbe(providerId: string): boolean { return resolveCliBackendLiveTest(providerId)?.defaultMcpProbe === true; } +export function resolveCliBackendLiveArgs(params: { + providerId: string; + defaultArgs?: string[]; + defaultResumeArgs?: string[]; +}): { args: string[]; resumeArgs?: string[] } { + const args = + parseJsonStringArray( + "OPENCLAW_LIVE_CLI_BACKEND_ARGS", + process.env.OPENCLAW_LIVE_CLI_BACKEND_ARGS, + ) ?? params.defaultArgs; + if (!args || args.length === 0) { + throw new Error( + `OPENCLAW_LIVE_CLI_BACKEND_ARGS is required for provider "${params.providerId}".`, + ); + } + const resumeArgs = + parseJsonStringArray( + "OPENCLAW_LIVE_CLI_BACKEND_RESUME_ARGS", + process.env.OPENCLAW_LIVE_CLI_BACKEND_RESUME_ARGS, + ) ?? params.defaultResumeArgs; + return { args, resumeArgs }; +} + export function resolveCliModelSwitchProbeTarget( providerId: string, modelRef: string, @@ -171,6 +215,259 @@ function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } +async function pollCliCronJobVisible(params: { + port: number; + token: string; + env: NodeJS.ProcessEnv; + expectedName: string; + expectedMessage: string; + polls?: number; + pollMs?: number; +}): Promise<{ job?: CronListJob; pollsUsed: number }> { + const polls = Math.max(1, params.polls ?? CLI_CRON_MCP_PROBE_VERIFY_POLLS); + const pollMs = Math.max(0, params.pollMs ?? CLI_CRON_MCP_PROBE_VERIFY_POLL_MS); + for (let verifyAttempt = 0; verifyAttempt < polls; verifyAttempt += 1) { + const job = await assertCronJobVisibleViaCli({ + port: params.port, + token: params.token, + env: params.env, + expectedName: params.expectedName, + expectedMessage: params.expectedMessage, + }); + if (job) { + return { job, pollsUsed: verifyAttempt + 1 }; + } + if (verifyAttempt < polls - 1) { + await sleep(pollMs); + } + } + return { pollsUsed: polls }; +} + +type LoopbackJsonRpcResponse = { + result?: unknown; + error?: { message?: string }; +}; + +async function callLoopbackJsonRpc(params: { + sessionKey: string; + senderIsOwner: boolean; + messageProvider?: string; + accountId?: string; + body: Record; +}): Promise { + const runtime = getActiveMcpLoopbackRuntime(); + if (!runtime) { + throw new Error("mcp loopback runtime is not active"); + } + const headers: Record = { + Authorization: `Bearer ${runtime.token}`, + "Content-Type": "application/json", + "x-session-key": params.sessionKey, + "x-openclaw-sender-is-owner": params.senderIsOwner ? "true" : "false", + }; + if (params.messageProvider) { + headers["x-openclaw-message-channel"] = params.messageProvider; + } + if (params.accountId) { + headers["x-openclaw-account-id"] = params.accountId; + } + const response = await fetch(`http://127.0.0.1:${runtime.port}/mcp`, { + method: "POST", + headers, + body: JSON.stringify(params.body), + }); + const text = await response.text(); + if (!response.ok) { + throw new Error(`mcp loopback http ${response.status}: ${text}`); + } + if (!text.trim()) { + return {}; + } + const parsed = JSON.parse(text) as LoopbackJsonRpcResponse; + if (parsed.error?.message) { + throw new Error(`mcp loopback json-rpc error: ${parsed.error.message}`); + } + return parsed; +} + +export async function verifyCliCronMcpLoopbackPreflight(params: { + sessionKey: string; + port: number; + token: string; + env: NodeJS.ProcessEnv; + senderIsOwner: boolean; + messageProvider?: string; + accountId?: string; +}): Promise { + const cronProbe = createLiveCronProbeSpec(); + logCliCronProbe("loopback-preflight:start", { + sessionKey: params.sessionKey, + senderIsOwner: params.senderIsOwner, + jobName: cronProbe.name, + }); + + await callLoopbackJsonRpc({ + sessionKey: params.sessionKey, + senderIsOwner: params.senderIsOwner, + messageProvider: params.messageProvider, + accountId: params.accountId, + body: { + jsonrpc: "2.0", + id: "init", + method: "initialize", + params: { protocolVersion: "2025-03-26", capabilities: {}, clientInfo: { name: "vitest" } }, + }, + }); + await callLoopbackJsonRpc({ + sessionKey: params.sessionKey, + senderIsOwner: params.senderIsOwner, + messageProvider: params.messageProvider, + accountId: params.accountId, + body: { jsonrpc: "2.0", method: "notifications/initialized" }, + }); + const toolsList = await callLoopbackJsonRpc({ + sessionKey: params.sessionKey, + senderIsOwner: params.senderIsOwner, + messageProvider: params.messageProvider, + accountId: params.accountId, + body: { jsonrpc: "2.0", id: "tools-list", method: "tools/list" }, + }); + const tools = Array.isArray((toolsList.result as { tools?: unknown[] } | undefined)?.tools) + ? (((toolsList.result as { tools?: unknown[] }).tools ?? []) as Array<{ name?: string }>) + : []; + const toolNames = tools + .map((tool) => (typeof tool.name === "string" ? tool.name : "")) + .filter(Boolean); + logCliCronProbe("loopback-preflight:tools", { + senderIsOwner: params.senderIsOwner, + toolCount: toolNames.length, + cronVisible: toolNames.includes("cron"), + }); + if (!toolNames.includes("cron")) { + throw new Error( + `mcp loopback tools/list did not expose cron (senderIsOwner=${String(params.senderIsOwner)})`, + ); + } + + const toolCall = await callLoopbackJsonRpc({ + sessionKey: params.sessionKey, + senderIsOwner: params.senderIsOwner, + messageProvider: params.messageProvider, + accountId: params.accountId, + body: { + jsonrpc: "2.0", + id: "cron-add", + method: "tools/call", + params: { + name: "cron", + arguments: JSON.parse(cronProbe.argsJson) as Record, + }, + }, + }); + const toolCallError = + (toolCall.result as { isError?: unknown } | undefined)?.isError === true || + !(toolCall.result as { content?: unknown } | undefined); + logCliCronProbe("loopback-preflight:call", { + isError: toolCallError, + jobName: cronProbe.name, + }); + if (toolCallError) { + throw new Error(`mcp loopback cron tools/call returned isError for job ${cronProbe.name}`); + } + + const { job: createdJob, pollsUsed } = await pollCliCronJobVisible({ + port: params.port, + token: params.token, + env: params.env, + expectedName: cronProbe.name, + expectedMessage: cronProbe.message, + }); + logCliCronProbe("loopback-preflight:verify", { + jobName: cronProbe.name, + pollsUsed, + createdJob: Boolean(createdJob), + }); + if (!createdJob) { + throw new Error(`mcp loopback cron tools/call did not create job ${cronProbe.name}`); + } + assertCronJobMatches({ + job: createdJob, + expectedName: cronProbe.name, + expectedMessage: cronProbe.message, + expectedSessionKey: params.sessionKey, + }); + if (createdJob.id) { + await runOpenClawCliJson( + [ + "cron", + "rm", + createdJob.id, + "--json", + "--url", + `ws://127.0.0.1:${params.port}`, + "--token", + params.token, + ], + params.env, + ); + } + logCliCronProbe("loopback-preflight:done", { jobName: cronProbe.name }); +} + +export function shouldRetryCliCronMcpProbeReply(text: string): boolean { + const normalized = normalizeLowercaseStringOrEmpty(text); + if (!normalized) { + return true; + } + const mentionsCancellation = + normalized.includes("tool call was cancelled") || + normalized.includes("tool call was canceled") || + normalized.includes("tool call was cancelled before completion") || + normalized.includes("tool call was canceled before completion") || + normalized.includes("attempts were cancelled") || + normalized.includes("attempts were canceled") || + normalized.includes("cancelled by the environment") || + normalized.includes("canceled by the environment") || + normalized.includes("mcp call was cancelled") || + normalized.includes("mcp call was canceled"); + const mentionsUserCancellation = + normalized.includes("user cancelled mcp tool call") || + normalized.includes("user canceled mcp tool call"); + const mentionsCreateFailure = + normalized.includes("could not create ") || + normalized.includes("couldn't create ") || + normalized.includes("couldn’t create ") || + normalized.includes("could not create the job") || + normalized.includes("couldn't create the job") || + normalized.includes("couldn’t create the job") || + normalized.includes("could not create job") || + normalized.includes("couldn't create job") || + normalized.includes("couldn’t create job"); + const mentionsRetryRequest = + normalized.includes("please retry") || + normalized.includes("i can try again") || + normalized.includes("i'll retry") || + normalized.includes("i’ll retry") || + normalized.includes("send the same request again"); + const mentionsMissingJob = + normalized.includes("job was not created") || + normalized.includes("job still was not created") || + normalized.includes("nothing was created") || + normalized.includes("verify the cron job was created") || + normalized.includes("was not created"); + if (mentionsUserCancellation) { + return true; + } + return ( + mentionsCancellation && (mentionsMissingJob || mentionsCreateFailure || mentionsRetryRequest) + ); +} + +function getCliBackendProbeThinking(providerId: string): "low" | undefined { + return normalizeLowercaseStringOrEmpty(providerId) === "codex-cli" ? "low" : undefined; +} + export async function connectTestGatewayClient(params: { url: string; token: string; @@ -368,6 +665,7 @@ export async function verifyCliBackendImageProbe(params: { tempDir: string; bootstrapWorkspace: BootstrapWorkspaceContext | null; }): Promise { + const thinking = getCliBackendProbeThinking(params.providerId); const imageBase64 = renderCatFacePngBase64(); const runIdImage = randomUUID(); const imageProbe = await params.client.request( @@ -389,6 +687,7 @@ export async function verifyCliBackendImageProbe(params: { }, ], deliver: false, + ...(thinking ? { thinking } : {}), }, { expectFinal: true }, ); @@ -407,11 +706,18 @@ export async function verifyCliCronMcpProbe(params: { env: NodeJS.ProcessEnv; }): Promise { const cronProbe = createLiveCronProbeSpec(); + const thinking = getCliBackendProbeThinking(params.providerId); let createdJob: CronListJob | undefined; let lastCronText = ""; - for (let attempt = 0; attempt < 2 && !createdJob; attempt += 1) { + for (let attempt = 0; attempt < CLI_CRON_MCP_PROBE_MAX_ATTEMPTS && !createdJob; attempt += 1) { + logCliCronProbe("agent-attempt:start", { + attempt, + providerId: params.providerId, + sessionKey: params.sessionKey, + expectedJob: cronProbe.name, + }); const runIdMcp = randomUUID(); const cronResult = await params.client.request( "agent", @@ -425,6 +731,7 @@ export async function verifyCliCronMcpProbe(params: { exactReply: cronProbe.name, }), deliver: false, + ...(thinking ? { thinking } : {}), }, { expectFinal: true }, ); @@ -432,22 +739,37 @@ export async function verifyCliCronMcpProbe(params: { throw new Error(`cron mcp probe failed: status=${String(cronResult?.status)}`); } lastCronText = extractPayloadText(cronResult?.result).trim(); - createdJob = await assertCronJobVisibleViaCli({ + const retryableReply = shouldRetryCliCronMcpProbeReply(lastCronText); + logCliCronProbe("agent-attempt:reply", { + attempt, + retryableReply, + reply: lastCronText, + }); + const verifyResult = await pollCliCronJobVisible({ port: params.port, token: params.token, env: params.env, expectedName: cronProbe.name, expectedMessage: cronProbe.message, }); - if (!createdJob && attempt === 1) { + createdJob = verifyResult.job; + logCliCronProbe("agent-attempt:verify", { + attempt, + pollsUsed: verifyResult.pollsUsed, + createdJob: Boolean(createdJob), + retryableReply, + }); + if (!createdJob && !retryableReply) { throw new Error( - `cron cli verify could not find job ${cronProbe.name}: reply=${JSON.stringify(lastCronText)}`, + `cron cli verify could not find job ${cronProbe.name} after attempt ${attempt + 1}: reply=${JSON.stringify(lastCronText)}`, ); } } if (!createdJob) { - throw new Error(`cron cli verify did not create job ${cronProbe.name}`); + throw new Error( + `cron cli verify did not create job ${cronProbe.name} after ${CLI_CRON_MCP_PROBE_MAX_ATTEMPTS} attempts: reply=${JSON.stringify(lastCronText)}`, + ); } assertCronJobMatches({ job: createdJob, diff --git a/src/gateway/gateway-cli-backend.live.test.ts b/src/gateway/gateway-cli-backend.live.test.ts index bce1ec7d336..287ddfd68cf 100644 --- a/src/gateway/gateway-cli-backend.live.test.ts +++ b/src/gateway/gateway-cli-backend.live.test.ts @@ -15,14 +15,16 @@ import { getFreeGatewayPort, matchesCliBackendReply, parseImageMode, - parseJsonStringArray, resolveCliModelSwitchProbeTarget, + resolveCliBackendLiveArgs, + parseJsonStringArray, restoreCliBackendLiveEnv, shouldRunCliImageProbe, shouldRunCliModelSwitchProbe, shouldRunCliMcpProbe, snapshotCliBackendLiveEnv, type SystemPromptReport, + verifyCliCronMcpLoopbackPreflight, verifyCliCronMcpProbe, verifyCliBackendImageProbe, withClaudeMcpConfigOverrides, @@ -40,7 +42,9 @@ const describeLive = LIVE && CLI_LIVE ? describe : describe.skip; const DEFAULT_PROVIDER = "claude-cli"; const DEFAULT_MODEL = resolveCliBackendLiveTest(DEFAULT_PROVIDER)?.defaultModelRef ?? "claude-cli/claude-sonnet-4-6"; -const CLI_BACKEND_LIVE_TIMEOUT_MS = 420_000; +// The cron/MCP live probe now tolerates more cancelled tool-call retries in CI, +// so the outer test budget needs enough headroom to finish those retries. +const CLI_BACKEND_LIVE_TIMEOUT_MS = 720_000; function logCliBackendLiveStep(step: string, details?: Record): void { if (!CLI_DEBUG) { @@ -104,14 +108,11 @@ describeLive("gateway live (cli backend)", () => { ); } - const baseCliArgs = - parseJsonStringArray( - "OPENCLAW_LIVE_CLI_BACKEND_ARGS", - process.env.OPENCLAW_LIVE_CLI_BACKEND_ARGS, - ) ?? providerDefaults?.args; - if (!baseCliArgs || baseCliArgs.length === 0) { - throw new Error(`OPENCLAW_LIVE_CLI_BACKEND_ARGS is required for provider "${providerId}".`); - } + const { args: baseCliArgs, resumeArgs: baseCliResumeArgs } = resolveCliBackendLiveArgs({ + providerId, + defaultArgs: providerDefaults?.args, + defaultResumeArgs: providerDefaults?.resumeArgs, + }); const cliClearEnv = parseJsonStringArray( @@ -190,6 +191,7 @@ describeLive("gateway live (cli backend)", () => { [providerId]: { command: cliCommand, args: cliArgs, + resumeArgs: baseCliResumeArgs, clearEnv: filteredCliClearEnv.length > 0 ? filteredCliClearEnv : undefined, env: Object.keys(preservedCliEnv).length > 0 ? preservedCliEnv : undefined, systemPromptWhen: providerDefaults?.systemPromptWhen ?? "never", @@ -352,6 +354,18 @@ describeLive("gateway live (cli backend)", () => { } if (enableCliMcpProbe) { + logCliBackendLiveStep("cron-mcp-loopback-preflight:start", { + sessionKey, + senderIsOwner: true, + }); + await verifyCliCronMcpLoopbackPreflight({ + sessionKey, + port, + token, + env: process.env, + senderIsOwner: true, + }); + logCliBackendLiveStep("cron-mcp-loopback-preflight:done"); logCliBackendLiveStep("cron-mcp-probe:start", { sessionKey }); await verifyCliCronMcpProbe({ client, diff --git a/src/gateway/gateway-codex-harness.live-helpers.test.ts b/src/gateway/gateway-codex-harness.live-helpers.test.ts new file mode 100644 index 00000000000..23bb426b514 --- /dev/null +++ b/src/gateway/gateway-codex-harness.live-helpers.test.ts @@ -0,0 +1,50 @@ +import { describe, expect, it } from "vitest"; +import { + EXPECTED_CODEX_MODELS_COMMAND_TEXT, + isExpectedCodexModelsCommandText, +} from "./gateway-codex-harness.live-helpers.js"; + +describe("gateway codex harness live helpers", () => { + it("accepts the interactive model-selection summary emitted by current codex", () => { + const text = [ + "`/codex models` opened an interactive model-selection prompt rather than printing a plain list.", + "", + "Visible options in this session:", + "- `GPT-5.4`", + "- `GPT-5.3-Codex` (listed as the existing model)", + "", + "Current active model is `codex/gpt-5.4`.", + ].join("\n"); + + expect( + EXPECTED_CODEX_MODELS_COMMAND_TEXT.some((expectedText) => text.includes(expectedText)), + ).toBe(true); + expect(isExpectedCodexModelsCommandText(text)).toBe(true); + }); + + it("accepts the configured-model fallback summary", () => { + const text = [ + "Configured models in this session:", + "- `codex/gpt-5.4`", + "Current session model is `codex/gpt-5.4`.", + ].join("\n"); + + expect(isExpectedCodexModelsCommandText(text)).toBe(true); + }); + + it("rejects unrelated codex command output", () => { + expect(isExpectedCodexModelsCommandText("Codex is healthy.")).toBe(false); + }); + + it("rejects generic current-status output that is not a model listing", () => { + const text = [ + "Current: waiting for the Codex CLI to finish booting.", + "Try again in a few seconds.", + ].join("\n"); + + expect( + EXPECTED_CODEX_MODELS_COMMAND_TEXT.some((expectedText) => text.includes(expectedText)), + ).toBe(false); + expect(isExpectedCodexModelsCommandText(text)).toBe(false); + }); +}); diff --git a/src/gateway/gateway-codex-harness.live-helpers.ts b/src/gateway/gateway-codex-harness.live-helpers.ts new file mode 100644 index 00000000000..7715699930d --- /dev/null +++ b/src/gateway/gateway-codex-harness.live-helpers.ts @@ -0,0 +1,95 @@ +export const EXPECTED_CODEX_MODELS_COMMAND_TEXT = [ + "Codex models:", + "Available Codex models", + "Available agent target:", + "Available agent targets:", + "opened an interactive trust prompt", + "opened an interactive model-selection prompt", + "running as Codex on `codex/", + "currently running on `codex/", + "stdin is not a terminal", + "The local `codex models` entrypoint is interactive in this environment", + "`codex models` did not run in this environment.", + "`codex models` failed in this sandbox", + "`codex models` could not be run in this sandbox.", + "`codex models` is not runnable in this sandboxed session.", + "I couldn’t get a direct `codex models` CLI listing because the local sandbox blocked that command.", + "I couldn’t list all installed/available Codex models from the local CLI because the sandboxed `codex` command failed to start in this environment.", + "I couldn’t get `codex models` from the CLI because the sandbox blocks the namespace setup it needs", + "I can only see the current session model from this environment", + "Available in this session:", + "Available models in this session:", + "Available models in this environment:", + "Available models in this Codex environment:", + "Available agent models:", + "Visible options in this session:", + "Current: `codex/", + "Current model:", + "Current model: `codex/", + "Current model is `codex/", + "Current session model: `codex/", + "Current session model is `codex/", + "The current session is using `codex/", + "Configured model from `~/.codex/config.toml`:", + "Configured models in this session:", + "Default model:", + "This harness is configured with a single Codex model: `codex/", + "Primary model: `codex/", + "Registered models: `codex/", + "Current active model is `codex/", + "Current OpenClaw session status reports the active model as:", +] as const; + +export function isExpectedCodexModelsCommandText(text: string): boolean { + const normalized = text.toLowerCase(); + const isSandboxFallback = + text.includes("`codex models`") && + (text.includes("did not run") || + text.includes("could not run") || + text.includes("could not be run") || + text.includes("failed in this sandbox") || + text.includes("failed with:") || + text.includes("repo-local fallback") || + text.includes("sandbox blocks") || + text.includes("interactive in this environment") || + text.includes("sandboxed session") || + text.includes("required user namespace")); + + const mentionsConfiguredModels = + normalized.includes("configured model") || + normalized.includes("configured codex model") || + normalized.includes("configured models"); + const mentionsSessionModel = + normalized.includes("current session is using") || + normalized.includes("current session model") || + normalized.includes("the current session is using"); + const mentionsConfigSummary = + normalized.includes("default model") || + normalized.includes("primary model") || + normalized.includes("registered models") || + normalized.includes("only listed model") || + normalized.includes("single codex model") || + normalized.includes("live openclaw config shows") || + normalized.includes("current gateway config"); + const isSessionConfigFallback = + text.includes("`codex/") && + ((mentionsConfiguredModels && mentionsSessionModel) || + (mentionsConfigSummary && (mentionsConfiguredModels || mentionsSessionModel))); + + const mentionsInteractiveSelection = + normalized.includes("interactive model-selection prompt") || + normalized.includes("interactive model selection prompt"); + const mentionsVisibleOptions = + normalized.includes("visible options in this session:") || + normalized.includes("visible options:"); + const mentionsCurrentActiveModel = + normalized.includes("current active model is `codex/") || + normalized.includes("current active model is codex/"); + const isInteractiveSelectionSummary = + text.includes("`/codex models`") && + mentionsInteractiveSelection && + mentionsVisibleOptions && + mentionsCurrentActiveModel; + + return isSandboxFallback || isSessionConfigFallback || isInteractiveSelectionSummary; +} diff --git a/src/gateway/gateway-codex-harness.live.test.ts b/src/gateway/gateway-codex-harness.live.test.ts index c84dea6c8b4..e6079317312 100644 --- a/src/gateway/gateway-codex-harness.live.test.ts +++ b/src/gateway/gateway-codex-harness.live.test.ts @@ -9,6 +9,10 @@ import type { OpenClawConfig } from "../config/config.js"; import type { DeviceIdentity } from "../infra/device-identity.js"; import { isTruthyEnvValue } from "../infra/env.js"; import type { GatewayClient } from "./client.js"; +import { + EXPECTED_CODEX_MODELS_COMMAND_TEXT, + isExpectedCodexModelsCommandText, +} from "./gateway-codex-harness.live-helpers.js"; import { assertCronJobMatches, assertCronJobVisibleViaCli, @@ -27,6 +31,8 @@ const CODEX_HARNESS_IMAGE_PROBE = isTruthyEnvValue( process.env.OPENCLAW_LIVE_CODEX_HARNESS_IMAGE_PROBE, ); const CODEX_HARNESS_MCP_PROBE = isTruthyEnvValue(process.env.OPENCLAW_LIVE_CODEX_HARNESS_MCP_PROBE); +const CODEX_HARNESS_AUTH_MODE = + process.env.OPENCLAW_LIVE_CODEX_HARNESS_AUTH === "api-key" ? "api-key" : "codex-auth"; const describeLive = LIVE && CODEX_HARNESS_LIVE ? describe : describe.skip; const describeDisabled = LIVE && !CODEX_HARNESS_LIVE ? describe : describe.skip; const CODEX_HARNESS_TIMEOUT_MS = 420_000; @@ -38,6 +44,7 @@ type EnvSnapshot = { configPath?: string; gatewayToken?: string; openaiApiKey?: string; + openaiBaseUrl?: string; skipBrowserControl?: string; skipCanvas?: string; skipChannels?: string; @@ -60,6 +67,7 @@ function snapshotEnv(): EnvSnapshot { configPath: process.env.OPENCLAW_CONFIG_PATH, gatewayToken: process.env.OPENCLAW_GATEWAY_TOKEN, openaiApiKey: process.env.OPENAI_API_KEY, + openaiBaseUrl: process.env.OPENAI_BASE_URL, skipBrowserControl: process.env.OPENCLAW_SKIP_BROWSER_CONTROL_SERVER, skipCanvas: process.env.OPENCLAW_SKIP_CANVAS_HOST, skipChannels: process.env.OPENCLAW_SKIP_CHANNELS, @@ -74,6 +82,7 @@ function restoreEnv(snapshot: EnvSnapshot): void { restoreEnvVar("OPENCLAW_CONFIG_PATH", snapshot.configPath); restoreEnvVar("OPENCLAW_GATEWAY_TOKEN", snapshot.gatewayToken); restoreEnvVar("OPENAI_API_KEY", snapshot.openaiApiKey); + restoreEnvVar("OPENAI_BASE_URL", snapshot.openaiBaseUrl); restoreEnvVar("OPENCLAW_SKIP_BROWSER_CONTROL_SERVER", snapshot.skipBrowserControl); restoreEnvVar("OPENCLAW_SKIP_CANVAS_HOST", snapshot.skipCanvas); restoreEnvVar("OPENCLAW_SKIP_CHANNELS", snapshot.skipChannels); @@ -273,6 +282,7 @@ async function requestCodexCommandText(params: { client: GatewayClient; command: string; expectedText: string | string[]; + isExpectedText?: (text: string) => boolean; sessionKey: string; }): Promise { const { extractPayloadText } = await import("./test-helpers.agent-results.js"); @@ -296,8 +306,10 @@ async function requestCodexCommandText(params: { const expectedTexts = Array.isArray(params.expectedText) ? params.expectedText : [params.expectedText]; + const matchedByText = expectedTexts.some((expectedText) => text.includes(expectedText)); + const matchedByPredicate = params.isExpectedText?.(text) ?? false; expect( - expectedTexts.some((expectedText) => text.includes(expectedText)), + matchedByText || matchedByPredicate, `Expected "${params.command}" response to contain one of: ${expectedTexts.join(", ")}\nReceived:\n${text}`, ).toBe(true); return text; @@ -411,10 +423,6 @@ describeLive("gateway live (Codex harness)", () => { "runs gateway agent turns through the plugin-owned Codex app-server harness", async () => { const modelKey = process.env.OPENCLAW_LIVE_CODEX_HARNESS_MODEL ?? DEFAULT_CODEX_MODEL; - const openaiKey = process.env.OPENAI_API_KEY?.trim(); - if (!openaiKey) { - throw new Error("OPENAI_API_KEY is required for the Codex harness live test."); - } const { clearRuntimeConfigSnapshot } = await import("../config/config.js"); const { startGatewayServer } = await import("./server.js"); @@ -429,6 +437,17 @@ describeLive("gateway live (Codex harness)", () => { clearRuntimeConfigSnapshot(); process.env.OPENCLAW_AGENT_RUNTIME = "codex"; process.env.OPENCLAW_AGENT_HARNESS_FALLBACK = "none"; + // Keep the runtime fixed on the plugin-owned Codex app-server harness. + // CI can opt into API-key auth to avoid stale OAuth refresh secrets, + // while local maintainer runs can continue exercising staged ~/.codex auth. + // Only the Codex-auth path should force-clear OpenAI overrides; API-key + // mode may intentionally point at a custom endpoint. + if (CODEX_HARNESS_AUTH_MODE !== "api-key") { + delete process.env.OPENAI_BASE_URL; + delete process.env.OPENAI_API_KEY; + } else if (!process.env.OPENAI_BASE_URL?.trim()) { + delete process.env.OPENAI_BASE_URL; + } process.env.OPENCLAW_CONFIG_PATH = configPath; process.env.OPENCLAW_GATEWAY_TOKEN = token; process.env.OPENCLAW_SKIP_BROWSER_CONTROL_SERVER = "1"; @@ -500,18 +519,8 @@ describeLive("gateway live (Codex harness)", () => { client, sessionKey, command: "/codex models", - expectedText: [ - "Codex models:", - "Available Codex models", - "Available agent target:", - "Available agent targets:", - "opened an interactive trust prompt", - "running as Codex on `codex/", - "currently running on `codex/", - "stdin is not a terminal", - "Configured model from `~/.codex/config.toml`:", - "Current OpenClaw session status reports the active model as:", - ], + expectedText: [...EXPECTED_CODEX_MODELS_COMMAND_TEXT], + isExpectedText: isExpectedCodexModelsCommandText, }); logCodexLiveStep("codex-models-command", { modelsText }); diff --git a/src/gateway/gateway-models.profiles.live.test.ts b/src/gateway/gateway-models.profiles.live.test.ts index e3a748a3e6b..5ed3a6e338d 100644 --- a/src/gateway/gateway-models.profiles.live.test.ts +++ b/src/gateway/gateway-models.profiles.live.test.ts @@ -7,7 +7,8 @@ import type { Api, Model } from "@mariozechner/pi-ai"; import { afterEach, describe, expect, it } from "vitest"; import { resolveOpenClawAgentDir } from "../agents/agent-paths.js"; import { resolveAgentWorkspaceDir } from "../agents/agent-scope.js"; -import type { AuthProfileStore } from "../agents/auth-profiles.js"; +import { ensureAuthProfileStore, saveAuthProfileStore } from "../agents/auth-profiles/store.js"; +import type { AuthProfileStore } from "../agents/auth-profiles/types.js"; import { collectAnthropicApiKeys, isAnthropicBillingError, @@ -20,6 +21,7 @@ import { isHighSignalLiveModelRef, resolveHighSignalLiveModelLimit, selectHighSignalLiveItems, + shouldExcludeProviderFromDefaultHighSignalLiveSweep, } from "../agents/live-model-filter.js"; import { createLiveTargetMatcher } from "../agents/live-target-matcher.js"; import { isLiveProfileKeyModeEnabled, isLiveTestEnabled } from "../agents/live-test-helpers.js"; @@ -29,7 +31,8 @@ import { shouldSuppressBuiltInModel } from "../agents/model-suppression.js"; import { ensureOpenClawModelsJson } from "../agents/models-config.js"; import { isRateLimitErrorMessage } from "../agents/pi-embedded-helpers/errors.js"; import { discoverAuthStorage, discoverModels } from "../agents/pi-model-discovery.js"; -import type { ModelsConfig, OpenClawConfig, ModelProviderConfig } from "../config/types.js"; +import { clearRuntimeConfigSnapshot, loadConfig } from "../config/io.js"; +import type { ModelsConfig, ModelProviderConfig, OpenClawConfig } from "../config/types.js"; import { isTruthyEnvValue } from "../infra/env.js"; import { normalizeGoogleModelId } from "../plugin-sdk/google-model-id.js"; import { DEFAULT_AGENT_ID } from "../routing/session-key.js"; @@ -43,7 +46,7 @@ import { shouldRetryExecReadProbe, shouldRetryToolReadProbe, } from "./live-tool-probe-utils.js"; -import { startGatewayServer } from "./server.js"; +import { startGatewayServer } from "./server.impl.js"; import { loadSessionEntry, readSessionMessages } from "./session-utils.js"; const ZAI_FALLBACK = isTruthyEnvValue(process.env.OPENCLAW_LIVE_GATEWAY_ZAI_FALLBACK); @@ -233,19 +236,6 @@ async function withGatewayLiveModelTimeout(operation: Promise, context: st }); } -let gatewayConfigModulePromise: Promise | undefined; -let authProfilesModulePromise: Promise | undefined; - -async function getGatewayConfigModule() { - gatewayConfigModulePromise ??= import("../config/config.js"); - return await gatewayConfigModulePromise; -} - -async function getAuthProfilesModule() { - authProfilesModulePromise ??= import("../agents/auth-profiles.js"); - return await authProfilesModulePromise; -} - function logProgress(message: string): void { process.stderr.write(`[live] ${message}\n`); } @@ -1250,7 +1240,6 @@ async function sanitizeAuthConfig(params: { if (!auth) { return auth; } - const { ensureAuthProfileStore } = await getAuthProfilesModule(); const store = ensureAuthProfileStore(params.agentDir, { allowKeychainPrompt: false, }); @@ -1311,7 +1300,7 @@ function buildMinimaxProviderOverride(params: { } async function runGatewayModelSuite(params: GatewayModelSuiteParams) { - (await getGatewayConfigModule()).clearRuntimeConfigSnapshot(); + clearRuntimeConfigSnapshot(); const runtimeEnv = enterProductionEnvForLiveRun(); const previous = { configPath: process.env.OPENCLAW_CONFIG_PATH, @@ -1343,7 +1332,6 @@ async function runGatewayModelSuite(params: GatewayModelSuiteParams) { const agentId = "dev"; const hostAgentDir = resolveOpenClawAgentDir(); - const { ensureAuthProfileStore, saveAuthProfileStore } = await getAuthProfilesModule(); const hostStore = ensureAuthProfileStore(hostAgentDir, { allowKeychainPrompt: false, }); @@ -1396,6 +1384,9 @@ async function runGatewayModelSuite(params: GatewayModelSuiteParams) { await fs.writeFile(modelsPath, `${JSON.stringify({ providers: liveProviders }, null, 2)}\n`); } + // Keep the broad live Docker suite on the impl entrypoint. The lazy public + // boundary (`./server.js`) is covered elsewhere, but under Vitest's live Docker + // worker this path can trip a Node module-status loader bug during startup. let server: Awaited> | undefined; let client: GatewayClient | undefined; try { @@ -2006,7 +1997,7 @@ async function runGatewayModelSuite(params: GatewayModelSuiteParams) { logProgress(`[${params.label}] skipped all models (missing profiles)`); } } finally { - (await getGatewayConfigModule()).clearRuntimeConfigSnapshot(); + clearRuntimeConfigSnapshot(); restoreProductionEnvForLiveRun(runtimeEnv); client.stop(); await server.close({ reason: "live test complete" }); @@ -2040,8 +2031,7 @@ describeLive("gateway live (dev agent, profile keys)", () => { "runs meaningful prompts across models with available keys", async () => await withSuppressedGatewayLiveWarnings(async () => { - const { loadConfig } = await getGatewayConfigModule(); - (await getGatewayConfigModule()).clearRuntimeConfigSnapshot(); + clearRuntimeConfigSnapshot(); const cfg = loadConfig(); await ensureOpenClawModelsJson(cfg); @@ -2063,7 +2053,16 @@ describeLive("gateway live (dev agent, profile keys)", () => { }); const wanted = filter ? all.filter((m) => targetMatcher.matchesModel(m.provider, m.id)) - : all.filter((m) => isHighSignalLiveModelRef({ provider: m.provider, id: m.id })); + : all.filter( + (m) => + !shouldExcludeProviderFromDefaultHighSignalLiveSweep({ + provider: m.provider, + useExplicitModels: useExplicit, + providerFilter: PROVIDERS, + config: cfg, + env: process.env, + }) && isHighSignalLiveModelRef({ provider: m.provider, id: m.id }), + ); const candidates: Array> = []; const skipped: Array<{ model: string; error: string }> = []; @@ -2164,8 +2163,7 @@ describeLive("gateway live (dev agent, profile keys)", () => { if (!ZAI_FALLBACK) { return; } - const { loadConfig } = await getGatewayConfigModule(); - (await getGatewayConfigModule()).clearRuntimeConfigSnapshot(); + clearRuntimeConfigSnapshot(); const runtimeEnv = enterProductionEnvForLiveRun(); const previous = { configPath: process.env.OPENCLAW_CONFIG_PATH, @@ -2323,10 +2321,7 @@ describeLive("gateway live (dev agent, profile keys)", () => { throw new Error(`zai followup missing nonce: ${followupText}`); } } finally { - { - const { clearRuntimeConfigSnapshot } = await getGatewayConfigModule(); - clearRuntimeConfigSnapshot(); - } + clearRuntimeConfigSnapshot(); restoreProductionEnvForLiveRun(runtimeEnv); client.stop(); await server.close({ reason: "live test complete" }); diff --git a/src/gateway/live-agent-probes.test.ts b/src/gateway/live-agent-probes.test.ts index 90bd4c77b56..e6786af2f01 100644 --- a/src/gateway/live-agent-probes.test.ts +++ b/src/gateway/live-agent-probes.test.ts @@ -28,7 +28,7 @@ describe("live-agent-probes", () => { attempt: 1, exactReply: spec.name, }), - ).toContain("Return only a tool call"); + ).toContain(`reply exactly: ${spec.name}`); expect( buildLiveCronProbeMessage({ agent: "codex", @@ -36,7 +36,15 @@ describe("live-agent-probes", () => { attempt: 1, exactReply: spec.name, }), - ).toContain("No prose before the tool call"); + ).toContain("ask me to retry"); + expect( + buildLiveCronProbeMessage({ + agent: "codex", + argsJson: spec.argsJson, + attempt: 1, + exactReply: spec.name, + }), + ).toContain("previous OpenClaw cron MCP tool call was cancelled"); }); it("validates cron cli job shape for the shared live probe", () => { diff --git a/src/gateway/live-agent-probes.ts b/src/gateway/live-agent-probes.ts index 38c6ae02e33..b340c049c8e 100644 --- a/src/gateway/live-agent-probes.ts +++ b/src/gateway/live-agent-probes.ts @@ -85,15 +85,22 @@ export function buildLiveCronProbeMessage(params: { } if (family === "claude") { return ( - "Return only a tool call for the OpenClaw MCP tool `cron`. " + + "Retry the OpenClaw MCP tool named `cron` now. " + `Use these exact JSON arguments: ${params.argsJson}. ` + - "No prose. I will verify externally with the OpenClaw cron CLI." + `If the cron job is created, reply exactly: ${params.exactReply}. ` + + "If the tool call is cancelled, the job is not created, or you cannot confirm creation, " + + "reply briefly saying that and ask me to retry. No markdown. " + + "I will verify externally with the OpenClaw cron CLI." ); } return ( - "Use the OpenClaw MCP tool named cron. " + + "Your previous OpenClaw cron MCP tool call was cancelled before the job was created. " + + "Retry the OpenClaw MCP tool named cron now. " + `Use these exact JSON arguments: ${params.argsJson}. ` + - "No prose before the tool call. I will verify externally with the OpenClaw cron CLI." + `If the cron job is created, reply exactly: ${params.exactReply}. ` + + "If the tool call is cancelled, the job is not created, or you cannot confirm creation, " + + "reply briefly saying that and ask me to retry. No markdown. " + + "I will verify externally with the OpenClaw cron CLI." ); } diff --git a/src/gateway/mcp-http.request.ts b/src/gateway/mcp-http.request.ts index 687f97f4591..412297eb23f 100644 --- a/src/gateway/mcp-http.request.ts +++ b/src/gateway/mcp-http.request.ts @@ -1,6 +1,7 @@ import type { IncomingMessage, ServerResponse } from "node:http"; import { resolveMainSessionKey } from "../config/sessions.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; +import { isTruthyEnvValue } from "../infra/env.js"; import { safeEqualSecret } from "../security/secret-equal.js"; import { normalizeOptionalLowercaseString, @@ -13,6 +14,20 @@ import { checkBrowserOrigin } from "./origin-check.js"; const MAX_MCP_BODY_BYTES = 1_048_576; +function shouldLogMcpLoopbackHttp(): boolean { + return ( + isTruthyEnvValue(process.env.OPENCLAW_CLI_BACKEND_LOG_OUTPUT) || + isTruthyEnvValue(process.env.OPENCLAW_LIVE_CLI_BACKEND_DEBUG) + ); +} + +function logMcpLoopbackHttp(step: string, details: Record): void { + if (!shouldLogMcpLoopbackHttp()) { + return; + } + console.error(`[mcp-loopback] ${step} ${JSON.stringify(details)}`); +} + export type McpRequestContext = { sessionKey: string; messageProvider: string | undefined; @@ -57,6 +72,7 @@ export function validateMcpLoopbackRequest(params: { try { url = new URL(params.req.url ?? "/", `http://${params.req.headers.host ?? "localhost"}`); } catch { + logMcpLoopbackHttp("reject", { reason: "bad_request_url", method: params.req.method ?? "" }); params.res.writeHead(400, { "Content-Type": "application/json" }); params.res.end(JSON.stringify({ error: "bad_request" })); return false; @@ -69,18 +85,33 @@ export function validateMcpLoopbackRequest(params: { } if (url.pathname !== "/mcp") { + logMcpLoopbackHttp("reject", { + reason: "not_found", + method: params.req.method ?? "", + path: url.pathname, + }); params.res.writeHead(404, { "Content-Type": "application/json" }); params.res.end(JSON.stringify({ error: "not_found" })); return false; } if (params.req.method !== "POST") { + logMcpLoopbackHttp("reject", { + reason: "method_not_allowed", + method: params.req.method ?? "", + path: url.pathname, + }); params.res.writeHead(405, { Allow: "POST" }); params.res.end(); return false; } if (rejectsBrowserLoopbackRequest(params.req)) { + logMcpLoopbackHttp("reject", { + reason: "forbidden_origin", + method: params.req.method ?? "", + origin: getHeader(params.req, "origin") ?? "", + }); params.res.writeHead(403, { "Content-Type": "application/json" }); params.res.end(JSON.stringify({ error: "forbidden" })); return false; @@ -88,6 +119,11 @@ export function validateMcpLoopbackRequest(params: { const authHeader = getHeader(params.req, "authorization") ?? ""; if (!safeEqualSecret(authHeader, `Bearer ${params.token}`)) { + logMcpLoopbackHttp("reject", { + reason: "unauthorized", + method: params.req.method ?? "", + hasAuthorization: authHeader.length > 0, + }); params.res.writeHead(401, { "Content-Type": "application/json" }); params.res.end(JSON.stringify({ error: "unauthorized" })); return false; @@ -95,6 +131,11 @@ export function validateMcpLoopbackRequest(params: { const contentType = getHeader(params.req, "content-type") ?? ""; if (!contentType.startsWith("application/json")) { + logMcpLoopbackHttp("reject", { + reason: "unsupported_media_type", + method: params.req.method ?? "", + contentType, + }); params.res.writeHead(415, { "Content-Type": "application/json" }); params.res.end(JSON.stringify({ error: "unsupported_media_type" })); return false; diff --git a/src/gateway/mcp-http.ts b/src/gateway/mcp-http.ts index 2b91e4e8921..2f8c38de05f 100644 --- a/src/gateway/mcp-http.ts +++ b/src/gateway/mcp-http.ts @@ -1,6 +1,7 @@ import crypto from "node:crypto"; import { createServer as createHttpServer } from "node:http"; import { loadConfig } from "../config/config.js"; +import { isTruthyEnvValue } from "../infra/env.js"; import { formatErrorMessage } from "../infra/errors.js"; import { logDebug, logWarn } from "../logger.js"; import { handleMcpJsonRpc } from "./mcp-http.handlers.js"; @@ -31,6 +32,24 @@ type McpLoopbackServer = { let activeMcpLoopbackServer: McpLoopbackServer | undefined; let activeMcpLoopbackServerPromise: Promise | null = null; +function shouldLogMcpLoopbackTraffic(): boolean { + return ( + isTruthyEnvValue(process.env.OPENCLAW_CLI_BACKEND_LOG_OUTPUT) || + isTruthyEnvValue(process.env.OPENCLAW_LIVE_CLI_BACKEND_DEBUG) + ); +} + +function logMcpLoopbackTraffic(step: string, details: Record): void { + if (!shouldLogMcpLoopbackTraffic()) { + return; + } + console.error(`[mcp-loopback] ${step} ${JSON.stringify(details)}`); +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + export async function startMcpLoopbackServer(port = 0): Promise<{ port: number; close: () => Promise; @@ -58,6 +77,14 @@ export async function startMcpLoopbackServer(port = 0): Promise<{ }); const messages = Array.isArray(parsed) ? parsed : [parsed]; + logMcpLoopbackTraffic("request", { + batchSize: messages.length, + methods: messages.map((message) => message.method), + sessionKey: requestContext.sessionKey, + senderIsOwner: requestContext.senderIsOwner, + toolCount: scopedTools.toolSchema.length, + cronVisible: scopedTools.toolSchema.some((tool) => tool.name === "cron"), + }); const responses: object[] = []; for (const message of messages) { const response = await handleMcpJsonRpc({ @@ -66,6 +93,17 @@ export async function startMcpLoopbackServer(port = 0): Promise<{ toolSchema: scopedTools.toolSchema, }); if (response !== null) { + const toolName = + message.method === "tools/call" && isRecord(message.params) + ? message.params.name + : undefined; + const isError = + isRecord(response) && isRecord(response.result) && response.result.isError === true; + logMcpLoopbackTraffic("response", { + method: message.method, + toolName: typeof toolName === "string" ? toolName : undefined, + isError, + }); responses.push(response); } } @@ -83,6 +121,9 @@ export async function startMcpLoopbackServer(port = 0): Promise<{ res.end(payload); } catch (error) { logWarn(`mcp loopback: request handling failed: ${formatErrorMessage(error)}`); + logMcpLoopbackTraffic("request-failed", { + message: formatErrorMessage(error), + }); if (!res.headersSent) { res.writeHead(400, { "Content-Type": "application/json" }); res.end(JSON.stringify(jsonRpcError(null, -32700, "Parse error"))); diff --git a/src/gateway/server.canvas-auth.test.ts b/src/gateway/server.canvas-auth.test.ts index 7391af0845a..e2af3d31a60 100644 --- a/src/gateway/server.canvas-auth.test.ts +++ b/src/gateway/server.canvas-auth.test.ts @@ -13,7 +13,7 @@ import { withTempConfig } from "./test-temp-config.js"; const WS_REJECT_TIMEOUT_MS = 2_000; const WS_CONNECT_TIMEOUT_MS = 5_000; -const HTTP_REQUEST_TIMEOUT_MS = 5_000; +const HTTP_REQUEST_TIMEOUT_MS = 15_000; const SERVER_CLOSE_TIMEOUT_MS = 5_000; async function fetchCanvas(input: string, init?: RequestInit): Promise { diff --git a/src/scripts/prepare-codex-ci-config.test.ts b/src/scripts/prepare-codex-ci-config.test.ts new file mode 100644 index 00000000000..d52cd850b1c --- /dev/null +++ b/src/scripts/prepare-codex-ci-config.test.ts @@ -0,0 +1,49 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { + buildCiSafeCodexConfig, + writeCiSafeCodexConfig, +} from "../../scripts/prepare-codex-ci-config.ts"; +import { withTempDir } from "../test-utils/temp-dir.js"; + +describe("prepare-codex-ci-config", () => { + it("renders a minimal trusted non-interactive Codex config for the target repo", () => { + expect( + buildCiSafeCodexConfig({ + projectPath: "/tmp/openclaw-pr-sync.xph5uu", + }), + ).toBe( + [ + "# Generated for Codex CI runs.", + "# Keep the checked-out repo trusted while avoiding maintainer-local", + "# provider/profile overrides that do not exist on CI runners.", + 'approval_policy = "never"', + 'sandbox_mode = "workspace-write"', + "", + '[projects."/tmp/openclaw-pr-sync.xph5uu"]', + 'trust_level = "trusted"', + "", + ].join("\n"), + ); + }); + + it("writes the generated config to disk", async () => { + await withTempDir("codex-ci-config-", async (tempDir) => { + const outputPath = path.join(tempDir, ".codex", "config.toml"); + const projectPath = path.join(tempDir, "repo"); + + await writeCiSafeCodexConfig({ + outputPath, + projectPath, + }); + + await expect(fs.readFile(outputPath, "utf-8")).resolves.toContain( + `approval_policy = "never"`, + ); + await expect(fs.readFile(outputPath, "utf-8")).resolves.toContain( + `[projects."${projectPath}"]`, + ); + }); + }); +}); diff --git a/test/scripts/test-live-cli-backend-docker.test.ts b/test/scripts/test-live-cli-backend-docker.test.ts new file mode 100644 index 00000000000..807a175ce17 --- /dev/null +++ b/test/scripts/test-live-cli-backend-docker.test.ts @@ -0,0 +1,22 @@ +import fs from "node:fs"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; + +const SCRIPT_PATH = path.resolve( + import.meta.dirname, + "../../scripts/test-live-cli-backend-docker.sh", +); + +function readForwardedDockerEnvVars(): string[] { + const script = fs.readFileSync(SCRIPT_PATH, "utf8"); + return Array.from(script.matchAll(/-e\s+([A-Z0-9_]+)=/g), (match) => match[1] ?? ""); +} + +describe("scripts/test-live-cli-backend-docker.sh", () => { + it("forwards both fresh and resume CLI arg overrides into the Docker container", () => { + const forwardedVars = readForwardedDockerEnvVars(); + + expect(forwardedVars).toContain("OPENCLAW_LIVE_CLI_BACKEND_ARGS"); + expect(forwardedVars).toContain("OPENCLAW_LIVE_CLI_BACKEND_RESUME_ARGS"); + }); +});