From 8326349939bcfe5b161e4ed9fc656e60d3a0ad47 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 6 Apr 2026 15:30:57 +0100 Subject: [PATCH] fix(test): stabilize docker claude cli live lane --- scripts/gateway-cli-bootstrap-live-probe.ts | 59 +- scripts/test-live-cli-backend-docker.sh | 8 +- src/gateway/gateway-cli-backend.live.test.ts | 575 ++++++++++--------- 3 files changed, 354 insertions(+), 288 deletions(-) diff --git a/scripts/gateway-cli-bootstrap-live-probe.ts b/scripts/gateway-cli-bootstrap-live-probe.ts index 4431e2bc130..6b859cd11d5 100644 --- a/scripts/gateway-cli-bootstrap-live-probe.ts +++ b/scripts/gateway-cli-bootstrap-live-probe.ts @@ -2,7 +2,7 @@ import { randomUUID } from "node:crypto"; import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; -import { clearRuntimeConfigSnapshot, loadConfig } from "../src/config/config.js"; +import { clearRuntimeConfigSnapshot, type OpenClawConfig } from "../src/config/config.js"; import { GatewayClient } from "../src/gateway/client.js"; import { startGatewayServer } from "../src/gateway/server.js"; import { extractPayloadText } from "../src/gateway/test-helpers.agent-results.js"; @@ -19,6 +19,12 @@ const DEFAULT_CLAUDE_ARGS = [ "bypassPermissions", ]; const DEFAULT_CLEAR_ENV = ["ANTHROPIC_API_KEY", "ANTHROPIC_API_KEY_OLD"]; +const CLI_BOOTSTRAP_TIMEOUT_MS = 300_000; +const GATEWAY_CONNECT_TIMEOUT_MS = 30_000; + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} function withMcpConfigOverrides(args: string[], mcpConfigPath: string): string[] { const next = [...args]; @@ -32,8 +38,37 @@ function withMcpConfigOverrides(args: string[], mcpConfigPath: string): string[] } async function connectClient(params: { url: string; token: string }) { + const startedAt = Date.now(); + let attempt = 0; + let lastError: Error | null = null; + + while (Date.now() - startedAt < GATEWAY_CONNECT_TIMEOUT_MS) { + attempt += 1; + const remainingMs = GATEWAY_CONNECT_TIMEOUT_MS - (Date.now() - startedAt); + if (remainingMs <= 0) { + break; + } + try { + return await connectClientOnce({ + ...params, + timeoutMs: Math.min(remainingMs, 10_000), + }); + } catch (error) { + lastError = error instanceof Error ? error : new Error(String(error)); + if (!isRetryableGatewayConnectError(lastError) || remainingMs <= 2_000) { + throw lastError; + } + await sleep(Math.min(500 * attempt, 2_000)); + } + } + + throw lastError ?? new Error("gateway connect timeout"); +} + +async function connectClientOnce(params: { url: string; token: string; timeoutMs: number }) { return await new Promise((resolve, reject) => { let done = false; + let client: GatewayClient | undefined; const finish = (result: { client?: GatewayClient; error?: Error }) => { if (done) { return; @@ -41,17 +76,22 @@ async function connectClient(params: { url: string; token: string }) { done = true; clearTimeout(connectTimeout); if (result.error) { + if (client) { + void client.stopAndWait({ timeoutMs: 1_000 }).catch(() => {}); + } reject(result.error); return; } resolve(result.client as GatewayClient); }; - const client = new GatewayClient({ + client = new GatewayClient({ url: params.url, token: params.token, clientName: GATEWAY_CLIENT_NAMES.TEST, clientVersion: "dev", mode: "test", + requestTimeoutMs: params.timeoutMs, + connectChallengeTimeoutMs: params.timeoutMs, onHelloOk: () => finish({ client }), onConnectError: (error) => finish({ error }), onClose: (code, reason) => @@ -59,13 +99,22 @@ async function connectClient(params: { url: string; token: string }) { }); const connectTimeout = setTimeout( () => finish({ error: new Error("gateway connect timeout") }), - 10_000, + params.timeoutMs, ); connectTimeout.unref(); client.start(); }); } +function isRetryableGatewayConnectError(error: Error): boolean { + const message = error.message.toLowerCase(); + return ( + message.includes("gateway closed during connect (1000)") || + message.includes("gateway connect timeout") || + message.includes("gateway connect challenge timeout") + ); +} + async function getFreeGatewayPort(): Promise { return await getFreePortBlockWithPermissionFallback({ offsets: [0, 1, 2, 4], @@ -98,7 +147,7 @@ async function main() { await fs.writeFile(path.join(workspaceDir, "IDENTITY.md"), `${identitySecret}\n`); await fs.writeFile(path.join(workspaceDir, "USER.md"), `${userSecret}\n`); - const cfg = loadConfig(); + const cfg: OpenClawConfig = {}; const existingBackends = cfg.agents?.defaults?.cliBackends ?? {}; const claudeBackend = existingBackends["claude-cli"] ?? {}; const cliCommand = @@ -166,7 +215,7 @@ async function main() { message: `BOOTSTRAP_CHECK ${randomUUID()}`, deliver: false, }, - { expectFinal: true, timeoutMs: 60_000 }, + { expectFinal: true, timeoutMs: CLI_BOOTSTRAP_TIMEOUT_MS }, ); const text = extractPayloadText(payload?.result); process.stdout.write( diff --git a/scripts/test-live-cli-backend-docker.sh b/scripts/test-live-cli-backend-docker.sh index 766323f9360..5e43696e26f 100644 --- a/scripts/test-live-cli-backend-docker.sh +++ b/scripts/test-live-cli-backend-docker.sh @@ -144,7 +144,13 @@ tar -C /src \ --exclude=ui/dist \ --exclude=ui/node_modules \ -cf - . | tar -C "$tmp_dir" -xf - -ln -s /app/node_modules "$tmp_dir/node_modules" +# Use a writable node_modules overlay in the temp repo. Vite writes bundled +# config artifacts under the nearest node_modules/.vite-temp path, and the +# build-stage /app/node_modules tree is root-owned in this Docker lane. +mkdir -p "$tmp_dir/node_modules" +cp -aRs /app/node_modules/. "$tmp_dir/node_modules" +rm -rf "$tmp_dir/node_modules/.vite-temp" +mkdir -p "$tmp_dir/node_modules/.vite-temp" ln -s /app/dist "$tmp_dir/dist" if [ -d /app/dist-runtime/extensions ]; then export OPENCLAW_BUNDLED_PLUGINS_DIR=/app/dist-runtime/extensions diff --git a/src/gateway/gateway-cli-backend.live.test.ts b/src/gateway/gateway-cli-backend.live.test.ts index 2ebd4dd5499..b734467cc72 100644 --- a/src/gateway/gateway-cli-backend.live.test.ts +++ b/src/gateway/gateway-cli-backend.live.test.ts @@ -6,7 +6,7 @@ import path from "node:path"; import { describe, expect, it } from "vitest"; import { isLiveTestEnabled } from "../agents/live-test-helpers.js"; import { parseModelRef } from "../agents/model-selection.js"; -import { clearRuntimeConfigSnapshot, loadConfig, type OpenClawConfig } from "../config/config.js"; +import { clearRuntimeConfigSnapshot, type OpenClawConfig } from "../config/config.js"; import { isTruthyEnvValue } from "../infra/env.js"; import { getFreePortBlockWithPermissionFallback } from "../test-utils/ports.js"; import { GATEWAY_CLIENT_NAMES } from "../utils/message-channel.js"; @@ -22,6 +22,9 @@ const CLI_RESUME = isTruthyEnvValue(process.env.OPENCLAW_LIVE_CLI_BACKEND_RESUME const describeLive = LIVE && CLI_LIVE ? describe : describe.skip; const DEFAULT_MODEL = "claude-cli/claude-sonnet-4-6"; +const CLI_BACKEND_LIVE_TIMEOUT_MS = 180_000; +const CLI_BOOTSTRAP_LIVE_TIMEOUT_MS = 300_000; +const CLI_GATEWAY_CONNECT_TIMEOUT_MS = 30_000; const BOOTSTRAP_LIVE_MODEL = process.env.OPENCLAW_LIVE_CLI_BACKEND_MODEL ?? DEFAULT_MODEL; const describeClaudeBootstrapLive = LIVE && CLI_LIVE && BOOTSTRAP_LIVE_MODEL.startsWith("claude-cli/") ? describe : describe.skip; @@ -186,7 +189,7 @@ async function connectClient(params: { url: string; token: string }) { const connectTimeout = setTimeout( () => finish({ error: new Error("gateway connect timeout") }), - 10_000, + CLI_GATEWAY_CONNECT_TIMEOUT_MS, ); connectTimeout.unref(); client.start(); @@ -218,7 +221,7 @@ async function runGatewayCliBootstrapLiveProbe(): Promise<{ const timeout = setTimeout(() => { child.kill("SIGTERM"); reject(new Error(`bootstrap probe timed out\nstdout:\n${stdout}\nstderr:\n${stderr}`)); - }, 120_000); + }, CLI_BOOTSTRAP_LIVE_TIMEOUT_MS); timeout.unref(); child.stdout.setEncoding("utf8"); child.stderr.setEncoding("utf8"); @@ -257,306 +260,314 @@ async function runGatewayCliBootstrapLiveProbe(): Promise<{ } describeLive("gateway live (cli backend)", () => { - it("runs the agent pipeline against the local CLI backend", async () => { - const preservedEnv = new Set( - parseJsonStringArray( - "OPENCLAW_LIVE_CLI_BACKEND_PRESERVE_ENV", - process.env.OPENCLAW_LIVE_CLI_BACKEND_PRESERVE_ENV, - ) ?? [], - ); - - clearRuntimeConfigSnapshot(); - const previous = { - configPath: process.env.OPENCLAW_CONFIG_PATH, - token: process.env.OPENCLAW_GATEWAY_TOKEN, - skipChannels: process.env.OPENCLAW_SKIP_CHANNELS, - skipGmail: process.env.OPENCLAW_SKIP_GMAIL_WATCHER, - skipCron: process.env.OPENCLAW_SKIP_CRON, - skipCanvas: process.env.OPENCLAW_SKIP_CANVAS_HOST, - anthropicApiKey: process.env.ANTHROPIC_API_KEY, - anthropicApiKeyOld: process.env.ANTHROPIC_API_KEY_OLD, - }; - - process.env.OPENCLAW_SKIP_CHANNELS = "1"; - process.env.OPENCLAW_SKIP_GMAIL_WATCHER = "1"; - process.env.OPENCLAW_SKIP_CRON = "1"; - process.env.OPENCLAW_SKIP_CANVAS_HOST = "1"; - if (!preservedEnv.has("ANTHROPIC_API_KEY")) { - delete process.env.ANTHROPIC_API_KEY; - } - if (!preservedEnv.has("ANTHROPIC_API_KEY_OLD")) { - delete process.env.ANTHROPIC_API_KEY_OLD; - } - - const token = `test-${randomUUID()}`; - process.env.OPENCLAW_GATEWAY_TOKEN = token; - - const rawModel = process.env.OPENCLAW_LIVE_CLI_BACKEND_MODEL ?? DEFAULT_MODEL; - const parsed = parseModelRef(rawModel, "claude-cli"); - if (!parsed) { - throw new Error( - `OPENCLAW_LIVE_CLI_BACKEND_MODEL must resolve to a CLI backend model. Got: ${rawModel}`, + it( + "runs the agent pipeline against the local CLI backend", + async () => { + const preservedEnv = new Set( + parseJsonStringArray( + "OPENCLAW_LIVE_CLI_BACKEND_PRESERVE_ENV", + process.env.OPENCLAW_LIVE_CLI_BACKEND_PRESERVE_ENV, + ) ?? [], ); - } - const providerId = parsed.provider; - const modelKey = `${providerId}/${parsed.model}`; - const providerDefaults = - providerId === "claude-cli" - ? { command: "claude", args: DEFAULT_CLAUDE_ARGS } - : providerId === "codex-cli" - ? { command: "codex", args: DEFAULT_CODEX_ARGS } - : null; + clearRuntimeConfigSnapshot(); + const previous = { + configPath: process.env.OPENCLAW_CONFIG_PATH, + token: process.env.OPENCLAW_GATEWAY_TOKEN, + skipChannels: process.env.OPENCLAW_SKIP_CHANNELS, + skipGmail: process.env.OPENCLAW_SKIP_GMAIL_WATCHER, + skipCron: process.env.OPENCLAW_SKIP_CRON, + skipCanvas: process.env.OPENCLAW_SKIP_CANVAS_HOST, + anthropicApiKey: process.env.ANTHROPIC_API_KEY, + anthropicApiKeyOld: process.env.ANTHROPIC_API_KEY_OLD, + }; - const cliCommand = process.env.OPENCLAW_LIVE_CLI_BACKEND_COMMAND ?? providerDefaults?.command; - if (!cliCommand) { - throw new Error( - `OPENCLAW_LIVE_CLI_BACKEND_COMMAND is required for provider "${providerId}".`, + process.env.OPENCLAW_SKIP_CHANNELS = "1"; + process.env.OPENCLAW_SKIP_GMAIL_WATCHER = "1"; + process.env.OPENCLAW_SKIP_CRON = "1"; + process.env.OPENCLAW_SKIP_CANVAS_HOST = "1"; + if (!preservedEnv.has("ANTHROPIC_API_KEY")) { + delete process.env.ANTHROPIC_API_KEY; + } + if (!preservedEnv.has("ANTHROPIC_API_KEY_OLD")) { + delete process.env.ANTHROPIC_API_KEY_OLD; + } + + const token = `test-${randomUUID()}`; + process.env.OPENCLAW_GATEWAY_TOKEN = token; + + const rawModel = process.env.OPENCLAW_LIVE_CLI_BACKEND_MODEL ?? DEFAULT_MODEL; + const parsed = parseModelRef(rawModel, "claude-cli"); + if (!parsed) { + throw new Error( + `OPENCLAW_LIVE_CLI_BACKEND_MODEL must resolve to a CLI backend model. Got: ${rawModel}`, + ); + } + const providerId = parsed.provider; + const modelKey = `${providerId}/${parsed.model}`; + + const providerDefaults = + providerId === "claude-cli" + ? { command: "claude", args: DEFAULT_CLAUDE_ARGS } + : providerId === "codex-cli" + ? { command: "codex", args: DEFAULT_CODEX_ARGS } + : null; + + const cliCommand = process.env.OPENCLAW_LIVE_CLI_BACKEND_COMMAND ?? providerDefaults?.command; + if (!cliCommand) { + throw new Error( + `OPENCLAW_LIVE_CLI_BACKEND_COMMAND is required for provider "${providerId}".`, + ); + } + const baseCliArgs = + parseJsonStringArray( + "OPENCLAW_LIVE_CLI_BACKEND_ARGS", + process.env.OPENCLAW_LIVE_CLI_BACKEND_ARGS, + ) ?? providerDefaults?.args; + if (!baseCliArgs || baseCliArgs.length === 0) { + throw new Error(`OPENCLAW_LIVE_CLI_BACKEND_ARGS is required for provider "${providerId}".`); + } + const cliClearEnv = + parseJsonStringArray( + "OPENCLAW_LIVE_CLI_BACKEND_CLEAR_ENV", + process.env.OPENCLAW_LIVE_CLI_BACKEND_CLEAR_ENV, + ) ?? (providerId === "claude-cli" ? DEFAULT_CLEAR_ENV : []); + const filteredCliClearEnv = cliClearEnv.filter((name) => !preservedEnv.has(name)); + const preservedCliEnv = Object.fromEntries( + [...preservedEnv] + .map((name) => [name, process.env[name]]) + .filter((entry): entry is [string, string] => typeof entry[1] === "string"), ); - } - const baseCliArgs = - parseJsonStringArray( - "OPENCLAW_LIVE_CLI_BACKEND_ARGS", - process.env.OPENCLAW_LIVE_CLI_BACKEND_ARGS, - ) ?? providerDefaults?.args; - if (!baseCliArgs || baseCliArgs.length === 0) { - throw new Error(`OPENCLAW_LIVE_CLI_BACKEND_ARGS is required for provider "${providerId}".`); - } - const cliClearEnv = - parseJsonStringArray( - "OPENCLAW_LIVE_CLI_BACKEND_CLEAR_ENV", - process.env.OPENCLAW_LIVE_CLI_BACKEND_CLEAR_ENV, - ) ?? (providerId === "claude-cli" ? DEFAULT_CLEAR_ENV : []); - const filteredCliClearEnv = cliClearEnv.filter((name) => !preservedEnv.has(name)); - const preservedCliEnv = Object.fromEntries( - [...preservedEnv] - .map((name) => [name, process.env[name]]) - .filter((entry): entry is [string, string] => typeof entry[1] === "string"), - ); - const cliImageArg = process.env.OPENCLAW_LIVE_CLI_BACKEND_IMAGE_ARG?.trim() || undefined; - const cliImageMode = parseImageMode(process.env.OPENCLAW_LIVE_CLI_BACKEND_IMAGE_MODE); + const cliImageArg = process.env.OPENCLAW_LIVE_CLI_BACKEND_IMAGE_ARG?.trim() || undefined; + const cliImageMode = parseImageMode(process.env.OPENCLAW_LIVE_CLI_BACKEND_IMAGE_MODE); - if (cliImageMode && !cliImageArg) { - throw new Error( - "OPENCLAW_LIVE_CLI_BACKEND_IMAGE_MODE requires OPENCLAW_LIVE_CLI_BACKEND_IMAGE_ARG.", - ); - } + if (cliImageMode && !cliImageArg) { + throw new Error( + "OPENCLAW_LIVE_CLI_BACKEND_IMAGE_MODE requires OPENCLAW_LIVE_CLI_BACKEND_IMAGE_ARG.", + ); + } - const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-live-cli-")); - const disableMcpConfig = process.env.OPENCLAW_LIVE_CLI_BACKEND_DISABLE_MCP_CONFIG !== "0"; - let cliArgs = baseCliArgs; - if (providerId === "claude-cli" && disableMcpConfig) { - const mcpConfigPath = path.join(tempDir, "claude-mcp.json"); - await fs.writeFile(mcpConfigPath, `${JSON.stringify({ mcpServers: {} }, null, 2)}\n`); - cliArgs = withMcpConfigOverrides(baseCliArgs, mcpConfigPath); - } + const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-live-cli-")); + const disableMcpConfig = process.env.OPENCLAW_LIVE_CLI_BACKEND_DISABLE_MCP_CONFIG !== "0"; + let cliArgs = baseCliArgs; + if (providerId === "claude-cli" && disableMcpConfig) { + const mcpConfigPath = path.join(tempDir, "claude-mcp.json"); + await fs.writeFile(mcpConfigPath, `${JSON.stringify({ mcpServers: {} }, null, 2)}\n`); + cliArgs = withMcpConfigOverrides(baseCliArgs, mcpConfigPath); + } - const cfg = loadConfig(); - const cfgWithCliBackends = cfg as OpenClawConfig & { - agents?: { - defaults?: { - cliBackends?: Record>; + const cfg: OpenClawConfig = {}; + const cfgWithCliBackends = cfg as OpenClawConfig & { + agents?: { + defaults?: { + cliBackends?: Record>; + }; }; }; - }; - const existingBackends = cfgWithCliBackends.agents?.defaults?.cliBackends ?? {}; - const nextCfg = { - ...cfg, - agents: { - ...cfg.agents, - defaults: { - ...cfg.agents?.defaults, - model: { primary: modelKey }, - models: { - [modelKey]: {}, - }, - cliBackends: { - ...existingBackends, - [providerId]: { - command: cliCommand, - args: cliArgs, - clearEnv: filteredCliClearEnv.length > 0 ? filteredCliClearEnv : undefined, - env: Object.keys(preservedCliEnv).length > 0 ? preservedCliEnv : undefined, - systemPromptWhen: "never", - ...(cliImageArg ? { imageArg: cliImageArg, imageMode: cliImageMode } : {}), + const existingBackends = cfgWithCliBackends.agents?.defaults?.cliBackends ?? {}; + const nextCfg = { + ...cfg, + agents: { + ...cfg.agents, + defaults: { + ...cfg.agents?.defaults, + model: { primary: modelKey }, + models: { + [modelKey]: {}, }, - }, - sandbox: { mode: "off" }, - }, - }, - }; - const tempConfigPath = path.join(tempDir, "openclaw.json"); - await fs.writeFile(tempConfigPath, `${JSON.stringify(nextCfg, null, 2)}\n`); - process.env.OPENCLAW_CONFIG_PATH = tempConfigPath; - - const port = await getFreeGatewayPort(); - const server = await startGatewayServer(port, { - bind: "loopback", - auth: { mode: "token", token }, - controlUiEnabled: false, - }); - - const client = await connectClient({ - url: `ws://127.0.0.1:${port}`, - token, - }); - - try { - const sessionKey = "agent:dev:live-cli-backend"; - const runId = randomUUID(); - const nonce = randomBytes(3).toString("hex").toUpperCase(); - const message = - providerId === "codex-cli" - ? `Please include the token CLI-BACKEND-${nonce} in your reply.` - : `Reply with exactly: CLI backend OK ${nonce}.`; - const payload = await client.request( - "agent", - { - sessionKey, - idempotencyKey: `idem-${runId}`, - message, - deliver: false, - }, - { expectFinal: true }, - ); - if (payload?.status !== "ok") { - throw new Error(`agent status=${String(payload?.status)}`); - } - const text = extractPayloadText(payload?.result); - if (providerId === "codex-cli") { - expect(text).toContain(`CLI-BACKEND-${nonce}`); - } else { - expect(text).toContain(`CLI backend OK ${nonce}.`); - } - - if (CLI_RESUME) { - const runIdResume = randomUUID(); - const resumeNonce = randomBytes(3).toString("hex").toUpperCase(); - const resumeMessage = - providerId === "codex-cli" - ? `Please include the token CLI-RESUME-${resumeNonce} in your reply.` - : `Reply with exactly: CLI backend RESUME OK ${resumeNonce}.`; - const resumePayload = await client.request( - "agent", - { - sessionKey, - idempotencyKey: `idem-${runIdResume}`, - message: resumeMessage, - deliver: false, - }, - { expectFinal: true }, - ); - if (resumePayload?.status !== "ok") { - throw new Error(`resume status=${String(resumePayload?.status)}`); - } - const resumeText = extractPayloadText(resumePayload?.result); - if (providerId === "codex-cli") { - expect(resumeText).toContain(`CLI-RESUME-${resumeNonce}`); - } else { - expect(resumeText).toContain(`CLI backend RESUME OK ${resumeNonce}.`); - } - } - - if (CLI_IMAGE) { - // Shorter code => less OCR flake across providers, still tests image attachments end-to-end. - const imageCode = randomImageProbeCode(); - const imageBase64 = renderCatNoncePngBase64(imageCode); - const runIdImage = randomUUID(); - - const imageProbe = await client.request( - "agent", - { - sessionKey, - idempotencyKey: `idem-${runIdImage}-image`, - message: - "Look at the attached image. Reply with exactly two tokens separated by a single space: " + - "(1) the animal shown or written in the image, lowercase; " + - "(2) the code printed in the image, uppercase. No extra text.", - attachments: [ - { - mimeType: "image/png", - fileName: `probe-${runIdImage}.png`, - content: imageBase64, + cliBackends: { + ...existingBackends, + [providerId]: { + command: cliCommand, + args: cliArgs, + clearEnv: filteredCliClearEnv.length > 0 ? filteredCliClearEnv : undefined, + env: Object.keys(preservedCliEnv).length > 0 ? preservedCliEnv : undefined, + systemPromptWhen: "never", + ...(cliImageArg ? { imageArg: cliImageArg, imageMode: cliImageMode } : {}), }, - ], + }, + sandbox: { mode: "off" }, + }, + }, + }; + const tempConfigPath = path.join(tempDir, "openclaw.json"); + await fs.writeFile(tempConfigPath, `${JSON.stringify(nextCfg, null, 2)}\n`); + process.env.OPENCLAW_CONFIG_PATH = tempConfigPath; + + const port = await getFreeGatewayPort(); + const server = await startGatewayServer(port, { + bind: "loopback", + auth: { mode: "token", token }, + controlUiEnabled: false, + }); + + const client = await connectClient({ + url: `ws://127.0.0.1:${port}`, + token, + }); + + try { + const sessionKey = "agent:dev:live-cli-backend"; + const runId = randomUUID(); + const nonce = randomBytes(3).toString("hex").toUpperCase(); + const message = + providerId === "codex-cli" + ? `Please include the token CLI-BACKEND-${nonce} in your reply.` + : `Reply with exactly: CLI backend OK ${nonce}.`; + const payload = await client.request( + "agent", + { + sessionKey, + idempotencyKey: `idem-${runId}`, + message, deliver: false, }, { expectFinal: true }, ); - if (imageProbe?.status !== "ok") { - throw new Error(`image probe failed: status=${String(imageProbe?.status)}`); + if (payload?.status !== "ok") { + throw new Error(`agent status=${String(payload?.status)}`); } - const imageText = extractPayloadText(imageProbe?.result); - if (!/\bcat\b/i.test(imageText)) { - throw new Error(`image probe missing 'cat': ${imageText}`); + const text = extractPayloadText(payload?.result); + if (providerId === "codex-cli") { + expect(text).toContain(`CLI-BACKEND-${nonce}`); + } else { + expect(text).toContain(`CLI backend OK ${nonce}.`); } - const candidates = imageText.toUpperCase().match(/[A-Z0-9]{6,20}/g) ?? []; - const bestDistance = candidates.reduce((best, cand) => { - if (Math.abs(cand.length - imageCode.length) > 2) { - return best; + + if (CLI_RESUME) { + const runIdResume = randomUUID(); + const resumeNonce = randomBytes(3).toString("hex").toUpperCase(); + const resumeMessage = + providerId === "codex-cli" + ? `Please include the token CLI-RESUME-${resumeNonce} in your reply.` + : `Reply with exactly: CLI backend RESUME OK ${resumeNonce}.`; + const resumePayload = await client.request( + "agent", + { + sessionKey, + idempotencyKey: `idem-${runIdResume}`, + message: resumeMessage, + deliver: false, + }, + { expectFinal: true }, + ); + if (resumePayload?.status !== "ok") { + throw new Error(`resume status=${String(resumePayload?.status)}`); } - return Math.min(best, editDistance(cand, imageCode)); - }, Number.POSITIVE_INFINITY); - if (!(bestDistance <= 5)) { - throw new Error(`image probe missing code (${imageCode}): ${imageText}`); + const resumeText = extractPayloadText(resumePayload?.result); + if (providerId === "codex-cli") { + expect(resumeText).toContain(`CLI-RESUME-${resumeNonce}`); + } else { + expect(resumeText).toContain(`CLI backend RESUME OK ${resumeNonce}.`); + } + } + + if (CLI_IMAGE) { + // Shorter code => less OCR flake across providers, still tests image attachments end-to-end. + const imageCode = randomImageProbeCode(); + const imageBase64 = renderCatNoncePngBase64(imageCode); + const runIdImage = randomUUID(); + + const imageProbe = await client.request( + "agent", + { + sessionKey, + idempotencyKey: `idem-${runIdImage}-image`, + message: + "Look at the attached image. Reply with exactly two tokens separated by a single space: " + + "(1) the animal shown or written in the image, lowercase; " + + "(2) the code printed in the image, uppercase. No extra text.", + attachments: [ + { + mimeType: "image/png", + fileName: `probe-${runIdImage}.png`, + content: imageBase64, + }, + ], + deliver: false, + }, + { expectFinal: true }, + ); + if (imageProbe?.status !== "ok") { + throw new Error(`image probe failed: status=${String(imageProbe?.status)}`); + } + const imageText = extractPayloadText(imageProbe?.result); + if (!/\bcat\b/i.test(imageText)) { + throw new Error(`image probe missing 'cat': ${imageText}`); + } + const candidates = imageText.toUpperCase().match(/[A-Z0-9]{6,20}/g) ?? []; + const bestDistance = candidates.reduce((best, cand) => { + if (Math.abs(cand.length - imageCode.length) > 2) { + return best; + } + return Math.min(best, editDistance(cand, imageCode)); + }, Number.POSITIVE_INFINITY); + if (!(bestDistance <= 5)) { + throw new Error(`image probe missing code (${imageCode}): ${imageText}`); + } + } + } finally { + clearRuntimeConfigSnapshot(); + await client.stopAndWait(); + await server.close(); + await fs.rm(tempDir, { recursive: true, force: true }); + if (previous.configPath === undefined) { + delete process.env.OPENCLAW_CONFIG_PATH; + } else { + process.env.OPENCLAW_CONFIG_PATH = previous.configPath; + } + if (previous.token === undefined) { + delete process.env.OPENCLAW_GATEWAY_TOKEN; + } else { + process.env.OPENCLAW_GATEWAY_TOKEN = previous.token; + } + if (previous.skipChannels === undefined) { + delete process.env.OPENCLAW_SKIP_CHANNELS; + } else { + process.env.OPENCLAW_SKIP_CHANNELS = previous.skipChannels; + } + if (previous.skipGmail === undefined) { + delete process.env.OPENCLAW_SKIP_GMAIL_WATCHER; + } else { + process.env.OPENCLAW_SKIP_GMAIL_WATCHER = previous.skipGmail; + } + if (previous.skipCron === undefined) { + delete process.env.OPENCLAW_SKIP_CRON; + } else { + process.env.OPENCLAW_SKIP_CRON = previous.skipCron; + } + if (previous.skipCanvas === undefined) { + delete process.env.OPENCLAW_SKIP_CANVAS_HOST; + } else { + process.env.OPENCLAW_SKIP_CANVAS_HOST = previous.skipCanvas; + } + if (previous.anthropicApiKey === undefined) { + delete process.env.ANTHROPIC_API_KEY; + } else { + process.env.ANTHROPIC_API_KEY = previous.anthropicApiKey; + } + if (previous.anthropicApiKeyOld === undefined) { + delete process.env.ANTHROPIC_API_KEY_OLD; + } else { + process.env.ANTHROPIC_API_KEY_OLD = previous.anthropicApiKeyOld; } } - } finally { - clearRuntimeConfigSnapshot(); - await client.stopAndWait(); - await server.close(); - await fs.rm(tempDir, { recursive: true, force: true }); - if (previous.configPath === undefined) { - delete process.env.OPENCLAW_CONFIG_PATH; - } else { - process.env.OPENCLAW_CONFIG_PATH = previous.configPath; - } - if (previous.token === undefined) { - delete process.env.OPENCLAW_GATEWAY_TOKEN; - } else { - process.env.OPENCLAW_GATEWAY_TOKEN = previous.token; - } - if (previous.skipChannels === undefined) { - delete process.env.OPENCLAW_SKIP_CHANNELS; - } else { - process.env.OPENCLAW_SKIP_CHANNELS = previous.skipChannels; - } - if (previous.skipGmail === undefined) { - delete process.env.OPENCLAW_SKIP_GMAIL_WATCHER; - } else { - process.env.OPENCLAW_SKIP_GMAIL_WATCHER = previous.skipGmail; - } - if (previous.skipCron === undefined) { - delete process.env.OPENCLAW_SKIP_CRON; - } else { - process.env.OPENCLAW_SKIP_CRON = previous.skipCron; - } - if (previous.skipCanvas === undefined) { - delete process.env.OPENCLAW_SKIP_CANVAS_HOST; - } else { - process.env.OPENCLAW_SKIP_CANVAS_HOST = previous.skipCanvas; - } - if (previous.anthropicApiKey === undefined) { - delete process.env.ANTHROPIC_API_KEY; - } else { - process.env.ANTHROPIC_API_KEY = previous.anthropicApiKey; - } - if (previous.anthropicApiKeyOld === undefined) { - delete process.env.ANTHROPIC_API_KEY_OLD; - } else { - process.env.ANTHROPIC_API_KEY_OLD = previous.anthropicApiKeyOld; - } - } - }, 60_000); + }, + CLI_BOOTSTRAP_LIVE_TIMEOUT_MS, + ); }); describeClaudeBootstrapLive("gateway live (claude-cli bootstrap context)", () => { - it("injects AGENTS, SOUL, IDENTITY, and USER files into the first Claude CLI turn", async () => { - const result = await runGatewayCliBootstrapLiveProbe(); - expect(result.ok).toBe(true); - expect(result.text).toBe(result.expectedText); - expect( - result.systemPromptReport?.injectedWorkspaceFiles?.map((entry) => entry.name) ?? [], - ).toEqual(expect.arrayContaining(["AGENTS.md", "SOUL.md", "IDENTITY.md", "USER.md"])); - }, 60_000); + it( + "injects AGENTS, SOUL, IDENTITY, and USER files into the first Claude CLI turn", + async () => { + const result = await runGatewayCliBootstrapLiveProbe(); + expect(result.ok).toBe(true); + expect(result.text).toBe(result.expectedText); + expect( + result.systemPromptReport?.injectedWorkspaceFiles?.map((entry) => entry.name) ?? [], + ).toEqual(expect.arrayContaining(["AGENTS.md", "SOUL.md", "IDENTITY.md", "USER.md"])); + }, + CLI_BACKEND_LIVE_TIMEOUT_MS, + ); });