diff --git a/scripts/e2e/mcp-channels-harness.ts b/scripts/e2e/mcp-channels-harness.ts index f90190c138c..18cabd5a8d1 100644 --- a/scripts/e2e/mcp-channels-harness.ts +++ b/scripts/e2e/mcp-channels-harness.ts @@ -38,6 +38,9 @@ export type McpClientHandle = { rawMessages: unknown[]; }; +const GATEWAY_WS_TIMEOUT_MS = 30_000; +const GATEWAY_CONNECT_RETRY_WINDOW_MS = 45_000; + export function assert(condition: unknown, message: string): asserts condition { if (!condition) { throw new Error(message); @@ -85,10 +88,37 @@ export async function waitFor( export async function connectGateway(params: { url: string; token: string; +}): Promise { + const startedAt = Date.now(); + let attempt = 0; + let lastError: Error | null = null; + + while (Date.now() - startedAt < GATEWAY_CONNECT_RETRY_WINDOW_MS) { + attempt += 1; + try { + return await connectGatewayOnce(params); + } catch (error) { + lastError = error instanceof Error ? error : new Error(String(error)); + if (!isRetryableGatewayConnectError(lastError)) { + throw lastError; + } + await delay(Math.min(500 * attempt, 2_000)); + } + } + + throw lastError ?? new Error("gateway ws open timeout"); +} + +async function connectGatewayOnce(params: { + url: string; + token: string; }): Promise { const ws = new WebSocket(params.url); await new Promise((resolve, reject) => { - const timeout = setTimeout(() => reject(new Error("gateway ws open timeout")), 10_000); + const timeout = setTimeout( + () => reject(new Error("gateway ws open timeout")), + GATEWAY_WS_TIMEOUT_MS, + ); timeout.unref?.(); ws.once("open", () => { clearTimeout(timeout); @@ -196,7 +226,7 @@ export async function connectGateway(params: { const timeout = setTimeout(() => { pending.delete(connectId); reject(new Error("gateway connect timeout")); - }, 10_000); + }, GATEWAY_WS_TIMEOUT_MS); timeout.unref?.(); pending.set(connectId, { resolve: () => { @@ -215,7 +245,7 @@ export async function connectGateway(params: { const timeout = setTimeout(() => { pending.delete(id); reject(new Error("gateway sessions.subscribe timeout")); - }, 10_000); + }, GATEWAY_WS_TIMEOUT_MS); timeout.unref?.(); pending.set(id, { resolve: () => { @@ -284,6 +314,17 @@ export async function connectGateway(params: { }; } +function isRetryableGatewayConnectError(error: Error): boolean { + const message = error.message.toLowerCase(); + return ( + message.includes("gateway ws open timeout") || + message.includes("gateway connect timeout") || + message.includes("gateway closed") || + message.includes("econnrefused") || + message.includes("socket hang up") + ); +} + export async function connectMcpClient(params: { gatewayUrl: string; gatewayToken: string; diff --git a/scripts/e2e/plugins-docker.sh b/scripts/e2e/plugins-docker.sh index ebdd56bb0f7..27ea1921a9a 100755 --- a/scripts/e2e/plugins-docker.sh +++ b/scripts/e2e/plugins-docker.sh @@ -142,14 +142,17 @@ run_gateway_chat_json() { local session_key="$1" local message="$2" local output_file="$3" - local timeout_ms="${4:-15000}" + local timeout_ms="${4:-45000}" node - <<'NODE' "$OPENCLAW_ENTRY" "$session_key" "$message" "$output_file" "$timeout_ms" const { execFileSync } = require("node:child_process"); const fs = require("node:fs"); const { randomUUID } = require("node:crypto"); const [, , entry, sessionKey, message, outputFile, timeoutRaw] = process.argv; -const timeoutMs = Number(timeoutRaw) > 0 ? Number(timeoutRaw) : 15000; +const timeoutMs = Number(timeoutRaw) > 0 ? Number(timeoutRaw) : 45000; +const gatewayCallTimeoutMs = Math.max(15000, Math.min(timeoutMs, 30000)); +const retryableGatewayErrorPattern = + /gateway ws open timeout|gateway connect timeout|gateway closed|ECONNREFUSED|socket hang up|gateway timeout after/i; const gatewayArgs = [ entry, "gateway", @@ -159,11 +162,11 @@ const gatewayArgs = [ "--token", "plugin-e2e-token", "--timeout", - "10000", + String(gatewayCallTimeoutMs), "--json", ]; -const callGateway = (method, params) => { +const callGatewayOnce = (method, params) => { try { return { ok: true, @@ -181,6 +184,9 @@ const callGateway = (method, params) => { } }; +const isRetryableGatewayError = (error) => + retryableGatewayErrorPattern.test(error instanceof Error ? error.message : String(error)); + const extractText = (messageLike) => { if (!messageLike || typeof messageLike !== "object") { return ""; @@ -220,6 +226,22 @@ const findLatestAssistantText = (history) => { const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); +const callGateway = async (method, params, deadline = Date.now() + gatewayCallTimeoutMs) => { + let lastFailure = null; + while (Date.now() < deadline) { + const result = callGatewayOnce(method, params); + if (result.ok) { + return result; + } + lastFailure = result; + if (!isRetryableGatewayError(result.error)) { + return result; + } + await sleep(250); + } + return lastFailure ?? callGatewayOnce(method, params); +}; + async function main() { const runId = `plugin-e2e-${randomUUID()}`; const sendParams = { @@ -227,18 +249,25 @@ async function main() { message, idempotencyKey: runId, }; - const sendResult = callGateway("chat.send", sendParams); + let lastGatewayError = null; + const sendResult = await callGateway( + "chat.send", + sendParams, + Date.now() + Math.min(timeoutMs, gatewayCallTimeoutMs), + ); if (!sendResult.ok) { throw sendResult.error; } const deadline = Date.now() + timeoutMs; while (Date.now() < deadline) { - const historyResult = callGateway("chat.history", { sessionKey }); + const historyResult = await callGateway("chat.history", { sessionKey }, Date.now() + 5000); if (!historyResult.ok) { + lastGatewayError = String(historyResult.error); await sleep(150); continue; } + lastGatewayError = null; const history = historyResult.value; const latestAssistant = findLatestAssistantText(history); if (latestAssistant) { @@ -259,22 +288,10 @@ async function main() { ); return; } - const statusResult = callGateway("chat.send", sendParams); - if (statusResult.ok) { - const status = statusResult.value; - if (status?.status === "error") { - const summary = - typeof status.summary === "string" && status.summary.trim() - ? status.summary.trim() - : JSON.stringify(status); - throw new Error(`gateway run failed for ${sessionKey}: ${summary}`); - } - } await sleep(100); } - const finalHistory = callGateway("chat.history", { sessionKey }); - const finalStatus = callGateway("chat.send", sendParams); + const finalHistory = await callGateway("chat.history", { sessionKey }, Date.now() + 3000); fs.writeFileSync( outputFile, `${JSON.stringify( @@ -284,15 +301,15 @@ async function main() { error: "timeout", history: finalHistory.ok ? finalHistory.value : null, historyError: finalHistory.ok ? null : String(finalHistory.error), - status: finalStatus.ok ? finalStatus.value : null, - statusError: finalStatus.ok ? null : String(finalStatus.error), + lastGatewayError, }, null, 2, )}\n`, "utf8", ); - throw new Error(`timed out waiting for assistant reply for ${sessionKey}`); + const retrySummary = lastGatewayError ? `; last gateway error: ${lastGatewayError}` : ""; + throw new Error(`timed out waiting for assistant reply for ${sessionKey}${retrySummary}`); } main().catch((error) => { @@ -696,7 +713,11 @@ if (!text.includes("[disabled]")) { console.log("ok"); NODE -run_gateway_chat_json "plugin-e2e-enable" "/plugin enable claude-bundle-e2e" /tmp/plugin-command-enable.json +run_gateway_chat_json \ + "plugin-e2e-enable" \ + "/plugin enable claude-bundle-e2e" \ + /tmp/plugin-command-enable.json \ + 60000 node - <<'NODE' const fs = require("node:fs"); const payload = JSON.parse(fs.readFileSync("/tmp/plugin-command-enable.json", "utf8")); diff --git a/scripts/lib/live-docker-stage.sh b/scripts/lib/live-docker-stage.sh index dba5a75f653..61dc3d993e8 100644 --- a/scripts/lib/live-docker-stage.sh +++ b/scripts/lib/live-docker-stage.sh @@ -33,3 +33,30 @@ openclaw_live_link_runtime_tree() { export OPENCLAW_BUNDLED_PLUGINS_DIR=/app/dist/extensions fi } + +openclaw_live_stage_state_dir() { + local dest_dir="${1:?destination directory required}" + local source_dir="${HOME}/.openclaw" + + mkdir -p "$dest_dir" + if [ -d "$source_dir" ]; then + tar -C "$source_dir" --exclude=workspace -cf - . | tar -C "$dest_dir" -xf - + if [ -d "$source_dir/workspace" ] && [ ! -e "$dest_dir/workspace" ]; then + ln -s "$source_dir/workspace" "$dest_dir/workspace" + fi + fi + + export OPENCLAW_STATE_DIR="$dest_dir" + export OPENCLAW_CONFIG_PATH="$dest_dir/openclaw.json" +} + +openclaw_live_prepare_staged_config() { + if [ ! -f "${OPENCLAW_CONFIG_PATH:-}" ]; then + return 0 + fi + + ( + cd /app + node --import tsx /src/scripts/live-docker-normalize-config.ts + ) +} diff --git a/scripts/live-docker-normalize-config.ts b/scripts/live-docker-normalize-config.ts new file mode 100644 index 00000000000..a21e9ba2a1c --- /dev/null +++ b/scripts/live-docker-normalize-config.ts @@ -0,0 +1,15 @@ +import { loadAndMaybeMigrateDoctorConfig } from "../src/commands/doctor-config-flow.js"; +import { writeConfigFile } from "../src/config/config.js"; + +const result = await loadAndMaybeMigrateDoctorConfig({ + options: { + nonInteractive: true, + repair: true, + yes: true, + }, + confirm: async () => false, +}); + +if (result.shouldWriteConfig) { + await writeConfigFile(result.cfg); +} diff --git a/scripts/test-live-acp-bind-docker.sh b/scripts/test-live-acp-bind-docker.sh index d3935729da9..90da8a47300 100644 --- a/scripts/test-live-acp-bind-docker.sh +++ b/scripts/test-live-acp-bind-docker.sh @@ -150,9 +150,11 @@ cleanup() { rm -rf "$tmp_dir" } trap cleanup EXIT -source /app/scripts/lib/live-docker-stage.sh +source /src/scripts/lib/live-docker-stage.sh openclaw_live_stage_source_tree "$tmp_dir" openclaw_live_link_runtime_tree "$tmp_dir" +openclaw_live_stage_state_dir "$tmp_dir/.openclaw-state" +openclaw_live_prepare_staged_config cd "$tmp_dir" export OPENCLAW_LIVE_ACP_BIND_AGENT_COMMAND="${OPENCLAW_LIVE_ACP_BIND_AGENT_COMMAND:-}" pnpm test:live src/gateway/gateway-acp-bind.live.test.ts diff --git a/scripts/test-live-cli-backend-docker.sh b/scripts/test-live-cli-backend-docker.sh index f17ee73da07..90a9fecc9b9 100644 --- a/scripts/test-live-cli-backend-docker.sh +++ b/scripts/test-live-cli-backend-docker.sh @@ -138,13 +138,8 @@ cleanup() { rm -rf "$tmp_dir" } trap cleanup EXIT -tar -C /src \ - --exclude=.git \ - --exclude=node_modules \ - --exclude=dist \ - --exclude=ui/dist \ - --exclude=ui/node_modules \ - -cf - . | tar -C "$tmp_dir" -xf - +source /src/scripts/lib/live-docker-stage.sh +openclaw_live_stage_source_tree "$tmp_dir" # Use a writable node_modules overlay in the temp repo. Vite writes bundled # config artifacts under the nearest node_modules/.vite-temp path, and the # build-stage /app/node_modules tree is root-owned in this Docker lane. @@ -152,12 +147,9 @@ mkdir -p "$tmp_dir/node_modules" cp -aRs /app/node_modules/. "$tmp_dir/node_modules" rm -rf "$tmp_dir/node_modules/.vite-temp" mkdir -p "$tmp_dir/node_modules/.vite-temp" -ln -s /app/dist "$tmp_dir/dist" -if [ -d /app/dist-runtime/extensions ]; then - export OPENCLAW_BUNDLED_PLUGINS_DIR=/app/dist-runtime/extensions -elif [ -d /app/dist/extensions ]; then - export OPENCLAW_BUNDLED_PLUGINS_DIR=/app/dist/extensions -fi +openclaw_live_link_runtime_tree "$tmp_dir" +openclaw_live_stage_state_dir "$tmp_dir/.openclaw-state" +openclaw_live_prepare_staged_config cd "$tmp_dir" pnpm test:live src/gateway/gateway-cli-backend.live.test.ts EOF diff --git a/scripts/test-live-gateway-models-docker.sh b/scripts/test-live-gateway-models-docker.sh index c4119a1c7ee..13417828bef 100755 --- a/scripts/test-live-gateway-models-docker.sh +++ b/scripts/test-live-gateway-models-docker.sh @@ -101,9 +101,11 @@ cleanup() { rm -rf "$tmp_dir" } trap cleanup EXIT -source /app/scripts/lib/live-docker-stage.sh +source /src/scripts/lib/live-docker-stage.sh openclaw_live_stage_source_tree "$tmp_dir" openclaw_live_link_runtime_tree "$tmp_dir" +openclaw_live_stage_state_dir "$tmp_dir/.openclaw-state" +openclaw_live_prepare_staged_config cd "$tmp_dir" pnpm test:live:gateway-profiles EOF diff --git a/scripts/test-live-models-docker.sh b/scripts/test-live-models-docker.sh index 5915c7aa7bb..bcf22dbf1df 100755 --- a/scripts/test-live-models-docker.sh +++ b/scripts/test-live-models-docker.sh @@ -111,9 +111,11 @@ cleanup() { rm -rf "$tmp_dir" } trap cleanup EXIT -source /app/scripts/lib/live-docker-stage.sh +source /src/scripts/lib/live-docker-stage.sh openclaw_live_stage_source_tree "$tmp_dir" openclaw_live_link_runtime_tree "$tmp_dir" +openclaw_live_stage_state_dir "$tmp_dir/.openclaw-state" +openclaw_live_prepare_staged_config cd "$tmp_dir" pnpm test:live:models-profiles EOF diff --git a/src/agents/cli-runner.bundle-mcp.e2e.test.ts b/src/agents/cli-runner.bundle-mcp.e2e.test.ts index eda3edff80f..40202fbc14e 100644 --- a/src/agents/cli-runner.bundle-mcp.e2e.test.ts +++ b/src/agents/cli-runner.bundle-mcp.e2e.test.ts @@ -23,8 +23,9 @@ vi.mock("./cli-runner/helpers.js", async () => { }); // This e2e spins a real stdio MCP server plus a spawned CLI process, which is -// notably slower under Docker and cold Vitest imports. -const E2E_TIMEOUT_MS = 40_000; +// notably slower under Docker and cold Vitest imports. The plugins Docker lane +// also reaches this test after several gateway/plugin restart exercises. +const E2E_TIMEOUT_MS = 90_000; describe("runCliAgent bundle MCP e2e", () => { it( @@ -76,7 +77,7 @@ describe("runCliAgent bundle MCP e2e", () => { prompt: "Use your configured MCP tools and report the bundle probe text.", provider: "claude-cli", model: "test-bundle", - timeoutMs: 10_000, + timeoutMs: 20_000, runId: "bundle-mcp-e2e", }); diff --git a/src/agents/model-auth.profiles.test.ts b/src/agents/model-auth.profiles.test.ts index 03305096421..dbee02ff39a 100644 --- a/src/agents/model-auth.profiles.test.ts +++ b/src/agents/model-auth.profiles.test.ts @@ -262,6 +262,49 @@ describe("getApiKeyForModel", () => { ); }); + it("keeps stored provider auth ahead of env by default", async () => { + await withEnvAsync({ OPENAI_API_KEY: "env-openai-key" }, async () => { + const resolved = await resolveApiKeyForProvider({ + provider: "openai", + store: { + version: 1, + profiles: { + "openai:default": { + type: "api_key", + provider: "openai", + key: "stored-openai-key", + }, + }, + }, + }); + expect(resolved.apiKey).toBe("stored-openai-key"); + expect(resolved.source).toBe("profile:openai:default"); + expect(resolved.profileId).toBe("openai:default"); + }); + }); + + it("supports env-first precedence for live auth probes", async () => { + await withEnvAsync({ OPENAI_API_KEY: "env-openai-key" }, async () => { + const resolved = await resolveApiKeyForProvider({ + provider: "openai", + credentialPrecedence: "env-first", + store: { + version: 1, + profiles: { + "openai:default": { + type: "api_key", + provider: "openai", + key: "stored-openai-key", + }, + }, + }, + }); + expect(resolved.apiKey).toBe("env-openai-key"); + expect(resolved.source).toContain("OPENAI_API_KEY"); + expect(resolved.profileId).toBeUndefined(); + }); + }); + it("hasAvailableAuthForProvider('google') accepts GOOGLE_API_KEY fallback", async () => { await withEnvAsync( { diff --git a/src/agents/model-auth.ts b/src/agents/model-auth.ts index 5f94bb16dde..5acf04a10a1 100644 --- a/src/agents/model-auth.ts +++ b/src/agents/model-auth.ts @@ -38,6 +38,7 @@ import { normalizeProviderId } from "./model-selection.js"; export { ensureAuthProfileStore, resolveAuthProfileOrder } from "./auth-profiles.js"; export { requireApiKey, resolveAwsSdkEnvVarName } from "./model-auth-runtime-shared.js"; export type { ResolvedProviderAuth } from "./model-auth-runtime-shared.js"; +export type ProviderCredentialPrecedence = "profile-first" | "env-first"; const log = createSubsystemLogger("model-auth"); function resolveProviderConfig( @@ -347,6 +348,7 @@ export async function resolveApiKeyForProvider(params: { /** When true, treat profileId as a user-locked selection that must not be * silently overridden by env/config credentials (e.g. ollama-local). */ lockedProfile?: boolean; + credentialPrecedence?: ProviderCredentialPrecedence; }): Promise { const { provider, cfg, profileId, preferredProfile } = params; const store = params.store ?? ensureAuthProfileStore(params.agentDir); @@ -402,6 +404,20 @@ export async function resolveApiKeyForProvider(params: { } } + if (params.credentialPrecedence === "env-first") { + const envResolved = resolveEnvApiKey(provider); + if (envResolved) { + const resolvedMode: ResolvedProviderAuth["mode"] = envResolved.source.includes("OAUTH_TOKEN") + ? "oauth" + : "api-key"; + return { + apiKey: envResolved.apiKey, + source: envResolved.source, + mode: resolvedMode, + }; + } + } + const providerConfig = resolveProviderConfig(cfg, provider); const order = resolveAuthProfileOrder({ cfg, @@ -633,6 +649,7 @@ export async function getApiKeyForModel(params: { store?: AuthProfileStore; agentDir?: string; lockedProfile?: boolean; + credentialPrecedence?: ProviderCredentialPrecedence; }): Promise { return resolveApiKeyForProvider({ provider: params.model.provider, @@ -642,6 +659,7 @@ export async function getApiKeyForModel(params: { store: params.store, agentDir: params.agentDir, lockedProfile: params.lockedProfile, + credentialPrecedence: params.credentialPrecedence, }); } diff --git a/src/agents/models.profiles.live.test.ts b/src/agents/models.profiles.live.test.ts index 77c453a2b32..31d79d728c9 100644 --- a/src/agents/models.profiles.live.test.ts +++ b/src/agents/models.profiles.live.test.ts @@ -20,6 +20,7 @@ import { discoverAuthStorage, discoverModels } from "./pi-model-discovery.js"; const LIVE = isLiveTestEnabled(); const DIRECT_ENABLED = Boolean(process.env.OPENCLAW_LIVE_MODELS?.trim()); const REQUIRE_PROFILE_KEYS = isLiveProfileKeyModeEnabled(); +const LIVE_CREDENTIAL_PRECEDENCE = REQUIRE_PROFILE_KEYS ? "profile-first" : "env-first"; const LIVE_HEARTBEAT_MS = Math.max(1_000, toInt(process.env.OPENCLAW_LIVE_HEARTBEAT_MS, 30_000)); const LIVE_SETUP_TIMEOUT_MS = Math.max( 1_000, @@ -450,7 +451,11 @@ describeLive("live models (profile keys)", () => { } } try { - const apiKeyInfo = await getApiKeyForModel({ model, cfg }); + const apiKeyInfo = await getApiKeyForModel({ + model, + cfg, + credentialPrecedence: LIVE_CREDENTIAL_PRECEDENCE, + }); if (REQUIRE_PROFILE_KEYS && !apiKeyInfo.source.startsWith("profile:")) { skipped.push({ model: id, diff --git a/src/gateway/gateway-models.profiles.live.test.ts b/src/gateway/gateway-models.profiles.live.test.ts index 01e57d61b24..0650200e336 100644 --- a/src/gateway/gateway-models.profiles.live.test.ts +++ b/src/gateway/gateway-models.profiles.live.test.ts @@ -25,7 +25,8 @@ import { } from "../agents/live-model-filter.js"; import { createLiveTargetMatcher } from "../agents/live-target-matcher.js"; import { isLiveProfileKeyModeEnabled, isLiveTestEnabled } from "../agents/live-test-helpers.js"; -import { getApiKeyForModel } from "../agents/model-auth.js"; +import { getApiKeyForModel, resolveEnvApiKey } from "../agents/model-auth.js"; +import { normalizeProviderId } from "../agents/model-selection.js"; import { shouldSuppressBuiltInModel } from "../agents/model-suppression.js"; import { ensureOpenClawModelsJson } from "../agents/models-config.js"; import { isRateLimitErrorMessage } from "../agents/pi-embedded-helpers/errors.js"; @@ -50,6 +51,7 @@ import { loadSessionEntry, readSessionMessages } from "./session-utils.js"; const ZAI_FALLBACK = isTruthyEnvValue(process.env.OPENCLAW_LIVE_GATEWAY_ZAI_FALLBACK); const REQUIRE_PROFILE_KEYS = isLiveProfileKeyModeEnabled(); +const LIVE_CREDENTIAL_PRECEDENCE = REQUIRE_PROFILE_KEYS ? "profile-first" : "env-first"; const PROVIDERS = parseFilter(process.env.OPENCLAW_LIVE_GATEWAY_PROVIDERS); const GATEWAY_LIVE_SMOKE = isTruthyEnvValue(process.env.OPENCLAW_LIVE_GATEWAY_SMOKE); const THINKING_LEVEL = GATEWAY_LIVE_SMOKE ? "low" : "high"; @@ -824,43 +826,98 @@ async function getFreeGatewayPort(): Promise { throw new Error("failed to acquire a free gateway port block"); } -function sleep(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)); +async function sleep(ms: number): Promise { + await new Promise((resolve) => setTimeout(resolve, ms)); } -async function connectClient(params: { url: string; token: string }) { +function sanitizeAuthProfileStoreForLiveGateway(store: AuthProfileStore): AuthProfileStore { + if (REQUIRE_PROFILE_KEYS) { + return store; + } + + const envBackedProviders = new Set(); + for (const profile of Object.values(store.profiles)) { + if (resolveEnvApiKey(profile.provider)?.apiKey) { + envBackedProviders.add(normalizeProviderId(profile.provider)); + } + } + if (envBackedProviders.size === 0) { + return store; + } + + const profiles = Object.fromEntries( + Object.entries(store.profiles).filter(([, profile]) => { + return !envBackedProviders.has(normalizeProviderId(profile.provider)); + }), + ); + const keepProfileIds = new Set(Object.keys(profiles)); + + const order = store.order + ? Object.fromEntries( + Object.entries(store.order) + .filter(([provider]) => !envBackedProviders.has(normalizeProviderId(provider))) + .map(([provider, ids]) => [provider, ids.filter((id) => keepProfileIds.has(id))]) + .filter(([, ids]) => ids.length > 0), + ) + : undefined; + + const lastGood = store.lastGood + ? Object.fromEntries( + Object.entries(store.lastGood).filter(([provider, id]) => { + return !envBackedProviders.has(normalizeProviderId(provider)) && keepProfileIds.has(id); + }), + ) + : undefined; + + const usageStats = store.usageStats + ? Object.fromEntries(Object.entries(store.usageStats).filter(([id]) => keepProfileIds.has(id))) + : undefined; + + return { + ...store, + profiles, + order: order && Object.keys(order).length > 0 ? order : undefined, + lastGood: lastGood && Object.keys(lastGood).length > 0 ? lastGood : undefined, + usageStats: usageStats && Object.keys(usageStats).length > 0 ? usageStats : undefined, + }; +} + +async function connectClient(params: { url: string; token: string; timeoutMs?: number }) { + const timeoutMs = params.timeoutMs ?? GATEWAY_LIVE_PROBE_TIMEOUT_MS; const startedAt = Date.now(); let attempt = 0; let lastError: Error | null = null; - while (Date.now() - startedAt < GATEWAY_LIVE_PROBE_TIMEOUT_MS) { + while (Date.now() - startedAt < timeoutMs) { attempt += 1; - const remainingMs = GATEWAY_LIVE_PROBE_TIMEOUT_MS - (Date.now() - startedAt); + const remainingMs = timeoutMs - (Date.now() - startedAt); if (remainingMs <= 0) { break; } try { return await connectClientOnce({ ...params, - timeoutMs: Math.min(remainingMs, 10_000), + timeoutMs: Math.min(remainingMs, 35_000), }); } catch (error) { lastError = error instanceof Error ? error : new Error(String(error)); - if (!isRetryableGatewayConnectError(lastError) || remainingMs <= 2_000) { + if (!isRetryableGatewayConnectError(lastError) || remainingMs <= 5_000) { throw lastError; } - await sleep(Math.min(500 * attempt, 2_000)); + logProgress(`gateway connect warmup retry ${attempt}: ${lastError.message}`); + await sleep(Math.min(1_000 * attempt, 5_000)); } } throw lastError ?? new Error("gateway connect timeout"); } -async function connectClientOnce(params: { url: string; token: string; timeoutMs: number }) { +async function connectClientOnce(params: { url: string; token: string; timeoutMs?: number }) { + const timeoutMs = params.timeoutMs ?? 10_000; return await new Promise((resolve, reject) => { let settled = false; let client: GatewayClient | undefined; - const stop = (err?: Error, connectedClient?: GatewayClient) => { + const stop = (err?: Error, nextClient?: GatewayClient) => { if (settled) { return; } @@ -872,24 +929,24 @@ async function connectClientOnce(params: { url: string; token: string; timeoutMs } reject(err); } else { - resolve(connectedClient as GatewayClient); + resolve(nextClient as GatewayClient); } }; client = new GatewayClient({ url: params.url, token: params.token, + requestTimeoutMs: Math.max(timeoutMs, GATEWAY_LIVE_MODEL_TIMEOUT_MS), + connectChallengeTimeoutMs: timeoutMs, clientName: GATEWAY_CLIENT_NAMES.TEST, clientDisplayName: "vitest-live", clientVersion: "dev", mode: GATEWAY_CLIENT_MODES.TEST, - requestTimeoutMs: params.timeoutMs, - connectChallengeTimeoutMs: params.timeoutMs, onHelloOk: () => stop(undefined, client), onConnectError: (err) => stop(err), onClose: (code, reason) => stop(new Error(`gateway closed during connect (${code}): ${reason}`)), }); - const timer = setTimeout(() => stop(new Error("gateway connect timeout")), params.timeoutMs); + const timer = setTimeout(() => stop(new Error("gateway connect timeout")), timeoutMs); timer.unref(); client.start(); }); @@ -905,6 +962,56 @@ function isRetryableGatewayConnectError(error: Error): boolean { ); } +describe("sanitizeAuthProfileStoreForLiveGateway", () => { + it("drops env-backed provider profiles when live auth should prefer env", () => { + const store: AuthProfileStore = { + version: 1, + profiles: { + openaiProfile: { + type: "api_key", + provider: "openai", + key: "sk-openai-test", + }, + codexProfile: { + type: "oauth", + provider: "openai-codex", + access: "access", + refresh: "refresh", + expires: 1, + }, + }, + order: { + openai: ["openaiProfile"], + "openai-codex": ["codexProfile"], + }, + lastGood: { + openai: "openaiProfile", + "openai-codex": "codexProfile", + }, + usageStats: { + openaiProfile: { lastUsed: 1 }, + codexProfile: { lastUsed: 2 }, + }, + }; + + const previousOpenAiKey = process.env.OPENAI_API_KEY; + process.env.OPENAI_API_KEY = "sk-live-openai"; + try { + const sanitized = sanitizeAuthProfileStoreForLiveGateway(store); + expect(sanitized.profiles.openaiProfile).toBeUndefined(); + expect(sanitized.profiles.codexProfile).toBeDefined(); + expect(sanitized.order).toEqual({ "openai-codex": ["codexProfile"] }); + expect(sanitized.lastGood).toEqual({ "openai-codex": "codexProfile" }); + expect(sanitized.usageStats).toEqual({ codexProfile: { lastUsed: 2 } }); + } finally { + if (previousOpenAiKey === undefined) { + delete process.env.OPENAI_API_KEY; + } else { + process.env.OPENAI_API_KEY = previousOpenAiKey; + } + } + }); +}); function extractTranscriptMessageText(message: unknown): string { if (!message || typeof message !== "object") { return ""; @@ -1188,7 +1295,7 @@ async function runGatewayModelSuite(params: GatewayModelSuiteParams) { const hostStore = ensureAuthProfileStore(hostAgentDir, { allowKeychainPrompt: false, }); - const sanitizedStore: AuthProfileStore = { + const sanitizedStore = sanitizeAuthProfileStoreForLiveGateway({ version: hostStore.version, profiles: { ...hostStore.profiles }, // Keep selection state so the gateway picks the same known-good profiles @@ -1196,7 +1303,7 @@ async function runGatewayModelSuite(params: GatewayModelSuiteParams) { order: hostStore.order ? { ...hostStore.order } : undefined, lastGood: hostStore.lastGood ? { ...hostStore.lastGood } : undefined, usageStats: hostStore.usageStats ? { ...hostStore.usageStats } : undefined, - }; + }); tempStateDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-live-state-")); process.env.OPENCLAW_STATE_DIR = tempStateDir; tempAgentDir = path.join(tempStateDir, "agents", DEFAULT_AGENT_ID, "agent"); @@ -1911,7 +2018,11 @@ describeLive("gateway live (dev agent, profile keys)", () => { } const modelRef = `${model.provider}/${model.id}`; try { - const apiKeyInfo = await getApiKeyForModel({ model, cfg }); + const apiKeyInfo = await getApiKeyForModel({ + model, + cfg, + credentialPrecedence: LIVE_CREDENTIAL_PRECEDENCE, + }); if (REQUIRE_PROFILE_KEYS && !apiKeyInfo.source.startsWith("profile:")) { skipped.push({ model: modelRef, @@ -2027,8 +2138,16 @@ describeLive("gateway live (dev agent, profile keys)", () => { return; } try { - await getApiKeyForModel({ model: anthropic, cfg }); - await getApiKeyForModel({ model: zai, cfg }); + await getApiKeyForModel({ + model: anthropic, + cfg, + credentialPrecedence: LIVE_CREDENTIAL_PRECEDENCE, + }); + await getApiKeyForModel({ + model: zai, + cfg, + credentialPrecedence: LIVE_CREDENTIAL_PRECEDENCE, + }); } catch { return; }