import type { OpenClawConfig } from "../config/config.js"; import type { FailoverReason } from "./pi-embedded-helpers.js"; import { ensureAuthProfileStore, isProfileInCooldown, resolveAuthProfileOrder, } from "./auth-profiles.js"; import { DEFAULT_MODEL, DEFAULT_PROVIDER } from "./defaults.js"; import { coerceToFailoverError, describeFailoverError, isFailoverError, isTimeoutError, } from "./failover-error.js"; import { buildConfiguredAllowlistKeys, buildModelAliasIndex, modelKey, resolveConfiguredModelRef, resolveModelRefFromString, } from "./model-selection.js"; type ModelCandidate = { provider: string; model: string; }; type FallbackAttempt = { provider: string; model: string; error: string; reason?: FailoverReason; status?: number; code?: string; }; function isAbortError(err: unknown): boolean { if (!err || typeof err !== "object") { return false; } if (isFailoverError(err)) { return false; } const name = "name" in err ? String(err.name) : ""; // Only treat explicit AbortError names as user aborts. // Message-based checks (e.g., "aborted") can mask timeouts and skip fallback. return name === "AbortError"; } function shouldRethrowAbort(err: unknown): boolean { return isAbortError(err) && !isTimeoutError(err); } function resolveImageFallbackCandidates(params: { cfg: OpenClawConfig | undefined; defaultProvider: string; modelOverride?: string; }): ModelCandidate[] { const aliasIndex = buildModelAliasIndex({ cfg: params.cfg ?? {}, defaultProvider: params.defaultProvider, }); const allowlist = buildConfiguredAllowlistKeys({ cfg: params.cfg, defaultProvider: params.defaultProvider, }); const seen = new Set(); const candidates: ModelCandidate[] = []; const addCandidate = (candidate: ModelCandidate, enforceAllowlist: boolean) => { if (!candidate.provider || !candidate.model) { return; } const key = modelKey(candidate.provider, candidate.model); if (seen.has(key)) { return; } if (enforceAllowlist && allowlist && !allowlist.has(key)) { return; } seen.add(key); candidates.push(candidate); }; const addRaw = (raw: string, enforceAllowlist: boolean) => { const resolved = resolveModelRefFromString({ raw: String(raw ?? ""), defaultProvider: params.defaultProvider, aliasIndex, }); if (!resolved) { return; } addCandidate(resolved.ref, enforceAllowlist); }; if (params.modelOverride?.trim()) { addRaw(params.modelOverride, false); } else { const imageModel = params.cfg?.agents?.defaults?.imageModel as | { primary?: string } | string | undefined; const primary = typeof imageModel === "string" ? imageModel.trim() : imageModel?.primary; if (primary?.trim()) { addRaw(primary, false); } } const imageFallbacks = (() => { const imageModel = params.cfg?.agents?.defaults?.imageModel as | { fallbacks?: string[] } | string | undefined; if (imageModel && typeof imageModel === "object") { return imageModel.fallbacks ?? []; } return []; })(); for (const raw of imageFallbacks) { addRaw(raw, true); } return candidates; } function resolveFallbackCandidates(params: { cfg: OpenClawConfig | undefined; provider: string; model: string; /** Optional explicit fallbacks list; when provided (even empty), replaces agents.defaults.model.fallbacks. */ fallbacksOverride?: string[]; }): ModelCandidate[] { const primary = params.cfg ? resolveConfiguredModelRef({ cfg: params.cfg, defaultProvider: DEFAULT_PROVIDER, defaultModel: DEFAULT_MODEL, }) : null; const defaultProvider = primary?.provider ?? DEFAULT_PROVIDER; const defaultModel = primary?.model ?? DEFAULT_MODEL; const provider = String(params.provider ?? "").trim() || defaultProvider; const model = String(params.model ?? "").trim() || defaultModel; const aliasIndex = buildModelAliasIndex({ cfg: params.cfg ?? {}, defaultProvider, }); const allowlist = buildConfiguredAllowlistKeys({ cfg: params.cfg, defaultProvider, }); const seen = new Set(); const candidates: ModelCandidate[] = []; const addCandidate = (candidate: ModelCandidate, enforceAllowlist: boolean) => { if (!candidate.provider || !candidate.model) { return; } const key = modelKey(candidate.provider, candidate.model); if (seen.has(key)) { return; } if (enforceAllowlist && allowlist && !allowlist.has(key)) { return; } seen.add(key); candidates.push(candidate); }; addCandidate({ provider, model }, false); const modelFallbacks = (() => { if (params.fallbacksOverride !== undefined) { return params.fallbacksOverride; } const model = params.cfg?.agents?.defaults?.model as | { fallbacks?: string[] } | string | undefined; if (model && typeof model === "object") { return model.fallbacks ?? []; } return []; })(); for (const raw of modelFallbacks) { const resolved = resolveModelRefFromString({ raw: String(raw ?? ""), defaultProvider, aliasIndex, }); if (!resolved) { continue; } addCandidate(resolved.ref, true); } if (params.fallbacksOverride === undefined && primary?.provider && primary.model) { addCandidate({ provider: primary.provider, model: primary.model }, false); } return candidates; } export async function runWithModelFallback(params: { cfg: OpenClawConfig | undefined; provider: string; model: string; agentDir?: string; /** Optional explicit fallbacks list; when provided (even empty), replaces agents.defaults.model.fallbacks. */ fallbacksOverride?: string[]; run: (provider: string, model: string) => Promise; onError?: (attempt: { provider: string; model: string; error: unknown; attempt: number; total: number; }) => void | Promise; }): Promise<{ result: T; provider: string; model: string; attempts: FallbackAttempt[]; }> { const candidates = resolveFallbackCandidates({ cfg: params.cfg, provider: params.provider, model: params.model, fallbacksOverride: params.fallbacksOverride, }); const authStore = params.cfg ? ensureAuthProfileStore(params.agentDir, { allowKeychainPrompt: false }) : null; const attempts: FallbackAttempt[] = []; let lastError: unknown; for (let i = 0; i < candidates.length; i += 1) { const candidate = candidates[i]; if (authStore) { const profileIds = resolveAuthProfileOrder({ cfg: params.cfg, store: authStore, provider: candidate.provider, }); const isAnyProfileAvailable = profileIds.some((id) => !isProfileInCooldown(authStore, id)); if (profileIds.length > 0 && !isAnyProfileAvailable) { // All profiles for this provider are in cooldown; skip without attempting attempts.push({ provider: candidate.provider, model: candidate.model, error: `Provider ${candidate.provider} is in cooldown (all profiles unavailable)`, reason: "rate_limit", }); continue; } } try { const result = await params.run(candidate.provider, candidate.model); return { result, provider: candidate.provider, model: candidate.model, attempts, }; } catch (err) { if (shouldRethrowAbort(err)) { throw err; } const normalized = coerceToFailoverError(err, { provider: candidate.provider, model: candidate.model, }) ?? err; if (!isFailoverError(normalized)) { throw err; } lastError = normalized; const described = describeFailoverError(normalized); attempts.push({ provider: candidate.provider, model: candidate.model, error: described.message, reason: described.reason, status: described.status, code: described.code, }); await params.onError?.({ provider: candidate.provider, model: candidate.model, error: normalized, attempt: i + 1, total: candidates.length, }); } } if (attempts.length <= 1 && lastError) { throw lastError; } const summary = attempts.length > 0 ? attempts .map( (attempt) => `${attempt.provider}/${attempt.model}: ${attempt.error}${ attempt.reason ? ` (${attempt.reason})` : "" }`, ) .join(" | ") : "unknown"; throw new Error(`All models failed (${attempts.length || candidates.length}): ${summary}`, { cause: lastError instanceof Error ? lastError : undefined, }); } export async function runWithImageModelFallback(params: { cfg: OpenClawConfig | undefined; modelOverride?: string; run: (provider: string, model: string) => Promise; onError?: (attempt: { provider: string; model: string; error: unknown; attempt: number; total: number; }) => void | Promise; }): Promise<{ result: T; provider: string; model: string; attempts: FallbackAttempt[]; }> { const candidates = resolveImageFallbackCandidates({ cfg: params.cfg, defaultProvider: DEFAULT_PROVIDER, modelOverride: params.modelOverride, }); if (candidates.length === 0) { throw new Error( "No image model configured. Set agents.defaults.imageModel.primary or agents.defaults.imageModel.fallbacks.", ); } const attempts: FallbackAttempt[] = []; let lastError: unknown; for (let i = 0; i < candidates.length; i += 1) { const candidate = candidates[i]; try { const result = await params.run(candidate.provider, candidate.model); return { result, provider: candidate.provider, model: candidate.model, attempts, }; } catch (err) { if (shouldRethrowAbort(err)) { throw err; } lastError = err; attempts.push({ provider: candidate.provider, model: candidate.model, error: err instanceof Error ? err.message : String(err), }); await params.onError?.({ provider: candidate.provider, model: candidate.model, error: err, attempt: i + 1, total: candidates.length, }); } } if (attempts.length <= 1 && lastError) { throw lastError; } const summary = attempts.length > 0 ? attempts .map((attempt) => `${attempt.provider}/${attempt.model}: ${attempt.error}`) .join(" | ") : "unknown"; throw new Error(`All image models failed (${attempts.length || candidates.length}): ${summary}`, { cause: lastError instanceof Error ? lastError : undefined, }); }