Files
openclaw/src/agents/pi-embedded-helpers/errors.ts
Altay 6e962d8b9e fix(agents): handle overloaded failover separately (#38301)
* fix(agents): skip auth-profile failure on overload

* fix(agents): note overload auth-profile fallback fix

* fix(agents): classify overloaded failures separately

* fix(agents): back off before overload failover

* fix(agents): tighten overload probe and backoff state

* fix(agents): persist overloaded cooldown across runs

* fix(agents): tighten overloaded status handling

* test(agents): add overload regression coverage

* fix(agents): restore runner imports after rebase

* test(agents): add overload fallback integration coverage

* fix(agents): harden overloaded failover abort handling

* test(agents): tighten overload classifier coverage

* test(agents): cover all-overloaded fallback exhaustion

* fix(cron): retry overloaded fallback summaries

* fix(cron): treat HTTP 529 as overloaded retry
2026-03-07 01:42:11 +03:00

910 lines
28 KiB
TypeScript

import type { AssistantMessage } from "@mariozechner/pi-ai";
import type { OpenClawConfig } from "../../config/config.js";
import { createSubsystemLogger } from "../../logging/subsystem.js";
import { formatSandboxToolPolicyBlockedMessage } from "../sandbox.js";
import { stableStringify } from "../stable-stringify.js";
import {
isAuthErrorMessage,
isAuthPermanentErrorMessage,
isBillingErrorMessage,
isOverloadedErrorMessage,
isPeriodicUsageLimitErrorMessage,
isRateLimitErrorMessage,
isTimeoutErrorMessage,
matchesFormatErrorPattern,
} from "./failover-matches.js";
import type { FailoverReason } from "./types.js";
export {
isAuthErrorMessage,
isAuthPermanentErrorMessage,
isBillingErrorMessage,
isOverloadedErrorMessage,
isRateLimitErrorMessage,
isTimeoutErrorMessage,
} from "./failover-matches.js";
const log = createSubsystemLogger("errors");
export function formatBillingErrorMessage(provider?: string, model?: string): string {
const providerName = provider?.trim();
const modelName = model?.trim();
const providerLabel =
providerName && modelName ? `${providerName} (${modelName})` : providerName || undefined;
if (providerLabel) {
return `⚠️ ${providerLabel} returned a billing error — your API key has run out of credits or has an insufficient balance. Check your ${providerName} billing dashboard and top up or switch to a different API key.`;
}
return "⚠️ API provider returned a billing error — your API key has run out of credits or has an insufficient balance. Check your provider's billing dashboard and top up or switch to a different API key.";
}
export const BILLING_ERROR_USER_MESSAGE = formatBillingErrorMessage();
const RATE_LIMIT_ERROR_USER_MESSAGE = "⚠️ API rate limit reached. Please try again later.";
const OVERLOADED_ERROR_USER_MESSAGE =
"The AI service is temporarily overloaded. Please try again in a moment.";
function formatRateLimitOrOverloadedErrorCopy(raw: string): string | undefined {
if (isRateLimitErrorMessage(raw)) {
return RATE_LIMIT_ERROR_USER_MESSAGE;
}
if (isOverloadedErrorMessage(raw)) {
return OVERLOADED_ERROR_USER_MESSAGE;
}
return undefined;
}
function isReasoningConstraintErrorMessage(raw: string): boolean {
if (!raw) {
return false;
}
const lower = raw.toLowerCase();
return (
lower.includes("reasoning is mandatory") ||
lower.includes("reasoning is required") ||
lower.includes("requires reasoning") ||
(lower.includes("reasoning") && lower.includes("cannot be disabled"))
);
}
function hasRateLimitTpmHint(raw: string): boolean {
const lower = raw.toLowerCase();
return /\btpm\b/i.test(lower) || lower.includes("tokens per minute");
}
export function isContextOverflowError(errorMessage?: string): boolean {
if (!errorMessage) {
return false;
}
const lower = errorMessage.toLowerCase();
// Groq uses 413 for TPM (tokens per minute) limits, which is a rate limit, not context overflow.
if (hasRateLimitTpmHint(errorMessage)) {
return false;
}
if (isReasoningConstraintErrorMessage(errorMessage)) {
return false;
}
const hasRequestSizeExceeds = lower.includes("request size exceeds");
const hasContextWindow =
lower.includes("context window") ||
lower.includes("context length") ||
lower.includes("maximum context length");
return (
lower.includes("request_too_large") ||
lower.includes("request exceeds the maximum size") ||
lower.includes("context length exceeded") ||
lower.includes("maximum context length") ||
lower.includes("prompt is too long") ||
lower.includes("exceeds model context window") ||
lower.includes("model token limit") ||
(hasRequestSizeExceeds && hasContextWindow) ||
lower.includes("context overflow:") ||
lower.includes("exceed context limit") ||
lower.includes("exceeds the model's maximum context") ||
(lower.includes("max_tokens") && lower.includes("exceed") && lower.includes("context")) ||
(lower.includes("input length") && lower.includes("exceed") && lower.includes("context")) ||
(lower.includes("413") && lower.includes("too large")) ||
// Anthropic API and OpenAI-compatible providers (e.g. ZhipuAI/GLM) return this stop reason
// when the context window is exceeded. pi-ai surfaces it as "Unhandled stop reason: model_context_window_exceeded".
lower.includes("context_window_exceeded") ||
// Chinese proxy error messages for context overflow
errorMessage.includes("上下文过长") ||
errorMessage.includes("上下文超出") ||
errorMessage.includes("上下文长度超") ||
errorMessage.includes("超出最大上下文") ||
errorMessage.includes("请压缩上下文")
);
}
const CONTEXT_WINDOW_TOO_SMALL_RE = /context window.*(too small|minimum is)/i;
const CONTEXT_OVERFLOW_HINT_RE =
/context.*overflow|context window.*(too (?:large|long)|exceed|over|limit|max(?:imum)?|requested|sent|tokens)|prompt.*(too (?:large|long)|exceed|over|limit|max(?:imum)?)|(?:request|input).*(?:context|window|length|token).*(too (?:large|long)|exceed|over|limit|max(?:imum)?)/i;
const RATE_LIMIT_HINT_RE =
/rate limit|too many requests|requests per (?:minute|hour|day)|quota|throttl|429\b/i;
export function isLikelyContextOverflowError(errorMessage?: string): boolean {
if (!errorMessage) {
return false;
}
// Groq uses 413 for TPM (tokens per minute) limits, which is a rate limit, not context overflow.
if (hasRateLimitTpmHint(errorMessage)) {
return false;
}
if (isReasoningConstraintErrorMessage(errorMessage)) {
return false;
}
if (CONTEXT_WINDOW_TOO_SMALL_RE.test(errorMessage)) {
return false;
}
// Rate limit errors can match the broad CONTEXT_OVERFLOW_HINT_RE pattern
// (e.g., "request reached organization TPD rate limit" matches request.*limit).
// Exclude them before checking context overflow heuristics.
if (isRateLimitErrorMessage(errorMessage)) {
return false;
}
if (isContextOverflowError(errorMessage)) {
return true;
}
if (RATE_LIMIT_HINT_RE.test(errorMessage)) {
return false;
}
return CONTEXT_OVERFLOW_HINT_RE.test(errorMessage);
}
export function isCompactionFailureError(errorMessage?: string): boolean {
if (!errorMessage) {
return false;
}
const lower = errorMessage.toLowerCase();
const hasCompactionTerm =
lower.includes("summarization failed") ||
lower.includes("auto-compaction") ||
lower.includes("compaction failed") ||
lower.includes("compaction");
if (!hasCompactionTerm) {
return false;
}
// Treat any likely overflow shape as a compaction failure when compaction terms are present.
// Providers often vary wording (e.g. "context window exceeded") across APIs.
if (isLikelyContextOverflowError(errorMessage)) {
return true;
}
// Keep explicit fallback for bare "context overflow" strings.
return lower.includes("context overflow");
}
const ERROR_PAYLOAD_PREFIX_RE =
/^(?:error|api\s*error|apierror|openai\s*error|anthropic\s*error|gateway\s*error)[:\s-]+/i;
const FINAL_TAG_RE = /<\s*\/?\s*final\s*>/gi;
const ERROR_PREFIX_RE =
/^(?:error|api\s*error|openai\s*error|anthropic\s*error|gateway\s*error|request failed|failed|exception)[:\s-]+/i;
const CONTEXT_OVERFLOW_ERROR_HEAD_RE =
/^(?:context overflow:|request_too_large\b|request size exceeds\b|request exceeds the maximum size\b|context length exceeded\b|maximum context length\b|prompt is too long\b|exceeds model context window\b)/i;
const HTTP_STATUS_PREFIX_RE = /^(?:http\s*)?(\d{3})\s+(.+)$/i;
const HTTP_STATUS_CODE_PREFIX_RE = /^(?:http\s*)?(\d{3})(?:\s+([\s\S]+))?$/i;
const HTML_ERROR_PREFIX_RE = /^\s*(?:<!doctype\s+html\b|<html\b)/i;
const CLOUDFLARE_HTML_ERROR_CODES = new Set([521, 522, 523, 524, 525, 526, 530]);
const TRANSIENT_HTTP_ERROR_CODES = new Set([500, 502, 503, 504, 521, 522, 523, 524, 529]);
const HTTP_ERROR_HINTS = [
"error",
"bad request",
"not found",
"unauthorized",
"forbidden",
"internal server",
"service unavailable",
"gateway",
"rate limit",
"overloaded",
"timeout",
"timed out",
"invalid",
"too many requests",
"permission",
];
function extractLeadingHttpStatus(raw: string): { code: number; rest: string } | null {
const match = raw.match(HTTP_STATUS_CODE_PREFIX_RE);
if (!match) {
return null;
}
const code = Number(match[1]);
if (!Number.isFinite(code)) {
return null;
}
return { code, rest: (match[2] ?? "").trim() };
}
export function isCloudflareOrHtmlErrorPage(raw: string): boolean {
const trimmed = raw.trim();
if (!trimmed) {
return false;
}
const status = extractLeadingHttpStatus(trimmed);
if (!status || status.code < 500) {
return false;
}
if (CLOUDFLARE_HTML_ERROR_CODES.has(status.code)) {
return true;
}
return (
status.code < 600 && HTML_ERROR_PREFIX_RE.test(status.rest) && /<\/html>/i.test(status.rest)
);
}
export function isTransientHttpError(raw: string): boolean {
const trimmed = raw.trim();
if (!trimmed) {
return false;
}
const status = extractLeadingHttpStatus(trimmed);
if (!status) {
return false;
}
return TRANSIENT_HTTP_ERROR_CODES.has(status.code);
}
export function classifyFailoverReasonFromHttpStatus(
status: number | undefined,
message?: string,
): FailoverReason | null {
if (typeof status !== "number" || !Number.isFinite(status)) {
return null;
}
if (status === 402) {
// Some providers (e.g. Anthropic Claude Max plan) surface temporary
// usage/rate-limit failures as HTTP 402. Use a narrow matcher for
// temporary limits to avoid misclassifying billing failures (#30484).
if (message) {
const lower = message.toLowerCase();
// Temporary usage limit signals: retry language + usage/limit terminology
const hasTemporarySignal =
(lower.includes("try again") ||
lower.includes("retry") ||
lower.includes("temporary") ||
lower.includes("cooldown")) &&
(lower.includes("usage limit") ||
lower.includes("rate limit") ||
lower.includes("organization usage"));
if (hasTemporarySignal) {
return "rate_limit";
}
}
return "billing";
}
if (status === 429) {
return "rate_limit";
}
if (status === 401 || status === 403) {
if (message && isAuthPermanentErrorMessage(message)) {
return "auth_permanent";
}
return "auth";
}
if (status === 408) {
return "timeout";
}
if (status === 503) {
if (message && isOverloadedErrorMessage(message)) {
return "overloaded";
}
return "timeout";
}
if (status === 502 || status === 504) {
return "timeout";
}
if (status === 529) {
return "overloaded";
}
if (status === 400) {
// Some providers return quota/balance errors under HTTP 400, so do not
// let the generic format fallback mask an explicit billing signal.
if (message && isBillingErrorMessage(message)) {
return "billing";
}
return "format";
}
return null;
}
function stripFinalTagsFromText(text: string): string {
if (!text) {
return text;
}
return text.replace(FINAL_TAG_RE, "");
}
function collapseConsecutiveDuplicateBlocks(text: string): string {
const trimmed = text.trim();
if (!trimmed) {
return text;
}
const blocks = trimmed.split(/\n{2,}/);
if (blocks.length < 2) {
return text;
}
const normalizeBlock = (value: string) => value.trim().replace(/\s+/g, " ");
const result: string[] = [];
let lastNormalized: string | null = null;
for (const block of blocks) {
const normalized = normalizeBlock(block);
if (lastNormalized && normalized === lastNormalized) {
continue;
}
result.push(block.trim());
lastNormalized = normalized;
}
if (result.length === blocks.length) {
return text;
}
return result.join("\n\n");
}
function isLikelyHttpErrorText(raw: string): boolean {
if (isCloudflareOrHtmlErrorPage(raw)) {
return true;
}
const match = raw.match(HTTP_STATUS_PREFIX_RE);
if (!match) {
return false;
}
const code = Number(match[1]);
if (!Number.isFinite(code) || code < 400) {
return false;
}
const message = match[2].toLowerCase();
return HTTP_ERROR_HINTS.some((hint) => message.includes(hint));
}
function shouldRewriteContextOverflowText(raw: string): boolean {
if (!isContextOverflowError(raw)) {
return false;
}
return (
isRawApiErrorPayload(raw) ||
isLikelyHttpErrorText(raw) ||
ERROR_PREFIX_RE.test(raw) ||
CONTEXT_OVERFLOW_ERROR_HEAD_RE.test(raw)
);
}
type ErrorPayload = Record<string, unknown>;
function isErrorPayloadObject(payload: unknown): payload is ErrorPayload {
if (!payload || typeof payload !== "object" || Array.isArray(payload)) {
return false;
}
const record = payload as ErrorPayload;
if (record.type === "error") {
return true;
}
if (typeof record.request_id === "string" || typeof record.requestId === "string") {
return true;
}
if ("error" in record) {
const err = record.error;
if (err && typeof err === "object" && !Array.isArray(err)) {
const errRecord = err as ErrorPayload;
if (
typeof errRecord.message === "string" ||
typeof errRecord.type === "string" ||
typeof errRecord.code === "string"
) {
return true;
}
}
}
return false;
}
function parseApiErrorPayload(raw: string): ErrorPayload | null {
if (!raw) {
return null;
}
const trimmed = raw.trim();
if (!trimmed) {
return null;
}
const candidates = [trimmed];
if (ERROR_PAYLOAD_PREFIX_RE.test(trimmed)) {
candidates.push(trimmed.replace(ERROR_PAYLOAD_PREFIX_RE, "").trim());
}
for (const candidate of candidates) {
if (!candidate.startsWith("{") || !candidate.endsWith("}")) {
continue;
}
try {
const parsed = JSON.parse(candidate) as unknown;
if (isErrorPayloadObject(parsed)) {
return parsed;
}
} catch {
// ignore parse errors
}
}
return null;
}
export function getApiErrorPayloadFingerprint(raw?: string): string | null {
if (!raw) {
return null;
}
const payload = parseApiErrorPayload(raw);
if (!payload) {
return null;
}
return stableStringify(payload);
}
export function isRawApiErrorPayload(raw?: string): boolean {
return getApiErrorPayloadFingerprint(raw) !== null;
}
export type ApiErrorInfo = {
httpCode?: string;
type?: string;
message?: string;
requestId?: string;
};
export function parseApiErrorInfo(raw?: string): ApiErrorInfo | null {
if (!raw) {
return null;
}
const trimmed = raw.trim();
if (!trimmed) {
return null;
}
let httpCode: string | undefined;
let candidate = trimmed;
const httpPrefixMatch = candidate.match(/^(\d{3})\s+(.+)$/s);
if (httpPrefixMatch) {
httpCode = httpPrefixMatch[1];
candidate = httpPrefixMatch[2].trim();
}
const payload = parseApiErrorPayload(candidate);
if (!payload) {
return null;
}
const requestId =
typeof payload.request_id === "string"
? payload.request_id
: typeof payload.requestId === "string"
? payload.requestId
: undefined;
const topType = typeof payload.type === "string" ? payload.type : undefined;
const topMessage = typeof payload.message === "string" ? payload.message : undefined;
let errType: string | undefined;
let errMessage: string | undefined;
if (payload.error && typeof payload.error === "object" && !Array.isArray(payload.error)) {
const err = payload.error as Record<string, unknown>;
if (typeof err.type === "string") {
errType = err.type;
}
if (typeof err.code === "string" && !errType) {
errType = err.code;
}
if (typeof err.message === "string") {
errMessage = err.message;
}
}
return {
httpCode,
type: errType ?? topType,
message: errMessage ?? topMessage,
requestId,
};
}
export function formatRawAssistantErrorForUi(raw?: string): string {
const trimmed = (raw ?? "").trim();
if (!trimmed) {
return "LLM request failed with an unknown error.";
}
const leadingStatus = extractLeadingHttpStatus(trimmed);
if (leadingStatus && isCloudflareOrHtmlErrorPage(trimmed)) {
return `The AI service is temporarily unavailable (HTTP ${leadingStatus.code}). Please try again in a moment.`;
}
const httpMatch = trimmed.match(HTTP_STATUS_PREFIX_RE);
if (httpMatch) {
const rest = httpMatch[2].trim();
if (!rest.startsWith("{")) {
return `HTTP ${httpMatch[1]}: ${rest}`;
}
}
const info = parseApiErrorInfo(trimmed);
if (info?.message) {
const prefix = info.httpCode ? `HTTP ${info.httpCode}` : "LLM error";
const type = info.type ? ` ${info.type}` : "";
const requestId = info.requestId ? ` (request_id: ${info.requestId})` : "";
return `${prefix}${type}: ${info.message}${requestId}`;
}
return trimmed.length > 600 ? `${trimmed.slice(0, 600)}` : trimmed;
}
export function formatAssistantErrorText(
msg: AssistantMessage,
opts?: { cfg?: OpenClawConfig; sessionKey?: string; provider?: string; model?: string },
): string | undefined {
// Also format errors if errorMessage is present, even if stopReason isn't "error"
const raw = (msg.errorMessage ?? "").trim();
if (msg.stopReason !== "error" && !raw) {
return undefined;
}
if (!raw) {
return "LLM request failed with an unknown error.";
}
const unknownTool =
raw.match(/unknown tool[:\s]+["']?([a-z0-9_-]+)["']?/i) ??
raw.match(/tool\s+["']?([a-z0-9_-]+)["']?\s+(?:not found|is not available)/i);
if (unknownTool?.[1]) {
const rewritten = formatSandboxToolPolicyBlockedMessage({
cfg: opts?.cfg,
sessionKey: opts?.sessionKey,
toolName: unknownTool[1],
});
if (rewritten) {
return rewritten;
}
}
if (isContextOverflowError(raw)) {
return (
"Context overflow: prompt too large for the model. " +
"Try /reset (or /new) to start a fresh session, or use a larger-context model."
);
}
if (isReasoningConstraintErrorMessage(raw)) {
return (
"Reasoning is required for this model endpoint. " +
"Use /think minimal (or any non-off level) and try again."
);
}
// Catch role ordering errors - including JSON-wrapped and "400" prefix variants
if (
/incorrect role information|roles must alternate|400.*role|"message".*role.*information/i.test(
raw,
)
) {
return (
"Message ordering conflict - please try again. " +
"If this persists, use /new to start a fresh session."
);
}
if (isMissingToolCallInputError(raw)) {
return (
"Session history looks corrupted (tool call input missing). " +
"Use /new to start a fresh session. " +
"If this keeps happening, reset the session or delete the corrupted session transcript."
);
}
const invalidRequest = raw.match(/"type":"invalid_request_error".*?"message":"([^"]+)"/);
if (invalidRequest?.[1]) {
return `LLM request rejected: ${invalidRequest[1]}`;
}
const transientCopy = formatRateLimitOrOverloadedErrorCopy(raw);
if (transientCopy) {
return transientCopy;
}
if (isTimeoutErrorMessage(raw)) {
return "LLM request timed out.";
}
if (isBillingErrorMessage(raw)) {
return formatBillingErrorMessage(opts?.provider, opts?.model ?? msg.model);
}
if (isLikelyHttpErrorText(raw) || isRawApiErrorPayload(raw)) {
return formatRawAssistantErrorForUi(raw);
}
// Never return raw unhandled errors - log for debugging but return safe message
if (raw.length > 600) {
log.warn(`Long error truncated: ${raw.slice(0, 200)}`);
}
return raw.length > 600 ? `${raw.slice(0, 600)}` : raw;
}
export function sanitizeUserFacingText(text: string, opts?: { errorContext?: boolean }): string {
if (!text) {
return text;
}
const errorContext = opts?.errorContext ?? false;
const stripped = stripFinalTagsFromText(text);
const trimmed = stripped.trim();
if (!trimmed) {
return "";
}
// Only apply error-pattern rewrites when the caller knows this text is an error payload.
// Otherwise we risk swallowing legitimate assistant text that merely *mentions* these errors.
if (errorContext) {
if (/incorrect role information|roles must alternate/i.test(trimmed)) {
return (
"Message ordering conflict - please try again. " +
"If this persists, use /new to start a fresh session."
);
}
if (shouldRewriteContextOverflowText(trimmed)) {
return (
"Context overflow: prompt too large for the model. " +
"Try /reset (or /new) to start a fresh session, or use a larger-context model."
);
}
if (isBillingErrorMessage(trimmed)) {
return BILLING_ERROR_USER_MESSAGE;
}
if (isRawApiErrorPayload(trimmed) || isLikelyHttpErrorText(trimmed)) {
return formatRawAssistantErrorForUi(trimmed);
}
if (ERROR_PREFIX_RE.test(trimmed)) {
const prefixedCopy = formatRateLimitOrOverloadedErrorCopy(trimmed);
if (prefixedCopy) {
return prefixedCopy;
}
if (isTimeoutErrorMessage(trimmed)) {
return "LLM request timed out.";
}
return formatRawAssistantErrorForUi(trimmed);
}
}
// Strip leading blank lines (including whitespace-only lines) without clobbering indentation on
// the first content line (e.g. markdown/code blocks).
const withoutLeadingEmptyLines = stripped.replace(/^(?:[ \t]*\r?\n)+/, "");
return collapseConsecutiveDuplicateBlocks(withoutLeadingEmptyLines);
}
export function isRateLimitAssistantError(msg: AssistantMessage | undefined): boolean {
if (!msg || msg.stopReason !== "error") {
return false;
}
return isRateLimitErrorMessage(msg.errorMessage ?? "");
}
const TOOL_CALL_INPUT_MISSING_RE =
/tool_(?:use|call)\.(?:input|arguments).*?(?:field required|required)/i;
const TOOL_CALL_INPUT_PATH_RE =
/messages\.\d+\.content\.\d+\.tool_(?:use|call)\.(?:input|arguments)/i;
const IMAGE_DIMENSION_ERROR_RE =
/image dimensions exceed max allowed size for many-image requests:\s*(\d+)\s*pixels/i;
const IMAGE_DIMENSION_PATH_RE = /messages\.(\d+)\.content\.(\d+)\.image/i;
const IMAGE_SIZE_ERROR_RE = /image exceeds\s*(\d+(?:\.\d+)?)\s*mb/i;
export function isMissingToolCallInputError(raw: string): boolean {
if (!raw) {
return false;
}
return TOOL_CALL_INPUT_MISSING_RE.test(raw) || TOOL_CALL_INPUT_PATH_RE.test(raw);
}
export function isBillingAssistantError(msg: AssistantMessage | undefined): boolean {
if (!msg || msg.stopReason !== "error") {
return false;
}
return isBillingErrorMessage(msg.errorMessage ?? "");
}
function isJsonApiInternalServerError(raw: string): boolean {
if (!raw) {
return false;
}
const value = raw.toLowerCase();
// Anthropic often wraps transient 500s in JSON payloads like:
// {"type":"error","error":{"type":"api_error","message":"Internal server error"}}
return value.includes('"type":"api_error"') && value.includes("internal server error");
}
export function parseImageDimensionError(raw: string): {
maxDimensionPx?: number;
messageIndex?: number;
contentIndex?: number;
raw: string;
} | null {
if (!raw) {
return null;
}
const lower = raw.toLowerCase();
if (!lower.includes("image dimensions exceed max allowed size")) {
return null;
}
const limitMatch = raw.match(IMAGE_DIMENSION_ERROR_RE);
const pathMatch = raw.match(IMAGE_DIMENSION_PATH_RE);
return {
maxDimensionPx: limitMatch?.[1] ? Number.parseInt(limitMatch[1], 10) : undefined,
messageIndex: pathMatch?.[1] ? Number.parseInt(pathMatch[1], 10) : undefined,
contentIndex: pathMatch?.[2] ? Number.parseInt(pathMatch[2], 10) : undefined,
raw,
};
}
export function isImageDimensionErrorMessage(raw: string): boolean {
return Boolean(parseImageDimensionError(raw));
}
export function parseImageSizeError(raw: string): {
maxMb?: number;
raw: string;
} | null {
if (!raw) {
return null;
}
const lower = raw.toLowerCase();
if (!lower.includes("image exceeds") || !lower.includes("mb")) {
return null;
}
const match = raw.match(IMAGE_SIZE_ERROR_RE);
return {
maxMb: match?.[1] ? Number.parseFloat(match[1]) : undefined,
raw,
};
}
export function isImageSizeError(errorMessage?: string): boolean {
if (!errorMessage) {
return false;
}
return Boolean(parseImageSizeError(errorMessage));
}
export function isCloudCodeAssistFormatError(raw: string): boolean {
return !isImageDimensionErrorMessage(raw) && matchesFormatErrorPattern(raw);
}
export function isAuthAssistantError(msg: AssistantMessage | undefined): boolean {
if (!msg || msg.stopReason !== "error") {
return false;
}
return isAuthErrorMessage(msg.errorMessage ?? "");
}
export function isModelNotFoundErrorMessage(raw: string): boolean {
if (!raw) {
return false;
}
const lower = raw.toLowerCase();
// Direct pattern matches from OpenClaw internals and common providers.
if (
lower.includes("unknown model") ||
lower.includes("model not found") ||
lower.includes("model_not_found") ||
lower.includes("not_found_error") ||
(lower.includes("does not exist") && lower.includes("model")) ||
(lower.includes("invalid model") && !lower.includes("invalid model reference"))
) {
return true;
}
// Google Gemini: "models/X is not found for api version"
if (/models\/[^\s]+ is not found/i.test(raw)) {
return true;
}
// JSON error payloads: {"status": "NOT_FOUND"} or {"code": 404} combined with not-found text.
if (/\b404\b/.test(raw) && /not[-_ ]?found/i.test(raw)) {
return true;
}
return false;
}
function isCliSessionExpiredErrorMessage(raw: string): boolean {
if (!raw) {
return false;
}
const lower = raw.toLowerCase();
return (
lower.includes("session not found") ||
lower.includes("session does not exist") ||
lower.includes("session expired") ||
lower.includes("session invalid") ||
lower.includes("conversation not found") ||
lower.includes("conversation does not exist") ||
lower.includes("conversation expired") ||
lower.includes("conversation invalid") ||
lower.includes("no such session") ||
lower.includes("invalid session") ||
lower.includes("session id not found") ||
lower.includes("conversation id not found")
);
}
export function classifyFailoverReason(raw: string): FailoverReason | null {
if (isImageDimensionErrorMessage(raw)) {
return null;
}
if (isImageSizeError(raw)) {
return null;
}
if (isCliSessionExpiredErrorMessage(raw)) {
return "session_expired";
}
if (isModelNotFoundErrorMessage(raw)) {
return "model_not_found";
}
if (isPeriodicUsageLimitErrorMessage(raw)) {
return isBillingErrorMessage(raw) ? "billing" : "rate_limit";
}
if (isRateLimitErrorMessage(raw)) {
return "rate_limit";
}
if (isOverloadedErrorMessage(raw)) {
return "overloaded";
}
if (isTransientHttpError(raw)) {
// 529 is always overloaded, even without explicit overload keywords in the body.
const status = extractLeadingHttpStatus(raw.trim());
if (status?.code === 529) {
return "overloaded";
}
// Treat remaining transient 5xx provider failures as retryable transport issues.
return "timeout";
}
if (isJsonApiInternalServerError(raw)) {
return "timeout";
}
if (isCloudCodeAssistFormatError(raw)) {
return "format";
}
if (isBillingErrorMessage(raw)) {
return "billing";
}
if (isTimeoutErrorMessage(raw)) {
return "timeout";
}
if (isAuthPermanentErrorMessage(raw)) {
return "auth_permanent";
}
if (isAuthErrorMessage(raw)) {
return "auth";
}
return null;
}
export function isFailoverErrorMessage(raw: string): boolean {
return classifyFailoverReason(raw) !== null;
}
export function isFailoverAssistantError(msg: AssistantMessage | undefined): boolean {
if (!msg || msg.stopReason !== "error") {
return false;
}
return isFailoverErrorMessage(msg.errorMessage ?? "");
}