From d1ccc1e5428acdb97c4c9bbbea1f49daa1e547c5 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sun, 18 Jan 2026 03:31:38 +0000 Subject: [PATCH] fix: align prompt failover fallback gating (#1136) (thanks @cheeeee) --- CHANGELOG.md | 3 ++ src/agents/model-fallback.test.ts | 72 ++++++++++++++++++++++++++- src/agents/model-fallback.ts | 10 ++++ src/agents/pi-embedded-runner/run.ts | 45 +++++++++++------ src/plugins/runtime/index.ts | 73 ++++++++++++++++++++++++---- 5 files changed, 178 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ee31fe2393..c394f076fb1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,9 @@ Docs: https://docs.clawd.bot - Docs: document plugin slots and memory plugin behavior. - Plugins: migrate bundled messaging extensions to the plugin SDK; resolve plugin-sdk imports in loader. +### Fixes +- Agents: trigger model fallback for prompt-phase failover errors, respecting per-agent overrides. (#1136) — thanks @cheeeee. + ## 2026.1.17-5 ### Changes diff --git a/src/agents/model-fallback.test.ts b/src/agents/model-fallback.test.ts index 381da47ff22..3e027952be1 100644 --- a/src/agents/model-fallback.test.ts +++ b/src/agents/model-fallback.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it, vi } from "vitest"; import type { ClawdbotConfig } from "../config/config.js"; -import { runWithModelFallback } from "./model-fallback.js"; +import { hasModelFallbackCandidates, runWithModelFallback } from "./model-fallback.js"; function makeCfg(overrides: Partial = {}): ClawdbotConfig { return { @@ -310,3 +310,73 @@ describe("runWithModelFallback", () => { expect(result.model).toBe("gpt-4.1-mini"); }); }); + +describe("hasModelFallbackCandidates", () => { + it("returns false when only the primary candidate is available", () => { + const cfg = makeCfg({ + agents: { + defaults: { + model: { + primary: "openai/gpt-4.1-mini", + fallbacks: [], + }, + }, + }, + }); + + expect( + hasModelFallbackCandidates({ + cfg, + provider: "openai", + model: "gpt-4.1-mini", + }), + ).toBe(false); + }); + + it("returns true when the configured primary differs from the requested model", () => { + const cfg = makeCfg({ + agents: { + defaults: { + model: { + primary: "openai/gpt-4.1-mini", + fallbacks: [], + }, + }, + }, + }); + + expect( + hasModelFallbackCandidates({ + cfg, + provider: "openrouter", + model: "meta-llama/llama-3.3-70b:free", + }), + ).toBe(true); + }); + + it("honors an explicit empty fallbacksOverride", () => { + const cfg = makeCfg(); + + expect( + hasModelFallbackCandidates({ + cfg, + provider: "openai", + model: "gpt-4.1-mini", + fallbacksOverride: [], + }), + ).toBe(false); + }); + + it("returns true when fallbacksOverride provides extra candidates", () => { + const cfg = makeCfg(); + + expect( + hasModelFallbackCandidates({ + cfg, + provider: "openai", + model: "gpt-4.1-mini", + fallbacksOverride: ["openai/gpt-4.1"], + }), + ).toBe(true); + }); +}); diff --git a/src/agents/model-fallback.ts b/src/agents/model-fallback.ts index 469f45a2c27..fe8b39c5aee 100644 --- a/src/agents/model-fallback.ts +++ b/src/agents/model-fallback.ts @@ -176,6 +176,16 @@ function resolveFallbackCandidates(params: { return candidates; } +export function hasModelFallbackCandidates(params: { + cfg: ClawdbotConfig | undefined; + provider: string; + model: string; + /** Optional explicit fallbacks list; when provided (even empty), replaces agents.defaults.model.fallbacks. */ + fallbacksOverride?: string[]; +}): boolean { + return resolveFallbackCandidates(params).length > 1; +} + export async function runWithModelFallback(params: { cfg: ClawdbotConfig | undefined; provider: string; diff --git a/src/agents/pi-embedded-runner/run.ts b/src/agents/pi-embedded-runner/run.ts index 25f36a712c7..b1edc560b03 100644 --- a/src/agents/pi-embedded-runner/run.ts +++ b/src/agents/pi-embedded-runner/run.ts @@ -4,6 +4,7 @@ import { enqueueCommandInLane } from "../../process/command-queue.js"; import { resolveUserPath } from "../../utils.js"; import { isMarkdownCapableMessageChannel } from "../../utils/message-channel.js"; import { resolveClawdbotAgentDir } from "../agent-paths.js"; +import { resolveAgentModelFallbacksOverride, resolveSessionAgentId } from "../agent-scope.js"; import { markAuthProfileFailure, markAuthProfileGood, @@ -16,12 +17,13 @@ import { resolveContextWindowInfo, } from "../context-window-guard.js"; import { DEFAULT_CONTEXT_TOKENS, DEFAULT_MODEL, DEFAULT_PROVIDER } from "../defaults.js"; -import { FailoverError, resolveFailoverStatus } from "../failover-error.js"; +import { FailoverError, coerceToFailoverError, resolveFailoverStatus } from "../failover-error.js"; import { ensureAuthProfileStore, getApiKeyForModel, resolveAuthProfileOrder, } from "../model-auth.js"; +import { hasModelFallbackCandidates } from "../model-fallback.js"; import { ensureClawdbotModelsJson } from "../models-config.js"; import { classifyFailoverReason, @@ -30,7 +32,6 @@ import { isCompactionFailureError, isContextOverflowError, isFailoverAssistantError, - isFailoverErrorMessage, isRateLimitAssistantError, isTimeoutErrorMessage, pickFallbackThinkingLevel, @@ -88,6 +89,20 @@ export async function runEmbeddedPiAgent( if (!model) { throw new Error(error ?? `Unknown model: ${provider}/${modelId}`); } + const sessionAgentId = resolveSessionAgentId({ + sessionKey: params.sessionKey, + config: params.config, + }); + const fallbacksOverride = resolveAgentModelFallbacksOverride( + params.config ?? {}, + sessionAgentId, + ); + const hasModelFallbacks = hasModelFallbackCandidates({ + cfg: params.config, + provider, + model: modelId, + fallbacksOverride, + }); const ctxInfo = resolveContextWindowInfo({ cfg: params.config, @@ -290,7 +305,14 @@ export async function runEmbeddedPiAgent( }, }; } - const promptFailoverReason = classifyFailoverReason(errorText); + const promptFailoverError = + coerceToFailoverError(promptError, { + provider, + model: modelId, + profileId: lastProfileId, + }) ?? null; + const promptFailoverReason = + promptFailoverError?.reason ?? classifyFailoverReason(errorText); if (promptFailoverReason && promptFailoverReason !== "timeout" && lastProfileId) { await markAuthProfileFailure({ store: authStore, @@ -301,7 +323,7 @@ export async function runEmbeddedPiAgent( }); } if ( - isFailoverErrorMessage(errorText) && + promptFailoverReason && promptFailoverReason !== "timeout" && (await advanceAuthProfile()) ) { @@ -318,17 +340,14 @@ export async function runEmbeddedPiAgent( thinkLevel = fallbackThinking; continue; } - // FIX: Throw FailoverError for prompt errors when fallbacks configured - // This enables model fallback for quota/rate limit errors during prompt submission - const promptFallbackConfigured = - (params.config?.agents?.defaults?.model?.fallbacks?.length ?? 0) > 0; - if (promptFallbackConfigured && isFailoverErrorMessage(errorText)) { + if (promptFailoverReason && hasModelFallbacks) { + if (promptFailoverError) throw promptFailoverError; throw new FailoverError(errorText, { - reason: promptFailoverReason ?? "unknown", + reason: promptFailoverReason, provider, model: modelId, profileId: lastProfileId, - status: resolveFailoverStatus(promptFailoverReason ?? "unknown"), + status: resolveFailoverStatus(promptFailoverReason), }); } throw promptError; @@ -346,8 +365,6 @@ export async function runEmbeddedPiAgent( continue; } - const fallbackConfigured = - (params.config?.agents?.defaults?.model?.fallbacks?.length ?? 0) > 0; const authFailure = isAuthAssistantError(lastAssistant); const rateLimitFailure = isRateLimitAssistantError(lastAssistant); const failoverFailure = isFailoverAssistantError(lastAssistant); @@ -385,7 +402,7 @@ export async function runEmbeddedPiAgent( const rotated = await advanceAuthProfile(); if (rotated) continue; - if (fallbackConfigured) { + if (hasModelFallbacks) { // Prefer formatted error message (user-friendly) over raw errorMessage const message = (lastAssistant diff --git a/src/plugins/runtime/index.ts b/src/plugins/runtime/index.ts index 0ded31ea617..e34f3eadd33 100644 --- a/src/plugins/runtime/index.ts +++ b/src/plugins/runtime/index.ts @@ -45,13 +45,21 @@ export function createPluginRuntime(): PluginRuntime { hasControlCommand, }, reply: { - dispatchReplyWithBufferedBlockDispatcher, - createReplyDispatcherWithTyping, + dispatchReplyWithBufferedBlockDispatcher: async (params) => { + await dispatchReplyWithBufferedBlockDispatcher( + params as Parameters[0], + ); + }, + createReplyDispatcherWithTyping: (...args) => + createReplyDispatcherWithTyping( + ...(args as Parameters), + ), resolveEffectiveMessagesConfig, resolveHumanDelayConfig, }, routing: { - resolveAgentRoute, + resolveAgentRoute: (params) => + resolveAgentRoute(params as Parameters[0]), }, pairing: { buildPairingReply, @@ -60,19 +68,61 @@ export function createPluginRuntime(): PluginRuntime { }, media: { fetchRemoteMedia, - saveMediaBuffer, + saveMediaBuffer: (buffer, contentType, direction, maxBytes) => + saveMediaBuffer( + Buffer.isBuffer(buffer) ? buffer : Buffer.from(buffer), + contentType, + direction, + maxBytes, + ), }, mentions: { buildMentionRegexes, matchesMentionPatterns, }, groups: { - resolveGroupPolicy: resolveChannelGroupPolicy, - resolveRequireMention: resolveChannelGroupRequireMention, + resolveGroupPolicy: (cfg, channel, accountId, groupId) => + resolveChannelGroupPolicy({ + cfg, + channel: channel as Parameters[0]["channel"], + accountId, + groupId, + }), + resolveRequireMention: (cfg, channel, accountId, groupId, override) => + resolveChannelGroupRequireMention({ + cfg, + channel: channel as Parameters[0]["channel"], + accountId, + groupId, + requireMentionOverride: override, + }), }, debounce: { - createInboundDebouncer, - resolveInboundDebounceMs, + createInboundDebouncer: (opts) => { + const keys = new Set(); + const debouncer = createInboundDebouncer({ + debounceMs: opts.debounceMs, + buildKey: (item) => { + const key = opts.buildKey(item); + if (key) keys.add(key); + return key; + }, + shouldDebounce: opts.shouldDebounce, + onFlush: opts.onFlush, + onError: opts.onError ? (err) => opts.onError?.(err) : undefined, + }); + return { + push: (value) => { + void debouncer.enqueue(value); + }, + flush: async () => { + const pending = Array.from(keys); + keys.clear(); + await Promise.all(pending.map((key) => debouncer.flushKey(key))); + }, + }; + }, + resolveInboundDebounceMs: (cfg, channel) => resolveInboundDebounceMs({ cfg, channel }), }, commands: { resolveCommandAuthorizedFromAuthorizers, @@ -81,7 +131,10 @@ export function createPluginRuntime(): PluginRuntime { logging: { shouldLogVerbose, getChildLogger: (bindings, opts) => { - const logger = getChildLogger(bindings, opts); + const logger = getChildLogger( + bindings, + opts as Parameters[1], + ); return { debug: (message) => logger.debug?.(message), info: (message) => logger.info(message), @@ -91,7 +144,7 @@ export function createPluginRuntime(): PluginRuntime { }, }, state: { - resolveStateDir, + resolveStateDir: () => resolveStateDir(), }, }; }