diff --git a/CHANGELOG.md b/CHANGELOG.md index 953fc047542..d160aea4897 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -133,6 +133,7 @@ Docs: https://docs.openclaw.ai - Providers/catalogs: reject malformed successful LM Studio, GitHub Copilot, DeepInfra, Vercel AI Gateway, and Kilocode model-list responses with provider-owned errors instead of raw parser/type failures or silent fallback catalogs. - Providers/polling: reject array, null, or scalar successful operation status responses with provider-owned malformed JSON errors instead of waiting until timeout. - ACPX/Codex: reap plugin-local Codex ACP adapter orphans on startup after wrapper crashes while keeping direct adapter commands out of launch-lease injection. Fixes #82364. (#82459) Thanks @joshavant. +- Agents/model fallback: periodically probe the configured primary for auto-pinned fallback sessions and clear the pin when it recovers, preventing sessions from staying on a fallback model indefinitely. Fixes #82544. Thanks @crpol. - Telegram: send presentation-only payloads by rendering fallback text and inline buttons instead of treating them as empty. Fixes #82404. (#82449) Thanks @joshavant. - Providers/Kimi: preserve Kimi Coding `reasoning_content` replay and backfill assistant tool-call placeholders when thinking is enabled, so `kimi-for-coding` follow-up tool turns no longer fail after prior tool use. Fixes #82161. Thanks @amknight. - Providers/search tools: reject malformed successful xAI, Gemini, and Kimi web/code search responses with provider-owned errors instead of silent `No response` payloads or ungrounded fallback state. diff --git a/docs/concepts/model-failover.md b/docs/concepts/model-failover.md index 423d5369c2b..9283c3d0310 100644 --- a/docs/concepts/model-failover.md +++ b/docs/concepts/model-failover.md @@ -60,7 +60,7 @@ OpenClaw separates the selected provider/model from why it was selected. That so - **Configured default**: `agents.defaults.model.primary` uses `agents.defaults.model.fallbacks`. - **Agent primary**: `agents.list[].model` is strict unless that agent model object includes its own `fallbacks`. Use `fallbacks: []` to make the strict behavior explicit, or provide a non-empty list to opt that agent into model fallback. -- **Auto fallback override**: a runtime fallback writes `providerOverride`, `modelOverride`, `modelOverrideSource: "auto"`, and the selected origin model before retrying. That auto override can keep walking the configured fallback chain and is cleared by `/new`, `/reset`, and `sessions.reset`. Heartbeat runs without an explicit `heartbeat.model` also clear a direct auto override when its origin no longer matches the current configured default. +- **Auto fallback override**: a runtime fallback writes `providerOverride`, `modelOverride`, `modelOverrideSource: "auto"`, and the selected origin model before retrying. That auto override can keep walking the configured fallback chain without probing the primary on every message, but OpenClaw periodically probes the configured origin again and clears the auto override when it recovers. `/new`, `/reset`, and `sessions.reset` also clear auto-sourced overrides. Heartbeat runs without an explicit `heartbeat.model` clear direct auto overrides when their origin no longer matches the current configured default. - **User session override**: `/model`, the model picker, `session_status(model=...)`, and `sessions.patch` write `modelOverrideSource: "user"`. That is an exact session selection. If the selected provider/model fails before producing a reply, OpenClaw reports the failure instead of answering from an unrelated configured fallback. - **Legacy session override**: older session entries may have `modelOverride` without `modelOverrideSource`. OpenClaw treats those as user overrides so an explicit old selection is not silently converted into fallback behavior. - **Cron payload model**: a cron job `payload.model` / `--model` is a job primary, not a user session override. It uses configured fallbacks unless the job provides `payload.fallbacks`; `payload.fallbacks: []` makes the cron run strict. @@ -305,7 +305,7 @@ That means fallback retries have to coordinate with live model switching: - System-driven model changes such as fallback rotation, heartbeat overrides, or compaction never mark a pending live switch on their own. - User-driven model overrides are treated as exact selections for fallback policy, so an unreachable selected provider surfaces as a failure instead of being masked by `agents.defaults.model.fallbacks`. - Before a fallback retry starts, the reply runner persists the selected fallback override fields to the session entry. -- Auto fallback overrides remain selected on subsequent turns so OpenClaw does not probe a known-bad primary on every message. `/new`, `/reset`, and `sessions.reset` clear auto-sourced overrides and return the session to the configured default. +- Auto fallback overrides remain selected on subsequent turns so OpenClaw does not probe a known-bad primary on every message. OpenClaw periodically probes the configured origin again and clears the auto override when it recovers; `/new`, `/reset`, and `sessions.reset` clear auto-sourced overrides immediately. - `/status` shows the selected model and, when fallback state differs, the active fallback model and reason. - Live-session reconciliation prefers persisted session overrides over stale runtime model fields. - If a live-switch error points at a later candidate in the active fallback chain, OpenClaw jumps directly to that selected model instead of walking unrelated candidates first. diff --git a/docs/concepts/models.md b/docs/concepts/models.md index 0f0c8409e6d..83e686bff76 100644 --- a/docs/concepts/models.md +++ b/docs/concepts/models.md @@ -59,7 +59,7 @@ OpenClaw selects models in this order: The same `provider/model` can mean different things depending on where it came from: - Configured defaults (`agents.defaults.model.primary` and agent-specific primaries) are the normal starting point and use `agents.defaults.model.fallbacks`. -- Auto fallback selections are temporary recovery state. They are stored with `modelOverrideSource: "auto"` so later turns can keep using the fallback chain without probing a known-bad primary first. +- Auto fallback selections are temporary recovery state. They are stored with `modelOverrideSource: "auto"` so later turns can keep using the fallback chain without probing a known-bad primary every time; OpenClaw periodically probes the original primary again and clears the auto selection when it recovers. - User session selections are exact. `/model`, the model picker, `session_status(model=...)`, and `sessions.patch` store `modelOverrideSource: "user"`; if that selected provider/model is unreachable, OpenClaw fails visibly instead of falling through to another configured model. - Cron `--model` / payload `model` is a per-job primary. It still uses configured fallbacks unless the job supplies explicit payload `fallbacks` (use `fallbacks: []` for a strict cron run). - CLI default-model and allowlist pickers respect `models.mode: "replace"` by listing explicit `models.providers.*.models` instead of loading the full built-in catalog. diff --git a/src/agents/agent-command.live-model-switch.test.ts b/src/agents/agent-command.live-model-switch.test.ts index f3ab2623fdd..d50e7ebd5d6 100644 --- a/src/agents/agent-command.live-model-switch.test.ts +++ b/src/agents/agent-command.live-model-switch.test.ts @@ -1,4 +1,5 @@ import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import type { SessionEntry } from "../config/sessions.js"; import { INTERNAL_RUNTIME_CONTEXT_BEGIN, INTERNAL_RUNTIME_CONTEXT_END } from "./internal-events.js"; import { LiveSessionModelSwitchError } from "./live-model-switch-error.js"; @@ -33,6 +34,7 @@ const state = vi.hoisted(() => ({ deliverAgentCommandResultMock: vi.fn(), trajectoryRecordEventMock: vi.fn(), trajectoryFlushMock: vi.fn(async () => undefined), + persistSessionEntryMock: vi.fn(async (..._args: unknown[]): Promise => undefined), clearSessionAuthProfileOverrideMock: vi.fn(), isThinkingLevelSupportedMock: vi.fn((_args: unknown) => true), resolveThinkingDefaultMock: vi.fn((_args: unknown) => "low"), @@ -46,6 +48,7 @@ const state = vi.hoisted(() => ({ authProfileStoreMock: { profiles: {} } as { profiles: Record }, sessionEntryMock: undefined as unknown, sessionStoreMock: undefined as unknown, + storePathMock: undefined as string | undefined, })); vi.mock("./model-fallback.js", () => ({ @@ -66,6 +69,16 @@ vi.mock("./command/attempt-execution.runtime.js", () => ({ sessionFileHasContent: vi.fn(async () => false), })); +vi.mock("./command/attempt-execution.shared.js", async () => { + const actual = await vi.importActual( + "./command/attempt-execution.shared.js", + ); + return { + ...actual, + persistSessionEntry: (...args: unknown[]) => state.persistSessionEntryMock(...args), + }; +}); + vi.mock("./command/delivery.runtime.js", () => ({ deliverAgentCommandResult: (...args: unknown[]) => state.deliverAgentCommandResultMock(...args), })); @@ -99,7 +112,7 @@ vi.mock("./command/session.js", () => ({ skillsSnapshot: { prompt: "", skills: [], version: 0 }, }, sessionStore: state.sessionStoreMock, - storePath: undefined, + storePath: state.storePathMock, isNewSession: false, persistedThinking: undefined, persistedVerbose: undefined, @@ -108,6 +121,10 @@ vi.mock("./command/session.js", () => ({ vi.mock("./command/types.js", () => ({})); +vi.mock("./harness/runtime-plugin.js", () => ({ + ensureSelectedAgentHarnessPlugin: vi.fn(async () => undefined), +})); + vi.mock("../acp/policy.js", () => ({ resolveAcpAgentPolicyError: (...args: unknown[]) => state.resolveAcpAgentPolicyErrorMock(...args), resolveAcpDispatchPolicyError: (...args: unknown[]) => @@ -266,9 +283,13 @@ vi.mock("../utils/message-channel.js", () => ({ })); vi.mock("./agent-scope.js", () => ({ + clearAutoFallbackPrimaryProbeSelection: vi.fn(), + entryMatchesAutoFallbackPrimaryProbe: () => true, hasSessionAutoModelFallbackProvenance: () => false, listAgentEntries: () => [], listAgentIds: () => ["default"], + markAutoFallbackPrimaryProbe: vi.fn(), + resolveAutoFallbackPrimaryProbe: () => undefined, resolveAgentConfig: () => undefined, resolveAgentDir: () => "/tmp/agent", resolveEffectiveModelFallbacks: state.resolveEffectiveModelFallbacksMock, @@ -735,6 +756,30 @@ describe("agentCommand – LiveSessionModelSwitchError retry", () => { state.authProfileStoreMock = { profiles: {} }; state.sessionEntryMock = undefined; state.sessionStoreMock = undefined; + state.storePathMock = undefined; + state.persistSessionEntryMock.mockImplementation(async (...args: unknown[]) => { + const params = args[0] as { + sessionStore?: Record; + sessionKey?: string; + entry?: unknown; + shouldPersist?: (entry: unknown) => boolean; + }; + const current = + params.sessionStore && params.sessionKey + ? params.sessionStore[params.sessionKey] + : undefined; + if (params.shouldPersist && !params.shouldPersist(current)) { + if (current === undefined && params.sessionStore && params.sessionKey) { + delete params.sessionStore[params.sessionKey]; + } + return current; + } + if (params.sessionStore && params.sessionKey && params.entry) { + params.sessionStore[params.sessionKey] = params.entry; + return params.entry; + } + return current; + }); state.buildWorkspaceSkillSnapshotMock.mockReturnValue({ prompt: "", skills: [], @@ -1033,6 +1078,90 @@ describe("agentCommand – LiveSessionModelSwitchError retry", () => { expect(state.runWithModelFallbackMock).toHaveBeenCalledTimes(2); }); + it("does not persist a user live switch as an auto fallback probe result", async () => { + const sessionEntry: SessionEntry = { + sessionId: "session-1", + updatedAt: Date.now(), + providerOverride: "openai", + modelOverride: "claude", + modelOverrideSource: "auto", + modelOverrideFallbackOriginProvider: "anthropic", + modelOverrideFallbackOriginModel: "claude", + skillsSnapshot: { prompt: "", skills: [], version: 0 }, + }; + state.sessionEntryMock = sessionEntry; + const sessionStore: Record = { "agent:main": sessionEntry }; + state.sessionStoreMock = sessionStore; + state.storePathMock = "/tmp/openclaw-session-store.json"; + setupModelSwitchRetry({ + provider: "openai", + model: "gpt-5.4", + authProfileId: "openai:primary", + authProfileIdSource: "user", + }); + state.runAgentAttemptMock.mockResolvedValue(makeSuccessResult("openai", "gpt-5.4")); + + await runBasicAgentCommand(); + + const autoPinnedSwitchWrites = state.persistSessionEntryMock.mock.calls.filter((call) => { + const entry = (call[0] as { entry?: Record } | undefined)?.entry; + return ( + entry?.providerOverride === "openai" && + entry?.modelOverride === "gpt-5.4" && + entry?.modelOverrideSource === "auto" && + entry?.modelOverrideFallbackOriginProvider === "anthropic" + ); + }); + expect(autoPinnedSwitchWrites).toHaveLength(0); + expectRecordFields(mockCallArg(state.updateSessionStoreAfterAgentRunMock), { + fallbackProvider: "openai", + fallbackModel: "gpt-5.4", + }); + }); + + it("does not overwrite a concurrent user model switch after a primary probe", async () => { + const sessionEntry: SessionEntry = { + sessionId: "session-1", + updatedAt: Date.now(), + providerOverride: "openai", + modelOverride: "claude", + modelOverrideSource: "auto", + modelOverrideFallbackOriginProvider: "anthropic", + modelOverrideFallbackOriginModel: "claude", + skillsSnapshot: { prompt: "", skills: [], version: 0 }, + }; + state.sessionEntryMock = sessionEntry; + const sessionStore: Record = { "agent:main": sessionEntry }; + state.sessionStoreMock = sessionStore; + state.storePathMock = "/tmp/openclaw-session-store.json"; + state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => { + const result = await params.run(params.provider, params.model); + sessionStore["agent:main"] = { + sessionId: "session-1", + updatedAt: Date.now(), + providerOverride: "google", + modelOverride: "gemini-3-pro", + modelOverrideSource: "user", + skillsSnapshot: { prompt: "", skills: [], version: 0 }, + }; + return { + result, + provider: params.provider, + model: params.model, + attempts: [], + }; + }); + state.runAgentAttemptMock.mockResolvedValue(makeSuccessResult("anthropic", "claude")); + + await runBasicAgentCommand(); + + expectRecordFields(sessionStore["agent:main"], { + providerOverride: "google", + modelOverride: "gemini-3-pro", + modelOverrideSource: "user", + }); + }); + it("keeps aliased session auth profiles for codex-cli runs", async () => { let capturedAuthProfileProvider: string | undefined; const sessionEntry = { diff --git a/src/agents/agent-command.ts b/src/agents/agent-command.ts index bc4d329c4f7..3c069bdfc96 100644 --- a/src/agents/agent-command.ts +++ b/src/agents/agent-command.ts @@ -39,8 +39,12 @@ import { createTrajectoryRuntimeRecorder } from "../trajectory/runtime.js"; import { resolveMessageChannel } from "../utils/message-channel.js"; import { resolveAgentRuntimeConfig } from "./agent-runtime-config.js"; import { + clearAutoFallbackPrimaryProbeSelection, + entryMatchesAutoFallbackPrimaryProbe, hasSessionAutoModelFallbackProvenance, listAgentIds, + markAutoFallbackPrimaryProbe, + resolveAutoFallbackPrimaryProbe, resolveAgentDir, resolveEffectiveModelFallbacks, resolveSessionAgentId, @@ -226,6 +230,9 @@ type PersistSessionEntryParams = { type OverrideFieldClearedByDelete = | "providerOverride" | "modelOverride" + | "modelOverrideSource" + | "modelOverrideFallbackOriginProvider" + | "modelOverrideFallbackOriginModel" | "authProfileOverride" | "authProfileOverrideSource" | "authProfileOverrideCompactionCount" @@ -237,6 +244,9 @@ type OverrideFieldClearedByDelete = const OVERRIDE_FIELDS_CLEARED_BY_DELETE: OverrideFieldClearedByDelete[] = [ "providerOverride", "modelOverride", + "modelOverrideSource", + "modelOverrideFallbackOriginProvider", + "modelOverrideFallbackOriginModel", "authProfileOverride", "authProfileOverrideSource", "authProfileOverrideCompactionCount", @@ -248,8 +258,12 @@ const OVERRIDE_FIELDS_CLEARED_BY_DELETE: OverrideFieldClearedByDelete[] = [ const OVERRIDE_VALUE_MAX_LENGTH = 256; -async function persistSessionEntry(params: PersistSessionEntryParams): Promise { - await persistSessionEntryBase({ +async function persistSessionEntry( + params: PersistSessionEntryParams & { + shouldPersist?: (entry: SessionEntry | undefined) => boolean; + }, +): Promise { + return await persistSessionEntryBase({ ...params, clearedFields: OVERRIDE_FIELDS_CLEARED_BY_DELETE, }); @@ -834,6 +848,21 @@ async function agentCommandInternal( model = normalizedStored.model; } } + const autoFallbackPrimaryProbe = !hasExplicitRunOverride + ? resolveAutoFallbackPrimaryProbe({ + entry: sessionEntry, + sessionKey, + primaryProvider: defaultProvider, + primaryModel: defaultModel, + }) + : undefined; + let autoFallbackPrimaryProbeSessionEntry: SessionEntry | undefined; + if (autoFallbackPrimaryProbe && sessionEntry) { + provider = autoFallbackPrimaryProbe.provider; + model = autoFallbackPrimaryProbe.model; + autoFallbackPrimaryProbeSessionEntry = { ...sessionEntry }; + clearAutoFallbackPrimaryProbeSelection(autoFallbackPrimaryProbeSessionEntry); + } let providerForAuthProfileValidation = provider; if (hasExplicitRunOverride) { const explicitRef = explicitModelOverride @@ -877,11 +906,11 @@ async function agentCommandInternal( workspaceDir, }); - let sessionEntryForAttempt = sessionEntry; - if (sessionEntry) { - const authProfileId = sessionEntry.authProfileOverride; + let sessionEntryForAttempt = autoFallbackPrimaryProbeSessionEntry ?? sessionEntry; + if (sessionEntryForAttempt) { + const authProfileId = sessionEntryForAttempt.authProfileOverride; if (authProfileId) { - const entry = sessionEntry; + const entry = sessionEntryForAttempt; const store = ensureAuthProfileStore(); const profile = store.profiles[authProfileId]; const validationHarnessPolicy = resolveAvailableAgentHarnessPolicy({ @@ -908,7 +937,7 @@ async function agentCommandInternal( }), ); if (!profileMatchesRuntime) { - if (hasExplicitRunOverride) { + if (hasExplicitRunOverride || autoFallbackPrimaryProbe) { sessionEntryForAttempt = { ...entry, authProfileOverride: undefined, @@ -1025,6 +1054,7 @@ async function agentCommandInternal( let fallbackModel = model; const MAX_LIVE_SWITCH_RETRIES = 5; let liveSwitchRetries = 0; + let autoFallbackPrimaryProbeInterruptedByLiveSwitch = false; const fallbackTrajectoryRecorder = createTrajectoryRuntimeRecorder({ cfg, runId, @@ -1068,6 +1098,22 @@ async function agentCommandInternal( result, }), run: async (providerOverride, modelOverride, runOptions) => { + const isAutoFallbackPrimaryProbeCandidate = + autoFallbackPrimaryProbe && + providerOverride === autoFallbackPrimaryProbe.provider && + modelOverride === autoFallbackPrimaryProbe.model; + const attemptSessionEntry = + autoFallbackPrimaryProbe && + providerOverride === autoFallbackPrimaryProbe.fallbackProvider && + !isAutoFallbackPrimaryProbeCandidate + ? sessionEntry + : sessionEntryForAttempt; + if (isAutoFallbackPrimaryProbeCandidate) { + markAutoFallbackPrimaryProbe({ + probe: autoFallbackPrimaryProbe, + sessionKey, + }); + } const isFallbackRetry = fallbackAttemptIndex > 0; fallbackAttemptIndex += 1; opts.onActiveModelSelected?.({ @@ -1080,7 +1126,7 @@ async function agentCommandInternal( modelFallbacksOverride: effectiveFallbacksOverride, originalProvider: provider, cfg, - sessionEntry: sessionEntryForAttempt, + sessionEntry: attemptSessionEntry, sessionId, sessionKey, sessionAgentId, @@ -1132,6 +1178,49 @@ async function agentCommandInternal( result = fallbackResult.result; fallbackProvider = fallbackResult.provider; fallbackModel = fallbackResult.model; + if ( + autoFallbackPrimaryProbe && + !autoFallbackPrimaryProbeInterruptedByLiveSwitch && + sessionEntry && + sessionStore && + sessionKey && + entryMatchesAutoFallbackPrimaryProbe(sessionEntry, autoFallbackPrimaryProbe) + ) { + const nextSessionEntry = { ...sessionEntry }; + if ( + fallbackProvider === autoFallbackPrimaryProbe.provider && + fallbackModel === autoFallbackPrimaryProbe.model + ) { + clearAutoFallbackPrimaryProbeSelection(nextSessionEntry); + } else { + nextSessionEntry.providerOverride = fallbackProvider; + nextSessionEntry.modelOverride = fallbackModel; + nextSessionEntry.modelOverrideSource = "auto"; + nextSessionEntry.modelOverrideFallbackOriginProvider = + autoFallbackPrimaryProbe.provider; + nextSessionEntry.modelOverrideFallbackOriginModel = autoFallbackPrimaryProbe.model; + if ( + nextSessionEntry.authProfileOverrideSource === "auto" && + fallbackProvider !== autoFallbackPrimaryProbe.fallbackProvider + ) { + delete nextSessionEntry.authProfileOverride; + delete nextSessionEntry.authProfileOverrideSource; + delete nextSessionEntry.authProfileOverrideCompactionCount; + } + nextSessionEntry.updatedAt = Date.now(); + } + const persistedEntry = await persistSessionEntry({ + sessionStore, + sessionKey, + storePath, + entry: nextSessionEntry, + shouldPersist: (current) => + Boolean( + current && entryMatchesAutoFallbackPrimaryProbe(current, autoFallbackPrimaryProbe), + ), + }); + sessionEntry = persistedEntry ?? sessionEntry; + } if (fallbackResult.attempts.length > 0 && result.meta.agentMeta) { result = { ...result, @@ -1214,6 +1303,9 @@ async function agentCommandInternal( } const previousProvider = provider; const previousModel = model; + if (autoFallbackPrimaryProbe) { + autoFallbackPrimaryProbeInterruptedByLiveSwitch = true; + } provider = err.provider; model = err.model; fallbackProvider = err.provider; diff --git a/src/agents/agent-scope.test.ts b/src/agents/agent-scope.test.ts index 0e08f4ae231..4269adffd76 100644 --- a/src/agents/agent-scope.test.ts +++ b/src/agents/agent-scope.test.ts @@ -3,7 +3,10 @@ import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; import type { OpenClawConfig } from "../config/config.js"; +import type { SessionEntry } from "../config/sessions.js"; import { + clearAutoFallbackPrimaryProbeSelection, + markAutoFallbackPrimaryProbe, hasConfiguredModelFallbacks, resolveAgentConfig, resolveDefaultAgentDir, @@ -19,6 +22,7 @@ import { resolveSubagentModelConfigSelection, resolveSubagentModelFallbacksOverride, resolveAgentWorkspaceDir, + resolveAutoFallbackPrimaryProbe, resolveAgentIdByWorkspacePath, resolveAgentIdsByWorkspacePath, setAgentEffectiveModelPrimary, @@ -482,6 +486,276 @@ describe("resolveAgentConfig", () => { ).toEqual(["openai/gpt-5.4"]); }); + it("resolves throttled primary probes for auto fallback selections", () => { + const probeState = new Map(); + const entry: SessionEntry = { + sessionId: "session", + updatedAt: 1, + providerOverride: "google", + modelOverride: "gemini-3-pro", + modelOverrideSource: "auto", + modelOverrideFallbackOriginProvider: "anthropic", + modelOverrideFallbackOriginModel: "claude-sonnet-4-6", + authProfileOverride: "google:fallback", + authProfileOverrideSource: "auto", + }; + + expect( + resolveAutoFallbackPrimaryProbe({ + entry, + sessionKey: "agent:main:session", + primaryProvider: "anthropic", + primaryModel: "claude-sonnet-4-6", + now: 1_000, + minIntervalMs: 60_000, + probeState, + }), + ).toMatchObject({ + provider: "anthropic", + model: "claude-sonnet-4-6", + fallbackAuthProfileId: "google:fallback", + fallbackAuthProfileIdSource: "auto", + }); + markAutoFallbackPrimaryProbe({ + probe: { + provider: "anthropic", + model: "claude-sonnet-4-6", + fallbackProvider: "google", + fallbackModel: "gemini-3-pro", + }, + sessionKey: "agent:main:session", + now: 1_000, + probeState, + }); + expect( + resolveAutoFallbackPrimaryProbe({ + entry, + sessionKey: "agent:main:session", + primaryProvider: "anthropic", + primaryModel: "claude-sonnet-4-6", + now: 30_000, + minIntervalMs: 60_000, + probeState, + }), + ).toBeUndefined(); + expect( + resolveAutoFallbackPrimaryProbe({ + entry: { + ...entry, + providerOverride: "openai", + modelOverride: "gpt-5.4", + }, + sessionKey: "agent:main:session", + primaryProvider: "anthropic", + primaryModel: "claude-sonnet-4-6", + now: 30_000, + minIntervalMs: 60_000, + probeState, + }), + ).toBeUndefined(); + expect( + resolveAutoFallbackPrimaryProbe({ + entry, + sessionKey: "agent:main:session", + primaryProvider: "anthropic", + primaryModel: "claude-sonnet-4-6", + now: 70_000, + minIntervalMs: 60_000, + probeState, + }), + ).toMatchObject({ provider: "anthropic", model: "claude-sonnet-4-6" }); + }); + + it("prunes stale and excess primary probe throttle entries", () => { + const probeState = new Map(); + const probe = { + provider: "anthropic", + model: "claude-sonnet-4-6", + fallbackProvider: "google", + fallbackModel: "gemini-3-pro", + }; + markAutoFallbackPrimaryProbe({ + probe, + sessionKey: "old", + now: 1_000, + minIntervalMs: 100, + maxTrackedProbeKeys: 3, + probeState, + }); + for (let index = 0; index < 4; index += 1) { + markAutoFallbackPrimaryProbe({ + probe, + sessionKey: `new-${index}`, + now: 2_000 + index, + minIntervalMs: 100, + maxTrackedProbeKeys: 3, + probeState, + }); + } + + expect(probeState.size).toBe(3); + expect( + resolveAutoFallbackPrimaryProbe({ + entry: { + providerOverride: "google", + modelOverride: "gemini-3-pro", + modelOverrideSource: "auto", + modelOverrideFallbackOriginProvider: "anthropic", + modelOverrideFallbackOriginModel: "claude-sonnet-4-6", + }, + sessionKey: "old", + primaryProvider: "anthropic", + primaryModel: "claude-sonnet-4-6", + now: 2_004, + minIntervalMs: 100, + maxTrackedProbeKeys: 3, + probeState, + }), + ).toMatchObject({ provider: "anthropic", model: "claude-sonnet-4-6" }); + }); + + it("skips primary probes for strict or stale fallback selections", () => { + const baseEntry: SessionEntry = { + sessionId: "session", + updatedAt: 1, + providerOverride: "google", + modelOverride: "gemini-3-pro", + modelOverrideSource: "auto", + modelOverrideFallbackOriginProvider: "anthropic", + modelOverrideFallbackOriginModel: "claude-sonnet-4-6", + }; + + expect( + resolveAutoFallbackPrimaryProbe({ + entry: { ...baseEntry, modelOverrideSource: "user" }, + primaryProvider: "anthropic", + primaryModel: "claude-sonnet-4-6", + probeState: new Map(), + }), + ).toBeUndefined(); + expect( + resolveAutoFallbackPrimaryProbe({ + entry: baseEntry, + primaryProvider: "openai", + primaryModel: "gpt-5.4", + probeState: new Map(), + }), + ).toBeUndefined(); + expect( + resolveAutoFallbackPrimaryProbe({ + entry: { + ...baseEntry, + providerOverride: "anthropic", + modelOverride: "claude-sonnet-4-6", + }, + primaryProvider: "anthropic", + primaryModel: "claude-sonnet-4-6", + probeState: new Map(), + }), + ).toBeUndefined(); + }); + + it("recognizes recovered auto fallback provenance without a source marker", () => { + expect( + resolveAutoFallbackPrimaryProbe({ + entry: { + providerOverride: "google", + modelOverride: "gemini-3-pro", + modelOverrideFallbackOriginProvider: "anthropic", + modelOverrideFallbackOriginModel: "claude-sonnet-4-6", + }, + primaryProvider: "anthropic", + primaryModel: "claude-sonnet-4-6", + probeState: new Map(), + }), + ).toMatchObject({ provider: "anthropic", model: "claude-sonnet-4-6" }); + }); + + it("preserves legacy auto auth provenance on primary probes", () => { + expect( + resolveAutoFallbackPrimaryProbe({ + entry: { + providerOverride: "google", + modelOverride: "gemini-3-pro", + modelOverrideSource: "auto", + modelOverrideFallbackOriginProvider: "anthropic", + modelOverrideFallbackOriginModel: "claude-sonnet-4-6", + authProfileOverride: "fallback-key", + authProfileOverrideCompactionCount: 1, + }, + primaryProvider: "anthropic", + primaryModel: "claude-sonnet-4-6", + probeState: new Map(), + }), + ).toMatchObject({ + fallbackAuthProfileId: "fallback-key", + fallbackAuthProfileIdSource: "auto", + }); + }); + + it("clears only auto-owned fallback selection state for a primary probe", () => { + const entry: SessionEntry = { + sessionId: "session", + updatedAt: 1, + providerOverride: "google", + modelOverride: "gemini-3-pro", + modelOverrideSource: "auto", + modelOverrideFallbackOriginProvider: "anthropic", + modelOverrideFallbackOriginModel: "claude-sonnet-4-6", + authProfileOverride: "fallback-key", + authProfileOverrideSource: "auto", + authProfileOverrideCompactionCount: 1, + fallbackNoticeSelectedModel: "google/gemini-3-pro", + fallbackNoticeActiveModel: "google/gemini-3-pro", + fallbackNoticeReason: "rate_limit", + }; + + clearAutoFallbackPrimaryProbeSelection(entry, 2); + + expect(entry).toEqual({ sessionId: "session", updatedAt: 2 }); + }); + + it("clears legacy auto auth selection when clearing primary probe state", () => { + const entry: SessionEntry = { + sessionId: "session", + updatedAt: 1, + providerOverride: "google", + modelOverride: "gemini-3-pro", + modelOverrideSource: "auto", + modelOverrideFallbackOriginProvider: "anthropic", + modelOverrideFallbackOriginModel: "claude-sonnet-4-6", + authProfileOverride: "fallback-key", + authProfileOverrideCompactionCount: 1, + }; + + clearAutoFallbackPrimaryProbeSelection(entry, 2); + + expect(entry).toEqual({ sessionId: "session", updatedAt: 2 }); + }); + + it("preserves user-owned auth selection when clearing primary probe state", () => { + const entry: SessionEntry = { + sessionId: "session", + updatedAt: 1, + providerOverride: "google", + modelOverride: "gemini-3-pro", + modelOverrideSource: "auto", + modelOverrideFallbackOriginProvider: "anthropic", + modelOverrideFallbackOriginModel: "claude-sonnet-4-6", + authProfileOverride: "selected-key", + authProfileOverrideSource: "user", + }; + + clearAutoFallbackPrimaryProbeSelection(entry, 2); + + expect(entry).toEqual({ + sessionId: "session", + updatedAt: 2, + authProfileOverride: "selected-key", + authProfileOverrideSource: "user", + }); + }); + it("computes whether any model fallbacks are configured via shared helper", () => { const cfgDefaultsOnly: OpenClawConfig = { agents: { diff --git a/src/agents/agent-scope.ts b/src/agents/agent-scope.ts index 793a7651393..b3aca0ba0ce 100644 --- a/src/agents/agent-scope.ts +++ b/src/agents/agent-scope.ts @@ -1,7 +1,9 @@ import fs from "node:fs"; import path from "node:path"; import { resolveAgentModelFallbackValues } from "../config/model-input.js"; +import { hasSessionAutoModelFallbackProvenance } from "../config/sessions/model-override-provenance.js"; export { hasSessionAutoModelFallbackProvenance } from "../config/sessions/model-override-provenance.js"; +import type { SessionEntry } from "../config/sessions/types.js"; import type { AgentDefaultsConfig } from "../config/types.agent-defaults.js"; import type { AgentModelConfig } from "../config/types.agents-shared.js"; import type { AgentConfig } from "../config/types.agents.js"; @@ -44,6 +46,234 @@ function stripNullBytes(s: string): string { return s.replace(/\0/g, ""); } +const AUTO_FALLBACK_PRIMARY_PROBE_INTERVAL_MS = 15 * 60 * 1000; +const AUTO_FALLBACK_PRIMARY_PROBE_MAX_KEYS = 4096; +const autoFallbackPrimaryProbeState = new Map(); + +function autoFallbackPrimaryProbeStateKey(params: { + sessionKey?: string | null; + primaryProvider: string; + primaryModel: string; +}): string { + return [ + normalizeOptionalString(params.sessionKey) ?? "", + `${params.primaryProvider}/${params.primaryModel}`, + ].join("\0"); +} + +function pruneAutoFallbackPrimaryProbeState(params: { + state: Map; + now: number; + minIntervalMs: number; + maxKeys?: number; +}): void { + const maxKeys = Math.max(1, Math.trunc(params.maxKeys ?? AUTO_FALLBACK_PRIMARY_PROBE_MAX_KEYS)); + const staleBefore = params.now - params.minIntervalMs; + for (const [key, lastProbeAt] of params.state) { + if (!Number.isFinite(lastProbeAt) || lastProbeAt < staleBefore) { + params.state.delete(key); + } + } + if (params.state.size <= maxKeys) { + return; + } + const removeCount = params.state.size - maxKeys; + let removed = 0; + for (const key of params.state.keys()) { + params.state.delete(key); + removed += 1; + if (removed >= removeCount) { + break; + } + } +} + +export type AutoFallbackPrimaryProbe = { + provider: string; + model: string; + fallbackProvider: string; + fallbackModel: string; + fallbackAuthProfileId?: string; + fallbackAuthProfileIdSource?: "auto" | "user"; +}; + +export function resolveAutoFallbackPrimaryProbe(params: { + entry: + | Pick< + SessionEntry, + | "providerOverride" + | "modelOverride" + | "modelOverrideSource" + | "modelOverrideFallbackOriginProvider" + | "modelOverrideFallbackOriginModel" + | "authProfileOverride" + | "authProfileOverrideSource" + | "authProfileOverrideCompactionCount" + > + | null + | undefined; + sessionKey?: string | null; + primaryProvider: string; + primaryModel: string; + now?: number; + minIntervalMs?: number; + maxTrackedProbeKeys?: number; + probeState?: Map; +}): AutoFallbackPrimaryProbe | undefined { + const entry = params.entry; + if (!entry) { + return undefined; + } + const recoveredAutoFallbackOverride = + entry.modelOverrideSource === undefined && hasSessionAutoModelFallbackProvenance(entry); + if (entry.modelOverrideSource !== "auto" && !recoveredAutoFallbackOverride) { + return undefined; + } + + const originProvider = normalizeOptionalString(entry.modelOverrideFallbackOriginProvider); + const originModel = normalizeOptionalString(entry.modelOverrideFallbackOriginModel); + const overrideProvider = normalizeOptionalString(entry.providerOverride); + const overrideModel = normalizeOptionalString(entry.modelOverride); + const primaryProvider = normalizeOptionalString(params.primaryProvider); + const primaryModel = normalizeOptionalString(params.primaryModel); + if (!originProvider || !originModel || !overrideProvider || !overrideModel) { + return undefined; + } + if (!primaryProvider || !primaryModel) { + return undefined; + } + if (originProvider !== primaryProvider || originModel !== primaryModel) { + return undefined; + } + if (overrideProvider === originProvider && overrideModel === originModel) { + return undefined; + } + + const now = params.now ?? Date.now(); + const minIntervalMs = params.minIntervalMs ?? AUTO_FALLBACK_PRIMARY_PROBE_INTERVAL_MS; + const state = params.probeState ?? autoFallbackPrimaryProbeState; + pruneAutoFallbackPrimaryProbeState({ + state, + now, + minIntervalMs, + maxKeys: params.maxTrackedProbeKeys, + }); + const key = autoFallbackPrimaryProbeStateKey({ + sessionKey: params.sessionKey, + primaryProvider: originProvider, + primaryModel: originModel, + }); + const lastProbeAt = state.get(key); + if ( + typeof lastProbeAt === "number" && + Number.isFinite(lastProbeAt) && + now - lastProbeAt < minIntervalMs + ) { + return undefined; + } + const fallbackAuthProfileId = normalizeOptionalString(entry.authProfileOverride); + const fallbackAuthProfileIdSource = + entry.authProfileOverrideSource ?? + (entry.authProfileOverrideCompactionCount !== undefined ? "auto" : undefined); + return { + provider: originProvider, + model: originModel, + fallbackProvider: overrideProvider, + fallbackModel: overrideModel, + ...(fallbackAuthProfileId + ? { + fallbackAuthProfileId, + ...(fallbackAuthProfileIdSource ? { fallbackAuthProfileIdSource } : {}), + } + : {}), + }; +} + +export function markAutoFallbackPrimaryProbe(params: { + probe: AutoFallbackPrimaryProbe; + sessionKey?: string | null; + now?: number; + minIntervalMs?: number; + maxTrackedProbeKeys?: number; + probeState?: Map; +}): void { + const now = params.now ?? Date.now(); + const minIntervalMs = params.minIntervalMs ?? AUTO_FALLBACK_PRIMARY_PROBE_INTERVAL_MS; + const state = params.probeState ?? autoFallbackPrimaryProbeState; + pruneAutoFallbackPrimaryProbeState({ + state, + now, + minIntervalMs, + maxKeys: params.maxTrackedProbeKeys, + }); + const key = autoFallbackPrimaryProbeStateKey({ + sessionKey: params.sessionKey, + primaryProvider: params.probe.provider, + primaryModel: params.probe.model, + }); + state.set(key, now); + pruneAutoFallbackPrimaryProbeState({ + state, + now, + minIntervalMs, + maxKeys: params.maxTrackedProbeKeys, + }); +} + +export function entryMatchesAutoFallbackPrimaryProbe( + entry: + | Pick< + SessionEntry, + | "providerOverride" + | "modelOverride" + | "modelOverrideSource" + | "modelOverrideFallbackOriginProvider" + | "modelOverrideFallbackOriginModel" + > + | null + | undefined, + probe: AutoFallbackPrimaryProbe, +): boolean { + if (!entry) { + return false; + } + const recoveredAutoFallbackOverride = + entry.modelOverrideSource === undefined && hasSessionAutoModelFallbackProvenance(entry); + if (entry.modelOverrideSource !== "auto" && !recoveredAutoFallbackOverride) { + return false; + } + return ( + normalizeOptionalString(entry.providerOverride) === probe.fallbackProvider && + normalizeOptionalString(entry.modelOverride) === probe.fallbackModel && + normalizeOptionalString(entry.modelOverrideFallbackOriginProvider) === probe.provider && + normalizeOptionalString(entry.modelOverrideFallbackOriginModel) === probe.model + ); +} + +export function clearAutoFallbackPrimaryProbeSelection( + entry: SessionEntry, + now = Date.now(), +): void { + delete entry.providerOverride; + delete entry.modelOverride; + delete entry.modelOverrideSource; + delete entry.modelOverrideFallbackOriginProvider; + delete entry.modelOverrideFallbackOriginModel; + if ( + entry.authProfileOverrideSource === "auto" || + (entry.authProfileOverrideSource === undefined && + entry.authProfileOverrideCompactionCount !== undefined) + ) { + delete entry.authProfileOverride; + delete entry.authProfileOverrideSource; + delete entry.authProfileOverrideCompactionCount; + } + delete entry.fallbackNoticeSelectedModel; + delete entry.fallbackNoticeActiveModel; + delete entry.fallbackNoticeReason; + entry.updatedAt = now; +} + export { resolveAgentIdFromSessionKey }; export function resolveSessionAgentIds(params: { diff --git a/src/agents/command/attempt-execution.shared.test.ts b/src/agents/command/attempt-execution.shared.test.ts index 0fe7c34e27f..a40906e2bf3 100644 --- a/src/agents/command/attempt-execution.shared.test.ts +++ b/src/agents/command/attempt-execution.shared.test.ts @@ -1,9 +1,13 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; import { describe, expect, it } from "vitest"; import { INTERNAL_RUNTIME_CONTEXT_BEGIN, INTERNAL_RUNTIME_CONTEXT_END, } from "../internal-events.js"; import { + persistSessionEntry, resolveAcpPromptBody, resolveInternalEventTranscriptBody, } from "./attempt-execution.shared.js"; @@ -75,3 +79,34 @@ describe("attempt execution prompt materialization", () => { expect(transcriptBody).not.toContain(INTERNAL_RUNTIME_CONTEXT_END); }); }); + +describe("persistSessionEntry", () => { + it("clears stale local entries when guarded persistence sees no persisted entry", async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-session-store-")); + try { + const storePath = path.join(dir, "sessions.json"); + const sessionStore = { + main: { + sessionId: "stale", + updatedAt: 1, + }, + }; + + const persisted = await persistSessionEntry({ + sessionStore, + sessionKey: "main", + storePath, + entry: { + sessionId: "stale", + updatedAt: 2, + }, + shouldPersist: (entry) => Boolean(entry), + }); + + expect(persisted).toBeUndefined(); + expect(sessionStore.main).toBeUndefined(); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/src/agents/command/attempt-execution.shared.ts b/src/agents/command/attempt-execution.shared.ts index 24ed2139b82..b9b1b2f1b50 100644 --- a/src/agents/command/attempt-execution.shared.ts +++ b/src/agents/command/attempt-execution.shared.ts @@ -16,10 +16,17 @@ export type PersistSessionEntryParams = { storePath: string; entry: SessionEntry; clearedFields?: string[]; + shouldPersist?: (entry: SessionEntry | undefined) => boolean; }; -export async function persistSessionEntry(params: PersistSessionEntryParams): Promise { +export async function persistSessionEntry( + params: PersistSessionEntryParams, +): Promise { const persisted = await updateSessionStore(params.storePath, (store) => { + const current = store[params.sessionKey]; + if (params.shouldPersist && !params.shouldPersist(current)) { + return current; + } const merged = mergeSessionEntry(store[params.sessionKey], params.entry); for (const field of params.clearedFields ?? []) { if (!Object.hasOwn(params.entry, field)) { @@ -29,7 +36,12 @@ export async function persistSessionEntry(params: PersistSessionEntryParams): Pr store[params.sessionKey] = merged; return merged; }); - params.sessionStore[params.sessionKey] = persisted; + if (persisted) { + params.sessionStore[params.sessionKey] = persisted; + } else { + delete params.sessionStore[params.sessionKey]; + } + return persisted; } export function prependInternalEventContext( diff --git a/src/auto-reply/reply/agent-runner-execution.test.ts b/src/auto-reply/reply/agent-runner-execution.test.ts index b9a69aea8aa..226c7f3ef40 100644 --- a/src/auto-reply/reply/agent-runner-execution.test.ts +++ b/src/auto-reply/reply/agent-runner-execution.test.ts @@ -10,6 +10,7 @@ import type { GetReplyOptions, ReplyPayload } from "../types.js"; import { buildContextOverflowRecoveryText, MAX_LIVE_SWITCH_RETRIES, + resolveRunAfterAutoFallbackPrimaryProbeRecheck, } from "./agent-runner-execution.js"; import type { FollowupRun } from "./queue.js"; import type { ReplyOperation } from "./reply-run-registry.js"; @@ -153,7 +154,14 @@ vi.mock("./agent-runner-utils.js", () => ({ }, }), resolveQueuedReplyRuntimeConfig: (config: T) => config, - resolveModelFallbackOptions: vi.fn(() => ({})), + resolveModelFallbackOptions: vi.fn( + (run: { provider?: string; model?: string; config?: unknown; agentDir?: string }) => ({ + provider: run.provider, + model: run.model, + cfg: run.config, + agentDir: run.agentDir, + }), + ), })); vi.mock("./reply-delivery.js", () => ({ @@ -177,6 +185,8 @@ async function getApplyFallbackCandidateSelectionToEntry() { } type FallbackRunnerParams = { + provider: string; + model: string; run: (provider: string, model: string) => Promise; classifyResult?: (params: { result: { payloads?: Array<{ text?: string; isError?: boolean; isReasoning?: boolean }> }; @@ -486,6 +496,614 @@ describe("runAgentTurnWithFallback", () => { vi.clearAllMocks(); }); + it("rechecks queued auto fallback primary probes before running", async () => { + const { markAutoFallbackPrimaryProbe } = await import("../../agents/agent-scope.js"); + const probe = { + provider: "anthropic", + model: "claude-sonnet-4-6", + fallbackProvider: "google", + fallbackModel: "gemini-3-pro", + fallbackAuthProfileId: "google:fallback", + fallbackAuthProfileIdSource: "auto" as const, + }; + markAutoFallbackPrimaryProbe({ + probe, + sessionKey: "main", + now: Date.now(), + }); + const sessionEntry: SessionEntry = { + sessionId: "session", + updatedAt: 1, + providerOverride: "google", + modelOverride: "gemini-3-pro", + modelOverrideSource: "auto", + modelOverrideFallbackOriginProvider: "anthropic", + modelOverrideFallbackOriginModel: "claude-sonnet-4-6", + authProfileOverride: "google:fallback", + authProfileOverrideSource: "auto", + }; + const run = createFollowupRun().run; + run.provider = "anthropic"; + run.model = "claude-sonnet-4-6"; + run.authProfileId = "anthropic:primary"; + run.authProfileIdSource = "auto"; + run.autoFallbackPrimaryProbe = probe; + + expect( + resolveRunAfterAutoFallbackPrimaryProbeRecheck({ + run, + entry: sessionEntry, + sessionKey: "main", + }), + ).toMatchObject({ + provider: "google", + model: "gemini-3.1-pro-preview", + authProfileId: "google:fallback", + authProfileIdSource: "auto", + autoFallbackPrimaryProbe: undefined, + }); + }); + + it("drops stale queued primary probes after a user model switch", async () => { + const probe = { + provider: "anthropic", + model: "claude-sonnet-4-6", + fallbackProvider: "google", + fallbackModel: "gemini-3-pro", + }; + const sessionEntry: SessionEntry = { + sessionId: "session", + updatedAt: 1, + modelOverride: "openai/gpt-5.4", + modelOverrideSource: "user", + authProfileOverride: "openai:work", + authProfileOverrideSource: "user", + }; + const run = createFollowupRun().run; + run.provider = "anthropic"; + run.model = "claude-sonnet-4-6"; + run.autoFallbackPrimaryProbe = probe; + + expect( + resolveRunAfterAutoFallbackPrimaryProbeRecheck({ + run, + entry: sessionEntry, + sessionKey: "main", + }), + ).toMatchObject({ + provider: "openai", + model: "gpt-5.4", + authProfileId: "openai:work", + authProfileIdSource: "user", + modelOverrideSource: "user", + autoFallbackPrimaryProbe: undefined, + }); + }); + + it("propagates rechecked user selections to post-run state", async () => { + const sessionKey = "rechecked-user-selection"; + const sessionEntry: SessionEntry = { + sessionId: "session", + updatedAt: 1, + providerOverride: "openai", + modelOverride: "gpt-5.4", + modelOverrideSource: "user", + authProfileOverride: "openai:work", + authProfileOverrideSource: "user", + }; + const activeSessionStore = { [sessionKey]: sessionEntry }; + const staleAutoEntry: SessionEntry = { + sessionId: "session", + updatedAt: 1, + providerOverride: "google", + modelOverride: "gemini-3-pro", + modelOverrideSource: "auto", + modelOverrideFallbackOriginProvider: "anthropic", + modelOverrideFallbackOriginModel: "claude-sonnet-4-6", + }; + const followupRun = createFollowupRun(); + followupRun.run.provider = "anthropic"; + followupRun.run.model = "claude-sonnet-4-6"; + followupRun.run.autoFallbackPrimaryProbe = { + provider: "anthropic", + model: "claude-sonnet-4-6", + fallbackProvider: "google", + fallbackModel: "gemini-3-pro", + }; + state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ + result: await params.run(params.provider, params.model), + provider: params.provider, + model: params.model, + attempts: [], + })); + state.runEmbeddedPiAgentMock.mockResolvedValueOnce({ + payloads: [{ text: "user model" }], + meta: { + agentMeta: { + provider: "openai", + model: "gpt-5.4", + }, + }, + }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + await runAgentTurnWithFallback({ + ...createMinimalRunAgentTurnParams({ followupRun }), + sessionKey, + activeSessionStore, + getActiveSessionEntry: () => staleAutoEntry, + }); + + expectRecordFields(followupRun.run as unknown as Record, { + provider: "openai", + model: "gpt-5.4", + authProfileId: "openai:work", + authProfileIdSource: "user", + modelOverrideSource: "user", + }); + expect(followupRun.run.autoFallbackPrimaryProbe).toBeUndefined(); + expectRecordFields(activeSessionStore[sessionKey] as unknown as Record, { + providerOverride: "openai", + modelOverride: "gpt-5.4", + modelOverrideSource: "user", + }); + }); + + it("drops stale queued probe metadata after the auto fallback pin is cleared", () => { + const probe = { + provider: "anthropic", + model: "claude-sonnet-4-6", + fallbackProvider: "google", + fallbackModel: "gemini-3-pro", + }; + const sessionEntry: SessionEntry = { + sessionId: "session", + updatedAt: 1, + authProfileOverride: "google:fallback", + authProfileOverrideSource: "user", + }; + const run = createFollowupRun().run; + run.provider = "anthropic"; + run.model = "claude-sonnet-4-6"; + run.hasSessionModelOverride = true; + run.modelOverrideSource = "auto"; + run.hasAutoFallbackProvenance = true; + run.autoFallbackPrimaryProbe = probe; + + expect( + resolveRunAfterAutoFallbackPrimaryProbeRecheck({ + run, + entry: sessionEntry, + sessionKey: "main", + }), + ).toMatchObject({ + provider: "anthropic", + model: "claude-sonnet-4-6", + autoFallbackPrimaryProbe: undefined, + }); + const rechecked = resolveRunAfterAutoFallbackPrimaryProbeRecheck({ + run, + entry: sessionEntry, + sessionKey: "main", + }); + expect(rechecked.authProfileId).toBeUndefined(); + expect(rechecked.authProfileIdSource).toBeUndefined(); + expect(rechecked.hasSessionModelOverride).toBeUndefined(); + expect(rechecked.modelOverrideSource).toBeUndefined(); + expect(rechecked.hasAutoFallbackProvenance).toBeUndefined(); + }); + + it("keeps fallback auth available when a primary probe falls back", async () => { + const probe = { + provider: "anthropic", + model: "claude-sonnet-4-6", + fallbackProvider: "google", + fallbackModel: "gemini-3-pro", + fallbackAuthProfileId: "google:fallback", + fallbackAuthProfileIdSource: "auto" as const, + }; + const followupRun = createFollowupRun(); + followupRun.run.provider = "anthropic"; + followupRun.run.model = "claude-sonnet-4-6"; + followupRun.run.authProfileId = "anthropic:primary"; + followupRun.run.authProfileIdSource = "auto"; + followupRun.run.autoFallbackPrimaryProbe = probe; + state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ + result: await params.run("google", "gemini-3-pro"), + provider: "google", + model: "gemini-3-pro", + attempts: [{ provider: "anthropic", model: "claude-sonnet-4-6", error: "rate limit" }], + })); + state.runEmbeddedPiAgentMock.mockResolvedValueOnce({ + payloads: [{ text: "fallback" }], + meta: {}, + }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + await runAgentTurnWithFallback(createMinimalRunAgentTurnParams({ followupRun })); + + expectMockCallArgFields(state.runEmbeddedPiAgentMock, 0, "embedded run", { + provider: "google", + model: "gemini-3-pro", + authProfileId: "google:fallback", + authProfileIdSource: "auto", + }); + }); + + it("keeps fallback auth available for later same-provider fallback models", async () => { + const probe = { + provider: "anthropic", + model: "claude-sonnet-4-6", + fallbackProvider: "openai", + fallbackModel: "gpt-5.4", + fallbackAuthProfileId: "openai:fallback", + fallbackAuthProfileIdSource: "auto" as const, + }; + const followupRun = createFollowupRun(); + followupRun.run.provider = "anthropic"; + followupRun.run.model = "claude-sonnet-4-6"; + followupRun.run.authProfileId = "anthropic:primary"; + followupRun.run.authProfileIdSource = "auto"; + followupRun.run.autoFallbackPrimaryProbe = probe; + const sessionKey = "same-provider-fallback-auth"; + const sessionEntry: SessionEntry = { + sessionId: "session", + updatedAt: 1, + providerOverride: "openai", + modelOverride: "gpt-5.4", + modelOverrideSource: "auto", + modelOverrideFallbackOriginProvider: "anthropic", + modelOverrideFallbackOriginModel: "claude-sonnet-4-6", + authProfileOverride: "openai:fallback", + authProfileOverrideSource: "auto", + }; + const activeSessionStore = { [sessionKey]: sessionEntry }; + state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ + result: await params.run("openai", "gpt-5.5"), + provider: "openai", + model: "gpt-5.5", + attempts: [ + { provider: "anthropic", model: "claude-sonnet-4-6", error: "rate limit" }, + { provider: "openai", model: "gpt-5.4", error: "rate limit" }, + ], + })); + state.runEmbeddedPiAgentMock.mockResolvedValueOnce({ + payloads: [{ text: "fallback" }], + meta: {}, + }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + await runAgentTurnWithFallback({ + ...createMinimalRunAgentTurnParams({ followupRun }), + sessionKey, + activeSessionStore, + getActiveSessionEntry: () => activeSessionStore[sessionKey], + }); + + expectMockCallArgFields(state.runEmbeddedPiAgentMock, 0, "embedded run", { + provider: "openai", + model: "gpt-5.5", + authProfileId: "openai:fallback", + authProfileIdSource: "auto", + }); + expectRecordFields(sessionEntry as unknown as Record, { + providerOverride: "openai", + modelOverride: "gpt-5.5", + modelOverrideSource: "auto", + modelOverrideFallbackOriginProvider: "anthropic", + modelOverrideFallbackOriginModel: "claude-sonnet-4-6", + authProfileOverride: "openai:fallback", + authProfileOverrideSource: "auto", + }); + }); + + it("keeps the primary origin when an auto pin is cleared before fallback persists", async () => { + const probe = { + provider: "anthropic", + model: "claude-sonnet-4-6", + fallbackProvider: "openai", + fallbackModel: "gpt-5.4", + fallbackAuthProfileId: "openai:fallback", + fallbackAuthProfileIdSource: "auto" as const, + }; + const followupRun = createFollowupRun(); + followupRun.run.provider = "anthropic"; + followupRun.run.model = "claude-sonnet-4-6"; + followupRun.run.autoFallbackPrimaryProbe = probe; + const sessionKey = "cleared-before-fallback-persists"; + const sessionEntry: SessionEntry = { + sessionId: "session", + updatedAt: 1, + providerOverride: "openai", + modelOverride: "gpt-5.4", + modelOverrideSource: "auto", + modelOverrideFallbackOriginProvider: "anthropic", + modelOverrideFallbackOriginModel: "claude-sonnet-4-6", + authProfileOverride: "openai:fallback", + authProfileOverrideSource: "auto", + }; + const activeSessionStore: Record = { [sessionKey]: sessionEntry }; + state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => { + activeSessionStore[sessionKey] = { + sessionId: "session", + updatedAt: 2, + }; + return { + result: await params.run("openai", "gpt-5.5"), + provider: "openai", + model: "gpt-5.5", + attempts: [{ provider: "anthropic", model: "claude-sonnet-4-6", error: "rate limit" }], + }; + }); + state.runEmbeddedPiAgentMock.mockResolvedValueOnce({ + payloads: [{ text: "fallback" }], + meta: {}, + }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + await runAgentTurnWithFallback({ + ...createMinimalRunAgentTurnParams({ followupRun }), + sessionKey, + activeSessionStore, + getActiveSessionEntry: () => activeSessionStore[sessionKey], + }); + + expectRecordFields(activeSessionStore[sessionKey] as unknown as Record, { + providerOverride: "openai", + modelOverride: "gpt-5.5", + modelOverrideSource: "auto", + modelOverrideFallbackOriginProvider: "anthropic", + modelOverrideFallbackOriginModel: "claude-sonnet-4-6", + authProfileOverride: "openai:fallback", + authProfileOverrideSource: "auto", + }); + }); + + it("re-persists cross-provider same-model fallback pins after an in-flight clear", async () => { + const probe = { + provider: "openai", + model: "gpt-5.5", + fallbackProvider: "azure", + fallbackModel: "gpt-5.5", + fallbackAuthProfileId: "azure:fallback", + fallbackAuthProfileIdSource: "auto" as const, + }; + const followupRun = createFollowupRun(); + followupRun.run.provider = "openai"; + followupRun.run.model = "gpt-5.5"; + followupRun.run.autoFallbackPrimaryProbe = probe; + const sessionKey = "cleared-cross-provider-same-model"; + const sessionEntry: SessionEntry = { + sessionId: "session", + updatedAt: 1, + providerOverride: "azure", + modelOverride: "gpt-5.5", + modelOverrideSource: "auto", + modelOverrideFallbackOriginProvider: "openai", + modelOverrideFallbackOriginModel: "gpt-5.5", + authProfileOverride: "azure:fallback", + authProfileOverrideSource: "auto", + }; + const activeSessionStore: Record = { [sessionKey]: sessionEntry }; + state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => { + activeSessionStore[sessionKey] = { + sessionId: "session", + updatedAt: 2, + }; + return { + result: await params.run("azure", "gpt-5.5"), + provider: "azure", + model: "gpt-5.5", + attempts: [{ provider: "openai", model: "gpt-5.5", error: "rate limit" }], + }; + }); + state.runEmbeddedPiAgentMock.mockResolvedValueOnce({ + payloads: [{ text: "fallback" }], + meta: {}, + }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + await runAgentTurnWithFallback({ + ...createMinimalRunAgentTurnParams({ followupRun }), + sessionKey, + activeSessionStore, + getActiveSessionEntry: () => activeSessionStore[sessionKey], + }); + + expectRecordFields(activeSessionStore[sessionKey] as unknown as Record, { + providerOverride: "azure", + modelOverride: "gpt-5.5", + modelOverrideSource: "auto", + modelOverrideFallbackOriginProvider: "openai", + modelOverrideFallbackOriginModel: "gpt-5.5", + authProfileOverride: "azure:fallback", + authProfileOverrideSource: "auto", + }); + }); + + it("keeps primary auth on same-provider primary probes", async () => { + const probe = { + provider: "openai", + model: "gpt-5.5", + fallbackProvider: "openai", + fallbackModel: "gpt-5.4", + fallbackAuthProfileId: "openai:fallback", + fallbackAuthProfileIdSource: "auto" as const, + }; + const followupRun = createFollowupRun(); + followupRun.run.provider = "openai"; + followupRun.run.model = "gpt-5.5"; + followupRun.run.authProfileId = "openai:primary"; + followupRun.run.authProfileIdSource = "auto"; + followupRun.run.autoFallbackPrimaryProbe = probe; + state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => { + await params.run("openai", "gpt-5.5"); + return { + result: await params.run("openai", "gpt-5.4"), + provider: "openai", + model: "gpt-5.4", + attempts: [{ provider: "openai", model: "gpt-5.5", error: "rate limit" }], + }; + }); + state.runEmbeddedPiAgentMock + .mockResolvedValueOnce({ payloads: [], meta: {} }) + .mockResolvedValueOnce({ payloads: [{ text: "fallback" }], meta: {} }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + await runAgentTurnWithFallback(createMinimalRunAgentTurnParams({ followupRun })); + + expectMockCallArgFields(state.runEmbeddedPiAgentMock, 0, "primary run", { + provider: "openai", + model: "gpt-5.5", + authProfileId: "openai:primary", + authProfileIdSource: "auto", + }); + expectMockCallArgFields(state.runEmbeddedPiAgentMock, 1, "fallback run", { + provider: "openai", + model: "gpt-5.4", + authProfileId: "openai:fallback", + authProfileIdSource: "auto", + }); + }); + + it("does not clear a concurrent user selection after primary probe success", async () => { + const probe = { + provider: "anthropic", + model: "claude-sonnet-4-6", + fallbackProvider: "google", + fallbackModel: "gemini-3-pro", + }; + const sessionKey = "concurrent-user-switch-during-probe"; + const staleAutoEntry: SessionEntry = { + sessionId: "session", + updatedAt: 1, + providerOverride: "google", + modelOverride: "gemini-3-pro", + modelOverrideSource: "auto", + modelOverrideFallbackOriginProvider: "anthropic", + modelOverrideFallbackOriginModel: "claude-sonnet-4-6", + }; + const activeSessionStore = { [sessionKey]: staleAutoEntry }; + const followupRun = createFollowupRun(); + followupRun.run.sessionKey = sessionKey; + followupRun.run.provider = "anthropic"; + followupRun.run.model = "claude-sonnet-4-6"; + followupRun.run.autoFallbackPrimaryProbe = probe; + state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => { + const result = await params.run(params.provider, params.model); + activeSessionStore[sessionKey] = { + sessionId: "session", + updatedAt: 2, + providerOverride: "openai", + modelOverride: "gpt-5.4", + modelOverrideSource: "user", + }; + return { + result, + provider: params.provider, + model: params.model, + attempts: [], + }; + }); + state.runEmbeddedPiAgentMock.mockResolvedValueOnce({ + payloads: [{ text: "primary recovered" }], + meta: { + agentMeta: { + provider: "anthropic", + model: "claude-sonnet-4-6", + }, + }, + }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + await runAgentTurnWithFallback({ + ...createMinimalRunAgentTurnParams({ followupRun }), + sessionKey, + activeSessionStore, + getActiveSessionEntry: () => staleAutoEntry, + }); + + expectRecordFields(activeSessionStore[sessionKey] as unknown as Record, { + providerOverride: "openai", + modelOverride: "gpt-5.4", + modelOverrideSource: "user", + }); + }); + + it("keeps rechecked primary probe runs in sync after live model switches", async () => { + const probe = { + provider: "anthropic", + model: "claude-sonnet-4-6", + fallbackProvider: "openai", + fallbackModel: "gpt-5.5", + fallbackAuthProfileId: "openai:fallback", + fallbackAuthProfileIdSource: "auto" as const, + }; + const sessionEntry: SessionEntry = { + sessionId: "session", + updatedAt: 1, + providerOverride: "openai", + modelOverride: "gpt-5.5", + modelOverrideSource: "auto", + modelOverrideFallbackOriginProvider: "anthropic", + modelOverrideFallbackOriginModel: "claude-sonnet-4-6", + }; + const sessionKey = "live-switch-probe"; + const activeSessionStore = { [sessionKey]: sessionEntry }; + const followupRun = createFollowupRun(); + followupRun.run.sessionKey = sessionKey; + followupRun.run.provider = "anthropic"; + followupRun.run.model = "claude-sonnet-4-6"; + followupRun.run.autoFallbackPrimaryProbe = probe; + const attemptedProviders: Array = []; + state.runWithModelFallbackMock.mockImplementation(async (params: FallbackRunnerParams) => { + attemptedProviders.push(params.provider); + const provider = params.provider ?? "anthropic"; + const model = params.model ?? "claude-sonnet-4-6"; + return { + result: await params.run(provider, model), + provider, + model, + attempts: [], + }; + }); + state.runEmbeddedPiAgentMock + .mockImplementationOnce(async () => { + throw new LiveSessionModelSwitchError({ + provider: "openai", + model: "gpt-5.4", + authProfileId: "openai:primary", + authProfileIdSource: "auto", + }); + }) + .mockResolvedValueOnce({ + payloads: [{ text: "switched" }], + meta: { + agentMeta: { + provider: "openai", + model: "gpt-5.4", + }, + }, + }); + + const runAgentTurnWithFallback = await getRunAgentTurnWithFallback(); + const result = await runAgentTurnWithFallback({ + ...createMinimalRunAgentTurnParams({ followupRun }), + sessionKey, + activeSessionStore, + getActiveSessionEntry: () => activeSessionStore[sessionKey], + }); + + expect(result.kind).toBe("success"); + expect(attemptedProviders).toEqual(["anthropic", "openai"]); + expectMockCallArgFields(state.runEmbeddedPiAgentMock, 1, "embedded run", { + provider: "openai", + model: "gpt-5.4", + authProfileId: "openai:primary", + authProfileIdSource: "auto", + }); + }); + it("forwards the static extra system prompt to CLI backends", async () => { state.isCliProviderMock.mockReturnValue(true); state.runWithModelFallbackMock.mockImplementationOnce(async (params: FallbackRunnerParams) => ({ diff --git a/src/auto-reply/reply/agent-runner-execution.ts b/src/auto-reply/reply/agent-runner-execution.ts index 6d2a71b2a0c..388028e6924 100644 --- a/src/auto-reply/reply/agent-runner-execution.ts +++ b/src/auto-reply/reply/agent-runner-execution.ts @@ -4,7 +4,13 @@ import { hasOutboundReplyContent, resolveSendableOutboundReplyParts, } from "openclaw/plugin-sdk/reply-payload"; -import { hasSessionAutoModelFallbackProvenance } from "../../agents/agent-scope.js"; +import { + clearAutoFallbackPrimaryProbeSelection, + entryMatchesAutoFallbackPrimaryProbe, + hasSessionAutoModelFallbackProvenance, + markAutoFallbackPrimaryProbe, + resolveAutoFallbackPrimaryProbe, +} from "../../agents/agent-scope.js"; import { buildOAuthRefreshFailureLoginCommand, classifyOAuthRefreshFailure, @@ -20,7 +26,11 @@ import { listLegacyRuntimeModelProviderAliases, resolveCliRuntimeExecutionProvider, } from "../../agents/model-runtime-aliases.js"; -import { isCliProvider, resolveModelRefFromString } from "../../agents/model-selection.js"; +import { + isCliProvider, + resolveModelRefFromString, + resolvePersistedOverrideModelRef, +} from "../../agents/model-selection.js"; import { resolveOpenAIRuntimeProviderForPi } from "../../agents/openai-codex-routing.js"; import { BILLING_ERROR_USER_MESSAGE, @@ -279,13 +289,20 @@ export function applyFallbackCandidateSelectionToEntry(params: { run: FollowupRun["run"]; provider: string; model: string; + origin?: { provider: string; model: string }; + force?: boolean; now?: number; }): { updated: boolean; nextState?: FallbackSelectionState } { - if (params.provider === params.run.provider && params.model === params.run.model) { + if ( + !params.force && + params.provider === params.run.provider && + params.model === params.run.model + ) { return { updated: false }; } const scopedAuthProfile = resolveRunAuthProfile(params.run, params.provider); - const origin = resolveFallbackSelectionOrigin({ entry: params.entry, run: params.run }); + const origin = + params.origin ?? resolveFallbackSelectionOrigin({ entry: params.entry, run: params.run }); const nextState = buildFallbackSelectionState({ provider: params.provider, model: params.model, @@ -1124,6 +1141,75 @@ function resolveSessionRuntimeOverrideForProvider(params: { )?.runtime; } +export function resolveRunAfterAutoFallbackPrimaryProbeRecheck(params: { + run: FollowupRun["run"]; + entry?: SessionEntry; + sessionKey?: string; +}): FollowupRun["run"] { + const probe = params.run.autoFallbackPrimaryProbe; + if (!probe || !params.sessionKey) { + return params.run; + } + if (!params.entry) { + return params.run; + } + const resolveEntrySelectionRun = (): FollowupRun["run"] => { + const entryRef = resolvePersistedOverrideModelRef({ + defaultProvider: params.run.provider, + overrideProvider: params.entry?.providerOverride, + overrideModel: params.entry?.modelOverride, + }); + const hasEntryModelOverride = Boolean(entryRef); + const authProfileId = normalizeOptionalString(params.entry?.authProfileOverride); + const fallbackRun: FollowupRun["run"] = { + ...params.run, + provider: entryRef?.provider ?? params.run.provider, + model: entryRef?.model ?? params.run.model, + autoFallbackPrimaryProbe: undefined, + }; + if (hasEntryModelOverride) { + fallbackRun.hasSessionModelOverride = true; + fallbackRun.hasAutoFallbackProvenance = + hasSessionAutoModelFallbackProvenance(params.entry) || undefined; + } else { + delete fallbackRun.hasSessionModelOverride; + delete fallbackRun.hasAutoFallbackProvenance; + } + if (hasEntryModelOverride && params.entry?.modelOverrideSource) { + fallbackRun.modelOverrideSource = params.entry.modelOverrideSource; + } else { + delete fallbackRun.modelOverrideSource; + } + if (hasEntryModelOverride && authProfileId) { + fallbackRun.authProfileId = authProfileId; + if (params.entry?.authProfileOverrideSource) { + fallbackRun.authProfileIdSource = params.entry.authProfileOverrideSource; + } else { + delete fallbackRun.authProfileIdSource; + } + } else if (hasEntryModelOverride) { + delete fallbackRun.authProfileId; + delete fallbackRun.authProfileIdSource; + } + return fallbackRun; + }; + const refreshedProbe = resolveAutoFallbackPrimaryProbe({ + entry: params.entry, + sessionKey: params.sessionKey, + primaryProvider: probe.provider, + primaryModel: probe.model, + }); + if (!refreshedProbe) { + return resolveEntrySelectionRun(); + } + return { + ...params.run, + provider: refreshedProbe.provider, + model: refreshedProbe.model, + autoFallbackPrimaryProbe: refreshedProbe, + }; +} + export async function runAgentTurnWithFallback(params: { commandBody: string; transcriptCommandBody?: string; @@ -1163,14 +1249,56 @@ export async function runAgentTurnWithFallback(params: { let autoCompactionCount = 0; // Track payloads sent directly (not via pipeline) during tool flush to avoid duplicates. const directlySentBlockKeys = new Set(); - const runtimeConfig = resolveQueuedReplyRuntimeConfig(params.followupRun.run.config); - const effectiveRun = - runtimeConfig === params.followupRun.run.config - ? params.followupRun.run + let runnableRun = resolveRunAfterAutoFallbackPrimaryProbeRecheck({ + run: params.followupRun.run, + entry: params.activeSessionStore?.[params.sessionKey ?? ""] ?? params.getActiveSessionEntry(), + sessionKey: params.sessionKey, + }); + if (runnableRun !== params.followupRun.run) { + params.followupRun.run = runnableRun; + } + const runtimeConfig = resolveQueuedReplyRuntimeConfig(runnableRun.config); + let effectiveRun = + runtimeConfig === runnableRun.config + ? runnableRun : { - ...params.followupRun.run, + ...runnableRun, config: runtimeConfig, }; + const resolveRunForFallbackCandidate = (provider: string, model: string): FollowupRun["run"] => { + const probe = effectiveRun.autoFallbackPrimaryProbe; + const isPrimaryProbeCandidate = probe && provider === probe.provider && model === probe.model; + if ( + probe && + provider === probe.fallbackProvider && + !isPrimaryProbeCandidate && + probe.fallbackAuthProfileId + ) { + const candidateRun: FollowupRun["run"] = { + ...effectiveRun, + provider, + model, + authProfileId: probe.fallbackAuthProfileId, + }; + if (probe.fallbackAuthProfileIdSource) { + candidateRun.authProfileIdSource = probe.fallbackAuthProfileIdSource; + } else { + delete candidateRun.authProfileIdSource; + } + return candidateRun; + } + return effectiveRun; + }; + const applyLiveModelSwitchToRun = ( + run: FollowupRun["run"], + err: LiveSessionModelSwitchError, + ): void => { + run.provider = err.provider; + run.model = err.model; + run.authProfileId = err.authProfileId; + run.authProfileIdSource = err.authProfileId ? err.authProfileIdSource : undefined; + run.autoFallbackPrimaryProbe = undefined; + }; const runId = params.opts?.runId ?? crypto.randomUUID(); const replyMediaContext = @@ -1301,20 +1429,21 @@ export async function runAgentTurnWithFallback(params: { const persistFallbackCandidateSelection = async ( provider: string, model: string, + candidateRun: FollowupRun["run"], ): Promise<(() => Promise) | undefined> => { - if (params.followupRun.run.hasOneTurnModelOverride === true) { + if (effectiveRun.hasOneTurnModelOverride === true) { return undefined; } if ( !params.sessionKey || !params.activeSessionStore || - (provider === params.followupRun.run.provider && model === params.followupRun.run.model) + (provider === effectiveRun.provider && model === effectiveRun.model) ) { return undefined; } const activeSessionEntry = - params.getActiveSessionEntry() ?? params.activeSessionStore[params.sessionKey]; + params.activeSessionStore[params.sessionKey] ?? params.getActiveSessionEntry(); if (!activeSessionEntry) { return undefined; } @@ -1341,11 +1470,28 @@ export async function runAgentTurnWithFallback(params: { } const previousState = snapshotFallbackSelectionState(activeSessionEntry); + const selectionRun = + candidateRun !== effectiveRun && effectiveRun.autoFallbackPrimaryProbe + ? { + ...candidateRun, + provider: candidateRun.provider, + model: effectiveRun.model, + } + : candidateRun; const applied = applyFallbackCandidateSelectionToEntry({ entry: activeSessionEntry, - run: params.followupRun.run, + run: selectionRun, provider, model, + force: candidateRun !== effectiveRun && Boolean(effectiveRun.autoFallbackPrimaryProbe), + ...(effectiveRun.autoFallbackPrimaryProbe + ? { + origin: { + provider: effectiveRun.autoFallbackPrimaryProbe.provider, + model: effectiveRun.autoFallbackPrimaryProbe.model, + }, + } + : {}), }); const nextState = applied.nextState; if (!applied.updated || !nextState) { @@ -1393,6 +1539,45 @@ export async function runAgentTurnWithFallback(params: { }); }; }; + const clearRecoveredAutoFallbackPrimaryProbe = async (paramsForClear: { + provider: string; + model: string; + }): Promise => { + const probe = effectiveRun.autoFallbackPrimaryProbe; + if (!probe) { + return; + } + if (paramsForClear.provider !== probe.provider || paramsForClear.model !== probe.model) { + return; + } + if (!params.sessionKey || !params.activeSessionStore) { + return; + } + const activeSessionEntry = + params.activeSessionStore[params.sessionKey] ?? params.getActiveSessionEntry(); + if (!activeSessionEntry) { + return; + } + if (!entryMatchesAutoFallbackPrimaryProbe(activeSessionEntry, probe)) { + return; + } + clearAutoFallbackPrimaryProbeSelection(activeSessionEntry); + params.activeSessionStore[params.sessionKey] = activeSessionEntry; + if (!params.storePath) { + return; + } + await updateSessionStore(params.storePath, (store) => { + const persistedEntry = store[params.sessionKey!]; + if (!persistedEntry) { + return; + } + if (!entryMatchesAutoFallbackPrimaryProbe(persistedEntry, probe)) { + return; + } + clearAutoFallbackPrimaryProbeSelection(persistedEntry); + store[params.sessionKey!] = persistedEntry; + }); + }; while (true) { try { @@ -1503,6 +1688,14 @@ export async function runAgentTurnWithFallback(params: { return classification; }, run: async (provider, model, runOptions) => { + const candidateRun = resolveRunForFallbackCandidate(provider, model); + const activeProbe = effectiveRun.autoFallbackPrimaryProbe; + if (activeProbe && provider === activeProbe.provider && model === activeProbe.model) { + markAutoFallbackPrimaryProbe({ + probe: activeProbe, + sessionKey: params.sessionKey, + }); + } // Notify that model selection is complete (including after fallback). // This allows responsePrefix template interpolation with the actual model. params.opts?.onModelSelected?.({ @@ -1515,6 +1708,7 @@ export async function runAgentTurnWithFallback(params: { rollbackFallbackCandidateSelection = await persistFallbackCandidateSelection( provider, model, + candidateRun, ); if (rollbackFallbackCandidateSelection) { pendingFallbackCandidateRollback = { @@ -1562,13 +1756,9 @@ export async function runAgentTurnWithFallback(params: { const cliSessionBinding = isRoomEventCliTurn ? undefined : getCliSessionBinding(params.getActiveSessionEntry(), cliExecutionProvider); - const authProfile = resolveRunAuthProfile( - params.followupRun.run, - cliExecutionProvider, - { - config: runtimeConfig, - }, - ); + const authProfile = resolveRunAuthProfile(candidateRun, cliExecutionProvider, { + config: runtimeConfig, + }); const hookMessageProvider = resolveOriginMessageProvider({ originatingChannel: params.followupRun.originatingChannel, provider: params.sessionCtx.Provider, @@ -1765,7 +1955,7 @@ export async function runAgentTurnWithFallback(params: { } const { embeddedContext, senderContext, runBaseParams } = buildEmbeddedRunExecutionParams( { - run: effectiveRun, + run: candidateRun, sessionCtx: params.sessionCtx, hasRepliedRef: params.opts?.hasRepliedRef, provider, @@ -2139,6 +2329,10 @@ export async function runAgentTurnWithFallback(params: { code: attempt.code || undefined, })) : []; + await clearRecoveredAutoFallbackPrimaryProbe({ + provider: fallbackProvider, + model: fallbackModel, + }); // Some embedded runs surface context overflow as an error payload instead of throwing. // Treat those as a session-level failure and auto-recover by starting a fresh session. @@ -2212,12 +2406,13 @@ export async function runAgentTurnWithFallback(params: { }), }; } - params.followupRun.run.provider = err.provider; - params.followupRun.run.model = err.model; - params.followupRun.run.authProfileId = err.authProfileId; - params.followupRun.run.authProfileIdSource = err.authProfileId - ? err.authProfileIdSource - : undefined; + applyLiveModelSwitchToRun(params.followupRun.run, err); + if (runnableRun !== params.followupRun.run) { + applyLiveModelSwitchToRun(runnableRun, err); + } + if (effectiveRun !== runnableRun && effectiveRun !== params.followupRun.run) { + applyLiveModelSwitchToRun(effectiveRun, err); + } fallbackProvider = err.provider; fallbackModel = err.model; continue; diff --git a/src/auto-reply/reply/followup-runner.test.ts b/src/auto-reply/reply/followup-runner.test.ts index e61bc843f2d..9e8368d4136 100644 --- a/src/auto-reply/reply/followup-runner.test.ts +++ b/src/auto-reply/reply/followup-runner.test.ts @@ -533,6 +533,129 @@ function createAsyncReplySpy() { return vi.fn(async () => {}); } +describe("createFollowupRunner auto fallback primary probes", () => { + it("clears queued auto fallback pins after a successful primary probe", async () => { + const sessionKey = "probe-clear"; + const sessionEntry: SessionEntry = { + sessionId: "session-1", + updatedAt: Date.now(), + providerOverride: "openai", + modelOverride: "gpt-5.4", + modelOverrideSource: "auto", + modelOverrideFallbackOriginProvider: "anthropic", + modelOverrideFallbackOriginModel: "claude", + }; + const sessionStore = { [sessionKey]: sessionEntry }; + runEmbeddedPiAgentMock.mockResolvedValueOnce({ + payloads: [], + meta: { agentMeta: { provider: "anthropic", model: "claude" } }, + }); + + const runner = createFollowupRunner({ + typing: createMockTypingController(), + typingMode: "instant", + sessionEntry, + sessionStore, + sessionKey, + defaultModel: "anthropic/claude", + }); + + await runner( + createQueuedRun({ + run: { + sessionKey, + provider: "anthropic", + model: "claude", + autoFallbackPrimaryProbe: { + provider: "anthropic", + model: "claude", + fallbackProvider: "openai", + fallbackModel: "gpt-5.4", + }, + }, + }), + ); + + const call = requireLastMockCallArg(runEmbeddedPiAgentMock, "run embedded pi agent"); + expect(call.provider).toBe("anthropic"); + expect(call.model).toBe("claude"); + expect(sessionEntry.providerOverride).toBeUndefined(); + expect(sessionEntry.modelOverride).toBeUndefined(); + expect(sessionEntry.modelOverrideSource).toBeUndefined(); + expect(sessionEntry.modelOverrideFallbackOriginProvider).toBeUndefined(); + expect(sessionEntry.modelOverrideFallbackOriginModel).toBeUndefined(); + }); + + it("rechecks queued probe throttle and keeps fallback auth when probe is not due", async () => { + const sessionKey = "probe-skip"; + const probe = { + provider: "anthropic", + model: "claude", + fallbackProvider: "openai", + fallbackModel: "gpt-5.4", + fallbackAuthProfileId: "openai:fallback", + fallbackAuthProfileIdSource: "auto" as const, + }; + const sessionEntry: SessionEntry = { + sessionId: "session-1", + updatedAt: Date.now(), + providerOverride: "openai", + modelOverride: "gpt-5.4", + modelOverrideSource: "auto", + modelOverrideFallbackOriginProvider: "anthropic", + modelOverrideFallbackOriginModel: "claude", + authProfileOverride: "openai:fallback", + authProfileOverrideSource: "auto", + }; + const sessionStore = { [sessionKey]: sessionEntry }; + const { markAutoFallbackPrimaryProbe } = await import("../../agents/agent-scope.js"); + markAutoFallbackPrimaryProbe({ probe, sessionKey }); + runEmbeddedPiAgentMock.mockResolvedValueOnce({ + payloads: [], + meta: { agentMeta: { provider: "openai", model: "gpt-5.4" } }, + }); + runPreflightCompactionIfNeededMock.mockImplementationOnce( + async (params: { followupRun: FollowupRun; sessionEntry?: SessionEntry }) => { + expect(params.followupRun.run.provider).toBe("openai"); + expect(params.followupRun.run.model).toBe("gpt-5.4"); + expect(params.followupRun.run.autoFallbackPrimaryProbe).toBeUndefined(); + return params.sessionEntry; + }, + ); + + const runner = createFollowupRunner({ + typing: createMockTypingController(), + typingMode: "instant", + sessionEntry, + sessionStore, + sessionKey, + defaultModel: "anthropic/claude", + }); + + await runner( + createQueuedRun({ + run: { + sessionKey, + provider: "anthropic", + model: "claude", + authProfileId: "anthropic:primary", + authProfileIdSource: "auto", + autoFallbackPrimaryProbe: probe, + }, + }), + ); + + const call = requireLastMockCallArg(runEmbeddedPiAgentMock, "run embedded pi agent"); + expect(call.provider).toBe("openai"); + expect(call.model).toBe("gpt-5.4"); + expect(call.authProfileId).toBe("openai:fallback"); + expect(call.authProfileIdSource).toBe("auto"); + expect(sessionEntry.providerOverride).toBe("openai"); + expect(sessionEntry.modelOverride).toBe("gpt-5.4"); + expect(sessionEntry.modelOverrideSource).toBe("auto"); + }); +}); + describe("createFollowupRunner runtime config", () => { it("uses the active runtime snapshot for queued embedded followup runs", async () => { const sourceConfig: OpenClawConfig = { diff --git a/src/auto-reply/reply/followup-runner.ts b/src/auto-reply/reply/followup-runner.ts index fe70eb1f847..7452a9af5fb 100644 --- a/src/auto-reply/reply/followup-runner.ts +++ b/src/auto-reply/reply/followup-runner.ts @@ -1,5 +1,10 @@ import crypto from "node:crypto"; import { hasOutboundReplyContent } from "openclaw/plugin-sdk/reply-payload"; +import { + clearAutoFallbackPrimaryProbeSelection, + entryMatchesAutoFallbackPrimaryProbe, + markAutoFallbackPrimaryProbe, +} from "../../agents/agent-scope.js"; import { resolveBootstrapWarningSignaturesSeen } from "../../agents/bootstrap-budget.js"; import { resolveContextTokensForModel } from "../../agents/context.js"; import { DEFAULT_CONTEXT_TOKENS } from "../../agents/defaults.js"; @@ -9,7 +14,7 @@ import { buildAgentRuntimeDeliveryPlan, buildAgentRuntimeOutcomePlan, } from "../../agents/runtime-plan/build.js"; -import type { SessionEntry } from "../../config/sessions.js"; +import { updateSessionStore, type SessionEntry } from "../../config/sessions.js"; import type { TypingMode } from "../../config/types.js"; import { logVerbose } from "../../globals.js"; import { registerAgentRunContext } from "../../infra/agent-events.js"; @@ -17,6 +22,7 @@ import { formatErrorMessage } from "../../infra/errors.js"; import { defaultRuntime } from "../../runtime.js"; import { isInternalMessageChannel } from "../../utils/message-channel.js"; import type { GetReplyOptions, ReplyPayload } from "../types.js"; +import { resolveRunAfterAutoFallbackPrimaryProbeRecheck } from "./agent-runner-execution.js"; import { runPreflightCompactionIfNeeded } from "./agent-runner-memory.js"; import { resolveQueuedReplyExecutionConfig, @@ -222,11 +228,22 @@ export function createFollowupRunner(params: { }); const replySessionKey = queued.run.sessionKey ?? sessionKey; const runtimeConfig = resolveQueuedReplyRuntimeConfig(queued.run.config); - const effectiveQueued = + let effectiveQueued = runtimeConfig === queued.run.config ? queued : { ...queued, run: { ...queued.run, config: runtimeConfig } }; - const run = effectiveQueued.run; + let run = effectiveQueued.run; + let activeSessionEntry = + (replySessionKey ? sessionStore?.[replySessionKey] : undefined) ?? + (replySessionKey === sessionKey ? sessionEntry : undefined); + run = resolveRunAfterAutoFallbackPrimaryProbeRecheck({ + run, + entry: activeSessionEntry, + sessionKey: replySessionKey, + }); + if (run !== effectiveQueued.run) { + effectiveQueued = { ...effectiveQueued, run }; + } replyOperation = createReplyOperation({ sessionId: run.sessionId, sessionKey: replySessionKey ?? "", @@ -251,8 +268,6 @@ export function createFollowupRunner(params: { let runResult: Awaited>; let fallbackProvider = run.provider; let fallbackModel = run.model; - let activeSessionEntry = - (sessionKey ? sessionStore?.[sessionKey] : undefined) ?? sessionEntry; activeSessionEntry = await runPreflightCompactionIfNeeded({ cfg: runtimeConfig, followupRun: effectiveQueued, @@ -261,7 +276,7 @@ export function createFollowupRunner(params: { agentCfgContextTokens, sessionEntry: activeSessionEntry, sessionStore, - sessionKey, + sessionKey: replySessionKey, storePath, isHeartbeat: opts?.isHeartbeat === true, replyOperation, @@ -269,6 +284,72 @@ export function createFollowupRunner(params: { let bootstrapPromptWarningSignaturesSeen = resolveBootstrapWarningSignaturesSeen( activeSessionEntry?.systemPromptReport, ); + const resolveRunForFallbackCandidate = ( + provider: string, + model: string, + ): FollowupRun["run"] => { + const probe = run.autoFallbackPrimaryProbe; + const isPrimaryProbeCandidate = + probe && provider === probe.provider && model === probe.model; + if ( + probe && + provider === probe.fallbackProvider && + !isPrimaryProbeCandidate && + probe.fallbackAuthProfileId + ) { + const candidateRun: FollowupRun["run"] = { + ...run, + provider, + model, + authProfileId: probe.fallbackAuthProfileId, + }; + if (probe.fallbackAuthProfileIdSource) { + candidateRun.authProfileIdSource = probe.fallbackAuthProfileIdSource; + } else { + delete candidateRun.authProfileIdSource; + } + return candidateRun; + } + return run; + }; + const clearRecoveredAutoFallbackPrimaryProbe = async (paramsForClear: { + provider: string; + model: string; + }): Promise => { + const probe = run.autoFallbackPrimaryProbe; + if (!probe) { + return; + } + if (paramsForClear.provider !== probe.provider || paramsForClear.model !== probe.model) { + return; + } + if (!replySessionKey || !sessionStore) { + return; + } + const entry = sessionStore[replySessionKey] ?? activeSessionEntry; + if (!entry || !entryMatchesAutoFallbackPrimaryProbe(entry, probe)) { + return; + } + clearAutoFallbackPrimaryProbeSelection(entry); + sessionStore[replySessionKey] = entry; + activeSessionEntry = entry; + if (!storePath) { + return; + } + await updateSessionStore(storePath, (store) => { + const persistedEntry = store[replySessionKey]; + if (!persistedEntry) { + return; + } + if (!entryMatchesAutoFallbackPrimaryProbe(persistedEntry, probe)) { + return; + } + clearAutoFallbackPrimaryProbeSelection(persistedEntry); + store[replySessionKey] = persistedEntry; + }); + }; + fallbackProvider = run.provider; + fallbackModel = run.model; replyOperation.setPhase("running"); try { const outcomePlan = buildAgentRuntimeOutcomePlan(); @@ -279,7 +360,17 @@ export function createFollowupRunner(params: { classifyResult: ({ result, provider, model }) => outcomePlan.classifyRunResult({ result, provider, model }), run: async (provider, model, runOptions) => { - const authProfile = resolveRunAuthProfile(run, provider, { config: runtimeConfig }); + const candidateRun = resolveRunForFallbackCandidate(provider, model); + const activeProbe = run.autoFallbackPrimaryProbe; + if (activeProbe && provider === activeProbe.provider && model === activeProbe.model) { + markAutoFallbackPrimaryProbe({ + probe: activeProbe, + sessionKey: replySessionKey, + }); + } + const authProfile = resolveRunAuthProfile(candidateRun, provider, { + config: runtimeConfig, + }); let attemptCompactionCount = 0; try { const result = await runEmbeddedPiAgent({ @@ -375,6 +466,10 @@ export function createFollowupRunner(params: { runResult = fallbackResult.result; fallbackProvider = fallbackResult.provider; fallbackModel = fallbackResult.model; + await clearRecoveredAutoFallbackPrimaryProbe({ + provider: fallbackProvider, + model: fallbackModel, + }); } catch (err) { const message = formatErrorMessage(err); replyOperation.fail("run_failed", err); @@ -393,14 +488,14 @@ export function createFollowupRunner(params: { provider: providerUsed, model: modelUsed, contextTokensOverride: agentCfgContextTokens, - fallbackContextTokens: sessionEntry?.contextTokens ?? DEFAULT_CONTEXT_TOKENS, + fallbackContextTokens: activeSessionEntry?.contextTokens ?? DEFAULT_CONTEXT_TOKENS, allowAsyncLoad: false, }) ?? DEFAULT_CONTEXT_TOKENS; - if (storePath && sessionKey) { + if (storePath && replySessionKey) { await persistRunSessionUsage({ storePath, - sessionKey, + sessionKey: replySessionKey, cfg: runtimeConfig, usage, lastCallUsage: runResult.meta?.agentMeta?.lastCallUsage, @@ -441,9 +536,9 @@ export function createFollowupRunner(params: { const previousSessionId = run.sessionId; const count = await incrementRunCompactionCount({ cfg: runtimeConfig, - sessionEntry, + sessionEntry: activeSessionEntry, sessionStore, - sessionKey, + sessionKey: replySessionKey, storePath, amount: autoCompactionCount, compactionTokensAfter: runResult.meta?.agentMeta?.compactionTokensAfter, @@ -453,7 +548,7 @@ export function createFollowupRunner(params: { newSessionFile: runResult.meta?.agentMeta?.sessionFile, }); const refreshedSessionEntry = - sessionKey && sessionStore ? sessionStore[sessionKey] : undefined; + replySessionKey && sessionStore ? sessionStore[replySessionKey] : undefined; if (refreshedSessionEntry) { const queueKey = run.sessionKey ?? sessionKey; if (queueKey) { diff --git a/src/auto-reply/reply/get-reply-directives.ts b/src/auto-reply/reply/get-reply-directives.ts index 4508019a1a9..c0060a4e9f2 100644 --- a/src/auto-reply/reply/get-reply-directives.ts +++ b/src/auto-reply/reply/get-reply-directives.ts @@ -172,6 +172,7 @@ export async function resolveReplyDirectives(params: { provider: string; model: string; hasOneTurnModelOverride?: boolean; + skipStoredModelOverride?: boolean; hasResolvedHeartbeatModelOverride: boolean; typing: TypingController; opts?: GetReplyOptions; @@ -202,6 +203,7 @@ export async function resolveReplyDirectives(params: { provider: initialProvider, model: initialModel, hasOneTurnModelOverride, + skipStoredModelOverride, hasResolvedHeartbeatModelOverride, typing, opts, @@ -538,6 +540,7 @@ export async function resolveReplyDirectives(params: { model, hasModelDirective: directives.hasModelDirective, hasOneTurnModelOverride, + skipStoredModelOverride, hasResolvedHeartbeatModelOverride, isHeartbeat: opts?.isHeartbeat === true, }); diff --git a/src/auto-reply/reply/get-reply-run.ts b/src/auto-reply/reply/get-reply-run.ts index ef3ca174d94..3a32164ae59 100644 --- a/src/auto-reply/reply/get-reply-run.ts +++ b/src/auto-reply/reply/get-reply-run.ts @@ -1,5 +1,9 @@ import crypto from "node:crypto"; -import { hasSessionAutoModelFallbackProvenance } from "../../agents/agent-scope.js"; +import { + clearAutoFallbackPrimaryProbeSelection, + hasSessionAutoModelFallbackProvenance, + type AutoFallbackPrimaryProbe, +} from "../../agents/agent-scope.js"; import { resolveSessionAuthProfileOverride } from "../../agents/auth-profiles/session-override.js"; import type { ExecToolDefaults } from "../../agents/bash-tools.js"; import { resolveFastModeState } from "../../agents/fast-mode.js"; @@ -354,6 +358,7 @@ type RunPreparedReplyParams = { hasAppliedImageModelOverride?: boolean; imageModelOverrideBaseProvider?: string; imageModelFallbacksOverride?: string[]; + autoFallbackPrimaryProbe?: AutoFallbackPrimaryProbe; }; export async function runPreparedReply( @@ -916,12 +921,17 @@ export async function runPreparedReply( authProfileIdSource: preparedSessionState.sessionEntry?.authProfileOverrideSource, }; } - const shouldUseEphemeralSession = shouldResolveEphemeralAuthProfileForImageOverride(); + const shouldUseEphemeralSession = + shouldResolveEphemeralAuthProfileForImageOverride() || + params.autoFallbackPrimaryProbe !== undefined; const authSessionKey = shouldUseEphemeralSession ? (sessionKey ?? sessionIdFinal) : sessionKey; const authSessionEntry = shouldUseEphemeralSession && preparedSessionState.sessionEntry ? { ...preparedSessionState.sessionEntry } : preparedSessionState.sessionEntry; + if (params.autoFallbackPrimaryProbe && authSessionEntry) { + clearAutoFallbackPrimaryProbeSelection(authSessionEntry); + } const authSessionStore = shouldUseEphemeralSession && authSessionEntry ? { [authSessionKey]: authSessionEntry } @@ -1082,6 +1092,7 @@ export async function runPreparedReply( modelOverrideSource: runModelOverrideSource, hasAutoFallbackProvenance: runHasAutoFallbackProvenance || undefined, imageModelFallbacksOverride, + autoFallbackPrimaryProbe: params.autoFallbackPrimaryProbe, authProfileId, authProfileIdSource, thinkLevel: resolvedThinkLevel, diff --git a/src/auto-reply/reply/get-reply.ts b/src/auto-reply/reply/get-reply.ts index 7651c4d202c..bd8b636cef6 100644 --- a/src/auto-reply/reply/get-reply.ts +++ b/src/auto-reply/reply/get-reply.ts @@ -1,5 +1,6 @@ import fs from "node:fs/promises"; import { + resolveAutoFallbackPrimaryProbe, resolveAgentConfig, resolveAgentDir, resolveAgentWorkspaceDir, @@ -44,7 +45,7 @@ import { runPreparedReply } from "./get-reply-run.js"; import { finalizeInboundContext } from "./inbound-context.js"; import { hasInboundMedia } from "./inbound-media.js"; import { emitPreAgentMessageHooks } from "./message-preprocess-hooks.js"; -import { createFastTestModelSelectionState } from "./model-selection.js"; +import { createFastTestModelSelectionState, createModelSelectionState } from "./model-selection.js"; import { sanitizePendingFinalDeliveryText } from "./pending-final-delivery.js"; import { initSessionState } from "./session.js"; import { @@ -557,6 +558,18 @@ export async function getReplyFromConfig( provider = storedModelOverride.provider ?? defaultProvider; model = storedModelOverride.model; } + const canApplyAutoFallbackPrimaryProbe = + !hasResolvedHeartbeatModelOverride && + !hasAppliedImageModelOverride && + !staleHeartbeatAutoFallbackOverride; + const autoFallbackPrimaryProbe = canApplyAutoFallbackPrimaryProbe + ? resolveAutoFallbackPrimaryProbe({ + entry: sessionEntry, + sessionKey, + primaryProvider, + primaryModel, + }) + : undefined; const hasEffectiveSessionModelOverride = hasSessionModelOverride && !staleHeartbeatAutoFallbackOverride; if ( @@ -640,11 +653,11 @@ export async function getReplyFromConfig( resolvedBlockStreamingBreak: "text_end", modelState: createFastTestModelSelectionState({ agentCfg, - provider, - model, + provider: autoFallbackPrimaryProbe?.provider ?? provider, + model: autoFallbackPrimaryProbe?.model ?? model, }), - provider, - model, + provider: autoFallbackPrimaryProbe?.provider ?? provider, + model: autoFallbackPrimaryProbe?.model ?? model, perMessageQueueMode: undefined, perMessageQueueOptions: undefined, typing, @@ -665,6 +678,7 @@ export async function getReplyFromConfig( hasAppliedImageModelOverride, imageModelOverrideBaseProvider, imageModelFallbacksOverride, + autoFallbackPrimaryProbe, }), ); } @@ -809,6 +823,62 @@ export async function getReplyFromConfig( directives = inlineActionResult.directives; cleanedBody = inlineActionResult.cleanedBody; abortedLastRun = inlineActionResult.abortedLastRun ?? abortedLastRun; + const runAutoFallbackPrimaryProbe = directives.hasModelDirective + ? undefined + : autoFallbackPrimaryProbe; + const runProvider = runAutoFallbackPrimaryProbe?.provider ?? provider; + const runModel = runAutoFallbackPrimaryProbe?.model ?? model; + let runModelState = modelState; + if (runAutoFallbackPrimaryProbe) { + runModelState = await createModelSelectionState({ + cfg, + agentId, + agentCfg, + sessionEntry, + sessionStore, + sessionKey, + parentSessionKey: + sessionEntry.parentSessionKey ?? + sessionCtx.ModelParentSessionKey ?? + sessionCtx.ParentSessionKey, + storePath, + defaultProvider, + defaultModel, + primaryProvider, + primaryModel, + provider: runProvider, + model: runModel, + hasModelDirective: false, + hasOneTurnModelOverride: hasAppliedImageModelOverride, + skipStoredModelOverride: true, + hasResolvedHeartbeatModelOverride, + isHeartbeat: opts?.isHeartbeat === true, + }); + const hasExplicitThinkLevel = + resolvedOpts?.thinkingLevelOverride !== undefined || + directives.thinkLevel !== undefined || + (!directives.clearThinkLevel && sessionEntry.thinkingLevel !== undefined) || + agentCfg?.thinkingDefault !== undefined; + if (!hasExplicitThinkLevel) { + resolvedThinkLevel = await runModelState.resolveDefaultThinkingLevel(); + } + const agentEntry = resolveAgentConfig(cfg, agentId); + const rawSessionReasoningLevel = sessionEntry.reasoningLevel; + const canUseReasoningState = + command.isAuthorizedSender || + command.senderIsOwner || + (Array.isArray(ctx.GatewayClientScopes) && + ctx.GatewayClientScopes.includes("operator.admin")); + const hasExplicitReasoningLevel = + directives.reasoningLevel !== undefined || + (rawSessionReasoningLevel != null && canUseReasoningState) || + (rawSessionReasoningLevel != null && !canUseReasoningState) || + agentEntry?.reasoningDefault != null || + agentCfg?.reasoningDefault != null; + if (!hasExplicitReasoningLevel && resolvedThinkLevel === "off") { + resolvedReasoningLevel = await runModelState.resolveDefaultReasoningLevel(); + } + } // Allow plugins to intercept and return a synthetic reply before the LLM runs. if (!useFastTestBootstrap) { @@ -886,9 +956,9 @@ export async function getReplyFromConfig( blockStreamingEnabled, blockReplyChunking, resolvedBlockStreamingBreak, - modelState, - provider, - model, + modelState: runModelState, + provider: runProvider, + model: runModel, perMessageQueueMode, perMessageQueueOptions, typing, @@ -909,6 +979,7 @@ export async function getReplyFromConfig( hasAppliedImageModelOverride, imageModelOverrideBaseProvider, imageModelFallbacksOverride, + autoFallbackPrimaryProbe: runAutoFallbackPrimaryProbe, }), ); } diff --git a/src/auto-reply/reply/model-selection.test.ts b/src/auto-reply/reply/model-selection.test.ts index 09a0d9d06ed..9a6ca3161e8 100644 --- a/src/auto-reply/reply/model-selection.test.ts +++ b/src/auto-reply/reply/model-selection.test.ts @@ -901,6 +901,7 @@ describe("createModelSelectionState auto-failover overrides", () => { primaryProvider?: string; primaryModel?: string; isHeartbeat?: boolean; + skipStoredModelOverride?: boolean; }) { const cfg = {} as OpenClawConfig; const sessionEntry = makeEntry({ @@ -928,6 +929,7 @@ describe("createModelSelectionState auto-failover overrides", () => { model: params.model ?? defaultModel, hasModelDirective: false, isHeartbeat: params.isHeartbeat, + skipStoredModelOverride: params.skipStoredModelOverride, }); return { state, sessionEntry, sessionStore }; } @@ -1011,6 +1013,29 @@ describe("createModelSelectionState auto-failover overrides", () => { expect(state.resetModelOverride).toBe(false); }); + it("can suppress a stored auto-failover override for a primary recovery probe", async () => { + const { state, sessionStore } = await resolveStateWithOverride({ + providerOverride: "openrouter", + modelOverride: "minimax/minimax-m2.7", + modelOverrideSource: "auto", + modelOverrideFallbackOriginProvider: defaultProvider, + modelOverrideFallbackOriginModel: defaultModel, + authProfileOverride: "openrouter:fallback", + authProfileOverrideSource: "auto", + provider: defaultProvider, + model: defaultModel, + skipStoredModelOverride: true, + }); + + expect(state.provider).toBe(defaultProvider); + expect(state.model).toBe(defaultModel); + expect(state.resetModelOverride).toBe(false); + expect(sessionStore[sessionKey]?.providerOverride).toBe("openrouter"); + expect(sessionStore[sessionKey]?.modelOverride).toBe("minimax/minimax-m2.7"); + expect(sessionStore[sessionKey]?.authProfileOverride).toBe("openrouter:fallback"); + expect(sessionStore[sessionKey]?.authProfileOverrideSource).toBe("auto"); + }); + it("clears stale heartbeat auto-failover override when the fallback origin changed", async () => { const { state, sessionStore } = await resolveStateWithOverride({ providerOverride: "openrouter", diff --git a/src/auto-reply/reply/model-selection.ts b/src/auto-reply/reply/model-selection.ts index 48627c6f126..23958e572e8 100644 --- a/src/auto-reply/reply/model-selection.ts +++ b/src/auto-reply/reply/model-selection.ts @@ -104,6 +104,7 @@ export async function createModelSelectionState(params: { model: string; hasModelDirective: boolean; hasOneTurnModelOverride?: boolean; + skipStoredModelOverride?: boolean; /** True when heartbeat.model was explicitly resolved for this run. * In that case, skip session-stored overrides so the heartbeat selection wins. */ hasResolvedHeartbeatModelOverride?: boolean; @@ -280,6 +281,7 @@ export async function createModelSelectionState(params: { // overrides unless a direct auto fallback override is stale for the current // configured default. const skipStoredOverride = + params.skipStoredModelOverride === true || hasOneTurnModelOverride || params.hasResolvedHeartbeatModelOverride === true || (staleHeartbeatAutoFallbackOverride && storedOverride?.source === "session"); @@ -310,7 +312,13 @@ export async function createModelSelectionState(params: { model = allowedInitialSelection.model; } - if (sessionEntry && sessionStore && sessionKey && sessionEntry.authProfileOverride) { + if ( + !params.skipStoredModelOverride && + sessionEntry && + sessionStore && + sessionKey && + sessionEntry.authProfileOverride + ) { const { ensureAuthProfileStore } = await import("../../agents/auth-profiles.runtime.js"); const store = ensureAuthProfileStore(undefined, { allowKeychainPrompt: false, diff --git a/src/auto-reply/reply/queue/types.ts b/src/auto-reply/reply/queue/types.ts index f3755c1a810..657d011f15d 100644 --- a/src/auto-reply/reply/queue/types.ts +++ b/src/auto-reply/reply/queue/types.ts @@ -1,3 +1,4 @@ +import type { AutoFallbackPrimaryProbe } from "../../../agents/agent-scope.js"; import type { ExecToolDefaults } from "../../../agents/bash-tools.js"; import type { CurrentTurnPromptContext } from "../../../agents/pi-embedded-runner/run/params.js"; import type { SkillSnapshot } from "../../../agents/skills.js"; @@ -90,6 +91,7 @@ export type FollowupRun = { modelOverrideSource?: "auto" | "user"; hasAutoFallbackProvenance?: boolean; imageModelFallbacksOverride?: string[]; + autoFallbackPrimaryProbe?: AutoFallbackPrimaryProbe; authProfileId?: string; authProfileIdSource?: "auto" | "user"; thinkLevel?: ThinkLevel;