From ea1a0277d53bb24262d6ef06a8edc4463d6721e5 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 2 May 2026 07:59:47 +0100 Subject: [PATCH] fix: report model run fallback metadata --- CHANGELOG.md | 1 + .../agent-command.live-model-switch.test.ts | 16 ++++++++ src/agents/agent-command.ts | 12 ++++++ src/agents/pi-embedded-runner/types.ts | 2 + src/cli/capability-cli.test.ts | 41 +++++++++++++++++++ src/cli/capability-cli.ts | 10 ++++- src/commands/models/list.status-command.ts | 3 +- src/commands/models/list.status.test.ts | 27 ++++++++++++ 8 files changed, 109 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6394829760b..f3979c921f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,6 +49,7 @@ Docs: https://docs.openclaw.ai - Web search: keep public provider requests on the strict SSRF guard and reserve private-network access for explicit self-hosted SearXNG/Firecrawl endpoints. Fixes #74357 and supersedes #74360. Thanks @fede-kamel. - Firecrawl: reject private, loopback, metadata, and non-HTTP(S) `firecrawl_scrape` target URLs before forwarding them to Firecrawl. Supersedes #48133. Thanks @kn1ghtc. - Web search/Firecrawl: allow self-hosted private/internal Firecrawl `baseUrl` endpoints, including HTTP for private targets, while keeping hosted Firecrawl on the strict official endpoint. Fixes #63877 and supersedes #59666, #63941, and #74013. Thanks @jhthompson12, @jzakirov, @Mlightsnow, and @shad0wca7. +- CLI/models: report gateway model fallback attempts in `infer model run --json` and avoid double-prefixing provider-qualified defaults such as `openrouter/auto` in `models status`. Partially fixes #69527. Thanks @alexifra. - Providers/OpenRouter: strip trailing assistant prefill turns from verified OpenRouter Anthropic model requests when reasoning is enabled, so Claude 4.6 routes no longer fail with Anthropic's prefill rejection through the OpenAI-compatible adapter. Fixes #75395. Thanks @sbmilburn. - Feishu: preserve Feishu/Lark HTTP error bodies for message sends, media sends, and chat member lookups, so HTTP 400 failures include vendor code, message, log id, and troubleshooter details. Fixes #73860. Thanks @desksk. - Agents/transcripts: avoid reopening large Pi transcript files through the synchronous session manager for maintenance rewrites, persisted tool-result truncation, manual compaction boundary hardening, and queued compaction rotation. Thanks @mariozechner. diff --git a/src/agents/agent-command.live-model-switch.test.ts b/src/agents/agent-command.live-model-switch.test.ts index 36684f20789..e3685dcf9b2 100644 --- a/src/agents/agent-command.live-model-switch.test.ts +++ b/src/agents/agent-command.live-model-switch.test.ts @@ -864,6 +864,7 @@ describe("agentCommand – LiveSessionModelSwitchError retry", () => { { provider: params.provider, model: params.model, + error: "empty result", reason: "format", code: "empty_result", }, @@ -886,6 +887,21 @@ describe("agentCommand – LiveSessionModelSwitchError retry", () => { modelOverride: "gpt-5.4", isFallbackRetry: true, }); + expect(state.deliverAgentCommandResultMock.mock.calls[0]?.[0]).toMatchObject({ + result: { + meta: { + agentMeta: { + fallbackAttempts: [ + expect.objectContaining({ + provider: "anthropic", + model: "claude", + reason: "format", + }), + ], + }, + }, + }, + }); }); it("updates hasSessionModelOverride for fallback resolution after switch", async () => { diff --git a/src/agents/agent-command.ts b/src/agents/agent-command.ts index 7e965b65d0d..a5d30a634fe 100644 --- a/src/agents/agent-command.ts +++ b/src/agents/agent-command.ts @@ -1028,6 +1028,18 @@ async function agentCommandInternal( result = fallbackResult.result; fallbackProvider = fallbackResult.provider; fallbackModel = fallbackResult.model; + if (fallbackResult.attempts.length > 0 && result.meta.agentMeta) { + result = { + ...result, + meta: { + ...result.meta, + agentMeta: { + ...result.meta.agentMeta, + fallbackAttempts: fallbackResult.attempts, + }, + }, + }; + } if (!lifecycleEnded) { const stopReason = result.meta.stopReason; if (stopReason && stopReason !== "end_turn") { diff --git a/src/agents/pi-embedded-runner/types.ts b/src/agents/pi-embedded-runner/types.ts index 008a7f4714d..3b8401436ab 100644 --- a/src/agents/pi-embedded-runner/types.ts +++ b/src/agents/pi-embedded-runner/types.ts @@ -1,6 +1,7 @@ import type { HeartbeatToolResponse } from "../../auto-reply/heartbeat-tool-response.js"; import type { CliSessionBinding, SessionSystemPromptReport } from "../../config/sessions/types.js"; import type { DiagnosticTraceContext } from "../../infra/diagnostic-trace-context.js"; +import type { FallbackAttempt } from "../model-fallback.types.js"; import type { MessagingToolSend } from "../pi-embedded-messaging.types.js"; export type EmbeddedPiAgentMeta = { @@ -10,6 +11,7 @@ export type EmbeddedPiAgentMeta = { model: string; contextTokens?: number; agentHarnessId?: string; + fallbackAttempts?: FallbackAttempt[]; cliSessionBinding?: CliSessionBinding; compactionCount?: number; /** diff --git a/src/cli/capability-cli.test.ts b/src/cli/capability-cli.test.ts index c6f1fee96a8..ae6b6ad459d 100644 --- a/src/cli/capability-cli.test.ts +++ b/src/cli/capability-cli.test.ts @@ -585,6 +585,47 @@ describe("capability cli", () => { ); }); + it("surfaces gateway model fallback attempts in model probe JSON", async () => { + mocks.callGateway.mockResolvedValueOnce({ + result: { + payloads: [{ text: "gateway fallback reply" }], + meta: { + agentMeta: { + provider: "openai", + model: "gpt-4.1-mini", + fallbackAttempts: [ + { + provider: "openrouter", + model: "openrouter/auto", + error: "model unavailable", + reason: "model_not_found", + }, + ], + }, + }, + }, + } as never); + + await runRegisteredCli({ + register: registerCapabilityCli as (program: Command) => void, + argv: ["capability", "model", "run", "--prompt", "hello", "--gateway", "--json"], + }); + + expect(mocks.runtime.writeJson).toHaveBeenCalledWith( + expect.objectContaining({ + provider: "openai", + model: "gpt-4.1-mini", + attempts: [ + expect.objectContaining({ + provider: "openrouter", + model: "openrouter/auto", + reason: "model_not_found", + }), + ], + }), + ); + }); + it("requests admin scope for gateway model probes with provider/model overrides", async () => { await runRegisteredCli({ register: registerCapabilityCli as (program: Command) => void, diff --git a/src/cli/capability-cli.ts b/src/cli/capability-cli.ts index f74e693bace..bf7b12b6eaa 100644 --- a/src/cli/capability-cli.ts +++ b/src/cli/capability-cli.ts @@ -711,7 +711,13 @@ async function runModelRun(params: { const response: { result?: { payloads?: Array<{ text?: string; mediaUrl?: string | null; mediaUrls?: string[] }>; - meta?: { agentMeta?: { provider?: string; model?: string } }; + meta?: { + agentMeta?: { + provider?: string; + model?: string; + fallbackAttempts?: Array>; + }; + }; }; } = await callGateway({ method: "agent", @@ -746,7 +752,7 @@ async function runModelRun(params: { transport: "gateway" as const, provider: response?.result?.meta?.agentMeta?.provider, model: response?.result?.meta?.agentMeta?.model, - attempts: [], + attempts: response?.result?.meta?.agentMeta?.fallbackAttempts ?? [], outputs: (response?.result?.payloads ?? []).map((payload) => ({ text: payload.text, mediaUrl: payload.mediaUrl, diff --git a/src/commands/models/list.status-command.ts b/src/commands/models/list.status-command.ts index 7203108f94c..805bf95e91f 100644 --- a/src/commands/models/list.status-command.ts +++ b/src/commands/models/list.status-command.ts @@ -25,6 +25,7 @@ import { resolveEnvApiKey } from "../../agents/model-auth.js"; import { buildModelAliasIndex, isCliProvider, + modelKey, normalizeProviderId, parseModelRef, resolveConfiguredModelRef, @@ -202,7 +203,7 @@ export async function modelsStatusCommand( const rawDefaultsModel = resolveAgentModelPrimaryValue(cfg.agents?.defaults?.model) ?? ""; const rawModel = agentModelPrimary ?? rawDefaultsModel; - const resolvedLabel = `${resolved.provider}/${resolved.model}`; + const resolvedLabel = modelKey(resolved.provider, resolved.model); const defaultLabel = rawModel || resolvedLabel; const defaultsFallbacks = resolveAgentModelFallbackValues(cfg.agents?.defaults?.model); const fallbacks = agentFallbacksOverride ?? defaultsFallbacks; diff --git a/src/commands/models/list.status.test.ts b/src/commands/models/list.status.test.ts index b79dc65907d..82d16db8b3b 100644 --- a/src/commands/models/list.status.test.ts +++ b/src/commands/models/list.status.test.ts @@ -396,6 +396,33 @@ describe("modelsStatusCommand auth overview", () => { ); }); + it("does not double-prefix provider-qualified resolved default models", async () => { + const localRuntime = createRuntime(); + const originalLoadConfig = mocks.loadConfig.getMockImplementation(); + mocks.loadConfig.mockReturnValue({ + agents: { + defaults: { + model: { primary: "openrouter/auto", fallbacks: [] }, + models: { "openrouter/auto": {} }, + }, + }, + models: { providers: {} }, + env: { shellEnv: { enabled: true } }, + }); + + try { + await modelsStatusCommand({ json: true }, localRuntime as never); + const payload = JSON.parse(String((localRuntime.log as Mock).mock.calls[0]?.[0])); + + expect(payload.defaultModel).toBe("openrouter/auto"); + expect(payload.resolvedDefault).toBe("openrouter/auto"); + } finally { + if (originalLoadConfig) { + mocks.loadConfig.mockImplementation(originalLoadConfig); + } + } + }); + it("handles cli backend and aliased provider auth summaries", async () => { const localRuntime = createRuntime(); const originalLoadConfig = mocks.loadConfig.getMockImplementation();