fix: report model run fallback metadata

This commit is contained in:
Peter Steinberger
2026-05-02 07:59:47 +01:00
parent 11560f8d3a
commit ea1a0277d5
8 changed files with 109 additions and 3 deletions

View File

@@ -49,6 +49,7 @@ Docs: https://docs.openclaw.ai
- Web search: keep public provider requests on the strict SSRF guard and reserve private-network access for explicit self-hosted SearXNG/Firecrawl endpoints. Fixes #74357 and supersedes #74360. Thanks @fede-kamel.
- Firecrawl: reject private, loopback, metadata, and non-HTTP(S) `firecrawl_scrape` target URLs before forwarding them to Firecrawl. Supersedes #48133. Thanks @kn1ghtc.
- Web search/Firecrawl: allow self-hosted private/internal Firecrawl `baseUrl` endpoints, including HTTP for private targets, while keeping hosted Firecrawl on the strict official endpoint. Fixes #63877 and supersedes #59666, #63941, and #74013. Thanks @jhthompson12, @jzakirov, @Mlightsnow, and @shad0wca7.
- CLI/models: report gateway model fallback attempts in `infer model run --json` and avoid double-prefixing provider-qualified defaults such as `openrouter/auto` in `models status`. Partially fixes #69527. Thanks @alexifra.
- Providers/OpenRouter: strip trailing assistant prefill turns from verified OpenRouter Anthropic model requests when reasoning is enabled, so Claude 4.6 routes no longer fail with Anthropic's prefill rejection through the OpenAI-compatible adapter. Fixes #75395. Thanks @sbmilburn.
- Feishu: preserve Feishu/Lark HTTP error bodies for message sends, media sends, and chat member lookups, so HTTP 400 failures include vendor code, message, log id, and troubleshooter details. Fixes #73860. Thanks @desksk.
- Agents/transcripts: avoid reopening large Pi transcript files through the synchronous session manager for maintenance rewrites, persisted tool-result truncation, manual compaction boundary hardening, and queued compaction rotation. Thanks @mariozechner.

View File

@@ -864,6 +864,7 @@ describe("agentCommand LiveSessionModelSwitchError retry", () => {
{
provider: params.provider,
model: params.model,
error: "empty result",
reason: "format",
code: "empty_result",
},
@@ -886,6 +887,21 @@ describe("agentCommand LiveSessionModelSwitchError retry", () => {
modelOverride: "gpt-5.4",
isFallbackRetry: true,
});
expect(state.deliverAgentCommandResultMock.mock.calls[0]?.[0]).toMatchObject({
result: {
meta: {
agentMeta: {
fallbackAttempts: [
expect.objectContaining({
provider: "anthropic",
model: "claude",
reason: "format",
}),
],
},
},
},
});
});
it("updates hasSessionModelOverride for fallback resolution after switch", async () => {

View File

@@ -1028,6 +1028,18 @@ async function agentCommandInternal(
result = fallbackResult.result;
fallbackProvider = fallbackResult.provider;
fallbackModel = fallbackResult.model;
if (fallbackResult.attempts.length > 0 && result.meta.agentMeta) {
result = {
...result,
meta: {
...result.meta,
agentMeta: {
...result.meta.agentMeta,
fallbackAttempts: fallbackResult.attempts,
},
},
};
}
if (!lifecycleEnded) {
const stopReason = result.meta.stopReason;
if (stopReason && stopReason !== "end_turn") {

View File

@@ -1,6 +1,7 @@
import type { HeartbeatToolResponse } from "../../auto-reply/heartbeat-tool-response.js";
import type { CliSessionBinding, SessionSystemPromptReport } from "../../config/sessions/types.js";
import type { DiagnosticTraceContext } from "../../infra/diagnostic-trace-context.js";
import type { FallbackAttempt } from "../model-fallback.types.js";
import type { MessagingToolSend } from "../pi-embedded-messaging.types.js";
export type EmbeddedPiAgentMeta = {
@@ -10,6 +11,7 @@ export type EmbeddedPiAgentMeta = {
model: string;
contextTokens?: number;
agentHarnessId?: string;
fallbackAttempts?: FallbackAttempt[];
cliSessionBinding?: CliSessionBinding;
compactionCount?: number;
/**

View File

@@ -585,6 +585,47 @@ describe("capability cli", () => {
);
});
it("surfaces gateway model fallback attempts in model probe JSON", async () => {
mocks.callGateway.mockResolvedValueOnce({
result: {
payloads: [{ text: "gateway fallback reply" }],
meta: {
agentMeta: {
provider: "openai",
model: "gpt-4.1-mini",
fallbackAttempts: [
{
provider: "openrouter",
model: "openrouter/auto",
error: "model unavailable",
reason: "model_not_found",
},
],
},
},
},
} as never);
await runRegisteredCli({
register: registerCapabilityCli as (program: Command) => void,
argv: ["capability", "model", "run", "--prompt", "hello", "--gateway", "--json"],
});
expect(mocks.runtime.writeJson).toHaveBeenCalledWith(
expect.objectContaining({
provider: "openai",
model: "gpt-4.1-mini",
attempts: [
expect.objectContaining({
provider: "openrouter",
model: "openrouter/auto",
reason: "model_not_found",
}),
],
}),
);
});
it("requests admin scope for gateway model probes with provider/model overrides", async () => {
await runRegisteredCli({
register: registerCapabilityCli as (program: Command) => void,

View File

@@ -711,7 +711,13 @@ async function runModelRun(params: {
const response: {
result?: {
payloads?: Array<{ text?: string; mediaUrl?: string | null; mediaUrls?: string[] }>;
meta?: { agentMeta?: { provider?: string; model?: string } };
meta?: {
agentMeta?: {
provider?: string;
model?: string;
fallbackAttempts?: Array<Record<string, unknown>>;
};
};
};
} = await callGateway({
method: "agent",
@@ -746,7 +752,7 @@ async function runModelRun(params: {
transport: "gateway" as const,
provider: response?.result?.meta?.agentMeta?.provider,
model: response?.result?.meta?.agentMeta?.model,
attempts: [],
attempts: response?.result?.meta?.agentMeta?.fallbackAttempts ?? [],
outputs: (response?.result?.payloads ?? []).map((payload) => ({
text: payload.text,
mediaUrl: payload.mediaUrl,

View File

@@ -25,6 +25,7 @@ import { resolveEnvApiKey } from "../../agents/model-auth.js";
import {
buildModelAliasIndex,
isCliProvider,
modelKey,
normalizeProviderId,
parseModelRef,
resolveConfiguredModelRef,
@@ -202,7 +203,7 @@ export async function modelsStatusCommand(
const rawDefaultsModel = resolveAgentModelPrimaryValue(cfg.agents?.defaults?.model) ?? "";
const rawModel = agentModelPrimary ?? rawDefaultsModel;
const resolvedLabel = `${resolved.provider}/${resolved.model}`;
const resolvedLabel = modelKey(resolved.provider, resolved.model);
const defaultLabel = rawModel || resolvedLabel;
const defaultsFallbacks = resolveAgentModelFallbackValues(cfg.agents?.defaults?.model);
const fallbacks = agentFallbacksOverride ?? defaultsFallbacks;

View File

@@ -396,6 +396,33 @@ describe("modelsStatusCommand auth overview", () => {
);
});
it("does not double-prefix provider-qualified resolved default models", async () => {
const localRuntime = createRuntime();
const originalLoadConfig = mocks.loadConfig.getMockImplementation();
mocks.loadConfig.mockReturnValue({
agents: {
defaults: {
model: { primary: "openrouter/auto", fallbacks: [] },
models: { "openrouter/auto": {} },
},
},
models: { providers: {} },
env: { shellEnv: { enabled: true } },
});
try {
await modelsStatusCommand({ json: true }, localRuntime as never);
const payload = JSON.parse(String((localRuntime.log as Mock).mock.calls[0]?.[0]));
expect(payload.defaultModel).toBe("openrouter/auto");
expect(payload.resolvedDefault).toBe("openrouter/auto");
} finally {
if (originalLoadConfig) {
mocks.loadConfig.mockImplementation(originalLoadConfig);
}
}
});
it("handles cli backend and aliased provider auth summaries", async () => {
const localRuntime = createRuntime();
const originalLoadConfig = mocks.loadConfig.getMockImplementation();