diff --git a/docs/concepts/model-failover.md b/docs/concepts/model-failover.md index 3a3641979982..c54476ce925d 100644 --- a/docs/concepts/model-failover.md +++ b/docs/concepts/model-failover.md @@ -314,6 +314,7 @@ OpenClaw builds the candidate list from the currently requested `provider/model` - explicit aborts that are not timeout/failover-shaped - context overflow errors that should stay inside compaction/retry logic (for example `request_too_large`, `INVALID_ARGUMENT: input exceeds the maximum number of tokens`, `input token count exceeds the maximum number of input tokens`, `The input is too long for the model`, or `ollama error: context length exceeded`) - a final unknown error when there are no candidates left + - Claude Fable 5 safety refusals; direct API-key requests handle those at the provider level via Anthropic's server-side fallback to `claude-opus-4-8` instead (see [Anthropic](/providers/anthropic#safety-refusal-fallback-claude-fable-5)) diff --git a/docs/docs_map.md b/docs/docs_map.md index 3fb15f796f4b..72d641f6affc 100644 --- a/docs/docs_map.md +++ b/docs/docs_map.md @@ -7203,6 +7203,11 @@ Do not edit it by hand; run `pnpm docs:map:gen`. - Headings: - H2: Getting started - H2: Thinking defaults (Claude Fable 5, 4.8, and 4.6) + - H2: Safety refusal fallback (Claude Fable 5) + - H3: Why this exists + - H3: How it works + - H3: Observability and billing + - H3: Scope - H2: Prompt caching - H2: Advanced configuration - H2: Troubleshooting diff --git a/docs/providers/anthropic.md b/docs/providers/anthropic.md index 0a44acdac919..b515a364acf3 100644 --- a/docs/providers/anthropic.md +++ b/docs/providers/anthropic.md @@ -206,6 +206,77 @@ Related Anthropic docs: +## Safety refusal fallback (Claude Fable 5) + + +Using Claude Fable 5 means also using Claude Opus 4.8. Fable 5 ships with +safety classifiers that can decline a request, and Anthropic's sanctioned +recovery is to have `claude-opus-4-8` serve that turn. OpenClaw opts into this +automatically for direct API-key requests, so some Fable turns are answered +and billed as Claude Opus 4.8. If your policy or budget cannot accept +Opus-served turns, do not select `anthropic/claude-fable-5`. + + +### Why this exists + +Fable 5 classifiers return `stop_reason: "refusal"` on requests in restricted +domains, and they also false-positive on benign-adjacent work (security +tooling, life sciences, or even asking the model to reproduce its raw +reasoning). Without a fallback, the turn dies with an error even though +another Claude model would happily serve it — Anthropic's own refusal message +tells API integrators to configure a fallback model. + +### How it works + +1. For every direct API-key request to `anthropic/claude-fable-5`, OpenClaw + sends Anthropic's server-side fallback opt-in: the + `server-side-fallback-2026-06-01` beta header plus + `fallbacks: [{"model": "claude-opus-4-8"}]`. Claude Opus 4.8 is the only + fallback target Anthropic permits for Fable 5. +2. Only a safety-classifier decline triggers the fallback. Rate limits, + overloads, and server errors behave exactly as before and go through + OpenClaw's normal [model failover](/concepts/model-failover). +3. The rescue happens inside the same call. A decline before any output is + invisible apart from latency; the whole answer comes from Opus 4.8. On a + mid-stream decline the partial text is kept as the prefix the fallback + model continues from, while the declined model's reasoning and tool calls + are discarded per Anthropic's replay rules (they must not be echoed back or + executed). +4. If Claude Opus 4.8 declines as well, the turn surfaces the refusal as an + error, exactly like before this feature. + +The fallback happens at the Anthropic API level, so `claude-opus-4-8` does not +need to be in your configured model list or fallback chain — a Fable-capable +API key can always serve Opus. + +### Observability and billing + +- A fallback-served turn records a `provider_fallback` diagnostic on the + assistant message naming `fromModel` and `toModel`, and the message's + `responseModel` reports `claude-opus-4-8`. +- Anthropic bills per attempt: a decline before output is free, and the rescue + bills at Claude Opus 4.8 rates (currently half of Fable 5 rates). OpenClaw's + per-turn cost estimate prices fallback-served turns at Opus rates to match. +- A mid-stream decline additionally bills the already-streamed Fable partial + on Anthropic's side; that portion is reported in the API's per-attempt + usage but not folded into OpenClaw's per-turn estimate. + +### Scope + +Applies to `anthropic/claude-fable-5` with API-key auth against +`api.anthropic.com`. OAuth (Claude CLI subscription reuse), proxy base URLs, +Bedrock, Vertex, and Foundry requests are unchanged and still surface +refusals as errors there. + +Verified live: a benign prompt asking Fable 5 to reproduce its raw chain of +thought is declined with `category: "reasoning_extraction"` when sent without +fallbacks, and the same prompt through OpenClaw returns a normal Opus-served +answer with the `provider_fallback` diagnostic attached. + +See Anthropic's [refusals and fallback +guide](https://platform.claude.com/docs/en/build-with-claude/refusals-and-fallback) +for the underlying behavior. + ## Prompt caching OpenClaw supports Anthropic's prompt caching feature for API-key auth. diff --git a/src/agents/anthropic-transport-stream.test.ts b/src/agents/anthropic-transport-stream.test.ts index 6f4b6e8125d1..1184abbe61b7 100644 --- a/src/agents/anthropic-transport-stream.test.ts +++ b/src/agents/anthropic-transport-stream.test.ts @@ -300,6 +300,193 @@ describe("anthropic transport stream", () => { ); }); + it("sends server-side fallback params for direct Fable API-key requests", async () => { + guardedFetchMock.mockResolvedValueOnce( + createSseResponse([ + { + type: "message_start", + message: { id: "msg_fb", usage: { input_tokens: 1, output_tokens: 0 } }, + }, + { + type: "message_delta", + delta: { stop_reason: "end_turn" }, + usage: { input_tokens: 1, output_tokens: 1 }, + }, + { type: "message_stop" }, + ]), + ); + + await runTransportStream( + makeAnthropicTransportModel({ id: "claude-fable-5", name: "Claude Fable 5" }), + { + messages: [{ role: "user", content: "hello" }], + } as AnthropicStreamContext, + { + apiKey: "sk-ant-api", + } as AnthropicStreamOptions, + ); + + expect(latestAnthropicRequest().payload.fallbacks).toEqual([{ model: "claude-opus-4-8" }]); + expect(latestAnthropicRequestHeaders().get("anthropic-beta")).toBe( + "fine-grained-tool-streaming-2025-05-14,server-side-fallback-2026-06-01", + ); + }); + + it.each([ + { + label: "OAuth tokens", + model: { id: "claude-fable-5", name: "Claude Fable 5" }, + apiKey: "sk-ant-oat01-token", + }, + { + label: "custom proxy endpoints", + model: { + id: "claude-fable-5", + name: "Claude Fable 5", + baseUrl: "https://proxy.example.com/v1", + }, + apiKey: "sk-ant-api", + }, + { + label: "non-Fable models", + model: { id: "claude-opus-4-8", name: "Claude Opus 4.8" }, + apiKey: "sk-ant-api", + }, + ])("omits server-side fallback params for $label", async ({ model, apiKey }) => { + guardedFetchMock.mockResolvedValueOnce( + createSseResponse([ + { + type: "message_start", + message: { id: "msg_no_fb", usage: { input_tokens: 1, output_tokens: 0 } }, + }, + { + type: "message_delta", + delta: { stop_reason: "end_turn" }, + usage: { input_tokens: 1, output_tokens: 1 }, + }, + { type: "message_stop" }, + ]), + ); + + await runTransportStream( + makeAnthropicTransportModel(model), + { + messages: [{ role: "user", content: "hello" }], + } as AnthropicStreamContext, + { + apiKey, + } as AnthropicStreamOptions, + ); + + expect(latestAnthropicRequest().payload.fallbacks).toBeUndefined(); + expect(latestAnthropicRequestHeaders().get("anthropic-beta") ?? "").not.toContain( + "server-side-fallback", + ); + }); + + it("rebuilds Fable output at a mid-stream server-side fallback boundary", async () => { + guardedFetchMock.mockResolvedValueOnce( + createSseResponse([ + { + type: "message_start", + message: { + id: "msg_fb", + model: "claude-fable-5", + usage: { input_tokens: 5, output_tokens: 0 }, + }, + }, + { + type: "content_block_start", + index: 0, + content_block: { type: "thinking", thinking: "" }, + }, + { + type: "content_block_delta", + index: 0, + delta: { type: "thinking_delta", thinking: "pre-boundary reasoning" }, + }, + { type: "content_block_stop", index: 0 }, + { + type: "content_block_start", + index: 1, + content_block: { type: "text", text: "partial " }, + }, + { type: "content_block_stop", index: 1 }, + { + // Starting a tool call tags the preceding text as commentary before + // the classifier declines mid-turn. + type: "content_block_start", + index: 2, + content_block: { type: "tool_use", id: "call_1", name: "lookup", input: {} }, + }, + { type: "content_block_stop", index: 2 }, + { + type: "content_block_start", + index: 3, + content_block: { + type: "fallback", + from: { model: "claude-fable-5" }, + to: { model: "claude-opus-4-8" }, + }, + }, + { type: "content_block_stop", index: 3 }, + { + type: "content_block_start", + index: 4, + content_block: { type: "text", text: "" }, + }, + { + type: "content_block_delta", + index: 4, + delta: { type: "text_delta", text: "continued" }, + }, + { type: "content_block_stop", index: 4 }, + { + type: "message_delta", + delta: { stop_reason: "end_turn" }, + usage: { input_tokens: 5, output_tokens: 9 }, + }, + { type: "message_stop" }, + ]), + ); + + const model = makeAnthropicTransportModel({ id: "claude-fable-5", name: "Claude Fable 5" }); + model.cost = { input: 10, output: 50, cacheRead: 1, cacheWrite: 12.5 }; + const result = await runTransportStream( + model, + { + messages: [{ role: "user", content: "hello" }], + } as AnthropicStreamContext, + { + apiKey: "sk-ant-api", + } as AnthropicStreamOptions, + ); + + // Pre-boundary thinking/tool blocks must not replay or execute; text is + // the continuation prefix, and the commentary tag added for the dropped + // tool call must not survive (it would hide the prefix from the visible + // final answer). + expect(result.stopReason).toBe("stop"); + expect(result.content).toEqual([ + { type: "text", text: "partial " }, + { type: "text", text: "continued" }, + ]); + expect(result.responseModel).toBe("claude-opus-4-8"); + expect(result.diagnostics).toEqual([ + expect.objectContaining({ + type: "provider_fallback", + details: { + provider: "anthropic", + fromModel: "claude-fable-5", + toModel: "claude-opus-4-8", + }, + }), + ]); + // Fallback-served turns bill at the serving model's rates, not Fable's: + // 5 input tokens at $5/MTok plus 9 output tokens at $25/MTok. + expect(result.usage.cost.total).toBeCloseTo(0.00025, 10); + }); + it("uses bearer auth for Microsoft Foundry Anthropic transport requests", async () => { const model = makeAnthropicTransportModel({ provider: "microsoft-foundry", diff --git a/src/agents/anthropic-transport-stream.ts b/src/agents/anthropic-transport-stream.ts index c96d8935e035..ffa8b5d9ddc2 100644 --- a/src/agents/anthropic-transport-stream.ts +++ b/src/agents/anthropic-transport-stream.ts @@ -40,6 +40,13 @@ import { usesClaudeFable5MessagesContract, } from "../shared/anthropic-model-contract.js"; import { applyAnthropicRefusal } from "../shared/anthropic-refusal.js"; +import { + ANTHROPIC_SERVER_SIDE_FALLBACK_BETA, + CLAUDE_FABLE_5_FALLBACK_MODEL_COST, + applyAnthropicFallbackBoundary, + buildAnthropicServerSideFallbacks, + readAnthropicFallbackBoundary, +} from "../shared/anthropic-server-fallback.js"; import { MALFORMED_STREAMING_FRAGMENT_ERROR_MESSAGE } from "../shared/assistant-error-format.js"; import { createDeferredEventBuffer } from "../shared/deferred-event-buffer.js"; import { notifyLlmRequestActivity } from "../shared/llm-request-activity.js"; @@ -279,6 +286,15 @@ function isKimiAnthropicProvider(provider: string | undefined): boolean { return /^kimi(?:-|$)/.test(normalizeLowercaseStringOrEmpty(provider ?? "")); } +/** + * Server-side refusal fallback is a first-party Claude API beta: proxies and + * Bedrock/Vertex/Foundry reject the `fallbacks` param, and OAuth (Claude Code + * identity) requests are excluded until the beta is verified there. + */ +function useAnthropicServerSideFallback(model: AnthropicTransportModel): boolean { + return usesClaudeFable5MessagesContract(model) && isDirectAnthropicModel(model); +} + function supportsReasoningContentReplay( model: Pick, ): boolean { @@ -902,6 +918,9 @@ function createAnthropicTransportClient(params: { isOAuthToken: true, }; } + if (useAnthropicServerSideFallback(model)) { + betaFeatures.push(ANTHROPIC_SERVER_SIDE_FALLBACK_BETA); + } const betaHeader = buildAnthropicBetaHeader(model, betaFeatures, { oauth: false }); return { client: createAnthropicMessagesClient({ @@ -960,6 +979,12 @@ function buildAnthropicParams( max_tokens: maxTokens, stream: true, }; + // Fable safety classifiers can decline benign-adjacent work; server-side + // fallback re-serves the same call on claude-opus-4-8 instead of failing + // the turn. Requires the matching beta header from the transport client. + if (!isOAuthToken && useAnthropicServerSideFallback(model)) { + params.fallbacks = buildAnthropicServerSideFallbacks(); + } if (isOAuthToken) { params.system = [ { @@ -1125,6 +1150,9 @@ export function createAnthropicMessagesTransportStreamFn(): StreamFn { ) : undefined; const eventSink = refusalBuffer ?? stream; + // Fallback-served turns bill at the serving model's rates; a boundary + // swaps this to the fallback model's cost table. + let costModel = model; try { const apiKey = options?.apiKey ?? getEnvApiKey(model.provider) ?? ""; if (!apiKey) { @@ -1295,7 +1323,7 @@ export function createAnthropicMessagesTransportStreamFn(): StreamFn { output.usage.output + output.usage.cacheRead + output.usage.cacheWrite; - calculateCost(model, output.usage); + calculateCost(costModel, output.usage); // Defer start until after message_start so that pre-stream SSE errors // (e.g. invalid thinking signatures) arrive before any non-error event // is yielded, keeping yieldedOutput=false in pumpStreamWithRecovery @@ -1310,6 +1338,56 @@ export function createAnthropicMessagesTransportStreamFn(): StreamFn { if (event.type === "content_block_start") { const contentBlock = event.content_block as Record | undefined; const index = typeof event.index === "number" ? event.index : -1; + const fallbackBoundary = refusalBuffer + ? readAnthropicFallbackBoundary(contentBlock) + : null; + if (fallbackBoundary) { + // Server-side fallback boundary: pre-boundary thinking/tool + // blocks must not replay or execute, and the buffered preview + // events reference them, so rebuild the deferred timeline from + // the surviving text prefix the fallback model continued from. + refusalBuffer?.discard(); + pendingTextEnds.length = 0; + blockIndexes.clear(); + applyAnthropicFallbackBoundary({ + output, + boundary: fallbackBoundary, + provider: model.provider, + }); + // Cost intentionally mirrors top-level usage (serving attempt at + // serving-model rates). A mid-stream decline's billed partial is + // only in usage.iterations and is not folded in here. + costModel = { ...model, cost: CLAUDE_FABLE_5_FALLBACK_MODEL_COST }; + calculateCost(costModel, output.usage); + eventSink.push({ type: "start", partial: output as never }); + for (let i = 0; i < output.content.length; i += 1) { + const block = output.content[i]; + if (block.type !== "text") { + continue; + } + delete block.index; + eventSink.push({ + type: "text_start", + contentIndex: i, + partial: output as never, + }); + if (block.text) { + eventSink.push({ + type: "text_delta", + contentIndex: i, + delta: block.text, + partial: output as never, + }); + } + pendingTextEnds.push({ + type: "text_end", + contentIndex: i, + content: block.text, + partial: output as never, + }); + } + continue; + } if (contentBlock?.type === "text") { const text = typeof contentBlock.text === "string" @@ -1595,7 +1673,7 @@ export function createAnthropicMessagesTransportStreamFn(): StreamFn { output.usage.output + output.usage.cacheRead + output.usage.cacheWrite; - calculateCost(model, output.usage); + calculateCost(costModel, output.usage); // Gate on the turn CONTAINING a tool call, not the provider's stop_reason // label: Bedrock/Vertex-proxied routes (e.g. pioneer) report "end_turn" on // tool-using turns. No-op for direct Anthropic (already "toolUse" here). diff --git a/src/llm/providers/anthropic.test.ts b/src/llm/providers/anthropic.test.ts index d4a77d0995de..c29a0f879ac1 100644 --- a/src/llm/providers/anthropic.test.ts +++ b/src/llm/providers/anthropic.test.ts @@ -540,6 +540,247 @@ describe("Anthropic provider", () => { ]); }); + it("sends server-side fallback params for direct Fable API-key requests", async () => { + let capturedPayload: unknown; + const stream = streamAnthropic( + makeAnthropicModel({ id: "claude-fable-5", name: "Claude Fable 5" }), + { messages: [{ role: "user", content: "hello", timestamp: 0 }] }, + { + apiKey: "sk-ant-provider", + onPayload: (payload) => { + capturedPayload = payload; + throw new Error("stop before network"); + }, + }, + ); + await stream.result(); + + expect((capturedPayload as { fallbacks?: unknown }).fallbacks).toEqual([ + { model: "claude-opus-4-8" }, + ]); + await vi.waitFor(() => expect(anthropicMockState.configs).toHaveLength(1)); + const config = anthropicMockState.configs[0] as { + defaultHeaders?: Record; + }; + expect(config.defaultHeaders?.["anthropic-beta"]).toContain("server-side-fallback-2026-06-01"); + }); + + it.each([ + { label: "OAuth tokens", overrides: {}, apiKey: "sk-ant-oat01-token" }, + { + label: "custom proxy endpoints", + overrides: { baseUrl: "https://proxy.example.com/v1" }, + apiKey: "sk-ant-provider", + }, + { + label: "Anthropic Vertex models", + overrides: { provider: "anthropic-vertex" }, + apiKey: "vertex-token", + }, + { + label: "non-Fable models", + overrides: { id: "claude-opus-4-8", name: "Claude Opus 4.8" }, + apiKey: "sk-ant-provider", + }, + ])("omits server-side fallback params for $label", async ({ overrides, apiKey }) => { + let capturedPayload: unknown; + const stream = streamAnthropic( + makeAnthropicModel({ id: "claude-fable-5", name: "Claude Fable 5", ...overrides }), + { messages: [{ role: "user", content: "hello", timestamp: 0 }] }, + { + apiKey, + onPayload: (payload) => { + capturedPayload = payload; + throw new Error("stop before network"); + }, + }, + ); + await stream.result(); + + expect((capturedPayload as { fallbacks?: unknown }).fallbacks).toBeUndefined(); + }); + + it("rebuilds Fable output at a mid-stream server-side fallback boundary", async () => { + const client = { + messages: { + create: vi.fn(() => ({ + asResponse: () => + Promise.resolve( + createSseResponse([ + { + type: "message_start", + message: { + id: "msg_fallback", + model: "claude-fable-5", + usage: { input_tokens: 5, output_tokens: 0 }, + }, + }, + { + type: "content_block_start", + index: 0, + content_block: { type: "thinking", thinking: "" }, + }, + { + type: "content_block_delta", + index: 0, + delta: { type: "thinking_delta", thinking: "pre-boundary reasoning" }, + }, + { type: "content_block_stop", index: 0 }, + { + type: "content_block_start", + index: 1, + content_block: { type: "text", text: "" }, + }, + { + type: "content_block_delta", + index: 1, + delta: { type: "text_delta", text: "partial " }, + }, + { type: "content_block_stop", index: 1 }, + { + type: "content_block_start", + index: 2, + content_block: { type: "tool_use", id: "call_1", name: "lookup", input: {} }, + }, + { type: "content_block_stop", index: 2 }, + { + type: "content_block_start", + index: 3, + content_block: { + type: "fallback", + from: { model: "claude-fable-5" }, + to: { model: "claude-opus-4-8" }, + }, + }, + { type: "content_block_stop", index: 3 }, + { + type: "content_block_start", + index: 4, + content_block: { type: "text", text: "" }, + }, + { + type: "content_block_delta", + index: 4, + delta: { type: "text_delta", text: "continued" }, + }, + { type: "content_block_stop", index: 4 }, + { + type: "message_delta", + delta: { stop_reason: "end_turn" }, + usage: { input_tokens: 5, output_tokens: 9 }, + }, + { type: "message_stop" }, + ]), + ), + })), + }, + }; + + const stream = streamAnthropic( + makeAnthropicModel({ + id: "claude-fable-5", + name: "Claude Fable 5", + cost: { input: 10, output: 50, cacheRead: 1, cacheWrite: 12.5 }, + }), + { messages: [{ role: "user", content: "hello", timestamp: 0 }] }, + { apiKey: "sk-ant-provider", client: client as never }, + ); + const eventTypes: string[] = []; + for await (const event of stream) { + eventTypes.push(event.type); + } + const result = await stream.result(); + + // Pre-boundary thinking/tool blocks must not replay or execute; text is + // the continuation prefix the fallback model built on. + expect(result.stopReason).toBe("stop"); + expect(result.content).toEqual([ + { type: "text", text: "partial " }, + { type: "text", text: "continued" }, + ]); + expect(result.responseModel).toBe("claude-opus-4-8"); + expect(result.diagnostics).toEqual([ + expect.objectContaining({ + type: "provider_fallback", + details: { + provider: "anthropic", + fromModel: "claude-fable-5", + toModel: "claude-opus-4-8", + }, + }), + ]); + expect(eventTypes).not.toContain("thinking_start"); + expect(eventTypes).not.toContain("toolcall_start"); + expect(eventTypes.filter((type) => type === "start")).toHaveLength(1); + // Fallback-served turns bill at the serving model's rates, not Fable's: + // 5 input tokens at $5/MTok plus 9 output tokens at $25/MTok. + expect(result.usage.cost.input).toBeCloseTo(0.000025, 10); + expect(result.usage.cost.output).toBeCloseTo(0.000225, 10); + expect(result.usage.cost.total).toBeCloseTo(0.00025, 10); + }); + + it("records a pre-output server-side fallback and keeps the continuation", async () => { + const client = { + messages: { + create: vi.fn(() => ({ + asResponse: () => + Promise.resolve( + createSseResponse([ + { + type: "message_start", + message: { + id: "msg_fallback", + // Pre-output declines: message_start names the fallback model. + model: "claude-opus-4-8", + usage: { input_tokens: 5, output_tokens: 0 }, + }, + }, + { + type: "content_block_start", + index: 0, + content_block: { + type: "fallback", + from: { model: "claude-fable-5" }, + to: { model: "claude-opus-4-8" }, + }, + }, + { type: "content_block_stop", index: 0 }, + { + type: "content_block_start", + index: 1, + content_block: { type: "text", text: "" }, + }, + { + type: "content_block_delta", + index: 1, + delta: { type: "text_delta", text: "Hi!" }, + }, + { type: "content_block_stop", index: 1 }, + { + type: "message_delta", + delta: { stop_reason: "end_turn" }, + usage: { input_tokens: 5, output_tokens: 2 }, + }, + { type: "message_stop" }, + ]), + ), + })), + }, + }; + + const stream = streamAnthropic( + makeAnthropicModel({ id: "claude-fable-5", name: "Claude Fable 5" }), + { messages: [{ role: "user", content: "hello", timestamp: 0 }] }, + { apiKey: "sk-ant-provider", client: client as never }, + ); + const result = await stream.result(); + + expect(result.stopReason).toBe("stop"); + expect(result.content).toEqual([{ type: "text", text: "Hi!" }]); + expect(result.responseModel).toBe("claude-opus-4-8"); + expect(result.diagnostics).toEqual([expect.objectContaining({ type: "provider_fallback" })]); + }); + it("routes interleaved active content blocks by their event indexes", async () => { const client = { messages: { diff --git a/src/llm/providers/anthropic.ts b/src/llm/providers/anthropic.ts index 9c58e0e19212..094c13417422 100644 --- a/src/llm/providers/anthropic.ts +++ b/src/llm/providers/anthropic.ts @@ -15,6 +15,7 @@ import { type AnthropicProjectedToolChoice, type AnthropicToolProjection, } from "../../agents/anthropic-tool-projection.js"; +import { resolveProviderEndpoint } from "../../agents/provider-attribution.js"; import { buildGuardedModelFetch } from "../../agents/provider-transport-fetch.js"; import { splitSystemPromptCacheBoundary, @@ -33,6 +34,13 @@ import { usesClaudeFable5MessagesContract, } from "../../shared/anthropic-model-contract.js"; import { applyAnthropicRefusal } from "../../shared/anthropic-refusal.js"; +import { + ANTHROPIC_SERVER_SIDE_FALLBACK_BETA, + CLAUDE_FABLE_5_FALLBACK_MODEL_COST, + applyAnthropicFallbackBoundary, + buildAnthropicServerSideFallbacks, + readAnthropicFallbackBoundary, +} from "../../shared/anthropic-server-fallback.js"; import { createDeferredEventBuffer } from "../../shared/deferred-event-buffer.js"; import { notifyLlmRequestActivity } from "../../shared/llm-request-activity.js"; import { getEnvApiKey } from "../env-api-keys.js"; @@ -506,10 +514,17 @@ export const streamAnthropic: StreamFunction<"anthropic-messages", AnthropicOpti ) : undefined; const eventSink = refusalBuffer ?? stream; + // Fallback-served turns bill at the serving model's rates; a boundary + // swaps this to the fallback model's cost table. + let costModel = model; try { let client: Anthropic; let isOAuth: boolean; + // The beta-gated fallbacks param may only ship on clients we built, + // where the matching beta header is guaranteed; injected clients carry + // caller-owned headers. + let serverSideFallback = false; if (options?.client) { client = options.client; @@ -540,8 +555,9 @@ export const streamAnthropic: StreamFunction<"anthropic-messages", AnthropicOpti ); client = created.client; isOAuth = created.isOAuthToken; + serverSideFallback = created.serverSideFallback; } - const builtParams = buildParams(model, context, isOAuth, options); + const builtParams = buildParams(model, context, isOAuth, options, serverSideFallback); let params = builtParams.params; const toolProjection = builtParams.toolProjection; const nextParams = await options?.onPayload?.(params, model); @@ -584,14 +600,57 @@ export const streamAnthropic: StreamFunction<"anthropic-messages", AnthropicOpti output.usage.output + output.usage.cacheRead + output.usage.cacheWrite; - calculateCost(model, output.usage); + calculateCost(costModel, output.usage); // Defer start until after message_start so that pre-stream SSE errors // (e.g. invalid thinking signatures) arrive before any non-error event // is yielded, keeping yieldedOutput=false in pumpStreamWithRecovery // and allowing the thinking-block recovery retry to fire. eventSink.push({ type: "start", partial: output }); } else if (event.type === "content_block_start") { - if (event.content_block.type === "text") { + const fallbackBoundary = refusalBuffer + ? readAnthropicFallbackBoundary(event.content_block) + : null; + if (fallbackBoundary) { + // Server-side fallback boundary: pre-boundary thinking/tool blocks + // must not replay or execute, and the buffered preview events + // reference them, so rebuild the deferred timeline from the + // surviving text prefix the fallback model continued from. + refusalBuffer?.discard(); + blockIndexes.clear(); + applyAnthropicFallbackBoundary({ + output, + boundary: fallbackBoundary, + provider: model.provider, + }); + // Cost intentionally mirrors top-level usage (serving attempt at + // serving-model rates). A mid-stream decline's billed partial is + // only in usage.iterations and is not folded in here. + costModel = { ...model, cost: CLAUDE_FABLE_5_FALLBACK_MODEL_COST }; + calculateCost(costModel, output.usage); + eventSink.push({ type: "start", partial: output }); + for (let i = 0; i < blocks.length; i += 1) { + const block = blocks[i]; + if (block.type !== "text") { + continue; + } + delete (block as Partial).index; + eventSink.push({ type: "text_start", contentIndex: i, partial: output }); + if (block.text) { + eventSink.push({ + type: "text_delta", + contentIndex: i, + delta: block.text, + partial: output, + }); + } + eventSink.push({ + type: "text_end", + contentIndex: i, + content: block.text, + partial: output, + }); + } + } else if (event.content_block.type === "text") { const block: Block = { type: "text", text: "", @@ -759,7 +818,7 @@ export const streamAnthropic: StreamFunction<"anthropic-messages", AnthropicOpti output.usage.output + output.usage.cacheRead + output.usage.cacheWrite; - calculateCost(model, output.usage); + calculateCost(costModel, output.usage); } } @@ -912,6 +971,19 @@ function isOAuthToken(apiKey: string): boolean { return apiKey.includes("sk-ant-oat"); } +/** + * Server-side refusal fallback is a first-party Claude API beta: proxies and + * Bedrock/Vertex/Foundry reject the `fallbacks` param, and OAuth (Claude Code + * identity) requests are excluded until the beta is verified there. + */ +function supportsAnthropicServerSideFallback(model: Model<"anthropic-messages">): boolean { + if (!usesClaudeFable5MessagesContract(model) || model.provider !== "anthropic") { + return false; + } + const endpointClass = resolveProviderEndpoint(model.baseUrl).endpointClass; + return endpointClass === "default" || endpointClass === "anthropic-public"; +} + function createClient( model: Model<"anthropic-messages">, apiKey: string, @@ -920,7 +992,7 @@ function createClient( optionsHeaders?: Record, dynamicHeaders?: Record, sessionId?: string, -): { client: Anthropic; isOAuthToken: boolean } { +): { client: Anthropic; isOAuthToken: boolean; serverSideFallback: boolean } { // Adaptive thinking models (Opus 4.6, Sonnet 4.6) have interleaved thinking built-in. // The beta header is deprecated on Opus 4.6 and redundant on Sonnet 4.6, so skip it. const needsInterleavedBeta = interleavedThinking && !supportsAdaptiveThinking(model); @@ -951,7 +1023,7 @@ function createClient( fetch: buildGuardedModelFetch(model), }); - return { client, isOAuthToken: false }; + return { client, isOAuthToken: false, serverSideFallback: false }; } // Copilot: Bearer auth, selective betas. @@ -973,7 +1045,7 @@ function createClient( ), }); - return { client, isOAuthToken: false }; + return { client, isOAuthToken: false, serverSideFallback: false }; } if (usesFoundryBearerAuth(model)) { @@ -994,7 +1066,7 @@ function createClient( ), }); - return { client, isOAuthToken: false }; + return { client, isOAuthToken: false, serverSideFallback: false }; } // OAuth: Bearer auth, Claude Code identity headers @@ -1017,10 +1089,14 @@ function createClient( ), }); - return { client, isOAuthToken: true }; + return { client, isOAuthToken: true, serverSideFallback: false }; } // API key auth + const serverSideFallback = supportsAnthropicServerSideFallback(model); + if (serverSideFallback) { + betaFeatures.push(ANTHROPIC_SERVER_SIDE_FALLBACK_BETA); + } const sessionAffinityHeaders: Record = sessionId && getAnthropicCompat(model).sendSessionAffinityHeaders ? { "x-session-affinity": sessionId } @@ -1042,7 +1118,7 @@ function createClient( ), }); - return { client, isOAuthToken: false }; + return { client, isOAuthToken: false, serverSideFallback }; } function buildParams( @@ -1050,6 +1126,7 @@ function buildParams( context: Context, isOAuthTokenResult: boolean, options?: AnthropicOptions, + serverSideFallback = false, ): { params: MessageCreateParamsStreaming; toolProjection?: AnthropicToolProjection; @@ -1094,6 +1171,14 @@ function buildParams( params.system = system; } + // Fable safety classifiers can decline benign-adjacent work; server-side + // fallback re-serves the same call on claude-opus-4-8 instead of failing + // the turn. Only set when createClient added the matching beta header. + if (serverSideFallback) { + (params as { fallbacks?: Array<{ model: string }> }).fallbacks = + buildAnthropicServerSideFallbacks(); + } + // Thinking and post-4.6 Claude models reject custom temperature values. if ( options?.temperature !== undefined && diff --git a/src/shared/anthropic-server-fallback.ts b/src/shared/anthropic-server-fallback.ts new file mode 100644 index 000000000000..ee3e6029fa40 --- /dev/null +++ b/src/shared/anthropic-server-fallback.ts @@ -0,0 +1,97 @@ +import type { AssistantMessageDiagnostic } from "../llm/types.js"; + +/** + * Anthropic server-side refusal fallback (`server-side-fallback-2026-06-01`). + * When Claude Fable 5 safety classifiers decline a request, the API re-serves + * the same call on a permitted fallback model inside the same stream instead + * of returning `stop_reason: "refusal"`. + * https://platform.claude.com/docs/en/build-with-claude/refusals-and-fallback + */ +export const ANTHROPIC_SERVER_SIDE_FALLBACK_BETA = "server-side-fallback-2026-06-01"; + +// Anthropic contract: claude-opus-4-8 is the only entry in claude-fable-5's +// published `allowed_fallback_models`; other targets are rejected up front. +export const CLAUDE_FABLE_5_FALLBACK_MODEL = "claude-opus-4-8"; + +// Fallback-served turns bill at the serving model's rates (top-level usage +// covers only that attempt), so cost math must switch off the Fable table. +// Claude Opus 4.8 per-MTok pricing, same shape as the bundled Fable table. +export const CLAUDE_FABLE_5_FALLBACK_MODEL_COST = { + input: 5, + output: 25, + cacheRead: 0.5, + cacheWrite: 6.25, +} as const; + +export function buildAnthropicServerSideFallbacks(): Array<{ model: string }> { + return [{ model: CLAUDE_FABLE_5_FALLBACK_MODEL }]; +} + +export type AnthropicFallbackBoundary = { + fromModel: string | null; + toModel: string | null; +}; + +function readBoundaryModel(value: unknown): string | null { + if (!value || typeof value !== "object") { + return null; + } + const model = (value as { model?: unknown }).model; + return typeof model === "string" && model.trim() ? model : null; +} + +/** Reads a `fallback` content block marking where one model's output gives way to the next. */ +export function readAnthropicFallbackBoundary(block: unknown): AnthropicFallbackBoundary | null { + if (!block || typeof block !== "object") { + return null; + } + const record = block as { type?: unknown; from?: unknown; to?: unknown }; + if (record.type !== "fallback") { + return null; + } + return { + fromModel: readBoundaryModel(record.from), + toModel: readBoundaryModel(record.to), + }; +} + +/** + * Applies a mid-stream fallback boundary to the accumulated assistant output. + * Anthropic's replay contract: pre-boundary thinking/tool_use blocks must not + * be echoed on later turns (and dropped tool calls must never execute); the + * pre-boundary text is the continuation prefix the fallback model built on. + */ +export function applyAnthropicFallbackBoundary(params: { + output: { + content: Array<{ type: string }>; + responseModel?: string; + diagnostics?: AssistantMessageDiagnostic[]; + }; + boundary: AnthropicFallbackBoundary; + provider: string; +}): void { + const { output, boundary } = params; + const survivors = output.content.filter((block) => block.type === "text"); + for (const survivor of survivors) { + // Commentary phase tags refer to dropped pre-boundary tool calls; the + // prefix is now the start of the final answer, and phase-aware display + // extraction would otherwise hide it from the visible response. + delete (survivor as { textSignature?: string }).textSignature; + } + output.content.splice(0, output.content.length, ...survivors); + if (boundary.toModel) { + output.responseModel = boundary.toModel; + } + output.diagnostics = [ + ...(output.diagnostics ?? []), + { + type: "provider_fallback", + timestamp: Date.now(), + details: { + provider: params.provider, + fromModel: boundary.fromModel, + toModel: boundary.toModel, + }, + }, + ]; +}