fix: remap thinking levels on model switch

This commit is contained in:
Peter Steinberger
2026-04-21 18:51:34 +01:00
parent 24db09a19b
commit bcfa781a1b
12 changed files with 373 additions and 18 deletions

View File

@@ -50,6 +50,7 @@ Docs: https://docs.openclaw.ai
- Webchat/images: treat inline image attachments as media for empty-turn gating while still ignoring metadata-only blank turns. (#69474) Thanks @Jaswir.
- Discord/think: only show `adaptive` in `/think` autocomplete for provider/model pairs that actually support provider-managed adaptive thinking, so GPT/OpenAI models no longer advertise an Anthropic-only option.
- Thinking: only expose `max` for models that explicitly support provider max reasoning, and remap stored `max` settings to the largest supported thinking mode when users switch to another model.
- Thinking/slash: remap stored `/think adaptive` to `medium` when switching to non-adaptive OpenAI models, remap unsupported `xhigh` to the nearest supported level, and cover the provider-specific option list with a live QA Lab scenario.
- Thinking/UI: drive `/think` options and chat/Sessions pickers from provider-owned thinking profiles, so custom model level sets such as binary `on/off`, Gemini 3 Pro `off/low/high`, Anthropic `adaptive/max`, and OpenAI `xhigh` stay in one runtime contract.
- Gateway/usage: bound the cost usage cache with FIFO eviction so date/range lookups cannot grow unbounded. (#68842) Thanks @Feelw00.
- OpenAI/Responses: resolve `/think` levels against each GPT model's supported reasoning efforts so `/think off` no longer becomes high reasoning or sends unsupported `reasoning.effort: "none"` payloads.

View File

@@ -23,7 +23,7 @@ title: "Thinking Levels"
- Provider notes:
- Thinking menus and pickers are provider-profile driven. Provider plugins declare the exact level set for the selected model, including labels such as binary `on`.
- `adaptive`, `xhigh`, and `max` are only advertised for provider/model profiles that support them. Typed directives for unsupported levels are rejected with that model's valid options.
- Existing stored unsupported levels, including old `max` values after switching models, are remapped to the largest supported level for the selected model.
- Existing stored unsupported levels are remapped by provider profile rank. `adaptive` falls back to `medium` on non-adaptive models, while `xhigh` and `max` fall back to the largest supported non-off level for the selected model.
- Anthropic Claude 4.6 models default to `adaptive` when no explicit thinking level is set.
- Anthropic Claude Opus 4.7 does not default to adaptive thinking. Its API effort default remains provider-owned unless you explicitly set a thinking level.
- Anthropic Claude Opus 4.7 maps `/think xhigh` to adaptive thinking plus `output_config.effort: "xhigh"`, because `/think` is a thinking directive and `xhigh` is the Opus 4.7 effort setting.

View File

@@ -149,6 +149,29 @@ describe("qa scenario catalog", () => {
]);
});
it("includes the thinking slash model remap scenario", () => {
const scenario = readQaScenarioById("thinking-slash-model-remap");
const config = readQaScenarioExecutionConfig("thinking-slash-model-remap") as
| {
requiredProviderMode?: string;
anthropicModelRef?: string;
openAiXhighModelRef?: string;
noXhighModelRef?: string;
}
| undefined;
expect(scenario.sourcePath).toBe("qa/scenarios/models/thinking-slash-model-remap.md");
expect(config?.requiredProviderMode).toBe("live-frontier");
expect(config?.anthropicModelRef).toBe("anthropic/claude-sonnet-4-6");
expect(config?.openAiXhighModelRef).toBe("openai/gpt-5.4");
expect(config?.noXhighModelRef).toBe("anthropic/claude-sonnet-4-6");
expect(scenario.execution.flow?.steps.map((step) => step.name)).toEqual([
"selects Anthropic and verifies adaptive options",
"maps adaptive to medium when switching to OpenAI",
"maps xhigh to high on a model without xhigh",
]);
});
it("includes the seeded mock-only broken-turn scenarios in the markdown pack", () => {
const scenarioIds = [
"reasoning-only-recovery-replay-safe-read",

View File

@@ -0,0 +1,236 @@
# Thinking slash model remap
```yaml qa-scenario
id: thinking-slash-model-remap
title: Thinking slash model remap
surface: models
coverage:
primary:
- models.thinking
secondary:
- models.switching
- runtime.session-continuity
objective: Verify /think lists provider-owned levels and remaps stored thinking levels when /model changes provider capabilities.
successCriteria:
- Anthropic Claude Sonnet 4.6 advertises adaptive but not OpenAI-only xhigh or Opus max.
- A stored adaptive level remaps to medium when switching to OpenAI GPT-5.4.
- OpenAI GPT-5.4 advertises xhigh but not adaptive or max.
- A stored xhigh level remaps to high when switching to an Anthropic model without xhigh support.
docsRefs:
- docs/tools/thinking.md
- docs/help/testing.md
- docs/concepts/qa-e2e-automation.md
codeRefs:
- src/auto-reply/thinking.ts
- src/auto-reply/thinking.shared.ts
- src/auto-reply/reply/directive-handling.impl.ts
- src/gateway/sessions-patch.ts
- extensions/anthropic/register.runtime.ts
- extensions/openai/openai-provider.ts
execution:
kind: flow
summary: Select Anthropic, set adaptive, switch to OpenAI and verify medium fallback, then set xhigh and verify high fallback on a non-xhigh model.
config:
requiredProviderMode: live-frontier
anthropicModelRef: anthropic/claude-sonnet-4-6
openAiXhighModelRef: openai/gpt-5.4
noXhighModelRef: anthropic/claude-sonnet-4-6
conversationId: qa-thinking-slash-remap
```
```yaml qa-flow
steps:
- name: selects Anthropic and verifies adaptive options
actions:
- call: waitForGatewayHealthy
args:
- ref: env
- 60000
- call: waitForQaChannelReady
args:
- ref: env
- 60000
- call: reset
- assert:
expr: "env.providerMode === config.requiredProviderMode"
message:
expr: "`thinking remap scenario requires ${config.requiredProviderMode}; got ${env.providerMode}`"
- set: cursor
value:
expr: state.getSnapshot().messages.length
- call: state.addInboundMessage
args:
- conversation:
id:
expr: config.conversationId
kind: direct
senderId: qa-operator
senderName: QA Operator
text:
expr: "`/model ${config.anthropicModelRef}`"
- call: waitForCondition
saveAs: anthropicModelAck
args:
- lambda:
expr: "state.getSnapshot().messages.slice(cursor).filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === config.conversationId && candidate.text.includes(`Model set to ${config.anthropicModelRef}`)).at(-1)"
- expr: liveTurnTimeoutMs(env, 20000)
- set: cursor
value:
expr: state.getSnapshot().messages.length
- call: state.addInboundMessage
args:
- conversation:
id:
expr: config.conversationId
kind: direct
senderId: qa-operator
senderName: QA Operator
text: /think
- call: waitForCondition
saveAs: anthropicThinkStatus
args:
- lambda:
expr: "state.getSnapshot().messages.slice(cursor).filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === config.conversationId && /Current thinking level:/i.test(candidate.text)).at(-1)"
- expr: liveTurnTimeoutMs(env, 20000)
- assert:
expr: "/Options: .*adaptive/i.test(anthropicThinkStatus.text)"
message:
expr: "`expected Anthropic /think options to include adaptive, got ${anthropicThinkStatus.text}`"
- assert:
expr: "!/Options: .*\\bxhigh\\b/i.test(anthropicThinkStatus.text) && !/Options: .*\\bmax\\b/i.test(anthropicThinkStatus.text)"
message:
expr: "`expected Sonnet /think options to omit xhigh/max, got ${anthropicThinkStatus.text}`"
detailsExpr: "`model=${anthropicModelAck.text}; think=${anthropicThinkStatus.text}`"
- name: maps adaptive to medium when switching to OpenAI
actions:
- set: cursor
value:
expr: state.getSnapshot().messages.length
- call: state.addInboundMessage
args:
- conversation:
id:
expr: config.conversationId
kind: direct
senderId: qa-operator
senderName: QA Operator
text: /think adaptive
- call: waitForCondition
saveAs: adaptiveAck
args:
- lambda:
expr: "state.getSnapshot().messages.slice(cursor).filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === config.conversationId && /Thinking level set to adaptive/i.test(candidate.text)).at(-1)"
- expr: liveTurnTimeoutMs(env, 20000)
- set: cursor
value:
expr: state.getSnapshot().messages.length
- call: state.addInboundMessage
args:
- conversation:
id:
expr: config.conversationId
kind: direct
senderId: qa-operator
senderName: QA Operator
text:
expr: "`/model ${config.openAiXhighModelRef}`"
- call: waitForCondition
saveAs: openAiModelAck
args:
- lambda:
expr: "state.getSnapshot().messages.slice(cursor).filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === config.conversationId && candidate.text.includes(config.openAiXhighModelRef) && /Model (set to|reset to default)/i.test(candidate.text)).at(-1)"
- expr: liveTurnTimeoutMs(env, 20000)
- assert:
expr: "/Thinking level set to medium \\(adaptive not supported for openai\\/gpt-5\\.4\\)/i.test(openAiModelAck.text)"
message:
expr: "`expected adaptive->medium remap, got ${openAiModelAck.text}`"
- set: cursor
value:
expr: state.getSnapshot().messages.length
- call: state.addInboundMessage
args:
- conversation:
id:
expr: config.conversationId
kind: direct
senderId: qa-operator
senderName: QA Operator
text: /think
- call: waitForCondition
saveAs: openAiThinkStatus
args:
- lambda:
expr: "state.getSnapshot().messages.slice(cursor).filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === config.conversationId && /Current thinking level: medium/i.test(candidate.text)).at(-1)"
- expr: liveTurnTimeoutMs(env, 20000)
- assert:
expr: "/Options: .*\\bxhigh\\b/i.test(openAiThinkStatus.text) && !/Options: .*\\badaptive\\b/i.test(openAiThinkStatus.text) && !/Options: .*\\bmax\\b/i.test(openAiThinkStatus.text)"
message:
expr: "`expected OpenAI GPT-5.4 /think options to include xhigh only, got ${openAiThinkStatus.text}`"
detailsExpr: "`adaptive=${adaptiveAck.text}; switch=${openAiModelAck.text}; think=${openAiThinkStatus.text}`"
- name: maps xhigh to high on a model without xhigh
actions:
- set: cursor
value:
expr: state.getSnapshot().messages.length
- call: state.addInboundMessage
args:
- conversation:
id:
expr: config.conversationId
kind: direct
senderId: qa-operator
senderName: QA Operator
text: /think xhigh
- call: waitForCondition
saveAs: xhighAck
args:
- lambda:
expr: "state.getSnapshot().messages.slice(cursor).filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === config.conversationId && /Thinking level set to xhigh/i.test(candidate.text)).at(-1)"
- expr: liveTurnTimeoutMs(env, 20000)
- set: cursor
value:
expr: state.getSnapshot().messages.length
- call: state.addInboundMessage
args:
- conversation:
id:
expr: config.conversationId
kind: direct
senderId: qa-operator
senderName: QA Operator
text:
expr: "`/model ${config.noXhighModelRef}`"
- call: waitForCondition
saveAs: noXhighModelAck
args:
- lambda:
expr: "state.getSnapshot().messages.slice(cursor).filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === config.conversationId && candidate.text.includes(config.noXhighModelRef) && /Model (set to|reset to default)/i.test(candidate.text)).at(-1)"
- expr: liveTurnTimeoutMs(env, 20000)
- assert:
expr: "/Thinking level set to high \\(xhigh not supported for anthropic\\/claude-sonnet-4-6\\)/i.test(noXhighModelAck.text)"
message:
expr: "`expected xhigh->high remap, got ${noXhighModelAck.text}`"
- set: cursor
value:
expr: state.getSnapshot().messages.length
- call: state.addInboundMessage
args:
- conversation:
id:
expr: config.conversationId
kind: direct
senderId: qa-operator
senderName: QA Operator
text: /think
- call: waitForCondition
saveAs: noXhighThinkStatus
args:
- lambda:
expr: "state.getSnapshot().messages.slice(cursor).filter((candidate) => candidate.direction === 'outbound' && candidate.conversation.id === config.conversationId && /Current thinking level: high/i.test(candidate.text)).at(-1)"
- expr: liveTurnTimeoutMs(env, 20000)
- assert:
expr: "/Options: .*\\badaptive\\b/i.test(noXhighThinkStatus.text) && !/Options: .*\\bxhigh\\b/i.test(noXhighThinkStatus.text) && !/Options: .*\\bmax\\b/i.test(noXhighThinkStatus.text)"
message:
expr: "`expected non-xhigh model /think options to include adaptive and omit xhigh/max, got ${noXhighThinkStatus.text}`"
detailsExpr: "`xhigh=${xhighAck.text}; switch=${noXhighModelAck.text}; think=${noXhighThinkStatus.text}`"
```

View File

@@ -680,6 +680,23 @@ describe("handleDirectiveOnly model persist behavior (fixes #1435)", () => {
expect(sessionEntry.liveModelSwitchPending).toBe(true);
});
it("remaps unsupported stored thinking levels when persisting a model switch", async () => {
const sessionEntry = createSessionEntry({ thinkingLevel: "adaptive" });
const { persisted } = await persistModelDirectiveForTest({
command: "/model openai/gpt-4o",
allowedModelKeys: ["anthropic/claude-opus-4-6", "openai/gpt-4o"],
sessionEntry,
});
expect(sessionEntry.thinkingLevel).toBe("medium");
expect(persisted.thinkingRemap).toEqual({
from: "adaptive",
to: "medium",
provider: "openai",
model: "gpt-4o",
});
});
it("does not request a live restart when /model mutates an active session", async () => {
const directives = parseInlineDirectives("/model openai/gpt-4o");
const sessionEntry = createSessionEntry();
@@ -811,7 +828,7 @@ describe("handleDirectiveOnly model persist behavior (fixes #1435)", () => {
);
expect(result?.text).toContain("Current thinking level: low");
expect(result?.text).toContain("Options: off, minimal, low, medium, high.");
expect(result?.text).toContain("Options: off, minimal, low, medium, adaptive, high.");
});
it("persists verbose on and off directives", async () => {

View File

@@ -12,6 +12,7 @@ import type { OpenClawConfig } from "../../config/types.openclaw.js";
import { enqueueSystemEvent } from "../../infra/system-events.js";
import { applyTraceOverride, applyVerboseOverride } from "../../sessions/level-overrides.js";
import { applyModelOverrideToSessionEntry } from "../../sessions/model-overrides.js";
import { isThinkingLevelSupported, resolveSupportedThinkingLevel } from "../thinking.js";
import { resolveModelSelectionFromDirective } from "./directive-handling.model-selection.js";
import type { InlineDirectives } from "./directive-handling.parse.js";
import {
@@ -19,7 +20,14 @@ import {
canPersistInternalVerboseDirective,
enqueueModeSwitchEvents,
} from "./directive-handling.shared.js";
import type { ElevatedLevel, ReasoningLevel } from "./directives.js";
import type { ElevatedLevel, ReasoningLevel, ThinkLevel } from "./directives.js";
export type PersistedThinkingLevelRemap = {
from: ThinkLevel;
to: ThinkLevel;
provider: string;
model: string;
};
export async function persistInlineDirectives(params: {
directives: InlineDirectives;
@@ -46,7 +54,12 @@ export async function persistInlineDirectives(params: {
gatewayClientScopes?: string[];
senderIsOwner?: boolean;
markLiveSwitchPending?: boolean;
}): Promise<{ provider: string; model: string; contextTokens: number }> {
}): Promise<{
provider: string;
model: string;
contextTokens: number;
thinkingRemap?: PersistedThinkingLevelRemap;
}> {
const {
directives,
cfg,
@@ -65,6 +78,7 @@ export async function persistInlineDirectives(params: {
agentCfg,
} = params;
let { provider, model } = params;
let thinkingRemap: PersistedThinkingLevelRemap | undefined;
const allowInternalExecPersistence = canPersistInternalExecDirective({
messageProvider: params.messageProvider,
surface: params.surface,
@@ -190,6 +204,32 @@ export async function persistInlineDirectives(params: {
});
provider = modelResolution.modelSelection.provider;
model = modelResolution.modelSelection.model;
const currentThinkingLevel = sessionEntry.thinkingLevel as ThinkLevel | undefined;
if (
currentThinkingLevel &&
!directives.hasThinkDirective &&
!isThinkingLevelSupported({
provider,
model,
level: currentThinkingLevel,
})
) {
const remappedThinkingLevel = resolveSupportedThinkingLevel({
provider,
model,
level: currentThinkingLevel,
});
if (remappedThinkingLevel !== currentThinkingLevel) {
sessionEntry.thinkingLevel = remappedThinkingLevel;
thinkingRemap = {
from: currentThinkingLevel,
to: remappedThinkingLevel,
provider,
model,
};
updated = true;
}
}
const nextLabel = `${provider}/${model}`;
if (nextLabel !== initialModelLabel) {
enqueueSystemEvent(
@@ -232,6 +272,7 @@ export async function persistInlineDirectives(params: {
return {
provider,
model,
thinkingRemap,
contextTokens:
resolveContextTokensForModel({
cfg,

View File

@@ -248,7 +248,7 @@ export async function applyInlineDirectiveOverrides(params: {
}
const modelSelection = modelResolution.modelSelection;
if (modelSelection) {
await (
const persisted = await (
await loadDirectivePersist()
).persistInlineDirectives({
directives,
@@ -279,6 +279,9 @@ export async function applyInlineDirectiveOverrides(params: {
const label = `${modelSelection.provider}/${modelSelection.model}`;
const labelWithAlias = modelSelection.alias ? `${modelSelection.alias} (${label})` : label;
const parts = [
persisted.thinkingRemap
? `Thinking level set to ${persisted.thinkingRemap.to} (${persisted.thinkingRemap.from} not supported for ${persisted.thinkingRemap.provider}/${persisted.thinkingRemap.model}).`
: undefined,
modelSelection.isDefault
? `Model reset to default (${labelWithAlias}).`
: `Model set to ${labelWithAlias}.`,

View File

@@ -450,24 +450,25 @@ export async function createModelSelectionState(params: {
if (defaultThinkingLevel) {
return defaultThinkingLevel;
}
let catalogForThinking = modelCatalog ?? allowedModelCatalog;
if (!catalogForThinking || catalogForThinking.length === 0) {
const agentThinkingDefault = agentEntry?.thinkingDefault as ThinkLevel | undefined;
const configuredThinkingDefault = agentCfg?.thinkingDefault as ThinkLevel | undefined;
const explicitThinkingDefault = agentThinkingDefault ?? configuredThinkingDefault;
if (explicitThinkingDefault) {
defaultThinkingLevel = explicitThinkingDefault;
return defaultThinkingLevel;
}
if (!modelCatalog) {
modelCatalog = await (await loadModelCatalogRuntime()).loadModelCatalog({ config: cfg });
logStage("catalog-loaded-for-thinking", `entries=${modelCatalog.length}`);
catalogForThinking = modelCatalog;
}
const catalogForThinking = modelCatalog.length > 0 ? modelCatalog : allowedModelCatalog;
const resolved = resolveThinkingDefault({
cfg,
provider,
model,
catalog: catalogForThinking,
});
const agentThinkingDefault = agentEntry?.thinkingDefault as ThinkLevel | undefined;
defaultThinkingLevel =
agentThinkingDefault ??
resolved ??
(agentCfg?.thinkingDefault as ThinkLevel | undefined) ??
"off";
defaultThinkingLevel = resolved ?? "off";
return defaultThinkingLevel;
};

View File

@@ -35,7 +35,7 @@ export const THINKING_LEVEL_RANKS: Record<ThinkLevel, number> = {
low: 20,
medium: 30,
high: 40,
adaptive: 50,
adaptive: 30,
xhigh: 60,
max: 70,
};

View File

@@ -168,6 +168,27 @@ describe("listThinkingLevels", () => {
}),
).toBe("high");
});
it("maps unsupported adaptive to medium and unsupported xhigh to high", () => {
providerRuntimeMocks.resolveProviderThinkingProfile.mockReturnValue({
levels: [{ id: "off" }, { id: "minimal" }, { id: "low" }, { id: "medium" }, { id: "high" }],
});
expect(
resolveSupportedThinkingLevel({
provider: "openai",
model: "gpt-5.4",
level: "adaptive",
}),
).toBe("medium");
expect(
resolveSupportedThinkingLevel({
provider: "openai",
model: "gpt-4.1-mini",
level: "xhigh",
}),
).toBe("high");
});
});
describe("listThinkingLevelLabels", () => {

View File

@@ -257,8 +257,15 @@ export function resolveSupportedThinkingLevel(params: {
model?: string | null;
level: ThinkLevel;
}): ThinkLevel {
if (isThinkingLevelSupported(params)) {
const profile = resolveThinkingProfile({ provider: params.provider, model: params.model });
if (profile.levels.some((entry) => entry.id === params.level)) {
return params.level;
}
return resolveLargestSupportedThinkingLevel(params.provider, params.model);
const requestedRank = THINKING_LEVEL_RANKS[params.level];
const ranked = profile.levels.toSorted((a, b) => b.rank - a.rank);
return (
ranked.find((level) => level.id !== "off" && level.rank <= requestedRank)?.id ??
ranked.find((level) => level.id !== "off")?.id ??
"off"
);
}

View File

@@ -1,4 +1,5 @@
import { normalizeProviderId } from "../agents/provider-id.js";
import { resolveProviderRuntimePlugin } from "./provider-hook-runtime.js";
import type {
ProviderDefaultThinkingPolicyContext,
ProviderThinkingProfile,
@@ -43,9 +44,13 @@ function resolveActiveThinkingProvider(providerId: string): ThinkingProviderPlug
const state = (
globalThis as typeof globalThis & { [PLUGIN_REGISTRY_STATE]?: ThinkingRegistryState }
)[PLUGIN_REGISTRY_STATE];
return state?.activeRegistry?.providers?.find((entry) => {
const activeProvider = state?.activeRegistry?.providers?.find((entry) => {
return matchesProviderId(entry.provider, providerId);
})?.provider;
if (activeProvider) {
return activeProvider;
}
return resolveProviderRuntimePlugin({ provider: providerId });
}
type ThinkingHookParams<TContext> = {