diff --git a/CHANGELOG.md b/CHANGELOG.md index 4559b929d38..b38ca8697b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,6 +49,7 @@ Docs: https://docs.openclaw.ai ### Fixes - Channels/Telegram: keep Bot API network fallbacks sticky after failed attempts and retry timed-out startup control calls once on the fallback route, so `deleteWebhook` IPv6 stalls no longer trigger slow multi-account retry storms. Fixes #73255. Thanks @ttomiczek and @sktbrd. +- Gateway/models: merge explicit `models.providers.*.models` rows into the Gateway model catalog with normalized provider/model dedupe, and use normalized image-capability lookup so custom vision models keep native image attachments even when Pi discovery omits them or model ID casing differs. Fixes #64213 and #65165. Thanks @billonese and @202233a. - Export/session: keep inline export HTML scripts and vendor libraries injected after template formatting so generated session exports open with the app code, markdown renderer, and syntax highlighter present. Fixes #41862 and #49957; carries forward #41861 and #68947. Thanks @briannewman, @martenzi, and @armanddp. - Agents/ACPX: stage the patched Claude ACP adapter as an ACPX runtime dependency and route known Codex/Claude ACP commands through local wrappers, so Gateway runtime no longer depends on live `npx` adapter resolution. Fixes #73202. Thanks @joerod26. - Memory/compaction: let pre-compaction memory flush use an exact `agents.defaults.compaction.memoryFlush.model` override such as `ollama/qwen3:8b` without inheriting the active session fallback chain, so local housekeeping can avoid paid conversation models. Fixes #53772. Thanks @limen96. diff --git a/docs/concepts/model-providers.md b/docs/concepts/model-providers.md index cdeeaa69935..fbfe15a4ad8 100644 --- a/docs/concepts/model-providers.md +++ b/docs/concepts/model-providers.md @@ -329,6 +329,8 @@ Use `models.providers` (or `models.json`) to add **custom** providers or OpenAI/ Many of the bundled provider plugins below already publish a default catalog. Use explicit `models.providers.` entries only when you want to override the default base URL, headers, or model list. +Gateway model capability checks also read explicit `models.providers..models[]` metadata. If a custom or proxy model accepts images, set `input: ["text", "image"]` on that model so WebChat and node-origin attachment paths pass images as native model inputs instead of text-only media refs. + ### Moonshot AI (Kimi) Moonshot ships as a bundled provider plugin. Use the built-in provider by default, and add an explicit `models.providers.moonshot` entry only when you need to override the base URL or model metadata: diff --git a/src/agents/model-catalog.test.ts b/src/agents/model-catalog.test.ts index bc03223722b..d029a64c4ee 100644 --- a/src/agents/model-catalog.test.ts +++ b/src/agents/model-catalog.test.ts @@ -366,6 +366,75 @@ describe("loadModelCatalog", () => { ).toHaveLength(1); }); + it("includes configured provider models missing from discovery", async () => { + mockSingleOpenAiCatalogModel(); + + const result = await loadModelCatalog({ + config: { + models: { + providers: { + modelscope: { + baseUrl: "https://api-inference.modelscope.cn/v1", + models: [ + { + id: "Qwen/Qwen3.5-35B-A3B", + name: "Qwen3.5 35B", + input: ["text", "image"], + reasoning: true, + contextWindow: 128_000, + maxTokens: 8192, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + }, + ], + }, + }, + }, + } as OpenClawConfig, + }); + + expect(result).toContainEqual( + expect.objectContaining({ + provider: "modelscope", + id: "Qwen/Qwen3.5-35B-A3B", + name: "Qwen3.5 35B", + input: ["text", "image"], + reasoning: true, + contextWindow: 128_000, + }), + ); + }); + + it("dedupes configured models against discovered provider aliases", async () => { + mockPiDiscoveryModels([{ id: "glm-5", provider: "z.ai", name: "GLM-5" }]); + + const result = await loadModelCatalog({ + config: { + models: { + providers: { + "z-ai": { + baseUrl: "https://api.z.ai/v1", + models: [ + { + id: "glm-5", + name: "Configured GLM-5", + input: ["text", "image"], + reasoning: false, + contextWindow: 128_000, + maxTokens: 8192, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + }, + ], + }, + }, + }, + } as OpenClawConfig, + }); + + const matches = result.filter((entry) => findModelInCatalog([entry], "z-ai", "glm-5")); + expect(matches).toHaveLength(1); + expect(matches[0]).toMatchObject({ provider: "z.ai", id: "glm-5", name: "GLM-5" }); + }); + it("does not add unrelated models when provider plugins return nothing", async () => { mockSingleOpenAiCatalogModel(); diff --git a/src/agents/model-catalog.ts b/src/agents/model-catalog.ts index 743306c6bfa..29580eb551d 100644 --- a/src/agents/model-catalog.ts +++ b/src/agents/model-catalog.ts @@ -9,6 +9,7 @@ import { } from "../shared/string-coerce.js"; import { resolveOpenClawAgentDir } from "./agent-paths.js"; import type { ModelCatalogEntry, ModelInputType } from "./model-catalog.types.js"; +import { buildConfiguredModelCatalog } from "./model-selection-shared.js"; import { ensureOpenClawModelsJson } from "./models-config.js"; import { normalizeProviderId } from "./provider-id.js"; @@ -78,6 +79,25 @@ function instantiatePiModelRegistry( return new Registry(authStorage, modelsFile); } +function catalogEntryDedupeKey(provider: string, id: string): string { + return `${normalizeProviderId(provider)}::${normalizeLowercaseStringOrEmpty(id)}`; +} + +function appendCatalogEntriesIfAbsent( + models: ModelCatalogEntry[], + entries: ModelCatalogEntry[], +): void { + const seen = new Set(models.map((entry) => catalogEntryDedupeKey(entry.provider, entry.id))); + for (const entry of entries) { + const key = catalogEntryDedupeKey(entry.provider, entry.id); + if (seen.has(key)) { + continue; + } + models.push(entry); + seen.add(key); + } +} + export async function loadModelCatalog(params?: { config?: OpenClawConfig; useCache?: boolean; @@ -170,23 +190,16 @@ export async function loadModelCatalog(params?: { }, }); if (supplemental.length > 0) { - const seen = new Set( - models.map( - (entry) => - `${normalizeLowercaseStringOrEmpty(entry.provider)}::${normalizeLowercaseStringOrEmpty(entry.id)}`, - ), - ); - for (const entry of supplemental) { - const key = `${normalizeLowercaseStringOrEmpty(entry.provider)}::${normalizeLowercaseStringOrEmpty(entry.id)}`; - if (seen.has(key)) { - continue; - } - models.push(entry); - seen.add(key); - } + appendCatalogEntriesIfAbsent(models, supplemental); } logStage("plugin-models-merged", `entries=${models.length}`); + const configuredModels = buildConfiguredModelCatalog({ cfg }); + if (configuredModels.length > 0) { + appendCatalogEntriesIfAbsent(models, configuredModels); + } + logStage("configured-models-merged", `entries=${models.length}`); + if (models.length === 0) { // If we found nothing, don't cache this result so we can try again. if (!readOnly) { diff --git a/src/gateway/session-utils.test.ts b/src/gateway/session-utils.test.ts index a5985d4f765..1922dfca0fc 100644 --- a/src/gateway/session-utils.test.ts +++ b/src/gateway/session-utils.test.ts @@ -1262,4 +1262,66 @@ describe("resolveGatewayModelSupportsImages", () => { }), ).resolves.toBe(true); }); + + test("matches catalog model ids case-insensitively for explicit providers", async () => { + await expect( + resolveGatewayModelSupportsImages({ + model: "Qwen/Qwen3.5-35B-A3B", + provider: "modelscope", + loadGatewayModelCatalog: async () => [ + { + id: "qwen/qwen3.5-35b-a3b", + name: "Qwen3.5 35B", + provider: "modelscope", + input: ["text", "image"], + }, + ], + }), + ).resolves.toBe(true); + }); + + test("does not borrow image support from another provider when provider is explicit", async () => { + await expect( + resolveGatewayModelSupportsImages({ + model: "gpt-4", + provider: "openai", + loadGatewayModelCatalog: async () => [ + { id: "gpt-4", name: "GPT-4", provider: "other", input: ["text", "image"] }, + ], + }), + ).resolves.toBe(false); + }); + + test("uses a unique providerless catalog match", async () => { + await expect( + resolveGatewayModelSupportsImages({ + model: "Qwen/Qwen3.5-35B-A3B", + loadGatewayModelCatalog: async () => [ + { + id: "qwen/qwen3.5-35b-a3b", + name: "Qwen3.5 35B", + provider: "modelscope", + input: ["text", "image"], + }, + ], + }), + ).resolves.toBe(true); + }); + + test("fails closed on ambiguous providerless catalog matches", async () => { + await expect( + resolveGatewayModelSupportsImages({ + model: "shared-vision", + loadGatewayModelCatalog: async () => [ + { id: "shared-vision", name: "Shared Vision", provider: "first", input: ["text"] }, + { + id: "shared-vision", + name: "Shared Vision", + provider: "second", + input: ["text", "image"], + }, + ], + }), + ).resolves.toBe(false); + }); }); diff --git a/src/gateway/session-utils.ts b/src/gateway/session-utils.ts index 9d6707941a8..3a9703acdd9 100644 --- a/src/gateway/session-utils.ts +++ b/src/gateway/session-utils.ts @@ -10,7 +10,7 @@ import { } from "../agents/agent-scope.js"; import { lookupContextTokens, resolveContextTokensForModel } from "../agents/context.js"; import { DEFAULT_CONTEXT_TOKENS, DEFAULT_MODEL, DEFAULT_PROVIDER } from "../agents/defaults.js"; -import type { ModelCatalogEntry } from "../agents/model-catalog.js"; +import { findModelInCatalog, type ModelCatalogEntry } from "../agents/model-catalog.js"; import { inferUniqueProviderFromConfiguredModels, normalizeStoredOverrideModel, @@ -1115,6 +1115,23 @@ export function resolveSessionModelRef( return resolved; } +function findGatewayImageSupportCatalogEntry(params: { + catalog: ModelCatalogEntry[]; + provider?: string; + model: string; +}): ModelCatalogEntry | undefined { + const provider = normalizeOptionalString(params.provider); + if (provider) { + return findModelInCatalog(params.catalog, provider, params.model); + } + + const normalizedModel = normalizeLowercaseStringOrEmpty(params.model); + const matches = params.catalog.filter( + (entry) => normalizeLowercaseStringOrEmpty(entry.id) === normalizedModel, + ); + return matches.length === 1 ? matches[0] : undefined; +} + export async function resolveGatewayModelSupportsImages(params: { loadGatewayModelCatalog: () => Promise; provider?: string; @@ -1126,11 +1143,14 @@ export async function resolveGatewayModelSupportsImages(params: { try { const catalog = await params.loadGatewayModelCatalog(); - const modelEntry = catalog.find( - (entry) => - entry.id === params.model && (!params.provider || entry.provider === params.provider), + const modelEntry = findGatewayImageSupportCatalogEntry({ + catalog, + provider: params.provider, + model: params.model, + }); + const normalizedProvider = normalizeOptionalLowercaseString( + params.provider ?? modelEntry?.provider, ); - const normalizedProvider = normalizeOptionalLowercaseString(params.provider); const normalizedCandidates = [ normalizeLowercaseStringOrEmpty(params.model), normalizeLowercaseStringOrEmpty(modelEntry?.name),