diff --git a/src/agents/model-catalog-lookup.ts b/src/agents/model-catalog-lookup.ts new file mode 100644 index 00000000000..810635efdd2 --- /dev/null +++ b/src/agents/model-catalog-lookup.ts @@ -0,0 +1,48 @@ +import { + normalizeLowercaseStringOrEmpty, + normalizeOptionalString, +} from "../shared/string-coerce.js"; +import type { ModelCatalogEntry, ModelInputType } from "./model-catalog.types.js"; +import { normalizeProviderId } from "./provider-id.js"; + +export function modelSupportsInput( + entry: ModelCatalogEntry | undefined, + input: ModelInputType, +): boolean { + return entry?.input?.includes(input) ?? false; +} + +export function findModelInCatalog( + catalog: ModelCatalogEntry[], + provider: string, + modelId: string, +): ModelCatalogEntry | undefined { + const normalizedProvider = normalizeProviderId(provider); + const normalizedModelId = normalizeLowercaseStringOrEmpty(modelId); + return catalog.find( + (entry) => + normalizeProviderId(entry.provider) === normalizedProvider && + normalizeLowercaseStringOrEmpty(entry.id) === normalizedModelId, + ); +} + +export function findModelCatalogEntry( + catalog: ModelCatalogEntry[], + params: { provider?: string; modelId: string }, +): ModelCatalogEntry | undefined { + const modelId = normalizeOptionalString(params.modelId) ?? ""; + if (!modelId) { + return undefined; + } + + const provider = normalizeOptionalString(params.provider); + if (provider) { + return findModelInCatalog(catalog, provider, modelId); + } + + const normalizedModelId = normalizeLowercaseStringOrEmpty(modelId); + const matches = catalog.filter( + (entry) => normalizeLowercaseStringOrEmpty(entry.id) === normalizedModelId, + ); + return matches.length === 1 ? matches[0] : undefined; +} diff --git a/src/agents/model-catalog.test.ts b/src/agents/model-catalog.test.ts index d029a64c4ee..0eeca0d6977 100644 --- a/src/agents/model-catalog.test.ts +++ b/src/agents/model-catalog.test.ts @@ -5,8 +5,10 @@ import { resetLogger, setLoggerOverride } from "../logging/logger.js"; type PiSdkModule = typeof import("./pi-model-discovery.js"); let __setModelCatalogImportForTest: typeof import("./model-catalog.js").__setModelCatalogImportForTest; +let findModelCatalogEntry: typeof import("./model-catalog.js").findModelCatalogEntry; let findModelInCatalog: typeof import("./model-catalog.js").findModelInCatalog; let loadModelCatalog: typeof import("./model-catalog.js").loadModelCatalog; +let modelSupportsInput: typeof import("./model-catalog.js").modelSupportsInput; let resetModelCatalogCacheForTest: typeof import("./model-catalog.js").resetModelCatalogCacheForTest; let augmentCatalogMock: ReturnType; let ensureOpenClawModelsJsonMock: ReturnType; @@ -73,8 +75,10 @@ describe("loadModelCatalog", () => { ({ __setModelCatalogImportForTest, + findModelCatalogEntry, findModelInCatalog, loadModelCatalog, + modelSupportsInput, resetModelCatalogCacheForTest, } = await import("./model-catalog.js")); const providerRuntime = await import("../plugins/provider-runtime.runtime.js"); @@ -482,4 +486,23 @@ describe("loadModelCatalog", () => { name: "GLM-5", }); }); + + it("resolves catalog entries with explicit providers and unique providerless matches", () => { + const catalog = [ + { provider: "first", id: "shared", name: "First", input: ["text"] }, + { provider: "second", id: "shared", name: "Second", input: ["text", "image"] }, + { provider: "modelscope", id: "qwen/qwen3.5-35b-a3b", name: "Qwen", input: ["text"] }, + ] satisfies Awaited>; + + expect(findModelCatalogEntry(catalog, { provider: "second", modelId: "SHARED" })).toEqual( + catalog[1], + ); + expect( + findModelCatalogEntry(catalog, { provider: "modelscope", modelId: "Qwen/Qwen3.5-35B-A3B" }), + ).toEqual(catalog[2]); + expect(findModelCatalogEntry(catalog, { modelId: "shared" })).toBeUndefined(); + expect(findModelCatalogEntry(catalog, { modelId: "Qwen/Qwen3.5-35B-A3B" })).toEqual(catalog[2]); + expect(modelSupportsInput(catalog[1], "image")).toBe(true); + expect(modelSupportsInput(catalog[2], "image")).toBe(false); + }); }); diff --git a/src/agents/model-catalog.ts b/src/agents/model-catalog.ts index 29580eb551d..a039b2a6a37 100644 --- a/src/agents/model-catalog.ts +++ b/src/agents/model-catalog.ts @@ -8,6 +8,7 @@ import { normalizeOptionalString, } from "../shared/string-coerce.js"; import { resolveOpenClawAgentDir } from "./agent-paths.js"; +import { modelSupportsInput as modelCatalogEntrySupportsInput } from "./model-catalog-lookup.js"; import type { ModelCatalogEntry, ModelInputType } from "./model-catalog.types.js"; import { buildConfiguredModelCatalog } from "./model-selection-shared.js"; import { ensureOpenClawModelsJson } from "./models-config.js"; @@ -16,6 +17,11 @@ import { normalizeProviderId } from "./provider-id.js"; const log = createSubsystemLogger("model-catalog"); export type { ModelCatalogEntry, ModelInputType } from "./model-catalog.types.js"; +export { + findModelCatalogEntry, + findModelInCatalog, + modelSupportsInput, +} from "./model-catalog-lookup.js"; type DiscoveredModel = { id: string; @@ -238,29 +244,12 @@ export async function loadModelCatalog(params?: { * Check if a model supports image input based on its catalog entry. */ export function modelSupportsVision(entry: ModelCatalogEntry | undefined): boolean { - return entry?.input?.includes("image") ?? false; + return modelCatalogEntrySupportsInput(entry, "image"); } /** * Check if a model supports native document/PDF input based on its catalog entry. */ export function modelSupportsDocument(entry: ModelCatalogEntry | undefined): boolean { - return entry?.input?.includes("document") ?? false; -} - -/** - * Find a model in the catalog by provider and model ID. - */ -export function findModelInCatalog( - catalog: ModelCatalogEntry[], - provider: string, - modelId: string, -): ModelCatalogEntry | undefined { - const normalizedProvider = normalizeProviderId(provider); - const normalizedModelId = normalizeLowercaseStringOrEmpty(modelId); - return catalog.find( - (entry) => - normalizeProviderId(entry.provider) === normalizedProvider && - normalizeLowercaseStringOrEmpty(entry.id) === normalizedModelId, - ); + return modelCatalogEntrySupportsInput(entry, "document"); } diff --git a/src/agents/model-selection-shared.ts b/src/agents/model-selection-shared.ts index 58cceb3b05b..84d6326b5f7 100644 --- a/src/agents/model-selection-shared.ts +++ b/src/agents/model-selection-shared.ts @@ -8,6 +8,7 @@ import { import { sanitizeForLog, stripAnsi } from "../terminal/ansi.js"; import { resolveConfiguredProviderFallback } from "./configured-provider-fallback.js"; import { DEFAULT_PROVIDER } from "./defaults.js"; +import { findModelCatalogEntry } from "./model-catalog-lookup.js"; import type { ModelCatalogEntry } from "./model-catalog.types.js"; import { splitTrailingAuthProfile } from "./model-ref-profile.js"; import { normalizeStaticProviderModelId } from "./model-ref-shared.js"; @@ -573,7 +574,18 @@ export function buildAllowedModelSetWithFallbacks(params: { } const allowedKeys = new Set(); + const allowedRefs: ModelRef[] = []; const syntheticCatalogEntries = new Map(); + const addAllowedCatalogRef = (ref: ModelRef) => { + if ( + !allowedRefs.some( + (existing) => + modelKey(existing.provider, existing.model) === modelKey(ref.provider, ref.model), + ) + ) { + allowedRefs.push(ref); + } + }; const addAllowedModelRef = (raw: string) => { const trimmed = raw.trim(); const defaultProvider = !trimmed.includes("/") @@ -594,8 +606,12 @@ export function buildAllowedModelSetWithFallbacks(params: { } const key = modelKey(parsed.provider, parsed.model); allowedKeys.add(key); + addAllowedCatalogRef(parsed); - if (!catalogKeys.has(key) && !syntheticCatalogEntries.has(key)) { + if ( + !findModelCatalogEntry(catalog, { provider: parsed.provider, modelId: parsed.model }) && + !syntheticCatalogEntries.has(key) + ) { syntheticCatalogEntries.set(key, buildSyntheticAllowedCatalogEntry({ parsed, metadata })); } }; @@ -610,10 +626,18 @@ export function buildAllowedModelSetWithFallbacks(params: { if (defaultKey) { allowedKeys.add(defaultKey); + if (defaultRef) { + addAllowedCatalogRef(defaultRef); + } } const allowedCatalog = [ - ...catalog.filter((entry) => allowedKeys.has(modelKey(entry.provider, entry.id))), + ...catalog.filter((entry) => + allowedRefs.some( + (ref) => + findModelCatalogEntry([entry], { provider: ref.provider, modelId: ref.model }) === entry, + ), + ), ...syntheticCatalogEntries.values(), ]; @@ -655,7 +679,12 @@ export function getModelRefStatusFromAllowedSet(params: { const key = modelKey(params.ref.provider, params.ref.model); return { key, - inCatalog: params.catalog.some((entry) => modelKey(entry.provider, entry.id) === key), + inCatalog: Boolean( + findModelCatalogEntry(params.catalog, { + provider: params.ref.provider, + modelId: params.ref.model, + }), + ), allowAny: params.allowed.allowAny, allowed: params.allowed.allowAny || params.allowed.allowedKeys.has(key), }; diff --git a/src/agents/model-selection.test.ts b/src/agents/model-selection.test.ts index a4c0ccb5d1b..5a52174fa21 100644 --- a/src/agents/model-selection.test.ts +++ b/src/agents/model-selection.test.ts @@ -654,6 +654,39 @@ describe("model-selection", () => { ]); }); + it("matches allowlisted catalog entries with normalized provider and model ids", () => { + const cfg: OpenClawConfig = { + agents: { + defaults: { + models: { + "modelscope/Qwen/Qwen3.5-35B-A3B": {}, + }, + }, + }, + } as unknown as OpenClawConfig; + + const result = buildAllowedModelSet({ + cfg, + catalog: [ + { + provider: "modelscope", + id: "qwen/qwen3.5-35b-a3b", + name: "Qwen3.5 35B", + input: ["text", "image"], + }, + ], + defaultProvider: "anthropic", + }); + + expect(result.allowedCatalog).toEqual([ + expect.objectContaining({ + provider: "modelscope", + id: "qwen/qwen3.5-35b-a3b", + input: ["text", "image"], + }), + ]); + }); + it("applies configured provider metadata and alias to synthetic allowlist entries", () => { const cfg: OpenClawConfig = { agents: { diff --git a/src/gateway/server-methods/chat.directive-tags.test.ts b/src/gateway/server-methods/chat.directive-tags.test.ts index f9a16040671..ff6f5c767a4 100644 --- a/src/gateway/server-methods/chat.directive-tags.test.ts +++ b/src/gateway/server-methods/chat.directive-tags.test.ts @@ -2403,6 +2403,54 @@ describe("chat directive tag stripping for non-streaming final payloads", () => ]); }); + it("keeps image attachments inline for configured custom vision models", async () => { + createTranscriptFixture("openclaw-chat-send-configured-custom-vision-"); + mockState.finalText = "ok"; + mockState.sessionEntry = { + modelProvider: "modelscope", + model: "Qwen/Qwen3.5-35B-A3B", + }; + mockState.modelCatalog = [ + { + provider: "modelscope", + id: "qwen/qwen3.5-35b-a3b", + name: "Qwen3.5 35B", + input: ["text", "image"], + }, + ]; + const respond = vi.fn(); + const context = createChatContext(); + + await runNonStreamingChatSend({ + context, + respond, + idempotencyKey: "idem-configured-custom-vision", + message: "describe image", + requestParams: { + attachments: [ + { + mimeType: "image/png", + content: + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/woAAn8B9FD5fHAAAAAASUVORK5CYII=", + }, + ], + }, + expectBroadcast: false, + }); + + expect(mockState.lastDispatchImages).toEqual([ + expect.objectContaining({ + mimeType: "image/png", + data: expect.any(String), + }), + ]); + expect(mockState.lastDispatchImageOrder).toEqual(["inline"]); + expect(mockState.lastDispatchCtx?.Body).toBe("describe image"); + expect(mockState.savedMediaCalls).toEqual([ + expect.objectContaining({ contentType: "image/png", subdir: "inbound" }), + ]); + }); + it("keeps image attachments for text-only sessions bound to ACP", async () => { createTranscriptFixture("openclaw-chat-send-text-only-acp-bound-attachments-"); mockState.finalText = "ok"; diff --git a/src/gateway/session-utils.ts b/src/gateway/session-utils.ts index 3a9703acdd9..bb1b1db3155 100644 --- a/src/gateway/session-utils.ts +++ b/src/gateway/session-utils.ts @@ -10,7 +10,11 @@ import { } from "../agents/agent-scope.js"; import { lookupContextTokens, resolveContextTokensForModel } from "../agents/context.js"; import { DEFAULT_CONTEXT_TOKENS, DEFAULT_MODEL, DEFAULT_PROVIDER } from "../agents/defaults.js"; -import { findModelInCatalog, type ModelCatalogEntry } from "../agents/model-catalog.js"; +import { + findModelCatalogEntry, + modelSupportsInput, + type ModelCatalogEntry, +} from "../agents/model-catalog.js"; import { inferUniqueProviderFromConfiguredModels, normalizeStoredOverrideModel, @@ -68,8 +72,8 @@ import { } from "../shared/avatar-policy.js"; import { normalizeLowercaseStringOrEmpty, - normalizeOptionalLowercaseString, normalizeOptionalString, + normalizeOptionalLowercaseString, } from "../shared/string-coerce.js"; import { normalizeSessionDeliveryFields } from "../utils/delivery-context.shared.js"; import { estimateUsageCost, resolveModelCostConfig } from "../utils/usage-format.js"; @@ -1115,23 +1119,6 @@ export function resolveSessionModelRef( return resolved; } -function findGatewayImageSupportCatalogEntry(params: { - catalog: ModelCatalogEntry[]; - provider?: string; - model: string; -}): ModelCatalogEntry | undefined { - const provider = normalizeOptionalString(params.provider); - if (provider) { - return findModelInCatalog(params.catalog, provider, params.model); - } - - const normalizedModel = normalizeLowercaseStringOrEmpty(params.model); - const matches = params.catalog.filter( - (entry) => normalizeLowercaseStringOrEmpty(entry.id) === normalizedModel, - ); - return matches.length === 1 ? matches[0] : undefined; -} - export async function resolveGatewayModelSupportsImages(params: { loadGatewayModelCatalog: () => Promise; provider?: string; @@ -1143,10 +1130,9 @@ export async function resolveGatewayModelSupportsImages(params: { try { const catalog = await params.loadGatewayModelCatalog(); - const modelEntry = findGatewayImageSupportCatalogEntry({ - catalog, + const modelEntry = findModelCatalogEntry(catalog, { provider: params.provider, - model: params.model, + modelId: params.model, }); const normalizedProvider = normalizeOptionalLowercaseString( params.provider ?? modelEntry?.provider, @@ -1156,7 +1142,7 @@ export async function resolveGatewayModelSupportsImages(params: { normalizeLowercaseStringOrEmpty(modelEntry?.name), ].filter(Boolean); if (modelEntry) { - if (modelEntry.input?.includes("image")) { + if (modelSupportsInput(modelEntry, "image")) { return true; } // Legacy safety shim for stale persisted Foundry rows that predate