From 0841bcdc012a6f5c15c268f01d9774182a3692d0 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sat, 2 May 2026 14:25:55 -0700 Subject: [PATCH] fix(config): reject unavailable provider refs --- .../config.model-ref-validation.test.ts | 86 ++++++++++ src/config/config.web-search-provider.test.ts | 29 ++++ src/config/validation.ts | 155 +++++++++++++++++- 3 files changed, 264 insertions(+), 6 deletions(-) create mode 100644 src/config/config.model-ref-validation.test.ts diff --git a/src/config/config.model-ref-validation.test.ts b/src/config/config.model-ref-validation.test.ts new file mode 100644 index 00000000000..8881047a73c --- /dev/null +++ b/src/config/config.model-ref-validation.test.ts @@ -0,0 +1,86 @@ +import { describe, expect, it } from "vitest"; +import type { PluginManifestRegistry } from "../plugins/manifest-registry.js"; +import { validateConfigObjectWithPlugins } from "./validation.js"; + +function createModelSuppressionRegistry(): PluginManifestRegistry { + return { + diagnostics: [], + plugins: [ + { + id: "openai", + origin: "bundled", + channels: [], + providers: ["openai", "openai-codex"], + contracts: {}, + cliBackends: [], + skills: [], + hooks: [], + rootDir: "/tmp/plugins/openai", + source: "test", + manifestPath: "/tmp/plugins/openai/openclaw.plugin.json", + modelCatalog: { + suppressions: [ + { + provider: "openai-codex", + model: "gpt-5.3-codex-spark", + reason: + "gpt-5.3-codex-spark is no longer exposed by the OpenAI or Codex catalogs. Use openai/gpt-5.5.", + }, + ], + }, + }, + ], + }; +} + +describe("config model reference validation", () => { + it("rejects statically suppressed provider/model pairs during config validation", () => { + const res = validateConfigObjectWithPlugins( + { + agents: { + defaults: { + model: { + primary: "openai-codex/gpt-5.3-codex-spark", + }, + }, + }, + }, + { + pluginMetadataSnapshot: { + manifestRegistry: createModelSuppressionRegistry(), + }, + }, + ); + + expect(res.ok).toBe(false); + if (res.ok) { + return; + } + expect(res.issues).toContainEqual({ + path: "agents.defaults.model.primary", + message: + "Unknown model: openai-codex/gpt-5.3-codex-spark. gpt-5.3-codex-spark is no longer exposed by the OpenAI or Codex catalogs. Use openai/gpt-5.5.", + }); + }); + + it("accepts supported openai-codex provider/model pairs", () => { + const res = validateConfigObjectWithPlugins( + { + agents: { + defaults: { + model: { + primary: "openai-codex/gpt-5.4-mini", + }, + }, + }, + }, + { + pluginMetadataSnapshot: { + manifestRegistry: createModelSuppressionRegistry(), + }, + }, + ); + + expect(res.ok).toBe(true); + }); +}); diff --git a/src/config/config.web-search-provider.test.ts b/src/config/config.web-search-provider.test.ts index 630c3e4d052..6afd7d8b176 100644 --- a/src/config/config.web-search-provider.test.ts +++ b/src/config/config.web-search-provider.test.ts @@ -434,6 +434,35 @@ describe("web search provider config", () => { expect(res.ok).toBe(true); }); + it("rejects installable provider ids when the plugin is not active", () => { + const res = validateConfigObjectWithPlugins( + buildWebSearchProviderConfig({ + provider: "brave", + }), + { + pluginMetadataSnapshot: { + manifestRegistry: { + plugins: [], + diagnostics: [], + }, + }, + }, + ); + + expect(res.ok).toBe(false); + if (res.ok) { + return; + } + expect(res.issues).toContainEqual( + expect.objectContaining({ + path: "tools.web.search.provider", + message: + 'web_search provider is not available: brave (install or enable plugin "brave", then run openclaw doctor --fix)', + allowedValues: expect.arrayContaining(["brave"]), + }), + ); + }); + it("rejects unknown provider ids without plugin evidence", () => { const res = validateConfigObjectWithPlugins({ tools: { diff --git a/src/config/validation.ts b/src/config/validation.ts index 56c4eabd192..300d56c5977 100644 --- a/src/config/validation.ts +++ b/src/config/validation.ts @@ -1,6 +1,7 @@ import path from "node:path"; import { resolveAgentWorkspaceDir, resolveDefaultAgentId } from "../agents/agent-scope.js"; import { CHANNEL_IDS, normalizeChatChannelId } from "../channels/ids.js"; +import { planManifestModelCatalogSuppressions } from "../model-catalog/index.js"; import { withBundledPluginAllowlistCompat } from "../plugins/bundled-compat.js"; import { normalizePluginsConfig, @@ -49,6 +50,10 @@ type AllowedValuesCollection = { hasValues: boolean; }; type JsonSchemaLike = Record; +type ConfiguredModelRef = { + path: string; + value: string; +}; function stripDeprecatedValidationKeys(raw: unknown): unknown { if (!isRecord(raw) || !isRecord(raw.commands) || !Object.hasOwn(raw.commands, "modelsWrite")) { @@ -978,20 +983,29 @@ function validateConfigObjectWithPluginsBase( return ensureInstalledPluginRecordIds().has(normalizedChannelId); }; - const collectKnownWebSearchProviderIds = (): string[] => { + const collectActiveWebSearchProviderIds = (): string[] => { const { registry } = ensureRegistry(); return [ ...new Set( - [ - ...registry.plugins.flatMap((record) => record.contracts?.webSearchProviders ?? []), - ...resolveWebSearchInstallCatalogEntries().map((entry) => entry.provider.id), - ] + registry.plugins + .flatMap((record) => record.contracts?.webSearchProviders ?? []) .map((providerId) => providerId.trim()) .filter((providerId) => providerId.length > 0), ), ].toSorted((left, right) => left.localeCompare(right)); }; + const collectKnownWebSearchProviderIds = (): string[] => { + return [ + ...new Set([ + ...collectActiveWebSearchProviderIds(), + ...resolveWebSearchInstallCatalogEntries() + .map((entry) => entry.provider.id.trim()) + .filter((providerId) => providerId.length > 0), + ]), + ].toSorted((left, right) => left.localeCompare(right)); + }; + const hasStalePluginEvidenceForUnknownWebSearchProvider = (providerId: string): boolean => { const normalizedProviderId = normalizePluginId(providerId); if (!normalizedProviderId || ensureKnownIds().has(normalizedProviderId)) { @@ -1034,8 +1048,23 @@ function validateConfigObjectWithPluginsBase( issues.push({ path, message: "web_search provider must not be empty" }); return; } + const activeProviderIds = collectActiveWebSearchProviderIds(); + if (activeProviderIds.includes(trimmed)) { + return; + } + const installCatalogEntry = resolveWebSearchInstallCatalogEntries().find( + (entry) => entry.provider.id === trimmed, + ); + if (installCatalogEntry) { + issues.push({ + path, + message: `web_search provider is not available: ${trimmed} (install or enable plugin "${installCatalogEntry.pluginId}", then run openclaw doctor --fix)`, + allowedValues: collectKnownWebSearchProviderIds(), + }); + return; + } const allowedValues = collectKnownWebSearchProviderIds(); - if (allowedValues.length === 0 || allowedValues.includes(trimmed)) { + if (allowedValues.length === 0) { return; } const issue = { @@ -1053,6 +1082,119 @@ function validateConfigObjectWithPluginsBase( issues.push(issue); }; + const collectConfiguredModelRefs = (): ConfiguredModelRef[] => { + const refs: ConfiguredModelRef[] = []; + const pushModelRef = (path: string, value: unknown) => { + if (typeof value === "string" && value.trim()) { + refs.push({ path, value: value.trim() }); + } + }; + const collectModelConfig = (path: string, value: unknown) => { + if (typeof value === "string") { + pushModelRef(path, value); + return; + } + if (!isRecord(value)) { + return; + } + pushModelRef(`${path}.primary`, value.primary); + if (Array.isArray(value.fallbacks)) { + for (const [index, entry] of value.fallbacks.entries()) { + pushModelRef(`${path}.fallbacks.${index}`, entry); + } + } + }; + const collectFromAgent = (path: string, agent: unknown) => { + if (!isRecord(agent)) { + return; + } + for (const key of [ + "model", + "imageModel", + "imageGenerationModel", + "videoGenerationModel", + "musicGenerationModel", + "pdfModel", + ]) { + collectModelConfig(`${path}.${key}`, agent[key]); + } + if (isRecord(agent.models)) { + for (const modelRef of Object.keys(agent.models)) { + pushModelRef(`${path}.models.${modelRef}`, modelRef); + } + } + }; + + collectFromAgent("agents.defaults", config.agents?.defaults); + if (Array.isArray(config.agents?.list)) { + for (const [index, entry] of config.agents.list.entries()) { + collectFromAgent(`agents.list.${index}`, entry); + } + } + return refs; + }; + + const parseProviderModelRef = (value: string): { provider: string; model: string } | null => { + const slashIndex = value.indexOf("/"); + if (slashIndex <= 0 || slashIndex >= value.length - 1) { + return null; + } + const provider = normalizeLowercaseStringOrEmpty(value.slice(0, slashIndex)); + const model = normalizeLowercaseStringOrEmpty(value.slice(slashIndex + 1)); + return provider && model ? { provider, model } : null; + }; + + const validateConfiguredModelRefs = () => { + const configuredRefs = collectConfiguredModelRefs(); + if (configuredRefs.length === 0) { + return; + } + const { registry } = ensureRegistry(); + const suppressedModels = new Map< + string, + { provider: string; model: string; reason?: string } + >(); + for (const suppression of planManifestModelCatalogSuppressions({ registry }).suppressions) { + if (suppression.when) { + continue; + } + const key = `${suppression.provider}/${suppression.model}`; + if (!suppressedModels.has(key)) { + suppressedModels.set(key, { + provider: suppression.provider, + model: suppression.model, + ...(suppression.reason ? { reason: suppression.reason } : {}), + }); + } + } + if (suppressedModels.size === 0) { + return; + } + const seen = new Set(); + for (const ref of configuredRefs) { + const parsed = parseProviderModelRef(ref.value); + if (!parsed) { + continue; + } + const suppression = suppressedModels.get(`${parsed.provider}/${parsed.model}`); + if (!suppression) { + continue; + } + const issueKey = `${ref.path}\0${parsed.provider}/${parsed.model}`; + if (seen.has(issueKey)) { + continue; + } + seen.add(issueKey); + const modelRef = `${suppression.provider}/${suppression.model}`; + issues.push({ + path: ref.path, + message: suppression.reason + ? `Unknown model: ${modelRef}. ${suppression.reason}` + : `Unknown model: ${modelRef}.`, + }); + } + }; + const replaceChannelConfig = (channelId: string, nextValue: unknown) => { if (!channelsCloned) { mutatedConfig = { @@ -1200,6 +1342,7 @@ function validateConfigObjectWithPluginsBase( } validateWebSearchProvider(); + validateConfiguredModelRefs(); if (!hasExplicitPluginsConfig) { if (issues.length > 0) {