From 639cd50261d44fbf2fc98e4d64a24c34f8b0e9a2 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sat, 25 Apr 2026 22:31:52 -0700 Subject: [PATCH] fix(models): preserve provider index catalog fallback (#71985) * fix(models): preserve provider index catalog fallback * fix(models): mark provider index rows as previews --- CHANGELOG.md | 1 + .../list.list-command.forward-compat.test.ts | 17 +++++ src/commands/models/list.list-command.ts | 16 ++++- .../models/list.provider-index-catalog.ts | 19 +++++ src/commands/models/list.row-sources.ts | 20 +++++- src/commands/models/list.rows.ts | 22 ++++-- src/model-catalog/authority.test.ts | 53 ++++++++++++++ src/model-catalog/authority.ts | 31 +++++++++ src/model-catalog/index.ts | 9 +++ .../provider-index-planner.test.ts | 56 +++++++++++++++ src/model-catalog/provider-index-planner.ts | 69 +++++++++++++++++++ .../provider-index/openclaw-provider-index.ts | 3 + 12 files changed, 308 insertions(+), 8 deletions(-) create mode 100644 src/commands/models/list.provider-index-catalog.ts create mode 100644 src/model-catalog/authority.test.ts create mode 100644 src/model-catalog/authority.ts create mode 100644 src/model-catalog/provider-index-planner.test.ts create mode 100644 src/model-catalog/provider-index-planner.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 96027a254c1..abd16fc7de0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,7 @@ Docs: https://docs.openclaw.ai - Plugins/registry: keep installed plugin index records focused on install/state/load paths and resolve plugin capabilities from manifests scoped to indexed plugins. Thanks @shakkernerd. - Plugins/registry: route cold manifest and capability lookups through the installed plugin index so setup, channels, config, secrets, doctor, and provider metadata paths avoid broad plugin-root scans before runtime execution. Thanks @shakkernerd. - CLI/models: speed up `models list --all --provider ` for static manifest-backed providers by loading catalog rows through the installed plugin index instead of broad manifest scans or runtime suppression hooks. Thanks @shakkernerd. +- CLI/models: use OpenClaw Provider Index preview rows as the final cold fallback for installable providers, while keeping user config, installed manifests, and refreshed cache rows above provider-index metadata. Thanks @vincentkoc. - Plugins/chat commands: refresh the persisted plugin registry after `/plugins enable` and `/plugins disable`, matching the CLI mutation path. Thanks @vincentkoc. - Plugins/compat: mark `OPENCLAW_DISABLE_PERSISTED_PLUGIN_REGISTRY` as a deprecated break-glass switch and point operators at registry repair instead. Thanks @vincentkoc. - Plugins/registry: ignore stale persisted registry reads when plugin policy no longer matches current config, and stamp generated registry files with a do-not-edit warning. Thanks @vincentkoc. diff --git a/src/commands/models/list.list-command.forward-compat.test.ts b/src/commands/models/list.list-command.forward-compat.test.ts index 7947789d642..0e539e0689c 100644 --- a/src/commands/models/list.list-command.forward-compat.test.ts +++ b/src/commands/models/list.list-command.forward-compat.test.ts @@ -507,6 +507,23 @@ describe("modelsListCommand forward-compat", () => { ]); }); + it("uses provider index preview rows when an installable provider is not installed", async () => { + mocks.resolveConfiguredEntries.mockReturnValueOnce({ entries: [] }); + mocks.hasProviderStaticCatalogForFilter.mockResolvedValueOnce(false); + const runtime = createRuntime(); + + await modelsListCommand({ all: true, provider: "moonshot", json: true }, runtime as never); + + expect(mocks.loadModelRegistry).not.toHaveBeenCalled(); + expect(mocks.loadProviderCatalogModelsForList).not.toHaveBeenCalled(); + expect(lastPrintedRows<{ key: string; available: boolean }>()).toEqual([ + expect.objectContaining({ + key: "moonshot/kimi-k2.6", + available: false, + }), + ]); + }); + it("falls back to registry-backed rows when the fast-path catalog is empty", async () => { mocks.resolveConfiguredEntries.mockReturnValueOnce({ entries: [] }); mocks.hasProviderStaticCatalogForFilter.mockResolvedValueOnce(true); diff --git a/src/commands/models/list.list-command.ts b/src/commands/models/list.list-command.ts index 0fff709b079..07dab4c2a7c 100644 --- a/src/commands/models/list.list-command.ts +++ b/src/commands/models/list.list-command.ts @@ -63,6 +63,7 @@ export async function modelsListCommand( const { entries } = resolveConfiguredEntries(cfg); const configuredByKey = new Map(entries.map((entry) => [entry.key, entry])); let manifestCatalogRows: readonly NormalizedModelCatalogRow[] = []; + let providerIndexCatalogRows: readonly NormalizedModelCatalogRow[] = []; if (opts.all && providerFilter) { const { loadStaticManifestCatalogRowsForList } = await import("./list.manifest-catalog.js"); manifestCatalogRows = loadStaticManifestCatalogRowsForList({ cfg, providerFilter }); @@ -72,11 +73,18 @@ export async function modelsListCommand( !useManifestCatalogFastPath && opts.all && providerFilter ? await hasProviderStaticCatalogForFilter({ cfg, providerFilter }) : false; + if (!useManifestCatalogFastPath && !useProviderCatalogFastPath && opts.all && providerFilter) { + const { loadProviderIndexCatalogRowsForList } = + await import("./list.provider-index-catalog.js"); + providerIndexCatalogRows = loadProviderIndexCatalogRowsForList({ providerFilter }); + } + const useProviderIndexCatalogFastPath = providerIndexCatalogRows.length > 0; const shouldLoadRegistry = modelRowSourcesRequireRegistry({ all: opts.all, providerFilter, useManifestCatalogFastPath, useProviderCatalogFastPath, + useProviderIndexCatalogFastPath, }); const loadRegistryState = async () => { const loaded = await loadListModelRegistry(cfg, { providerFilter }); @@ -115,14 +123,18 @@ export async function modelsListCommand( const rows: ModelRow[] = []; if (opts.all) { - let rowContext = buildRowContext(useManifestCatalogFastPath || useProviderCatalogFastPath); + let rowContext = buildRowContext( + useManifestCatalogFastPath || useProviderCatalogFastPath || useProviderIndexCatalogFastPath, + ); const initialAppend = await appendAllModelRowSources({ rows, context: rowContext, modelRegistry, manifestCatalogRows, + providerIndexCatalogRows, useManifestCatalogFastPath, useProviderCatalogFastPath, + useProviderIndexCatalogFastPath, }); if (initialAppend.requiresRegistryFallback) { try { @@ -139,8 +151,10 @@ export async function modelsListCommand( context: rowContext, modelRegistry, manifestCatalogRows: [], + providerIndexCatalogRows: [], useManifestCatalogFastPath: false, useProviderCatalogFastPath: false, + useProviderIndexCatalogFastPath: false, }); } } else { diff --git a/src/commands/models/list.provider-index-catalog.ts b/src/commands/models/list.provider-index-catalog.ts new file mode 100644 index 00000000000..5371fe64027 --- /dev/null +++ b/src/commands/models/list.provider-index-catalog.ts @@ -0,0 +1,19 @@ +import { + loadOpenClawProviderIndex, + normalizeModelCatalogProviderId, + planProviderIndexModelCatalogRows, +} from "../../model-catalog/index.js"; +import type { NormalizedModelCatalogRow } from "../../model-catalog/index.js"; + +export function loadProviderIndexCatalogRowsForList(params: { + providerFilter: string; +}): readonly NormalizedModelCatalogRow[] { + const providerFilter = normalizeModelCatalogProviderId(params.providerFilter); + if (!providerFilter) { + return []; + } + return planProviderIndexModelCatalogRows({ + index: loadOpenClawProviderIndex(), + providerFilter, + }).rows; +} diff --git a/src/commands/models/list.row-sources.ts b/src/commands/models/list.row-sources.ts index fbaaadf625e..bb868a5706d 100644 --- a/src/commands/models/list.row-sources.ts +++ b/src/commands/models/list.row-sources.ts @@ -6,6 +6,7 @@ import { appendConfiguredRows, appendDiscoveredRows, appendManifestCatalogRows, + appendModelCatalogRows, appendProviderCatalogRows, type RowBuilderContext, } from "./list.rows.js"; @@ -16,8 +17,10 @@ type AllModelRowSources = { context: RowBuilderContext; modelRegistry?: ModelRegistry; manifestCatalogRows?: readonly NormalizedModelCatalogRow[]; + providerIndexCatalogRows?: readonly NormalizedModelCatalogRow[]; useManifestCatalogFastPath: boolean; useProviderCatalogFastPath: boolean; + useProviderIndexCatalogFastPath: boolean; }; type AppendAllModelRowSourcesResult = { @@ -29,13 +32,16 @@ export function modelRowSourcesRequireRegistry(params: { providerFilter?: string; useManifestCatalogFastPath: boolean; useProviderCatalogFastPath: boolean; + useProviderIndexCatalogFastPath: boolean; }): boolean { if (!params.all) { return false; } if ( params.providerFilter && - (params.useManifestCatalogFastPath || params.useProviderCatalogFastPath) + (params.useManifestCatalogFastPath || + params.useProviderCatalogFastPath || + params.useProviderIndexCatalogFastPath) ) { return false; } @@ -47,7 +53,9 @@ export async function appendAllModelRowSources( ): Promise { if ( params.context.filter.provider && - (params.useManifestCatalogFastPath || params.useProviderCatalogFastPath) + (params.useManifestCatalogFastPath || + params.useProviderCatalogFastPath || + params.useProviderIndexCatalogFastPath) ) { let seenKeys = new Set(); appendConfiguredProviderRows({ @@ -72,6 +80,14 @@ export async function appendAllModelRowSources( staticOnly: true, }); } + if (catalogRows === 0 && params.useProviderIndexCatalogFastPath) { + catalogRows = appendModelCatalogRows({ + rows: params.rows, + context: params.context, + seenKeys, + catalogRows: params.providerIndexCatalogRows ?? [], + }); + } if (catalogRows === 0) { if (!params.modelRegistry) { return { requiresRegistryFallback: true }; diff --git a/src/commands/models/list.rows.ts b/src/commands/models/list.rows.ts index af38385c1c5..18d6a0097a5 100644 --- a/src/commands/models/list.rows.ts +++ b/src/commands/models/list.rows.ts @@ -225,19 +225,19 @@ export function appendConfiguredProviderRows(params: { } } -export function appendManifestCatalogRows(params: { +export function appendModelCatalogRows(params: { rows: ModelRow[]; context: RowBuilderContext; seenKeys: Set; - manifestRows: readonly NormalizedModelCatalogRow[]; + catalogRows: readonly NormalizedModelCatalogRow[]; }): number { let appended = 0; - for (const manifestRow of params.manifestRows) { - const key = modelKey(manifestRow.provider, manifestRow.id); + for (const catalogRow of params.catalogRows) { + const key = modelKey(catalogRow.provider, catalogRow.id); if ( appendVisibleRow({ rows: params.rows, - model: toManifestCatalogListModel(manifestRow), + model: toManifestCatalogListModel(catalogRow), key, context: params.context, seenKeys: params.seenKeys, @@ -250,6 +250,18 @@ export function appendManifestCatalogRows(params: { return appended; } +export function appendManifestCatalogRows(params: { + rows: ModelRow[]; + context: RowBuilderContext; + seenKeys: Set; + manifestRows: readonly NormalizedModelCatalogRow[]; +}): number { + return appendModelCatalogRows({ + ...params, + catalogRows: params.manifestRows, + }); +} + export async function appendCatalogSupplementRows(params: { rows: ModelRow[]; modelRegistry: ModelRegistry; diff --git a/src/model-catalog/authority.test.ts b/src/model-catalog/authority.test.ts new file mode 100644 index 00000000000..019ea45b69c --- /dev/null +++ b/src/model-catalog/authority.test.ts @@ -0,0 +1,53 @@ +import { describe, expect, it } from "vitest"; +import { mergeModelCatalogRowsByAuthority } from "./index.js"; +import type { ModelCatalogSource, NormalizedModelCatalogRow } from "./index.js"; + +function row(source: ModelCatalogSource, name: string): NormalizedModelCatalogRow { + return { + provider: "moonshot", + id: "kimi-k2.6", + ref: "moonshot/kimi-k2.6", + mergeKey: "moonshot::kimi-k2.6", + name, + source, + input: ["text"], + reasoning: false, + status: source === "provider-index" ? "preview" : "available", + }; +} + +describe("model catalog authority", () => { + it("keeps user config above manifest, cache, and provider-index preview rows", () => { + expect( + mergeModelCatalogRowsByAuthority([ + row("provider-index", "Preview"), + row("cache", "Cached"), + row("manifest", "Manifest"), + row("config", "Configured"), + ]), + ).toEqual([expect.objectContaining({ name: "Configured", source: "config" })]); + }); + + it("keeps installed manifest rows above cache and provider-index preview rows", () => { + expect( + mergeModelCatalogRowsByAuthority([ + row("provider-index", "Preview"), + row("runtime-refresh", "Refreshed"), + row("cache", "Cached"), + row("manifest", "Manifest"), + ]), + ).toEqual([expect.objectContaining({ name: "Manifest", source: "manifest" })]); + }); + + it("uses cache rows above provider-index preview rows", () => { + expect( + mergeModelCatalogRowsByAuthority([row("provider-index", "Preview"), row("cache", "Cached")]), + ).toEqual([expect.objectContaining({ name: "Cached", source: "cache" })]); + }); + + it("uses provider-index preview rows when no higher-authority row exists", () => { + expect(mergeModelCatalogRowsByAuthority([row("provider-index", "Preview")])).toEqual([ + expect.objectContaining({ name: "Preview", source: "provider-index" }), + ]); + }); +}); diff --git a/src/model-catalog/authority.ts b/src/model-catalog/authority.ts new file mode 100644 index 00000000000..376fb22441d --- /dev/null +++ b/src/model-catalog/authority.ts @@ -0,0 +1,31 @@ +import type { ModelCatalogSource, NormalizedModelCatalogRow } from "./types.js"; + +const MODEL_CATALOG_SOURCE_AUTHORITY: Readonly> = { + config: 0, + manifest: 1, + cache: 2, + "runtime-refresh": 2, + "provider-index": 3, +}; + +export function compareModelCatalogSourceAuthority( + left: ModelCatalogSource, + right: ModelCatalogSource, +): number { + return MODEL_CATALOG_SOURCE_AUTHORITY[left] - MODEL_CATALOG_SOURCE_AUTHORITY[right]; +} + +export function mergeModelCatalogRowsByAuthority( + rows: Iterable, +): NormalizedModelCatalogRow[] { + const byMergeKey = new Map(); + for (const row of rows) { + const existing = byMergeKey.get(row.mergeKey); + if (!existing || compareModelCatalogSourceAuthority(row.source, existing.source) < 0) { + byMergeKey.set(row.mergeKey, row); + } + } + return [...byMergeKey.values()].toSorted( + (left, right) => left.provider.localeCompare(right.provider) || left.id.localeCompare(right.id), + ); +} diff --git a/src/model-catalog/index.ts b/src/model-catalog/index.ts index 5261f03ebaf..49bc6d58b96 100644 --- a/src/model-catalog/index.ts +++ b/src/model-catalog/index.ts @@ -1,3 +1,7 @@ +export { + compareModelCatalogSourceAuthority, + mergeModelCatalogRowsByAuthority, +} from "./authority.js"; export { buildModelCatalogMergeKey, buildModelCatalogRef, @@ -13,6 +17,11 @@ export { normalizeOpenClawProviderIndex, } from "./provider-index/index.js"; export { planManifestModelCatalogRows } from "./manifest-planner.js"; +export { planProviderIndexModelCatalogRows } from "./provider-index-planner.js"; +export type { + ProviderIndexModelCatalogPlan, + ProviderIndexModelCatalogPlanEntry, +} from "./provider-index-planner.js"; export type { ManifestModelCatalogConflict, ManifestModelCatalogPlan, diff --git a/src/model-catalog/provider-index-planner.test.ts b/src/model-catalog/provider-index-planner.test.ts new file mode 100644 index 00000000000..786f5c41d2d --- /dev/null +++ b/src/model-catalog/provider-index-planner.test.ts @@ -0,0 +1,56 @@ +import { describe, expect, it } from "vitest"; +import { planProviderIndexModelCatalogRows } from "./index.js"; + +describe("provider index model catalog planner", () => { + it("builds preview rows from installable provider metadata", () => { + const plan = planProviderIndexModelCatalogRows({ + providerFilter: "Moonshot", + index: { + version: 1, + providers: { + moonshot: { + id: "moonshot", + name: "Moonshot AI", + plugin: { + id: "moonshot", + package: "@openclaw/plugin-moonshot", + }, + previewCatalog: { + models: [{ id: "kimi-k2.6", name: "Kimi K2.6", contextWindow: 262144 }], + }, + }, + deepseek: { + id: "deepseek", + name: "DeepSeek", + plugin: { id: "deepseek" }, + previewCatalog: { + models: [{ id: "deepseek-chat" }], + }, + }, + }, + }, + }); + + expect(plan.entries).toEqual([ + { + provider: "moonshot", + pluginId: "moonshot", + rows: [ + { + provider: "moonshot", + id: "kimi-k2.6", + ref: "moonshot/kimi-k2.6", + mergeKey: "moonshot::kimi-k2.6", + name: "Kimi K2.6", + source: "provider-index", + input: ["text"], + reasoning: false, + status: "preview", + contextWindow: 262144, + }, + ], + }, + ]); + expect(plan.rows.map((row) => row.ref)).toEqual(["moonshot/kimi-k2.6"]); + }); +}); diff --git a/src/model-catalog/provider-index-planner.ts b/src/model-catalog/provider-index-planner.ts new file mode 100644 index 00000000000..4998f7cb10c --- /dev/null +++ b/src/model-catalog/provider-index-planner.ts @@ -0,0 +1,69 @@ +import { normalizeModelCatalogProviderRows } from "./normalize.js"; +import type { OpenClawProviderIndex } from "./provider-index/index.js"; +import { normalizeModelCatalogProviderId } from "./refs.js"; +import type { ModelCatalogProvider, NormalizedModelCatalogRow } from "./types.js"; + +export type ProviderIndexModelCatalogPlanEntry = { + provider: string; + pluginId: string; + rows: readonly NormalizedModelCatalogRow[]; +}; + +export type ProviderIndexModelCatalogPlan = { + rows: readonly NormalizedModelCatalogRow[]; + entries: readonly ProviderIndexModelCatalogPlanEntry[]; +}; + +function withPreviewStatusDefaults(providerCatalog: ModelCatalogProvider): ModelCatalogProvider { + return { + ...providerCatalog, + models: providerCatalog.models.map((model) => ({ + ...model, + status: model.status ?? "preview", + })), + }; +} + +export function planProviderIndexModelCatalogRows(params: { + index: OpenClawProviderIndex; + providerFilter?: string; +}): ProviderIndexModelCatalogPlan { + const providerFilter = params.providerFilter + ? normalizeModelCatalogProviderId(params.providerFilter) + : undefined; + const entries: ProviderIndexModelCatalogPlanEntry[] = []; + + for (const [providerId, provider] of Object.entries(params.index.providers)) { + const normalizedProvider = normalizeModelCatalogProviderId(providerId); + if ( + !normalizedProvider || + (providerFilter && normalizedProvider !== providerFilter) || + !provider.previewCatalog + ) { + continue; + } + const rows = normalizeModelCatalogProviderRows({ + provider: normalizedProvider, + providerCatalog: withPreviewStatusDefaults(provider.previewCatalog), + source: "provider-index", + }); + if (rows.length === 0) { + continue; + } + entries.push({ + provider: normalizedProvider, + pluginId: provider.plugin.id, + rows, + }); + } + + return { + entries, + rows: entries + .flatMap((entry) => entry.rows) + .toSorted( + (left, right) => + left.provider.localeCompare(right.provider) || left.id.localeCompare(right.id), + ), + }; +} diff --git a/src/model-catalog/provider-index/openclaw-provider-index.ts b/src/model-catalog/provider-index/openclaw-provider-index.ts index 493f451869b..359b6aced58 100644 --- a/src/model-catalog/provider-index/openclaw-provider-index.ts +++ b/src/model-catalog/provider-index/openclaw-provider-index.ts @@ -6,6 +6,9 @@ import type { OpenClawProviderIndex } from "./types.js"; // Preview catalogs use the shared model catalog type, but intentionally keep to // stable display fields unless runtime adapter metadata is kept in sync with // the installed plugin manifest. +// When a bundled provider moves to an external package, keep its provider id +// here and add plugin package metadata so pre-install surfaces do not disappear +// before the user installs the new package. export const OPENCLAW_PROVIDER_INDEX = { version: 1, providers: {