refactor: centralize plugin model discovery

This commit is contained in:
Peter Steinberger
2026-05-29 09:21:30 +01:00
parent 189a7962b2
commit b78ebacb18
8 changed files with 164 additions and 143 deletions

View File

@@ -1,6 +1,6 @@
import path from "node:path";
import type { OpenClawConfig } from "../config/types.openclaw.js";
import type { Model } from "../llm/types.js";
import type { PluginMetadataSnapshot } from "../plugins/plugin-metadata-snapshot.types.js";
import { normalizeModelCompat } from "../plugins/provider-model-compat.js";
import {
applyProviderResolvedTransportWithPlugin,
@@ -12,6 +12,8 @@ import {
scrubLegacyStaticAuthJsonEntriesForDiscovery,
type DiscoverAuthStorageOptions,
} from "./agent-auth-discovery.js";
import { resolveModelPluginMetadataSnapshot } from "./model-discovery-context.js";
import type { PluginModelCatalogMetadataSnapshot } from "./plugin-model-catalog.js";
import { normalizeProviderId } from "./provider-id.js";
import {
AuthStorage,
@@ -31,8 +33,9 @@ type DiscoveredProviderRuntimeModelLike = Omit<ProviderRuntimeModelLike, "api">
};
type DiscoverModelsOptions = {
config?: OpenClawConfig;
providerFilter?: string;
pluginMetadataSnapshot?: Pick<PluginMetadataSnapshot, "owners">;
pluginMetadataSnapshot?: PluginModelCatalogMetadataSnapshot;
workspaceDir?: string;
normalizeModels?: boolean;
};
@@ -87,12 +90,17 @@ function createOpenClawModelRegistry(
agentDir: string,
options?: DiscoverModelsOptions,
): AgentModelRegistry {
const registry = ModelRegistry.create(authStorage, modelsJsonPath, {
const pluginMetadataSnapshot = resolveModelPluginMetadataSnapshot({
...(options?.config ? { config: options.config } : {}),
...(options?.pluginMetadataSnapshot
? { pluginMetadataSnapshot: options.pluginMetadataSnapshot }
: {}),
...(options?.workspaceDir ? { workspaceDir: options.workspaceDir } : {}),
allowWorkspaceScopedCurrent: options?.workspaceDir === undefined,
useRuntimeConfig: options?.config === undefined,
});
const registryOptions = pluginMetadataSnapshot ? { pluginMetadataSnapshot } : {};
const registry = ModelRegistry.create(authStorage, modelsJsonPath, registryOptions);
const getAll = registry.getAll.bind(registry);
const getAvailable = registry.getAvailable.bind(registry);
const find = registry.find.bind(registry);

View File

@@ -1,8 +1,6 @@
import { statSync } from "node:fs";
import path from "node:path";
import type { OpenClawConfig } from "../../config/types.openclaw.js";
import { getCurrentPluginMetadataSnapshot } from "../../plugins/current-plugin-metadata-snapshot.js";
import { resolvePluginMetadataSnapshot } from "../../plugins/plugin-metadata-snapshot.js";
import type { PluginMetadataSnapshot } from "../../plugins/plugin-metadata-snapshot.types.js";
import {
resolveRuntimeExternalAuthProviderRefs,
@@ -11,7 +9,8 @@ import {
import { discoverAuthStorage, discoverModels } from "../agent-model-discovery.js";
import { resolveDefaultAgentDir } from "../agent-scope.js";
import { hasAnyRuntimeAuthProfileStoreSource } from "../auth-profiles/runtime-snapshots.js";
import { listPluginModelCatalogPaths } from "../plugin-model-catalog.js";
import { resolveModelPluginMetadataSnapshot } from "../model-discovery-context.js";
import { listPluginModelCatalogFiles } from "../plugin-model-catalog.js";
import type { AuthStorage, ModelRegistry } from "../sessions/index.js";
type DiscoveryStores = {
@@ -57,9 +56,9 @@ function authFingerprint(agentDir: string): object {
function pluginModelCatalogFingerprint(
agentDir: string,
): Array<[string, ReturnType<typeof fileFingerprint>]> {
return listPluginModelCatalogPaths(agentDir).map((catalogPath) => [
path.relative(agentDir, catalogPath),
fileFingerprint(catalogPath),
return listPluginModelCatalogFiles(agentDir).map((catalogFile) => [
catalogFile.relativePath,
fileFingerprint(catalogFile.path),
]);
}
@@ -107,23 +106,11 @@ function pruneDiscoveryStoreCache(): void {
function resolvePluginMetadataSnapshotForDiscovery(
options: DiscoverCachedAgentStoresOptions,
): PluginMetadataSnapshot | undefined {
try {
return (
getCurrentPluginMetadataSnapshot({
allowWorkspaceScopedSnapshot: true,
config: options.config,
env: process.env,
...(options.workspaceDir ? { workspaceDir: options.workspaceDir } : {}),
}) ??
resolvePluginMetadataSnapshot({
config: options.config ?? {},
env: process.env,
...(options.workspaceDir ? { workspaceDir: options.workspaceDir } : {}),
})
);
} catch {
return undefined;
}
return resolveModelPluginMetadataSnapshot({
...(options.config ? { config: options.config } : {}),
...(options.workspaceDir ? { workspaceDir: options.workspaceDir } : {}),
useRuntimeConfig: options.config === undefined,
}) as PluginMetadataSnapshot | undefined;
}
function pluginMetadataFingerprint(snapshot: PluginMetadataSnapshot | undefined): object {
@@ -136,11 +123,12 @@ function pluginMetadataFingerprint(snapshot: PluginMetadataSnapshot | undefined)
function discoverFreshAgentStores(
agentDir: string,
options: Pick<DiscoverCachedAgentStoresOptions, "workspaceDir">,
options: Pick<DiscoverCachedAgentStoresOptions, "config" | "workspaceDir">,
pluginMetadataSnapshot: PluginMetadataSnapshot | undefined,
): DiscoveryStores {
const authStorage = discoverAuthStorage(agentDir);
const modelRegistry = discoverModels(authStorage, agentDir, {
...(options.config ? { config: options.config } : {}),
...(pluginMetadataSnapshot ? { pluginMetadataSnapshot } : {}),
...(options.workspaceDir ? { workspaceDir: options.workspaceDir } : {}),
});

View File

@@ -13,13 +13,10 @@ import {
shouldPreferProviderRuntimeResolvedModel,
} from "../../plugins/provider-runtime.js";
import { discoverAuthStorage, discoverModels } from "../agent-model-discovery.js";
import {
resolveAgentWorkspaceDir,
resolveDefaultAgentDir,
resolveDefaultAgentId,
} from "../agent-scope.js";
import { resolveDefaultAgentDir } from "../agent-scope.js";
import { DEFAULT_CONTEXT_TOKENS } from "../defaults.js";
import { buildModelAliasLines } from "../model-alias-lines.js";
import { resolveModelWorkspaceDir } from "../model-discovery-context.js";
import { modelKey, normalizeStaticProviderModelId } from "../model-ref-shared.js";
import { findNormalizedProviderValue, normalizeProviderId } from "../model-selection.js";
import {
@@ -153,16 +150,6 @@ function discoverCachedAgentStoresForAgent(
});
}
function resolveModelWorkspaceDir(
cfg: OpenClawConfig | undefined,
explicitWorkspaceDir: string | undefined,
): string | undefined {
if (explicitWorkspaceDir !== undefined || !cfg) {
return explicitWorkspaceDir;
}
return resolveAgentWorkspaceDir(cfg, resolveDefaultAgentId(cfg));
}
function canonicalizeLegacyResolvedModel(params: { provider: string; model: Model }): Model {
if (
normalizeProviderId(params.provider) !== "openai-codex" ||

View File

@@ -1,5 +1,5 @@
import { readFile } from "node:fs/promises";
import { join, relative } from "node:path";
import { join } from "node:path";
import { getRuntimeConfig } from "../config/config.js";
import type { OpenClawConfig } from "../config/types.openclaw.js";
import { createSubsystemLogger } from "../logging/subsystem.js";
@@ -17,14 +17,11 @@ import {
normalizeLowercaseStringOrEmpty,
normalizeOptionalString,
} from "../shared/string-coerce.js";
import {
resolveAgentWorkspaceDir,
resolveDefaultAgentDir,
resolveDefaultAgentId,
} from "./agent-scope.js";
import { resolveDefaultAgentDir } from "./agent-scope.js";
import { ensureAuthProfileStoreWithoutExternalProfiles } from "./auth-profiles.js";
import { modelSupportsInput as modelCatalogEntrySupportsInput } from "./model-catalog-lookup.js";
import type { ModelCatalogEntry, ModelInputType } from "./model-catalog.types.js";
import { resolveModelWorkspaceDir } from "./model-discovery-context.js";
import {
modelKey,
normalizeConfiguredProviderCatalogModelId,
@@ -36,10 +33,9 @@ import {
} from "./model-selection-shared.js";
import { ensureOpenClawModelsJson } from "./models-config.js";
import {
decodePluginModelCatalogRelativePathPluginId,
isGeneratedPluginModelCatalog,
listPluginModelCatalogPaths,
resolvePluginModelCatalogOwnerPluginId,
filterGeneratedPluginModelCatalogProviders,
listPluginModelCatalogFiles,
type PluginModelCatalogMetadataSnapshot,
} from "./plugin-model-catalog.js";
import { normalizeProviderId } from "./provider-id.js";
@@ -328,18 +324,12 @@ function readProviderCatalogRows(parsed: unknown): Record<string, Record<string,
async function loadReadOnlyPersistedProviderRows(
agentDir: string,
getPluginMetadataSnapshot: () => Pick<PluginMetadataSnapshot, "owners">,
getPluginMetadataSnapshot: () => PluginModelCatalogMetadataSnapshot,
): Promise<Record<string, Record<string, unknown>>> {
const raw = await readFile(join(agentDir, "models.json"), "utf8");
const providers = { ...readProviderCatalogRows(JSON.parse(raw) as unknown) };
for (const catalogPath of listPluginModelCatalogPaths(agentDir)) {
const catalogPluginId = decodePluginModelCatalogRelativePathPluginId(
relative(agentDir, catalogPath),
);
if (!catalogPluginId) {
continue;
}
const catalogRaw = await readFile(catalogPath, "utf8").catch(() => undefined);
for (const catalogFile of listPluginModelCatalogFiles(agentDir)) {
const catalogRaw = await readFile(catalogFile.path, "utf8").catch(() => undefined);
if (!catalogRaw) {
continue;
}
@@ -349,17 +339,15 @@ async function loadReadOnlyPersistedProviderRows(
} catch {
continue;
}
if (isGeneratedPluginModelCatalog(parsed)) {
for (const [providerId, provider] of Object.entries(readProviderCatalogRows(parsed))) {
const ownerPluginId = resolvePluginModelCatalogOwnerPluginId({
providerId,
pluginMetadataSnapshot: getPluginMetadataSnapshot(),
});
if (ownerPluginId === catalogPluginId) {
providers[providerId] = provider;
}
}
}
Object.assign(
providers,
filterGeneratedPluginModelCatalogProviders({
catalogPluginId: catalogFile.pluginId,
parsedCatalog: parsed,
pluginMetadataSnapshot: getPluginMetadataSnapshot(),
providers: readProviderCatalogRows(parsed),
}),
);
}
return providers;
}
@@ -370,7 +358,7 @@ async function loadReadOnlyPersistedModelCatalog(params?: {
}): Promise<ModelCatalogEntry[]> {
const cfg = params?.config ?? getRuntimeConfig();
const agentDir = resolveDefaultAgentDir(cfg);
const workspaceDir = resolveAgentWorkspaceDir(cfg, resolveDefaultAgentId(cfg));
const workspaceDir = resolveModelWorkspaceDir(cfg, undefined);
const models: ModelCatalogEntry[] = [];
const { buildShouldSuppressBuiltInModel } = await loadModelSuppression();
const shouldSuppressBuiltInModel = buildShouldSuppressBuiltInModel({ config: cfg });
@@ -519,7 +507,7 @@ export async function loadModelCatalog(params?: {
const sortModels = sortModelCatalogEntries;
try {
const cfg = params?.config ?? getRuntimeConfig();
const workspaceDir = resolveAgentWorkspaceDir(cfg, resolveDefaultAgentId(cfg));
const workspaceDir = resolveModelWorkspaceDir(cfg, undefined);
let manifestMetadataSnapshot: PluginMetadataSnapshot | undefined;
let manifestPlugins: ProviderModelIdNormalizationOptions["manifestPlugins"];
const getManifestMetadataSnapshot = () => {

View File

@@ -0,0 +1,51 @@
import { getRuntimeConfig } from "../config/config.js";
import type { OpenClawConfig } from "../config/types.openclaw.js";
import { getCurrentPluginMetadataSnapshot } from "../plugins/current-plugin-metadata-snapshot.js";
import { resolvePluginMetadataSnapshot } from "../plugins/plugin-metadata-snapshot.js";
import { resolveAgentWorkspaceDir, resolveDefaultAgentId } from "./agent-scope.js";
import type { PluginModelCatalogMetadataSnapshot } from "./plugin-model-catalog.js";
export function resolveModelWorkspaceDir(
cfg: OpenClawConfig | undefined,
explicitWorkspaceDir: string | undefined,
): string | undefined {
if (explicitWorkspaceDir !== undefined || !cfg) {
return explicitWorkspaceDir;
}
return resolveAgentWorkspaceDir(cfg, resolveDefaultAgentId(cfg));
}
export function resolveModelPluginMetadataSnapshot(params: {
allowWorkspaceScopedCurrent?: boolean;
config?: OpenClawConfig;
env?: NodeJS.ProcessEnv;
pluginMetadataSnapshot?: PluginModelCatalogMetadataSnapshot;
useRuntimeConfig?: boolean;
workspaceDir?: string;
}): PluginModelCatalogMetadataSnapshot | undefined {
if (params.pluginMetadataSnapshot) {
return params.pluginMetadataSnapshot;
}
const env = params.env ?? process.env;
try {
const config = params.config ?? (params.useRuntimeConfig ? getRuntimeConfig() : undefined);
return (
getCurrentPluginMetadataSnapshot({
allowWorkspaceScopedSnapshot: true,
env,
...(config ? { config } : {}),
...(params.workspaceDir ? { workspaceDir: params.workspaceDir } : {}),
}) ??
resolvePluginMetadataSnapshot({
config: config ?? {},
env,
...(params.workspaceDir ? { workspaceDir: params.workspaceDir } : {}),
...(params.allowWorkspaceScopedCurrent !== undefined
? { allowWorkspaceScopedCurrent: params.allowWorkspaceScopedCurrent }
: {}),
})
);
} catch {
return undefined;
}
}

View File

@@ -1,7 +1,7 @@
import type { OpenClawConfig } from "../config/types.openclaw.js";
import { resolvePluginMetadataSnapshot } from "../plugins/plugin-metadata-snapshot.js";
import { discoverAuthStorage, discoverModels } from "./agent-model-discovery.js";
import { resolveDefaultAgentDir } from "./agent-scope.js";
import { resolveModelPluginMetadataSnapshot } from "./model-discovery-context.js";
import type { ModelRegistry } from "./sessions/index.js";
export type LoadAgentModelRegistryOptions = {
@@ -23,12 +23,13 @@ export function loadAgentModelRegistry(
config,
workspaceDir: options.workspaceDir,
});
const pluginMetadataSnapshot = resolveModelPluginMetadataSnapshot({
config,
workspaceDir: options.workspaceDir,
});
const registry = discoverModels(authStorage, agentDir, {
pluginMetadataSnapshot: resolvePluginMetadataSnapshot({
config,
env: process.env,
...(options.workspaceDir ? { workspaceDir: options.workspaceDir } : {}),
}),
config,
...(pluginMetadataSnapshot ? { pluginMetadataSnapshot } : {}),
providerFilter: options.providerFilter,
...(options.workspaceDir ? { workspaceDir: options.workspaceDir } : {}),
normalizeModels: options.normalizeModels,

View File

@@ -16,6 +16,12 @@ export type PluginModelCatalogMetadataSnapshot = Pick<PluginMetadataSnapshot, "o
normalizePluginId?: (pluginId: string) => string;
};
export type PluginModelCatalogFile = {
path: string;
pluginId: string;
relativePath: string;
};
export function encodePluginModelCatalogRelativePath(pluginId: string): string {
return `plugins/${encodeURIComponent(pluginId)}/${PLUGIN_MODEL_CATALOG_FILE}`;
}
@@ -62,10 +68,20 @@ export function listPluginModelCatalogRelativePaths(agentDir: string): string[]
.toSorted((left, right) => left.localeCompare(right));
}
export function listPluginModelCatalogPaths(agentDir: string): string[] {
export function listPluginModelCatalogFiles(agentDir: string): PluginModelCatalogFile[] {
return listPluginModelCatalogRelativePaths(agentDir)
.map((relativePath) => path.join(agentDir, relativePath))
.filter((catalogPath) => existsSync(catalogPath));
.map((relativePath) => {
const pluginId = decodePluginModelCatalogRelativePathPluginId(relativePath);
return pluginId
? {
path: path.join(agentDir, relativePath),
pluginId,
relativePath,
}
: undefined;
})
.filter((entry): entry is PluginModelCatalogFile => entry !== undefined)
.filter((entry) => existsSync(entry.path));
}
export function isGeneratedPluginModelCatalog(value: unknown): boolean {
@@ -106,3 +122,28 @@ export function resolvePluginModelCatalogOwnerPluginId(params: {
? normalizedPluginId
: undefined;
}
export function filterGeneratedPluginModelCatalogProviders<T>(params: {
catalogPluginId?: string;
parsedCatalog?: unknown;
pluginMetadataSnapshot?: PluginModelCatalogMetadataSnapshot;
providers: Record<string, T>;
}): Record<string, T> {
if (
!params.catalogPluginId ||
!params.pluginMetadataSnapshot ||
(params.parsedCatalog !== undefined && !isGeneratedPluginModelCatalog(params.parsedCatalog))
) {
return {};
}
return Object.fromEntries(
Object.entries(params.providers).filter(([providerId]) => {
return (
resolvePluginModelCatalogOwnerPluginId({
providerId,
pluginMetadataSnapshot: params.pluginMetadataSnapshot,
}) === params.catalogPluginId
);
}),
);
}

View File

@@ -3,11 +3,10 @@
*/
import { existsSync, readFileSync } from "node:fs";
import { dirname, join, relative } from "node:path";
import { dirname, join } from "node:path";
import { type Static, Type } from "typebox";
import { Compile } from "typebox/compile";
import type { TLocalizedValidationError } from "typebox/error";
import { getRuntimeConfig } from "../../config/config.js";
import { registerApiProvider } from "../../llm/api-registry.js";
import { resetApiProviders } from "../../llm/providers/register-builtins.js";
import {
@@ -22,15 +21,13 @@ import {
} from "../../llm/types.js";
import { registerOAuthProvider, resetOAuthProviders } from "../../llm/utils/oauth/index.js";
import type { OAuthProviderInterface } from "../../llm/utils/oauth/types.js";
import { getCurrentPluginMetadataSnapshot } from "../../plugins/current-plugin-metadata-snapshot.js";
import { loadPluginMetadataSnapshot } from "../../plugins/plugin-metadata-snapshot.js";
import { getAgentDir } from "../config.js";
import { resolveModelPluginMetadataSnapshot } from "../model-discovery-context.js";
import {
decodePluginModelCatalogRelativePathPluginId,
filterGeneratedPluginModelCatalogProviders,
isGeneratedPluginModelCatalog,
listPluginModelCatalogPaths,
listPluginModelCatalogFiles,
type PluginModelCatalogMetadataSnapshot,
resolvePluginModelCatalogOwnerPluginId,
} from "../plugin-model-catalog.js";
import type { AuthStatus, AuthStorage } from "./auth-storage.js";
import { BUILT_IN_PROVIDER_DISPLAY_NAMES } from "./provider-display-names.js";
@@ -256,49 +253,6 @@ type ModelRegistryOptions = {
workspaceDir?: string;
};
function resolvePluginMetadataSnapshotForModelRegistry(
options: Pick<ModelRegistryOptions, "workspaceDir"> = {},
): PluginModelCatalogMetadataSnapshot | undefined {
try {
const config = getRuntimeConfig();
return (
getCurrentPluginMetadataSnapshot({
allowWorkspaceScopedSnapshot: true,
config,
env: process.env,
...(options.workspaceDir ? { workspaceDir: options.workspaceDir } : {}),
}) ??
loadPluginMetadataSnapshot({
config,
env: process.env,
...(options.workspaceDir ? { workspaceDir: options.workspaceDir } : {}),
})
);
} catch {
return undefined;
}
}
function filterGeneratedPluginCatalogProviders(params: {
catalogPluginId?: string;
pluginMetadataSnapshot?: PluginModelCatalogMetadataSnapshot;
providers: ModelsConfig["providers"];
}): ModelsConfig["providers"] {
if (!params.catalogPluginId || !params.pluginMetadataSnapshot) {
return {};
}
return Object.fromEntries(
Object.entries(params.providers).filter(([providerId]) => {
return (
resolvePluginModelCatalogOwnerPluginId({
providerId,
pluginMetadataSnapshot: params.pluginMetadataSnapshot,
}) === params.catalogPluginId
);
}),
);
}
function mergeCompat(
baseCompat: Model["compat"],
overrideCompat: Model["compat"],
@@ -358,8 +312,14 @@ export class ModelRegistry {
) {
this.authStorage = authStorage;
this.modelsJsonPath = modelsJsonPath;
this.pluginMetadataSnapshot =
options.pluginMetadataSnapshot ?? resolvePluginMetadataSnapshotForModelRegistry(options);
this.pluginMetadataSnapshot = resolveModelPluginMetadataSnapshot({
...(options.pluginMetadataSnapshot
? { pluginMetadataSnapshot: options.pluginMetadataSnapshot }
: {}),
...(options.workspaceDir ? { workspaceDir: options.workspaceDir } : {}),
allowWorkspaceScopedCurrent: true,
useRuntimeConfig: true,
});
this.loadModels();
}
@@ -461,8 +421,9 @@ export class ModelRegistry {
const config = parsed;
const providers =
options.requireGeneratedCatalog === true
? filterGeneratedPluginCatalogProviders({
? filterGeneratedPluginModelCatalogProviders({
catalogPluginId: options.catalogPluginId,
parsedCatalog: parsed,
pluginMetadataSnapshot: this.pluginMetadataSnapshot,
providers: config.providers,
})
@@ -483,13 +444,9 @@ export class ModelRegistry {
const models = this.parseModels(configForUse);
if (options.includePluginCatalogs !== false) {
const agentDir = dirname(modelsJsonPath);
for (const pluginCatalogPath of listPluginModelCatalogPaths(dirname(modelsJsonPath))) {
const catalogPluginId = decodePluginModelCatalogRelativePathPluginId(
relative(agentDir, pluginCatalogPath),
);
const pluginResult = this.loadCustomModels(pluginCatalogPath, {
catalogPluginId,
for (const pluginCatalog of listPluginModelCatalogFiles(dirname(modelsJsonPath))) {
const pluginResult = this.loadCustomModels(pluginCatalog.path, {
catalogPluginId: pluginCatalog.pluginId,
includePluginCatalogs: false,
requireGeneratedCatalog: true,
});