mirror of
https://github.com/openclaw/openclaw.git
synced 2026-08-02 09:11:34 +00:00
fix(models): keep dynamic model lists consistent across CLI and Gateway (#114393)
* fix(models): unify prepared catalogs and safe migration * refactor(models): remove obsolete prepared catalog paths
This commit is contained in:
committed by
GitHub
parent
3c67fcc45d
commit
498a2caaf2
@@ -652,7 +652,6 @@ src/commands/models/auth.test.ts
|
||||
src/commands/models/auth.ts
|
||||
src/commands/models/list.list-command.forward-compat.test.ts
|
||||
src/commands/models/list.probe.ts
|
||||
src/commands/models/list.rows.ts
|
||||
src/commands/models/list.status-command.ts
|
||||
src/commands/models/list.status.test.ts
|
||||
src/commands/onboard-channels.e2e.test.ts
|
||||
|
||||
@@ -11,11 +11,13 @@ import {
|
||||
replaceRuntimeAuthProfileStoreSnapshots,
|
||||
saveAuthProfileStore,
|
||||
} from "../auth-profiles.js";
|
||||
import { PLUGIN_MODEL_CATALOG_GENERATED_BY } from "../plugin-model-catalog.js";
|
||||
import {
|
||||
encodePluginModelCatalogRelativePath,
|
||||
PLUGIN_MODEL_CATALOG_GENERATED_BY,
|
||||
replacePersistedPluginModelCatalogs,
|
||||
} from "../plugin-model-catalog.js";
|
||||
import { createProviderRuntimeTestMock } from "./model.provider-runtime.test-support.js";
|
||||
|
||||
const PLUGIN_MODEL_CATALOG_FILE = "catalog.json";
|
||||
|
||||
const resolveBundledStaticCatalogModelMock = vi.hoisted(() => vi.fn());
|
||||
const resolveBundledProviderStaticCatalogModelMock = vi.hoisted(() => vi.fn());
|
||||
const resolveManifestModelCatalogProviderAliasMetadataMock = vi.hoisted(() =>
|
||||
@@ -536,15 +538,15 @@ describe("resolveModel", () => {
|
||||
const first = await resolveModelAsync("zai", "glm-5.1", agentDir, undefined, {
|
||||
runtimeHooks: createRuntimeHooks(),
|
||||
});
|
||||
const catalogPath = path.join(agentDir, "plugins", "zai", PLUGIN_MODEL_CATALOG_FILE);
|
||||
fs.mkdirSync(path.dirname(catalogPath), { recursive: true });
|
||||
fs.writeFileSync(
|
||||
catalogPath,
|
||||
JSON.stringify({
|
||||
generatedBy: PLUGIN_MODEL_CATALOG_GENERATED_BY,
|
||||
providers: {},
|
||||
}),
|
||||
);
|
||||
replacePersistedPluginModelCatalogs({
|
||||
agentDir,
|
||||
pluginCatalogWrites: {
|
||||
[encodePluginModelCatalogRelativePath("zai")]: JSON.stringify({
|
||||
generatedBy: PLUGIN_MODEL_CATALOG_GENERATED_BY,
|
||||
providers: {},
|
||||
}),
|
||||
},
|
||||
});
|
||||
const second = await resolveModelAsync("zai", "glm-5.1", agentDir, undefined, {
|
||||
runtimeHooks: createRuntimeHooks(),
|
||||
});
|
||||
|
||||
@@ -14,9 +14,16 @@ import {
|
||||
withModelsTempHome as withTempHome,
|
||||
} from "./models-config.e2e-harness.js";
|
||||
import type { ProviderConfig as ModelsProviderConfig } from "./models-config.providers.secrets.js";
|
||||
import { PLUGIN_MODEL_CATALOG_GENERATED_BY } from "./plugin-model-catalog.js";
|
||||
import {
|
||||
encodePluginModelCatalogRelativePath,
|
||||
loadPersistedPluginModelCatalogs,
|
||||
PLUGIN_MODEL_CATALOG_GENERATED_BY,
|
||||
replacePersistedPluginModelCatalogs,
|
||||
} from "./plugin-model-catalog.js";
|
||||
|
||||
const PLUGIN_MODEL_CATALOG_FILE = "catalog.json";
|
||||
function listPersistedPluginModelCatalogs(agentDir: string) {
|
||||
return loadPersistedPluginModelCatalogs(agentDir).catalogs;
|
||||
}
|
||||
|
||||
vi.mock("./auth-profiles/external-cli-sync.js", () => ({
|
||||
resolveExternalCliAuthProfiles: () => [],
|
||||
@@ -108,27 +115,12 @@ type ParsedProviderConfig = {
|
||||
async function readGeneratedProviders(
|
||||
agentDir: string,
|
||||
): Promise<Record<string, ParsedProviderConfig>> {
|
||||
// Generated plugin catalogs are separate files but part of the effective provider set.
|
||||
// Generated plugin catalogs live in the agent database but remain part of the effective provider set.
|
||||
const raw = await fs.readFile(path.join(agentDir, "models.json"), "utf8");
|
||||
const parsed = JSON.parse(raw) as { providers?: Record<string, ParsedProviderConfig> };
|
||||
const providers = { ...parsed.providers };
|
||||
const pluginsDir = path.join(agentDir, "plugins");
|
||||
let pluginDirs: Array<import("node:fs").Dirent>;
|
||||
try {
|
||||
pluginDirs = await fs.readdir(pluginsDir, { withFileTypes: true });
|
||||
} catch {
|
||||
return providers;
|
||||
}
|
||||
for (const entry of pluginDirs) {
|
||||
if (!entry.isDirectory()) {
|
||||
continue;
|
||||
}
|
||||
const catalogPath = path.join(pluginsDir, entry.name, PLUGIN_MODEL_CATALOG_FILE);
|
||||
const catalogRaw = await fs.readFile(catalogPath, "utf8").catch(() => undefined);
|
||||
if (!catalogRaw) {
|
||||
continue;
|
||||
}
|
||||
const catalog = JSON.parse(catalogRaw) as {
|
||||
for (const { contents } of listPersistedPluginModelCatalogs(agentDir)) {
|
||||
const catalog = JSON.parse(contents) as {
|
||||
generatedBy?: string;
|
||||
providers?: Record<string, ParsedProviderConfig>;
|
||||
};
|
||||
@@ -242,27 +234,28 @@ describe("models-config", () => {
|
||||
it("preserves existing generated plugin catalog secrets in merge mode", async () => {
|
||||
await withTempHome(async (home) => {
|
||||
const agentDir = path.join(home, "agent-plugin-merge");
|
||||
const catalogPath = path.join(agentDir, "plugins", "deepseek", PLUGIN_MODEL_CATALOG_FILE);
|
||||
await fs.mkdir(path.dirname(catalogPath), { recursive: true });
|
||||
await fs.mkdir(agentDir, { recursive: true });
|
||||
await fs.writeFile(path.join(agentDir, "models.json"), JSON.stringify({ providers: {} }));
|
||||
await fs.writeFile(
|
||||
catalogPath,
|
||||
JSON.stringify(
|
||||
{
|
||||
generatedBy: PLUGIN_MODEL_CATALOG_GENERATED_BY,
|
||||
providers: {
|
||||
deepseek: {
|
||||
baseUrl: "https://persisted.example/v1",
|
||||
api: "openai-completions",
|
||||
apiKey: "persisted-key",
|
||||
models: [{ id: "test-model" }],
|
||||
replacePersistedPluginModelCatalogs({
|
||||
agentDir,
|
||||
pluginCatalogWrites: {
|
||||
[encodePluginModelCatalogRelativePath("deepseek")]: JSON.stringify(
|
||||
{
|
||||
generatedBy: PLUGIN_MODEL_CATALOG_GENERATED_BY,
|
||||
providers: {
|
||||
deepseek: {
|
||||
baseUrl: "https://persisted.example/v1",
|
||||
api: "openai-completions",
|
||||
apiKey: "persisted-key",
|
||||
models: [{ id: "test-model" }],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
);
|
||||
null,
|
||||
2,
|
||||
),
|
||||
},
|
||||
});
|
||||
const pluginMetadataSnapshot = {
|
||||
index: { plugins: [{ pluginId: "deepseek", enabled: true }] },
|
||||
normalizePluginId: (pluginId: string) => pluginId,
|
||||
@@ -278,8 +271,11 @@ describe("models-config", () => {
|
||||
pluginMetadataSnapshot,
|
||||
});
|
||||
|
||||
const raw = await fs.readFile(catalogPath, "utf8");
|
||||
const parsed = JSON.parse(raw) as {
|
||||
const persistedCatalog = listPersistedPluginModelCatalogs(agentDir).find(
|
||||
(catalog) => catalog.pluginId === "deepseek",
|
||||
);
|
||||
expect(persistedCatalog).toBeDefined();
|
||||
const parsed = JSON.parse(persistedCatalog!.contents) as {
|
||||
providers: Record<string, ParsedProviderConfig>;
|
||||
};
|
||||
expect(parsed.providers.deepseek?.baseUrl).toBe("https://persisted.example/v1");
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
/**
|
||||
* Ensures the agent-local models.json and plugin model catalog sidecars match
|
||||
* runtime config, discovered providers, auth-profile state, and generated
|
||||
* catalog ownership.
|
||||
* Ensures agent-local models.json and the SQLite-backed plugin model catalog
|
||||
* match runtime config, discovered providers, auth-profile state, and
|
||||
* generated catalog ownership.
|
||||
*/
|
||||
import { createHash } from "node:crypto";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import {
|
||||
@@ -32,10 +33,9 @@ import {
|
||||
} from "./models-config-state.js";
|
||||
import { planOpenClawModelsJson } from "./models-config.plan.js";
|
||||
import {
|
||||
decodePluginModelCatalogRelativePathPluginId,
|
||||
isGeneratedPluginModelCatalog,
|
||||
isPluginModelCatalogRelativePath,
|
||||
listPluginModelCatalogRelativePaths,
|
||||
loadPersistedPluginModelCatalogs,
|
||||
replacePersistedPluginModelCatalogs,
|
||||
resolvePluginModelCatalogOwnerPluginId,
|
||||
} from "./plugin-model-catalog.js";
|
||||
import { stableStringify } from "./stable-stringify.js";
|
||||
@@ -54,6 +54,16 @@ type EnsureOpenClawModelsJsonOptions = {
|
||||
providerDiscoveryEntriesOnly?: boolean;
|
||||
};
|
||||
|
||||
function listPreparedPluginModelCatalogs(agentDir: string) {
|
||||
const { catalogs, warnings } = loadPersistedPluginModelCatalogs(agentDir);
|
||||
if (warnings.length > 0) {
|
||||
throw new Error(
|
||||
`Cannot safely prepare provider models until legacy catalog migration succeeds: ${warnings.join("; ")}. Run openclaw doctor --fix.`,
|
||||
);
|
||||
}
|
||||
return catalogs;
|
||||
}
|
||||
|
||||
async function readFileMtimeMs(pathname: string): Promise<number | null> {
|
||||
try {
|
||||
const stat = await fs.stat(pathname);
|
||||
@@ -63,18 +73,6 @@ async function readFileMtimeMs(pathname: string): Promise<number | null> {
|
||||
}
|
||||
}
|
||||
|
||||
async function readPluginCatalogMtimes(agentDir: string): Promise<Array<[string, number | null]>> {
|
||||
const entries = await Promise.all(
|
||||
listPluginModelCatalogRelativePaths(agentDir).map(async (relativePath) => {
|
||||
return [relativePath, await readFileMtimeMs(path.join(agentDir, relativePath))] satisfies [
|
||||
string,
|
||||
number | null,
|
||||
];
|
||||
}),
|
||||
);
|
||||
return entries.toSorted(([left], [right]) => left.localeCompare(right));
|
||||
}
|
||||
|
||||
async function buildModelsJsonFingerprint(params: {
|
||||
config: OpenClawConfig;
|
||||
sourceConfigForSecrets: OpenClawConfig;
|
||||
@@ -90,7 +88,9 @@ async function buildModelsJsonFingerprint(params: {
|
||||
const authProfilesMtimeMs = await readFileMtimeMs(authProfilesSqlitePath);
|
||||
const authProfilesWalMtimeMs = await readFileMtimeMs(`${authProfilesSqlitePath}-wal`);
|
||||
const modelsFileMtimeMs = await readFileMtimeMs(path.join(params.agentDir, "models.json"));
|
||||
const pluginCatalogMtimes = await readPluginCatalogMtimes(params.agentDir);
|
||||
const pluginCatalogFingerprint = createHash("sha256")
|
||||
.update(stableStringify(listPreparedPluginModelCatalogs(params.agentDir)))
|
||||
.digest("base64url");
|
||||
const envShape = createConfigRuntimeEnv(params.config, params.env ?? {});
|
||||
const pluginMetadataSnapshotIndexFingerprint = params.pluginMetadataSnapshot
|
||||
? resolveInstalledManifestRegistryIndexFingerprint(params.pluginMetadataSnapshot.index)
|
||||
@@ -102,7 +102,7 @@ async function buildModelsJsonFingerprint(params: {
|
||||
authProfilesMtimeMs,
|
||||
authProfilesWalMtimeMs,
|
||||
modelsFileMtimeMs,
|
||||
pluginCatalogMtimes,
|
||||
pluginCatalogFingerprint,
|
||||
workspaceDir: params.workspaceDir,
|
||||
pluginMetadataSnapshotIndexFingerprint,
|
||||
providerDiscoveryProviderIds: params.providerDiscoveryProviderIds,
|
||||
@@ -141,7 +141,7 @@ async function readExistingModelsFile(pathname: string): Promise<{
|
||||
}
|
||||
}
|
||||
|
||||
/** Best-effort chmod for generated models.json and plugin catalog files. */
|
||||
/** Best-effort chmod for the user-visible generated models.json file. */
|
||||
async function ensureModelsFileModeForModelsJson(pathname: string): Promise<void> {
|
||||
await fs.chmod(pathname, 0o600).catch(() => {
|
||||
// best-effort
|
||||
@@ -163,16 +163,6 @@ if (process.env.VITEST || process.env.NODE_ENV === "test") {
|
||||
};
|
||||
}
|
||||
|
||||
async function isGeneratedPluginCatalogFile(targetPath: string): Promise<boolean> {
|
||||
return (await readGeneratedPluginCatalog(targetPath)) !== undefined;
|
||||
}
|
||||
|
||||
async function readGeneratedPluginCatalog(targetPath: string): Promise<unknown> {
|
||||
const existing = await readExistingModelsFile(targetPath);
|
||||
const parsed = existing.parsed;
|
||||
return isGeneratedPluginModelCatalog(parsed) ? parsed : undefined;
|
||||
}
|
||||
|
||||
function isRecordLike(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
}
|
||||
@@ -185,13 +175,20 @@ async function mergeGeneratedPluginCatalogProvidersIntoExistingParsed(params: {
|
||||
const root = isRecordLike(params.existingParsed) ? params.existingParsed : {};
|
||||
const providers = isRecordLike(root.providers) ? { ...root.providers } : {};
|
||||
let changed = false;
|
||||
for (const relativePath of listPluginModelCatalogRelativePaths(params.agentDir)) {
|
||||
const catalogPluginId = decodePluginModelCatalogRelativePathPluginId(relativePath);
|
||||
if (!catalogPluginId) {
|
||||
for (const { pluginId: catalogPluginId, contents } of listPreparedPluginModelCatalogs(
|
||||
params.agentDir,
|
||||
)) {
|
||||
let catalog: unknown;
|
||||
try {
|
||||
catalog = JSON.parse(contents) as unknown;
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
const catalog = await readGeneratedPluginCatalog(path.join(params.agentDir, relativePath));
|
||||
if (!isRecordLike(catalog) || !isRecordLike(catalog.providers)) {
|
||||
if (
|
||||
!isGeneratedPluginModelCatalog(catalog) ||
|
||||
!isRecordLike(catalog) ||
|
||||
!isRecordLike(catalog.providers)
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
for (const [providerId, provider] of Object.entries(catalog.providers)) {
|
||||
@@ -212,60 +209,17 @@ async function mergeGeneratedPluginCatalogProvidersIntoExistingParsed(params: {
|
||||
return { ...root, providers };
|
||||
}
|
||||
|
||||
async function removeStalePluginCatalogs(params: {
|
||||
agentDir: string;
|
||||
activeRelativePaths: ReadonlySet<string>;
|
||||
}): Promise<boolean> {
|
||||
let wrote = false;
|
||||
for (const relativePath of listPluginModelCatalogRelativePaths(params.agentDir)) {
|
||||
if (params.activeRelativePaths.has(path.normalize(relativePath))) {
|
||||
continue;
|
||||
}
|
||||
const targetPath = path.join(params.agentDir, relativePath);
|
||||
if (!(await isGeneratedPluginCatalogFile(targetPath))) {
|
||||
continue;
|
||||
}
|
||||
await fs.unlink(targetPath).catch((error: unknown) => {
|
||||
if ((error as NodeJS.ErrnoException).code === "ENOENT") {
|
||||
return;
|
||||
}
|
||||
throw error;
|
||||
});
|
||||
wrote = true;
|
||||
}
|
||||
return wrote;
|
||||
}
|
||||
|
||||
async function writePluginCatalogsForModelsJson(params: {
|
||||
function writePluginCatalogsForModelsJson(params: {
|
||||
agentDir: string;
|
||||
pluginCatalogWrites?: Record<string, string>;
|
||||
}): Promise<boolean> {
|
||||
}): boolean {
|
||||
if (!params.pluginCatalogWrites) {
|
||||
return false;
|
||||
}
|
||||
let wrote = false;
|
||||
const activeRelativePaths = new Set<string>();
|
||||
for (const [relativePath, contents] of Object.entries(params.pluginCatalogWrites)) {
|
||||
if (!isPluginModelCatalogRelativePath(relativePath)) {
|
||||
continue;
|
||||
}
|
||||
activeRelativePaths.add(path.normalize(relativePath));
|
||||
const targetPath = path.join(params.agentDir, relativePath);
|
||||
const existing = await readExistingModelsFile(targetPath);
|
||||
if (existing.raw === contents) {
|
||||
await ensureModelsFileModeForModelsJson(targetPath);
|
||||
continue;
|
||||
}
|
||||
await fs.mkdir(path.dirname(targetPath), { recursive: true, mode: 0o700 });
|
||||
await writeModelsFileAtomicForModelsJson(targetPath, contents);
|
||||
await ensureModelsFileModeForModelsJson(targetPath);
|
||||
wrote = true;
|
||||
}
|
||||
const removedStale = await removeStalePluginCatalogs({
|
||||
return replacePersistedPluginModelCatalogs({
|
||||
agentDir: params.agentDir,
|
||||
activeRelativePaths,
|
||||
pluginCatalogWrites: params.pluginCatalogWrites,
|
||||
});
|
||||
return wrote || removedStale;
|
||||
}
|
||||
|
||||
function resolveModelsConfigInput(config?: OpenClawConfig): {
|
||||
@@ -353,7 +307,7 @@ async function withModelsJsonWriteLock<T>(targetPath: string, run: () => Promise
|
||||
return await MODELS_JSON_STATE.writeQueue.enqueue(targetPath, run);
|
||||
}
|
||||
|
||||
/** Ensures models.json and plugin catalog sidecars are current for an agent. */
|
||||
/** Ensures models.json and the agent SQLite catalog cache are current. */
|
||||
async function prepareOpenClawModelsJsonSource(
|
||||
config?: OpenClawConfig,
|
||||
agentDirOverride?: string,
|
||||
@@ -421,7 +375,7 @@ async function prepareOpenClawModelsJsonSource(
|
||||
});
|
||||
|
||||
if (plan.action === "skip") {
|
||||
const wrotePluginCatalog = await writePluginCatalogsForModelsJson({
|
||||
const wrotePluginCatalog = writePluginCatalogsForModelsJson({
|
||||
agentDir,
|
||||
pluginCatalogWrites: plan.pluginCatalogWrites,
|
||||
});
|
||||
@@ -429,7 +383,7 @@ async function prepareOpenClawModelsJsonSource(
|
||||
}
|
||||
|
||||
if (plan.action === "noop") {
|
||||
const wrotePluginCatalog = await writePluginCatalogsForModelsJson({
|
||||
const wrotePluginCatalog = writePluginCatalogsForModelsJson({
|
||||
agentDir,
|
||||
pluginCatalogWrites: plan.pluginCatalogWrites,
|
||||
});
|
||||
@@ -444,7 +398,7 @@ async function prepareOpenClawModelsJsonSource(
|
||||
await writeModelsFileAtomicForModelsJson(targetPath, plan.contents);
|
||||
}
|
||||
await ensureModelsFileModeForModelsJson(targetPath);
|
||||
const wrotePluginCatalog = await writePluginCatalogsForModelsJson({
|
||||
const wrotePluginCatalog = writePluginCatalogsForModelsJson({
|
||||
agentDir,
|
||||
pluginCatalogWrites: plan.pluginCatalogWrites,
|
||||
});
|
||||
@@ -491,7 +445,7 @@ async function prepareOpenClawModelsJsonSource(
|
||||
}
|
||||
}
|
||||
|
||||
/** Ensures models.json and plugin catalog sidecars are current for an agent. */
|
||||
/** Ensures models.json and the agent SQLite catalog cache are current. */
|
||||
export async function ensureOpenClawModelsJson(
|
||||
config?: OpenClawConfig,
|
||||
agentDirOverride?: string,
|
||||
|
||||
@@ -13,10 +13,14 @@ import {
|
||||
import { readGeneratedModelsJson } from "./models-config.test-utils.js";
|
||||
import {
|
||||
encodePluginModelCatalogRelativePath,
|
||||
loadPersistedPluginModelCatalogs,
|
||||
PLUGIN_MODEL_CATALOG_GENERATED_BY,
|
||||
replacePersistedPluginModelCatalogs,
|
||||
} from "./plugin-model-catalog.js";
|
||||
|
||||
const PLUGIN_MODEL_CATALOG_FILE = "catalog.json";
|
||||
function listPersistedPluginModelCatalogs(agentDir: string) {
|
||||
return loadPersistedPluginModelCatalogs(agentDir).catalogs;
|
||||
}
|
||||
|
||||
const planOpenClawModelsJsonMock = vi.fn();
|
||||
const writePrivateStoreTextWriteMock = vi.fn();
|
||||
@@ -221,7 +225,114 @@ describe("models-config write serialization", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("writes plugin-owned model catalogs beside the agent plugin state", async () => {
|
||||
it("migrates released provider credentials before model planning can regenerate a catalog", async () => {
|
||||
await withModelsTempHome(async (home) => {
|
||||
const agentDir = path.join(home, "agent");
|
||||
const relativePath = encodePluginModelCatalogRelativePath("zai");
|
||||
const sourcePath = path.join(agentDir, relativePath);
|
||||
const contents = `${JSON.stringify({
|
||||
generatedBy: PLUGIN_MODEL_CATALOG_GENERATED_BY,
|
||||
providers: {
|
||||
zai: {
|
||||
baseUrl: "https://api.z.ai/api/paas/v4",
|
||||
api: "openai-completions",
|
||||
apiKey: "released-zai-provider-test-key",
|
||||
models: [{ id: "glm-5.1", name: "GLM 5.1" }],
|
||||
},
|
||||
},
|
||||
})}\n`;
|
||||
await fs.mkdir(path.dirname(sourcePath), { recursive: true });
|
||||
await fs.writeFile(sourcePath, contents, "utf8");
|
||||
planOpenClawModelsJsonMock.mockImplementation(async () => {
|
||||
expect(listPersistedPluginModelCatalogs(agentDir)).toEqual([{ pluginId: "zai", contents }]);
|
||||
return { action: "skip" };
|
||||
});
|
||||
|
||||
await ensureOpenClawModelsJson({}, agentDir);
|
||||
|
||||
expect(planOpenClawModelsJsonMock).toHaveBeenCalledOnce();
|
||||
await expectMissingPath(fs.access(sourcePath));
|
||||
expect(listPersistedPluginModelCatalogs(agentDir)).toEqual([{ pluginId: "zai", contents }]);
|
||||
});
|
||||
});
|
||||
|
||||
it("refuses to regenerate provider catalogs while released credentials cannot be read", async () => {
|
||||
if (process.getuid?.() === 0) {
|
||||
return;
|
||||
}
|
||||
await withModelsTempHome(async (home) => {
|
||||
const agentDir = path.join(home, "agent");
|
||||
const sourcePath = path.join(agentDir, encodePluginModelCatalogRelativePath("zai"));
|
||||
await fs.mkdir(path.dirname(sourcePath), { recursive: true });
|
||||
await fs.writeFile(
|
||||
sourcePath,
|
||||
JSON.stringify({
|
||||
generatedBy: PLUGIN_MODEL_CATALOG_GENERATED_BY,
|
||||
providers: { zai: { apiKey: "unreadable-released-provider-test-key" } },
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
await fs.chmod(sourcePath, 0o000);
|
||||
|
||||
try {
|
||||
await expect(ensureOpenClawModelsJson({}, agentDir)).rejects.toThrow(
|
||||
"Cannot safely prepare provider models until legacy catalog migration succeeds",
|
||||
);
|
||||
expect(planOpenClawModelsJsonMock).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
await fs.chmod(sourcePath, 0o600);
|
||||
}
|
||||
|
||||
await expect(fs.access(sourcePath)).resolves.toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
it("never promotes an unmarked cached plugin catalog into model planning", async () => {
|
||||
await withModelsTempHome(async (home) => {
|
||||
const agentDir = path.join(home, "agent");
|
||||
await fs.mkdir(agentDir, { recursive: true });
|
||||
await fs.writeFile(path.join(agentDir, "models.json"), JSON.stringify({ providers: {} }));
|
||||
replacePersistedPluginModelCatalogs({
|
||||
agentDir,
|
||||
pluginCatalogWrites: {
|
||||
[encodePluginModelCatalogRelativePath("zai")]: JSON.stringify({
|
||||
providers: {
|
||||
zai: {
|
||||
baseUrl: "https://unmarked.example/v1",
|
||||
api: "openai-completions",
|
||||
apiKey: "unmarked-provider-test-key",
|
||||
models: [{ id: "unmarked-model" }],
|
||||
},
|
||||
},
|
||||
}),
|
||||
},
|
||||
});
|
||||
const pluginMetadataSnapshot = {
|
||||
index: { plugins: [{ pluginId: "zai", enabled: true }] },
|
||||
normalizePluginId: (pluginId: string) => pluginId,
|
||||
manifestRegistry: { plugins: [], diagnostics: [] },
|
||||
owners: {
|
||||
providers: new Map([["zai", ["zai"]]]),
|
||||
modelCatalogProviders: new Map([["zai", ["zai"]]]),
|
||||
setupProviders: new Map(),
|
||||
},
|
||||
} as unknown as Pick<PluginMetadataSnapshot, "index" | "manifestRegistry" | "owners">;
|
||||
planOpenClawModelsJsonMock.mockImplementation(
|
||||
async (params: { existingParsed?: unknown }) => {
|
||||
expect(params.existingParsed).toEqual({ providers: {} });
|
||||
return { action: "skip" };
|
||||
},
|
||||
);
|
||||
|
||||
await ensureOpenClawModelsJson({ models: { providers: {} } }, agentDir, {
|
||||
pluginMetadataSnapshot,
|
||||
});
|
||||
|
||||
expect(planOpenClawModelsJsonMock).toHaveBeenCalledOnce();
|
||||
});
|
||||
});
|
||||
|
||||
it("writes plugin-owned model catalogs into the agent SQLite cache", async () => {
|
||||
await withModelsTempHome(async (home) => {
|
||||
const agentDir = path.join(home, "agent");
|
||||
planOpenClawModelsJsonMock.mockImplementation(async () => ({
|
||||
@@ -251,33 +362,33 @@ describe("models-config write serialization", () => {
|
||||
const root = JSON.parse(await fs.readFile(path.join(agentDir, "models.json"), "utf8")) as {
|
||||
providers?: Record<string, unknown>;
|
||||
};
|
||||
const catalog = JSON.parse(
|
||||
await fs.readFile(path.join(agentDir, "plugins", "zai", PLUGIN_MODEL_CATALOG_FILE), "utf8"),
|
||||
) as { providers?: Record<string, unknown> };
|
||||
const stored = listPersistedPluginModelCatalogs(agentDir);
|
||||
expect(stored).toHaveLength(1);
|
||||
expect(stored[0]?.pluginId).toBe("zai");
|
||||
const catalog = JSON.parse(stored[0]?.contents ?? "{}") as {
|
||||
providers?: Record<string, unknown>;
|
||||
};
|
||||
expect(root.providers).toEqual({});
|
||||
expect(root).not.toHaveProperty("pluginCatalogs");
|
||||
expect(Object.keys(catalog.providers ?? {})).toEqual(["zai"]);
|
||||
await expect(fs.access(path.join(agentDir, "plugins"))).rejects.toMatchObject({
|
||||
code: "ENOENT",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it("removes stale plugin-owned model catalogs", async () => {
|
||||
it("removes stale plugin-owned model catalogs from the agent SQLite cache", async () => {
|
||||
await withModelsTempHome(async (home) => {
|
||||
const agentDir = path.join(home, "agent");
|
||||
const staleCatalog = path.join(
|
||||
replacePersistedPluginModelCatalogs({
|
||||
agentDir,
|
||||
"plugins",
|
||||
"old-provider",
|
||||
PLUGIN_MODEL_CATALOG_FILE,
|
||||
);
|
||||
await fs.mkdir(path.dirname(staleCatalog), { recursive: true });
|
||||
await fs.writeFile(
|
||||
staleCatalog,
|
||||
`${JSON.stringify(
|
||||
{ generatedBy: PLUGIN_MODEL_CATALOG_GENERATED_BY, providers: {} },
|
||||
null,
|
||||
2,
|
||||
)}\n`,
|
||||
);
|
||||
pluginCatalogWrites: {
|
||||
[encodePluginModelCatalogRelativePath("old-provider")]: `${JSON.stringify({
|
||||
generatedBy: PLUGIN_MODEL_CATALOG_GENERATED_BY,
|
||||
providers: {},
|
||||
})}\n`,
|
||||
},
|
||||
});
|
||||
planOpenClawModelsJsonMock.mockImplementation(async () => ({
|
||||
action: "noop",
|
||||
pluginCatalogWrites: {},
|
||||
@@ -291,29 +402,30 @@ describe("models-config write serialization", () => {
|
||||
const result = await ensureOpenClawModelsJson({}, agentDir);
|
||||
|
||||
expect(result.wrote).toBe(true);
|
||||
await expect(fs.access(staleCatalog)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
expect(listPersistedPluginModelCatalogs(agentDir)).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps generated plugin catalogs on non-authoritative skip plans", async () => {
|
||||
it("keeps generated SQLite plugin catalogs on non-authoritative skip plans", async () => {
|
||||
await withModelsTempHome(async (home) => {
|
||||
const agentDir = path.join(home, "agent");
|
||||
const catalogPath = path.join(agentDir, "plugins", "zai", PLUGIN_MODEL_CATALOG_FILE);
|
||||
await fs.mkdir(path.dirname(catalogPath), { recursive: true });
|
||||
await fs.writeFile(
|
||||
catalogPath,
|
||||
`${JSON.stringify(
|
||||
{ generatedBy: PLUGIN_MODEL_CATALOG_GENERATED_BY, providers: {} },
|
||||
null,
|
||||
2,
|
||||
)}\n`,
|
||||
);
|
||||
replacePersistedPluginModelCatalogs({
|
||||
agentDir,
|
||||
pluginCatalogWrites: {
|
||||
[encodePluginModelCatalogRelativePath("zai")]: `${JSON.stringify({
|
||||
generatedBy: PLUGIN_MODEL_CATALOG_GENERATED_BY,
|
||||
providers: {},
|
||||
})}\n`,
|
||||
},
|
||||
});
|
||||
planOpenClawModelsJsonMock.mockImplementation(async () => ({ action: "skip" }));
|
||||
|
||||
const result = await ensureOpenClawModelsJson({}, agentDir);
|
||||
|
||||
expect(result.wrote).toBe(false);
|
||||
await expect(fs.access(catalogPath)).resolves.toBeUndefined();
|
||||
expect(listPersistedPluginModelCatalogs(agentDir)).toEqual([
|
||||
expect.objectContaining({ pluginId: "zai" }),
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
728
src/agents/plugin-model-catalog.test.ts
Normal file
728
src/agents/plugin-model-catalog.test.ts
Normal file
@@ -0,0 +1,728 @@
|
||||
// Generated plugin catalogs reuse the existing per-agent SQLite cache.
|
||||
import {
|
||||
chmodSync,
|
||||
existsSync,
|
||||
mkdirSync,
|
||||
mkdtempSync,
|
||||
readFileSync,
|
||||
renameSync,
|
||||
rmSync,
|
||||
unlinkSync,
|
||||
writeFileSync,
|
||||
} from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { DatabaseSync } from "node:sqlite";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { closeOpenClawAgentDatabasesForTest } from "../state/openclaw-agent-db.js";
|
||||
import {
|
||||
decodePluginModelCatalogRelativePathPluginId,
|
||||
encodePluginModelCatalogRelativePath,
|
||||
loadPersistedPluginModelCatalogs,
|
||||
migrateLegacyPluginModelCatalogs,
|
||||
PLUGIN_MODEL_CATALOG_GENERATED_BY,
|
||||
replacePersistedPluginModelCatalogs,
|
||||
} from "./plugin-model-catalog.js";
|
||||
|
||||
function listPersistedPluginModelCatalogs(agentDir: string) {
|
||||
return loadPersistedPluginModelCatalogs(agentDir).catalogs;
|
||||
}
|
||||
|
||||
const tempDirs: string[] = [];
|
||||
|
||||
function createAgentDir(): string {
|
||||
const agentDir = mkdtempSync(join(tmpdir(), "openclaw-plugin-model-catalog-"));
|
||||
tempDirs.push(agentDir);
|
||||
return agentDir;
|
||||
}
|
||||
|
||||
function catalogContents(provider: string, apiKey?: string): string {
|
||||
return JSON.stringify({
|
||||
generatedBy: PLUGIN_MODEL_CATALOG_GENERATED_BY,
|
||||
providers: {
|
||||
[provider]: {
|
||||
baseUrl: `https://${provider}.example/v1`,
|
||||
api: "openai-completions",
|
||||
...(apiKey ? { apiKey } : {}),
|
||||
models: [{ id: `${provider}-model`, name: `${provider} model` }],
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
closeOpenClawAgentDatabasesForTest();
|
||||
for (const agentDir of tempDirs.splice(0)) {
|
||||
rmSync(agentDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
describe("SQLite-backed plugin model catalogs", () => {
|
||||
it("does not create an agent database for a read-only cache miss", () => {
|
||||
const agentDir = createAgentDir();
|
||||
|
||||
expect(listPersistedPluginModelCatalogs(agentDir)).toEqual([]);
|
||||
expect(existsSync(join(agentDir, "openclaw-agent.sqlite"))).toBe(false);
|
||||
expect(existsSync(join(agentDir, "plugins"))).toBe(false);
|
||||
});
|
||||
|
||||
it("does not create agent state when an empty catalog is already current", () => {
|
||||
const agentDir = createAgentDir();
|
||||
|
||||
expect(replacePersistedPluginModelCatalogs({ agentDir, pluginCatalogWrites: {} })).toBe(false);
|
||||
expect(existsSync(join(agentDir, "openclaw-agent.sqlite"))).toBe(false);
|
||||
expect(existsSync(join(agentDir, "plugins"))).toBe(false);
|
||||
});
|
||||
|
||||
it("automatically migrates a released catalog before the first SQLite read", () => {
|
||||
const agentDir = createAgentDir();
|
||||
const contents = catalogContents("zai", "released-provider-test-key");
|
||||
const sourcePath = join(agentDir, encodePluginModelCatalogRelativePath("zai"));
|
||||
mkdirSync(join(agentDir, "plugins", "zai"), { recursive: true });
|
||||
writeFileSync(sourcePath, contents, "utf8");
|
||||
|
||||
expect(listPersistedPluginModelCatalogs(agentDir)).toEqual([{ pluginId: "zai", contents }]);
|
||||
expect(existsSync(join(agentDir, "openclaw-agent.sqlite"))).toBe(true);
|
||||
expect(existsSync(sourcePath)).toBe(false);
|
||||
});
|
||||
|
||||
it("retires migrated credential recovery data after verifying the canonical catalog", () => {
|
||||
const agentDir = createAgentDir();
|
||||
const contents = catalogContents("zai", "retired-released-provider-test-key");
|
||||
const sourcePath = join(agentDir, encodePluginModelCatalogRelativePath("zai"));
|
||||
mkdirSync(join(agentDir, "plugins", "zai"), { recursive: true });
|
||||
writeFileSync(sourcePath, contents, "utf8");
|
||||
|
||||
expect(listPersistedPluginModelCatalogs(agentDir)).toEqual([{ pluginId: "zai", contents }]);
|
||||
const database = new DatabaseSync(join(agentDir, "openclaw-agent.sqlite"), {
|
||||
readOnly: true,
|
||||
});
|
||||
try {
|
||||
expect(
|
||||
database
|
||||
.prepare("SELECT value_json FROM cache_entries WHERE scope = ?")
|
||||
.all("plugin-model-catalog-migration-v1"),
|
||||
).toEqual([]);
|
||||
} finally {
|
||||
database.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("retires an orphaned recovery credential after an interrupted migration", () => {
|
||||
const agentDir = createAgentDir();
|
||||
const contents = catalogContents("zai", "interrupted-released-provider-test-key");
|
||||
replacePersistedPluginModelCatalogs({
|
||||
agentDir,
|
||||
pluginCatalogWrites: {
|
||||
[encodePluginModelCatalogRelativePath("zai")]: contents,
|
||||
},
|
||||
});
|
||||
const database = new DatabaseSync(join(agentDir, "openclaw-agent.sqlite"));
|
||||
try {
|
||||
database
|
||||
.prepare(
|
||||
"INSERT INTO cache_entries (scope, key, value_json, blob, expires_at, updated_at) VALUES (?, ?, ?, NULL, NULL, ?)",
|
||||
)
|
||||
.run("plugin-model-catalog-migration-v1", "zai", contents, Date.now());
|
||||
} finally {
|
||||
database.close();
|
||||
}
|
||||
|
||||
expect(listPersistedPluginModelCatalogs(agentDir)).toEqual([{ pluginId: "zai", contents }]);
|
||||
const verified = new DatabaseSync(join(agentDir, "openclaw-agent.sqlite"), {
|
||||
readOnly: true,
|
||||
});
|
||||
try {
|
||||
expect(
|
||||
verified
|
||||
.prepare("SELECT value_json FROM cache_entries WHERE scope = ?")
|
||||
.all("plugin-model-catalog-migration-v1"),
|
||||
).toEqual([]);
|
||||
} finally {
|
||||
verified.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("protects migration recovery credentials when a plugin directory cannot be inspected", () => {
|
||||
if (process.getuid?.() === 0) {
|
||||
return;
|
||||
}
|
||||
const agentDir = createAgentDir();
|
||||
const contents = catalogContents("zai", "protected-released-provider-test-key");
|
||||
const pluginDir = join(agentDir, "plugins", "zai");
|
||||
replacePersistedPluginModelCatalogs({
|
||||
agentDir,
|
||||
pluginCatalogWrites: {
|
||||
[encodePluginModelCatalogRelativePath("zai")]: contents,
|
||||
},
|
||||
});
|
||||
const database = new DatabaseSync(join(agentDir, "openclaw-agent.sqlite"));
|
||||
try {
|
||||
database
|
||||
.prepare(
|
||||
"INSERT INTO cache_entries (scope, key, value_json, blob, expires_at, updated_at) VALUES (?, ?, ?, NULL, NULL, ?)",
|
||||
)
|
||||
.run("plugin-model-catalog-migration-v1", "zai", contents, Date.now());
|
||||
} finally {
|
||||
database.close();
|
||||
}
|
||||
mkdirSync(pluginDir, { recursive: true });
|
||||
writeFileSync(join(pluginDir, "catalog.json"), contents, "utf8");
|
||||
chmodSync(pluginDir, 0o000);
|
||||
|
||||
try {
|
||||
expect(migrateLegacyPluginModelCatalogs({ agentDir })).toEqual({
|
||||
detected: 0,
|
||||
migrated: 0,
|
||||
warnings: [expect.stringContaining("Could not inspect legacy provider catalogs")],
|
||||
});
|
||||
const verified = new DatabaseSync(join(agentDir, "openclaw-agent.sqlite"), {
|
||||
readOnly: true,
|
||||
});
|
||||
try {
|
||||
expect(
|
||||
verified
|
||||
.prepare("SELECT value_json FROM cache_entries WHERE scope = ? AND key = ?")
|
||||
.all("plugin-model-catalog-migration-v1", "zai"),
|
||||
).toEqual([{ value_json: contents }]);
|
||||
} finally {
|
||||
verified.close();
|
||||
}
|
||||
} finally {
|
||||
chmodSync(pluginDir, 0o700);
|
||||
}
|
||||
|
||||
expect(listPersistedPluginModelCatalogs(agentDir)).toEqual([{ pluginId: "zai", contents }]);
|
||||
});
|
||||
|
||||
it("detects a released catalog that appears after an earlier empty scan", () => {
|
||||
const agentDir = createAgentDir();
|
||||
expect(listPersistedPluginModelCatalogs(agentDir)).toEqual([]);
|
||||
|
||||
const contents = catalogContents("zai", "late-released-provider-test-key");
|
||||
const sourcePath = join(agentDir, encodePluginModelCatalogRelativePath("zai"));
|
||||
mkdirSync(join(agentDir, "plugins", "zai"), { recursive: true });
|
||||
writeFileSync(sourcePath, contents, "utf8");
|
||||
|
||||
expect(listPersistedPluginModelCatalogs(agentDir)).toEqual([{ pluginId: "zai", contents }]);
|
||||
expect(existsSync(sourcePath)).toBe(false);
|
||||
});
|
||||
|
||||
it("migrates a later provider catalog after an earlier migration completed", () => {
|
||||
const agentDir = createAgentDir();
|
||||
const zai = catalogContents("zai", "released-zai-provider-test-key");
|
||||
const zaiPath = join(agentDir, encodePluginModelCatalogRelativePath("zai"));
|
||||
mkdirSync(join(agentDir, "plugins", "zai"), { recursive: true });
|
||||
writeFileSync(zaiPath, zai, "utf8");
|
||||
|
||||
expect(listPersistedPluginModelCatalogs(agentDir)).toEqual([
|
||||
{ pluginId: "zai", contents: zai },
|
||||
]);
|
||||
|
||||
const anthropic = catalogContents("anthropic", "late-anthropic-provider-test-key");
|
||||
const anthropicPath = join(agentDir, encodePluginModelCatalogRelativePath("anthropic"));
|
||||
mkdirSync(join(agentDir, "plugins", "anthropic"), { recursive: true });
|
||||
writeFileSync(anthropicPath, anthropic, "utf8");
|
||||
|
||||
expect(listPersistedPluginModelCatalogs(agentDir)).toEqual([
|
||||
{ pluginId: "anthropic", contents: anthropic },
|
||||
{ pluginId: "zai", contents: zai },
|
||||
]);
|
||||
expect(existsSync(zaiPath)).toBe(false);
|
||||
expect(existsSync(anthropicPath)).toBe(false);
|
||||
});
|
||||
|
||||
it("migrates a changed later sidecar for an already migrated provider", () => {
|
||||
const agentDir = createAgentDir();
|
||||
const pluginDir = join(agentDir, "plugins", "zai");
|
||||
const sourcePath = join(agentDir, encodePluginModelCatalogRelativePath("zai"));
|
||||
const original = catalogContents("zai", "first-released-provider-test-key");
|
||||
const later = catalogContents("zai", "later-released-provider-test-key");
|
||||
mkdirSync(pluginDir, { recursive: true });
|
||||
writeFileSync(sourcePath, original, "utf8");
|
||||
expect(listPersistedPluginModelCatalogs(agentDir)).toEqual([
|
||||
{ pluginId: "zai", contents: original },
|
||||
]);
|
||||
|
||||
writeFileSync(sourcePath, later, "utf8");
|
||||
|
||||
expect(listPersistedPluginModelCatalogs(agentDir)).toEqual([
|
||||
{ pluginId: "zai", contents: later },
|
||||
]);
|
||||
expect(existsSync(sourcePath)).toBe(false);
|
||||
});
|
||||
|
||||
it("preserves released credentials and unrelated regenerated SQLite catalogs", () => {
|
||||
const agentDir = createAgentDir();
|
||||
const regenerated = catalogContents("zai", "regenerated-provider-test-key");
|
||||
const released = catalogContents("zai", "released-provider-test-key");
|
||||
const unrelated = catalogContents("anthropic", "unrelated-provider-test-key");
|
||||
replacePersistedPluginModelCatalogs({
|
||||
agentDir,
|
||||
pluginCatalogWrites: {
|
||||
[encodePluginModelCatalogRelativePath("zai")]: regenerated,
|
||||
[encodePluginModelCatalogRelativePath("anthropic")]: unrelated,
|
||||
},
|
||||
});
|
||||
const sourcePath = join(agentDir, encodePluginModelCatalogRelativePath("zai"));
|
||||
mkdirSync(join(agentDir, "plugins", "zai"), { recursive: true });
|
||||
writeFileSync(sourcePath, released, "utf8");
|
||||
|
||||
expect(
|
||||
migrateLegacyPluginModelCatalogs({
|
||||
agentDir,
|
||||
expectedContents: new Map([["zai", released]]),
|
||||
}),
|
||||
).toEqual({ detected: 1, migrated: 1, warnings: [] });
|
||||
expect(listPersistedPluginModelCatalogs(agentDir)).toEqual([
|
||||
{ pluginId: "anthropic", contents: unrelated },
|
||||
{ pluginId: "zai", contents: released },
|
||||
]);
|
||||
expect(existsSync(sourcePath)).toBe(false);
|
||||
});
|
||||
|
||||
it("recovers a released sidecar that appears after a catalog replacement was planned", () => {
|
||||
const agentDir = createAgentDir();
|
||||
const released = catalogContents("zai", "released-provider-test-key");
|
||||
const preplanned = catalogContents("zai", "preplanned-provider-test-key");
|
||||
const sourcePath = join(agentDir, encodePluginModelCatalogRelativePath("zai"));
|
||||
mkdirSync(join(agentDir, "plugins", "zai"), { recursive: true });
|
||||
writeFileSync(sourcePath, released, "utf8");
|
||||
|
||||
replacePersistedPluginModelCatalogs({
|
||||
agentDir,
|
||||
pluginCatalogWrites: {
|
||||
[encodePluginModelCatalogRelativePath("zai")]: preplanned,
|
||||
},
|
||||
});
|
||||
|
||||
expect(existsSync(sourcePath)).toBe(true);
|
||||
expect(listPersistedPluginModelCatalogs(agentDir)).toEqual([
|
||||
{ pluginId: "zai", contents: released },
|
||||
]);
|
||||
expect(existsSync(sourcePath)).toBe(false);
|
||||
});
|
||||
|
||||
it("treats an already-completed legacy migration as safe to repeat", () => {
|
||||
const agentDir = createAgentDir();
|
||||
const contents = catalogContents("zai", "released-provider-test-key");
|
||||
const sourcePath = join(agentDir, encodePluginModelCatalogRelativePath("zai"));
|
||||
mkdirSync(join(agentDir, "plugins", "zai"), { recursive: true });
|
||||
writeFileSync(sourcePath, contents, "utf8");
|
||||
|
||||
expect(migrateLegacyPluginModelCatalogs({ agentDir })).toEqual({
|
||||
detected: 1,
|
||||
migrated: 1,
|
||||
warnings: [],
|
||||
});
|
||||
expect(migrateLegacyPluginModelCatalogs({ agentDir })).toEqual({
|
||||
detected: 0,
|
||||
migrated: 0,
|
||||
warnings: [],
|
||||
});
|
||||
expect(listPersistedPluginModelCatalogs(agentDir)).toEqual([{ pluginId: "zai", contents }]);
|
||||
});
|
||||
|
||||
it("accepts a sidecar already migrated and removed by another process", () => {
|
||||
const agentDir = createAgentDir();
|
||||
const contents = catalogContents("zai", "concurrently-migrated-provider-test-key");
|
||||
replacePersistedPluginModelCatalogs({
|
||||
agentDir,
|
||||
pluginCatalogWrites: {
|
||||
[encodePluginModelCatalogRelativePath("zai")]: contents,
|
||||
},
|
||||
});
|
||||
|
||||
expect(
|
||||
migrateLegacyPluginModelCatalogs({
|
||||
agentDir,
|
||||
expectedContents: new Map([["zai", contents]]),
|
||||
}),
|
||||
).toEqual({ detected: 0, migrated: 0, warnings: [] });
|
||||
expect(listPersistedPluginModelCatalogs(agentDir)).toEqual([{ pluginId: "zai", contents }]);
|
||||
});
|
||||
|
||||
it("keeps committed catalogs available when a released sidecar cannot be removed", () => {
|
||||
if (process.getuid?.() === 0) {
|
||||
return;
|
||||
}
|
||||
const agentDir = createAgentDir();
|
||||
const contents = catalogContents("zai", "released-provider-test-key");
|
||||
const pluginDir = join(agentDir, "plugins", "zai");
|
||||
const sourcePath = join(agentDir, encodePluginModelCatalogRelativePath("zai"));
|
||||
mkdirSync(pluginDir, { recursive: true });
|
||||
writeFileSync(sourcePath, contents, "utf8");
|
||||
chmodSync(pluginDir, 0o500);
|
||||
|
||||
try {
|
||||
expect(migrateLegacyPluginModelCatalogs({ agentDir })).toEqual({
|
||||
detected: 1,
|
||||
migrated: 0,
|
||||
warnings: [expect.stringContaining("Could not remove migrated legacy provider catalog")],
|
||||
});
|
||||
expect(listPersistedPluginModelCatalogs(agentDir)).toEqual([{ pluginId: "zai", contents }]);
|
||||
expect(existsSync(sourcePath)).toBe(true);
|
||||
|
||||
const refreshed = catalogContents("zai", "refreshed-provider-test-key");
|
||||
replacePersistedPluginModelCatalogs({
|
||||
agentDir,
|
||||
pluginCatalogWrites: {
|
||||
[encodePluginModelCatalogRelativePath("zai")]: refreshed,
|
||||
},
|
||||
});
|
||||
expect(listPersistedPluginModelCatalogs(agentDir)).toEqual([
|
||||
{ pluginId: "zai", contents: refreshed },
|
||||
]);
|
||||
expect(existsSync(sourcePath)).toBe(true);
|
||||
} finally {
|
||||
chmodSync(pluginDir, 0o700);
|
||||
}
|
||||
});
|
||||
|
||||
it("never deletes retained migrated credentials after a newer catalog replaces them", () => {
|
||||
if (process.getuid?.() === 0) {
|
||||
return;
|
||||
}
|
||||
const agentDir = createAgentDir();
|
||||
const original = catalogContents("zai", "released-provider-test-key");
|
||||
const refreshed = catalogContents("zai", "refreshed-provider-test-key");
|
||||
const pluginDir = join(agentDir, "plugins", "zai");
|
||||
const sourcePath = join(agentDir, encodePluginModelCatalogRelativePath("zai"));
|
||||
mkdirSync(pluginDir, { recursive: true });
|
||||
writeFileSync(sourcePath, original, "utf8");
|
||||
chmodSync(pluginDir, 0o500);
|
||||
|
||||
try {
|
||||
expect(migrateLegacyPluginModelCatalogs({ agentDir })).toEqual({
|
||||
detected: 1,
|
||||
migrated: 0,
|
||||
warnings: [expect.stringContaining("Could not remove migrated legacy provider catalog")],
|
||||
});
|
||||
replacePersistedPluginModelCatalogs({
|
||||
agentDir,
|
||||
pluginCatalogWrites: {
|
||||
[encodePluginModelCatalogRelativePath("zai")]: refreshed,
|
||||
},
|
||||
});
|
||||
} finally {
|
||||
chmodSync(pluginDir, 0o700);
|
||||
}
|
||||
|
||||
expect(migrateLegacyPluginModelCatalogs({ agentDir })).toEqual({
|
||||
detected: 1,
|
||||
migrated: 0,
|
||||
warnings: [expect.stringContaining("Left superseded legacy provider catalog in place")],
|
||||
});
|
||||
expect(existsSync(sourcePath)).toBe(true);
|
||||
expect(listPersistedPluginModelCatalogs(agentDir)).toEqual([
|
||||
{ pluginId: "zai", contents: refreshed },
|
||||
]);
|
||||
expect(existsSync(sourcePath)).toBe(true);
|
||||
});
|
||||
|
||||
it("atomically preserves a provider catalog refreshed immediately before migration claims it", () => {
|
||||
const agentDir = createAgentDir();
|
||||
const original = catalogContents("zai", "released-provider-test-key");
|
||||
const refreshed = catalogContents("zai", "concurrently-refreshed-provider-test-key");
|
||||
const sourcePath = join(agentDir, encodePluginModelCatalogRelativePath("zai"));
|
||||
mkdirSync(join(agentDir, "plugins", "zai"), { recursive: true });
|
||||
writeFileSync(sourcePath, original, "utf8");
|
||||
|
||||
expect(
|
||||
migrateLegacyPluginModelCatalogs({
|
||||
agentDir,
|
||||
beforeLegacyCatalogClaim: (pathname) => {
|
||||
writeFileSync(pathname, refreshed, "utf8");
|
||||
},
|
||||
}),
|
||||
).toEqual({
|
||||
detected: 1,
|
||||
migrated: 0,
|
||||
warnings: [expect.stringContaining("Left changed legacy provider catalog in place")],
|
||||
});
|
||||
expect(readFileSync(sourcePath, "utf8")).toBe(refreshed);
|
||||
expect(listPersistedPluginModelCatalogs(agentDir)).toEqual([
|
||||
{ pluginId: "zai", contents: refreshed },
|
||||
]);
|
||||
expect(existsSync(sourcePath)).toBe(false);
|
||||
});
|
||||
|
||||
it("never publishes a stale scan after another process removes the legacy source", () => {
|
||||
const agentDir = createAgentDir();
|
||||
const original = catalogContents("zai", "stale-released-provider-test-key");
|
||||
const refreshed = catalogContents("zai", "current-regenerated-provider-test-key");
|
||||
const sourcePath = join(agentDir, encodePluginModelCatalogRelativePath("zai"));
|
||||
mkdirSync(join(agentDir, "plugins", "zai"), { recursive: true });
|
||||
writeFileSync(sourcePath, original, "utf8");
|
||||
|
||||
expect(
|
||||
migrateLegacyPluginModelCatalogs({
|
||||
agentDir,
|
||||
beforeLegacyCatalogClaim: (pathname) => {
|
||||
replacePersistedPluginModelCatalogs({
|
||||
agentDir,
|
||||
pluginCatalogWrites: {
|
||||
[encodePluginModelCatalogRelativePath("zai")]: refreshed,
|
||||
},
|
||||
});
|
||||
unlinkSync(pathname);
|
||||
},
|
||||
}),
|
||||
).toEqual({
|
||||
detected: 1,
|
||||
migrated: 0,
|
||||
warnings: [
|
||||
expect.stringContaining(
|
||||
"Legacy provider catalog was claimed before its migration was committed",
|
||||
),
|
||||
],
|
||||
});
|
||||
expect(listPersistedPluginModelCatalogs(agentDir)).toEqual([
|
||||
{ pluginId: "zai", contents: refreshed },
|
||||
]);
|
||||
});
|
||||
|
||||
it("blocks preparation until another process commits its claimed provider catalog", () => {
|
||||
const agentDir = createAgentDir();
|
||||
const contents = catalogContents("zai", "in-flight-released-provider-test-key");
|
||||
const sourcePath = join(agentDir, encodePluginModelCatalogRelativePath("zai"));
|
||||
const claimPath = `${sourcePath}.doctor-importing-concurrent-process`;
|
||||
mkdirSync(join(agentDir, "plugins", "zai"), { recursive: true });
|
||||
writeFileSync(sourcePath, contents, "utf8");
|
||||
|
||||
expect(
|
||||
migrateLegacyPluginModelCatalogs({
|
||||
agentDir,
|
||||
beforeLegacyCatalogClaim: (pathname) => {
|
||||
renameSync(pathname, claimPath);
|
||||
},
|
||||
}),
|
||||
).toEqual({
|
||||
detected: 1,
|
||||
migrated: 0,
|
||||
warnings: [
|
||||
expect.stringContaining(
|
||||
"Legacy provider catalog was claimed before its migration was committed",
|
||||
),
|
||||
],
|
||||
});
|
||||
expect(existsSync(claimPath)).toBe(true);
|
||||
expect(existsSync(join(agentDir, "openclaw-agent.sqlite"))).toBe(false);
|
||||
expect(listPersistedPluginModelCatalogs(agentDir)).toEqual([{ pluginId: "zai", contents }]);
|
||||
expect(existsSync(claimPath)).toBe(false);
|
||||
});
|
||||
|
||||
it("never migrates a provider while one retained claim cannot be read", () => {
|
||||
if (process.getuid?.() === 0) {
|
||||
return;
|
||||
}
|
||||
const agentDir = createAgentDir();
|
||||
const pluginDir = join(agentDir, "plugins", "zai");
|
||||
const claimPath = join(pluginDir, "catalog.json.doctor-importing-previous-process");
|
||||
const sourcePath = join(pluginDir, "catalog.json");
|
||||
const retained = catalogContents("zai", "unreadable-retained-provider-test-key");
|
||||
const refreshed = catalogContents("zai", "current-canonical-provider-test-key");
|
||||
mkdirSync(pluginDir, { recursive: true });
|
||||
writeFileSync(claimPath, retained, "utf8");
|
||||
writeFileSync(sourcePath, refreshed, "utf8");
|
||||
chmodSync(claimPath, 0o000);
|
||||
|
||||
try {
|
||||
expect(migrateLegacyPluginModelCatalogs({ agentDir })).toEqual({
|
||||
detected: 0,
|
||||
migrated: 0,
|
||||
warnings: [expect.stringContaining("Could not read legacy provider catalog")],
|
||||
});
|
||||
expect(existsSync(claimPath)).toBe(true);
|
||||
expect(existsSync(sourcePath)).toBe(true);
|
||||
expect(existsSync(join(agentDir, "openclaw-agent.sqlite"))).toBe(false);
|
||||
} finally {
|
||||
chmodSync(claimPath, 0o600);
|
||||
}
|
||||
|
||||
expect(listPersistedPluginModelCatalogs(agentDir)).toEqual([
|
||||
{ pluginId: "zai", contents: refreshed },
|
||||
]);
|
||||
});
|
||||
|
||||
it("recovers a retained atomic migration claim after an interrupted upgrade", () => {
|
||||
const agentDir = createAgentDir();
|
||||
const contents = catalogContents("zai", "interrupted-released-provider-test-key");
|
||||
const pluginDir = join(agentDir, "plugins", "zai");
|
||||
const claimPath = join(pluginDir, "catalog.json.doctor-importing-previous-process");
|
||||
mkdirSync(pluginDir, { recursive: true });
|
||||
writeFileSync(claimPath, contents, "utf8");
|
||||
|
||||
expect(listPersistedPluginModelCatalogs(agentDir)).toEqual([{ pluginId: "zai", contents }]);
|
||||
expect(existsSync(claimPath)).toBe(false);
|
||||
});
|
||||
|
||||
it("recovers a retained claim before applying a newer canonical provider catalog", () => {
|
||||
const agentDir = createAgentDir();
|
||||
const retained = catalogContents("zai", "interrupted-released-provider-test-key");
|
||||
const refreshed = catalogContents("zai", "concurrently-refreshed-provider-test-key");
|
||||
const pluginDir = join(agentDir, "plugins", "zai");
|
||||
const sourcePath = join(pluginDir, "catalog.json");
|
||||
const claimPath = join(pluginDir, "catalog.json.doctor-importing-previous-process");
|
||||
mkdirSync(pluginDir, { recursive: true });
|
||||
writeFileSync(claimPath, retained, "utf8");
|
||||
writeFileSync(sourcePath, refreshed, "utf8");
|
||||
|
||||
expect(migrateLegacyPluginModelCatalogs({ agentDir })).toEqual({
|
||||
detected: 2,
|
||||
migrated: 2,
|
||||
warnings: [],
|
||||
});
|
||||
expect(listPersistedPluginModelCatalogs(agentDir)).toEqual([
|
||||
{ pluginId: "zai", contents: refreshed },
|
||||
]);
|
||||
expect(existsSync(claimPath)).toBe(false);
|
||||
expect(existsSync(sourcePath)).toBe(false);
|
||||
});
|
||||
|
||||
it("preserves conflicting retained claims without guessing which credential is newer", () => {
|
||||
const agentDir = createAgentDir();
|
||||
const pluginDir = join(agentDir, "plugins", "zai");
|
||||
const olderPath = join(pluginDir, "catalog.json.doctor-importing-zzz");
|
||||
const newerPath = join(pluginDir, "catalog.json.doctor-importing-aaa");
|
||||
mkdirSync(pluginDir, { recursive: true });
|
||||
writeFileSync(olderPath, catalogContents("zai", "older-provider-test-key"), "utf8");
|
||||
writeFileSync(newerPath, catalogContents("zai", "newer-provider-test-key"), "utf8");
|
||||
|
||||
expect(migrateLegacyPluginModelCatalogs({ agentDir })).toEqual({
|
||||
detected: 0,
|
||||
migrated: 0,
|
||||
warnings: [expect.stringContaining("Conflicting retained legacy provider catalogs")],
|
||||
});
|
||||
expect(existsSync(olderPath)).toBe(true);
|
||||
expect(existsSync(newerPath)).toBe(true);
|
||||
expect(existsSync(join(agentDir, "openclaw-agent.sqlite"))).toBe(false);
|
||||
});
|
||||
|
||||
it("does not let an unreadable sidecar hide other committed provider catalogs", () => {
|
||||
if (process.getuid?.() === 0) {
|
||||
return;
|
||||
}
|
||||
const agentDir = createAgentDir();
|
||||
const anthropic = catalogContents("anthropic", "available-anthropic-provider-test-key");
|
||||
replacePersistedPluginModelCatalogs({
|
||||
agentDir,
|
||||
pluginCatalogWrites: {
|
||||
[encodePluginModelCatalogRelativePath("anthropic")]: anthropic,
|
||||
},
|
||||
});
|
||||
const pluginDir = join(agentDir, "plugins", "zai");
|
||||
const sourcePath = join(agentDir, encodePluginModelCatalogRelativePath("zai"));
|
||||
mkdirSync(pluginDir, { recursive: true });
|
||||
writeFileSync(sourcePath, catalogContents("zai", "unreadable-provider-test-key"), "utf8");
|
||||
chmodSync(sourcePath, 0o000);
|
||||
|
||||
try {
|
||||
expect(migrateLegacyPluginModelCatalogs({ agentDir })).toEqual({
|
||||
detected: 0,
|
||||
migrated: 0,
|
||||
warnings: [expect.stringContaining("Could not read legacy provider catalog")],
|
||||
});
|
||||
expect(loadPersistedPluginModelCatalogs(agentDir)).toEqual({
|
||||
catalogs: [{ pluginId: "anthropic", contents: anthropic }],
|
||||
warnings: [expect.stringContaining("Could not read legacy provider catalog")],
|
||||
});
|
||||
expect(listPersistedPluginModelCatalogs(agentDir)).toEqual([
|
||||
{ pluginId: "anthropic", contents: anthropic },
|
||||
]);
|
||||
} finally {
|
||||
chmodSync(sourcePath, 0o600);
|
||||
}
|
||||
});
|
||||
|
||||
it("never migrates or deletes an unmarked user-authored catalog", () => {
|
||||
const agentDir = createAgentDir();
|
||||
const sourcePath = join(agentDir, encodePluginModelCatalogRelativePath("zai"));
|
||||
const contents = JSON.stringify({ providers: { zai: { apiKey: "user-authored-test-key" } } });
|
||||
mkdirSync(join(agentDir, "plugins", "zai"), { recursive: true });
|
||||
writeFileSync(sourcePath, contents, "utf8");
|
||||
|
||||
expect(listPersistedPluginModelCatalogs(agentDir)).toEqual([]);
|
||||
expect(existsSync(join(agentDir, "openclaw-agent.sqlite"))).toBe(false);
|
||||
expect(existsSync(sourcePath)).toBe(true);
|
||||
});
|
||||
|
||||
it("persists provider-owned catalogs in deterministic SQLite order", () => {
|
||||
const agentDir = createAgentDir();
|
||||
const zai = catalogContents("zai");
|
||||
const anthropic = catalogContents("anthropic");
|
||||
|
||||
expect(
|
||||
replacePersistedPluginModelCatalogs({
|
||||
agentDir,
|
||||
pluginCatalogWrites: {
|
||||
[encodePluginModelCatalogRelativePath("zai")]: zai,
|
||||
[encodePluginModelCatalogRelativePath("anthropic")]: anthropic,
|
||||
},
|
||||
}),
|
||||
).toBe(true);
|
||||
|
||||
expect(listPersistedPluginModelCatalogs(agentDir)).toEqual([
|
||||
{ pluginId: "anthropic", contents: anthropic },
|
||||
{ pluginId: "zai", contents: zai },
|
||||
]);
|
||||
expect(existsSync(join(agentDir, "openclaw-agent.sqlite"))).toBe(true);
|
||||
expect(existsSync(join(agentDir, "plugins"))).toBe(false);
|
||||
});
|
||||
|
||||
it("recognizes unchanged payloads and atomically removes stale provider rows", () => {
|
||||
const agentDir = createAgentDir();
|
||||
const zai = catalogContents("zai");
|
||||
const anthropic = catalogContents("anthropic");
|
||||
const both = {
|
||||
[encodePluginModelCatalogRelativePath("zai")]: zai,
|
||||
[encodePluginModelCatalogRelativePath("anthropic")]: anthropic,
|
||||
};
|
||||
|
||||
expect(replacePersistedPluginModelCatalogs({ agentDir, pluginCatalogWrites: both })).toBe(true);
|
||||
expect(replacePersistedPluginModelCatalogs({ agentDir, pluginCatalogWrites: both })).toBe(
|
||||
false,
|
||||
);
|
||||
expect(
|
||||
replacePersistedPluginModelCatalogs({
|
||||
agentDir,
|
||||
pluginCatalogWrites: {
|
||||
[encodePluginModelCatalogRelativePath("zai")]: zai,
|
||||
},
|
||||
}),
|
||||
).toBe(true);
|
||||
expect(listPersistedPluginModelCatalogs(agentDir)).toEqual([
|
||||
{ pluginId: "zai", contents: zai },
|
||||
]);
|
||||
});
|
||||
|
||||
it("rejects invalid planning keys without deleting the committed catalog", () => {
|
||||
const agentDir = createAgentDir();
|
||||
const zai = catalogContents("zai");
|
||||
replacePersistedPluginModelCatalogs({
|
||||
agentDir,
|
||||
pluginCatalogWrites: {
|
||||
[encodePluginModelCatalogRelativePath("zai")]: zai,
|
||||
},
|
||||
});
|
||||
|
||||
expect(() =>
|
||||
replacePersistedPluginModelCatalogs({
|
||||
agentDir,
|
||||
pluginCatalogWrites: { "../catalog.json": catalogContents("anthropic") },
|
||||
}),
|
||||
).toThrow("Invalid generated plugin model catalog key: ../catalog.json");
|
||||
expect(listPersistedPluginModelCatalogs(agentDir)).toEqual([
|
||||
{ pluginId: "zai", contents: zai },
|
||||
]);
|
||||
});
|
||||
|
||||
it("round-trips encoded plugin ownership without filesystem discovery", () => {
|
||||
const pluginId = "provider/with spaces";
|
||||
const key = encodePluginModelCatalogRelativePath(pluginId);
|
||||
|
||||
expect(key).toBe("plugins/provider%2Fwith%20spaces/catalog.json");
|
||||
expect(decodePluginModelCatalogRelativePathPluginId(key)).toBe(pluginId);
|
||||
expect(decodePluginModelCatalogRelativePathPluginId("../catalog.json")).toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -1,17 +1,572 @@
|
||||
/**
|
||||
* Generated plugin model catalog discovery.
|
||||
*
|
||||
* Catalog files live under agent profiles and let provider discovery reuse plugin-owned catalogs without loading runtimes.
|
||||
* The existing agent SQLite cache lets provider discovery reuse plugin-owned
|
||||
* catalogs without loading runtimes or creating parallel state files.
|
||||
*/
|
||||
import { existsSync, readdirSync } from "node:fs";
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { linkSync, readFileSync, readdirSync, renameSync, unlinkSync, type Dirent } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { normalizeProviderId } from "@openclaw/model-catalog-core/provider-id";
|
||||
import { executeSqliteQuerySync, getNodeSqliteKysely } from "../infra/kysely-sync.js";
|
||||
import type { PluginMetadataSnapshot } from "../plugins/plugin-metadata-snapshot.types.js";
|
||||
import { withOpenClawAgentDatabaseReadOnly } from "../state/openclaw-agent-db-readonly.js";
|
||||
import type { DB as OpenClawAgentKyselyDatabase } from "../state/openclaw-agent-db.generated.js";
|
||||
import { runOpenClawAgentWriteTransaction } from "../state/openclaw-agent-db.js";
|
||||
import {
|
||||
resolveAuthProfileDatabaseOwnerId,
|
||||
resolveAuthProfileDatabasePath,
|
||||
} from "./auth-profiles/sqlite.js";
|
||||
|
||||
// Generated catalog files live under each agent profile so provider model
|
||||
// discovery can reuse plugin-owned catalogs without loading plugin runtimes.
|
||||
// The in-memory planning key retains the established owner encoding; generated
|
||||
// payloads themselves are persisted only in the agent SQLite cache.
|
||||
const PLUGIN_MODEL_CATALOG_FILE = "catalog.json";
|
||||
export const PLUGIN_MODEL_CATALOG_GENERATED_BY = "openclaw-plugin-model-catalog-v1";
|
||||
const PLUGIN_MODEL_CATALOG_CACHE_SCOPE = "plugin-model-catalog-v1";
|
||||
const PLUGIN_MODEL_CATALOG_MIGRATION_SCOPE = "plugin-model-catalog-migration-v1";
|
||||
|
||||
/** Recognizes canonical catalogs and recoverable atomic migration claims. */
|
||||
export function isPluginModelCatalogMigrationFile(filename: string): boolean {
|
||||
return (
|
||||
filename === PLUGIN_MODEL_CATALOG_FILE ||
|
||||
filename.startsWith(`${PLUGIN_MODEL_CATALOG_FILE}.doctor-importing-`)
|
||||
);
|
||||
}
|
||||
|
||||
type PluginModelCatalogDatabase = Pick<OpenClawAgentKyselyDatabase, "cache_entries">;
|
||||
|
||||
type PersistedPluginModelCatalog = {
|
||||
pluginId: string;
|
||||
contents: string;
|
||||
};
|
||||
|
||||
type PersistedPluginModelCatalogLoadResult = {
|
||||
catalogs: PersistedPluginModelCatalog[];
|
||||
warnings: string[];
|
||||
};
|
||||
|
||||
function pluginModelCatalogDatabaseOptions(agentDir: string) {
|
||||
return {
|
||||
agentId: resolveAuthProfileDatabaseOwnerId(agentDir),
|
||||
path: resolveAuthProfileDatabasePath(agentDir),
|
||||
};
|
||||
}
|
||||
|
||||
function readPersistedPluginModelCatalogEntries(
|
||||
agentDir: string,
|
||||
scope: string,
|
||||
): PersistedPluginModelCatalog[] {
|
||||
const result = withOpenClawAgentDatabaseReadOnly((database) => {
|
||||
const kysely = getNodeSqliteKysely<PluginModelCatalogDatabase>(database.db);
|
||||
return executeSqliteQuerySync(
|
||||
database.db,
|
||||
kysely
|
||||
.selectFrom("cache_entries")
|
||||
.select(["key", "value_json"])
|
||||
.where("scope", "=", scope)
|
||||
.orderBy("key"),
|
||||
).rows.flatMap((row) =>
|
||||
row.value_json === null ? [] : [{ pluginId: row.key, contents: row.value_json }],
|
||||
);
|
||||
}, pluginModelCatalogDatabaseOptions(agentDir));
|
||||
return result.found ? result.value : [];
|
||||
}
|
||||
|
||||
function readPersistedPluginModelCatalogs(agentDir: string): PersistedPluginModelCatalog[] {
|
||||
return readPersistedPluginModelCatalogEntries(agentDir, PLUGIN_MODEL_CATALOG_CACHE_SCOPE);
|
||||
}
|
||||
|
||||
function readPersistedPluginModelCatalogMigrationPayloads(
|
||||
agentDir: string,
|
||||
): ReadonlyMap<string, string> {
|
||||
return new Map(
|
||||
readPersistedPluginModelCatalogEntries(agentDir, PLUGIN_MODEL_CATALOG_MIGRATION_SCOPE).map(
|
||||
(catalog) => [catalog.pluginId, catalog.contents],
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
function replacePersistedPluginModelCatalogEntries(params: {
|
||||
agentDir: string;
|
||||
planned: ReadonlyMap<string, string>;
|
||||
migrationPayloads?: ReadonlyMap<string, string>;
|
||||
deleteMissing?: boolean;
|
||||
}): boolean {
|
||||
if (
|
||||
params.planned.size === 0 &&
|
||||
(params.deleteMissing === false ||
|
||||
readPersistedPluginModelCatalogs(params.agentDir).length === 0)
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
const updatedAt = Date.now();
|
||||
return runOpenClawAgentWriteTransaction(
|
||||
(database) => {
|
||||
const kysely = getNodeSqliteKysely<PluginModelCatalogDatabase>(database.db);
|
||||
const existing = executeSqliteQuerySync(
|
||||
database.db,
|
||||
kysely
|
||||
.selectFrom("cache_entries")
|
||||
.select(["key", "value_json"])
|
||||
.where("scope", "=", PLUGIN_MODEL_CATALOG_CACHE_SCOPE),
|
||||
).rows;
|
||||
const existingByPluginId = new Map(existing.map((row) => [row.key, row.value_json]));
|
||||
const existingMigrationPayloads = params.migrationPayloads
|
||||
? new Map(
|
||||
executeSqliteQuerySync(
|
||||
database.db,
|
||||
kysely
|
||||
.selectFrom("cache_entries")
|
||||
.select(["key", "value_json"])
|
||||
.where("scope", "=", PLUGIN_MODEL_CATALOG_MIGRATION_SCOPE),
|
||||
).rows.map((row) => [row.key, row.value_json]),
|
||||
)
|
||||
: undefined;
|
||||
const upsertCacheEntry = (scope: string, pluginId: string, contents: string): void => {
|
||||
executeSqliteQuerySync(
|
||||
database.db,
|
||||
kysely
|
||||
.insertInto("cache_entries")
|
||||
.values({
|
||||
scope,
|
||||
key: pluginId,
|
||||
value_json: contents,
|
||||
blob: null,
|
||||
expires_at: null,
|
||||
updated_at: updatedAt,
|
||||
})
|
||||
.onConflict((conflict) =>
|
||||
conflict.columns(["scope", "key"]).doUpdateSet({
|
||||
value_json: contents,
|
||||
blob: null,
|
||||
expires_at: null,
|
||||
updated_at: updatedAt,
|
||||
}),
|
||||
),
|
||||
);
|
||||
};
|
||||
let changed = false;
|
||||
for (const [pluginId, contents] of params.planned) {
|
||||
const migrationPayload = params.migrationPayloads?.get(pluginId);
|
||||
if (migrationPayload && existingMigrationPayloads?.get(pluginId) === migrationPayload) {
|
||||
continue;
|
||||
}
|
||||
if (existingByPluginId.get(pluginId) !== contents) {
|
||||
upsertCacheEntry(PLUGIN_MODEL_CATALOG_CACHE_SCOPE, pluginId, contents);
|
||||
changed = true;
|
||||
}
|
||||
if (migrationPayload) {
|
||||
upsertCacheEntry(PLUGIN_MODEL_CATALOG_MIGRATION_SCOPE, pluginId, migrationPayload);
|
||||
changed = true;
|
||||
}
|
||||
}
|
||||
if (params.deleteMissing !== false) {
|
||||
for (const pluginId of existingByPluginId.keys()) {
|
||||
if (params.planned.has(pluginId)) {
|
||||
continue;
|
||||
}
|
||||
executeSqliteQuerySync(
|
||||
database.db,
|
||||
kysely
|
||||
.deleteFrom("cache_entries")
|
||||
.where("scope", "=", PLUGIN_MODEL_CATALOG_CACHE_SCOPE)
|
||||
.where("key", "=", pluginId),
|
||||
);
|
||||
changed = true;
|
||||
}
|
||||
}
|
||||
return changed;
|
||||
},
|
||||
pluginModelCatalogDatabaseOptions(params.agentDir),
|
||||
{
|
||||
operationLabel:
|
||||
params.deleteMissing === false
|
||||
? "plugin-model-catalog.migrate"
|
||||
: "plugin-model-catalog.replace",
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
type PluginModelCatalogMigrationResult = {
|
||||
detected: number;
|
||||
migrated: number;
|
||||
warnings: string[];
|
||||
};
|
||||
|
||||
function readLegacyPluginModelCatalog(pathname: string): string | null {
|
||||
try {
|
||||
return readFileSync(pathname, "utf8");
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code === "ENOENT") {
|
||||
return null;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
function hasCommittedExpectedPluginModelCatalogs(
|
||||
agentDir: string,
|
||||
expectedContents: ReadonlyMap<string, string>,
|
||||
): boolean {
|
||||
const committed = new Map(
|
||||
readPersistedPluginModelCatalogs(agentDir).map((catalog) => [
|
||||
catalog.pluginId,
|
||||
catalog.contents,
|
||||
]),
|
||||
);
|
||||
const migrationPayloads = readPersistedPluginModelCatalogMigrationPayloads(agentDir);
|
||||
for (const [pluginId, contents] of expectedContents) {
|
||||
if (committed.get(pluginId) !== contents && migrationPayloads.get(pluginId) !== contents) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
function hasCommittedMigratedPluginModelCatalog(
|
||||
agentDir: string,
|
||||
pluginId: string,
|
||||
contents: string,
|
||||
): boolean {
|
||||
const committed = readPersistedPluginModelCatalogs(agentDir).find(
|
||||
(catalog) => catalog.pluginId === pluginId,
|
||||
);
|
||||
return (
|
||||
committed?.contents === contents &&
|
||||
readPersistedPluginModelCatalogMigrationPayloads(agentDir).get(pluginId) === contents
|
||||
);
|
||||
}
|
||||
|
||||
function retireCommittedPluginModelCatalogMigration(params: {
|
||||
agentDir: string;
|
||||
pluginId: string;
|
||||
contents: string;
|
||||
}): boolean {
|
||||
return runOpenClawAgentWriteTransaction(
|
||||
(database) => {
|
||||
const kysely = getNodeSqliteKysely<PluginModelCatalogDatabase>(database.db);
|
||||
const committed = executeSqliteQuerySync(
|
||||
database.db,
|
||||
kysely
|
||||
.selectFrom("cache_entries")
|
||||
.select(["scope", "value_json"])
|
||||
.where("key", "=", params.pluginId)
|
||||
.where("scope", "in", [
|
||||
PLUGIN_MODEL_CATALOG_CACHE_SCOPE,
|
||||
PLUGIN_MODEL_CATALOG_MIGRATION_SCOPE,
|
||||
]),
|
||||
).rows;
|
||||
const contentsByScope = new Map(committed.map((row) => [row.scope, row.value_json]));
|
||||
if (
|
||||
contentsByScope.get(PLUGIN_MODEL_CATALOG_CACHE_SCOPE) !== params.contents ||
|
||||
contentsByScope.get(PLUGIN_MODEL_CATALOG_MIGRATION_SCOPE) !== params.contents
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
executeSqliteQuerySync(
|
||||
database.db,
|
||||
kysely
|
||||
.deleteFrom("cache_entries")
|
||||
.where("scope", "=", PLUGIN_MODEL_CATALOG_MIGRATION_SCOPE)
|
||||
.where("key", "=", params.pluginId),
|
||||
);
|
||||
return true;
|
||||
},
|
||||
pluginModelCatalogDatabaseOptions(params.agentDir),
|
||||
{ operationLabel: "plugin-model-catalog.retire-migration" },
|
||||
);
|
||||
}
|
||||
|
||||
function retireOrphanedPluginModelCatalogMigrations(params: {
|
||||
agentDir: string;
|
||||
protectedPluginIds?: ReadonlySet<string>;
|
||||
}): void {
|
||||
for (const [pluginId, contents] of readPersistedPluginModelCatalogMigrationPayloads(
|
||||
params.agentDir,
|
||||
)) {
|
||||
if (params.protectedPluginIds?.has(pluginId)) {
|
||||
continue;
|
||||
}
|
||||
// A crashed claim can leave recovery bytes after its source was removed.
|
||||
// The transaction retires them only while the canonical catalog still matches.
|
||||
retireCommittedPluginModelCatalogMigration({
|
||||
agentDir: params.agentDir,
|
||||
pluginId,
|
||||
contents,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/** Migrates released sidecars before runtime can read or replace agent SQLite state. */
|
||||
export function migrateLegacyPluginModelCatalogs(params: {
|
||||
agentDir: string;
|
||||
expectedContents?: ReadonlyMap<string, string>;
|
||||
beforeLegacyCatalogClaim?: (pathname: string) => void;
|
||||
}): PluginModelCatalogMigrationResult {
|
||||
const agentDir = path.resolve(params.agentDir);
|
||||
const pluginsDir = path.join(agentDir, "plugins");
|
||||
const warnings: string[] = [];
|
||||
let pluginDirs: Dirent[];
|
||||
try {
|
||||
pluginDirs = readdirSync(pluginsDir, { withFileTypes: true });
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code !== "ENOENT") {
|
||||
if (
|
||||
params.expectedContents &&
|
||||
!hasCommittedExpectedPluginModelCatalogs(agentDir, params.expectedContents)
|
||||
) {
|
||||
throw new Error("Could not inspect expected legacy provider catalogs", { cause: error });
|
||||
}
|
||||
return {
|
||||
detected: 0,
|
||||
migrated: 0,
|
||||
warnings: [`Could not inspect legacy provider catalogs: ${pluginsDir}`],
|
||||
};
|
||||
}
|
||||
if (
|
||||
params.expectedContents &&
|
||||
params.expectedContents.size > 0 &&
|
||||
!hasCommittedExpectedPluginModelCatalogs(agentDir, params.expectedContents)
|
||||
) {
|
||||
throw new Error("Legacy provider catalogs disappeared before migration", { cause: error });
|
||||
}
|
||||
retireOrphanedPluginModelCatalogMigrations({ agentDir });
|
||||
return { detected: 0, migrated: 0, warnings: [] };
|
||||
}
|
||||
|
||||
const legacyCatalogs: Array<{ pluginId: string; pathname: string; contents: string }> = [];
|
||||
const protectedMigrationPluginIds = new Set<string>();
|
||||
for (const pluginDir of pluginDirs) {
|
||||
if (!pluginDir.isDirectory()) {
|
||||
continue;
|
||||
}
|
||||
const relativePath = path.join("plugins", pluginDir.name, PLUGIN_MODEL_CATALOG_FILE);
|
||||
const pluginId = decodePluginModelCatalogRelativePathPluginId(relativePath);
|
||||
if (!pluginId) {
|
||||
continue;
|
||||
}
|
||||
const pluginPath = path.join(agentDir, "plugins", pluginDir.name);
|
||||
let catalogFiles: Dirent[];
|
||||
try {
|
||||
catalogFiles = readdirSync(pluginPath, { withFileTypes: true });
|
||||
} catch {
|
||||
// An uninspected owner may still contain the source for a recovery row.
|
||||
// Never retire its credential backup based on an incomplete scan.
|
||||
protectedMigrationPluginIds.add(pluginId);
|
||||
warnings.push(`Could not inspect legacy provider catalogs: ${pluginPath}`);
|
||||
continue;
|
||||
}
|
||||
const sourceFiles = catalogFiles
|
||||
.filter((entry) => entry.isFile() && isPluginModelCatalogMigrationFile(entry.name))
|
||||
.toSorted((left, right) => {
|
||||
if (left.name === PLUGIN_MODEL_CATALOG_FILE) {
|
||||
return 1;
|
||||
}
|
||||
if (right.name === PLUGIN_MODEL_CATALOG_FILE) {
|
||||
return -1;
|
||||
}
|
||||
return left.name.localeCompare(right.name);
|
||||
});
|
||||
if (sourceFiles.length > 0) {
|
||||
protectedMigrationPluginIds.add(pluginId);
|
||||
}
|
||||
const pluginLegacyCatalogs: Array<{
|
||||
pluginId: string;
|
||||
pathname: string;
|
||||
contents: string;
|
||||
}> = [];
|
||||
let hasUnreadableCatalog = false;
|
||||
for (const sourceFile of sourceFiles) {
|
||||
const pathname = path.join(pluginPath, sourceFile.name);
|
||||
let contents: string | null;
|
||||
try {
|
||||
contents = readLegacyPluginModelCatalog(pathname);
|
||||
} catch {
|
||||
hasUnreadableCatalog = true;
|
||||
warnings.push(`Could not read legacy provider catalog: ${pathname}`);
|
||||
continue;
|
||||
}
|
||||
if (contents === null) {
|
||||
warnings.push(`Legacy provider catalog disappeared before migration: ${pathname}`);
|
||||
continue;
|
||||
}
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(contents) as unknown;
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
if (isGeneratedPluginModelCatalog(parsed)) {
|
||||
pluginLegacyCatalogs.push({ pluginId, pathname, contents });
|
||||
}
|
||||
}
|
||||
if (hasUnreadableCatalog) {
|
||||
// One inaccessible claim can hold a newer credential than the readable
|
||||
// source; migrating either side would make later recovery roll back.
|
||||
continue;
|
||||
}
|
||||
if (
|
||||
!pluginLegacyCatalogs.some(
|
||||
(catalog) => path.basename(catalog.pathname) === PLUGIN_MODEL_CATALOG_FILE,
|
||||
) &&
|
||||
new Set(pluginLegacyCatalogs.map((catalog) => catalog.contents)).size > 1
|
||||
) {
|
||||
// Random claim names carry no generation order. Preserve every divergent
|
||||
// credential until Doctor or its operator can select an authoritative source.
|
||||
warnings.push(`Conflicting retained legacy provider catalogs: ${pluginPath}`);
|
||||
continue;
|
||||
}
|
||||
legacyCatalogs.push(...pluginLegacyCatalogs);
|
||||
}
|
||||
|
||||
retireOrphanedPluginModelCatalogMigrations({
|
||||
agentDir,
|
||||
protectedPluginIds: protectedMigrationPluginIds,
|
||||
});
|
||||
|
||||
if (params.expectedContents) {
|
||||
const observed = new Map(legacyCatalogs.map((catalog) => [catalog.pluginId, catalog.contents]));
|
||||
for (const [pluginId, contents] of params.expectedContents) {
|
||||
const observedContents = observed.get(pluginId);
|
||||
if (observedContents === contents) {
|
||||
continue;
|
||||
}
|
||||
if (
|
||||
observedContents === undefined &&
|
||||
hasCommittedExpectedPluginModelCatalogs(agentDir, new Map([[pluginId, contents]]))
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
if (observedContents !== contents) {
|
||||
throw new Error(`Legacy provider catalog changed before migration: ${pluginId}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (legacyCatalogs.length === 0) {
|
||||
return { detected: 0, migrated: 0, warnings };
|
||||
}
|
||||
|
||||
let migrated = 0;
|
||||
for (const catalog of legacyCatalogs) {
|
||||
if (
|
||||
readPersistedPluginModelCatalogMigrationPayloads(agentDir).get(catalog.pluginId) ===
|
||||
catalog.contents &&
|
||||
!hasCommittedMigratedPluginModelCatalog(agentDir, catalog.pluginId, catalog.contents)
|
||||
) {
|
||||
warnings.push(`Left superseded legacy provider catalog in place: ${catalog.pathname}`);
|
||||
continue;
|
||||
}
|
||||
params.beforeLegacyCatalogClaim?.(catalog.pathname);
|
||||
const claimPath = `${catalog.pathname}.doctor-importing-${process.pid}-${randomUUID()}`;
|
||||
try {
|
||||
// Claim the exact inode before checking or deleting it; a legacy writer
|
||||
// can safely recreate catalog.json without losing its newer credentials.
|
||||
renameSync(catalog.pathname, claimPath);
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code === "ENOENT") {
|
||||
if (
|
||||
hasCommittedExpectedPluginModelCatalogs(
|
||||
agentDir,
|
||||
new Map([[catalog.pluginId, catalog.contents]]),
|
||||
)
|
||||
) {
|
||||
migrated += 1;
|
||||
} else {
|
||||
warnings.push(
|
||||
`Legacy provider catalog was claimed before its migration was committed: ${catalog.pathname}`,
|
||||
);
|
||||
}
|
||||
} else {
|
||||
try {
|
||||
if (readLegacyPluginModelCatalog(catalog.pathname) === catalog.contents) {
|
||||
const migrationPayloads = new Map([[catalog.pluginId, catalog.contents]]);
|
||||
// A directory can permit reads while forbidding rename. Publish the
|
||||
// verified credential, retain its source, and retry cleanup later.
|
||||
replacePersistedPluginModelCatalogEntries({
|
||||
agentDir,
|
||||
planned: migrationPayloads,
|
||||
migrationPayloads,
|
||||
deleteMissing: false,
|
||||
});
|
||||
}
|
||||
} catch {
|
||||
// Preserve the original source and surface its migration warning.
|
||||
}
|
||||
warnings.push(`Could not remove migrated legacy provider catalog: ${catalog.pathname}`);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
if (readLegacyPluginModelCatalog(claimPath) !== catalog.contents) {
|
||||
throw new Error("legacy provider catalog changed before migration could claim it");
|
||||
}
|
||||
const migrationPayloads = new Map([[catalog.pluginId, catalog.contents]]);
|
||||
// Never publish scanned bytes until the exact source inode is claimed.
|
||||
// Catalog and temporary credential recovery commit in one transaction.
|
||||
replacePersistedPluginModelCatalogEntries({
|
||||
agentDir,
|
||||
planned: migrationPayloads,
|
||||
migrationPayloads,
|
||||
deleteMissing: false,
|
||||
});
|
||||
if (!hasCommittedMigratedPluginModelCatalog(agentDir, catalog.pluginId, catalog.contents)) {
|
||||
throw new Error("committed provider catalog changed before migration could remove it");
|
||||
}
|
||||
unlinkSync(claimPath);
|
||||
} catch {
|
||||
// linkSync restores without replacing a newer catalog.json created by a
|
||||
// concurrent writer; retain the claim if that canonical path already exists.
|
||||
let retainedPath = claimPath;
|
||||
try {
|
||||
linkSync(claimPath, catalog.pathname);
|
||||
retainedPath = catalog.pathname;
|
||||
unlinkSync(claimPath);
|
||||
} catch {
|
||||
// Either canonical or claimed bytes remain available for Doctor repair.
|
||||
}
|
||||
warnings.push(`Left changed legacy provider catalog in place: ${retainedPath}`);
|
||||
continue;
|
||||
}
|
||||
// Retire the temporary secret only when the same SQLite transaction proves
|
||||
// the authoritative catalog still contains every migrated credential.
|
||||
retireCommittedPluginModelCatalogMigration({
|
||||
agentDir,
|
||||
pluginId: catalog.pluginId,
|
||||
contents: catalog.contents,
|
||||
});
|
||||
migrated += 1;
|
||||
}
|
||||
return { detected: legacyCatalogs.length, migrated, warnings };
|
||||
}
|
||||
|
||||
/** Reads available provider catalogs without discarding legacy migration diagnostics. */
|
||||
export function loadPersistedPluginModelCatalogs(
|
||||
agentDir: string,
|
||||
): PersistedPluginModelCatalogLoadResult {
|
||||
const migration = migrateLegacyPluginModelCatalogs({ agentDir });
|
||||
return {
|
||||
catalogs: readPersistedPluginModelCatalogs(agentDir),
|
||||
warnings: migration.warnings,
|
||||
};
|
||||
}
|
||||
|
||||
/** Replaces rebuildable provider catalogs in the existing per-agent SQLite cache. */
|
||||
export function replacePersistedPluginModelCatalogs(params: {
|
||||
agentDir: string;
|
||||
pluginCatalogWrites: Readonly<Record<string, string>>;
|
||||
}): boolean {
|
||||
const planned = new Map<string, string>();
|
||||
for (const [relativePath, contents] of Object.entries(params.pluginCatalogWrites)) {
|
||||
const pluginId = decodePluginModelCatalogRelativePathPluginId(relativePath);
|
||||
if (!pluginId) {
|
||||
throw new Error(`Invalid generated plugin model catalog key: ${relativePath}`);
|
||||
}
|
||||
planned.set(pluginId, contents);
|
||||
}
|
||||
return replacePersistedPluginModelCatalogEntries({ agentDir: params.agentDir, planned });
|
||||
}
|
||||
|
||||
export type PluginModelCatalogMetadataSnapshot = Pick<PluginMetadataSnapshot, "owners"> & {
|
||||
index?: {
|
||||
@@ -23,19 +578,13 @@ export type PluginModelCatalogMetadataSnapshot = Pick<PluginMetadataSnapshot, "o
|
||||
normalizePluginId?: (pluginId: string) => string;
|
||||
};
|
||||
|
||||
type PluginModelCatalogFile = {
|
||||
path: string;
|
||||
pluginId: string;
|
||||
relativePath: string;
|
||||
};
|
||||
|
||||
/** Encodes the profile-relative path for a plugin-owned generated model catalog. */
|
||||
export function encodePluginModelCatalogRelativePath(pluginId: string): string {
|
||||
return `plugins/${encodeURIComponent(pluginId)}/${PLUGIN_MODEL_CATALOG_FILE}`;
|
||||
}
|
||||
|
||||
/** Returns true only for canonical profile-relative generated catalog paths. */
|
||||
export function isPluginModelCatalogRelativePath(relativePath: string): boolean {
|
||||
function isPluginModelCatalogRelativePath(relativePath: string): boolean {
|
||||
const parts = relativePath.split(/[\\/]/);
|
||||
return (
|
||||
!path.isAbsolute(relativePath) &&
|
||||
@@ -66,39 +615,6 @@ export function decodePluginModelCatalogRelativePathPluginId(
|
||||
}
|
||||
}
|
||||
|
||||
/** Lists deterministic generated catalog paths present in an agent profile. */
|
||||
export function listPluginModelCatalogRelativePaths(agentDir: string): string[] {
|
||||
const pluginsDir = path.join(agentDir, "plugins");
|
||||
let pluginDirs: Array<import("node:fs").Dirent>;
|
||||
try {
|
||||
pluginDirs = readdirSync(pluginsDir, { withFileTypes: true });
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
return pluginDirs
|
||||
.filter((entry) => entry.isDirectory())
|
||||
.map((entry) => path.join("plugins", entry.name, PLUGIN_MODEL_CATALOG_FILE))
|
||||
.filter(isPluginModelCatalogRelativePath)
|
||||
.toSorted((left, right) => left.localeCompare(right));
|
||||
}
|
||||
|
||||
/** Lists existing generated catalog files with decoded plugin ownership. */
|
||||
export function listPluginModelCatalogFiles(agentDir: string): PluginModelCatalogFile[] {
|
||||
return listPluginModelCatalogRelativePaths(agentDir)
|
||||
.map((relativePath) => {
|
||||
const pluginId = decodePluginModelCatalogRelativePathPluginId(relativePath);
|
||||
return pluginId
|
||||
? {
|
||||
path: path.join(agentDir, relativePath),
|
||||
pluginId,
|
||||
relativePath,
|
||||
}
|
||||
: undefined;
|
||||
})
|
||||
.filter((entry): entry is PluginModelCatalogFile => entry !== undefined)
|
||||
.filter((entry) => existsSync(entry.path));
|
||||
}
|
||||
|
||||
/** Detects model catalogs generated by OpenClaw rather than user-authored JSON. */
|
||||
export function isGeneratedPluginModelCatalog(value: unknown): boolean {
|
||||
return (
|
||||
|
||||
@@ -1,15 +1,22 @@
|
||||
// Model registry tests cover models.json auth modes and plugin-owned model
|
||||
// catalog shards.
|
||||
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
// Model registry tests cover models.json auth modes and SQLite-cached plugin catalogs.
|
||||
import { chmodSync, existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { dirname, join } from "node:path";
|
||||
import { getApiProvider } from "@openclaw/ai/internal/runtime";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { PLUGIN_MODEL_CATALOG_GENERATED_BY } from "../plugin-model-catalog.js";
|
||||
import {
|
||||
loadPersistedPluginModelCatalogs,
|
||||
PLUGIN_MODEL_CATALOG_GENERATED_BY,
|
||||
replacePersistedPluginModelCatalogs,
|
||||
} from "../plugin-model-catalog.js";
|
||||
import { AuthStorage } from "./auth-storage.js";
|
||||
import { getModelRegistryRuntime } from "./model-registry-runtime.js";
|
||||
import { ModelRegistry, type ProviderConfigInput } from "./model-registry.js";
|
||||
|
||||
function listPersistedPluginModelCatalogs(agentDir: string) {
|
||||
return loadPersistedPluginModelCatalogs(agentDir).catalogs;
|
||||
}
|
||||
|
||||
const PLUGIN_MODEL_CATALOG_FILE = "catalog.json";
|
||||
|
||||
const tempDirs: string[] = [];
|
||||
@@ -49,11 +56,15 @@ function writeModelsJsonWithPluginCatalogs(params: {
|
||||
tempDirs.push(dir);
|
||||
const file = join(dir, "models.json");
|
||||
writeFileSync(file, JSON.stringify(params.root, null, 2), "utf-8");
|
||||
for (const pluginCatalog of params.pluginCatalogs) {
|
||||
const pluginFile = join(dir, pluginCatalog.pluginRelativePath);
|
||||
mkdirSync(dirname(pluginFile), { recursive: true });
|
||||
writeFileSync(pluginFile, JSON.stringify(pluginCatalog.pluginCatalog, null, 2), "utf-8");
|
||||
}
|
||||
replacePersistedPluginModelCatalogs({
|
||||
agentDir: dir,
|
||||
pluginCatalogWrites: Object.fromEntries(
|
||||
params.pluginCatalogs.map((pluginCatalog) => [
|
||||
pluginCatalog.pluginRelativePath,
|
||||
JSON.stringify(pluginCatalog.pluginCatalog, null, 2),
|
||||
]),
|
||||
),
|
||||
});
|
||||
return file;
|
||||
}
|
||||
|
||||
@@ -64,7 +75,7 @@ function pluginOwnerSnapshot(providerId: string, pluginId: string, enabled = tru
|
||||
function pluginOwnerSnapshotEntries(
|
||||
entries: Array<{ providerId: string; pluginId: string; enabled?: boolean }>,
|
||||
) {
|
||||
// The registry only trusts generated provider shards that are still owned by
|
||||
// The registry only trusts generated provider catalogs that are still owned by
|
||||
// an enabled plugin in the current metadata snapshot.
|
||||
return {
|
||||
index: {
|
||||
@@ -314,7 +325,38 @@ describe("ModelRegistry models.json auth", () => {
|
||||
expect(registry.getAvailable().map((model) => model.id)).toEqual(["example-model"]);
|
||||
});
|
||||
|
||||
it("loads provider models from generated plugin catalog shards", () => {
|
||||
it("automatically migrates released provider models before the first registry load", async () => {
|
||||
const modelsPath = writeModelsJson({ providers: {} });
|
||||
const agentDir = dirname(modelsPath);
|
||||
const catalogPath = join(agentDir, "plugins", "zai", PLUGIN_MODEL_CATALOG_FILE);
|
||||
const contents = JSON.stringify({
|
||||
generatedBy: PLUGIN_MODEL_CATALOG_GENERATED_BY,
|
||||
providers: {
|
||||
zai: {
|
||||
baseUrl: "https://api.z.ai/api/paas/v4",
|
||||
api: "openai-completions",
|
||||
apiKey: "released-zai-provider-test-key",
|
||||
models: [{ id: "glm-5.1", name: "GLM 5.1" }],
|
||||
},
|
||||
},
|
||||
});
|
||||
mkdirSync(dirname(catalogPath), { recursive: true });
|
||||
writeFileSync(catalogPath, contents, "utf8");
|
||||
|
||||
const registry = ModelRegistry.create(AuthStorage.inMemory(), modelsPath, {
|
||||
pluginMetadataSnapshot: pluginOwnerSnapshot("zai", "zai"),
|
||||
});
|
||||
|
||||
expect(registry.getError()).toBeUndefined();
|
||||
expect(registry.find("zai", "glm-5.1")?.name).toBe("GLM 5.1");
|
||||
await expect(registry.getApiKeyForProvider("zai")).resolves.toBe(
|
||||
"released-zai-provider-test-key",
|
||||
);
|
||||
expect(listPersistedPluginModelCatalogs(agentDir)).toEqual([{ pluginId: "zai", contents }]);
|
||||
expect(existsSync(catalogPath)).toBe(false);
|
||||
});
|
||||
|
||||
it("loads provider models from the SQLite-backed generated plugin catalog", () => {
|
||||
const modelsPath = writeModelsJsonWithPluginCatalog({
|
||||
root: { providers: {} },
|
||||
pluginRelativePath: join("plugins", "zai", PLUGIN_MODEL_CATALOG_FILE),
|
||||
@@ -341,6 +383,81 @@ describe("ModelRegistry models.json auth", () => {
|
||||
expect(registry.find("zai", "glm-5.1")?.name).toBe("GLM 5.1");
|
||||
});
|
||||
|
||||
it("reports an unreadable legacy catalog while preserving healthy provider models", () => {
|
||||
if (process.getuid?.() === 0) {
|
||||
return;
|
||||
}
|
||||
const modelsPath = writeModelsJsonWithPluginCatalog({
|
||||
root: {
|
||||
providers: {
|
||||
custom: {
|
||||
baseUrl: "https://models.example/v1",
|
||||
api: "openai-completions",
|
||||
apiKey: "authored-provider-test-key",
|
||||
models: [{ id: "authored-model", name: "Authored Model" }],
|
||||
},
|
||||
},
|
||||
},
|
||||
pluginRelativePath: join("plugins", "anthropic", PLUGIN_MODEL_CATALOG_FILE),
|
||||
pluginCatalog: {
|
||||
generatedBy: PLUGIN_MODEL_CATALOG_GENERATED_BY,
|
||||
providers: {
|
||||
anthropic: {
|
||||
baseUrl: "https://anthropic.example/v1",
|
||||
api: "anthropic-messages",
|
||||
apiKey: "healthy-provider-test-key",
|
||||
models: [{ id: "healthy-model", name: "Healthy Model" }],
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
const sourcePath = join(dirname(modelsPath), "plugins", "zai", PLUGIN_MODEL_CATALOG_FILE);
|
||||
mkdirSync(dirname(sourcePath), { recursive: true });
|
||||
writeFileSync(
|
||||
sourcePath,
|
||||
JSON.stringify({
|
||||
generatedBy: PLUGIN_MODEL_CATALOG_GENERATED_BY,
|
||||
providers: { zai: { apiKey: "unreadable-released-provider-test-key" } },
|
||||
}),
|
||||
);
|
||||
chmodSync(sourcePath, 0o000);
|
||||
|
||||
try {
|
||||
const registry = ModelRegistry.create(AuthStorage.inMemory(), modelsPath, {
|
||||
pluginMetadataSnapshot: pluginOwnerSnapshotEntries([
|
||||
{ providerId: "anthropic", pluginId: "anthropic" },
|
||||
{ providerId: "zai", pluginId: "zai" },
|
||||
]),
|
||||
});
|
||||
|
||||
expect(registry.getError()).toContain("Could not read legacy provider catalog");
|
||||
expect(registry.find("custom", "authored-model")?.name).toBe("Authored Model");
|
||||
expect(registry.find("anthropic", "healthy-model")?.name).toBe("Healthy Model");
|
||||
expect(existsSync(sourcePath)).toBe(true);
|
||||
} finally {
|
||||
chmodSync(sourcePath, 0o600);
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps authored provider models available when the plugin catalog database is corrupt", () => {
|
||||
const modelsPath = writeModelsJson({
|
||||
providers: {
|
||||
custom: {
|
||||
baseUrl: "https://models.example/v1",
|
||||
api: "openai-completions",
|
||||
apiKey: "authored-provider-test-key",
|
||||
models: [{ id: "authored-model", name: "Authored Model" }],
|
||||
},
|
||||
},
|
||||
});
|
||||
writeFileSync(join(dirname(modelsPath), "openclaw-agent.sqlite"), "not a SQLite database");
|
||||
|
||||
const registry = ModelRegistry.create(AuthStorage.inMemory(), modelsPath);
|
||||
|
||||
expect(registry.find("custom", "authored-model")?.name).toBe("Authored Model");
|
||||
expect(registry.getError()).toContain("Failed to load generated plugin model catalogs");
|
||||
});
|
||||
|
||||
it("tracks explicit max-token provenance across authored and generated catalogs", () => {
|
||||
const modelsPath = writeModelsJsonWithPluginCatalog({
|
||||
root: {
|
||||
@@ -485,7 +602,7 @@ describe("ModelRegistry models.json auth", () => {
|
||||
expect(availableRefs).toContain("nvidia/explicit-empty");
|
||||
});
|
||||
|
||||
it("isolates invalid generated plugin catalog shards from valid models", () => {
|
||||
it("isolates invalid SQLite-cached plugin catalogs from valid models", () => {
|
||||
const modelsPath = writeModelsJsonWithPluginCatalogs({
|
||||
root: {
|
||||
providers: {
|
||||
@@ -543,7 +660,7 @@ describe("ModelRegistry models.json auth", () => {
|
||||
expect(registry.find("google-vertex", "gemini-3.1-pro-preview")).toBeUndefined();
|
||||
});
|
||||
|
||||
it("preserves model params from generated plugin catalog shards", () => {
|
||||
it("preserves model params from SQLite-cached plugin catalogs", () => {
|
||||
const modelsPath = writeModelsJsonWithPluginCatalog({
|
||||
root: { providers: {} },
|
||||
pluginRelativePath: join("plugins", "amazon-bedrock", PLUGIN_MODEL_CATALOG_FILE),
|
||||
@@ -576,8 +693,8 @@ describe("ModelRegistry models.json auth", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("ignores non-generated plugin catalog files", () => {
|
||||
// Plugin catalog shards are codegen artifacts; hand-written lookalikes must
|
||||
it("ignores non-generated SQLite plugin catalog entries", () => {
|
||||
// Plugin catalogs are codegen artifacts; unmarked lookalikes must
|
||||
// not extend the provider registry.
|
||||
const modelsPath = writeModelsJsonWithPluginCatalog({
|
||||
root: { providers: {} },
|
||||
|
||||
@@ -24,7 +24,7 @@ import { resolveModelPluginMetadataSnapshot } from "../model-discovery-context.j
|
||||
import {
|
||||
filterGeneratedPluginModelCatalogProviders,
|
||||
isGeneratedPluginModelCatalog,
|
||||
listPluginModelCatalogFiles,
|
||||
loadPersistedPluginModelCatalogs,
|
||||
type PluginModelCatalogMetadataSnapshot,
|
||||
} from "../plugin-model-catalog.js";
|
||||
import { getAuthStorageOAuthProviderRegistry } from "./auth-storage-oauth-registry.js";
|
||||
@@ -448,8 +448,8 @@ export class ModelRegistry {
|
||||
}
|
||||
|
||||
private loadModels(): void {
|
||||
// Load configured models and request settings from models.json plus
|
||||
// generated plugin-owned catalog shards under the agent plugin state.
|
||||
// Keep authored models.json separate from rebuildable provider catalogs
|
||||
// owned by the agent SQLite cache.
|
||||
const { models: customModels, error } = this.modelsJsonPath
|
||||
? this.loadCustomModels(this.modelsJsonPath)
|
||||
: emptyCustomModelsResult();
|
||||
@@ -477,18 +477,19 @@ export class ModelRegistry {
|
||||
modelsJsonPath: string,
|
||||
options: {
|
||||
catalogPluginId?: string;
|
||||
contents?: string;
|
||||
includePluginCatalogs?: boolean;
|
||||
requireGeneratedCatalog?: boolean;
|
||||
} = {
|
||||
includePluginCatalogs: true,
|
||||
},
|
||||
): CustomModelsResult {
|
||||
if (!existsSync(modelsJsonPath)) {
|
||||
if (options.contents === undefined && !existsSync(modelsJsonPath)) {
|
||||
return emptyCustomModelsResult();
|
||||
}
|
||||
|
||||
try {
|
||||
const content = readFileSync(modelsJsonPath, "utf-8");
|
||||
const content = options.contents ?? readFileSync(modelsJsonPath, "utf-8");
|
||||
const parsed = JSON.parse(stripJsonComments(content)) as unknown;
|
||||
if (options.requireGeneratedCatalog === true && !isGeneratedPluginModelCatalog(parsed)) {
|
||||
return emptyCustomModelsResult();
|
||||
@@ -537,12 +538,28 @@ export class ModelRegistry {
|
||||
);
|
||||
const pluginCatalogErrors: string[] = [];
|
||||
if (options.includePluginCatalogs !== false) {
|
||||
for (const pluginCatalog of listPluginModelCatalogFiles(dirname(modelsJsonPath))) {
|
||||
const pluginResult = this.loadCustomModels(pluginCatalog.path, {
|
||||
catalogPluginId: pluginCatalog.pluginId,
|
||||
includePluginCatalogs: false,
|
||||
requireGeneratedCatalog: true,
|
||||
});
|
||||
let pluginCatalogs: ReturnType<typeof loadPersistedPluginModelCatalogs>["catalogs"] = [];
|
||||
try {
|
||||
const loaded = loadPersistedPluginModelCatalogs(dirname(modelsJsonPath));
|
||||
pluginCatalogs = loaded.catalogs;
|
||||
pluginCatalogErrors.push(...loaded.warnings);
|
||||
} catch (error) {
|
||||
pluginCatalogErrors.push(
|
||||
`Failed to load generated plugin model catalogs: ${
|
||||
error instanceof Error ? error.message : String(error)
|
||||
}`,
|
||||
);
|
||||
}
|
||||
for (const pluginCatalog of pluginCatalogs) {
|
||||
const pluginResult = this.loadCustomModels(
|
||||
`sqlite:plugin-model-catalog/${pluginCatalog.pluginId}`,
|
||||
{
|
||||
catalogPluginId: pluginCatalog.pluginId,
|
||||
contents: pluginCatalog.contents,
|
||||
includePluginCatalogs: false,
|
||||
requireGeneratedCatalog: true,
|
||||
},
|
||||
);
|
||||
if (pluginResult.error) {
|
||||
pluginCatalogErrors.push(pluginResult.error);
|
||||
continue;
|
||||
|
||||
279
src/commands/doctor-plugin-model-catalog.test.ts
Normal file
279
src/commands/doctor-plugin-model-catalog.test.ts
Normal file
@@ -0,0 +1,279 @@
|
||||
// Doctor preserves provider credentials while migrating released catalog sidecars to SQLite.
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
encodePluginModelCatalogRelativePath,
|
||||
loadPersistedPluginModelCatalogs,
|
||||
PLUGIN_MODEL_CATALOG_GENERATED_BY,
|
||||
replacePersistedPluginModelCatalogs,
|
||||
} from "../agents/plugin-model-catalog.js";
|
||||
import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
import type { RuntimeEnv } from "../runtime.js";
|
||||
import { closeOpenClawAgentDatabasesForTest } from "../state/openclaw-agent-db.js";
|
||||
import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js";
|
||||
import { maybeMigrateLegacyPluginModelCatalogs } from "./doctor-plugin-model-catalog.js";
|
||||
import type { DoctorPrompter } from "./doctor-prompter.js";
|
||||
|
||||
function listPersistedPluginModelCatalogs(agentDir: string) {
|
||||
return loadPersistedPluginModelCatalogs(agentDir).catalogs;
|
||||
}
|
||||
|
||||
const tempDirs: string[] = [];
|
||||
|
||||
function createAgentDir(): string {
|
||||
const agentDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-doctor-model-catalog-"));
|
||||
tempDirs.push(agentDir);
|
||||
return agentDir;
|
||||
}
|
||||
|
||||
function generatedCatalog(provider: string, apiKey = "persisted-test-key"): string {
|
||||
return `${JSON.stringify(
|
||||
{
|
||||
generatedBy: PLUGIN_MODEL_CATALOG_GENERATED_BY,
|
||||
providers: {
|
||||
[provider]: {
|
||||
api: "openai-completions",
|
||||
baseUrl: `https://${provider}.example/v1`,
|
||||
apiKey,
|
||||
models: [{ id: `${provider}-model`, name: `${provider} model` }],
|
||||
},
|
||||
},
|
||||
},
|
||||
null,
|
||||
2,
|
||||
)}\n`;
|
||||
}
|
||||
|
||||
function writeLegacyCatalog(agentDir: string, pluginId: string, contents: string): string {
|
||||
const sourcePath = path.join(agentDir, encodePluginModelCatalogRelativePath(pluginId));
|
||||
fs.mkdirSync(path.dirname(sourcePath), { recursive: true });
|
||||
fs.writeFileSync(sourcePath, contents, "utf8");
|
||||
return sourcePath;
|
||||
}
|
||||
|
||||
function prompter(shouldRepair: boolean): DoctorPrompter {
|
||||
return {
|
||||
confirmAutoFix: vi.fn(async () => shouldRepair),
|
||||
shouldRepair,
|
||||
} as unknown as DoctorPrompter;
|
||||
}
|
||||
|
||||
function migrationParams(agentDirs: string[], shouldRepair = true) {
|
||||
return {
|
||||
cfg: {} as OpenClawConfig,
|
||||
agentDirs,
|
||||
prompter: prompter(shouldRepair),
|
||||
runtime: { log: vi.fn(), error: vi.fn(), exit: vi.fn() } as unknown as RuntimeEnv,
|
||||
note: vi.fn(),
|
||||
};
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
closeOpenClawAgentDatabasesForTest();
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
for (const agentDir of tempDirs.splice(0)) {
|
||||
fs.rmSync(agentDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
describe("doctor generated plugin model catalog migration", () => {
|
||||
it("does not create agent SQLite for a legacy-free profile", async () => {
|
||||
const agentDir = createAgentDir();
|
||||
|
||||
await expect(
|
||||
maybeMigrateLegacyPluginModelCatalogs(migrationParams([agentDir])),
|
||||
).resolves.toEqual({ detected: 0, migrated: 0, warnings: [] });
|
||||
expect(fs.existsSync(path.join(agentDir, "openclaw-agent.sqlite"))).toBe(false);
|
||||
});
|
||||
|
||||
it("verifies a shipped sidecar and preserves its provider credential in SQLite", async () => {
|
||||
const agentDir = createAgentDir();
|
||||
const contents = generatedCatalog("zai", "persisted-zai-test-key");
|
||||
const sourcePath = writeLegacyCatalog(agentDir, "zai", contents);
|
||||
|
||||
await expect(
|
||||
maybeMigrateLegacyPluginModelCatalogs(migrationParams([agentDir])),
|
||||
).resolves.toEqual({ detected: 1, migrated: 1, warnings: [] });
|
||||
|
||||
expect(listPersistedPluginModelCatalogs(agentDir)).toEqual([{ pluginId: "zai", contents }]);
|
||||
expect(fs.existsSync(sourcePath)).toBe(false);
|
||||
});
|
||||
|
||||
it("discovers and repairs a retained migration claim after an interrupted upgrade", async () => {
|
||||
const agentDir = createAgentDir();
|
||||
const contents = generatedCatalog("zai", "interrupted-zai-provider-test-key");
|
||||
const pluginDir = path.join(agentDir, "plugins", "zai");
|
||||
const claimPath = path.join(pluginDir, "catalog.json.doctor-importing-previous-process");
|
||||
fs.mkdirSync(pluginDir, { recursive: true });
|
||||
fs.writeFileSync(claimPath, contents, "utf8");
|
||||
|
||||
await expect(
|
||||
maybeMigrateLegacyPluginModelCatalogs(migrationParams([agentDir])),
|
||||
).resolves.toEqual({ detected: 1, migrated: 1, warnings: [] });
|
||||
expect(listPersistedPluginModelCatalogs(agentDir)).toEqual([{ pluginId: "zai", contents }]);
|
||||
expect(fs.existsSync(claimPath)).toBe(false);
|
||||
});
|
||||
|
||||
it("reports and preserves conflicting retained migration claims", async () => {
|
||||
const agentDir = createAgentDir();
|
||||
const pluginDir = path.join(agentDir, "plugins", "zai");
|
||||
const firstPath = path.join(pluginDir, "catalog.json.doctor-importing-first");
|
||||
const secondPath = path.join(pluginDir, "catalog.json.doctor-importing-second");
|
||||
fs.mkdirSync(pluginDir, { recursive: true });
|
||||
fs.writeFileSync(firstPath, generatedCatalog("zai", "first-provider-test-key"), "utf8");
|
||||
fs.writeFileSync(secondPath, generatedCatalog("zai", "second-provider-test-key"), "utf8");
|
||||
const params = migrationParams([agentDir]);
|
||||
|
||||
await expect(maybeMigrateLegacyPluginModelCatalogs(params)).resolves.toEqual({
|
||||
detected: 0,
|
||||
migrated: 0,
|
||||
warnings: [expect.stringContaining("Conflicting retained legacy provider catalogs")],
|
||||
});
|
||||
expect(params.runtime.error).toHaveBeenCalledWith(
|
||||
expect.stringContaining("Conflicting retained legacy provider catalogs"),
|
||||
);
|
||||
expect(fs.existsSync(firstPath)).toBe(true);
|
||||
expect(fs.existsSync(secondPath)).toBe(true);
|
||||
expect(fs.existsSync(path.join(agentDir, "openclaw-agent.sqlite"))).toBe(false);
|
||||
});
|
||||
|
||||
it("does not migrate a provider while one of its retained claims is unreadable", async () => {
|
||||
if (process.getuid?.() === 0) {
|
||||
return;
|
||||
}
|
||||
const agentDir = createAgentDir();
|
||||
const pluginDir = path.join(agentDir, "plugins", "zai");
|
||||
const claimPath = path.join(pluginDir, "catalog.json.doctor-importing-previous-process");
|
||||
const sourcePath = path.join(pluginDir, "catalog.json");
|
||||
fs.mkdirSync(pluginDir, { recursive: true });
|
||||
fs.writeFileSync(claimPath, generatedCatalog("zai", "retained-provider-test-key"), "utf8");
|
||||
fs.writeFileSync(sourcePath, generatedCatalog("zai", "canonical-provider-test-key"), "utf8");
|
||||
fs.chmodSync(claimPath, 0o000);
|
||||
|
||||
try {
|
||||
await expect(
|
||||
maybeMigrateLegacyPluginModelCatalogs(migrationParams([agentDir])),
|
||||
).resolves.toEqual({
|
||||
detected: 0,
|
||||
migrated: 0,
|
||||
warnings: [expect.stringContaining("Could not read legacy provider catalog")],
|
||||
});
|
||||
expect(fs.existsSync(claimPath)).toBe(true);
|
||||
expect(fs.existsSync(sourcePath)).toBe(true);
|
||||
expect(fs.existsSync(path.join(agentDir, "openclaw-agent.sqlite"))).toBe(false);
|
||||
} finally {
|
||||
fs.chmodSync(claimPath, 0o600);
|
||||
}
|
||||
});
|
||||
|
||||
it("migrates every configured agent without mixing provider ownership", async () => {
|
||||
const mainDir = createAgentDir();
|
||||
const workerDir = createAgentDir();
|
||||
const mainContents = generatedCatalog("openai", "main-provider-test-key");
|
||||
const workerContents = generatedCatalog("anthropic", "worker-provider-test-key");
|
||||
const mainPath = writeLegacyCatalog(mainDir, "openai", mainContents);
|
||||
const workerPath = writeLegacyCatalog(workerDir, "anthropic", workerContents);
|
||||
|
||||
await expect(
|
||||
maybeMigrateLegacyPluginModelCatalogs(migrationParams([workerDir, mainDir])),
|
||||
).resolves.toEqual({ detected: 2, migrated: 2, warnings: [] });
|
||||
|
||||
expect(listPersistedPluginModelCatalogs(mainDir)).toEqual([
|
||||
{ pluginId: "openai", contents: mainContents },
|
||||
]);
|
||||
expect(listPersistedPluginModelCatalogs(workerDir)).toEqual([
|
||||
{ pluginId: "anthropic", contents: workerContents },
|
||||
]);
|
||||
expect(fs.existsSync(mainPath)).toBe(false);
|
||||
expect(fs.existsSync(workerPath)).toBe(false);
|
||||
});
|
||||
|
||||
it("preserves legacy credentials and does not create SQLite when repair is declined", async () => {
|
||||
const agentDir = createAgentDir();
|
||||
const contents = generatedCatalog("zai");
|
||||
const sourcePath = writeLegacyCatalog(agentDir, "zai", contents);
|
||||
const params = migrationParams([agentDir], false);
|
||||
|
||||
await expect(maybeMigrateLegacyPluginModelCatalogs(params)).resolves.toEqual({
|
||||
detected: 1,
|
||||
migrated: 0,
|
||||
warnings: [],
|
||||
});
|
||||
|
||||
expect(params.prompter.confirmAutoFix).toHaveBeenCalledOnce();
|
||||
expect(fs.readFileSync(sourcePath, "utf8")).toBe(contents);
|
||||
expect(fs.existsSync(path.join(agentDir, "openclaw-agent.sqlite"))).toBe(false);
|
||||
});
|
||||
|
||||
it("ignores user-authored and malformed catalog lookalikes", async () => {
|
||||
const agentDir = createAgentDir();
|
||||
const authoredPath = writeLegacyCatalog(
|
||||
agentDir,
|
||||
"authored",
|
||||
JSON.stringify({ providers: { authored: { apiKey: "do-not-touch" } } }),
|
||||
);
|
||||
const malformedPath = writeLegacyCatalog(agentDir, "malformed", "{not-json");
|
||||
|
||||
await expect(
|
||||
maybeMigrateLegacyPluginModelCatalogs(migrationParams([agentDir])),
|
||||
).resolves.toEqual({ detected: 0, migrated: 0, warnings: [] });
|
||||
|
||||
expect(fs.existsSync(authoredPath)).toBe(true);
|
||||
expect(fs.existsSync(malformedPath)).toBe(true);
|
||||
expect(fs.existsSync(path.join(agentDir, "openclaw-agent.sqlite"))).toBe(false);
|
||||
});
|
||||
|
||||
it("migrates readable providers when another legacy catalog is unreadable", async () => {
|
||||
if (process.getuid?.() === 0) {
|
||||
return;
|
||||
}
|
||||
const agentDir = createAgentDir();
|
||||
const unreadablePath = writeLegacyCatalog(agentDir, "zai", generatedCatalog("zai"));
|
||||
const readableContents = generatedCatalog("anthropic", "readable-provider-test-key");
|
||||
const readablePath = writeLegacyCatalog(agentDir, "anthropic", readableContents);
|
||||
fs.chmodSync(unreadablePath, 0o000);
|
||||
|
||||
try {
|
||||
const params = migrationParams([agentDir]);
|
||||
await expect(maybeMigrateLegacyPluginModelCatalogs(params)).resolves.toEqual({
|
||||
detected: 1,
|
||||
migrated: 1,
|
||||
warnings: [expect.stringContaining("Could not read legacy provider catalog")],
|
||||
});
|
||||
expect(params.runtime.error).toHaveBeenCalledWith(
|
||||
expect.stringContaining("Could not read legacy provider catalog"),
|
||||
);
|
||||
expect(listPersistedPluginModelCatalogs(agentDir)).toEqual([
|
||||
{ pluginId: "anthropic", contents: readableContents },
|
||||
]);
|
||||
expect(fs.existsSync(unreadablePath)).toBe(true);
|
||||
expect(fs.existsSync(readablePath)).toBe(false);
|
||||
} finally {
|
||||
fs.chmodSync(unreadablePath, 0o600);
|
||||
}
|
||||
});
|
||||
|
||||
it("preserves the released provider credential over a conflicting regenerated catalog", async () => {
|
||||
const agentDir = createAgentDir();
|
||||
const regenerated = generatedCatalog("zai", "regenerated-sqlite-test-key");
|
||||
const released = generatedCatalog("zai", "released-sidecar-test-key");
|
||||
replacePersistedPluginModelCatalogs({
|
||||
agentDir,
|
||||
pluginCatalogWrites: {
|
||||
[encodePluginModelCatalogRelativePath("zai")]: regenerated,
|
||||
},
|
||||
});
|
||||
const sourcePath = writeLegacyCatalog(agentDir, "zai", released);
|
||||
|
||||
await expect(
|
||||
maybeMigrateLegacyPluginModelCatalogs(migrationParams([agentDir])),
|
||||
).resolves.toEqual({ detected: 1, migrated: 1, warnings: [] });
|
||||
|
||||
expect(listPersistedPluginModelCatalogs(agentDir)).toEqual([
|
||||
{ pluginId: "zai", contents: released },
|
||||
]);
|
||||
expect(fs.existsSync(sourcePath)).toBe(false);
|
||||
});
|
||||
});
|
||||
226
src/commands/doctor-plugin-model-catalog.ts
Normal file
226
src/commands/doctor-plugin-model-catalog.ts
Normal file
@@ -0,0 +1,226 @@
|
||||
/** Doctor-owned migration of shipped generated provider catalogs into agent SQLite. */
|
||||
import type { Dirent } from "node:fs";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { note } from "../../packages/terminal-core/src/note.js";
|
||||
import { listAgentIds, resolveAgentDir, resolveDefaultAgentDir } from "../agents/agent-scope.js";
|
||||
import {
|
||||
decodePluginModelCatalogRelativePathPluginId,
|
||||
isGeneratedPluginModelCatalog,
|
||||
isPluginModelCatalogMigrationFile,
|
||||
migrateLegacyPluginModelCatalogs,
|
||||
} from "../agents/plugin-model-catalog.js";
|
||||
import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
import { privateFileStore } from "../infra/private-file-store.js";
|
||||
import type { RuntimeEnv } from "../runtime.js";
|
||||
import { shortenHomePath } from "../utils.js";
|
||||
import type { DoctorPrompter } from "./doctor-prompter.js";
|
||||
|
||||
type LegacyPluginModelCatalogMigration = {
|
||||
agentDir: string;
|
||||
pluginId: string;
|
||||
relativePath: string;
|
||||
contents: string;
|
||||
};
|
||||
|
||||
function resolveMigrationAgentDirs(params: {
|
||||
cfg: OpenClawConfig;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
agentDirs?: readonly string[];
|
||||
}): string[] {
|
||||
if (params.agentDirs) {
|
||||
return [...new Set(params.agentDirs)].toSorted((left, right) => left.localeCompare(right));
|
||||
}
|
||||
const env = params.env ?? process.env;
|
||||
return [
|
||||
...new Set([
|
||||
resolveDefaultAgentDir(params.cfg, env),
|
||||
...listAgentIds(params.cfg).map((agentId) => resolveAgentDir(params.cfg, agentId, env)),
|
||||
]),
|
||||
].toSorted((left, right) => left.localeCompare(right));
|
||||
}
|
||||
|
||||
async function readLegacyPluginCatalogContents(params: {
|
||||
agentDir: string;
|
||||
relativePath: string;
|
||||
}): Promise<string | null> {
|
||||
const pluginDir = path.dirname(path.join(params.agentDir, params.relativePath));
|
||||
return await privateFileStore(pluginDir).readTextIfExists(path.basename(params.relativePath));
|
||||
}
|
||||
|
||||
/** Detects only marker-backed catalogs produced by tagged OpenClaw releases. */
|
||||
async function collectLegacyPluginModelCatalogMigrations(params: {
|
||||
cfg: OpenClawConfig;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
agentDirs?: readonly string[];
|
||||
warnings?: string[];
|
||||
}): Promise<LegacyPluginModelCatalogMigration[]> {
|
||||
const migrations: LegacyPluginModelCatalogMigration[] = [];
|
||||
for (const agentDir of resolveMigrationAgentDirs(params)) {
|
||||
const pluginsDir = path.join(agentDir, "plugins");
|
||||
let pluginDirs: Dirent[];
|
||||
try {
|
||||
pluginDirs = await fs.readdir(pluginsDir, { withFileTypes: true });
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code === "ENOENT") {
|
||||
continue;
|
||||
}
|
||||
params.warnings?.push(
|
||||
`Could not inspect legacy provider catalogs: ${shortenHomePath(pluginsDir)}`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
for (const pluginDir of pluginDirs) {
|
||||
if (!pluginDir.isDirectory()) {
|
||||
continue;
|
||||
}
|
||||
const canonicalRelativePath = path.join("plugins", pluginDir.name, "catalog.json");
|
||||
const pluginId = decodePluginModelCatalogRelativePathPluginId(canonicalRelativePath);
|
||||
if (!pluginId) {
|
||||
continue;
|
||||
}
|
||||
const pluginPath = path.join(pluginsDir, pluginDir.name);
|
||||
let catalogFiles: Dirent[];
|
||||
try {
|
||||
catalogFiles = await fs.readdir(pluginPath, { withFileTypes: true });
|
||||
} catch {
|
||||
params.warnings?.push(
|
||||
`Could not inspect legacy provider catalogs: ${shortenHomePath(pluginPath)}`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
const sourceFiles = catalogFiles
|
||||
.filter((entry) => entry.isFile() && isPluginModelCatalogMigrationFile(entry.name))
|
||||
.toSorted((left, right) => {
|
||||
if (left.name === "catalog.json") {
|
||||
return 1;
|
||||
}
|
||||
if (right.name === "catalog.json") {
|
||||
return -1;
|
||||
}
|
||||
return left.name.localeCompare(right.name);
|
||||
});
|
||||
const pluginMigrations: LegacyPluginModelCatalogMigration[] = [];
|
||||
let hasUnreadableCatalog = false;
|
||||
for (const sourceFile of sourceFiles) {
|
||||
const relativePath = path.join("plugins", pluginDir.name, sourceFile.name);
|
||||
let contents: string | null;
|
||||
try {
|
||||
contents = await readLegacyPluginCatalogContents({ agentDir, relativePath });
|
||||
} catch {
|
||||
hasUnreadableCatalog = true;
|
||||
params.warnings?.push(
|
||||
`Could not read legacy provider catalog: ${shortenHomePath(path.join(agentDir, relativePath))}`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
if (contents === null) {
|
||||
continue;
|
||||
}
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(contents) as unknown;
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
if (isGeneratedPluginModelCatalog(parsed)) {
|
||||
pluginMigrations.push({ agentDir, pluginId, relativePath, contents });
|
||||
}
|
||||
}
|
||||
if (hasUnreadableCatalog) {
|
||||
continue;
|
||||
}
|
||||
if (
|
||||
!pluginMigrations.some(
|
||||
(migration) => path.basename(migration.relativePath) === "catalog.json",
|
||||
) &&
|
||||
new Set(pluginMigrations.map((migration) => migration.contents)).size > 1
|
||||
) {
|
||||
params.warnings?.push(
|
||||
`Conflicting retained legacy provider catalogs: ${shortenHomePath(pluginPath)}`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
migrations.push(...pluginMigrations);
|
||||
}
|
||||
}
|
||||
return migrations.toSorted((left, right) => {
|
||||
const agentOrder = left.agentDir.localeCompare(right.agentDir);
|
||||
return agentOrder !== 0 ? agentOrder : left.pluginId.localeCompare(right.pluginId);
|
||||
});
|
||||
}
|
||||
|
||||
/** Imports and verifies released sidecars before Doctor removes any legacy bytes. */
|
||||
export async function maybeMigrateLegacyPluginModelCatalogs(params: {
|
||||
cfg: OpenClawConfig;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
agentDirs?: readonly string[];
|
||||
prompter: DoctorPrompter;
|
||||
runtime: RuntimeEnv;
|
||||
note?: typeof note;
|
||||
}): Promise<{ detected: number; migrated: number; warnings: string[] }> {
|
||||
const warnings: string[] = [];
|
||||
const migrations = await collectLegacyPluginModelCatalogMigrations({ ...params, warnings });
|
||||
for (const warning of warnings) {
|
||||
params.runtime.error(warning);
|
||||
}
|
||||
if (migrations.length === 0) {
|
||||
return { detected: 0, migrated: 0, warnings };
|
||||
}
|
||||
|
||||
const emitNote = params.note ?? note;
|
||||
emitNote(
|
||||
[
|
||||
"Legacy generated provider catalogs contain model and credential state.",
|
||||
...migrations.map(
|
||||
(migration) =>
|
||||
`- ${shortenHomePath(path.join(migration.agentDir, migration.relativePath))}`,
|
||||
),
|
||||
"Run openclaw doctor --fix to verify and migrate these catalogs into agent SQLite.",
|
||||
].join("\n"),
|
||||
"Plugin model catalogs",
|
||||
);
|
||||
const shouldRepair =
|
||||
params.prompter.shouldRepair ||
|
||||
(await params.prompter.confirmAutoFix({
|
||||
message: "Migrate generated provider model catalogs into agent SQLite now?",
|
||||
initialValue: true,
|
||||
}));
|
||||
if (!shouldRepair) {
|
||||
return { detected: migrations.length, migrated: 0, warnings };
|
||||
}
|
||||
|
||||
const grouped = new Map<string, LegacyPluginModelCatalogMigration[]>();
|
||||
for (const migration of migrations) {
|
||||
const agentMigrations = grouped.get(migration.agentDir) ?? [];
|
||||
agentMigrations.push(migration);
|
||||
grouped.set(migration.agentDir, agentMigrations);
|
||||
}
|
||||
|
||||
let migrated = 0;
|
||||
for (const [agentDir, agentMigrations] of grouped) {
|
||||
const result = migrateLegacyPluginModelCatalogs({
|
||||
agentDir,
|
||||
expectedContents: new Map(
|
||||
agentMigrations.map((migration) => [migration.pluginId, migration.contents]),
|
||||
),
|
||||
});
|
||||
migrated += result.migrated;
|
||||
for (const warning of result.warnings) {
|
||||
const displayWarning = shortenHomePath(warning);
|
||||
if (warnings.includes(displayWarning)) {
|
||||
continue;
|
||||
}
|
||||
warnings.push(displayWarning);
|
||||
params.runtime.error(displayWarning);
|
||||
}
|
||||
}
|
||||
|
||||
if (migrated > 0) {
|
||||
emitNote(
|
||||
`Migrated and verified ${migrated} generated provider catalog${migrated === 1 ? "" : "s"} in agent SQLite.`,
|
||||
"Doctor changes",
|
||||
);
|
||||
}
|
||||
return { detected: migrations.length, migrated, warnings };
|
||||
}
|
||||
@@ -21,16 +21,9 @@ const resolveEnvApiKey = vi.fn().mockReturnValue(undefined);
|
||||
const resolveAwsSdkEnvVarName = vi.fn().mockReturnValue(undefined);
|
||||
const hasUsableCustomProviderApiKey = vi.fn().mockReturnValue(false);
|
||||
const hasSyntheticLocalProviderAuthConfig = vi.fn().mockReturnValue(false);
|
||||
const loadModelCatalog = vi.fn<(_params?: unknown) => Promise<never[]>>(async () => []);
|
||||
const loadProviderCatalogModelsForList = vi.fn<() => Promise<Array<Record<string, unknown>>>>(
|
||||
const loadModelCatalog = vi.fn<(_params?: unknown) => Promise<Array<Record<string, unknown>>>>(
|
||||
async () => [],
|
||||
);
|
||||
const loadStaticManifestCatalogRowsForList = vi.fn<() => Array<Record<string, unknown>>>(() => []);
|
||||
const loadSupplementalManifestCatalogRowsForList = vi.fn<() => Array<Record<string, unknown>>>(
|
||||
() => [],
|
||||
);
|
||||
const loadProviderIndexCatalogRowsForList = vi.fn<() => Array<Record<string, unknown>>>(() => []);
|
||||
const hasProviderStaticCatalogForFilter = vi.fn().mockResolvedValue(false);
|
||||
const shouldSuppressBuiltInModel = vi.fn().mockReturnValue(false);
|
||||
const shouldSuppressBuiltInModelFromManifest = vi.fn().mockReturnValue(false);
|
||||
const normalizeProviderResolvedModelWithPlugin = vi.hoisted(() =>
|
||||
@@ -188,24 +181,6 @@ vi.mock("../plugins/synthetic-auth.runtime.js", () => ({
|
||||
resolveRuntimeSyntheticAuthProviderRefs: () => [],
|
||||
}));
|
||||
|
||||
vi.mock("./models/list.provider-catalog.js", async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import("./models/list.provider-catalog.js")>();
|
||||
return {
|
||||
...actual,
|
||||
hasProviderStaticCatalogForFilter,
|
||||
loadProviderCatalogModelsForList,
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock("./models/list.manifest-catalog.js", () => ({
|
||||
loadStaticManifestCatalogRowsForList,
|
||||
loadSupplementalManifestCatalogRowsForList,
|
||||
}));
|
||||
|
||||
vi.mock("./models/list.provider-index-catalog.js", () => ({
|
||||
loadProviderIndexCatalogRowsForList,
|
||||
}));
|
||||
|
||||
vi.mock("../agents/model-suppression.js", async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import("../agents/model-suppression.js")>()),
|
||||
shouldSuppressBuiltInModel,
|
||||
@@ -281,16 +256,6 @@ beforeEach(() => {
|
||||
listProfilesForProvider.mockReturnValue([]);
|
||||
loadModelCatalog.mockClear();
|
||||
loadModelCatalog.mockResolvedValue([]);
|
||||
loadProviderCatalogModelsForList.mockReset();
|
||||
loadProviderCatalogModelsForList.mockResolvedValue([]);
|
||||
loadStaticManifestCatalogRowsForList.mockReset();
|
||||
loadStaticManifestCatalogRowsForList.mockReturnValue([]);
|
||||
loadSupplementalManifestCatalogRowsForList.mockReset();
|
||||
loadSupplementalManifestCatalogRowsForList.mockReturnValue([]);
|
||||
loadProviderIndexCatalogRowsForList.mockReset();
|
||||
loadProviderIndexCatalogRowsForList.mockReturnValue([]);
|
||||
hasProviderStaticCatalogForFilter.mockReset();
|
||||
hasProviderStaticCatalogForFilter.mockResolvedValue(false);
|
||||
shouldSuppressBuiltInModel.mockReset();
|
||||
shouldSuppressBuiltInModel.mockReturnValue(false);
|
||||
normalizeProviderResolvedModelWithPlugin.mockClear();
|
||||
@@ -404,7 +369,6 @@ describe("models list/status", () => {
|
||||
|
||||
async function expectZaiProviderFilter(provider: string) {
|
||||
setDefaultZaiRegistry();
|
||||
loadProviderIndexCatalogRowsForList.mockReturnValueOnce([ZAI_MODEL]);
|
||||
const runtime = makeRuntime();
|
||||
|
||||
await modelsListCommand({ all: true, provider, json: true }, runtime);
|
||||
@@ -630,8 +594,7 @@ describe("models list/status", () => {
|
||||
|
||||
it("models list all includes catalog rows with unknown auth availability", async () => {
|
||||
setDefaultZaiRegistry({ available: false });
|
||||
hasProviderStaticCatalogForFilter.mockResolvedValueOnce(true);
|
||||
loadProviderCatalogModelsForList.mockResolvedValueOnce([MOONSHOT_MODEL]);
|
||||
loadModelCatalog.mockResolvedValueOnce([MOONSHOT_MODEL]);
|
||||
const runtime = makeRuntime();
|
||||
|
||||
await withEnvAsync(
|
||||
@@ -640,7 +603,7 @@ describe("models list/status", () => {
|
||||
);
|
||||
|
||||
const payload = parseJsonLog(runtime);
|
||||
expect(loadModelCatalog).not.toHaveBeenCalled();
|
||||
expect(loadModelCatalog).toHaveBeenCalledOnce();
|
||||
expect(payload.models).toHaveLength(1);
|
||||
const model = payload.models[0];
|
||||
expect(model.key).toBe("moonshot/kimi-k2.6");
|
||||
@@ -660,18 +623,18 @@ describe("models list/status", () => {
|
||||
);
|
||||
expect(runtime.log).not.toHaveBeenCalled();
|
||||
expect(loadModelCatalog).not.toHaveBeenCalled();
|
||||
expect(loadProviderCatalogModelsForList).not.toHaveBeenCalled();
|
||||
expect(process.exitCode).toBe(1);
|
||||
});
|
||||
|
||||
it("models list all local skips unauthenticated provider catalog rows", async () => {
|
||||
setDefaultZaiRegistry({ available: false });
|
||||
loadProviderCatalogModelsForList.mockResolvedValueOnce([MOONSHOT_MODEL]);
|
||||
loadModelCatalog.mockResolvedValueOnce([MOONSHOT_MODEL]);
|
||||
const runtime = makeRuntime();
|
||||
|
||||
await modelsListCommand({ all: true, local: true, json: true }, runtime);
|
||||
|
||||
expect(loadProviderCatalogModelsForList).not.toHaveBeenCalled();
|
||||
expect(loadModelCatalog).toHaveBeenCalledOnce();
|
||||
expect(parseJsonLog(runtime).models).toEqual([]);
|
||||
});
|
||||
|
||||
it("models list default does not enumerate all registry models", async () => {
|
||||
|
||||
@@ -90,12 +90,6 @@ const mocks = vi.hoisted(() => {
|
||||
resolveDefaultAgentDir: vi.fn(),
|
||||
loadModelRegistry: vi.fn(),
|
||||
loadModelCatalog: vi.fn(),
|
||||
loadProviderCatalogModelsForList: vi.fn(),
|
||||
loadStaticManifestCatalogRowsForList: vi.fn(),
|
||||
loadSupplementalManifestCatalogRowsForList: vi.fn(),
|
||||
loadProviderIndexCatalogRowsForList: vi.fn(),
|
||||
hasProviderRuntimeCatalogForFilter: vi.fn(),
|
||||
hasProviderStaticCatalogForFilter: vi.fn(),
|
||||
resolveConfiguredEntries: vi.fn(),
|
||||
printModelTable: vi.fn(),
|
||||
resolveModelWithRegistry: vi.fn(),
|
||||
@@ -122,11 +116,6 @@ function resetMocks() {
|
||||
},
|
||||
});
|
||||
mocks.loadModelCatalog.mockResolvedValue([]);
|
||||
mocks.loadProviderCatalogModelsForList.mockResolvedValue([]);
|
||||
mocks.loadStaticManifestCatalogRowsForList.mockReturnValue([]);
|
||||
mocks.loadSupplementalManifestCatalogRowsForList.mockReturnValue([]);
|
||||
mocks.loadProviderIndexCatalogRowsForList.mockReturnValue([]);
|
||||
mocks.hasProviderStaticCatalogForFilter.mockResolvedValue(false);
|
||||
mocks.resolveConfiguredEntries.mockReturnValue({
|
||||
entries: [
|
||||
{
|
||||
@@ -193,14 +182,6 @@ function modelRegistryOptions(index = 0): Record<string, unknown> {
|
||||
return options as Record<string, unknown>;
|
||||
}
|
||||
|
||||
function providerCatalogOptions(index = 0): Record<string, unknown> {
|
||||
const options = mocks.loadProviderCatalogModelsForList.mock.calls[index]?.[0];
|
||||
if (!options || typeof options !== "object") {
|
||||
throw new Error(`expected provider catalog options ${index}`);
|
||||
}
|
||||
return options as Record<string, unknown>;
|
||||
}
|
||||
|
||||
let modelsListCommand: typeof import("./list.list-command.js").modelsListCommand;
|
||||
let listRowsModule: typeof import("./list.rows.js");
|
||||
let listRegistryModule: typeof import("./list.registry.js");
|
||||
@@ -236,21 +217,6 @@ function installModelsListCommandForwardCompatMocks() {
|
||||
printModelTable: mocks.printModelTable,
|
||||
}));
|
||||
|
||||
vi.doMock("./list.provider-catalog.js", () => ({
|
||||
hasProviderRuntimeCatalogForFilter: mocks.hasProviderRuntimeCatalogForFilter,
|
||||
hasProviderStaticCatalogForFilter: mocks.hasProviderStaticCatalogForFilter,
|
||||
loadProviderCatalogModelsForList: mocks.loadProviderCatalogModelsForList,
|
||||
}));
|
||||
|
||||
vi.doMock("./list.manifest-catalog.js", () => ({
|
||||
loadStaticManifestCatalogRowsForList: mocks.loadStaticManifestCatalogRowsForList,
|
||||
loadSupplementalManifestCatalogRowsForList: mocks.loadSupplementalManifestCatalogRowsForList,
|
||||
}));
|
||||
|
||||
vi.doMock("./list.provider-index-catalog.js", () => ({
|
||||
loadProviderIndexCatalogRowsForList: mocks.loadProviderIndexCatalogRowsForList,
|
||||
}));
|
||||
|
||||
vi.doMock("./list.registry-load.js", () => ({
|
||||
loadListModelRegistry: async (
|
||||
cfg: unknown,
|
||||
@@ -373,9 +339,8 @@ async function buildAllOpenAiCodexRows(opts: { supplementCatalog?: boolean } = {
|
||||
context: context as never,
|
||||
});
|
||||
if (opts.supplementCatalog !== false) {
|
||||
await listRowsModule.appendCatalogSupplementRows({
|
||||
await listRowsModule.appendPreparedModelCatalogRows({
|
||||
rows: rows as never,
|
||||
modelRegistry: loaded.registry as never,
|
||||
context: context as never,
|
||||
seenKeys,
|
||||
});
|
||||
@@ -416,9 +381,9 @@ describe("modelsListCommand forward-compat", () => {
|
||||
});
|
||||
|
||||
describe("configured rows", () => {
|
||||
it("returns manifest catalog rows for provider filters without --all", async () => {
|
||||
it("projects prepared catalog rows for provider filters without --all", async () => {
|
||||
mocks.resolveConfiguredEntries.mockReturnValueOnce({ entries: [] });
|
||||
mocks.loadStaticManifestCatalogRowsForList.mockReturnValueOnce([
|
||||
mocks.loadModelCatalog.mockResolvedValueOnce([
|
||||
{
|
||||
provider: "moonshot",
|
||||
id: "kimi-k2.6",
|
||||
@@ -437,11 +402,50 @@ describe("modelsListCommand forward-compat", () => {
|
||||
|
||||
await modelsListCommand({ json: true, provider: "moonshot" }, runtime as never);
|
||||
|
||||
expect(mocks.loadModelRegistry).not.toHaveBeenCalled();
|
||||
expect(mocks.loadModelRegistry).toHaveBeenCalledOnce();
|
||||
expect(mocks.loadModelCatalog).toHaveBeenCalledOnce();
|
||||
expect(runtime.log).not.toHaveBeenCalledWith("No models found.");
|
||||
expectRowKeys(lastPrintedRows<{ key: string }>(), ["moonshot/kimi-k2.6"]);
|
||||
});
|
||||
|
||||
it("canonicalizes a manifest provider alias before reading the prepared catalog", async () => {
|
||||
mocks.resolveConfiguredEntries.mockReturnValueOnce({ entries: [] });
|
||||
mocks.loadManifestMetadataSnapshot.mockReturnValueOnce({
|
||||
...mocks.emptyPluginMetadataSnapshot,
|
||||
manifestRegistry: {
|
||||
diagnostics: [],
|
||||
plugins: [
|
||||
{
|
||||
id: "moonshot",
|
||||
origin: "bundled",
|
||||
rootDir: "/tmp/openclaw-moonshot",
|
||||
modelCatalog: {
|
||||
aliases: {
|
||||
kimi: { provider: "moonshot" },
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
mocks.loadModelCatalog.mockResolvedValueOnce([
|
||||
{
|
||||
provider: "moonshot",
|
||||
id: "kimi-k2.6",
|
||||
name: "Kimi K2.6",
|
||||
input: ["text", "image"],
|
||||
baseUrl: "https://api.moonshot.ai/v1",
|
||||
contextWindow: 262_144,
|
||||
},
|
||||
]);
|
||||
const runtime = createRuntime();
|
||||
|
||||
await modelsListCommand({ json: true, provider: "kimi" }, runtime as never);
|
||||
|
||||
expect(modelRegistryOptions().providerFilter).toBe("moonshot");
|
||||
expectRowKeys(lastPrintedRows<{ key: string }>(), ["moonshot/kimi-k2.6"]);
|
||||
});
|
||||
|
||||
it("keeps catalog metadata when provider-filtered configured entries overlap", async () => {
|
||||
mocks.resolveConfiguredEntries.mockReturnValueOnce({
|
||||
entries: [
|
||||
@@ -453,7 +457,7 @@ describe("modelsListCommand forward-compat", () => {
|
||||
},
|
||||
],
|
||||
});
|
||||
mocks.loadStaticManifestCatalogRowsForList.mockReturnValueOnce([
|
||||
mocks.loadModelCatalog.mockResolvedValueOnce([
|
||||
{
|
||||
provider: "moonshot",
|
||||
id: "kimi-k2.6",
|
||||
@@ -472,7 +476,8 @@ describe("modelsListCommand forward-compat", () => {
|
||||
|
||||
await modelsListCommand({ json: true, provider: "moonshot" }, runtime as never);
|
||||
|
||||
expect(mocks.loadModelRegistry).not.toHaveBeenCalled();
|
||||
expect(mocks.loadModelRegistry).toHaveBeenCalledOnce();
|
||||
expect(mocks.loadModelCatalog).toHaveBeenCalledOnce();
|
||||
const rows = lastPrintedRows<{ key: string; name: string; tags: string[] }>();
|
||||
expectRowKeys(rows, ["moonshot/kimi-k2.6"]);
|
||||
expectRowFields(rows, "moonshot/kimi-k2.6", {
|
||||
@@ -555,10 +560,9 @@ describe("modelsListCommand forward-compat", () => {
|
||||
expectRowKeys(lastPrintedRows<{ key: string }>(), ["openai/gpt-5.5"]);
|
||||
});
|
||||
|
||||
it("uses provider static catalog rows for provider filters without --all", async () => {
|
||||
it("projects static provider rows from the committed catalog", async () => {
|
||||
mocks.resolveConfiguredEntries.mockReturnValueOnce({ entries: [] });
|
||||
mocks.hasProviderStaticCatalogForFilter.mockResolvedValueOnce(true);
|
||||
mocks.loadProviderCatalogModelsForList.mockResolvedValueOnce([
|
||||
mocks.loadModelCatalog.mockResolvedValueOnce([
|
||||
{
|
||||
provider: "google",
|
||||
id: "gemini-2.5-pro",
|
||||
@@ -575,35 +579,20 @@ describe("modelsListCommand forward-compat", () => {
|
||||
|
||||
await modelsListCommand({ json: true, provider: "google" }, runtime as never);
|
||||
|
||||
expect(mocks.loadModelRegistry).not.toHaveBeenCalled();
|
||||
expect(providerCatalogOptions().providerFilter).toBe("google");
|
||||
expect(providerCatalogOptions().staticOnly).toBe(true);
|
||||
expect(mocks.loadModelRegistry).toHaveBeenCalledOnce();
|
||||
expect(mocks.loadModelCatalog).toHaveBeenCalledOnce();
|
||||
expectRowKeys(lastPrintedRows<{ key: string }>(), ["google/gemini-2.5-pro"]);
|
||||
});
|
||||
|
||||
it("uses provider-index catalog rows for provider filters without --all", async () => {
|
||||
it("does not invent installable preview rows outside the committed catalog", async () => {
|
||||
mocks.resolveConfiguredEntries.mockReturnValueOnce({ entries: [] });
|
||||
mocks.loadProviderIndexCatalogRowsForList.mockReturnValueOnce([
|
||||
{
|
||||
provider: "moonshot",
|
||||
id: "kimi-k2.6",
|
||||
ref: "moonshot/kimi-k2.6",
|
||||
mergeKey: "moonshot::kimi-k2.6",
|
||||
name: "Kimi K2.6",
|
||||
source: "provider-index",
|
||||
input: ["text", "image"],
|
||||
reasoning: false,
|
||||
status: "available",
|
||||
baseUrl: "https://api.moonshot.ai/v1",
|
||||
contextWindow: 262_144,
|
||||
},
|
||||
]);
|
||||
const runtime = createRuntime();
|
||||
|
||||
await modelsListCommand({ json: true, provider: "moonshot" }, runtime as never);
|
||||
|
||||
expect(mocks.loadModelRegistry).not.toHaveBeenCalled();
|
||||
expectRowKeys(lastPrintedRows<{ key: string }>(), ["moonshot/kimi-k2.6"]);
|
||||
expect(mocks.loadModelRegistry).toHaveBeenCalledOnce();
|
||||
expect(mocks.loadModelCatalog).toHaveBeenCalledOnce();
|
||||
expectRowKeys(lastPrintedRows<{ key: string }>(), []);
|
||||
});
|
||||
|
||||
it("includes configured provider model rows for provider-filtered lists", async () => {
|
||||
@@ -642,7 +631,7 @@ describe("modelsListCommand forward-compat", () => {
|
||||
|
||||
await modelsListCommand({ json: true, provider: "ollama" }, runtime as never);
|
||||
|
||||
expect(mocks.loadModelRegistry).not.toHaveBeenCalled();
|
||||
expect(mocks.loadModelRegistry).toHaveBeenCalledOnce();
|
||||
const rows = lastPrintedRows<{ key: string; name: string; tags: string[] }>();
|
||||
expectRowKeys(rows, ["ollama/qwen2.5:7b", "ollama/llama3.2:3b"]);
|
||||
expectRowFields(rows, "ollama/qwen2.5:7b", {
|
||||
@@ -898,10 +887,9 @@ describe("modelsListCommand forward-compat", () => {
|
||||
});
|
||||
|
||||
describe("--all catalog supplementation", () => {
|
||||
it("includes refreshed manifest models for runtime-backed provider lists", async () => {
|
||||
it("includes refreshed runtime models from the committed provider generation", async () => {
|
||||
mocks.resolveConfiguredEntries.mockReturnValueOnce({ entries: [] });
|
||||
mocks.hasProviderRuntimeCatalogForFilter.mockResolvedValueOnce(true);
|
||||
mocks.loadProviderCatalogModelsForList.mockResolvedValueOnce([
|
||||
mocks.loadModelCatalog.mockResolvedValueOnce([
|
||||
{
|
||||
provider: "anthropic",
|
||||
id: "claude-live",
|
||||
@@ -913,8 +901,6 @@ describe("modelsListCommand forward-compat", () => {
|
||||
maxTokens: 4096,
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
},
|
||||
]);
|
||||
mocks.loadSupplementalManifestCatalogRowsForList.mockReturnValueOnce([
|
||||
{
|
||||
provider: "anthropic",
|
||||
id: "claude-refreshed",
|
||||
@@ -933,7 +919,8 @@ describe("modelsListCommand forward-compat", () => {
|
||||
|
||||
await modelsListCommand({ all: true, provider: "anthropic", json: true }, runtime as never);
|
||||
|
||||
expect(mocks.loadModelRegistry).not.toHaveBeenCalled();
|
||||
expect(mocks.loadModelRegistry).toHaveBeenCalledOnce();
|
||||
expect(mocks.loadModelCatalog).toHaveBeenCalledOnce();
|
||||
expectRowKeys(lastPrintedRows<{ key: string }>(), [
|
||||
"anthropic/claude-live",
|
||||
"anthropic/claude-refreshed",
|
||||
@@ -942,8 +929,7 @@ describe("modelsListCommand forward-compat", () => {
|
||||
|
||||
it("keeps OpenAI runtime rows authoritative while adding refreshed models once", async () => {
|
||||
mocks.resolveConfiguredEntries.mockReturnValueOnce({ entries: [] });
|
||||
mocks.hasProviderRuntimeCatalogForFilter.mockResolvedValueOnce(true);
|
||||
mocks.loadProviderCatalogModelsForList.mockResolvedValueOnce([
|
||||
mocks.loadModelCatalog.mockResolvedValueOnce([
|
||||
{
|
||||
provider: "openai",
|
||||
id: "gpt-live",
|
||||
@@ -955,21 +941,6 @@ describe("modelsListCommand forward-compat", () => {
|
||||
maxTokens: 4096,
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
},
|
||||
]);
|
||||
mocks.loadSupplementalManifestCatalogRowsForList.mockReturnValueOnce([
|
||||
{
|
||||
provider: "openai",
|
||||
id: "gpt-live",
|
||||
ref: "openai/gpt-live",
|
||||
mergeKey: "openai::gpt-live",
|
||||
name: "Refreshed Duplicate GPT",
|
||||
source: "runtime-refresh",
|
||||
input: ["text"],
|
||||
reasoning: false,
|
||||
status: "available",
|
||||
baseUrl: "https://api.openai.com/v1",
|
||||
contextWindow: 200_000,
|
||||
},
|
||||
{
|
||||
provider: "openai",
|
||||
id: "gpt-refreshed",
|
||||
@@ -988,7 +959,8 @@ describe("modelsListCommand forward-compat", () => {
|
||||
|
||||
await modelsListCommand({ all: true, provider: "openai", json: true }, runtime as never);
|
||||
|
||||
expect(mocks.loadModelRegistry).not.toHaveBeenCalled();
|
||||
expect(mocks.loadModelRegistry).toHaveBeenCalledOnce();
|
||||
expect(mocks.loadModelCatalog).toHaveBeenCalledOnce();
|
||||
const rows = lastPrintedRows<{ key: string; name: string }>();
|
||||
expectRowKeys(rows, ["openai/gpt-live", "openai/gpt-refreshed"]);
|
||||
expect(requireRow(rows, "openai/gpt-live").name).toBe("Live GPT");
|
||||
@@ -996,8 +968,7 @@ describe("modelsListCommand forward-compat", () => {
|
||||
|
||||
it("keeps provider-catalog Codex availability indeterminate without model auth", async () => {
|
||||
mocks.resolveConfiguredEntries.mockReturnValueOnce({ entries: [] });
|
||||
mocks.hasProviderStaticCatalogForFilter.mockResolvedValueOnce(true);
|
||||
mocks.loadProviderCatalogModelsForList.mockResolvedValueOnce([
|
||||
mocks.loadModelCatalog.mockResolvedValueOnce([
|
||||
{
|
||||
provider: "codex",
|
||||
id: "gpt-5.4",
|
||||
@@ -1022,23 +993,16 @@ describe("modelsListCommand forward-compat", () => {
|
||||
await modelsListCommand({ all: true, provider: "codex", json: true }, runtime as never);
|
||||
|
||||
expect(mocks.ensureOpenClawModelsJson).not.toHaveBeenCalled();
|
||||
expect(mocks.loadModelRegistry).not.toHaveBeenCalled();
|
||||
expect(mocks.loadProviderCatalogModelsForList).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
cfg: mocks.resolvedConfig,
|
||||
agentDir: "/tmp/openclaw-agent",
|
||||
providerFilter: "codex",
|
||||
staticOnly: true,
|
||||
}),
|
||||
);
|
||||
expect(mocks.loadModelRegistry).toHaveBeenCalledOnce();
|
||||
expect(mocks.loadModelCatalog).toHaveBeenCalledOnce();
|
||||
const rows = lastPrintedRows<{ key: string; available: boolean | null }>();
|
||||
expectRowKeys(rows, ["codex/gpt-5.4"]);
|
||||
expectRowFields(rows, "codex/gpt-5.4", { available: null });
|
||||
});
|
||||
|
||||
it("uses manifest catalog rows before provider runtime catalog rows", async () => {
|
||||
it("uses committed catalog rows without separately loading provider runtimes", async () => {
|
||||
mocks.resolveConfiguredEntries.mockReturnValueOnce({ entries: [] });
|
||||
mocks.loadStaticManifestCatalogRowsForList.mockReturnValueOnce([
|
||||
mocks.loadModelCatalog.mockResolvedValueOnce([
|
||||
{
|
||||
provider: "moonshot",
|
||||
id: "kimi-k2.6",
|
||||
@@ -1057,15 +1021,14 @@ describe("modelsListCommand forward-compat", () => {
|
||||
|
||||
await modelsListCommand({ all: true, provider: "moonshot", json: true }, runtime as never);
|
||||
|
||||
expect(mocks.loadModelRegistry).not.toHaveBeenCalled();
|
||||
expect(mocks.hasProviderStaticCatalogForFilter).not.toHaveBeenCalled();
|
||||
expect(mocks.loadProviderCatalogModelsForList).not.toHaveBeenCalled();
|
||||
expect(mocks.loadModelRegistry).toHaveBeenCalledOnce();
|
||||
expect(mocks.loadModelCatalog).toHaveBeenCalledOnce();
|
||||
expectRowKeys(lastPrintedRows<{ key: string }>(), ["moonshot/kimi-k2.6"]);
|
||||
});
|
||||
|
||||
it("keeps refreshable manifest catalog rows on the registry-backed provider path", async () => {
|
||||
mocks.resolveConfiguredEntries.mockReturnValueOnce({ entries: [] });
|
||||
mocks.loadSupplementalManifestCatalogRowsForList.mockReturnValueOnce([
|
||||
mocks.loadModelCatalog.mockResolvedValueOnce([
|
||||
{
|
||||
provider: "openai",
|
||||
id: "gpt-5.5-pro",
|
||||
@@ -1125,31 +1088,15 @@ describe("modelsListCommand forward-compat", () => {
|
||||
expectRowKeys(lastPrintedRows<{ key: string }>(), ["openai/gpt-5.4", "openai/gpt-5.5-pro"]);
|
||||
});
|
||||
|
||||
it("uses provider index preview rows when an installable provider is not installed", async () => {
|
||||
it("keeps uninstalled provider previews out of the authoritative all-model view", async () => {
|
||||
mocks.resolveConfiguredEntries.mockReturnValueOnce({ entries: [] });
|
||||
mocks.loadProviderIndexCatalogRowsForList.mockReturnValueOnce([
|
||||
{
|
||||
provider: "moonshot",
|
||||
id: "kimi-k2.6",
|
||||
ref: "moonshot/kimi-k2.6",
|
||||
mergeKey: "moonshot::kimi-k2.6",
|
||||
name: "Kimi K2.6",
|
||||
source: "provider-index",
|
||||
input: ["text", "image"],
|
||||
reasoning: false,
|
||||
status: "available",
|
||||
baseUrl: "https://api.moonshot.ai/v1",
|
||||
contextWindow: 262_144,
|
||||
},
|
||||
]);
|
||||
const runtime = createRuntime();
|
||||
|
||||
await modelsListCommand({ all: true, provider: "moonshot", json: true }, runtime as never);
|
||||
|
||||
expect(mocks.loadModelRegistry).not.toHaveBeenCalled();
|
||||
expect(mocks.hasProviderStaticCatalogForFilter).not.toHaveBeenCalled();
|
||||
expect(mocks.loadProviderCatalogModelsForList).not.toHaveBeenCalled();
|
||||
expectRowKeys(lastPrintedRows<{ key: string }>(), ["moonshot/kimi-k2.6"]);
|
||||
expect(mocks.loadModelRegistry).toHaveBeenCalledOnce();
|
||||
expect(mocks.loadModelCatalog).toHaveBeenCalledOnce();
|
||||
expectRowKeys(lastPrintedRows<{ key: string }>(), []);
|
||||
});
|
||||
|
||||
it("does not load broad provider runtime catalogs for unfiltered all-model lists", async () => {
|
||||
@@ -1173,7 +1120,7 @@ describe("modelsListCommand forward-compat", () => {
|
||||
],
|
||||
},
|
||||
});
|
||||
mocks.loadSupplementalManifestCatalogRowsForList.mockReturnValueOnce([
|
||||
mocks.loadModelCatalog.mockResolvedValueOnce([
|
||||
{
|
||||
provider: "moonshot",
|
||||
id: "kimi-k2.6",
|
||||
@@ -1188,7 +1135,6 @@ describe("modelsListCommand forward-compat", () => {
|
||||
contextWindow: 262_144,
|
||||
},
|
||||
]);
|
||||
mocks.loadModelCatalog.mockResolvedValueOnce([]);
|
||||
const runtime = createRuntime();
|
||||
|
||||
await modelsListCommand({ all: true, json: true }, runtime as never);
|
||||
@@ -1196,16 +1142,13 @@ describe("modelsListCommand forward-compat", () => {
|
||||
expectFirstRegistryConfig();
|
||||
expect(modelRegistryOptions().providerFilter).toBeUndefined();
|
||||
expect(modelRegistryOptions().normalizeModels).toBe(false);
|
||||
expect(mocks.loadProviderCatalogModelsForList).not.toHaveBeenCalled();
|
||||
expect(mocks.resolveModelWithRegistry).not.toHaveBeenCalled();
|
||||
expect(mocks.loadModelCatalog).not.toHaveBeenCalled();
|
||||
expect(mocks.loadModelCatalog).toHaveBeenCalledOnce();
|
||||
expectRowKeys(lastPrintedRows<{ key: string }>(), ["openai/gpt-5.4", "moonshot/kimi-k2.6"]);
|
||||
});
|
||||
|
||||
it("falls back to registry-backed rows when the fast-path catalog is empty", async () => {
|
||||
mocks.resolveConfiguredEntries.mockReturnValueOnce({ entries: [] });
|
||||
mocks.hasProviderStaticCatalogForFilter.mockResolvedValueOnce(true);
|
||||
mocks.loadProviderCatalogModelsForList.mockResolvedValueOnce([]).mockResolvedValueOnce([]);
|
||||
mocks.loadModelRegistry.mockResolvedValueOnce({
|
||||
models: [{ ...OPENAI_CODEX_MODEL }],
|
||||
availableKeys: new Set(["openai/gpt-5.4"]),
|
||||
@@ -1220,24 +1163,7 @@ describe("modelsListCommand forward-compat", () => {
|
||||
expectFirstRegistryConfig();
|
||||
expect(modelRegistryOptions().providerFilter).toBe("openai");
|
||||
expect(modelRegistryOptions().normalizeModels).toBe(true);
|
||||
expect(mocks.loadProviderCatalogModelsForList).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
expect.objectContaining({
|
||||
cfg: mocks.resolvedConfig,
|
||||
agentDir: "/tmp/openclaw-agent",
|
||||
providerFilter: "openai",
|
||||
staticOnly: true,
|
||||
}),
|
||||
);
|
||||
expect(mocks.loadProviderCatalogModelsForList).toHaveBeenNthCalledWith(
|
||||
2,
|
||||
expect.objectContaining({
|
||||
cfg: mocks.resolvedConfig,
|
||||
agentDir: "/tmp/openclaw-agent",
|
||||
providerFilter: "openai",
|
||||
staticOnly: undefined,
|
||||
}),
|
||||
);
|
||||
expect(mocks.loadModelCatalog).toHaveBeenCalledOnce();
|
||||
const rows = lastPrintedRows<{ key: string; available: boolean }>();
|
||||
expectRowKeys(rows, ["openai/gpt-5.4"]);
|
||||
expectRowFields(rows, "openai/gpt-5.4", { available: true });
|
||||
@@ -1245,7 +1171,6 @@ describe("modelsListCommand forward-compat", () => {
|
||||
|
||||
it("falls back to registry rows for provider filters without catalog coverage", async () => {
|
||||
mocks.resolveConfiguredEntries.mockReturnValueOnce({ entries: [] });
|
||||
mocks.hasProviderStaticCatalogForFilter.mockResolvedValueOnce(false);
|
||||
mocks.loadModelRegistry.mockResolvedValueOnce({
|
||||
models: [
|
||||
{
|
||||
@@ -1283,14 +1208,13 @@ describe("modelsListCommand forward-compat", () => {
|
||||
|
||||
expectFirstRegistryConfig();
|
||||
expect(modelRegistryOptions().providerFilter).toBe("anthropic");
|
||||
expect(modelRegistryOptions().normalizeModels).toBe(false);
|
||||
expect(modelRegistryOptions().loadAvailability).toBe(false);
|
||||
expect(modelRegistryOptions().normalizeModels).toBe(true);
|
||||
expect(mocks.loadModelCatalog).toHaveBeenCalledOnce();
|
||||
expectRowKeys(lastPrintedRows<{ key: string }>(), ["anthropic/claude-opus-4-7"]);
|
||||
});
|
||||
|
||||
it("includes provider-owned supplemental catalog rows with provider filters", async () => {
|
||||
mocks.resolveConfiguredEntries.mockReturnValueOnce({ entries: [] });
|
||||
mocks.hasProviderStaticCatalogForFilter.mockResolvedValueOnce(true);
|
||||
mocks.loadModelRegistry.mockResolvedValueOnce({
|
||||
models: [],
|
||||
availableKeys: new Set(["opencode-go/deepseek-v4-pro"]),
|
||||
@@ -1348,21 +1272,6 @@ describe("modelsListCommand forward-compat", () => {
|
||||
contextWindow: 400000,
|
||||
},
|
||||
]);
|
||||
mocks.resolveModelWithRegistry.mockImplementation(
|
||||
({ provider, modelId }: { provider: string; modelId: string }) => {
|
||||
if (provider !== "openai") {
|
||||
return undefined;
|
||||
}
|
||||
if (modelId === "gpt-5.4") {
|
||||
return { ...OPENAI_CODEX_53_MODEL };
|
||||
}
|
||||
return undefined;
|
||||
},
|
||||
);
|
||||
mocks.resolveModelWithRegistry.mockImplementationOnce(
|
||||
({ provider, modelId }: { provider: string; modelId: string }) =>
|
||||
provider === "openai" && modelId === "gpt-5.4" ? { ...OPENAI_CODEX_53_MODEL } : undefined,
|
||||
);
|
||||
const rows = await buildAllOpenAiCodexRows();
|
||||
expectRowKeys(rows as Array<{ key: string }>, ["openai/gpt-5.4"]);
|
||||
expectRowFields(rows as Array<{ key: string; available: boolean }>, "openai/gpt-5.4", {
|
||||
@@ -1372,7 +1281,6 @@ describe("modelsListCommand forward-compat", () => {
|
||||
|
||||
it("uses provider runtime metadata for discovered codex gpt-5.5 rows", async () => {
|
||||
mocks.resolveConfiguredEntries.mockReturnValueOnce({ entries: [] });
|
||||
mocks.hasProviderStaticCatalogForFilter.mockResolvedValueOnce(true);
|
||||
const oauthConfig = {
|
||||
agents: { defaults: { model: { primary: "openai/gpt-5.5" } } },
|
||||
models: { providers: { openai: {} } },
|
||||
@@ -1508,7 +1416,6 @@ describe("modelsListCommand forward-compat", () => {
|
||||
describe("provider filter matching", () => {
|
||||
it("matches discovered providers against exact provider filters", async () => {
|
||||
mocks.resolveConfiguredEntries.mockReturnValueOnce({ entries: [] });
|
||||
mocks.hasProviderStaticCatalogForFilter.mockResolvedValueOnce(true);
|
||||
mocks.loadModelRegistry.mockResolvedValueOnce({
|
||||
models: [
|
||||
{
|
||||
|
||||
@@ -21,7 +21,6 @@ const DISPLAY_MODEL_PARSE_OPTIONS = { allowPluginNormalization: false } as const
|
||||
type PromotionsModule = typeof import("./list.promotions.js");
|
||||
type RegistryLoadModule = typeof import("./list.registry-load.js");
|
||||
type RowSourcesModule = typeof import("./list.row-sources.js");
|
||||
type SourcePlanModule = typeof import("./list.source-plan.js");
|
||||
|
||||
const promotionsModuleLoader = createLazyImportLoader<PromotionsModule>(
|
||||
() => import("./list.promotions.js"),
|
||||
@@ -32,9 +31,6 @@ const registryLoadModuleLoader = createLazyImportLoader<RegistryLoadModule>(
|
||||
const rowSourcesModuleLoader = createLazyImportLoader<RowSourcesModule>(
|
||||
() => import("./list.row-sources.js"),
|
||||
);
|
||||
const sourcePlanModuleLoader = createLazyImportLoader<SourcePlanModule>(
|
||||
() => import("./list.source-plan.js"),
|
||||
);
|
||||
|
||||
function loadRegistryLoadModule(): Promise<RegistryLoadModule> {
|
||||
return registryLoadModuleLoader.load();
|
||||
@@ -44,10 +40,6 @@ function loadRowSourcesModule(): Promise<RowSourcesModule> {
|
||||
return rowSourcesModuleLoader.load();
|
||||
}
|
||||
|
||||
function loadSourcePlanModule(): Promise<SourcePlanModule> {
|
||||
return sourcePlanModuleLoader.load();
|
||||
}
|
||||
|
||||
/** Lists configured, catalog, and runtime-discovered models as text, plain, or JSON. */
|
||||
export async function modelsListCommand(
|
||||
opts: {
|
||||
@@ -124,22 +116,9 @@ export async function modelsListCommand(
|
||||
let availableKeys: Set<string> | undefined;
|
||||
let availabilityErrorMessage: string | undefined;
|
||||
const configuredByKey = new Map(entries.map((entry) => [entry.key, entry]));
|
||||
const enableSourcePlanCascade = Boolean(opts.all) || Boolean(providerFilter);
|
||||
// Full/provider-filtered lists may need runtime, manifest, and registry rows.
|
||||
// Defer that planning so default configured-only output stays cheap.
|
||||
const sourcePlanModule = enableSourcePlanCascade ? await loadSourcePlanModule() : undefined;
|
||||
const sourcePlan = sourcePlanModule
|
||||
? await sourcePlanModule.planAllModelListSources({
|
||||
all: opts.all,
|
||||
enableCascade: enableSourcePlanCascade,
|
||||
providerFilter,
|
||||
cfg,
|
||||
agentId,
|
||||
agentDir,
|
||||
metadataSnapshot,
|
||||
})
|
||||
: undefined;
|
||||
const shouldLoadRegistry = sourcePlan?.kind === "registry";
|
||||
// The default configured view remains lazy; full and filtered views share
|
||||
// the registry and the same committed model generation as the Gateway.
|
||||
const includePreparedCatalog = Boolean(opts.all || providerFilter);
|
||||
const loadRegistryState = async (optsLocal?: {
|
||||
normalizeModels?: boolean;
|
||||
loadAvailability?: boolean;
|
||||
@@ -160,7 +139,7 @@ export async function modelsListCommand(
|
||||
availabilityErrorMessage = loaded.availabilityErrorMessage;
|
||||
};
|
||||
try {
|
||||
if (shouldLoadRegistry) {
|
||||
if (includePreparedCatalog) {
|
||||
await loadRegistryState();
|
||||
} else if (!opts.all && opts.local) {
|
||||
const { loadConfiguredListModelRegistry } = await loadRegistryLoadModule();
|
||||
@@ -197,55 +176,15 @@ export async function modelsListCommand(
|
||||
});
|
||||
const rows: ModelRow[] = [];
|
||||
|
||||
if (enableSourcePlanCascade) {
|
||||
if (includePreparedCatalog) {
|
||||
const { appendAllModelRowSources } = await loadRowSourcesModule();
|
||||
if (!sourcePlan || !sourcePlanModule) {
|
||||
throw new Error("models list source plan was not initialized");
|
||||
}
|
||||
let rowContext = buildRowContext(
|
||||
sourcePlan.kind === "manifest" ||
|
||||
sourcePlan.kind === "provider-index" ||
|
||||
sourcePlan.kind === "provider-runtime-static",
|
||||
);
|
||||
const initialAppend = await appendAllModelRowSources({
|
||||
await appendAllModelRowSources({
|
||||
rows,
|
||||
entries,
|
||||
context: rowContext,
|
||||
context: buildRowContext(false),
|
||||
modelRegistry,
|
||||
registryModels,
|
||||
sourcePlan,
|
||||
});
|
||||
if (initialAppend.requiresRegistryFallback) {
|
||||
const useScopedRegistryFallback = sourcePlan.kind === "provider-runtime-scoped";
|
||||
// Runtime-scoped providers can fail catalog availability while still being
|
||||
// useful for a provider-filtered list; retry through the registry fallback.
|
||||
try {
|
||||
await loadRegistryState(
|
||||
useScopedRegistryFallback
|
||||
? {
|
||||
normalizeModels: false,
|
||||
loadAvailability: false,
|
||||
}
|
||||
: undefined,
|
||||
);
|
||||
} catch (err) {
|
||||
runtime.error(`Model registry unavailable:\n${formatErrorWithStack(err)}`);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
rows.length = 0;
|
||||
rowContext = buildRowContext(useScopedRegistryFallback);
|
||||
await appendAllModelRowSources({
|
||||
rows,
|
||||
entries,
|
||||
context: rowContext,
|
||||
modelRegistry,
|
||||
registryModels,
|
||||
sourcePlan: useScopedRegistryFallback
|
||||
? sourcePlan
|
||||
: sourcePlanModule.createRegistryModelListSourcePlan(),
|
||||
});
|
||||
}
|
||||
} else {
|
||||
const { appendConfiguredModelRowSources } = await loadRowSourcesModule();
|
||||
await appendConfiguredModelRowSources({
|
||||
|
||||
@@ -100,9 +100,8 @@ describe("loadStaticManifestCatalogRowsForList", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("loads refreshable manifest rows as registry-backed supplements", async () => {
|
||||
const { loadSupplementalManifestCatalogRowsForList } =
|
||||
await import("./list.manifest-catalog.js");
|
||||
it("does not expose refreshable provider previews as prepared models", async () => {
|
||||
const { loadStaticManifestCatalogRowsForList } = await import("./list.manifest-catalog.js");
|
||||
const manifestRegistry = {
|
||||
plugins: [openrouterPlugin, moonshotPlugin],
|
||||
diagnostics: [],
|
||||
@@ -114,15 +113,14 @@ describe("loadStaticManifestCatalogRowsForList", () => {
|
||||
});
|
||||
|
||||
expect(
|
||||
loadSupplementalManifestCatalogRowsForList({
|
||||
loadStaticManifestCatalogRowsForList({
|
||||
cfg: {},
|
||||
}).map((row) => row.ref),
|
||||
).toEqual(["moonshot/kimi-k2.6", "openrouter/auto"]);
|
||||
).toEqual(["moonshot/kimi-k2.6"]);
|
||||
});
|
||||
|
||||
it("supplements runtime-owned providers with refreshed rows only", async () => {
|
||||
const { loadStaticManifestCatalogRowsForList, loadSupplementalManifestCatalogRowsForList } =
|
||||
await import("./list.manifest-catalog.js");
|
||||
it("does not expose runtime overlay rows as static manifest models", async () => {
|
||||
const { loadStaticManifestCatalogRowsForList } = await import("./list.manifest-catalog.js");
|
||||
const manifestRegistry = {
|
||||
plugins: [openaiRuntimePlugin],
|
||||
diagnostics: [],
|
||||
@@ -144,19 +142,11 @@ describe("loadStaticManifestCatalogRowsForList", () => {
|
||||
cfg: {},
|
||||
providerFilter: "openai",
|
||||
metadataSnapshot: metadataSnapshot as unknown as Parameters<
|
||||
typeof loadSupplementalManifestCatalogRowsForList
|
||||
typeof loadStaticManifestCatalogRowsForList
|
||||
>[0]["metadataSnapshot"],
|
||||
};
|
||||
|
||||
expect(loadStaticManifestCatalogRowsForList(params)).toEqual([]);
|
||||
expect(loadSupplementalManifestCatalogRowsForList(params)).toMatchObject([
|
||||
{
|
||||
provider: "openai",
|
||||
id: "gpt-refreshed",
|
||||
ref: "openai/gpt-refreshed",
|
||||
source: "runtime-refresh",
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("uses an injected metadata snapshot instead of loading metadata again", async () => {
|
||||
|
||||
@@ -2,10 +2,7 @@
|
||||
import { normalizeModelCatalogProviderId } from "@openclaw/model-catalog-core/model-catalog-refs";
|
||||
import type { NormalizedModelCatalogRow } from "@openclaw/model-catalog-core/model-catalog-types";
|
||||
import type { OpenClawConfig } from "../../config/types.openclaw.js";
|
||||
import {
|
||||
planEffectiveModelCatalogRows,
|
||||
type ManifestModelCatalogRowSelection,
|
||||
} from "../../model-catalog/index.js";
|
||||
import { planEffectiveModelCatalogRows } from "../../model-catalog/index.js";
|
||||
import { loadManifestMetadataSnapshot } from "../../plugins/manifest-contract-eligibility.js";
|
||||
import type { PluginManifestRegistry } from "../../plugins/manifest-registry.js";
|
||||
import type { PluginMetadataSnapshot } from "../../plugins/plugin-metadata-snapshot.types.js";
|
||||
@@ -19,7 +16,6 @@ import {
|
||||
function loadManifestCatalogRowsForPluginIds(params: {
|
||||
cfg: OpenClawConfig;
|
||||
registry: PluginManifestRegistry;
|
||||
mode: ManifestModelCatalogRowSelection;
|
||||
pluginIds?: readonly string[];
|
||||
providerFilter?: string;
|
||||
}): readonly NormalizedModelCatalogRow[] {
|
||||
@@ -37,7 +33,7 @@ function loadManifestCatalogRowsForPluginIds(params: {
|
||||
registry,
|
||||
config: params.cfg,
|
||||
...(params.providerFilter ? { providerFilter: params.providerFilter } : {}),
|
||||
selection: params.mode,
|
||||
selection: "static",
|
||||
}).rows;
|
||||
}
|
||||
|
||||
@@ -76,17 +72,16 @@ function resolveDeclaredModelCatalogPluginIds(params: {
|
||||
});
|
||||
}
|
||||
|
||||
function loadManifestCatalogRowsForList(params: {
|
||||
/** Loads authoritative static manifest catalog rows for model-list output. */
|
||||
export function loadStaticManifestCatalogRowsForList(params: {
|
||||
cfg: OpenClawConfig;
|
||||
providerFilter?: string;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
mode?: ManifestModelCatalogRowSelection;
|
||||
metadataSnapshot?: PluginMetadataSnapshot;
|
||||
}): readonly NormalizedModelCatalogRow[] {
|
||||
const providerFilter = params.providerFilter
|
||||
? normalizeModelCatalogProviderId(params.providerFilter)
|
||||
: undefined;
|
||||
const mode = params.mode ?? "static";
|
||||
const snapshot =
|
||||
params.metadataSnapshot ??
|
||||
loadManifestMetadataSnapshot({
|
||||
@@ -98,13 +93,11 @@ function loadManifestCatalogRowsForList(params: {
|
||||
return loadManifestCatalogRowsForPluginIds({
|
||||
cfg: params.cfg,
|
||||
registry: snapshot.manifestRegistry,
|
||||
mode,
|
||||
});
|
||||
}
|
||||
const conventionRows = loadManifestCatalogRowsForPluginIds({
|
||||
cfg: params.cfg,
|
||||
registry: snapshot.manifestRegistry,
|
||||
mode,
|
||||
pluginIds: resolveConventionModelCatalogPluginIds({
|
||||
cfg: params.cfg,
|
||||
index,
|
||||
@@ -118,7 +111,6 @@ function loadManifestCatalogRowsForList(params: {
|
||||
return loadManifestCatalogRowsForPluginIds({
|
||||
cfg: params.cfg,
|
||||
registry: snapshot.manifestRegistry,
|
||||
mode,
|
||||
pluginIds: resolveDeclaredModelCatalogPluginIds({
|
||||
cfg: params.cfg,
|
||||
index,
|
||||
@@ -127,29 +119,3 @@ function loadManifestCatalogRowsForList(params: {
|
||||
providerFilter,
|
||||
});
|
||||
}
|
||||
|
||||
/** Loads authoritative static manifest catalog rows for model-list output. */
|
||||
export function loadStaticManifestCatalogRowsForList(params: {
|
||||
cfg: OpenClawConfig;
|
||||
providerFilter?: string;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
metadataSnapshot?: PluginMetadataSnapshot;
|
||||
}): readonly NormalizedModelCatalogRow[] {
|
||||
return loadManifestCatalogRowsForList({
|
||||
...params,
|
||||
mode: "static",
|
||||
});
|
||||
}
|
||||
|
||||
/** Loads supplemental non-runtime manifest catalog rows for fallback list sources. */
|
||||
export function loadSupplementalManifestCatalogRowsForList(params: {
|
||||
cfg: OpenClawConfig;
|
||||
providerFilter?: string;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
metadataSnapshot?: PluginMetadataSnapshot;
|
||||
}): readonly NormalizedModelCatalogRow[] {
|
||||
return loadManifestCatalogRowsForList({
|
||||
...params,
|
||||
mode: "supplemental",
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1,341 +0,0 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import type {
|
||||
PluginMetadataSnapshot,
|
||||
PluginMetadataSnapshotOwnerMaps,
|
||||
} from "../../plugins/plugin-metadata-snapshot.types.js";
|
||||
|
||||
const mocks = vi.hoisted(() => ({
|
||||
loadAuthStore: vi.fn(() => ({ version: 1, profiles: {} })),
|
||||
loadMetadata: vi.fn(),
|
||||
loadOwner: vi.fn(),
|
||||
resolveImplicitProviders: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("../../agents/auth-profiles/store.js", () => ({
|
||||
loadAuthProfileStoreForSecretsRuntime: mocks.loadAuthStore,
|
||||
}));
|
||||
|
||||
vi.mock("../../agents/models-config.providers.implicit.js", () => ({
|
||||
resolveImplicitProviders: mocks.resolveImplicitProviders,
|
||||
}));
|
||||
|
||||
vi.mock("../../agents/prepared-model-catalog.js", () => ({
|
||||
loadPreparedModelCatalogOwnerSnapshot: mocks.loadOwner,
|
||||
}));
|
||||
|
||||
vi.mock("../../plugins/manifest-contract-eligibility.js", () => ({
|
||||
loadManifestMetadataSnapshot: mocks.loadMetadata,
|
||||
}));
|
||||
|
||||
import {
|
||||
hasProviderRuntimeCatalogForFilter,
|
||||
hasProviderStaticCatalogForFilter,
|
||||
loadProviderCatalogModelsForList,
|
||||
} from "./list.provider-catalog.js";
|
||||
|
||||
const emptyOwners: PluginMetadataSnapshotOwnerMaps = {
|
||||
channels: new Map(),
|
||||
channelConfigs: new Map(),
|
||||
providers: new Map(),
|
||||
modelCatalogProviders: new Map(),
|
||||
cliBackends: new Map(),
|
||||
setupProviders: new Map(),
|
||||
commandAliases: new Map(),
|
||||
contracts: new Map(),
|
||||
};
|
||||
|
||||
const emptyMetadataSnapshot = {
|
||||
manifestRegistry: { plugins: [] },
|
||||
owners: emptyOwners,
|
||||
} as unknown as PluginMetadataSnapshot;
|
||||
|
||||
function metadataWithCatalogOwner(provider: string, pluginId: string): PluginMetadataSnapshot {
|
||||
return {
|
||||
...emptyMetadataSnapshot,
|
||||
owners: {
|
||||
...emptyOwners,
|
||||
modelCatalogProviders: new Map([[provider, [pluginId]]]),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function ownerSnapshot(modelCatalog: unknown, metadataSnapshot = emptyMetadataSnapshot) {
|
||||
return {
|
||||
agentDir: "/tmp/agent",
|
||||
metadataSnapshot,
|
||||
modelCatalog,
|
||||
};
|
||||
}
|
||||
|
||||
describe("model-list provider catalog", () => {
|
||||
beforeEach(() => {
|
||||
mocks.loadAuthStore.mockClear();
|
||||
mocks.loadMetadata.mockReset();
|
||||
mocks.loadMetadata.mockReturnValue(emptyMetadataSnapshot);
|
||||
mocks.loadOwner.mockReset();
|
||||
mocks.resolveImplicitProviders.mockReset();
|
||||
});
|
||||
|
||||
it("projects a filtered provider through targeted plugin discovery", async () => {
|
||||
mocks.resolveImplicitProviders.mockResolvedValue({
|
||||
moonshot: {
|
||||
baseUrl: "https://api.moonshot.ai",
|
||||
api: "openai-completions",
|
||||
models: [{ id: "kimi-k2.6", name: "Kimi K2.6", contextWindow: 262_144 }],
|
||||
},
|
||||
});
|
||||
|
||||
await expect(
|
||||
loadProviderCatalogModelsForList({
|
||||
cfg: {},
|
||||
agentDir: "/tmp/agent",
|
||||
providerFilter: "moonshot",
|
||||
}),
|
||||
).resolves.toEqual([
|
||||
expect.objectContaining({
|
||||
provider: "moonshot",
|
||||
id: "kimi-k2.6",
|
||||
name: "Kimi K2.6",
|
||||
contextWindow: 262_144,
|
||||
maxTokens: 200_000,
|
||||
}),
|
||||
]);
|
||||
expect(mocks.resolveImplicitProviders).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
authStore: { version: 1, profiles: {} },
|
||||
providerDiscoveryProviderIds: ["moonshot"],
|
||||
}),
|
||||
);
|
||||
expect(mocks.loadAuthStore).toHaveBeenCalledWith("/tmp/agent", {
|
||||
config: {},
|
||||
externalCliProviderIds: ["moonshot"],
|
||||
});
|
||||
expect(mocks.loadOwner).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("resolves the effective agent context for filtered discovery", async () => {
|
||||
const cfg = {
|
||||
agents: {
|
||||
list: [
|
||||
{
|
||||
id: "worker",
|
||||
agentDir: "/tmp/model-list-worker-agent",
|
||||
workspace: "/tmp/model-list-worker-workspace",
|
||||
},
|
||||
],
|
||||
},
|
||||
};
|
||||
mocks.resolveImplicitProviders.mockResolvedValue({ moonshot: { models: [] } });
|
||||
|
||||
await loadProviderCatalogModelsForList({
|
||||
cfg,
|
||||
agentId: "worker",
|
||||
providerFilter: "moonshot",
|
||||
});
|
||||
|
||||
expect(mocks.loadMetadata).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
config: cfg,
|
||||
workspaceDir: "/tmp/model-list-worker-workspace",
|
||||
}),
|
||||
);
|
||||
expect(mocks.loadAuthStore).toHaveBeenCalledWith("/tmp/model-list-worker-agent", {
|
||||
config: cfg,
|
||||
externalCliProviderIds: ["moonshot"],
|
||||
});
|
||||
expect(mocks.resolveImplicitProviders).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ agentDir: "/tmp/model-list-worker-agent" }),
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps unfiltered runtime output on the lifecycle owner", async () => {
|
||||
mocks.loadOwner.mockResolvedValue(
|
||||
ownerSnapshot({
|
||||
entries: [
|
||||
{ provider: "moonshot", id: "kimi-k2.6", name: "Kimi K2.6" },
|
||||
{ provider: "ollama", id: "local-model", name: "Local Model" },
|
||||
],
|
||||
routeVariants: [],
|
||||
}),
|
||||
);
|
||||
|
||||
await expect(
|
||||
loadProviderCatalogModelsForList({
|
||||
cfg: {},
|
||||
agentDir: "/tmp/agent",
|
||||
}),
|
||||
).resolves.not.toContainEqual(expect.objectContaining({ provider: "ollama" }));
|
||||
});
|
||||
|
||||
it("keeps static provider-hook rows separate from runtime ownership", async () => {
|
||||
mocks.loadOwner.mockResolvedValue(
|
||||
ownerSnapshot({
|
||||
entries: [{ provider: "moonshot", id: "kimi-runtime", name: "Kimi Runtime" }],
|
||||
staticEntries: [{ provider: "nvidia", id: "nemotron-static", name: "Nemotron Static" }],
|
||||
routeVariants: [],
|
||||
}),
|
||||
);
|
||||
|
||||
await expect(
|
||||
hasProviderRuntimeCatalogForFilter({
|
||||
cfg: {},
|
||||
agentId: "worker",
|
||||
agentDir: "/tmp/agent",
|
||||
providerFilter: "nvidia",
|
||||
}),
|
||||
).resolves.toBe(false);
|
||||
await expect(
|
||||
hasProviderStaticCatalogForFilter({
|
||||
cfg: {},
|
||||
agentDir: "/tmp/agent",
|
||||
providerFilter: "nvidia",
|
||||
}),
|
||||
).resolves.toBe(true);
|
||||
await expect(
|
||||
hasProviderStaticCatalogForFilter({
|
||||
cfg: {},
|
||||
agentDir: "/tmp/agent",
|
||||
}),
|
||||
).resolves.toBe(true);
|
||||
await expect(
|
||||
loadProviderCatalogModelsForList({
|
||||
cfg: {},
|
||||
agentDir: "/tmp/agent",
|
||||
staticOnly: true,
|
||||
}),
|
||||
).resolves.toEqual([{ provider: "nvidia", id: "nemotron-static", name: "Nemotron Static" }]);
|
||||
expect(mocks.loadOwner).toHaveBeenCalledWith(expect.objectContaining({ readOnly: true }));
|
||||
});
|
||||
|
||||
it("uses model-catalog manifest ownership without activating a prepared runtime", async () => {
|
||||
const env = { OPENCLAW_STATE_DIR: "/tmp/model-list-state" };
|
||||
mocks.loadMetadata.mockReturnValue(metadataWithCatalogOwner("moonshot", "moonshot"));
|
||||
|
||||
await expect(
|
||||
hasProviderRuntimeCatalogForFilter({
|
||||
cfg: {},
|
||||
agentId: "worker",
|
||||
agentDir: "/tmp/agent",
|
||||
env,
|
||||
providerFilter: "moonshot",
|
||||
}),
|
||||
).resolves.toBe(true);
|
||||
expect(mocks.loadOwner).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does not treat generic provider ownership as runtime catalog ownership", async () => {
|
||||
mocks.loadMetadata.mockReturnValue({
|
||||
...emptyMetadataSnapshot,
|
||||
owners: {
|
||||
...emptyMetadataSnapshot.owners,
|
||||
providers: new Map([["auth-only", ["auth-only"]]]),
|
||||
cliBackends: new Map([["auth-only", ["auth-only"]]]),
|
||||
},
|
||||
});
|
||||
|
||||
await expect(
|
||||
hasProviderRuntimeCatalogForFilter({
|
||||
cfg: {},
|
||||
agentDir: "/tmp/agent",
|
||||
providerFilter: "auth-only",
|
||||
}),
|
||||
).resolves.toBe(false);
|
||||
});
|
||||
|
||||
it("derives the matching directory for an explicit agent", async () => {
|
||||
const cfg = {
|
||||
agents: {
|
||||
list: [{ id: "worker", agentDir: "/tmp/model-list-worker-agent" }],
|
||||
},
|
||||
};
|
||||
mocks.loadOwner.mockResolvedValue(
|
||||
ownerSnapshot({
|
||||
entries: [],
|
||||
staticEntries: [{ provider: "nvidia", id: "worker-model", name: "Worker Model" }],
|
||||
routeVariants: [],
|
||||
}),
|
||||
);
|
||||
|
||||
await expect(
|
||||
hasProviderStaticCatalogForFilter({
|
||||
cfg,
|
||||
agentId: "worker",
|
||||
providerFilter: "nvidia",
|
||||
}),
|
||||
).resolves.toBe(true);
|
||||
expect(mocks.loadOwner).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
agentId: "worker",
|
||||
agentDir: "/tmp/model-list-worker-agent",
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("resolves provider aliases without a caller-supplied metadata snapshot", async () => {
|
||||
mocks.loadMetadata.mockReturnValue({
|
||||
manifestRegistry: {
|
||||
plugins: [
|
||||
{
|
||||
id: "moonshot",
|
||||
modelCatalog: {
|
||||
aliases: { kimi: { provider: "moonshot" } },
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
mocks.resolveImplicitProviders.mockResolvedValue({
|
||||
moonshot: {
|
||||
baseUrl: "https://api.moonshot.ai",
|
||||
api: "openai-completions",
|
||||
models: [{ id: "kimi-k2.6", name: "Kimi K2.6" }],
|
||||
},
|
||||
});
|
||||
|
||||
await loadProviderCatalogModelsForList({
|
||||
cfg: {},
|
||||
agentDir: "/tmp/agent",
|
||||
providerFilter: "kimi",
|
||||
});
|
||||
|
||||
expect(mocks.resolveImplicitProviders).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ providerDiscoveryProviderIds: ["moonshot"] }),
|
||||
);
|
||||
});
|
||||
|
||||
it("matches provider aliases from the captured metadata generation", async () => {
|
||||
const metadataSnapshot = {
|
||||
manifestRegistry: {
|
||||
plugins: [
|
||||
{
|
||||
id: "moonshot",
|
||||
modelCatalog: {
|
||||
aliases: { kimi: { provider: "moonshot" } },
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
} as never;
|
||||
mocks.resolveImplicitProviders.mockResolvedValue({
|
||||
moonshot: {
|
||||
baseUrl: "https://api.moonshot.ai",
|
||||
api: "openai-completions",
|
||||
models: [{ id: "kimi-k2.6", name: "Kimi K2.6" }],
|
||||
},
|
||||
});
|
||||
|
||||
await expect(
|
||||
loadProviderCatalogModelsForList({
|
||||
cfg: {},
|
||||
agentDir: "/tmp/agent",
|
||||
providerFilter: "kimi",
|
||||
metadataSnapshot,
|
||||
}),
|
||||
).resolves.toEqual([
|
||||
expect.objectContaining({ provider: "moonshot", id: "kimi-k2.6", name: "Kimi K2.6" }),
|
||||
]);
|
||||
expect(mocks.resolveImplicitProviders).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ providerDiscoveryProviderIds: ["moonshot"] }),
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -1,188 +0,0 @@
|
||||
/** Provider catalog projection for model-list output. */
|
||||
import { normalizeProviderId } from "@openclaw/model-catalog-core/provider-id";
|
||||
import {
|
||||
resolveAgentDir,
|
||||
resolveAgentWorkspaceDir,
|
||||
resolveDefaultAgentDir,
|
||||
resolveDefaultAgentId,
|
||||
} from "../../agents/agent-scope.js";
|
||||
import { loadAuthProfileStoreForSecretsRuntime } from "../../agents/auth-profiles/store.js";
|
||||
import { DEFAULT_CONTEXT_TOKENS } from "../../agents/defaults.js";
|
||||
import { buildInlineProviderModels } from "../../agents/embedded-agent-runner/model.inline-provider.js";
|
||||
import { resolveImplicitProviders } from "../../agents/models-config.providers.implicit.js";
|
||||
import { loadPreparedModelCatalogOwnerSnapshot } from "../../agents/prepared-model-catalog.js";
|
||||
import { resolveDefaultAgentWorkspaceDir } from "../../agents/workspace.js";
|
||||
import type { OpenClawConfig } from "../../config/types.openclaw.js";
|
||||
import type { Model } from "../../llm/types.js";
|
||||
import { loadManifestMetadataSnapshot } from "../../plugins/manifest-contract-eligibility.js";
|
||||
import type { PluginMetadataSnapshot } from "../../plugins/plugin-metadata-snapshot.types.js";
|
||||
import { canonicalizeModelCatalogProviderAlias } from "./provider-aliases.js";
|
||||
|
||||
type ProviderCatalogListParams = {
|
||||
cfg: OpenClawConfig;
|
||||
agentId?: string;
|
||||
agentDir?: string;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
providerFilter?: string;
|
||||
staticOnly?: boolean;
|
||||
metadataSnapshot?: PluginMetadataSnapshot;
|
||||
};
|
||||
|
||||
const SELF_HOSTED_DISCOVERY_PROVIDER_IDS = new Set(["lmstudio", "ollama", "sglang", "vllm"]);
|
||||
|
||||
async function loadProviderCatalogSnapshot(
|
||||
params: ProviderCatalogListParams,
|
||||
options: { readOnly?: boolean } = {},
|
||||
) {
|
||||
const input = {
|
||||
config: params.cfg,
|
||||
...(params.agentId ? { agentId: params.agentId } : {}),
|
||||
agentDir: resolveProviderCatalogAgentDir(params),
|
||||
...(params.metadataSnapshot?.workspaceDir
|
||||
? { workspaceDir: params.metadataSnapshot.workspaceDir }
|
||||
: {}),
|
||||
...(params.env ? { env: params.env } : {}),
|
||||
...(options.readOnly ? { readOnly: true } : {}),
|
||||
};
|
||||
return await loadPreparedModelCatalogOwnerSnapshot(input);
|
||||
}
|
||||
|
||||
function resolveProviderFilter(
|
||||
params: ProviderCatalogListParams,
|
||||
metadataSnapshot?: PluginMetadataSnapshot,
|
||||
): string {
|
||||
const providerFilter = normalizeProviderId(params.providerFilter ?? "");
|
||||
return providerFilter
|
||||
? normalizeProviderId(
|
||||
canonicalizeModelCatalogProviderAlias(providerFilter, {
|
||||
cfg: params.cfg,
|
||||
metadataSnapshot,
|
||||
}),
|
||||
)
|
||||
: providerFilter;
|
||||
}
|
||||
|
||||
function resolveProviderCatalogMetadataSnapshot(
|
||||
params: ProviderCatalogListParams,
|
||||
): PluginMetadataSnapshot {
|
||||
if (params.metadataSnapshot) {
|
||||
return params.metadataSnapshot;
|
||||
}
|
||||
const agentId = params.agentId ?? resolveDefaultAgentId(params.cfg);
|
||||
const workspaceDir =
|
||||
resolveAgentWorkspaceDir(params.cfg, agentId) ?? resolveDefaultAgentWorkspaceDir();
|
||||
return loadManifestMetadataSnapshot({
|
||||
config: params.cfg,
|
||||
env: params.env ?? process.env,
|
||||
workspaceDir,
|
||||
});
|
||||
}
|
||||
|
||||
function resolveProviderCatalogAgentDir(
|
||||
params: Omit<ProviderCatalogListParams, "agentDir"> & { agentDir?: string },
|
||||
): string {
|
||||
return (
|
||||
params.agentDir ??
|
||||
(params.agentId
|
||||
? resolveAgentDir(params.cfg, params.agentId, params.env)
|
||||
: resolveDefaultAgentDir(params.cfg, params.env))
|
||||
);
|
||||
}
|
||||
|
||||
function completeCatalogModel(model: ReturnType<typeof buildInlineProviderModels>[number]): Model {
|
||||
const contextWindow = model.contextWindow ?? DEFAULT_CONTEXT_TOKENS;
|
||||
return {
|
||||
...model,
|
||||
name: model.name || model.id,
|
||||
api: model.api ?? "openai-responses",
|
||||
baseUrl: model.baseUrl ?? "",
|
||||
reasoning: model.reasoning ?? false,
|
||||
input: model.input ?? ["text"],
|
||||
cost: model.cost ?? { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow,
|
||||
maxTokens: model.maxTokens ?? DEFAULT_CONTEXT_TOKENS,
|
||||
} as Model;
|
||||
}
|
||||
|
||||
async function loadScopedProviderCatalogModels(
|
||||
params: ProviderCatalogListParams,
|
||||
): Promise<Model[]> {
|
||||
const metadataSnapshot = resolveProviderCatalogMetadataSnapshot(params);
|
||||
const providerFilter = resolveProviderFilter(params, metadataSnapshot);
|
||||
if (!providerFilter) {
|
||||
return [];
|
||||
}
|
||||
const agentDir = resolveProviderCatalogAgentDir(params);
|
||||
const providers = await resolveImplicitProviders({
|
||||
agentDir,
|
||||
authStore: loadAuthProfileStoreForSecretsRuntime(agentDir, {
|
||||
config: params.cfg,
|
||||
externalCliProviderIds: [providerFilter],
|
||||
}),
|
||||
config: params.cfg,
|
||||
explicitProviders: params.cfg.models?.providers ?? null,
|
||||
providerDiscoveryProviderIds: [providerFilter],
|
||||
...(params.env ? { env: params.env } : {}),
|
||||
...(metadataSnapshot.workspaceDir ? { workspaceDir: metadataSnapshot.workspaceDir } : {}),
|
||||
pluginMetadataSnapshot: metadataSnapshot,
|
||||
});
|
||||
return buildInlineProviderModels(providers ?? {})
|
||||
.filter((model) => normalizeProviderId(model.provider) === providerFilter)
|
||||
.map(completeCatalogModel);
|
||||
}
|
||||
|
||||
/** Returns true when manifest ownership exposes a runtime catalog for the provider filter. */
|
||||
export async function hasProviderRuntimeCatalogForFilter(
|
||||
params: Omit<ProviderCatalogListParams, "agentDir"> & { agentDir?: string },
|
||||
): Promise<boolean> {
|
||||
const resolvedParams = {
|
||||
...params,
|
||||
agentDir: resolveProviderCatalogAgentDir(params),
|
||||
};
|
||||
const metadataSnapshot = resolveProviderCatalogMetadataSnapshot(resolvedParams);
|
||||
const providerFilter = resolveProviderFilter(resolvedParams, metadataSnapshot);
|
||||
if (!providerFilter) {
|
||||
return false;
|
||||
}
|
||||
return Boolean(metadataSnapshot.owners.modelCatalogProviders.get(providerFilter)?.length);
|
||||
}
|
||||
|
||||
/** Returns true when the prepared generation captured static provider-hook rows. */
|
||||
export async function hasProviderStaticCatalogForFilter(
|
||||
params: Omit<ProviderCatalogListParams, "agentDir"> & { agentDir?: string },
|
||||
): Promise<boolean> {
|
||||
const resolvedParams = {
|
||||
...params,
|
||||
agentDir: resolveProviderCatalogAgentDir(params),
|
||||
};
|
||||
const owner = await loadProviderCatalogSnapshot(resolvedParams, { readOnly: true });
|
||||
const providerFilter = resolveProviderFilter(resolvedParams, owner.metadataSnapshot);
|
||||
return (owner.modelCatalog.staticEntries ?? []).some(
|
||||
(entry) => !providerFilter || normalizeProviderId(entry.provider) === providerFilter,
|
||||
);
|
||||
}
|
||||
|
||||
/** Projects provider rows from the committed model catalog without discovery or cache IO. */
|
||||
export async function loadProviderCatalogModelsForList(
|
||||
params: ProviderCatalogListParams,
|
||||
): Promise<Model[]> {
|
||||
if (params.providerFilter && params.staticOnly !== true) {
|
||||
return await loadScopedProviderCatalogModels(params);
|
||||
}
|
||||
const owner = await loadProviderCatalogSnapshot(params, {
|
||||
readOnly: params.staticOnly === true,
|
||||
});
|
||||
const providerFilter = resolveProviderFilter(params, owner.metadataSnapshot);
|
||||
const entries = params.staticOnly
|
||||
? (owner.modelCatalog.staticEntries ?? [])
|
||||
: owner.modelCatalog.entries;
|
||||
return entries
|
||||
.filter((entry) => {
|
||||
const provider = normalizeProviderId(entry.provider);
|
||||
if (!providerFilter && SELF_HOSTED_DISCOVERY_PROVIDER_IDS.has(provider)) {
|
||||
return false;
|
||||
}
|
||||
return !providerFilter || provider === providerFilter;
|
||||
})
|
||||
.map((entry) => Object.assign({}, entry) as Model);
|
||||
}
|
||||
@@ -1,53 +0,0 @@
|
||||
// Model provider index catalog tests cover model list catalog indexing and provider grouping.
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { OpenClawConfig } from "../../config/types.openclaw.js";
|
||||
import { loadProviderIndexCatalogRowsForList } from "./list.provider-index-catalog.js";
|
||||
|
||||
const baseConfig = {} satisfies OpenClawConfig;
|
||||
|
||||
describe("loadProviderIndexCatalogRowsForList", () => {
|
||||
it("returns provider-index preview rows when the provider plugin is enabled", () => {
|
||||
expect(
|
||||
loadProviderIndexCatalogRowsForList({
|
||||
cfg: baseConfig,
|
||||
providerFilter: "moonshot",
|
||||
}).map((row) => row.ref),
|
||||
).toEqual([
|
||||
"moonshot/kimi-k2.6",
|
||||
"moonshot/kimi-k2.7-code",
|
||||
"moonshot/kimi-k2.7-code-highspeed",
|
||||
"moonshot/kimi-k3",
|
||||
]);
|
||||
});
|
||||
|
||||
it("returns all enabled provider-index preview rows without a provider filter", () => {
|
||||
const refs = loadProviderIndexCatalogRowsForList({
|
||||
cfg: baseConfig,
|
||||
}).map((row) => row.ref);
|
||||
expect(refs).toEqual([
|
||||
"deepseek/deepseek-chat",
|
||||
"deepseek/deepseek-reasoner",
|
||||
"deepseek/deepseek-v4-flash",
|
||||
"deepseek/deepseek-v4-pro",
|
||||
"moonshot/kimi-k2.6",
|
||||
"moonshot/kimi-k2.7-code",
|
||||
"moonshot/kimi-k2.7-code-highspeed",
|
||||
"moonshot/kimi-k3",
|
||||
]);
|
||||
});
|
||||
|
||||
it("suppresses provider-index preview rows when the provider plugin is disabled", () => {
|
||||
expect(
|
||||
loadProviderIndexCatalogRowsForList({
|
||||
cfg: {
|
||||
plugins: {
|
||||
entries: {
|
||||
moonshot: { enabled: false },
|
||||
},
|
||||
},
|
||||
},
|
||||
providerFilter: "moonshot",
|
||||
}),
|
||||
).toStrictEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -1,35 +0,0 @@
|
||||
/** Provider-index-backed model catalog rows for bundled model-list output. */
|
||||
import { normalizeModelCatalogProviderId } from "@openclaw/model-catalog-core/model-catalog-refs";
|
||||
import type { NormalizedModelCatalogRow } from "@openclaw/model-catalog-core/model-catalog-types";
|
||||
import type { OpenClawConfig } from "../../config/types.openclaw.js";
|
||||
import {
|
||||
loadOpenClawProviderIndex,
|
||||
planProviderIndexModelCatalogRows,
|
||||
} from "../../model-catalog/index.js";
|
||||
import { normalizePluginsConfig, resolveEffectiveEnableState } from "../../plugins/config-state.js";
|
||||
|
||||
/** Loads enabled bundled provider-index catalog rows, optionally scoped by provider. */
|
||||
export function loadProviderIndexCatalogRowsForList(params: {
|
||||
providerFilter?: string;
|
||||
cfg: OpenClawConfig;
|
||||
}): readonly NormalizedModelCatalogRow[] {
|
||||
const providerFilter = params.providerFilter
|
||||
? normalizeModelCatalogProviderId(params.providerFilter)
|
||||
: undefined;
|
||||
const index = loadOpenClawProviderIndex();
|
||||
return planProviderIndexModelCatalogRows({
|
||||
index,
|
||||
...(providerFilter ? { providerFilter } : {}),
|
||||
})
|
||||
.entries.filter(
|
||||
(entry) =>
|
||||
resolveEffectiveEnableState({
|
||||
id: entry.pluginId,
|
||||
origin: "bundled",
|
||||
config: normalizePluginsConfig(params.cfg.plugins),
|
||||
rootConfig: params.cfg,
|
||||
enabledByDefault: true,
|
||||
}).enabled,
|
||||
)
|
||||
.flatMap((entry) => entry.rows);
|
||||
}
|
||||
@@ -1,17 +1,13 @@
|
||||
/** Orchestrates model-list row sources across registry, manifests, catalogs, and config. */
|
||||
/** Projects the prepared model generation together with configured model rows. */
|
||||
import type { ModelRegistry } from "../../llm/model-registry.js";
|
||||
import {
|
||||
appendCatalogSupplementRows,
|
||||
appendAuthenticatedCatalogRows,
|
||||
appendConfiguredProviderRows,
|
||||
appendConfiguredRows,
|
||||
appendDiscoveredRows,
|
||||
appendManifestCatalogRows,
|
||||
appendModelCatalogRows,
|
||||
appendProviderCatalogRows,
|
||||
appendPreparedModelCatalogRows,
|
||||
type RowBuilderContext,
|
||||
} from "./list.rows.js";
|
||||
import type { ModelListSourcePlan } from "./list.source-plan.js";
|
||||
import type { ConfiguredEntry, ModelRow } from "./list.types.js";
|
||||
|
||||
type AllModelRowSources = {
|
||||
@@ -20,99 +16,10 @@ type AllModelRowSources = {
|
||||
context: RowBuilderContext;
|
||||
modelRegistry?: ModelRegistry;
|
||||
registryModels?: ReturnType<ModelRegistry["getAll"]>;
|
||||
sourcePlan: ModelListSourcePlan;
|
||||
};
|
||||
|
||||
type AppendAllModelRowSourcesResult = {
|
||||
requiresRegistryFallback: boolean;
|
||||
};
|
||||
|
||||
/** Appends all rows requested by `models list --all` or a provider-filtered list. */
|
||||
export async function appendAllModelRowSources(
|
||||
params: AllModelRowSources,
|
||||
): Promise<AppendAllModelRowSourcesResult> {
|
||||
if (params.context.filter.provider && params.sourcePlan.kind !== "registry") {
|
||||
const seenKeys = new Set<string>();
|
||||
let catalogRows = 0;
|
||||
if (params.sourcePlan.kind === "manifest") {
|
||||
catalogRows = await appendManifestCatalogRows({
|
||||
rows: params.rows,
|
||||
context: params.context,
|
||||
seenKeys,
|
||||
manifestRows: params.sourcePlan.manifestCatalogRows,
|
||||
});
|
||||
}
|
||||
if (catalogRows === 0 && params.sourcePlan.kind === "provider-index") {
|
||||
catalogRows = await appendModelCatalogRows({
|
||||
rows: params.rows,
|
||||
context: params.context,
|
||||
seenKeys,
|
||||
catalogRows: params.sourcePlan.providerIndexCatalogRows,
|
||||
});
|
||||
}
|
||||
if (
|
||||
catalogRows === 0 &&
|
||||
(params.sourcePlan.kind === "provider-runtime-static" ||
|
||||
params.sourcePlan.kind === "provider-runtime-scoped")
|
||||
) {
|
||||
catalogRows = await appendProviderCatalogRows({
|
||||
rows: params.rows,
|
||||
context: params.context,
|
||||
seenKeys,
|
||||
staticOnly: params.sourcePlan.kind === "provider-runtime-static",
|
||||
});
|
||||
if (params.sourcePlan.manifestCatalogRows.length > 0) {
|
||||
// Runtime discovery keeps precedence; refreshed manifest rows fill only
|
||||
// model refs the provider runtime has not materialized yet.
|
||||
catalogRows += await appendManifestCatalogRows({
|
||||
rows: params.rows,
|
||||
context: { ...params.context, skipRuntimeModelSuppression: true },
|
||||
seenKeys,
|
||||
manifestRows: params.sourcePlan.manifestCatalogRows,
|
||||
});
|
||||
}
|
||||
}
|
||||
if (params.entries && params.entries.length > 0) {
|
||||
const missingEntries = params.entries.filter((entry) => !seenKeys.has(entry.key));
|
||||
if (missingEntries.length > 0) {
|
||||
await appendConfiguredRows({
|
||||
rows: params.rows,
|
||||
entries: missingEntries,
|
||||
modelRegistry: params.modelRegistry,
|
||||
context: params.context,
|
||||
});
|
||||
for (const row of params.rows) {
|
||||
seenKeys.add(row.key);
|
||||
}
|
||||
}
|
||||
}
|
||||
await appendConfiguredProviderRows({
|
||||
rows: params.rows,
|
||||
context: params.context,
|
||||
seenKeys,
|
||||
});
|
||||
if (
|
||||
catalogRows === 0 &&
|
||||
params.rows.length === 0 &&
|
||||
(params.sourcePlan.kind === "provider-runtime-static" ||
|
||||
params.sourcePlan.kind === "provider-runtime-scoped")
|
||||
) {
|
||||
// Provider-scoped static sources can be empty when a plugin exposes only
|
||||
// runtime discovery; tell the caller to load the registry and retry.
|
||||
if (!params.modelRegistry) {
|
||||
return { requiresRegistryFallback: true };
|
||||
}
|
||||
await appendDiscoveredRows({
|
||||
rows: params.rows,
|
||||
models: params.registryModels ?? params.modelRegistry.getAll(),
|
||||
modelRegistry: params.modelRegistry,
|
||||
context: params.context,
|
||||
resolveWithRegistry: false,
|
||||
});
|
||||
}
|
||||
return { requiresRegistryFallback: false };
|
||||
}
|
||||
|
||||
export async function appendAllModelRowSources(params: AllModelRowSources): Promise<void> {
|
||||
const seenKeys = await appendDiscoveredRows({
|
||||
rows: params.rows,
|
||||
models: params.registryModels ?? params.modelRegistry?.getAll() ?? [],
|
||||
@@ -122,6 +29,22 @@ export async function appendAllModelRowSources(
|
||||
skipSuppression: Boolean(params.modelRegistry),
|
||||
});
|
||||
|
||||
// Registry rows win when already discovered; every remaining catalog row and
|
||||
// route variant must come from the single committed lifecycle generation.
|
||||
await appendPreparedModelCatalogRows({
|
||||
rows: params.rows,
|
||||
context: params.context,
|
||||
seenKeys,
|
||||
});
|
||||
|
||||
// Authored provider definitions are authoritative for configured metadata;
|
||||
// only synthesize missing configured references after real rows were tried.
|
||||
await appendConfiguredProviderRows({
|
||||
rows: params.rows,
|
||||
context: params.context,
|
||||
seenKeys,
|
||||
});
|
||||
|
||||
if (params.context.filter.provider && params.entries && params.entries.length > 0) {
|
||||
const missingEntries = params.entries.filter((entry) => !seenKeys.has(entry.key));
|
||||
if (missingEntries.length > 0) {
|
||||
@@ -137,49 +60,6 @@ export async function appendAllModelRowSources(
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
await appendConfiguredProviderRows({
|
||||
rows: params.rows,
|
||||
context: params.context,
|
||||
seenKeys,
|
||||
});
|
||||
|
||||
if (params.sourcePlan.manifestCatalogRows.length > 0) {
|
||||
await appendManifestCatalogRows({
|
||||
rows: params.rows,
|
||||
context: { ...params.context, skipRuntimeModelSuppression: true },
|
||||
seenKeys,
|
||||
manifestRows: params.sourcePlan.manifestCatalogRows,
|
||||
});
|
||||
}
|
||||
|
||||
if (params.sourcePlan.providerIndexCatalogRows.length > 0) {
|
||||
await appendModelCatalogRows({
|
||||
rows: params.rows,
|
||||
context: { ...params.context, skipRuntimeModelSuppression: true },
|
||||
seenKeys,
|
||||
catalogRows: params.sourcePlan.providerIndexCatalogRows,
|
||||
});
|
||||
}
|
||||
|
||||
if (params.modelRegistry && params.context.filter.provider) {
|
||||
await appendCatalogSupplementRows({
|
||||
rows: params.rows,
|
||||
modelRegistry: params.modelRegistry,
|
||||
context: params.context,
|
||||
seenKeys,
|
||||
});
|
||||
}
|
||||
if (params.modelRegistry) {
|
||||
return { requiresRegistryFallback: false };
|
||||
}
|
||||
|
||||
await appendProviderCatalogRows({
|
||||
rows: params.rows,
|
||||
context: params.context,
|
||||
seenKeys,
|
||||
});
|
||||
return { requiresRegistryFallback: false };
|
||||
}
|
||||
|
||||
/** Appends the configured/default rows used by the cheap default list path. */
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
// Model list row tests cover rendered row construction for model listing output.
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import type { ModelCatalogEntry } from "../../agents/model-catalog.types.js";
|
||||
import type { ModelRow } from "./list.types.js";
|
||||
|
||||
const mocks = vi.hoisted(() => ({
|
||||
@@ -29,7 +30,8 @@ import {
|
||||
appendConfiguredRows,
|
||||
appendConfiguredProviderRows,
|
||||
appendDiscoveredRows,
|
||||
appendProviderCatalogRows,
|
||||
appendPreparedModelCatalogRows,
|
||||
type RowBuilderContext,
|
||||
} from "./list.rows.js";
|
||||
|
||||
const authIndex = {
|
||||
@@ -52,10 +54,219 @@ function requireOnlyRow(rows: ModelRow[]): ModelRow {
|
||||
return row;
|
||||
}
|
||||
|
||||
async function appendCommittedProviderCatalogRows(params: {
|
||||
rows: ModelRow[];
|
||||
seenKeys: Set<string>;
|
||||
catalogModels: ModelCatalogEntry[];
|
||||
context: RowBuilderContext;
|
||||
}): Promise<void> {
|
||||
const { catalogModels, ...projection } = params;
|
||||
await appendPreparedModelCatalogRows({
|
||||
...projection,
|
||||
catalogSnapshot: {
|
||||
entries: catalogModels,
|
||||
routeVariants: catalogModels,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
describe("appendPreparedModelCatalogRows", () => {
|
||||
it("projects the committed OpenAI route variants without rediscovering the catalog", async () => {
|
||||
const platform = {
|
||||
id: "gpt-5.5",
|
||||
name: "Platform GPT-5.5",
|
||||
provider: "openai",
|
||||
api: "openai-responses" as const,
|
||||
baseUrl: "https://api.openai.com/v1",
|
||||
input: ["text", "image"] as ("text" | "image")[],
|
||||
contextWindow: 1_000_000,
|
||||
};
|
||||
const chatgpt = {
|
||||
...platform,
|
||||
name: "ChatGPT GPT-5.5",
|
||||
api: "openai-chatgpt-responses" as const,
|
||||
baseUrl: "https://chatgpt.com/backend-api/codex",
|
||||
input: ["text"] as ("text" | "image")[],
|
||||
contextWindow: 400_000,
|
||||
};
|
||||
const selectedRoute = {
|
||||
api: chatgpt.api,
|
||||
baseUrl: chatgpt.baseUrl,
|
||||
authRequirement: "subscription" as const,
|
||||
requestTransportOverrides: "none" as const,
|
||||
};
|
||||
const evaluateModelAuth = vi.fn(() => ({
|
||||
availability: true,
|
||||
routeResolution: {
|
||||
kind: "routes" as const,
|
||||
routes: [selectedRoute] as [typeof selectedRoute],
|
||||
},
|
||||
selectedRoute,
|
||||
}));
|
||||
const rows: ModelRow[] = [];
|
||||
|
||||
await appendPreparedModelCatalogRows({
|
||||
rows,
|
||||
seenKeys: new Set(),
|
||||
catalogSnapshot: {
|
||||
entries: [platform],
|
||||
routeVariants: [platform, chatgpt],
|
||||
staticEntries: [platform],
|
||||
},
|
||||
context: {
|
||||
cfg: {},
|
||||
agentDir: "/tmp/openclaw-agent",
|
||||
authIndex: { evaluateModelAuth },
|
||||
configuredByKey: new Map(),
|
||||
discoveredKeys: new Set(),
|
||||
filter: { provider: "openai" },
|
||||
skipRuntimeModelSuppression: true,
|
||||
},
|
||||
});
|
||||
|
||||
expect(requireOnlyRow(rows)).toMatchObject({
|
||||
key: "openai/gpt-5.5",
|
||||
name: "ChatGPT GPT-5.5",
|
||||
input: "text",
|
||||
contextWindow: 400_000,
|
||||
available: true,
|
||||
});
|
||||
expect(evaluateModelAuth).toHaveBeenCalledWith("openai", {
|
||||
modelId: "gpt-5.5",
|
||||
observedRoutes: [
|
||||
{ api: platform.api, baseUrl: platform.baseUrl },
|
||||
{ api: chatgpt.api, baseUrl: chatgpt.baseUrl },
|
||||
],
|
||||
});
|
||||
expect(mocks.loadModelCatalogSnapshot).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("projects static-only provider hooks from the same committed generation", async () => {
|
||||
const entry = {
|
||||
id: "nemotron-static",
|
||||
name: "Nemotron Static",
|
||||
provider: "nvidia",
|
||||
api: "openai-completions" as const,
|
||||
baseUrl: "https://integrate.api.nvidia.com/v1",
|
||||
input: ["text"] as "text"[],
|
||||
};
|
||||
const evaluateModelAuth = vi.fn(() => authEvaluation(true));
|
||||
const rows: ModelRow[] = [];
|
||||
|
||||
await appendPreparedModelCatalogRows({
|
||||
rows,
|
||||
seenKeys: new Set(),
|
||||
catalogSnapshot: {
|
||||
entries: [],
|
||||
routeVariants: [],
|
||||
staticEntries: [entry, entry],
|
||||
},
|
||||
context: {
|
||||
cfg: {},
|
||||
agentDir: "/tmp/openclaw-agent",
|
||||
authIndex: { evaluateModelAuth },
|
||||
configuredByKey: new Map(),
|
||||
discoveredKeys: new Set(),
|
||||
filter: { provider: "nvidia" },
|
||||
skipRuntimeModelSuppression: true,
|
||||
},
|
||||
});
|
||||
|
||||
expect(requireOnlyRow(rows)).toMatchObject({
|
||||
key: "nvidia/nemotron-static",
|
||||
name: "Nemotron Static",
|
||||
available: true,
|
||||
});
|
||||
expect(evaluateModelAuth).toHaveBeenCalledWith("nvidia", {
|
||||
modelId: "nemotron-static",
|
||||
observedRoutes: [{ api: entry.api, baseUrl: entry.baseUrl }],
|
||||
});
|
||||
expect(mocks.loadModelCatalogSnapshot).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("filters and deduplicates local rows from the committed generation", async () => {
|
||||
const local = {
|
||||
id: "qwen2.5:7b",
|
||||
name: "Qwen 2.5 7B",
|
||||
provider: "ollama",
|
||||
baseUrl: "http://127.0.0.1:11434/v1",
|
||||
input: ["text"] as "text"[],
|
||||
};
|
||||
const rows: ModelRow[] = [];
|
||||
const seenKeys = new Set<string>();
|
||||
const params = {
|
||||
rows,
|
||||
seenKeys,
|
||||
catalogSnapshot: {
|
||||
entries: [local, { ...local, id: "remote", baseUrl: "https://models.example/v1" }],
|
||||
routeVariants: [local],
|
||||
},
|
||||
context: {
|
||||
cfg: {},
|
||||
agentDir: "/tmp/openclaw-agent",
|
||||
authIndex: { evaluateModelAuth: () => authEvaluation(true) },
|
||||
configuredByKey: new Map(),
|
||||
discoveredKeys: new Set<string>(),
|
||||
filter: { provider: "ollama", local: true },
|
||||
skipRuntimeModelSuppression: true,
|
||||
},
|
||||
};
|
||||
|
||||
await appendPreparedModelCatalogRows(params);
|
||||
await appendPreparedModelCatalogRows(params);
|
||||
|
||||
expect(requireOnlyRow(rows)).toMatchObject({
|
||||
key: "ollama/qwen2.5:7b",
|
||||
local: true,
|
||||
available: true,
|
||||
});
|
||||
expect(seenKeys).toEqual(new Set(["ollama/qwen2.5:7b"]));
|
||||
});
|
||||
|
||||
it("acquires exactly one read-only prepared generation when none was supplied", async () => {
|
||||
const entry = {
|
||||
id: "claude-opus-4-7",
|
||||
name: "Claude Opus 4.7",
|
||||
provider: "anthropic",
|
||||
input: ["text"] as "text"[],
|
||||
};
|
||||
mocks.loadModelCatalogSnapshot.mockResolvedValueOnce({
|
||||
entries: [entry],
|
||||
routeVariants: [entry],
|
||||
});
|
||||
const rows: ModelRow[] = [];
|
||||
|
||||
await appendPreparedModelCatalogRows({
|
||||
rows,
|
||||
seenKeys: new Set(),
|
||||
context: {
|
||||
cfg: {},
|
||||
agentId: "worker",
|
||||
agentDir: "/tmp/openclaw-worker",
|
||||
workspaceDir: "/tmp/openclaw-workspace",
|
||||
authIndex: { evaluateModelAuth: () => authEvaluation(true) },
|
||||
configuredByKey: new Map(),
|
||||
discoveredKeys: new Set(),
|
||||
filter: { provider: "anthropic" },
|
||||
skipRuntimeModelSuppression: true,
|
||||
},
|
||||
});
|
||||
|
||||
expect(requireOnlyRow(rows).key).toBe("anthropic/claude-opus-4-7");
|
||||
expect(mocks.loadModelCatalogSnapshot).toHaveBeenCalledExactlyOnceWith({
|
||||
config: {},
|
||||
agentId: "worker",
|
||||
agentDir: "/tmp/openclaw-worker",
|
||||
workspaceDir: "/tmp/openclaw-workspace",
|
||||
readOnly: true,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("appendDiscoveredRows", () => {
|
||||
it("does not borrow provider registry auth when an OpenAI route is unknown", async () => {
|
||||
const rows: ModelRow[] = [];
|
||||
@@ -71,9 +282,7 @@ describe("appendDiscoveredRows", () => {
|
||||
baseUrl: "https://api.openai.com/v1",
|
||||
input: ["text"],
|
||||
reasoning: false,
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 8192,
|
||||
maxTokens: 4096,
|
||||
},
|
||||
] as never,
|
||||
context: {
|
||||
@@ -249,12 +458,12 @@ describe("appendConfiguredRows", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("appendProviderCatalogRows", () => {
|
||||
describe("prepared provider catalog projection", () => {
|
||||
it("applies manifest suppression when runtime model-suppression hooks are skipped", async () => {
|
||||
mocks.shouldSuppressBuiltInModelFromManifest.mockReturnValueOnce(true);
|
||||
const rows: ModelRow[] = [];
|
||||
|
||||
await appendProviderCatalogRows({
|
||||
await appendCommittedProviderCatalogRows({
|
||||
rows,
|
||||
seenKeys: new Set(),
|
||||
catalogModels: [
|
||||
@@ -266,9 +475,7 @@ describe("appendProviderCatalogRows", () => {
|
||||
baseUrl: "https://api.openai.com/v1",
|
||||
input: ["text", "image"],
|
||||
reasoning: false,
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 8192,
|
||||
maxTokens: 4096,
|
||||
},
|
||||
],
|
||||
context: {
|
||||
@@ -306,7 +513,7 @@ describe("appendProviderCatalogRows", () => {
|
||||
authEvaluation(ref.modelId === "gpt-5.5"),
|
||||
);
|
||||
|
||||
await appendProviderCatalogRows({
|
||||
await appendCommittedProviderCatalogRows({
|
||||
rows,
|
||||
seenKeys: new Set(),
|
||||
catalogModels: [
|
||||
@@ -318,9 +525,7 @@ describe("appendProviderCatalogRows", () => {
|
||||
baseUrl: "https://api.openai.com/v1",
|
||||
input: ["text", "image"],
|
||||
reasoning: false,
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 8192,
|
||||
maxTokens: 4096,
|
||||
},
|
||||
],
|
||||
context: {
|
||||
@@ -369,7 +574,7 @@ describe("appendProviderCatalogRows", () => {
|
||||
it("preserves unknown route auth instead of borrowing provider registry availability", async () => {
|
||||
const rows: ModelRow[] = [];
|
||||
|
||||
await appendProviderCatalogRows({
|
||||
await appendCommittedProviderCatalogRows({
|
||||
rows,
|
||||
seenKeys: new Set(),
|
||||
catalogModels: [
|
||||
@@ -381,9 +586,7 @@ describe("appendProviderCatalogRows", () => {
|
||||
baseUrl: "https://api.openai.com/v1",
|
||||
input: ["text", "image"],
|
||||
reasoning: false,
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 8192,
|
||||
maxTokens: 4096,
|
||||
},
|
||||
],
|
||||
context: {
|
||||
@@ -406,7 +609,7 @@ describe("appendProviderCatalogRows", () => {
|
||||
it("preserves registry-negative availability for non-route provider auth", async () => {
|
||||
const rows: ModelRow[] = [];
|
||||
|
||||
await appendProviderCatalogRows({
|
||||
await appendCommittedProviderCatalogRows({
|
||||
rows,
|
||||
seenKeys: new Set(),
|
||||
catalogModels: [
|
||||
@@ -418,9 +621,7 @@ describe("appendProviderCatalogRows", () => {
|
||||
baseUrl: "https://api.anthropic.com",
|
||||
input: ["text"],
|
||||
reasoning: false,
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 8192,
|
||||
maxTokens: 4096,
|
||||
},
|
||||
],
|
||||
context: {
|
||||
@@ -443,7 +644,7 @@ describe("appendProviderCatalogRows", () => {
|
||||
it("keeps unresolved native route auth unknown without positive registry evidence", async () => {
|
||||
const rows: ModelRow[] = [];
|
||||
|
||||
await appendProviderCatalogRows({
|
||||
await appendCommittedProviderCatalogRows({
|
||||
rows,
|
||||
seenKeys: new Set(),
|
||||
catalogModels: [
|
||||
@@ -455,9 +656,7 @@ describe("appendProviderCatalogRows", () => {
|
||||
baseUrl: "https://api.openai.com/v1",
|
||||
input: ["text", "image"],
|
||||
reasoning: false,
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 8192,
|
||||
maxTokens: 4096,
|
||||
},
|
||||
],
|
||||
context: {
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
/** Row builders used by `openclaw models list` source orchestration. */
|
||||
import type { NormalizedModelCatalogRow } from "@openclaw/model-catalog-core/model-catalog-types";
|
||||
import {
|
||||
normalizeProviderId,
|
||||
normalizeProviderIdForAuth,
|
||||
@@ -9,7 +8,7 @@ import {
|
||||
projectModelCatalogEntryForRoute,
|
||||
resolveConfiguredModelCatalogOverrides,
|
||||
} from "../../agents/model-catalog-route.js";
|
||||
import type { ModelCatalogEntry } from "../../agents/model-catalog.types.js";
|
||||
import type { ModelCatalogEntry, ModelCatalogSnapshot } from "../../agents/model-catalog.types.js";
|
||||
import { modelCatalogLogicalKey } from "../../agents/model-selection-shared.js";
|
||||
import {
|
||||
shouldSuppressBuiltInModel,
|
||||
@@ -39,7 +38,6 @@ import { modelKey } from "./shared.js";
|
||||
type ConfiguredByKey = Map<string, ConfiguredEntry>;
|
||||
type ModelCatalogModule = typeof import("../../agents/prepared-model-catalog.js");
|
||||
type ModelResolverModule = typeof import("../../agents/embedded-agent-runner/model.js");
|
||||
type ProviderCatalogModule = typeof import("./list.provider-catalog.js");
|
||||
|
||||
type RowFilter = {
|
||||
provider?: string;
|
||||
@@ -67,10 +65,6 @@ const modelCatalogModuleLoader = createLazyImportLoader<ModelCatalogModule>(
|
||||
const modelResolverModuleLoader = createLazyImportLoader<ModelResolverModule>(
|
||||
() => import("../../agents/embedded-agent-runner/model.js"),
|
||||
);
|
||||
const providerCatalogModuleLoader = createLazyImportLoader<ProviderCatalogModule>(
|
||||
() => import("./list.provider-catalog.js"),
|
||||
);
|
||||
|
||||
function loadPreparedModelCatalogModule(): Promise<ModelCatalogModule> {
|
||||
return modelCatalogModuleLoader.load();
|
||||
}
|
||||
@@ -79,10 +73,6 @@ function loadModelResolverModule(): Promise<ModelResolverModule> {
|
||||
return modelResolverModuleLoader.load();
|
||||
}
|
||||
|
||||
function loadProviderCatalogModule(): Promise<ProviderCatalogModule> {
|
||||
return providerCatalogModuleLoader.load();
|
||||
}
|
||||
|
||||
function matchesProviderFilter(context: RowBuilderContext, provider: string): boolean {
|
||||
const providerFilter = context.filter.provider;
|
||||
if (!providerFilter) {
|
||||
@@ -390,9 +380,9 @@ function toListRowInput(input: readonly string[] | undefined): ListRowModel["inp
|
||||
return parsed?.length ? parsed : ["text"];
|
||||
}
|
||||
|
||||
function toManifestCatalogListModel(
|
||||
function toPreparedCatalogListModel(
|
||||
row: Pick<
|
||||
NormalizedModelCatalogRow,
|
||||
ModelCatalogEntry,
|
||||
"provider" | "id" | "name" | "api" | "baseUrl" | "contextWindow" | "contextTokens"
|
||||
> & {
|
||||
input?: readonly string[];
|
||||
@@ -564,7 +554,7 @@ export async function appendAuthenticatedCatalogRows(params: {
|
||||
});
|
||||
const routeIndex = createModelCatalogLogicalRouteIndex(routeVariants);
|
||||
for (const entry of catalog) {
|
||||
const model = toManifestCatalogListModel(entry);
|
||||
const model = toPreparedCatalogListModel(entry);
|
||||
const authEvaluation = params.context.authIndex.evaluateModelAuth(
|
||||
entry.provider,
|
||||
toModelAuthRef(model, routeIndex),
|
||||
@@ -589,155 +579,59 @@ export async function appendAuthenticatedCatalogRows(params: {
|
||||
}
|
||||
}
|
||||
|
||||
/** Appends normalized model catalog rows into the shared row list. */
|
||||
export async function appendModelCatalogRows(params: {
|
||||
/** Projects every model from the same lifecycle generation used by the Gateway. */
|
||||
export async function appendPreparedModelCatalogRows(params: {
|
||||
rows: ModelRow[];
|
||||
context: RowBuilderContext;
|
||||
seenKeys: Set<string>;
|
||||
catalogRows: readonly NormalizedModelCatalogRow[];
|
||||
}): Promise<number> {
|
||||
let appended = 0;
|
||||
const projectionCatalog = params.catalogRows.map((row) =>
|
||||
toCatalogProjectionEntry(toManifestCatalogListModel(row)),
|
||||
catalogSnapshot?: ModelCatalogSnapshot;
|
||||
}): Promise<void> {
|
||||
const catalogSnapshot =
|
||||
params.catalogSnapshot ??
|
||||
(await (
|
||||
await loadPreparedModelCatalogModule()
|
||||
).loadPreparedModelCatalogSnapshot({
|
||||
config: params.context.cfg,
|
||||
...(params.context.agentId ? { agentId: params.context.agentId } : {}),
|
||||
agentDir: params.context.agentDir,
|
||||
...((params.context.workspaceDir ?? params.context.metadataSnapshot?.workspaceDir)
|
||||
? {
|
||||
workspaceDir:
|
||||
params.context.workspaceDir ?? params.context.metadataSnapshot?.workspaceDir,
|
||||
}
|
||||
: {}),
|
||||
readOnly: true,
|
||||
}));
|
||||
const staticEntries = catalogSnapshot.staticEntries ?? [];
|
||||
const routeVariants = [...catalogSnapshot.routeVariants];
|
||||
const seenRouteVariants = new Set(
|
||||
routeVariants.map(
|
||||
(entry) => `${resolveCatalogLogicalKey(entry)}\0${entry.api ?? ""}\0${entry.baseUrl ?? ""}`,
|
||||
),
|
||||
);
|
||||
const routeIndex = createModelCatalogLogicalRouteIndex(projectionCatalog);
|
||||
for (const catalogRow of params.catalogRows) {
|
||||
const key = modelKey(catalogRow.provider, catalogRow.id);
|
||||
if (
|
||||
await appendVisibleRow({
|
||||
rows: params.rows,
|
||||
model: toManifestCatalogListModel(catalogRow),
|
||||
key,
|
||||
context: params.context,
|
||||
seenKeys: params.seenKeys,
|
||||
routeIndex,
|
||||
allowAuthAvailabilityOverride: true,
|
||||
})
|
||||
) {
|
||||
appended += 1;
|
||||
for (const entry of staticEntries) {
|
||||
const routeKey = `${resolveCatalogLogicalKey(entry)}\0${entry.api ?? ""}\0${entry.baseUrl ?? ""}`;
|
||||
if (!seenRouteVariants.has(routeKey)) {
|
||||
routeVariants.push(entry);
|
||||
seenRouteVariants.add(routeKey);
|
||||
}
|
||||
}
|
||||
return appended;
|
||||
}
|
||||
|
||||
/** Appends manifest catalog rows through the generic catalog-row path. */
|
||||
export function appendManifestCatalogRows(params: {
|
||||
rows: ModelRow[];
|
||||
context: RowBuilderContext;
|
||||
seenKeys: Set<string>;
|
||||
manifestRows: readonly NormalizedModelCatalogRow[];
|
||||
}): Promise<number> {
|
||||
return appendModelCatalogRows({
|
||||
...params,
|
||||
catalogRows: params.manifestRows,
|
||||
});
|
||||
}
|
||||
|
||||
/** Appends catalog rows that are resolvable by the registry but missing from registry output. */
|
||||
export async function appendCatalogSupplementRows(params: {
|
||||
rows: ModelRow[];
|
||||
modelRegistry: ModelRegistry;
|
||||
context: RowBuilderContext;
|
||||
seenKeys: Set<string>;
|
||||
}): Promise<void> {
|
||||
const [modelCatalog, { resolveModelWithRegistry }] = await Promise.all([
|
||||
loadPreparedModelCatalogModule(),
|
||||
loadModelResolverModule(),
|
||||
]);
|
||||
const { entries: catalog, routeVariants } = await modelCatalog.loadPreparedModelCatalogSnapshot({
|
||||
config: params.context.cfg,
|
||||
...(params.context.agentId ? { agentId: params.context.agentId } : {}),
|
||||
agentDir: params.context.agentDir,
|
||||
...((params.context.workspaceDir ?? params.context.metadataSnapshot?.workspaceDir)
|
||||
? {
|
||||
workspaceDir:
|
||||
params.context.workspaceDir ?? params.context.metadataSnapshot?.workspaceDir,
|
||||
}
|
||||
: {}),
|
||||
readOnly: true,
|
||||
});
|
||||
const routeIndex = createModelCatalogLogicalRouteIndex(routeVariants);
|
||||
for (const entry of catalog) {
|
||||
if (!matchesProviderFilter(params.context, entry.provider)) {
|
||||
continue;
|
||||
}
|
||||
const key = modelKey(entry.provider, entry.id);
|
||||
if (params.seenKeys.has(key)) {
|
||||
continue;
|
||||
}
|
||||
const model = resolveModelWithRegistry({
|
||||
provider: entry.provider,
|
||||
modelId: entry.id,
|
||||
modelRegistry: params.modelRegistry,
|
||||
cfg: params.context.cfg,
|
||||
});
|
||||
if (!model) {
|
||||
continue;
|
||||
}
|
||||
// Static provider hooks belong to this same published generation; omitting
|
||||
// them hides valid plugin-owned models from filtered and complete listings.
|
||||
for (const entry of [...catalogSnapshot.entries, ...staticEntries]) {
|
||||
await appendVisibleRow({
|
||||
rows: params.rows,
|
||||
model,
|
||||
key,
|
||||
model: toPreparedCatalogListModel(entry),
|
||||
key: modelKey(entry.provider, entry.id),
|
||||
context: params.context,
|
||||
seenKeys: params.seenKeys,
|
||||
routeIndex,
|
||||
allowAuthAvailabilityOverride: !params.context.discoveredKeys.has(key),
|
||||
allowAuthAvailabilityOverride: !params.context.discoveredKeys.has(
|
||||
modelKey(entry.provider, entry.id),
|
||||
),
|
||||
});
|
||||
}
|
||||
|
||||
if (params.context.filter.local || !params.context.filter.provider) {
|
||||
return;
|
||||
}
|
||||
|
||||
await appendProviderCatalogRows({
|
||||
rows: params.rows,
|
||||
context: params.context,
|
||||
seenKeys: params.seenKeys,
|
||||
});
|
||||
}
|
||||
|
||||
/** Appends model rows returned by provider catalog hooks. */
|
||||
export async function appendProviderCatalogRows(params: {
|
||||
rows: ModelRow[];
|
||||
context: RowBuilderContext;
|
||||
seenKeys: Set<string>;
|
||||
staticOnly?: boolean;
|
||||
catalogModels?: readonly Model[];
|
||||
}): Promise<number> {
|
||||
let appended = 0;
|
||||
let catalogModels = params.catalogModels;
|
||||
if (catalogModels == null) {
|
||||
const { loadProviderCatalogModelsForList } = await loadProviderCatalogModule();
|
||||
catalogModels = await loadProviderCatalogModelsForList({
|
||||
cfg: params.context.cfg,
|
||||
...(params.context.agentId ? { agentId: params.context.agentId } : {}),
|
||||
agentDir: params.context.agentDir,
|
||||
providerFilter: params.context.filter.provider,
|
||||
staticOnly: params.staticOnly,
|
||||
metadataSnapshot: params.context.metadataSnapshot,
|
||||
});
|
||||
}
|
||||
const projectionCatalog = catalogModels.map((model) =>
|
||||
toCatalogProjectionEntry(model as ListRowModel),
|
||||
);
|
||||
const routeIndex = createModelCatalogLogicalRouteIndex(projectionCatalog);
|
||||
for (const model of catalogModels) {
|
||||
const key = modelKey(model.provider, model.id);
|
||||
if (
|
||||
await appendVisibleRow({
|
||||
rows: params.rows,
|
||||
model,
|
||||
key,
|
||||
context: params.context,
|
||||
seenKeys: params.seenKeys,
|
||||
routeIndex,
|
||||
allowAuthAvailabilityOverride: !params.context.discoveredKeys.has(key),
|
||||
})
|
||||
) {
|
||||
appended += 1;
|
||||
}
|
||||
}
|
||||
return appended;
|
||||
}
|
||||
|
||||
/** Appends rows from default/fallback/configured model references. */
|
||||
@@ -807,4 +701,3 @@ export async function appendConfiguredRows(params: {
|
||||
);
|
||||
}
|
||||
}
|
||||
/* oxlint-disable max-lines -- TODO: split this grandfathered oversized file. */
|
||||
|
||||
@@ -1,185 +0,0 @@
|
||||
// Model list source-plan tests cover catalog source selection and fallback planning.
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const mocks = vi.hoisted(() => ({
|
||||
loadStaticManifestCatalogRowsForList: vi.fn(),
|
||||
loadSupplementalManifestCatalogRowsForList: vi.fn(),
|
||||
loadProviderIndexCatalogRowsForList: vi.fn(),
|
||||
hasProviderRuntimeCatalogForFilter: vi.fn(),
|
||||
hasProviderStaticCatalogForFilter: vi.fn(),
|
||||
}));
|
||||
|
||||
const catalogRow = {
|
||||
provider: "moonshot",
|
||||
id: "kimi-k2.6",
|
||||
ref: "moonshot/kimi-k2.6",
|
||||
mergeKey: "moonshot::kimi-k2.6",
|
||||
name: "Kimi K2.6",
|
||||
source: "manifest",
|
||||
input: ["text"],
|
||||
reasoning: false,
|
||||
status: "available",
|
||||
} as const;
|
||||
|
||||
describe("planAllModelListSources", () => {
|
||||
beforeEach(() => {
|
||||
vi.resetAllMocks();
|
||||
mocks.loadStaticManifestCatalogRowsForList.mockReturnValue([]);
|
||||
mocks.loadSupplementalManifestCatalogRowsForList.mockReturnValue([]);
|
||||
mocks.loadProviderIndexCatalogRowsForList.mockReturnValue([]);
|
||||
mocks.hasProviderRuntimeCatalogForFilter.mockResolvedValue(false);
|
||||
mocks.hasProviderStaticCatalogForFilter.mockResolvedValue(false);
|
||||
});
|
||||
|
||||
it("uses installed manifest rows before provider index or runtime catalog sources", async () => {
|
||||
const { planAllModelListSources } = await import("./list.source-plan.js");
|
||||
mocks.loadStaticManifestCatalogRowsForList.mockReturnValueOnce([catalogRow]);
|
||||
|
||||
const plan = await planAllModelListSources({
|
||||
all: true,
|
||||
providerFilter: "moonshot",
|
||||
cfg: {},
|
||||
dependencies: mocks,
|
||||
});
|
||||
|
||||
expect(plan.kind).toBe("manifest");
|
||||
expect(plan.manifestCatalogRows).toEqual([catalogRow]);
|
||||
expect(mocks.loadStaticManifestCatalogRowsForList).toHaveBeenCalledWith({
|
||||
cfg: {},
|
||||
providerFilter: "moonshot",
|
||||
});
|
||||
expect(mocks.hasProviderRuntimeCatalogForFilter).not.toHaveBeenCalled();
|
||||
expect(mocks.loadSupplementalManifestCatalogRowsForList).not.toHaveBeenCalled();
|
||||
expect(mocks.loadProviderIndexCatalogRowsForList).not.toHaveBeenCalled();
|
||||
expect(mocks.hasProviderStaticCatalogForFilter).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("retains refreshed manifest supplements for runtime-backed providers", async () => {
|
||||
const { planAllModelListSources } = await import("./list.source-plan.js");
|
||||
mocks.hasProviderRuntimeCatalogForFilter.mockResolvedValueOnce(true);
|
||||
mocks.loadSupplementalManifestCatalogRowsForList.mockReturnValueOnce([catalogRow]);
|
||||
|
||||
const plan = await planAllModelListSources({
|
||||
all: true,
|
||||
providerFilter: "openai",
|
||||
cfg: {},
|
||||
dependencies: mocks,
|
||||
});
|
||||
|
||||
expect(plan.kind).toBe("provider-runtime-scoped");
|
||||
expect(plan.manifestCatalogRows).toEqual([catalogRow]);
|
||||
expect(mocks.loadStaticManifestCatalogRowsForList).toHaveBeenCalledWith({
|
||||
cfg: {},
|
||||
providerFilter: "openai",
|
||||
});
|
||||
expect(mocks.loadSupplementalManifestCatalogRowsForList).toHaveBeenCalledWith({
|
||||
cfg: {},
|
||||
providerFilter: "openai",
|
||||
metadataSnapshot: undefined,
|
||||
});
|
||||
expect(mocks.hasProviderStaticCatalogForFilter).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("uses provider index rows only when installed manifest rows are unavailable", async () => {
|
||||
const { planAllModelListSources } = await import("./list.source-plan.js");
|
||||
const providerIndexRow = { ...catalogRow, source: "provider-index" };
|
||||
mocks.loadProviderIndexCatalogRowsForList.mockReturnValueOnce([providerIndexRow]);
|
||||
|
||||
const plan = await planAllModelListSources({
|
||||
all: true,
|
||||
providerFilter: "moonshot",
|
||||
cfg: {},
|
||||
dependencies: mocks,
|
||||
});
|
||||
|
||||
expect(plan.kind).toBe("provider-index");
|
||||
expect(plan.providerIndexCatalogRows).toEqual([providerIndexRow]);
|
||||
expect(mocks.hasProviderStaticCatalogForFilter).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("keeps provider-filtered refreshable manifest rows registry-backed", async () => {
|
||||
const { planAllModelListSources } = await import("./list.source-plan.js");
|
||||
mocks.loadSupplementalManifestCatalogRowsForList.mockReturnValueOnce([catalogRow]);
|
||||
|
||||
const plan = await planAllModelListSources({
|
||||
all: true,
|
||||
providerFilter: "openai",
|
||||
cfg: {},
|
||||
dependencies: mocks,
|
||||
});
|
||||
|
||||
expect(plan.kind).toBe("registry");
|
||||
expect(plan.manifestCatalogRows).toEqual([catalogRow]);
|
||||
expect(mocks.loadStaticManifestCatalogRowsForList).toHaveBeenCalledWith({
|
||||
cfg: {},
|
||||
providerFilter: "openai",
|
||||
});
|
||||
expect(mocks.loadSupplementalManifestCatalogRowsForList).toHaveBeenCalledWith({
|
||||
cfg: {},
|
||||
providerFilter: "openai",
|
||||
});
|
||||
expect(mocks.loadProviderIndexCatalogRowsForList).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("allows scoped runtime catalog plans to fall back to registry rows", async () => {
|
||||
const { planAllModelListSources } = await import("./list.source-plan.js");
|
||||
|
||||
const plan = await planAllModelListSources({
|
||||
all: true,
|
||||
providerFilter: "openrouter",
|
||||
cfg: {},
|
||||
dependencies: mocks,
|
||||
});
|
||||
expect(plan.kind).toBe("provider-runtime-scoped");
|
||||
});
|
||||
|
||||
it("keeps broad all-model lists on the registry path with cheap catalog supplements", async () => {
|
||||
const { planAllModelListSources } = await import("./list.source-plan.js");
|
||||
const providerIndexRow = { ...catalogRow, source: "provider-index" };
|
||||
mocks.loadSupplementalManifestCatalogRowsForList.mockReturnValueOnce([catalogRow]);
|
||||
mocks.loadProviderIndexCatalogRowsForList.mockReturnValueOnce([providerIndexRow]);
|
||||
|
||||
const plan = await planAllModelListSources({
|
||||
all: true,
|
||||
cfg: {},
|
||||
dependencies: mocks,
|
||||
});
|
||||
|
||||
expect(plan.kind).toBe("registry");
|
||||
expect(plan.manifestCatalogRows).toEqual([catalogRow]);
|
||||
expect(plan.providerIndexCatalogRows).toEqual([providerIndexRow]);
|
||||
expect(mocks.loadSupplementalManifestCatalogRowsForList).toHaveBeenCalledWith({
|
||||
cfg: {},
|
||||
});
|
||||
expect(mocks.loadStaticManifestCatalogRowsForList).not.toHaveBeenCalled();
|
||||
expect(mocks.hasProviderStaticCatalogForFilter).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("falls back to registry only for provider static fast paths that return no rows", async () => {
|
||||
const { planAllModelListSources } = await import("./list.source-plan.js");
|
||||
mocks.hasProviderStaticCatalogForFilter.mockResolvedValueOnce(true);
|
||||
|
||||
const plan = await planAllModelListSources({
|
||||
all: true,
|
||||
providerFilter: "codex",
|
||||
cfg: {},
|
||||
dependencies: mocks,
|
||||
});
|
||||
expect(plan.kind).toBe("provider-runtime-static");
|
||||
});
|
||||
|
||||
it("uses runtime-scoped plans for providers with live and static catalog hooks", async () => {
|
||||
const { planAllModelListSources } = await import("./list.source-plan.js");
|
||||
mocks.hasProviderRuntimeCatalogForFilter.mockResolvedValueOnce(true);
|
||||
mocks.hasProviderStaticCatalogForFilter.mockResolvedValueOnce(true);
|
||||
|
||||
const plan = await planAllModelListSources({
|
||||
all: true,
|
||||
providerFilter: "openai",
|
||||
cfg: {},
|
||||
dependencies: mocks,
|
||||
});
|
||||
expect(plan.kind).toBe("provider-runtime-scoped");
|
||||
expect(mocks.hasProviderStaticCatalogForFilter).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -1,177 +0,0 @@
|
||||
/** Chooses which source family should back a model-list invocation. */
|
||||
import type { NormalizedModelCatalogRow } from "@openclaw/model-catalog-core/model-catalog-types";
|
||||
import type { OpenClawConfig } from "../../config/types.openclaw.js";
|
||||
import type { PluginMetadataSnapshot } from "../../plugins/plugin-metadata-snapshot.types.js";
|
||||
import { createLazyImportLoader } from "../../shared/lazy-promise.js";
|
||||
|
||||
/** Source family selected for a model-list run. */
|
||||
export type ModelListSourcePlanKind =
|
||||
| "registry"
|
||||
| "manifest"
|
||||
| "provider-index"
|
||||
| "provider-runtime-static"
|
||||
| "provider-runtime-scoped";
|
||||
|
||||
/** Concrete source plan plus preloaded catalog rows and fallback flags. */
|
||||
export type ModelListSourcePlan = {
|
||||
kind: ModelListSourcePlanKind;
|
||||
manifestCatalogRows: readonly NormalizedModelCatalogRow[];
|
||||
providerIndexCatalogRows: readonly NormalizedModelCatalogRow[];
|
||||
};
|
||||
|
||||
type ProviderIndexCatalogModule = typeof import("./list.provider-index-catalog.js");
|
||||
type ManifestCatalogModule = typeof import("./list.manifest-catalog.js");
|
||||
type ProviderCatalogModule = typeof import("./list.provider-catalog.js");
|
||||
|
||||
type ModelListSourcePlanDependencies = Pick<
|
||||
ManifestCatalogModule,
|
||||
"loadStaticManifestCatalogRowsForList" | "loadSupplementalManifestCatalogRowsForList"
|
||||
> &
|
||||
Pick<ProviderIndexCatalogModule, "loadProviderIndexCatalogRowsForList"> &
|
||||
Pick<
|
||||
ProviderCatalogModule,
|
||||
"hasProviderRuntimeCatalogForFilter" | "hasProviderStaticCatalogForFilter"
|
||||
>;
|
||||
|
||||
const providerIndexCatalogLoader = createLazyImportLoader<ProviderIndexCatalogModule>(
|
||||
() => import("./list.provider-index-catalog.js"),
|
||||
);
|
||||
|
||||
function createSourcePlan(params: {
|
||||
kind: ModelListSourcePlanKind;
|
||||
manifestCatalogRows?: readonly NormalizedModelCatalogRow[];
|
||||
providerIndexCatalogRows?: readonly NormalizedModelCatalogRow[];
|
||||
}): ModelListSourcePlan {
|
||||
return {
|
||||
kind: params.kind,
|
||||
manifestCatalogRows: params.manifestCatalogRows ?? [],
|
||||
providerIndexCatalogRows: params.providerIndexCatalogRows ?? [],
|
||||
};
|
||||
}
|
||||
|
||||
/** Creates the baseline plan that loads the runtime model registry. */
|
||||
export function createRegistryModelListSourcePlan(): ModelListSourcePlan {
|
||||
return createSourcePlan({ kind: "registry" });
|
||||
}
|
||||
|
||||
/** Plans source precedence for all/provider-filtered model-list output. */
|
||||
export async function planAllModelListSources(params: {
|
||||
all?: boolean;
|
||||
enableCascade?: boolean;
|
||||
providerFilter?: string;
|
||||
cfg: OpenClawConfig;
|
||||
agentId?: string;
|
||||
agentDir?: string;
|
||||
metadataSnapshot?: PluginMetadataSnapshot;
|
||||
dependencies?: Partial<ModelListSourcePlanDependencies>;
|
||||
}): Promise<ModelListSourcePlan> {
|
||||
const enableCascade = params.enableCascade ?? params.all;
|
||||
if (!enableCascade) {
|
||||
return createRegistryModelListSourcePlan();
|
||||
}
|
||||
|
||||
const manifestCatalog = await import("./list.manifest-catalog.js");
|
||||
const loadStaticManifestCatalogRowsForList =
|
||||
params.dependencies?.loadStaticManifestCatalogRowsForList ??
|
||||
manifestCatalog.loadStaticManifestCatalogRowsForList;
|
||||
const loadSupplementalManifestCatalogRowsForList =
|
||||
params.dependencies?.loadSupplementalManifestCatalogRowsForList ??
|
||||
manifestCatalog.loadSupplementalManifestCatalogRowsForList;
|
||||
if (!params.providerFilter) {
|
||||
const providerIndexCatalog = await providerIndexCatalogLoader.load();
|
||||
const loadProviderIndexCatalogRowsForList =
|
||||
params.dependencies?.loadProviderIndexCatalogRowsForList ??
|
||||
providerIndexCatalog.loadProviderIndexCatalogRowsForList;
|
||||
return createSourcePlan({
|
||||
kind: "registry",
|
||||
manifestCatalogRows: loadSupplementalManifestCatalogRowsForList({
|
||||
cfg: params.cfg,
|
||||
metadataSnapshot: params.metadataSnapshot,
|
||||
}),
|
||||
providerIndexCatalogRows: loadProviderIndexCatalogRowsForList({
|
||||
cfg: params.cfg,
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
const providerCatalog = await import("./list.provider-catalog.js");
|
||||
const hasProviderRuntimeCatalogForFilter =
|
||||
params.dependencies?.hasProviderRuntimeCatalogForFilter ??
|
||||
providerCatalog.hasProviderRuntimeCatalogForFilter;
|
||||
const hasProviderStaticCatalogForFilter =
|
||||
params.dependencies?.hasProviderStaticCatalogForFilter ??
|
||||
providerCatalog.hasProviderStaticCatalogForFilter;
|
||||
|
||||
const staticManifestCatalogRows = loadStaticManifestCatalogRowsForList({
|
||||
cfg: params.cfg,
|
||||
providerFilter: params.providerFilter,
|
||||
metadataSnapshot: params.metadataSnapshot,
|
||||
});
|
||||
if (staticManifestCatalogRows.length > 0) {
|
||||
return createSourcePlan({
|
||||
kind: "manifest",
|
||||
manifestCatalogRows: staticManifestCatalogRows,
|
||||
});
|
||||
}
|
||||
|
||||
const hasProviderRuntimeCatalog = await hasProviderRuntimeCatalogForFilter({
|
||||
cfg: params.cfg,
|
||||
...(params.agentId ? { agentId: params.agentId } : {}),
|
||||
...(params.agentDir ? { agentDir: params.agentDir } : {}),
|
||||
providerFilter: params.providerFilter,
|
||||
metadataSnapshot: params.metadataSnapshot,
|
||||
});
|
||||
if (hasProviderRuntimeCatalog) {
|
||||
return createSourcePlan({
|
||||
kind: "provider-runtime-scoped",
|
||||
manifestCatalogRows: loadSupplementalManifestCatalogRowsForList({
|
||||
cfg: params.cfg,
|
||||
providerFilter: params.providerFilter,
|
||||
metadataSnapshot: params.metadataSnapshot,
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
const manifestCatalogRows = loadSupplementalManifestCatalogRowsForList({
|
||||
cfg: params.cfg,
|
||||
providerFilter: params.providerFilter,
|
||||
metadataSnapshot: params.metadataSnapshot,
|
||||
});
|
||||
|
||||
if (manifestCatalogRows.length > 0) {
|
||||
// Supplemental manifest rows still need the registry for runtime-backed
|
||||
// availability and suppression decisions.
|
||||
return createSourcePlan({
|
||||
kind: "registry",
|
||||
manifestCatalogRows,
|
||||
});
|
||||
}
|
||||
|
||||
const providerIndexCatalog = await providerIndexCatalogLoader.load();
|
||||
const loadProviderIndexCatalogRowsForList =
|
||||
params.dependencies?.loadProviderIndexCatalogRowsForList ??
|
||||
providerIndexCatalog.loadProviderIndexCatalogRowsForList;
|
||||
const providerIndexCatalogRows = loadProviderIndexCatalogRowsForList({
|
||||
cfg: params.cfg,
|
||||
providerFilter: params.providerFilter,
|
||||
});
|
||||
if (providerIndexCatalogRows.length > 0) {
|
||||
return createSourcePlan({
|
||||
kind: "provider-index",
|
||||
providerIndexCatalogRows,
|
||||
});
|
||||
}
|
||||
|
||||
const hasProviderStaticCatalog = await hasProviderStaticCatalogForFilter({
|
||||
cfg: params.cfg,
|
||||
...(params.agentId ? { agentId: params.agentId } : {}),
|
||||
...(params.agentDir ? { agentDir: params.agentDir } : {}),
|
||||
providerFilter: params.providerFilter,
|
||||
metadataSnapshot: params.metadataSnapshot,
|
||||
});
|
||||
if (hasProviderStaticCatalog) {
|
||||
return createSourcePlan({ kind: "provider-runtime-static" });
|
||||
}
|
||||
|
||||
return createSourcePlan({ kind: "provider-runtime-scoped" });
|
||||
}
|
||||
@@ -23,6 +23,11 @@ const mocks = vi.hoisted(() => ({
|
||||
runDoctorHealthRepairs: vi.fn(),
|
||||
maybeRepairLegacyFlatAuthProfileStores: vi.fn().mockResolvedValue(undefined),
|
||||
maybeRepairCanonicalApiKeyFieldAlias: vi.fn().mockResolvedValue(undefined),
|
||||
maybeMigrateLegacyPluginModelCatalogs: vi.fn().mockResolvedValue({
|
||||
detected: 0,
|
||||
migrated: 0,
|
||||
warnings: [],
|
||||
}),
|
||||
maybeRepairGatewayDaemon: vi.fn().mockResolvedValue(undefined),
|
||||
maybeRepairLegacyOAuthProfileIds: vi.fn(async (cfg: unknown) => cfg),
|
||||
maybeRepairLegacyOAuthSidecarProfiles: vi.fn().mockResolvedValue(undefined),
|
||||
@@ -207,6 +212,10 @@ vi.mock("../commands/doctor-auth-flat-profiles.js", () => ({
|
||||
maybeRepairCanonicalApiKeyFieldAlias: mocks.maybeRepairCanonicalApiKeyFieldAlias,
|
||||
}));
|
||||
|
||||
vi.mock("../commands/doctor-plugin-model-catalog.js", () => ({
|
||||
maybeMigrateLegacyPluginModelCatalogs: mocks.maybeMigrateLegacyPluginModelCatalogs,
|
||||
}));
|
||||
|
||||
vi.mock("../commands/doctor-gateway-daemon-flow.js", () => ({
|
||||
maybeRepairGatewayDaemon: mocks.maybeRepairGatewayDaemon,
|
||||
}));
|
||||
@@ -540,6 +549,12 @@ describe("doctor health contributions", () => {
|
||||
mocks.maybeRepairLegacyFlatAuthProfileStores.mockResolvedValue(undefined);
|
||||
mocks.maybeRepairCanonicalApiKeyFieldAlias.mockClear();
|
||||
mocks.maybeRepairCanonicalApiKeyFieldAlias.mockResolvedValue(undefined);
|
||||
mocks.maybeMigrateLegacyPluginModelCatalogs.mockClear();
|
||||
mocks.maybeMigrateLegacyPluginModelCatalogs.mockResolvedValue({
|
||||
detected: 0,
|
||||
migrated: 0,
|
||||
warnings: [],
|
||||
});
|
||||
mocks.maybeRepairGatewayDaemon.mockClear();
|
||||
mocks.maybeRepairGatewayDaemon.mockResolvedValue(undefined);
|
||||
mocks.maybeRepairLegacyOAuthProfileIds.mockClear();
|
||||
@@ -1715,6 +1730,11 @@ describe("doctor health contributions", () => {
|
||||
cfg: ctx.cfg,
|
||||
prompter: ctx.prompter,
|
||||
});
|
||||
expect(mocks.maybeMigrateLegacyPluginModelCatalogs).toHaveBeenCalledWith({
|
||||
cfg: ctx.cfg,
|
||||
prompter: ctx.prompter,
|
||||
runtime: ctx.runtime,
|
||||
});
|
||||
});
|
||||
|
||||
it("registers auth profile health as an opt-in structured check", async () => {
|
||||
|
||||
@@ -60,6 +60,8 @@ async function runAuthProfileHealth(ctx: DoctorHealthFlowContext): Promise<void>
|
||||
await import("../commands/doctor-auth-legacy-oauth.js");
|
||||
const { maybeRepairLegacyOAuthSidecarProfiles } =
|
||||
await import("../commands/doctor-auth-oauth-sidecar.js");
|
||||
const { maybeMigrateLegacyPluginModelCatalogs } =
|
||||
await import("../commands/doctor-plugin-model-catalog.js");
|
||||
const { noteAuthProfileHealth, noteLegacyCodexProviderOverride } =
|
||||
await import("../commands/doctor-auth.js");
|
||||
const { buildGatewayConnectionDetails } = await import("../gateway/call.js");
|
||||
@@ -76,6 +78,12 @@ async function runAuthProfileHealth(ctx: DoctorHealthFlowContext): Promise<void>
|
||||
cfg: ctx.cfg,
|
||||
prompter: ctx.prompter,
|
||||
});
|
||||
await maybeMigrateLegacyPluginModelCatalogs({
|
||||
cfg: ctx.cfg,
|
||||
...(ctx.env ? { env: ctx.env } : {}),
|
||||
prompter: ctx.prompter,
|
||||
runtime: ctx.runtime,
|
||||
});
|
||||
ctx.cfg = await maybeRepairLegacyOAuthProfileIds(ctx.cfg, ctx.prompter);
|
||||
await noteAuthProfileHealth({
|
||||
cfg: ctx.cfg,
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
// normalized planning APIs instead of reaching into provider-index internals.
|
||||
export { loadOpenClawProviderIndex } from "./provider-index/index.js";
|
||||
export { planManifestModelCatalogSuppressions } from "./manifest-planner.js";
|
||||
export { planProviderIndexModelCatalogRows } from "./provider-index-planner.js";
|
||||
import type { ModelCatalogProvider } from "@openclaw/model-catalog-core/model-catalog-types";
|
||||
import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
import {
|
||||
@@ -26,8 +25,5 @@ export function planEffectiveModelCatalogRows(params: {
|
||||
...(params.selection ? { selection: params.selection } : {}),
|
||||
});
|
||||
}
|
||||
export type {
|
||||
ManifestModelCatalogRowSelection,
|
||||
ManifestModelCatalogSuppressionEntry,
|
||||
} from "./manifest-planner.js";
|
||||
export type { ManifestModelCatalogSuppressionEntry } from "./manifest-planner.js";
|
||||
export type { OpenClawProviderIndexProvider } from "./provider-index/index.js";
|
||||
|
||||
@@ -1,57 +0,0 @@
|
||||
// Provider-index planner tests cover preview catalog row generation and provider filtering.
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { planProviderIndexModelCatalogRows } from "./index.js";
|
||||
|
||||
describe("provider index model catalog planner", () => {
|
||||
it("builds preview rows from installable provider metadata", () => {
|
||||
const plan = planProviderIndexModelCatalogRows({
|
||||
providerFilter: "Moonshot",
|
||||
index: {
|
||||
version: 1,
|
||||
providers: {
|
||||
moonshot: {
|
||||
id: "moonshot",
|
||||
name: "Moonshot AI",
|
||||
plugin: {
|
||||
id: "moonshot",
|
||||
package: "@openclaw/plugin-moonshot",
|
||||
},
|
||||
previewCatalog: {
|
||||
models: [{ id: "kimi-k2.6", name: "Kimi K2.6", contextWindow: 262144 }],
|
||||
},
|
||||
},
|
||||
deepseek: {
|
||||
id: "deepseek",
|
||||
name: "DeepSeek",
|
||||
plugin: { id: "deepseek" },
|
||||
previewCatalog: {
|
||||
models: [{ id: "deepseek-chat" }],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(plan.entries).toEqual([
|
||||
{
|
||||
provider: "moonshot",
|
||||
pluginId: "moonshot",
|
||||
rows: [
|
||||
{
|
||||
provider: "moonshot",
|
||||
id: "kimi-k2.6",
|
||||
ref: "moonshot/kimi-k2.6",
|
||||
mergeKey: "moonshot::kimi-k2.6",
|
||||
name: "Kimi K2.6",
|
||||
source: "provider-index",
|
||||
input: ["text"],
|
||||
reasoning: false,
|
||||
status: "preview",
|
||||
contextWindow: 262144,
|
||||
},
|
||||
],
|
||||
},
|
||||
]);
|
||||
expect(plan.rows.map((row) => row.ref)).toEqual(["moonshot/kimi-k2.6"]);
|
||||
});
|
||||
});
|
||||
@@ -1,77 +0,0 @@
|
||||
// Provider-index model-catalog planner converts installable provider previews into normalized discovery rows.
|
||||
import { normalizeModelCatalogProviderRows } from "@openclaw/model-catalog-core/model-catalog-normalize";
|
||||
import { normalizeModelCatalogProviderId } from "@openclaw/model-catalog-core/model-catalog-refs";
|
||||
import type {
|
||||
ModelCatalogProvider,
|
||||
NormalizedModelCatalogRow,
|
||||
} from "@openclaw/model-catalog-core/model-catalog-types";
|
||||
import type { OpenClawProviderIndex } from "./provider-index/index.js";
|
||||
|
||||
// Provider-index planner converts ClawHub-style preview catalog entries into
|
||||
// normalized model rows for discovery before a plugin is installed.
|
||||
type ProviderIndexModelCatalogPlanEntry = {
|
||||
provider: string;
|
||||
pluginId: string;
|
||||
rows: readonly NormalizedModelCatalogRow[];
|
||||
};
|
||||
|
||||
type ProviderIndexModelCatalogPlan = {
|
||||
rows: readonly NormalizedModelCatalogRow[];
|
||||
entries: readonly ProviderIndexModelCatalogPlanEntry[];
|
||||
};
|
||||
|
||||
function withPreviewStatusDefaults(providerCatalog: ModelCatalogProvider): ModelCatalogProvider {
|
||||
// Provider-index rows are advisory discovery data, so unspecified model
|
||||
// statuses default to preview instead of stable.
|
||||
return {
|
||||
...providerCatalog,
|
||||
models: providerCatalog.models.map((model) => ({
|
||||
...model,
|
||||
status: model.status ?? "preview",
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
export function planProviderIndexModelCatalogRows(params: {
|
||||
index: OpenClawProviderIndex;
|
||||
providerFilter?: string;
|
||||
}): ProviderIndexModelCatalogPlan {
|
||||
const providerFilter = params.providerFilter
|
||||
? normalizeModelCatalogProviderId(params.providerFilter)
|
||||
: undefined;
|
||||
const entries: ProviderIndexModelCatalogPlanEntry[] = [];
|
||||
|
||||
for (const [providerId, provider] of Object.entries(params.index.providers)) {
|
||||
const normalizedProvider = normalizeModelCatalogProviderId(providerId);
|
||||
if (
|
||||
!normalizedProvider ||
|
||||
(providerFilter && normalizedProvider !== providerFilter) ||
|
||||
!provider.previewCatalog
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
const rows = normalizeModelCatalogProviderRows({
|
||||
provider: normalizedProvider,
|
||||
providerCatalog: withPreviewStatusDefaults(provider.previewCatalog),
|
||||
source: "provider-index",
|
||||
});
|
||||
if (rows.length === 0) {
|
||||
continue;
|
||||
}
|
||||
entries.push({
|
||||
provider: normalizedProvider,
|
||||
pluginId: provider.plugin.id,
|
||||
rows,
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
entries,
|
||||
rows: entries
|
||||
.flatMap((entry) => entry.rows)
|
||||
.toSorted(
|
||||
(left, right) =>
|
||||
left.provider.localeCompare(right.provider) || left.id.localeCompare(right.id),
|
||||
),
|
||||
};
|
||||
}
|
||||
@@ -1,3 +1,3 @@
|
||||
// Provider-index public facade for normalized provider discovery metadata.
|
||||
export { loadOpenClawProviderIndex } from "./load.js";
|
||||
export type { OpenClawProviderIndex, OpenClawProviderIndexProvider } from "./types.js";
|
||||
export type { OpenClawProviderIndexProvider } from "./types.js";
|
||||
|
||||
@@ -95,7 +95,7 @@ describe("resolveInstalledPluginProviderContributionIds", () => {
|
||||
it("keeps current production callers off the ambiguous runtime-discovery alias", () => {
|
||||
const callerPaths = [
|
||||
"src/agents/models-config.providers.implicit.ts",
|
||||
"src/commands/models/list.provider-catalog.ts",
|
||||
"src/commands/models/list.row-sources.ts",
|
||||
];
|
||||
|
||||
for (const callerPath of callerPaths) {
|
||||
|
||||
Reference in New Issue
Block a user