diff --git a/CHANGELOG.md b/CHANGELOG.md index d440c967f42..bb3bf5298d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,7 @@ Docs: https://docs.openclaw.ai - Gateway/sessions: keep async `sessions.list` title and preview hydration bounded to transcript head/tail reads so Control UI polling cannot full-scan large session transcripts every refresh. Thanks @vincentkoc. - Gateway/sessions: keep agent runtime metadata on lightweight `sessions.list` rows so model-only session patches do not make Control UI lose runtime identity. Thanks @vincentkoc. - Gateway/sessions: keep bulk `sessions.list` rows lightweight by skipping per-row transcript usage fallback, display model inference, and plugin projection, avoiding event-loop stalls in large session stores. Thanks @Marvinthebored and @vincentkoc. +- Gateway/models: keep read-only `models.list` fallbacks on persisted/current metadata and configured rows while using static auth checks, so missing `models.json` files no longer runtime-load provider discovery or stall gateway after restart. Fixes #76382; refs #76360 and #75707. Thanks @trojy13, @RayWoo, @AnathemaOfficial, and @vincentkoc. - CLI/plugins: reject missing plugin ids before config writes in `plugins enable` and `plugins disable` so a typo no longer persists a stale config entry. (#73554) Thanks @ai-hpc. - Agents/sessions: preserve delivered trailing assistant replies during session-file repair so Telegram/WebChat history is not rewritten to drop already-delivered responses. Fixes #76329. Thanks @obviyus. - Gateway/chat history: preserve oversized transcript turns as explicit omitted-message placeholders while avoiding large JSONL parse stalls. Thanks @Marvinthebored and @vincentkoc. diff --git a/src/agents/model-auth.ts b/src/agents/model-auth.ts index 9cb23dbbf45..7c704842bbe 100644 --- a/src/agents/model-auth.ts +++ b/src/agents/model-auth.ts @@ -324,6 +324,7 @@ export function hasRuntimeAvailableProviderAuth(params: { cfg?: OpenClawConfig; workspaceDir?: string; env?: NodeJS.ProcessEnv; + allowPluginSyntheticAuth?: boolean; }): boolean { const provider = normalizeProviderId(params.provider); const authOverride = resolveProviderAuthOverride(params.cfg, provider); @@ -347,7 +348,10 @@ export function hasRuntimeAvailableProviderAuth(params: { if (hasSyntheticLocalProviderAuthConfig({ cfg: params.cfg, provider })) { return true; } - if (resolveSyntheticLocalProviderAuth({ cfg: params.cfg, provider })) { + if ( + params.allowPluginSyntheticAuth !== false && + resolveSyntheticLocalProviderAuth({ cfg: params.cfg, provider }) + ) { return true; } return false; diff --git a/src/agents/model-catalog-visibility.test.ts b/src/agents/model-catalog-visibility.test.ts new file mode 100644 index 00000000000..abf9d10f036 --- /dev/null +++ b/src/agents/model-catalog-visibility.test.ts @@ -0,0 +1,43 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { OpenClawConfig } from "../config/types.openclaw.js"; +import { resolveVisibleModelCatalog } from "./model-catalog-visibility.js"; +import type { ModelCatalogEntry } from "./model-catalog.types.js"; +import { createProviderAuthChecker } from "./model-provider-auth.js"; + +vi.mock("./model-provider-auth.js", () => ({ + createProviderAuthChecker: vi.fn(), +})); + +const createProviderAuthCheckerMock = vi.mocked(createProviderAuthChecker); + +describe("resolveVisibleModelCatalog", () => { + beforeEach(() => { + createProviderAuthCheckerMock.mockReset(); + }); + + it("can use static auth checks for gateway read-only model lists", () => { + const authChecker = vi.fn((provider: string) => provider === "openai"); + createProviderAuthCheckerMock.mockReturnValue(authChecker); + const catalog: ModelCatalogEntry[] = [ + { provider: "anthropic", id: "claude-test", name: "Claude Test" }, + { provider: "openai", id: "gpt-test", name: "GPT Test" }, + ]; + + const result = resolveVisibleModelCatalog({ + cfg: {} as OpenClawConfig, + catalog, + defaultProvider: "openai", + runtimeAuthDiscovery: false, + }); + + expect(createProviderAuthCheckerMock).toHaveBeenCalledWith( + expect.objectContaining({ + allowPluginSyntheticAuth: false, + discoverExternalCliAuth: false, + }), + ); + expect(authChecker).toHaveBeenCalledWith("anthropic"); + expect(authChecker).toHaveBeenCalledWith("openai"); + expect(result).toEqual([{ provider: "openai", id: "gpt-test", name: "GPT Test" }]); + }); +}); diff --git a/src/agents/model-catalog-visibility.ts b/src/agents/model-catalog-visibility.ts index bd46625c622..6a6134ac3ad 100644 --- a/src/agents/model-catalog-visibility.ts +++ b/src/agents/model-catalog-visibility.ts @@ -35,6 +35,7 @@ export function resolveVisibleModelCatalog(params: { workspaceDir?: string; env?: NodeJS.ProcessEnv; view?: ModelCatalogVisibilityView; + runtimeAuthDiscovery?: boolean; }): ModelCatalogEntry[] { if (params.view === "all") { return params.catalog; @@ -59,6 +60,8 @@ export function resolveVisibleModelCatalog(params: { workspaceDir: params.workspaceDir, agentDir: params.agentDir, env: params.env, + allowPluginSyntheticAuth: params.runtimeAuthDiscovery, + discoverExternalCliAuth: params.runtimeAuthDiscovery, }); const authBackedCatalog = params.catalog.filter((entry) => hasAuth(entry.provider)); return sortModelCatalogEntries( diff --git a/src/agents/model-catalog.test.ts b/src/agents/model-catalog.test.ts index af955d6cfaf..9088a04a800 100644 --- a/src/agents/model-catalog.test.ts +++ b/src/agents/model-catalog.test.ts @@ -69,6 +69,18 @@ function mockSingleOpenAiCatalogModel() { mockPiDiscoveryModels([{ id: "gpt-4.1", provider: "openai", name: "GPT-4.1" }]); } +function emptyPluginMetadataSnapshot() { + return { + policyHash: "test-policy", + configFingerprint: "test-config", + index: { + policyHash: "test-policy", + plugins: [], + }, + plugins: [], + }; +} + describe("loadModelCatalog", () => { beforeAll(async () => { readFileMock = vi.fn(); @@ -117,7 +129,9 @@ describe("loadModelCatalog", () => { ensureOpenClawModelsJsonMock.mockClear(); augmentCatalogMock.mockClear(); currentPluginMetadataSnapshotMock.mockReset(); + currentPluginMetadataSnapshotMock.mockReturnValue(emptyPluginMetadataSnapshot()); loadPluginMetadataSnapshotMock.mockReset(); + loadPluginMetadataSnapshotMock.mockReturnValue(emptyPluginMetadataSnapshot()); }); afterEach(() => { @@ -206,26 +220,46 @@ describe("loadModelCatalog", () => { } }); - it("does not prepare models.json when loading catalog in read-only mode", async () => { - const discoverAuthStorage = vi.fn(() => ({})); - __setModelCatalogImportForTest( - async () => - ({ - discoverAuthStorage, - AuthStorage: function AuthStorage() {}, - ModelRegistry: class { - getAll() { - return [{ id: "gpt-4.1", name: "GPT-4.1", provider: "openai" }]; - } + it("does not prepare models.json or import provider discovery when loading fallback catalog in read-only mode", async () => { + const importPiSdk = vi.fn(async () => { + throw new Error("provider discovery should not load"); + }); + __setModelCatalogImportForTest(importPiSdk as unknown as () => Promise); + currentPluginMetadataSnapshotMock.mockReturnValueOnce(undefined); + loadPluginMetadataSnapshotMock.mockImplementationOnce(() => { + throw new Error("metadata scan should not run"); + }); + + const result = await loadModelCatalog({ + config: { + models: { + providers: { + openai: { + baseUrl: "https://openai.example.com/v1", + models: [ + { + id: "gpt-test", + name: "GPT Test", + reasoning: false, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 200_000, + maxTokens: 8192, + }, + ], + }, }, - }) as unknown as PiSdkModule, + }, + } as OpenClawConfig, + readOnly: true, + }); + + expect(result).toContainEqual( + expect.objectContaining({ id: "gpt-test", name: "GPT Test", provider: "openai" }), ); - - const result = await loadModelCatalog({ config: {} as OpenClawConfig, readOnly: true }); - - expect(result).toEqual([{ id: "gpt-4.1", name: "GPT-4.1", provider: "openai" }]); expect(ensureOpenClawModelsJsonMock).not.toHaveBeenCalled(); - expect(discoverAuthStorage).toHaveBeenCalledWith("/tmp/openclaw", { readOnly: true }); + expect(importPiSdk).not.toHaveBeenCalled(); + expect(loadPluginMetadataSnapshotMock).not.toHaveBeenCalled(); }); it("filters suppressed built-ins from persisted read-only catalog rows", async () => { @@ -279,7 +313,7 @@ describe("loadModelCatalog", () => { expect(augmentCatalogMock).not.toHaveBeenCalled(); }); - it("falls back to the registry when persisted read-only catalog has no model rows", async () => { + it("falls back to manifest catalog rows when persisted read-only catalog has no model rows", async () => { readFileMock.mockResolvedValueOnce( JSON.stringify({ providers: { @@ -293,27 +327,50 @@ describe("loadModelCatalog", () => { }, }), ); - const discoverAuthStorage = vi.fn(() => ({ - getOAuthProviders: () => [], - })); - __setModelCatalogImportForTest( - async () => - ({ - discoverAuthStorage, - AuthStorage: function AuthStorage() {}, - ModelRegistry: class { - getAll() { - return [{ id: "gpt-4.1", name: "GPT-4.1", provider: "openai" }]; - } + currentPluginMetadataSnapshotMock.mockReturnValueOnce({ + policyHash: "policy", + index: { + policyHash: "policy", + plugins: [ + { + pluginId: "external-provider", + enabled: true, + origin: "global", }, - }) as unknown as PiSdkModule, - ); + ], + }, + plugins: [ + { + id: "external-provider", + origin: "global", + modelCatalog: { + providers: { + external: { + models: [{ id: "external-fast", name: "External Fast" }], + }, + }, + }, + }, + ], + }); + const importPiSdk = vi.fn(async () => { + throw new Error("provider discovery should not load"); + }); + __setModelCatalogImportForTest(importPiSdk as unknown as () => Promise); const result = await loadModelCatalog({ config: {} as OpenClawConfig, readOnly: true }); - expect(result).toEqual([{ id: "gpt-4.1", name: "GPT-4.1", provider: "openai" }]); + expect(result).toEqual([ + { + provider: "external", + id: "external-fast", + name: "External Fast", + input: ["text"], + reasoning: false, + }, + ]); expect(ensureOpenClawModelsJsonMock).not.toHaveBeenCalled(); - expect(discoverAuthStorage).toHaveBeenCalledWith("/tmp/openclaw", { readOnly: true }); + expect(importPiSdk).not.toHaveBeenCalled(); }); it("preserves registry defaults for minimal persisted read-only catalog rows", async () => { diff --git a/src/agents/model-catalog.ts b/src/agents/model-catalog.ts index 449a5212a0a..e773e08433b 100644 --- a/src/agents/model-catalog.ts +++ b/src/agents/model-catalog.ts @@ -53,6 +53,7 @@ type PiRegistryClassLike = { let modelCatalogPromise: Promise | null = null; let hasLoggedModelCatalogError = false; +let hasLoggedReadOnlyStaticCatalogError = false; const defaultImportPiSdk = () => import("./pi-model-discovery-runtime.js"); let importPiSdk = defaultImportPiSdk; const modelSuppressionLoader = createLazyImportLoader( @@ -70,6 +71,7 @@ function loadModelSuppression() { export function resetModelCatalogCache() { modelCatalogPromise = null; hasLoggedModelCatalogError = false; + hasLoggedReadOnlyStaticCatalogError = false; } export function resetModelCatalogCacheForTest() { @@ -117,22 +119,29 @@ export function loadManifestModelCatalog(params: { config: OpenClawConfig; workspaceDir?: string; env?: NodeJS.ProcessEnv; + fallbackToMetadataScan?: boolean; }): ModelCatalogEntry[] { - const snapshot = - getCurrentPluginMetadataSnapshot({ - config: params.config, - ...(params.workspaceDir !== undefined ? { workspaceDir: params.workspaceDir } : {}), - }) ?? - loadPluginMetadataSnapshot({ - config: params.config, - ...(params.workspaceDir !== undefined ? { workspaceDir: params.workspaceDir } : {}), - env: params.env ?? process.env, - }); - const eligiblePlugins = snapshot.plugins.filter( + const snapshot = getCurrentPluginMetadataSnapshot({ + config: params.config, + ...(params.workspaceDir !== undefined ? { workspaceDir: params.workspaceDir } : {}), + }); + const resolvedSnapshot = + snapshot ?? + (params.fallbackToMetadataScan === false + ? undefined + : loadPluginMetadataSnapshot({ + config: params.config, + ...(params.workspaceDir !== undefined ? { workspaceDir: params.workspaceDir } : {}), + env: params.env ?? process.env, + })); + if (!resolvedSnapshot) { + return []; + } + const eligiblePlugins = resolvedSnapshot.plugins.filter( (plugin) => plugin.modelCatalog && isManifestPluginAvailableForControlPlane({ - snapshot, + snapshot: resolvedSnapshot, plugin, config: params.config, }), @@ -250,6 +259,32 @@ async function loadReadOnlyPersistedModelCatalog(params?: { return sortModelCatalogEntries(models); } +function loadReadOnlyStaticModelCatalog(params?: { config?: OpenClawConfig }): ModelCatalogEntry[] { + const cfg = params?.config ?? getRuntimeConfig(); + const models: ModelCatalogEntry[] = []; + try { + appendCatalogEntriesIfAbsent( + models, + loadManifestModelCatalog({ + config: cfg, + env: process.env, + fallbackToMetadataScan: false, + }), + ); + } catch (error) { + if (!hasLoggedReadOnlyStaticCatalogError) { + hasLoggedReadOnlyStaticCatalogError = true; + log.warn(`Failed to load read-only manifest model catalog: ${String(error)}`); + } + } + + const configuredModels = buildConfiguredModelCatalog({ cfg }); + if (configuredModels.length > 0) { + appendCatalogEntriesIfAbsent(models, configuredModels); + } + return sortModelCatalogEntries(models); +} + export async function loadModelCatalog(params?: { config?: OpenClawConfig; useCache?: boolean; @@ -260,7 +295,9 @@ export async function loadModelCatalog(params?: { try { return await loadReadOnlyPersistedModelCatalog(params); } catch { - // fall through to full catalog path + // Keep gateway models.list on side-effect-free sources. The RPC timeout + // cannot fire while provider discovery blocks the event loop. + return loadReadOnlyStaticModelCatalog(params); } } if (!readOnly && params?.useCache === false) { diff --git a/src/agents/model-provider-auth.ts b/src/agents/model-provider-auth.ts index 0b4ce41a54d..0352528c2a7 100644 --- a/src/agents/model-provider-auth.ts +++ b/src/agents/model-provider-auth.ts @@ -2,6 +2,7 @@ import type { OpenClawConfig } from "../config/types.openclaw.js"; import { externalCliDiscoveryForProviderAuth, ensureAuthProfileStore, + ensureAuthProfileStoreWithoutExternalProfiles, listProfilesForProvider, type AuthProfileStore, } from "./auth-profiles.js"; @@ -15,6 +16,8 @@ export function hasAuthForModelProvider(params: { agentDir?: string; env?: NodeJS.ProcessEnv; store?: AuthProfileStore; + allowPluginSyntheticAuth?: boolean; + discoverExternalCliAuth?: boolean; }): boolean { const provider = normalizeProviderId(params.provider); if ( @@ -23,15 +26,20 @@ export function hasAuthForModelProvider(params: { cfg: params.cfg, workspaceDir: params.workspaceDir, env: params.env, + allowPluginSyntheticAuth: params.allowPluginSyntheticAuth, }) ) { return true; } const store = params.store ?? - ensureAuthProfileStore(params.agentDir, { - externalCli: externalCliDiscoveryForProviderAuth({ cfg: params.cfg, provider }), - }); + (params.discoverExternalCliAuth === false + ? ensureAuthProfileStoreWithoutExternalProfiles(params.agentDir, { + allowKeychainPrompt: false, + }) + : ensureAuthProfileStore(params.agentDir, { + externalCli: externalCliDiscoveryForProviderAuth({ cfg: params.cfg, provider }), + })); if (listProfilesForProvider(store, provider).length > 0) { return true; } @@ -43,6 +51,8 @@ export function createProviderAuthChecker(params: { workspaceDir?: string; agentDir?: string; env?: NodeJS.ProcessEnv; + allowPluginSyntheticAuth?: boolean; + discoverExternalCliAuth?: boolean; }): (provider: string) => boolean { const authCache = new Map(); return (provider: string) => { @@ -57,6 +67,8 @@ export function createProviderAuthChecker(params: { workspaceDir: params.workspaceDir, agentDir: params.agentDir, env: params.env, + allowPluginSyntheticAuth: params.allowPluginSyntheticAuth, + discoverExternalCliAuth: params.discoverExternalCliAuth, }); authCache.set(key, value); return value; diff --git a/src/gateway/server-methods/models.ts b/src/gateway/server-methods/models.ts index 8cf6236d4e5..16fbf071c1d 100644 --- a/src/gateway/server-methods/models.ts +++ b/src/gateway/server-methods/models.ts @@ -85,6 +85,7 @@ export const modelsHandlers: GatewayRequestHandlers = { defaultProvider: DEFAULT_PROVIDER, workspaceDir, view, + runtimeAuthDiscovery: false, }); respond(true, { models }, undefined); } catch (err) {