From 62a5963d243c33d099f124936d7bc8015f6bdd08 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sat, 25 Apr 2026 22:47:13 -0700 Subject: [PATCH] feat(providers): add provider index install metadata --- CHANGELOG.md | 1 + docs/plugins/manifest.md | 11 +- src/model-catalog/provider-index/index.ts | 2 + .../provider-index/normalize.test.ts | 43 +- src/model-catalog/provider-index/normalize.ts | 108 ++++ src/model-catalog/provider-index/types.ts | 26 + src/plugins/provider-install-catalog.test.ts | 571 +++++++++--------- src/plugins/provider-install-catalog.ts | 257 +++++--- 8 files changed, 633 insertions(+), 386 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d4677c6d094..ff8aa448e42 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,7 @@ Docs: https://docs.openclaw.ai - Plugins/registry: route cold manifest and capability lookups through the installed plugin index so setup, channels, config, secrets, doctor, and provider metadata paths avoid broad plugin-root scans before runtime execution. Thanks @shakkernerd. - CLI/models: speed up `models list --all --provider ` for static manifest-backed providers by loading catalog rows through the installed plugin index instead of broad manifest scans or runtime suppression hooks. Thanks @shakkernerd. - CLI/models: use OpenClaw Provider Index preview rows as the final cold fallback for installable providers, while keeping user config, installed manifests, and refreshed cache rows above provider-index metadata. Thanks @vincentkoc. +- Providers/plugins: keep onboarding and auth-choice setup lists on cold manifest/install metadata and add Provider Index install metadata for not-yet-installed provider plugins. Thanks @vincentkoc. - Plugins/chat commands: refresh the persisted plugin registry after `/plugins enable` and `/plugins disable`, matching the CLI mutation path. Thanks @vincentkoc. - Plugins/compat: mark `OPENCLAW_DISABLE_PERSISTED_PLUGIN_REGISTRY` as a deprecated break-glass switch and point operators at registry repair instead. Thanks @vincentkoc. - Plugins/registry: ignore stale persisted registry reads when plugin policy no longer matches current config, and stamp generated registry files with a do-not-edit warning. Thanks @vincentkoc. diff --git a/docs/plugins/manifest.md b/docs/plugins/manifest.md index 40f9cda5397..f1703ac2107 100644 --- a/docs/plugins/manifest.md +++ b/docs/plugins/manifest.md @@ -169,8 +169,8 @@ or npm install metadata. Those belong in your plugin code and `package.json`. Each `providerAuthChoices` entry describes one onboarding or auth choice. OpenClaw reads this before provider runtime loads. -Provider setup flow prefers these manifest choices, then falls back to runtime -wizard metadata and install-catalog choices for compatibility. +Provider setup lists use these manifest choices, descriptor-derived setup +choices, and install-catalog metadata without loading provider runtime. | Field | Required | Type | What it means | | --------------------- | -------- | ----------------------------------------------- | -------------------------------------------------------------------------------------------------------- | @@ -749,6 +749,13 @@ the installed plugin manifest. Providers with live `/models` discovery should write refreshed rows through the explicit model catalog cache path instead of making normal listing or onboarding call provider APIs. +Provider Index entries may also carry installable-plugin metadata for providers +whose plugin has moved out of core or is otherwise not installed yet. This +metadata mirrors the channel catalog pattern: package name, npm install spec, +expected integrity, and cheap auth-choice labels are enough to show an +installable setup option. Once the plugin is installed, its manifest wins and +the Provider Index entry is ignored for that provider. + Legacy top-level capability keys are deprecated. Use `openclaw doctor --fix` to move `speechProviders`, `realtimeTranscriptionProviders`, `realtimeVoiceProviders`, `mediaUnderstandingProviders`, diff --git a/src/model-catalog/provider-index/index.ts b/src/model-catalog/provider-index/index.ts index 753ea309be9..8122d0fa9a2 100644 --- a/src/model-catalog/provider-index/index.ts +++ b/src/model-catalog/provider-index/index.ts @@ -2,6 +2,8 @@ export { loadOpenClawProviderIndex } from "./load.js"; export { normalizeOpenClawProviderIndex } from "./normalize.js"; export type { OpenClawProviderIndex, + OpenClawProviderIndexPluginInstall, OpenClawProviderIndexPlugin, + OpenClawProviderIndexProviderAuthChoice, OpenClawProviderIndexProvider, } from "./types.js"; diff --git a/src/model-catalog/provider-index/normalize.test.ts b/src/model-catalog/provider-index/normalize.test.ts index 09e933f1276..ae39507058c 100644 --- a/src/model-catalog/provider-index/normalize.test.ts +++ b/src/model-catalog/provider-index/normalize.test.ts @@ -9,9 +9,33 @@ describe("OpenClaw provider index", () => { Moonshot: { id: "moonshot", name: "Moonshot AI", - plugin: { id: "moonshot", package: " @openclaw/plugin-moonshot " }, + plugin: { + id: "moonshot", + package: " @openclaw/plugin-moonshot ", + install: { + npmSpec: " @openclaw/plugin-moonshot@1.2.3 ", + defaultChoice: "npm", + expectedIntegrity: " sha512-moonshot ", + }, + }, docs: "/providers/moonshot", categories: ["cloud", "llm"], + authChoices: [ + { + method: " api-key ", + choiceId: " moonshot-api-key ", + choiceLabel: " Moonshot API key ", + groupLabel: " Moonshot AI ", + assistantPriority: -1, + assistantVisibility: "visible", + onboardingScopes: ["text-inference", "bad-scope"], + }, + { + method: "__proto__", + choiceId: "bad", + choiceLabel: "Bad", + }, + ], previewCatalog: { api: "openai-responses", baseUrl: "https://api.moonshot.ai/v1", @@ -38,9 +62,26 @@ describe("OpenClaw provider index", () => { plugin: { id: "moonshot", package: "@openclaw/plugin-moonshot", + install: { + npmSpec: "@openclaw/plugin-moonshot@1.2.3", + defaultChoice: "npm", + expectedIntegrity: "sha512-moonshot", + }, }, docs: "/providers/moonshot", categories: ["cloud", "llm"], + authChoices: [ + { + method: "api-key", + choiceId: "moonshot-api-key", + choiceLabel: "Moonshot API key", + assistantPriority: -1, + assistantVisibility: "visible", + groupId: "moonshot", + groupLabel: "Moonshot AI", + onboardingScopes: ["text-inference"], + }, + ], previewCatalog: { api: "openai-responses", baseUrl: "https://api.moonshot.ai/v1", diff --git a/src/model-catalog/provider-index/normalize.ts b/src/model-catalog/provider-index/normalize.ts index 0f7ceee0dc7..956a9b4ceea 100644 --- a/src/model-catalog/provider-index/normalize.ts +++ b/src/model-catalog/provider-index/normalize.ts @@ -1,3 +1,4 @@ +import { parseRegistryNpmSpec } from "../../infra/npm-registry-spec.js"; import { isBlockedObjectKey } from "../../infra/prototype-keys.js"; import { normalizeOptionalString } from "../../shared/string-coerce.js"; import { normalizeTrimmedStringList } from "../../shared/string-normalization.js"; @@ -7,7 +8,9 @@ import { normalizeModelCatalogProviderId } from "../refs.js"; import type { ModelCatalogProvider } from "../types.js"; import type { OpenClawProviderIndex, + OpenClawProviderIndexPluginInstall, OpenClawProviderIndexPlugin, + OpenClawProviderIndexProviderAuthChoice, OpenClawProviderIndexProvider, } from "./types.js"; @@ -18,6 +21,26 @@ function normalizeSafeKey(value: unknown): string { return key && !isBlockedObjectKey(key) ? key : ""; } +function normalizeInstall(value: unknown): OpenClawProviderIndexPluginInstall | undefined { + if (!isRecord(value)) { + return undefined; + } + const npmSpec = normalizeOptionalString(value.npmSpec); + const parsed = npmSpec ? parseRegistryNpmSpec(npmSpec) : null; + if (!parsed) { + return undefined; + } + const defaultChoice = value.defaultChoice === "npm" ? "npm" : undefined; + const minHostVersion = normalizeOptionalString(value.minHostVersion); + const expectedIntegrity = normalizeOptionalString(value.expectedIntegrity); + return { + npmSpec: parsed.raw, + ...(defaultChoice ? { defaultChoice } : {}), + ...(minHostVersion ? { minHostVersion } : {}), + ...(expectedIntegrity ? { expectedIntegrity } : {}), + }; +} + function normalizePlugin(value: unknown): OpenClawProviderIndexPlugin | undefined { if (!isRecord(value)) { return undefined; @@ -28,10 +51,12 @@ function normalizePlugin(value: unknown): OpenClawProviderIndexPlugin | undefine } const packageName = normalizeOptionalString(value.package) ?? ""; const source = normalizeOptionalString(value.source) ?? ""; + const install = normalizeInstall(value.install); return { id, ...(packageName ? { package: packageName } : {}), ...(source ? { source } : {}), + ...(install ? { install } : {}), }; } @@ -57,6 +82,83 @@ function normalizePreviewCatalog(params: { return provider; } +function normalizeOnboardingScopes( + value: unknown, +): OpenClawProviderIndexProviderAuthChoice["onboardingScopes"] | undefined { + const scopes = normalizeTrimmedStringList(value).filter( + (scope): scope is "text-inference" | "image-generation" => + scope === "text-inference" || scope === "image-generation", + ); + return scopes.length > 0 ? [...new Set(scopes)] : undefined; +} + +function normalizeAssistantVisibility( + value: unknown, +): OpenClawProviderIndexProviderAuthChoice["assistantVisibility"] | undefined { + return value === "visible" || value === "manual-only" ? value : undefined; +} + +function normalizeFiniteNumber(value: unknown): number | undefined { + return typeof value === "number" && Number.isFinite(value) ? value : undefined; +} + +function normalizeAuthChoice(params: { + providerId: string; + providerName: string; + value: unknown; +}): OpenClawProviderIndexProviderAuthChoice | undefined { + if (!isRecord(params.value)) { + return undefined; + } + const method = normalizeSafeKey(params.value.method); + const choiceId = normalizeSafeKey(params.value.choiceId); + const choiceLabel = normalizeOptionalString(params.value.choiceLabel) ?? ""; + if (!method || !choiceId || !choiceLabel) { + return undefined; + } + const choiceHint = normalizeOptionalString(params.value.choiceHint); + const groupId = normalizeSafeKey(params.value.groupId) || params.providerId; + const groupLabel = normalizeOptionalString(params.value.groupLabel) ?? params.providerName; + const groupHint = normalizeOptionalString(params.value.groupHint); + const optionKey = normalizeSafeKey(params.value.optionKey); + const cliFlag = normalizeOptionalString(params.value.cliFlag); + const cliOption = normalizeOptionalString(params.value.cliOption); + const cliDescription = normalizeOptionalString(params.value.cliDescription); + const assistantPriority = normalizeFiniteNumber(params.value.assistantPriority); + const assistantVisibility = normalizeAssistantVisibility(params.value.assistantVisibility); + const onboardingScopes = normalizeOnboardingScopes(params.value.onboardingScopes); + return { + method, + choiceId, + choiceLabel, + ...(choiceHint ? { choiceHint } : {}), + ...(assistantPriority !== undefined ? { assistantPriority } : {}), + ...(assistantVisibility ? { assistantVisibility } : {}), + ...(groupId ? { groupId } : {}), + ...(groupLabel ? { groupLabel } : {}), + ...(groupHint ? { groupHint } : {}), + ...(optionKey ? { optionKey } : {}), + ...(cliFlag ? { cliFlag } : {}), + ...(cliOption ? { cliOption } : {}), + ...(cliDescription ? { cliDescription } : {}), + ...(onboardingScopes ? { onboardingScopes } : {}), + }; +} + +function normalizeAuthChoices(params: { + providerId: string; + providerName: string; + value: unknown; +}): readonly OpenClawProviderIndexProviderAuthChoice[] | undefined { + if (!Array.isArray(params.value)) { + return undefined; + } + const choices = params.value + .map((value) => normalizeAuthChoice({ ...params, value })) + .filter((choice): choice is OpenClawProviderIndexProviderAuthChoice => Boolean(choice)); + return choices.length > 0 ? choices : undefined; +} + function normalizeProvider( rawProviderId: string, value: unknown, @@ -79,6 +181,11 @@ function normalizeProvider( } const docs = normalizeOptionalString(value.docs) ?? ""; const categories = normalizeCategories(value.categories); + const authChoices = normalizeAuthChoices({ + providerId, + providerName: name, + value: value.authChoices, + }); const previewCatalog = normalizePreviewCatalog({ providerId, value: value.previewCatalog, @@ -89,6 +196,7 @@ function normalizeProvider( plugin, ...(docs ? { docs } : {}), ...(categories.length > 0 ? { categories } : {}), + ...(authChoices ? { authChoices } : {}), ...(previewCatalog ? { previewCatalog } : {}), }; } diff --git a/src/model-catalog/provider-index/types.ts b/src/model-catalog/provider-index/types.ts index 873888c4aec..18f1f84d174 100644 --- a/src/model-catalog/provider-index/types.ts +++ b/src/model-catalog/provider-index/types.ts @@ -1,9 +1,34 @@ import type { ModelCatalogProvider } from "../types.js"; +export type OpenClawProviderIndexPluginInstall = { + npmSpec: string; + defaultChoice?: "npm"; + minHostVersion?: string; + expectedIntegrity?: string; +}; + export type OpenClawProviderIndexPlugin = { id: string; package?: string; source?: string; + install?: OpenClawProviderIndexPluginInstall; +}; + +export type OpenClawProviderIndexProviderAuthChoice = { + method: string; + choiceId: string; + choiceLabel: string; + choiceHint?: string; + assistantPriority?: number; + assistantVisibility?: "visible" | "manual-only"; + groupId?: string; + groupLabel?: string; + groupHint?: string; + optionKey?: string; + cliFlag?: string; + cliOption?: string; + cliDescription?: string; + onboardingScopes?: readonly ("text-inference" | "image-generation")[]; }; export type OpenClawProviderIndexProvider = { @@ -12,6 +37,7 @@ export type OpenClawProviderIndexProvider = { plugin: OpenClawProviderIndexPlugin; docs?: string; categories?: readonly string[]; + authChoices?: readonly OpenClawProviderIndexProviderAuthChoice[]; previewCatalog?: ModelCatalogProvider; }; diff --git a/src/plugins/provider-install-catalog.test.ts b/src/plugins/provider-install-catalog.test.ts index a2619a10908..b0398badabc 100644 --- a/src/plugins/provider-install-catalog.test.ts +++ b/src/plugins/provider-install-catalog.test.ts @@ -1,26 +1,41 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; -type DiscoverOpenClawPlugins = typeof import("./discovery.js").discoverOpenClawPlugins; -type LoadPluginManifest = typeof import("./manifest.js").loadPluginManifest; +type LoadOpenClawProviderIndex = + typeof import("../model-catalog/index.js").loadOpenClawProviderIndex; +type LoadPluginRegistrySnapshot = typeof import("./plugin-registry.js").loadPluginRegistrySnapshot; type ResolveManifestProviderAuthChoices = typeof import("./provider-auth-choices.js").resolveManifestProviderAuthChoices; -const discoverOpenClawPlugins = vi.hoisted(() => - vi.fn(() => ({ candidates: [], diagnostics: [] })), +const loadOpenClawProviderIndex = vi.hoisted(() => + vi.fn(() => ({ version: 1, providers: {} })), ); -vi.mock("./discovery.js", () => ({ - discoverOpenClawPlugins, -})); - -const loadPluginManifest = vi.hoisted(() => vi.fn()); -vi.mock("./manifest.js", async () => { - const actual = await vi.importActual("./manifest.js"); +vi.mock("../model-catalog/index.js", async () => { + const actual = await vi.importActual( + "../model-catalog/index.js", + ); return { ...actual, - loadPluginManifest, + loadOpenClawProviderIndex, }; }); +const loadPluginRegistrySnapshot = vi.hoisted(() => + vi.fn(() => ({ + version: 1, + hostContractVersion: "test", + compatRegistryVersion: "test", + migrationVersion: 1, + policyHash: "test", + generatedAtMs: 0, + installRecords: {}, + plugins: [], + diagnostics: [], + })), +); +vi.mock("./plugin-registry.js", () => ({ + loadPluginRegistrySnapshot, +})); + const resolveManifestProviderAuthChoices = vi.hoisted(() => vi.fn(() => []), ); @@ -36,45 +51,66 @@ import { describe("provider install catalog", () => { beforeEach(() => { vi.clearAllMocks(); - discoverOpenClawPlugins.mockReturnValue({ - candidates: [], + loadOpenClawProviderIndex.mockReturnValue({ version: 1, providers: {} }); + loadPluginRegistrySnapshot.mockReturnValue({ + version: 1, + hostContractVersion: "test", + compatRegistryVersion: "test", + migrationVersion: 1, + policyHash: "test", + generatedAtMs: 0, + installRecords: {}, + plugins: [], diagnostics: [], }); resolveManifestProviderAuthChoices.mockReturnValue([]); }); - it("merges manifest auth-choice metadata with discovery install metadata", () => { - discoverOpenClawPlugins.mockReturnValue({ - candidates: [ + it("merges manifest auth-choice metadata with registry install metadata", () => { + loadPluginRegistrySnapshot.mockReturnValue({ + version: 1, + hostContractVersion: "test", + compatRegistryVersion: "test", + migrationVersion: 1, + policyHash: "test", + generatedAtMs: 0, + installRecords: {}, + plugins: [ { - idHint: "openai", + pluginId: "openai", origin: "bundled", + manifestPath: "/repo/extensions/openai/openclaw.plugin.json", + manifestHash: "hash", rootDir: "/repo/extensions/openai", - source: "/repo/extensions/openai/index.ts", - workspaceDir: "/repo", + enabled: true, + startup: { + sidecar: false, + memory: false, + deferConfiguredChannelFullLoadUntilAfterListen: false, + agentHarnesses: [], + }, + compat: [], packageName: "@openclaw/openai", - packageDir: "/repo/extensions/openai", - packageManifest: { - install: { - npmSpec: "@openclaw/openai@1.2.3", - defaultChoice: "npm", + packageInstall: { + defaultChoice: "npm", + npm: { + spec: "@openclaw/openai@1.2.3", + packageName: "@openclaw/openai", + selector: "1.2.3", + selectorKind: "exact-version", + exactVersion: true, expectedIntegrity: "sha512-openai", + pinState: "exact-with-integrity", }, + local: { + path: "extensions/openai", + }, + warnings: [], }, }, ], diagnostics: [], }); - loadPluginManifest.mockReturnValue({ - ok: true, - manifestPath: "/repo/extensions/openai/openclaw.plugin.json", - manifest: { - id: "openai", - configSchema: { - type: "object", - }, - }, - }); resolveManifestProviderAuthChoices.mockReturnValue([ { pluginId: "openai", @@ -124,96 +160,54 @@ describe("provider install catalog", () => { ]); }); - it("falls back to workspace-relative local path when install metadata is sparse", () => { - discoverOpenClawPlugins.mockReturnValue({ - candidates: [ + it("prefers durable install records over package-authored install intent", () => { + loadPluginRegistrySnapshot.mockReturnValue({ + version: 1, + hostContractVersion: "test", + compatRegistryVersion: "test", + migrationVersion: 1, + policyHash: "test", + generatedAtMs: 0, + installRecords: { + vllm: { + source: "npm", + spec: "@openclaw/vllm", + resolvedSpec: "@openclaw/vllm@2.0.0", + integrity: "sha512-vllm", + }, + }, + plugins: [ { - idHint: "demo-provider", - origin: "workspace", - rootDir: "/repo/extensions/demo-provider", - source: "/repo/extensions/demo-provider/index.ts", - workspaceDir: "/repo", - packageName: "@vendor/demo-provider", - packageDir: "/repo/extensions/demo-provider", - packageManifest: {}, - }, - ], - diagnostics: [], - }); - loadPluginManifest.mockReturnValue({ - ok: true, - manifestPath: "/repo/extensions/demo-provider/openclaw.plugin.json", - manifest: { - id: "demo-provider", - configSchema: { - type: "object", - }, - }, - }); - resolveManifestProviderAuthChoices.mockReturnValue([ - { - pluginId: "demo-provider", - providerId: "demo-provider", - methodId: "api-key", - choiceId: "demo-provider-api-key", - choiceLabel: "Demo Provider API key", - }, - ]); - - expect(resolveProviderInstallCatalogEntries()).toEqual([ - { - pluginId: "demo-provider", - providerId: "demo-provider", - methodId: "api-key", - choiceId: "demo-provider-api-key", - choiceLabel: "Demo Provider API key", - label: "Demo Provider API key", - origin: "workspace", - install: { - localPath: "extensions/demo-provider", - defaultChoice: "local", - }, - installSource: { - defaultChoice: "local", - local: { - path: "extensions/demo-provider", + pluginId: "vllm", + origin: "global", + manifestPath: "/Users/test/.openclaw/plugins/vllm/openclaw.plugin.json", + manifestHash: "hash", + rootDir: "/Users/test/.openclaw/plugins/vllm", + enabled: true, + startup: { + sidecar: false, + memory: false, + deferConfiguredChannelFullLoadUntilAfterListen: false, + agentHarnesses: [], }, - warnings: [], - }, - }, - ]); - }); - - it("resolves one installable auth choice by id", () => { - discoverOpenClawPlugins.mockReturnValue({ - candidates: [ - { - idHint: "vllm", - origin: "config", - rootDir: "/Users/test/.openclaw/extensions/vllm", - source: "/Users/test/.openclaw/extensions/vllm/index.js", + compat: [], packageName: "@openclaw/vllm", - packageDir: "/Users/test/.openclaw/extensions/vllm", - packageManifest: { - install: { - npmSpec: "@openclaw/vllm@2.0.0", - expectedIntegrity: "sha512-vllm", + packageInstall: { + npm: { + spec: "@openclaw/vllm-fork@1.0.0", + packageName: "@openclaw/vllm-fork", + selector: "1.0.0", + selectorKind: "exact-version", + exactVersion: true, + expectedIntegrity: "sha512-old", + pinState: "exact-with-integrity", }, + warnings: [], }, }, ], diagnostics: [], }); - loadPluginManifest.mockReturnValue({ - ok: true, - manifestPath: "/Users/test/.openclaw/extensions/vllm/openclaw.plugin.json", - manifest: { - id: "vllm", - configSchema: { - type: "object", - }, - }, - }); resolveManifestProviderAuthChoices.mockReturnValue([ { pluginId: "vllm", @@ -233,7 +227,7 @@ describe("provider install catalog", () => { choiceLabel: "vLLM", groupLabel: "vLLM", label: "vLLM", - origin: "config", + origin: "global", install: { npmSpec: "@openclaw/vllm@2.0.0", expectedIntegrity: "sha512-vllm", @@ -255,157 +249,47 @@ describe("provider install catalog", () => { }); }); - it("exposes trusted registry npm specs without requiring an exact version or integrity pin", () => { - discoverOpenClawPlugins.mockReturnValue({ - candidates: [ + it("does not expose untrusted global package install intent without an install record", () => { + loadPluginRegistrySnapshot.mockReturnValue({ + version: 1, + hostContractVersion: "test", + compatRegistryVersion: "test", + migrationVersion: 1, + policyHash: "test", + generatedAtMs: 0, + installRecords: {}, + plugins: [ { - idHint: "vllm", - origin: "config", - rootDir: "/Users/test/.openclaw/extensions/vllm", - source: "/Users/test/.openclaw/extensions/vllm/index.js", - packageName: "@openclaw/vllm", - packageDir: "/Users/test/.openclaw/extensions/vllm", - packageManifest: { - install: { - npmSpec: "@openclaw/vllm", - }, - }, - }, - ], - diagnostics: [], - }); - loadPluginManifest.mockReturnValue({ - ok: true, - manifestPath: "/Users/test/.openclaw/extensions/vllm/openclaw.plugin.json", - manifest: { - id: "vllm", - configSchema: { - type: "object", - }, - }, - }); - resolveManifestProviderAuthChoices.mockReturnValue([ - { - pluginId: "vllm", - providerId: "vllm", - methodId: "server", - choiceId: "vllm", - choiceLabel: "vLLM", - }, - ]); - - expect(resolveProviderInstallCatalogEntry("vllm")).toEqual({ - pluginId: "vllm", - providerId: "vllm", - methodId: "server", - choiceId: "vllm", - choiceLabel: "vLLM", - label: "vLLM", - origin: "config", - install: { - npmSpec: "@openclaw/vllm", - defaultChoice: "npm", - }, - installSource: { - defaultChoice: "npm", - npm: { - spec: "@openclaw/vllm", - packageName: "@openclaw/vllm", - selectorKind: "none", - exactVersion: false, - pinState: "floating-without-integrity", - }, - warnings: ["npm-spec-floating", "npm-spec-missing-integrity"], - }, - }); - }); - - it("warns when provider install npmSpec drifts from package identity", () => { - discoverOpenClawPlugins.mockReturnValue({ - candidates: [ - { - idHint: "vllm", - origin: "config", - rootDir: "/Users/test/.openclaw/extensions/vllm", - source: "/Users/test/.openclaw/extensions/vllm/index.js", - packageName: "@openclaw/vllm", - packageDir: "/Users/test/.openclaw/extensions/vllm", - packageManifest: { - install: { - npmSpec: "@openclaw/vllm-fork@2.0.0", - expectedIntegrity: "sha512-vllm", - }, - }, - }, - ], - diagnostics: [], - }); - loadPluginManifest.mockReturnValue({ - ok: true, - manifestPath: "/Users/test/.openclaw/extensions/vllm/openclaw.plugin.json", - manifest: { - id: "vllm", - configSchema: { - type: "object", - }, - }, - }); - resolveManifestProviderAuthChoices.mockReturnValue([ - { - pluginId: "vllm", - providerId: "vllm", - methodId: "server", - choiceId: "vllm", - choiceLabel: "vLLM", - }, - ]); - - expect(resolveProviderInstallCatalogEntry("vllm")?.installSource).toEqual({ - defaultChoice: "npm", - npm: { - spec: "@openclaw/vllm-fork@2.0.0", - packageName: "@openclaw/vllm-fork", - expectedPackageName: "@openclaw/vllm", - selector: "2.0.0", - selectorKind: "exact-version", - exactVersion: true, - expectedIntegrity: "sha512-vllm", - pinState: "exact-with-integrity", - }, - warnings: ["npm-spec-package-name-mismatch"], - }); - }); - - it("does not expose npm install specs from untrusted package metadata", () => { - discoverOpenClawPlugins.mockReturnValue({ - candidates: [ - { - idHint: "demo-provider", + pluginId: "demo-provider", origin: "global", - rootDir: "/Users/test/.openclaw/extensions/demo-provider", - source: "/Users/test/.openclaw/extensions/demo-provider/index.js", + manifestPath: "/Users/test/.openclaw/plugins/demo-provider/openclaw.plugin.json", + manifestHash: "hash", + rootDir: "/Users/test/.openclaw/plugins/demo-provider", + enabled: true, + startup: { + sidecar: false, + memory: false, + deferConfiguredChannelFullLoadUntilAfterListen: false, + agentHarnesses: [], + }, + compat: [], packageName: "@vendor/demo-provider", - packageDir: "/Users/test/.openclaw/extensions/demo-provider", - packageManifest: { - install: { - npmSpec: "@vendor/demo-provider@1.2.3", + packageInstall: { + npm: { + spec: "@vendor/demo-provider@1.2.3", + packageName: "@vendor/demo-provider", + selector: "1.2.3", + selectorKind: "exact-version", + exactVersion: true, expectedIntegrity: "sha512-demo", + pinState: "exact-with-integrity", }, + warnings: [], }, }, ], diagnostics: [], }); - loadPluginManifest.mockReturnValue({ - ok: true, - manifestPath: "/Users/test/.openclaw/extensions/demo-provider/openclaw.plugin.json", - manifest: { - id: "demo-provider", - configSchema: { - type: "object", - }, - }, - }); resolveManifestProviderAuthChoices.mockReturnValue([ { pluginId: "demo-provider", @@ -419,26 +303,49 @@ describe("provider install catalog", () => { expect(resolveProviderInstallCatalogEntries()).toEqual([]); }); - it("skips untrusted workspace install candidates when requested", () => { - discoverOpenClawPlugins.mockReturnValue({ - candidates: [ + it("skips untrusted workspace package install metadata when the plugin is disabled", () => { + loadPluginRegistrySnapshot.mockReturnValue({ + version: 1, + hostContractVersion: "test", + compatRegistryVersion: "test", + migrationVersion: 1, + policyHash: "test", + generatedAtMs: 0, + installRecords: {}, + plugins: [ { - idHint: "demo-provider", + pluginId: "demo-provider", origin: "workspace", + manifestPath: "/repo/extensions/demo-provider/openclaw.plugin.json", + manifestHash: "hash", rootDir: "/repo/extensions/demo-provider", - source: "/repo/extensions/demo-provider/index.ts", - workspaceDir: "/repo", - packageName: "@vendor/demo-provider", - packageDir: "/repo/extensions/demo-provider", - packageManifest: { - install: { - npmSpec: "@vendor/demo-provider", + enabled: false, + startup: { + sidecar: false, + memory: false, + deferConfiguredChannelFullLoadUntilAfterListen: false, + agentHarnesses: [], + }, + compat: [], + packageInstall: { + local: { + path: "extensions/demo-provider", }, + warnings: [], }, }, ], diagnostics: [], }); + resolveManifestProviderAuthChoices.mockReturnValue([ + { + pluginId: "demo-provider", + providerId: "demo-provider", + methodId: "api-key", + choiceId: "demo-provider-api-key", + choiceLabel: "Demo Provider API key", + }, + ]); expect( resolveProviderInstallCatalogEntries({ @@ -450,33 +357,123 @@ describe("provider install catalog", () => { includeUntrustedWorkspacePlugins: false, }), ).toEqual([]); - expect(loadPluginManifest).not.toHaveBeenCalled(); }); - it("skips untrusted workspace candidates without id hints before manifest load", () => { - discoverOpenClawPlugins.mockReturnValue({ - candidates: [ - { - idHint: "", - origin: "workspace", - rootDir: "/repo/extensions/demo-provider", - source: "/repo/extensions/demo-provider/index.ts", - workspaceDir: "/repo", - packageName: "@vendor/demo-provider", - packageDir: "/repo/extensions/demo-provider", - packageManifest: { + it("surfaces provider-index install metadata when the provider plugin is not installed", () => { + loadOpenClawProviderIndex.mockReturnValue({ + version: 1, + providers: { + moonshot: { + id: "moonshot", + name: "Moonshot AI", + plugin: { + id: "moonshot", + package: "@openclaw/plugin-moonshot", install: { - npmSpec: "@vendor/demo-provider", + npmSpec: "@openclaw/plugin-moonshot@1.2.3", + defaultChoice: "npm", + expectedIntegrity: "sha512-moonshot", }, }, + authChoices: [ + { + method: "api-key", + choiceId: "moonshot-api-key", + choiceLabel: "Moonshot API key", + groupId: "moonshot", + groupLabel: "Moonshot AI", + onboardingScopes: ["text-inference"], + }, + ], + }, + }, + }); + + expect(resolveProviderInstallCatalogEntry("moonshot-api-key")).toEqual({ + pluginId: "moonshot", + providerId: "moonshot", + methodId: "api-key", + choiceId: "moonshot-api-key", + choiceLabel: "Moonshot API key", + groupId: "moonshot", + groupLabel: "Moonshot AI", + onboardingScopes: ["text-inference"], + label: "Moonshot AI", + origin: "bundled", + install: { + npmSpec: "@openclaw/plugin-moonshot@1.2.3", + defaultChoice: "npm", + expectedIntegrity: "sha512-moonshot", + }, + installSource: { + defaultChoice: "npm", + npm: { + spec: "@openclaw/plugin-moonshot@1.2.3", + packageName: "@openclaw/plugin-moonshot", + selector: "1.2.3", + selectorKind: "exact-version", + exactVersion: true, + expectedIntegrity: "sha512-moonshot", + pinState: "exact-with-integrity", + }, + warnings: [], + }, + }); + }); + + it("keeps provider-index entries hidden when the plugin is already installed", () => { + loadPluginRegistrySnapshot.mockReturnValue({ + version: 1, + hostContractVersion: "test", + compatRegistryVersion: "test", + migrationVersion: 1, + policyHash: "test", + generatedAtMs: 0, + installRecords: {}, + plugins: [ + { + pluginId: "moonshot", + origin: "bundled", + manifestPath: "/repo/extensions/moonshot/openclaw.plugin.json", + manifestHash: "hash", + rootDir: "/repo/extensions/moonshot", + enabled: true, + startup: { + sidecar: false, + memory: false, + deferConfiguredChannelFullLoadUntilAfterListen: false, + agentHarnesses: [], + }, + compat: [], }, ], diagnostics: [], }); + loadOpenClawProviderIndex.mockReturnValue({ + version: 1, + providers: { + moonshot: { + id: "moonshot", + name: "Moonshot AI", + plugin: { + id: "moonshot", + package: "@openclaw/plugin-moonshot", + install: { + npmSpec: "@openclaw/plugin-moonshot@1.2.3", + expectedIntegrity: "sha512-moonshot", + }, + }, + authChoices: [ + { + method: "api-key", + choiceId: "moonshot-api-key", + choiceLabel: "Moonshot API key", + }, + ], + }, + }, + }); - expect( - resolveProviderInstallCatalogEntries({ includeUntrustedWorkspacePlugins: false }), - ).toEqual([]); - expect(loadPluginManifest).not.toHaveBeenCalled(); + expect(resolveProviderInstallCatalogEntry("moonshot-api-key")).toBeUndefined(); }); }); diff --git a/src/plugins/provider-install-catalog.ts b/src/plugins/provider-install-catalog.ts index 002d12c01cf..3a49085a511 100644 --- a/src/plugins/provider-install-catalog.ts +++ b/src/plugins/provider-install-catalog.ts @@ -1,17 +1,16 @@ -import path from "node:path"; -import { parseRegistryNpmSpec } from "../infra/npm-registry-spec.js"; +import { + loadOpenClawProviderIndex, + type OpenClawProviderIndexProvider, +} from "../model-catalog/index.js"; import { normalizePluginsConfig, resolveEffectiveEnableState } from "./config-state.js"; -import { discoverOpenClawPlugins } from "./discovery.js"; import { describePluginInstallSource, type PluginInstallSourceInfo, } from "./install-source-info.js"; -import { - loadPluginManifest, - type PluginPackageInstall, - type PluginManifestLoadResult, -} from "./manifest.js"; +import type { InstalledPluginInstallRecordInfo } from "./installed-plugin-index.js"; +import type { PluginPackageInstall } from "./manifest.js"; import type { PluginOrigin } from "./plugin-origin.types.js"; +import { loadPluginRegistrySnapshot, type PluginRegistryRecord } from "./plugin-registry.js"; import { resolveManifestProviderAuthChoices, type ProviderAuthChoiceMetadata, @@ -36,6 +35,10 @@ type PreferredInstallSource = { install: PluginPackageInstall; packageName?: string; }; +type PreferredInstallSources = { + installedPluginIds: ReadonlySet; + installsByPluginId: Map; +}; const INSTALL_ORIGIN_PRIORITY: Readonly> = { config: 0, @@ -45,136 +48,190 @@ const INSTALL_ORIGIN_PRIORITY: Readonly> = { }; function isPreferredOrigin(candidate: PluginOrigin, current: PluginOrigin | undefined): boolean { - if (!current) { - return true; + return !current || INSTALL_ORIGIN_PRIORITY[candidate] < INSTALL_ORIGIN_PRIORITY[current]; +} + +function normalizeDefaultChoice(value: unknown): PluginPackageInstall["defaultChoice"] | undefined { + return value === "npm" || value === "local" ? value : undefined; +} + +function resolveInstallInfoFromInstallRecord( + record: InstalledPluginInstallRecordInfo | undefined, +): PluginPackageInstall | null { + if (!record) { + return null; } - return INSTALL_ORIGIN_PRIORITY[candidate] < INSTALL_ORIGIN_PRIORITY[current]; + const npmSpec = (record.resolvedSpec ?? record.spec)?.trim(); + const localPath = (record.installPath ?? record.sourcePath)?.trim(); + if (record.source === "npm" && npmSpec) { + return { + npmSpec, + defaultChoice: "npm", + ...(record.integrity ? { expectedIntegrity: record.integrity } : {}), + }; + } + if (record.source === "path" && localPath) { + return { + localPath, + defaultChoice: "local", + }; + } + return null; } -function resolvePluginManifest( - rootDir: Parameters[0], - rejectHardlinks: boolean, -): Extract | null { - const manifest = loadPluginManifest(rootDir, rejectHardlinks); - return manifest.ok ? manifest : null; -} - -function resolveTrustedNpmSpec(params: { +function resolveInstallInfoFromPackageSource(params: { origin: PluginOrigin; - install?: PluginPackageInstall; -}): string | undefined { - if (params.origin !== "bundled" && params.origin !== "config") { - return undefined; - } - const npmSpec = params.install?.npmSpec?.trim(); - if (!npmSpec) { - return undefined; - } - const parsed = parseRegistryNpmSpec(npmSpec); - return parsed ? npmSpec : undefined; -} - -function resolveInstallInfo(params: { - origin: PluginOrigin; - install?: PluginPackageInstall; - packageDir?: string; - workspaceDir?: string; + source?: PluginInstallSourceInfo; }): PluginPackageInstall | null { - const npmSpec = resolveTrustedNpmSpec({ - origin: params.origin, - install: params.install, - }); - let localPath = params.install?.localPath?.trim(); - if (!localPath && params.workspaceDir && params.packageDir) { - const relative = path.relative(params.workspaceDir, params.packageDir); - localPath = relative || undefined; - } + const npmSpec = + params.origin === "bundled" || params.origin === "config" + ? params.source?.npm?.spec + : undefined; + const localPath = params.source?.local?.path; if (!npmSpec && !localPath) { return null; } - const defaultChoice = - params.install?.defaultChoice ?? (localPath ? "local" : npmSpec ? "npm" : undefined); + const defaultChoice = normalizeDefaultChoice(params.source?.defaultChoice); return { ...(npmSpec ? { npmSpec } : {}), ...(localPath ? { localPath } : {}), - ...(defaultChoice ? { defaultChoice } : {}), - ...(params.install?.minHostVersion ? { minHostVersion: params.install.minHostVersion } : {}), - ...(npmSpec && params.install?.expectedIntegrity - ? { expectedIntegrity: params.install.expectedIntegrity } - : {}), - ...(params.install?.allowInvalidConfigRecovery === true - ? { allowInvalidConfigRecovery: true } + ...(defaultChoice ? { defaultChoice } : npmSpec ? { defaultChoice: "npm" as const } : {}), + ...(npmSpec && params.source?.npm?.expectedIntegrity + ? { expectedIntegrity: params.source.npm.expectedIntegrity } : {}), }; } +function resolveInstallInfoFromRegistryRecord(params: { + record: PluginRegistryRecord; + installRecord?: InstalledPluginInstallRecordInfo; +}): PluginPackageInstall | null { + return ( + resolveInstallInfoFromInstallRecord(params.installRecord) ?? + resolveInstallInfoFromPackageSource({ + origin: params.record.origin, + source: params.record.packageInstall, + }) + ); +} + +function resolveInstallInfoFromProviderIndex( + provider: OpenClawProviderIndexProvider, +): PluginPackageInstall | null { + const install = provider.plugin.install; + const npmSpec = install?.npmSpec?.trim(); + if (!npmSpec) { + return null; + } + const defaultChoice = normalizeDefaultChoice(install.defaultChoice) ?? "npm"; + return { + npmSpec, + defaultChoice, + ...(install.minHostVersion ? { minHostVersion: install.minHostVersion } : {}), + ...(install.expectedIntegrity ? { expectedIntegrity: install.expectedIntegrity } : {}), + }; +} + function resolvePreferredInstallsByPluginId( params: ProviderInstallCatalogParams, -): Map { +): PreferredInstallSources { const preferredByPluginId = new Map(); - const normalizedConfig = normalizePluginsConfig(params.config?.plugins); - for (const candidate of discoverOpenClawPlugins({ + const index = loadPluginRegistrySnapshot({ + config: params.config, workspaceDir: params.workspaceDir, env: params.env, - }).candidates) { - const idHint = candidate.idHint.trim(); - if (candidate.origin === "workspace" && params.includeUntrustedWorkspacePlugins === false) { - if (!idHint) { - continue; - } - if ( - !resolveEffectiveEnableState({ - id: idHint, - origin: candidate.origin, - config: normalizedConfig, - rootConfig: params.config, - }).enabled - ) { - continue; - } - } - const manifest = resolvePluginManifest(candidate.rootDir, candidate.origin !== "bundled"); - if (!manifest) { - continue; - } + }); + const installedPluginIds = new Set(index.plugins.map((record) => record.pluginId)); + const normalizedConfig = normalizePluginsConfig(params.config?.plugins); + for (const record of index.plugins) { if ( - candidate.origin === "workspace" && + record.origin === "workspace" && params.includeUntrustedWorkspacePlugins === false && !resolveEffectiveEnableState({ - id: manifest.manifest.id, - origin: candidate.origin, + id: record.pluginId, + origin: record.origin, config: normalizedConfig, rootConfig: params.config, + enabledByDefault: record.enabledByDefault, }).enabled ) { continue; } - const install = resolveInstallInfo({ - origin: candidate.origin, - install: candidate.packageManifest?.install, - packageDir: candidate.packageDir, - workspaceDir: candidate.workspaceDir, + const install = resolveInstallInfoFromRegistryRecord({ + record, + installRecord: index.installRecords[record.pluginId], }); if (!install) { continue; } - const existing = preferredByPluginId.get(manifest.manifest.id); - if (!existing || isPreferredOrigin(candidate.origin, existing.origin)) { - preferredByPluginId.set(manifest.manifest.id, { - origin: candidate.origin, + const existing = preferredByPluginId.get(record.pluginId); + if (!existing || isPreferredOrigin(record.origin, existing.origin)) { + preferredByPluginId.set(record.pluginId, { + origin: record.origin, install, - ...(candidate.packageName ? { packageName: candidate.packageName } : {}), + ...(record.packageName ? { packageName: record.packageName } : {}), }); } } - return preferredByPluginId; + return { installedPluginIds, installsByPluginId: preferredByPluginId }; +} + +function resolveProviderIndexInstallCatalogEntries(params: { + installedPluginIds: ReadonlySet; + seenChoiceIds: ReadonlySet; +}): ProviderInstallCatalogEntry[] { + const entries: ProviderInstallCatalogEntry[] = []; + const index = loadOpenClawProviderIndex(); + for (const provider of Object.values(index.providers)) { + if (params.installedPluginIds.has(provider.plugin.id)) { + continue; + } + const install = resolveInstallInfoFromProviderIndex(provider); + if (!install) { + continue; + } + for (const choice of provider.authChoices ?? []) { + if (params.seenChoiceIds.has(choice.choiceId)) { + continue; + } + entries.push({ + pluginId: provider.plugin.id, + providerId: provider.id, + methodId: choice.method, + choiceId: choice.choiceId, + choiceLabel: choice.choiceLabel, + ...(choice.choiceHint ? { choiceHint: choice.choiceHint } : {}), + ...(choice.assistantPriority !== undefined + ? { assistantPriority: choice.assistantPriority } + : {}), + ...(choice.assistantVisibility ? { assistantVisibility: choice.assistantVisibility } : {}), + ...(choice.groupId ? { groupId: choice.groupId } : {}), + ...(choice.groupLabel ? { groupLabel: choice.groupLabel } : {}), + ...(choice.groupHint ? { groupHint: choice.groupHint } : {}), + ...(choice.optionKey ? { optionKey: choice.optionKey } : {}), + ...(choice.cliFlag ? { cliFlag: choice.cliFlag } : {}), + ...(choice.cliOption ? { cliOption: choice.cliOption } : {}), + ...(choice.cliDescription ? { cliDescription: choice.cliDescription } : {}), + ...(choice.onboardingScopes ? { onboardingScopes: [...choice.onboardingScopes] } : {}), + label: provider.name, + origin: "bundled", + install, + installSource: describePluginInstallSource(install, { + expectedPackageName: provider.plugin.package, + }), + }); + } + } + return entries; } export function resolveProviderInstallCatalogEntries( params?: ProviderInstallCatalogParams, ): ProviderInstallCatalogEntry[] { - const installsByPluginId = resolvePreferredInstallsByPluginId(params ?? {}); - return resolveManifestProviderAuthChoices(params) + const installParams = params ?? {}; + const { installedPluginIds, installsByPluginId } = + resolvePreferredInstallsByPluginId(installParams); + const manifestEntries = resolveManifestProviderAuthChoices(params) .flatMap((choice) => { const install = installsByPluginId.get(choice.pluginId); if (!install) { @@ -193,6 +250,14 @@ export function resolveProviderInstallCatalogEntries( ]; }) .toSorted((left, right) => left.choiceLabel.localeCompare(right.choiceLabel)); + const seenChoiceIds = new Set(manifestEntries.map((entry) => entry.choiceId)); + const indexEntries = resolveProviderIndexInstallCatalogEntries({ + installedPluginIds, + seenChoiceIds, + }); + return [...manifestEntries, ...indexEntries].toSorted((left, right) => + left.choiceLabel.localeCompare(right.choiceLabel), + ); } export function resolveProviderInstallCatalogEntry(