From 8e5b535d489dc9dc4b206f8fe698bb8cb5ff5fe8 Mon Sep 17 00:00:00 2001 From: Tak Hoffman <781889+Takhoffman@users.noreply.github.com> Date: Fri, 13 Mar 2026 00:57:16 -0500 Subject: [PATCH] refactor: isolate bundled search provider implementations --- extensions/search-brave/index.ts | 12 - extensions/search-brave/openclaw.plugin.json | 4 + extensions/search-brave/package.json | 2 +- extensions/search-brave/src/index.ts | 13 + extensions/search-brave/src/provider.ts | 590 ++++++++++++ extensions/search-gemini/index.ts | 12 - extensions/search-gemini/openclaw.plugin.json | 4 + extensions/search-gemini/package.json | 2 +- extensions/search-gemini/src/index.ts | 13 + extensions/search-gemini/src/provider.ts | 255 ++++++ extensions/search-grok/openclaw.plugin.json | 4 + extensions/search-grok/package.json | 2 +- extensions/search-grok/{ => src}/index.ts | 5 +- extensions/search-grok/src/provider.ts | 266 ++++++ extensions/search-kimi/index.ts | 12 - extensions/search-kimi/openclaw.plugin.json | 4 + extensions/search-kimi/package.json | 2 +- extensions/search-kimi/src/index.ts | 13 + extensions/search-kimi/src/provider.ts | 317 +++++++ .../search-perplexity/openclaw.plugin.json | 4 + extensions/search-perplexity/package.json | 2 +- .../search-perplexity/{ => src}/index.ts | 5 +- extensions/search-perplexity/src/provider.ts | 574 ++++++++++++ package.json | 4 + .../tools/web-search-provider-catalog.ts | 62 ++ src/agents/tools/web-search.ts | 865 ++---------------- .../tools/web-tools.enabled-defaults.test.ts | 71 +- src/commands/onboard-search.test.ts | 1 - src/commands/onboard-search.ts | 261 ++++-- src/plugin-sdk/index.test.ts | 7 +- src/plugin-sdk/index.ts | 1 - src/plugin-sdk/web-search.ts | 218 +++++ src/plugins/types.ts | 23 +- src/secrets/runtime-web-tools.ts | 216 ++--- vitest.config.ts | 1 + 35 files changed, 2821 insertions(+), 1026 deletions(-) delete mode 100644 extensions/search-brave/index.ts create mode 100644 extensions/search-brave/src/index.ts create mode 100644 extensions/search-brave/src/provider.ts delete mode 100644 extensions/search-gemini/index.ts create mode 100644 extensions/search-gemini/src/index.ts create mode 100644 extensions/search-gemini/src/provider.ts rename extensions/search-grok/{ => src}/index.ts (50%) create mode 100644 extensions/search-grok/src/provider.ts delete mode 100644 extensions/search-kimi/index.ts create mode 100644 extensions/search-kimi/src/index.ts create mode 100644 extensions/search-kimi/src/provider.ts rename extensions/search-perplexity/{ => src}/index.ts (50%) create mode 100644 extensions/search-perplexity/src/provider.ts create mode 100644 src/plugin-sdk/web-search.ts diff --git a/extensions/search-brave/index.ts b/extensions/search-brave/index.ts deleted file mode 100644 index aa34520a7c8..00000000000 --- a/extensions/search-brave/index.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { createBundledBuiltinSearchProvider, type OpenClawPluginApi } from "openclaw/plugin-sdk"; - -const plugin = { - id: "search-brave", - name: "Brave Search", - description: "Bundled Brave web search provider for OpenClaw.", - register(api: OpenClawPluginApi) { - api.registerSearchProvider(createBundledBuiltinSearchProvider("brave")); - }, -}; - -export default plugin; diff --git a/extensions/search-brave/openclaw.plugin.json b/extensions/search-brave/openclaw.plugin.json index 060c043872e..457dcab3cfa 100644 --- a/extensions/search-brave/openclaw.plugin.json +++ b/extensions/search-brave/openclaw.plugin.json @@ -1,4 +1,8 @@ { "id": "search-brave", + "configSchema": { + "type": "object", + "properties": {} + }, "provides": ["providers.search.brave"] } diff --git a/extensions/search-brave/package.json b/extensions/search-brave/package.json index 0c3cf7af6fe..c3017ebe3a9 100644 --- a/extensions/search-brave/package.json +++ b/extensions/search-brave/package.json @@ -6,7 +6,7 @@ "type": "module", "openclaw": { "extensions": [ - "./index.ts" + "./src/index.ts" ] } } diff --git a/extensions/search-brave/src/index.ts b/extensions/search-brave/src/index.ts new file mode 100644 index 00000000000..4cc85c07d60 --- /dev/null +++ b/extensions/search-brave/src/index.ts @@ -0,0 +1,13 @@ +import type { OpenClawPluginApi } from "openclaw/plugin-sdk/core"; +import { createBundledBraveSearchProvider } from "./provider.js"; + +const plugin = { + id: "search-brave", + name: "Brave Search", + description: "Bundled Brave web search provider for OpenClaw.", + register(api: OpenClawPluginApi) { + api.registerSearchProvider(createBundledBraveSearchProvider()); + }, +}; + +export default plugin; diff --git a/extensions/search-brave/src/provider.ts b/extensions/search-brave/src/provider.ts new file mode 100644 index 00000000000..cfd851d6585 --- /dev/null +++ b/extensions/search-brave/src/provider.ts @@ -0,0 +1,590 @@ +import { + CacheEntry, + createMissingSearchKeyPayload, + formatCliCommand, + normalizeCacheKey, + normalizeResolvedSecretInputString, + normalizeSecretInput, + readCache, + readResponseText, + readSearchProviderApiKeyValue, + resolveSearchConfig, + resolveSiteName, + type OpenClawConfig, + type SearchProviderContext, + type SearchProviderErrorResult, + type SearchProviderExecutionResult, + type SearchProviderLegacyUiMetadata, + type SearchProviderPlugin, + type SearchProviderRequest, + withTrustedWebToolsEndpoint, + wrapWebContent, + writeCache, +} from "openclaw/plugin-sdk/web-search"; + +const BRAVE_SEARCH_ENDPOINT = "https://api.search.brave.com/res/v1/web/search"; +const BRAVE_LLM_CONTEXT_ENDPOINT = "https://api.search.brave.com/res/v1/llm/context"; + +const BRAVE_SEARCH_CACHE = new Map>>(); +const BRAVE_FRESHNESS_SHORTCUTS = new Set(["pd", "pw", "pm", "py"]); +const BRAVE_FRESHNESS_RANGE = /^(\d{4}-\d{2}-\d{2})to(\d{4}-\d{2}-\d{2})$/; +const BRAVE_SEARCH_LANG_CODES = new Set([ + "ar", + "eu", + "bn", + "bg", + "ca", + "zh-hans", + "zh-hant", + "hr", + "cs", + "da", + "nl", + "en", + "en-gb", + "et", + "fi", + "fr", + "gl", + "de", + "el", + "gu", + "he", + "hi", + "hu", + "is", + "it", + "jp", + "kn", + "ko", + "lv", + "lt", + "ms", + "ml", + "mr", + "nb", + "pl", + "pt-br", + "pt-pt", + "pa", + "ro", + "ru", + "sr", + "sk", + "sl", + "es", + "sv", + "ta", + "te", + "th", + "tr", + "uk", + "vi", +]); +const BRAVE_SEARCH_LANG_ALIASES: Record = { + ja: "jp", + zh: "zh-hans", + "zh-cn": "zh-hans", + "zh-hk": "zh-hant", + "zh-sg": "zh-hans", + "zh-tw": "zh-hant", +}; +const BRAVE_UI_LANG_LOCALE = /^([a-z]{2})-([a-z]{2})$/i; + +type BraveSearchResult = { + title?: string; + url?: string; + description?: string; + age?: string; +}; + +type BraveSearchResponse = { + web?: { + results?: BraveSearchResult[]; + }; +}; + +type BraveLlmContextResult = { url: string; title: string; snippets: string[] }; +type BraveLlmContextResponse = { + grounding: { generic?: BraveLlmContextResult[] }; + sources?: { url?: string; hostname?: string; date?: string }[]; +}; + +type BraveConfig = { + mode?: string; +}; + +type WebSearchConfig = NonNullable["web"] extends infer Web + ? Web extends { search?: infer Search } + ? Search + : undefined + : undefined; + +function resolveBraveConfig(search?: WebSearchConfig): BraveConfig { + if (!search || typeof search !== "object") { + return {}; + } + const brave = "brave" in search ? search.brave : undefined; + if (!brave || typeof brave !== "object") { + return {}; + } + return brave as BraveConfig; +} + +function resolveBraveMode(brave: BraveConfig): "web" | "llm-context" { + return brave.mode === "llm-context" ? "llm-context" : "web"; +} + +function resolveBraveApiKey(search?: WebSearchConfig): string | undefined { + const fromConfigRaw = search + ? normalizeResolvedSecretInputString({ + value: readSearchProviderApiKeyValue(search as Record, "brave"), + path: "tools.web.search.apiKey", + }) + : undefined; + const fromConfig = normalizeSecretInput(fromConfigRaw); + const fromEnv = normalizeSecretInput(process.env.BRAVE_API_KEY); + return fromConfig || fromEnv || undefined; +} + +function normalizeBraveSearchLang(value: string | undefined): string | undefined { + if (!value) { + return undefined; + } + const trimmed = value.trim(); + if (!trimmed) { + return undefined; + } + const canonical = BRAVE_SEARCH_LANG_ALIASES[trimmed.toLowerCase()] ?? trimmed.toLowerCase(); + if (!BRAVE_SEARCH_LANG_CODES.has(canonical)) { + return undefined; + } + return canonical; +} + +function normalizeBraveUiLang(value: string | undefined): string | undefined { + if (!value) { + return undefined; + } + const trimmed = value.trim(); + if (!trimmed) { + return undefined; + } + const match = trimmed.match(BRAVE_UI_LANG_LOCALE); + if (!match) { + return undefined; + } + const [, language, region] = match; + return `${language.toLowerCase()}-${region.toUpperCase()}`; +} + +function normalizeBraveLanguageParams(params: { search_lang?: string; ui_lang?: string }): { + search_lang?: string; + ui_lang?: string; + invalidField?: "search_lang" | "ui_lang"; +} { + const rawSearchLang = params.search_lang?.trim() || undefined; + const rawUiLang = params.ui_lang?.trim() || undefined; + let searchLangCandidate = rawSearchLang; + let uiLangCandidate = rawUiLang; + + if (normalizeBraveUiLang(rawSearchLang) && normalizeBraveSearchLang(rawUiLang)) { + searchLangCandidate = rawUiLang; + uiLangCandidate = rawSearchLang; + } + + const search_lang = normalizeBraveSearchLang(searchLangCandidate); + if (searchLangCandidate && !search_lang) { + return { invalidField: "search_lang" }; + } + + const ui_lang = normalizeBraveUiLang(uiLangCandidate); + if (uiLangCandidate && !ui_lang) { + return { invalidField: "ui_lang" }; + } + + return { search_lang, ui_lang }; +} + +function isValidIsoDate(value: string): boolean { + if (!/^\d{4}-\d{2}-\d{2}$/.test(value)) { + return false; + } + const [year, month, day] = value.split("-").map((part) => Number.parseInt(part, 10)); + if (!Number.isFinite(year) || !Number.isFinite(month) || !Number.isFinite(day)) { + return false; + } + const date = new Date(Date.UTC(year, month - 1, day)); + return ( + date.getUTCFullYear() === year && date.getUTCMonth() === month - 1 && date.getUTCDate() === day + ); +} + +function normalizeFreshness(value: string | undefined): string | undefined { + if (!value) { + return undefined; + } + const trimmed = value.trim(); + if (!trimmed) { + return undefined; + } + const lower = trimmed.toLowerCase(); + if (BRAVE_FRESHNESS_SHORTCUTS.has(lower)) { + return lower; + } + const match = trimmed.match(BRAVE_FRESHNESS_RANGE); + if (match) { + const [, start, end] = match; + if (isValidIsoDate(start) && isValidIsoDate(end) && start <= end) { + return `${start}to${end}`; + } + } + return undefined; +} + +function buildBraveCacheIdentity(params: { + query: string; + count: number; + country?: string; + search_lang?: string; + ui_lang?: string; + freshness?: string; + dateAfter?: string; + dateBefore?: string; + braveMode: "web" | "llm-context"; +}): string { + return [ + params.query, + params.count, + params.country || "default", + params.search_lang || "default", + params.ui_lang || "default", + params.freshness || "default", + params.dateAfter || "default", + params.dateBefore || "default", + params.braveMode, + ].join(":"); +} + +async function throwBraveApiError(res: Response, label: string): Promise { + const detailResult = await readResponseText(res, { maxBytes: 64_000 }); + const detail = detailResult.text; + throw new Error(`${label} API error (${res.status}): ${detail || res.statusText}`); +} + +function mapBraveLlmContextResults( + data: BraveLlmContextResponse, +): { url: string; title: string; snippets: string[]; siteName?: string }[] { + const genericResults = Array.isArray(data.grounding?.generic) ? data.grounding.generic : []; + return genericResults.map((entry) => ({ + url: entry.url ?? "", + title: entry.title ?? "", + snippets: (entry.snippets ?? []).filter((s) => typeof s === "string" && s.length > 0), + siteName: resolveSiteName(entry.url) || undefined, + })); +} + +async function runBraveLlmContextSearch(params: { + query: string; + apiKey: string; + timeoutSeconds: number; + country?: string; + search_lang?: string; + freshness?: string; +}) { + const url = new URL(BRAVE_LLM_CONTEXT_ENDPOINT); + url.searchParams.set("q", params.query); + if (params.country) { + url.searchParams.set("country", params.country); + } + if (params.search_lang) { + url.searchParams.set("search_lang", params.search_lang); + } + if (params.freshness) { + url.searchParams.set("freshness", params.freshness); + } + + return withTrustedWebToolsEndpoint( + { + url: url.toString(), + timeoutSeconds: params.timeoutSeconds, + init: { + method: "GET", + headers: { + Accept: "application/json", + "X-Subscription-Token": params.apiKey, + }, + }, + }, + async ({ response }) => { + if (!response.ok) { + return await throwBraveApiError(response, "Brave LLM Context"); + } + const data = (await response.json()) as BraveLlmContextResponse; + return { results: mapBraveLlmContextResults(data), sources: data.sources }; + }, + ); +} + +async function runBraveWebSearch(params: { + query: string; + count: number; + apiKey: string; + timeoutSeconds: number; + country?: string; + search_lang?: string; + ui_lang?: string; + freshness?: string; + dateAfter?: string; + dateBefore?: string; +}) { + const url = new URL(BRAVE_SEARCH_ENDPOINT); + url.searchParams.set("q", params.query); + url.searchParams.set("count", String(params.count)); + if (params.country) { + url.searchParams.set("country", params.country); + } + if (params.search_lang) { + url.searchParams.set("search_lang", params.search_lang); + } + if (params.ui_lang) { + url.searchParams.set("ui_lang", params.ui_lang); + } + if (params.freshness) { + url.searchParams.set("freshness", params.freshness); + } else if (params.dateAfter && params.dateBefore) { + url.searchParams.set("freshness", `${params.dateAfter}to${params.dateBefore}`); + } else if (params.dateAfter) { + url.searchParams.set( + "freshness", + `${params.dateAfter}to${new Date().toISOString().slice(0, 10)}`, + ); + } else if (params.dateBefore) { + url.searchParams.set("freshness", `1970-01-01to${params.dateBefore}`); + } + + return withTrustedWebToolsEndpoint( + { + url: url.toString(), + timeoutSeconds: params.timeoutSeconds, + init: { + method: "GET", + headers: { + Accept: "application/json", + "X-Subscription-Token": params.apiKey, + }, + }, + }, + async ({ response }) => { + if (!response.ok) { + return await throwBraveApiError(response, "Brave Search"); + } + const data = (await response.json()) as BraveSearchResponse; + const results = Array.isArray(data.web?.results) ? (data.web?.results ?? []) : []; + return results.map((entry) => { + const description = entry.description ?? ""; + const title = entry.title ?? ""; + const url = entry.url ?? ""; + return { + title: title ? wrapWebContent(title, "web_search") : "", + url, + description: description ? wrapWebContent(description, "web_search") : "", + published: entry.age || undefined, + siteName: resolveSiteName(url) || undefined, + }; + }); + }, + ); +} + +export const BRAVE_SEARCH_PROVIDER_METADATA: SearchProviderLegacyUiMetadata = { + label: "Brave Search", + hint: "Structured results · country/language/time filters", + envKeys: ["BRAVE_API_KEY"], + placeholder: "BSA...", + signupUrl: "https://brave.com/search/api/", + apiKeyConfigPath: "tools.web.search.apiKey", + readApiKeyValue: (search) => readSearchProviderApiKeyValue(search, "brave"), + writeApiKeyValue: (search, value) => void ((search.apiKey = value) as unknown), +}; + +export function createBundledBraveSearchProvider(): SearchProviderPlugin { + return { + id: "brave", + name: BRAVE_SEARCH_PROVIDER_METADATA.label, + description: + "Search the web using Brave Search. Supports web and llm-context modes, region-specific search, and localized search parameters.", + pluginOwnedExecution: true, + docsUrl: BRAVE_SEARCH_PROVIDER_METADATA.signupUrl, + legacyConfig: BRAVE_SEARCH_PROVIDER_METADATA, + isAvailable: (config) => { + const search = config?.tools?.web?.search; + return Boolean( + resolveBraveApiKey(resolveSearchConfig(search as Record)), + ); + }, + search: async (request, ctx): Promise => { + const search = resolveSearchConfig(request.providerConfig); + const braveConfig = resolveBraveConfig(search); + const braveMode = resolveBraveMode(braveConfig); + const apiKey = resolveBraveApiKey(search); + + if (!apiKey) { + return createMissingSearchKeyPayload( + "missing_brave_api_key", + `web_search (brave) needs a Brave Search API key. Run \`${formatCliCommand("openclaw configure --section web")}\` to store it, or set BRAVE_API_KEY in the Gateway environment.`, + ); + } + + const normalizedLanguageParams = normalizeBraveLanguageParams({ + search_lang: request.search_lang || request.language, + ui_lang: request.ui_lang, + }); + if (normalizedLanguageParams.invalidField === "search_lang") { + return { + error: "invalid_search_lang", + message: + "search_lang must be a Brave-supported language code like 'en', 'en-gb', 'zh-hans', or 'zh-hant'.", + docs: "https://docs.openclaw.ai/tools/web", + }; + } + if (normalizedLanguageParams.invalidField === "ui_lang") { + return { + error: "invalid_ui_lang", + message: "ui_lang must be a language-region locale like 'en-US'.", + docs: "https://docs.openclaw.ai/tools/web", + }; + } + if (normalizedLanguageParams.ui_lang && braveMode === "llm-context") { + return { + error: "unsupported_ui_lang", + message: + "ui_lang is not supported by Brave llm-context mode. Remove ui_lang or use Brave web mode for locale-based UI hints.", + docs: "https://docs.openclaw.ai/tools/web", + }; + } + if (request.freshness && braveMode === "llm-context") { + return { + error: "unsupported_freshness", + message: + "freshness filtering is not supported by Brave llm-context mode. Remove freshness or use Brave web mode.", + docs: "https://docs.openclaw.ai/tools/web", + }; + } + const normalizedFreshness = request.freshness + ? normalizeFreshness(request.freshness) + : undefined; + if (request.freshness && !normalizedFreshness) { + return { + error: "invalid_freshness", + message: "freshness must be day, week, month, or year.", + docs: "https://docs.openclaw.ai/tools/web", + }; + } + if ((request.dateAfter || request.dateBefore) && braveMode === "llm-context") { + return { + error: "unsupported_date_filter", + message: + "date_after/date_before filtering is not supported by Brave llm-context mode. Use Brave web mode for date filters.", + docs: "https://docs.openclaw.ai/tools/web", + }; + } + + const cacheKey = normalizeCacheKey( + `brave:${buildBraveCacheIdentity({ + query: request.query, + count: request.count, + country: request.country, + search_lang: normalizedLanguageParams.search_lang, + ui_lang: normalizedLanguageParams.ui_lang, + freshness: normalizedFreshness, + dateAfter: request.dateAfter, + dateBefore: request.dateBefore, + braveMode, + })}`, + ); + const cached = readCache(BRAVE_SEARCH_CACHE, cacheKey); + if (cached) { + return { ...cached.value, cached: true } as Record< + string, + unknown + > as SearchProviderExecutionResult; + } + + const startedAt = Date.now(); + if (braveMode === "llm-context") { + const { results, sources } = await runBraveLlmContextSearch({ + query: request.query, + apiKey, + timeoutSeconds: ctx.timeoutSeconds, + country: request.country, + search_lang: normalizedLanguageParams.search_lang, + freshness: normalizedFreshness, + }); + const mappedResults = results.map( + (entry: { title: string; url: string; snippets: string[]; siteName?: string }) => ({ + title: entry.title ? wrapWebContent(entry.title, "web_search") : "", + url: entry.url, + snippets: entry.snippets.map((s: string) => wrapWebContent(s, "web_search")), + siteName: entry.siteName, + }), + ); + const payload = { + query: request.query, + provider: "brave", + mode: "llm-context" as const, + count: mappedResults.length, + tookMs: Date.now() - startedAt, + externalContent: { + untrusted: true, + source: "web_search", + provider: "brave", + wrapped: true, + }, + results: mappedResults, + sources, + }; + writeCache(BRAVE_SEARCH_CACHE, cacheKey, payload, ctx.cacheTtlMs); + return payload as Record as SearchProviderExecutionResult; + } + + const results = await runBraveWebSearch({ + query: request.query, + count: request.count, + apiKey, + timeoutSeconds: ctx.timeoutSeconds, + country: request.country, + search_lang: normalizedLanguageParams.search_lang, + ui_lang: normalizedLanguageParams.ui_lang, + freshness: normalizedFreshness, + dateAfter: request.dateAfter, + dateBefore: request.dateBefore, + }); + const payload = { + query: request.query, + provider: "brave", + count: results.length, + tookMs: Date.now() - startedAt, + externalContent: { + untrusted: true, + source: "web_search", + provider: "brave", + wrapped: true, + }, + results, + }; + writeCache(BRAVE_SEARCH_CACHE, cacheKey, payload, ctx.cacheTtlMs); + return payload as Record as SearchProviderExecutionResult; + }, + }; +} + +export const __testing = { + resolveBraveApiKey, + resolveBraveMode, + normalizeBraveLanguageParams, + normalizeFreshness, + clearSearchProviderCaches() { + BRAVE_SEARCH_CACHE.clear(); + }, +}; diff --git a/extensions/search-gemini/index.ts b/extensions/search-gemini/index.ts deleted file mode 100644 index c55b8a04e2d..00000000000 --- a/extensions/search-gemini/index.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { createBundledBuiltinSearchProvider, type OpenClawPluginApi } from "openclaw/plugin-sdk"; - -const plugin = { - id: "search-gemini", - name: "Gemini Search", - description: "Bundled Gemini web search provider for OpenClaw.", - register(api: OpenClawPluginApi) { - api.registerSearchProvider(createBundledBuiltinSearchProvider("gemini")); - }, -}; - -export default plugin; diff --git a/extensions/search-gemini/openclaw.plugin.json b/extensions/search-gemini/openclaw.plugin.json index a774390b13a..2f08b7a333c 100644 --- a/extensions/search-gemini/openclaw.plugin.json +++ b/extensions/search-gemini/openclaw.plugin.json @@ -1,4 +1,8 @@ { "id": "search-gemini", + "configSchema": { + "type": "object", + "properties": {} + }, "provides": ["providers.search.gemini"] } diff --git a/extensions/search-gemini/package.json b/extensions/search-gemini/package.json index 8f03543e77b..0749797bd00 100644 --- a/extensions/search-gemini/package.json +++ b/extensions/search-gemini/package.json @@ -6,7 +6,7 @@ "type": "module", "openclaw": { "extensions": [ - "./index.ts" + "./src/index.ts" ] } } diff --git a/extensions/search-gemini/src/index.ts b/extensions/search-gemini/src/index.ts new file mode 100644 index 00000000000..81217d14af6 --- /dev/null +++ b/extensions/search-gemini/src/index.ts @@ -0,0 +1,13 @@ +import type { OpenClawPluginApi } from "openclaw/plugin-sdk/core"; +import { createBundledGeminiSearchProvider } from "./provider.js"; + +const plugin = { + id: "search-gemini", + name: "Gemini Search", + description: "Bundled Gemini web search provider for OpenClaw.", + register(api: OpenClawPluginApi) { + api.registerSearchProvider(createBundledGeminiSearchProvider()); + }, +}; + +export default plugin; diff --git a/extensions/search-gemini/src/provider.ts b/extensions/search-gemini/src/provider.ts new file mode 100644 index 00000000000..e648e749360 --- /dev/null +++ b/extensions/search-gemini/src/provider.ts @@ -0,0 +1,255 @@ +import { + buildSearchRequestCacheIdentity, + createMissingSearchKeyPayload, + normalizeCacheKey, + normalizeSecretInput, + readCache, + readResponseText, + readSearchProviderApiKeyValue, + rejectUnsupportedSearchFilters, + resolveCitationRedirectUrl, + resolveSearchConfig, + type OpenClawConfig, + type SearchProviderExecutionResult, + type SearchProviderLegacyUiMetadata, + type SearchProviderPlugin, + withTrustedWebToolsEndpoint, + wrapWebContent, + writeCache, + writeSearchProviderApiKeyValue, +} from "openclaw/plugin-sdk/web-search"; + +const DEFAULT_GEMINI_MODEL = "gemini-2.5-flash"; +const GEMINI_API_BASE = "https://generativelanguage.googleapis.com/v1beta"; + +const GEMINI_SEARCH_CACHE = new Map< + string, + { value: Record; expiresAt: number } +>(); + +type WebSearchConfig = NonNullable["web"] extends infer Web + ? Web extends { search?: infer Search } + ? Search + : undefined + : undefined; + +type GeminiConfig = { + apiKey?: string; + model?: string; +}; + +type GeminiGroundingResponse = { + candidates?: Array<{ + content?: { + parts?: Array<{ + text?: string; + }>; + }; + groundingMetadata?: { + groundingChunks?: Array<{ + web?: { + uri?: string; + title?: string; + }; + }>; + }; + }>; + error?: { + code?: number; + message?: string; + status?: string; + }; +}; + +function resolveGeminiConfig(search?: WebSearchConfig): GeminiConfig { + if (!search || typeof search !== "object") return {}; + const gemini = "gemini" in search ? search.gemini : undefined; + if (!gemini || typeof gemini !== "object") return {}; + return gemini as GeminiConfig; +} + +function resolveGeminiApiKey(gemini?: GeminiConfig): string | undefined { + return ( + normalizeSecretInput(gemini?.apiKey) || + normalizeSecretInput(process.env.GEMINI_API_KEY) || + undefined + ); +} + +function resolveGeminiModel(gemini?: GeminiConfig): string { + const fromConfig = + gemini && "model" in gemini && typeof gemini.model === "string" ? gemini.model.trim() : ""; + return fromConfig || DEFAULT_GEMINI_MODEL; +} + +async function runGeminiSearch(params: { + query: string; + apiKey: string; + model: string; + timeoutSeconds: number; +}): Promise<{ content: string; citations: Array<{ url: string; title?: string }> }> { + const endpoint = `${GEMINI_API_BASE}/models/${params.model}:generateContent`; + return withTrustedWebToolsEndpoint( + { + url: endpoint, + timeoutSeconds: params.timeoutSeconds, + init: { + method: "POST", + headers: { + "Content-Type": "application/json", + "x-goog-api-key": params.apiKey, + }, + body: JSON.stringify({ + contents: [{ parts: [{ text: params.query }] }], + tools: [{ google_search: {} }], + }), + }, + }, + async ({ response }) => { + if (!response.ok) { + const detailResult = await readResponseText(response, { maxBytes: 64_000 }); + const safeDetail = (detailResult.text || response.statusText).replace( + /key=[^&\s]+/gi, + "key=***", + ); + throw new Error(`Gemini API error (${response.status}): ${safeDetail}`); + } + let data: GeminiGroundingResponse; + try { + data = (await response.json()) as GeminiGroundingResponse; + } catch (err) { + const safeError = String(err).replace(/key=[^&\s]+/gi, "key=***"); + throw new Error(`Gemini API returned invalid JSON: ${safeError}`, { cause: err }); + } + if (data.error) { + const rawMsg = data.error.message || data.error.status || "unknown"; + const safeMsg = rawMsg.replace(/key=[^&\s]+/gi, "key=***"); + throw new Error(`Gemini API error (${data.error.code}): ${safeMsg}`); + } + const candidate = data.candidates?.[0]; + const content = + candidate?.content?.parts + ?.map((p) => p.text) + .filter(Boolean) + .join("\n") ?? "No response"; + const groundingChunks = candidate?.groundingMetadata?.groundingChunks ?? []; + const rawCitations = groundingChunks + .filter((chunk) => chunk.web?.uri) + .map((chunk) => ({ + url: chunk.web!.uri!, + title: chunk.web?.title || undefined, + })); + const citations: Array<{ url: string; title?: string }> = []; + const MAX_CONCURRENT_REDIRECTS = 10; + for (let i = 0; i < rawCitations.length; i += MAX_CONCURRENT_REDIRECTS) { + const batch = rawCitations.slice(i, i + MAX_CONCURRENT_REDIRECTS); + const resolved = await Promise.all( + batch.map(async (citation) => ({ + ...citation, + url: await resolveCitationRedirectUrl(citation.url), + })), + ); + citations.push(...resolved); + } + return { content, citations }; + }, + ); +} + +export const GEMINI_SEARCH_PROVIDER_METADATA: SearchProviderLegacyUiMetadata = { + label: "Gemini (Google Search)", + hint: "Google Search grounding · AI-synthesized", + envKeys: ["GEMINI_API_KEY"], + placeholder: "AIza...", + signupUrl: "https://aistudio.google.com/apikey", + apiKeyConfigPath: "tools.web.search.gemini.apiKey", + readApiKeyValue: (search) => readSearchProviderApiKeyValue(search, "gemini"), + writeApiKeyValue: (search, value) => + writeSearchProviderApiKeyValue({ search, provider: "gemini", value }), +}; + +export function createBundledGeminiSearchProvider(): SearchProviderPlugin { + return { + id: "gemini", + name: "Gemini Search", + description: + "Search the web using Gemini with Google Search grounding. Returns AI-synthesized answers with citations from Google Search.", + pluginOwnedExecution: true, + legacyConfig: GEMINI_SEARCH_PROVIDER_METADATA, + isAvailable: (config) => + Boolean( + resolveGeminiApiKey( + resolveGeminiConfig( + resolveSearchConfig( + config?.tools?.web?.search as Record, + ), + ), + ), + ), + search: async (request, ctx): Promise => { + const search = resolveSearchConfig(request.providerConfig); + const geminiConfig = resolveGeminiConfig(search); + const apiKey = resolveGeminiApiKey(geminiConfig); + if (!apiKey) { + return createMissingSearchKeyPayload( + "missing_gemini_api_key", + "web_search (gemini) needs an API key. Set GEMINI_API_KEY in the Gateway environment, or configure tools.web.search.gemini.apiKey.", + ); + } + const unsupportedFilter = rejectUnsupportedSearchFilters({ + providerName: "gemini", + request, + support: { + country: false, + language: false, + freshness: false, + date: false, + domainFilter: false, + }, + }); + if (unsupportedFilter) { + return unsupportedFilter; + } + + const model = resolveGeminiModel(geminiConfig); + const cacheKey = normalizeCacheKey( + `gemini:${model}:${buildSearchRequestCacheIdentity({ + query: request.query, + count: request.count, + })}`, + ); + const cached = readCache(GEMINI_SEARCH_CACHE, cacheKey); + if (cached) return { ...cached.value, cached: true } as SearchProviderExecutionResult; + const startedAt = Date.now(); + const result = await runGeminiSearch({ + query: request.query, + apiKey, + model, + timeoutSeconds: ctx.timeoutSeconds, + }); + const payload = { + query: request.query, + provider: "gemini", + model, + tookMs: Date.now() - startedAt, + externalContent: { + untrusted: true, + source: "web_search", + provider: "gemini", + wrapped: true, + }, + content: wrapWebContent(result.content), + citations: result.citations, + }; + writeCache(GEMINI_SEARCH_CACHE, cacheKey, payload, ctx.cacheTtlMs); + return payload as SearchProviderExecutionResult; + }, + }; +} + +export const __testing = { + GEMINI_SEARCH_CACHE, + clearSearchProviderCaches() { + GEMINI_SEARCH_CACHE.clear(); + }, +} as const; diff --git a/extensions/search-grok/openclaw.plugin.json b/extensions/search-grok/openclaw.plugin.json index f28e593ec25..e6b9706cd7d 100644 --- a/extensions/search-grok/openclaw.plugin.json +++ b/extensions/search-grok/openclaw.plugin.json @@ -1,4 +1,8 @@ { "id": "search-grok", + "configSchema": { + "type": "object", + "properties": {} + }, "provides": ["providers.search.grok"] } diff --git a/extensions/search-grok/package.json b/extensions/search-grok/package.json index ace1527f22c..7e8a28af810 100644 --- a/extensions/search-grok/package.json +++ b/extensions/search-grok/package.json @@ -6,7 +6,7 @@ "type": "module", "openclaw": { "extensions": [ - "./index.ts" + "./src/index.ts" ] } } diff --git a/extensions/search-grok/index.ts b/extensions/search-grok/src/index.ts similarity index 50% rename from extensions/search-grok/index.ts rename to extensions/search-grok/src/index.ts index 88802da23c2..19d9f41dbde 100644 --- a/extensions/search-grok/index.ts +++ b/extensions/search-grok/src/index.ts @@ -1,11 +1,12 @@ -import { createBundledBuiltinSearchProvider, type OpenClawPluginApi } from "openclaw/plugin-sdk"; +import type { OpenClawPluginApi } from "openclaw/plugin-sdk/core"; +import { createBundledGrokSearchProvider } from "./provider.js"; const plugin = { id: "search-grok", name: "Grok Search", description: "Bundled xAI Grok web search provider for OpenClaw.", register(api: OpenClawPluginApi) { - api.registerSearchProvider(createBundledBuiltinSearchProvider("grok")); + api.registerSearchProvider(createBundledGrokSearchProvider()); }, }; diff --git a/extensions/search-grok/src/provider.ts b/extensions/search-grok/src/provider.ts new file mode 100644 index 00000000000..028aee2e11a --- /dev/null +++ b/extensions/search-grok/src/provider.ts @@ -0,0 +1,266 @@ +import { + buildSearchRequestCacheIdentity, + createMissingSearchKeyPayload, + normalizeCacheKey, + normalizeSecretInput, + readCache, + readSearchProviderApiKeyValue, + rejectUnsupportedSearchFilters, + resolveSearchConfig, + throwWebSearchApiError, + type OpenClawConfig, + type SearchProviderExecutionResult, + type SearchProviderLegacyUiMetadata, + type SearchProviderPlugin, + withTrustedWebToolsEndpoint, + wrapWebContent, + writeCache, + writeSearchProviderApiKeyValue, +} from "openclaw/plugin-sdk/web-search"; + +const XAI_API_ENDPOINT = "https://api.x.ai/v1/responses"; +const DEFAULT_GROK_MODEL = "grok-4-1-fast"; + +const GROK_SEARCH_CACHE = new Map; expiresAt: number }>(); + +type WebSearchConfig = NonNullable["web"] extends infer Web + ? Web extends { search?: infer Search } + ? Search + : undefined + : undefined; + +type GrokConfig = { + apiKey?: string; + model?: string; + inlineCitations?: boolean; +}; + +type GrokSearchResponse = { + output?: Array<{ + type?: string; + role?: string; + text?: string; + content?: Array<{ + type?: string; + text?: string; + annotations?: Array<{ + type?: string; + url?: string; + start_index?: number; + end_index?: number; + }>; + }>; + annotations?: Array<{ + type?: string; + url?: string; + start_index?: number; + end_index?: number; + }>; + }>; + output_text?: string; + citations?: string[]; + inline_citations?: Array<{ + start_index: number; + end_index: number; + url: string; + }>; +}; + +function resolveGrokConfig(search?: WebSearchConfig): GrokConfig { + if (!search || typeof search !== "object") return {}; + const grok = "grok" in search ? search.grok : undefined; + if (!grok || typeof grok !== "object") return {}; + return grok as GrokConfig; +} + +function resolveGrokApiKey(grok?: GrokConfig): string | undefined { + return ( + normalizeSecretInput(grok?.apiKey) || normalizeSecretInput(process.env.XAI_API_KEY) || undefined + ); +} + +function resolveGrokModel(grok?: GrokConfig): string { + const fromConfig = + grok && "model" in grok && typeof grok.model === "string" ? grok.model.trim() : ""; + return fromConfig || DEFAULT_GROK_MODEL; +} + +function resolveGrokInlineCitations(grok?: GrokConfig): boolean { + return grok?.inlineCitations === true; +} + +function extractGrokContent(data: GrokSearchResponse): { + text: string | undefined; + annotationCitations: string[]; +} { + for (const output of data.output ?? []) { + if (output.type === "message") { + for (const block of output.content ?? []) { + if (block.type === "output_text" && typeof block.text === "string" && block.text) { + const urls = (block.annotations ?? []) + .filter((a) => a.type === "url_citation" && typeof a.url === "string") + .map((a) => a.url as string); + return { text: block.text, annotationCitations: [...new Set(urls)] }; + } + } + } + if ( + output.type === "output_text" && + "text" in output && + typeof output.text === "string" && + output.text + ) { + const rawAnnotations = + "annotations" in output && Array.isArray(output.annotations) ? output.annotations : []; + const urls = rawAnnotations + .filter( + (a: Record) => a.type === "url_citation" && typeof a.url === "string", + ) + .map((a: Record) => a.url as string); + return { text: output.text, annotationCitations: [...new Set(urls)] }; + } + } + const text = typeof data.output_text === "string" ? data.output_text : undefined; + return { text, annotationCitations: [] }; +} + +async function runGrokSearch(params: { + query: string; + apiKey: string; + model: string; + timeoutSeconds: number; + inlineCitations: boolean; +}) { + const body: Record = { + model: params.model, + input: [{ role: "user", content: params.query }], + tools: [{ type: "web_search" }], + }; + return withTrustedWebToolsEndpoint( + { + url: XAI_API_ENDPOINT, + timeoutSeconds: params.timeoutSeconds, + init: { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${params.apiKey}`, + }, + body: JSON.stringify(body), + }, + }, + async ({ response }) => { + if (!response.ok) { + return await throwWebSearchApiError(response, "xAI"); + } + const data = (await response.json()) as GrokSearchResponse; + const { text: extractedText, annotationCitations } = extractGrokContent(data); + return { + content: extractedText ?? "No response", + citations: (data.citations ?? []).length > 0 ? data.citations! : annotationCitations, + inlineCitations: data.inline_citations, + }; + }, + ); +} + +export const GROK_SEARCH_PROVIDER_METADATA: SearchProviderLegacyUiMetadata = { + label: "Grok (xAI)", + hint: "xAI web-grounded responses", + envKeys: ["XAI_API_KEY"], + placeholder: "xai-...", + signupUrl: "https://console.x.ai/", + apiKeyConfigPath: "tools.web.search.grok.apiKey", + readApiKeyValue: (search) => readSearchProviderApiKeyValue(search, "grok"), + writeApiKeyValue: (search, value) => + writeSearchProviderApiKeyValue({ search, provider: "grok", value }), +}; + +export function createBundledGrokSearchProvider(): SearchProviderPlugin { + return { + id: "grok", + name: "xAI Grok", + description: + "Search the web using xAI Grok. Returns AI-synthesized answers with citations from real-time web search.", + pluginOwnedExecution: true, + legacyConfig: GROK_SEARCH_PROVIDER_METADATA, + isAvailable: (config) => + Boolean( + resolveGrokApiKey( + resolveGrokConfig( + resolveSearchConfig( + config?.tools?.web?.search as Record, + ), + ), + ), + ), + search: async (request, ctx): Promise => { + const search = resolveSearchConfig(request.providerConfig); + const grokConfig = resolveGrokConfig(search); + const apiKey = resolveGrokApiKey(grokConfig); + if (!apiKey) { + return createMissingSearchKeyPayload( + "missing_xai_api_key", + "web_search (grok) needs an xAI API key. Set XAI_API_KEY in the Gateway environment, or configure tools.web.search.grok.apiKey.", + ); + } + const unsupportedFilter = rejectUnsupportedSearchFilters({ + providerName: "grok", + request, + support: { + country: false, + language: false, + freshness: false, + date: false, + domainFilter: false, + }, + }); + if (unsupportedFilter) { + return unsupportedFilter; + } + + const model = resolveGrokModel(grokConfig); + const inlineCitationsEnabled = resolveGrokInlineCitations(grokConfig); + const cacheKey = normalizeCacheKey( + `grok:${model}:${String(inlineCitationsEnabled)}:${buildSearchRequestCacheIdentity({ + query: request.query, + count: request.count, + })}`, + ); + const cached = readCache(GROK_SEARCH_CACHE, cacheKey); + if (cached) return { ...cached.value, cached: true } as SearchProviderExecutionResult; + const startedAt = Date.now(); + const result = await runGrokSearch({ + query: request.query, + apiKey, + model, + timeoutSeconds: ctx.timeoutSeconds, + inlineCitations: inlineCitationsEnabled, + }); + const payload = { + query: request.query, + provider: "grok", + model, + tookMs: Date.now() - startedAt, + externalContent: { + untrusted: true, + source: "web_search", + provider: "grok", + wrapped: true, + }, + content: wrapWebContent(result.content), + citations: result.citations, + inlineCitations: result.inlineCitations, + }; + writeCache(GROK_SEARCH_CACHE, cacheKey, payload, ctx.cacheTtlMs); + return payload as SearchProviderExecutionResult; + }, + }; +} + +export const __testing = { + GROK_SEARCH_CACHE, + clearSearchProviderCaches() { + GROK_SEARCH_CACHE.clear(); + }, +} as const; diff --git a/extensions/search-kimi/index.ts b/extensions/search-kimi/index.ts deleted file mode 100644 index 908f8704bfd..00000000000 --- a/extensions/search-kimi/index.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { createBundledBuiltinSearchProvider, type OpenClawPluginApi } from "openclaw/plugin-sdk"; - -const plugin = { - id: "search-kimi", - name: "Kimi Search", - description: "Bundled Kimi web search provider for OpenClaw.", - register(api: OpenClawPluginApi) { - api.registerSearchProvider(createBundledBuiltinSearchProvider("kimi")); - }, -}; - -export default plugin; diff --git a/extensions/search-kimi/openclaw.plugin.json b/extensions/search-kimi/openclaw.plugin.json index 5811bfdbeff..86f27a5c4b1 100644 --- a/extensions/search-kimi/openclaw.plugin.json +++ b/extensions/search-kimi/openclaw.plugin.json @@ -1,4 +1,8 @@ { "id": "search-kimi", + "configSchema": { + "type": "object", + "properties": {} + }, "provides": ["providers.search.kimi"] } diff --git a/extensions/search-kimi/package.json b/extensions/search-kimi/package.json index 1a19b0ad14f..19e51871443 100644 --- a/extensions/search-kimi/package.json +++ b/extensions/search-kimi/package.json @@ -6,7 +6,7 @@ "type": "module", "openclaw": { "extensions": [ - "./index.ts" + "./src/index.ts" ] } } diff --git a/extensions/search-kimi/src/index.ts b/extensions/search-kimi/src/index.ts new file mode 100644 index 00000000000..69f6e30eabb --- /dev/null +++ b/extensions/search-kimi/src/index.ts @@ -0,0 +1,13 @@ +import type { OpenClawPluginApi } from "openclaw/plugin-sdk/core"; +import { createBundledKimiSearchProvider } from "./provider.js"; + +const plugin = { + id: "search-kimi", + name: "Kimi Search", + description: "Bundled Kimi web search provider for OpenClaw.", + register(api: OpenClawPluginApi) { + api.registerSearchProvider(createBundledKimiSearchProvider()); + }, +}; + +export default plugin; diff --git a/extensions/search-kimi/src/provider.ts b/extensions/search-kimi/src/provider.ts new file mode 100644 index 00000000000..43bbf2aae2d --- /dev/null +++ b/extensions/search-kimi/src/provider.ts @@ -0,0 +1,317 @@ +import { + buildSearchRequestCacheIdentity, + createMissingSearchKeyPayload, + normalizeCacheKey, + normalizeSecretInput, + readCache, + readSearchProviderApiKeyValue, + rejectUnsupportedSearchFilters, + resolveSearchConfig, + type OpenClawConfig, + type SearchProviderExecutionResult, + type SearchProviderLegacyUiMetadata, + type SearchProviderPlugin, + withTrustedWebToolsEndpoint, + wrapWebContent, + writeCache, + writeSearchProviderApiKeyValue, +} from "openclaw/plugin-sdk/web-search"; + +const DEFAULT_KIMI_BASE_URL = "https://api.moonshot.ai/v1"; +const DEFAULT_KIMI_MODEL = "moonshot-v1-128k"; +const KIMI_WEB_SEARCH_TOOL = { + type: "builtin_function", + function: { name: "$web_search" }, +} as const; + +const KIMI_SEARCH_CACHE = new Map; expiresAt: number }>(); + +type WebSearchConfig = NonNullable["web"] extends infer Web + ? Web extends { search?: infer Search } + ? Search + : undefined + : undefined; + +type KimiConfig = { + apiKey?: string; + baseUrl?: string; + model?: string; +}; + +type KimiToolCall = { + id?: string; + type?: string; + function?: { + name?: string; + arguments?: string; + }; +}; + +type KimiMessage = { + role?: string; + content?: string; + reasoning_content?: string; + tool_calls?: KimiToolCall[]; +}; + +type KimiSearchResponse = { + choices?: Array<{ + finish_reason?: string; + message?: KimiMessage; + }>; + search_results?: Array<{ + title?: string; + url?: string; + content?: string; + }>; +}; + +function resolveKimiConfig(search?: WebSearchConfig): KimiConfig { + if (!search || typeof search !== "object") return {}; + const kimi = "kimi" in search ? search.kimi : undefined; + if (!kimi || typeof kimi !== "object") return {}; + return kimi as KimiConfig; +} + +function resolveKimiApiKey(kimi?: KimiConfig): string | undefined { + return ( + normalizeSecretInput(kimi?.apiKey) || + normalizeSecretInput(process.env.KIMI_API_KEY) || + normalizeSecretInput(process.env.MOONSHOT_API_KEY) || + undefined + ); +} + +function resolveKimiModel(kimi?: KimiConfig): string { + const fromConfig = + kimi && "model" in kimi && typeof kimi.model === "string" ? kimi.model.trim() : ""; + return fromConfig || DEFAULT_KIMI_MODEL; +} + +function resolveKimiBaseUrl(kimi?: KimiConfig): string { + const fromConfig = + kimi && "baseUrl" in kimi && typeof kimi.baseUrl === "string" ? kimi.baseUrl.trim() : ""; + return fromConfig || DEFAULT_KIMI_BASE_URL; +} + +function extractKimiMessageText(message: KimiMessage | undefined): string | undefined { + const content = message?.content?.trim(); + if (content) return content; + const reasoning = message?.reasoning_content?.trim(); + return reasoning || undefined; +} + +function extractKimiCitations(data: KimiSearchResponse): string[] { + const citations = (data.search_results ?? []) + .map((entry) => entry.url?.trim()) + .filter((url): url is string => Boolean(url)); + for (const toolCall of data.choices?.[0]?.message?.tool_calls ?? []) { + const rawArguments = toolCall.function?.arguments; + if (!rawArguments) continue; + try { + const parsed = JSON.parse(rawArguments) as { + search_results?: Array<{ url?: string }>; + url?: string; + }; + if (typeof parsed.url === "string" && parsed.url.trim()) citations.push(parsed.url.trim()); + for (const result of parsed.search_results ?? []) { + if (typeof result.url === "string" && result.url.trim()) citations.push(result.url.trim()); + } + } catch { + // ignore malformed tool arguments + } + } + return [...new Set(citations)]; +} + +function buildKimiToolResultContent(data: KimiSearchResponse): string { + return JSON.stringify({ + search_results: (data.search_results ?? []).map((entry) => ({ + title: entry.title ?? "", + url: entry.url ?? "", + content: entry.content ?? "", + })), + }); +} + +async function runKimiSearch(params: { + query: string; + apiKey: string; + baseUrl: string; + model: string; + timeoutSeconds: number; +}) { + const baseUrl = params.baseUrl.trim().replace(/\/$/, ""); + const endpoint = `${baseUrl}/chat/completions`; + const messages: Array> = [{ role: "user", content: params.query }]; + const collectedCitations = new Set(); + const MAX_ROUNDS = 3; + for (let round = 0; round < MAX_ROUNDS; round += 1) { + const nextResult = await withTrustedWebToolsEndpoint( + { + url: endpoint, + timeoutSeconds: params.timeoutSeconds, + init: { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${params.apiKey}`, + }, + body: JSON.stringify({ + model: params.model, + messages, + tools: [KIMI_WEB_SEARCH_TOOL], + }), + }, + }, + async ({ + response, + }): Promise<{ done: true; content: string; citations: string[] } | { done: false }> => { + if (!response.ok) { + const detail = await response.text().catch(() => ""); + throw new Error(`Kimi API error (${response.status}): ${detail || response.statusText}`); + } + const data = (await response.json()) as KimiSearchResponse; + for (const citation of extractKimiCitations(data)) { + collectedCitations.add(citation); + } + const choice = data.choices?.[0]; + const message = choice?.message; + const text = extractKimiMessageText(message); + const toolCalls = message?.tool_calls ?? []; + if (choice?.finish_reason !== "tool_calls" || toolCalls.length === 0) { + return { done: true, content: text ?? "No response", citations: [...collectedCitations] }; + } + messages.push({ + role: "assistant", + content: message?.content ?? "", + ...(message?.reasoning_content ? { reasoning_content: message.reasoning_content } : {}), + tool_calls: toolCalls, + }); + const toolContent = buildKimiToolResultContent(data); + let pushedToolResult = false; + for (const toolCall of toolCalls) { + const toolCallId = toolCall.id?.trim(); + if (!toolCallId) continue; + pushedToolResult = true; + messages.push({ + role: "tool", + tool_call_id: toolCallId, + content: toolContent, + }); + } + if (!pushedToolResult) { + return { done: true, content: text ?? "No response", citations: [...collectedCitations] }; + } + return { done: false }; + }, + ); + if (nextResult.done) { + return { content: nextResult.content, citations: nextResult.citations }; + } + } + return { + content: "Search completed but no final answer was produced.", + citations: [...collectedCitations], + }; +} + +export const KIMI_SEARCH_PROVIDER_METADATA: SearchProviderLegacyUiMetadata = { + label: "Kimi (Moonshot)", + hint: "Moonshot web search", + envKeys: ["KIMI_API_KEY", "MOONSHOT_API_KEY"], + placeholder: "sk-...", + signupUrl: "https://platform.moonshot.cn/", + apiKeyConfigPath: "tools.web.search.kimi.apiKey", + readApiKeyValue: (search) => readSearchProviderApiKeyValue(search, "kimi"), + writeApiKeyValue: (search, value) => + writeSearchProviderApiKeyValue({ search, provider: "kimi", value }), +}; + +export function createBundledKimiSearchProvider(): SearchProviderPlugin { + return { + id: "kimi", + name: "Kimi by Moonshot", + description: + "Search the web using Kimi by Moonshot. Returns AI-synthesized answers with citations from native $web_search.", + pluginOwnedExecution: true, + legacyConfig: KIMI_SEARCH_PROVIDER_METADATA, + isAvailable: (config) => + Boolean( + resolveKimiApiKey( + resolveKimiConfig( + resolveSearchConfig( + config?.tools?.web?.search as Record, + ), + ), + ), + ), + search: async (request, ctx): Promise => { + const search = resolveSearchConfig(request.providerConfig); + const kimiConfig = resolveKimiConfig(search); + const apiKey = resolveKimiApiKey(kimiConfig); + if (!apiKey) { + return createMissingSearchKeyPayload( + "missing_kimi_api_key", + "web_search (kimi) needs a Moonshot API key. Set KIMI_API_KEY or MOONSHOT_API_KEY in the Gateway environment, or configure tools.web.search.kimi.apiKey.", + ); + } + const unsupportedFilter = rejectUnsupportedSearchFilters({ + providerName: "kimi", + request, + support: { + country: false, + language: false, + freshness: false, + date: false, + domainFilter: false, + }, + }); + if (unsupportedFilter) { + return unsupportedFilter; + } + + const baseUrl = resolveKimiBaseUrl(kimiConfig); + const model = resolveKimiModel(kimiConfig); + const cacheKey = normalizeCacheKey( + `kimi:${baseUrl}:${model}:${buildSearchRequestCacheIdentity({ + query: request.query, + count: request.count, + })}`, + ); + const cached = readCache(KIMI_SEARCH_CACHE, cacheKey); + if (cached) return { ...cached.value, cached: true } as SearchProviderExecutionResult; + const startedAt = Date.now(); + const result = await runKimiSearch({ + query: request.query, + apiKey, + baseUrl, + model, + timeoutSeconds: ctx.timeoutSeconds, + }); + const payload = { + query: request.query, + provider: "kimi", + model, + tookMs: Date.now() - startedAt, + externalContent: { + untrusted: true, + source: "web_search", + provider: "kimi", + wrapped: true, + }, + content: wrapWebContent(result.content), + citations: result.citations, + }; + writeCache(KIMI_SEARCH_CACHE, cacheKey, payload, ctx.cacheTtlMs); + return payload as SearchProviderExecutionResult; + }, + }; +} + +export const __testing = { + KIMI_SEARCH_CACHE, + clearSearchProviderCaches() { + KIMI_SEARCH_CACHE.clear(); + }, +} as const; diff --git a/extensions/search-perplexity/openclaw.plugin.json b/extensions/search-perplexity/openclaw.plugin.json index e2cb10c2642..b412ca28256 100644 --- a/extensions/search-perplexity/openclaw.plugin.json +++ b/extensions/search-perplexity/openclaw.plugin.json @@ -1,4 +1,8 @@ { "id": "search-perplexity", + "configSchema": { + "type": "object", + "properties": {} + }, "provides": ["providers.search.perplexity"] } diff --git a/extensions/search-perplexity/package.json b/extensions/search-perplexity/package.json index 32cdc1a19eb..0f9abb18293 100644 --- a/extensions/search-perplexity/package.json +++ b/extensions/search-perplexity/package.json @@ -6,7 +6,7 @@ "type": "module", "openclaw": { "extensions": [ - "./index.ts" + "./src/index.ts" ] } } diff --git a/extensions/search-perplexity/index.ts b/extensions/search-perplexity/src/index.ts similarity index 50% rename from extensions/search-perplexity/index.ts rename to extensions/search-perplexity/src/index.ts index 351742b0675..3560d9d4dab 100644 --- a/extensions/search-perplexity/index.ts +++ b/extensions/search-perplexity/src/index.ts @@ -1,11 +1,12 @@ -import { createBundledBuiltinSearchProvider, type OpenClawPluginApi } from "openclaw/plugin-sdk"; +import type { OpenClawPluginApi } from "openclaw/plugin-sdk/core"; +import { createBundledPerplexitySearchProvider } from "./provider.js"; const plugin = { id: "search-perplexity", name: "Perplexity Search", description: "Bundled Perplexity web search provider for OpenClaw.", register(api: OpenClawPluginApi) { - api.registerSearchProvider(createBundledBuiltinSearchProvider("perplexity")); + api.registerSearchProvider(createBundledPerplexitySearchProvider()); }, }; diff --git a/extensions/search-perplexity/src/provider.ts b/extensions/search-perplexity/src/provider.ts new file mode 100644 index 00000000000..66de63f4f8b --- /dev/null +++ b/extensions/search-perplexity/src/provider.ts @@ -0,0 +1,574 @@ +import { + buildSearchRequestCacheIdentity, + createMissingSearchKeyPayload, + createSearchProviderErrorResult, + normalizeCacheKey, + normalizeDateInputToIso, + normalizeResolvedSecretInputString, + normalizeSecretInput, + readCache, + readSearchProviderApiKeyValue, + resolveSearchConfig, + resolveSiteName, + throwWebSearchApiError, + type OpenClawConfig, + type SearchProviderExecutionResult, + type SearchProviderLegacyUiMetadata, + type SearchProviderPlugin, + withTrustedWebToolsEndpoint, + wrapWebContent, + writeCache, + writeSearchProviderApiKeyValue, +} from "openclaw/plugin-sdk/web-search"; + +const DEFAULT_PERPLEXITY_BASE_URL = "https://openrouter.ai/api/v1"; +const PERPLEXITY_DIRECT_BASE_URL = "https://api.perplexity.ai"; +const PERPLEXITY_SEARCH_ENDPOINT = "https://api.perplexity.ai/search"; +const DEFAULT_PERPLEXITY_MODEL = "perplexity/sonar-pro"; +const PERPLEXITY_KEY_PREFIXES = ["pplx-"]; +const OPENROUTER_KEY_PREFIXES = ["sk-or-"]; +const PERPLEXITY_RECENCY_VALUES = new Set(["day", "week", "month", "year"]); +const ISO_DATE_PATTERN = /^(\d{4})-(\d{2})-(\d{2})$/; + +const PERPLEXITY_SEARCH_CACHE = new Map< + string, + { value: Record; expiresAt: number } +>(); + +type WebSearchConfig = NonNullable["web"] extends infer Web + ? Web extends { search?: infer Search } + ? Search + : undefined + : undefined; + +type PerplexityConfig = { + apiKey?: string; + baseUrl?: string; + model?: string; +}; + +type PerplexityApiKeySource = "config" | "perplexity_env" | "openrouter_env" | "none"; +type PerplexityTransport = "search_api" | "chat_completions"; +type PerplexityBaseUrlHint = "direct" | "openrouter"; + +type PerplexitySearchResponse = { + choices?: Array<{ + message?: { + content?: string; + annotations?: Array<{ + type?: string; + url?: string; + url_citation?: { + url?: string; + }; + }>; + }; + }>; + citations?: string[]; +}; + +type PerplexitySearchApiResult = { + title?: string; + url?: string; + snippet?: string; + date?: string; +}; + +type PerplexitySearchApiResponse = { + results?: PerplexitySearchApiResult[]; +}; + +function normalizeApiKey(key: unknown): string { + return normalizeSecretInput(key); +} + +function extractPerplexityCitations(data: PerplexitySearchResponse): string[] { + const normalizeUrl = (value: unknown): string | undefined => { + if (typeof value !== "string") { + return undefined; + } + const trimmed = value.trim(); + return trimmed ? trimmed : undefined; + }; + const topLevel = (data.citations ?? []) + .map(normalizeUrl) + .filter((url): url is string => Boolean(url)); + if (topLevel.length > 0) { + return [...new Set(topLevel)]; + } + const citations: string[] = []; + for (const choice of data.choices ?? []) { + for (const annotation of choice.message?.annotations ?? []) { + if (annotation.type !== "url_citation") { + continue; + } + const url = normalizeUrl(annotation.url_citation?.url ?? annotation.url); + if (url) { + citations.push(url); + } + } + } + return [...new Set(citations)]; +} + +function resolvePerplexityConfig(search?: WebSearchConfig): PerplexityConfig { + if (!search || typeof search !== "object") { + return {}; + } + const perplexity = "perplexity" in search ? search.perplexity : undefined; + if (!perplexity || typeof perplexity !== "object") { + return {}; + } + return perplexity as PerplexityConfig; +} + +function resolvePerplexityApiKey(perplexity?: PerplexityConfig): { + apiKey?: string; + source: PerplexityApiKeySource; +} { + const fromConfig = normalizeApiKey(perplexity?.apiKey); + if (fromConfig) { + return { apiKey: fromConfig, source: "config" }; + } + const fromEnvPerplexity = normalizeApiKey(process.env.PERPLEXITY_API_KEY); + if (fromEnvPerplexity) { + return { apiKey: fromEnvPerplexity, source: "perplexity_env" }; + } + const fromEnvOpenRouter = normalizeApiKey(process.env.OPENROUTER_API_KEY); + if (fromEnvOpenRouter) { + return { apiKey: fromEnvOpenRouter, source: "openrouter_env" }; + } + return { apiKey: undefined, source: "none" }; +} + +function inferPerplexityBaseUrlFromApiKey(apiKey?: string): PerplexityBaseUrlHint | undefined { + if (!apiKey) { + return undefined; + } + const normalized = apiKey.toLowerCase(); + if (PERPLEXITY_KEY_PREFIXES.some((prefix) => normalized.startsWith(prefix))) { + return "direct"; + } + if (OPENROUTER_KEY_PREFIXES.some((prefix) => normalized.startsWith(prefix))) { + return "openrouter"; + } + return undefined; +} + +function resolvePerplexityBaseUrl( + perplexity?: PerplexityConfig, + authSource: PerplexityApiKeySource = "none", + configuredKey?: string, +): string { + const fromConfig = + perplexity && "baseUrl" in perplexity && typeof perplexity.baseUrl === "string" + ? perplexity.baseUrl.trim() + : ""; + if (fromConfig) { + return fromConfig; + } + if (authSource === "perplexity_env") { + return PERPLEXITY_DIRECT_BASE_URL; + } + if (authSource === "openrouter_env") { + return DEFAULT_PERPLEXITY_BASE_URL; + } + if (authSource === "config") { + const inferred = inferPerplexityBaseUrlFromApiKey(configuredKey); + if (inferred === "openrouter") { + return DEFAULT_PERPLEXITY_BASE_URL; + } + return PERPLEXITY_DIRECT_BASE_URL; + } + return DEFAULT_PERPLEXITY_BASE_URL; +} + +function resolvePerplexityModel(perplexity?: PerplexityConfig): string { + const fromConfig = + perplexity && "model" in perplexity && typeof perplexity.model === "string" + ? perplexity.model.trim() + : ""; + return fromConfig || DEFAULT_PERPLEXITY_MODEL; +} + +function isDirectPerplexityBaseUrl(baseUrl: string): boolean { + const trimmed = baseUrl.trim(); + if (!trimmed) { + return false; + } + try { + return new URL(trimmed).hostname.toLowerCase() === "api.perplexity.ai"; + } catch { + return false; + } +} + +function resolvePerplexityRequestModel(baseUrl: string, model: string): string { + if (!isDirectPerplexityBaseUrl(baseUrl)) { + return model; + } + return model.startsWith("perplexity/") ? model.slice("perplexity/".length) : model; +} + +function resolvePerplexityTransport(perplexity?: PerplexityConfig): { + apiKey?: string; + source: PerplexityApiKeySource; + baseUrl: string; + model: string; + transport: PerplexityTransport; +} { + const auth = resolvePerplexityApiKey(perplexity); + const baseUrl = resolvePerplexityBaseUrl(perplexity, auth.source, auth.apiKey); + const model = resolvePerplexityModel(perplexity); + const hasLegacyOverride = Boolean( + (perplexity?.baseUrl && perplexity.baseUrl.trim()) || + (perplexity?.model && perplexity.model.trim()), + ); + return { + ...auth, + baseUrl, + model, + transport: + hasLegacyOverride || !isDirectPerplexityBaseUrl(baseUrl) ? "chat_completions" : "search_api", + }; +} + +async function runPerplexitySearchApi(params: { + query: string; + apiKey: string; + count: number; + timeoutSeconds: number; + country?: string; + searchDomainFilter?: string[]; + searchRecencyFilter?: string; + searchLanguageFilter?: string[]; + searchAfterDate?: string; + searchBeforeDate?: string; + maxTokens?: number; + maxTokensPerPage?: number; +}) { + const body: Record = { query: params.query, max_results: params.count }; + if (params.country) body.country = params.country; + if (params.searchDomainFilter?.length) body.search_domain_filter = params.searchDomainFilter; + if (params.searchRecencyFilter) body.search_recency_filter = params.searchRecencyFilter; + if (params.searchLanguageFilter?.length) + body.search_language_filter = params.searchLanguageFilter; + if (params.searchAfterDate) body.search_after_date = params.searchAfterDate; + if (params.searchBeforeDate) body.search_before_date = params.searchBeforeDate; + if (params.maxTokens !== undefined) body.max_tokens = params.maxTokens; + if (params.maxTokensPerPage !== undefined) body.max_tokens_per_page = params.maxTokensPerPage; + + return withTrustedWebToolsEndpoint( + { + url: PERPLEXITY_SEARCH_ENDPOINT, + timeoutSeconds: params.timeoutSeconds, + init: { + method: "POST", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + Authorization: `Bearer ${params.apiKey}`, + "HTTP-Referer": "https://openclaw.ai", + "X-Title": "OpenClaw Web Search", + }, + body: JSON.stringify(body), + }, + }, + async ({ response }) => { + if (!response.ok) { + return await throwWebSearchApiError(response, "Perplexity Search"); + } + const data = (await response.json()) as PerplexitySearchApiResponse; + const results = Array.isArray(data.results) ? data.results : []; + return results.map((entry) => { + const title = entry.title ?? ""; + const url = entry.url ?? ""; + const snippet = entry.snippet ?? ""; + return { + title: title ? wrapWebContent(title, "web_search") : "", + url, + description: snippet ? wrapWebContent(snippet, "web_search") : "", + published: entry.date ?? undefined, + siteName: resolveSiteName(url) || undefined, + }; + }); + }, + ); +} + +async function runPerplexitySearch(params: { + query: string; + apiKey: string; + baseUrl: string; + model: string; + timeoutSeconds: number; + freshness?: string; +}) { + const baseUrl = params.baseUrl.trim().replace(/\/$/, ""); + const endpoint = `${baseUrl}/chat/completions`; + const model = resolvePerplexityRequestModel(baseUrl, params.model); + const body: Record = { + model, + messages: [{ role: "user", content: params.query }], + }; + if (params.freshness) { + body.search_recency_filter = params.freshness; + } + return withTrustedWebToolsEndpoint( + { + url: endpoint, + timeoutSeconds: params.timeoutSeconds, + init: { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${params.apiKey}`, + "HTTP-Referer": "https://openclaw.ai", + "X-Title": "OpenClaw Web Search", + }, + body: JSON.stringify(body), + }, + }, + async ({ response }) => { + if (!response.ok) { + return await throwWebSearchApiError(response, "Perplexity"); + } + const data = (await response.json()) as PerplexitySearchResponse; + return { + content: data.choices?.[0]?.message?.content ?? "No response", + citations: extractPerplexityCitations(data), + }; + }, + ); +} + +function isoToPerplexityDate(iso: string): string | undefined { + const match = iso.match(ISO_DATE_PATTERN); + if (!match) { + return undefined; + } + const [, year, month, day] = match; + return `${Number.parseInt(month, 10)}/${Number.parseInt(day, 10)}/${year}`; +} + +function createPerplexityPayload(params: { + request: { query: string }; + startedAt: number; + model?: string; + results?: unknown[]; + content?: string; + citations?: string[]; +}) { + const payload: Record = { + query: params.request.query, + provider: "perplexity", + tookMs: Date.now() - params.startedAt, + externalContent: { + untrusted: true, + source: "web_search", + provider: "perplexity", + wrapped: true, + }, + }; + if (params.model) payload.model = params.model; + if (params.results) { + payload.results = params.results; + payload.count = params.results.length; + } + if (params.content) payload.content = wrapWebContent(params.content, "web_search"); + if (params.citations) payload.citations = params.citations; + return payload; +} + +export const PERPLEXITY_SEARCH_PROVIDER_METADATA: SearchProviderLegacyUiMetadata = { + label: "Perplexity Search", + hint: "Structured results · domain/country/language/time filters", + envKeys: ["PERPLEXITY_API_KEY"], + placeholder: "pplx-...", + signupUrl: "https://www.perplexity.ai/settings/api", + apiKeyConfigPath: "tools.web.search.perplexity.apiKey", + readApiKeyValue: (search) => readSearchProviderApiKeyValue(search, "perplexity"), + writeApiKeyValue: (search, value) => + writeSearchProviderApiKeyValue({ search, provider: "perplexity", value }), + resolveRuntimeMetadata: (params) => ({ + perplexityTransport: resolvePerplexityTransport( + resolvePerplexityConfig(resolveSearchConfig(params.search)), + ).transport, + }), +}; + +export function createBundledPerplexitySearchProvider(): SearchProviderPlugin { + return { + id: "perplexity", + name: "Perplexity", + description: + "Search the web using Perplexity. Runtime routing decides between native Search API and Sonar chat-completions compatibility. Structured filters are available on the native Search API path.", + pluginOwnedExecution: true, + legacyConfig: PERPLEXITY_SEARCH_PROVIDER_METADATA, + resolveRuntimeMetadata: PERPLEXITY_SEARCH_PROVIDER_METADATA.resolveRuntimeMetadata, + isAvailable: (config) => + Boolean( + resolvePerplexityApiKey( + resolvePerplexityConfig( + resolveSearchConfig( + config?.tools?.web?.search as Record, + ), + ), + ).apiKey, + ), + search: async (request, ctx): Promise => { + const search = resolveSearchConfig(request.providerConfig); + const runtime = resolvePerplexityTransport(resolvePerplexityConfig(search)); + if (!runtime.apiKey) { + return createMissingSearchKeyPayload( + "missing_perplexity_api_key", + "web_search (perplexity) needs an API key. Set PERPLEXITY_API_KEY or OPENROUTER_API_KEY in the Gateway environment, or configure tools.web.search.perplexity.apiKey.", + ); + } + const supportsStructured = runtime.transport === "search_api"; + if (request.country && !supportsStructured) { + return createSearchProviderErrorResult( + "unsupported_country", + "country filtering is only supported by the native Perplexity Search API path. Remove Perplexity baseUrl/model overrides or use a direct PERPLEXITY_API_KEY to enable it.", + ); + } + if (request.language && !supportsStructured) { + return createSearchProviderErrorResult( + "unsupported_language", + "language filtering is only supported by the native Perplexity Search API path. Remove Perplexity baseUrl/model overrides or use a direct PERPLEXITY_API_KEY to enable it.", + ); + } + if (request.language && !/^[a-z]{2}$/i.test(request.language)) { + return createSearchProviderErrorResult( + "invalid_language", + "language must be a 2-letter ISO 639-1 code like 'en', 'de', or 'fr'.", + ); + } + const normalizedFreshness = request.freshness + ? PERPLEXITY_RECENCY_VALUES.has(request.freshness.trim().toLowerCase()) + ? request.freshness.trim().toLowerCase() + : undefined + : undefined; + if (request.freshness && !normalizedFreshness) { + return createSearchProviderErrorResult( + "invalid_freshness", + "freshness must be day, week, month, or year.", + ); + } + if ((request.dateAfter || request.dateBefore) && !supportsStructured) { + return createSearchProviderErrorResult( + "unsupported_date_filter", + "date_after/date_before are only supported by the native Perplexity Search API path. Remove Perplexity baseUrl/model overrides or use a direct PERPLEXITY_API_KEY to enable them.", + ); + } + if (request.domainFilter && request.domainFilter.length > 0 && !supportsStructured) { + return createSearchProviderErrorResult( + "unsupported_domain_filter", + "domain_filter is only supported by the native Perplexity Search API path. Remove Perplexity baseUrl/model overrides or use a direct PERPLEXITY_API_KEY to enable it.", + ); + } + if (request.domainFilter && request.domainFilter.length > 0) { + const hasDenylist = request.domainFilter.some((domain) => domain.startsWith("-")); + const hasAllowlist = request.domainFilter.some((domain) => !domain.startsWith("-")); + if (hasDenylist && hasAllowlist) { + return createSearchProviderErrorResult( + "invalid_domain_filter", + "domain_filter cannot mix allowlist and denylist entries. Use either all positive entries (allowlist) or all entries prefixed with '-' (denylist).", + ); + } + if (request.domainFilter.length > 20) { + return createSearchProviderErrorResult( + "invalid_domain_filter", + "domain_filter supports a maximum of 20 domains.", + ); + } + } + if ( + runtime.transport === "chat_completions" && + (request.maxTokens !== undefined || request.maxTokensPerPage !== undefined) + ) { + return createSearchProviderErrorResult( + "unsupported_content_budget", + "max_tokens and max_tokens_per_page are only supported by the native Perplexity Search API path. Remove Perplexity baseUrl/model overrides or use a direct PERPLEXITY_API_KEY to enable them.", + ); + } + if (request.dateAfter && !normalizeDateInputToIso(request.dateAfter)) { + return createSearchProviderErrorResult( + "invalid_date_after", + "date_after must be a valid YYYY-MM-DD date.", + ); + } + if (request.dateBefore && !normalizeDateInputToIso(request.dateBefore)) { + return createSearchProviderErrorResult( + "invalid_date_before", + "date_before must be a valid YYYY-MM-DD date.", + ); + } + + const cacheKey = normalizeCacheKey( + `perplexity:${runtime.transport}:${runtime.baseUrl}:${runtime.model}:${buildSearchRequestCacheIdentity( + { + query: request.query, + count: request.count, + country: request.country, + language: request.language, + freshness: normalizedFreshness, + dateAfter: request.dateAfter, + dateBefore: request.dateBefore, + domainFilter: request.domainFilter, + maxTokens: request.maxTokens, + maxTokensPerPage: request.maxTokensPerPage, + }, + )}`, + ); + const cached = readCache(PERPLEXITY_SEARCH_CACHE, cacheKey); + if (cached) return { ...cached.value, cached: true } as SearchProviderExecutionResult; + const startedAt = Date.now(); + let payload: Record; + if (runtime.transport === "chat_completions") { + const result = await runPerplexitySearch({ + query: request.query, + apiKey: runtime.apiKey, + baseUrl: runtime.baseUrl, + model: runtime.model, + timeoutSeconds: ctx.timeoutSeconds, + freshness: normalizedFreshness, + }); + payload = createPerplexityPayload({ + request, + startedAt, + model: runtime.model, + content: result.content, + citations: result.citations, + }); + } else { + const results = await runPerplexitySearchApi({ + query: request.query, + apiKey: runtime.apiKey, + count: request.count, + timeoutSeconds: ctx.timeoutSeconds, + country: request.country, + searchDomainFilter: request.domainFilter, + searchRecencyFilter: normalizedFreshness, + searchLanguageFilter: request.language ? [request.language] : undefined, + searchAfterDate: request.dateAfter ? isoToPerplexityDate(request.dateAfter) : undefined, + searchBeforeDate: request.dateBefore + ? isoToPerplexityDate(request.dateBefore) + : undefined, + maxTokens: request.maxTokens, + maxTokensPerPage: request.maxTokensPerPage, + }); + payload = createPerplexityPayload({ request, startedAt, results }); + } + writeCache(PERPLEXITY_SEARCH_CACHE, cacheKey, payload, ctx.cacheTtlMs); + return payload as SearchProviderExecutionResult; + }, + }; +} + +export const __testing = { + PERPLEXITY_SEARCH_CACHE, + clearSearchProviderCaches() { + PERPLEXITY_SEARCH_CACHE.clear(); + }, +} as const; diff --git a/package.json b/package.json index 6cde8d84431..5bc7a8f2258 100644 --- a/package.json +++ b/package.json @@ -44,6 +44,10 @@ "types": "./dist/plugin-sdk/core.d.ts", "default": "./dist/plugin-sdk/core.js" }, + "./plugin-sdk/web-search": { + "types": "./dist/plugin-sdk/web-search.d.ts", + "default": "./dist/plugin-sdk/web-search.js" + }, "./plugin-sdk/compat": { "types": "./dist/plugin-sdk/compat.d.ts", "default": "./dist/plugin-sdk/compat.js" diff --git a/src/agents/tools/web-search-provider-catalog.ts b/src/agents/tools/web-search-provider-catalog.ts index d8c82f9b5cc..075fa39d3dd 100644 --- a/src/agents/tools/web-search-provider-catalog.ts +++ b/src/agents/tools/web-search-provider-catalog.ts @@ -27,6 +27,7 @@ export type BuiltinWebSearchProviderEntry = { envKeys: readonly string[]; placeholder: string; signupUrl: string; + apiKeyConfigPath: string; }; const BUILTIN_WEB_SEARCH_PROVIDER_CATALOG: Record< @@ -39,6 +40,7 @@ const BUILTIN_WEB_SEARCH_PROVIDER_CATALOG: Record< envKeys: ["BRAVE_API_KEY"], placeholder: "BSA...", signupUrl: "https://brave.com/search/api/", + apiKeyConfigPath: "tools.web.search.apiKey", }, gemini: { label: "Gemini (Google Search)", @@ -46,6 +48,7 @@ const BUILTIN_WEB_SEARCH_PROVIDER_CATALOG: Record< envKeys: ["GEMINI_API_KEY"], placeholder: "AIza...", signupUrl: "https://aistudio.google.com/apikey", + apiKeyConfigPath: "tools.web.search.gemini.apiKey", }, grok: { label: "Grok (xAI)", @@ -53,6 +56,7 @@ const BUILTIN_WEB_SEARCH_PROVIDER_CATALOG: Record< envKeys: ["XAI_API_KEY"], placeholder: "xai-...", signupUrl: "https://console.x.ai/", + apiKeyConfigPath: "tools.web.search.grok.apiKey", }, kimi: { label: "Kimi (Moonshot)", @@ -60,6 +64,7 @@ const BUILTIN_WEB_SEARCH_PROVIDER_CATALOG: Record< envKeys: ["KIMI_API_KEY", "MOONSHOT_API_KEY"], placeholder: "sk-...", signupUrl: "https://platform.moonshot.cn/", + apiKeyConfigPath: "tools.web.search.kimi.apiKey", }, perplexity: { label: "Perplexity Search", @@ -67,6 +72,7 @@ const BUILTIN_WEB_SEARCH_PROVIDER_CATALOG: Record< envKeys: ["PERPLEXITY_API_KEY"], placeholder: "pplx-...", signupUrl: "https://www.perplexity.ai/settings/api", + apiKeyConfigPath: "tools.web.search.perplexity.apiKey", }, }; @@ -79,3 +85,59 @@ export const BUILTIN_WEB_SEARCH_PROVIDER_OPTIONS: readonly BuiltinWebSearchProvi export function isBuiltinWebSearchProviderId(value: string): value is BuiltinWebSearchProviderId { return BUILTIN_WEB_SEARCH_PROVIDER_IDS.includes(value as BuiltinWebSearchProviderId); } + +export function normalizeBuiltinWebSearchProvider( + value: unknown, +): BuiltinWebSearchProviderId | undefined { + if (typeof value !== "string") { + return undefined; + } + const normalized = value.trim().toLowerCase(); + return isBuiltinWebSearchProviderId(normalized) ? normalized : undefined; +} + +export function getBuiltinWebSearchProviderEntry( + provider: BuiltinWebSearchProviderId, +): BuiltinWebSearchProviderEntry { + return BUILTIN_WEB_SEARCH_PROVIDER_OPTIONS.find((entry) => entry.value === provider)!; +} + +function getScopedSearchConfig( + search: Record, + provider: BuiltinWebSearchProviderId, +): Record | undefined { + if (provider === "brave") { + return search; + } + const scoped = search[provider]; + return typeof scoped === "object" && scoped !== null && !Array.isArray(scoped) + ? (scoped as Record) + : undefined; +} + +export function readBuiltinWebSearchApiKeyValue( + search: Record | undefined, + provider: BuiltinWebSearchProviderId, +): unknown { + if (!search) { + return undefined; + } + return getScopedSearchConfig(search, provider)?.apiKey; +} + +export function writeBuiltinWebSearchApiKeyValue(params: { + search: Record; + provider: BuiltinWebSearchProviderId; + value: unknown; +}): void { + if (params.provider === "brave") { + params.search.apiKey = params.value; + return; + } + const current = getScopedSearchConfig(params.search, params.provider); + if (current) { + current.apiKey = params.value; + return; + } + params.search[params.provider] = { apiKey: params.value }; +} diff --git a/src/agents/tools/web-search.ts b/src/agents/tools/web-search.ts index d4320577adb..5eba98205ff 100644 --- a/src/agents/tools/web-search.ts +++ b/src/agents/tools/web-search.ts @@ -1,9 +1,10 @@ import { Type } from "@sinclair/typebox"; -import { formatCliCommand } from "../../cli/command-format.js"; +import { formatCliCommand as _formatCliCommand } from "../../cli/command-format.js"; import type { OpenClawConfig } from "../../config/config.js"; import { normalizeResolvedSecretInputString } from "../../config/types.secrets.js"; import { logVerbose } from "../../globals.js"; import { resolveCapabilitySlotSelection } from "../../plugins/capability-slots.js"; +import { loadOpenClawPlugins } from "../../plugins/loader.js"; import { getActivePluginRegistry } from "../../plugins/runtime.js"; import type { SearchProviderContext, @@ -21,7 +22,7 @@ import { withTrustedWebToolsEndpoint } from "./web-guarded-fetch.js"; import { resolveCitationRedirectUrl } from "./web-search-citation-redirect.js"; import { type BuiltinWebSearchProviderId, - isBuiltinWebSearchProviderId, + isBuiltinWebSearchProviderId as isBuiltinWebSearchProviderIdFromCatalog, } from "./web-search-provider-catalog.js"; import { CacheEntry, @@ -39,8 +40,8 @@ const DEFAULT_SEARCH_COUNT = 5; const MAX_SEARCH_COUNT = 10; const DEFAULT_PROVIDER = "brave"; -const BRAVE_SEARCH_ENDPOINT = "https://api.search.brave.com/res/v1/web/search"; -const BRAVE_LLM_CONTEXT_ENDPOINT = "https://api.search.brave.com/res/v1/llm/context"; +const _BRAVE_SEARCH_ENDPOINT = "https://api.search.brave.com/res/v1/web/search"; +const _BRAVE_LLM_CONTEXT_ENDPOINT = "https://api.search.brave.com/res/v1/llm/context"; const DEFAULT_PERPLEXITY_BASE_URL = "https://openrouter.ai/api/v1"; const PERPLEXITY_DIRECT_BASE_URL = "https://api.perplexity.ai"; const PERPLEXITY_SEARCH_ENDPOINT = "https://api.perplexity.ai/search"; @@ -338,7 +339,7 @@ type BraveSearchResult = { age?: string; }; -type BraveSearchResponse = { +type _BraveSearchResponse = { web?: { results?: BraveSearchResult[]; }; @@ -570,7 +571,7 @@ type GeminiGroundingResponse = { }; }; -const DEFAULT_GEMINI_MODEL = "gemini-2.5-flash"; +const _DEFAULT_GEMINI_MODEL = "gemini-2.5-flash"; const GEMINI_API_BASE = "https://generativelanguage.googleapis.com/v1beta"; function resolveSearchConfig(cfg?: OpenClawConfig): WebSearchConfig { @@ -604,52 +605,12 @@ function resolveSearchApiKey(search?: WebSearchConfig): string | undefined { return fromConfig || fromEnv || undefined; } -function missingSearchKeyPayload(provider: BuiltinWebSearchProviderId) { - if (provider === "brave") { - return { - error: "missing_brave_api_key", - message: `web_search (brave) needs a Brave Search API key. Run \`${formatCliCommand("openclaw configure --section web")}\` to store it, or set BRAVE_API_KEY in the Gateway environment.`, - docs: "https://docs.openclaw.ai/tools/web", - }; - } - if (provider === "gemini") { - return { - error: "missing_gemini_api_key", - message: - "web_search (gemini) needs an API key. Set GEMINI_API_KEY in the Gateway environment, or configure tools.web.search.gemini.apiKey.", - docs: "https://docs.openclaw.ai/tools/web", - }; - } - if (provider === "grok") { - return { - error: "missing_xai_api_key", - message: - "web_search (grok) needs an xAI API key. Set XAI_API_KEY in the Gateway environment, or configure tools.web.search.grok.apiKey.", - docs: "https://docs.openclaw.ai/tools/web", - }; - } - if (provider === "kimi") { - return { - error: "missing_kimi_api_key", - message: - "web_search (kimi) needs a Moonshot API key. Set KIMI_API_KEY or MOONSHOT_API_KEY in the Gateway environment, or configure tools.web.search.kimi.apiKey.", - docs: "https://docs.openclaw.ai/tools/web", - }; - } - return { - error: "missing_perplexity_api_key", - message: - "web_search (perplexity) needs an API key. Set PERPLEXITY_API_KEY or OPENROUTER_API_KEY in the Gateway environment, or configure tools.web.search.perplexity.apiKey.", - docs: "https://docs.openclaw.ai/tools/web", - }; -} - function resolveBuiltinSearchProvider(search?: WebSearchConfig): BuiltinWebSearchProviderId { const raw = search && "provider" in search && typeof search.provider === "string" ? search.provider.trim().toLowerCase() : ""; - if (isBuiltinWebSearchProviderId(raw)) { + if (isBuiltinSearchProviderId(raw)) { return raw; } @@ -700,17 +661,6 @@ function resolveBuiltinSearchProvider(search?: WebSearchConfig): BuiltinWebSearc return "brave"; } -function resolveBraveConfig(search?: WebSearchConfig): BraveConfig { - if (!search || typeof search !== "object") { - return {}; - } - const brave = "brave" in search ? search.brave : undefined; - if (!brave || typeof brave !== "object") { - return {}; - } - return brave as BraveConfig; -} - function resolveBraveMode(brave: BraveConfig): "web" | "llm-context" { return brave.mode === "llm-context" ? "llm-context" : "web"; } @@ -940,12 +890,6 @@ function resolveGeminiApiKey(gemini?: GeminiConfig): string | undefined { return fromEnv || undefined; } -function resolveGeminiModel(gemini?: GeminiConfig): string { - const fromConfig = - gemini && "model" in gemini && typeof gemini.model === "string" ? gemini.model.trim() : ""; - return fromConfig || DEFAULT_GEMINI_MODEL; -} - async function withTrustedWebSearchEndpoint( params: { url: string; @@ -964,7 +908,7 @@ async function withTrustedWebSearchEndpoint( ); } -async function runGeminiSearch(params: { +async function _runGeminiSearch(params: { query: string; apiKey: string; model: string; @@ -1191,7 +1135,7 @@ async function throwWebSearchApiError(res: Response, providerLabel: string): Pro throw new Error(`${providerLabel} API error (${res.status}): ${detail || res.statusText}`); } -async function runPerplexitySearchApi(params: { +async function _runPerplexitySearchApi(params: { query: string; apiKey: string; count: number; @@ -1277,7 +1221,7 @@ async function runPerplexitySearchApi(params: { ); } -async function runPerplexitySearch(params: { +async function _runPerplexitySearch(params: { query: string; apiKey: string; baseUrl: string; @@ -1333,7 +1277,7 @@ async function runPerplexitySearch(params: { ); } -async function runGrokSearch(params: { +async function _runGrokSearch(params: { query: string; apiKey: string; model: string; @@ -1440,7 +1384,7 @@ function buildKimiToolResultContent(data: KimiSearchResponse): string { }); } -async function runKimiSearch(params: { +async function _runKimiSearch(params: { query: string; apiKey: string; baseUrl: string; @@ -1553,7 +1497,7 @@ function mapBraveLlmContextResults( })); } -async function runBraveLlmContextSearch(params: { +async function _runBraveLlmContextSearch(params: { query: string; apiKey: string; timeoutSeconds: number; @@ -1608,350 +1552,12 @@ async function runBraveLlmContextSearch(params: { ); } -async function runWebSearch(params: { - query: string; - count: number; - apiKey: string; - timeoutSeconds: number; - cacheTtlMs: number; - provider: BuiltinWebSearchProviderId; - country?: string; - language?: string; - search_lang?: string; - ui_lang?: string; - freshness?: string; - dateAfter?: string; - dateBefore?: string; - searchDomainFilter?: string[]; - maxTokens?: number; - maxTokensPerPage?: number; - perplexityBaseUrl?: string; - perplexityModel?: string; - perplexityTransport?: PerplexityTransport; - grokModel?: string; - grokInlineCitations?: boolean; - geminiModel?: string; - kimiBaseUrl?: string; - kimiModel?: string; - braveMode?: "web" | "llm-context"; -}): Promise> { - const effectiveBraveMode = params.braveMode ?? "web"; - const providerSpecificKey = - params.provider === "perplexity" - ? `${params.perplexityTransport ?? "search_api"}:${params.perplexityBaseUrl ?? PERPLEXITY_DIRECT_BASE_URL}:${params.perplexityModel ?? DEFAULT_PERPLEXITY_MODEL}` - : params.provider === "grok" - ? `${params.grokModel ?? DEFAULT_GROK_MODEL}:${String(params.grokInlineCitations ?? false)}` - : params.provider === "gemini" - ? (params.geminiModel ?? DEFAULT_GEMINI_MODEL) - : params.provider === "kimi" - ? `${params.kimiBaseUrl ?? DEFAULT_KIMI_BASE_URL}:${params.kimiModel ?? DEFAULT_KIMI_MODEL}` - : ""; - const requestCacheIdentity = buildSearchRequestCacheIdentity({ - query: params.query, - count: params.count, - country: params.country, - language: params.language, - search_lang: params.search_lang, - ui_lang: params.ui_lang, - freshness: params.freshness, - dateAfter: params.dateAfter, - dateBefore: params.dateBefore, - domainFilter: params.searchDomainFilter, - maxTokens: params.maxTokens, - maxTokensPerPage: params.maxTokensPerPage, - }); - const cacheKey = normalizeCacheKey( - params.provider === "brave" && effectiveBraveMode === "llm-context" - ? `${params.provider}:llm-context:${buildSearchRequestCacheIdentity({ - query: params.query, - count: params.count, - country: params.country, - language: params.language, - search_lang: params.search_lang, - ui_lang: undefined, - freshness: params.freshness, - dateAfter: undefined, - dateBefore: undefined, - domainFilter: undefined, - maxTokens: undefined, - maxTokensPerPage: undefined, - })}` - : `${params.provider}:${effectiveBraveMode}:${requestCacheIdentity}:${providerSpecificKey}`, - ); - const cached = readCache(SEARCH_CACHE, cacheKey); - if (cached) { - return { ...cached.value, cached: true }; - } - - const start = Date.now(); - - if (params.provider === "perplexity") { - if (params.perplexityTransport === "chat_completions") { - const { content, citations } = await runPerplexitySearch({ - query: params.query, - apiKey: params.apiKey, - baseUrl: params.perplexityBaseUrl ?? DEFAULT_PERPLEXITY_BASE_URL, - model: params.perplexityModel ?? DEFAULT_PERPLEXITY_MODEL, - timeoutSeconds: params.timeoutSeconds, - freshness: params.freshness, - }); - - const payload = { - query: params.query, - provider: params.provider, - model: params.perplexityModel ?? DEFAULT_PERPLEXITY_MODEL, - tookMs: Date.now() - start, - externalContent: { - untrusted: true, - source: "web_search", - provider: params.provider, - wrapped: true, - }, - content: wrapWebContent(content, "web_search"), - citations, - }; - writeCache(SEARCH_CACHE, cacheKey, payload, params.cacheTtlMs); - return payload; - } - - const results = await runPerplexitySearchApi({ - query: params.query, - apiKey: params.apiKey, - count: params.count, - timeoutSeconds: params.timeoutSeconds, - country: params.country, - searchDomainFilter: params.searchDomainFilter, - searchRecencyFilter: params.freshness, - searchLanguageFilter: params.language ? [params.language] : undefined, - searchAfterDate: params.dateAfter ? isoToPerplexityDate(params.dateAfter) : undefined, - searchBeforeDate: params.dateBefore ? isoToPerplexityDate(params.dateBefore) : undefined, - maxTokens: params.maxTokens, - maxTokensPerPage: params.maxTokensPerPage, - }); - - const payload = { - query: params.query, - provider: params.provider, - count: results.length, - tookMs: Date.now() - start, - externalContent: { - untrusted: true, - source: "web_search", - provider: params.provider, - wrapped: true, - }, - results, - }; - writeCache(SEARCH_CACHE, cacheKey, payload, params.cacheTtlMs); - return payload; - } - - if (params.provider === "grok") { - const { content, citations, inlineCitations } = await runGrokSearch({ - query: params.query, - apiKey: params.apiKey, - model: params.grokModel ?? DEFAULT_GROK_MODEL, - timeoutSeconds: params.timeoutSeconds, - inlineCitations: params.grokInlineCitations ?? false, - }); - - const payload = { - query: params.query, - provider: params.provider, - model: params.grokModel ?? DEFAULT_GROK_MODEL, - tookMs: Date.now() - start, - externalContent: { - untrusted: true, - source: "web_search", - provider: params.provider, - wrapped: true, - }, - content: wrapWebContent(content), - citations, - inlineCitations, - }; - writeCache(SEARCH_CACHE, cacheKey, payload, params.cacheTtlMs); - return payload; - } - - if (params.provider === "kimi") { - const { content, citations } = await runKimiSearch({ - query: params.query, - apiKey: params.apiKey, - baseUrl: params.kimiBaseUrl ?? DEFAULT_KIMI_BASE_URL, - model: params.kimiModel ?? DEFAULT_KIMI_MODEL, - timeoutSeconds: params.timeoutSeconds, - }); - - const payload = { - query: params.query, - provider: params.provider, - model: params.kimiModel ?? DEFAULT_KIMI_MODEL, - tookMs: Date.now() - start, - externalContent: { - untrusted: true, - source: "web_search", - provider: params.provider, - wrapped: true, - }, - content: wrapWebContent(content), - citations, - }; - writeCache(SEARCH_CACHE, cacheKey, payload, params.cacheTtlMs); - return payload; - } - - if (params.provider === "gemini") { - const geminiResult = await runGeminiSearch({ - query: params.query, - apiKey: params.apiKey, - model: params.geminiModel ?? DEFAULT_GEMINI_MODEL, - timeoutSeconds: params.timeoutSeconds, - }); - - const payload = { - query: params.query, - provider: params.provider, - model: params.geminiModel ?? DEFAULT_GEMINI_MODEL, - tookMs: Date.now() - start, // Includes redirect URL resolution time - externalContent: { - untrusted: true, - source: "web_search", - provider: params.provider, - wrapped: true, - }, - content: wrapWebContent(geminiResult.content), - citations: geminiResult.citations, - }; - writeCache(SEARCH_CACHE, cacheKey, payload, params.cacheTtlMs); - return payload; - } - - if (params.provider !== "brave") { - throw new Error("Unsupported web search provider."); - } - - if (effectiveBraveMode === "llm-context") { - const { results: llmResults, sources } = await runBraveLlmContextSearch({ - query: params.query, - apiKey: params.apiKey, - timeoutSeconds: params.timeoutSeconds, - country: params.country, - search_lang: params.search_lang, - freshness: params.freshness, - }); - - const mapped = llmResults.map((entry) => ({ - title: entry.title ? wrapWebContent(entry.title, "web_search") : "", - url: entry.url, - snippets: entry.snippets.map((s) => wrapWebContent(s, "web_search")), - siteName: entry.siteName, - })); - - const payload = { - query: params.query, - provider: params.provider, - mode: "llm-context" as const, - count: mapped.length, - tookMs: Date.now() - start, - externalContent: { - untrusted: true, - source: "web_search", - provider: params.provider, - wrapped: true, - }, - results: mapped, - sources, - }; - writeCache(SEARCH_CACHE, cacheKey, payload, params.cacheTtlMs); - return payload; - } - - const url = new URL(BRAVE_SEARCH_ENDPOINT); - url.searchParams.set("q", params.query); - url.searchParams.set("count", String(params.count)); - if (params.country) { - url.searchParams.set("country", params.country); - } - if (params.search_lang || params.language) { - url.searchParams.set("search_lang", (params.search_lang || params.language)!); - } - if (params.ui_lang) { - url.searchParams.set("ui_lang", params.ui_lang); - } - if (params.freshness) { - url.searchParams.set("freshness", params.freshness); - } else if (params.dateAfter && params.dateBefore) { - url.searchParams.set("freshness", `${params.dateAfter}to${params.dateBefore}`); - } else if (params.dateAfter) { - url.searchParams.set( - "freshness", - `${params.dateAfter}to${new Date().toISOString().slice(0, 10)}`, - ); - } else if (params.dateBefore) { - url.searchParams.set("freshness", `1970-01-01to${params.dateBefore}`); - } - - const mapped = await withTrustedWebSearchEndpoint( - { - url: url.toString(), - timeoutSeconds: params.timeoutSeconds, - init: { - method: "GET", - headers: { - Accept: "application/json", - "X-Subscription-Token": params.apiKey, - }, - }, - }, - async (res) => { - if (!res.ok) { - const detailResult = await readResponseText(res, { maxBytes: 64_000 }); - const detail = detailResult.text; - throw new Error(`Brave Search API error (${res.status}): ${detail || res.statusText}`); - } - - const data = (await res.json()) as BraveSearchResponse; - const results = Array.isArray(data.web?.results) ? (data.web?.results ?? []) : []; - return results.map((entry) => { - const description = entry.description ?? ""; - const title = entry.title ?? ""; - const url = entry.url ?? ""; - const rawSiteName = resolveSiteName(url); - return { - title: title ? wrapWebContent(title, "web_search") : "", - url, // Keep raw for tool chaining - description: description ? wrapWebContent(description, "web_search") : "", - published: entry.age || undefined, - siteName: rawSiteName || undefined, - }; - }); - }, - ); - - const payload = { - query: params.query, - provider: params.provider, - count: mapped.length, - tookMs: Date.now() - start, - externalContent: { - untrusted: true, - source: "web_search", - provider: params.provider, - wrapped: true, - }, - results: mapped, - }; - writeCache(SEARCH_CACHE, cacheKey, payload, params.cacheTtlMs); - return payload; -} - function normalizeSearchProviderId(value: string | undefined): string { return value?.trim().toLowerCase() ?? ""; } function isBuiltinSearchProviderId(value: string): value is BuiltinWebSearchProviderId { - return isBuiltinWebSearchProviderId(value); + return isBuiltinWebSearchProviderIdFromCatalog(value); } function stableSerializeForCache(value: unknown): string { @@ -2049,6 +1655,24 @@ function executePluginSearchProvider(params: { request: SearchProviderRequest; context: SearchProviderContext; }): Promise> { + if (params.provider.pluginOwnedExecution) { + return params.provider.search(params.request, params.context).then((result) => { + if ("error" in result && typeof result.error === "string") { + const errorResult: SearchProviderErrorResult = result; + return { + ...errorResult, + provider: params.provider.id, + ...(typeof errorResult.message === "string" + ? {} + : { + message: `Search provider "${params.provider.id}" returned error "${errorResult.error}".`, + }), + }; + } + return result as Record; + }); + } + const pluginConfigKey = params.context.pluginConfig ? stableSerializeForCache(params.context.pluginConfig) : "no-plugin-config"; @@ -2168,347 +1792,19 @@ function executePluginSearchProvider(params: { })); } -function executeBuiltinSearchProvider(params: { - provider: BuiltinWebSearchProviderId; - request: SearchProviderRequest; - context: SearchProviderContext; -}): Promise> { - const search = params.request.providerConfig as WebSearchConfig | undefined; - const provider = params.provider; - const perplexityConfig = resolvePerplexityConfig(search); - const grokConfig = resolveGrokConfig(search); - const geminiConfig = resolveGeminiConfig(search); - const kimiConfig = resolveKimiConfig(search); - const braveConfig = resolveBraveConfig(search); - const braveMode = resolveBraveMode(braveConfig); - - const perplexityRuntime = - provider === "perplexity" ? resolvePerplexityTransport(perplexityConfig) : undefined; - const apiKey = - provider === "perplexity" - ? perplexityRuntime?.apiKey - : provider === "grok" - ? resolveGrokApiKey(grokConfig) - : provider === "kimi" - ? resolveKimiApiKey(kimiConfig) - : provider === "gemini" - ? resolveGeminiApiKey(geminiConfig) - : resolveSearchApiKey(search); - - if (!apiKey) { - return Promise.resolve(missingSearchKeyPayload(provider)); +function getRegisteredSearchProviders(config?: OpenClawConfig): SearchProviderPlugin[] { + let registry = getActivePluginRegistry(); + if (!registry || registry.searchProviders.length === 0) { + registry = loadOpenClawPlugins({ config }); } - - const supportsStructuredPerplexityFilters = - provider === "perplexity" && perplexityRuntime?.transport === "search_api"; - if ( - params.request.country && - provider !== "brave" && - !(provider === "perplexity" && supportsStructuredPerplexityFilters) - ) { - return Promise.resolve({ - error: "unsupported_country", - message: - provider === "perplexity" - ? "country filtering is only supported by the native Perplexity Search API path. Remove Perplexity baseUrl/model overrides or use a direct PERPLEXITY_API_KEY to enable it." - : `country filtering is not supported by the ${provider} provider. Only Brave and Perplexity support country filtering.`, - docs: "https://docs.openclaw.ai/tools/web", - }); - } - if ( - params.request.language && - provider !== "brave" && - !(provider === "perplexity" && supportsStructuredPerplexityFilters) - ) { - return Promise.resolve({ - error: "unsupported_language", - message: - provider === "perplexity" - ? "language filtering is only supported by the native Perplexity Search API path. Remove Perplexity baseUrl/model overrides or use a direct PERPLEXITY_API_KEY to enable it." - : `language filtering is not supported by the ${provider} provider. Only Brave and Perplexity support language filtering.`, - docs: "https://docs.openclaw.ai/tools/web", - }); - } - if ( - params.request.language && - provider === "perplexity" && - !/^[a-z]{2}$/i.test(params.request.language) - ) { - return Promise.resolve({ - error: "invalid_language", - message: "language must be a 2-letter ISO 639-1 code like 'en', 'de', or 'fr'.", - docs: "https://docs.openclaw.ai/tools/web", - }); - } - const normalizedBraveLanguageParams = - provider === "brave" - ? normalizeBraveLanguageParams({ - search_lang: params.request.search_lang || params.request.language, - ui_lang: params.request.ui_lang, - }) - : { search_lang: params.request.language, ui_lang: params.request.ui_lang }; - if (normalizedBraveLanguageParams.invalidField === "search_lang") { - return Promise.resolve({ - error: "invalid_search_lang", - message: - "search_lang must be a Brave-supported language code like 'en', 'en-gb', 'zh-hans', or 'zh-hant'.", - docs: "https://docs.openclaw.ai/tools/web", - }); - } - if (normalizedBraveLanguageParams.invalidField === "ui_lang") { - return Promise.resolve({ - error: "invalid_ui_lang", - message: "ui_lang must be a language-region locale like 'en-US'.", - docs: "https://docs.openclaw.ai/tools/web", - }); - } - if ( - normalizedBraveLanguageParams.ui_lang && - provider === "brave" && - braveMode === "llm-context" - ) { - return Promise.resolve({ - error: "unsupported_ui_lang", - message: - "ui_lang is not supported by Brave llm-context mode. Remove ui_lang or use Brave web mode for locale-based UI hints.", - docs: "https://docs.openclaw.ai/tools/web", - }); - } - if (params.request.freshness && provider !== "brave" && provider !== "perplexity") { - return Promise.resolve({ - error: "unsupported_freshness", - message: `freshness filtering is not supported by the ${provider} provider. Only Brave and Perplexity support freshness.`, - docs: "https://docs.openclaw.ai/tools/web", - }); - } - if (params.request.freshness && provider === "brave" && braveMode === "llm-context") { - return Promise.resolve({ - error: "unsupported_freshness", - message: - "freshness filtering is not supported by Brave llm-context mode. Remove freshness or use Brave web mode.", - docs: "https://docs.openclaw.ai/tools/web", - }); - } - const normalizedFreshness = params.request.freshness - ? normalizeFreshness(params.request.freshness, provider) - : undefined; - if (params.request.freshness && !normalizedFreshness) { - return Promise.resolve({ - error: "invalid_freshness", - message: "freshness must be day, week, month, or year.", - docs: "https://docs.openclaw.ai/tools/web", - }); - } - if ( - (params.request.dateAfter || params.request.dateBefore) && - provider !== "brave" && - !(provider === "perplexity" && supportsStructuredPerplexityFilters) - ) { - return Promise.resolve({ - error: "unsupported_date_filter", - message: - provider === "perplexity" - ? "date_after/date_before are only supported by the native Perplexity Search API path. Remove Perplexity baseUrl/model overrides or use a direct PERPLEXITY_API_KEY to enable them." - : `date_after/date_before filtering is not supported by the ${provider} provider. Only Brave and Perplexity support date filtering.`, - docs: "https://docs.openclaw.ai/tools/web", - }); - } - if ( - (params.request.dateAfter || params.request.dateBefore) && - provider === "brave" && - braveMode === "llm-context" - ) { - return Promise.resolve({ - error: "unsupported_date_filter", - message: - "date_after/date_before filtering is not supported by Brave llm-context mode. Use Brave web mode for date filters.", - docs: "https://docs.openclaw.ai/tools/web", - }); - } - if (params.request.domainFilter && params.request.domainFilter.length > 0) { - const hasDenylist = params.request.domainFilter.some((domain) => domain.startsWith("-")); - const hasAllowlist = params.request.domainFilter.some((domain) => !domain.startsWith("-")); - if (hasDenylist && hasAllowlist) { - return Promise.resolve({ - error: "invalid_domain_filter", - message: - "domain_filter cannot mix allowlist and denylist entries. Use either all positive entries (allowlist) or all entries prefixed with '-' (denylist).", - docs: "https://docs.openclaw.ai/tools/web", - }); - } - if (params.request.domainFilter.length > 20) { - return Promise.resolve({ - error: "invalid_domain_filter", - message: "domain_filter supports a maximum of 20 domains.", - docs: "https://docs.openclaw.ai/tools/web", - }); - } - } - if ( - params.request.domainFilter && - params.request.domainFilter.length > 0 && - !(provider === "perplexity" && supportsStructuredPerplexityFilters) - ) { - return Promise.resolve({ - error: "unsupported_domain_filter", - message: - provider === "perplexity" - ? "domain_filter is only supported by the native Perplexity Search API path. Remove Perplexity baseUrl/model overrides or use a direct PERPLEXITY_API_KEY to enable it." - : `domain_filter is not supported by the ${provider} provider. Only Perplexity supports domain filtering.`, - docs: "https://docs.openclaw.ai/tools/web", - }); - } - if ( - provider === "perplexity" && - perplexityRuntime?.transport === "chat_completions" && - (params.request.maxTokens !== undefined || params.request.maxTokensPerPage !== undefined) - ) { - return Promise.resolve({ - error: "unsupported_content_budget", - message: - "max_tokens and max_tokens_per_page are only supported by the native Perplexity Search API path. Remove Perplexity baseUrl/model overrides or use a direct PERPLEXITY_API_KEY to enable them.", - docs: "https://docs.openclaw.ai/tools/web", - }); - } - - return runWebSearch({ - query: params.request.query, - count: params.request.count, - apiKey, - timeoutSeconds: params.context.timeoutSeconds, - cacheTtlMs: params.context.cacheTtlMs, - provider, - country: params.request.country, - language: params.request.language, - search_lang: normalizedBraveLanguageParams.search_lang, - ui_lang: normalizedBraveLanguageParams.ui_lang, - freshness: normalizedFreshness, - dateAfter: params.request.dateAfter, - dateBefore: params.request.dateBefore, - searchDomainFilter: params.request.domainFilter, - maxTokens: params.request.maxTokens, - maxTokensPerPage: params.request.maxTokensPerPage, - perplexityBaseUrl: perplexityRuntime?.baseUrl, - perplexityModel: perplexityRuntime?.model, - perplexityTransport: perplexityRuntime?.transport, - grokModel: resolveGrokModel(grokConfig), - grokInlineCitations: resolveGrokInlineCitations(grokConfig), - geminiModel: resolveGeminiModel(geminiConfig), - kimiBaseUrl: resolveKimiBaseUrl(kimiConfig), - kimiModel: resolveKimiModel(kimiConfig), - braveMode, - }); -} - -function getBuiltinSearchProviders(search?: WebSearchConfig): SearchProviderPlugin[] { - const braveMode = resolveBraveMode(resolveBraveConfig(search)); - return [ - { - id: "brave", - name: braveMode === "llm-context" ? "Brave LLM Context" : "Brave Search", - description: - braveMode === "llm-context" - ? "Search the web using Brave Search LLM Context API. Returns pre-extracted page content (text chunks, tables, code blocks) optimized for LLM grounding." - : "Search the web using Brave Search API. Supports region-specific and localized search via country and language parameters. Returns titles, URLs, and snippets for fast research.", - isAvailable: (config) => Boolean(resolveSearchApiKey(resolveSearchConfig(config))), - search: (request, context) => - executeBuiltinSearchProvider({ provider: "brave", request, context }), - }, - { - id: "gemini", - name: "Gemini Search", - description: - "Search the web using Gemini with Google Search grounding. Returns AI-synthesized answers with citations from Google Search.", - isAvailable: (config) => - Boolean(resolveGeminiApiKey(resolveGeminiConfig(resolveSearchConfig(config)))), - search: (request, context) => - executeBuiltinSearchProvider({ provider: "gemini", request, context }), - }, - { - id: "grok", - name: "xAI Grok", - description: - "Search the web using xAI Grok. Returns AI-synthesized answers with citations from real-time web search.", - isAvailable: (config) => - Boolean(resolveGrokApiKey(resolveGrokConfig(resolveSearchConfig(config)))), - search: (request, context) => - executeBuiltinSearchProvider({ provider: "grok", request, context }), - }, - { - id: "kimi", - name: "Kimi by Moonshot", - description: - "Search the web using Kimi by Moonshot. Returns AI-synthesized answers with citations from native $web_search.", - isAvailable: (config) => - Boolean(resolveKimiApiKey(resolveKimiConfig(resolveSearchConfig(config)))), - search: (request, context) => - executeBuiltinSearchProvider({ provider: "kimi", request, context }), - }, - { - id: "perplexity", - name: "Perplexity", - description: - resolvePerplexitySchemaTransportHint(resolvePerplexityConfig(search)) === "chat_completions" - ? "Search the web using Perplexity Sonar via Perplexity/OpenRouter chat completions. Returns AI-synthesized answers with citations from web-grounded search." - : "Search the web using Perplexity. Runtime routing decides between native Search API and Sonar chat-completions compatibility. Structured filters are available on the native Search API path.", - isAvailable: (config) => { - const searchConfig = resolveSearchConfig(config); - return Boolean(resolvePerplexityApiKey(resolvePerplexityConfig(searchConfig)).apiKey); - }, - search: (request, context) => - executeBuiltinSearchProvider({ provider: "perplexity", request, context }), - }, - ]; -} - -export function createBundledBuiltinSearchProvider( - providerId: BuiltinWebSearchProviderId, -): SearchProviderPlugin { - const providers = getBuiltinSearchProviders(); - switch (providerId) { - case "brave": - return { - ...providers[0], - builtinProviderId: "brave", - }; - case "gemini": - return { - ...providers[1], - builtinProviderId: "gemini", - }; - case "grok": - return { - ...providers[2], - builtinProviderId: "grok", - }; - case "kimi": - return { - ...providers[3], - builtinProviderId: "kimi", - }; - case "perplexity": - return { - ...providers[4], - builtinProviderId: "perplexity", - }; - } -} - -function getPluginSearchProviders(): SearchProviderPlugin[] { - return getActivePluginRegistry()?.searchProviders.map((entry) => entry.provider) ?? []; + return registry.searchProviders.map((entry) => entry.provider); } function resolveBuiltinSchemaProviderId( provider: SearchProviderPlugin, ): BuiltinWebSearchProviderId | undefined { - if (provider.builtinProviderId && isBuiltinSearchProviderId(provider.builtinProviderId)) { - return provider.builtinProviderId; - } - if (!provider.pluginId) { - const candidate = normalizeSearchProviderId(provider.id); - return isBuiltinSearchProviderId(candidate) ? candidate : undefined; - } - return undefined; + const candidate = normalizeSearchProviderId(provider.id); + return isBuiltinSearchProviderId(candidate) ? candidate : undefined; } function resolveConfiguredSearchProviderId(params: { @@ -2570,23 +1866,24 @@ function resolveRegisteredSearchProvider(params: { search: params.search, }) ?? undefined, ); - const builtinProviders = new Map( - getBuiltinSearchProviders(params.search).map((provider) => [provider.id, provider]), - ); - const pluginProviders = new Map( - getPluginSearchProviders().map((provider) => [ + const registeredProviders = new Map( + getRegisteredSearchProviders(params.config).map((provider) => [ normalizeSearchProviderId(provider.id), provider, ]), ); if (configuredProviderId) { - const pluginProvider = pluginProviders.get(configuredProviderId); - if (pluginProvider) { - return pluginProvider; + const registeredProvider = registeredProviders.get(configuredProviderId); + if (registeredProvider) { + return registeredProvider; } + logVerbose( + `web_search: configured provider "${configuredProviderId}" is not registered; failing closed`, + ); + return createMissingSearchProviderPlugin(configuredProviderId); } else { - for (const provider of pluginProviders.values()) { + for (const provider of registeredProviders.values()) { let isAvailable = false; try { isAvailable = provider.isAvailable?.(params.config) ?? false; @@ -2604,22 +1901,15 @@ function resolveRegisteredSearchProvider(params: { } } } - - if (configuredProviderId && !isBuiltinSearchProviderId(configuredProviderId)) { - logVerbose( - `web_search: configured plugin provider "${configuredProviderId}" is not registered; failing closed`, - ); - return createMissingSearchProviderPlugin(configuredProviderId); - } - + const preferredBuiltinProvider = resolvePreferredBuiltinSearchProvider({ + config: params.config, + search: params.search, + runtimeWebSearch: params.runtimeWebSearch, + }); return ( - builtinProviders.get( - resolvePreferredBuiltinSearchProvider({ - config: params.config, - search: params.search, - runtimeWebSearch: params.runtimeWebSearch, - }), - ) ?? builtinProviders.get(DEFAULT_PROVIDER)! + registeredProviders.get(preferredBuiltinProvider) ?? + registeredProviders.get(DEFAULT_PROVIDER) ?? + createMissingSearchProviderPlugin(preferredBuiltinProvider) ); } @@ -2765,35 +2055,18 @@ export function createWebSearchTool(options?: { }); } - const builtinProviderId = resolveBuiltinSchemaProviderId(provider); logVerbose(formatWebSearchExecutionLog(provider)); - const result = builtinProviderId - ? await executeBuiltinSearchProvider({ - provider: builtinProviderId, - request, - context: { - config: options?.config ?? {}, - timeoutSeconds: resolveTimeoutSeconds( - search?.timeoutSeconds, - DEFAULT_TIMEOUT_SECONDS, - ), - cacheTtlMs: resolveCacheTtlMs(search?.cacheTtlMinutes, DEFAULT_CACHE_TTL_MINUTES), - pluginConfig: resolveSearchProviderPluginConfig(options?.config, provider), - }, - }) - : await executePluginSearchProvider({ - provider, - request, - context: { - config: options?.config ?? {}, - timeoutSeconds: resolveTimeoutSeconds( - search?.timeoutSeconds, - DEFAULT_TIMEOUT_SECONDS, - ), - cacheTtlMs: resolveCacheTtlMs(search?.cacheTtlMinutes, DEFAULT_CACHE_TTL_MINUTES), - pluginConfig: resolveSearchProviderPluginConfig(options?.config, provider), - }, - }); + const context = { + config: options?.config ?? {}, + timeoutSeconds: resolveTimeoutSeconds(search?.timeoutSeconds, DEFAULT_TIMEOUT_SECONDS), + cacheTtlMs: resolveCacheTtlMs(search?.cacheTtlMinutes, DEFAULT_CACHE_TTL_MINUTES), + pluginConfig: resolveSearchProviderPluginConfig(options?.config, provider), + }; + const result = await executePluginSearchProvider({ + provider, + request, + context, + }); return jsonResult(result); }, }; diff --git a/src/agents/tools/web-tools.enabled-defaults.test.ts b/src/agents/tools/web-tools.enabled-defaults.test.ts index be3f138d396..91ab14c196a 100644 --- a/src/agents/tools/web-tools.enabled-defaults.test.ts +++ b/src/agents/tools/web-tools.enabled-defaults.test.ts @@ -1,5 +1,25 @@ import { EnvHttpProxyAgent } from "undici"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { + createBundledBraveSearchProvider, + __testing as bundledBraveTesting, +} from "../../../extensions/search-brave/src/provider.js"; +import { + createBundledGeminiSearchProvider, + __testing as bundledGeminiTesting, +} from "../../../extensions/search-gemini/src/provider.js"; +import { + createBundledGrokSearchProvider, + __testing as bundledGrokTesting, +} from "../../../extensions/search-grok/src/provider.js"; +import { + createBundledKimiSearchProvider, + __testing as bundledKimiTesting, +} from "../../../extensions/search-kimi/src/provider.js"; +import { + createBundledPerplexitySearchProvider, + __testing as bundledPerplexityTesting, +} from "../../../extensions/search-perplexity/src/provider.js"; import { createEmptyPluginRegistry } from "../../plugins/registry.js"; import { getActivePluginRegistry, setActivePluginRegistry } from "../../plugins/runtime.js"; import { withFetchPreconnect } from "../../test-utils/fetch-mock.js"; @@ -8,9 +28,41 @@ import { createWebFetchTool, createWebSearchTool } from "./web-tools.js"; let previousPluginRegistry = getActivePluginRegistry(); +const BUNDLED_PROVIDER_CREATORS = { + brave: createBundledBraveSearchProvider, + gemini: createBundledGeminiSearchProvider, + grok: createBundledGrokSearchProvider, + kimi: createBundledKimiSearchProvider, + perplexity: createBundledPerplexitySearchProvider, +} as const; + beforeEach(() => { previousPluginRegistry = getActivePluginRegistry(); - setActivePluginRegistry(createEmptyPluginRegistry()); + const registry = createEmptyPluginRegistry(); + ( + Object.entries(BUNDLED_PROVIDER_CREATORS) as Array< + [ + keyof typeof BUNDLED_PROVIDER_CREATORS, + (typeof BUNDLED_PROVIDER_CREATORS)[keyof typeof BUNDLED_PROVIDER_CREATORS], + ] + > + ).forEach(([providerId, createProvider]) => { + registry.searchProviders.push({ + pluginId: `search-${providerId}`, + source: `/plugins/search-${providerId}`, + provider: { + ...createProvider(), + pluginId: `search-${providerId}`, + }, + }); + }); + setActivePluginRegistry(registry); + webSearchTesting.SEARCH_CACHE.clear(); + bundledBraveTesting.clearSearchProviderCaches(); + bundledPerplexityTesting.clearSearchProviderCaches(); + bundledGrokTesting.clearSearchProviderCaches(); + bundledGeminiTesting.clearSearchProviderCaches(); + bundledKimiTesting.clearSearchProviderCaches(); }); afterEach(() => { @@ -214,16 +266,13 @@ describe("web_search plugin providers", () => { "resolves configured built-in provider %s through bundled plugin registrations when available", async (providerId) => { const registry = createEmptyPluginRegistry(); + const bundledProvider = BUNDLED_PROVIDER_CREATORS[providerId](); registry.searchProviders.push({ pluginId: `search-${providerId}`, source: `/plugins/search-${providerId}`, provider: { - id: providerId, - name: `${providerId} bundled provider`, + ...bundledProvider, pluginId: `search-${providerId}`, - builtinProviderId: providerId, - isAvailable: () => true, - search: async () => ({ content: "unused" }), }, }); setActivePluginRegistry(registry); @@ -706,6 +755,11 @@ describe("web_search perplexity Search API", () => { vi.unstubAllEnvs(); global.fetch = priorFetch; webSearchTesting.SEARCH_CACHE.clear(); + bundledBraveTesting.clearSearchProviderCaches(); + bundledPerplexityTesting.clearSearchProviderCaches(); + bundledGrokTesting.clearSearchProviderCaches(); + bundledGeminiTesting.clearSearchProviderCaches(); + bundledKimiTesting.clearSearchProviderCaches(); }); it("uses Perplexity Search API when PERPLEXITY_API_KEY is set", async () => { @@ -843,6 +897,11 @@ describe("web_search perplexity OpenRouter compatibility", () => { vi.unstubAllEnvs(); global.fetch = priorFetch; webSearchTesting.SEARCH_CACHE.clear(); + bundledBraveTesting.clearSearchProviderCaches(); + bundledPerplexityTesting.clearSearchProviderCaches(); + bundledGrokTesting.clearSearchProviderCaches(); + bundledGeminiTesting.clearSearchProviderCaches(); + bundledKimiTesting.clearSearchProviderCaches(); }); it("routes OPENROUTER_API_KEY through chat completions", async () => { diff --git a/src/commands/onboard-search.test.ts b/src/commands/onboard-search.test.ts index 192158c9d1c..a03f09f1216 100644 --- a/src/commands/onboard-search.test.ts +++ b/src/commands/onboard-search.test.ts @@ -260,7 +260,6 @@ describe("setupSearch", () => { name: providerLabel, description: `Bundled ${providerLabel} provider`, pluginId: `search-${providerId}`, - builtinProviderId: providerId, isAvailable: () => true, search: async () => ({ content: "ok" }), }, diff --git a/src/commands/onboard-search.ts b/src/commands/onboard-search.ts index 110a4a2b161..9220c2d09b5 100644 --- a/src/commands/onboard-search.ts +++ b/src/commands/onboard-search.ts @@ -13,6 +13,10 @@ import { hasConfiguredSecretInput, normalizeSecretInputString, } from "../config/types.secrets.js"; +import { + readSearchProviderApiKeyValue, + writeSearchProviderApiKeyValue, +} from "../plugin-sdk/web-search.js"; import { applyCapabilitySlotSelection, resolveCapabilitySlotSelection, @@ -22,7 +26,11 @@ import { createHookRunner, type HookRunner } from "../plugins/hooks.js"; import { loadOpenClawPlugins } from "../plugins/loader.js"; import { loadPluginManifestRegistry } from "../plugins/manifest-registry.js"; import { validateJsonSchemaValue } from "../plugins/schema-validator.js"; -import type { PluginConfigUiHint, PluginOrigin } from "../plugins/types.js"; +import type { + PluginConfigUiHint, + PluginOrigin, + SearchProviderLegacyConfigMetadata, +} from "../plugins/types.js"; import type { RuntimeEnv } from "../runtime.js"; import type { WizardPrompter } from "../wizard/prompts.js"; import type { SecretInputMode } from "./onboard-types.js"; @@ -67,6 +75,7 @@ type PluginSearchProviderEntry = { configFieldOrder?: string[]; configJsonSchema?: Record; configUiHints?: Record; + legacyConfig?: SearchProviderLegacyConfigMetadata; }; export type SearchProviderPickerEntry = @@ -110,6 +119,21 @@ type SearchProviderHookDetails = { configured: boolean; }; +function legacyConfigFromBuiltinEntry( + entry: SearchProviderEntry, +): SearchProviderLegacyConfigMetadata { + return { + hint: entry.hint, + envKeys: entry.envKeys, + placeholder: entry.placeholder, + signupUrl: entry.signupUrl, + apiKeyConfigPath: entry.apiKeyConfigPath, + readApiKeyValue: (search) => readSearchProviderApiKeyValue(search, entry.value), + writeApiKeyValue: (search, value) => + writeSearchProviderApiKeyValue({ search, provider: entry.value, value }), + }; +} + const HOOK_RUNNER_LOGGER = { warn: () => {}, error: () => {}, @@ -527,7 +551,7 @@ export async function resolveSearchProviderPickerEntries( ).map((entry) => ({ ...entry, kind: "builtin", - configured: hasExistingKey(config, entry.value) || hasKeyInEnv(entry), + configured: hasExistingKey(config, legacyConfigFromBuiltinEntry(entry)) || hasKeyInEnv(entry), })); let pluginEntries: PluginSearchProviderEntry[] = []; @@ -553,6 +577,7 @@ export async function resolveSearchProviderPickerEntries( const sourceHint = formatPluginSourceHint(pluginRecord.origin); const baseHint = + registration.provider.legacyConfig?.hint?.trim() || registration.provider.description?.trim() || pluginRecord.description?.trim() || "Plugin-provided web search"; @@ -573,6 +598,7 @@ export async function resolveSearchProviderPickerEntries( configFieldOrder: registration.provider.configFieldOrder, configJsonSchema: pluginRecord.configJsonSchema, configUiHints: pluginRecord.configUiHints, + legacyConfig: registration.provider.legacyConfig, }; }) .filter((entry) => { @@ -1022,6 +1048,27 @@ export async function configureSearchProviderSelection( intent === "switch-active" ? setWebSearchProvider(enabled.config, selectedEntry.value) : enabled.config; + const legacyConfig = selectedEntry.legacyConfig; + const existingKey = legacyConfig ? resolveExistingKey(config, legacyConfig) : undefined; + const keyConfigured = legacyConfig ? hasExistingKey(config, legacyConfig) : false; + const envAvailable = + legacyConfig?.envKeys?.some((key) => Boolean(process.env[key]?.trim())) ?? false; + + if (legacyConfig && intent === "switch-active" && (keyConfigured || envAvailable)) { + const result = existingKey + ? applySearchKey(config, selectedEntry.value as SearchProvider, legacyConfig, existingKey) + : applyProviderOnly(config, selectedEntry.value as SearchProvider); + const nextConfig = preserveSearchProviderIntent(config, result, intent, selectedEntry.value); + await runAfterSearchProviderHooks({ + hookRunner, + originalConfig: config, + resultConfig: nextConfig, + provider: providerDetails, + intent, + workspaceDir: opts?.workspaceDir, + }); + return nextConfig; + } if (selectedEntry.configured) { const result = preserveSearchProviderIntent(config, next, intent, selectedEntry.value); await runAfterSearchProviderHooks({ @@ -1054,6 +1101,127 @@ export async function configureSearchProviderSelection( prompter, workspaceDir: opts?.workspaceDir, }); + if (legacyConfig) { + const useSecretRefMode = opts?.secretInputMode === "ref"; // pragma: allowlist secret + if (useSecretRefMode) { + if (keyConfigured) { + return preserveSearchProviderIntent( + config, + applyProviderOnly(config, selectedEntry.value as SearchProvider), + intent, + selectedEntry.value, + ); + } + const ref = buildSearchEnvRef(legacyConfig); + await prompter.note( + [ + "Secret references enabled — OpenClaw will store a reference instead of the API key.", + `Env var: ${ref.id}${envAvailable ? " (detected)" : ""}.`, + ...(envAvailable ? [] : [`Set ${ref.id} in the Gateway environment.`]), + "Docs: https://docs.openclaw.ai/tools/web", + ].join("\n"), + "Web search", + ); + const result = preserveSearchProviderIntent( + config, + applySearchKey(config, selectedEntry.value as SearchProvider, legacyConfig, ref), + intent, + selectedEntry.value, + ); + await runAfterSearchProviderHooks({ + hookRunner, + originalConfig: config, + resultConfig: result, + provider: providerDetails, + intent, + workspaceDir: opts?.workspaceDir, + }); + return result; + } + + const keyInput = await prompter.text({ + message: keyConfigured + ? `${selectedEntry.label} API key (leave blank to keep current)` + : envAvailable + ? `${selectedEntry.label} API key (leave blank to use env var)` + : `${selectedEntry.label} API key`, + placeholder: keyConfigured ? "Leave blank to keep current" : legacyConfig.placeholder, + }); + + const key = keyInput?.trim() ?? ""; + if (key) { + const secretInput = resolveSearchSecretInput( + selectedEntry.value as SearchProvider, + legacyConfig, + key, + opts?.secretInputMode, + ); + const result = preserveSearchProviderIntent( + config, + applySearchKey(config, selectedEntry.value as SearchProvider, legacyConfig, secretInput), + intent, + selectedEntry.value, + ); + await runAfterSearchProviderHooks({ + hookRunner, + originalConfig: config, + resultConfig: result, + provider: providerDetails, + intent, + workspaceDir: opts?.workspaceDir, + }); + return result; + } + + if (existingKey) { + const result = preserveSearchProviderIntent( + config, + applySearchKey(config, selectedEntry.value as SearchProvider, legacyConfig, existingKey), + intent, + selectedEntry.value, + ); + await runAfterSearchProviderHooks({ + hookRunner, + originalConfig: config, + resultConfig: result, + provider: providerDetails, + intent, + workspaceDir: opts?.workspaceDir, + }); + return result; + } + + if (keyConfigured || envAvailable) { + const result = preserveSearchProviderIntent( + config, + applyProviderOnly(config, selectedEntry.value as SearchProvider), + intent, + selectedEntry.value, + ); + await runAfterSearchProviderHooks({ + hookRunner, + originalConfig: config, + resultConfig: result, + provider: providerDetails, + intent, + workspaceDir: opts?.workspaceDir, + }); + return result; + } + + await prompter.note( + [ + `Get your key at: ${legacyConfig.signupUrl}`, + envAvailable + ? `OpenClaw can also use ${legacyConfig.envKeys?.find((k) => Boolean(process.env[k]?.trim()))}.` + : undefined, + ] + .filter(Boolean) + .join("\n"), + selectedEntry.label, + ); + return config; + } const pluginConfigResult = await promptPluginSearchProviderConfig( next, selectedEntry, @@ -1084,19 +1252,20 @@ export async function configureSearchProviderSelection( return config; } const hookRunner = createSearchProviderHookRunner(config, opts?.workspaceDir); + const builtinLegacyConfig = legacyConfigFromBuiltinEntry(entry); const providerDetails: SearchProviderHookDetails = { providerId: builtinChoice, providerLabel: entry.label, providerSource: "builtin", - configured: hasExistingKey(config, builtinChoice) || hasKeyInEnv(entry), + configured: hasExistingKey(config, builtinLegacyConfig) || hasKeyInEnv(entry), }; - const existingKey = resolveExistingKey(config, builtinChoice); - const keyConfigured = hasExistingKey(config, builtinChoice); + const existingKey = resolveExistingKey(config, builtinLegacyConfig); + const keyConfigured = hasExistingKey(config, builtinLegacyConfig); const envAvailable = hasKeyInEnv(entry); if (intent === "switch-active" && (keyConfigured || envAvailable)) { const result = existingKey - ? applySearchKey(config, builtinChoice, existingKey) + ? applySearchKey(config, builtinChoice, builtinLegacyConfig, existingKey) : applyProviderOnly(config, builtinChoice); const next = preserveSearchProviderIntent(config, result, intent, builtinChoice); await runAfterSearchProviderHooks({ @@ -1112,7 +1281,7 @@ export async function configureSearchProviderSelection( if (opts?.quickstartDefaults && (keyConfigured || envAvailable)) { const result = existingKey - ? applySearchKey(config, builtinChoice, existingKey) + ? applySearchKey(config, builtinChoice, builtinLegacyConfig, existingKey) : applyProviderOnly(config, builtinChoice); const next = preserveSearchProviderIntent(config, result, intent, builtinChoice); await runAfterSearchProviderHooks({ @@ -1140,7 +1309,7 @@ export async function configureSearchProviderSelection( if (keyConfigured) { return preserveDisabledState(config, applyProviderOnly(config, builtinChoice)); } - const ref = buildSearchEnvRef(builtinChoice); + const ref = buildSearchEnvRef(builtinLegacyConfig); await prompter.note( [ "Secret references enabled — OpenClaw will store a reference instead of the API key.", @@ -1152,7 +1321,7 @@ export async function configureSearchProviderSelection( ); const result = preserveSearchProviderIntent( config, - applySearchKey(config, builtinChoice, ref), + applySearchKey(config, builtinChoice, builtinLegacyConfig, ref), intent, builtinChoice, ); @@ -1178,10 +1347,15 @@ export async function configureSearchProviderSelection( const key = keyInput?.trim() ?? ""; if (key) { - const secretInput = resolveSearchSecretInput(builtinChoice, key, opts?.secretInputMode); + const secretInput = resolveSearchSecretInput( + builtinChoice, + builtinLegacyConfig, + key, + opts?.secretInputMode, + ); const result = preserveSearchProviderIntent( config, - applySearchKey(config, builtinChoice, secretInput), + applySearchKey(config, builtinChoice, builtinLegacyConfig, secretInput), intent, builtinChoice, ); @@ -1199,7 +1373,7 @@ export async function configureSearchProviderSelection( if (existingKey) { const result = preserveSearchProviderIntent( config, - applySearchKey(config, builtinChoice, existingKey), + applySearchKey(config, builtinChoice, builtinLegacyConfig, existingKey), intent, builtinChoice, ); @@ -1355,43 +1529,38 @@ export function hasKeyInEnv(entry: SearchProviderEntry): boolean { return entry.envKeys.some((k) => Boolean(process.env[k]?.trim())); } -function rawKeyValue(config: OpenClawConfig, provider: SearchProvider): unknown { +function rawKeyValue( + config: OpenClawConfig, + metadata: SearchProviderLegacyConfigMetadata, +): unknown { const search = config.tools?.web?.search; - switch (provider) { - case "brave": - return search?.apiKey; - case "gemini": - return search?.gemini?.apiKey; - case "grok": - return search?.grok?.apiKey; - case "kimi": - return search?.kimi?.apiKey; - case "perplexity": - return search?.perplexity?.apiKey; - } + return search && typeof search === "object" && metadata.readApiKeyValue + ? metadata.readApiKeyValue(search as Record) + : undefined; } /** Returns the plaintext key string, or undefined for SecretRefs/missing. */ export function resolveExistingKey( config: OpenClawConfig, - provider: SearchProvider, + metadata: SearchProviderLegacyConfigMetadata, ): string | undefined { - return normalizeSecretInputString(rawKeyValue(config, provider)); + return normalizeSecretInputString(rawKeyValue(config, metadata)); } /** Returns true if a key is configured (plaintext string or SecretRef). */ -export function hasExistingKey(config: OpenClawConfig, provider: SearchProvider): boolean { - return hasConfiguredSecretInput(rawKeyValue(config, provider)); +export function hasExistingKey( + config: OpenClawConfig, + metadata: SearchProviderLegacyConfigMetadata, +): boolean { + return hasConfiguredSecretInput(rawKeyValue(config, metadata)); } /** Build an env-backed SecretRef for a search provider. */ -function buildSearchEnvRef(provider: SearchProvider): SecretRef { - const entry = SEARCH_PROVIDER_OPTIONS.find((e) => e.value === provider); - const envVar = entry?.envKeys.find((k) => Boolean(process.env[k]?.trim())) ?? entry?.envKeys[0]; +function buildSearchEnvRef(metadata: SearchProviderLegacyConfigMetadata): SecretRef { + const envVar = + metadata.envKeys?.find((k) => Boolean(process.env[k]?.trim())) ?? metadata.envKeys?.[0]; if (!envVar) { - throw new Error( - `No env var mapping for search provider "${provider}" in secret-input-mode=ref.`, - ); + throw new Error("No env var mapping for search provider in secret-input-mode=ref."); } return { source: "env", provider: DEFAULT_SECRET_PROVIDER_ALIAS, id: envVar }; } @@ -1399,12 +1568,13 @@ function buildSearchEnvRef(provider: SearchProvider): SecretRef { /** Resolve a plaintext key into the appropriate SecretInput based on mode. */ function resolveSearchSecretInput( provider: SearchProvider, + metadata: SearchProviderLegacyConfigMetadata, key: string, secretInputMode?: SecretInputMode, ): SecretInput { const useSecretRefMode = secretInputMode === "ref"; // pragma: allowlist secret if (useSecretRefMode) { - return buildSearchEnvRef(provider); + return buildSearchEnvRef(metadata); } return key; } @@ -1412,26 +1582,11 @@ function resolveSearchSecretInput( export function applySearchKey( config: OpenClawConfig, provider: SearchProvider, + metadata: SearchProviderLegacyConfigMetadata, key: SecretInput, ): OpenClawConfig { const search = { ...config.tools?.web?.search, provider, enabled: true }; - switch (provider) { - case "brave": - search.apiKey = key; - break; - case "gemini": - search.gemini = { ...search.gemini, apiKey: key }; - break; - case "grok": - search.grok = { ...search.grok, apiKey: key }; - break; - case "kimi": - search.kimi = { ...search.kimi, apiKey: key }; - break; - case "perplexity": - search.perplexity = { ...search.perplexity, apiKey: key }; - break; - } + metadata.writeApiKeyValue?.(search as Record, key); return { ...config, tools: { diff --git a/src/plugin-sdk/index.test.ts b/src/plugin-sdk/index.test.ts index 6cbeb9440e7..ab2bafac1e4 100644 --- a/src/plugin-sdk/index.test.ts +++ b/src/plugin-sdk/index.test.ts @@ -107,6 +107,12 @@ describe("plugin-sdk exports", () => { "probeTelegram", "probeIMessage", "probeSignal", + "createBundledSearchProviderAdapter", + "createBundledBraveSearchProvider", + "createBundledGeminiSearchProvider", + "createBundledGrokSearchProvider", + "createBundledKimiSearchProvider", + "createBundledPerplexitySearchProvider", ]; for (const key of forbidden) { @@ -139,7 +145,6 @@ describe("plugin-sdk exports", () => { "formatInboundFromLabel", "resolveRuntimeGroupPolicy", "emptyPluginConfigSchema", - "createBundledBuiltinSearchProvider", "normalizePluginHttpPath", "registerPluginHttpRoute", "buildBaseAccountStatusSnapshot", diff --git a/src/plugin-sdk/index.ts b/src/plugin-sdk/index.ts index b7a6d3e965e..eaae5d08968 100644 --- a/src/plugin-sdk/index.ts +++ b/src/plugin-sdk/index.ts @@ -125,7 +125,6 @@ export type { export { normalizePluginHttpPath } from "../plugins/http-path.js"; export { registerPluginHttpRoute } from "../plugins/http-registry.js"; export { emptyPluginConfigSchema } from "../plugins/config-schema.js"; -export { createBundledBuiltinSearchProvider } from "../agents/tools/web-search.js"; export type { OpenClawConfig } from "../config/config.js"; /** @deprecated Use OpenClawConfig instead */ export type { OpenClawConfig as ClawdbotConfig } from "../config/config.js"; diff --git a/src/plugin-sdk/web-search.ts b/src/plugin-sdk/web-search.ts new file mode 100644 index 00000000000..5394eba4423 --- /dev/null +++ b/src/plugin-sdk/web-search.ts @@ -0,0 +1,218 @@ +export { formatCliCommand } from "../cli/command-format.js"; +export type { OpenClawConfig } from "../config/config.js"; +export { normalizeResolvedSecretInputString } from "../config/types.secrets.js"; +export { wrapWebContent } from "../security/external-content.js"; +export { normalizeSecretInput } from "../utils/normalize-secret-input.js"; +export type { + CacheEntry, + SearchProviderContext, + SearchProviderExecutionResult, + SearchProviderRequest, + SearchProviderPlugin, + SearchProviderRuntimeMetadataResolver, + SearchProviderSuccessResult, +} from "../plugins/types.js"; +export { + normalizeCacheKey, + readCache, + readResponseText, + writeCache, +} from "../agents/tools/web-shared.js"; +export { withTrustedWebToolsEndpoint } from "../agents/tools/web-guarded-fetch.js"; +export { resolveCitationRedirectUrl } from "../agents/tools/web-search-citation-redirect.js"; + +export type SearchProviderLegacyUiMetadata = { + label: string; + hint: string; + envKeys: readonly string[]; + placeholder: string; + signupUrl: string; + apiKeyConfigPath: string; + readApiKeyValue?: (search: Record | undefined) => unknown; + writeApiKeyValue?: (search: Record, value: unknown) => void; +}; + +export type SearchProviderFilterSupport = { + country?: boolean; + language?: boolean; + freshness?: boolean; + date?: boolean; + domainFilter?: boolean; +}; + +const WEB_SEARCH_DOCS_URL = "https://docs.openclaw.ai/tools/web"; + +export function resolveSearchConfig(search?: Record): T { + return search as T; +} + +export function createSearchProviderErrorResult( + error: string, + message: string, + docs: string = WEB_SEARCH_DOCS_URL, +): { error: string; message: string; docs: string } { + return { error, message, docs }; +} + +export function createMissingSearchKeyPayload( + error: string, + message: string, +): { error: string; message: string; docs: string } { + return createSearchProviderErrorResult(error, message); +} + +export function rejectUnsupportedSearchFilters(params: { + providerName: string; + request: Pick< + SearchProviderRequest, + "country" | "language" | "freshness" | "dateAfter" | "dateBefore" | "domainFilter" + >; + support: SearchProviderFilterSupport; +}): { error: string; message: string; docs: string } | undefined { + const provider = params.providerName; + if (params.request.country && params.support.country !== true) { + return createSearchProviderErrorResult( + "unsupported_country", + `country filtering is not supported by the ${provider} provider. Only Brave and Perplexity support country filtering.`, + ); + } + if (params.request.language && params.support.language !== true) { + return createSearchProviderErrorResult( + "unsupported_language", + `language filtering is not supported by the ${provider} provider. Only Brave and Perplexity support language filtering.`, + ); + } + if (params.request.freshness && params.support.freshness !== true) { + return createSearchProviderErrorResult( + "unsupported_freshness", + `freshness filtering is not supported by the ${provider} provider. Only Brave and Perplexity support freshness.`, + ); + } + if ((params.request.dateAfter || params.request.dateBefore) && params.support.date !== true) { + return createSearchProviderErrorResult( + "unsupported_date_filter", + `date_after/date_before filtering is not supported by the ${provider} provider. Only Brave and Perplexity support date filtering.`, + ); + } + if (params.request.domainFilter?.length && params.support.domainFilter !== true) { + return createSearchProviderErrorResult( + "unsupported_domain_filter", + `domain_filter is not supported by the ${provider} provider. Only Perplexity supports domain filtering.`, + ); + } + return undefined; +} + +export function resolveSiteName(url: string | undefined): string | undefined { + if (!url) { + return undefined; + } + try { + return new URL(url).hostname; + } catch { + return undefined; + } +} + +export async function throwWebSearchApiError(res: Response, providerLabel: string): Promise { + const detailResult = await readResponseText(res, { maxBytes: 64_000 }); + const detail = detailResult.text; + throw new Error(`${providerLabel} API error (${res.status}): ${detail || res.statusText}`); +} + +export function buildSearchRequestCacheIdentity(params: { + query: string; + count: number; + country?: string; + language?: string; + freshness?: string; + dateAfter?: string; + dateBefore?: string; + domainFilter?: string[]; + maxTokens?: number; + maxTokensPerPage?: number; +}): string { + return [ + params.query, + params.count, + params.country || "default", + params.language || "default", + params.freshness || "default", + params.dateAfter || "default", + params.dateBefore || "default", + params.domainFilter?.join(",") || "default", + params.maxTokens || "default", + params.maxTokensPerPage || "default", + ].join(":"); +} + +function isValidIsoDate(value: string): boolean { + if (!/^\d{4}-\d{2}-\d{2}$/.test(value)) { + return false; + } + const [year, month, day] = value.split("-").map((part) => Number.parseInt(part, 10)); + if (!Number.isFinite(year) || !Number.isFinite(month) || !Number.isFinite(day)) { + return false; + } + const date = new Date(Date.UTC(year, month - 1, day)); + return ( + date.getUTCFullYear() === year && date.getUTCMonth() === month - 1 && date.getUTCDate() === day + ); +} + +const ISO_DATE_PATTERN = /^(\d{4})-(\d{2})-(\d{2})$/; +const PERPLEXITY_DATE_PATTERN = /^(\d{1,2})\/(\d{1,2})\/(\d{4})$/; + +export function normalizeDateInputToIso(value: string): string | undefined { + const trimmed = value.trim(); + if (ISO_DATE_PATTERN.test(trimmed)) { + return isValidIsoDate(trimmed) ? trimmed : undefined; + } + const match = trimmed.match(PERPLEXITY_DATE_PATTERN); + if (match) { + const [, month, day, year] = match; + const iso = `${year}-${month.padStart(2, "0")}-${day.padStart(2, "0")}`; + return isValidIsoDate(iso) ? iso : undefined; + } + return undefined; +} + +function getScopedSearchConfig( + search: Record, + provider: string, +): Record | undefined { + if (provider === "brave") { + return search; + } + const scoped = search[provider]; + return typeof scoped === "object" && scoped !== null && !Array.isArray(scoped) + ? (scoped as Record) + : undefined; +} + +export function readSearchProviderApiKeyValue( + search: Record | undefined, + provider: string, +): unknown { + if (!search) { + return undefined; + } + return getScopedSearchConfig(search, provider)?.apiKey; +} + +export function writeSearchProviderApiKeyValue(params: { + search: Record; + provider: string; + value: unknown; +}): void { + if (params.provider === "brave") { + params.search.apiKey = params.value; + return; + } + const current = getScopedSearchConfig(params.search, params.provider); + if (current) { + current.apiKey = params.value; + return; + } + params.search[params.provider] = { apiKey: params.value }; +} diff --git a/src/plugins/types.ts b/src/plugins/types.ts index a0e0fa6c886..da095ca332b 100644 --- a/src/plugins/types.ts +++ b/src/plugins/types.ts @@ -294,14 +294,35 @@ export type SearchProviderContext = { pluginConfig?: Record; }; +export type SearchProviderLegacyConfigMetadata = { + hint?: string; + envKeys?: readonly string[]; + placeholder?: string; + signupUrl?: string; + apiKeyConfigPath?: string; + readApiKeyValue?: (search: Record | undefined) => unknown; + writeApiKeyValue?: (search: Record, value: unknown) => void; +}; + +export type SearchProviderRuntimeMetadata = Record; + +export type SearchProviderRuntimeMetadataResolver = (params: { + search: Record | undefined; + keyValue?: string; + keySource: "config" | "secretRef" | "env" | "missing"; + fallbackEnvVar?: string; +}) => SearchProviderRuntimeMetadata; + export type SearchProviderPlugin = { id: string; name: string; description?: string; pluginId?: string; - builtinProviderId?: string; + pluginOwnedExecution?: boolean; docsUrl?: string; configFieldOrder?: string[]; + legacyConfig?: SearchProviderLegacyConfigMetadata; + resolveRuntimeMetadata?: SearchProviderRuntimeMetadataResolver; isAvailable?: (config?: OpenClawConfig) => boolean; search: ( params: SearchProviderRequest, diff --git a/src/secrets/runtime-web-tools.ts b/src/secrets/runtime-web-tools.ts index 883aac6bd02..0e6124f644a 100644 --- a/src/secrets/runtime-web-tools.ts +++ b/src/secrets/runtime-web-tools.ts @@ -1,5 +1,12 @@ +import { + BUILTIN_WEB_SEARCH_PROVIDER_IDS, + type BuiltinWebSearchProviderId, + normalizeBuiltinWebSearchProvider, +} from "../agents/tools/web-search-provider-catalog.js"; import type { OpenClawConfig } from "../config/config.js"; import { resolveSecretInputRef } from "../config/types.secrets.js"; +import { loadOpenClawPlugins } from "../plugins/loader.js"; +import type { SearchProviderLegacyConfigMetadata, SearchProviderPlugin } from "../plugins/types.js"; import { normalizeSecretInput } from "../utils/normalize-secret-input.js"; import { secretRefKey } from "./ref-contract.js"; import { resolveSecretRefValues } from "./resolve.js"; @@ -10,13 +17,7 @@ import { type SecretDefaults, } from "./runtime-shared.js"; -const WEB_SEARCH_PROVIDERS = ["brave", "gemini", "grok", "kimi", "perplexity"] as const; -const PERPLEXITY_DIRECT_BASE_URL = "https://api.perplexity.ai"; -const DEFAULT_PERPLEXITY_BASE_URL = "https://openrouter.ai/api/v1"; -const PERPLEXITY_KEY_PREFIXES = ["pplx-"]; -const OPENROUTER_KEY_PREFIXES = ["sk-or-"]; - -type WebSearchProvider = (typeof WEB_SEARCH_PROVIDERS)[number]; +type WebSearchProvider = BuiltinWebSearchProviderId; type SecretResolutionSource = "config" | "secretRef" | "env" | "missing"; // pragma: allowlist secret type RuntimeWebProviderSource = "configured" | "auto-detect" | "none"; @@ -78,20 +79,45 @@ function isRecord(value: unknown): value is Record { } function normalizeProvider(value: unknown): WebSearchProvider | undefined { - if (typeof value !== "string") { - return undefined; + return normalizeBuiltinWebSearchProvider(value); +} + +type RegisteredSearchProviderRuntimeSupport = { + legacyConfig: SearchProviderLegacyConfigMetadata; + resolveRuntimeMetadata?: SearchProviderPlugin["resolveRuntimeMetadata"]; +}; + +function resolveRegisteredSearchProviderMetadata( + config: OpenClawConfig, +): Map { + try { + const registry = loadOpenClawPlugins({ + config, + cache: false, + suppressOpenAllowlistWarning: true, + }); + return new Map( + registry.searchProviders + .filter( + ( + entry, + ): entry is typeof entry & { + provider: typeof entry.provider & { legacyConfig: SearchProviderLegacyConfigMetadata }; + } => + normalizeProvider(entry.provider.id) !== undefined && + Boolean(entry.provider.legacyConfig), + ) + .map((entry) => [ + entry.provider.id as WebSearchProvider, + { + legacyConfig: entry.provider.legacyConfig, + resolveRuntimeMetadata: entry.provider.resolveRuntimeMetadata, + }, + ]), + ); + } catch { + return new Map(); } - const normalized = value.trim().toLowerCase(); - if ( - normalized === "brave" || - normalized === "gemini" || - normalized === "grok" || - normalized === "kimi" || - normalized === "perplexity" - ) { - return normalized; - } - return undefined; } function readNonEmptyEnvValue( @@ -225,60 +251,6 @@ async function resolveSecretInputWithEnvFallback(params: { }; } -function inferPerplexityBaseUrlFromApiKey(apiKey?: string): "direct" | "openrouter" | undefined { - if (!apiKey) { - return undefined; - } - const normalized = apiKey.toLowerCase(); - if (PERPLEXITY_KEY_PREFIXES.some((prefix) => normalized.startsWith(prefix))) { - return "direct"; - } - if (OPENROUTER_KEY_PREFIXES.some((prefix) => normalized.startsWith(prefix))) { - return "openrouter"; - } - return undefined; -} - -function resolvePerplexityRuntimeTransport(params: { - keyValue?: string; - keySource: SecretResolutionSource; - fallbackEnvVar?: string; - configValue: unknown; -}): "search_api" | "chat_completions" | undefined { - const config = isRecord(params.configValue) ? params.configValue : undefined; - const configuredBaseUrl = typeof config?.baseUrl === "string" ? config.baseUrl.trim() : ""; - const configuredModel = typeof config?.model === "string" ? config.model.trim() : ""; - - const baseUrl = (() => { - if (configuredBaseUrl) { - return configuredBaseUrl; - } - if (params.keySource === "env") { - if (params.fallbackEnvVar === "PERPLEXITY_API_KEY") { - return PERPLEXITY_DIRECT_BASE_URL; - } - if (params.fallbackEnvVar === "OPENROUTER_API_KEY") { - return DEFAULT_PERPLEXITY_BASE_URL; - } - } - if ((params.keySource === "config" || params.keySource === "secretRef") && params.keyValue) { - const inferred = inferPerplexityBaseUrlFromApiKey(params.keyValue); - return inferred === "openrouter" ? DEFAULT_PERPLEXITY_BASE_URL : PERPLEXITY_DIRECT_BASE_URL; - } - return DEFAULT_PERPLEXITY_BASE_URL; - })(); - - const hasLegacyOverride = Boolean(configuredBaseUrl || configuredModel); - const direct = (() => { - try { - return new URL(baseUrl).hostname.toLowerCase() === "api.perplexity.ai"; - } catch { - return false; - } - })(); - return hasLegacyOverride || !direct ? "chat_completions" : "search_api"; -} - function ensureObject(target: Record, key: string): Record { const current = target[key]; if (isRecord(current)) { @@ -292,17 +264,13 @@ function ensureObject(target: Record, key: string): Record, "tools"); const web = ensureObject(tools, "web"); const search = ensureObject(web, "search"); - if (params.provider === "brave") { - search.apiKey = params.value; - return; - } - const providerConfig = ensureObject(search, params.provider); - providerConfig.apiKey = params.value; + params.metadata.legacyConfig.writeApiKeyValue?.(search, params.value); } function setResolvedFirecrawlApiKey(params: { @@ -316,34 +284,28 @@ function setResolvedFirecrawlApiKey(params: { firecrawl.apiKey = params.value; } -function envVarsForProvider(provider: WebSearchProvider): string[] { - if (provider === "brave") { - return ["BRAVE_API_KEY"]; - } - if (provider === "gemini") { - return ["GEMINI_API_KEY"]; - } - if (provider === "grok") { - return ["XAI_API_KEY"]; - } - if (provider === "kimi") { - return ["KIMI_API_KEY", "MOONSHOT_API_KEY"]; - } - return ["PERPLEXITY_API_KEY", "OPENROUTER_API_KEY"]; +function envVarsForProvider( + metadataByProvider: Map, + provider: WebSearchProvider, +): string[] { + return [...(metadataByProvider.get(provider)?.legacyConfig.envKeys ?? [])]; } function resolveProviderKeyValue( + metadataByProvider: Map, search: Record, provider: WebSearchProvider, ): unknown { - if (provider === "brave") { - return search.apiKey; - } - const scoped = search[provider]; - if (!isRecord(scoped)) { - return undefined; - } - return scoped.apiKey; + return metadataByProvider.get(provider)?.legacyConfig.readApiKeyValue?.(search); +} + +function providerConfigPath( + metadataByProvider: Map, + provider: WebSearchProvider, +): string { + return ( + metadataByProvider.get(provider)?.legacyConfig.apiKeyConfigPath ?? "tools.web.search.provider" + ); } function hasConfiguredSecretRef(value: unknown, defaults: SecretDefaults | undefined): boolean { @@ -366,6 +328,7 @@ export async function resolveRuntimeWebTools(params: { const tools = isRecord(params.sourceConfig.tools) ? params.sourceConfig.tools : undefined; const web = isRecord(tools?.web) ? tools.web : undefined; const search = isRecord(web?.search) ? web.search : undefined; + const searchProviderMetadata = resolveRegisteredSearchProviderMetadata(params.sourceConfig); const searchMetadata: RuntimeWebSearchMetadata = { providerSource: "none", @@ -398,7 +361,9 @@ export async function resolveRuntimeWebTools(params: { } if (searchEnabled && search) { - const candidates = configuredProvider ? [configuredProvider] : [...WEB_SEARCH_PROVIDERS]; + const candidates = configuredProvider + ? [configuredProvider] + : [...BUILTIN_WEB_SEARCH_PROVIDER_IDS]; const unresolvedWithoutFallback: Array<{ provider: WebSearchProvider; path: string; @@ -409,16 +374,15 @@ export async function resolveRuntimeWebTools(params: { let selectedResolution: SecretResolutionResult | undefined; for (const provider of candidates) { - const path = - provider === "brave" ? "tools.web.search.apiKey" : `tools.web.search.${provider}.apiKey`; - const value = resolveProviderKeyValue(search, provider); + const path = providerConfigPath(searchProviderMetadata, provider); + const value = resolveProviderKeyValue(searchProviderMetadata, search, provider); const resolution = await resolveSecretInputWithEnvFallback({ sourceConfig: params.sourceConfig, context: params.context, defaults, value, path, - envVars: envVarsForProvider(provider), + envVars: envVarsForProvider(searchProviderMetadata, provider), }); if (resolution.secretRefConfigured && resolution.fallbackUsedAfterRefFailure) { @@ -450,9 +414,11 @@ export async function resolveRuntimeWebTools(params: { selectedProvider = provider; selectedResolution = resolution; if (resolution.value) { + const metadata = searchProviderMetadata.get(provider); setResolvedWebSearchApiKey({ resolvedConfig: params.resolvedConfig, provider, + metadata: metadata ?? { legacyConfig: {} }, value: resolution.value, }); } @@ -462,9 +428,11 @@ export async function resolveRuntimeWebTools(params: { if (resolution.value) { selectedProvider = provider; selectedResolution = resolution; + const metadata = searchProviderMetadata.get(provider); setResolvedWebSearchApiKey({ resolvedConfig: params.resolvedConfig, provider, + metadata: metadata ?? { legacyConfig: {} }, value: resolution.value, }); break; @@ -514,25 +482,31 @@ export async function resolveRuntimeWebTools(params: { if (!configuredProvider) { searchMetadata.providerSource = "auto-detect"; } - if (selectedProvider === "perplexity") { - searchMetadata.perplexityTransport = resolvePerplexityRuntimeTransport({ + const runtimeMetadata = searchProviderMetadata + .get(selectedProvider) + ?.resolveRuntimeMetadata?.({ + search, keyValue: selectedResolution?.value, keySource: selectedResolution?.source ?? "missing", fallbackEnvVar: selectedResolution?.fallbackEnvVar, - configValue: search.perplexity, }); + const perplexityTransport = + runtimeMetadata && typeof runtimeMetadata.perplexityTransport === "string" + ? runtimeMetadata.perplexityTransport + : undefined; + if (perplexityTransport === "search_api" || perplexityTransport === "chat_completions") { + searchMetadata.perplexityTransport = perplexityTransport; } } } if (searchEnabled && search && !configuredProvider && searchMetadata.selectedProvider) { - for (const provider of WEB_SEARCH_PROVIDERS) { + for (const provider of BUILTIN_WEB_SEARCH_PROVIDER_IDS) { if (provider === searchMetadata.selectedProvider) { continue; } - const path = - provider === "brave" ? "tools.web.search.apiKey" : `tools.web.search.${provider}.apiKey`; - const value = resolveProviderKeyValue(search, provider); + const path = providerConfigPath(searchProviderMetadata, provider); + const value = resolveProviderKeyValue(searchProviderMetadata, search, provider); if (!hasConfiguredSecretRef(value, defaults)) { continue; } @@ -543,10 +517,9 @@ export async function resolveRuntimeWebTools(params: { }); } } else if (search && !searchEnabled) { - for (const provider of WEB_SEARCH_PROVIDERS) { - const path = - provider === "brave" ? "tools.web.search.apiKey" : `tools.web.search.${provider}.apiKey`; - const value = resolveProviderKeyValue(search, provider); + for (const provider of BUILTIN_WEB_SEARCH_PROVIDER_IDS) { + const path = providerConfigPath(searchProviderMetadata, provider); + const value = resolveProviderKeyValue(searchProviderMetadata, search, provider); if (!hasConfiguredSecretRef(value, defaults)) { continue; } @@ -559,13 +532,12 @@ export async function resolveRuntimeWebTools(params: { } if (searchEnabled && search && configuredProvider) { - for (const provider of WEB_SEARCH_PROVIDERS) { + for (const provider of BUILTIN_WEB_SEARCH_PROVIDER_IDS) { if (provider === configuredProvider) { continue; } - const path = - provider === "brave" ? "tools.web.search.apiKey" : `tools.web.search.${provider}.apiKey`; - const value = resolveProviderKeyValue(search, provider); + const path = providerConfigPath(searchProviderMetadata, provider); + const value = resolveProviderKeyValue(searchProviderMetadata, search, provider); if (!hasConfiguredSecretRef(value, defaults)) { continue; } diff --git a/vitest.config.ts b/vitest.config.ts index 5e0a192d5a3..ac5f060ac56 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -11,6 +11,7 @@ const ciWorkers = isWindows ? 2 : 3; const pluginSdkSubpaths = [ "account-id", "core", + "web-search", "compat", "telegram", "discord",