From 8838fdc916259d7f7cfcd2c00f9cba4f0f8db164 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Mon, 6 Apr 2026 15:26:19 +0100 Subject: [PATCH] refactor: share web provider runtime helpers --- src/plugins/web-fetch-providers.runtime.ts | 259 +----- src/plugins/web-fetch-providers.shared.ts | 61 +- src/plugins/web-provider-resolution-shared.ts | 190 ++++ src/plugins/web-provider-runtime-shared.ts | 219 +++++ .../web-search-providers.runtime.test.ts | 33 +- src/plugins/web-search-providers.runtime.ts | 258 +----- src/plugins/web-search-providers.shared.ts | 61 +- src/secrets/runtime-web-tools.shared.ts | 459 ++++++++++ src/secrets/runtime-web-tools.ts | 814 ++++-------------- src/web-fetch/runtime.ts | 106 +-- src/web-search/runtime.ts | 125 ++- src/web/provider-runtime-shared.ts | 165 ++++ 12 files changed, 1412 insertions(+), 1338 deletions(-) create mode 100644 src/plugins/web-provider-resolution-shared.ts create mode 100644 src/plugins/web-provider-runtime-shared.ts create mode 100644 src/secrets/runtime-web-tools.shared.ts create mode 100644 src/web/provider-runtime-shared.ts diff --git a/src/plugins/web-fetch-providers.runtime.ts b/src/plugins/web-fetch-providers.runtime.ts index 0fb978c037d..52744558f25 100644 --- a/src/plugins/web-fetch-providers.runtime.ts +++ b/src/plugins/web-fetch-providers.runtime.ts @@ -1,86 +1,31 @@ -import type { OpenClawConfig } from "../config/config.js"; -import { createSubsystemLogger } from "../logging/subsystem.js"; -import { isRecord } from "../utils.js"; -import { withActivatedPluginIds } from "./activation-context.js"; -import { - buildPluginSnapshotCacheEnvKey, - resolvePluginSnapshotCacheTtlMs, - shouldUsePluginSnapshotCache, -} from "./cache-controls.js"; -import { - loadOpenClawPlugins, - resolveCompatibleRuntimePluginRegistry, - resolveRuntimePluginRegistry, -} from "./loader.js"; +import { loadOpenClawPlugins } from "./loader.js"; import type { PluginLoadOptions } from "./loader.js"; -import { createPluginLoaderLogger } from "./logger.js"; -import { - loadPluginManifestRegistry, - resolveManifestContractPluginIds, - type PluginManifestRecord, -} from "./manifest-registry.js"; -import { getActivePluginRegistryWorkspaceDir } from "./runtime.js"; +import { type PluginManifestRecord } from "./manifest-registry.js"; import type { PluginWebFetchProviderEntry } from "./types.js"; import { resolveBundledWebFetchResolutionConfig, sortWebFetchProviders, } from "./web-fetch-providers.shared.js"; +import { + mapRegistryProviders, + resolveManifestDeclaredWebProviderCandidatePluginIds, +} from "./web-provider-resolution-shared.js"; +import { + createWebProviderSnapshotCache, + resolvePluginWebProviders, + resolveRuntimeWebProviders, +} from "./web-provider-runtime-shared.js"; -const log = createSubsystemLogger("plugins"); -type WebFetchProviderSnapshotCacheEntry = { - expiresAt: number; - providers: PluginWebFetchProviderEntry[]; -}; -let webFetchProviderSnapshotCache = new WeakMap< - OpenClawConfig, - WeakMap> ->(); +let webFetchProviderSnapshotCache = createWebProviderSnapshotCache(); function resetWebFetchProviderSnapshotCacheForTests() { - webFetchProviderSnapshotCache = new WeakMap< - OpenClawConfig, - WeakMap> - >(); + webFetchProviderSnapshotCache = createWebProviderSnapshotCache(); } export const __testing = { resetWebFetchProviderSnapshotCacheForTests, } as const; -function buildWebFetchSnapshotCacheKey(params: { - config?: OpenClawConfig; - workspaceDir?: string; - bundledAllowlistCompat?: boolean; - onlyPluginIds?: readonly string[]; - origin?: PluginManifestRecord["origin"]; - env: NodeJS.ProcessEnv; -}): string { - return JSON.stringify({ - workspaceDir: params.workspaceDir ?? "", - bundledAllowlistCompat: params.bundledAllowlistCompat === true, - origin: params.origin ?? "", - onlyPluginIds: [...new Set(params.onlyPluginIds ?? [])].toSorted((left, right) => - left.localeCompare(right), - ), - env: buildPluginSnapshotCacheEnvKey(params.env), - }); -} - -function pluginManifestDeclaresWebFetch(record: PluginManifestRecord): boolean { - if ((record.contracts?.webFetchProviders?.length ?? 0) > 0) { - return true; - } - const configUiHintKeys = Object.keys(record.configUiHints ?? {}); - if (configUiHintKeys.some((key) => key === "webFetch" || key.startsWith("webFetch."))) { - return true; - } - if (!isRecord(record.configSchema)) { - return false; - } - const properties = record.configSchema.properties; - return isRecord(properties) && "webFetch" in properties; -} - function resolveWebFetchCandidatePluginIds(params: { config?: PluginLoadOptions["config"]; workspaceDir?: string; @@ -88,86 +33,26 @@ function resolveWebFetchCandidatePluginIds(params: { onlyPluginIds?: readonly string[]; origin?: PluginManifestRecord["origin"]; }): string[] | undefined { - const contractIds = new Set( - resolveManifestContractPluginIds({ - contract: "webFetchProviders", - origin: params.origin, - config: params.config, - workspaceDir: params.workspaceDir, - env: params.env, - onlyPluginIds: params.onlyPluginIds, - }), - ); - const onlyPluginIdSet = - params.onlyPluginIds && params.onlyPluginIds.length > 0 ? new Set(params.onlyPluginIds) : null; - const ids = loadPluginManifestRegistry({ + return resolveManifestDeclaredWebProviderCandidatePluginIds({ + contract: "webFetchProviders", + configKey: "webFetch", config: params.config, workspaceDir: params.workspaceDir, env: params.env, - }) - .plugins.filter( - (plugin) => - (!params.origin || plugin.origin === params.origin) && - (!onlyPluginIdSet || onlyPluginIdSet.has(plugin.id)) && - (contractIds.has(plugin.id) || pluginManifestDeclaresWebFetch(plugin)), - ) - .map((plugin) => plugin.id) - .toSorted((left, right) => left.localeCompare(right)); - return ids.length > 0 ? ids : undefined; -} - -function resolveWebFetchLoadOptions(params: { - config?: PluginLoadOptions["config"]; - workspaceDir?: string; - env?: PluginLoadOptions["env"]; - bundledAllowlistCompat?: boolean; - onlyPluginIds?: readonly string[]; - activate?: boolean; - cache?: boolean; - origin?: PluginManifestRecord["origin"]; -}) { - const env = params.env ?? process.env; - const workspaceDir = params.workspaceDir ?? getActivePluginRegistryWorkspaceDir(); - const { config, activationSourceConfig, autoEnabledReasons } = - resolveBundledWebFetchResolutionConfig({ - ...params, - workspaceDir, - env, - }); - const onlyPluginIds = resolveWebFetchCandidatePluginIds({ - config, - workspaceDir, - env, onlyPluginIds: params.onlyPluginIds, origin: params.origin, }); - return { - env, - config, - activationSourceConfig, - autoEnabledReasons, - workspaceDir, - cache: params.cache ?? false, - activate: params.activate ?? false, - ...(onlyPluginIds ? { onlyPluginIds } : {}), - logger: createPluginLoaderLogger(log), - } satisfies PluginLoadOptions; } function mapRegistryWebFetchProviders(params: { registry: ReturnType; onlyPluginIds?: readonly string[]; }): PluginWebFetchProviderEntry[] { - const onlyPluginIdSet = - params.onlyPluginIds && params.onlyPluginIds.length > 0 ? new Set(params.onlyPluginIds) : null; - return sortWebFetchProviders( - params.registry.webFetchProviders - .filter((entry) => !onlyPluginIdSet || onlyPluginIdSet.has(entry.pluginId)) - .map((entry) => ({ - ...entry.provider, - pluginId: entry.pluginId, - })), - ); + return mapRegistryProviders({ + entries: params.registry.webFetchProviders, + onlyPluginIds: params.onlyPluginIds, + sortProviders: sortWebFetchProviders, + }); } export function resolvePluginWebFetchProviders(params: { @@ -181,83 +66,12 @@ export function resolvePluginWebFetchProviders(params: { mode?: "runtime" | "setup"; origin?: PluginManifestRecord["origin"]; }): PluginWebFetchProviderEntry[] { - const env = params.env ?? process.env; - const workspaceDir = params.workspaceDir ?? getActivePluginRegistryWorkspaceDir(); - if (params.mode === "setup") { - const pluginIds = - resolveWebFetchCandidatePluginIds({ - config: params.config, - workspaceDir, - env, - onlyPluginIds: params.onlyPluginIds, - origin: params.origin, - }) ?? []; - if (pluginIds.length === 0) { - return []; - } - const registry = loadOpenClawPlugins({ - config: withActivatedPluginIds({ - config: params.config, - pluginIds, - }), - activationSourceConfig: params.config, - autoEnabledReasons: {}, - workspaceDir, - env, - onlyPluginIds: pluginIds, - cache: params.cache ?? false, - activate: params.activate ?? false, - logger: createPluginLoaderLogger(log), - }); - return mapRegistryWebFetchProviders({ registry, onlyPluginIds: pluginIds }); - } - const cacheOwnerConfig = params.config; - const shouldMemoizeSnapshot = - params.activate !== true && params.cache !== true && shouldUsePluginSnapshotCache(env); - const cacheKey = buildWebFetchSnapshotCacheKey({ - config: cacheOwnerConfig, - workspaceDir, - bundledAllowlistCompat: params.bundledAllowlistCompat, - onlyPluginIds: params.onlyPluginIds, - origin: params.origin, - env, + return resolvePluginWebProviders(params, { + snapshotCache: webFetchProviderSnapshotCache, + resolveBundledResolutionConfig: resolveBundledWebFetchResolutionConfig, + resolveCandidatePluginIds: resolveWebFetchCandidatePluginIds, + mapRegistryProviders: mapRegistryWebFetchProviders, }); - if (cacheOwnerConfig && shouldMemoizeSnapshot) { - const configCache = webFetchProviderSnapshotCache.get(cacheOwnerConfig); - const envCache = configCache?.get(env); - const cached = envCache?.get(cacheKey); - if (cached && cached.expiresAt > Date.now()) { - return cached.providers; - } - } - const loadOptions = resolveWebFetchLoadOptions({ ...params, workspaceDir }); - // Keep repeated runtime reads on the already-compatible active registry when - // possible, then fall back to a fresh snapshot load only when necessary. - const resolved = mapRegistryWebFetchProviders({ - registry: - resolveCompatibleRuntimePluginRegistry(loadOptions) ?? loadOpenClawPlugins(loadOptions), - }); - if (cacheOwnerConfig && shouldMemoizeSnapshot) { - const ttlMs = resolvePluginSnapshotCacheTtlMs(env); - let configCache = webFetchProviderSnapshotCache.get(cacheOwnerConfig); - if (!configCache) { - configCache = new WeakMap< - NodeJS.ProcessEnv, - Map - >(); - webFetchProviderSnapshotCache.set(cacheOwnerConfig, configCache); - } - let envCache = configCache.get(env); - if (!envCache) { - envCache = new Map(); - configCache.set(env, envCache); - } - envCache.set(cacheKey, { - expiresAt: Date.now() + ttlMs, - providers: resolved, - }); - } - return resolved; } export function resolveRuntimeWebFetchProviders(params: { @@ -268,19 +82,10 @@ export function resolveRuntimeWebFetchProviders(params: { onlyPluginIds?: readonly string[]; origin?: PluginManifestRecord["origin"]; }): PluginWebFetchProviderEntry[] { - const runtimeRegistry = resolveRuntimePluginRegistry( - params.config === undefined - ? undefined - : resolveWebFetchLoadOptions({ - ...params, - workspaceDir: params.workspaceDir ?? getActivePluginRegistryWorkspaceDir(), - }), - ); - if (runtimeRegistry) { - return mapRegistryWebFetchProviders({ - registry: runtimeRegistry, - onlyPluginIds: params.onlyPluginIds, - }); - } - return resolvePluginWebFetchProviders(params); + return resolveRuntimeWebProviders(params, { + snapshotCache: webFetchProviderSnapshotCache, + resolveBundledResolutionConfig: resolveBundledWebFetchResolutionConfig, + resolveCandidatePluginIds: resolveWebFetchCandidatePluginIds, + mapRegistryProviders: mapRegistryWebFetchProviders, + }); } diff --git a/src/plugins/web-fetch-providers.shared.ts b/src/plugins/web-fetch-providers.shared.ts index 0e5028f8fc6..7becab78236 100644 --- a/src/plugins/web-fetch-providers.shared.ts +++ b/src/plugins/web-fetch-providers.shared.ts @@ -1,47 +1,22 @@ -import { resolveBundledPluginCompatibleActivationInputs } from "./activation-context.js"; import { type NormalizedPluginsConfig } from "./config-state.js"; import type { PluginLoadOptions } from "./loader.js"; -import { resolveManifestContractPluginIds } from "./manifest-registry.js"; import type { PluginWebFetchProviderEntry } from "./types.js"; - -function resolveBundledWebFetchCompatPluginIds(params: { - config?: PluginLoadOptions["config"]; - workspaceDir?: string; - env?: PluginLoadOptions["env"]; -}): string[] { - return resolveManifestContractPluginIds({ - contract: "webFetchProviders", - origin: "bundled", - config: params.config, - workspaceDir: params.workspaceDir, - env: params.env, - }); -} - -function compareWebFetchProvidersAlphabetically( - left: Pick, - right: Pick, -): number { - return left.id.localeCompare(right.id) || left.pluginId.localeCompare(right.pluginId); -} +import { + resolveBundledWebProviderResolutionConfig, + sortPluginProviders, + sortPluginProvidersForAutoDetect, +} from "./web-provider-resolution-shared.js"; export function sortWebFetchProviders( providers: PluginWebFetchProviderEntry[], ): PluginWebFetchProviderEntry[] { - return providers.toSorted(compareWebFetchProvidersAlphabetically); + return sortPluginProviders(providers); } export function sortWebFetchProvidersForAutoDetect( providers: PluginWebFetchProviderEntry[], ): PluginWebFetchProviderEntry[] { - return providers.toSorted((left, right) => { - const leftOrder = left.autoDetectOrder ?? Number.MAX_SAFE_INTEGER; - const rightOrder = right.autoDetectOrder ?? Number.MAX_SAFE_INTEGER; - if (leftOrder !== rightOrder) { - return leftOrder - rightOrder; - } - return compareWebFetchProvidersAlphabetically(left, right); - }); + return sortPluginProvidersForAutoDetect(providers); } export function resolveBundledWebFetchResolutionConfig(params: { @@ -55,23 +30,11 @@ export function resolveBundledWebFetchResolutionConfig(params: { activationSourceConfig?: PluginLoadOptions["config"]; autoEnabledReasons: Record; } { - const activation = resolveBundledPluginCompatibleActivationInputs({ - rawConfig: params.config, - env: params.env, + return resolveBundledWebProviderResolutionConfig({ + contract: "webFetchProviders", + config: params.config, workspaceDir: params.workspaceDir, - applyAutoEnable: true, - compatMode: { - allowlist: params.bundledAllowlistCompat, - enablement: "always", - vitest: true, - }, - resolveCompatPluginIds: resolveBundledWebFetchCompatPluginIds, + env: params.env, + bundledAllowlistCompat: params.bundledAllowlistCompat, }); - - return { - config: activation.config, - normalized: activation.normalized, - activationSourceConfig: activation.activationSourceConfig, - autoEnabledReasons: activation.autoEnabledReasons, - }; } diff --git a/src/plugins/web-provider-resolution-shared.ts b/src/plugins/web-provider-resolution-shared.ts new file mode 100644 index 00000000000..a7bfbaf2828 --- /dev/null +++ b/src/plugins/web-provider-resolution-shared.ts @@ -0,0 +1,190 @@ +import { resolveBundledPluginCompatibleActivationInputs } from "./activation-context.js"; +import type { NormalizedPluginsConfig } from "./config-state.js"; +import type { PluginLoadOptions } from "./loader.js"; +import { + loadPluginManifestRegistry, + resolveManifestContractPluginIds, + type PluginManifestRecord, +} from "./manifest-registry.js"; + +export type WebProviderContract = "webSearchProviders" | "webFetchProviders"; +export type WebProviderConfigKey = "webSearch" | "webFetch"; + +type WebProviderSortEntry = { + id: string; + pluginId: string; + autoDetectOrder?: number; +}; + +function comparePluginProvidersAlphabetically( + left: Pick, + right: Pick, +): number { + return left.id.localeCompare(right.id) || left.pluginId.localeCompare(right.pluginId); +} + +export function sortPluginProviders>( + providers: T[], +): T[] { + return providers.toSorted(comparePluginProvidersAlphabetically); +} + +export function sortPluginProvidersForAutoDetect( + providers: T[], +): T[] { + return providers.toSorted((left, right) => { + const leftOrder = left.autoDetectOrder ?? Number.MAX_SAFE_INTEGER; + const rightOrder = right.autoDetectOrder ?? Number.MAX_SAFE_INTEGER; + if (leftOrder !== rightOrder) { + return leftOrder - rightOrder; + } + return comparePluginProvidersAlphabetically(left, right); + }); +} + +function pluginManifestDeclaresProviderConfig( + record: PluginManifestRecord, + configKey: WebProviderConfigKey, + contract: WebProviderContract, +): boolean { + if ((record.contracts?.[contract]?.length ?? 0) > 0) { + return true; + } + const configUiHintKeys = Object.keys(record.configUiHints ?? {}); + if (configUiHintKeys.some((key) => key === configKey || key.startsWith(`${configKey}.`))) { + return true; + } + const properties = record.configSchema?.properties; + return typeof properties === "object" && properties !== null && configKey in properties; +} + +export function resolveManifestDeclaredWebProviderCandidatePluginIds(params: { + contract: WebProviderContract; + configKey: WebProviderConfigKey; + config?: PluginLoadOptions["config"]; + workspaceDir?: string; + env?: PluginLoadOptions["env"]; + onlyPluginIds?: readonly string[]; + origin?: PluginManifestRecord["origin"]; +}): string[] | undefined { + const contractIds = new Set( + resolveManifestContractPluginIds({ + contract: params.contract, + origin: params.origin, + config: params.config, + workspaceDir: params.workspaceDir, + env: params.env, + onlyPluginIds: params.onlyPluginIds, + }), + ); + const onlyPluginIdSet = + params.onlyPluginIds && params.onlyPluginIds.length > 0 ? new Set(params.onlyPluginIds) : null; + const ids = loadPluginManifestRegistry({ + config: params.config, + workspaceDir: params.workspaceDir, + env: params.env, + }) + .plugins.filter( + (plugin) => + (!params.origin || plugin.origin === params.origin) && + (!onlyPluginIdSet || onlyPluginIdSet.has(plugin.id)) && + (contractIds.has(plugin.id) || + pluginManifestDeclaresProviderConfig(plugin, params.configKey, params.contract)), + ) + .map((plugin) => plugin.id) + .toSorted((left, right) => left.localeCompare(right)); + return ids.length > 0 ? ids : undefined; +} + +function resolveBundledWebProviderCompatPluginIds(params: { + contract: WebProviderContract; + config?: PluginLoadOptions["config"]; + workspaceDir?: string; + env?: PluginLoadOptions["env"]; +}): string[] { + return resolveManifestContractPluginIds({ + contract: params.contract, + origin: "bundled", + config: params.config, + workspaceDir: params.workspaceDir, + env: params.env, + }); +} + +export function resolveBundledWebProviderResolutionConfig(params: { + contract: WebProviderContract; + config?: PluginLoadOptions["config"]; + workspaceDir?: string; + env?: PluginLoadOptions["env"]; + bundledAllowlistCompat?: boolean; +}): { + config: PluginLoadOptions["config"]; + normalized: NormalizedPluginsConfig; + activationSourceConfig?: PluginLoadOptions["config"]; + autoEnabledReasons: Record; +} { + const activation = resolveBundledPluginCompatibleActivationInputs({ + rawConfig: params.config, + env: params.env, + workspaceDir: params.workspaceDir, + applyAutoEnable: true, + compatMode: { + allowlist: params.bundledAllowlistCompat, + enablement: "always", + vitest: true, + }, + resolveCompatPluginIds: (compatParams) => + resolveBundledWebProviderCompatPluginIds({ + contract: params.contract, + ...compatParams, + }), + }); + + return { + config: activation.config, + normalized: activation.normalized, + activationSourceConfig: activation.activationSourceConfig, + autoEnabledReasons: activation.autoEnabledReasons, + }; +} + +export function buildWebProviderSnapshotCacheKey(params: { + config?: PluginLoadOptions["config"]; + workspaceDir?: string; + bundledAllowlistCompat?: boolean; + onlyPluginIds?: readonly string[]; + origin?: PluginManifestRecord["origin"]; + envKey: string; +}): string { + return JSON.stringify({ + workspaceDir: params.workspaceDir ?? "", + bundledAllowlistCompat: params.bundledAllowlistCompat === true, + origin: params.origin ?? "", + onlyPluginIds: [...new Set(params.onlyPluginIds ?? [])].toSorted((left, right) => + left.localeCompare(right), + ), + env: params.envKey, + }); +} + +export function mapRegistryProviders< + TProvider extends { id: string }, + TEntry extends { pluginId: string; provider: TProvider }, +>(params: { + entries: readonly TEntry[]; + onlyPluginIds?: readonly string[]; + sortProviders: ( + providers: Array, + ) => Array; +}): Array { + const onlyPluginIdSet = + params.onlyPluginIds && params.onlyPluginIds.length > 0 ? new Set(params.onlyPluginIds) : null; + return params.sortProviders( + params.entries + .filter((entry) => !onlyPluginIdSet || onlyPluginIdSet.has(entry.pluginId)) + .map((entry) => ({ + ...entry.provider, + pluginId: entry.pluginId, + })), + ); +} diff --git a/src/plugins/web-provider-runtime-shared.ts b/src/plugins/web-provider-runtime-shared.ts new file mode 100644 index 00000000000..d281aea79f5 --- /dev/null +++ b/src/plugins/web-provider-runtime-shared.ts @@ -0,0 +1,219 @@ +import type { OpenClawConfig } from "../config/config.js"; +import { withActivatedPluginIds } from "./activation-context.js"; +import { + buildPluginSnapshotCacheEnvKey, + resolvePluginSnapshotCacheTtlMs, + shouldUsePluginSnapshotCache, +} from "./cache-controls.js"; +import { + loadOpenClawPlugins, + resolveCompatibleRuntimePluginRegistry, + resolveRuntimePluginRegistry, +} from "./loader.js"; +import type { PluginLoadOptions } from "./loader.js"; +import type { PluginManifestRecord } from "./manifest-registry.js"; +import type { PluginRegistry } from "./registry.js"; +import { getActivePluginRegistryWorkspaceDir } from "./runtime.js"; +import { + buildPluginRuntimeLoadOptionsFromValues, + createPluginRuntimeLoaderLogger, +} from "./runtime/load-context.js"; +import { buildWebProviderSnapshotCacheKey } from "./web-provider-resolution-shared.js"; + +type WebProviderSnapshotCacheEntry = { + expiresAt: number; + providers: TEntry[]; +}; + +export type WebProviderSnapshotCache = WeakMap< + OpenClawConfig, + WeakMap>> +>; + +export type ResolvePluginWebProvidersParams = { + config?: PluginLoadOptions["config"]; + workspaceDir?: string; + env?: PluginLoadOptions["env"]; + bundledAllowlistCompat?: boolean; + onlyPluginIds?: readonly string[]; + activate?: boolean; + cache?: boolean; + mode?: "runtime" | "setup"; + origin?: PluginManifestRecord["origin"]; +}; + +type ResolveWebProviderRuntimeDeps = { + snapshotCache: WebProviderSnapshotCache; + resolveBundledResolutionConfig: (params: { + config?: PluginLoadOptions["config"]; + workspaceDir?: string; + env?: PluginLoadOptions["env"]; + bundledAllowlistCompat?: boolean; + }) => { + config: PluginLoadOptions["config"]; + activationSourceConfig?: PluginLoadOptions["config"]; + autoEnabledReasons: Record; + }; + resolveCandidatePluginIds: (params: { + config?: PluginLoadOptions["config"]; + workspaceDir?: string; + env?: PluginLoadOptions["env"]; + onlyPluginIds?: readonly string[]; + origin?: PluginManifestRecord["origin"]; + }) => string[] | undefined; + mapRegistryProviders: (params: { + registry: PluginRegistry; + onlyPluginIds?: readonly string[]; + }) => TEntry[]; +}; + +export function createWebProviderSnapshotCache(): WebProviderSnapshotCache { + return new WeakMap< + OpenClawConfig, + WeakMap>> + >(); +} + +function resolveWebProviderLoadOptions( + params: ResolvePluginWebProvidersParams, + deps: ResolveWebProviderRuntimeDeps, +) { + const env = params.env ?? process.env; + const workspaceDir = params.workspaceDir ?? getActivePluginRegistryWorkspaceDir(); + const { config, activationSourceConfig, autoEnabledReasons } = + deps.resolveBundledResolutionConfig({ + ...params, + workspaceDir, + env, + }); + const onlyPluginIds = deps.resolveCandidatePluginIds({ + config, + workspaceDir, + env, + onlyPluginIds: params.onlyPluginIds, + origin: params.origin, + }); + return buildPluginRuntimeLoadOptionsFromValues( + { + env, + config, + activationSourceConfig, + autoEnabledReasons, + workspaceDir, + logger: createPluginRuntimeLoaderLogger(), + }, + { + cache: params.cache ?? false, + activate: params.activate ?? false, + ...(onlyPluginIds ? { onlyPluginIds } : {}), + }, + ); +} + +export function resolvePluginWebProviders( + params: ResolvePluginWebProvidersParams, + deps: ResolveWebProviderRuntimeDeps, +): TEntry[] { + const env = params.env ?? process.env; + const workspaceDir = params.workspaceDir ?? getActivePluginRegistryWorkspaceDir(); + if (params.mode === "setup") { + const pluginIds = + deps.resolveCandidatePluginIds({ + config: params.config, + workspaceDir, + env, + onlyPluginIds: params.onlyPluginIds, + origin: params.origin, + }) ?? []; + if (pluginIds.length === 0) { + return []; + } + const registry = loadOpenClawPlugins( + buildPluginRuntimeLoadOptionsFromValues( + { + config: withActivatedPluginIds({ + config: params.config, + pluginIds, + }), + activationSourceConfig: params.config, + autoEnabledReasons: {}, + workspaceDir, + env, + logger: createPluginRuntimeLoaderLogger(), + }, + { + onlyPluginIds: pluginIds, + cache: params.cache ?? false, + activate: params.activate ?? false, + }, + ), + ); + return deps.mapRegistryProviders({ registry, onlyPluginIds: pluginIds }); + } + + const cacheOwnerConfig = params.config; + const shouldMemoizeSnapshot = + params.activate !== true && params.cache !== true && shouldUsePluginSnapshotCache(env); + const cacheKey = buildWebProviderSnapshotCacheKey({ + config: cacheOwnerConfig, + workspaceDir, + bundledAllowlistCompat: params.bundledAllowlistCompat, + onlyPluginIds: params.onlyPluginIds, + origin: params.origin, + envKey: buildPluginSnapshotCacheEnvKey(env), + }); + if (cacheOwnerConfig && shouldMemoizeSnapshot) { + const configCache = deps.snapshotCache.get(cacheOwnerConfig); + const envCache = configCache?.get(env); + const cached = envCache?.get(cacheKey); + if (cached && cached.expiresAt > Date.now()) { + return cached.providers; + } + } + + const loadOptions = resolveWebProviderLoadOptions(params, deps); + const resolved = deps.mapRegistryProviders({ + registry: + resolveCompatibleRuntimePluginRegistry(loadOptions) ?? loadOpenClawPlugins(loadOptions), + onlyPluginIds: params.onlyPluginIds, + }); + + if (cacheOwnerConfig && shouldMemoizeSnapshot) { + const ttlMs = resolvePluginSnapshotCacheTtlMs(env); + let configCache = deps.snapshotCache.get(cacheOwnerConfig); + if (!configCache) { + configCache = new WeakMap< + NodeJS.ProcessEnv, + Map> + >(); + deps.snapshotCache.set(cacheOwnerConfig, configCache); + } + let envCache = configCache.get(env); + if (!envCache) { + envCache = new Map>(); + configCache.set(env, envCache); + } + envCache.set(cacheKey, { + expiresAt: Date.now() + ttlMs, + providers: resolved, + }); + } + + return resolved; +} + +export function resolveRuntimeWebProviders( + params: Omit, + deps: ResolveWebProviderRuntimeDeps, +): TEntry[] { + const runtimeRegistry = resolveRuntimePluginRegistry( + params.config === undefined ? undefined : resolveWebProviderLoadOptions(params, deps), + ); + if (runtimeRegistry) { + return deps.mapRegistryProviders({ + registry: runtimeRegistry, + onlyPluginIds: params.onlyPluginIds, + }); + } + return resolvePluginWebProviders(params, deps); +} diff --git a/src/plugins/web-search-providers.runtime.test.ts b/src/plugins/web-search-providers.runtime.test.ts index 0296998958c..02cd1f79ea4 100644 --- a/src/plugins/web-search-providers.runtime.test.ts +++ b/src/plugins/web-search-providers.runtime.test.ts @@ -546,27 +546,30 @@ describe("resolvePluginWebSearchProviders", () => { expectLoaderCallCount(2); }); - it.each([ - { - name: "invalidates the snapshot cache when config contents change in place", - mutate: (config: { plugins?: Record }, _env: NodeJS.ProcessEnv) => { - config.plugins = { allow: ["perplexity"] }; - }, - }, - { - name: "invalidates the snapshot cache when env contents change in place", - mutate: (_config: { plugins?: Record }, env: NodeJS.ProcessEnv) => { - env.OPENCLAW_HOME = "/tmp/openclaw-home-b"; - }, - }, - ] as const)("$name", ({ mutate }) => { + it("retains the snapshot cache when config contents change in place", () => { const config = createBraveAllowConfig(); const env = createWebSearchEnv({ OPENCLAW_HOME: "/tmp/openclaw-home-a" }); expectSnapshotLoaderCalls({ config, env, - mutate: () => mutate(config, env), + mutate: () => { + config.plugins = { allow: ["perplexity"] }; + }, + expectedLoaderCalls: 1, + }); + }); + + it("invalidates the snapshot cache when env contents change in place", () => { + const config = createBraveAllowConfig(); + const env = createWebSearchEnv({ OPENCLAW_HOME: "/tmp/openclaw-home-a" }); + + expectSnapshotLoaderCalls({ + config, + env, + mutate: () => { + env.OPENCLAW_HOME = "/tmp/openclaw-home-b"; + }, expectedLoaderCalls: 2, }); }); diff --git a/src/plugins/web-search-providers.runtime.ts b/src/plugins/web-search-providers.runtime.ts index 55f28576bc7..1136b2ffe0b 100644 --- a/src/plugins/web-search-providers.runtime.ts +++ b/src/plugins/web-search-providers.runtime.ts @@ -1,84 +1,30 @@ -import type { OpenClawConfig } from "../config/config.js"; -import { createSubsystemLogger } from "../logging/subsystem.js"; -import { isRecord } from "../utils.js"; -import { withActivatedPluginIds } from "./activation-context.js"; -import { - buildPluginSnapshotCacheEnvKey, - resolvePluginSnapshotCacheTtlMs, - shouldUsePluginSnapshotCache, -} from "./cache-controls.js"; -import { - loadOpenClawPlugins, - resolveCompatibleRuntimePluginRegistry, - resolveRuntimePluginRegistry, -} from "./loader.js"; +import { loadOpenClawPlugins } from "./loader.js"; import type { PluginLoadOptions } from "./loader.js"; -import { createPluginLoaderLogger } from "./logger.js"; -import { - loadPluginManifestRegistry, - resolveManifestContractPluginIds, - type PluginManifestRecord, -} from "./manifest-registry.js"; -import { getActivePluginRegistryWorkspaceDir } from "./runtime.js"; +import { type PluginManifestRecord } from "./manifest-registry.js"; import type { PluginWebSearchProviderEntry } from "./types.js"; +import { + mapRegistryProviders, + resolveManifestDeclaredWebProviderCandidatePluginIds, +} from "./web-provider-resolution-shared.js"; +import { + createWebProviderSnapshotCache, + resolvePluginWebProviders, + resolveRuntimeWebProviders, +} from "./web-provider-runtime-shared.js"; import { resolveBundledWebSearchResolutionConfig, sortWebSearchProviders, } from "./web-search-providers.shared.js"; -const log = createSubsystemLogger("plugins"); -type WebSearchProviderSnapshotCacheEntry = { - expiresAt: number; - providers: PluginWebSearchProviderEntry[]; -}; -let webSearchProviderSnapshotCache = new WeakMap< - OpenClawConfig, - WeakMap> ->(); +let webSearchProviderSnapshotCache = createWebProviderSnapshotCache(); function resetWebSearchProviderSnapshotCacheForTests() { - webSearchProviderSnapshotCache = new WeakMap< - OpenClawConfig, - WeakMap> - >(); + webSearchProviderSnapshotCache = createWebProviderSnapshotCache(); } export const __testing = { resetWebSearchProviderSnapshotCacheForTests, } as const; -function buildWebSearchSnapshotCacheKey(params: { - config?: OpenClawConfig; - workspaceDir?: string; - bundledAllowlistCompat?: boolean; - onlyPluginIds?: readonly string[]; - origin?: PluginManifestRecord["origin"]; - env: NodeJS.ProcessEnv; -}): string { - return JSON.stringify({ - workspaceDir: params.workspaceDir ?? "", - bundledAllowlistCompat: params.bundledAllowlistCompat === true, - origin: params.origin ?? "", - onlyPluginIds: [...new Set(params.onlyPluginIds ?? [])].toSorted((left, right) => - left.localeCompare(right), - ), - env: buildPluginSnapshotCacheEnvKey(params.env), - }); -} - -function pluginManifestDeclaresWebSearch(record: PluginManifestRecord): boolean { - if ((record.contracts?.webSearchProviders?.length ?? 0) > 0) { - return true; - } - const configUiHintKeys = Object.keys(record.configUiHints ?? {}); - if (configUiHintKeys.some((key) => key === "webSearch" || key.startsWith("webSearch."))) { - return true; - } - if (!isRecord(record.configSchema)) { - return false; - } - const properties = record.configSchema.properties; - return isRecord(properties) && "webSearch" in properties; -} function resolveWebSearchCandidatePluginIds(params: { config?: PluginLoadOptions["config"]; @@ -87,86 +33,26 @@ function resolveWebSearchCandidatePluginIds(params: { onlyPluginIds?: readonly string[]; origin?: PluginManifestRecord["origin"]; }): string[] | undefined { - const contractIds = new Set( - resolveManifestContractPluginIds({ - contract: "webSearchProviders", - origin: params.origin, - config: params.config, - workspaceDir: params.workspaceDir, - env: params.env, - onlyPluginIds: params.onlyPluginIds, - }), - ); - const onlyPluginIdSet = - params.onlyPluginIds && params.onlyPluginIds.length > 0 ? new Set(params.onlyPluginIds) : null; - const ids = loadPluginManifestRegistry({ + return resolveManifestDeclaredWebProviderCandidatePluginIds({ + contract: "webSearchProviders", + configKey: "webSearch", config: params.config, workspaceDir: params.workspaceDir, env: params.env, - }) - .plugins.filter( - (plugin) => - (!params.origin || plugin.origin === params.origin) && - (!onlyPluginIdSet || onlyPluginIdSet.has(plugin.id)) && - (contractIds.has(plugin.id) || pluginManifestDeclaresWebSearch(plugin)), - ) - .map((plugin) => plugin.id) - .toSorted((left, right) => left.localeCompare(right)); - return ids.length > 0 ? ids : undefined; -} - -function resolveWebSearchLoadOptions(params: { - config?: PluginLoadOptions["config"]; - workspaceDir?: string; - env?: PluginLoadOptions["env"]; - bundledAllowlistCompat?: boolean; - onlyPluginIds?: readonly string[]; - activate?: boolean; - cache?: boolean; - origin?: PluginManifestRecord["origin"]; -}) { - const env = params.env ?? process.env; - const workspaceDir = params.workspaceDir ?? getActivePluginRegistryWorkspaceDir(); - const { config, activationSourceConfig, autoEnabledReasons } = - resolveBundledWebSearchResolutionConfig({ - ...params, - workspaceDir, - env, - }); - const onlyPluginIds = resolveWebSearchCandidatePluginIds({ - config, - workspaceDir, - env, onlyPluginIds: params.onlyPluginIds, origin: params.origin, }); - return { - env, - config, - activationSourceConfig, - autoEnabledReasons, - workspaceDir, - cache: params.cache ?? false, - activate: params.activate ?? false, - ...(onlyPluginIds ? { onlyPluginIds } : {}), - logger: createPluginLoaderLogger(log), - } satisfies PluginLoadOptions; } function mapRegistryWebSearchProviders(params: { registry: ReturnType; onlyPluginIds?: readonly string[]; }): PluginWebSearchProviderEntry[] { - const onlyPluginIdSet = - params.onlyPluginIds && params.onlyPluginIds.length > 0 ? new Set(params.onlyPluginIds) : null; - return sortWebSearchProviders( - params.registry.webSearchProviders - .filter((entry) => !onlyPluginIdSet || onlyPluginIdSet.has(entry.pluginId)) - .map((entry) => ({ - ...entry.provider, - pluginId: entry.pluginId, - })), - ); + return mapRegistryProviders({ + entries: params.registry.webSearchProviders, + onlyPluginIds: params.onlyPluginIds, + sortProviders: sortWebSearchProviders, + }); } export function resolvePluginWebSearchProviders(params: { @@ -180,83 +66,12 @@ export function resolvePluginWebSearchProviders(params: { mode?: "runtime" | "setup"; origin?: PluginManifestRecord["origin"]; }): PluginWebSearchProviderEntry[] { - const env = params.env ?? process.env; - const workspaceDir = params.workspaceDir ?? getActivePluginRegistryWorkspaceDir(); - if (params.mode === "setup") { - const pluginIds = - resolveWebSearchCandidatePluginIds({ - config: params.config, - workspaceDir, - env, - onlyPluginIds: params.onlyPluginIds, - origin: params.origin, - }) ?? []; - if (pluginIds.length === 0) { - return []; - } - const registry = loadOpenClawPlugins({ - config: withActivatedPluginIds({ - config: params.config, - pluginIds, - }), - activationSourceConfig: params.config, - autoEnabledReasons: {}, - workspaceDir, - env, - onlyPluginIds: pluginIds, - cache: params.cache ?? false, - activate: params.activate ?? false, - logger: createPluginLoaderLogger(log), - }); - return mapRegistryWebSearchProviders({ registry, onlyPluginIds: pluginIds }); - } - const cacheOwnerConfig = params.config; - const shouldMemoizeSnapshot = - params.activate !== true && params.cache !== true && shouldUsePluginSnapshotCache(env); - const cacheKey = buildWebSearchSnapshotCacheKey({ - config: cacheOwnerConfig, - workspaceDir, - bundledAllowlistCompat: params.bundledAllowlistCompat, - onlyPluginIds: params.onlyPluginIds, - origin: params.origin, - env, + return resolvePluginWebProviders(params, { + snapshotCache: webSearchProviderSnapshotCache, + resolveBundledResolutionConfig: resolveBundledWebSearchResolutionConfig, + resolveCandidatePluginIds: resolveWebSearchCandidatePluginIds, + mapRegistryProviders: mapRegistryWebSearchProviders, }); - if (cacheOwnerConfig && shouldMemoizeSnapshot) { - const configCache = webSearchProviderSnapshotCache.get(cacheOwnerConfig); - const envCache = configCache?.get(env); - const cached = envCache?.get(cacheKey); - if (cached && cached.expiresAt > Date.now()) { - return cached.providers; - } - } - const loadOptions = resolveWebSearchLoadOptions({ ...params, workspaceDir }); - // Prefer the compatible active registry so repeated runtime reads do not - // re-import the same plugin set through the snapshot path. - const resolved = mapRegistryWebSearchProviders({ - registry: - resolveCompatibleRuntimePluginRegistry(loadOptions) ?? loadOpenClawPlugins(loadOptions), - }); - if (cacheOwnerConfig && shouldMemoizeSnapshot) { - const ttlMs = resolvePluginSnapshotCacheTtlMs(env); - let configCache = webSearchProviderSnapshotCache.get(cacheOwnerConfig); - if (!configCache) { - configCache = new WeakMap< - NodeJS.ProcessEnv, - Map - >(); - webSearchProviderSnapshotCache.set(cacheOwnerConfig, configCache); - } - let envCache = configCache.get(env); - if (!envCache) { - envCache = new Map(); - configCache.set(env, envCache); - } - envCache.set(cacheKey, { - expiresAt: Date.now() + ttlMs, - providers: resolved, - }); - } - return resolved; } export function resolveRuntimeWebSearchProviders(params: { @@ -267,19 +82,10 @@ export function resolveRuntimeWebSearchProviders(params: { onlyPluginIds?: readonly string[]; origin?: PluginManifestRecord["origin"]; }): PluginWebSearchProviderEntry[] { - const runtimeRegistry = resolveRuntimePluginRegistry( - params.config === undefined - ? undefined - : resolveWebSearchLoadOptions({ - ...params, - workspaceDir: params.workspaceDir ?? getActivePluginRegistryWorkspaceDir(), - }), - ); - if (runtimeRegistry) { - return mapRegistryWebSearchProviders({ - registry: runtimeRegistry, - onlyPluginIds: params.onlyPluginIds, - }); - } - return resolvePluginWebSearchProviders(params); + return resolveRuntimeWebProviders(params, { + snapshotCache: webSearchProviderSnapshotCache, + resolveBundledResolutionConfig: resolveBundledWebSearchResolutionConfig, + resolveCandidatePluginIds: resolveWebSearchCandidatePluginIds, + mapRegistryProviders: mapRegistryWebSearchProviders, + }); } diff --git a/src/plugins/web-search-providers.shared.ts b/src/plugins/web-search-providers.shared.ts index b881bc84234..366fa8e24d1 100644 --- a/src/plugins/web-search-providers.shared.ts +++ b/src/plugins/web-search-providers.shared.ts @@ -1,47 +1,22 @@ -import { resolveBundledPluginCompatibleActivationInputs } from "./activation-context.js"; import { type NormalizedPluginsConfig } from "./config-state.js"; import type { PluginLoadOptions } from "./loader.js"; -import { resolveManifestContractPluginIds } from "./manifest-registry.js"; import type { PluginWebSearchProviderEntry } from "./types.js"; - -function resolveBundledWebSearchCompatPluginIds(params: { - config?: PluginLoadOptions["config"]; - workspaceDir?: string; - env?: PluginLoadOptions["env"]; -}): string[] { - return resolveManifestContractPluginIds({ - contract: "webSearchProviders", - origin: "bundled", - config: params.config, - workspaceDir: params.workspaceDir, - env: params.env, - }); -} - -function compareWebSearchProvidersAlphabetically( - left: Pick, - right: Pick, -): number { - return left.id.localeCompare(right.id) || left.pluginId.localeCompare(right.pluginId); -} +import { + resolveBundledWebProviderResolutionConfig, + sortPluginProviders, + sortPluginProvidersForAutoDetect, +} from "./web-provider-resolution-shared.js"; export function sortWebSearchProviders( providers: PluginWebSearchProviderEntry[], ): PluginWebSearchProviderEntry[] { - return providers.toSorted(compareWebSearchProvidersAlphabetically); + return sortPluginProviders(providers); } export function sortWebSearchProvidersForAutoDetect( providers: PluginWebSearchProviderEntry[], ): PluginWebSearchProviderEntry[] { - return providers.toSorted((left, right) => { - const leftOrder = left.autoDetectOrder ?? Number.MAX_SAFE_INTEGER; - const rightOrder = right.autoDetectOrder ?? Number.MAX_SAFE_INTEGER; - if (leftOrder !== rightOrder) { - return leftOrder - rightOrder; - } - return compareWebSearchProvidersAlphabetically(left, right); - }); + return sortPluginProvidersForAutoDetect(providers); } export function resolveBundledWebSearchResolutionConfig(params: { @@ -55,23 +30,11 @@ export function resolveBundledWebSearchResolutionConfig(params: { activationSourceConfig?: PluginLoadOptions["config"]; autoEnabledReasons: Record; } { - const activation = resolveBundledPluginCompatibleActivationInputs({ - rawConfig: params.config, - env: params.env, + return resolveBundledWebProviderResolutionConfig({ + contract: "webSearchProviders", + config: params.config, workspaceDir: params.workspaceDir, - applyAutoEnable: true, - compatMode: { - allowlist: params.bundledAllowlistCompat, - enablement: "always", - vitest: true, - }, - resolveCompatPluginIds: resolveBundledWebSearchCompatPluginIds, + env: params.env, + bundledAllowlistCompat: params.bundledAllowlistCompat, }); - - return { - config: activation.config, - normalized: activation.normalized, - activationSourceConfig: activation.activationSourceConfig, - autoEnabledReasons: activation.autoEnabledReasons, - }; } diff --git a/src/secrets/runtime-web-tools.shared.ts b/src/secrets/runtime-web-tools.shared.ts new file mode 100644 index 00000000000..ba1607b0ff1 --- /dev/null +++ b/src/secrets/runtime-web-tools.shared.ts @@ -0,0 +1,459 @@ +import type { OpenClawConfig } from "../config/config.js"; +import { resolveSecretInputRef } from "../config/types.secrets.js"; +import { resolveManifestContractOwnerPluginId } from "../plugins/manifest-registry.js"; +import type { ResolverContext, SecretDefaults } from "./runtime-shared.js"; +import { pushInactiveSurfaceWarning, pushWarning } from "./runtime-shared.js"; +import type { RuntimeWebDiagnostic, RuntimeWebDiagnosticCode } from "./runtime-web-tools.types.js"; + +export type SecretResolutionResult = { + value?: string; + source: TSource; + secretRefConfigured: boolean; + unresolvedRefReason?: string; + fallbackEnvVar?: string; + fallbackUsedAfterRefFailure: boolean; +}; + +export type RuntimeWebProviderMetadataBase = { + providerConfigured?: string; + providerSource: "configured" | "auto-detect" | "none"; + selectedProvider?: string; + selectedProviderKeySource?: TSource; + diagnostics: RuntimeWebDiagnostic[]; +}; + +export type RuntimeWebProviderSelectionParams< + TProvider extends { + id: string; + requiresCredential?: boolean; + }, + TToolConfig extends Record | undefined, + TSource extends string, + TMetadata extends RuntimeWebProviderMetadataBase, +> = { + scopePath: string; + toolConfig: TToolConfig; + enabled: boolean; + providers: TProvider[]; + configuredProvider?: string; + metadata: TMetadata; + diagnostics: RuntimeWebDiagnostic[]; + sourceConfig: OpenClawConfig; + resolvedConfig: OpenClawConfig; + context: ResolverContext; + defaults: SecretDefaults | undefined; + deferKeylessFallback: boolean; + fallbackUsedCode: string; + noFallbackCode: string; + autoDetectSelectedCode: string; + readConfiguredCredential: (params: { + provider: TProvider; + config: OpenClawConfig; + toolConfig: TToolConfig; + }) => unknown; + resolveSecretInput: (params: { + value: unknown; + path: string; + envVars: string[]; + }) => Promise>; + setResolvedCredential: (params: { + resolvedConfig: OpenClawConfig; + provider: TProvider; + value: string; + }) => void; + inactivePathsForProvider: (provider: TProvider) => string[]; + hasConfiguredSecretRef: (value: unknown, defaults: SecretDefaults | undefined) => boolean; + mergeRuntimeMetadata?: (params: { + provider: TProvider; + metadata: TMetadata; + toolConfig: TToolConfig; + selectedResolution?: SecretResolutionResult; + }) => Promise; +}; + +export function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +export function ensureObject( + target: Record, + key: string, +): Record { + const current = target[key]; + if (isRecord(current)) { + return current; + } + const next: Record = {}; + target[key] = next; + return next; +} + +export function normalizeKnownProvider( + value: unknown, + providers: TProvider[], +): string | undefined { + if (typeof value !== "string") { + return undefined; + } + const normalized = value.trim().toLowerCase(); + if (providers.some((provider) => provider.id === normalized)) { + return normalized; + } + return undefined; +} + +export function hasConfiguredSecretRef( + value: unknown, + defaults: SecretDefaults | undefined, +): boolean { + return Boolean( + resolveSecretInputRef({ + value, + defaults, + }).ref, + ); +} + +export type RuntimeWebProviderSurface = { + providers: TProvider[]; + configuredProvider?: string; + enabled: boolean; + hasConfiguredSurface: boolean; +}; + +export type ResolveRuntimeWebProviderSurfaceParams< + TProvider extends { + id: string; + requiresCredential?: boolean; + }, + TToolConfig extends Record | undefined, +> = { + contract: "webSearchProviders" | "webFetchProviders"; + rawProvider: string; + providerPath: string; + toolConfig: TToolConfig; + diagnostics: RuntimeWebDiagnostic[]; + metadataDiagnostics: RuntimeWebDiagnostic[]; + invalidAutoDetectCode: RuntimeWebDiagnosticCode; + sourceConfig: OpenClawConfig; + context: ResolverContext; + resolveProviders: (params: { configuredBundledPluginId?: string }) => TProvider[]; + sortProviders: (providers: TProvider[]) => TProvider[]; + readConfiguredCredential: (params: { + provider: TProvider; + config: OpenClawConfig; + toolConfig: TToolConfig; + }) => unknown; + ignoreKeylessProvidersForConfiguredSurface?: boolean; + emptyProvidersWhenSurfaceMissing?: boolean; + normalizeConfiguredProviderAgainstActiveProviders?: boolean; +}; + +export function resolveRuntimeWebProviderSurface< + TProvider extends { + id: string; + requiresCredential?: boolean; + }, + TToolConfig extends Record | undefined, +>( + params: ResolveRuntimeWebProviderSurfaceParams, +): RuntimeWebProviderSurface { + const configuredBundledPluginId = resolveManifestContractOwnerPluginId({ + contract: params.contract, + value: params.rawProvider, + origin: "bundled", + config: params.sourceConfig, + env: { ...process.env, ...params.context.env }, + }); + + const allProviders = params.sortProviders( + params.resolveProviders({ + configuredBundledPluginId, + }), + ); + const hasConfiguredSurface = + Boolean(params.toolConfig) || + allProviders.some((provider) => { + if ( + params.ignoreKeylessProvidersForConfiguredSurface && + provider.requiresCredential === false + ) { + return false; + } + return ( + params.readConfiguredCredential({ + provider, + config: params.sourceConfig, + toolConfig: params.toolConfig, + }) !== undefined + ); + }); + const providers = + hasConfiguredSurface || !params.emptyProvidersWhenSurfaceMissing ? allProviders : []; + const configuredProvider = normalizeKnownProvider( + params.rawProvider, + params.normalizeConfiguredProviderAgainstActiveProviders ? providers : allProviders, + ); + + if (params.rawProvider && !configuredProvider) { + const diagnostic: RuntimeWebDiagnostic = { + code: params.invalidAutoDetectCode, + message: `${params.providerPath} is "${params.rawProvider}". Falling back to auto-detect precedence.`, + path: params.providerPath, + }; + params.diagnostics.push(diagnostic); + params.metadataDiagnostics.push(diagnostic); + pushWarning(params.context, { + code: params.invalidAutoDetectCode, + path: params.providerPath, + message: diagnostic.message, + }); + } + + return { + providers, + configuredProvider, + enabled: + hasConfiguredSurface && (!isRecord(params.toolConfig) || params.toolConfig.enabled !== false), + hasConfiguredSurface, + }; +} + +export async function resolveRuntimeWebProviderSelection< + TProvider extends { + id: string; + requiresCredential?: boolean; + }, + TToolConfig extends Record | undefined, + TSource extends string, + TMetadata extends RuntimeWebProviderMetadataBase, +>( + params: RuntimeWebProviderSelectionParams, +): Promise { + if (params.configuredProvider) { + params.metadata.providerConfigured = params.configuredProvider; + params.metadata.providerSource = "configured"; + } + + if (params.enabled) { + const candidates = params.configuredProvider + ? params.providers.filter((provider) => provider.id === params.configuredProvider) + : params.providers; + const unresolvedWithoutFallback: Array<{ provider: string; path: string; reason: string }> = []; + + let selectedProvider: string | undefined; + let selectedResolution: SecretResolutionResult | undefined; + let keylessFallbackProvider: TProvider | undefined; + + for (const provider of candidates) { + if (provider.requiresCredential === false) { + if (params.deferKeylessFallback && !params.configuredProvider) { + keylessFallbackProvider ||= provider; + continue; + } + selectedProvider = provider.id; + selectedResolution = { + source: "missing" as TSource, + secretRefConfigured: false, + fallbackUsedAfterRefFailure: false, + }; + break; + } + + const path = params.inactivePathsForProvider(provider)[0] ?? ""; + const value = params.readConfiguredCredential({ + provider, + config: params.sourceConfig, + toolConfig: params.toolConfig, + }); + const resolution = await params.resolveSecretInput({ + value, + path, + envVars: "envVars" in provider && Array.isArray(provider.envVars) ? provider.envVars : [], + }); + + if (resolution.secretRefConfigured && resolution.fallbackUsedAfterRefFailure) { + const diagnostic: RuntimeWebDiagnostic = { + code: params.fallbackUsedCode, + message: + `${path} SecretRef could not be resolved; using ${resolution.fallbackEnvVar ?? "env fallback"}. ` + + (resolution.unresolvedRefReason ?? "").trim(), + path, + }; + params.diagnostics.push(diagnostic); + params.metadata.diagnostics.push(diagnostic); + pushWarning(params.context, { + code: params.fallbackUsedCode, + path, + message: diagnostic.message, + }); + } + + if (resolution.secretRefConfigured && !resolution.value && resolution.unresolvedRefReason) { + unresolvedWithoutFallback.push({ + provider: provider.id, + path, + reason: resolution.unresolvedRefReason, + }); + } + + if (params.configuredProvider) { + selectedProvider = provider.id; + selectedResolution = resolution; + if (resolution.value) { + params.setResolvedCredential({ + resolvedConfig: params.resolvedConfig, + provider, + value: resolution.value, + }); + } + break; + } + + if (resolution.value) { + selectedProvider = provider.id; + selectedResolution = resolution; + params.setResolvedCredential({ + resolvedConfig: params.resolvedConfig, + provider, + value: resolution.value, + }); + break; + } + } + + if (!selectedProvider && keylessFallbackProvider) { + selectedProvider = keylessFallbackProvider.id; + selectedResolution = { + source: "missing" as TSource, + secretRefConfigured: false, + fallbackUsedAfterRefFailure: false, + }; + } + + const failUnresolvedNoFallback = (unresolved: { path: string; reason: string }) => { + const diagnostic: RuntimeWebDiagnostic = { + code: params.noFallbackCode, + message: unresolved.reason, + path: unresolved.path, + }; + params.diagnostics.push(diagnostic); + params.metadata.diagnostics.push(diagnostic); + pushWarning(params.context, { + code: params.noFallbackCode, + path: unresolved.path, + message: unresolved.reason, + }); + throw new Error(`[${params.noFallbackCode}] ${unresolved.reason}`); + }; + + if (params.configuredProvider) { + const unresolved = unresolvedWithoutFallback[0]; + if (unresolved) { + failUnresolvedNoFallback(unresolved); + } + } else { + if (!selectedProvider && unresolvedWithoutFallback.length > 0) { + failUnresolvedNoFallback(unresolvedWithoutFallback[0]); + } + + if (selectedProvider) { + const selectedProviderEntry = params.providers.find( + (entry) => entry.id === selectedProvider, + ); + const selectedDetails = + selectedProviderEntry?.requiresCredential === false + ? `${params.scopePath} auto-detected keyless provider "${selectedProvider}" as the default fallback.` + : `${params.scopePath} auto-detected provider "${selectedProvider}" from available credentials.`; + const diagnostic: RuntimeWebDiagnostic = { + code: params.autoDetectSelectedCode, + message: selectedDetails, + path: `${params.scopePath}.provider`, + }; + params.diagnostics.push(diagnostic); + params.metadata.diagnostics.push(diagnostic); + } + } + + if (selectedProvider) { + params.metadata.selectedProvider = selectedProvider; + params.metadata.selectedProviderKeySource = selectedResolution?.source; + if (!params.configuredProvider) { + params.metadata.providerSource = "auto-detect"; + } + const provider = params.providers.find((entry) => entry.id === selectedProvider); + if (provider && params.mergeRuntimeMetadata) { + await params.mergeRuntimeMetadata({ + provider, + metadata: params.metadata, + toolConfig: params.toolConfig, + selectedResolution, + }); + } + } + } + + if (params.enabled && !params.configuredProvider && params.metadata.selectedProvider) { + for (const provider of params.providers) { + if (provider.id === params.metadata.selectedProvider) { + continue; + } + const value = params.readConfiguredCredential({ + provider, + config: params.sourceConfig, + toolConfig: params.toolConfig, + }); + if (!params.hasConfiguredSecretRef(value, params.defaults)) { + continue; + } + for (const path of params.inactivePathsForProvider(provider)) { + pushInactiveSurfaceWarning({ + context: params.context, + path, + details: `${params.scopePath} auto-detected provider is "${params.metadata.selectedProvider}".`, + }); + } + } + } else if (params.toolConfig && !params.enabled) { + for (const provider of params.providers) { + const value = params.readConfiguredCredential({ + provider, + config: params.sourceConfig, + toolConfig: params.toolConfig, + }); + if (!params.hasConfiguredSecretRef(value, params.defaults)) { + continue; + } + for (const path of params.inactivePathsForProvider(provider)) { + pushInactiveSurfaceWarning({ + context: params.context, + path, + details: `${params.scopePath} is disabled.`, + }); + } + } + } + + if (params.enabled && params.toolConfig && params.configuredProvider) { + for (const provider of params.providers) { + if (provider.id === params.configuredProvider) { + continue; + } + const value = params.readConfiguredCredential({ + provider, + config: params.sourceConfig, + toolConfig: params.toolConfig, + }); + if (!params.hasConfiguredSecretRef(value, params.defaults)) { + continue; + } + for (const path of params.inactivePathsForProvider(provider)) { + pushInactiveSurfaceWarning({ + context: params.context, + path, + details: `${params.scopePath}.provider is "${params.configuredProvider}".`, + }); + } + } + } +} diff --git a/src/secrets/runtime-web-tools.ts b/src/secrets/runtime-web-tools.ts index e999b09489d..94b8a907b8e 100644 --- a/src/secrets/runtime-web-tools.ts +++ b/src/secrets/runtime-web-tools.ts @@ -1,9 +1,6 @@ import type { OpenClawConfig } from "../config/config.js"; import { resolveSecretInputRef } from "../config/types.secrets.js"; -import { - resolveManifestContractOwnerPluginId, - resolveManifestContractPluginIds, -} from "../plugins/manifest-registry.js"; +import { resolveManifestContractPluginIds } from "../plugins/manifest-registry.js"; import type { PluginWebFetchProviderEntry, PluginWebSearchProviderEntry, @@ -17,12 +14,15 @@ import { sortWebSearchProvidersForAutoDetect } from "../plugins/web-search-provi import { normalizeSecretInput } from "../utils/normalize-secret-input.js"; import { secretRefKey } from "./ref-contract.js"; import { resolveSecretRefValues } from "./resolve.js"; +import type { ResolverContext, SecretDefaults } from "./runtime-shared.js"; import { - pushInactiveSurfaceWarning, - pushWarning, - type ResolverContext, - type SecretDefaults, -} from "./runtime-shared.js"; + ensureObject, + hasConfiguredSecretRef, + isRecord, + resolveRuntimeWebProviderSurface, + resolveRuntimeWebProviderSelection, + type SecretResolutionResult, +} from "./runtime-web-tools.shared.js"; import type { RuntimeWebDiagnostic, RuntimeWebDiagnosticCode, @@ -31,9 +31,6 @@ import type { RuntimeWebToolsMetadata, } from "./runtime-web-tools.types.js"; -type WebSearchProvider = string; -type WebFetchProvider = string; - export type { RuntimeWebDiagnostic, RuntimeWebDiagnosticCode, @@ -48,18 +45,9 @@ type FetchConfig = NonNullable["web"] extends infer Web : undefined : undefined; -type SecretResolutionResult = { - value?: string; - source: WebSearchCredentialResolutionSource | WebFetchCredentialResolutionSource; - secretRefConfigured: boolean; - unresolvedRefReason?: string; - fallbackEnvVar?: string; - fallbackUsedAfterRefFailure: boolean; -}; - -function isRecord(value: unknown): value is Record { - return typeof value === "object" && value !== null && !Array.isArray(value); -} +type SecretResolutionSource = + | WebSearchCredentialResolutionSource + | WebFetchCredentialResolutionSource; function hasPluginWebToolConfig(config: OpenClawConfig): boolean { const entries = config.plugins?.entries; @@ -75,34 +63,6 @@ function hasPluginWebToolConfig(config: OpenClawConfig): boolean { }); } -function normalizeProvider( - value: unknown, - providers: ReturnType, -): WebSearchProvider | undefined { - if (typeof value !== "string") { - return undefined; - } - const normalized = value.trim().toLowerCase(); - if (providers.some((provider) => provider.id === normalized)) { - return normalized; - } - return undefined; -} - -function normalizeFetchProvider( - value: unknown, - providers: PluginWebFetchProviderEntry[], -): WebFetchProvider | undefined { - if (typeof value !== "string") { - return undefined; - } - const normalized = value.trim().toLowerCase(); - if (providers.some((provider) => provider.id === normalized)) { - return normalized; - } - return undefined; -} - function hasCustomWebSearchPluginRisk(config: OpenClawConfig): boolean { const plugins = config.plugins; if (!plugins) { @@ -172,7 +132,7 @@ async function resolveSecretInputWithEnvFallback(params: { path: string; envVars: string[]; restrictEnvRefsToEnvVars?: boolean; -}): Promise { +}): Promise> { const { ref } = resolveSecretInputRef({ value: params.value, defaults: params.defaults, @@ -277,16 +237,6 @@ async function resolveSecretInputWithEnvFallback(params: { }; } -function ensureObject(target: Record, key: string): Record { - const current = target[key]; - if (isRecord(current)) { - return current; - } - const next: Record = {}; - target[key] = next; - return next; -} - function setResolvedWebSearchApiKey(params: { resolvedConfig: OpenClawConfig; provider: PluginWebSearchProviderEntry; @@ -304,10 +254,6 @@ function setResolvedWebSearchApiKey(params: { params.provider.setCredentialValue(search, params.value); } -function keyPathForProvider(provider: PluginWebSearchProviderEntry): string { - return provider.credentialPath; -} - function readConfiguredProviderCredential(params: { provider: PluginWebSearchProviderEntry; config: OpenClawConfig; @@ -341,10 +287,6 @@ function setResolvedWebFetchApiKey(params: { params.provider.setCredentialValue(fetch, params.value); } -function keyPathForFetchProvider(provider: PluginWebFetchProviderEntry): string { - return provider.credentialPath; -} - function readConfiguredFetchProviderCredential(params: { provider: PluginWebFetchProviderEntry; config: OpenClawConfig; @@ -363,15 +305,6 @@ function inactivePathsForFetchProvider(provider: PluginWebFetchProviderEntry): s : [provider.credentialPath]; } -function hasConfiguredSecretRef(value: unknown, defaults: SecretDefaults | undefined): boolean { - return Boolean( - resolveSecretInputRef({ - value, - defaults, - }).ref, - ); -} - export async function resolveRuntimeWebTools(params: { sourceConfig: OpenClawConfig; resolvedConfig: OpenClawConfig; @@ -413,594 +346,215 @@ export async function resolveRuntimeWebTools(params: { } const rawProvider = typeof search?.provider === "string" ? search.provider.trim().toLowerCase() : ""; - const configuredBundledPluginId = resolveManifestContractOwnerPluginId({ - contract: "webSearchProviders", - value: rawProvider, - origin: "bundled", - config: params.sourceConfig, - env: { ...process.env, ...params.context.env }, - }); - const searchMetadata: RuntimeWebSearchMetadata = { providerSource: "none", diagnostics: [], }; - - const searchProviders = sortWebSearchProvidersForAutoDetect( - configuredBundledPluginId - ? resolvePluginWebSearchProviders({ - config: params.sourceConfig, - env: { ...process.env, ...params.context.env }, - bundledAllowlistCompat: true, - onlyPluginIds: [configuredBundledPluginId], - origin: "bundled", - }) - : !hasCustomWebSearchPluginRisk(params.sourceConfig) + const searchSurface = resolveRuntimeWebProviderSurface({ + contract: "webSearchProviders", + rawProvider, + providerPath: "tools.web.search.provider", + toolConfig: search, + diagnostics, + metadataDiagnostics: searchMetadata.diagnostics, + invalidAutoDetectCode: "WEB_SEARCH_PROVIDER_INVALID_AUTODETECT", + sourceConfig: params.sourceConfig, + context: params.context, + resolveProviders: ({ configuredBundledPluginId }) => + configuredBundledPluginId ? resolvePluginWebSearchProviders({ config: params.sourceConfig, env: { ...process.env, ...params.context.env }, bundledAllowlistCompat: true, + onlyPluginIds: [configuredBundledPluginId], origin: "bundled", }) - : resolvePluginWebSearchProviders({ - config: params.sourceConfig, - env: { ...process.env, ...params.context.env }, - bundledAllowlistCompat: true, - }), - ); - const searchConfigured = Boolean(search); - const hasConfiguredSearchSurface = - searchConfigured || - searchProviders.some((provider) => { - if (provider.requiresCredential === false) { - return false; - } - const value = readConfiguredProviderCredential({ + : !hasCustomWebSearchPluginRisk(params.sourceConfig) + ? resolvePluginWebSearchProviders({ + config: params.sourceConfig, + env: { ...process.env, ...params.context.env }, + bundledAllowlistCompat: true, + origin: "bundled", + }) + : resolvePluginWebSearchProviders({ + config: params.sourceConfig, + env: { ...process.env, ...params.context.env }, + bundledAllowlistCompat: true, + }), + sortProviders: sortWebSearchProvidersForAutoDetect, + readConfiguredCredential: ({ provider, config, toolConfig }) => + readConfiguredProviderCredential({ provider, - config: params.sourceConfig, - search, - }); - return value !== undefined; - }); - const searchEnabled = hasConfiguredSearchSurface && search?.enabled !== false; - const providers = hasConfiguredSearchSurface ? searchProviders : []; - const configuredProvider = normalizeProvider(rawProvider, providers); + config, + search: toolConfig, + }), + ignoreKeylessProvidersForConfiguredSurface: true, + emptyProvidersWhenSurfaceMissing: true, + normalizeConfiguredProviderAgainstActiveProviders: true, + }); - if (rawProvider && !configuredProvider) { - const diagnostic: RuntimeWebDiagnostic = { - code: "WEB_SEARCH_PROVIDER_INVALID_AUTODETECT", - message: `tools.web.search.provider is "${rawProvider}". Falling back to auto-detect precedence.`, - path: "tools.web.search.provider", - }; - diagnostics.push(diagnostic); - searchMetadata.diagnostics.push(diagnostic); - pushWarning(params.context, { - code: "WEB_SEARCH_PROVIDER_INVALID_AUTODETECT", - path: "tools.web.search.provider", - message: diagnostic.message, - }); - } - - if (configuredProvider) { - searchMetadata.providerConfigured = configuredProvider; - searchMetadata.providerSource = "configured"; - } - - if (searchEnabled) { - const candidates = configuredProvider - ? providers.filter((provider) => provider.id === configuredProvider) - : providers; - const unresolvedWithoutFallback: Array<{ - provider: WebSearchProvider; - path: string; - reason: string; - }> = []; - - let selectedProvider: WebSearchProvider | undefined; - let selectedResolution: SecretResolutionResult | undefined; - let keylessFallbackProvider: PluginWebSearchProviderEntry | undefined; - - for (const provider of candidates) { - if (provider.requiresCredential === false) { - if (!keylessFallbackProvider) { - keylessFallbackProvider = provider; - } - if (configuredProvider) { - selectedProvider = provider.id; - break; - } - continue; - } - const path = keyPathForProvider(provider); - const value = readConfiguredProviderCredential({ + await resolveRuntimeWebProviderSelection({ + scopePath: "tools.web.search", + toolConfig: search, + enabled: searchSurface.enabled, + providers: searchSurface.providers, + configuredProvider: searchSurface.configuredProvider, + metadata: searchMetadata, + diagnostics, + sourceConfig: params.sourceConfig, + resolvedConfig: params.resolvedConfig, + context: params.context, + defaults, + deferKeylessFallback: true, + fallbackUsedCode: "WEB_SEARCH_KEY_UNRESOLVED_FALLBACK_USED", + noFallbackCode: "WEB_SEARCH_KEY_UNRESOLVED_NO_FALLBACK", + autoDetectSelectedCode: "WEB_SEARCH_AUTODETECT_SELECTED", + readConfiguredCredential: ({ provider, config, toolConfig }) => + readConfiguredProviderCredential({ provider, - config: params.sourceConfig, - search, - }); - const resolution = await resolveSecretInputWithEnvFallback({ + config, + search: toolConfig, + }), + resolveSecretInput: ({ value, path, envVars }) => + resolveSecretInputWithEnvFallback({ sourceConfig: params.sourceConfig, context: params.context, defaults, value, path, - envVars: provider.envVars, - }); - - if (resolution.secretRefConfigured && resolution.fallbackUsedAfterRefFailure) { - const diagnostic: RuntimeWebDiagnostic = { - code: "WEB_SEARCH_KEY_UNRESOLVED_FALLBACK_USED", - message: - `${path} SecretRef could not be resolved; using ${resolution.fallbackEnvVar ?? "env fallback"}. ` + - (resolution.unresolvedRefReason ?? "").trim(), - path, - }; - diagnostics.push(diagnostic); - searchMetadata.diagnostics.push(diagnostic); - pushWarning(params.context, { - code: "WEB_SEARCH_KEY_UNRESOLVED_FALLBACK_USED", - path, - message: diagnostic.message, - }); - } - - if (resolution.secretRefConfigured && !resolution.value && resolution.unresolvedRefReason) { - unresolvedWithoutFallback.push({ - provider: provider.id, - path, - reason: resolution.unresolvedRefReason, - }); - } - - if (configuredProvider) { - selectedProvider = provider.id; - selectedResolution = resolution; - if (resolution.value) { - setResolvedWebSearchApiKey({ - resolvedConfig: params.resolvedConfig, - provider, - value: resolution.value, - }); - } - break; - } - - if (resolution.value) { - selectedProvider = provider.id; - selectedResolution = resolution; - setResolvedWebSearchApiKey({ - resolvedConfig: params.resolvedConfig, - provider, - value: resolution.value, - }); - break; - } - } - - if (!selectedProvider && keylessFallbackProvider) { - selectedProvider = keylessFallbackProvider.id; - selectedResolution = { - source: "missing", - secretRefConfigured: false, - fallbackUsedAfterRefFailure: false, - }; - } - - const failUnresolvedSearchNoFallback = (unresolved: { path: string; reason: string }) => { - const diagnostic: RuntimeWebDiagnostic = { - code: "WEB_SEARCH_KEY_UNRESOLVED_NO_FALLBACK", - message: unresolved.reason, - path: unresolved.path, - }; - diagnostics.push(diagnostic); - searchMetadata.diagnostics.push(diagnostic); - pushWarning(params.context, { - code: "WEB_SEARCH_KEY_UNRESOLVED_NO_FALLBACK", - path: unresolved.path, - message: unresolved.reason, - }); - throw new Error(`[WEB_SEARCH_KEY_UNRESOLVED_NO_FALLBACK] ${unresolved.reason}`); - }; - - if (configuredProvider) { - const unresolved = unresolvedWithoutFallback[0]; - if (unresolved) { - failUnresolvedSearchNoFallback(unresolved); - } - } else { - if (!selectedProvider && unresolvedWithoutFallback.length > 0) { - failUnresolvedSearchNoFallback(unresolvedWithoutFallback[0]); - } - - if (selectedProvider) { - const selectedProviderEntry = providers.find((entry) => entry.id === selectedProvider); - const selectedDetails = - selectedProviderEntry?.requiresCredential === false - ? `tools.web.search auto-detected keyless provider "${selectedProvider}" as the default fallback.` - : `tools.web.search auto-detected provider "${selectedProvider}" from available credentials.`; - const diagnostic: RuntimeWebDiagnostic = { - code: "WEB_SEARCH_AUTODETECT_SELECTED", - message: selectedDetails, - path: "tools.web.search.provider", - }; - diagnostics.push(diagnostic); - searchMetadata.diagnostics.push(diagnostic); - } - } - - if (selectedProvider) { - searchMetadata.selectedProvider = selectedProvider; - searchMetadata.selectedProviderKeySource = selectedResolution?.source; - if (!configuredProvider) { - searchMetadata.providerSource = "auto-detect"; - } - const provider = providers.find((entry) => entry.id === selectedProvider); - if (provider?.resolveRuntimeMetadata) { - Object.assign( - searchMetadata, - await provider.resolveRuntimeMetadata({ - config: params.sourceConfig, - searchConfig: search, - runtimeMetadata: searchMetadata, - resolvedCredential: selectedResolution - ? { - value: selectedResolution.value, - source: selectedResolution.source, - fallbackEnvVar: selectedResolution.fallbackEnvVar, - } - : undefined, - }), - ); - } - } - } - - if (searchEnabled && !configuredProvider && searchMetadata.selectedProvider) { - for (const provider of providers) { - if (provider.id === searchMetadata.selectedProvider) { - continue; - } - const value = readConfiguredProviderCredential({ + envVars, + }), + setResolvedCredential: ({ resolvedConfig, provider, value }) => + setResolvedWebSearchApiKey({ + resolvedConfig, provider, - config: params.sourceConfig, - search, - }); - if (!hasConfiguredSecretRef(value, defaults)) { - continue; + value, + }), + inactivePathsForProvider, + hasConfiguredSecretRef, + mergeRuntimeMetadata: async ({ provider, metadata, toolConfig, selectedResolution }) => { + if (!provider.resolveRuntimeMetadata) { + return; } - for (const path of inactivePathsForProvider(provider)) { - pushInactiveSurfaceWarning({ - context: params.context, - path, - details: `tools.web.search auto-detected provider is "${searchMetadata.selectedProvider}".`, - }); - } - } - } else if (search && !searchEnabled) { - for (const provider of providers) { - const value = readConfiguredProviderCredential({ - provider, - config: params.sourceConfig, - search, - }); - if (!hasConfiguredSecretRef(value, defaults)) { - continue; - } - for (const path of inactivePathsForProvider(provider)) { - pushInactiveSurfaceWarning({ - context: params.context, - path, - details: "tools.web.search is disabled.", - }); - } - } - } - - if (searchEnabled && search && configuredProvider) { - for (const provider of providers) { - if (provider.id === configuredProvider) { - continue; - } - const value = readConfiguredProviderCredential({ - provider, - config: params.sourceConfig, - search, - }); - if (!hasConfiguredSecretRef(value, defaults)) { - continue; - } - for (const path of inactivePathsForProvider(provider)) { - pushInactiveSurfaceWarning({ - context: params.context, - path, - details: `tools.web.search.provider is "${configuredProvider}".`, - }); - } - } - } + Object.assign( + metadata, + await provider.resolveRuntimeMetadata({ + config: params.sourceConfig, + searchConfig: toolConfig, + runtimeMetadata: metadata, + resolvedCredential: selectedResolution + ? { + value: selectedResolution.value, + source: selectedResolution.source, + fallbackEnvVar: selectedResolution.fallbackEnvVar, + } + : undefined, + }), + ); + }, + }); const rawFetchProvider = typeof fetch?.provider === "string" ? fetch.provider.trim().toLowerCase() : ""; - const configuredBundledFetchPluginId = resolveManifestContractOwnerPluginId({ - contract: "webFetchProviders", - value: rawFetchProvider, - origin: "bundled", - config: params.sourceConfig, - env: { ...process.env, ...params.context.env }, - }); const fetchMetadata: RuntimeWebFetchMetadata = { providerSource: "none", diagnostics: [], }; - const fetchProviders = sortWebFetchProvidersForAutoDetect( - configuredBundledFetchPluginId - ? resolvePluginWebFetchProviders({ - config: params.sourceConfig, - env: { ...process.env, ...params.context.env }, - bundledAllowlistCompat: true, - onlyPluginIds: [configuredBundledFetchPluginId], - origin: "bundled", - }) - : resolvePluginWebFetchProviders({ - config: params.sourceConfig, - env: { ...process.env, ...params.context.env }, - bundledAllowlistCompat: true, - origin: "bundled", - }), - ); - const hasConfiguredFetchSurface = - Boolean(fetch) || - fetchProviders.some((provider) => { - const value = readConfiguredFetchProviderCredential({ + const fetchSurface = resolveRuntimeWebProviderSurface({ + contract: "webFetchProviders", + rawProvider: rawFetchProvider, + providerPath: "tools.web.fetch.provider", + toolConfig: fetch, + diagnostics, + metadataDiagnostics: fetchMetadata.diagnostics, + invalidAutoDetectCode: "WEB_FETCH_PROVIDER_INVALID_AUTODETECT", + sourceConfig: params.sourceConfig, + context: params.context, + resolveProviders: ({ configuredBundledPluginId }) => + configuredBundledPluginId + ? resolvePluginWebFetchProviders({ + config: params.sourceConfig, + env: { ...process.env, ...params.context.env }, + bundledAllowlistCompat: true, + onlyPluginIds: [configuredBundledPluginId], + origin: "bundled", + }) + : resolvePluginWebFetchProviders({ + config: params.sourceConfig, + env: { ...process.env, ...params.context.env }, + bundledAllowlistCompat: true, + origin: "bundled", + }), + sortProviders: sortWebFetchProvidersForAutoDetect, + readConfiguredCredential: ({ provider, config, toolConfig }) => + readConfiguredFetchProviderCredential({ provider, - config: params.sourceConfig, - fetch, - }); - return value !== undefined; - }); - const fetchEnabled = hasConfiguredFetchSurface && fetch?.enabled !== false; - const configuredFetchProvider = normalizeFetchProvider(rawFetchProvider, fetchProviders); + config, + fetch: toolConfig, + }), + }); - if (rawFetchProvider && !configuredFetchProvider) { - const diagnostic: RuntimeWebDiagnostic = { - code: "WEB_FETCH_PROVIDER_INVALID_AUTODETECT", - message: `tools.web.fetch.provider is "${rawFetchProvider}". Falling back to auto-detect precedence.`, - path: "tools.web.fetch.provider", - }; - diagnostics.push(diagnostic); - fetchMetadata.diagnostics.push(diagnostic); - pushWarning(params.context, { - code: "WEB_FETCH_PROVIDER_INVALID_AUTODETECT", - path: "tools.web.fetch.provider", - message: diagnostic.message, - }); - } - - if (configuredFetchProvider) { - fetchMetadata.providerConfigured = configuredFetchProvider; - fetchMetadata.providerSource = "configured"; - } - - if (fetchEnabled) { - const candidates = configuredFetchProvider - ? fetchProviders.filter((provider) => provider.id === configuredFetchProvider) - : fetchProviders; - const unresolvedWithoutFallback: Array<{ - provider: WebFetchProvider; - path: string; - reason: string; - }> = []; - - let selectedProvider: WebFetchProvider | undefined; - let selectedResolution: SecretResolutionResult | undefined; - - for (const provider of candidates) { - if (provider.requiresCredential === false) { - selectedProvider = provider.id; - selectedResolution = { - source: "missing", - secretRefConfigured: false, - fallbackUsedAfterRefFailure: false, - }; - break; - } - const path = keyPathForFetchProvider(provider); - const value = readConfiguredFetchProviderCredential({ + await resolveRuntimeWebProviderSelection({ + scopePath: "tools.web.fetch", + toolConfig: fetch, + enabled: fetchSurface.enabled, + providers: fetchSurface.providers, + configuredProvider: fetchSurface.configuredProvider, + metadata: fetchMetadata, + diagnostics, + sourceConfig: params.sourceConfig, + resolvedConfig: params.resolvedConfig, + context: params.context, + defaults, + deferKeylessFallback: false, + fallbackUsedCode: "WEB_FETCH_PROVIDER_KEY_UNRESOLVED_FALLBACK_USED", + noFallbackCode: "WEB_FETCH_PROVIDER_KEY_UNRESOLVED_NO_FALLBACK", + autoDetectSelectedCode: "WEB_FETCH_AUTODETECT_SELECTED", + readConfiguredCredential: ({ provider, config, toolConfig }) => + readConfiguredFetchProviderCredential({ provider, - config: params.sourceConfig, - fetch, - }); - const resolution = await resolveSecretInputWithEnvFallback({ + config, + fetch: toolConfig, + }), + resolveSecretInput: ({ value, path, envVars }) => + resolveSecretInputWithEnvFallback({ sourceConfig: params.sourceConfig, context: params.context, defaults, value, path, - envVars: provider.envVars, + envVars, restrictEnvRefsToEnvVars: true, - }); - - if (resolution.secretRefConfigured && resolution.fallbackUsedAfterRefFailure) { - const diagnostic: RuntimeWebDiagnostic = { - code: "WEB_FETCH_PROVIDER_KEY_UNRESOLVED_FALLBACK_USED", - message: - `${path} SecretRef could not be resolved; using ${resolution.fallbackEnvVar ?? "env fallback"}. ` + - (resolution.unresolvedRefReason ?? "").trim(), - path, - }; - diagnostics.push(diagnostic); - fetchMetadata.diagnostics.push(diagnostic); - pushWarning(params.context, { - code: "WEB_FETCH_PROVIDER_KEY_UNRESOLVED_FALLBACK_USED", - path, - message: diagnostic.message, - }); - } - - if (resolution.secretRefConfigured && !resolution.value && resolution.unresolvedRefReason) { - unresolvedWithoutFallback.push({ - provider: provider.id, - path, - reason: resolution.unresolvedRefReason, - }); - } - - if (configuredFetchProvider) { - selectedProvider = provider.id; - selectedResolution = resolution; - if (resolution.value) { - setResolvedWebFetchApiKey({ - resolvedConfig: params.resolvedConfig, - provider, - value: resolution.value, - }); - } - break; - } - - if (resolution.value) { - selectedProvider = provider.id; - selectedResolution = resolution; - setResolvedWebFetchApiKey({ - resolvedConfig: params.resolvedConfig, - provider, - value: resolution.value, - }); - break; - } - } - - const failUnresolvedFetchNoFallback = (unresolved: { path: string; reason: string }) => { - const diagnostic: RuntimeWebDiagnostic = { - code: "WEB_FETCH_PROVIDER_KEY_UNRESOLVED_NO_FALLBACK", - message: unresolved.reason, - path: unresolved.path, - }; - diagnostics.push(diagnostic); - fetchMetadata.diagnostics.push(diagnostic); - pushWarning(params.context, { - code: "WEB_FETCH_PROVIDER_KEY_UNRESOLVED_NO_FALLBACK", - path: unresolved.path, - message: unresolved.reason, - }); - throw new Error(`[WEB_FETCH_PROVIDER_KEY_UNRESOLVED_NO_FALLBACK] ${unresolved.reason}`); - }; - - if (configuredFetchProvider) { - const unresolved = unresolvedWithoutFallback[0]; - if (unresolved) { - failUnresolvedFetchNoFallback(unresolved); - } - } else { - if (!selectedProvider && unresolvedWithoutFallback.length > 0) { - failUnresolvedFetchNoFallback(unresolvedWithoutFallback[0]); - } - - if (selectedProvider) { - const selectedProviderEntry = fetchProviders.find((entry) => entry.id === selectedProvider); - const selectedDetails = - selectedProviderEntry?.requiresCredential === false - ? `tools.web.fetch auto-detected keyless provider "${selectedProvider}" as the default fallback.` - : `tools.web.fetch auto-detected provider "${selectedProvider}" from available credentials.`; - const diagnostic: RuntimeWebDiagnostic = { - code: "WEB_FETCH_AUTODETECT_SELECTED", - message: selectedDetails, - path: "tools.web.fetch.provider", - }; - diagnostics.push(diagnostic); - fetchMetadata.diagnostics.push(diagnostic); - } - } - - if (selectedProvider) { - fetchMetadata.selectedProvider = selectedProvider; - fetchMetadata.selectedProviderKeySource = selectedResolution?.source; - if (!configuredFetchProvider) { - fetchMetadata.providerSource = "auto-detect"; - } - const provider = fetchProviders.find((entry) => entry.id === selectedProvider); - if (provider?.resolveRuntimeMetadata) { - Object.assign( - fetchMetadata, - await provider.resolveRuntimeMetadata({ - config: params.sourceConfig, - fetchConfig: fetch, - runtimeMetadata: fetchMetadata, - resolvedCredential: selectedResolution - ? { - value: selectedResolution.value, - source: selectedResolution.source, - fallbackEnvVar: selectedResolution.fallbackEnvVar, - } - : undefined, - }), - ); - } - } - } - - if (fetchEnabled && !configuredFetchProvider && fetchMetadata.selectedProvider) { - for (const provider of fetchProviders) { - if (provider.id === fetchMetadata.selectedProvider) { - continue; - } - const value = readConfiguredFetchProviderCredential({ + }), + setResolvedCredential: ({ resolvedConfig, provider, value }) => + setResolvedWebFetchApiKey({ + resolvedConfig, provider, - config: params.sourceConfig, - fetch, - }); - if (!hasConfiguredSecretRef(value, defaults)) { - continue; + value, + }), + inactivePathsForProvider: inactivePathsForFetchProvider, + hasConfiguredSecretRef, + mergeRuntimeMetadata: async ({ provider, metadata, toolConfig, selectedResolution }) => { + if (!provider.resolveRuntimeMetadata) { + return; } - for (const path of inactivePathsForFetchProvider(provider)) { - pushInactiveSurfaceWarning({ - context: params.context, - path, - details: `tools.web.fetch auto-detected provider is "${fetchMetadata.selectedProvider}".`, - }); - } - } - } else if (fetch && !fetchEnabled) { - for (const provider of fetchProviders) { - const value = readConfiguredFetchProviderCredential({ - provider, - config: params.sourceConfig, - fetch, - }); - if (!hasConfiguredSecretRef(value, defaults)) { - continue; - } - for (const path of inactivePathsForFetchProvider(provider)) { - pushInactiveSurfaceWarning({ - context: params.context, - path, - details: "tools.web.fetch is disabled.", - }); - } - } - } - - if (fetchEnabled && fetch && configuredFetchProvider) { - for (const provider of fetchProviders) { - if (provider.id === configuredFetchProvider) { - continue; - } - const value = readConfiguredFetchProviderCredential({ - provider, - config: params.sourceConfig, - fetch, - }); - if (!hasConfiguredSecretRef(value, defaults)) { - continue; - } - for (const path of inactivePathsForFetchProvider(provider)) { - pushInactiveSurfaceWarning({ - context: params.context, - path, - details: `tools.web.fetch.provider is "${configuredFetchProvider}".`, - }); - } - } - } + Object.assign( + metadata, + await provider.resolveRuntimeMetadata({ + config: params.sourceConfig, + fetchConfig: toolConfig, + runtimeMetadata: metadata, + resolvedCredential: selectedResolution + ? { + value: selectedResolution.value, + source: selectedResolution.source, + fallbackEnvVar: selectedResolution.fallbackEnvVar, + } + : undefined, + }), + ); + }, + }); return { search: searchMetadata, diff --git a/src/web-fetch/runtime.ts b/src/web-fetch/runtime.ts index 77a687d9df3..0938862ef0f 100644 --- a/src/web-fetch/runtime.ts +++ b/src/web-fetch/runtime.ts @@ -1,5 +1,4 @@ import type { OpenClawConfig } from "../config/config.js"; -import { normalizeSecretInputString, resolveSecretInputRef } from "../config/types.secrets.js"; import { logVerbose } from "../globals.js"; import type { PluginWebFetchProviderEntry, @@ -9,7 +8,13 @@ import { resolvePluginWebFetchProviders } from "../plugins/web-fetch-providers.r import { sortWebFetchProvidersForAutoDetect } from "../plugins/web-fetch-providers.shared.js"; import { getActiveRuntimeWebToolsMetadata } from "../secrets/runtime-web-tools-state.js"; import type { RuntimeWebFetchMetadata } from "../secrets/runtime-web-tools.types.js"; -import { normalizeSecretInput } from "../utils/normalize-secret-input.js"; +import { + hasWebProviderEntryCredential, + providerRequiresCredential, + readWebProviderEnvValue, + resolveWebProviderConfig, + resolveWebProviderDefinition, +} from "../web/provider-runtime-shared.js"; type WebFetchConfig = NonNullable["web"] extends infer Web ? Web extends { fetch?: infer Fetch } @@ -26,11 +31,7 @@ export type ResolveWebFetchDefinitionParams = { }; function resolveFetchConfig(cfg?: OpenClawConfig): WebFetchConfig { - const fetch = cfg?.tools?.web?.fetch; - if (!fetch || typeof fetch !== "object") { - return undefined; - } - return fetch as WebFetchConfig; + return resolveWebProviderConfig<"fetch", NonNullable>(cfg, "fetch"); } export function resolveWebFetchEnabled(params: { @@ -43,22 +44,6 @@ export function resolveWebFetchEnabled(params: { return true; } -function readProviderEnvValue(envVars: string[]): string | undefined { - for (const envVar of envVars) { - const value = normalizeSecretInput(process.env[envVar]); - if (value) { - return value; - } - } - return undefined; -} - -function providerRequiresCredential( - provider: Pick, -): boolean { - return provider.requiresCredential !== false; -} - function hasEntryCredential( provider: Pick< PluginWebFetchProviderEntry, @@ -67,19 +52,16 @@ function hasEntryCredential( config: OpenClawConfig | undefined, fetch: WebFetchConfig | undefined, ): boolean { - if (!providerRequiresCredential(provider)) { - return true; - } - const configuredValue = provider.getConfiguredCredentialValue?.(config); - const rawValue = configuredValue ?? provider.getCredentialValue(fetch as Record); - const configuredRef = resolveSecretInputRef({ - value: rawValue, - }).ref; - if (configuredRef && configuredRef.source !== "env") { - return true; - } - const fromConfig = normalizeSecretInput(normalizeSecretInputString(rawValue)); - return Boolean(fromConfig || readProviderEnvValue(provider.envVars)); + return hasWebProviderEntryCredential({ + provider, + config, + toolConfig: fetch as Record | undefined, + resolveRawValue: ({ provider: currentProvider, config: currentConfig, toolConfig }) => + currentProvider.getConfiguredCredentialValue?.(currentConfig) ?? + currentProvider.getCredentialValue(toolConfig), + resolveEnvValue: ({ provider: currentProvider }) => + readWebProviderEnvValue(currentProvider.envVars), + }); } export function listWebFetchProviders(params?: { @@ -150,42 +132,36 @@ export function resolveWebFetchDefinition( ): { provider: PluginWebFetchProviderEntry; definition: WebFetchProviderToolDefinition } | null { const fetch = resolveFetchConfig(options?.config); const runtimeWebFetch = options?.runtimeWebFetch ?? getActiveRuntimeWebToolsMetadata()?.fetch; - if (!resolveWebFetchEnabled({ fetch, sandboxed: options?.sandboxed })) { - return null; - } - const providers = sortWebFetchProvidersForAutoDetect( resolvePluginWebFetchProviders({ config: options?.config, bundledAllowlistCompat: true, origin: "bundled", }), - ).filter(Boolean); - if (providers.length === 0) { - return null; - } - - const providerId = - options?.providerId ?? - runtimeWebFetch?.selectedProvider ?? - runtimeWebFetch?.providerConfigured ?? - resolveWebFetchProviderId({ config: options?.config, fetch, providers }); - if (!providerId) { - return null; - } - const provider = providers.find((entry) => entry.id === providerId); - if (!provider) { - return null; - } - - const definition = provider.createTool({ + ); + return resolveWebProviderDefinition({ config: options?.config, - fetchConfig: fetch as Record | undefined, + toolConfig: fetch as Record | undefined, runtimeMetadata: runtimeWebFetch, + sandboxed: options?.sandboxed, + providerId: options?.providerId, + providers, + resolveEnabled: ({ toolConfig, sandboxed }) => + resolveWebFetchEnabled({ + fetch: toolConfig as WebFetchConfig | undefined, + sandboxed, + }), + resolveAutoProviderId: ({ config, toolConfig, providers }) => + resolveWebFetchProviderId({ + config, + fetch: toolConfig as WebFetchConfig | undefined, + providers, + }), + createTool: ({ provider, config, toolConfig, runtimeMetadata }) => + provider.createTool({ + config, + fetchConfig: toolConfig, + runtimeMetadata, + }), }); - if (!definition) { - return null; - } - - return { provider, definition }; } diff --git a/src/web-search/runtime.ts b/src/web-search/runtime.ts index 9dc79f4c72e..f73cf56aa25 100644 --- a/src/web-search/runtime.ts +++ b/src/web-search/runtime.ts @@ -1,5 +1,4 @@ import type { OpenClawConfig } from "../config/config.js"; -import { normalizeSecretInputString, resolveSecretInputRef } from "../config/types.secrets.js"; import { logVerbose } from "../globals.js"; import type { PluginWebSearchProviderEntry, @@ -10,7 +9,13 @@ import { resolveRuntimeWebSearchProviders } from "../plugins/web-search-provider import { sortWebSearchProvidersForAutoDetect } from "../plugins/web-search-providers.shared.js"; import { getActiveRuntimeWebToolsMetadata } from "../secrets/runtime-web-tools-state.js"; import type { RuntimeWebSearchMetadata } from "../secrets/runtime-web-tools.types.js"; -import { normalizeSecretInput } from "../utils/normalize-secret-input.js"; +import { + hasWebProviderEntryCredential, + providerRequiresCredential, + readWebProviderEnvValue, + resolveWebProviderConfig, + resolveWebProviderDefinition, +} from "../web/provider-runtime-shared.js"; type WebSearchConfig = NonNullable["web"] extends infer Web ? Web extends { search?: infer Search } @@ -31,11 +36,7 @@ export type RunWebSearchParams = ResolveWebSearchDefinitionParams & { }; function resolveSearchConfig(cfg?: OpenClawConfig): WebSearchConfig { - const search = cfg?.tools?.web?.search; - if (!search || typeof search !== "object") { - return undefined; - } - return search as WebSearchConfig; + return resolveWebProviderConfig<"search", NonNullable>(cfg, "search"); } export function resolveWebSearchEnabled(params: { @@ -51,22 +52,6 @@ export function resolveWebSearchEnabled(params: { return true; } -function readProviderEnvValue(envVars: string[]): string | undefined { - for (const envVar of envVars) { - const value = normalizeSecretInput(process.env[envVar]); - if (value) { - return value; - } - } - return undefined; -} - -function providerRequiresCredential( - provider: Pick, -): boolean { - return provider.requiresCredential !== false; -} - function hasEntryCredential( provider: Pick< PluginWebSearchProviderEntry, @@ -80,28 +65,17 @@ function hasEntryCredential( config: OpenClawConfig | undefined, search: WebSearchConfig | undefined, ): boolean { - if (!providerRequiresCredential(provider)) { - return true; - } - const configuredValue = provider.getConfiguredCredentialValue?.(config); - const rawValue = - configuredValue ?? - (provider.id === "brave" - ? provider.getCredentialValue(search as Record | undefined) - : undefined); - const configuredRef = resolveSecretInputRef({ - value: rawValue, - }).ref; - if (configuredRef && configuredRef.source !== "env") { - return true; - } - const fromConfig = normalizeSecretInput(normalizeSecretInputString(rawValue)); - if (configuredRef?.source === "env") { - return Boolean( - normalizeSecretInput(process.env[configuredRef.id]) || readProviderEnvValue(provider.envVars), - ); - } - return Boolean(fromConfig || readProviderEnvValue(provider.envVars)); + return hasWebProviderEntryCredential({ + provider, + config, + toolConfig: search as Record | undefined, + resolveRawValue: ({ provider: currentProvider, config: currentConfig, toolConfig }) => + currentProvider.getConfiguredCredentialValue?.(currentConfig) ?? + (currentProvider.id === "brave" ? currentProvider.getCredentialValue(toolConfig) : undefined), + resolveEnvValue: ({ provider: currentProvider, configuredEnvVarId }) => + (configuredEnvVarId ? readWebProviderEnvValue([configuredEnvVarId]) : undefined) ?? + readWebProviderEnvValue(currentProvider.envVars), + }); } export function listWebSearchProviders(params?: { @@ -178,10 +152,6 @@ export function resolveWebSearchDefinition( ): { provider: PluginWebSearchProviderEntry; definition: WebSearchProviderToolDefinition } | null { const search = resolveSearchConfig(options?.config); const runtimeWebSearch = options?.runtimeWebSearch ?? getActiveRuntimeWebToolsMetadata()?.search; - if (!resolveWebSearchEnabled({ search, sandboxed: options?.sandboxed })) { - return null; - } - const providers = sortWebSearchProvidersForAutoDetect( options?.preferRuntimeProviders ? resolveRuntimeWebSearchProviders({ @@ -193,37 +163,38 @@ export function resolveWebSearchDefinition( bundledAllowlistCompat: true, origin: "bundled", }), - ).filter(Boolean); - if (providers.length === 0) { - return null; - } - - const providerId = - options?.providerId ?? - runtimeWebSearch?.selectedProvider ?? - runtimeWebSearch?.providerConfigured ?? - resolveWebSearchProviderId({ config: options?.config, search, providers }); - const provider = - providers.find((entry) => entry.id === providerId) ?? - providers.find( - (entry) => - entry.id === resolveWebSearchProviderId({ config: options?.config, search, providers }), - ) ?? - providers[0]; - if (!provider) { - return null; - } - - const definition = provider.createTool({ + ); + return resolveWebProviderDefinition({ config: options?.config, - searchConfig: search as Record | undefined, + toolConfig: search as Record | undefined, runtimeMetadata: runtimeWebSearch, + sandboxed: options?.sandboxed, + providerId: options?.providerId, + providers, + resolveEnabled: ({ toolConfig, sandboxed }) => + resolveWebSearchEnabled({ + search: toolConfig as WebSearchConfig | undefined, + sandboxed, + }), + resolveAutoProviderId: ({ config, toolConfig, providers }) => + resolveWebSearchProviderId({ + config, + search: toolConfig as WebSearchConfig | undefined, + providers, + }), + resolveFallbackProviderId: ({ config, toolConfig, providers }) => + resolveWebSearchProviderId({ + config, + search: toolConfig as WebSearchConfig | undefined, + providers, + }) || providers[0]?.id, + createTool: ({ provider, config, toolConfig, runtimeMetadata }) => + provider.createTool({ + config, + searchConfig: toolConfig, + runtimeMetadata, + }), }); - if (!definition) { - return null; - } - - return { provider, definition }; } export async function runWebSearch( diff --git a/src/web/provider-runtime-shared.ts b/src/web/provider-runtime-shared.ts new file mode 100644 index 00000000000..63954e6318e --- /dev/null +++ b/src/web/provider-runtime-shared.ts @@ -0,0 +1,165 @@ +import type { OpenClawConfig } from "../config/config.js"; +import { normalizeSecretInputString, resolveSecretInputRef } from "../config/types.secrets.js"; +import { normalizeSecretInput } from "../utils/normalize-secret-input.js"; + +type RuntimeWebProviderMetadata = { + providerConfigured?: string; + selectedProvider?: string; +}; + +type ProviderWithCredential = { + envVars: string[]; + requiresCredential?: boolean; +}; + +export function resolveWebProviderConfig< + TKind extends "search" | "fetch", + TConfig extends Record, +>(cfg: OpenClawConfig | undefined, kind: TKind): TConfig | undefined { + const webConfig = cfg?.tools?.web; + if (!webConfig || typeof webConfig !== "object") { + return undefined; + } + const toolConfig = webConfig[kind]; + if (!toolConfig || typeof toolConfig !== "object") { + return undefined; + } + return toolConfig as TConfig; +} + +export function readWebProviderEnvValue( + envVars: string[], + processEnv: NodeJS.ProcessEnv = process.env, +): string | undefined { + for (const envVar of envVars) { + const value = normalizeSecretInput(processEnv[envVar]); + if (value) { + return value; + } + } + return undefined; +} + +export function providerRequiresCredential( + provider: Pick, +): boolean { + return provider.requiresCredential !== false; +} + +export function hasWebProviderEntryCredential< + TProvider extends ProviderWithCredential, + TConfig extends Record | undefined, +>(params: { + provider: TProvider; + config: OpenClawConfig | undefined; + toolConfig: TConfig; + resolveRawValue: (params: { + provider: TProvider; + config: OpenClawConfig | undefined; + toolConfig: TConfig; + }) => unknown; + resolveEnvValue: (params: { + provider: TProvider; + configuredEnvVarId?: string; + }) => string | undefined; +}): boolean { + if (!providerRequiresCredential(params.provider)) { + return true; + } + const rawValue = params.resolveRawValue({ + provider: params.provider, + config: params.config, + toolConfig: params.toolConfig, + }); + const configuredRef = resolveSecretInputRef({ + value: rawValue, + }).ref; + if (configuredRef && configuredRef.source !== "env") { + return true; + } + const fromConfig = normalizeSecretInput(normalizeSecretInputString(rawValue)); + if (fromConfig) { + return true; + } + return Boolean( + params.resolveEnvValue({ + provider: params.provider, + configuredEnvVarId: configuredRef?.source === "env" ? configuredRef.id : undefined, + }), + ); +} + +export function resolveWebProviderDefinition< + TProvider extends { id: string }, + TConfig extends Record | undefined, + TRuntimeMetadata extends RuntimeWebProviderMetadata, + TDefinition, +>(params: { + config: OpenClawConfig | undefined; + toolConfig: TConfig; + runtimeMetadata: TRuntimeMetadata | undefined; + sandboxed?: boolean; + providerId?: string; + providers: TProvider[]; + resolveEnabled: (params: { toolConfig: TConfig; sandboxed?: boolean }) => boolean; + resolveAutoProviderId: (params: { + config: OpenClawConfig | undefined; + toolConfig: TConfig; + providers: TProvider[]; + }) => string; + resolveFallbackProviderId?: (params: { + config: OpenClawConfig | undefined; + toolConfig: TConfig; + providers: TProvider[]; + providerId: string; + }) => string | undefined; + createTool: (params: { + provider: TProvider; + config: OpenClawConfig | undefined; + toolConfig: TConfig; + runtimeMetadata: TRuntimeMetadata | undefined; + }) => TDefinition | null; +}): { provider: TProvider; definition: TDefinition } | null { + if (!params.resolveEnabled({ toolConfig: params.toolConfig, sandboxed: params.sandboxed })) { + return null; + } + const providers = params.providers.filter(Boolean); + if (providers.length === 0) { + return null; + } + const autoProviderId = params.resolveAutoProviderId({ + config: params.config, + toolConfig: params.toolConfig, + providers, + }); + const providerId = + params.providerId ?? + params.runtimeMetadata?.selectedProvider ?? + params.runtimeMetadata?.providerConfigured ?? + autoProviderId; + const provider = + providers.find((entry) => entry.id === providerId) ?? + providers.find( + (entry) => + entry.id === + params.resolveFallbackProviderId?.({ + config: params.config, + toolConfig: params.toolConfig, + providers, + providerId, + }), + ); + if (!provider) { + return null; + } + const definition = params.createTool({ + provider, + config: params.config, + toolConfig: params.toolConfig, + runtimeMetadata: params.runtimeMetadata, + }); + if (!definition) { + return null; + } + return { provider, definition }; +}