From f0eb67923cd74b9278b408e868b80b0db40a23e9 Mon Sep 17 00:00:00 2001 From: Josh Avant <830519+joshavant@users.noreply.github.com> Date: Mon, 9 Mar 2026 22:57:03 -0500 Subject: [PATCH] fix(secrets): resolve web tool SecretRefs atomically at runtime --- CHANGELOG.md | 1 + docs/gateway/secrets.md | 3 +- docs/help/faq.md | 15 +- docs/perplexity.md | 3 + docs/reference/api-usage-costs.md | 8 +- .../reference/secretref-credential-surface.md | 4 +- ...tref-user-supplied-credentials-matrix.json | 7 + docs/tools/firecrawl.md | 3 +- docs/tools/web.md | 32 +- src/agents/openclaw-tools.ts | 4 + src/agents/openclaw-tools.web-runtime.test.ts | 135 ++++ .../tools/web-fetch.cf-markdown.test.ts | 41 + src/agents/tools/web-fetch.ts | 27 +- src/agents/tools/web-search.ts | 62 +- .../tools/web-tools.enabled-defaults.test.ts | 140 +++- src/cli/command-secret-gateway.test.ts | 113 +++ src/cli/command-secret-gateway.ts | 60 +- src/cli/command-secret-targets.test.ts | 1 + src/cli/command-secret-targets.ts | 1 + src/config/types.tools.ts | 2 +- src/gateway/server.reload.test.ts | 93 +++ src/secrets/runtime-config-collectors-core.ts | 62 -- src/secrets/runtime-shared.ts | 7 +- src/secrets/runtime-web-tools.test.ts | 451 +++++++++++ src/secrets/runtime-web-tools.ts | 705 ++++++++++++++++++ src/secrets/runtime.test.ts | 164 +++- src/secrets/runtime.ts | 16 +- src/secrets/target-registry-data.ts | 11 + 28 files changed, 2059 insertions(+), 112 deletions(-) create mode 100644 src/agents/openclaw-tools.web-runtime.test.ts create mode 100644 src/secrets/runtime-web-tools.test.ts create mode 100644 src/secrets/runtime-web-tools.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 95f3ab600cb..c19a5c2eda7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ Docs: https://docs.openclaw.ai ### Fixes +- Resolve web tool SecretRefs atomically at runtime. (#41599) Thanks @joshavant. - Feishu/local image auto-convert: pass `mediaLocalRoots` through the `sendText` local-image shim so allowed local image paths upload as Feishu images again instead of falling back to raw path text. (#40623) Thanks @ayanesakura. - macOS/LaunchAgent install: tighten LaunchAgent directory and plist permissions during install so launchd bootstrap does not fail when the target home path or generated plist inherited group/world-writable modes. - Gateway/Control UI: keep dashboard auth tokens in session-scoped browser storage so same-tab refreshes preserve remote token auth without restoring long-lived localStorage token persistence, while scoping tokens to the selected gateway URL and fragment-only bootstrap flow. (#40892) thanks @velvet-shark. diff --git a/docs/gateway/secrets.md b/docs/gateway/secrets.md index 3ef08267618..e9d75343147 100644 --- a/docs/gateway/secrets.md +++ b/docs/gateway/secrets.md @@ -38,7 +38,8 @@ Examples of inactive surfaces: - Top-level channel credentials that no enabled account inherits. - Disabled tool/feature surfaces. - Web search provider-specific keys that are not selected by `tools.web.search.provider`. - In auto mode (provider unset), provider-specific keys are also active for provider auto-detection. + In auto mode (provider unset), keys are consulted by precedence for provider auto-detection until one resolves. + After selection, non-selected provider keys are treated as inactive until selected. - `gateway.remote.token` / `gateway.remote.password` SecretRefs are active (when `gateway.remote.enabled` is not `false`) if one of these is true: - `gateway.mode=remote` - `gateway.remote.url` is configured diff --git a/docs/help/faq.md b/docs/help/faq.md index 7dad0548fd4..a43e91f4396 100644 --- a/docs/help/faq.md +++ b/docs/help/faq.md @@ -1489,10 +1489,16 @@ Set `cli.banner.taglineMode` in config: ### How do I enable web search and web fetch -`web_fetch` works without an API key. `web_search` requires a Brave Search API -key. **Recommended:** run `openclaw configure --section web` to store it in -`tools.web.search.apiKey`. Environment alternative: set `BRAVE_API_KEY` for the -Gateway process. +`web_fetch` works without an API key. `web_search` requires a key for your +selected provider (Brave, Gemini, Grok, Kimi, or Perplexity). +**Recommended:** run `openclaw configure --section web` and choose a provider. +Environment alternatives: + +- Brave: `BRAVE_API_KEY` +- Gemini: `GEMINI_API_KEY` +- Grok: `XAI_API_KEY` +- Kimi: `KIMI_API_KEY` or `MOONSHOT_API_KEY` +- Perplexity: `PERPLEXITY_API_KEY` or `OPENROUTER_API_KEY` ```json5 { @@ -1500,6 +1506,7 @@ Gateway process. web: { search: { enabled: true, + provider: "brave", apiKey: "BRAVE_API_KEY_HERE", maxResults: 5, }, diff --git a/docs/perplexity.md b/docs/perplexity.md index bb1acef49c8..f7eccc9453e 100644 --- a/docs/perplexity.md +++ b/docs/perplexity.md @@ -71,11 +71,14 @@ Optional legacy controls: **Via config:** run `openclaw configure --section web`. It stores the key in `~/.openclaw/openclaw.json` under `tools.web.search.perplexity.apiKey`. +That field also accepts SecretRef objects. **Via environment:** set `PERPLEXITY_API_KEY` or `OPENROUTER_API_KEY` in the Gateway process environment. For a gateway install, put it in `~/.openclaw/.env` (or your service environment). See [Env vars](/help/faq#how-does-openclaw-load-environment-variables). +If `provider: "perplexity"` is configured and the Perplexity key SecretRef is unresolved with no env fallback, startup/reload fails fast. + ## Tool parameters These parameters apply to the native Perplexity Search API path. diff --git a/docs/reference/api-usage-costs.md b/docs/reference/api-usage-costs.md index dba017aacc1..baf4302ac0d 100644 --- a/docs/reference/api-usage-costs.md +++ b/docs/reference/api-usage-costs.md @@ -80,10 +80,10 @@ See [Memory](/concepts/memory). `web_search` uses API keys and may incur usage charges depending on your provider: - **Brave Search API**: `BRAVE_API_KEY` or `tools.web.search.apiKey` -- **Gemini (Google Search)**: `GEMINI_API_KEY` -- **Grok (xAI)**: `XAI_API_KEY` -- **Kimi (Moonshot)**: `KIMI_API_KEY` or `MOONSHOT_API_KEY` -- **Perplexity Search API**: `PERPLEXITY_API_KEY` +- **Gemini (Google Search)**: `GEMINI_API_KEY` or `tools.web.search.gemini.apiKey` +- **Grok (xAI)**: `XAI_API_KEY` or `tools.web.search.grok.apiKey` +- **Kimi (Moonshot)**: `KIMI_API_KEY`, `MOONSHOT_API_KEY`, or `tools.web.search.kimi.apiKey` +- **Perplexity Search API**: `PERPLEXITY_API_KEY`, `OPENROUTER_API_KEY`, or `tools.web.search.perplexity.apiKey` **Brave Search free credit:** Each Brave plan includes $5/month in renewing free credit. The Search plan costs $5 per 1,000 requests, so the credit covers diff --git a/docs/reference/secretref-credential-surface.md b/docs/reference/secretref-credential-surface.md index dd1b5f1fd2f..2a5fc5a66ac 100644 --- a/docs/reference/secretref-credential-surface.md +++ b/docs/reference/secretref-credential-surface.md @@ -31,6 +31,7 @@ Scope intent: - `talk.providers.*.apiKey` - `messages.tts.elevenlabs.apiKey` - `messages.tts.openai.apiKey` +- `tools.web.fetch.firecrawl.apiKey` - `tools.web.search.apiKey` - `tools.web.search.gemini.apiKey` - `tools.web.search.grok.apiKey` @@ -102,7 +103,8 @@ Notes: - For SecretRef-managed model providers, generated `agents/*/agent/models.json` entries persist non-secret markers (not resolved secret values) for `apiKey`/header surfaces. - For web search: - In explicit provider mode (`tools.web.search.provider` set), only the selected provider key is active. - - In auto mode (`tools.web.search.provider` unset), `tools.web.search.apiKey` and provider-specific keys are active. + - In auto mode (`tools.web.search.provider` unset), only the first provider key that resolves by precedence is active. + - In auto mode, non-selected provider refs are treated as inactive until selected. ## Unsupported credentials diff --git a/docs/reference/secretref-user-supplied-credentials-matrix.json b/docs/reference/secretref-user-supplied-credentials-matrix.json index 773ef8ab162..6d4b05d2822 100644 --- a/docs/reference/secretref-user-supplied-credentials-matrix.json +++ b/docs/reference/secretref-user-supplied-credentials-matrix.json @@ -454,6 +454,13 @@ "secretShape": "secret_input", "optIn": true }, + { + "id": "tools.web.fetch.firecrawl.apiKey", + "configFile": "openclaw.json", + "path": "tools.web.fetch.firecrawl.apiKey", + "secretShape": "secret_input", + "optIn": true + }, { "id": "tools.web.search.apiKey", "configFile": "openclaw.json", diff --git a/docs/tools/firecrawl.md b/docs/tools/firecrawl.md index e859eb2dcb1..2cd90a06bf5 100644 --- a/docs/tools/firecrawl.md +++ b/docs/tools/firecrawl.md @@ -40,7 +40,8 @@ with JS-heavy sites or pages that block plain HTTP fetches. Notes: -- `firecrawl.enabled` defaults to true when an API key is present. +- `firecrawl.enabled` defaults to `true` unless explicitly set to `false`. +- Firecrawl fallback attempts run only when an API key is available (`tools.web.fetch.firecrawl.apiKey` or `FIRECRAWL_API_KEY`). - `maxAgeMs` controls how old cached results can be (ms). Default is 2 days. ## Stealth / bot circumvention diff --git a/docs/tools/web.md b/docs/tools/web.md index 1eeb4eba7db..e77d046ce5b 100644 --- a/docs/tools/web.md +++ b/docs/tools/web.md @@ -2,7 +2,7 @@ summary: "Web search + fetch tools (Brave, Gemini, Grok, Kimi, and Perplexity providers)" read_when: - You want to enable web_search or web_fetch - - You need Brave or Perplexity Search API key setup + - You need provider API key setup - You want to use Gemini with Google Search grounding title: "Web Tools" --- @@ -49,6 +49,12 @@ The table above is alphabetical. If no `provider` is explicitly set, runtime aut If no keys are found, it falls back to Brave (you'll get a missing-key error prompting you to configure one). +Runtime SecretRef behavior: + +- Web tool SecretRefs are resolved atomically at gateway startup/reload. +- In auto-detect mode, OpenClaw resolves only the selected provider key. Non-selected provider SecretRefs stay inactive until selected. +- If the selected provider SecretRef is unresolved and no provider env fallback exists, startup/reload fails fast. + ## Setting up web search Use `openclaw configure --section web` to set up your API key and choose a provider. @@ -77,9 +83,25 @@ See [Perplexity Search API Docs](https://docs.perplexity.ai/guides/search-quicks ### Where to store the key -**Via config:** run `openclaw configure --section web`. It stores the key under `tools.web.search.apiKey` or `tools.web.search.perplexity.apiKey`, depending on provider. +**Via config:** run `openclaw configure --section web`. It stores the key under the provider-specific config path: -**Via environment:** set `PERPLEXITY_API_KEY`, `OPENROUTER_API_KEY`, or `BRAVE_API_KEY` in the Gateway process environment. For a gateway install, put it in `~/.openclaw/.env` (or your service environment). See [Env vars](/help/faq#how-does-openclaw-load-environment-variables). +- Brave: `tools.web.search.apiKey` +- Gemini: `tools.web.search.gemini.apiKey` +- Grok: `tools.web.search.grok.apiKey` +- Kimi: `tools.web.search.kimi.apiKey` +- Perplexity: `tools.web.search.perplexity.apiKey` + +All of these fields also support SecretRef objects. + +**Via environment:** set provider env vars in the Gateway process environment: + +- Brave: `BRAVE_API_KEY` +- Gemini: `GEMINI_API_KEY` +- Grok: `XAI_API_KEY` +- Kimi: `KIMI_API_KEY` or `MOONSHOT_API_KEY` +- Perplexity: `PERPLEXITY_API_KEY` or `OPENROUTER_API_KEY` + +For a gateway install, put these in `~/.openclaw/.env` (or your service environment). See [Env vars](/help/faq#how-does-openclaw-load-environment-variables). ### Config examples @@ -216,6 +238,7 @@ Search the web using your configured provider. - **Grok**: `XAI_API_KEY` or `tools.web.search.grok.apiKey` - **Kimi**: `KIMI_API_KEY`, `MOONSHOT_API_KEY`, or `tools.web.search.kimi.apiKey` - **Perplexity**: `PERPLEXITY_API_KEY`, `OPENROUTER_API_KEY`, or `tools.web.search.perplexity.apiKey` +- All provider key fields above support SecretRef objects. ### Config @@ -310,6 +333,7 @@ Fetch a URL and extract readable content. - `tools.web.fetch.enabled` must not be `false` (default: enabled) - Optional Firecrawl fallback: set `tools.web.fetch.firecrawl.apiKey` or `FIRECRAWL_API_KEY`. +- `tools.web.fetch.firecrawl.apiKey` supports SecretRef objects. ### web_fetch config @@ -351,6 +375,8 @@ Notes: - `web_fetch` uses Readability (main-content extraction) first, then Firecrawl (if configured). If both fail, the tool returns an error. - Firecrawl requests use bot-circumvention mode and cache results by default. +- Firecrawl SecretRefs are resolved only when Firecrawl is active (`tools.web.fetch.enabled !== false` and `tools.web.fetch.firecrawl.enabled !== false`). +- If Firecrawl is active and its SecretRef is unresolved with no `FIRECRAWL_API_KEY` fallback, startup/reload fails fast. - `web_fetch` sends a Chrome-like User-Agent and `Accept-Language` by default; override `userAgent` if needed. - `web_fetch` blocks private/internal hostnames and re-checks redirects (limit with `maxRedirects`). - `maxChars` is clamped to `tools.web.fetch.maxCharsCap`. diff --git a/src/agents/openclaw-tools.ts b/src/agents/openclaw-tools.ts index 17f8e6dadb4..56d0801d13c 100644 --- a/src/agents/openclaw-tools.ts +++ b/src/agents/openclaw-tools.ts @@ -1,5 +1,6 @@ import type { OpenClawConfig } from "../config/config.js"; import { resolvePluginTools } from "../plugins/tools.js"; +import { getActiveRuntimeWebToolsMetadata } from "../secrets/runtime.js"; import type { GatewayMessageChannel } from "../utils/message-channel.js"; import { resolveSessionAgentId } from "./agent-scope.js"; import type { SandboxFsBridge } from "./sandbox/fs-bridge.js"; @@ -72,6 +73,7 @@ export function createOpenClawTools( } & SpawnedToolContext, ): AnyAgentTool[] { const workspaceDir = resolveWorkspaceRoot(options?.workspaceDir); + const runtimeWebTools = getActiveRuntimeWebToolsMetadata(); const imageTool = options?.agentDir?.trim() ? createImageTool({ config: options?.config, @@ -100,10 +102,12 @@ export function createOpenClawTools( const webSearchTool = createWebSearchTool({ config: options?.config, sandboxed: options?.sandboxed, + runtimeWebSearch: runtimeWebTools?.search, }); const webFetchTool = createWebFetchTool({ config: options?.config, sandboxed: options?.sandboxed, + runtimeFirecrawl: runtimeWebTools?.fetch.firecrawl, }); const messageTool = options?.disableMessageTool ? null diff --git a/src/agents/openclaw-tools.web-runtime.test.ts b/src/agents/openclaw-tools.web-runtime.test.ts new file mode 100644 index 00000000000..94478930cf1 --- /dev/null +++ b/src/agents/openclaw-tools.web-runtime.test.ts @@ -0,0 +1,135 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import type { OpenClawConfig } from "../config/config.js"; +import { + activateSecretsRuntimeSnapshot, + clearSecretsRuntimeSnapshot, + prepareSecretsRuntimeSnapshot, +} from "../secrets/runtime.js"; +import { withFetchPreconnect } from "../test-utils/fetch-mock.js"; +import { createOpenClawTools } from "./openclaw-tools.js"; + +vi.mock("../plugins/tools.js", () => ({ + resolvePluginTools: () => [], +})); + +function asConfig(value: unknown): OpenClawConfig { + return value as OpenClawConfig; +} + +function findTool(name: string, config: OpenClawConfig) { + const allTools = createOpenClawTools({ config, sandboxed: true }); + const tool = allTools.find((candidate) => candidate.name === name); + expect(tool).toBeDefined(); + if (!tool) { + throw new Error(`missing ${name} tool`); + } + return tool; +} + +function makeHeaders(map: Record): { get: (key: string) => string | null } { + return { + get: (key) => map[key.toLowerCase()] ?? null, + }; +} + +async function prepareAndActivate(params: { config: OpenClawConfig; env?: NodeJS.ProcessEnv }) { + const snapshot = await prepareSecretsRuntimeSnapshot({ + config: params.config, + env: params.env, + agentDirs: ["/tmp/openclaw-agent-main"], + loadAuthStore: () => ({ version: 1, profiles: {} }), + }); + activateSecretsRuntimeSnapshot(snapshot); + return snapshot; +} + +describe("openclaw tools runtime web metadata wiring", () => { + const priorFetch = global.fetch; + + afterEach(() => { + global.fetch = priorFetch; + clearSecretsRuntimeSnapshot(); + }); + + it("uses runtime-selected provider when higher-precedence provider ref is unresolved", async () => { + const snapshot = await prepareAndActivate({ + config: asConfig({ + tools: { + web: { + search: { + apiKey: { source: "env", provider: "default", id: "MISSING_BRAVE_KEY_REF" }, + gemini: { + apiKey: { source: "env", provider: "default", id: "GEMINI_WEB_KEY_REF" }, + }, + }, + }, + }, + }), + env: { + GEMINI_WEB_KEY_REF: "gemini-runtime-key", + }, + }); + + expect(snapshot.webTools.search.selectedProvider).toBe("gemini"); + + const mockFetch = vi.fn((_input?: unknown, _init?: unknown) => + Promise.resolve({ + ok: true, + json: () => + Promise.resolve({ + candidates: [ + { + content: { parts: [{ text: "runtime gemini ok" }] }, + groundingMetadata: { groundingChunks: [] }, + }, + ], + }), + } as Response), + ); + global.fetch = withFetchPreconnect(mockFetch); + + const webSearch = findTool("web_search", snapshot.config); + const result = await webSearch.execute("call-runtime-search", { query: "runtime search" }); + + expect(mockFetch).toHaveBeenCalled(); + expect(String(mockFetch.mock.calls[0]?.[0])).toContain("generativelanguage.googleapis.com"); + expect((result.details as { provider?: string }).provider).toBe("gemini"); + }); + + it("skips Firecrawl key resolution when runtime marks Firecrawl inactive", async () => { + const snapshot = await prepareAndActivate({ + config: asConfig({ + tools: { + web: { + fetch: { + firecrawl: { + enabled: false, + apiKey: { source: "env", provider: "default", id: "MISSING_FIRECRAWL_KEY_REF" }, + }, + }, + }, + }, + }), + }); + + const mockFetch = vi.fn((_input?: unknown, _init?: unknown) => + Promise.resolve({ + ok: true, + status: 200, + headers: makeHeaders({ "content-type": "text/html; charset=utf-8" }), + text: () => + Promise.resolve( + "

Runtime Off

Use direct fetch.

", + ), + } as Response), + ); + global.fetch = withFetchPreconnect(mockFetch); + + const webFetch = findTool("web_fetch", snapshot.config); + await webFetch.execute("call-runtime-fetch", { url: "https://example.com/runtime-off" }); + + expect(mockFetch).toHaveBeenCalled(); + expect(mockFetch.mock.calls[0]?.[0]).toBe("https://example.com/runtime-off"); + expect(String(mockFetch.mock.calls[0]?.[0])).not.toContain("api.firecrawl.dev"); + }); +}); diff --git a/src/agents/tools/web-fetch.cf-markdown.test.ts b/src/agents/tools/web-fetch.cf-markdown.test.ts index 6e7768fc43a..e235177a309 100644 --- a/src/agents/tools/web-fetch.cf-markdown.test.ts +++ b/src/agents/tools/web-fetch.cf-markdown.test.ts @@ -84,6 +84,47 @@ describe("web_fetch Cloudflare Markdown for Agents", () => { expect(details?.contentType).toBe("text/html"); }); + it("bypasses Firecrawl when runtime metadata marks Firecrawl inactive", async () => { + const fetchSpy = vi + .fn() + .mockResolvedValue( + htmlResponse( + "

Runtime Off

Use direct fetch.

", + ), + ); + global.fetch = withFetchPreconnect(fetchSpy); + + const tool = createWebFetchTool({ + config: { + tools: { + web: { + fetch: { + firecrawl: { + enabled: true, + apiKey: { + source: "env", + provider: "default", + id: "MISSING_FIRECRAWL_KEY_REF", + }, + }, + }, + }, + }, + }, + sandboxed: false, + runtimeFirecrawl: { + active: false, + apiKeySource: "secretRef", + diagnostics: [], + }, + }); + + await tool?.execute?.("call", { url: "https://example.com/runtime-firecrawl-off" }); + + expect(fetchSpy).toHaveBeenCalled(); + expect(fetchSpy.mock.calls[0]?.[0]).toBe("https://example.com/runtime-firecrawl-off"); + }); + it("logs x-markdown-tokens when header is present", async () => { const logSpy = vi.spyOn(logger, "logDebug").mockImplementation(() => {}); const fetchSpy = vi diff --git a/src/agents/tools/web-fetch.ts b/src/agents/tools/web-fetch.ts index 4ac7a1d7bfd..f4cc88e2d83 100644 --- a/src/agents/tools/web-fetch.ts +++ b/src/agents/tools/web-fetch.ts @@ -1,7 +1,9 @@ import { Type } from "@sinclair/typebox"; import type { OpenClawConfig } from "../../config/config.js"; +import { normalizeResolvedSecretInputString } from "../../config/types.secrets.js"; import { SsrFBlockedError } from "../../infra/net/ssrf.js"; import { logDebug } from "../../logger.js"; +import type { RuntimeWebFetchFirecrawlMetadata } from "../../secrets/runtime-web-tools.js"; import { wrapExternalContent, wrapWebContent } from "../../security/external-content.js"; import { normalizeSecretInput } from "../../utils/normalize-secret-input.js"; import { stringEnum } from "../schema/typebox.js"; @@ -71,7 +73,7 @@ type WebFetchConfig = NonNullable["web"] extends infer type FirecrawlFetchConfig = | { enabled?: boolean; - apiKey?: string; + apiKey?: unknown; baseUrl?: string; onlyMainContent?: boolean; maxAgeMs?: number; @@ -136,10 +138,14 @@ function resolveFirecrawlConfig(fetch?: WebFetchConfig): FirecrawlFetchConfig { } function resolveFirecrawlApiKey(firecrawl?: FirecrawlFetchConfig): string | undefined { - const fromConfig = - firecrawl && "apiKey" in firecrawl && typeof firecrawl.apiKey === "string" - ? normalizeSecretInput(firecrawl.apiKey) - : ""; + const fromConfigRaw = + firecrawl && "apiKey" in firecrawl + ? normalizeResolvedSecretInputString({ + value: firecrawl.apiKey, + path: "tools.web.fetch.firecrawl.apiKey", + }) + : undefined; + const fromConfig = normalizeSecretInput(fromConfigRaw); const fromEnv = normalizeSecretInput(process.env.FIRECRAWL_API_KEY); return fromConfig || fromEnv || undefined; } @@ -712,6 +718,7 @@ function resolveFirecrawlEndpoint(baseUrl: string): string { export function createWebFetchTool(options?: { config?: OpenClawConfig; sandboxed?: boolean; + runtimeFirecrawl?: RuntimeWebFetchFirecrawlMetadata; }): AnyAgentTool | null { const fetch = resolveFetchConfig(options?.config); if (!resolveFetchEnabled({ fetch, sandboxed: options?.sandboxed })) { @@ -719,8 +726,14 @@ export function createWebFetchTool(options?: { } const readabilityEnabled = resolveFetchReadabilityEnabled(fetch); const firecrawl = resolveFirecrawlConfig(fetch); - const firecrawlApiKey = resolveFirecrawlApiKey(firecrawl); - const firecrawlEnabled = resolveFirecrawlEnabled({ firecrawl, apiKey: firecrawlApiKey }); + const runtimeFirecrawlActive = options?.runtimeFirecrawl?.active; + const shouldResolveFirecrawlApiKey = + runtimeFirecrawlActive === undefined ? firecrawl?.enabled !== false : runtimeFirecrawlActive; + const firecrawlApiKey = shouldResolveFirecrawlApiKey + ? resolveFirecrawlApiKey(firecrawl) + : undefined; + const firecrawlEnabled = + runtimeFirecrawlActive ?? resolveFirecrawlEnabled({ firecrawl, apiKey: firecrawlApiKey }); const firecrawlBaseUrl = resolveFirecrawlBaseUrl(firecrawl); const firecrawlOnlyMainContent = resolveFirecrawlOnlyMainContent(firecrawl); const firecrawlMaxAgeMs = resolveFirecrawlMaxAgeMsOrDefault(firecrawl); diff --git a/src/agents/tools/web-search.ts b/src/agents/tools/web-search.ts index d4f88caea61..4fbbfa95e43 100644 --- a/src/agents/tools/web-search.ts +++ b/src/agents/tools/web-search.ts @@ -3,6 +3,7 @@ import { formatCliCommand } from "../../cli/command-format.js"; import type { OpenClawConfig } from "../../config/config.js"; import { normalizeResolvedSecretInputString } from "../../config/types.secrets.js"; import { logVerbose } from "../../globals.js"; +import type { RuntimeWebSearchMetadata } from "../../secrets/runtime-web-tools.js"; import { wrapWebContent } from "../../security/external-content.js"; import { normalizeSecretInput } from "../../utils/normalize-secret-input.js"; import type { AnyAgentTool } from "./common.js"; @@ -193,6 +194,33 @@ function createWebSearchSchema(params: { ), } as const; + const perplexityStructuredFilterSchema = { + country: Type.Optional( + Type.String({ + description: + "Native Perplexity Search API only. 2-letter country code for region-specific results (e.g., 'DE', 'US', 'ALL'). Default: 'US'.", + }), + ), + language: Type.Optional( + Type.String({ + description: + "Native Perplexity Search API only. ISO 639-1 language code for results (e.g., 'en', 'de', 'fr').", + }), + ), + date_after: Type.Optional( + Type.String({ + description: + "Native Perplexity Search API only. Only results published after this date (YYYY-MM-DD).", + }), + ), + date_before: Type.Optional( + Type.String({ + description: + "Native Perplexity Search API only. Only results published before this date (YYYY-MM-DD).", + }), + ), + } as const; + if (params.provider === "brave") { return Type.Object({ ...querySchema, @@ -221,7 +249,8 @@ function createWebSearchSchema(params: { } return Type.Object({ ...querySchema, - ...filterSchema, + freshness: filterSchema.freshness, + ...perplexityStructuredFilterSchema, domain_filter: Type.Optional( Type.Array(Type.String(), { description: @@ -742,6 +771,16 @@ function resolvePerplexityTransport(perplexity?: PerplexityConfig): { }; } +function resolvePerplexitySchemaTransportHint( + perplexity?: PerplexityConfig, +): PerplexityTransport | undefined { + const hasLegacyOverride = Boolean( + (perplexity?.baseUrl && perplexity.baseUrl.trim()) || + (perplexity?.model && perplexity.model.trim()), + ); + return hasLegacyOverride ? "chat_completions" : undefined; +} + function resolveGrokConfig(search?: WebSearchConfig): GrokConfig { if (!search || typeof search !== "object") { return {}; @@ -1809,15 +1848,21 @@ async function runWebSearch(params: { export function createWebSearchTool(options?: { config?: OpenClawConfig; sandboxed?: boolean; + runtimeWebSearch?: RuntimeWebSearchMetadata; }): AnyAgentTool | null { const search = resolveSearchConfig(options?.config); if (!resolveSearchEnabled({ search, sandboxed: options?.sandboxed })) { return null; } - const provider = resolveSearchProvider(search); + const provider = + options?.runtimeWebSearch?.selectedProvider ?? + options?.runtimeWebSearch?.providerConfigured ?? + resolveSearchProvider(search); const perplexityConfig = resolvePerplexityConfig(search); - const perplexityTransport = resolvePerplexityTransport(perplexityConfig); + const perplexitySchemaTransportHint = + options?.runtimeWebSearch?.perplexityTransport ?? + resolvePerplexitySchemaTransportHint(perplexityConfig); const grokConfig = resolveGrokConfig(search); const geminiConfig = resolveGeminiConfig(search); const kimiConfig = resolveKimiConfig(search); @@ -1826,9 +1871,9 @@ export function createWebSearchTool(options?: { const description = provider === "perplexity" - ? perplexityTransport.transport === "chat_completions" + ? perplexitySchemaTransportHint === "chat_completions" ? "Search the web using Perplexity Sonar via Perplexity/OpenRouter chat completions. Returns AI-synthesized answers with citations from web-grounded search." - : "Search the web using the Perplexity Search API. Returns structured results (title, URL, snippet) for fast research. Supports domain, region, language, and freshness filtering." + : "Search the web using Perplexity. Runtime routing decides between native Search API and Sonar chat-completions compatibility. Structured filters are available on the native Search API path." : provider === "grok" ? "Search the web using xAI Grok. Returns AI-synthesized answers with citations from real-time web search." : provider === "kimi" @@ -1845,10 +1890,13 @@ export function createWebSearchTool(options?: { description, parameters: createWebSearchSchema({ provider, - perplexityTransport: provider === "perplexity" ? perplexityTransport.transport : undefined, + perplexityTransport: provider === "perplexity" ? perplexitySchemaTransportHint : undefined, }), execute: async (_toolCallId, args) => { - const perplexityRuntime = provider === "perplexity" ? perplexityTransport : undefined; + // Resolve Perplexity auth/transport lazily at execution time so unrelated providers + // do not touch Perplexity-only credential surfaces during tool construction. + const perplexityRuntime = + provider === "perplexity" ? resolvePerplexityTransport(perplexityConfig) : undefined; const apiKey = provider === "perplexity" ? perplexityRuntime?.apiKey diff --git a/src/agents/tools/web-tools.enabled-defaults.test.ts b/src/agents/tools/web-tools.enabled-defaults.test.ts index 80dcd6a025d..4951f1c6b5a 100644 --- a/src/agents/tools/web-tools.enabled-defaults.test.ts +++ b/src/agents/tools/web-tools.enabled-defaults.test.ts @@ -166,6 +166,39 @@ describe("web tools defaults", () => { const tool = createWebSearchTool({ config: {}, sandboxed: false }); expect(tool?.name).toBe("web_search"); }); + + it("prefers runtime-selected web_search provider over local provider config", async () => { + const mockFetch = installMockFetch(createProviderSuccessPayload("gemini")); + const tool = createWebSearchTool({ + config: { + tools: { + web: { + search: { + provider: "brave", + apiKey: "brave-config-test", // pragma: allowlist secret + gemini: { + apiKey: "gemini-config-test", // pragma: allowlist secret + }, + }, + }, + }, + }, + sandboxed: true, + runtimeWebSearch: { + providerConfigured: "brave", + providerSource: "auto-detect", + selectedProvider: "gemini", + selectedProviderKeySource: "secretRef", + diagnostics: [], + }, + }); + + const result = await tool?.execute?.("call-runtime-provider", { query: "runtime override" }); + + expect(mockFetch).toHaveBeenCalled(); + expect(String(mockFetch.mock.calls[0]?.[0])).toContain("generativelanguage.googleapis.com"); + expect((result?.details as { provider?: string } | undefined)?.provider).toBe("gemini"); + }); }); describe("web_search country and language parameters", () => { @@ -489,20 +522,56 @@ describe("web_search perplexity OpenRouter compatibility", () => { expect(result?.details).toMatchObject({ error: "unsupported_domain_filter" }); }); - it("hides Search API-only schema params on the compatibility path", () => { + it("keeps Search API schema params visible before runtime auth routing", () => { vi.stubEnv("OPENROUTER_API_KEY", "sk-or-v1-test"); // pragma: allowlist secret const tool = createPerplexitySearchTool(); const properties = (tool?.parameters as { properties?: Record } | undefined) ?.properties; expect(properties?.freshness).toBeDefined(); - expect(properties?.country).toBeUndefined(); - expect(properties?.language).toBeUndefined(); - expect(properties?.date_after).toBeUndefined(); - expect(properties?.date_before).toBeUndefined(); - expect(properties?.domain_filter).toBeUndefined(); - expect(properties?.max_tokens).toBeUndefined(); - expect(properties?.max_tokens_per_page).toBeUndefined(); + expect(properties?.country).toBeDefined(); + expect(properties?.language).toBeDefined(); + expect(properties?.date_after).toBeDefined(); + expect(properties?.date_before).toBeDefined(); + expect(properties?.domain_filter).toBeDefined(); + expect(properties?.max_tokens).toBeDefined(); + expect(properties?.max_tokens_per_page).toBeDefined(); + expect( + ( + properties?.country as + | { + description?: string; + } + | undefined + )?.description, + ).toContain("Native Perplexity Search API only."); + expect( + ( + properties?.language as + | { + description?: string; + } + | undefined + )?.description, + ).toContain("Native Perplexity Search API only."); + expect( + ( + properties?.date_after as + | { + description?: string; + } + | undefined + )?.description, + ).toContain("Native Perplexity Search API only."); + expect( + ( + properties?.date_before as + | { + description?: string; + } + | undefined + )?.description, + ).toContain("Native Perplexity Search API only."); }); it("keeps structured schema params on the native Search API path", () => { @@ -522,6 +591,61 @@ describe("web_search perplexity OpenRouter compatibility", () => { }); }); +describe("web_search Perplexity lazy resolution", () => { + const priorFetch = global.fetch; + + afterEach(() => { + vi.unstubAllEnvs(); + global.fetch = priorFetch; + }); + + it("does not read Perplexity credentials while creating non-Perplexity tools", () => { + const perplexityConfig: Record = {}; + Object.defineProperty(perplexityConfig, "apiKey", { + enumerable: true, + get() { + throw new Error("perplexity-apiKey-getter-called"); + }, + }); + + const tool = createWebSearchTool({ + config: { + tools: { + web: { + search: { + provider: "gemini", + gemini: { apiKey: "gemini-config-test" }, + perplexity: perplexityConfig as { apiKey?: string; baseUrl?: string; model?: string }, + }, + }, + }, + }, + sandboxed: true, + }); + + expect(tool?.name).toBe("web_search"); + }); + + it("defers Perplexity credential reads until execute", async () => { + const perplexityConfig: Record = {}; + Object.defineProperty(perplexityConfig, "apiKey", { + enumerable: true, + get() { + throw new Error("perplexity-apiKey-getter-called"); + }, + }); + + const tool = createPerplexitySearchTool( + perplexityConfig as { apiKey?: string; baseUrl?: string; model?: string }, + ); + + expect(tool?.name).toBe("web_search"); + await expect(tool?.execute?.("call-1", { query: "test" })).rejects.toThrow( + /perplexity-apiKey-getter-called/, + ); + }); +}); + describe("web_search kimi provider", () => { const priorFetch = global.fetch; diff --git a/src/cli/command-secret-gateway.test.ts b/src/cli/command-secret-gateway.test.ts index 7929cdbdafc..6d0f89f6349 100644 --- a/src/cli/command-secret-gateway.test.ts +++ b/src/cli/command-secret-gateway.test.ts @@ -206,6 +206,119 @@ describe("resolveCommandSecretRefsViaGateway", () => { } }); + it("falls back to local resolution for web search SecretRefs when gateway is unavailable", async () => { + const envKey = "WEB_SEARCH_GEMINI_API_KEY_LOCAL_FALLBACK"; + const priorValue = process.env[envKey]; + process.env[envKey] = "gemini-local-fallback-key"; + callGateway.mockRejectedValueOnce(new Error("gateway closed")); + + try { + const result = await resolveCommandSecretRefsViaGateway({ + config: { + tools: { + web: { + search: { + provider: "gemini", + gemini: { + apiKey: { source: "env", provider: "default", id: envKey }, + }, + }, + }, + }, + } as OpenClawConfig, + commandName: "agent", + targetIds: new Set(["tools.web.search.gemini.apiKey"]), + }); + + expect(result.resolvedConfig.tools?.web?.search?.gemini?.apiKey).toBe( + "gemini-local-fallback-key", + ); + expect(result.targetStatesByPath["tools.web.search.gemini.apiKey"]).toBe("resolved_local"); + expect( + result.diagnostics.some((entry) => entry.includes("gateway secrets.resolve unavailable")), + ).toBe(true); + expect( + result.diagnostics.some((entry) => entry.includes("resolved command secrets locally")), + ).toBe(true); + } finally { + if (priorValue === undefined) { + delete process.env[envKey]; + } else { + process.env[envKey] = priorValue; + } + } + }); + + it("falls back to local resolution for Firecrawl SecretRefs when gateway is unavailable", async () => { + const envKey = "WEB_FETCH_FIRECRAWL_API_KEY_LOCAL_FALLBACK"; + const priorValue = process.env[envKey]; + process.env[envKey] = "firecrawl-local-fallback-key"; + callGateway.mockRejectedValueOnce(new Error("gateway closed")); + + try { + const result = await resolveCommandSecretRefsViaGateway({ + config: { + tools: { + web: { + fetch: { + firecrawl: { + apiKey: { source: "env", provider: "default", id: envKey }, + }, + }, + }, + }, + } as OpenClawConfig, + commandName: "agent", + targetIds: new Set(["tools.web.fetch.firecrawl.apiKey"]), + }); + + expect(result.resolvedConfig.tools?.web?.fetch?.firecrawl?.apiKey).toBe( + "firecrawl-local-fallback-key", + ); + expect(result.targetStatesByPath["tools.web.fetch.firecrawl.apiKey"]).toBe("resolved_local"); + expect( + result.diagnostics.some((entry) => entry.includes("gateway secrets.resolve unavailable")), + ).toBe(true); + expect( + result.diagnostics.some((entry) => entry.includes("resolved command secrets locally")), + ).toBe(true); + } finally { + if (priorValue === undefined) { + delete process.env[envKey]; + } else { + process.env[envKey] = priorValue; + } + } + }); + + it("marks web SecretRefs inactive when the web surface is disabled during local fallback", async () => { + callGateway.mockRejectedValueOnce(new Error("gateway closed")); + const result = await resolveCommandSecretRefsViaGateway({ + config: { + tools: { + web: { + search: { + enabled: false, + gemini: { + apiKey: { source: "env", provider: "default", id: "WEB_SEARCH_DISABLED_KEY" }, + }, + }, + }, + }, + } as OpenClawConfig, + commandName: "agent", + targetIds: new Set(["tools.web.search.gemini.apiKey"]), + }); + + expect(result.hadUnresolvedTargets).toBe(false); + expect(result.targetStatesByPath["tools.web.search.gemini.apiKey"]).toBe("inactive_surface"); + expect( + result.diagnostics.some((entry) => + entry.includes("tools.web.search.gemini.apiKey: tools.web.search is disabled."), + ), + ).toBe(true); + }); + it("returns a version-skew hint when gateway does not support secrets.resolve", async () => { const envKey = "TALK_API_KEY_UNSUPPORTED"; callGateway.mockRejectedValueOnce(new Error("unknown method: secrets.resolve")); diff --git a/src/cli/command-secret-gateway.ts b/src/cli/command-secret-gateway.ts index 89b8c78a3e3..03e578b642c 100644 --- a/src/cli/command-secret-gateway.ts +++ b/src/cli/command-secret-gateway.ts @@ -10,6 +10,7 @@ import { getPath, setPathExistingStrict } from "../secrets/path-utils.js"; import { resolveSecretRefValue } from "../secrets/resolve.js"; import { collectConfigAssignments } from "../secrets/runtime-config-collectors.js"; import { createResolverContext } from "../secrets/runtime-shared.js"; +import { resolveRuntimeWebTools } from "../secrets/runtime-web-tools.js"; import { assertExpectedResolvedSecretValue } from "../secrets/secret-value.js"; import { describeUnknownError } from "../secrets/shared.js"; import { @@ -44,6 +45,15 @@ type GatewaySecretsResolveResult = { inactiveRefPaths?: string[]; }; +const WEB_RUNTIME_SECRET_TARGET_ID_PREFIXES = [ + "tools.web.search", + "tools.web.fetch.firecrawl", +] as const; +const WEB_RUNTIME_SECRET_PATH_PREFIXES = [ + "tools.web.search.", + "tools.web.fetch.firecrawl.", +] as const; + function dedupeDiagnostics(entries: readonly string[]): string[] { const seen = new Set(); const ordered: string[] = []; @@ -58,6 +68,30 @@ function dedupeDiagnostics(entries: readonly string[]): string[] { return ordered; } +function targetsRuntimeWebPath(path: string): boolean { + return WEB_RUNTIME_SECRET_PATH_PREFIXES.some((prefix) => path.startsWith(prefix)); +} + +function targetsRuntimeWebResolution(params: { + targetIds: ReadonlySet; + allowedPaths?: ReadonlySet; +}): boolean { + if (params.allowedPaths) { + for (const path of params.allowedPaths) { + if (targetsRuntimeWebPath(path)) { + return true; + } + } + return false; + } + for (const targetId of params.targetIds) { + if (WEB_RUNTIME_SECRET_TARGET_ID_PREFIXES.some((prefix) => targetId.startsWith(prefix))) { + return true; + } + } + return false; +} + function collectConfiguredTargetRefPaths(params: { config: OpenClawConfig; targetIds: Set; @@ -193,17 +227,40 @@ async function resolveCommandSecretRefsLocally(params: { sourceConfig, env: process.env, }); + const localResolutionDiagnostics: string[] = []; collectConfigAssignments({ config: structuredClone(params.config), context, }); + if ( + targetsRuntimeWebResolution({ targetIds: params.targetIds, allowedPaths: params.allowedPaths }) + ) { + try { + await resolveRuntimeWebTools({ + sourceConfig, + resolvedConfig, + context, + }); + } catch (error) { + if (params.mode === "strict") { + throw error; + } + localResolutionDiagnostics.push( + `${params.commandName}: failed to resolve web tool secrets locally (${describeUnknownError(error)}).`, + ); + } + } const inactiveRefPaths = new Set( context.warnings .filter((warning) => warning.code === "SECRETS_REF_IGNORED_INACTIVE_SURFACE") + .filter((warning) => !params.allowedPaths || params.allowedPaths.has(warning.path)) .map((warning) => warning.path), ); + const inactiveWarningDiagnostics = context.warnings + .filter((warning) => warning.code === "SECRETS_REF_IGNORED_INACTIVE_SURFACE") + .filter((warning) => !params.allowedPaths || params.allowedPaths.has(warning.path)) + .map((warning) => warning.message); const activePaths = new Set(context.assignments.map((assignment) => assignment.path)); - const localResolutionDiagnostics: string[] = []; for (const target of discoverConfigSecretTargetsByIds(sourceConfig, params.targetIds)) { if (params.allowedPaths && !params.allowedPaths.has(target.path)) { continue; @@ -244,6 +301,7 @@ async function resolveCommandSecretRefsLocally(params: { resolvedConfig, diagnostics: dedupeDiagnostics([ ...params.preflightDiagnostics, + ...inactiveWarningDiagnostics, ...filterInactiveSurfaceDiagnostics({ diagnostics: analyzed.diagnostics, inactiveRefPaths, diff --git a/src/cli/command-secret-targets.test.ts b/src/cli/command-secret-targets.test.ts index 3a7de543a02..a71ac5e00c4 100644 --- a/src/cli/command-secret-targets.test.ts +++ b/src/cli/command-secret-targets.test.ts @@ -9,6 +9,7 @@ describe("command secret target ids", () => { const ids = getAgentRuntimeCommandSecretTargetIds(); expect(ids.has("agents.defaults.memorySearch.remote.apiKey")).toBe(true); expect(ids.has("agents.list[].memorySearch.remote.apiKey")).toBe(true); + expect(ids.has("tools.web.fetch.firecrawl.apiKey")).toBe(true); }); it("keeps memory command target set focused on memorySearch remote credentials", () => { diff --git a/src/cli/command-secret-targets.ts b/src/cli/command-secret-targets.ts index c4a4fb5ea4a..e1c2c49e0ae 100644 --- a/src/cli/command-secret-targets.ts +++ b/src/cli/command-secret-targets.ts @@ -23,6 +23,7 @@ const COMMAND_SECRET_TARGETS = { "skills.entries.", "messages.tts.", "tools.web.search", + "tools.web.fetch.firecrawl.", ]), status: idsByPrefix([ "channels.", diff --git a/src/config/types.tools.ts b/src/config/types.tools.ts index 89775758411..e352f858c39 100644 --- a/src/config/types.tools.ts +++ b/src/config/types.tools.ts @@ -512,7 +512,7 @@ export type ToolsConfig = { /** Enable Firecrawl fallback (default: true when apiKey is set). */ enabled?: boolean; /** Firecrawl API key (optional; defaults to FIRECRAWL_API_KEY env var). */ - apiKey?: string; + apiKey?: SecretInput; /** Firecrawl base URL (default: https://api.firecrawl.dev). */ baseUrl?: string; /** Whether to keep only main content (default: true). */ diff --git a/src/gateway/server.reload.test.ts b/src/gateway/server.reload.test.ts index e691256d70f..b3a603fa287 100644 --- a/src/gateway/server.reload.test.ts +++ b/src/gateway/server.reload.test.ts @@ -175,12 +175,14 @@ describe("gateway hot reload", () => { let prevSkipGmail: string | undefined; let prevSkipProviders: string | undefined; let prevOpenAiApiKey: string | undefined; + let prevGeminiApiKey: string | undefined; beforeEach(() => { prevSkipChannels = process.env.OPENCLAW_SKIP_CHANNELS; prevSkipGmail = process.env.OPENCLAW_SKIP_GMAIL_WATCHER; prevSkipProviders = process.env.OPENCLAW_SKIP_PROVIDERS; prevOpenAiApiKey = process.env.OPENAI_API_KEY; + prevGeminiApiKey = process.env.GEMINI_API_KEY; process.env.OPENCLAW_SKIP_CHANNELS = "0"; delete process.env.OPENCLAW_SKIP_GMAIL_WATCHER; delete process.env.OPENCLAW_SKIP_PROVIDERS; @@ -207,6 +209,11 @@ describe("gateway hot reload", () => { } else { process.env.OPENAI_API_KEY = prevOpenAiApiKey; } + if (prevGeminiApiKey === undefined) { + delete process.env.GEMINI_API_KEY; + } else { + process.env.GEMINI_API_KEY = prevGeminiApiKey; + } }); async function writeEnvRefConfig() { @@ -328,6 +335,34 @@ describe("gateway hot reload", () => { ); } + async function writeWebSearchGeminiRefConfig() { + const configPath = process.env.OPENCLAW_CONFIG_PATH; + if (!configPath) { + throw new Error("OPENCLAW_CONFIG_PATH is not set"); + } + await fs.writeFile( + configPath, + `${JSON.stringify( + { + tools: { + web: { + search: { + enabled: true, + provider: "gemini", + gemini: { + apiKey: { source: "env", provider: "default", id: "GEMINI_API_KEY" }, + }, + }, + }, + }, + }, + null, + 2, + )}\n`, + "utf8", + ); + } + async function removeMainAuthProfileStore() { const stateDir = process.env.OPENCLAW_STATE_DIR; if (!stateDir) { @@ -540,6 +575,64 @@ describe("gateway hot reload", () => { }); }); + it("emits one-shot degraded and recovered system events for web search secret reload transitions", async () => { + await writeWebSearchGeminiRefConfig(); + process.env.GEMINI_API_KEY = "gemini-startup-key"; // pragma: allowlist secret + + await withGatewayServer(async () => { + const onHotReload = hoisted.getOnHotReload(); + expect(onHotReload).toBeTypeOf("function"); + const sessionKey = resolveMainSessionKeyFromConfig(); + const plan = { + changedPaths: ["tools.web.search.gemini.apiKey"], + restartGateway: false, + restartReasons: [], + hotReasons: ["tools.web.search.gemini.apiKey"], + reloadHooks: false, + restartGmailWatcher: false, + restartBrowserControl: false, + restartCron: false, + restartHeartbeat: false, + restartChannels: new Set(), + noopPaths: [], + }; + const nextConfig = { + tools: { + web: { + search: { + enabled: true, + provider: "gemini", + gemini: { + apiKey: { source: "env", provider: "default", id: "GEMINI_API_KEY" }, + }, + }, + }, + }, + }; + + delete process.env.GEMINI_API_KEY; + await expect(onHotReload?.(plan, nextConfig)).rejects.toThrow( + "[WEB_SEARCH_KEY_UNRESOLVED_NO_FALLBACK]", + ); + const degradedEvents = drainSystemEvents(sessionKey); + expect(degradedEvents.some((event) => event.includes("[SECRETS_RELOADER_DEGRADED]"))).toBe( + true, + ); + + await expect(onHotReload?.(plan, nextConfig)).rejects.toThrow( + "[WEB_SEARCH_KEY_UNRESOLVED_NO_FALLBACK]", + ); + expect(drainSystemEvents(sessionKey)).toEqual([]); + + process.env.GEMINI_API_KEY = "gemini-recovered-key"; // pragma: allowlist secret + await expect(onHotReload?.(plan, nextConfig)).resolves.toBeUndefined(); + const recoveredEvents = drainSystemEvents(sessionKey); + expect(recoveredEvents.some((event) => event.includes("[SECRETS_RELOADER_RECOVERED]"))).toBe( + true, + ); + }); + }); + it("serves secrets.reload immediately after startup without race failures", async () => { await writeEnvRefConfig(); process.env.OPENAI_API_KEY = "sk-startup"; // pragma: allowlist secret diff --git a/src/secrets/runtime-config-collectors-core.ts b/src/secrets/runtime-config-collectors-core.ts index 504331f0a96..99668371ad1 100644 --- a/src/secrets/runtime-config-collectors-core.ts +++ b/src/secrets/runtime-config-collectors-core.ts @@ -292,67 +292,6 @@ function collectMessagesTtsAssignments(params: { }); } -function collectToolsWebSearchAssignments(params: { - config: OpenClawConfig; - defaults: SecretDefaults | undefined; - context: ResolverContext; -}): void { - const tools = params.config.tools as Record | undefined; - if (!isRecord(tools) || !isRecord(tools.web) || !isRecord(tools.web.search)) { - return; - } - const search = tools.web.search; - const searchEnabled = search.enabled !== false; - const rawProvider = - typeof search.provider === "string" ? search.provider.trim().toLowerCase() : ""; - const selectedProvider = - rawProvider === "brave" || - rawProvider === "gemini" || - rawProvider === "grok" || - rawProvider === "kimi" || - rawProvider === "perplexity" - ? rawProvider - : undefined; - const paths = [ - "apiKey", - "gemini.apiKey", - "grok.apiKey", - "kimi.apiKey", - "perplexity.apiKey", - ] as const; - for (const path of paths) { - const [scope, field] = path.includes(".") ? path.split(".", 2) : [undefined, path]; - const target = scope ? search[scope] : search; - if (!isRecord(target)) { - continue; - } - const active = scope - ? searchEnabled && (selectedProvider === undefined || selectedProvider === scope) - : searchEnabled && (selectedProvider === undefined || selectedProvider === "brave"); - const inactiveReason = !searchEnabled - ? "tools.web.search is disabled." - : scope - ? selectedProvider === undefined - ? undefined - : `tools.web.search.provider is "${selectedProvider}".` - : selectedProvider === undefined - ? undefined - : `tools.web.search.provider is "${selectedProvider}".`; - collectSecretInputAssignment({ - value: target[field], - path: `tools.web.search.${path}`, - expected: "string", - defaults: params.defaults, - context: params.context, - active, - inactiveReason, - apply: (value) => { - target[field] = value; - }, - }); - } -} - function collectCronAssignments(params: { config: OpenClawConfig; defaults: SecretDefaults | undefined; @@ -401,6 +340,5 @@ export function collectCoreConfigAssignments(params: { collectTalkAssignments(params); collectGatewayAssignments(params); collectMessagesTtsAssignments(params); - collectToolsWebSearchAssignments(params); collectCronAssignments(params); } diff --git a/src/secrets/runtime-shared.ts b/src/secrets/runtime-shared.ts index 8374f642de8..77dcb3c051c 100644 --- a/src/secrets/runtime-shared.ts +++ b/src/secrets/runtime-shared.ts @@ -7,7 +7,12 @@ import { isRecord } from "./shared.js"; export type SecretResolverWarningCode = | "SECRETS_REF_OVERRIDES_PLAINTEXT" - | "SECRETS_REF_IGNORED_INACTIVE_SURFACE"; + | "SECRETS_REF_IGNORED_INACTIVE_SURFACE" + | "WEB_SEARCH_PROVIDER_INVALID_AUTODETECT" + | "WEB_SEARCH_KEY_UNRESOLVED_FALLBACK_USED" + | "WEB_SEARCH_KEY_UNRESOLVED_NO_FALLBACK" + | "WEB_FETCH_FIRECRAWL_KEY_UNRESOLVED_FALLBACK_USED" + | "WEB_FETCH_FIRECRAWL_KEY_UNRESOLVED_NO_FALLBACK"; export type SecretResolverWarning = { code: SecretResolverWarningCode; diff --git a/src/secrets/runtime-web-tools.test.ts b/src/secrets/runtime-web-tools.test.ts new file mode 100644 index 00000000000..b8c1e679ba6 --- /dev/null +++ b/src/secrets/runtime-web-tools.test.ts @@ -0,0 +1,451 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import type { OpenClawConfig } from "../config/config.js"; +import * as secretResolve from "./resolve.js"; +import { createResolverContext } from "./runtime-shared.js"; +import { resolveRuntimeWebTools } from "./runtime-web-tools.js"; + +type ProviderUnderTest = "brave" | "gemini" | "grok" | "kimi" | "perplexity"; + +function asConfig(value: unknown): OpenClawConfig { + return value as OpenClawConfig; +} + +async function runRuntimeWebTools(params: { config: OpenClawConfig; env?: NodeJS.ProcessEnv }) { + const sourceConfig = structuredClone(params.config); + const resolvedConfig = structuredClone(params.config); + const context = createResolverContext({ + sourceConfig, + env: params.env ?? {}, + }); + const metadata = await resolveRuntimeWebTools({ + sourceConfig, + resolvedConfig, + context, + }); + return { metadata, resolvedConfig, context }; +} + +function createProviderSecretRefConfig( + provider: ProviderUnderTest, + envRefId: string, +): OpenClawConfig { + const search: Record = { + enabled: true, + provider, + }; + if (provider === "brave") { + search.apiKey = { source: "env", provider: "default", id: envRefId }; + } else { + search[provider] = { + apiKey: { source: "env", provider: "default", id: envRefId }, + }; + } + return asConfig({ + tools: { + web: { + search, + }, + }, + }); +} + +function readProviderKey(config: OpenClawConfig, provider: ProviderUnderTest): unknown { + if (provider === "brave") { + return config.tools?.web?.search?.apiKey; + } + if (provider === "gemini") { + return config.tools?.web?.search?.gemini?.apiKey; + } + if (provider === "grok") { + return config.tools?.web?.search?.grok?.apiKey; + } + if (provider === "kimi") { + return config.tools?.web?.search?.kimi?.apiKey; + } + return config.tools?.web?.search?.perplexity?.apiKey; +} + +describe("runtime web tools resolution", () => { + afterEach(() => { + vi.restoreAllMocks(); + }); + + it.each([ + { + provider: "brave" as const, + envRefId: "BRAVE_PROVIDER_REF", + resolvedKey: "brave-provider-key", + }, + { + provider: "gemini" as const, + envRefId: "GEMINI_PROVIDER_REF", + resolvedKey: "gemini-provider-key", + }, + { + provider: "grok" as const, + envRefId: "GROK_PROVIDER_REF", + resolvedKey: "grok-provider-key", + }, + { + provider: "kimi" as const, + envRefId: "KIMI_PROVIDER_REF", + resolvedKey: "kimi-provider-key", + }, + { + provider: "perplexity" as const, + envRefId: "PERPLEXITY_PROVIDER_REF", + resolvedKey: "pplx-provider-key", + }, + ])( + "resolves configured provider SecretRef for $provider", + async ({ provider, envRefId, resolvedKey }) => { + const { metadata, resolvedConfig, context } = await runRuntimeWebTools({ + config: createProviderSecretRefConfig(provider, envRefId), + env: { + [envRefId]: resolvedKey, + }, + }); + + expect(metadata.search.providerConfigured).toBe(provider); + expect(metadata.search.providerSource).toBe("configured"); + expect(metadata.search.selectedProvider).toBe(provider); + expect(metadata.search.selectedProviderKeySource).toBe("secretRef"); + expect(readProviderKey(resolvedConfig, provider)).toBe(resolvedKey); + expect(context.warnings.map((warning) => warning.code)).not.toContain( + "WEB_SEARCH_KEY_UNRESOLVED_NO_FALLBACK", + ); + if (provider === "perplexity") { + expect(metadata.search.perplexityTransport).toBe("search_api"); + } + }, + ); + + it("auto-detects provider precedence across all configured providers", async () => { + const { metadata, resolvedConfig, context } = await runRuntimeWebTools({ + config: asConfig({ + tools: { + web: { + search: { + apiKey: { source: "env", provider: "default", id: "BRAVE_REF" }, + gemini: { + apiKey: { source: "env", provider: "default", id: "GEMINI_REF" }, + }, + grok: { + apiKey: { source: "env", provider: "default", id: "GROK_REF" }, + }, + kimi: { + apiKey: { source: "env", provider: "default", id: "KIMI_REF" }, + }, + perplexity: { + apiKey: { source: "env", provider: "default", id: "PERPLEXITY_REF" }, + }, + }, + }, + }, + }), + env: { + BRAVE_REF: "brave-precedence-key", + GEMINI_REF: "gemini-precedence-key", + GROK_REF: "grok-precedence-key", + KIMI_REF: "kimi-precedence-key", + PERPLEXITY_REF: "pplx-precedence-key", + }, + }); + + expect(metadata.search.providerSource).toBe("auto-detect"); + expect(metadata.search.selectedProvider).toBe("brave"); + expect(resolvedConfig.tools?.web?.search?.apiKey).toBe("brave-precedence-key"); + expect(context.warnings).toEqual( + expect.arrayContaining([ + expect.objectContaining({ path: "tools.web.search.gemini.apiKey" }), + expect.objectContaining({ path: "tools.web.search.grok.apiKey" }), + expect.objectContaining({ path: "tools.web.search.kimi.apiKey" }), + expect.objectContaining({ path: "tools.web.search.perplexity.apiKey" }), + ]), + ); + }); + + it("auto-detects first available provider and keeps lower-priority refs inactive", async () => { + const { metadata, resolvedConfig, context } = await runRuntimeWebTools({ + config: asConfig({ + tools: { + web: { + search: { + apiKey: { source: "env", provider: "default", id: "BRAVE_API_KEY_REF" }, + gemini: { + apiKey: { + source: "env", + provider: "default", + id: "MISSING_GEMINI_API_KEY_REF", + }, + }, + }, + }, + }, + }), + env: { + BRAVE_API_KEY_REF: "brave-runtime-key", + }, + }); + + expect(metadata.search.providerSource).toBe("auto-detect"); + expect(metadata.search.selectedProvider).toBe("brave"); + expect(metadata.search.selectedProviderKeySource).toBe("secretRef"); + expect(resolvedConfig.tools?.web?.search?.apiKey).toBe("brave-runtime-key"); + expect(resolvedConfig.tools?.web?.search?.gemini?.apiKey).toEqual({ + source: "env", + provider: "default", + id: "MISSING_GEMINI_API_KEY_REF", + }); + expect(context.warnings).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + code: "SECRETS_REF_IGNORED_INACTIVE_SURFACE", + path: "tools.web.search.gemini.apiKey", + }), + ]), + ); + expect(context.warnings.map((warning) => warning.code)).not.toContain( + "WEB_SEARCH_KEY_UNRESOLVED_NO_FALLBACK", + ); + }); + + it("auto-detects the next provider when a higher-priority ref is unresolved", async () => { + const { metadata, resolvedConfig, context } = await runRuntimeWebTools({ + config: asConfig({ + tools: { + web: { + search: { + apiKey: { source: "env", provider: "default", id: "MISSING_BRAVE_API_KEY_REF" }, + gemini: { + apiKey: { source: "env", provider: "default", id: "GEMINI_API_KEY_REF" }, + }, + }, + }, + }, + }), + env: { + GEMINI_API_KEY_REF: "gemini-runtime-key", + }, + }); + + expect(metadata.search.providerSource).toBe("auto-detect"); + expect(metadata.search.selectedProvider).toBe("gemini"); + expect(resolvedConfig.tools?.web?.search?.gemini?.apiKey).toBe("gemini-runtime-key"); + expect(context.warnings).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + code: "SECRETS_REF_IGNORED_INACTIVE_SURFACE", + path: "tools.web.search.apiKey", + }), + ]), + ); + expect(context.warnings.map((warning) => warning.code)).not.toContain( + "WEB_SEARCH_KEY_UNRESOLVED_NO_FALLBACK", + ); + }); + + it("warns when provider is invalid and falls back to auto-detect", async () => { + const { metadata, resolvedConfig, context } = await runRuntimeWebTools({ + config: asConfig({ + tools: { + web: { + search: { + provider: "invalid-provider", + gemini: { + apiKey: { source: "env", provider: "default", id: "GEMINI_API_KEY_REF" }, + }, + }, + }, + }, + }), + env: { + GEMINI_API_KEY_REF: "gemini-runtime-key", + }, + }); + + expect(metadata.search.providerConfigured).toBeUndefined(); + expect(metadata.search.providerSource).toBe("auto-detect"); + expect(metadata.search.selectedProvider).toBe("gemini"); + expect(resolvedConfig.tools?.web?.search?.gemini?.apiKey).toBe("gemini-runtime-key"); + expect(metadata.search.diagnostics).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + code: "WEB_SEARCH_PROVIDER_INVALID_AUTODETECT", + path: "tools.web.search.provider", + }), + ]), + ); + expect(context.warnings).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + code: "WEB_SEARCH_PROVIDER_INVALID_AUTODETECT", + path: "tools.web.search.provider", + }), + ]), + ); + }); + + it("fails fast when configured provider ref is unresolved with no fallback", async () => { + const sourceConfig = asConfig({ + tools: { + web: { + search: { + provider: "gemini", + gemini: { + apiKey: { source: "env", provider: "default", id: "MISSING_GEMINI_API_KEY_REF" }, + }, + }, + }, + }, + }); + const resolvedConfig = structuredClone(sourceConfig); + const context = createResolverContext({ + sourceConfig, + env: {}, + }); + + await expect( + resolveRuntimeWebTools({ + sourceConfig, + resolvedConfig, + context, + }), + ).rejects.toThrow("[WEB_SEARCH_KEY_UNRESOLVED_NO_FALLBACK]"); + expect(context.warnings).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + code: "WEB_SEARCH_KEY_UNRESOLVED_NO_FALLBACK", + path: "tools.web.search.gemini.apiKey", + }), + ]), + ); + }); + + it("does not resolve Firecrawl SecretRef when Firecrawl is inactive", async () => { + const resolveSpy = vi.spyOn(secretResolve, "resolveSecretRefValues"); + const { metadata, context } = await runRuntimeWebTools({ + config: asConfig({ + tools: { + web: { + fetch: { + enabled: false, + firecrawl: { + apiKey: { source: "env", provider: "default", id: "MISSING_FIRECRAWL_REF" }, + }, + }, + }, + }, + }), + }); + + expect(resolveSpy).not.toHaveBeenCalled(); + expect(metadata.fetch.firecrawl.active).toBe(false); + expect(metadata.fetch.firecrawl.apiKeySource).toBe("secretRef"); + expect(context.warnings).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + code: "SECRETS_REF_IGNORED_INACTIVE_SURFACE", + path: "tools.web.fetch.firecrawl.apiKey", + }), + ]), + ); + }); + + it("does not resolve Firecrawl SecretRef when Firecrawl is disabled", async () => { + const resolveSpy = vi.spyOn(secretResolve, "resolveSecretRefValues"); + const { metadata, context } = await runRuntimeWebTools({ + config: asConfig({ + tools: { + web: { + fetch: { + enabled: true, + firecrawl: { + enabled: false, + apiKey: { source: "env", provider: "default", id: "MISSING_FIRECRAWL_REF" }, + }, + }, + }, + }, + }), + }); + + expect(resolveSpy).not.toHaveBeenCalled(); + expect(metadata.fetch.firecrawl.active).toBe(false); + expect(metadata.fetch.firecrawl.apiKeySource).toBe("secretRef"); + expect(context.warnings).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + code: "SECRETS_REF_IGNORED_INACTIVE_SURFACE", + path: "tools.web.fetch.firecrawl.apiKey", + }), + ]), + ); + }); + + it("uses env fallback for unresolved Firecrawl SecretRef when active", async () => { + const { metadata, resolvedConfig, context } = await runRuntimeWebTools({ + config: asConfig({ + tools: { + web: { + fetch: { + firecrawl: { + apiKey: { source: "env", provider: "default", id: "MISSING_FIRECRAWL_REF" }, + }, + }, + }, + }, + }), + env: { + FIRECRAWL_API_KEY: "firecrawl-fallback-key", + }, + }); + + expect(metadata.fetch.firecrawl.active).toBe(true); + expect(metadata.fetch.firecrawl.apiKeySource).toBe("env"); + expect(resolvedConfig.tools?.web?.fetch?.firecrawl?.apiKey).toBe("firecrawl-fallback-key"); + expect(context.warnings).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + code: "WEB_FETCH_FIRECRAWL_KEY_UNRESOLVED_FALLBACK_USED", + path: "tools.web.fetch.firecrawl.apiKey", + }), + ]), + ); + }); + + it("fails fast when active Firecrawl SecretRef is unresolved with no fallback", async () => { + const sourceConfig = asConfig({ + tools: { + web: { + fetch: { + firecrawl: { + apiKey: { source: "env", provider: "default", id: "MISSING_FIRECRAWL_REF" }, + }, + }, + }, + }, + }); + const resolvedConfig = structuredClone(sourceConfig); + const context = createResolverContext({ + sourceConfig, + env: {}, + }); + + await expect( + resolveRuntimeWebTools({ + sourceConfig, + resolvedConfig, + context, + }), + ).rejects.toThrow("[WEB_FETCH_FIRECRAWL_KEY_UNRESOLVED_NO_FALLBACK]"); + expect(context.warnings).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + code: "WEB_FETCH_FIRECRAWL_KEY_UNRESOLVED_NO_FALLBACK", + path: "tools.web.fetch.firecrawl.apiKey", + }), + ]), + ); + }); +}); diff --git a/src/secrets/runtime-web-tools.ts b/src/secrets/runtime-web-tools.ts new file mode 100644 index 00000000000..004af2bdfe2 --- /dev/null +++ b/src/secrets/runtime-web-tools.ts @@ -0,0 +1,705 @@ +import type { OpenClawConfig } from "../config/config.js"; +import { resolveSecretInputRef } from "../config/types.secrets.js"; +import { normalizeSecretInput } from "../utils/normalize-secret-input.js"; +import { secretRefKey } from "./ref-contract.js"; +import { resolveSecretRefValues } from "./resolve.js"; +import { + pushInactiveSurfaceWarning, + pushWarning, + type ResolverContext, + type SecretDefaults, +} from "./runtime-shared.js"; + +const WEB_SEARCH_PROVIDERS = ["brave", "gemini", "grok", "kimi", "perplexity"] as const; +const PERPLEXITY_DIRECT_BASE_URL = "https://api.perplexity.ai"; +const DEFAULT_PERPLEXITY_BASE_URL = "https://openrouter.ai/api/v1"; +const PERPLEXITY_KEY_PREFIXES = ["pplx-"]; +const OPENROUTER_KEY_PREFIXES = ["sk-or-"]; + +type WebSearchProvider = (typeof WEB_SEARCH_PROVIDERS)[number]; + +type SecretResolutionSource = "config" | "secretRef" | "env" | "missing"; +type RuntimeWebProviderSource = "configured" | "auto-detect" | "none"; + +export type RuntimeWebDiagnosticCode = + | "WEB_SEARCH_PROVIDER_INVALID_AUTODETECT" + | "WEB_SEARCH_AUTODETECT_SELECTED" + | "WEB_SEARCH_KEY_UNRESOLVED_FALLBACK_USED" + | "WEB_SEARCH_KEY_UNRESOLVED_NO_FALLBACK" + | "WEB_FETCH_FIRECRAWL_KEY_UNRESOLVED_FALLBACK_USED" + | "WEB_FETCH_FIRECRAWL_KEY_UNRESOLVED_NO_FALLBACK"; + +export type RuntimeWebDiagnostic = { + code: RuntimeWebDiagnosticCode; + message: string; + path?: string; +}; + +export type RuntimeWebSearchMetadata = { + providerConfigured?: WebSearchProvider; + providerSource: RuntimeWebProviderSource; + selectedProvider?: WebSearchProvider; + selectedProviderKeySource?: SecretResolutionSource; + perplexityTransport?: "search_api" | "chat_completions"; + diagnostics: RuntimeWebDiagnostic[]; +}; + +export type RuntimeWebFetchFirecrawlMetadata = { + active: boolean; + apiKeySource: SecretResolutionSource; + diagnostics: RuntimeWebDiagnostic[]; +}; + +export type RuntimeWebToolsMetadata = { + search: RuntimeWebSearchMetadata; + fetch: { + firecrawl: RuntimeWebFetchFirecrawlMetadata; + }; + diagnostics: RuntimeWebDiagnostic[]; +}; + +type FetchConfig = NonNullable["web"] extends infer Web + ? Web extends { fetch?: infer Fetch } + ? Fetch + : undefined + : undefined; + +type SecretResolutionResult = { + value?: string; + source: SecretResolutionSource; + secretRefConfigured: boolean; + unresolvedRefReason?: string; + fallbackEnvVar?: string; + fallbackUsedAfterRefFailure: boolean; +}; + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function normalizeProvider(value: unknown): WebSearchProvider | undefined { + if (typeof value !== "string") { + return undefined; + } + const normalized = value.trim().toLowerCase(); + if ( + normalized === "brave" || + normalized === "gemini" || + normalized === "grok" || + normalized === "kimi" || + normalized === "perplexity" + ) { + return normalized; + } + return undefined; +} + +function readNonEmptyEnvValue( + env: NodeJS.ProcessEnv, + names: string[], +): { value?: string; envVar?: string } { + for (const envVar of names) { + const value = normalizeSecretInput(env[envVar]); + if (value) { + return { value, envVar }; + } + } + return {}; +} + +function buildUnresolvedReason(params: { + path: string; + kind: "unresolved" | "non-string" | "empty"; + refLabel: string; +}): string { + if (params.kind === "non-string") { + return `${params.path} SecretRef resolved to a non-string value.`; + } + if (params.kind === "empty") { + return `${params.path} SecretRef resolved to an empty value.`; + } + return `${params.path} SecretRef is unresolved (${params.refLabel}).`; +} + +async function resolveSecretInputWithEnvFallback(params: { + sourceConfig: OpenClawConfig; + context: ResolverContext; + defaults: SecretDefaults | undefined; + value: unknown; + path: string; + envVars: string[]; +}): Promise { + const { ref } = resolveSecretInputRef({ + value: params.value, + defaults: params.defaults, + }); + + if (!ref) { + const configValue = normalizeSecretInput(params.value); + if (configValue) { + return { + value: configValue, + source: "config", + secretRefConfigured: false, + fallbackUsedAfterRefFailure: false, + }; + } + const fallback = readNonEmptyEnvValue(params.context.env, params.envVars); + if (fallback.value) { + return { + value: fallback.value, + source: "env", + fallbackEnvVar: fallback.envVar, + secretRefConfigured: false, + fallbackUsedAfterRefFailure: false, + }; + } + return { + source: "missing", + secretRefConfigured: false, + fallbackUsedAfterRefFailure: false, + }; + } + + const refLabel = `${ref.source}:${ref.provider}:${ref.id}`; + let resolvedFromRef: string | undefined; + let unresolvedRefReason: string | undefined; + + try { + const resolved = await resolveSecretRefValues([ref], { + config: params.sourceConfig, + env: params.context.env, + cache: params.context.cache, + }); + const resolvedValue = resolved.get(secretRefKey(ref)); + if (typeof resolvedValue !== "string") { + unresolvedRefReason = buildUnresolvedReason({ + path: params.path, + kind: "non-string", + refLabel, + }); + } else { + resolvedFromRef = normalizeSecretInput(resolvedValue); + if (!resolvedFromRef) { + unresolvedRefReason = buildUnresolvedReason({ + path: params.path, + kind: "empty", + refLabel, + }); + } + } + } catch { + unresolvedRefReason = buildUnresolvedReason({ + path: params.path, + kind: "unresolved", + refLabel, + }); + } + + if (resolvedFromRef) { + return { + value: resolvedFromRef, + source: "secretRef", + secretRefConfigured: true, + fallbackUsedAfterRefFailure: false, + }; + } + + const fallback = readNonEmptyEnvValue(params.context.env, params.envVars); + if (fallback.value) { + return { + value: fallback.value, + source: "env", + fallbackEnvVar: fallback.envVar, + unresolvedRefReason, + secretRefConfigured: true, + fallbackUsedAfterRefFailure: true, + }; + } + + return { + source: "missing", + unresolvedRefReason, + secretRefConfigured: true, + fallbackUsedAfterRefFailure: false, + }; +} + +function inferPerplexityBaseUrlFromApiKey(apiKey?: string): "direct" | "openrouter" | undefined { + if (!apiKey) { + return undefined; + } + const normalized = apiKey.toLowerCase(); + if (PERPLEXITY_KEY_PREFIXES.some((prefix) => normalized.startsWith(prefix))) { + return "direct"; + } + if (OPENROUTER_KEY_PREFIXES.some((prefix) => normalized.startsWith(prefix))) { + return "openrouter"; + } + return undefined; +} + +function resolvePerplexityRuntimeTransport(params: { + keyValue?: string; + keySource: SecretResolutionSource; + fallbackEnvVar?: string; + configValue: unknown; +}): "search_api" | "chat_completions" | undefined { + const config = isRecord(params.configValue) ? params.configValue : undefined; + const configuredBaseUrl = typeof config?.baseUrl === "string" ? config.baseUrl.trim() : ""; + const configuredModel = typeof config?.model === "string" ? config.model.trim() : ""; + + const baseUrl = (() => { + if (configuredBaseUrl) { + return configuredBaseUrl; + } + if (params.keySource === "env") { + if (params.fallbackEnvVar === "PERPLEXITY_API_KEY") { + return PERPLEXITY_DIRECT_BASE_URL; + } + if (params.fallbackEnvVar === "OPENROUTER_API_KEY") { + return DEFAULT_PERPLEXITY_BASE_URL; + } + } + if ((params.keySource === "config" || params.keySource === "secretRef") && params.keyValue) { + const inferred = inferPerplexityBaseUrlFromApiKey(params.keyValue); + return inferred === "openrouter" ? DEFAULT_PERPLEXITY_BASE_URL : PERPLEXITY_DIRECT_BASE_URL; + } + return DEFAULT_PERPLEXITY_BASE_URL; + })(); + + const hasLegacyOverride = Boolean(configuredBaseUrl || configuredModel); + const direct = (() => { + try { + return new URL(baseUrl).hostname.toLowerCase() === "api.perplexity.ai"; + } catch { + return false; + } + })(); + return hasLegacyOverride || !direct ? "chat_completions" : "search_api"; +} + +function ensureObject(target: Record, key: string): Record { + const current = target[key]; + if (isRecord(current)) { + return current; + } + const next: Record = {}; + target[key] = next; + return next; +} + +function setResolvedWebSearchApiKey(params: { + resolvedConfig: OpenClawConfig; + provider: WebSearchProvider; + value: string; +}): void { + const tools = ensureObject(params.resolvedConfig as Record, "tools"); + const web = ensureObject(tools, "web"); + const search = ensureObject(web, "search"); + if (params.provider === "brave") { + search.apiKey = params.value; + return; + } + const providerConfig = ensureObject(search, params.provider); + providerConfig.apiKey = params.value; +} + +function setResolvedFirecrawlApiKey(params: { + resolvedConfig: OpenClawConfig; + value: string; +}): void { + const tools = ensureObject(params.resolvedConfig as Record, "tools"); + const web = ensureObject(tools, "web"); + const fetch = ensureObject(web, "fetch"); + const firecrawl = ensureObject(fetch, "firecrawl"); + firecrawl.apiKey = params.value; +} + +function envVarsForProvider(provider: WebSearchProvider): string[] { + if (provider === "brave") { + return ["BRAVE_API_KEY"]; + } + if (provider === "gemini") { + return ["GEMINI_API_KEY"]; + } + if (provider === "grok") { + return ["XAI_API_KEY"]; + } + if (provider === "kimi") { + return ["KIMI_API_KEY", "MOONSHOT_API_KEY"]; + } + return ["PERPLEXITY_API_KEY", "OPENROUTER_API_KEY"]; +} + +function resolveProviderKeyValue( + search: Record, + provider: WebSearchProvider, +): unknown { + if (provider === "brave") { + return search.apiKey; + } + const scoped = search[provider]; + if (!isRecord(scoped)) { + return undefined; + } + return scoped.apiKey; +} + +function hasConfiguredSecretRef(value: unknown, defaults: SecretDefaults | undefined): boolean { + return Boolean( + resolveSecretInputRef({ + value, + defaults, + }).ref, + ); +} + +export async function resolveRuntimeWebTools(params: { + sourceConfig: OpenClawConfig; + resolvedConfig: OpenClawConfig; + context: ResolverContext; +}): Promise { + const defaults = params.sourceConfig.secrets?.defaults; + const diagnostics: RuntimeWebDiagnostic[] = []; + + const tools = isRecord(params.sourceConfig.tools) ? params.sourceConfig.tools : undefined; + const web = isRecord(tools?.web) ? tools.web : undefined; + const search = isRecord(web?.search) ? web.search : undefined; + + const searchMetadata: RuntimeWebSearchMetadata = { + providerSource: "none", + diagnostics: [], + }; + + const searchEnabled = search?.enabled !== false; + const rawProvider = + typeof search?.provider === "string" ? search.provider.trim().toLowerCase() : ""; + const configuredProvider = normalizeProvider(rawProvider); + + if (rawProvider && !configuredProvider) { + const diagnostic: RuntimeWebDiagnostic = { + code: "WEB_SEARCH_PROVIDER_INVALID_AUTODETECT", + message: `tools.web.search.provider is "${rawProvider}". Falling back to auto-detect precedence.`, + path: "tools.web.search.provider", + }; + diagnostics.push(diagnostic); + searchMetadata.diagnostics.push(diagnostic); + pushWarning(params.context, { + code: "WEB_SEARCH_PROVIDER_INVALID_AUTODETECT", + path: "tools.web.search.provider", + message: diagnostic.message, + }); + } + + if (configuredProvider) { + searchMetadata.providerConfigured = configuredProvider; + searchMetadata.providerSource = "configured"; + } + + if (searchEnabled && search) { + const candidates = configuredProvider ? [configuredProvider] : [...WEB_SEARCH_PROVIDERS]; + const unresolvedWithoutFallback: Array<{ + provider: WebSearchProvider; + path: string; + reason: string; + }> = []; + + let selectedProvider: WebSearchProvider | undefined; + let selectedResolution: SecretResolutionResult | undefined; + + for (const provider of candidates) { + const path = + provider === "brave" ? "tools.web.search.apiKey" : `tools.web.search.${provider}.apiKey`; + const value = resolveProviderKeyValue(search, provider); + const resolution = await resolveSecretInputWithEnvFallback({ + sourceConfig: params.sourceConfig, + context: params.context, + defaults, + value, + path, + envVars: envVarsForProvider(provider), + }); + + if (resolution.secretRefConfigured && resolution.fallbackUsedAfterRefFailure) { + const diagnostic: RuntimeWebDiagnostic = { + code: "WEB_SEARCH_KEY_UNRESOLVED_FALLBACK_USED", + message: + `${path} SecretRef could not be resolved; using ${resolution.fallbackEnvVar ?? "env fallback"}. ` + + (resolution.unresolvedRefReason ?? "").trim(), + path, + }; + diagnostics.push(diagnostic); + searchMetadata.diagnostics.push(diagnostic); + pushWarning(params.context, { + code: "WEB_SEARCH_KEY_UNRESOLVED_FALLBACK_USED", + path, + message: diagnostic.message, + }); + } + + if (resolution.secretRefConfigured && !resolution.value && resolution.unresolvedRefReason) { + unresolvedWithoutFallback.push({ + provider, + path, + reason: resolution.unresolvedRefReason, + }); + } + + if (configuredProvider) { + selectedProvider = provider; + selectedResolution = resolution; + if (resolution.value) { + setResolvedWebSearchApiKey({ + resolvedConfig: params.resolvedConfig, + provider, + value: resolution.value, + }); + } + break; + } + + if (resolution.value) { + selectedProvider = provider; + selectedResolution = resolution; + setResolvedWebSearchApiKey({ + resolvedConfig: params.resolvedConfig, + provider, + value: resolution.value, + }); + break; + } + } + + if (configuredProvider) { + const unresolved = unresolvedWithoutFallback[0]; + if (unresolved) { + const diagnostic: RuntimeWebDiagnostic = { + code: "WEB_SEARCH_KEY_UNRESOLVED_NO_FALLBACK", + message: unresolved.reason, + path: unresolved.path, + }; + diagnostics.push(diagnostic); + searchMetadata.diagnostics.push(diagnostic); + pushWarning(params.context, { + code: "WEB_SEARCH_KEY_UNRESOLVED_NO_FALLBACK", + path: unresolved.path, + message: unresolved.reason, + }); + throw new Error(`[WEB_SEARCH_KEY_UNRESOLVED_NO_FALLBACK] ${unresolved.reason}`); + } + } else { + if (!selectedProvider && unresolvedWithoutFallback.length > 0) { + const unresolved = unresolvedWithoutFallback[0]; + const diagnostic: RuntimeWebDiagnostic = { + code: "WEB_SEARCH_KEY_UNRESOLVED_NO_FALLBACK", + message: unresolved.reason, + path: unresolved.path, + }; + diagnostics.push(diagnostic); + searchMetadata.diagnostics.push(diagnostic); + pushWarning(params.context, { + code: "WEB_SEARCH_KEY_UNRESOLVED_NO_FALLBACK", + path: unresolved.path, + message: unresolved.reason, + }); + throw new Error(`[WEB_SEARCH_KEY_UNRESOLVED_NO_FALLBACK] ${unresolved.reason}`); + } + + if (selectedProvider) { + const diagnostic: RuntimeWebDiagnostic = { + code: "WEB_SEARCH_AUTODETECT_SELECTED", + message: `tools.web.search auto-detected provider "${selectedProvider}" from available credentials.`, + path: "tools.web.search.provider", + }; + diagnostics.push(diagnostic); + searchMetadata.diagnostics.push(diagnostic); + } + } + + if (selectedProvider) { + searchMetadata.selectedProvider = selectedProvider; + searchMetadata.selectedProviderKeySource = selectedResolution?.source; + if (!configuredProvider) { + searchMetadata.providerSource = "auto-detect"; + } + if (selectedProvider === "perplexity") { + searchMetadata.perplexityTransport = resolvePerplexityRuntimeTransport({ + keyValue: selectedResolution?.value, + keySource: selectedResolution?.source ?? "missing", + fallbackEnvVar: selectedResolution?.fallbackEnvVar, + configValue: search.perplexity, + }); + } + } + } + + if (searchEnabled && search && !configuredProvider && searchMetadata.selectedProvider) { + for (const provider of WEB_SEARCH_PROVIDERS) { + if (provider === searchMetadata.selectedProvider) { + continue; + } + const path = + provider === "brave" ? "tools.web.search.apiKey" : `tools.web.search.${provider}.apiKey`; + const value = resolveProviderKeyValue(search, provider); + if (!hasConfiguredSecretRef(value, defaults)) { + continue; + } + pushInactiveSurfaceWarning({ + context: params.context, + path, + details: `tools.web.search auto-detected provider is "${searchMetadata.selectedProvider}".`, + }); + } + } else if (search && !searchEnabled) { + for (const provider of WEB_SEARCH_PROVIDERS) { + const path = + provider === "brave" ? "tools.web.search.apiKey" : `tools.web.search.${provider}.apiKey`; + const value = resolveProviderKeyValue(search, provider); + if (!hasConfiguredSecretRef(value, defaults)) { + continue; + } + pushInactiveSurfaceWarning({ + context: params.context, + path, + details: "tools.web.search is disabled.", + }); + } + } + + if (searchEnabled && search && configuredProvider) { + for (const provider of WEB_SEARCH_PROVIDERS) { + if (provider === configuredProvider) { + continue; + } + const path = + provider === "brave" ? "tools.web.search.apiKey" : `tools.web.search.${provider}.apiKey`; + const value = resolveProviderKeyValue(search, provider); + if (!hasConfiguredSecretRef(value, defaults)) { + continue; + } + pushInactiveSurfaceWarning({ + context: params.context, + path, + details: `tools.web.search.provider is "${configuredProvider}".`, + }); + } + } + + const fetch = isRecord(web?.fetch) ? (web.fetch as FetchConfig) : undefined; + const firecrawl = isRecord(fetch?.firecrawl) ? fetch.firecrawl : undefined; + const fetchEnabled = fetch?.enabled !== false; + const firecrawlEnabled = firecrawl?.enabled !== false; + const firecrawlActive = Boolean(fetchEnabled && firecrawlEnabled); + const firecrawlPath = "tools.web.fetch.firecrawl.apiKey"; + let firecrawlResolution: SecretResolutionResult = { + source: "missing", + secretRefConfigured: false, + fallbackUsedAfterRefFailure: false, + }; + + const firecrawlDiagnostics: RuntimeWebDiagnostic[] = []; + + if (firecrawlActive) { + firecrawlResolution = await resolveSecretInputWithEnvFallback({ + sourceConfig: params.sourceConfig, + context: params.context, + defaults, + value: firecrawl?.apiKey, + path: firecrawlPath, + envVars: ["FIRECRAWL_API_KEY"], + }); + + if (firecrawlResolution.value) { + setResolvedFirecrawlApiKey({ + resolvedConfig: params.resolvedConfig, + value: firecrawlResolution.value, + }); + } + + if (firecrawlResolution.secretRefConfigured) { + if (firecrawlResolution.fallbackUsedAfterRefFailure) { + const diagnostic: RuntimeWebDiagnostic = { + code: "WEB_FETCH_FIRECRAWL_KEY_UNRESOLVED_FALLBACK_USED", + message: + `${firecrawlPath} SecretRef could not be resolved; using ${firecrawlResolution.fallbackEnvVar ?? "env fallback"}. ` + + (firecrawlResolution.unresolvedRefReason ?? "").trim(), + path: firecrawlPath, + }; + diagnostics.push(diagnostic); + firecrawlDiagnostics.push(diagnostic); + pushWarning(params.context, { + code: "WEB_FETCH_FIRECRAWL_KEY_UNRESOLVED_FALLBACK_USED", + path: firecrawlPath, + message: diagnostic.message, + }); + } + + if (!firecrawlResolution.value && firecrawlResolution.unresolvedRefReason) { + const diagnostic: RuntimeWebDiagnostic = { + code: "WEB_FETCH_FIRECRAWL_KEY_UNRESOLVED_NO_FALLBACK", + message: firecrawlResolution.unresolvedRefReason, + path: firecrawlPath, + }; + diagnostics.push(diagnostic); + firecrawlDiagnostics.push(diagnostic); + pushWarning(params.context, { + code: "WEB_FETCH_FIRECRAWL_KEY_UNRESOLVED_NO_FALLBACK", + path: firecrawlPath, + message: firecrawlResolution.unresolvedRefReason, + }); + throw new Error( + `[WEB_FETCH_FIRECRAWL_KEY_UNRESOLVED_NO_FALLBACK] ${firecrawlResolution.unresolvedRefReason}`, + ); + } + } + } else { + if (hasConfiguredSecretRef(firecrawl?.apiKey, defaults)) { + pushInactiveSurfaceWarning({ + context: params.context, + path: firecrawlPath, + details: !fetchEnabled + ? "tools.web.fetch is disabled." + : "tools.web.fetch.firecrawl.enabled is false.", + }); + firecrawlResolution = { + source: "secretRef", + secretRefConfigured: true, + fallbackUsedAfterRefFailure: false, + }; + } else { + const configuredInlineValue = normalizeSecretInput(firecrawl?.apiKey); + if (configuredInlineValue) { + firecrawlResolution = { + value: configuredInlineValue, + source: "config", + secretRefConfigured: false, + fallbackUsedAfterRefFailure: false, + }; + } else { + const envFallback = readNonEmptyEnvValue(params.context.env, ["FIRECRAWL_API_KEY"]); + if (envFallback.value) { + firecrawlResolution = { + value: envFallback.value, + source: "env", + fallbackEnvVar: envFallback.envVar, + secretRefConfigured: false, + fallbackUsedAfterRefFailure: false, + }; + } + } + } + } + + return { + search: searchMetadata, + fetch: { + firecrawl: { + active: firecrawlActive, + apiKeySource: firecrawlResolution.source, + diagnostics: firecrawlDiagnostics, + }, + }, + diagnostics, + }; +} diff --git a/src/secrets/runtime.test.ts b/src/secrets/runtime.test.ts index 463914bf899..f03ce73da3e 100644 --- a/src/secrets/runtime.test.ts +++ b/src/secrets/runtime.test.ts @@ -8,6 +8,7 @@ import { withTempHome } from "../config/home-env.test-harness.js"; import { activateSecretsRuntimeSnapshot, clearSecretsRuntimeSnapshot, + getActiveRuntimeWebToolsMetadata, getActiveSecretsRuntimeSnapshot, prepareSecretsRuntimeSnapshot, } from "./runtime.js"; @@ -342,7 +343,7 @@ describe("secrets runtime snapshot", () => { ); }); - it("resolves provider-specific refs in web search auto mode", async () => { + it("keeps non-selected provider refs inactive in web search auto mode", async () => { const snapshot = await prepareSecretsRuntimeSnapshot({ config: asConfig({ tools: { @@ -366,9 +367,19 @@ describe("secrets runtime snapshot", () => { }); expect(snapshot.config.tools?.web?.search?.apiKey).toBe("web-search-ref"); - expect(snapshot.config.tools?.web?.search?.gemini?.apiKey).toBe("web-search-gemini-ref"); - expect(snapshot.warnings.map((warning) => warning.path)).not.toContain( - "tools.web.search.gemini.apiKey", + expect(snapshot.config.tools?.web?.search?.gemini?.apiKey).toEqual({ + source: "env", + provider: "default", + id: "WEB_SEARCH_GEMINI_API_KEY", + }); + expect(snapshot.webTools.search.selectedProvider).toBe("brave"); + expect(snapshot.warnings).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + code: "SECRETS_REF_IGNORED_INACTIVE_SURFACE", + path: "tools.web.search.gemini.apiKey", + }), + ]), ); }); @@ -401,6 +412,71 @@ describe("secrets runtime snapshot", () => { ); }); + it("fails fast at startup when selected web search provider ref is unresolved", async () => { + await expect( + prepareSecretsRuntimeSnapshot({ + config: asConfig({ + tools: { + web: { + search: { + enabled: true, + provider: "gemini", + gemini: { + apiKey: { + source: "env", + provider: "default", + id: "MISSING_WEB_SEARCH_GEMINI_API_KEY", + }, + }, + }, + }, + }, + }), + env: {}, + agentDirs: ["/tmp/openclaw-agent-main"], + loadAuthStore: () => ({ version: 1, profiles: {} }), + }), + ).rejects.toThrow("[WEB_SEARCH_KEY_UNRESOLVED_NO_FALLBACK]"); + }); + + it("exposes active runtime web tool metadata as a defensive clone", async () => { + const snapshot = await prepareSecretsRuntimeSnapshot({ + config: asConfig({ + tools: { + web: { + search: { + provider: "gemini", + gemini: { + apiKey: { source: "env", provider: "default", id: "WEB_SEARCH_GEMINI_API_KEY" }, + }, + }, + }, + }, + }), + env: { + WEB_SEARCH_GEMINI_API_KEY: "web-search-gemini-ref", // pragma: allowlist secret + }, + agentDirs: ["/tmp/openclaw-agent-main"], + loadAuthStore: () => ({ version: 1, profiles: {} }), + }); + + activateSecretsRuntimeSnapshot(snapshot); + + const first = getActiveRuntimeWebToolsMetadata(); + expect(first?.search.providerConfigured).toBe("gemini"); + expect(first?.search.selectedProvider).toBe("gemini"); + expect(first?.search.selectedProviderKeySource).toBe("secretRef"); + if (!first) { + throw new Error("missing runtime web tools metadata"); + } + first.search.providerConfigured = "brave"; + first.search.selectedProvider = "brave"; + + const second = getActiveRuntimeWebToolsMetadata(); + expect(second?.search.providerConfigured).toBe("gemini"); + expect(second?.search.selectedProvider).toBe("gemini"); + }); + it("resolves file refs via configured file provider", async () => { if (process.platform === "win32") { return; @@ -615,7 +691,7 @@ describe("secrets runtime snapshot", () => { }); }); - it("clears active secrets runtime state and throws when refresh fails after a write", async () => { + it("keeps last-known-good runtime snapshot active when refresh fails after a write", async () => { if (os.platform() === "win32") { return; } @@ -704,9 +780,11 @@ describe("secrets runtime snapshot", () => { /runtime snapshot refresh failed: simulated secrets runtime refresh failure/i, ); - expect(getActiveSecretsRuntimeSnapshot()).toBeNull(); - expect(loadConfig().gateway?.auth).toEqual({ mode: "token" }); - expect(loadConfig().models?.providers?.openai?.apiKey).toEqual({ + const activeAfterFailure = getActiveSecretsRuntimeSnapshot(); + expect(activeAfterFailure).not.toBeNull(); + expect(loadConfig().gateway?.auth).toBeUndefined(); + expect(loadConfig().models?.providers?.openai?.apiKey).toBe("sk-file-runtime"); + expect(activeAfterFailure?.sourceConfig.models?.providers?.openai?.apiKey).toEqual({ source: "file", provider: "default", id: "/providers/openai/apiKey", @@ -715,9 +793,75 @@ describe("secrets runtime snapshot", () => { const persistedStore = ensureAuthProfileStore(agentDir).profiles["openai:default"]; expect(persistedStore).toMatchObject({ type: "api_key", - keyRef: { source: "file", provider: "default", id: "/providers/openai/apiKey" }, + key: "sk-file-runtime", + }); + }); + }); + + it("keeps last-known-good web runtime snapshot when reload introduces unresolved active web refs", async () => { + await withTempHome("openclaw-secrets-runtime-web-reload-lkg-", async (home) => { + const prepared = await prepareSecretsRuntimeSnapshot({ + config: asConfig({ + tools: { + web: { + search: { + provider: "gemini", + gemini: { + apiKey: { source: "env", provider: "default", id: "WEB_SEARCH_GEMINI_API_KEY" }, + }, + }, + }, + }, + }), + env: { + WEB_SEARCH_GEMINI_API_KEY: "web-search-gemini-runtime-key", // pragma: allowlist secret + }, + agentDirs: ["/tmp/openclaw-agent-main"], + loadAuthStore: () => ({ version: 1, profiles: {} }), + }); + + activateSecretsRuntimeSnapshot(prepared); + + await expect( + writeConfigFile({ + ...loadConfig(), + tools: { + web: { + search: { + provider: "gemini", + gemini: { + apiKey: { + source: "env", + provider: "default", + id: "MISSING_WEB_SEARCH_GEMINI_API_KEY", + }, + }, + }, + }, + }, + }), + ).rejects.toThrow( + /runtime snapshot refresh failed: .*WEB_SEARCH_KEY_UNRESOLVED_NO_FALLBACK/i, + ); + + const activeAfterFailure = getActiveSecretsRuntimeSnapshot(); + expect(activeAfterFailure).not.toBeNull(); + expect(loadConfig().tools?.web?.search?.gemini?.apiKey).toBe("web-search-gemini-runtime-key"); + expect(activeAfterFailure?.sourceConfig.tools?.web?.search?.gemini?.apiKey).toEqual({ + source: "env", + provider: "default", + id: "WEB_SEARCH_GEMINI_API_KEY", + }); + expect(getActiveRuntimeWebToolsMetadata()?.search.selectedProvider).toBe("gemini"); + + const persistedConfig = JSON.parse( + await fs.readFile(path.join(home, ".openclaw", "openclaw.json"), "utf8"), + ) as OpenClawConfig; + expect(persistedConfig.tools?.web?.search?.gemini?.apiKey).toEqual({ + source: "env", + provider: "default", + id: "MISSING_WEB_SEARCH_GEMINI_API_KEY", }); - expect("key" in persistedStore ? persistedStore.key : undefined).toBeUndefined(); }); }); diff --git a/src/secrets/runtime.ts b/src/secrets/runtime.ts index 9e69ffa60ad..903fe5a6d24 100644 --- a/src/secrets/runtime.ts +++ b/src/secrets/runtime.ts @@ -25,6 +25,7 @@ import { createResolverContext, type SecretResolverWarning, } from "./runtime-shared.js"; +import { resolveRuntimeWebTools, type RuntimeWebToolsMetadata } from "./runtime-web-tools.js"; export type { SecretResolverWarning } from "./runtime-shared.js"; @@ -33,6 +34,7 @@ export type PreparedSecretsRuntimeSnapshot = { config: OpenClawConfig; authStores: Array<{ agentDir: string; store: AuthProfileStore }>; warnings: SecretResolverWarning[]; + webTools: RuntimeWebToolsMetadata; }; type SecretsRuntimeRefreshContext = { @@ -57,6 +59,7 @@ function cloneSnapshot(snapshot: PreparedSecretsRuntimeSnapshot): PreparedSecret store: structuredClone(entry.store), })), warnings: snapshot.warnings.map((warning) => ({ ...warning })), + webTools: structuredClone(snapshot.webTools), }; } @@ -148,6 +151,11 @@ export async function prepareSecretsRuntimeSnapshot(params: { config: resolvedConfig, authStores, warnings: context.warnings, + webTools: await resolveRuntimeWebTools({ + sourceConfig, + resolvedConfig, + context, + }), }; preparedSnapshotRefreshContext.set(snapshot, { env: { ...(params.env ?? process.env) } as Record, @@ -185,7 +193,6 @@ export function activateSecretsRuntimeSnapshot(snapshot: PreparedSecretsRuntimeS activateSecretsRuntimeSnapshot(refreshed); return true; }, - clearOnRefreshFailure: clearActiveSecretsRuntimeState, }); } @@ -200,6 +207,13 @@ export function getActiveSecretsRuntimeSnapshot(): PreparedSecretsRuntimeSnapsho return snapshot; } +export function getActiveRuntimeWebToolsMetadata(): RuntimeWebToolsMetadata | null { + if (!activeSnapshot) { + return null; + } + return structuredClone(activeSnapshot.webTools); +} + export function resolveCommandSecretsFromActiveRuntimeSnapshot(params: { commandName: string; targetIds: ReadonlySet; diff --git a/src/secrets/target-registry-data.ts b/src/secrets/target-registry-data.ts index 3be4992d28f..f085c9981ab 100644 --- a/src/secrets/target-registry-data.ts +++ b/src/secrets/target-registry-data.ts @@ -689,6 +689,17 @@ const SECRET_TARGET_REGISTRY: SecretTargetRegistryEntry[] = [ includeInConfigure: true, includeInAudit: true, }, + { + id: "tools.web.fetch.firecrawl.apiKey", + targetType: "tools.web.fetch.firecrawl.apiKey", + configFile: "openclaw.json", + pathPattern: "tools.web.fetch.firecrawl.apiKey", + secretShape: SECRET_INPUT_SHAPE, + expectedResolvedValue: "string", + includeInPlan: true, + includeInConfigure: true, + includeInAudit: true, + }, { id: "tools.web.search.apiKey", targetType: "tools.web.search.apiKey",