diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ba8d8f34a4..f950cd953a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -121,7 +121,7 @@ Docs: https://docs.openclaw.ai - CLI/configure: let model-only section setup enter provider auth directly instead of first asking where the Gateway runs, unblocking OAuth/token setup in terminals where that unrelated prompt is unresponsive. Fixes #39223. Thanks @LevityLeads. - Providers/Anthropic-messages: extract `reasoning_content` from `thinking` blocks during assistant replay so proxy providers that route through the Anthropic-messages transport preserve reasoning context across tool-call follow-up turns. Thanks @Sunnyone2three. - Agents/GitHub Copilot: normalize replayed Responses tool-call IDs before dispatch so resumed sessions with historical overlong tool IDs continue instead of failing Copilot schema validation. (#82750) Thanks @galiniliev. -- CLI/web: resolve provider-scoped web search/fetch SecretRefs for `infer web ... --provider ...` while leaving unrelated plugin secrets untouched. Fixes #82621. Thanks @leno23. +- CLI/infer: resolve plugin-scoped web search and fetch SecretRefs on the exact command credential surface, keeping non-selected and unrelated plugin secrets inactive. Fixes #82621. (#82699) Thanks @leno23. - Providers/Anthropic Vertex: resolve installed provider public surfaces from package-local `dist/`, restoring `anthropic-vertex/*` model calls after plugin externalization. Fixes #82781. Thanks @0L1v3DaD. - Gateway/exec approvals: bind path-shaped allowlists, safe-bin trust, skill auto-allow, Allow Always persistence, and approval audit metadata to the executable realpath so symlinked binaries cannot keep approvals after retargeting. Fixes #45595. Thanks @jasonftl. - Mac app: reorganize Settings around a grouped sidebar, with separate Connection and Exec Approvals pages so everyday permissions and app toggles are easier to scan. diff --git a/docs/reference/secretref-credential-surface.md b/docs/reference/secretref-credential-surface.md index 1e7b27e613b..96ca1c1bd51 100644 --- a/docs/reference/secretref-credential-surface.md +++ b/docs/reference/secretref-credential-surface.md @@ -54,6 +54,7 @@ Scope intent: - `plugins.entries.voice-call.config.streaming.providers.*.apiKey` - `plugins.entries.voice-call.config.tts.providers.*.apiKey` - `plugins.entries.voice-call.config.twilio.authToken` +- `tools.web.search.*.apiKey` - `tools.web.search.apiKey` - `gateway.auth.password` - `gateway.auth.token` diff --git a/docs/reference/secretref-user-supplied-credentials-matrix.json b/docs/reference/secretref-user-supplied-credentials-matrix.json index 33aa6f1c05e..15a1529a346 100644 --- a/docs/reference/secretref-user-supplied-credentials-matrix.json +++ b/docs/reference/secretref-user-supplied-credentials-matrix.json @@ -645,6 +645,13 @@ "secretShape": "secret_input", "optIn": true }, + { + "id": "tools.web.search.*.apiKey", + "configFile": "openclaw.json", + "path": "tools.web.search.*.apiKey", + "secretShape": "secret_input", + "optIn": true + }, { "id": "tools.web.search.apiKey", "configFile": "openclaw.json", diff --git a/extensions/firecrawl/src/firecrawl-fetch-provider-shared.ts b/extensions/firecrawl/src/firecrawl-fetch-provider-shared.ts index 120d463d9ec..a76c4002e17 100644 --- a/extensions/firecrawl/src/firecrawl-fetch-provider-shared.ts +++ b/extensions/firecrawl/src/firecrawl-fetch-provider-shared.ts @@ -44,6 +44,19 @@ export const FIRECRAWL_WEB_FETCH_PROVIDER_SHARED = { getConfiguredCredentialValue: (config) => (config?.plugins?.entries?.firecrawl?.config as { webFetch?: { apiKey?: unknown } } | undefined) ?.webFetch?.apiKey, + getConfiguredCredentialFallback: (config) => { + const apiKey = ( + config?.plugins?.entries?.firecrawl?.config as + | { webSearch?: { apiKey?: unknown } } + | undefined + )?.webSearch?.apiKey; + return apiKey === undefined + ? undefined + : { + path: "plugins.entries.firecrawl.config.webSearch.apiKey", + value: apiKey, + }; + }, setConfiguredCredentialValue: (configTarget, value) => { const plugins = ensureRecord(configTarget as unknown as Record, "plugins"); const entries = ensureRecord(plugins, "entries"); diff --git a/extensions/firecrawl/src/firecrawl-search-provider.ts b/extensions/firecrawl/src/firecrawl-search-provider.ts index 973d8074f82..0207d3dee91 100644 --- a/extensions/firecrawl/src/firecrawl-search-provider.ts +++ b/extensions/firecrawl/src/firecrawl-search-provider.ts @@ -4,6 +4,7 @@ import { } from "openclaw/plugin-sdk/provider-web-search-contract"; const FIRECRAWL_CREDENTIAL_PATH = "plugins.entries.firecrawl.config.webSearch.apiKey"; +const FIRECRAWL_FETCH_CREDENTIAL_PATH = "plugins.entries.firecrawl.config.webFetch.apiKey"; type FirecrawlClientModule = typeof import("./firecrawl-client.js"); @@ -14,6 +15,20 @@ function loadFirecrawlClientModule(): Promise { return firecrawlClientModulePromise; } +function getConfiguredFetchCredentialFallback(config?: { + plugins?: { entries?: { firecrawl?: { config?: unknown } } }; +}) { + const apiKey = ( + config?.plugins?.entries?.firecrawl?.config as { webFetch?: { apiKey?: unknown } } | undefined + )?.webFetch?.apiKey; + return apiKey === undefined + ? undefined + : { + path: FIRECRAWL_FETCH_CREDENTIAL_PATH, + value: apiKey, + }; +} + const GenericFirecrawlSearchSchema = { type: "object", properties: { @@ -47,6 +62,7 @@ export function createFirecrawlWebSearchProvider(): WebSearchProviderPlugin { configuredCredential: { pluginId: "firecrawl" }, selectionPluginId: "firecrawl", }), + getConfiguredCredentialFallback: getConfiguredFetchCredentialFallback, createTool: (ctx) => ({ description: "Search the web using Firecrawl. Returns structured results with snippets from Firecrawl Search. Use firecrawl_search for Firecrawl-specific knobs like sources or categories.", diff --git a/extensions/firecrawl/src/firecrawl-tools.test.ts b/extensions/firecrawl/src/firecrawl-tools.test.ts index 8765c0068cd..53d8b17b531 100644 --- a/extensions/firecrawl/src/firecrawl-tools.test.ts +++ b/extensions/firecrawl/src/firecrawl-tools.test.ts @@ -86,6 +86,24 @@ describe("firecrawl tools", () => { expect(provider.id).toBe("firecrawl"); expect(provider.credentialPath).toBe("plugins.entries.firecrawl.config.webSearch.apiKey"); + expect( + provider.getConfiguredCredentialFallback?.({ + plugins: { + entries: { + firecrawl: { + config: { + webFetch: { + apiKey: { source: "env", provider: "default", id: "FIRECRAWL_API_KEY" }, + }, + }, + }, + }, + }, + } as never), + ).toEqual({ + path: "plugins.entries.firecrawl.config.webFetch.apiKey", + value: { source: "env", provider: "default", id: "FIRECRAWL_API_KEY" }, + }); const pluginEntry = applied.plugins?.entries?.firecrawl; if (!pluginEntry) { throw new Error("expected Firecrawl plugin entry"); diff --git a/extensions/firecrawl/web-search-contract-api.ts b/extensions/firecrawl/web-search-contract-api.ts index 6d2271f04e7..9f5390745f2 100644 --- a/extensions/firecrawl/web-search-contract-api.ts +++ b/extensions/firecrawl/web-search-contract-api.ts @@ -5,6 +5,7 @@ import { export function createFirecrawlWebSearchProvider(): WebSearchProviderPlugin { const credentialPath = "plugins.entries.firecrawl.config.webSearch.apiKey"; + const fetchCredentialPath = "plugins.entries.firecrawl.config.webFetch.apiKey"; return { id: "firecrawl", @@ -24,6 +25,19 @@ export function createFirecrawlWebSearchProvider(): WebSearchProviderPlugin { configuredCredential: { pluginId: "firecrawl" }, selectionPluginId: "firecrawl", }), + getConfiguredCredentialFallback: (config) => { + const apiKey = ( + config?.plugins?.entries?.firecrawl?.config as + | { webFetch?: { apiKey?: unknown } } + | undefined + )?.webFetch?.apiKey; + return apiKey === undefined + ? undefined + : { + path: fetchCredentialPath, + value: apiKey, + }; + }, createTool: () => null, }; } diff --git a/scripts/repro/cli-web-search-secret-refs-live-proof.mjs b/scripts/repro/cli-web-search-secret-refs-live-proof.mjs index 1ac4891067b..a8d1086fb90 100644 --- a/scripts/repro/cli-web-search-secret-refs-live-proof.mjs +++ b/scripts/repro/cli-web-search-secret-refs-live-proof.mjs @@ -4,7 +4,7 @@ * Run: TAVILY_API_KEY=resolved-live-proof pnpm exec tsx scripts/repro/cli-web-search-secret-refs-live-proof.mjs */ import { resolveCommandConfigWithSecrets } from "../../src/cli/command-config-resolution.js"; -import { getWebSearchCommandSecretTargetIds } from "../../src/cli/command-secret-targets.js"; +import { getCapabilityWebSearchCommandSecretTargetIds } from "../../src/cli/command-secret-targets.js"; const unresolvedConfig = { tools: { web: { search: { provider: "tavily", enabled: true } } }, @@ -26,7 +26,7 @@ process.env.TAVILY_API_KEY = process.env.TAVILY_API_KEY ?? "resolved-live-proof" const { effectiveConfig, diagnostics } = await resolveCommandConfigWithSecrets({ config: unresolvedConfig, commandName: "infer web search", - targetIds: getWebSearchCommandSecretTargetIds(), + targetIds: getCapabilityWebSearchCommandSecretTargetIds(), autoEnable: true, }); @@ -38,5 +38,8 @@ console.log( "resolveCommandConfigWithSecrets apiKey is string =", typeof apiKey === "string" && apiKey.length > 0, ); -console.log("resolved apiKey remains redacted =", typeof apiKey === "string"); +console.log( + "resolved apiKey prefix =", + typeof apiKey === "string" ? `${apiKey.slice(0, 8)}…` : apiKey, +); console.log("diagnostics count =", diagnostics.length); diff --git a/src/cli/capability-cli.test.ts b/src/cli/capability-cli.test.ts index a574d8b588a..5a4e8b85e69 100644 --- a/src/cli/capability-cli.test.ts +++ b/src/cli/capability-cli.test.ts @@ -20,8 +20,6 @@ const mocks = vi.hoisted(() => ({ writeStdout: vi.fn(), }, loadConfig: vi.fn(() => ({})), - getRuntimeConfigSourceSnapshot: vi.fn(() => null), - setRuntimeConfigSnapshot: vi.fn(), loadAuthProfileStoreForRuntime: vi.fn(() => ({ profiles: {}, order: {} })), listProfilesForProvider: vi.fn(() => []), updateAuthProfileStoreWithLock: vi.fn( @@ -135,23 +133,13 @@ const mocks = vi.hoisted(() => ({ convertHeicToJpeg: vi.fn(async () => Buffer.from("jpeg-normalized")), isWebSearchProviderConfigured: vi.fn(() => false), isWebFetchProviderConfigured: vi.fn(() => false), - resolveCommandConfigWithSecrets: vi.fn(async ({ config }: { config: unknown }) => ({ - resolvedConfig: config, - effectiveConfig: config, - diagnostics: [], - })), - getAgentRuntimeCommandSecretTargetIds: vi.fn(() => new Set(["agent-runtime-target"])), - getMemoryEmbeddingCommandSecretTargetIds: vi.fn(() => new Set(["memory-target"])), - getModelsCommandSecretTargetIds: vi.fn(() => new Set(["model-target"])), - getTtsCommandSecretTargetIds: vi.fn(() => new Set(["tts-target"])), - getWebFetchCommandSecretTargets: vi.fn(() => ({ - targetIds: new Set(["web-fetch-target"]), - allowedPaths: new Set(["plugins.entries.firecrawl.config.webFetch.apiKey"]), - })), - getWebSearchCommandSecretTargets: vi.fn(() => ({ - targetIds: new Set(["web-search-target"]), - allowedPaths: new Set(["plugins.entries.tavily.config.webSearch.apiKey"]), - })), + resolveCommandConfigWithSecrets: vi.fn( + async ({ config }: { config: Record }) => ({ + resolvedConfig: config, + effectiveConfig: config, + diagnostics: [], + }), + ), modelsStatusCommand: vi.fn( async (_opts: unknown, runtime: { log: (...args: unknown[]) => void }) => { runtime.log(JSON.stringify({ ok: true, providers: [{ id: "openai" }] })); @@ -166,12 +154,8 @@ vi.mock("../runtime.js", () => ({ })); vi.mock("../config/config.js", () => ({ - getRuntimeConfigSourceSnapshot: - mocks.getRuntimeConfigSourceSnapshot as typeof import("../config/config.js").getRuntimeConfigSourceSnapshot, getRuntimeConfig: mocks.loadConfig as typeof import("../config/config.js").getRuntimeConfig, loadConfig: mocks.loadConfig as typeof import("../config/config.js").loadConfig, - setRuntimeConfigSnapshot: - mocks.setRuntimeConfigSnapshot as typeof import("../config/config.js").setRuntimeConfigSnapshot, })); vi.mock("./command-config-resolution.js", () => ({ @@ -319,24 +303,81 @@ vi.mock("../web-fetch/runtime.js", () => ({ resolveWebFetchDefinition: vi.fn(), })); -vi.mock("./command-config-resolution.js", () => ({ - resolveCommandConfigWithSecrets: - mocks.resolveCommandConfigWithSecrets as typeof import("./command-config-resolution.js").resolveCommandConfigWithSecrets, +vi.mock("../plugins/web-fetch-providers.runtime.js", () => ({ + resolvePluginWebFetchProviders: vi.fn((params: { config?: Record }) => [ + { + pluginId: "firecrawl", + id: "firecrawl", + credentialPath: "plugins.entries.firecrawl.config.webFetch.apiKey", + getConfiguredCredentialValue: (config?: { + plugins?: { + entries?: { + firecrawl?: { config?: { webFetch?: { apiKey?: unknown } } }; + }; + }; + }) => config?.plugins?.entries?.firecrawl?.config?.webFetch?.apiKey, + getConfiguredCredentialFallback: () => ({ + path: "plugins.entries.firecrawl.config.webSearch.apiKey", + value: ( + params.config as { + plugins?: { + entries?: { + firecrawl?: { config?: { webSearch?: { apiKey?: unknown } } }; + }; + }; + } + )?.plugins?.entries?.firecrawl?.config?.webSearch?.apiKey, + }), + getCredentialValue: (): undefined => undefined, + }, + ]), })); -vi.mock("./command-secret-targets.js", () => ({ - getAgentRuntimeCommandSecretTargetIds: - mocks.getAgentRuntimeCommandSecretTargetIds as typeof import("./command-secret-targets.js").getAgentRuntimeCommandSecretTargetIds, - getMemoryEmbeddingCommandSecretTargetIds: - mocks.getMemoryEmbeddingCommandSecretTargetIds as typeof import("./command-secret-targets.js").getMemoryEmbeddingCommandSecretTargetIds, - getModelsCommandSecretTargetIds: - mocks.getModelsCommandSecretTargetIds as typeof import("./command-secret-targets.js").getModelsCommandSecretTargetIds, - getTtsCommandSecretTargetIds: - mocks.getTtsCommandSecretTargetIds as typeof import("./command-secret-targets.js").getTtsCommandSecretTargetIds, - getWebFetchCommandSecretTargets: - mocks.getWebFetchCommandSecretTargets as typeof import("./command-secret-targets.js").getWebFetchCommandSecretTargets, - getWebSearchCommandSecretTargets: - mocks.getWebSearchCommandSecretTargets as typeof import("./command-secret-targets.js").getWebSearchCommandSecretTargets, +vi.mock("../plugins/web-search-providers.runtime.js", () => ({ + resolvePluginWebSearchProviders: vi.fn(() => [ + { + pluginId: "tavily", + id: "tavily", + credentialPath: "plugins.entries.tavily.config.webSearch.apiKey", + getConfiguredCredentialValue: (config?: { + plugins?: { + entries?: { + tavily?: { config?: { webSearch?: { apiKey?: unknown } } }; + }; + }; + }) => config?.plugins?.entries?.tavily?.config?.webSearch?.apiKey, + getConfiguredCredentialFallback: (): undefined => undefined, + getCredentialValue: (): undefined => undefined, + }, + { + pluginId: "firecrawl", + id: "firecrawl", + credentialPath: "plugins.entries.firecrawl.config.webSearch.apiKey", + getConfiguredCredentialValue: (config?: { + plugins?: { + entries?: { + firecrawl?: { config?: { webSearch?: { apiKey?: unknown } } }; + }; + }; + }) => config?.plugins?.entries?.firecrawl?.config?.webSearch?.apiKey, + getConfiguredCredentialFallback: (): undefined => undefined, + getCredentialValue: (): undefined => undefined, + }, + { + pluginId: "exa", + id: "exa", + credentialPath: "plugins.entries.exa.config.webSearch.apiKey", + getConfiguredCredentialValue: (config?: { + plugins?: { + entries?: { + exa?: { config?: { webSearch?: { apiKey?: unknown } } }; + }; + }; + }) => config?.plugins?.entries?.exa?.config?.webSearch?.apiKey, + getConfiguredCredentialFallback: (): undefined => undefined, + getCredentialValue: (): undefined => undefined, + }, + ]), })); describe("capability cli", () => { @@ -349,8 +390,6 @@ describe("capability cli", () => { mocks.runtime.log.mockClear(); mocks.runtime.error.mockClear(); mocks.runtime.writeJson.mockClear(); - mocks.getRuntimeConfigSourceSnapshot.mockReset().mockReturnValue(null); - mocks.setRuntimeConfigSnapshot.mockClear(); mocks.loadModelCatalog .mockReset() .mockResolvedValue([{ id: "gpt-5.4", provider: "openai", name: "GPT-5.4" }] as never); @@ -401,29 +440,6 @@ describe("capability cli", () => { mocks.registerBuiltInMemoryEmbeddingProviders.mockClear(); mocks.isWebSearchProviderConfigured.mockReset().mockReturnValue(false); mocks.isWebFetchProviderConfigured.mockReset().mockReturnValue(false); - mocks.resolveCommandConfigWithSecrets - .mockReset() - .mockImplementation(async ({ config }: { config: unknown }) => ({ - resolvedConfig: config, - effectiveConfig: config, - diagnostics: [], - })); - mocks.getAgentRuntimeCommandSecretTargetIds - .mockReset() - .mockReturnValue(new Set(["agent-runtime-target"])); - mocks.getMemoryEmbeddingCommandSecretTargetIds - .mockReset() - .mockReturnValue(new Set(["memory-target"])); - mocks.getModelsCommandSecretTargetIds.mockReset().mockReturnValue(new Set(["model-target"])); - mocks.getTtsCommandSecretTargetIds.mockReset().mockReturnValue(new Set(["tts-target"])); - mocks.getWebFetchCommandSecretTargets.mockReset().mockReturnValue({ - targetIds: new Set(["web-fetch-target"]), - allowedPaths: new Set(["plugins.entries.firecrawl.config.webFetch.apiKey"]), - }); - mocks.getWebSearchCommandSecretTargets.mockReset().mockReturnValue({ - targetIds: new Set(["web-search-target"]), - allowedPaths: new Set(["plugins.entries.tavily.config.webSearch.apiKey"]), - }); mocks.modelsStatusCommand.mockClear(); mocks.callGateway.mockImplementation((async ({ method }: { method: string }) => { if (method === "tts.status") { @@ -482,7 +498,6 @@ describe("capability cli", () => { }; type ImageDescribeParams = { filePath?: string; - mediaUrl?: string; model?: unknown; prompt?: unknown; provider?: unknown; @@ -561,13 +576,6 @@ describe("capability cli", () => { return calls[0]?.[0]; } - function firstCommandConfigResolutionCall() { - const calls = mocks.resolveCommandConfigWithSecrets.mock.calls as unknown as Array< - [Record] - >; - return calls[0]?.[0]; - } - function expectModelRunDispatch(transport: "local" | "gateway", modelRef: string) { if (transport === "gateway") { const slash = modelRef.indexOf("/"); @@ -1196,26 +1204,6 @@ describe("capability cli", () => { expect(describeCall?.timeoutMs).toBe(90000); }); - it("keeps image describe URL files as remote media references", async () => { - await runRegisteredCli({ - register: registerCapabilityCli as (program: Command) => void, - argv: [ - "capability", - "image", - "describe", - "--file", - "https://example.com/photo.png", - "--json", - ], - }); - - const describeCall = imageDescribeCall(); - expect(describeCall?.filePath).toBe("https://example.com/photo.png"); - expect(describeCall?.mediaUrl).toBe("https://example.com/photo.png"); - const outputs = firstJsonOutput()?.outputs as Array>; - expect(outputs[0]?.path).toBe("https://example.com/photo.png"); - }); - it("uses the explicit media-understanding provider for image describe model overrides", async () => { await runRegisteredCli({ register: registerCapabilityCli as (program: Command) => void, @@ -2003,35 +1991,6 @@ describe("capability cli", () => { expect(firstJsonOutput()?.model).toBe("text-embedding-3-small"); }); - it("resolves command SecretRefs before local model capability execution", async () => { - const rawConfig = { agents: { defaults: { model: "openai/gpt-5.4" } } }; - const resolvedConfig = { agents: { defaults: { model: "openai/gpt-5.4" } }, resolved: true }; - const targetIds = new Set(["models.providers.*.apiKey"]); - mocks.loadConfig.mockReturnValue(rawConfig); - mocks.getModelsCommandSecretTargetIds.mockReturnValue(targetIds); - mocks.resolveCommandConfigWithSecrets.mockResolvedValueOnce({ - resolvedConfig, - effectiveConfig: resolvedConfig, - diagnostics: [], - } as never); - - await runRegisteredCli({ - register: registerCapabilityCli as (program: Command) => void, - argv: ["capability", "model", "run", "--prompt", "hello", "--json"], - }); - - expect(firstCommandConfigResolutionCall()).toEqual( - expect.objectContaining({ - config: rawConfig, - commandName: "infer model run", - targetIds, - runtime: mocks.runtime, - }), - ); - expect(firstPreparedModelParams()?.cfg).toBe(resolvedConfig); - expect(mocks.setRuntimeConfigSnapshot).toHaveBeenCalledWith(resolvedConfig); - }); - it("derives the embedding provider from a provider/model override", async () => { await runRegisteredCli({ register: registerCapabilityCli as (program: Command) => void, @@ -2307,9 +2266,13 @@ describe("capability cli", () => { argv: ["infer", "web", "search", "--query", "ping", "--json"], }); + const { getCapabilityWebSearchCommandSecretTargets } = + await import("./command-secret-targets.js"); + const scopedTargets = getCapabilityWebSearchCommandSecretTargets(unresolvedConfig as never); expect(mocks.resolveCommandConfigWithSecrets).toHaveBeenCalledWith( expect.objectContaining({ commandName: "infer web search", + targetIds: scopedTargets.targetIds, }), ); expect(webSearchRuntime.runWebSearch).toHaveBeenCalledWith( @@ -2319,6 +2282,229 @@ describe("capability cli", () => { ); }); + it("uses the infer web search provider override when resolving SecretRefs", async () => { + const unresolvedConfig = { + tools: { web: { search: { provider: "exa", enabled: true } } }, + plugins: { + entries: { + firecrawl: { + config: { + webSearch: { + apiKey: { source: "env", provider: "default", id: "FIRECRAWL_API_KEY" }, + }, + }, + }, + exa: { + config: { + webSearch: { + apiKey: { source: "env", provider: "default", id: "EXA_API_KEY" }, + }, + }, + }, + }, + }, + }; + const resolvedConfig = { + ...unresolvedConfig, + plugins: { + entries: { + ...unresolvedConfig.plugins.entries, + firecrawl: { + config: { + webSearch: { + apiKey: "resolved-firecrawl-key", + }, + }, + }, + }, + }, + }; + mocks.loadConfig.mockReturnValue(unresolvedConfig); + mocks.resolveCommandConfigWithSecrets.mockResolvedValueOnce({ + resolvedConfig, + effectiveConfig: resolvedConfig, + diagnostics: [], + }); + const webSearchRuntime = await import("../web-search/runtime.js"); + vi.mocked(webSearchRuntime.runWebSearch).mockResolvedValueOnce({ + provider: "firecrawl", + result: { results: [] }, + } as never); + + await runRegisteredCli({ + register: registerCapabilityCli as (program: Command) => void, + argv: ["infer", "web", "search", "--query", "ping", "--provider", "firecrawl", "--json"], + }); + + const { getCapabilityWebSearchCommandSecretTargets } = + await import("./command-secret-targets.js"); + const scopedTargets = getCapabilityWebSearchCommandSecretTargets(unresolvedConfig as never, { + providerId: "firecrawl", + }); + const configResolutionCall = mocks.resolveCommandConfigWithSecrets.mock.calls.at(-1)?.[0]; + expect(configResolutionCall).toEqual( + expect.objectContaining({ + commandName: "infer web search", + targetIds: scopedTargets.targetIds, + forcedActivePaths: scopedTargets.forcedActivePaths, + }), + ); + expect(configResolutionCall).not.toHaveProperty("allowedPaths"); + expect(webSearchRuntime.runWebSearch).toHaveBeenCalledWith( + expect.objectContaining({ + config: resolvedConfig, + providerId: "firecrawl", + }), + ); + }); + + it("resolves only plugin web fetch SecretRefs before running infer web fetch", async () => { + const unresolvedConfig = { + tools: { web: { fetch: { provider: "firecrawl", enabled: true } } }, + plugins: { + entries: { + exa: { + config: { + webSearch: { + apiKey: { source: "env", provider: "default", id: "EXA_API_KEY" }, + }, + }, + }, + firecrawl: { + config: { + webFetch: { + apiKey: { source: "env", provider: "default", id: "FIRECRAWL_API_KEY" }, + }, + }, + }, + }, + }, + }; + const resolvedConfig = { + ...unresolvedConfig, + plugins: { + entries: { + ...unresolvedConfig.plugins.entries, + firecrawl: { + config: { + webFetch: { + apiKey: "resolved-firecrawl-key", + }, + }, + }, + }, + }, + }; + mocks.loadConfig.mockReturnValue(unresolvedConfig); + mocks.resolveCommandConfigWithSecrets.mockResolvedValueOnce({ + resolvedConfig, + effectiveConfig: resolvedConfig, + diagnostics: [], + }); + const webFetchRuntime = await import("../web-fetch/runtime.js"); + vi.mocked(webFetchRuntime.resolveWebFetchDefinition).mockReturnValueOnce({ + provider: { id: "firecrawl" }, + definition: { execute: vi.fn(async () => ({ content: "ok" })) }, + } as never); + + await runRegisteredCli({ + register: registerCapabilityCli as (program: Command) => void, + argv: ["infer", "web", "fetch", "--url", "https://example.com", "--json"], + }); + + const { getCapabilityWebFetchCommandSecretTargets } = + await import("./command-secret-targets.js"); + expect(mocks.resolveCommandConfigWithSecrets).toHaveBeenCalledWith( + expect.objectContaining({ + commandName: "infer web fetch", + targetIds: getCapabilityWebFetchCommandSecretTargets(unresolvedConfig as never).targetIds, + }), + ); + expect(webFetchRuntime.resolveWebFetchDefinition).toHaveBeenCalledWith( + expect.objectContaining({ + config: resolvedConfig, + }), + ); + }); + + it("uses the infer web fetch provider override when resolving fallback SecretRefs", async () => { + const fallbackRef = { source: "env", provider: "default", id: "FIRECRAWL_API_KEY" }; + const unresolvedConfig = { + tools: { web: { fetch: { enabled: true } } }, + plugins: { + entries: { + firecrawl: { + config: { + webSearch: { + apiKey: fallbackRef, + }, + }, + }, + }, + }, + }; + const resolvedConfig = { + ...unresolvedConfig, + plugins: { + entries: { + firecrawl: { + config: { + webSearch: { + apiKey: "resolved-firecrawl-key", + }, + }, + }, + }, + }, + }; + mocks.loadConfig.mockReturnValue(unresolvedConfig); + mocks.resolveCommandConfigWithSecrets.mockResolvedValueOnce({ + resolvedConfig, + effectiveConfig: resolvedConfig, + diagnostics: [], + }); + const webFetchRuntime = await import("../web-fetch/runtime.js"); + vi.mocked(webFetchRuntime.resolveWebFetchDefinition).mockReturnValueOnce({ + provider: { id: "firecrawl" }, + definition: { execute: vi.fn(async () => ({ content: "ok" })) }, + } as never); + + await runRegisteredCli({ + register: registerCapabilityCli as (program: Command) => void, + argv: [ + "infer", + "web", + "fetch", + "--url", + "https://example.com", + "--provider", + "firecrawl", + "--json", + ], + }); + + const { getCapabilityWebFetchCommandSecretTargets } = + await import("./command-secret-targets.js"); + const scopedTargets = getCapabilityWebFetchCommandSecretTargets(unresolvedConfig as never, { + providerId: "firecrawl", + }); + const configResolutionCall = mocks.resolveCommandConfigWithSecrets.mock.calls.at(-1)?.[0]; + expect(configResolutionCall).toEqual( + expect.objectContaining({ + commandName: "infer web fetch", + targetIds: scopedTargets.targetIds, + forcedActivePaths: scopedTargets.forcedActivePaths, + }), + ); + expect(configResolutionCall).not.toHaveProperty("allowedPaths"); + expect(webFetchRuntime.resolveWebFetchDefinition).toHaveBeenCalledWith( + expect.objectContaining({ + config: resolvedConfig, + providerId: "firecrawl", + }), + ); + }); + it("surfaces available, configured, and selected for web providers", async () => { mocks.loadConfig.mockReturnValue({ tools: { @@ -2374,161 +2560,6 @@ describe("capability cli", () => { }); }); - it("resolves command SecretRefs before local web search execution", async () => { - const rawConfig = { - tools: { web: { search: { provider: "brave" } } }, - plugins: { - entries: { - tavily: { - config: { - webSearch: { - apiKey: { source: "env", provider: "default", id: "TAVILY_API_KEY" }, - }, - }, - }, - }, - }, - }; - const resolvedConfig = { - ...rawConfig, - tools: { web: { search: { provider: "tavily" } } }, - plugins: { - entries: { - tavily: { config: { webSearch: { apiKey: "resolved-tavily-key" } } }, - }, - }, - }; - const targetIds = new Set(["plugins.entries.tavily.config.webSearch.apiKey"]); - const allowedPaths = new Set(["plugins.entries.tavily.config.webSearch.apiKey"]); - mocks.loadConfig.mockReturnValue(rawConfig); - mocks.getWebSearchCommandSecretTargets.mockReturnValue({ - targetIds, - allowedPaths, - }); - mocks.resolveCommandConfigWithSecrets.mockResolvedValueOnce({ - resolvedConfig, - effectiveConfig: resolvedConfig, - diagnostics: [], - } as never); - const webSearchRuntime = await import("../web-search/runtime.js"); - vi.mocked(webSearchRuntime.runWebSearch).mockResolvedValueOnce({ - provider: "tavily", - result: { results: [] }, - } as never); - - await runRegisteredCli({ - register: registerCapabilityCli as (program: Command) => void, - argv: ["capability", "web", "search", "--provider", "tavily", "--query", "ping", "--json"], - }); - - expect(firstCommandConfigResolutionCall()).toEqual( - expect.objectContaining({ - commandName: "infer web search", - targetIds, - allowedPaths, - providerOverrides: { webSearch: "tavily" }, - runtime: mocks.runtime, - }), - ); - expect(firstCommandConfigResolutionCall()?.config).toEqual( - expect.objectContaining({ - tools: { web: { search: { provider: "tavily" } } }, - }), - ); - expect(rawConfig.tools.web.search.provider).toBe("brave"); - expect(vi.mocked(webSearchRuntime.runWebSearch).mock.calls[0]?.[0]).toEqual( - expect.objectContaining({ - config: resolvedConfig, - preferInputConfig: true, - providerId: "tavily", - }), - ); - }); - - it("resolves command SecretRefs before local web fetch execution", async () => { - const rawConfig = { - tools: { web: { fetch: { provider: "browser" } } }, - plugins: { - entries: { - firecrawl: { - config: { - webFetch: { - apiKey: { source: "env", provider: "default", id: "FIRECRAWL_API_KEY" }, - }, - }, - }, - }, - }, - }; - const resolvedConfig = { - ...rawConfig, - tools: { web: { fetch: { provider: "firecrawl" } } }, - plugins: { - entries: { - firecrawl: { config: { webFetch: { apiKey: "resolved-firecrawl-key" } } }, - }, - }, - }; - const targetIds = new Set(["plugins.entries.firecrawl.config.webFetch.apiKey"]); - const allowedPaths = new Set(["plugins.entries.firecrawl.config.webFetch.apiKey"]); - mocks.loadConfig.mockReturnValue(rawConfig); - mocks.getWebFetchCommandSecretTargets.mockReturnValue({ - targetIds, - allowedPaths, - }); - mocks.resolveCommandConfigWithSecrets.mockResolvedValueOnce({ - resolvedConfig, - effectiveConfig: resolvedConfig, - diagnostics: [], - } as never); - const webFetchRuntime = await import("../web-fetch/runtime.js"); - const execute = vi.fn(async () => ({ text: "ok" })); - vi.mocked(webFetchRuntime.resolveWebFetchDefinition).mockReturnValueOnce({ - provider: { id: "firecrawl" }, - definition: { execute }, - } as never); - - await runRegisteredCli({ - register: registerCapabilityCli as (program: Command) => void, - argv: [ - "capability", - "web", - "fetch", - "--provider", - "firecrawl", - "--url", - "https://example.com", - "--json", - ], - }); - - expect(firstCommandConfigResolutionCall()).toEqual( - expect.objectContaining({ - commandName: "infer web fetch", - targetIds, - allowedPaths, - providerOverrides: { webFetch: "firecrawl" }, - runtime: mocks.runtime, - }), - ); - expect(firstCommandConfigResolutionCall()?.config).toEqual( - expect.objectContaining({ - tools: { web: { fetch: { provider: "firecrawl" } } }, - }), - ); - expect(rawConfig.tools.web.fetch.provider).toBe("browser"); - expect(vi.mocked(webFetchRuntime.resolveWebFetchDefinition).mock.calls[0]?.[0]).toEqual( - expect.objectContaining({ - config: resolvedConfig, - providerId: "firecrawl", - }), - ); - expect(execute).toHaveBeenCalledWith({ - url: "https://example.com", - format: undefined, - }); - }); - it("surfaces selected and configured embedding provider state", async () => { mocks.loadConfig.mockReturnValue({}); mocks.resolveMemorySearchConfig.mockReturnValue({ diff --git a/src/cli/capability-cli.ts b/src/cli/capability-cli.ts index 3c6dfc93953..46c795f9c07 100644 --- a/src/cli/capability-cli.ts +++ b/src/cli/capability-cli.ts @@ -21,11 +21,7 @@ import { prepareSimpleCompletionModelForAgent, } from "../agents/simple-completion-runtime.js"; import { normalizeThinkLevel, type ThinkLevel } from "../auto-reply/thinking.js"; -import { - getRuntimeConfig, - getRuntimeConfigSourceSnapshot, - setRuntimeConfigSnapshot, -} from "../config/config.js"; +import { getRuntimeConfig } from "../config/config.js"; import { resolveAgentModelPrimaryValue } from "../config/model-input.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; import { callGateway, randomIdempotencyKey } from "../gateway/call.js"; @@ -95,11 +91,8 @@ import { import { runCommandWithRuntime } from "./cli-utils.js"; import { resolveCommandConfigWithSecrets } from "./command-config-resolution.js"; import { - getMemoryEmbeddingCommandSecretTargetIds, - getModelsCommandSecretTargetIds, - getTtsCommandSecretTargetIds, - getWebFetchCommandSecretTargets, - getWebSearchCommandSecretTargets, + getCapabilityWebFetchCommandSecretTargets, + getCapabilityWebSearchCommandSecretTargets, } from "./command-secret-targets.js"; import { removeCommandByName } from "./program/command-tree.js"; import { collectOption } from "./program/helpers.js"; @@ -681,54 +674,6 @@ function normalizeModelRunThinking(value: unknown): ThinkLevel | undefined { return normalized; } -async function resolveLocalCapabilityRuntimeConfig(params: { - commandName: string; - targetIds: Set; - allowedPaths?: Set; - providerOverrides?: { webSearch?: string; webFetch?: string }; - config?: OpenClawConfig; -}): Promise { - const cfg = params.config ?? getRuntimeConfig(); - const sourceConfig = getRuntimeConfigSourceSnapshot(); - const { resolvedConfig } = await resolveCommandConfigWithSecrets({ - config: cfg, - commandName: params.commandName, - targetIds: params.targetIds, - ...(params.allowedPaths ? { allowedPaths: params.allowedPaths } : {}), - ...(params.providerOverrides ? { providerOverrides: params.providerOverrides } : {}), - runtime: defaultRuntime, - }); - if (sourceConfig) { - setRuntimeConfigSnapshot(resolvedConfig, sourceConfig); - } else { - setRuntimeConfigSnapshot(resolvedConfig); - } - return resolvedConfig; -} - -function withWebProviderOverride( - config: OpenClawConfig, - kind: "search" | "fetch", - provider?: string, -): OpenClawConfig { - const normalizedProvider = normalizeOptionalString(provider); - if (!normalizedProvider) { - return config; - } - const next = structuredClone(config); - const tools = (next.tools ??= {}); - const web = (tools.web ??= {}); - const existing = web[kind]; - web[kind] = - existing && typeof existing === "object" - ? { - ...existing, - provider: normalizedProvider, - } - : { provider: normalizedProvider }; - return next; -} - async function runModelRun(params: { prompt: string; files?: string[]; @@ -736,13 +681,7 @@ async function runModelRun(params: { thinking?: ThinkLevel; transport: CapabilityTransport; }) { - const cfg = - params.transport === "local" - ? await resolveLocalCapabilityRuntimeConfig({ - commandName: "infer model run", - targetIds: getModelsCommandSecretTargetIds(), - }) - : getRuntimeConfig(); + const cfg = getRuntimeConfig(); const agentId = resolveDefaultAgentId(cfg); const modelRef = await canonicalizeModelRunRef({ raw: params.model, @@ -1022,10 +961,7 @@ async function runImageGenerate(params: { output?: string; timeoutMs?: number; }) { - const cfg = await resolveLocalCapabilityRuntimeConfig({ - commandName: `infer ${params.capability}`, - targetIds: getModelsCommandSecretTargetIds(), - }); + const cfg = getRuntimeConfig(); const agentDir = resolveAgentDir(cfg, resolveDefaultAgentId(cfg)); const inputImages = params.file && params.file.length > 0 @@ -1094,10 +1030,7 @@ async function runImageDescribe(params: { prompt?: string; timeoutMs?: number; }) { - const cfg = await resolveLocalCapabilityRuntimeConfig({ - commandName: `infer ${params.capability}`, - targetIds: getModelsCommandSecretTargetIds(), - }); + const cfg = getRuntimeConfig(); const agentDir = resolveAgentDir(cfg, resolveDefaultAgentId(cfg)); const activeModel = requireProviderModelOverride(params.model); const prompt = normalizeOptionalString(params.prompt); @@ -1108,7 +1041,6 @@ async function runImageDescribe(params: { const result = activeModel ? await describeImageFileWithModel({ filePath: resolvedPath, - ...(isRemoteUrl ? { mediaUrl: resolvedPath } : {}), cfg, agentDir, provider: activeModel.provider, @@ -1118,7 +1050,6 @@ async function runImageDescribe(params: { }) : await describeImageFile({ filePath: resolvedPath, - ...(isRemoteUrl ? { mediaUrl: resolvedPath } : {}), cfg, agentDir, prompt, @@ -1167,10 +1098,7 @@ async function runAudioTranscribe(params: { model?: string; prompt?: string; }) { - const cfg = await resolveLocalCapabilityRuntimeConfig({ - commandName: "infer audio transcribe", - targetIds: getModelsCommandSecretTargetIds(), - }); + const cfg = getRuntimeConfig(); const activeModel = requireProviderModelOverride(params.model); const result = await transcribeAudioFile({ filePath: path.resolve(params.file), @@ -1265,10 +1193,7 @@ async function runVideoGenerate(params: { watermark?: boolean; timeoutMs?: number; }) { - const cfg = await resolveLocalCapabilityRuntimeConfig({ - commandName: "infer video generate", - targetIds: getModelsCommandSecretTargetIds(), - }); + const cfg = getRuntimeConfig(); const agentDir = resolveAgentDir(cfg, resolveDefaultAgentId(cfg)); const result = await generateVideo({ cfg, @@ -1343,10 +1268,7 @@ async function runVideoGenerate(params: { } async function runVideoDescribe(params: { file: string; model?: string }) { - const cfg = await resolveLocalCapabilityRuntimeConfig({ - commandName: "infer video describe", - targetIds: getModelsCommandSecretTargetIds(), - }); + const cfg = getRuntimeConfig(); const activeModel = requireProviderModelOverride(params.model); const result = await describeVideoFile({ filePath: path.resolve(params.file), @@ -1425,10 +1347,7 @@ async function runTtsConvert(params: { } satisfies CapabilityEnvelope; } - const cfg = await resolveLocalCapabilityRuntimeConfig({ - commandName: "infer tts convert", - targetIds: getTtsCommandSecretTargetIds(), - }); + const cfg = getRuntimeConfig(); const overrides = resolveExplicitTtsOverrides({ cfg, provider: params.provider, @@ -1549,10 +1468,7 @@ async function runTtsPersonas(transport: CapabilityTransport) { } async function runTtsVoices(providerRaw?: string) { - const cfg = await resolveLocalCapabilityRuntimeConfig({ - commandName: "infer tts voices", - targetIds: getTtsCommandSecretTargetIds(), - }); + const cfg = getRuntimeConfig(); const config = resolveTtsConfig(cfg); const prefsPath = resolveTtsPrefsPath(config); const provider = normalizeOptionalString(providerRaw) || getTtsProvider(config, prefsPath); @@ -1627,24 +1543,39 @@ async function runTtsStateMutation(params: { return { provider }; } -async function runWebSearchCommand(params: { query: string; provider?: string; limit?: number }) { - const rawConfig = getRuntimeConfig(); - const config = withWebProviderOverride(rawConfig, "search", params.provider); - const provider = normalizeOptionalString(params.provider); - const secretTargets = getWebSearchCommandSecretTargets({ - config, - provider, +async function resolveCapabilityCommandConfig(params: { + commandName: string; + resolveTargets: (config: OpenClawConfig) => { + targetIds: Set; + allowedPaths?: Set; + forcedActivePaths?: Set; + }; + runtime?: RuntimeEnv; +}): Promise { + const cfg = getRuntimeConfig(); + const scopedTargets = params.resolveTargets(cfg); + const { effectiveConfig } = await resolveCommandConfigWithSecrets({ + config: cfg, + commandName: params.commandName, + targetIds: scopedTargets.targetIds, + ...(scopedTargets.allowedPaths ? { allowedPaths: scopedTargets.allowedPaths } : {}), + ...(scopedTargets.forcedActivePaths + ? { forcedActivePaths: scopedTargets.forcedActivePaths } + : {}), + runtime: params.runtime, + autoEnable: true, }); - const cfg = await resolveLocalCapabilityRuntimeConfig({ + return effectiveConfig; +} + +async function runWebSearchCommand(params: { query: string; provider?: string; limit?: number }) { + const cfg = await resolveCapabilityCommandConfig({ commandName: "infer web search", - targetIds: secretTargets.targetIds, - ...(secretTargets.allowedPaths ? { allowedPaths: secretTargets.allowedPaths } : {}), - ...(provider ? { providerOverrides: { webSearch: provider } } : {}), - config, + resolveTargets: (config) => + getCapabilityWebSearchCommandSecretTargets(config, { providerId: params.provider }), }); const result = await runWebSearch({ config: cfg, - preferInputConfig: true, providerId: params.provider, args: { query: params.query, @@ -1663,19 +1594,10 @@ async function runWebSearchCommand(params: { query: string; provider?: string; l } async function runWebFetchCommand(params: { url: string; provider?: string; format?: string }) { - const rawConfig = getRuntimeConfig(); - const config = withWebProviderOverride(rawConfig, "fetch", params.provider); - const provider = normalizeOptionalString(params.provider); - const secretTargets = getWebFetchCommandSecretTargets({ - config, - provider, - }); - const cfg = await resolveLocalCapabilityRuntimeConfig({ + const cfg = await resolveCapabilityCommandConfig({ commandName: "infer web fetch", - targetIds: secretTargets.targetIds, - ...(secretTargets.allowedPaths ? { allowedPaths: secretTargets.allowedPaths } : {}), - ...(provider ? { providerOverrides: { webFetch: provider } } : {}), - config, + resolveTargets: (config) => + getCapabilityWebFetchCommandSecretTargets(config, { providerId: params.provider }), }); const resolved = resolveWebFetchDefinition({ config: cfg, @@ -1704,10 +1626,7 @@ async function runMemoryEmbeddingCreate(params: { model?: string; }) { ensureMemoryEmbeddingProvidersRegistered(); - const cfg = await resolveLocalCapabilityRuntimeConfig({ - commandName: "infer embedding create", - targetIds: getMemoryEmbeddingCommandSecretTargetIds(), - }); + const cfg = getRuntimeConfig(); const modelRef = resolveModelRefOverride(params.model); const requestedProvider = normalizeOptionalString(params.provider) || modelRef.provider || "auto"; const result = await createEmbeddingProvider({ diff --git a/src/cli/command-config-resolution.test.ts b/src/cli/command-config-resolution.test.ts index 247bf94c789..9fed2cf4550 100644 --- a/src/cli/command-config-resolution.test.ts +++ b/src/cli/command-config-resolution.test.ts @@ -80,8 +80,10 @@ describe("resolveCommandConfigWithSecrets", () => { expect(result.effectiveConfig).toBe(effectiveConfig); }); - it("passes provider overrides to command secret resolution", async () => { + it("passes scoped target paths to command secret resolution", async () => { const config = { tools: { web: { search: { provider: "tavily" } } } }; + const allowedPaths = new Set(["plugins.entries.tavily.config.webSearch.apiKey"]); + const forcedActivePaths = new Set(["plugins.entries.tavily.config.webSearch.apiKey"]); mocks.resolveCommandSecretRefsViaGateway.mockResolvedValue({ resolvedConfig: config, diagnostics: [], @@ -91,12 +93,14 @@ describe("resolveCommandConfigWithSecrets", () => { config, commandName: "infer web search", targetIds: new Set(["plugins.entries.*.config.webSearch.apiKey"]), - providerOverrides: { webSearch: "tavily" }, + allowedPaths, + forcedActivePaths, }); expect(mocks.resolveCommandSecretRefsViaGateway).toHaveBeenCalledWith( expect.objectContaining({ - providerOverrides: { webSearch: "tavily" }, + allowedPaths, + forcedActivePaths, }), ); }); diff --git a/src/cli/command-config-resolution.ts b/src/cli/command-config-resolution.ts index 6d4526526dd..05f20a62570 100644 --- a/src/cli/command-config-resolution.ts +++ b/src/cli/command-config-resolution.ts @@ -3,7 +3,6 @@ import type { OpenClawConfig } from "../config/types.js"; import type { RuntimeEnv } from "../runtime.js"; import { type CommandSecretResolutionMode, - type CommandSecretsProviderOverrides, resolveCommandSecretRefsViaGateway, } from "./command-secret-gateway.js"; @@ -13,7 +12,7 @@ export async function resolveCommandConfigWithSecrets; mode?: CommandSecretResolutionMode; allowedPaths?: Set; - providerOverrides?: CommandSecretsProviderOverrides; + forcedActivePaths?: Set; runtime?: RuntimeEnv; autoEnable?: boolean; env?: NodeJS.ProcessEnv; @@ -28,7 +27,7 @@ export async function resolveCommandConfigWithSecrets { expect(readTalkProviderApiKey(result.resolvedConfig)).toBe("sk-live"); }); - it("passes command provider overrides to gateway secret resolution", async () => { - callGateway.mockResolvedValueOnce({ - assignments: [ - { - path: TALK_TEST_PROVIDER_API_KEY_PATH, - pathSegments: [...TALK_TEST_PROVIDER_API_KEY_PATH_SEGMENTS], - value: "sk-live", - }, - ], - diagnostics: [], - }); - const config = buildTalkTestProviderConfig({ - source: "env", - provider: "default", - id: "TALK_API_KEY", - }); - - await resolveCommandSecretRefsViaGateway({ - config, - commandName: "infer web search", - targetIds: new Set(["talk.providers.*.apiKey"]), - providerOverrides: { webSearch: "tavily" }, - }); - - expect(callGateway.mock.calls[0]?.[0]?.params).toEqual({ - commandName: "infer web search", - targetIds: ["talk.providers.*.apiKey"], - providerOverrides: { webSearch: "tavily" }, - }); - }); - - it("applies provider overrides during unavailable-gateway local fallback", async () => { - const restoreDeps = commandSecretGatewayTesting.setDepsForTest({ - collectConfigAssignments: ({ context }) => { - context.assignments.push({ - path: "plugins.entries.google.config.webSearch.apiKey", - } as never); - }, - resolveManifestContractOwnerPluginId: (params) => - params.contract === "webSearchProviders" && params.value === "gemini" - ? "google" - : undefined, - }); - const envKey = "WEB_SEARCH_GEMINI_OVERRIDE_LOCAL_FALLBACK"; - await withEnvValue(envKey, "gemini-override-local-fallback-key", async () => { - try { - callGateway.mockRejectedValueOnce(new Error("gateway closed")); - const result = await resolveCommandSecretRefsViaGateway({ - config: { - tools: { - web: { - search: { - provider: "brave", - apiKey: { - source: "env", - provider: "default", - id: "WEB_SEARCH_BRAVE_MISSING_LOCAL_FALLBACK", - }, - }, - }, - }, - plugins: { - entries: { - google: { - config: { - webSearch: { - apiKey: { source: "env", provider: "default", id: envKey }, - }, - }, - }, - }, - }, - } as unknown as OpenClawConfig, - commandName: "infer web search", - targetIds: new Set([ - "tools.web.search.apiKey", - "plugins.entries.google.config.webSearch.apiKey", - ]), - providerOverrides: { webSearch: "gemini" }, - }); - - const googleWebSearchConfig = result.resolvedConfig.plugins?.entries?.google?.config as - | { webSearch?: { apiKey?: unknown } } - | undefined; - expect(result.resolvedConfig.tools?.web?.search?.provider).toBe("gemini"); - expect(googleWebSearchConfig?.webSearch?.apiKey).toBe("gemini-override-local-fallback-key"); - expect(result.targetStatesByPath["plugins.entries.google.config.webSearch.apiKey"]).toBe( - "resolved_local", - ); - expect(result.targetStatesByPath["tools.web.search.apiKey"]).toBe("inactive_surface"); - expectGatewayUnavailableLocalFallbackDiagnostics(result); - } finally { - restoreDeps(); - } - }); - }); - it("enforces unresolved checks only for allowed paths when provided", async () => { const restoreDeps = commandSecretGatewayTesting.setDepsForTest({ analyzeCommandSecretAssignmentsFromSnapshot: () => @@ -313,8 +216,16 @@ describe("resolveCommandSecretRefsViaGateway", () => { pathSegments: ["channels", "discord", "accounts", "ops", "token"], value: "ops-token", }, + { + path: "channels.discord.accounts.chat.token", + pathSegments: ["channels", "discord", "accounts", "chat", "token"], + value: "chat-token", + }, + ], + diagnostics: [ + "channels.discord.accounts.ops.token: gateway note", + "channels.discord.accounts.chat.token: gateway note", ], - diagnostics: [], }); try { @@ -339,9 +250,127 @@ describe("resolveCommandSecretRefsViaGateway", () => { }); expect(result.resolvedConfig.channels?.discord?.accounts?.ops?.token).toBe("ops-token"); + expect(result.resolvedConfig.channels?.discord?.accounts?.chat?.token).toEqual({ + source: "env", + provider: "default", + id: "DISCORD_CHAT_TOKEN", + }); expect(result.targetStatesByPath).toEqual({ "channels.discord.accounts.ops.token": "resolved_gateway", }); + expect(callGateway.mock.calls[0]?.[0].params).toEqual({ + commandName: "message", + targetIds: ["channels.discord.accounts.*.token"], + allowedPaths: ["channels.discord.accounts.ops.token"], + }); + expect(result.diagnostics).toEqual(["channels.discord.accounts.ops.token: gateway note"]); + expect(result.hadUnresolvedTargets).toBe(false); + } finally { + restoreDeps(); + } + }); + + it("retries old gateways without allowed paths and still filters scoped results", async () => { + const restoreDeps = commandSecretGatewayTesting.setDepsForTest({ + analyzeCommandSecretAssignmentsFromSnapshot: () => + ({ + assignments: [ + { + path: "channels.discord.accounts.ops.token", + pathSegments: ["channels", "discord", "accounts", "ops", "token"], + value: "ops-token", + }, + ], + diagnostics: [], + inactive: [], + unresolved: [], + }) as never, + collectConfigAssignments: ({ context }) => { + context.assignments.push( + { path: "channels.discord.accounts.ops.token" } as never, + { path: "channels.discord.accounts.chat.token" } as never, + ); + }, + discoverConfigSecretTargetsByIds: () => + [ + { + entry: { expectedResolvedValue: "string" }, + path: "channels.discord.accounts.ops.token", + pathSegments: ["channels", "discord", "accounts", "ops", "token"], + value: { source: "env", provider: "default", id: "DISCORD_OPS_TOKEN" }, + }, + { + entry: { expectedResolvedValue: "string" }, + path: "channels.discord.accounts.chat.token", + pathSegments: ["channels", "discord", "accounts", "chat", "token"], + value: { source: "env", provider: "default", id: "DISCORD_CHAT_TOKEN" }, + }, + ] as never, + }); + callGateway + .mockRejectedValueOnce( + new Error("secrets.resolve invalid request: invalid secrets.resolve params"), + ) + .mockResolvedValueOnce({ + assignments: [ + { + path: "channels.discord.accounts.ops.token", + pathSegments: ["channels", "discord", "accounts", "ops", "token"], + value: "ops-token", + }, + { + path: "channels.discord.accounts.chat.token", + pathSegments: ["channels", "discord", "accounts", "chat", "token"], + value: "chat-token", + }, + ], + diagnostics: [ + "channels.discord.accounts.ops.token: gateway note", + "channels.discord.accounts.chat.token: gateway note", + ], + }); + + try { + const result = await resolveCommandSecretRefsViaGateway({ + config: { + channels: { + discord: { + accounts: { + ops: { + token: { source: "env", provider: "default", id: "DISCORD_OPS_TOKEN" }, + }, + chat: { + token: { source: "env", provider: "default", id: "DISCORD_CHAT_TOKEN" }, + }, + }, + }, + }, + } as OpenClawConfig, + commandName: "message", + targetIds: new Set(["channels.discord.accounts.*.token"]), + allowedPaths: new Set(["channels.discord.accounts.ops.token"]), + }); + + expect(callGateway).toHaveBeenCalledTimes(2); + expect(callGateway.mock.calls[0]?.[0].params).toEqual({ + commandName: "message", + targetIds: ["channels.discord.accounts.*.token"], + allowedPaths: ["channels.discord.accounts.ops.token"], + }); + expect(callGateway.mock.calls[1]?.[0].params).toEqual({ + commandName: "message", + targetIds: ["channels.discord.accounts.*.token"], + }); + expect(result.resolvedConfig.channels?.discord?.accounts?.ops?.token).toBe("ops-token"); + expect(result.resolvedConfig.channels?.discord?.accounts?.chat?.token).toEqual({ + source: "env", + provider: "default", + id: "DISCORD_CHAT_TOKEN", + }); + expect(result.targetStatesByPath).toEqual({ + "channels.discord.accounts.ops.token": "resolved_gateway", + }); + expect(result.diagnostics).toEqual(["channels.discord.accounts.ops.token: gateway note"]); expect(result.hadUnresolvedTargets).toBe(false); } finally { restoreDeps(); @@ -522,32 +551,139 @@ describe("resolveCommandSecretRefsViaGateway", () => { }); }); - it("falls back to local resolution for legacy web fetch SecretRefs when gateway is unavailable", async () => { - const envKey = "WEB_FETCH_FIRECRAWL_LEGACY_LOCAL_FALLBACK"; - await withEnvValue(envKey, "legacy-firecrawl-local-fallback-key", async () => { + it("treats command-scoped web fetch fallback SecretRefs as active even when web search is disabled", async () => { + const envKey = "WEB_FETCH_FIRECRAWL_SEARCH_FALLBACK_KEY"; + await withEnvValue(envKey, "firecrawl-search-fallback-key", async () => { callGateway.mockRejectedValueOnce(new Error("gateway closed")); const result = await resolveCommandSecretRefsViaGateway({ config: { tools: { web: { + search: { + enabled: false, + provider: "brave", + }, fetch: { provider: "firecrawl", - firecrawl: { - apiKey: { source: "env", provider: "default", id: envKey }, + }, + }, + }, + plugins: { + entries: { + firecrawl: { + enabled: true, + config: { + webSearch: { + apiKey: { source: "env", provider: "default", id: envKey }, + }, }, }, }, }, } as unknown as OpenClawConfig, commandName: "infer web fetch", - targetIds: new Set(["tools.web.fetch.firecrawl.apiKey"]), + targetIds: new Set(["plugins.entries.firecrawl.config.webSearch.apiKey"]), + allowedPaths: new Set(["plugins.entries.firecrawl.config.webSearch.apiKey"]), + forcedActivePaths: new Set(["plugins.entries.firecrawl.config.webSearch.apiKey"]), }); - const resolvedFetch = result.resolvedConfig.tools?.web?.fetch as - | { firecrawl?: { apiKey?: unknown } } + const firecrawlConfig = result.resolvedConfig.plugins?.entries?.firecrawl?.config as + | { webSearch?: { apiKey?: unknown } } | undefined; - expect(resolvedFetch?.firecrawl?.apiKey).toBe("legacy-firecrawl-local-fallback-key"); - expect(result.targetStatesByPath["tools.web.fetch.firecrawl.apiKey"]).toBe("resolved_local"); + expect(firecrawlConfig?.webSearch?.apiKey).toBe("firecrawl-search-fallback-key"); + expect(result.targetStatesByPath["plugins.entries.firecrawl.config.webSearch.apiKey"]).toBe( + "resolved_local", + ); + expectGatewayUnavailableLocalFallbackDiagnostics(result); + }); + }); + + it("drops gateway inactive diagnostics for forced active fallback paths", async () => { + const envKey = "WEB_FETCH_FIRECRAWL_FORCED_FALLBACK_KEY"; + await withEnvValue(envKey, "firecrawl-search-fallback-key", async () => { + callGateway.mockResolvedValueOnce({ + assignments: [], + diagnostics: [ + "plugins.entries.firecrawl.config.webSearch.apiKey: secret ref is configured on an inactive surface; tools.web.search is disabled.", + ], + inactiveRefPaths: ["plugins.entries.firecrawl.config.webSearch.apiKey"], + }); + const result = await resolveCommandSecretRefsViaGateway({ + config: { + tools: { + web: { + search: { + enabled: false, + provider: "brave", + }, + fetch: { + provider: "firecrawl", + }, + }, + }, + plugins: { + entries: { + firecrawl: { + enabled: true, + config: { + webSearch: { + apiKey: { source: "env", provider: "default", id: envKey }, + }, + }, + }, + }, + }, + } as unknown as OpenClawConfig, + commandName: "infer web fetch", + targetIds: new Set(["plugins.entries.firecrawl.config.webSearch.apiKey"]), + allowedPaths: new Set(["plugins.entries.firecrawl.config.webSearch.apiKey"]), + forcedActivePaths: new Set(["plugins.entries.firecrawl.config.webSearch.apiKey"]), + }); + + const firecrawlConfig = result.resolvedConfig.plugins?.entries?.firecrawl?.config as + | { webSearch?: { apiKey?: unknown } } + | undefined; + expect(firecrawlConfig?.webSearch?.apiKey).toBe("firecrawl-search-fallback-key"); + expect(result.targetStatesByPath["plugins.entries.firecrawl.config.webSearch.apiKey"]).toBe( + "resolved_local", + ); + expect(callGateway.mock.calls[0]?.[0].params).toEqual({ + commandName: "infer web fetch", + targetIds: ["plugins.entries.firecrawl.config.webSearch.apiKey"], + allowedPaths: ["plugins.entries.firecrawl.config.webSearch.apiKey"], + forcedActivePaths: ["plugins.entries.firecrawl.config.webSearch.apiKey"], + }); + expect(result.diagnostics).not.toContain( + "plugins.entries.firecrawl.config.webSearch.apiKey: secret ref is configured on an inactive surface; tools.web.search is disabled.", + ); + }); + }); + + it("honors forced active paths for non-web local fallback targets", async () => { + const envKey = "GOOGLE_MODEL_FALLBACK_API_KEY"; + await withEnvValue(envKey, "google-local-fallback-key", async () => { + callGateway.mockRejectedValueOnce(new Error("gateway closed")); + const result = await resolveCommandSecretRefsViaGateway({ + config: { + models: { + providers: { + google: { + enabled: false, + apiKey: { source: "env", provider: "default", id: envKey }, + }, + }, + }, + } as unknown as OpenClawConfig, + commandName: "infer web search", + targetIds: new Set(["models.providers.*.apiKey"]), + allowedPaths: new Set(["models.providers.google.apiKey"]), + forcedActivePaths: new Set(["models.providers.google.apiKey"]), + }); + + expect(result.resolvedConfig.models?.providers?.google?.apiKey).toBe( + "google-local-fallback-key", + ); + expect(result.targetStatesByPath["models.providers.google.apiKey"]).toBe("resolved_local"); expectGatewayUnavailableLocalFallbackDiagnostics(result); }); }); @@ -594,7 +730,6 @@ describe("resolveCommandSecretRefsViaGateway", () => { } as OpenClawConfig, commandName: "agent", targetIds: new Set(["plugins.entries.google.config.webSearch.apiKey"]), - providerOverrides: { webSearch: "gemini" }, }); expect(result.hadUnresolvedTargets).toBe(false); diff --git a/src/cli/command-secret-gateway.ts b/src/cli/command-secret-gateway.ts index 2403acd5841..c0ab909ad41 100644 --- a/src/cli/command-secret-gateway.ts +++ b/src/cli/command-secret-gateway.ts @@ -5,7 +5,6 @@ import { GATEWAY_CLIENT_MODES, GATEWAY_CLIENT_NAMES } from "../gateway/protocol/ import { validateSecretsResolveResult } from "../gateway/protocol/index.js"; import { formatErrorMessage } from "../infra/errors.js"; import { resolveManifestContractOwnerPluginId } from "../plugins/plugin-registry.js"; -import { resolveBundledExplicitWebSearchProvidersFromPublicArtifacts } from "../plugins/web-provider-public-artifacts.explicit.js"; import { analyzeCommandSecretAssignmentsFromSnapshot, type UnresolvedCommandSecretAssignment, @@ -20,10 +19,7 @@ import { discoverConfigSecretTargetsByIds, type DiscoveredConfigSecretTarget, } from "../secrets/target-registry.js"; -import { - normalizeLowercaseStringOrEmpty, - normalizeOptionalString, -} from "../shared/string-coerce.js"; +import { normalizeLowercaseStringOrEmpty } from "../shared/string-coerce.js"; type ResolveCommandSecretsResult = { resolvedConfig: OpenClawConfig; @@ -60,16 +56,8 @@ type GatewaySecretsResolveResult = { inactiveRefPaths?: string[]; }; -const WEB_RUNTIME_SECRET_TARGET_ID_PREFIXES = [ - "tools.web.search", - "tools.web.fetch", - "plugins.entries.", -] as const; -const WEB_RUNTIME_SECRET_PATH_PREFIXES = [ - "tools.web.search.", - "tools.web.fetch.", - "plugins.entries.", -] as const; +const WEB_RUNTIME_SECRET_TARGET_ID_PREFIXES = ["tools.web.search", "plugins.entries."] as const; +const WEB_RUNTIME_SECRET_PATH_PREFIXES = ["tools.web.search.", "plugins.entries."] as const; type CommandSecretGatewayDeps = { analyzeCommandSecretAssignmentsFromSnapshot: typeof analyzeCommandSecretAssignmentsFromSnapshot; @@ -79,11 +67,6 @@ type CommandSecretGatewayDeps = { resolveRuntimeWebTools: typeof resolveRuntimeWebTools; }; -export type CommandSecretsProviderOverrides = { - webSearch?: string; - webFetch?: string; -}; - const commandSecretGatewayDeps: CommandSecretGatewayDeps = { analyzeCommandSecretAssignmentsFromSnapshot, collectConfigAssignments, @@ -116,115 +99,6 @@ function pluginIdFromRuntimeWebPath(path: string): string | undefined { return match?.[1]; } -function applyProviderOverridesToConfig( - config: OpenClawConfig, - overrides: CommandSecretsProviderOverrides | undefined, -): OpenClawConfig { - if ( - !normalizeOptionalString(overrides?.webSearch) && - !normalizeOptionalString(overrides?.webFetch) - ) { - return config; - } - const next = structuredClone(config); - const tools = (next.tools ??= {}) as Record; - const web = (tools.web ??= {}) as Record; - const webSearch = normalizeOptionalString(overrides?.webSearch); - if (webSearch) { - const search = (web.search ??= {}) as Record; - search.provider = webSearch; - } - const webFetch = normalizeOptionalString(overrides?.webFetch); - if (webFetch) { - const fetch = (web.fetch ??= {}) as Record; - fetch.provider = webFetch; - } - return next; -} - -function webSearchProviderUsesSharedSearchCredential(params: { - config: OpenClawConfig; - provider: string; -}): boolean { - const sentinel = "__openclaw_shared_web_search_probe__"; - const pluginId = commandSecretGatewayDeps.resolveManifestContractOwnerPluginId({ - contract: "webSearchProviders", - value: params.provider, - origin: "bundled", - config: params.config, - }); - if (!pluginId) { - return false; - } - const providers = resolveBundledExplicitWebSearchProvidersFromPublicArtifacts({ - onlyPluginIds: [pluginId], - }); - const provider = providers?.find((entry) => entry.id === params.provider); - return ( - provider?.credentialPath === "tools.web.search.apiKey" || - provider?.getCredentialValue({ apiKey: sentinel }) === sentinel || - provider?.getConfiguredCredentialFallback?.(params.config)?.path === "tools.web.search.apiKey" - ); -} - -function isProviderOverridePath(params: { - config: OpenClawConfig; - path: string; - providerOverrides: CommandSecretsProviderOverrides | undefined; -}): boolean { - const webSearch = normalizeOptionalString(params.providerOverrides?.webSearch); - if (webSearch) { - if (params.config.tools?.web?.search?.enabled === false) { - return false; - } - if (params.path === "tools.web.search.apiKey") { - return webSearchProviderUsesSharedSearchCredential({ - config: params.config, - provider: webSearch, - }); - } - const directSearchProvider = /^tools\.web\.search\.([^.]+)\.apiKey$/.exec(params.path)?.[1]; - if (directSearchProvider) { - return directSearchProvider === webSearch; - } - const pluginId = pluginIdFromRuntimeWebPath(params.path); - if (pluginId && params.path.endsWith(".config.webSearch.apiKey")) { - return ( - commandSecretGatewayDeps.resolveManifestContractOwnerPluginId({ - contract: "webSearchProviders", - value: webSearch, - origin: "bundled", - config: params.config, - }) === pluginId - ); - } - } - - const webFetch = normalizeOptionalString(params.providerOverrides?.webFetch); - if (webFetch) { - if (params.config.tools?.web?.fetch?.enabled === false) { - return false; - } - const directFetchProvider = /^tools\.web\.fetch\.([^.]+)\.apiKey$/.exec(params.path)?.[1]; - if (directFetchProvider) { - return directFetchProvider === webFetch; - } - const pluginId = pluginIdFromRuntimeWebPath(params.path); - if (pluginId && params.path.endsWith(".config.webFetch.apiKey")) { - return ( - commandSecretGatewayDeps.resolveManifestContractOwnerPluginId({ - contract: "webFetchProviders", - value: webFetch, - origin: "bundled", - config: params.config, - }) === pluginId - ); - } - } - - return false; -} - function normalizeCommandSecretResolutionMode( mode?: CommandSecretResolutionModeInput, ): CommandSecretResolutionMode { @@ -262,22 +136,7 @@ function targetsRuntimeWebPath(path: string): boolean { function classifyRuntimeWebTargetPathState(params: { config: OpenClawConfig; path: string; - providerOverrides?: CommandSecretsProviderOverrides; }): "active" | "inactive" | "unknown" { - if ( - (normalizeOptionalString(params.providerOverrides?.webSearch) || - normalizeOptionalString(params.providerOverrides?.webFetch)) && - isDirectRuntimeWebTargetPath(params.path) - ) { - return isProviderOverridePath({ - config: params.config, - path: params.path, - providerOverrides: params.providerOverrides, - }) - ? "active" - : "inactive"; - } - if (params.path === "tools.web.search.apiKey") { return params.config.tools?.web?.search?.enabled !== false ? "active" : "inactive"; } @@ -320,74 +179,28 @@ function classifyRuntimeWebTargetPathState(params: { : "inactive"; } - const directSearchMatch = /^tools\.web\.search\.([^.]+)\.apiKey$/.exec(params.path); - if (directSearchMatch) { - const search = params.config.tools?.web?.search; - if (search?.enabled === false) { - return "inactive"; - } - - const configuredProvider = normalizeLowercaseStringOrEmpty(search?.provider); - if (!configuredProvider) { - return "active"; - } - - return configuredProvider === directSearchMatch[1] ? "active" : "inactive"; - } - - const directFetchMatch = /^tools\.web\.fetch\.([^.]+)\.apiKey$/.exec(params.path); - if (!directFetchMatch) { + const match = /^tools\.web\.search\.([^.]+)\.apiKey$/.exec(params.path); + if (!match) { return "unknown"; } - const fetch = params.config.tools?.web?.fetch; - if (fetch?.enabled === false) { + const search = params.config.tools?.web?.search; + if (search?.enabled === false) { return "inactive"; } - const configuredProvider = normalizeLowercaseStringOrEmpty(fetch?.provider); + const configuredProvider = normalizeLowercaseStringOrEmpty(search?.provider); if (!configuredProvider) { return "active"; } - return configuredProvider === directFetchMatch[1] ? "active" : "inactive"; + return configuredProvider === match[1] ? "active" : "inactive"; } function describeInactiveRuntimeWebTargetPath(params: { config: OpenClawConfig; path: string; - providerOverrides?: CommandSecretsProviderOverrides; }): string | undefined { - if ( - params.config.tools?.web?.search?.enabled === false && - (params.path === "tools.web.search.apiKey" || - params.path.startsWith("tools.web.search.") || - params.path.includes(".webSearch.")) - ) { - return "tools.web.search is disabled."; - } - if ( - params.config.tools?.web?.fetch?.enabled === false && - (params.path.startsWith("tools.web.fetch.") || params.path.includes(".webFetch.")) - ) { - return "tools.web.fetch is disabled."; - } - - const webSearchOverride = normalizeOptionalString(params.providerOverrides?.webSearch); - if (webSearchOverride && params.path.includes(".webSearch.")) { - return `tools.web.search.provider is "${webSearchOverride}".`; - } - if (webSearchOverride && params.path.startsWith("tools.web.search.")) { - return `tools.web.search.provider is "${webSearchOverride}".`; - } - const webFetchOverride = normalizeOptionalString(params.providerOverrides?.webFetch); - if (webFetchOverride && params.path.includes(".webFetch.")) { - return `tools.web.fetch.provider is "${webFetchOverride}".`; - } - if (webFetchOverride && params.path.startsWith("tools.web.fetch.")) { - return `tools.web.fetch.provider is "${webFetchOverride}".`; - } - if (params.path === "tools.web.search.apiKey") { return params.config.tools?.web?.search?.enabled === false ? "tools.web.search is disabled." @@ -426,34 +239,19 @@ function describeInactiveRuntimeWebTargetPath(params: { return undefined; } - const directSearchMatch = /^tools\.web\.search\.([^.]+)\.apiKey$/.exec(params.path); - if (directSearchMatch) { - const search = params.config.tools?.web?.search; - if (search?.enabled === false) { - return "tools.web.search is disabled."; - } - - const configuredProvider = normalizeLowercaseStringOrEmpty(search?.provider); - if (configuredProvider && configuredProvider !== directSearchMatch[1]) { - return `tools.web.search.provider is "${configuredProvider}".`; - } - + const match = /^tools\.web\.search\.([^.]+)\.apiKey$/.exec(params.path); + if (!match) { return undefined; } - const directFetchMatch = /^tools\.web\.fetch\.([^.]+)\.apiKey$/.exec(params.path); - if (!directFetchMatch) { - return undefined; + const search = params.config.tools?.web?.search; + if (search?.enabled === false) { + return "tools.web.search is disabled."; } - const fetch = params.config.tools?.web?.fetch; - if (fetch?.enabled === false) { - return "tools.web.fetch is disabled."; - } - - const configuredProvider = normalizeLowercaseStringOrEmpty(fetch?.provider); - if (configuredProvider && configuredProvider !== directFetchMatch[1]) { - return `tools.web.fetch.provider is "${configuredProvider}".`; + const configuredProvider = normalizeLowercaseStringOrEmpty(search?.provider); + if (configuredProvider && configuredProvider !== match[1]) { + return `tools.web.search.provider is "${configuredProvider}".`; } return undefined; @@ -508,6 +306,7 @@ function collectConfiguredTargetRefPaths(params: { function classifyConfiguredTargetRefs(params: { config: OpenClawConfig; configuredTargetRefPaths: Set; + forcedActivePaths?: ReadonlySet; }): { hasActiveConfiguredRef: boolean; hasUnknownConfiguredRef: boolean; @@ -543,7 +342,7 @@ function classifyConfiguredTargetRefs(params: { let hasUnknownConfiguredRef = false; for (const path of params.configuredTargetRefPaths) { - if (activePaths.has(path)) { + if (activePaths.has(path) || params.forcedActivePaths?.has(path)) { hasActiveConfiguredRef = true; continue; } @@ -594,6 +393,27 @@ function collectInactiveSurfacePathsFromDiagnostics(diagnostics: string[]): Set< return paths; } +function filterAllowedGatewayDiagnostics(params: { + allowedPaths?: ReadonlySet; + forcedActivePaths?: ReadonlySet; + diagnostics: string[]; +}): string[] { + return params.diagnostics.filter((diagnostic) => { + const markerIndex = diagnostic.indexOf(":"); + if (markerIndex <= 0) { + return true; + } + const path = diagnostic.slice(0, markerIndex).trim(); + if (!path.includes(".")) { + return true; + } + if (params.forcedActivePaths?.has(path)) { + return false; + } + return !params.allowedPaths || params.allowedPaths.has(path); + }); +} + function isUnsupportedSecretsResolveError(err: unknown): boolean { const message = normalizeLowercaseStringOrEmpty(formatErrorMessage(err)); if (!message.includes("secrets.resolve")) { @@ -607,11 +427,58 @@ function isUnsupportedSecretsResolveError(err: unknown): boolean { ); } +function isAllowedPathsSecretsResolveCompatError(err: unknown): boolean { + const message = normalizeLowercaseStringOrEmpty(formatErrorMessage(err)); + if (!message.includes("secrets.resolve")) { + return false; + } + return message.includes("invalid request") || message.includes("invalid secrets.resolve params"); +} + +async function callGatewaySecretsResolve(params: { + config: OpenClawConfig; + commandName: string; + targetIds: Set; + allowedPaths?: ReadonlySet; + forcedActivePaths?: ReadonlySet; +}): Promise { + const request = { + config: params.config, + method: "secrets.resolve", + requiredMethods: ["secrets.resolve"], + params: { + commandName: params.commandName, + targetIds: [...params.targetIds], + ...(params.allowedPaths ? { allowedPaths: [...params.allowedPaths] } : {}), + ...(params.forcedActivePaths ? { forcedActivePaths: [...params.forcedActivePaths] } : {}), + }, + timeoutMs: 30_000, + clientName: GATEWAY_CLIENT_NAMES.CLI, + mode: GATEWAY_CLIENT_MODES.CLI, + }; + try { + return await callGateway(request); + } catch (err) { + if ( + (!params.allowedPaths && !params.forcedActivePaths) || + !isAllowedPathsSecretsResolveCompatError(err) + ) { + throw err; + } + return callGateway({ + ...request, + params: { + commandName: params.commandName, + targetIds: [...params.targetIds], + }, + }); + } +} + function isDirectRuntimeWebTargetPath(path: string): boolean { return ( - path === "tools.web.search.apiKey" || /^plugins\.entries\.[^.]+\.config\.(webSearch|webFetch)\.apiKey$/.test(path) || - /^tools\.web\.(search|fetch)\.[^.]+\.apiKey$/.test(path) + /^tools\.web\.search\.[^.]+\.apiKey$/.test(path) ); } @@ -622,10 +489,10 @@ async function resolveCommandSecretRefsLocally(params: { preflightDiagnostics: string[]; mode: CommandSecretResolutionMode; allowedPaths?: ReadonlySet; - providerOverrides?: CommandSecretsProviderOverrides; + forcedActivePaths?: ReadonlySet; }): Promise { - const sourceConfig = applyProviderOverridesToConfig(params.config, params.providerOverrides); - const resolvedConfig = structuredClone(sourceConfig); + const sourceConfig = params.config; + const resolvedConfig = structuredClone(params.config); const context = createResolverContext({ sourceConfig, env: process.env, @@ -638,7 +505,7 @@ async function resolveCommandSecretRefsLocally(params: { targetsRuntimeWebPath(target.path), ); commandSecretGatewayDeps.collectConfigAssignments({ - config: structuredClone(sourceConfig), + config: structuredClone(params.config), context, }); if ( @@ -667,22 +534,25 @@ async function resolveCommandSecretRefsLocally(params: { context.warnings .filter((warning) => warning.code === "SECRETS_REF_IGNORED_INACTIVE_SURFACE") .filter((warning) => !params.allowedPaths || params.allowedPaths.has(warning.path)) + .filter((warning) => !params.forcedActivePaths?.has(warning.path)) .map((warning) => warning.path), ); const runtimeWebActivePaths = new Set(); const runtimeWebInactiveDiagnostics: string[] = []; for (const target of runtimeWebTargets) { + if (params.forcedActivePaths?.has(target.path)) { + runtimeWebActivePaths.add(target.path); + continue; + } const runtimeState = classifyRuntimeWebTargetPathState({ config: sourceConfig, path: target.path, - providerOverrides: params.providerOverrides, }); if (runtimeState === "inactive") { inactiveRefPaths.add(target.path); const inactiveDetail = describeInactiveRuntimeWebTargetPath({ config: sourceConfig, path: target.path, - providerOverrides: params.providerOverrides, }); if (inactiveDetail) { runtimeWebInactiveDiagnostics.push(`${target.path}: ${inactiveDetail}`); @@ -696,6 +566,7 @@ async function resolveCommandSecretRefsLocally(params: { const inactiveWarningDiagnostics = context.warnings .filter((warning) => warning.code === "SECRETS_REF_IGNORED_INACTIVE_SURFACE") .filter((warning) => !params.allowedPaths || params.allowedPaths.has(warning.path)) + .filter((warning) => !params.forcedActivePaths?.has(warning.path)) .map((warning) => warning.message); const activePaths = new Set(context.assignments.map((assignment) => assignment.path)); for (const target of discoveredTargets) { @@ -708,6 +579,7 @@ async function resolveCommandSecretRefsLocally(params: { activePaths, runtimeWebActivePaths, inactiveRefPaths, + forcedActivePaths: params.forcedActivePaths, mode: params.mode, commandName: params.commandName, localResolutionDiagnostics, @@ -814,6 +686,7 @@ async function resolveTargetSecretLocally(params: { activePaths: ReadonlySet; runtimeWebActivePaths: ReadonlySet; inactiveRefPaths: ReadonlySet; + forcedActivePaths?: ReadonlySet; mode: CommandSecretResolutionMode; commandName: string; localResolutionDiagnostics: string[]; @@ -828,7 +701,8 @@ async function resolveTargetSecretLocally(params: { !ref || params.inactiveRefPaths.has(params.target.path) || (!params.activePaths.has(params.target.path) && - !params.runtimeWebActivePaths.has(params.target.path)) + !params.runtimeWebActivePaths.has(params.target.path) && + !params.forcedActivePaths?.has(params.target.path)) ) { return; } @@ -863,30 +737,30 @@ export async function resolveCommandSecretRefsViaGateway(params: { targetIds: Set; mode?: CommandSecretResolutionModeInput; allowedPaths?: ReadonlySet; - providerOverrides?: CommandSecretsProviderOverrides; + forcedActivePaths?: ReadonlySet; }): Promise { const mode = normalizeCommandSecretResolutionMode(params.mode); - const commandConfig = applyProviderOverridesToConfig(params.config, params.providerOverrides); const configuredTargetRefPaths = collectConfiguredTargetRefPaths({ - config: commandConfig, + config: params.config, targetIds: params.targetIds, allowedPaths: params.allowedPaths, }); if (configuredTargetRefPaths.size === 0) { return { - resolvedConfig: commandConfig, + resolvedConfig: params.config, diagnostics: [], targetStatesByPath: {}, hadUnresolvedTargets: false, }; } const preflight = classifyConfiguredTargetRefs({ - config: commandConfig, + config: params.config, configuredTargetRefPaths, + forcedActivePaths: params.forcedActivePaths, }); if (!preflight.hasActiveConfiguredRef && !preflight.hasUnknownConfiguredRef) { return { - resolvedConfig: commandConfig, + resolvedConfig: params.config, diagnostics: preflight.diagnostics, targetStatesByPath: {}, hadUnresolvedTargets: false, @@ -895,18 +769,12 @@ export async function resolveCommandSecretRefsViaGateway(params: { let payload: GatewaySecretsResolveResult; try { - payload = await callGateway({ + payload = await callGatewaySecretsResolve({ config: params.config, - method: "secrets.resolve", - requiredMethods: ["secrets.resolve"], - params: { - commandName: params.commandName, - targetIds: [...params.targetIds], - ...(params.providerOverrides ? { providerOverrides: params.providerOverrides } : {}), - }, - timeoutMs: 30_000, - clientName: GATEWAY_CLIENT_NAMES.CLI, - mode: GATEWAY_CLIENT_MODES.CLI, + commandName: params.commandName, + targetIds: params.targetIds, + allowedPaths: params.allowedPaths, + forcedActivePaths: params.forcedActivePaths, }); } catch (err) { try { @@ -917,7 +785,7 @@ export async function resolveCommandSecretRefsViaGateway(params: { preflightDiagnostics: preflight.diagnostics, mode, allowedPaths: params.allowedPaths, - providerOverrides: params.providerOverrides, + forcedActivePaths: params.forcedActivePaths, }); const recoveredLocally = Object.values(fallback.targetStatesByPath).some( (state) => state === "resolved_local", @@ -951,8 +819,22 @@ export async function resolveCommandSecretRefsViaGateway(params: { } const parsed = parseGatewaySecretsResolveResult(payload); - const resolvedConfig = structuredClone(commandConfig); - for (const assignment of parsed.assignments) { + const gatewayDiagnostics = filterAllowedGatewayDiagnostics({ + allowedPaths: params.allowedPaths, + forcedActivePaths: params.forcedActivePaths, + diagnostics: parsed.diagnostics, + }); + const gatewayInactiveRefPaths = params.allowedPaths + ? parsed.inactiveRefPaths.filter((path) => params.allowedPaths?.has(path)) + : parsed.inactiveRefPaths; + const resolvedConfig = structuredClone(params.config); + const assignments = params.allowedPaths + ? parsed.assignments.filter((assignment) => { + const path = assignment.path ?? assignment.pathSegments.join("."); + return params.allowedPaths?.has(path); + }) + : parsed.assignments; + for (const assignment of assignments) { const pathSegments = assignment.pathSegments.filter((segment) => segment.length > 0); if (pathSegments.length === 0) { continue; @@ -967,18 +849,22 @@ export async function resolveCommandSecretRefsViaGateway(params: { ); } } - const inactiveRefPaths = - parsed.inactiveRefPaths.length > 0 - ? new Set(parsed.inactiveRefPaths) - : collectInactiveSurfacePathsFromDiagnostics(parsed.diagnostics); + const inactiveRefPaths = new Set( + gatewayInactiveRefPaths.length > 0 + ? gatewayInactiveRefPaths + : collectInactiveSurfacePathsFromDiagnostics(gatewayDiagnostics), + ); + for (const path of params.forcedActivePaths ?? []) { + inactiveRefPaths.delete(path); + } const analyzed = commandSecretGatewayDeps.analyzeCommandSecretAssignmentsFromSnapshot({ - sourceConfig: commandConfig, + sourceConfig: params.config, resolvedConfig, targetIds: params.targetIds, inactiveRefPaths, allowedPaths: params.allowedPaths, }); - let diagnostics = dedupeDiagnostics(parsed.diagnostics); + let diagnostics = dedupeDiagnostics(gatewayDiagnostics); const targetStatesByPath = buildTargetStatesByPath({ analyzed, resolvedState: "resolved_gateway", @@ -992,7 +878,7 @@ export async function resolveCommandSecretRefsViaGateway(params: { preflightDiagnostics: [], mode, allowedPaths: new Set(analyzed.unresolved.map((entry) => entry.path)), - providerOverrides: params.providerOverrides, + forcedActivePaths: params.forcedActivePaths, }); for (const unresolved of analyzed.unresolved) { if (localFallback.targetStatesByPath[unresolved.path] !== "resolved_local") { diff --git a/src/cli/command-secret-resolution.coverage.test.ts b/src/cli/command-secret-resolution.coverage.test.ts index 9ae2b06e9b1..d7d3abe68fd 100644 --- a/src/cli/command-secret-resolution.coverage.test.ts +++ b/src/cli/command-secret-resolution.coverage.test.ts @@ -4,7 +4,6 @@ import { readCommandSource } from "./command-source.test-helpers.js"; const SECRET_TARGET_CALLSITES = [ bundledPluginFile("memory-core", "src/cli.runtime.ts"), - "src/cli/capability-cli.ts", "src/cli/qr-cli.ts", "src/agents/agent-runtime-config.ts", "src/commands/agent.ts", @@ -22,6 +21,7 @@ function hasSupportedTargetIdsWiring(source: string): boolean { source.includes("resolveAgentRuntimeConfig(") || /targetIds:\s*get[A-Za-z0-9_]+\(\)/m.test(source) || /targetIds:\s*getAgentRuntimeCommandSecretTargetIds\(/m.test(source) || + /targetIds:\s*getCapabilityWeb(Fetch|Search)CommandSecretTargetIds\(/m.test(source) || /targetIds:\s*scopedTargets\.targetIds/m.test(source) || source.includes("collectStatusScanOverview({") ); diff --git a/src/cli/command-secret-targets.test.ts b/src/cli/command-secret-targets.test.ts index 8e3a24dccc1..3aa07d25ecf 100644 --- a/src/cli/command-secret-targets.test.ts +++ b/src/cli/command-secret-targets.test.ts @@ -11,110 +11,227 @@ const REGISTRY_IDS = [ "gateway.auth.password", "gateway.remote.token", "gateway.remote.password", - "models.providers.google.apiKey", - "models.providers.openai.apiKey", + "models.providers.*.apiKey", "messages.tts.providers.openai.apiKey", + "plugins.entries.voice-call.config.twilio.authToken", "plugins.entries.firecrawl.config.webFetch.apiKey", "plugins.entries.firecrawl.config.webSearch.apiKey", + "plugins.entries.brave.config.webSearch.apiKey", "plugins.entries.exa.config.webSearch.apiKey", - "plugins.entries.google.config.webSearch.apiKey", - "plugins.entries.readerlab.config.webFetch.apiKey", - "plugins.entries.searchlab.config.webSearch.apiKey", + "plugins.entries.gemini.config.webSearch.apiKey", + "plugins.entries.other-fetch.config.webFetch.apiKey", + "plugins.entries.other-fetch.config.webSearch.apiKey", "skills.entries.demo.apiKey", "tools.web.search.apiKey", + "tools.web.search.*.apiKey", ] as const; +function readPath(source: unknown, path: string): unknown { + let current = source; + for (const segment of path.split(".")) { + if (!current || typeof current !== "object" || Array.isArray(current)) { + return undefined; + } + current = (current as Record)[segment]; + } + return current; +} + vi.mock("../secrets/target-registry.js", () => ({ listSecretTargetRegistryEntries: vi.fn(() => REGISTRY_IDS.map((id) => ({ id, + pathPattern: id, })), ), discoverConfigSecretTargetsByIds: vi.fn((config: unknown, targetIds?: Iterable) => { const allowed = targetIds ? new Set(targetIds) : null; - const out: Array<{ path: string; pathSegments: string[] }> = []; - const isAllowed = (path: string) => - !allowed || - allowed.has(path) || - (allowed.has("models.providers.*.apiKey") && /^models\.providers\.[^.]+\.apiKey$/.test(path)); - const record = (path: string) => { - if (!isAllowed(path)) { + const out: Array<{ entry: { id: string }; path: string; pathSegments: string[] }> = []; + const matches = (pattern: string, path: string): boolean => { + const patternSegments = pattern.split("."); + const pathSegments = path.split("."); + if (patternSegments.length !== pathSegments.length) { + return false; + } + return patternSegments.every( + (segment, index) => segment === "*" || segment === pathSegments[index], + ); + }; + const collectPaths = (node: unknown, segments: string[], prefix: string[] = []): string[] => { + const [segment, ...rest] = segments; + if (!segment) { + return node === undefined ? [] : [prefix.join(".")]; + } + if (!node || typeof node !== "object" || Array.isArray(node)) { + return []; + } + if (segment === "*") { + return Object.entries(node).flatMap(([key, value]) => + collectPaths(value, rest, [...prefix, key]), + ); + } + return collectPaths((node as Record)[segment], rest, [...prefix, segment]); + }; + const record = (targetId: string, path: string) => { + if (allowed && !allowed.has(targetId)) { return; } - out.push({ path, pathSegments: path.split(".") }); + out.push({ entry: { id: targetId }, path, pathSegments: path.split(".") }); }; - - const channels = (config as { channels?: Record } | undefined)?.channels; - const discord = channels?.discord as - | { token?: unknown; accounts?: Record } - | undefined; - - if (discord?.token !== undefined) { - record("channels.discord.token"); - } - for (const [accountId, account] of Object.entries(discord?.accounts ?? {})) { - if (account?.token !== undefined) { - record(`channels.discord.accounts.${accountId}.token`); + for (const id of REGISTRY_IDS) { + if (id.includes("*")) { + for (const path of collectPaths(config, id.split("."))) { + if (matches(id, path)) { + record(id, path); + } + } + continue; } - } - const models = (config as { models?: { providers?: Record } }) - ?.models; - for (const [providerId, provider] of Object.entries(models?.providers ?? {})) { - if (provider?.apiKey !== undefined) { - record(`models.providers.${providerId}.apiKey`); + if (readPath(config, id) !== undefined) { + record(id, id); } } - const plugins = ( - config as { - plugins?: { - entries?: Record< - string, - { config?: { webSearch?: { apiKey?: unknown }; webFetch?: { apiKey?: unknown } } } - >; - }; - } - )?.plugins; - for (const [pluginId, entry] of Object.entries(plugins?.entries ?? {})) { - if (entry?.config?.webSearch?.apiKey !== undefined) { - record(`plugins.entries.${pluginId}.config.webSearch.apiKey`); - } - if (entry?.config?.webFetch?.apiKey !== undefined) { - record(`plugins.entries.${pluginId}.config.webFetch.apiKey`); - } - } - const tools = (config as { tools?: { web?: { fetch?: { firecrawl?: { apiKey?: unknown } } } } }) - ?.tools; - if (tools?.web?.fetch?.firecrawl?.apiKey !== undefined) { - record("tools.web.fetch.firecrawl.apiKey"); - } return out; }), })); -vi.mock("../plugins/plugin-registry.js", () => ({ - resolveManifestContractOwnerPluginId: vi.fn( - ({ value }: { value?: string }) => - ({ - firecrawl: "firecrawl", - gemini: "google", - pagefetch: "readerlab", - serpapi: "searchlab", - })[value ?? ""], - ), +vi.mock("../plugins/web-fetch-providers.runtime.js", () => ({ + resolvePluginWebFetchProviders: vi.fn((params: { config?: Record }) => [ + { + pluginId: "firecrawl", + id: "firecrawl", + credentialPath: "plugins.entries.firecrawl.config.webFetch.apiKey", + getConfiguredCredentialValue: (config?: { + plugins?: { + entries?: { + firecrawl?: { config?: { webFetch?: { apiKey?: unknown } } }; + }; + }; + }) => config?.plugins?.entries?.firecrawl?.config?.webFetch?.apiKey, + getConfiguredCredentialFallback: () => ({ + path: "plugins.entries.firecrawl.config.webSearch.apiKey", + value: ( + params.config as { + plugins?: { + entries?: { + firecrawl?: { config?: { webSearch?: { apiKey?: unknown } } }; + }; + }; + } + )?.plugins?.entries?.firecrawl?.config?.webSearch?.apiKey, + }), + getCredentialValue: (): undefined => undefined, + }, + { + pluginId: "other-fetch", + id: "other", + credentialPath: "plugins.entries.other-fetch.config.webFetch.apiKey", + getConfiguredCredentialValue: (config?: { + plugins?: { + entries?: { + "other-fetch"?: { config?: { webFetch?: { apiKey?: unknown } } }; + }; + }; + }) => config?.plugins?.entries?.["other-fetch"]?.config?.webFetch?.apiKey, + getConfiguredCredentialFallback: () => ({ + path: "plugins.entries.other-fetch.config.webSearch.apiKey", + value: undefined, + }), + getCredentialValue: (): undefined => undefined, + }, + ]), +})); + +vi.mock("../plugins/web-search-providers.runtime.js", () => ({ + resolvePluginWebSearchProviders: vi.fn(() => [ + { + pluginId: "brave", + id: "brave", + credentialPath: "plugins.entries.brave.config.webSearch.apiKey", + getConfiguredCredentialValue: (config?: { + tools?: { web?: { search?: { apiKey?: unknown } } }; + plugins?: { + entries?: { + brave?: { config?: { webSearch?: { apiKey?: unknown } } }; + }; + }; + }) => + config?.plugins?.entries?.brave?.config?.webSearch?.apiKey ?? + config?.tools?.web?.search?.apiKey, + getConfiguredCredentialFallback: (): undefined => undefined, + getCredentialValue: (searchConfig?: { apiKey?: unknown }) => searchConfig?.apiKey, + }, + { + pluginId: "firecrawl", + id: "firecrawl", + credentialPath: "plugins.entries.firecrawl.config.webSearch.apiKey", + getConfiguredCredentialValue: (config?: { + plugins?: { + entries?: { + firecrawl?: { + config?: { webFetch?: { apiKey?: unknown }; webSearch?: { apiKey?: unknown } }; + }; + }; + }; + }) => config?.plugins?.entries?.firecrawl?.config?.webSearch?.apiKey, + getConfiguredCredentialFallback: (config?: { + plugins?: { + entries?: { + firecrawl?: { config?: { webFetch?: { apiKey?: unknown } } }; + }; + }; + }) => { + const apiKey = config?.plugins?.entries?.firecrawl?.config?.webFetch?.apiKey; + return apiKey === undefined + ? undefined + : { + path: "plugins.entries.firecrawl.config.webFetch.apiKey", + value: apiKey, + }; + }, + getCredentialValue: (): undefined => undefined, + }, + { + pluginId: "exa", + id: "exa", + credentialPath: "plugins.entries.exa.config.webSearch.apiKey", + getConfiguredCredentialValue: (config?: { + plugins?: { + entries?: { + exa?: { config?: { webSearch?: { apiKey?: unknown } } }; + }; + }; + }) => config?.plugins?.entries?.exa?.config?.webSearch?.apiKey, + getConfiguredCredentialFallback: (): undefined => undefined, + getCredentialValue: (searchConfig?: { exa?: { apiKey?: unknown } }) => + searchConfig?.exa?.apiKey, + }, + { + pluginId: "gemini", + id: "gemini", + credentialPath: "plugins.entries.gemini.config.webSearch.apiKey", + getConfiguredCredentialValue: (): undefined => undefined, + getConfiguredCredentialFallback: (config?: { + models?: { providers?: { google?: { apiKey?: unknown } } }; + }) => ({ + path: "models.providers.google.apiKey", + value: config?.models?.providers?.google?.apiKey, + }), + getCredentialValue: (): undefined => undefined, + }, + ]), })); import { getAgentRuntimeCommandSecretTargetIds, - getMemoryEmbeddingCommandSecretTargetIds, + getCapabilityWebFetchCommandSecretTargets, + getCapabilityWebFetchCommandSecretTargetIds, + getCapabilityWebSearchCommandSecretTargets, + getCapabilityWebSearchCommandSecretTargetIds, getModelsCommandSecretTargetIds, getQrRemoteCommandSecretTargetIds, getScopedChannelsCommandSecretTargets, getSecurityAuditCommandSecretTargetIds, - getTtsCommandSecretTargetIds, - getWebFetchCommandSecretTargets, - getWebFetchCommandSecretTargetIds, - getWebSearchCommandSecretTargets, - getWebSearchCommandSecretTargetIds, } from "./command-secret-targets.js"; describe("command secret target ids", () => { @@ -136,223 +253,479 @@ describe("command secret target ids", () => { expect(ids.has("agents.list[].memorySearch.remote.apiKey")).toBe(true); expect(ids.has("plugins.entries.firecrawl.config.webFetch.apiKey")).toBe(true); expect(ids.has("plugins.entries.exa.config.webSearch.apiKey")).toBe(true); - expect(ids.has("tools.web.fetch.firecrawl.apiKey")).toBe(true); expect(ids.has("channels.discord.token")).toBe(false); }); - it("scopes capability target ids to the provider family", () => { - const webSearch = getWebSearchCommandSecretTargetIds(); - expect(webSearch).toEqual( - new Set([ - "plugins.entries.exa.config.webSearch.apiKey", - "plugins.entries.firecrawl.config.webSearch.apiKey", - "plugins.entries.google.config.webSearch.apiKey", - "plugins.entries.searchlab.config.webSearch.apiKey", - "tools.web.search.apiKey", - ]), - ); - expect(webSearch.has("plugins.entries.firecrawl.config.webFetch.apiKey")).toBe(false); - expect(webSearch.has("models.providers.*.apiKey")).toBe(false); - - const webFetch = getWebFetchCommandSecretTargetIds(); - expect(webFetch).toEqual( - new Set([ - "plugins.entries.firecrawl.config.webFetch.apiKey", - "plugins.entries.readerlab.config.webFetch.apiKey", - "tools.web.fetch.firecrawl.apiKey", - ]), - ); - expect(webFetch.has("plugins.entries.exa.config.webSearch.apiKey")).toBe(false); - - const tts = getTtsCommandSecretTargetIds(); - expect(tts.has("models.providers.*.apiKey")).toBe(true); - expect(tts.has("messages.tts.providers.*.apiKey")).toBe(true); - expect(tts.has("plugins.entries.exa.config.webSearch.apiKey")).toBe(false); - - const memory = getMemoryEmbeddingCommandSecretTargetIds(); - expect(memory.has("models.providers.*.apiKey")).toBe(true); - expect(memory.has("agents.defaults.memorySearch.remote.apiKey")).toBe(true); - expect(memory.has("messages.tts.providers.*.apiKey")).toBe(false); + it("scopes capability web search commands to search credential surfaces only", () => { + const ids = getCapabilityWebSearchCommandSecretTargetIds(); + expect(ids.has("tools.web.search.apiKey")).toBe(true); + expect(ids.has("tools.web.search.*.apiKey")).toBe(true); + expect(ids.has("plugins.entries.exa.config.webSearch.apiKey")).toBe(true); + expect(ids.has("plugins.entries.firecrawl.config.webFetch.apiKey")).toBe(false); + expect(ids.has("plugins.entries.voice-call.config.twilio.authToken")).toBe(false); + expect(ids.has("models.providers.openai.apiKey")).toBe(false); + expect(ids.has("agents.defaults.memorySearch.remote.apiKey")).toBe(false); + expect(ids.has("messages.tts.providers.openai.apiKey")).toBe(false); + expect(ids.has("skills.entries.demo.apiKey")).toBe(false); + expect(ids.has("channels.discord.token")).toBe(false); }); - it("selects model-provider fallback credentials for selected web search providers", () => { - const selected = getWebSearchCommandSecretTargets({ - config: { - tools: { web: { search: { provider: "gemini" } } }, - models: { - providers: { - google: { apiKey: { source: "env", id: "GEMINI_API_KEY" } }, - openai: { apiKey: { source: "env", id: "OPENAI_API_KEY" } }, - }, - }, - plugins: { - entries: { - firecrawl: { config: { webSearch: { apiKey: { source: "env", id: "FC" } } } }, - }, - }, - } as never, - provider: "gemini", - }); - - expect(selected.targetIds.has("models.providers.*.apiKey")).toBe(true); - expect(selected.allowedPaths).toEqual(new Set(["models.providers.google.apiKey"])); - - const pluginCredential = getWebSearchCommandSecretTargets({ - config: { - tools: { web: { search: { provider: "gemini" } } }, - models: { - providers: { - google: { apiKey: { source: "env", id: "GEMINI_API_KEY" } }, - }, - }, - plugins: { - entries: { - google: { config: { webSearch: { apiKey: { source: "env", id: "GOOGLE" } } } }, - }, - }, - } as never, - provider: "gemini", - }); - expect(pluginCredential.targetIds.has("models.providers.*.apiKey")).toBe(false); - expect(pluginCredential.allowedPaths).toEqual( - new Set(["plugins.entries.google.config.webSearch.apiKey"]), - ); - - const unselected = getWebSearchCommandSecretTargets({ - config: { - models: { - providers: { - google: { apiKey: { source: "env", id: "GEMINI_API_KEY" } }, - }, - }, - } as never, - provider: "tavily", - }); - expect(unselected.targetIds.has("models.providers.*.apiKey")).toBe(false); - expect(unselected.allowedPaths).toEqual(new Set()); - - const configuredOnly = getWebSearchCommandSecretTargets({ - config: { - tools: { web: { search: { provider: "gemini" } } }, - models: { - providers: { - google: { apiKey: { source: "env", id: "GEMINI_API_KEY" } }, - }, - }, - } as never, - }); - expect(configuredOnly.targetIds.has("models.providers.*.apiKey")).toBe(true); - expect(configuredOnly.allowedPaths).toEqual(new Set(["models.providers.google.apiKey"])); - - const externalOwner = getWebSearchCommandSecretTargets({ - config: { - plugins: { - entries: { - serpapi: { config: { webSearch: { apiKey: { source: "env", id: "WRONG" } } } }, - searchlab: { config: { webSearch: { apiKey: { source: "env", id: "SERP" } } } }, - }, - }, - } as never, - provider: "serpapi", - }); - expect(externalOwner.allowedPaths).toEqual( - new Set(["plugins.entries.searchlab.config.webSearch.apiKey"]), - ); + it("scopes capability web fetch commands to fetch credential surfaces only", () => { + const ids = getCapabilityWebFetchCommandSecretTargetIds(); + expect(ids.has("tools.web.search.apiKey")).toBe(false); + expect(ids.has("plugins.entries.exa.config.webSearch.apiKey")).toBe(false); + expect(ids.has("plugins.entries.firecrawl.config.webFetch.apiKey")).toBe(true); + expect(ids.has("plugins.entries.voice-call.config.twilio.authToken")).toBe(false); + expect(ids.has("models.providers.openai.apiKey")).toBe(false); + expect(ids.has("agents.defaults.memorySearch.remote.apiKey")).toBe(false); + expect(ids.has("messages.tts.providers.openai.apiKey")).toBe(false); + expect(ids.has("skills.entries.demo.apiKey")).toBe(false); + expect(ids.has("channels.discord.token")).toBe(false); }); - it("selects same-plugin web search fallback credentials for web fetch providers", () => { - const selected = getWebFetchCommandSecretTargets({ - config: { - tools: { web: { fetch: { provider: "firecrawl" } } }, - plugins: { - entries: { - exa: { config: { webSearch: { apiKey: { source: "env", id: "EXA" } } } }, - firecrawl: { config: { webSearch: { apiKey: { source: "env", id: "FC" } } } }, + it("scopes configured web search command targets to the selected provider", () => { + const scoped = getCapabilityWebSearchCommandSecretTargets({ + tools: { web: { search: { provider: "firecrawl", enabled: true } } }, + plugins: { + entries: { + firecrawl: { + config: { + webSearch: { + apiKey: { source: "env", provider: "default", id: "FIRECRAWL_API_KEY" }, + }, + }, + }, + exa: { + config: { + webSearch: { + apiKey: { source: "env", provider: "default", id: "EXA_API_KEY" }, + }, + }, }, }, - } as never, - provider: "firecrawl", - }); + }, + } as never); - expect(selected.targetIds.has("plugins.entries.firecrawl.config.webFetch.apiKey")).toBe(true); - expect(selected.targetIds.has("plugins.entries.firecrawl.config.webSearch.apiKey")).toBe(true); - expect(selected.allowedPaths).toEqual( + expect(scoped.targetIds).toEqual( new Set(["plugins.entries.firecrawl.config.webSearch.apiKey"]), ); + expect(scoped.forcedActivePaths).toBeUndefined(); + }); - const configuredOnly = getWebFetchCommandSecretTargets({ - config: { - tools: { web: { fetch: { provider: "firecrawl" } } }, - plugins: { - entries: { - firecrawl: { config: { webSearch: { apiKey: { source: "env", id: "FC" } } } }, - }, - }, - } as never, - }); - expect(configuredOnly.targetIds.has("plugins.entries.firecrawl.config.webSearch.apiKey")).toBe( - true, - ); - expect(configuredOnly.allowedPaths).toEqual( - new Set(["plugins.entries.firecrawl.config.webSearch.apiKey"]), - ); - - const fetchCredential = getWebFetchCommandSecretTargets({ - config: { - tools: { web: { fetch: { provider: "firecrawl" } } }, + it("uses an explicit search provider override when scoping command targets", () => { + const scoped = getCapabilityWebSearchCommandSecretTargets( + { + tools: { web: { search: { provider: "exa", enabled: true } } }, plugins: { entries: { firecrawl: { config: { - webFetch: { apiKey: { source: "env", id: "FC_FETCH" } }, - webSearch: { apiKey: { source: "env", id: "FC_SEARCH" } }, + webSearch: { + apiKey: { source: "env", provider: "default", id: "FIRECRAWL_API_KEY" }, + }, + }, + }, + exa: { + config: { + webSearch: { + apiKey: { source: "env", provider: "default", id: "EXA_API_KEY" }, + }, }, }, }, }, } as never, - provider: "firecrawl", - }); - expect(fetchCredential.targetIds.has("plugins.entries.firecrawl.config.webSearch.apiKey")).toBe( - false, - ); - expect(fetchCredential.allowedPaths).toEqual( - new Set(["plugins.entries.firecrawl.config.webFetch.apiKey"]), + { providerId: "firecrawl" }, ); - const externalOwner = getWebFetchCommandSecretTargets({ - config: { - plugins: { - entries: { - pagefetch: { config: { webFetch: { apiKey: { source: "env", id: "WRONG" } } } }, - readerlab: { config: { webFetch: { apiKey: { source: "env", id: "PAGE" } } } }, - }, - }, - } as never, - provider: "pagefetch", - }); - expect(externalOwner.allowedPaths).toEqual( - new Set(["plugins.entries.readerlab.config.webFetch.apiKey"]), + expect(scoped.targetIds).toEqual( + new Set(["plugins.entries.firecrawl.config.webSearch.apiKey"]), + ); + expect(scoped.forcedActivePaths).toEqual( + new Set(["plugins.entries.firecrawl.config.webSearch.apiKey"]), ); }); - it("keeps legacy Firecrawl web fetch targets available for selected fetch commands", () => { - const selected = getWebFetchCommandSecretTargets({ - config: { - tools: { - web: { - fetch: { - provider: "firecrawl", - firecrawl: { apiKey: { source: "env", id: "FIRECRAWL_API_KEY" } }, + it("keeps selected top-level web search credential refs in command targets", () => { + const scoped = getCapabilityWebSearchCommandSecretTargets({ + tools: { + web: { + search: { + provider: "brave", + enabled: true, + apiKey: { source: "env", provider: "default", id: "BRAVE_API_KEY" }, + }, + }, + }, + } as never); + + expect(scoped.targetIds).toEqual( + new Set(["plugins.entries.brave.config.webSearch.apiKey", "tools.web.search.apiKey"]), + ); + expect(scoped.forcedActivePaths).toBeUndefined(); + }); + + it("maps selected legacy scoped web search refs to registry targets", () => { + const scoped = getCapabilityWebSearchCommandSecretTargets({ + tools: { + web: { + search: { + provider: "exa", + enabled: true, + exa: { + apiKey: { source: "env", provider: "default", id: "EXA_API_KEY" }, + }, + }, + }, + }, + } as never); + + expect(scoped.targetIds).toEqual( + new Set(["plugins.entries.exa.config.webSearch.apiKey", "tools.web.search.*.apiKey"]), + ); + expect(scoped.allowedPaths).toEqual( + new Set(["plugins.entries.exa.config.webSearch.apiKey", "tools.web.search.exa.apiKey"]), + ); + expect(scoped.forcedActivePaths).toBeUndefined(); + }); + + it("skips stale legacy scoped web search refs when plugin credential wins", () => { + const scoped = getCapabilityWebSearchCommandSecretTargets({ + tools: { + web: { + search: { + provider: "exa", + enabled: true, + exa: { + apiKey: { source: "env", provider: "default", id: "STALE_EXA_API_KEY" }, + }, + }, + }, + }, + plugins: { + entries: { + exa: { + config: { + webSearch: { + apiKey: { source: "env", provider: "default", id: "EXA_API_KEY" }, + }, + }, + }, + }, + }, + } as never); + + expect(scoped.targetIds).toEqual(new Set(["plugins.entries.exa.config.webSearch.apiKey"])); + expect(scoped.allowedPaths).toBeUndefined(); + expect(scoped.forcedActivePaths).toBeUndefined(); + }); + + it("maps selected fallback credential paths to registry targets", () => { + const scoped = getCapabilityWebSearchCommandSecretTargets({ + tools: { web: { search: { provider: "gemini", enabled: true } } }, + models: { + providers: { + google: { + apiKey: { source: "env", provider: "default", id: "GOOGLE_API_KEY" }, + }, + }, + }, + } as never); + + expect(scoped.targetIds).toEqual( + new Set(["models.providers.*.apiKey", "plugins.entries.gemini.config.webSearch.apiKey"]), + ); + expect(scoped.allowedPaths).toEqual( + new Set(["models.providers.google.apiKey", "plugins.entries.gemini.config.webSearch.apiKey"]), + ); + expect(scoped.forcedActivePaths).toEqual(new Set(["models.providers.google.apiKey"])); + }); + + it("uses Firecrawl web fetch credentials as search fallback targets", () => { + const scoped = getCapabilityWebSearchCommandSecretTargets({ + tools: { web: { search: { provider: "firecrawl", enabled: true } } }, + plugins: { + entries: { + firecrawl: { + config: { + webFetch: { + apiKey: { source: "env", provider: "default", id: "FIRECRAWL_API_KEY" }, + }, + }, + }, + }, + }, + } as never); + + expect(scoped.targetIds).toEqual( + new Set([ + "plugins.entries.firecrawl.config.webFetch.apiKey", + "plugins.entries.firecrawl.config.webSearch.apiKey", + ]), + ); + expect(scoped.allowedPaths).toBeUndefined(); + expect(scoped.forcedActivePaths).toEqual( + new Set(["plugins.entries.firecrawl.config.webFetch.apiKey"]), + ); + }); + + it("includes configured search fallback targets for auto-detect", () => { + const scoped = getCapabilityWebSearchCommandSecretTargets({ + tools: { web: { search: { enabled: true } } }, + plugins: { + entries: { + firecrawl: { + config: { + webFetch: { + apiKey: { source: "env", provider: "default", id: "FIRECRAWL_API_KEY" }, + }, + }, + }, + }, + }, + } as never); + + expect(scoped.targetIds.has("plugins.entries.firecrawl.config.webFetch.apiKey")).toBe(true); + expect(scoped.allowedPaths).toEqual( + new Set(["plugins.entries.firecrawl.config.webFetch.apiKey"]), + ); + expect(scoped.forcedActivePaths).toEqual( + new Set(["plugins.entries.firecrawl.config.webFetch.apiKey"]), + ); + }); + + it("limits auto-detect wildcard fallback paths to the concrete configured path", () => { + const scoped = getCapabilityWebSearchCommandSecretTargets({ + tools: { web: { search: { enabled: true } } }, + models: { + providers: { + google: { + apiKey: { source: "env", provider: "default", id: "GOOGLE_API_KEY" }, + }, + openai: { + apiKey: { source: "env", provider: "default", id: "OPENAI_API_KEY" }, + }, + }, + }, + } as never); + + expect(scoped.targetIds.has("models.providers.*.apiKey")).toBe(true); + expect(scoped.allowedPaths).toEqual(new Set(["models.providers.google.apiKey"])); + expect(scoped.forcedActivePaths).toEqual(new Set(["models.providers.google.apiKey"])); + }); + + it("falls back to broad web search command targets for stale configured providers", () => { + const scoped = getCapabilityWebSearchCommandSecretTargets({ + tools: { web: { search: { provider: "stale", enabled: true } } }, + plugins: { + entries: { + exa: { + config: { + webSearch: { + apiKey: { source: "env", provider: "default", id: "EXA_API_KEY" }, + }, + }, + }, + }, + }, + } as never); + + expect(scoped.targetIds).toEqual(getCapabilityWebSearchCommandSecretTargetIds()); + expect(scoped.forcedActivePaths).toBeUndefined(); + }); + + it("includes configured search fallback targets for stale configured providers", () => { + const scoped = getCapabilityWebSearchCommandSecretTargets({ + tools: { web: { search: { provider: "stale", enabled: true } } }, + plugins: { + entries: { + firecrawl: { + config: { + webFetch: { + apiKey: { source: "env", provider: "default", id: "FIRECRAWL_API_KEY" }, + }, + }, + }, + }, + }, + } as never); + + expect(scoped.targetIds.has("plugins.entries.firecrawl.config.webFetch.apiKey")).toBe(true); + expect(scoped.allowedPaths).toEqual( + new Set(["plugins.entries.firecrawl.config.webFetch.apiKey"]), + ); + expect(scoped.forcedActivePaths).toEqual( + new Set(["plugins.entries.firecrawl.config.webFetch.apiKey"]), + ); + }); + + it("adds configured fetch fallback credential paths only when the fetch key is absent", () => { + const fallbackRef = { source: "env", provider: "default", id: "FIRECRAWL_API_KEY" }; + const fallbackOnly = getCapabilityWebFetchCommandSecretTargets({ + tools: { web: { fetch: { provider: "firecrawl", enabled: true } } }, + plugins: { + entries: { + firecrawl: { + config: { + webSearch: { + apiKey: fallbackRef, + }, + }, + }, + }, + }, + } as never); + + expect(fallbackOnly.targetIds.has("plugins.entries.firecrawl.config.webSearch.apiKey")).toBe( + true, + ); + expect(fallbackOnly.allowedPaths).toBeUndefined(); + expect(fallbackOnly.forcedActivePaths).toEqual( + new Set(["plugins.entries.firecrawl.config.webSearch.apiKey"]), + ); + + const fetchConfigured = getCapabilityWebFetchCommandSecretTargets({ + tools: { web: { fetch: { provider: "firecrawl", enabled: true } } }, + plugins: { + entries: { + firecrawl: { + config: { + webFetch: { + apiKey: { source: "env", provider: "default", id: "FIRECRAWL_API_KEY" }, + }, + webSearch: { + apiKey: fallbackRef, + }, + }, + }, + }, + }, + } as never); + + expect(fetchConfigured.targetIds.has("plugins.entries.firecrawl.config.webSearch.apiKey")).toBe( + false, + ); + expect(fetchConfigured.allowedPaths).toBeUndefined(); + expect(fetchConfigured.forcedActivePaths).toBeUndefined(); + }); + + it("does not add fallback credential paths for non-selected fetch providers", () => { + const scoped = getCapabilityWebFetchCommandSecretTargets({ + tools: { web: { fetch: { provider: "other", enabled: true } } }, + plugins: { + entries: { + firecrawl: { + config: { + webSearch: { + apiKey: { source: "env", provider: "default", id: "FIRECRAWL_API_KEY" }, + }, + }, + }, + }, + }, + } as never); + + expect(scoped.targetIds.has("plugins.entries.firecrawl.config.webSearch.apiKey")).toBe(false); + expect(scoped.targetIds.has("plugins.entries.firecrawl.config.webFetch.apiKey")).toBe(false); + expect(scoped.targetIds.has("plugins.entries.other-fetch.config.webFetch.apiKey")).toBe(true); + expect(scoped.allowedPaths).toBeUndefined(); + expect(scoped.forcedActivePaths).toBeUndefined(); + }); + + it("uses an explicit fetch provider override when scoping fallback credential paths", () => { + const scoped = getCapabilityWebFetchCommandSecretTargets( + { + tools: { web: { fetch: { enabled: true } } }, + plugins: { + entries: { + firecrawl: { + config: { + webSearch: { + apiKey: { source: "env", provider: "default", id: "FIRECRAWL_API_KEY" }, + }, + }, }, }, }, } as never, - provider: "firecrawl", - }); + { providerId: "firecrawl" }, + ); - expect(selected.targetIds.has("tools.web.fetch.firecrawl.apiKey")).toBe(true); - expect(selected.allowedPaths).toEqual(new Set(["tools.web.fetch.firecrawl.apiKey"])); + expect(scoped.targetIds.has("plugins.entries.firecrawl.config.webSearch.apiKey")).toBe(true); + expect(scoped.allowedPaths).toBeUndefined(); + expect(scoped.forcedActivePaths).toEqual( + new Set(["plugins.entries.firecrawl.config.webSearch.apiKey"]), + ); + }); + + it("includes configured fetch fallback targets for auto-detect", () => { + const scoped = getCapabilityWebFetchCommandSecretTargets({ + tools: { web: { fetch: { enabled: true } } }, + plugins: { + entries: { + firecrawl: { + config: { + webSearch: { + apiKey: { source: "env", provider: "default", id: "FIRECRAWL_API_KEY" }, + }, + }, + }, + }, + }, + } as never); + + expect(scoped.targetIds.has("plugins.entries.firecrawl.config.webSearch.apiKey")).toBe(true); + expect(scoped.allowedPaths).toEqual( + new Set(["plugins.entries.firecrawl.config.webSearch.apiKey"]), + ); + expect(scoped.forcedActivePaths).toEqual( + new Set(["plugins.entries.firecrawl.config.webSearch.apiKey"]), + ); + }); + + it("includes configured fetch fallback targets for stale configured providers", () => { + const scoped = getCapabilityWebFetchCommandSecretTargets({ + tools: { web: { fetch: { provider: "stale", enabled: true } } }, + plugins: { + entries: { + firecrawl: { + config: { + webSearch: { + apiKey: { source: "env", provider: "default", id: "FIRECRAWL_API_KEY" }, + }, + }, + }, + }, + }, + } as never); + + expect(scoped.targetIds.has("plugins.entries.firecrawl.config.webSearch.apiKey")).toBe(true); + expect(scoped.allowedPaths).toEqual( + new Set(["plugins.entries.firecrawl.config.webSearch.apiKey"]), + ); + expect(scoped.forcedActivePaths).toEqual( + new Set(["plugins.entries.firecrawl.config.webSearch.apiKey"]), + ); + }); + + it("falls back to broad web fetch command targets for stale configured providers", () => { + const scoped = getCapabilityWebFetchCommandSecretTargets({ + tools: { web: { fetch: { provider: "stale", enabled: true } } }, + plugins: { + entries: { + firecrawl: { + config: { + webFetch: { + apiKey: { source: "env", provider: "default", id: "FIRECRAWL_API_KEY" }, + }, + }, + }, + }, + }, + } as never); + + expect(scoped.targetIds).toEqual(getCapabilityWebFetchCommandSecretTargetIds()); + expect(scoped.forcedActivePaths).toBeUndefined(); }); it("includes channel targets for agent runtime when delivery needs them", () => { diff --git a/src/cli/command-secret-targets.ts b/src/cli/command-secret-targets.ts index ba6f15abc8a..876d4a56303 100644 --- a/src/cli/command-secret-targets.ts +++ b/src/cli/command-secret-targets.ts @@ -1,16 +1,18 @@ import { listReadOnlyChannelPluginsForConfig } from "../channels/plugins/read-only.js"; import type { OpenClawConfig } from "../config/types.openclaw.js"; -import { resolveManifestContractOwnerPluginId } from "../plugins/plugin-registry.js"; +import type { + PluginWebFetchProviderEntry, + PluginWebSearchProviderEntry, +} from "../plugins/types.js"; +import { resolvePluginWebFetchProviders } from "../plugins/web-fetch-providers.runtime.js"; +import { resolvePluginWebSearchProviders } from "../plugins/web-search-providers.runtime.js"; import { normalizeOptionalAccountId } from "../routing/session-key.js"; import { loadChannelSecretContractApi } from "../secrets/channel-contract-api.js"; import { discoverConfigSecretTargetsByIds, listSecretTargetRegistryEntries, } from "../secrets/target-registry.js"; -import { - normalizeOptionalLowercaseString, - normalizeOptionalString, -} from "../shared/string-coerce.js"; +import { normalizeOptionalString } from "../shared/string-coerce.js"; const STATIC_QR_REMOTE_TARGET_IDS = ["gateway.remote.token", "gateway.remote.password"] as const; const STATIC_MODEL_TARGET_IDS = [ @@ -36,20 +38,7 @@ const STATIC_AGENT_RUNTIME_BASE_TARGET_IDS = [ "messages.tts.providers.*.apiKey", "skills.entries.*.apiKey", "tools.web.search.apiKey", - "tools.web.fetch.firecrawl.apiKey", ] as const; -const STATIC_MEMORY_EMBEDDING_TARGET_IDS = [ - ...STATIC_MODEL_TARGET_IDS, - "agents.defaults.memorySearch.remote.apiKey", - "agents.list[].memorySearch.remote.apiKey", -] as const; -const STATIC_TTS_TARGET_IDS = [ - ...STATIC_MODEL_TARGET_IDS, - "agents.list[].tts.providers.*.apiKey", - "messages.tts.providers.*.apiKey", -] as const; -const STATIC_WEB_SEARCH_TARGET_IDS = ["tools.web.search.apiKey"] as const; -const STATIC_WEB_FETCH_TARGET_IDS = ["tools.web.fetch.firecrawl.apiKey"] as const; const STATIC_STATUS_TARGET_IDS = [ "agents.defaults.memorySearch.remote.apiKey", "agents.list[].memorySearch.remote.apiKey", @@ -74,14 +63,29 @@ type CommandSecretTargets = { status: string[]; securityAudit: string[]; }; - -type CommandSecretTargetSelection = { +type CommandSecretTargetScope = { targetIds: Set; allowedPaths?: Set; + forcedActivePaths?: Set; }; +type SelectedProviderTargetIds = { + matchedProvider: boolean; + targetIds: string[]; + targetPaths: string[]; + allowedPaths: string[]; + fallbackTargetIds: string[]; + fallbackPaths: string[]; +}; + +const STATIC_CAPABILITY_WEB_SEARCH_TARGET_IDS = [ + "tools.web.search.apiKey", + "tools.web.search.*.apiKey", +] as const; let cachedCommandSecretTargets: CommandSecretTargets | undefined; let cachedAgentRuntimeBaseTargetIds: string[] | undefined; +let cachedCapabilityWebFetchTargetIds: string[] | undefined; +let cachedCapabilityWebSearchTargetIds: string[] | undefined; let cachedChannelSecretTargetIds: string[] | undefined; function getChannelSecretTargetIds(): string[] { @@ -89,34 +93,464 @@ function getChannelSecretTargetIds(): string[] { return cachedChannelSecretTargetIds; } -function isPluginWebCredentialTargetId(id: string, configPathFilter?: string): boolean { +function isPluginWebCredentialTargetId(id: string): boolean { const segments = id.split("."); if (segments[0] !== "plugins" || segments[1] !== "entries" || segments[3] !== "config") { return false; } const configPath = segments.slice(4).join("."); - if (configPathFilter) { - return configPath === configPathFilter; - } return configPath === "webSearch.apiKey" || configPath === "webFetch.apiKey"; } -function getPluginWebCredentialTargetIds(configPath: "webSearch.apiKey" | "webFetch.apiKey") { - return listSecretTargetRegistryEntries() +function isPluginWebSearchCredentialTargetId(id: string): boolean { + const segments = id.split("."); + if (segments[0] !== "plugins" || segments[1] !== "entries" || segments[3] !== "config") { + return false; + } + return segments.slice(4).join(".") === "webSearch.apiKey"; +} + +function isPluginWebFetchCredentialTargetId(id: string): boolean { + const segments = id.split("."); + if (segments[0] !== "plugins" || segments[1] !== "entries" || segments[3] !== "config") { + return false; + } + return segments.slice(4).join(".") === "webFetch.apiKey"; +} + +function getCapabilityWebSearchTargetIds(): string[] { + cachedCapabilityWebSearchTargetIds ??= [ + ...new Set([ + ...STATIC_CAPABILITY_WEB_SEARCH_TARGET_IDS, + ...listSecretTargetRegistryEntries() + .map((entry) => entry.id) + .filter(isPluginWebSearchCredentialTargetId), + ]), + ].toSorted(); + return cachedCapabilityWebSearchTargetIds; +} + +function getCapabilityWebFetchTargetIds(): string[] { + cachedCapabilityWebFetchTargetIds ??= listSecretTargetRegistryEntries() + .map((entry) => entry.id) + .filter(isPluginWebFetchCredentialTargetId) + .toSorted(); + return cachedCapabilityWebFetchTargetIds; +} + +function isConfiguredSecretCandidate(value: unknown): boolean { + if (typeof value === "string") { + return value.trim().length > 0; + } + return value !== undefined && value !== null; +} + +function resolveFetchConfig(config: OpenClawConfig): Record | undefined { + const fetch = config.tools?.web?.fetch; + return fetch && typeof fetch === "object" && !Array.isArray(fetch) + ? (fetch as Record) + : undefined; +} + +function resolveSearchConfig(config: OpenClawConfig): Record | undefined { + const search = config.tools?.web?.search; + return search && typeof search === "object" && !Array.isArray(search) + ? (search as Record) + : undefined; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function pathPatternMatchesConcretePath(pathPattern: string, path: string): boolean { + const pathSegments = path.split("."); + const patternSegments = pathPattern.split("."); + let pathIndex = 0; + for (const segment of patternSegments) { + if (segment === "*") { + if (!pathSegments[pathIndex]) { + return false; + } + pathIndex += 1; + continue; + } + if (segment.endsWith("[]")) { + const field = segment.slice(0, -2); + if (pathSegments[pathIndex] !== field || !/^\d+$/.test(pathSegments[pathIndex + 1] ?? "")) { + return false; + } + pathIndex += 2; + continue; + } + if (pathSegments[pathIndex] !== segment) { + return false; + } + pathIndex += 1; + } + return pathIndex === pathSegments.length; +} + +function targetIdsForConfigPath(path: string): string[] { + return listSecretTargetRegistryEntries() + .filter((entry) => pathPatternMatchesConcretePath(entry.pathPattern ?? entry.id, path)) .map((entry) => entry.id) - .filter((id) => isPluginWebCredentialTargetId(id, configPath)) .toSorted(); } -function pluginIdFromWebCredentialPath( - path: string, - configPath: "webSearch.apiKey" | "webFetch.apiKey", -): string | undefined { - const match = /^plugins\.entries\.([^.]+)\.config\.(webSearch|webFetch)\.apiKey$/.exec(path); - if (!match) { - return undefined; +function addConfigPathTargets(params: { + path: string; + targetIds: Set; + targetPaths: Set; + allowedPaths: Set; +}): boolean { + const targetIds = targetIdsForConfigPath(params.path); + if (targetIds.length === 0) { + return false; } - return match[2] === configPath.split(".")[0] ? match[1] : undefined; + for (const targetId of targetIds) { + params.targetIds.add(targetId); + if (targetId !== params.path) { + params.allowedPaths.add(params.path); + } + } + params.targetPaths.add(params.path); + return true; +} + +function normalizeProviderId(value: unknown): string | undefined { + return typeof value === "string" && value.trim() ? value.trim().toLowerCase() : undefined; +} + +function discoverForcedActivePaths( + config: OpenClawConfig, + targetIds: ReadonlySet, + allowedPaths?: ReadonlySet, +): Set | undefined { + const forcedActivePaths = new Set(); + for (const target of discoverConfigSecretTargetsByIds(config, targetIds)) { + if (allowedPaths && !allowedPaths.has(target.path)) { + continue; + } + forcedActivePaths.add(target.path); + } + return forcedActivePaths.size > 0 ? forcedActivePaths : undefined; +} + +function discoverConfiguredAllowedPaths( + config: OpenClawConfig, + targetIds: ReadonlySet, +): Set | undefined { + const allowedPaths = new Set(); + for (const target of discoverConfigSecretTargetsByIds(config, targetIds)) { + allowedPaths.add(target.path); + } + return allowedPaths.size > 0 ? allowedPaths : undefined; +} + +function mergeConfiguredAllowedPaths(params: { + config: OpenClawConfig; + baseTargetIds: ReadonlySet; + concreteFallbackPaths: ReadonlySet; +}): Set | undefined { + const allowedPaths = new Set(); + for (const path of discoverConfiguredAllowedPaths(params.config, params.baseTargetIds) ?? []) { + allowedPaths.add(path); + } + for (const path of params.concreteFallbackPaths) { + allowedPaths.add(path); + } + return allowedPaths.size > 0 ? allowedPaths : undefined; +} + +function resolveSelectedWebFetchProviderId( + config: OpenClawConfig, + providerId?: string | null, +): string | undefined { + return ( + normalizeProviderId(providerId) ?? normalizeProviderId(resolveFetchConfig(config)?.provider) + ); +} + +function resolveSelectedWebSearchProviderId( + config: OpenClawConfig, + providerId?: string | null, +): string | undefined { + return ( + normalizeProviderId(providerId) ?? normalizeProviderId(resolveSearchConfig(config)?.provider) + ); +} + +function hasConfiguredFetchCredential(params: { + provider: PluginWebFetchProviderEntry; + config: OpenClawConfig; +}): boolean { + return ( + isConfiguredSecretCandidate(params.provider.getConfiguredCredentialValue?.(params.config)) || + isConfiguredSecretCandidate( + params.provider.getCredentialValue(resolveFetchConfig(params.config)), + ) + ); +} + +function hasConfiguredSearchCredential(params: { + provider: PluginWebSearchProviderEntry; + config: OpenClawConfig; +}): boolean { + return ( + isConfiguredSecretCandidate(params.provider.getConfiguredCredentialValue?.(params.config)) || + isConfiguredSecretCandidate( + params.provider.getCredentialValue(resolveSearchConfig(params.config)), + ) + ); +} + +function addConfiguredSearchCredentialTargetIds(params: { + config: OpenClawConfig; + provider: PluginWebSearchProviderEntry; + targetIds: Set; + targetPaths: Set; + allowedPaths: Set; +}): void { + const searchConfig = resolveSearchConfig(params.config); + if (!searchConfig) { + return; + } + const configuredCredential = params.provider.getCredentialValue(searchConfig); + if (!isConfiguredSecretCandidate(configuredCredential)) { + return; + } + const pluginCredential = params.provider.getConfiguredCredentialValue?.(params.config); + if (isConfiguredSecretCandidate(pluginCredential) && configuredCredential !== pluginCredential) { + return; + } + if (configuredCredential === searchConfig.apiKey) { + addConfigPathTargets({ ...params, path: "tools.web.search.apiKey" }); + } + const scopedConfig = searchConfig[params.provider.id]; + if (isRecord(scopedConfig) && configuredCredential === scopedConfig.apiKey) { + addConfigPathTargets({ + ...params, + path: `tools.web.search.${params.provider.id}.apiKey`, + }); + } +} + +function getCapabilityWebSearchSelectedProviderTargetIds( + config: OpenClawConfig, + providerId?: string | null, +): SelectedProviderTargetIds { + const selectedProviderId = resolveSelectedWebSearchProviderId(config, providerId); + if (!selectedProviderId) { + return { + matchedProvider: false, + targetIds: [], + targetPaths: [], + allowedPaths: [], + fallbackTargetIds: [], + fallbackPaths: [], + }; + } + const targetIds = new Set(); + const targetPaths = new Set(); + const allowedPaths = new Set(); + const fallbackTargetIds = new Set(); + const fallbackPaths = new Set(); + const providers = resolvePluginWebSearchProviders({ + config, + bundledAllowlistCompat: true, + }).filter((provider) => provider.id === selectedProviderId); + for (const provider of providers) { + if (provider.credentialPath.trim()) { + addConfigPathTargets({ + path: provider.credentialPath, + targetIds, + targetPaths, + allowedPaths, + }); + } + addConfiguredSearchCredentialTargetIds({ + config, + provider, + targetIds, + targetPaths, + allowedPaths, + }); + if (hasConfiguredSearchCredential({ provider, config })) { + continue; + } + const fallbackPath = provider.getConfiguredCredentialFallback?.(config)?.path?.trim(); + if (fallbackPath) { + const before = new Set(targetIds); + const added = addConfigPathTargets({ + path: fallbackPath, + targetIds, + targetPaths, + allowedPaths, + }); + for (const targetId of targetIds) { + if (!before.has(targetId)) { + fallbackTargetIds.add(targetId); + } + } + if (added) { + fallbackPaths.add(fallbackPath); + } + } + } + return { + matchedProvider: providers.length > 0, + targetIds: [...targetIds].toSorted(), + targetPaths: [...targetPaths].toSorted(), + allowedPaths: [...allowedPaths].toSorted(), + fallbackTargetIds: [...fallbackTargetIds].toSorted(), + fallbackPaths: [...fallbackPaths].toSorted(), + }; +} + +function getCapabilityWebFetchSelectedProviderTargetIds( + config: OpenClawConfig, + providerId?: string | null, +): SelectedProviderTargetIds { + const selectedProviderId = resolveSelectedWebFetchProviderId(config, providerId); + if (!selectedProviderId) { + return { + matchedProvider: false, + targetIds: [], + targetPaths: [], + allowedPaths: [], + fallbackTargetIds: [], + fallbackPaths: [], + }; + } + const targetIds = new Set(); + const targetPaths = new Set(); + const allowedPaths = new Set(); + const fallbackTargetIds = new Set(); + const fallbackPaths = new Set(); + const providers = resolvePluginWebFetchProviders({ + config, + bundledAllowlistCompat: true, + }).filter((provider) => provider.id === selectedProviderId); + for (const provider of providers) { + if (provider.credentialPath.trim()) { + addConfigPathTargets({ + path: provider.credentialPath, + targetIds, + targetPaths, + allowedPaths, + }); + } + if (hasConfiguredFetchCredential({ provider, config })) { + continue; + } + const fallbackPath = provider.getConfiguredCredentialFallback?.(config)?.path?.trim(); + if (fallbackPath) { + const before = new Set(targetIds); + const added = addConfigPathTargets({ + path: fallbackPath, + targetIds, + targetPaths, + allowedPaths, + }); + for (const targetId of targetIds) { + if (!before.has(targetId)) { + fallbackTargetIds.add(targetId); + } + } + if (added) { + fallbackPaths.add(fallbackPath); + } + } + } + return { + matchedProvider: providers.length > 0, + targetIds: [...targetIds].toSorted(), + targetPaths: [...targetPaths].toSorted(), + allowedPaths: [...allowedPaths].toSorted(), + fallbackTargetIds: [...fallbackTargetIds].toSorted(), + fallbackPaths: [...fallbackPaths].toSorted(), + }; +} + +function getCapabilityWebSearchAutoDetectTargets(config: OpenClawConfig): CommandSecretTargetScope { + const baseTargetIds = getCapabilityWebSearchCommandSecretTargetIds(); + const targetIds = new Set(baseTargetIds); + const fallbackTargetIds = new Set(); + const fallbackPaths = new Set(); + const providers = resolvePluginWebSearchProviders({ + config, + bundledAllowlistCompat: true, + }); + for (const provider of providers) { + if (hasConfiguredSearchCredential({ provider, config })) { + continue; + } + const fallback = provider.getConfiguredCredentialFallback?.(config); + const fallbackPath = fallback?.path?.trim(); + if (!fallbackPath || !isConfiguredSecretCandidate(fallback?.value)) { + continue; + } + for (const targetId of targetIdsForConfigPath(fallbackPath)) { + targetIds.add(targetId); + fallbackTargetIds.add(targetId); + } + fallbackPaths.add(fallbackPath); + } + if (fallbackTargetIds.size === 0) { + return { targetIds }; + } + const allowedPaths = mergeConfiguredAllowedPaths({ + config, + baseTargetIds, + concreteFallbackPaths: fallbackPaths, + }); + const forcedActivePaths = discoverForcedActivePaths(config, fallbackTargetIds, allowedPaths); + return { + targetIds, + ...(allowedPaths ? { allowedPaths } : {}), + ...(forcedActivePaths ? { forcedActivePaths } : {}), + }; +} + +function getCapabilityWebFetchAutoDetectTargets(config: OpenClawConfig): CommandSecretTargetScope { + const baseTargetIds = getCapabilityWebFetchCommandSecretTargetIds(); + const targetIds = new Set(baseTargetIds); + const fallbackTargetIds = new Set(); + const fallbackPaths = new Set(); + const providers = resolvePluginWebFetchProviders({ + config, + bundledAllowlistCompat: true, + }); + for (const provider of providers) { + if (hasConfiguredFetchCredential({ provider, config })) { + continue; + } + const fallback = provider.getConfiguredCredentialFallback?.(config); + const fallbackPath = fallback?.path?.trim(); + if (!fallbackPath || !isConfiguredSecretCandidate(fallback?.value)) { + continue; + } + for (const targetId of targetIdsForConfigPath(fallbackPath)) { + targetIds.add(targetId); + fallbackTargetIds.add(targetId); + } + fallbackPaths.add(fallbackPath); + } + if (fallbackTargetIds.size === 0) { + return { targetIds }; + } + const allowedPaths = mergeConfiguredAllowedPaths({ + config, + baseTargetIds, + concreteFallbackPaths: fallbackPaths, + }); + const forcedActivePaths = discoverForcedActivePaths(config, fallbackTargetIds, allowedPaths); + return { + targetIds, + ...(allowedPaths ? { allowedPaths } : {}), + ...(forcedActivePaths ? { forcedActivePaths } : {}), + }; } function getAgentRuntimeBaseTargetIds(): string[] { @@ -124,7 +558,7 @@ function getAgentRuntimeBaseTargetIds(): string[] { ...STATIC_AGENT_RUNTIME_BASE_TARGET_IDS, ...listSecretTargetRegistryEntries() .map((entry) => entry.id) - .filter((id) => isPluginWebCredentialTargetId(id)) + .filter(isPluginWebCredentialTargetId) .toSorted(), ]; return cachedAgentRuntimeBaseTargetIds; @@ -205,16 +639,6 @@ function toTargetIdSet(values: readonly string[]): Set { return new Set(values); } -function mergeTargetIdSets(...sets: ReadonlyArray>): Set { - const merged = new Set(); - for (const set of sets) { - for (const value of set) { - merged.add(value); - } - } - return merged; -} - function selectChannelTargetIds(channel?: string): Set { const commandSecretTargets = getCommandSecretTargets(); if (!channel) { @@ -289,204 +713,6 @@ export function getModelsCommandSecretTargetIds(): Set { return toTargetIdSet(STATIC_MODEL_TARGET_IDS); } -export function getMemoryEmbeddingCommandSecretTargetIds(): Set { - return toTargetIdSet(STATIC_MEMORY_EMBEDDING_TARGET_IDS); -} - -export function getTtsCommandSecretTargetIds(): Set { - return toTargetIdSet(STATIC_TTS_TARGET_IDS); -} - -export function getWebSearchCommandSecretTargetIds(): Set { - return toTargetIdSet([ - ...STATIC_WEB_SEARCH_TARGET_IDS, - ...getPluginWebCredentialTargetIds("webSearch.apiKey"), - ]); -} - -export function getWebFetchCommandSecretTargetIds(): Set { - return toTargetIdSet([ - ...STATIC_WEB_FETCH_TARGET_IDS, - ...getPluginWebCredentialTargetIds("webFetch.apiKey"), - ]); -} - -function getConfiguredWebProviderId( - config: OpenClawConfig, - kind: "search" | "fetch", -): string | undefined { - const webConfig = config.tools?.web?.[kind]; - return normalizeOptionalLowercaseString( - webConfig && typeof webConfig === "object" ? webConfig.provider : undefined, - ); -} - -function configuredTargetPaths(config: OpenClawConfig, targetIds: Set): Set { - return new Set(discoverConfigSecretTargetsByIds(config, targetIds).map((target) => target.path)); -} - -function modelProviderCredentialFallbackPathForWebSearchProvider( - providerId: string | undefined, -): string | undefined { - switch (providerId) { - case "gemini": - return "models.providers.google.apiKey"; - case "ollama": - return "models.providers.ollama.apiKey"; - default: - return undefined; - } -} - -function resolveSelectedWebProviderPluginId(params: { - config: OpenClawConfig; - providerId: string | undefined; - contract: "webSearchProviders" | "webFetchProviders"; -}): string | undefined { - if (!params.providerId) { - return undefined; - } - return ( - resolveManifestContractOwnerPluginId({ - config: params.config, - contract: params.contract, - value: params.providerId, - }) ?? params.providerId - ); -} - -function pathForPluginCredential( - paths: ReadonlySet, - pluginId: string | undefined, - configPath: "webSearch.apiKey" | "webFetch.apiKey", -): string | undefined { - if (!pluginId) { - return undefined; - } - for (const path of paths) { - if (pluginIdFromWebCredentialPath(path, configPath) === pluginId) { - return path; - } - } - return undefined; -} - -export function getWebSearchCommandSecretTargets(params: { - config: OpenClawConfig; - provider?: string | null; -}): CommandSecretTargetSelection { - const webSearchTargetIds = getWebSearchCommandSecretTargetIds(); - const targetIds = new Set(webSearchTargetIds); - const providerId = - normalizeOptionalLowercaseString(params.provider) ?? - getConfiguredWebProviderId(params.config, "search"); - const webSearchPaths = configuredTargetPaths(params.config, webSearchTargetIds); - if (!providerId) { - return { targetIds, allowedPaths: webSearchPaths }; - } - - const allowedPaths = new Set(); - const selectedPluginId = resolveSelectedWebProviderPluginId({ - config: params.config, - providerId, - contract: "webSearchProviders", - }); - const pluginCredentialPath = pathForPluginCredential( - webSearchPaths, - selectedPluginId, - "webSearch.apiKey", - ); - if (pluginCredentialPath) { - allowedPaths.add(pluginCredentialPath); - return { targetIds, allowedPaths }; - } - - const fallbackPath = modelProviderCredentialFallbackPathForWebSearchProvider(providerId); - if (fallbackPath) { - const modelPaths = configuredTargetPaths(params.config, getModelsCommandSecretTargetIds()); - if (modelPaths.has(fallbackPath)) { - targetIds.add("models.providers.*.apiKey"); - allowedPaths.add(fallbackPath); - return { targetIds, allowedPaths }; - } - } - - if (webSearchPaths.has("tools.web.search.apiKey")) { - allowedPaths.add("tools.web.search.apiKey"); - } - return { targetIds, allowedPaths }; -} - -export function getWebFetchCommandSecretTargets(params: { - config: OpenClawConfig; - provider?: string | null; -}): CommandSecretTargetSelection { - const webFetchTargetIds = getWebFetchCommandSecretTargetIds(); - const webSearchTargetIds = getWebSearchCommandSecretTargetIds(); - const webFetchPaths = configuredTargetPaths(params.config, webFetchTargetIds); - const webSearchPaths = configuredTargetPaths(params.config, webSearchTargetIds); - const providerId = - normalizeOptionalLowercaseString(params.provider) ?? - getConfiguredWebProviderId(params.config, "fetch"); - const selectedPluginId = resolveSelectedWebProviderPluginId({ - config: params.config, - providerId, - contract: "webFetchProviders", - }); - const webFetchPluginIds = new Set( - [...getPluginWebCredentialTargetIds("webFetch.apiKey")] - .map((id) => pluginIdFromWebCredentialPath(id, "webFetch.apiKey")) - .filter((id): id is string => Boolean(id)), - ); - const candidatePluginIds = new Set(); - if (selectedPluginId) { - candidatePluginIds.add(selectedPluginId); - } - for (const path of webFetchPaths) { - const pluginId = pluginIdFromWebCredentialPath(path, "webFetch.apiKey"); - if (!selectedPluginId && pluginId) { - candidatePluginIds.add(pluginId); - } - } - for (const path of webSearchPaths) { - const pluginId = pluginIdFromWebCredentialPath(path, "webSearch.apiKey"); - if (!selectedPluginId && pluginId && webFetchPluginIds.has(pluginId)) { - candidatePluginIds.add(pluginId); - } - } - - const allowedPaths = new Set(); - const pluginsWithFetchCredential = new Set(); - let hasWebSearchFallbackPath = false; - for (const path of webFetchPaths) { - const pluginId = pluginIdFromWebCredentialPath(path, "webFetch.apiKey"); - if (!selectedPluginId || (pluginId && candidatePluginIds.has(pluginId))) { - allowedPaths.add(path); - if (pluginId) { - pluginsWithFetchCredential.add(pluginId); - } - } - } - if ( - webFetchPaths.has("tools.web.fetch.firecrawl.apiKey") && - (!selectedPluginId || selectedPluginId === "firecrawl" || providerId === "firecrawl") - ) { - allowedPaths.add("tools.web.fetch.firecrawl.apiKey"); - } - for (const path of webSearchPaths) { - const pluginId = pluginIdFromWebCredentialPath(path, "webSearch.apiKey"); - if (pluginId && candidatePluginIds.has(pluginId) && !pluginsWithFetchCredential.has(pluginId)) { - allowedPaths.add(path); - hasWebSearchFallbackPath = true; - } - } - - const targetIds = hasWebSearchFallbackPath - ? mergeTargetIdSets(webFetchTargetIds, webSearchTargetIds) - : new Set(webFetchTargetIds); - return { targetIds, allowedPaths }; -} - export function getAgentRuntimeCommandSecretTargetIds(params?: { includeChannelTargets?: boolean; }): Set { @@ -496,6 +722,82 @@ export function getAgentRuntimeCommandSecretTargetIds(params?: { return toTargetIdSet(getCommandSecretTargets().agentRuntime); } +export function getCapabilityWebFetchCommandSecretTargetIds(): Set { + return toTargetIdSet(getCapabilityWebFetchTargetIds()); +} + +export function getCapabilityWebFetchCommandSecretTargets( + config: OpenClawConfig, + options?: { + providerId?: string | null; + }, +): CommandSecretTargetScope { + const selectedProviderId = resolveSelectedWebFetchProviderId(config, options?.providerId); + if (!selectedProviderId) { + return getCapabilityWebFetchAutoDetectTargets(config); + } + const selectedTargets = getCapabilityWebFetchSelectedProviderTargetIds( + config, + selectedProviderId, + ); + if (!selectedTargets.matchedProvider && !options?.providerId) { + return getCapabilityWebFetchAutoDetectTargets(config); + } + const targetIds = toTargetIdSet(selectedTargets.targetIds); + const allowedPaths = + selectedTargets.allowedPaths.length > 0 ? new Set(selectedTargets.targetPaths) : undefined; + const forcedActivePaths = discoverForcedActivePaths( + config, + toTargetIdSet( + options?.providerId ? selectedTargets.targetIds : selectedTargets.fallbackTargetIds, + ), + allowedPaths, + ); + return { + targetIds, + ...(allowedPaths ? { allowedPaths } : {}), + ...(forcedActivePaths ? { forcedActivePaths } : {}), + }; +} + +export function getCapabilityWebSearchCommandSecretTargetIds(): Set { + return toTargetIdSet(getCapabilityWebSearchTargetIds()); +} + +export function getCapabilityWebSearchCommandSecretTargets( + config: OpenClawConfig, + options?: { + providerId?: string | null; + }, +): CommandSecretTargetScope { + const selectedProviderId = resolveSelectedWebSearchProviderId(config, options?.providerId); + if (!selectedProviderId) { + return getCapabilityWebSearchAutoDetectTargets(config); + } + const selectedTargets = getCapabilityWebSearchSelectedProviderTargetIds( + config, + selectedProviderId, + ); + if (!selectedTargets.matchedProvider && !options?.providerId) { + return getCapabilityWebSearchAutoDetectTargets(config); + } + const targetIds = toTargetIdSet(selectedTargets.targetIds); + const allowedPaths = + selectedTargets.allowedPaths.length > 0 ? new Set(selectedTargets.targetPaths) : undefined; + const forcedActivePaths = discoverForcedActivePaths( + config, + toTargetIdSet( + options?.providerId ? selectedTargets.targetIds : selectedTargets.fallbackTargetIds, + ), + allowedPaths, + ); + return { + targetIds, + ...(allowedPaths ? { allowedPaths } : {}), + ...(forcedActivePaths ? { forcedActivePaths } : {}), + }; +} + export function getStatusCommandSecretTargetIds( config?: OpenClawConfig, env?: NodeJS.ProcessEnv, diff --git a/src/gateway/protocol/schema/secrets.ts b/src/gateway/protocol/schema/secrets.ts index 1a2de801a47..7a1214d3ef1 100644 --- a/src/gateway/protocol/schema/secrets.ts +++ b/src/gateway/protocol/schema/secrets.ts @@ -7,15 +7,8 @@ export const SecretsResolveParamsSchema = Type.Object( { commandName: NonEmptyString, targetIds: Type.Array(NonEmptyString), - providerOverrides: Type.Optional( - Type.Object( - { - webSearch: Type.Optional(NonEmptyString), - webFetch: Type.Optional(NonEmptyString), - }, - { additionalProperties: false }, - ), - ), + allowedPaths: Type.Optional(Type.Array(NonEmptyString)), + forcedActivePaths: Type.Optional(Type.Array(NonEmptyString)), }, { additionalProperties: false }, ); diff --git a/src/gateway/server-aux-handlers.ts b/src/gateway/server-aux-handlers.ts index e17ad3e2e24..7f9da30d72d 100644 --- a/src/gateway/server-aux-handlers.ts +++ b/src/gateway/server-aux-handlers.ts @@ -223,12 +223,13 @@ export function createGatewayAuxHandlers(params: { } }), log: params.log, - resolveSecrets: async ({ commandName, targetIds, providerOverrides }) => { + resolveSecrets: async ({ allowedPaths, commandName, forcedActivePaths, targetIds }) => { const { assignments, diagnostics, inactiveRefPaths } = - await resolveCommandSecretsFromActiveRuntimeSnapshot({ + resolveCommandSecretsFromActiveRuntimeSnapshot({ commandName, targetIds: new Set(targetIds), - ...(providerOverrides ? { providerOverrides } : {}), + ...(allowedPaths ? { allowedPaths: new Set(allowedPaths) } : {}), + ...(forcedActivePaths ? { forcedActivePaths: new Set(forcedActivePaths) } : {}), }); if (assignments.length === 0) { return { diff --git a/src/gateway/server-methods/secrets.test.ts b/src/gateway/server-methods/secrets.test.ts index 6645def775e..28a827593e5 100644 --- a/src/gateway/server-methods/secrets.test.ts +++ b/src/gateway/server-methods/secrets.test.ts @@ -26,14 +26,18 @@ async function invokeSecretsResolve(params: { respond: ReturnType; commandName: unknown; targetIds: unknown; - providerOverrides?: unknown; + allowedPaths?: unknown; + forcedActivePaths?: unknown; }) { await params.handlers["secrets.resolve"]({ req: { type: "req", id: "1", method: "secrets.resolve" }, params: { commandName: params.commandName, targetIds: params.targetIds, - ...(params.providerOverrides ? { providerOverrides: params.providerOverrides } : {}), + ...(params.allowedPaths !== undefined ? { allowedPaths: params.allowedPaths } : {}), + ...(params.forcedActivePaths !== undefined + ? { forcedActivePaths: params.forcedActivePaths } + : {}), }, client: null, isWebchatConnect: () => false, @@ -75,7 +79,8 @@ describe("secrets handlers", () => { resolveSecrets?: (params: { commandName: string; targetIds: string[]; - providerOverrides?: { webSearch?: string; webFetch?: string }; + allowedPaths?: string[]; + forcedActivePaths?: string[]; }) => Promise<{ assignments: Array<{ path: string; pathSegments: string[]; value: unknown }>; diagnostics: string[]; @@ -141,10 +146,14 @@ describe("secrets handlers", () => { respond, commandName: "memory status", targetIds: ["talk.providers.*.apiKey"], + allowedPaths: [TALK_TEST_PROVIDER_API_KEY_PATH], + forcedActivePaths: [TALK_TEST_PROVIDER_API_KEY_PATH], }); expect(resolveSecrets).toHaveBeenCalledWith({ commandName: "memory status", targetIds: ["talk.providers.*.apiKey"], + allowedPaths: [TALK_TEST_PROVIDER_API_KEY_PATH], + forcedActivePaths: [TALK_TEST_PROVIDER_API_KEY_PATH], }); expect(respond).toHaveBeenCalledWith(true, { ok: true, @@ -160,36 +169,6 @@ describe("secrets handlers", () => { }); }); - it("passes trimmed provider overrides to secrets.resolve", async () => { - const resolveSecrets = vi.fn().mockResolvedValue({ - assignments: [], - diagnostics: [], - inactiveRefPaths: [], - }); - const handlers = createHandlers({ resolveSecrets }); - const respond = vi.fn(); - - await invokeSecretsResolve({ - handlers, - respond, - commandName: "infer web search", - targetIds: ["talk.providers.*.apiKey"], - providerOverrides: { webSearch: " tavily ", webFetch: " " }, - }); - - expect(resolveSecrets).toHaveBeenCalledWith({ - commandName: "infer web search", - targetIds: ["talk.providers.*.apiKey"], - providerOverrides: { webSearch: "tavily" }, - }); - expect(respond).toHaveBeenCalledWith( - true, - expect.objectContaining({ - ok: true, - }), - ); - }); - it("rejects invalid secrets.resolve params", async () => { const handlers = createHandlers(); const respond = vi.fn(); diff --git a/src/gateway/server-methods/secrets.ts b/src/gateway/server-methods/secrets.ts index 991a7254d7c..8632344502f 100644 --- a/src/gateway/server-methods/secrets.ts +++ b/src/gateway/server-methods/secrets.ts @@ -8,18 +8,13 @@ import { } from "../protocol/index.js"; import type { GatewayRequestHandlers } from "./types.js"; -type SecretsResolveProviderOverrides = { - webSearch?: string; - webFetch?: string; -}; - function errorMessage(error: unknown): string { return error instanceof Error ? error.message : String(error); } function invalidSecretsResolveField( errors: ErrorObject[] | null | undefined, -): "commandName" | "targetIds" | "providerOverrides" { +): "allowedPaths" | "commandName" | "forcedActivePaths" | "targetIds" { for (const issue of errors ?? []) { if ( issue.instancePath === "/commandName" || @@ -28,33 +23,23 @@ function invalidSecretsResolveField( ) { return "commandName"; } - if (issue.instancePath.startsWith("/providerOverrides")) { - return "providerOverrides"; + if (issue.instancePath.startsWith("/allowedPaths")) { + return "allowedPaths"; + } + if (issue.instancePath.startsWith("/forcedActivePaths")) { + return "forcedActivePaths"; } } return "targetIds"; } -function normalizeSecretsResolveProviderOverrides( - overrides: { webSearch?: string; webFetch?: string } | undefined, -): SecretsResolveProviderOverrides | undefined { - const webSearch = overrides?.webSearch?.trim(); - const webFetch = overrides?.webFetch?.trim(); - if (!webSearch && !webFetch) { - return undefined; - } - return { - ...(webSearch ? { webSearch } : {}), - ...(webFetch ? { webFetch } : {}), - }; -} - export function createSecretsHandlers(params: { reloadSecrets: () => Promise<{ warningCount: number }>; resolveSecrets: (params: { commandName: string; targetIds: string[]; - providerOverrides?: SecretsResolveProviderOverrides; + allowedPaths?: string[]; + forcedActivePaths?: string[]; }) => Promise<{ assignments: Array<{ path: string; @@ -100,9 +85,12 @@ export function createSecretsHandlers(params: { const targetIds = requestParams.targetIds .map((entry) => entry.trim()) .filter((entry) => entry.length > 0); - const providerOverrides = normalizeSecretsResolveProviderOverrides( - requestParams.providerOverrides, - ); + const allowedPaths = requestParams.allowedPaths + ?.map((entry) => entry.trim()) + .filter((entry) => entry.length > 0); + const forcedActivePaths = requestParams.forcedActivePaths + ?.map((entry) => entry.trim()) + .filter((entry) => entry.length > 0); for (const targetId of targetIds) { if (!isKnownSecretTargetId(targetId)) { @@ -122,7 +110,8 @@ export function createSecretsHandlers(params: { const result = await params.resolveSecrets({ commandName, targetIds, - ...(providerOverrides ? { providerOverrides } : {}), + ...(allowedPaths ? { allowedPaths } : {}), + ...(forcedActivePaths ? { forcedActivePaths } : {}), }); const payload = { ok: true, diff --git a/src/plugins/web-provider-types.ts b/src/plugins/web-provider-types.ts index 16156f15067..e09a0306bbd 100644 --- a/src/plugins/web-provider-types.ts +++ b/src/plugins/web-provider-types.ts @@ -49,6 +49,11 @@ export type WebSearchProviderConfiguredCredentialFallback = { value: unknown; }; +export type WebFetchProviderConfiguredCredentialFallback = { + path: string; + value: unknown; +}; + export type WebSearchRuntimeMetadataContext = { config?: OpenClawConfig; searchConfig?: Record; @@ -133,6 +138,9 @@ export type WebFetchProviderPlugin = { setCredentialValue: (fetchConfigTarget: Record, value: unknown) => void; getConfiguredCredentialValue?: (config?: OpenClawConfig) => unknown; setConfiguredCredentialValue?: (configTarget: OpenClawConfig, value: unknown) => void; + getConfiguredCredentialFallback?: ( + config?: OpenClawConfig, + ) => WebFetchProviderConfiguredCredentialFallback | undefined; applySelectionConfig?: (config: OpenClawConfig) => OpenClawConfig; resolveRuntimeMetadata?: ( ctx: WebFetchRuntimeMetadataContext, diff --git a/src/secrets/runtime-command-secrets.test.ts b/src/secrets/runtime-command-secrets.test.ts index bdaef268e67..62a26cca2ff 100644 --- a/src/secrets/runtime-command-secrets.test.ts +++ b/src/secrets/runtime-command-secrets.test.ts @@ -1,418 +1,81 @@ -import { describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it } from "vitest"; +import type { OpenClawConfig } from "../config/types.openclaw.js"; import { resolveCommandSecretsFromActiveRuntimeSnapshot } from "./runtime-command-secrets.js"; -import { activateSecretsRuntimeSnapshot } from "./runtime.js"; -import { asConfig, setupSecretsRuntimeSnapshotTestHooks } from "./runtime.test-support.ts"; +import { + activateSecretsRuntimeSnapshot, + clearSecretsRuntimeSnapshot, + prepareSecretsRuntimeSnapshot, +} from "./runtime.js"; -const { prepareSecretsRuntimeSnapshot } = setupSecretsRuntimeSnapshotTestHooks(); +describe("runtime command secrets", () => { + const previousBundledPluginsDir = process.env.OPENCLAW_BUNDLED_PLUGINS_DIR; + const previousTrustBundledPluginsDir = process.env.OPENCLAW_TEST_TRUST_BUNDLED_PLUGINS_DIR; -describe("resolveCommandSecretsFromActiveRuntimeSnapshot", () => { - it("reruns web secret resolution for provider overrides", async () => { - const googlePath = "plugins.entries.google.config.webSearch.apiKey"; - const bravePath = "plugins.entries.brave.config.webSearch.apiKey"; - const config = asConfig({ - tools: { web: { search: { provider: "gemini", enabled: true } } }, - plugins: { - entries: { - google: { - config: { - webSearch: { - apiKey: { source: "env", provider: "default", id: "GEMINI_API_KEY" }, - }, - }, - }, - brave: { - config: { - webSearch: { - apiKey: { source: "env", provider: "default", id: "BRAVE_API_KEY" }, - }, - }, - }, - }, - }, - }); - const env = { - ...process.env, - OPENCLAW_BUNDLED_PLUGINS_DIR: "extensions", - OPENCLAW_TEST_TRUST_BUNDLED_PLUGINS_DIR: "1", - GEMINI_API_KEY: "gemini-live", - BRAVE_API_KEY: "brave-live", - }; - - const snapshot = await prepareSecretsRuntimeSnapshot({ - config, - env, - includeAuthStoreRefs: false, - }); - const googleConfig = snapshot.config.plugins?.entries?.google?.config as - | { webSearch?: { apiKey?: unknown } } - | undefined; - const braveConfig = snapshot.config.plugins?.entries?.brave?.config as - | { webSearch?: { apiKey?: unknown } } - | undefined; - expect(googleConfig?.webSearch?.apiKey).toBe("gemini-live"); - expect(braveConfig?.webSearch?.apiKey).toEqual({ - source: "env", - provider: "default", - id: "BRAVE_API_KEY", - }); - - activateSecretsRuntimeSnapshot(snapshot); - const result = await resolveCommandSecretsFromActiveRuntimeSnapshot({ - commandName: "infer web search", - targetIds: new Set([googlePath, bravePath]), - providerOverrides: { webSearch: "brave" }, - }); - - expect(result.assignments).toEqual([ - { - path: bravePath, - pathSegments: bravePath.split("."), - value: "brave-live", - }, - ]); - expect(result.inactiveRefPaths).toContain(googlePath); + afterEach(() => { + clearSecretsRuntimeSnapshot(); + if (previousBundledPluginsDir === undefined) { + delete process.env.OPENCLAW_BUNDLED_PLUGINS_DIR; + } else { + process.env.OPENCLAW_BUNDLED_PLUGINS_DIR = previousBundledPluginsDir; + } + if (previousTrustBundledPluginsDir === undefined) { + delete process.env.OPENCLAW_TEST_TRUST_BUNDLED_PLUGINS_DIR; + } else { + process.env.OPENCLAW_TEST_TRUST_BUNDLED_PLUGINS_DIR = previousTrustBundledPluginsDir; + } }); - it("returns legacy web fetch assignments for provider overrides", async () => { - const legacyPath = "tools.web.fetch.firecrawl.apiKey"; - const config = asConfig({ + it("returns forced fallback assignments from the active gateway snapshot", async () => { + process.env.OPENCLAW_BUNDLED_PLUGINS_DIR = "extensions"; + process.env.OPENCLAW_TEST_TRUST_BUNDLED_PLUGINS_DIR = "1"; + const config = { tools: { web: { - fetch: { - provider: "browser", - firecrawl: { - apiKey: { source: "env", provider: "default", id: "FIRECRAWL_API_KEY" }, + search: { enabled: false, provider: "brave" }, + fetch: { provider: "firecrawl" }, + }, + }, + plugins: { + entries: { + firecrawl: { + enabled: true, + config: { + webSearch: { + apiKey: { + source: "env", + provider: "default", + id: "FIRECRAWL_API_KEY", + }, + }, }, }, }, }, - }); - const env = { - ...process.env, - OPENCLAW_BUNDLED_PLUGINS_DIR: "extensions", - OPENCLAW_TEST_TRUST_BUNDLED_PLUGINS_DIR: "1", - FIRECRAWL_API_KEY: "firecrawl-live", - }; - + } as OpenClawConfig; const snapshot = await prepareSecretsRuntimeSnapshot({ config, - env, - includeAuthStoreRefs: false, + env: { + FIRECRAWL_API_KEY: "gateway-only-firecrawl-key", + HOME: process.env.HOME, + OPENCLAW_BUNDLED_PLUGINS_DIR: "extensions", + OPENCLAW_TEST_TRUST_BUNDLED_PLUGINS_DIR: "1", + }, }); - const fetchConfig = snapshot.config.tools?.web?.fetch as - | { firecrawl?: { apiKey?: unknown } } - | undefined; - expect(fetchConfig?.firecrawl?.apiKey).toEqual({ - source: "env", - provider: "default", - id: "FIRECRAWL_API_KEY", - }); - activateSecretsRuntimeSnapshot(snapshot); - const result = await resolveCommandSecretsFromActiveRuntimeSnapshot({ + + const resolved = resolveCommandSecretsFromActiveRuntimeSnapshot({ commandName: "infer web fetch", - targetIds: new Set([legacyPath]), - providerOverrides: { webFetch: "firecrawl" }, + targetIds: new Set(["plugins.entries.firecrawl.config.webSearch.apiKey"]), + forcedActivePaths: new Set(["plugins.entries.firecrawl.config.webSearch.apiKey"]), }); - expect(result.assignments).toEqual([ + expect(resolved.assignments).toMatchObject([ { - path: legacyPath, - pathSegments: legacyPath.split("."), - value: "firecrawl-live", - }, - ]); - }); - - it("returns legacy web fetch assignments for the configured provider", async () => { - const legacyPath = "tools.web.fetch.firecrawl.apiKey"; - const config = asConfig({ - tools: { - web: { - fetch: { - provider: "firecrawl", - firecrawl: { - apiKey: { source: "env", provider: "default", id: "FIRECRAWL_API_KEY" }, - }, - }, - }, - }, - }); - const env = { - ...process.env, - OPENCLAW_BUNDLED_PLUGINS_DIR: "extensions", - OPENCLAW_TEST_TRUST_BUNDLED_PLUGINS_DIR: "1", - FIRECRAWL_API_KEY: "firecrawl-live", - }; - - const snapshot = await prepareSecretsRuntimeSnapshot({ - config, - env, - includeAuthStoreRefs: false, - }); - - activateSecretsRuntimeSnapshot(snapshot); - const result = await resolveCommandSecretsFromActiveRuntimeSnapshot({ - commandName: "infer web fetch", - targetIds: new Set([legacyPath]), - }); - - expect(result.assignments).toEqual([ - { - path: legacyPath, - pathSegments: legacyPath.split("."), - value: "firecrawl-live", - }, - ]); - }); - - it("keeps legacy shared web search refs inactive for plugin-scoped provider overrides", async () => { - const sharedPath = "tools.web.search.apiKey"; - const googlePath = "plugins.entries.google.config.webSearch.apiKey"; - const config = asConfig({ - tools: { - web: { - search: { - provider: "brave", - apiKey: { source: "env", provider: "default", id: "BRAVE_API_KEY" }, - }, - }, - }, - plugins: { - entries: { - google: { - config: { - webSearch: { - apiKey: { source: "env", provider: "default", id: "GEMINI_API_KEY" }, - }, - }, - }, - }, - }, - }); - const env = { - ...process.env, - OPENCLAW_BUNDLED_PLUGINS_DIR: "extensions", - OPENCLAW_TEST_TRUST_BUNDLED_PLUGINS_DIR: "1", - BRAVE_API_KEY: "brave-live", - GEMINI_API_KEY: "gemini-live", - }; - - const snapshot = await prepareSecretsRuntimeSnapshot({ - config, - env, - includeAuthStoreRefs: false, - }); - const resolvedSearchConfig = snapshot.config.tools?.web?.search as { apiKey?: unknown }; - resolvedSearchConfig.apiKey = "brave-live"; - - activateSecretsRuntimeSnapshot(snapshot); - const result = await resolveCommandSecretsFromActiveRuntimeSnapshot({ - commandName: "infer web search", - targetIds: new Set([sharedPath, googlePath]), - providerOverrides: { webSearch: "gemini" }, - }); - - expect(result.assignments).toEqual([ - { - path: googlePath, - pathSegments: googlePath.split("."), - value: "gemini-live", - }, - ]); - expect(result.inactiveRefPaths).toContain(sharedPath); - }); - - it("keeps provider override refs inactive when the web search surface is disabled", async () => { - const googlePath = "plugins.entries.google.config.webSearch.apiKey"; - const config = asConfig({ - tools: { web: { search: { enabled: false, provider: "brave" } } }, - plugins: { - entries: { - google: { - config: { - webSearch: { - apiKey: { source: "env", provider: "default", id: "GEMINI_API_KEY" }, - }, - }, - }, - }, - }, - }); - const env = { - ...process.env, - OPENCLAW_BUNDLED_PLUGINS_DIR: "extensions", - OPENCLAW_TEST_TRUST_BUNDLED_PLUGINS_DIR: "1", - GEMINI_API_KEY: "gemini-live", - }; - - const snapshot = await prepareSecretsRuntimeSnapshot({ - config, - env, - includeAuthStoreRefs: false, - }); - - activateSecretsRuntimeSnapshot(snapshot); - const result = await resolveCommandSecretsFromActiveRuntimeSnapshot({ - commandName: "infer web search", - targetIds: new Set([googlePath]), - providerOverrides: { webSearch: "gemini" }, - }); - - expect(result.assignments).toEqual([]); - expect(result.inactiveRefPaths).toContain(googlePath); - }); - - it("returns legacy shared web search assignments for providers that read the shared key", async () => { - const sharedPath = "tools.web.search.apiKey"; - const config = asConfig({ - tools: { - web: { - search: { - provider: "gemini", - apiKey: { source: "env", provider: "default", id: "BRAVE_API_KEY" }, - }, - }, - }, - plugins: { - entries: { - google: { - config: { - webSearch: { - apiKey: { source: "env", provider: "default", id: "GEMINI_API_KEY" }, - }, - }, - }, - }, - }, - }); - const env = { - ...process.env, - OPENCLAW_BUNDLED_PLUGINS_DIR: "extensions", - OPENCLAW_TEST_TRUST_BUNDLED_PLUGINS_DIR: "1", - BRAVE_API_KEY: "brave-live", - GEMINI_API_KEY: "gemini-live", - }; - - const snapshot = await prepareSecretsRuntimeSnapshot({ - config, - env, - includeAuthStoreRefs: false, - }); - - activateSecretsRuntimeSnapshot(snapshot); - const result = await resolveCommandSecretsFromActiveRuntimeSnapshot({ - commandName: "infer web search", - targetIds: new Set([sharedPath]), - providerOverrides: { webSearch: "brave" }, - }); - - expect(result.assignments).toEqual([ - { - path: sharedPath, - pathSegments: sharedPath.split("."), - value: "brave-live", - }, - ]); - }); - - it("returns shared web search assignments for selected top-level credential providers", async () => { - const sharedPath = "tools.web.search.apiKey"; - const config = asConfig({ - tools: { - web: { - search: { - provider: "gemini", - apiKey: { source: "env", provider: "default", id: "MINIMAX_API_KEY" }, - }, - }, - }, - }); - const env = { - ...process.env, - OPENCLAW_BUNDLED_PLUGINS_DIR: "extensions", - OPENCLAW_TEST_TRUST_BUNDLED_PLUGINS_DIR: "1", - MINIMAX_API_KEY: "minimax-live", - }; - - const snapshot = await prepareSecretsRuntimeSnapshot({ - config, - env, - includeAuthStoreRefs: false, - }); - - activateSecretsRuntimeSnapshot(snapshot); - const result = await resolveCommandSecretsFromActiveRuntimeSnapshot({ - commandName: "infer web search", - targetIds: new Set([sharedPath]), - providerOverrides: { webSearch: "minimax" }, - }); - - expect(result.assignments).toEqual([ - { - path: sharedPath, - pathSegments: sharedPath.split("."), - value: "minimax-live", - }, - ]); - }); - - it("preserves non-web snapshot assignments when provider overrides are present", async () => { - const talkPath = "talk.providers.default.apiKey"; - const googlePath = "plugins.entries.google.config.webSearch.apiKey"; - const config = asConfig({ - talk: { - providers: { - default: { - apiKey: { source: "env", provider: "default", id: "TALK_API_KEY" }, - }, - }, - }, - plugins: { - entries: { - google: { - config: { - webSearch: { - apiKey: { source: "env", provider: "default", id: "GEMINI_API_KEY" }, - }, - }, - }, - }, - }, - }); - const env = { - ...process.env, - OPENCLAW_BUNDLED_PLUGINS_DIR: "extensions", - OPENCLAW_TEST_TRUST_BUNDLED_PLUGINS_DIR: "1", - TALK_API_KEY: "talk-live", - GEMINI_API_KEY: "gemini-live", - }; - - const snapshot = await prepareSecretsRuntimeSnapshot({ - config, - env, - includeAuthStoreRefs: false, - }); - expect(snapshot.config.talk?.providers?.default?.apiKey).toBe("talk-live"); - - activateSecretsRuntimeSnapshot(snapshot); - const result = await resolveCommandSecretsFromActiveRuntimeSnapshot({ - commandName: "infer web search", - targetIds: new Set(["talk.providers.*.apiKey", googlePath]), - providerOverrides: { webSearch: "gemini" }, - }); - - expect(result.assignments).toEqual([ - { - path: talkPath, - pathSegments: talkPath.split("."), - value: "talk-live", - }, - { - path: googlePath, - pathSegments: googlePath.split("."), - value: "gemini-live", + path: "plugins.entries.firecrawl.config.webSearch.apiKey", + value: "gateway-only-firecrawl-key", }, ]); + expect(resolved.diagnostics).toEqual([]); + expect(resolved.inactiveRefPaths).toEqual([]); }); }); diff --git a/src/secrets/runtime-command-secrets.ts b/src/secrets/runtime-command-secrets.ts index 37666a9cd08..a36125fb93e 100644 --- a/src/secrets/runtime-command-secrets.ts +++ b/src/secrets/runtime-command-secrets.ts @@ -1,454 +1,40 @@ -import type { OpenClawConfig } from "../config/types.openclaw.js"; -import { resolveSecretInputRef } from "../config/types.secrets.js"; -import { resolveManifestContractOwnerPluginId } from "../plugins/plugin-registry.js"; -import { resolveBundledExplicitWebSearchProvidersFromPublicArtifacts } from "../plugins/web-provider-public-artifacts.explicit.js"; -import { normalizeOptionalString } from "../shared/string-coerce.js"; import { - analyzeCommandSecretAssignmentsFromSnapshot, collectCommandSecretAssignmentsFromSnapshot, type CommandSecretAssignment, } from "./command-config.js"; -import { getPath, setPathExistingStrict } from "./path-utils.js"; -import { createResolverContext } from "./runtime-shared.js"; -import { resolveRuntimeWebTools } from "./runtime-web-tools.js"; -import { getActiveSecretsRuntimeEnv, getActiveSecretsRuntimeSnapshot } from "./runtime.js"; -import { discoverConfigSecretTargetsByIds } from "./target-registry.js"; +import { getActiveSecretsRuntimeSnapshot } from "./runtime.js"; export type { CommandSecretAssignment } from "./command-config.js"; -export type CommandSecretProviderOverrides = { - webSearch?: string; - webFetch?: string; -}; - -function hasProviderOverrides(overrides: CommandSecretProviderOverrides | undefined): boolean { - return ( - normalizeOptionalString(overrides?.webSearch) !== undefined || - normalizeOptionalString(overrides?.webFetch) !== undefined - ); -} - -function applyProviderOverridesToConfig( - config: OpenClawConfig, - overrides: CommandSecretProviderOverrides | undefined, -): OpenClawConfig { - if (!hasProviderOverrides(overrides)) { - return config; - } - const next = structuredClone(config); - const tools = (next.tools ??= {}) as Record; - const web = (tools.web ??= {}) as Record; - const webSearch = normalizeOptionalString(overrides?.webSearch); - if (webSearch) { - const search = (web.search ??= {}) as Record; - search.provider = webSearch; - } - const webFetch = normalizeOptionalString(overrides?.webFetch); - if (webFetch) { - const fetch = (web.fetch ??= {}) as Record; - fetch.provider = webFetch; - } - return next; -} - -function pluginIdFromRuntimeWebPath(path: string): string | undefined { - return /^plugins\.entries\.([^.]+)\.config\.(webSearch|webFetch)\.apiKey$/.exec(path)?.[1]; -} - -function searchProviderFromDirectWebPath(path: string): string | undefined { - return /^tools\.web\.search\.([^.]+)\.apiKey$/.exec(path)?.[1]; -} - -function fetchProviderFromDirectWebPath(path: string): string | undefined { - return /^tools\.web\.fetch\.([^.]+)\.apiKey$/.exec(path)?.[1]; -} - -function isWebCommandSecretPath(path: string): boolean { - return ( - path === "tools.web.search.apiKey" || - /^tools\.web\.(search|fetch)\.[^.]+\.apiKey$/.test(path) || - /^plugins\.entries\.[^.]+\.config\.(webSearch|webFetch)\.apiKey$/.test(path) - ); -} - -function webSearchProviderUsesSharedSearchCredential(params: { - config: OpenClawConfig; - provider: string; -}): boolean { - const sentinel = "__openclaw_shared_web_search_probe__"; - const pluginId = resolveManifestContractOwnerPluginId({ - contract: "webSearchProviders", - value: params.provider, - origin: "bundled", - config: params.config, - }); - if (!pluginId) { - return false; - } - const providers = resolveBundledExplicitWebSearchProvidersFromPublicArtifacts({ - onlyPluginIds: [pluginId], - }); - const provider = providers?.find((entry) => entry.id === params.provider); - return ( - provider?.credentialPath === "tools.web.search.apiKey" || - provider?.getCredentialValue({ apiKey: sentinel }) === sentinel || - provider?.getConfiguredCredentialFallback?.(params.config)?.path === "tools.web.search.apiKey" - ); -} - -function isProviderOverridePath(params: { - config: OpenClawConfig; - path: string; - providerOverrides: CommandSecretProviderOverrides | undefined; -}): boolean { - const webSearch = normalizeOptionalString(params.providerOverrides?.webSearch); - if (webSearch) { - if (params.config.tools?.web?.search?.enabled === false) { - return false; - } - if (params.path === "tools.web.search.apiKey") { - return webSearchProviderUsesSharedSearchCredential({ - config: params.config, - provider: webSearch, - }); - } - const directProvider = searchProviderFromDirectWebPath(params.path); - if (directProvider) { - return directProvider === webSearch; - } - const pluginId = pluginIdFromRuntimeWebPath(params.path); - if (pluginId && params.path.endsWith(".config.webSearch.apiKey")) { - return ( - resolveManifestContractOwnerPluginId({ - contract: "webSearchProviders", - value: webSearch, - origin: "bundled", - config: params.config, - }) === pluginId - ); - } - } - - const webFetch = normalizeOptionalString(params.providerOverrides?.webFetch); - if (webFetch) { - if (params.config.tools?.web?.fetch?.enabled === false) { - return false; - } - const directProvider = fetchProviderFromDirectWebPath(params.path); - if (directProvider) { - return directProvider === webFetch; - } - const pluginId = pluginIdFromRuntimeWebPath(params.path); - if (pluginId && params.path.endsWith(".config.webFetch.apiKey")) { - return ( - resolveManifestContractOwnerPluginId({ - contract: "webFetchProviders", - value: webFetch, - origin: "bundled", - config: params.config, - }) === pluginId - ); - } - } - - return false; -} - -function restoreInactiveWebCommandSecretTargets(params: { - sourceConfig: OpenClawConfig; - resolvedConfig: OpenClawConfig; - targetIds: ReadonlySet; - inactiveRefPaths: string[]; - providerOverrides: CommandSecretProviderOverrides | undefined; -}): string[] { - if (!hasProviderOverrides(params.providerOverrides)) { - return params.inactiveRefPaths; - } - const inactive = new Set(params.inactiveRefPaths); - const defaults = params.sourceConfig.secrets?.defaults; - for (const target of discoverConfigSecretTargetsByIds(params.sourceConfig, params.targetIds)) { - if (!isWebCommandSecretPath(target.path)) { - continue; - } - const { ref } = resolveSecretInputRef({ - value: target.value, - refValue: target.refValue, - defaults, - }); - if (!ref) { - continue; - } - if ( - isProviderOverridePath({ - config: params.sourceConfig, - path: target.path, - providerOverrides: params.providerOverrides, - }) - ) { - continue; - } - inactive.add(target.path); - setPathExistingStrict( - params.resolvedConfig as Record, - target.pathSegments, - target.value, - ); - } - return [...inactive]; -} - -function filterInactiveRefPathsForProviderOverrides(params: { - config: OpenClawConfig; - inactiveRefPaths: readonly string[]; - providerOverrides: CommandSecretProviderOverrides | undefined; -}): string[] { - if (!hasProviderOverrides(params.providerOverrides)) { - return [...params.inactiveRefPaths]; - } - return params.inactiveRefPaths.filter( - (path) => - !isProviderOverridePath({ - config: params.config, - path, - providerOverrides: params.providerOverrides, - }), - ); -} - -function mirrorResolvedProviderCredentialToDirectPath(params: { - config: OpenClawConfig; - resolvedConfig: OpenClawConfig; - contract: "webSearchProviders" | "webFetchProviders"; - provider: string | undefined; - directPathPrefix: string; - pluginConfigKey: "webSearch" | "webFetch"; -}): void { - const provider = normalizeOptionalString(params.provider); - if (!provider) { - return; - } - const pluginId = resolveManifestContractOwnerPluginId({ - contract: params.contract, - value: provider, - origin: "bundled", - config: params.config, - }); - if (!pluginId) { - return; - } - const directSegments = [...params.directPathPrefix.split("."), provider, "apiKey"]; - const directValue = getPath(params.config, directSegments); - if (directValue === undefined) { - return; - } - const resolvedValue = getPath(params.resolvedConfig, [ - "plugins", - "entries", - pluginId, - "config", - params.pluginConfigKey, - "apiKey", - ]); - if (typeof resolvedValue !== "string" || resolvedValue.length === 0) { - return; - } - setPathExistingStrict( - params.resolvedConfig as Record, - directSegments, - resolvedValue, - ); -} - -function mirrorResolvedProviderCredentialToDirectPaths(params: { - config: OpenClawConfig; - resolvedConfig: OpenClawConfig; - providerOverrides: CommandSecretProviderOverrides | undefined; -}): void { - const configuredSearchProvider = - normalizeOptionalString(params.providerOverrides?.webSearch) ?? - normalizeOptionalString(params.config.tools?.web?.search?.provider); - const configuredFetchProvider = - normalizeOptionalString(params.providerOverrides?.webFetch) ?? - normalizeOptionalString(params.config.tools?.web?.fetch?.provider); - mirrorResolvedProviderCredentialToDirectPath({ - config: params.config, - resolvedConfig: params.resolvedConfig, - contract: "webSearchProviders", - provider: configuredSearchProvider, - directPathPrefix: "tools.web.search", - pluginConfigKey: "webSearch", - }); - mirrorResolvedProviderCredentialToDirectPath({ - config: params.config, - resolvedConfig: params.resolvedConfig, - contract: "webFetchProviders", - provider: configuredFetchProvider, - directPathPrefix: "tools.web.fetch", - pluginConfigKey: "webFetch", - }); - const webSearch = configuredSearchProvider; - if ( - webSearch && - webSearchProviderUsesSharedSearchCredential({ - config: params.config, - provider: webSearch, - }) && - getPath(params.config, ["tools", "web", "search", "apiKey"]) !== undefined - ) { - const pluginId = resolveManifestContractOwnerPluginId({ - contract: "webSearchProviders", - value: webSearch, - origin: "bundled", - config: params.config, - }); - const resolvedValue = pluginId - ? getPath(params.resolvedConfig, [ - "plugins", - "entries", - pluginId, - "config", - "webSearch", - "apiKey", - ]) - : undefined; - if (typeof resolvedValue === "string" && resolvedValue.length > 0) { - setPathExistingStrict( - params.resolvedConfig as Record, - ["tools", "web", "search", "apiKey"], - resolvedValue, - ); - } - } -} - export function resolveCommandSecretsFromActiveRuntimeSnapshot(params: { commandName: string; targetIds: ReadonlySet; - providerOverrides?: CommandSecretProviderOverrides; -}): Promise<{ - assignments: CommandSecretAssignment[]; - diagnostics: string[]; - inactiveRefPaths: string[]; -}> { + allowedPaths?: ReadonlySet; + forcedActivePaths?: ReadonlySet; +}): { assignments: CommandSecretAssignment[]; diagnostics: string[]; inactiveRefPaths: string[] } { const activeSnapshot = getActiveSecretsRuntimeSnapshot(); if (!activeSnapshot) { throw new Error("Secrets runtime snapshot is not active."); } if (params.targetIds.size === 0) { - return Promise.resolve({ assignments: [], diagnostics: [], inactiveRefPaths: [] }); - } - return resolveCommandSecretsFromSnapshot({ - activeSnapshot, - commandName: params.commandName, - targetIds: params.targetIds, - providerOverrides: params.providerOverrides, - }); -} - -async function resolveCommandSecretsFromSnapshot(params: { - activeSnapshot: NonNullable>; - commandName: string; - targetIds: ReadonlySet; - providerOverrides?: CommandSecretProviderOverrides; -}): Promise<{ - assignments: CommandSecretAssignment[]; - diagnostics: string[]; - inactiveRefPaths: string[]; -}> { - const hasOverrides = hasProviderOverrides(params.providerOverrides); - const sourceConfig = applyProviderOverridesToConfig( - params.activeSnapshot.sourceConfig, - params.providerOverrides, - ); - const resolvedConfig = applyProviderOverridesToConfig( - params.activeSnapshot.config, - params.providerOverrides, - ); - const context = hasOverrides - ? createResolverContext({ - sourceConfig, - env: getActiveSecretsRuntimeEnv(), - }) - : undefined; - if (context) { - await resolveRuntimeWebTools({ - sourceConfig, - resolvedConfig, - context, - }); - } - mirrorResolvedProviderCredentialToDirectPaths({ - config: sourceConfig, - resolvedConfig, - providerOverrides: params.providerOverrides, - }); - const warningSource = context?.warnings ?? params.activeSnapshot.warnings; - let inactiveRefPaths = filterInactiveRefPathsForProviderOverrides({ - config: sourceConfig, - providerOverrides: params.providerOverrides, - inactiveRefPaths: [ - ...new Set( - warningSource - .filter((warning) => warning.code === "SECRETS_REF_IGNORED_INACTIVE_SURFACE") - .map((warning) => warning.path), - ), - ], - }); - inactiveRefPaths = restoreInactiveWebCommandSecretTargets({ - sourceConfig, - resolvedConfig, - targetIds: params.targetIds, - inactiveRefPaths, - providerOverrides: params.providerOverrides, - }); - let analyzed = analyzeCommandSecretAssignmentsFromSnapshot({ - sourceConfig, - resolvedConfig, - targetIds: params.targetIds, - inactiveRefPaths: new Set(inactiveRefPaths), - }); - if (hasOverrides) { - const impliedInactivePaths = analyzed.unresolved - .filter((entry) => isWebCommandSecretPath(entry.path)) - .filter( - (entry) => - !isProviderOverridePath({ - config: sourceConfig, - path: entry.path, - providerOverrides: params.providerOverrides, - }), - ) - .map((entry) => entry.path); - if (impliedInactivePaths.length > 0) { - inactiveRefPaths = [...new Set([...inactiveRefPaths, ...impliedInactivePaths])]; - analyzed = analyzeCommandSecretAssignmentsFromSnapshot({ - sourceConfig, - resolvedConfig, - targetIds: params.targetIds, - inactiveRefPaths: new Set(inactiveRefPaths), - }); - } - } - const selectedProviderUnresolved = analyzed.unresolved.filter((entry) => - isProviderOverridePath({ - config: sourceConfig, - path: entry.path, - providerOverrides: params.providerOverrides, - }), - ); - if (selectedProviderUnresolved.length > 0) { - return { - assignments: analyzed.assignments, - diagnostics: analyzed.diagnostics, - inactiveRefPaths, - }; + return { assignments: [], diagnostics: [], inactiveRefPaths: [] }; } + const inactiveRefPaths = [ + ...new Set( + activeSnapshot.warnings + .filter((warning) => warning.code === "SECRETS_REF_IGNORED_INACTIVE_SURFACE") + .filter((warning) => !params.allowedPaths || params.allowedPaths.has(warning.path)) + .filter((warning) => !params.forcedActivePaths?.has(warning.path)) + .map((warning) => warning.path), + ), + ]; const resolved = collectCommandSecretAssignmentsFromSnapshot({ - sourceConfig, - resolvedConfig, + sourceConfig: activeSnapshot.sourceConfig, + resolvedConfig: activeSnapshot.config, commandName: params.commandName, targetIds: params.targetIds, inactiveRefPaths: new Set(inactiveRefPaths), + ...(params.allowedPaths ? { allowedPaths: params.allowedPaths } : {}), }); return { assignments: resolved.assignments, diff --git a/src/secrets/runtime-web-tools.shared.ts b/src/secrets/runtime-web-tools.shared.ts index 930761867a3..84d7506ffb8 100644 --- a/src/secrets/runtime-web-tools.shared.ts +++ b/src/secrets/runtime-web-tools.shared.ts @@ -2,6 +2,7 @@ import type { OpenClawConfig } from "../config/types.openclaw.js"; import { resolveSecretInputRef } from "../config/types.secrets.js"; import { createLazyRuntimeNamedExport } from "../shared/lazy-runtime.js"; import { normalizeOptionalLowercaseString } from "../shared/string-coerce.js"; +import { setPathExistingStrict } from "./path-utils.js"; import type { ResolverContext, SecretDefaults, @@ -160,6 +161,33 @@ export function hasConfiguredSecretRef( ); } +function getProviderEnvVars(provider: object): string[] { + return "envVars" in provider && Array.isArray(provider.envVars) ? provider.envVars : []; +} + +function setResolvedCredentialPath(params: { + resolvedConfig: OpenClawConfig; + path: string; + value: string; +}): void { + const pathSegments = params.path + .split(".") + .map((segment) => segment.trim()) + .filter((segment) => segment.length > 0); + if (pathSegments.length === 0) { + return; + } + try { + setPathExistingStrict( + params.resolvedConfig as Record, + pathSegments, + params.value, + ); + } catch { + // Env-only provider defaults may not have a config path to mirror. + } +} + export type RuntimeWebProviderSurface = { providers: TProvider[]; configuredProvider?: string; @@ -356,7 +384,7 @@ export async function resolveRuntimeWebProviderSelection< const resolution = await params.resolveSecretInput({ value, path, - envVars: "envVars" in provider && Array.isArray(provider.envVars) ? provider.envVars : [], + envVars: getProviderEnvVars(provider), }); let selectedCandidatePath = path; let selectedCandidateResolution = resolution; @@ -372,9 +400,32 @@ export async function resolveRuntimeWebProviderSelection< selectedCandidateResolution = await params.resolveSecretInput({ value: fallback.value, path: fallback.path, - envVars: [], + envVars: getProviderEnvVars(provider), }); } + } else if (resolution.source === "env" && !resolution.secretRefConfigured) { + const fallback = params.readConfiguredCredentialFallback?.({ + provider, + config: params.sourceConfig, + toolConfig: params.toolConfig, + }); + if ( + fallback?.value !== undefined && + params.hasConfiguredSecretRef(fallback.value, params.defaults) + ) { + const fallbackResolution = await params.resolveSecretInput({ + value: fallback.value, + path: fallback.path, + envVars: getProviderEnvVars(provider), + }); + if (fallbackResolution.source === "secretRef" && fallbackResolution.value) { + setResolvedCredentialPath({ + resolvedConfig: params.resolvedConfig, + path: fallback.path, + value: fallbackResolution.value, + }); + } + } } if ( @@ -413,6 +464,11 @@ export async function resolveRuntimeWebProviderSelection< selectedProvider = provider.id; selectedResolution = selectedCandidateResolution; if (selectedCandidateResolution.value) { + setResolvedCredentialPath({ + resolvedConfig: params.resolvedConfig, + path: selectedCandidatePath, + value: selectedCandidateResolution.value, + }); params.setResolvedCredential({ resolvedConfig: params.resolvedConfig, provider, @@ -425,6 +481,11 @@ export async function resolveRuntimeWebProviderSelection< if (selectedCandidateResolution.value) { selectedProvider = provider.id; selectedResolution = selectedCandidateResolution; + setResolvedCredentialPath({ + resolvedConfig: params.resolvedConfig, + path: selectedCandidatePath, + value: selectedCandidateResolution.value, + }); params.setResolvedCredential({ resolvedConfig: params.resolvedConfig, provider, diff --git a/src/secrets/runtime-web-tools.test.ts b/src/secrets/runtime-web-tools.test.ts index 02a7172a988..605eccb02e0 100644 --- a/src/secrets/runtime-web-tools.test.ts +++ b/src/secrets/runtime-web-tools.test.ts @@ -271,6 +271,19 @@ function buildTestWebFetchProviders(): PluginWebFetchProviderEntry[] { ? (entryConfig as { webFetch?: { apiKey?: unknown } }).webFetch?.apiKey : undefined; }, + getConfiguredCredentialFallback: (config) => { + const entryConfig = config?.plugins?.entries?.firecrawl?.config; + const apiKey = + entryConfig && typeof entryConfig === "object" + ? (entryConfig as { webSearch?: { apiKey?: unknown } }).webSearch?.apiKey + : undefined; + return apiKey === undefined + ? undefined + : { + path: "plugins.entries.firecrawl.config.webSearch.apiKey", + value: apiKey, + }; + }, setConfiguredCredentialValue: (configTarget, value) => { setConfiguredFetchProviderKey(configTarget, value); }, @@ -913,6 +926,37 @@ describe("runtime web tools resolution", () => { expect(readProviderKey(resolvedConfig, "gemini")).toBe("gemini-env-runtime-key"); }); + it("does not mirror provider env fallback over configured fallback SecretRefs", async () => { + const { metadata, resolvedConfig } = await runRuntimeWebTools({ + config: asConfig({ + tools: { + web: { + search: { + enabled: true, + }, + }, + }, + models: { + providers: { + google: { + apiKey: { source: "env", provider: "default", id: "GOOGLE_PROVIDER_REF" }, + }, + }, + }, + }), + env: { + GEMINI_API_KEY: "gemini-env-runtime-key", + GOOGLE_PROVIDER_REF: "google-provider-ref-key", + }, + }); + + expect(metadata.search.providerSource).toBe("auto-detect"); + expect(metadata.search.selectedProvider).toBe("gemini"); + expect(metadata.search.selectedProviderKeySource).toBe("env"); + expect(readProviderKey(resolvedConfig, "gemini")).toBe("gemini-env-runtime-key"); + expect(resolvedConfig.models?.providers?.google?.apiKey).toBe("google-provider-ref-key"); + }); + it("warns when provider is invalid and falls back to auto-detect", async () => { const { metadata, resolvedConfig, context } = await runRuntimeWebTools({ config: asConfig({ @@ -1423,6 +1467,44 @@ describe("runtime web tools resolution", () => { }); }); + it("resolves web fetch fallback SecretRefs with provider env var allowlist", async () => { + const { metadata, resolvedConfig } = await runRuntimeWebTools({ + config: asConfig({ + plugins: { + entries: { + firecrawl: { + config: { + webSearch: { + apiKey: { source: "env", provider: "default", id: "FIRECRAWL_API_KEY" }, + }, + }, + }, + }, + }, + tools: { + web: { + fetch: { + provider: "firecrawl", + }, + }, + }, + }), + env: { + FIRECRAWL_API_KEY: "firecrawl-search-ref-key", + }, + }); + + expect(metadata.fetch.selectedProvider).toBe("firecrawl"); + expect(metadata.fetch.selectedProviderKeySource).toBe("env"); + expect( + ( + resolvedConfig.plugins?.entries?.firecrawl?.config as + | { webFetch?: { apiKey?: unknown } } + | undefined + )?.webFetch?.apiKey, + ).toBe("firecrawl-search-ref-key"); + }); + it("resolves plugin-owned web fetch SecretRefs without tools.web.fetch", async () => { const { metadata, resolvedConfig } = await runRuntimeWebTools({ config: asConfig({ diff --git a/src/secrets/runtime-web-tools.ts b/src/secrets/runtime-web-tools.ts index 70ea130d9cb..05966baa8c7 100644 --- a/src/secrets/runtime-web-tools.ts +++ b/src/secrets/runtime-web-tools.ts @@ -526,6 +526,14 @@ function readConfiguredFetchProviderCredential(params: { return configuredValue ?? params.provider.getCredentialValue(params.fetch); } +function readConfiguredFetchProviderCredentialFallback(params: { + provider: PluginWebFetchProviderEntry; + config: OpenClawConfig; + fetch: Record | undefined; +}): { path: string; value: unknown } | undefined { + return params.provider.getConfiguredCredentialFallback?.(params.config); +} + function inactivePathsForFetchProvider(provider: PluginWebFetchProviderEntry): string[] { if (provider.requiresCredential === false) { return []; @@ -783,6 +791,12 @@ export async function resolveRuntimeWebTools(params: { config, fetch: toolConfig, }), + readConfiguredCredentialFallback: ({ provider, config, toolConfig }) => + readConfiguredFetchProviderCredentialFallback({ + provider, + config, + fetch: toolConfig, + }), }); await resolveRuntimeWebProviderSelection({ @@ -807,6 +821,12 @@ export async function resolveRuntimeWebTools(params: { config, fetch: toolConfig, }), + readConfiguredCredentialFallback: ({ provider, config, toolConfig }) => + readConfiguredFetchProviderCredentialFallback({ + provider, + config, + fetch: toolConfig, + }), resolveSecretInput: ({ value, path, envVars }) => resolveSecretInputWithEnvFallback({ sourceConfig: params.sourceConfig, diff --git a/src/secrets/runtime.coverage.test.ts b/src/secrets/runtime.coverage.test.ts index e519d8c95c9..e15ade009b4 100644 --- a/src/secrets/runtime.coverage.test.ts +++ b/src/secrets/runtime.coverage.test.ts @@ -154,6 +154,14 @@ const COVERAGE_WEB_FETCH_PROVIDERS = new Map( ); vi.mock("../plugins/web-provider-public-artifacts.explicit.js", () => ({ + loadBundledWebFetchProviderEntriesFromDir: (params: { pluginId: string }) => { + const provider = COVERAGE_WEB_FETCH_PROVIDERS.get(params.pluginId); + return provider ? [provider] : null; + }, + loadBundledWebSearchProviderEntriesFromDir: (params: { pluginId: string }) => { + const provider = COVERAGE_WEB_SEARCH_PROVIDERS.get(params.pluginId); + return provider ? [provider] : null; + }, resolveBundledExplicitWebFetchProvidersFromPublicArtifacts: (params: { onlyPluginIds: readonly string[]; }) => { @@ -230,6 +238,7 @@ const PLUGIN_OWNED_OPENCLAW_COVERAGE_EXCLUSIONS = new Set([ "channels.googlechat.accounts.*.serviceAccount", // Doctor migrates legacy web search config into plugin-owned webSearch config. "tools.web.search.apiKey", + "tools.web.search.*.apiKey", "tools.web.fetch.firecrawl.apiKey", ]); diff --git a/src/secrets/target-registry-data.ts b/src/secrets/target-registry-data.ts index d55dd417961..8f0bb31e4a5 100644 --- a/src/secrets/target-registry-data.ts +++ b/src/secrets/target-registry-data.ts @@ -439,15 +439,16 @@ const CORE_SECRET_TARGET_REGISTRY: SecretTargetRegistryEntry[] = [ includeInAudit: true, }, { - id: "tools.web.fetch.firecrawl.apiKey", - targetType: "tools.web.fetch.firecrawl.apiKey", + id: "tools.web.search.*.apiKey", + targetType: "tools.web.search.*.apiKey", configFile: "openclaw.json", - pathPattern: "tools.web.fetch.firecrawl.apiKey", + pathPattern: "tools.web.search.*.apiKey", secretShape: SECRET_INPUT_SHAPE, expectedResolvedValue: "string", includeInPlan: true, - includeInConfigure: true, + includeInConfigure: false, includeInAudit: true, + providerIdPathSegmentIndex: 3, }, ]; diff --git a/src/test-utils/web-provider-runtime.test-helpers.ts b/src/test-utils/web-provider-runtime.test-helpers.ts index 7116b3c2d52..ca367478cc9 100644 --- a/src/test-utils/web-provider-runtime.test-helpers.ts +++ b/src/test-utils/web-provider-runtime.test-helpers.ts @@ -12,15 +12,19 @@ type CommonWebProviderTestParams = { requiresCredential?: boolean; getCredentialValue?: (config?: Record) => unknown; getConfiguredCredentialValue?: (config?: OpenClawConfig) => unknown; - getConfiguredCredentialFallback?: PluginWebSearchProviderEntry["getConfiguredCredentialFallback"]; + getConfiguredCredentialFallback?: + | PluginWebSearchProviderEntry["getConfiguredCredentialFallback"] + | PluginWebFetchProviderEntry["getConfiguredCredentialFallback"]; }; export type WebSearchTestProviderParams = CommonWebProviderTestParams & { createTool?: PluginWebSearchProviderEntry["createTool"]; + getConfiguredCredentialFallback?: PluginWebSearchProviderEntry["getConfiguredCredentialFallback"]; }; export type WebFetchTestProviderParams = CommonWebProviderTestParams & { createTool?: PluginWebFetchProviderEntry["createTool"]; + getConfiguredCredentialFallback?: PluginWebFetchProviderEntry["getConfiguredCredentialFallback"]; }; function createCommonProviderFields(params: CommonWebProviderTestParams) { diff --git a/src/web-fetch/runtime.test.ts b/src/web-fetch/runtime.test.ts index ba9ed22364e..6bceb198deb 100644 --- a/src/web-fetch/runtime.test.ts +++ b/src/web-fetch/runtime.test.ts @@ -177,6 +177,80 @@ describe("web fetch runtime", () => { expect(requireResolvedWebFetch(resolved).provider.id).toBe("firecrawl"); }); + it("auto-detects providers from configured fallback credentials", () => { + const provider = createFirecrawlProvider({ + getConfiguredCredentialFallback: (config) => { + const pluginConfig = config?.plugins?.entries?.firecrawl?.config as + | { webSearch?: { apiKey?: unknown } } + | undefined; + return pluginConfig?.webSearch?.apiKey === undefined + ? undefined + : { + path: "plugins.entries.firecrawl.config.webSearch.apiKey", + value: pluginConfig.webSearch.apiKey, + }; + }, + }); + resolvePluginWebFetchProvidersMock.mockReturnValue([provider]); + + const resolved = resolveWebFetchDefinition({ + config: { + plugins: { + entries: { + firecrawl: { + config: { + webSearch: { + apiKey: "shared-firecrawl-key", + }, + }, + }, + }, + }, + } as OpenClawConfig, + }); + + expect(requireResolvedWebFetch(resolved).provider.id).toBe("firecrawl"); + }); + + it("auto-detects fallback credentials when the primary fetch key is blank", () => { + const provider = createFirecrawlProvider({ + getConfiguredCredentialValue: getFirecrawlApiKey, + getConfiguredCredentialFallback: (config) => { + const pluginConfig = config?.plugins?.entries?.firecrawl?.config as + | { webSearch?: { apiKey?: unknown } } + | undefined; + return pluginConfig?.webSearch?.apiKey === undefined + ? undefined + : { + path: "plugins.entries.firecrawl.config.webSearch.apiKey", + value: pluginConfig.webSearch.apiKey, + }; + }, + }); + resolvePluginWebFetchProvidersMock.mockReturnValue([provider]); + + const resolved = resolveWebFetchDefinition({ + config: { + plugins: { + entries: { + firecrawl: { + config: { + webFetch: { + apiKey: "", + }, + webSearch: { + apiKey: "shared-firecrawl-key", + }, + }, + }, + }, + }, + } as OpenClawConfig, + }); + + expect(requireResolvedWebFetch(resolved).provider.id).toBe("firecrawl"); + }); + it("falls back to auto-detect when the configured provider is invalid", () => { const provider = createFirecrawlProvider({ getConfiguredCredentialValue: () => "firecrawl-key", diff --git a/src/web-fetch/runtime.ts b/src/web-fetch/runtime.ts index 4313a5ed130..df125df7fcf 100644 --- a/src/web-fetch/runtime.ts +++ b/src/web-fetch/runtime.ts @@ -51,7 +51,11 @@ function resolveFetchConfig(config: OpenClawConfig | undefined): WebFetchConfig function hasEntryCredential( provider: Pick< PluginWebFetchProviderEntry, - "envVars" | "getConfiguredCredentialValue" | "getCredentialValue" | "requiresCredential" + | "envVars" + | "getConfiguredCredentialFallback" + | "getConfiguredCredentialValue" + | "getCredentialValue" + | "requiresCredential" >, config: OpenClawConfig | undefined, fetch: WebFetchConfig | undefined, @@ -63,6 +67,8 @@ function hasEntryCredential( resolveRawValue: ({ provider: currentProvider, config: currentConfig, toolConfig }) => currentProvider.getConfiguredCredentialValue?.(currentConfig) ?? currentProvider.getCredentialValue(toolConfig), + resolveFallbackRawValue: ({ provider: currentProvider, config: currentConfig }) => + currentProvider.getConfiguredCredentialFallback?.(currentConfig)?.value, resolveEnvValue: ({ provider: currentProvider }) => readWebProviderEnvValue(currentProvider.envVars), }); @@ -71,7 +77,11 @@ function hasEntryCredential( export function isWebFetchProviderConfigured(params: { provider: Pick< PluginWebFetchProviderEntry, - "envVars" | "getConfiguredCredentialValue" | "getCredentialValue" | "requiresCredential" + | "envVars" + | "getConfiguredCredentialFallback" + | "getConfiguredCredentialValue" + | "getCredentialValue" + | "requiresCredential" >; config?: OpenClawConfig; }): boolean {