vultr/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-04-28 01:21:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

!feat(plugins): add web fetch provider boundary (#59465)

Browse Source 
* feat(plugins): add web fetch provider boundary

* feat(plugins): add web fetch provider modules

* refactor(web-fetch): remove remaining core firecrawl fetch config

* fix(web-fetch): address review follow-ups

* fix(web-fetch): harden provider runtime boundaries

* fix(web-fetch): restore firecrawl compare helper

* fix(web-fetch): restore env-based provider autodetect

* fix(web-fetch): tighten provider hardening

* fix(web-fetch): restore fetch autodetect and compat args

* chore(changelog): note firecrawl fetch config break
This commit is contained in:
Vincent Koc
2026-04-02 20:25:19 +09:00
committed by GitHub
parent 82d5e6a2f7
commit 38d2faee20
72 changed files with 3425 additions and 1119 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
src/secrets/runtime-shared.ts
View File

@@ -11,10 +11,11 @@ export type SecretResolverWarningCode =
| "WEB_SEARCH_PROVIDER_INVALID_AUTODETECT"
| "WEB_SEARCH_KEY_UNRESOLVED_FALLBACK_USED"
| "WEB_SEARCH_KEY_UNRESOLVED_NO_FALLBACK"
| "WEB_FETCH_PROVIDER_INVALID_AUTODETECT"
| "WEB_FETCH_PROVIDER_KEY_UNRESOLVED_FALLBACK_USED"
| "WEB_FETCH_PROVIDER_KEY_UNRESOLVED_NO_FALLBACK"
| "WEB_X_SEARCH_KEY_UNRESOLVED_FALLBACK_USED"
| "WEB_X_SEARCH_KEY_UNRESOLVED_NO_FALLBACK"
| "WEB_FETCH_FIRECRAWL_KEY_UNRESOLVED_FALLBACK_USED"
| "WEB_FETCH_FIRECRAWL_KEY_UNRESOLVED_NO_FALLBACK";
| "WEB_X_SEARCH_KEY_UNRESOLVED_NO_FALLBACK";
export type SecretResolverWarning = {
code: SecretResolverWarningCode;

311
src/secrets/runtime-web-tools.test.ts
View File

@@ -1,6 +1,9 @@
import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import type { OpenClawConfig } from "../config/config.js";
import type { PluginWebSearchProviderEntry } from "../plugins/types.js";
import type {
PluginWebFetchProviderEntry,
PluginWebSearchProviderEntry,
} from "../plugins/types.js";
type ProviderUnderTest = "brave" | "gemini" | "grok" | "kimi" | "perplexity" | "duckduckgo";
@@ -12,8 +15,18 @@ const { resolveBundledPluginWebSearchProvidersMock } = vi.hoisted(() => ({
resolveBundledPluginWebSearchProvidersMock: vi.fn(() => buildTestWebSearchProviders()),
}));
const { resolvePluginWebFetchProvidersMock } = vi.hoisted(() => ({
resolvePluginWebFetchProvidersMock: vi.fn(() => buildTestWebFetchProviders()),
}));
const { resolveBundledPluginWebFetchProvidersMock } = vi.hoisted(() => ({
resolveBundledPluginWebFetchProvidersMock: vi.fn(() => buildTestWebFetchProviders()),
}));
let bundledWebSearchProviders: typeof import("../plugins/web-search-providers.js");
let runtimeWebSearchProviders: typeof import("../plugins/web-search-providers.runtime.js");
let bundledWebFetchProviders: typeof import("../plugins/web-fetch-providers.js");
let runtimeWebFetchProviders: typeof import("../plugins/web-fetch-providers.runtime.js");
let secretResolve: typeof import("./resolve.js");
let createResolverContext: typeof import("./runtime-shared.js").createResolverContext;
let resolveRuntimeWebTools: typeof import("./runtime-web-tools.js").resolveRuntimeWebTools;
@@ -31,6 +44,18 @@ vi.mock("../plugins/web-search-providers.runtime.js", async (importOriginal) =>
};
});
vi.mock("../plugins/web-fetch-providers.js", () => ({
resolveBundledPluginWebFetchProviders: resolveBundledPluginWebFetchProvidersMock,
}));
vi.mock("../plugins/web-fetch-providers.runtime.js", async (importOriginal) => {
const actual = await importOriginal<typeof import("../plugins/web-fetch-providers.runtime.js")>();
return {
...actual,
resolvePluginWebFetchProviders: resolvePluginWebFetchProvidersMock,
};
});
function asConfig(value: unknown): OpenClawConfig {
return value as OpenClawConfig;
}
@@ -73,6 +98,15 @@ function setConfiguredProviderKey(
webSearch.apiKey = value;
}
function setConfiguredFetchProviderKey(configTarget: OpenClawConfig, value: unknown): void {
const plugins = ensureRecord(configTarget as Record<string, unknown>, "plugins");
const entries = ensureRecord(plugins, "entries");
const pluginEntry = ensureRecord(entries, "firecrawl");
const config = ensureRecord(pluginEntry, "config");
const webFetch = ensureRecord(config, "webFetch");
webFetch.apiKey = value;
}
function createTestProvider(params: {
provider: ProviderUnderTest;
pluginId: string;
@@ -126,6 +160,37 @@ function buildTestWebSearchProviders(): PluginWebSearchProviderEntry[] {
];
}
function buildTestWebFetchProviders(): PluginWebFetchProviderEntry[] {
return [
{
pluginId: "firecrawl",
id: "firecrawl",
label: "firecrawl",
hint: "firecrawl test provider",
envVars: ["FIRECRAWL_API_KEY"],
placeholder: "fc-...",
signupUrl: "https://example.com/firecrawl",
autoDetectOrder: 50,
credentialPath: "plugins.entries.firecrawl.config.webFetch.apiKey",
inactiveSecretPaths: ["plugins.entries.firecrawl.config.webFetch.apiKey"],
getCredentialValue: (fetchConfig) => fetchConfig?.apiKey,
setCredentialValue: (fetchConfigTarget, value) => {
fetchConfigTarget.apiKey = value;
},
getConfiguredCredentialValue: (config) => {
const entryConfig = config?.plugins?.entries?.firecrawl?.config;
return entryConfig && typeof entryConfig === "object"
? (entryConfig as { webFetch?: { apiKey?: unknown } }).webFetch?.apiKey
: undefined;
},
setConfiguredCredentialValue: (configTarget, value) => {
setConfiguredFetchProviderKey(configTarget, value);
},
createTool: () => null,
},
];
}
async function runRuntimeWebTools(params: { config: OpenClawConfig; env?: NodeJS.ProcessEnv }) {
const sourceConfig = structuredClone(params.config);
const resolvedConfig = structuredClone(params.config);
@@ -176,19 +241,19 @@ function readProviderKey(config: OpenClawConfig, provider: ProviderUnderTest): u
return pluginConfig?.webSearch?.apiKey;
}
function expectInactiveFirecrawlSecretRef(params: {
function expectInactiveWebFetchProviderSecretRef(params: {
resolveSpy: ReturnType<typeof vi.spyOn>;
metadata: Awaited<ReturnType<typeof runRuntimeWebTools>>["metadata"];
context: Awaited<ReturnType<typeof runRuntimeWebTools>>["context"];
}) {
expect(params.resolveSpy).not.toHaveBeenCalled();
expect(params.metadata.fetch.firecrawl.active).toBe(false);
expect(params.metadata.fetch.firecrawl.apiKeySource).toBe("secretRef");
expect(params.metadata.fetch.selectedProvider).toBeUndefined();
expect(params.metadata.fetch.selectedProviderKeySource).toBeUndefined();
expect(params.context.warnings).toEqual(
expect.arrayContaining([
expect.objectContaining({
code: "SECRETS_REF_IGNORED_INACTIVE_SURFACE",
path: "tools.web.fetch.firecrawl.apiKey",
path: "plugins.entries.firecrawl.config.webFetch.apiKey",
}),
]),
);
@@ -199,6 +264,8 @@ describe("runtime web tools resolution", () => {
vi.resetModules();
bundledWebSearchProviders = await import("../plugins/web-search-providers.js");
runtimeWebSearchProviders = await import("../plugins/web-search-providers.runtime.js");
bundledWebFetchProviders = await import("../plugins/web-fetch-providers.js");
runtimeWebFetchProviders = await import("../plugins/web-fetch-providers.runtime.js");
secretResolve = await import("./resolve.js");
({ createResolverContext } = await import("./runtime-shared.js"));
({ resolveRuntimeWebTools } = await import("./runtime-web-tools.js"));
@@ -208,6 +275,8 @@ describe("runtime web tools resolution", () => {
runtimeWebSearchProviders.__testing.resetWebSearchProviderSnapshotCacheForTests();
vi.mocked(bundledWebSearchProviders.resolveBundledPluginWebSearchProviders).mockClear();
vi.mocked(runtimeWebSearchProviders.resolvePluginWebSearchProviders).mockClear();
vi.mocked(bundledWebFetchProviders.resolveBundledPluginWebFetchProviders).mockClear();
vi.mocked(runtimeWebFetchProviders.resolvePluginWebFetchProviders).mockClear();
});
afterEach(() => {
@@ -222,12 +291,21 @@ describe("runtime web tools resolution", () => {
const { metadata } = await runRuntimeWebTools({
config: asConfig({
plugins: {
entries: {
firecrawl: {
config: {
webFetch: {
apiKey: { source: "env", provider: "default", id: "FIRECRAWL_API_KEY_REF" },
},
},
},
},
},
tools: {
web: {
fetch: {
firecrawl: {
apiKey: { source: "env", provider: "default", id: "FIRECRAWL_API_KEY_REF" },
},
provider: "firecrawl",
},
},
},
@@ -241,8 +319,8 @@ describe("runtime web tools resolution", () => {
expect(runtimeProviderSpy).not.toHaveBeenCalled();
expect(metadata.search.selectedProvider).toBeUndefined();
expect(metadata.search.providerSource).toBe("none");
expect(metadata.fetch.firecrawl.active).toBe(true);
expect(metadata.fetch.firecrawl.apiKeySource).toBe("env");
expect(metadata.fetch.selectedProvider).toBe("firecrawl");
expect(metadata.fetch.selectedProviderKeySource).toBe("env");
});
it("auto-selects a keyless provider when no credentials are configured", async () => {
@@ -634,45 +712,33 @@ describe("runtime web tools resolution", () => {
expect(genericSpy).not.toHaveBeenCalled();
});
it("does not resolve Firecrawl SecretRef when Firecrawl is inactive", async () => {
it("does not resolve web fetch provider SecretRef when web fetch is inactive", async () => {
const resolveSpy = vi.spyOn(secretResolve, "resolveSecretRefValues");
const { metadata, context } = await runRuntimeWebTools({
config: asConfig({
plugins: {
entries: {
firecrawl: {
config: {
webFetch: {
apiKey: { source: "env", provider: "default", id: "MISSING_FIRECRAWL_REF" },
},
},
},
},
},
tools: {
web: {
fetch: {
enabled: false,
firecrawl: {
apiKey: { source: "env", provider: "default", id: "MISSING_FIRECRAWL_REF" },
},
provider: "firecrawl",
},
},
},
}),
});
expectInactiveFirecrawlSecretRef({ resolveSpy, metadata, context });
});
it("does not resolve Firecrawl SecretRef when Firecrawl is disabled", async () => {
const resolveSpy = vi.spyOn(secretResolve, "resolveSecretRefValues");
const { metadata, context } = await runRuntimeWebTools({
config: asConfig({
tools: {
web: {
fetch: {
enabled: true,
firecrawl: {
enabled: false,
apiKey: { source: "env", provider: "default", id: "MISSING_FIRECRAWL_REF" },
},
},
},
},
}),
});
expectInactiveFirecrawlSecretRef({ resolveSpy, metadata, context });
expectInactiveWebFetchProviderSecretRef({ resolveSpy, metadata, context });
});
it("keeps configured provider metadata and inactive warnings when search is disabled", async () => {
@@ -722,15 +788,24 @@ describe("runtime web tools resolution", () => {
expect(metadata.search.selectedProvider).toBeUndefined();
});
it("uses env fallback for unresolved Firecrawl SecretRef when active", async () => {
it("uses env fallback for unresolved web fetch provider SecretRef when active", async () => {
const { metadata, resolvedConfig, context } = await runRuntimeWebTools({
config: asConfig({
plugins: {
entries: {
firecrawl: {
config: {
webFetch: {
apiKey: { source: "env", provider: "default", id: "MISSING_FIRECRAWL_REF" },
},
},
},
},
},
tools: {
web: {
fetch: {
firecrawl: {
apiKey: { source: "env", provider: "default", id: "MISSING_FIRECRAWL_REF" },
},
provider: "firecrawl",
},
},
},
@@ -740,27 +815,74 @@ describe("runtime web tools resolution", () => {
},
});
expect(metadata.fetch.firecrawl.active).toBe(true);
expect(metadata.fetch.firecrawl.apiKeySource).toBe("env");
expect(resolvedConfig.tools?.web?.fetch?.firecrawl?.apiKey).toBe("firecrawl-fallback-key");
expect(metadata.fetch.selectedProvider).toBe("firecrawl");
expect(metadata.fetch.selectedProviderKeySource).toBe("env");
expect(
(
resolvedConfig.plugins?.entries?.firecrawl?.config as
| { webFetch?: { apiKey?: unknown } }
| undefined
)?.webFetch?.apiKey,
).toBe("firecrawl-fallback-key");
expect(context.warnings).toEqual(
expect.arrayContaining([
expect.objectContaining({
code: "WEB_FETCH_FIRECRAWL_KEY_UNRESOLVED_FALLBACK_USED",
path: "tools.web.fetch.firecrawl.apiKey",
code: "WEB_FETCH_PROVIDER_KEY_UNRESOLVED_FALLBACK_USED",
path: "plugins.entries.firecrawl.config.webFetch.apiKey",
}),
]),
);
});
it("fails fast when active Firecrawl SecretRef is unresolved with no fallback", async () => {
it("resolves plugin-owned web fetch SecretRefs without tools.web.fetch", async () => {
const { metadata, resolvedConfig } = await runRuntimeWebTools({
config: asConfig({
plugins: {
entries: {
firecrawl: {
config: {
webFetch: {
apiKey: { source: "env", provider: "default", id: "FIRECRAWL_API_KEY" },
},
},
},
},
},
}),
env: {
FIRECRAWL_API_KEY: "firecrawl-runtime-key",
},
});
expect(metadata.fetch.providerSource).toBe("auto-detect");
expect(metadata.fetch.selectedProvider).toBe("firecrawl");
expect(metadata.fetch.selectedProviderKeySource).toBe("secretRef");
expect(
(
resolvedConfig.plugins?.entries?.firecrawl?.config as
| { webFetch?: { apiKey?: unknown } }
| undefined
)?.webFetch?.apiKey,
).toBe("firecrawl-runtime-key");
});
it("fails fast when active web fetch provider SecretRef is unresolved with no fallback", async () => {
const sourceConfig = asConfig({
plugins: {
entries: {
firecrawl: {
config: {
webFetch: {
apiKey: { source: "env", provider: "default", id: "MISSING_FIRECRAWL_REF" },
},
},
},
},
},
tools: {
web: {
fetch: {
firecrawl: {
apiKey: { source: "env", provider: "default", id: "MISSING_FIRECRAWL_REF" },
},
provider: "firecrawl",
},
},
},
@@ -777,17 +899,102 @@ describe("runtime web tools resolution", () => {
resolvedConfig,
context,
}),
).rejects.toThrow("[WEB_FETCH_FIRECRAWL_KEY_UNRESOLVED_NO_FALLBACK]");
).rejects.toThrow("[WEB_FETCH_PROVIDER_KEY_UNRESOLVED_NO_FALLBACK]");
expect(context.warnings).toEqual(
expect.arrayContaining([
expect.objectContaining({
code: "WEB_FETCH_FIRECRAWL_KEY_UNRESOLVED_NO_FALLBACK",
path: "tools.web.fetch.firecrawl.apiKey",
code: "WEB_FETCH_PROVIDER_KEY_UNRESOLVED_NO_FALLBACK",
path: "plugins.entries.firecrawl.config.webFetch.apiKey",
}),
]),
);
});
it("rejects env SecretRefs for web fetch provider keys outside provider allowlists", async () => {
const sourceConfig = asConfig({
plugins: {
entries: {
firecrawl: {
config: {
webFetch: {
apiKey: { source: "env", provider: "default", id: "AWS_SECRET_ACCESS_KEY" },
},
},
},
},
},
tools: {
web: {
fetch: {
provider: "firecrawl",
},
},
},
});
const resolvedConfig = structuredClone(sourceConfig);
const context = createResolverContext({
sourceConfig,
env: {
AWS_SECRET_ACCESS_KEY: "not-allowed",
},
});
await expect(
resolveRuntimeWebTools({
sourceConfig,
resolvedConfig,
context,
}),
).rejects.toThrow("[WEB_FETCH_PROVIDER_KEY_UNRESOLVED_NO_FALLBACK]");
expect(context.warnings).toEqual(
expect.arrayContaining([
expect.objectContaining({
code: "WEB_FETCH_PROVIDER_KEY_UNRESOLVED_NO_FALLBACK",
path: "plugins.entries.firecrawl.config.webFetch.apiKey",
message: expect.stringContaining(
'SecretRef env var "AWS_SECRET_ACCESS_KEY" is not allowed.',
),
}),
]),
);
});
it("keeps web fetch provider discovery bundled-only during runtime secret resolution", async () => {
const bundledSpy = vi.mocked(bundledWebFetchProviders.resolveBundledPluginWebFetchProviders);
const runtimeSpy = vi.mocked(runtimeWebFetchProviders.resolvePluginWebFetchProviders);
const { metadata } = await runRuntimeWebTools({
config: asConfig({
plugins: {
load: {
paths: ["/tmp/malicious-plugin"],
},
entries: {
firecrawl: {
enabled: true,
config: {
webFetch: {
apiKey: "firecrawl-config-key",
},
},
},
},
},
tools: {
web: {
fetch: {
provider: "firecrawl",
},
},
},
}),
});
expect(metadata.fetch.selectedProvider).toBe("firecrawl");
expect(bundledSpy).toHaveBeenCalled();
expect(runtimeSpy).not.toHaveBeenCalled();
});
it("resolves x_search SecretRef and writes the resolved key into runtime config", async () => {
const { metadata, resolvedConfig, context } = await runRuntimeWebTools({
config: asConfig({

474
src/secrets/runtime-web-tools.ts
View File

@@ -1,11 +1,16 @@
import type { OpenClawConfig } from "../config/config.js";
import { resolveSecretInputRef } from "../config/types.secrets.js";
import { resolveBundledWebFetchPluginId } from "../plugins/bundled-web-fetch-provider-ids.js";
import { listBundledWebSearchPluginIds } from "../plugins/bundled-web-search-ids.js";
import { resolveBundledWebSearchPluginId } from "../plugins/bundled-web-search-provider-ids.js";
import type {
PluginWebFetchProviderEntry,
PluginWebSearchProviderEntry,
WebFetchCredentialResolutionSource,
WebSearchCredentialResolutionSource,
} from "../plugins/types.js";
import { resolveBundledPluginWebFetchProviders } from "../plugins/web-fetch-providers.js";
import { sortWebFetchProvidersForAutoDetect } from "../plugins/web-fetch-providers.shared.js";
import { resolveBundledPluginWebSearchProviders } from "../plugins/web-search-providers.js";
import { resolvePluginWebSearchProviders } from "../plugins/web-search-providers.runtime.js";
import { sortWebSearchProvidersForAutoDetect } from "../plugins/web-search-providers.shared.js";
@@ -21,18 +26,19 @@ import {
import type {
RuntimeWebDiagnostic,
RuntimeWebDiagnosticCode,
RuntimeWebFetchFirecrawlMetadata,
RuntimeWebFetchMetadata,
RuntimeWebSearchMetadata,
RuntimeWebToolsMetadata,
RuntimeWebXSearchMetadata,
} from "./runtime-web-tools.types.js";
type WebSearchProvider = string;
type WebFetchProvider = string;
export type {
RuntimeWebDiagnostic,
RuntimeWebDiagnosticCode,
RuntimeWebFetchFirecrawlMetadata,
RuntimeWebFetchMetadata,
RuntimeWebSearchMetadata,
RuntimeWebToolsMetadata,
RuntimeWebXSearchMetadata,
@@ -46,7 +52,7 @@ type FetchConfig = NonNullable<OpenClawConfig["tools"]>["web"] extends infer Web
type SecretResolutionResult = {
value?: string;
source: WebSearchCredentialResolutionSource;
source: WebSearchCredentialResolutionSource | WebFetchCredentialResolutionSource;
secretRefConfigured: boolean;
unresolvedRefReason?: string;
fallbackEnvVar?: string;
@@ -71,6 +77,20 @@ function normalizeProvider(
return undefined;
}
function normalizeFetchProvider(
value: unknown,
providers: PluginWebFetchProviderEntry[],
): WebFetchProvider | undefined {
if (typeof value !== "string") {
return undefined;
}
const normalized = value.trim().toLowerCase();
if (providers.some((provider) => provider.id === normalized)) {
return normalized;
}
return undefined;
}
function hasCustomWebSearchPluginRisk(config: OpenClawConfig): boolean {
const plugins = config.plugins;
if (!plugins) {
@@ -132,6 +152,7 @@ async function resolveSecretInputWithEnvFallback(params: {
value: unknown;
path: string;
envVars: string[];
restrictEnvRefsToEnvVars?: boolean;
}): Promise<SecretResolutionResult> {
const { ref } = resolveSecretInputRef({
value: params.value,
@@ -169,35 +190,43 @@ async function resolveSecretInputWithEnvFallback(params: {
let resolvedFromRef: string | undefined;
let unresolvedRefReason: string | undefined;
try {
const resolved = await resolveSecretRefValues([ref], {
config: params.sourceConfig,
env: params.context.env,
cache: params.context.cache,
});
const resolvedValue = resolved.get(secretRefKey(ref));
if (typeof resolvedValue !== "string") {
unresolvedRefReason = buildUnresolvedReason({
path: params.path,
kind: "non-string",
refLabel,
if (
params.restrictEnvRefsToEnvVars === true &&
ref.source === "env" &&
!params.envVars.includes(ref.id)
) {
unresolvedRefReason = `${params.path} SecretRef env var "${ref.id}" is not allowed.`;
} else {
try {
const resolved = await resolveSecretRefValues([ref], {
config: params.sourceConfig,
env: params.context.env,
cache: params.context.cache,
});
} else {
resolvedFromRef = normalizeSecretInput(resolvedValue);
if (!resolvedFromRef) {
const resolvedValue = resolved.get(secretRefKey(ref));
if (typeof resolvedValue !== "string") {
unresolvedRefReason = buildUnresolvedReason({
path: params.path,
kind: "empty",
kind: "non-string",
refLabel,
});
} else {
resolvedFromRef = normalizeSecretInput(resolvedValue);
if (!resolvedFromRef) {
unresolvedRefReason = buildUnresolvedReason({
path: params.path,
kind: "empty",
refLabel,
});
}
}
} catch {
unresolvedRefReason = buildUnresolvedReason({
path: params.path,
kind: "unresolved",
refLabel,
});
}
} catch {
unresolvedRefReason = buildUnresolvedReason({
path: params.path,
kind: "unresolved",
refLabel,
});
}
if (resolvedFromRef) {
@@ -256,17 +285,6 @@ function setResolvedWebSearchApiKey(params: {
params.provider.setCredentialValue(search, params.value);
}
function setResolvedFirecrawlApiKey(params: {
resolvedConfig: OpenClawConfig;
value: string;
}): void {
const tools = ensureObject(params.resolvedConfig as Record<string, unknown>, "tools");
const web = ensureObject(tools, "web");
const fetch = ensureObject(web, "fetch");
const firecrawl = ensureObject(fetch, "firecrawl");
firecrawl.apiKey = params.value;
}
function setResolvedXSearchApiKey(params: { resolvedConfig: OpenClawConfig; value: string }): void {
const tools = ensureObject(params.resolvedConfig as Record<string, unknown>, "tools");
const web = ensureObject(tools, "web");
@@ -284,10 +302,7 @@ function readConfiguredProviderCredential(params: {
search: Record<string, unknown> | undefined;
}): unknown {
const configuredValue = params.provider.getConfiguredCredentialValue?.(params.config);
return (
configuredValue ??
(params.provider.id === "brave" ? params.provider.getCredentialValue(params.search) : undefined)
);
return configuredValue ?? params.provider.getCredentialValue(params.search);
}
function inactivePathsForProvider(provider: PluginWebSearchProviderEntry): string[] {
@@ -299,6 +314,43 @@ function inactivePathsForProvider(provider: PluginWebSearchProviderEntry): strin
: [provider.credentialPath];
}
function setResolvedWebFetchApiKey(params: {
resolvedConfig: OpenClawConfig;
provider: PluginWebFetchProviderEntry;
value: string;
}): void {
const tools = ensureObject(params.resolvedConfig as Record<string, unknown>, "tools");
const web = ensureObject(tools, "web");
const fetch = ensureObject(web, "fetch");
if (params.provider.setConfiguredCredentialValue) {
params.provider.setConfiguredCredentialValue(params.resolvedConfig, params.value);
return;
}
params.provider.setCredentialValue(fetch, params.value);
}
function keyPathForFetchProvider(provider: PluginWebFetchProviderEntry): string {
return provider.credentialPath;
}
function readConfiguredFetchProviderCredential(params: {
provider: PluginWebFetchProviderEntry;
config: OpenClawConfig;
fetch: Record<string, unknown> | undefined;
}): unknown {
const configuredValue = params.provider.getConfiguredCredentialValue?.(params.config);
return configuredValue ?? params.provider.getCredentialValue(params.fetch);
}
function inactivePathsForFetchProvider(provider: PluginWebFetchProviderEntry): string[] {
if (provider.requiresCredential === false) {
return [];
}
return provider.inactiveSecretPaths?.length
? provider.inactiveSecretPaths
: [provider.credentialPath];
}
function hasConfiguredSecretRef(value: unknown, defaults: SecretDefaults | undefined): boolean {
return Boolean(
resolveSecretInputRef({
@@ -704,106 +756,278 @@ export async function resolveRuntimeWebTools(params: {
}
const fetch = isRecord(web?.fetch) ? (web.fetch as FetchConfig) : undefined;
const firecrawl = isRecord(fetch?.firecrawl) ? fetch.firecrawl : undefined;
const fetchEnabled = fetch?.enabled !== false;
const firecrawlEnabled = firecrawl?.enabled !== false;
const firecrawlActive = Boolean(fetchEnabled && firecrawlEnabled);
const firecrawlPath = "tools.web.fetch.firecrawl.apiKey";
let firecrawlResolution: SecretResolutionResult = {
source: "missing",
secretRefConfigured: false,
fallbackUsedAfterRefFailure: false,
const rawFetchProvider =
typeof fetch?.provider === "string" ? fetch.provider.trim().toLowerCase() : "";
const configuredBundledFetchPluginId = resolveBundledWebFetchPluginId(rawFetchProvider);
const fetchMetadata: RuntimeWebFetchMetadata = {
providerSource: "none",
diagnostics: [],
};
const firecrawlDiagnostics: RuntimeWebDiagnostic[] = [];
if (firecrawlActive) {
firecrawlResolution = await resolveSecretInputWithEnvFallback({
sourceConfig: params.sourceConfig,
context: params.context,
defaults,
value: firecrawl?.apiKey,
path: firecrawlPath,
envVars: ["FIRECRAWL_API_KEY"],
});
if (firecrawlResolution.value) {
setResolvedFirecrawlApiKey({
resolvedConfig: params.resolvedConfig,
value: firecrawlResolution.value,
const fetchProviders = sortWebFetchProvidersForAutoDetect(
configuredBundledFetchPluginId
? resolveBundledPluginWebFetchProviders({
config: params.sourceConfig,
env: { ...process.env, ...params.context.env },
bundledAllowlistCompat: true,
onlyPluginIds: [configuredBundledFetchPluginId],
})
: resolveBundledPluginWebFetchProviders({
config: params.sourceConfig,
env: { ...process.env, ...params.context.env },
bundledAllowlistCompat: true,
}),
);
const hasConfiguredFetchSurface =
Boolean(fetch) ||
fetchProviders.some((provider) => {
const value = readConfiguredFetchProviderCredential({
provider,
config: params.sourceConfig,
fetch,
});
}
return value !== undefined;
});
const fetchEnabled = hasConfiguredFetchSurface && fetch?.enabled !== false;
const configuredFetchProvider = normalizeFetchProvider(rawFetchProvider, fetchProviders);
if (firecrawlResolution.secretRefConfigured) {
if (firecrawlResolution.fallbackUsedAfterRefFailure) {
if (rawFetchProvider && !configuredFetchProvider) {
const diagnostic: RuntimeWebDiagnostic = {
code: "WEB_FETCH_PROVIDER_INVALID_AUTODETECT",
message: `tools.web.fetch.provider is "${rawFetchProvider}". Falling back to auto-detect precedence.`,
path: "tools.web.fetch.provider",
};
diagnostics.push(diagnostic);
fetchMetadata.diagnostics.push(diagnostic);
pushWarning(params.context, {
code: "WEB_FETCH_PROVIDER_INVALID_AUTODETECT",
path: "tools.web.fetch.provider",
message: diagnostic.message,
});
}
if (configuredFetchProvider) {
fetchMetadata.providerConfigured = configuredFetchProvider;
fetchMetadata.providerSource = "configured";
}
if (fetchEnabled) {
const candidates = configuredFetchProvider
? fetchProviders.filter((provider) => provider.id === configuredFetchProvider)
: fetchProviders;
const unresolvedWithoutFallback: Array<{
provider: WebFetchProvider;
path: string;
reason: string;
}> = [];
let selectedProvider: WebFetchProvider | undefined;
let selectedResolution: SecretResolutionResult | undefined;
for (const provider of candidates) {
if (provider.requiresCredential === false) {
selectedProvider = provider.id;
selectedResolution = {
source: "missing",
secretRefConfigured: false,
fallbackUsedAfterRefFailure: false,
};
break;
}
const path = keyPathForFetchProvider(provider);
const value = readConfiguredFetchProviderCredential({
provider,
config: params.sourceConfig,
fetch,
});
const resolution = await resolveSecretInputWithEnvFallback({
sourceConfig: params.sourceConfig,
context: params.context,
defaults,
value,
path,
envVars: provider.envVars,
restrictEnvRefsToEnvVars: true,
});
if (resolution.secretRefConfigured && resolution.fallbackUsedAfterRefFailure) {
const diagnostic: RuntimeWebDiagnostic = {
code: "WEB_FETCH_FIRECRAWL_KEY_UNRESOLVED_FALLBACK_USED",
code: "WEB_FETCH_PROVIDER_KEY_UNRESOLVED_FALLBACK_USED",
message:
`${firecrawlPath} SecretRef could not be resolved; using ${firecrawlResolution.fallbackEnvVar ?? "env fallback"}. ` +
(firecrawlResolution.unresolvedRefReason ?? "").trim(),
path: firecrawlPath,
`${path} SecretRef could not be resolved; using ${resolution.fallbackEnvVar ?? "env fallback"}. ` +
(resolution.unresolvedRefReason ?? "").trim(),
path,
};
diagnostics.push(diagnostic);
firecrawlDiagnostics.push(diagnostic);
fetchMetadata.diagnostics.push(diagnostic);
pushWarning(params.context, {
code: "WEB_FETCH_FIRECRAWL_KEY_UNRESOLVED_FALLBACK_USED",
path: firecrawlPath,
code: "WEB_FETCH_PROVIDER_KEY_UNRESOLVED_FALLBACK_USED",
path,
message: diagnostic.message,
});
}
if (!firecrawlResolution.value && firecrawlResolution.unresolvedRefReason) {
if (resolution.secretRefConfigured && !resolution.value && resolution.unresolvedRefReason) {
unresolvedWithoutFallback.push({
provider: provider.id,
path,
reason: resolution.unresolvedRefReason,
});
}
if (configuredFetchProvider) {
selectedProvider = provider.id;
selectedResolution = resolution;
if (resolution.value) {
setResolvedWebFetchApiKey({
resolvedConfig: params.resolvedConfig,
provider,
value: resolution.value,
});
}
break;
}
if (resolution.value) {
selectedProvider = provider.id;
selectedResolution = resolution;
setResolvedWebFetchApiKey({
resolvedConfig: params.resolvedConfig,
provider,
value: resolution.value,
});
break;
}
}
const failUnresolvedFetchNoFallback = (unresolved: { path: string; reason: string }) => {
const diagnostic: RuntimeWebDiagnostic = {
code: "WEB_FETCH_PROVIDER_KEY_UNRESOLVED_NO_FALLBACK",
message: unresolved.reason,
path: unresolved.path,
};
diagnostics.push(diagnostic);
fetchMetadata.diagnostics.push(diagnostic);
pushWarning(params.context, {
code: "WEB_FETCH_PROVIDER_KEY_UNRESOLVED_NO_FALLBACK",
path: unresolved.path,
message: unresolved.reason,
});
throw new Error(`[WEB_FETCH_PROVIDER_KEY_UNRESOLVED_NO_FALLBACK] ${unresolved.reason}`);
};
if (configuredFetchProvider) {
const unresolved = unresolvedWithoutFallback[0];
if (unresolved) {
failUnresolvedFetchNoFallback(unresolved);
}
} else {
if (!selectedProvider && unresolvedWithoutFallback.length > 0) {
failUnresolvedFetchNoFallback(unresolvedWithoutFallback[0]);
}
if (selectedProvider) {
const selectedProviderEntry = fetchProviders.find((entry) => entry.id === selectedProvider);
const selectedDetails =
selectedProviderEntry?.requiresCredential === false
? `tools.web.fetch auto-detected keyless provider "${selectedProvider}" as the default fallback.`
: `tools.web.fetch auto-detected provider "${selectedProvider}" from available credentials.`;
const diagnostic: RuntimeWebDiagnostic = {
code: "WEB_FETCH_FIRECRAWL_KEY_UNRESOLVED_NO_FALLBACK",
message: firecrawlResolution.unresolvedRefReason,
path: firecrawlPath,
code: "WEB_FETCH_AUTODETECT_SELECTED",
message: selectedDetails,
path: "tools.web.fetch.provider",
};
diagnostics.push(diagnostic);
firecrawlDiagnostics.push(diagnostic);
pushWarning(params.context, {
code: "WEB_FETCH_FIRECRAWL_KEY_UNRESOLVED_NO_FALLBACK",
path: firecrawlPath,
message: firecrawlResolution.unresolvedRefReason,
});
throw new Error(
`[WEB_FETCH_FIRECRAWL_KEY_UNRESOLVED_NO_FALLBACK] ${firecrawlResolution.unresolvedRefReason}`,
fetchMetadata.diagnostics.push(diagnostic);
}
}
if (selectedProvider) {
fetchMetadata.selectedProvider = selectedProvider;
fetchMetadata.selectedProviderKeySource = selectedResolution?.source;
if (!configuredFetchProvider) {
fetchMetadata.providerSource = "auto-detect";
}
const provider = fetchProviders.find((entry) => entry.id === selectedProvider);
if (provider?.resolveRuntimeMetadata) {
Object.assign(
fetchMetadata,
await provider.resolveRuntimeMetadata({
config: params.sourceConfig,
fetchConfig: fetch,
runtimeMetadata: fetchMetadata,
resolvedCredential: selectedResolution
? {
value: selectedResolution.value,
source: selectedResolution.source,
fallbackEnvVar: selectedResolution.fallbackEnvVar,
}
: undefined,
}),
);
}
}
} else {
if (hasConfiguredSecretRef(firecrawl?.apiKey, defaults)) {
pushInactiveSurfaceWarning({
context: params.context,
path: firecrawlPath,
details: !fetchEnabled
? "tools.web.fetch is disabled."
: "tools.web.fetch.firecrawl.enabled is false.",
}
if (fetchEnabled && !configuredFetchProvider && fetchMetadata.selectedProvider) {
for (const provider of fetchProviders) {
if (provider.id === fetchMetadata.selectedProvider) {
continue;
}
const value = readConfiguredFetchProviderCredential({
provider,
config: params.sourceConfig,
fetch,
});
firecrawlResolution = {
source: "secretRef",
secretRefConfigured: true,
fallbackUsedAfterRefFailure: false,
};
} else {
const configuredInlineValue = normalizeSecretInput(firecrawl?.apiKey);
if (configuredInlineValue) {
firecrawlResolution = {
value: configuredInlineValue,
source: "config",
secretRefConfigured: false,
fallbackUsedAfterRefFailure: false,
};
} else {
const envFallback = readNonEmptyEnvValue(params.context.env, ["FIRECRAWL_API_KEY"]);
if (envFallback.value) {
firecrawlResolution = {
value: envFallback.value,
source: "env",
fallbackEnvVar: envFallback.envVar,
secretRefConfigured: false,
fallbackUsedAfterRefFailure: false,
};
}
if (!hasConfiguredSecretRef(value, defaults)) {
continue;
}
for (const path of inactivePathsForFetchProvider(provider)) {
pushInactiveSurfaceWarning({
context: params.context,
path,
details: `tools.web.fetch auto-detected provider is "${fetchMetadata.selectedProvider}".`,
});
}
}
} else if (fetch && !fetchEnabled) {
for (const provider of fetchProviders) {
const value = readConfiguredFetchProviderCredential({
provider,
config: params.sourceConfig,
fetch,
});
if (!hasConfiguredSecretRef(value, defaults)) {
continue;
}
for (const path of inactivePathsForFetchProvider(provider)) {
pushInactiveSurfaceWarning({
context: params.context,
path,
details: "tools.web.fetch is disabled.",
});
}
}
}
if (fetchEnabled && fetch && configuredFetchProvider) {
for (const provider of fetchProviders) {
if (provider.id === configuredFetchProvider) {
continue;
}
const value = readConfiguredFetchProviderCredential({
provider,
config: params.sourceConfig,
fetch,
});
if (!hasConfiguredSecretRef(value, defaults)) {
continue;
}
for (const path of inactivePathsForFetchProvider(provider)) {
pushInactiveSurfaceWarning({
context: params.context,
path,
details: `tools.web.fetch.provider is "${configuredFetchProvider}".`,
});
}
}
}
@@ -815,13 +1039,7 @@ export async function resolveRuntimeWebTools(params: {
apiKeySource: xSearchResolution.source,
diagnostics: xSearchDiagnostics,
},
fetch: {
firecrawl: {
active: firecrawlActive,
apiKeySource: firecrawlResolution.source,
diagnostics: firecrawlDiagnostics,
},
},
fetch: fetchMetadata,
diagnostics,
};
}

20
src/secrets/runtime-web-tools.types.ts
View File

@@ -3,10 +3,12 @@ export type RuntimeWebDiagnosticCode =
| "WEB_SEARCH_AUTODETECT_SELECTED"
| "WEB_SEARCH_KEY_UNRESOLVED_FALLBACK_USED"
| "WEB_SEARCH_KEY_UNRESOLVED_NO_FALLBACK"
| "WEB_FETCH_PROVIDER_INVALID_AUTODETECT"
| "WEB_FETCH_AUTODETECT_SELECTED"
| "WEB_FETCH_PROVIDER_KEY_UNRESOLVED_FALLBACK_USED"
| "WEB_FETCH_PROVIDER_KEY_UNRESOLVED_NO_FALLBACK"
| "WEB_X_SEARCH_KEY_UNRESOLVED_FALLBACK_USED"
| "WEB_X_SEARCH_KEY_UNRESOLVED_NO_FALLBACK"
| "WEB_FETCH_FIRECRAWL_KEY_UNRESOLVED_FALLBACK_USED"
| "WEB_FETCH_FIRECRAWL_KEY_UNRESOLVED_NO_FALLBACK";
| "WEB_X_SEARCH_KEY_UNRESOLVED_NO_FALLBACK";
export type RuntimeWebDiagnostic = {
code: RuntimeWebDiagnosticCode;
@@ -23,9 +25,11 @@ export type RuntimeWebSearchMetadata = {
diagnostics: RuntimeWebDiagnostic[];
};
export type RuntimeWebFetchFirecrawlMetadata = {
active: boolean;
apiKeySource: "config" | "secretRef" | "env" | "missing";
export type RuntimeWebFetchMetadata = {
providerConfigured?: string;
providerSource: "configured" | "auto-detect" | "none";
selectedProvider?: string;
selectedProviderKeySource?: "config" | "secretRef" | "env" | "missing";
diagnostics: RuntimeWebDiagnostic[];
};
@@ -38,8 +42,6 @@ export type RuntimeWebXSearchMetadata = {
export type RuntimeWebToolsMetadata = {
search: RuntimeWebSearchMetadata;
xSearch: RuntimeWebXSearchMetadata;
fetch: {
firecrawl: RuntimeWebFetchFirecrawlMetadata;
};
fetch: RuntimeWebFetchMetadata;
diagnostics: RuntimeWebDiagnostic[];
};

22
src/secrets/target-registry-data.ts
View File

@@ -703,17 +703,6 @@ const SECRET_TARGET_REGISTRY: SecretTargetRegistryEntry[] = [
includeInConfigure: true,
includeInAudit: true,
},
{
id: "tools.web.fetch.firecrawl.apiKey",
targetType: "tools.web.fetch.firecrawl.apiKey",
configFile: "openclaw.json",
pathPattern: "tools.web.fetch.firecrawl.apiKey",
secretShape: SECRET_INPUT_SHAPE,
expectedResolvedValue: "string",
includeInPlan: true,
includeInConfigure: true,
includeInAudit: true,
},
{
id: "tools.web.x_search.apiKey",
targetType: "tools.web.x_search.apiKey",
@@ -802,6 +791,17 @@ const SECRET_TARGET_REGISTRY: SecretTargetRegistryEntry[] = [
includeInConfigure: true,
includeInAudit: true,
},
{
id: "plugins.entries.firecrawl.config.webFetch.apiKey",
targetType: "plugins.entries.firecrawl.config.webFetch.apiKey",
configFile: "openclaw.json",
pathPattern: "plugins.entries.firecrawl.config.webFetch.apiKey",
secretShape: SECRET_INPUT_SHAPE,
expectedResolvedValue: "string",
includeInPlan: true,
includeInConfigure: true,
includeInAudit: true,
},
{
id: "plugins.entries.tavily.config.webSearch.apiKey",
targetType: "plugins.entries.tavily.config.webSearch.apiKey",