!feat(plugins): add web fetch provider boundary (#59465)

* feat(plugins): add web fetch provider boundary

* feat(plugins): add web fetch provider modules

* refactor(web-fetch): remove remaining core firecrawl fetch config

* fix(web-fetch): address review follow-ups

* fix(web-fetch): harden provider runtime boundaries

* fix(web-fetch): restore firecrawl compare helper

* fix(web-fetch): restore env-based provider autodetect

* fix(web-fetch): tighten provider hardening

* fix(web-fetch): restore fetch autodetect and compat args

* chore(changelog): note firecrawl fetch config break

src/cli/command-secret-gateway.test.ts

@@ -312,30 +312,42 @@ describe("resolveCommandSecretRefsViaGateway", () => {
     });
   }, 300_000);
 
-  it("falls back to local resolution for Firecrawl SecretRefs when gateway is unavailable", async () => {
+  it("falls back to local resolution for web fetch provider SecretRefs when gateway is unavailable", async () => {
     const envKey = "WEB_FETCH_FIRECRAWL_API_KEY_LOCAL_FALLBACK";
     await withEnvValue(envKey, "firecrawl-local-fallback-key", async () => {
       callGateway.mockRejectedValueOnce(new Error("gateway closed"));
       const result = await resolveCommandSecretRefsViaGateway({
         config: {
+          plugins: {
+            entries: {
+              firecrawl: {
+                config: {
+                  webFetch: {
+                    apiKey: { source: "env", provider: "default", id: envKey },
+                  },
+                },
+              },
+            },
+          },
           tools: {
             web: {
               fetch: {
-                firecrawl: {
-                  apiKey: { source: "env", provider: "default", id: envKey },
-                },
+                provider: "firecrawl",
               },
             },
           },
         } as OpenClawConfig,
         commandName: "agent",
-        targetIds: new Set(["tools.web.fetch.firecrawl.apiKey"]),
+        targetIds: new Set(["plugins.entries.firecrawl.config.webFetch.apiKey"]),
       });
 
-      expect(result.resolvedConfig.tools?.web?.fetch?.firecrawl?.apiKey).toBe(
-        "firecrawl-local-fallback-key",
+      const firecrawlConfig = result.resolvedConfig.plugins?.entries?.firecrawl?.config as
+        | { webFetch?: { apiKey?: unknown } }
+        | undefined;
+      expect(firecrawlConfig?.webFetch?.apiKey).toBe("firecrawl-local-fallback-key");
+      expect(result.targetStatesByPath["plugins.entries.firecrawl.config.webFetch.apiKey"]).toBe(
+        "resolved_local",
       );
-      expect(result.targetStatesByPath["tools.web.fetch.firecrawl.apiKey"]).toBe("resolved_local");
       expectGatewayUnavailableLocalFallbackDiagnostics(result);
     });
   });

src/cli/command-secret-gateway.ts

@@ -2,6 +2,7 @@ import type { OpenClawConfig } from "../config/config.js";
 import { resolveSecretInputRef } from "../config/types.secrets.js";
 import { callGateway } from "../gateway/call.js";
 import { validateSecretsResolveResult } from "../gateway/protocol/index.js";
+import { resolveBundledWebFetchPluginId } from "../plugins/bundled-web-fetch-provider-ids.js";
 import { resolveBundledWebSearchPluginId } from "../plugins/bundled-web-search-provider-ids.js";
 import {
   analyzeCommandSecretAssignmentsFromSnapshot,
@@ -58,18 +59,16 @@ type GatewaySecretsResolveResult = {
 const WEB_RUNTIME_SECRET_TARGET_ID_PREFIXES = [
   "tools.web.search",
   "plugins.entries.",
-  "tools.web.fetch.firecrawl",
   "tools.web.x_search",
 ] as const;
 const WEB_RUNTIME_SECRET_PATH_PREFIXES = [
   "tools.web.search.",
   "plugins.entries.",
-  "tools.web.fetch.firecrawl.",
   "tools.web.x_search.",
 ] as const;
 
 function pluginIdFromRuntimeWebPath(path: string): string | undefined {
-  const match = /^plugins\.entries\.([^.]+)\.config\.webSearch\.apiKey$/.exec(path);
+  const match = /^plugins\.entries\.([^.]+)\.config\.(webSearch|webFetch)\.apiKey$/.exec(path);
   return match?.[1];
 }
 
@@ -111,11 +110,6 @@ function classifyRuntimeWebTargetPathState(params: {
   config: OpenClawConfig;
   path: string;
 }): "active" | "inactive" | "unknown" {
-  if (params.path === "tools.web.fetch.firecrawl.apiKey") {
-    const fetch = params.config.tools?.web?.fetch;
-    return fetch?.enabled !== false && fetch?.firecrawl?.enabled !== false ? "active" : "inactive";
-  }
-
   if (params.path === "tools.web.x_search.apiKey") {
     return params.config.tools?.web?.x_search?.enabled !== false ? "active" : "inactive";
   }
@@ -126,6 +120,20 @@ function classifyRuntimeWebTargetPathState(params: {
 
   const pluginId = pluginIdFromRuntimeWebPath(params.path);
   if (pluginId) {
+    if (params.path.endsWith(".config.webFetch.apiKey")) {
+      const fetch = params.config.tools?.web?.fetch;
+      if (fetch?.enabled === false) {
+        return "inactive";
+      }
+      const configuredProvider =
+        typeof fetch?.provider === "string" ? fetch.provider.trim().toLowerCase() : "";
+      if (!configuredProvider) {
+        return "active";
+      }
+      return resolveBundledWebFetchPluginId(configuredProvider) === pluginId
+        ? "active"
+        : "inactive";
+    }
     const search = params.config.tools?.web?.search;
     if (search?.enabled === false) {
       return "inactive";
@@ -161,17 +169,6 @@ function describeInactiveRuntimeWebTargetPath(params: {
   config: OpenClawConfig;
   path: string;
 }): string | undefined {
-  if (params.path === "tools.web.fetch.firecrawl.apiKey") {
-    const fetch = params.config.tools?.web?.fetch;
-    if (fetch?.enabled === false) {
-      return "tools.web.fetch is disabled.";
-    }
-    if (fetch?.firecrawl?.enabled === false) {
-      return "tools.web.fetch.firecrawl.enabled is false.";
-    }
-    return undefined;
-  }
-
   if (params.path === "tools.web.x_search.apiKey") {
     return params.config.tools?.web?.x_search?.enabled === false
       ? "tools.web.x_search is disabled."
@@ -186,6 +183,18 @@ function describeInactiveRuntimeWebTargetPath(params: {
 
   const pluginId = pluginIdFromRuntimeWebPath(params.path);
   if (pluginId) {
+    if (params.path.endsWith(".config.webFetch.apiKey")) {
+      const fetch = params.config.tools?.web?.fetch;
+      if (fetch?.enabled === false) {
+        return "tools.web.fetch is disabled.";
+      }
+      const configuredProvider =
+        typeof fetch?.provider === "string" ? fetch.provider.trim().toLowerCase() : "";
+      if (configuredProvider) {
+        return `tools.web.fetch.provider is "${configuredProvider}".`;
+      }
+      return undefined;
+    }
     const search = params.config.tools?.web?.search;
     if (search?.enabled === false) {
       return "tools.web.search is disabled.";
@@ -367,7 +376,7 @@ function isUnsupportedSecretsResolveError(err: unknown): boolean {
 
 function isDirectRuntimeWebTargetPath(path: string): boolean {
   return (
-    path === "tools.web.fetch.firecrawl.apiKey" ||
+    /^plugins\.entries\.[^.]+\.config\.(webSearch|webFetch)\.apiKey$/.test(path) ||
     path === "tools.web.x_search.apiKey" ||
     /^tools\.web\.search\.[^.]+\.apiKey$/.test(path)
   );

src/cli/command-secret-targets.test.ts

@@ -10,7 +10,7 @@ describe("command secret target ids", () => {
     const ids = getAgentRuntimeCommandSecretTargetIds();
     expect(ids.has("agents.defaults.memorySearch.remote.apiKey")).toBe(true);
     expect(ids.has("agents.list[].memorySearch.remote.apiKey")).toBe(true);
-    expect(ids.has("tools.web.fetch.firecrawl.apiKey")).toBe(true);
+    expect(ids.has("plugins.entries.firecrawl.config.webFetch.apiKey")).toBe(true);
     expect(ids.has("tools.web.x_search.apiKey")).toBe(true);
   });
 

src/cli/command-secret-targets.ts

@@ -12,6 +12,17 @@ function idsByPrefix(prefixes: readonly string[]): string[] {
     .toSorted();
 }
 
+function idsByPredicate(predicate: (id: string) => boolean): string[] {
+  return listSecretTargetRegistryEntries()
+    .map((entry) => entry.id)
+    .filter(predicate)
+    .toSorted();
+}
+
+const WEB_PLUGIN_SECRET_TARGETS = idsByPredicate((id) =>
+  /^plugins\.entries\.[^.]+\.config\.(webSearch|webFetch)\.apiKey$/.test(id),
+);
+
 const COMMAND_SECRET_TARGETS = {
   qrRemote: ["gateway.remote.token", "gateway.remote.password"],
   channels: idsByPrefix(["channels."]),
@@ -24,9 +35,8 @@ const COMMAND_SECRET_TARGETS = {
     "skills.entries.",
     "messages.tts.",
     "tools.web.search",
-    "tools.web.fetch.firecrawl.",
     "tools.web.x_search",
-  ]),
+  ]).concat(WEB_PLUGIN_SECRET_TARGETS),
   status: idsByPrefix([
     "channels.",
     "agents.defaults.memorySearch.remote.",