From 49fbecbf1667f760af4dd255310732e9a327b3cf Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Tue, 7 Apr 2026 08:34:52 +0100 Subject: [PATCH] perf(plugin-sdk): add web fetch contract artifacts --- .../.generated/plugin-sdk-api-baseline.sha256 | 4 +- docs/plugins/sdk-overview.md | 1 + .../firecrawl/web-fetch-contract-api.ts | 65 +++++++++++++++++++ package.json | 4 ++ scripts/lib/plugin-sdk-entrypoints.json | 1 + src/plugin-sdk/provider-web-fetch-contract.ts | 6 ++ .../contracts/plugin-sdk-subpaths.test.ts | 9 +++ .../web-provider-public-artifacts.test.ts | 10 +++ src/plugins/web-provider-public-artifacts.ts | 6 +- 9 files changed, 103 insertions(+), 3 deletions(-) create mode 100644 extensions/firecrawl/web-fetch-contract-api.ts create mode 100644 src/plugin-sdk/provider-web-fetch-contract.ts diff --git a/docs/.generated/plugin-sdk-api-baseline.sha256 b/docs/.generated/plugin-sdk-api-baseline.sha256 index b54864ab9a3..258f9e38b31 100644 --- a/docs/.generated/plugin-sdk-api-baseline.sha256 +++ b/docs/.generated/plugin-sdk-api-baseline.sha256 @@ -1,2 +1,2 @@ -2375f50cfc8f29df7de7ec84d72ea654ef6eb203204f295fc287a11d63ed2cdb plugin-sdk-api-baseline.json -103ee7cf995fe6a0a3d109ad4bb785d342fa1eba83e3295dee3a5ee0bb315b4a plugin-sdk-api-baseline.jsonl +8592ce4554f44cd5325f44f86d37b3057548853cc9bc543b0611e5d9000f262e plugin-sdk-api-baseline.json +35393eceb6733d966430369c116511bb15a4b83eec93ebfd3b9a3e8f9ee29cec plugin-sdk-api-baseline.jsonl diff --git a/docs/plugins/sdk-overview.md b/docs/plugins/sdk-overview.md index 2af0276cc1f..0ccaff88751 100644 --- a/docs/plugins/sdk-overview.md +++ b/docs/plugins/sdk-overview.md @@ -133,6 +133,7 @@ explicitly promotes one as public. | `plugin-sdk/provider-model-shared` | `ProviderReplayFamily`, `buildProviderReplayFamilyHooks`, `normalizeModelCompat`, shared replay-policy builders, provider-endpoint helpers, and model-id normalization helpers such as `normalizeNativeXaiModelId` | | `plugin-sdk/provider-catalog-shared` | `findCatalogTemplate`, `buildSingleProviderApiKeyCatalog`, `supportsNativeStreamingUsageCompat`, `applyProviderNativeStreamingUsageCompat` | | `plugin-sdk/provider-http` | Generic provider HTTP/endpoint capability helpers | + | `plugin-sdk/provider-web-fetch-contract` | Narrow web-fetch config/selection contract helpers such as `enablePluginInConfig` and `WebFetchProviderPlugin` | | `plugin-sdk/provider-web-fetch` | Web-fetch provider registration/cache helpers | | `plugin-sdk/provider-web-search-contract` | Narrow web-search config/credential contract helpers such as `enablePluginInConfig`, `resolveProviderWebSearchPluginConfig`, and scoped credential setters/getters | | `plugin-sdk/provider-web-search` | Web-search provider registration/cache/runtime helpers | diff --git a/extensions/firecrawl/web-fetch-contract-api.ts b/extensions/firecrawl/web-fetch-contract-api.ts new file mode 100644 index 00000000000..b2de504a337 --- /dev/null +++ b/extensions/firecrawl/web-fetch-contract-api.ts @@ -0,0 +1,65 @@ +import { + enablePluginInConfig, + type WebFetchProviderPlugin, +} from "openclaw/plugin-sdk/provider-web-fetch-contract"; + +function ensureRecord(target: Record, key: string): Record { + const current = target[key]; + if (current && typeof current === "object" && !Array.isArray(current)) { + return current as Record; + } + const next: Record = {}; + target[key] = next; + return next; +} + +export function createFirecrawlWebFetchProvider(): WebFetchProviderPlugin { + return { + id: "firecrawl", + label: "Firecrawl", + hint: "Fetch pages with Firecrawl for JS-heavy or bot-protected sites.", + envVars: ["FIRECRAWL_API_KEY"], + placeholder: "fc-...", + signupUrl: "https://www.firecrawl.dev/", + docsUrl: "https://docs.firecrawl.dev", + autoDetectOrder: 50, + credentialPath: "plugins.entries.firecrawl.config.webFetch.apiKey", + inactiveSecretPaths: [ + "plugins.entries.firecrawl.config.webFetch.apiKey", + "tools.web.fetch.firecrawl.apiKey", + ], + getCredentialValue: (fetchConfig) => { + if (!fetchConfig || typeof fetchConfig !== "object") { + return undefined; + } + const legacy = fetchConfig.firecrawl; + if (!legacy || typeof legacy !== "object" || Array.isArray(legacy)) { + return undefined; + } + if ((legacy as { enabled?: boolean }).enabled === false) { + return undefined; + } + return (legacy as { apiKey?: unknown }).apiKey; + }, + setCredentialValue: (fetchConfigTarget, value) => { + const firecrawl = ensureRecord(fetchConfigTarget, "firecrawl"); + firecrawl.apiKey = value; + }, + getConfiguredCredentialValue: (config) => + ( + config?.plugins?.entries?.firecrawl?.config as + | { webFetch?: { apiKey?: unknown } } + | undefined + )?.webFetch?.apiKey, + setConfiguredCredentialValue: (configTarget, value) => { + const plugins = ensureRecord(configTarget as Record, "plugins"); + const entries = ensureRecord(plugins, "entries"); + const firecrawlEntry = ensureRecord(entries, "firecrawl"); + const pluginConfig = ensureRecord(firecrawlEntry, "config"); + const webFetch = ensureRecord(pluginConfig, "webFetch"); + webFetch.apiKey = value; + }, + applySelectionConfig: (config) => enablePluginInConfig(config, "firecrawl").config, + createTool: () => null, + }; +} diff --git a/package.json b/package.json index 7a1438b381b..81197a7212e 100644 --- a/package.json +++ b/package.json @@ -896,6 +896,10 @@ "types": "./dist/plugin-sdk/provider-usage.d.ts", "default": "./dist/plugin-sdk/provider-usage.js" }, + "./plugin-sdk/provider-web-fetch-contract": { + "types": "./dist/plugin-sdk/provider-web-fetch-contract.d.ts", + "default": "./dist/plugin-sdk/provider-web-fetch-contract.js" + }, "./plugin-sdk/provider-web-fetch": { "types": "./dist/plugin-sdk/provider-web-fetch.d.ts", "default": "./dist/plugin-sdk/provider-web-fetch.js" diff --git a/scripts/lib/plugin-sdk-entrypoints.json b/scripts/lib/plugin-sdk-entrypoints.json index c081f1c5f7f..22eb5e6fe47 100644 --- a/scripts/lib/plugin-sdk-entrypoints.json +++ b/scripts/lib/plugin-sdk-entrypoints.json @@ -213,6 +213,7 @@ "provider-stream", "provider-tools", "provider-usage", + "provider-web-fetch-contract", "provider-web-fetch", "provider-web-search-contract", "provider-web-search", diff --git a/src/plugin-sdk/provider-web-fetch-contract.ts b/src/plugin-sdk/provider-web-fetch-contract.ts new file mode 100644 index 00000000000..102aa5ee05b --- /dev/null +++ b/src/plugin-sdk/provider-web-fetch-contract.ts @@ -0,0 +1,6 @@ +// Narrow shared exports for web-fetch contract surfaces. + +import type { WebFetchProviderPlugin } from "../plugins/types.js"; + +export { enablePluginInConfig } from "../plugins/enable.js"; +export type { WebFetchProviderPlugin }; diff --git a/src/plugins/contracts/plugin-sdk-subpaths.test.ts b/src/plugins/contracts/plugin-sdk-subpaths.test.ts index 925a3f733e0..039bf385cce 100644 --- a/src/plugins/contracts/plugin-sdk-subpaths.test.ts +++ b/src/plugins/contracts/plugin-sdk-subpaths.test.ts @@ -351,6 +351,15 @@ describe("plugin-sdk subpath exports", () => { "resolveCitationRedirectUrl", ], }); + expectSourceContract("provider-web-fetch-contract", { + mentions: ["enablePluginInConfig", "WebFetchProviderPlugin"], + omits: [ + "withTrustedWebToolsEndpoint", + "readResponseText", + "resolveCacheTtlMs", + "wrapExternalContent", + ], + }); expectSourceMentions("compat", [ "createPluginRuntimeStore", "createScopedChannelConfigAdapter", diff --git a/src/plugins/web-provider-public-artifacts.test.ts b/src/plugins/web-provider-public-artifacts.test.ts index fbbd0433e34..26b959b4124 100644 --- a/src/plugins/web-provider-public-artifacts.test.ts +++ b/src/plugins/web-provider-public-artifacts.test.ts @@ -41,4 +41,14 @@ describe("web provider public artifacts", () => { }), ); }); + + it("prefers lightweight bundled web fetch contract artifacts", () => { + const provider = resolveBundledWebFetchProvidersFromPublicArtifacts({ + bundledAllowlistCompat: true, + onlyPluginIds: ["firecrawl"], + })?.[0]; + + expect(provider?.pluginId).toBe("firecrawl"); + expect(provider?.createTool({ config: {} as never })).toBeNull(); + }); }); diff --git a/src/plugins/web-provider-public-artifacts.ts b/src/plugins/web-provider-public-artifacts.ts index ffcd8e53c91..a8031bee5ad 100644 --- a/src/plugins/web-provider-public-artifacts.ts +++ b/src/plugins/web-provider-public-artifacts.ts @@ -18,7 +18,11 @@ const WEB_SEARCH_ARTIFACT_CANDIDATES = [ "web-search-provider.js", "web-search.js", ] as const; -const WEB_FETCH_ARTIFACT_CANDIDATES = ["web-fetch-provider.js", "web-fetch.js"] as const; +const WEB_FETCH_ARTIFACT_CANDIDATES = [ + "web-fetch-contract-api.js", + "web-fetch-provider.js", + "web-fetch.js", +] as const; type BundledWebProviderPublicArtifactParams = { config?: PluginLoadOptions["config"];