mirror of
https://github.com/openclaw/openclaw.git
synced 2026-04-07 23:31:07 +00:00
!feat(plugins): add web fetch provider boundary (#59465)
* feat(plugins): add web fetch provider boundary * feat(plugins): add web fetch provider modules * refactor(web-fetch): remove remaining core firecrawl fetch config * fix(web-fetch): address review follow-ups * fix(web-fetch): harden provider runtime boundaries * fix(web-fetch): restore firecrawl compare helper * fix(web-fetch): restore env-based provider autodetect * fix(web-fetch): tighten provider hardening * fix(web-fetch): restore fetch autodetect and compat args * chore(changelog): note firecrawl fetch config break
This commit is contained in:
@@ -30,6 +30,7 @@ export type BuildPluginApiParams = {
|
||||
| "registerSpeechProvider"
|
||||
| "registerMediaUnderstandingProvider"
|
||||
| "registerImageGenerationProvider"
|
||||
| "registerWebFetchProvider"
|
||||
| "registerWebSearchProvider"
|
||||
| "registerInteractiveHandler"
|
||||
| "onConversationBindingResolved"
|
||||
@@ -58,6 +59,7 @@ const noopRegisterMediaUnderstandingProvider: OpenClawPluginApi["registerMediaUn
|
||||
() => {};
|
||||
const noopRegisterImageGenerationProvider: OpenClawPluginApi["registerImageGenerationProvider"] =
|
||||
() => {};
|
||||
const noopRegisterWebFetchProvider: OpenClawPluginApi["registerWebFetchProvider"] = () => {};
|
||||
const noopRegisterWebSearchProvider: OpenClawPluginApi["registerWebSearchProvider"] = () => {};
|
||||
const noopRegisterInteractiveHandler: OpenClawPluginApi["registerInteractiveHandler"] = () => {};
|
||||
const noopOnConversationBindingResolved: OpenClawPluginApi["onConversationBindingResolved"] =
|
||||
@@ -99,6 +101,7 @@ export function buildPluginApi(params: BuildPluginApiParams): OpenClawPluginApi
|
||||
handlers.registerMediaUnderstandingProvider ?? noopRegisterMediaUnderstandingProvider,
|
||||
registerImageGenerationProvider:
|
||||
handlers.registerImageGenerationProvider ?? noopRegisterImageGenerationProvider,
|
||||
registerWebFetchProvider: handlers.registerWebFetchProvider ?? noopRegisterWebFetchProvider,
|
||||
registerWebSearchProvider: handlers.registerWebSearchProvider ?? noopRegisterWebSearchProvider,
|
||||
registerInteractiveHandler:
|
||||
handlers.registerInteractiveHandler ?? noopRegisterInteractiveHandler,
|
||||
|
||||
@@ -7,6 +7,7 @@ export type BundledPluginContractSnapshot = {
|
||||
speechProviderIds: string[];
|
||||
mediaUnderstandingProviderIds: string[];
|
||||
imageGenerationProviderIds: string[];
|
||||
webFetchProviderIds: string[];
|
||||
webSearchProviderIds: string[];
|
||||
toolNames: string[];
|
||||
};
|
||||
@@ -34,6 +35,7 @@ export const BUNDLED_PLUGIN_CONTRACT_SNAPSHOTS: readonly BundledPluginContractSn
|
||||
speechProviderIds: uniqueStrings(manifest.contracts?.speechProviders),
|
||||
mediaUnderstandingProviderIds: uniqueStrings(manifest.contracts?.mediaUnderstandingProviders),
|
||||
imageGenerationProviderIds: uniqueStrings(manifest.contracts?.imageGenerationProviders),
|
||||
webFetchProviderIds: uniqueStrings(manifest.contracts?.webFetchProviders),
|
||||
webSearchProviderIds: uniqueStrings(manifest.contracts?.webSearchProviders),
|
||||
toolNames: uniqueStrings(manifest.contracts?.tools),
|
||||
}))
|
||||
@@ -44,6 +46,7 @@ export const BUNDLED_PLUGIN_CONTRACT_SNAPSHOTS: readonly BundledPluginContractSn
|
||||
entry.speechProviderIds.length > 0 ||
|
||||
entry.mediaUnderstandingProviderIds.length > 0 ||
|
||||
entry.imageGenerationProviderIds.length > 0 ||
|
||||
entry.webFetchProviderIds.length > 0 ||
|
||||
entry.webSearchProviderIds.length > 0 ||
|
||||
entry.toolNames.length > 0,
|
||||
)
|
||||
@@ -69,6 +72,8 @@ export const BUNDLED_IMAGE_GENERATION_PLUGIN_IDS = collectPluginIds(
|
||||
(entry) => entry.imageGenerationProviderIds,
|
||||
);
|
||||
|
||||
export const BUNDLED_WEB_FETCH_PLUGIN_IDS = collectPluginIds((entry) => entry.webFetchProviderIds);
|
||||
|
||||
export const BUNDLED_RUNTIME_CONTRACT_PLUGIN_IDS = [
|
||||
...new Set(
|
||||
BUNDLED_PLUGIN_CONTRACT_SNAPSHOTS.filter(
|
||||
@@ -77,6 +82,7 @@ export const BUNDLED_RUNTIME_CONTRACT_PLUGIN_IDS = [
|
||||
entry.speechProviderIds.length > 0 ||
|
||||
entry.mediaUnderstandingProviderIds.length > 0 ||
|
||||
entry.imageGenerationProviderIds.length > 0 ||
|
||||
entry.webFetchProviderIds.length > 0 ||
|
||||
entry.webSearchProviderIds.length > 0,
|
||||
).map((entry) => entry.pluginId),
|
||||
),
|
||||
|
||||
@@ -124,6 +124,7 @@ function createCapabilityPluginRecord(params: {
|
||||
speechProviderIds: [],
|
||||
mediaUnderstandingProviderIds: [],
|
||||
imageGenerationProviderIds: [],
|
||||
webFetchProviderIds: [],
|
||||
webSearchProviderIds: [],
|
||||
gatewayMethods: [],
|
||||
cliCommands: [],
|
||||
@@ -277,6 +278,7 @@ export function loadBundledCapabilityRuntimeRegistry(params: {
|
||||
record.imageGenerationProviderIds.push(
|
||||
...captured.imageGenerationProviders.map((entry) => entry.id),
|
||||
);
|
||||
record.webFetchProviderIds.push(...captured.webFetchProviders.map((entry) => entry.id));
|
||||
record.webSearchProviderIds.push(...captured.webSearchProviders.map((entry) => entry.id));
|
||||
record.toolNames.push(...captured.tools.map((entry) => entry.name));
|
||||
|
||||
@@ -325,6 +327,15 @@ export function loadBundledCapabilityRuntimeRegistry(params: {
|
||||
rootDir: record.rootDir,
|
||||
})),
|
||||
);
|
||||
registry.webFetchProviders.push(
|
||||
...captured.webFetchProviders.map((provider) => ({
|
||||
pluginId: record.id,
|
||||
pluginName: record.name,
|
||||
provider,
|
||||
source: record.source,
|
||||
rootDir: record.rootDir,
|
||||
})),
|
||||
);
|
||||
registry.webSearchProviders.push(
|
||||
...captured.webSearchProviders.map((provider) => ({
|
||||
pluginId: record.id,
|
||||
|
||||
7
src/plugins/bundled-web-fetch-ids.ts
Normal file
7
src/plugins/bundled-web-fetch-ids.ts
Normal file
@@ -0,0 +1,7 @@
|
||||
import { BUNDLED_WEB_FETCH_PLUGIN_IDS as BUNDLED_WEB_FETCH_PLUGIN_IDS_FROM_METADATA } from "./bundled-capability-metadata.js";
|
||||
|
||||
export const BUNDLED_WEB_FETCH_PLUGIN_IDS = BUNDLED_WEB_FETCH_PLUGIN_IDS_FROM_METADATA;
|
||||
|
||||
export function listBundledWebFetchPluginIds(): string[] {
|
||||
return [...BUNDLED_WEB_FETCH_PLUGIN_IDS];
|
||||
}
|
||||
18
src/plugins/bundled-web-fetch-provider-ids.ts
Normal file
18
src/plugins/bundled-web-fetch-provider-ids.ts
Normal file
@@ -0,0 +1,18 @@
|
||||
import { BUNDLED_PLUGIN_CONTRACT_SNAPSHOTS } from "./bundled-capability-metadata.js";
|
||||
|
||||
const bundledWebFetchProviderPluginIds = Object.fromEntries(
|
||||
BUNDLED_PLUGIN_CONTRACT_SNAPSHOTS.flatMap((entry) =>
|
||||
entry.webFetchProviderIds.map((providerId) => [providerId, entry.pluginId] as const),
|
||||
).toSorted(([left], [right]) => left.localeCompare(right)),
|
||||
) as Readonly<Record<string, string>>;
|
||||
|
||||
export function resolveBundledWebFetchPluginId(providerId: string | undefined): string | undefined {
|
||||
if (!providerId) {
|
||||
return undefined;
|
||||
}
|
||||
const normalizedProviderId = providerId.trim().toLowerCase();
|
||||
if (!(normalizedProviderId in bundledWebFetchProviderPluginIds)) {
|
||||
return undefined;
|
||||
}
|
||||
return bundledWebFetchProviderPluginIds[normalizedProviderId];
|
||||
}
|
||||
49
src/plugins/bundled-web-fetch.ts
Normal file
49
src/plugins/bundled-web-fetch.ts
Normal file
@@ -0,0 +1,49 @@
|
||||
import { loadBundledCapabilityRuntimeRegistry } from "./bundled-capability-runtime.js";
|
||||
import { BUNDLED_WEB_FETCH_PLUGIN_IDS } from "./bundled-web-fetch-ids.js";
|
||||
import { resolveBundledWebFetchPluginId as resolveBundledWebFetchPluginIdFromMap } from "./bundled-web-fetch-provider-ids.js";
|
||||
import type { PluginLoadOptions } from "./loader.js";
|
||||
import { loadPluginManifestRegistry } from "./manifest-registry.js";
|
||||
import type { PluginWebFetchProviderEntry } from "./types.js";
|
||||
|
||||
type BundledWebFetchProviderEntry = PluginWebFetchProviderEntry & { pluginId: string };
|
||||
|
||||
let bundledWebFetchProvidersCache: BundledWebFetchProviderEntry[] | null = null;
|
||||
|
||||
function loadBundledWebFetchProviders(): BundledWebFetchProviderEntry[] {
|
||||
if (!bundledWebFetchProvidersCache) {
|
||||
bundledWebFetchProvidersCache = loadBundledCapabilityRuntimeRegistry({
|
||||
pluginIds: BUNDLED_WEB_FETCH_PLUGIN_IDS,
|
||||
pluginSdkResolution: "dist",
|
||||
}).webFetchProviders.map((entry) => ({
|
||||
pluginId: entry.pluginId,
|
||||
...entry.provider,
|
||||
}));
|
||||
}
|
||||
return bundledWebFetchProvidersCache;
|
||||
}
|
||||
|
||||
export function resolveBundledWebFetchPluginIds(params: {
|
||||
config?: PluginLoadOptions["config"];
|
||||
workspaceDir?: string;
|
||||
env?: PluginLoadOptions["env"];
|
||||
}): string[] {
|
||||
const bundledWebFetchPluginIdSet = new Set<string>(BUNDLED_WEB_FETCH_PLUGIN_IDS);
|
||||
return loadPluginManifestRegistry({
|
||||
config: params.config,
|
||||
workspaceDir: params.workspaceDir,
|
||||
env: params.env,
|
||||
})
|
||||
.plugins.filter(
|
||||
(plugin) => plugin.origin === "bundled" && bundledWebFetchPluginIdSet.has(plugin.id),
|
||||
)
|
||||
.map((plugin) => plugin.id)
|
||||
.toSorted((left, right) => left.localeCompare(right));
|
||||
}
|
||||
|
||||
export function listBundledWebFetchProviders(): PluginWebFetchProviderEntry[] {
|
||||
return loadBundledWebFetchProviders();
|
||||
}
|
||||
|
||||
export function resolveBundledWebFetchPluginId(providerId: string | undefined): string | undefined {
|
||||
return resolveBundledWebFetchPluginIdFromMap(providerId);
|
||||
}
|
||||
@@ -11,6 +11,7 @@ import type {
|
||||
OpenClawPluginCliRegistrar,
|
||||
ProviderPlugin,
|
||||
SpeechProviderPlugin,
|
||||
WebFetchProviderPlugin,
|
||||
WebSearchProviderPlugin,
|
||||
} from "./types.js";
|
||||
|
||||
@@ -28,6 +29,7 @@ export type CapturedPluginRegistration = {
|
||||
speechProviders: SpeechProviderPlugin[];
|
||||
mediaUnderstandingProviders: MediaUnderstandingProviderPlugin[];
|
||||
imageGenerationProviders: ImageGenerationProviderPlugin[];
|
||||
webFetchProviders: WebFetchProviderPlugin[];
|
||||
webSearchProviders: WebSearchProviderPlugin[];
|
||||
tools: AnyAgentTool[];
|
||||
};
|
||||
@@ -42,6 +44,7 @@ export function createCapturedPluginRegistration(params?: {
|
||||
const speechProviders: SpeechProviderPlugin[] = [];
|
||||
const mediaUnderstandingProviders: MediaUnderstandingProviderPlugin[] = [];
|
||||
const imageGenerationProviders: ImageGenerationProviderPlugin[] = [];
|
||||
const webFetchProviders: WebFetchProviderPlugin[] = [];
|
||||
const webSearchProviders: WebSearchProviderPlugin[] = [];
|
||||
const tools: AnyAgentTool[] = [];
|
||||
const noopLogger = {
|
||||
@@ -58,6 +61,7 @@ export function createCapturedPluginRegistration(params?: {
|
||||
speechProviders,
|
||||
mediaUnderstandingProviders,
|
||||
imageGenerationProviders,
|
||||
webFetchProviders,
|
||||
webSearchProviders,
|
||||
tools,
|
||||
api: buildPluginApi({
|
||||
@@ -108,6 +112,9 @@ export function createCapturedPluginRegistration(params?: {
|
||||
registerImageGenerationProvider(provider: ImageGenerationProviderPlugin) {
|
||||
imageGenerationProviders.push(provider);
|
||||
},
|
||||
registerWebFetchProvider(provider: WebFetchProviderPlugin) {
|
||||
webFetchProviders.push(provider);
|
||||
},
|
||||
registerWebSearchProvider(provider: WebSearchProviderPlugin) {
|
||||
webSearchProviders.push(provider);
|
||||
},
|
||||
|
||||
@@ -38,6 +38,7 @@ const pluginRegistrationContractTests: PluginRegistrationContractParams[] = [
|
||||
},
|
||||
{
|
||||
pluginId: "firecrawl",
|
||||
webFetchProviderIds: ["firecrawl"],
|
||||
webSearchProviderIds: ["firecrawl"],
|
||||
toolNames: ["firecrawl_search", "firecrawl_scrape"],
|
||||
},
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { resolveBundledWebFetchPluginIds } from "../bundled-web-fetch.js";
|
||||
import { resolveBundledWebSearchPluginIds } from "../bundled-web-search.js";
|
||||
import { loadPluginManifestRegistry } from "../manifest-registry.js";
|
||||
import {
|
||||
@@ -7,8 +8,10 @@ import {
|
||||
pluginRegistrationContractRegistry,
|
||||
providerContractLoadError,
|
||||
providerContractPluginIds,
|
||||
resolveWebFetchProviderContractEntriesForPluginId,
|
||||
resolveWebSearchProviderContractEntriesForPluginId,
|
||||
speechProviderContractRegistry,
|
||||
webFetchProviderContractRegistry,
|
||||
} from "./registry.js";
|
||||
import { uniqueSortedStrings } from "./testkit.js";
|
||||
|
||||
@@ -55,6 +58,10 @@ describe("plugin contract registry", () => {
|
||||
name: "does not duplicate bundled provider ids",
|
||||
ids: () => pluginRegistrationContractRegistry.flatMap((entry) => entry.providerIds),
|
||||
},
|
||||
{
|
||||
name: "does not duplicate bundled web fetch provider ids",
|
||||
ids: () => pluginRegistrationContractRegistry.flatMap((entry) => entry.webFetchProviderIds),
|
||||
},
|
||||
{
|
||||
name: "does not duplicate bundled web search provider ids",
|
||||
ids: () => pluginRegistrationContractRegistry.flatMap((entry) => entry.webSearchProviderIds),
|
||||
@@ -94,6 +101,31 @@ describe("plugin contract registry", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("covers every bundled web fetch plugin from the shared resolver", () => {
|
||||
const bundledWebFetchPluginIds = resolveBundledWebFetchPluginIds({});
|
||||
|
||||
expect(
|
||||
uniqueSortedStrings(
|
||||
pluginRegistrationContractRegistry
|
||||
.filter((entry) => entry.webFetchProviderIds.length > 0)
|
||||
.map((entry) => entry.pluginId),
|
||||
),
|
||||
).toEqual(bundledWebFetchPluginIds);
|
||||
});
|
||||
|
||||
it(
|
||||
"loads bundled web fetch providers for each shared-resolver plugin",
|
||||
{ timeout: REGISTRY_CONTRACT_TIMEOUT_MS },
|
||||
() => {
|
||||
for (const pluginId of resolveBundledWebFetchPluginIds({})) {
|
||||
expect(resolveWebFetchProviderContractEntriesForPluginId(pluginId).length).toBeGreaterThan(
|
||||
0,
|
||||
);
|
||||
}
|
||||
expect(webFetchProviderContractRegistry.length).toBeGreaterThan(0);
|
||||
},
|
||||
);
|
||||
|
||||
it("covers every bundled web search plugin from the shared resolver", () => {
|
||||
const bundledWebSearchPluginIds = resolveBundledWebSearchPluginIds({});
|
||||
|
||||
|
||||
@@ -1,11 +1,12 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import type { ProviderPlugin, WebSearchProviderPlugin } from "../types.js";
|
||||
import type { ProviderPlugin, WebFetchProviderPlugin, WebSearchProviderPlugin } from "../types.js";
|
||||
|
||||
type MockPluginRecord = {
|
||||
id: string;
|
||||
status: "loaded" | "error";
|
||||
error?: string;
|
||||
providerIds: string[];
|
||||
webFetchProviderIds: string[];
|
||||
webSearchProviderIds: string[];
|
||||
};
|
||||
|
||||
@@ -13,12 +14,14 @@ type MockRuntimeRegistry = {
|
||||
plugins: MockPluginRecord[];
|
||||
diagnostics: Array<{ pluginId?: string; message: string }>;
|
||||
providers: Array<{ pluginId: string; provider: ProviderPlugin }>;
|
||||
webFetchProviders: Array<{ pluginId: string; provider: WebFetchProviderPlugin }>;
|
||||
webSearchProviders: Array<{ pluginId: string; provider: WebSearchProviderPlugin }>;
|
||||
};
|
||||
|
||||
function createMockRuntimeRegistry(params: {
|
||||
plugin: MockPluginRecord;
|
||||
providers?: Array<{ pluginId: string; provider: ProviderPlugin }>;
|
||||
webFetchProviders?: Array<{ pluginId: string; provider: WebFetchProviderPlugin }>;
|
||||
webSearchProviders?: Array<{ pluginId: string; provider: WebSearchProviderPlugin }>;
|
||||
diagnostics?: Array<{ pluginId?: string; message: string }>;
|
||||
}): MockRuntimeRegistry {
|
||||
@@ -26,6 +29,7 @@ function createMockRuntimeRegistry(params: {
|
||||
plugins: [params.plugin],
|
||||
diagnostics: params.diagnostics ?? [],
|
||||
providers: params.providers ?? [],
|
||||
webFetchProviders: params.webFetchProviders ?? [],
|
||||
webSearchProviders: params.webSearchProviders ?? [],
|
||||
};
|
||||
}
|
||||
@@ -46,6 +50,7 @@ describe("plugin contract registry scoped retries", () => {
|
||||
status: "error",
|
||||
error: "transient xai load failure",
|
||||
providerIds: [],
|
||||
webFetchProviderIds: [],
|
||||
webSearchProviderIds: [],
|
||||
},
|
||||
diagnostics: [{ pluginId: "xai", message: "transient xai load failure" }],
|
||||
@@ -57,6 +62,7 @@ describe("plugin contract registry scoped retries", () => {
|
||||
id: "xai",
|
||||
status: "loaded",
|
||||
providerIds: ["xai"],
|
||||
webFetchProviderIds: [],
|
||||
webSearchProviderIds: ["grok"],
|
||||
},
|
||||
providers: [
|
||||
@@ -95,6 +101,7 @@ describe("plugin contract registry scoped retries", () => {
|
||||
status: "error",
|
||||
error: "transient grok load failure",
|
||||
providerIds: [],
|
||||
webFetchProviderIds: [],
|
||||
webSearchProviderIds: [],
|
||||
},
|
||||
diagnostics: [{ pluginId: "xai", message: "transient grok load failure" }],
|
||||
@@ -106,6 +113,7 @@ describe("plugin contract registry scoped retries", () => {
|
||||
id: "xai",
|
||||
status: "loaded",
|
||||
providerIds: ["xai"],
|
||||
webFetchProviderIds: [],
|
||||
webSearchProviderIds: ["grok"],
|
||||
},
|
||||
webSearchProviders: [
|
||||
@@ -152,6 +160,7 @@ describe("plugin contract registry scoped retries", () => {
|
||||
id: "byteplus",
|
||||
status: "loaded",
|
||||
providerIds: ["byteplus"],
|
||||
webFetchProviderIds: [],
|
||||
webSearchProviderIds: [],
|
||||
},
|
||||
providers: [
|
||||
@@ -177,4 +186,70 @@ describe("plugin contract registry scoped retries", () => {
|
||||
expect(requireProviderContractProvider("byteplus-plan").id).toBe("byteplus");
|
||||
expect(loadBundledCapabilityRuntimeRegistry).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("retries web fetch provider loads after a transient plugin-scoped runtime error", async () => {
|
||||
const loadBundledCapabilityRuntimeRegistry = vi
|
||||
.fn()
|
||||
.mockReturnValueOnce(
|
||||
createMockRuntimeRegistry({
|
||||
plugin: {
|
||||
id: "firecrawl",
|
||||
status: "error",
|
||||
error: "transient firecrawl fetch load failure",
|
||||
providerIds: [],
|
||||
webFetchProviderIds: [],
|
||||
webSearchProviderIds: [],
|
||||
},
|
||||
diagnostics: [
|
||||
{ pluginId: "firecrawl", message: "transient firecrawl fetch load failure" },
|
||||
],
|
||||
}),
|
||||
)
|
||||
.mockReturnValueOnce(
|
||||
createMockRuntimeRegistry({
|
||||
plugin: {
|
||||
id: "firecrawl",
|
||||
status: "loaded",
|
||||
providerIds: [],
|
||||
webFetchProviderIds: ["firecrawl"],
|
||||
webSearchProviderIds: ["firecrawl"],
|
||||
},
|
||||
webFetchProviders: [
|
||||
{
|
||||
pluginId: "firecrawl",
|
||||
provider: {
|
||||
id: "firecrawl",
|
||||
label: "Firecrawl",
|
||||
hint: "Fetch with Firecrawl",
|
||||
envVars: ["FIRECRAWL_API_KEY"],
|
||||
placeholder: "fc-...",
|
||||
signupUrl: "https://firecrawl.dev",
|
||||
credentialPath: "plugins.entries.firecrawl.config.webFetch.apiKey",
|
||||
requiresCredential: true,
|
||||
getCredentialValue: () => undefined,
|
||||
setCredentialValue() {},
|
||||
createTool: () => ({
|
||||
description: "fetch",
|
||||
parameters: {},
|
||||
execute: async () => ({}),
|
||||
}),
|
||||
} as WebFetchProviderPlugin,
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
|
||||
vi.doMock("../bundled-capability-runtime.js", () => ({
|
||||
loadBundledCapabilityRuntimeRegistry,
|
||||
}));
|
||||
|
||||
const { resolveWebFetchProviderContractEntriesForPluginId } = await import("./registry.js");
|
||||
|
||||
expect(
|
||||
resolveWebFetchProviderContractEntriesForPluginId("firecrawl").map(
|
||||
(entry) => entry.provider.id,
|
||||
),
|
||||
).toEqual(["firecrawl"]);
|
||||
expect(loadBundledCapabilityRuntimeRegistry).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -4,6 +4,7 @@ import {
|
||||
BUNDLED_PLUGIN_CONTRACT_SNAPSHOTS,
|
||||
BUNDLED_PROVIDER_PLUGIN_IDS,
|
||||
BUNDLED_SPEECH_PLUGIN_IDS,
|
||||
BUNDLED_WEB_FETCH_PLUGIN_IDS,
|
||||
BUNDLED_WEB_SEARCH_PLUGIN_IDS,
|
||||
} from "../bundled-capability-metadata.js";
|
||||
import { loadBundledCapabilityRuntimeRegistry } from "../bundled-capability-runtime.js";
|
||||
@@ -12,6 +13,7 @@ import type {
|
||||
MediaUnderstandingProviderPlugin,
|
||||
ProviderPlugin,
|
||||
SpeechProviderPlugin,
|
||||
WebFetchProviderPlugin,
|
||||
WebSearchProviderPlugin,
|
||||
} from "../types.js";
|
||||
import {
|
||||
@@ -31,6 +33,9 @@ type ProviderContractEntry = CapabilityContractEntry<ProviderPlugin>;
|
||||
type WebSearchProviderContractEntry = CapabilityContractEntry<WebSearchProviderPlugin> & {
|
||||
credentialValue: unknown;
|
||||
};
|
||||
type WebFetchProviderContractEntry = CapabilityContractEntry<WebFetchProviderPlugin> & {
|
||||
credentialValue: unknown;
|
||||
};
|
||||
|
||||
type SpeechProviderContractEntry = CapabilityContractEntry<SpeechProviderPlugin>;
|
||||
type MediaUnderstandingProviderContractEntry =
|
||||
@@ -44,6 +49,7 @@ type PluginRegistrationContractEntry = {
|
||||
speechProviderIds: string[];
|
||||
mediaUnderstandingProviderIds: string[];
|
||||
imageGenerationProviderIds: string[];
|
||||
webFetchProviderIds: string[];
|
||||
webSearchProviderIds: string[];
|
||||
toolNames: string[];
|
||||
};
|
||||
@@ -77,6 +83,11 @@ function uniqueStrings(values: readonly string[]): string[] {
|
||||
|
||||
let providerContractRegistryCache: ProviderContractEntry[] | null = null;
|
||||
let providerContractRegistryByPluginIdCache: Map<string, ProviderContractEntry[]> | null = null;
|
||||
let webFetchProviderContractRegistryCache: WebFetchProviderContractEntry[] | null = null;
|
||||
let webFetchProviderContractRegistryByPluginIdCache: Map<
|
||||
string,
|
||||
WebFetchProviderContractEntry[]
|
||||
> | null = null;
|
||||
let webSearchProviderContractRegistryCache: WebSearchProviderContractEntry[] | null = null;
|
||||
let webSearchProviderContractRegistryByPluginIdCache: Map<
|
||||
string,
|
||||
@@ -106,6 +117,7 @@ function formatBundledCapabilityPluginLoadError(params: {
|
||||
`status=${plugin.status}`,
|
||||
...(plugin.error ? [`error=${plugin.error}`] : []),
|
||||
`providerIds=[${plugin.providerIds.join(", ")}]`,
|
||||
`webFetchProviderIds=[${plugin.webFetchProviderIds.join(", ")}]`,
|
||||
`webSearchProviderIds=[${plugin.webSearchProviderIds.join(", ")}]`,
|
||||
]
|
||||
: ["plugin record missing"];
|
||||
@@ -253,6 +265,65 @@ function resolveWebSearchCredentialValue(provider: WebSearchProviderPlugin): unk
|
||||
return envVar.toLowerCase().includes("api_key") ? `${provider.id}-test` : "sk-test";
|
||||
}
|
||||
|
||||
function resolveWebFetchCredentialValue(provider: WebFetchProviderPlugin): unknown {
|
||||
if (provider.requiresCredential === false) {
|
||||
return `${provider.id}-no-key-needed`;
|
||||
}
|
||||
const envVar = provider.envVars.find((entry) => entry.trim().length > 0);
|
||||
if (!envVar) {
|
||||
return `${provider.id}-test`;
|
||||
}
|
||||
return envVar.toLowerCase().includes("api_key") ? `${provider.id}-test` : "sk-test";
|
||||
}
|
||||
|
||||
function loadWebFetchProviderContractRegistry(): WebFetchProviderContractEntry[] {
|
||||
if (!webFetchProviderContractRegistryCache) {
|
||||
const registry = loadBundledCapabilityRuntimeRegistry({
|
||||
pluginIds: BUNDLED_WEB_FETCH_PLUGIN_IDS,
|
||||
pluginSdkResolution: "dist",
|
||||
});
|
||||
webFetchProviderContractRegistryCache = registry.webFetchProviders.map((entry) => ({
|
||||
pluginId: entry.pluginId,
|
||||
provider: entry.provider,
|
||||
credentialValue: resolveWebFetchCredentialValue(entry.provider),
|
||||
}));
|
||||
}
|
||||
return webFetchProviderContractRegistryCache;
|
||||
}
|
||||
|
||||
export function resolveWebFetchProviderContractEntriesForPluginId(
|
||||
pluginId: string,
|
||||
): WebFetchProviderContractEntry[] {
|
||||
if (webFetchProviderContractRegistryCache) {
|
||||
return webFetchProviderContractRegistryCache.filter((entry) => entry.pluginId === pluginId);
|
||||
}
|
||||
|
||||
const cache =
|
||||
webFetchProviderContractRegistryByPluginIdCache ??
|
||||
new Map<string, WebFetchProviderContractEntry[]>();
|
||||
webFetchProviderContractRegistryByPluginIdCache = cache;
|
||||
const cached = cache.get(pluginId);
|
||||
if (cached) {
|
||||
return cached;
|
||||
}
|
||||
|
||||
const entries = loadScopedCapabilityRuntimeRegistryEntries({
|
||||
pluginId,
|
||||
capabilityLabel: "web fetch provider",
|
||||
loadEntries: (registry) =>
|
||||
registry.webFetchProviders
|
||||
.filter((entry) => entry.pluginId === pluginId)
|
||||
.map((entry) => ({
|
||||
pluginId: entry.pluginId,
|
||||
provider: entry.provider,
|
||||
credentialValue: resolveWebFetchCredentialValue(entry.provider),
|
||||
})),
|
||||
loadDeclaredIds: (plugin) => plugin.webFetchProviderIds,
|
||||
});
|
||||
cache.set(pluginId, entries);
|
||||
return entries;
|
||||
}
|
||||
|
||||
function loadWebSearchProviderContractRegistry(): WebSearchProviderContractEntry[] {
|
||||
if (!webSearchProviderContractRegistryCache) {
|
||||
const registry = loadBundledCapabilityRuntimeRegistry({
|
||||
@@ -441,6 +512,9 @@ export function resolveProviderContractProvidersForPluginIds(
|
||||
export const webSearchProviderContractRegistry: WebSearchProviderContractEntry[] =
|
||||
createLazyArrayView(loadWebSearchProviderContractRegistry);
|
||||
|
||||
export const webFetchProviderContractRegistry: WebFetchProviderContractEntry[] =
|
||||
createLazyArrayView(loadWebFetchProviderContractRegistry);
|
||||
|
||||
export const speechProviderContractRegistry: SpeechProviderContractEntry[] = createLazyArrayView(
|
||||
loadSpeechProviderContractRegistry,
|
||||
);
|
||||
@@ -459,6 +533,7 @@ function loadPluginRegistrationContractRegistry(): PluginRegistrationContractEnt
|
||||
speechProviderIds: uniqueStrings(entry.speechProviderIds),
|
||||
mediaUnderstandingProviderIds: uniqueStrings(entry.mediaUnderstandingProviderIds),
|
||||
imageGenerationProviderIds: uniqueStrings(entry.imageGenerationProviderIds),
|
||||
webFetchProviderIds: uniqueStrings(entry.webFetchProviderIds),
|
||||
webSearchProviderIds: uniqueStrings(entry.webSearchProviderIds),
|
||||
toolNames: uniqueStrings(entry.toolNames),
|
||||
}));
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { expect, it } from "vitest";
|
||||
import type { OpenClawConfig } from "../../config/config.js";
|
||||
import type { ProviderPlugin, WebSearchProviderPlugin } from "../types.js";
|
||||
import type { ProviderPlugin, WebFetchProviderPlugin, WebSearchProviderPlugin } from "../types.js";
|
||||
|
||||
type Lazy<T> = T | (() => T);
|
||||
|
||||
@@ -132,3 +132,46 @@ export function installWebSearchProviderContractSuite(params: {
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
export function installWebFetchProviderContractSuite(params: {
|
||||
provider: Lazy<WebFetchProviderPlugin>;
|
||||
credentialValue: Lazy<unknown>;
|
||||
}) {
|
||||
it("satisfies the base web fetch provider contract", () => {
|
||||
const provider = resolveLazy(params.provider);
|
||||
const credentialValue = resolveLazy(params.credentialValue);
|
||||
|
||||
expect(provider.id).toMatch(/^[a-z0-9][a-z0-9-]*$/);
|
||||
expect(provider.label.trim()).not.toBe("");
|
||||
expect(provider.hint.trim()).not.toBe("");
|
||||
expect(provider.placeholder.trim()).not.toBe("");
|
||||
expect(provider.signupUrl.startsWith("https://")).toBe(true);
|
||||
if (provider.docsUrl) {
|
||||
expect(provider.docsUrl.startsWith("http")).toBe(true);
|
||||
}
|
||||
|
||||
expect(provider.envVars).toEqual([...new Set(provider.envVars)]);
|
||||
expect(provider.envVars.every((entry) => entry.trim().length > 0)).toBe(true);
|
||||
|
||||
const fetchConfigTarget: Record<string, unknown> = {};
|
||||
provider.setCredentialValue(fetchConfigTarget, credentialValue);
|
||||
expect(provider.getCredentialValue(fetchConfigTarget)).toEqual(credentialValue);
|
||||
|
||||
const config = {
|
||||
tools: {
|
||||
web: {
|
||||
fetch: {
|
||||
provider: provider.id,
|
||||
...fetchConfigTarget,
|
||||
},
|
||||
},
|
||||
},
|
||||
} as OpenClawConfig;
|
||||
const tool = provider.createTool({ config, fetchConfig: fetchConfigTarget });
|
||||
|
||||
expect(tool).not.toBeNull();
|
||||
expect(tool?.description.trim()).not.toBe("");
|
||||
expect(tool?.parameters).toEqual(expect.any(Object));
|
||||
expect(typeof tool?.execute).toBe("function");
|
||||
});
|
||||
}
|
||||
|
||||
10
src/plugins/contracts/web-fetch-provider.contract.test.ts
Normal file
10
src/plugins/contracts/web-fetch-provider.contract.test.ts
Normal file
@@ -0,0 +1,10 @@
|
||||
import { describeWebFetchProviderContracts } from "../../../test/helpers/plugins/web-fetch-provider-contract.js";
|
||||
import { pluginRegistrationContractRegistry } from "./registry.js";
|
||||
|
||||
const webFetchProviderContractTests = pluginRegistrationContractRegistry.filter(
|
||||
(entry) => entry.webFetchProviderIds.length > 0,
|
||||
);
|
||||
|
||||
for (const entry of webFetchProviderContractTests) {
|
||||
describeWebFetchProviderContracts(entry.pluginId);
|
||||
}
|
||||
@@ -505,6 +505,7 @@ function createPluginRecord(params: {
|
||||
speechProviderIds: [],
|
||||
mediaUnderstandingProviderIds: [],
|
||||
imageGenerationProviderIds: [],
|
||||
webFetchProviderIds: [],
|
||||
webSearchProviderIds: [],
|
||||
gatewayMethods: [],
|
||||
cliCommands: [],
|
||||
|
||||
@@ -53,6 +53,7 @@ export type PluginManifestContracts = {
|
||||
speechProviders?: string[];
|
||||
mediaUnderstandingProviders?: string[];
|
||||
imageGenerationProviders?: string[];
|
||||
webFetchProviders?: string[];
|
||||
webSearchProviders?: string[];
|
||||
tools?: string[];
|
||||
};
|
||||
@@ -125,12 +126,14 @@ function normalizeManifestContracts(value: unknown): PluginManifestContracts | u
|
||||
const speechProviders = normalizeStringList(value.speechProviders);
|
||||
const mediaUnderstandingProviders = normalizeStringList(value.mediaUnderstandingProviders);
|
||||
const imageGenerationProviders = normalizeStringList(value.imageGenerationProviders);
|
||||
const webFetchProviders = normalizeStringList(value.webFetchProviders);
|
||||
const webSearchProviders = normalizeStringList(value.webSearchProviders);
|
||||
const tools = normalizeStringList(value.tools);
|
||||
const contracts = {
|
||||
...(speechProviders.length > 0 ? { speechProviders } : {}),
|
||||
...(mediaUnderstandingProviders.length > 0 ? { mediaUnderstandingProviders } : {}),
|
||||
...(imageGenerationProviders.length > 0 ? { imageGenerationProviders } : {}),
|
||||
...(webFetchProviders.length > 0 ? { webFetchProviders } : {}),
|
||||
...(webSearchProviders.length > 0 ? { webSearchProviders } : {}),
|
||||
...(tools.length > 0 ? { tools } : {}),
|
||||
} satisfies PluginManifestContracts;
|
||||
|
||||
@@ -13,6 +13,7 @@ export function createEmptyPluginRegistry(): PluginRegistry {
|
||||
speechProviders: [],
|
||||
mediaUnderstandingProviders: [],
|
||||
imageGenerationProviders: [],
|
||||
webFetchProviders: [],
|
||||
webSearchProviders: [],
|
||||
gatewayHandlers: {},
|
||||
gatewayMethodScopes: {},
|
||||
|
||||
@@ -37,6 +37,7 @@ import {
|
||||
import type {
|
||||
CliBackendPlugin,
|
||||
ImageGenerationProviderPlugin,
|
||||
WebFetchProviderPlugin,
|
||||
OpenClawPluginApi,
|
||||
OpenClawPluginChannelRegistration,
|
||||
OpenClawPluginCliCommandDescriptor,
|
||||
@@ -144,6 +145,8 @@ export type PluginMediaUnderstandingProviderRegistration =
|
||||
PluginOwnedProviderRegistration<MediaUnderstandingProviderPlugin>;
|
||||
export type PluginImageGenerationProviderRegistration =
|
||||
PluginOwnedProviderRegistration<ImageGenerationProviderPlugin>;
|
||||
export type PluginWebFetchProviderRegistration =
|
||||
PluginOwnedProviderRegistration<WebFetchProviderPlugin>;
|
||||
export type PluginWebSearchProviderRegistration =
|
||||
PluginOwnedProviderRegistration<WebSearchProviderPlugin>;
|
||||
|
||||
@@ -204,6 +207,7 @@ export type PluginRecord = {
|
||||
speechProviderIds: string[];
|
||||
mediaUnderstandingProviderIds: string[];
|
||||
imageGenerationProviderIds: string[];
|
||||
webFetchProviderIds: string[];
|
||||
webSearchProviderIds: string[];
|
||||
gatewayMethods: string[];
|
||||
cliCommands: string[];
|
||||
@@ -229,6 +233,7 @@ export type PluginRegistry = {
|
||||
speechProviders: PluginSpeechProviderRegistration[];
|
||||
mediaUnderstandingProviders: PluginMediaUnderstandingProviderRegistration[];
|
||||
imageGenerationProviders: PluginImageGenerationProviderRegistration[];
|
||||
webFetchProviders: PluginWebFetchProviderRegistration[];
|
||||
webSearchProviders: PluginWebSearchProviderRegistration[];
|
||||
gatewayHandlers: GatewayRequestHandlers;
|
||||
gatewayMethodScopes?: Partial<Record<string, OperatorScope>>;
|
||||
@@ -712,6 +717,16 @@ export function createPluginRegistry(registryParams: PluginRegistryParams) {
|
||||
});
|
||||
};
|
||||
|
||||
const registerWebFetchProvider = (record: PluginRecord, provider: WebFetchProviderPlugin) => {
|
||||
registerUniqueProviderLike({
|
||||
record,
|
||||
provider,
|
||||
kindLabel: "web fetch provider",
|
||||
registrations: registry.webFetchProviders,
|
||||
ownedIds: record.webFetchProviderIds,
|
||||
});
|
||||
};
|
||||
|
||||
const registerWebSearchProvider = (record: PluginRecord, provider: WebSearchProviderPlugin) => {
|
||||
registerUniqueProviderLike({
|
||||
record,
|
||||
@@ -990,6 +1005,7 @@ export function createPluginRegistry(registryParams: PluginRegistryParams) {
|
||||
registerMediaUnderstandingProvider(record, provider),
|
||||
registerImageGenerationProvider: (provider) =>
|
||||
registerImageGenerationProvider(record, provider),
|
||||
registerWebFetchProvider: (provider) => registerWebFetchProvider(record, provider),
|
||||
registerWebSearchProvider: (provider) => registerWebSearchProvider(record, provider),
|
||||
registerGatewayMethod: (method, handler, opts) =>
|
||||
registerGatewayMethod(record, method, handler, opts),
|
||||
|
||||
@@ -48,6 +48,7 @@ export function createPluginRecord(
|
||||
speechProviderIds: [],
|
||||
mediaUnderstandingProviderIds: [],
|
||||
imageGenerationProviderIds: [],
|
||||
webFetchProviderIds: [],
|
||||
webSearchProviderIds: [],
|
||||
gatewayMethods: [],
|
||||
cliCommands: [],
|
||||
@@ -111,6 +112,7 @@ export function createPluginLoadResult(
|
||||
speechProviders: [],
|
||||
mediaUnderstandingProviders: [],
|
||||
imageGenerationProviders: [],
|
||||
webFetchProviders: [],
|
||||
webSearchProviders: [],
|
||||
tools: [],
|
||||
hooks: [],
|
||||
|
||||
@@ -35,7 +35,10 @@ import type { ImageGenerationProvider } from "../image-generation/types.js";
|
||||
import type { ProviderUsageSnapshot } from "../infra/provider-usage.types.js";
|
||||
import type { MediaUnderstandingProvider } from "../media-understanding/types.js";
|
||||
import type { RuntimeEnv } from "../runtime.js";
|
||||
import type { RuntimeWebSearchMetadata } from "../secrets/runtime-web-tools.types.js";
|
||||
import type {
|
||||
RuntimeWebFetchMetadata,
|
||||
RuntimeWebSearchMetadata,
|
||||
} from "../secrets/runtime-web-tools.types.js";
|
||||
import type {
|
||||
SpeechDirectiveTokenParseContext,
|
||||
SpeechDirectiveTokenParseResult,
|
||||
@@ -1309,6 +1312,7 @@ export type ProviderPlugin = {
|
||||
};
|
||||
|
||||
export type WebSearchProviderId = string;
|
||||
export type WebFetchProviderId = string;
|
||||
|
||||
export type WebSearchProviderToolDefinition = {
|
||||
description: string;
|
||||
@@ -1316,12 +1320,24 @@ export type WebSearchProviderToolDefinition = {
|
||||
execute: (args: Record<string, unknown>) => Promise<Record<string, unknown>>;
|
||||
};
|
||||
|
||||
export type WebFetchProviderToolDefinition = {
|
||||
description: string;
|
||||
parameters: Record<string, unknown>;
|
||||
execute: (args: Record<string, unknown>) => Promise<Record<string, unknown>>;
|
||||
};
|
||||
|
||||
export type WebSearchProviderContext = {
|
||||
config?: OpenClawConfig;
|
||||
searchConfig?: Record<string, unknown>;
|
||||
runtimeMetadata?: RuntimeWebSearchMetadata;
|
||||
};
|
||||
|
||||
export type WebFetchProviderContext = {
|
||||
config?: OpenClawConfig;
|
||||
fetchConfig?: Record<string, unknown>;
|
||||
runtimeMetadata?: RuntimeWebFetchMetadata;
|
||||
};
|
||||
|
||||
export type WebSearchCredentialResolutionSource = "config" | "secretRef" | "env" | "missing";
|
||||
|
||||
export type WebSearchRuntimeMetadataContext = {
|
||||
@@ -1343,6 +1359,19 @@ export type WebSearchProviderSetupContext = {
|
||||
secretInputMode?: SecretInputMode;
|
||||
};
|
||||
|
||||
export type WebFetchCredentialResolutionSource = "config" | "secretRef" | "env" | "missing";
|
||||
|
||||
export type WebFetchRuntimeMetadataContext = {
|
||||
config?: OpenClawConfig;
|
||||
fetchConfig?: Record<string, unknown>;
|
||||
runtimeMetadata?: RuntimeWebFetchMetadata;
|
||||
resolvedCredential?: {
|
||||
value?: string;
|
||||
source: WebFetchCredentialResolutionSource;
|
||||
fallbackEnvVar?: string;
|
||||
};
|
||||
};
|
||||
|
||||
export type WebSearchProviderPlugin = {
|
||||
id: WebSearchProviderId;
|
||||
label: string;
|
||||
@@ -1381,6 +1410,34 @@ export type PluginWebSearchProviderEntry = WebSearchProviderPlugin & {
|
||||
pluginId: string;
|
||||
};
|
||||
|
||||
export type WebFetchProviderPlugin = {
|
||||
id: WebFetchProviderId;
|
||||
label: string;
|
||||
hint: string;
|
||||
requiresCredential?: boolean;
|
||||
credentialLabel?: string;
|
||||
envVars: string[];
|
||||
placeholder: string;
|
||||
signupUrl: string;
|
||||
docsUrl?: string;
|
||||
autoDetectOrder?: number;
|
||||
credentialPath: string;
|
||||
inactiveSecretPaths?: string[];
|
||||
getCredentialValue: (fetchConfig?: Record<string, unknown>) => unknown;
|
||||
setCredentialValue: (fetchConfigTarget: Record<string, unknown>, value: unknown) => void;
|
||||
getConfiguredCredentialValue?: (config?: OpenClawConfig) => unknown;
|
||||
setConfiguredCredentialValue?: (configTarget: OpenClawConfig, value: unknown) => void;
|
||||
applySelectionConfig?: (config: OpenClawConfig) => OpenClawConfig;
|
||||
resolveRuntimeMetadata?: (
|
||||
ctx: WebFetchRuntimeMetadataContext,
|
||||
) => Partial<RuntimeWebFetchMetadata> | Promise<Partial<RuntimeWebFetchMetadata>>;
|
||||
createTool: (ctx: WebFetchProviderContext) => WebFetchProviderToolDefinition | null;
|
||||
};
|
||||
|
||||
export type PluginWebFetchProviderEntry = WebFetchProviderPlugin & {
|
||||
pluginId: string;
|
||||
};
|
||||
|
||||
/** Speech capability registered by a plugin. */
|
||||
export type SpeechProviderPlugin = {
|
||||
id: SpeechProviderId;
|
||||
@@ -1873,6 +1930,8 @@ export type OpenClawPluginApi = {
|
||||
registerMediaUnderstandingProvider: (provider: MediaUnderstandingProviderPlugin) => void;
|
||||
/** Register an image generation provider (image generation capability). */
|
||||
registerImageGenerationProvider: (provider: ImageGenerationProviderPlugin) => void;
|
||||
/** Register a web fetch provider (web fetch capability). */
|
||||
registerWebFetchProvider: (provider: WebFetchProviderPlugin) => void;
|
||||
/** Register a web search provider (web search capability). */
|
||||
registerWebSearchProvider: (provider: WebSearchProviderPlugin) => void;
|
||||
registerInteractiveHandler: (registration: PluginInteractiveHandlerRegistration) => void;
|
||||
|
||||
216
src/plugins/web-fetch-providers.runtime.ts
Normal file
216
src/plugins/web-fetch-providers.runtime.ts
Normal file
@@ -0,0 +1,216 @@
|
||||
import type { OpenClawConfig } from "../config/config.js";
|
||||
import { createSubsystemLogger } from "../logging/subsystem.js";
|
||||
import { isRecord } from "../utils.js";
|
||||
import {
|
||||
buildPluginSnapshotCacheEnvKey,
|
||||
resolvePluginSnapshotCacheTtlMs,
|
||||
shouldUsePluginSnapshotCache,
|
||||
} from "./cache-controls.js";
|
||||
import { loadOpenClawPlugins, resolveRuntimePluginRegistry } from "./loader.js";
|
||||
import type { PluginLoadOptions } from "./loader.js";
|
||||
import { createPluginLoaderLogger } from "./logger.js";
|
||||
import { loadPluginManifestRegistry, type PluginManifestRecord } from "./manifest-registry.js";
|
||||
import type { PluginWebFetchProviderEntry } from "./types.js";
|
||||
import {
|
||||
resolveBundledWebFetchResolutionConfig,
|
||||
sortWebFetchProviders,
|
||||
} from "./web-fetch-providers.shared.js";
|
||||
|
||||
const log = createSubsystemLogger("plugins");
|
||||
type WebFetchProviderSnapshotCacheEntry = {
|
||||
expiresAt: number;
|
||||
providers: PluginWebFetchProviderEntry[];
|
||||
};
|
||||
let webFetchProviderSnapshotCache = new WeakMap<
|
||||
OpenClawConfig,
|
||||
WeakMap<NodeJS.ProcessEnv, Map<string, WebFetchProviderSnapshotCacheEntry>>
|
||||
>();
|
||||
|
||||
function resetWebFetchProviderSnapshotCacheForTests() {
|
||||
webFetchProviderSnapshotCache = new WeakMap<
|
||||
OpenClawConfig,
|
||||
WeakMap<NodeJS.ProcessEnv, Map<string, WebFetchProviderSnapshotCacheEntry>>
|
||||
>();
|
||||
}
|
||||
|
||||
export const __testing = {
|
||||
resetWebFetchProviderSnapshotCacheForTests,
|
||||
} as const;
|
||||
|
||||
function buildWebFetchSnapshotCacheKey(params: {
|
||||
config?: OpenClawConfig;
|
||||
workspaceDir?: string;
|
||||
bundledAllowlistCompat?: boolean;
|
||||
onlyPluginIds?: readonly string[];
|
||||
env: NodeJS.ProcessEnv;
|
||||
}): string {
|
||||
return JSON.stringify({
|
||||
workspaceDir: params.workspaceDir ?? "",
|
||||
bundledAllowlistCompat: params.bundledAllowlistCompat === true,
|
||||
onlyPluginIds: [...new Set(params.onlyPluginIds ?? [])].toSorted((left, right) =>
|
||||
left.localeCompare(right),
|
||||
),
|
||||
config: params.config ?? null,
|
||||
env: buildPluginSnapshotCacheEnvKey(params.env),
|
||||
});
|
||||
}
|
||||
|
||||
function pluginManifestDeclaresWebFetch(record: PluginManifestRecord): boolean {
|
||||
if ((record.contracts?.webFetchProviders?.length ?? 0) > 0) {
|
||||
return true;
|
||||
}
|
||||
const configUiHintKeys = Object.keys(record.configUiHints ?? {});
|
||||
if (configUiHintKeys.some((key) => key === "webFetch" || key.startsWith("webFetch."))) {
|
||||
return true;
|
||||
}
|
||||
if (!isRecord(record.configSchema)) {
|
||||
return false;
|
||||
}
|
||||
const properties = record.configSchema.properties;
|
||||
return isRecord(properties) && "webFetch" in properties;
|
||||
}
|
||||
|
||||
function resolveWebFetchCandidatePluginIds(params: {
|
||||
config?: PluginLoadOptions["config"];
|
||||
workspaceDir?: string;
|
||||
env?: PluginLoadOptions["env"];
|
||||
onlyPluginIds?: readonly string[];
|
||||
}): string[] | undefined {
|
||||
const registry = loadPluginManifestRegistry({
|
||||
config: params.config,
|
||||
workspaceDir: params.workspaceDir,
|
||||
env: params.env,
|
||||
});
|
||||
const onlyPluginIdSet =
|
||||
params.onlyPluginIds && params.onlyPluginIds.length > 0 ? new Set(params.onlyPluginIds) : null;
|
||||
const ids = registry.plugins
|
||||
.filter(
|
||||
(plugin) =>
|
||||
pluginManifestDeclaresWebFetch(plugin) &&
|
||||
(!onlyPluginIdSet || onlyPluginIdSet.has(plugin.id)),
|
||||
)
|
||||
.map((plugin) => plugin.id)
|
||||
.toSorted((left, right) => left.localeCompare(right));
|
||||
return ids.length > 0 ? ids : undefined;
|
||||
}
|
||||
|
||||
function resolveWebFetchLoadOptions(params: {
|
||||
config?: PluginLoadOptions["config"];
|
||||
workspaceDir?: string;
|
||||
env?: PluginLoadOptions["env"];
|
||||
bundledAllowlistCompat?: boolean;
|
||||
onlyPluginIds?: readonly string[];
|
||||
activate?: boolean;
|
||||
cache?: boolean;
|
||||
}) {
|
||||
const env = params.env ?? process.env;
|
||||
const { config } = resolveBundledWebFetchResolutionConfig({
|
||||
...params,
|
||||
env,
|
||||
});
|
||||
const onlyPluginIds = resolveWebFetchCandidatePluginIds({
|
||||
config,
|
||||
workspaceDir: params.workspaceDir,
|
||||
env,
|
||||
onlyPluginIds: params.onlyPluginIds,
|
||||
});
|
||||
return {
|
||||
env,
|
||||
config,
|
||||
workspaceDir: params.workspaceDir,
|
||||
cache: params.cache ?? false,
|
||||
activate: params.activate ?? false,
|
||||
...(onlyPluginIds ? { onlyPluginIds } : {}),
|
||||
logger: createPluginLoaderLogger(log),
|
||||
} satisfies PluginLoadOptions;
|
||||
}
|
||||
|
||||
function mapRegistryWebFetchProviders(params: {
|
||||
registry: ReturnType<typeof loadOpenClawPlugins>;
|
||||
onlyPluginIds?: readonly string[];
|
||||
}): PluginWebFetchProviderEntry[] {
|
||||
const onlyPluginIdSet =
|
||||
params.onlyPluginIds && params.onlyPluginIds.length > 0 ? new Set(params.onlyPluginIds) : null;
|
||||
return sortWebFetchProviders(
|
||||
params.registry.webFetchProviders
|
||||
.filter((entry) => !onlyPluginIdSet || onlyPluginIdSet.has(entry.pluginId))
|
||||
.map((entry) => ({
|
||||
...entry.provider,
|
||||
pluginId: entry.pluginId,
|
||||
})),
|
||||
);
|
||||
}
|
||||
|
||||
export function resolvePluginWebFetchProviders(params: {
|
||||
config?: PluginLoadOptions["config"];
|
||||
workspaceDir?: string;
|
||||
env?: PluginLoadOptions["env"];
|
||||
bundledAllowlistCompat?: boolean;
|
||||
onlyPluginIds?: readonly string[];
|
||||
activate?: boolean;
|
||||
cache?: boolean;
|
||||
}): PluginWebFetchProviderEntry[] {
|
||||
const env = params.env ?? process.env;
|
||||
const cacheOwnerConfig = params.config;
|
||||
const shouldMemoizeSnapshot =
|
||||
params.activate !== true && params.cache !== true && shouldUsePluginSnapshotCache(env);
|
||||
const cacheKey = buildWebFetchSnapshotCacheKey({
|
||||
config: cacheOwnerConfig,
|
||||
workspaceDir: params.workspaceDir,
|
||||
bundledAllowlistCompat: params.bundledAllowlistCompat,
|
||||
onlyPluginIds: params.onlyPluginIds,
|
||||
env,
|
||||
});
|
||||
if (cacheOwnerConfig && shouldMemoizeSnapshot) {
|
||||
const configCache = webFetchProviderSnapshotCache.get(cacheOwnerConfig);
|
||||
const envCache = configCache?.get(env);
|
||||
const cached = envCache?.get(cacheKey);
|
||||
if (cached && cached.expiresAt > Date.now()) {
|
||||
return cached.providers;
|
||||
}
|
||||
}
|
||||
const loadOptions = resolveWebFetchLoadOptions(params);
|
||||
const resolved = mapRegistryWebFetchProviders({
|
||||
registry: loadOpenClawPlugins(loadOptions),
|
||||
});
|
||||
if (cacheOwnerConfig && shouldMemoizeSnapshot) {
|
||||
const ttlMs = resolvePluginSnapshotCacheTtlMs(env);
|
||||
let configCache = webFetchProviderSnapshotCache.get(cacheOwnerConfig);
|
||||
if (!configCache) {
|
||||
configCache = new WeakMap<
|
||||
NodeJS.ProcessEnv,
|
||||
Map<string, WebFetchProviderSnapshotCacheEntry>
|
||||
>();
|
||||
webFetchProviderSnapshotCache.set(cacheOwnerConfig, configCache);
|
||||
}
|
||||
let envCache = configCache.get(env);
|
||||
if (!envCache) {
|
||||
envCache = new Map<string, WebFetchProviderSnapshotCacheEntry>();
|
||||
configCache.set(env, envCache);
|
||||
}
|
||||
envCache.set(cacheKey, {
|
||||
expiresAt: Date.now() + ttlMs,
|
||||
providers: resolved,
|
||||
});
|
||||
}
|
||||
return resolved;
|
||||
}
|
||||
|
||||
export function resolveRuntimeWebFetchProviders(params: {
|
||||
config?: PluginLoadOptions["config"];
|
||||
workspaceDir?: string;
|
||||
env?: PluginLoadOptions["env"];
|
||||
bundledAllowlistCompat?: boolean;
|
||||
onlyPluginIds?: readonly string[];
|
||||
}): PluginWebFetchProviderEntry[] {
|
||||
const runtimeRegistry = resolveRuntimePluginRegistry(
|
||||
params.config === undefined ? undefined : resolveWebFetchLoadOptions(params),
|
||||
);
|
||||
if (runtimeRegistry) {
|
||||
return mapRegistryWebFetchProviders({
|
||||
registry: runtimeRegistry,
|
||||
onlyPluginIds: params.onlyPluginIds,
|
||||
});
|
||||
}
|
||||
return resolvePluginWebFetchProviders(params);
|
||||
}
|
||||
91
src/plugins/web-fetch-providers.shared.ts
Normal file
91
src/plugins/web-fetch-providers.shared.ts
Normal file
@@ -0,0 +1,91 @@
|
||||
import { applyPluginAutoEnable } from "../config/plugin-auto-enable.js";
|
||||
import {
|
||||
withBundledPluginAllowlistCompat,
|
||||
withBundledPluginEnablementCompat,
|
||||
withBundledPluginVitestCompat,
|
||||
} from "./bundled-compat.js";
|
||||
import { resolveBundledWebFetchPluginIds } from "./bundled-web-fetch.js";
|
||||
import { normalizePluginsConfig, type NormalizedPluginsConfig } from "./config-state.js";
|
||||
import type { PluginLoadOptions } from "./loader.js";
|
||||
import type { PluginWebFetchProviderEntry } from "./types.js";
|
||||
|
||||
function resolveBundledWebFetchCompatPluginIds(params: {
|
||||
config?: PluginLoadOptions["config"];
|
||||
workspaceDir?: string;
|
||||
env?: PluginLoadOptions["env"];
|
||||
}): string[] {
|
||||
return resolveBundledWebFetchPluginIds({
|
||||
config: params.config,
|
||||
workspaceDir: params.workspaceDir,
|
||||
env: params.env,
|
||||
});
|
||||
}
|
||||
|
||||
function compareWebFetchProvidersAlphabetically(
|
||||
left: Pick<PluginWebFetchProviderEntry, "id" | "pluginId">,
|
||||
right: Pick<PluginWebFetchProviderEntry, "id" | "pluginId">,
|
||||
): number {
|
||||
return left.id.localeCompare(right.id) || left.pluginId.localeCompare(right.pluginId);
|
||||
}
|
||||
|
||||
export function sortWebFetchProviders(
|
||||
providers: PluginWebFetchProviderEntry[],
|
||||
): PluginWebFetchProviderEntry[] {
|
||||
return providers.toSorted(compareWebFetchProvidersAlphabetically);
|
||||
}
|
||||
|
||||
export function sortWebFetchProvidersForAutoDetect(
|
||||
providers: PluginWebFetchProviderEntry[],
|
||||
): PluginWebFetchProviderEntry[] {
|
||||
return providers.toSorted((left, right) => {
|
||||
const leftOrder = left.autoDetectOrder ?? Number.MAX_SAFE_INTEGER;
|
||||
const rightOrder = right.autoDetectOrder ?? Number.MAX_SAFE_INTEGER;
|
||||
if (leftOrder !== rightOrder) {
|
||||
return leftOrder - rightOrder;
|
||||
}
|
||||
return compareWebFetchProvidersAlphabetically(left, right);
|
||||
});
|
||||
}
|
||||
|
||||
export function resolveBundledWebFetchResolutionConfig(params: {
|
||||
config?: PluginLoadOptions["config"];
|
||||
workspaceDir?: string;
|
||||
env?: PluginLoadOptions["env"];
|
||||
bundledAllowlistCompat?: boolean;
|
||||
}): {
|
||||
config: PluginLoadOptions["config"];
|
||||
normalized: NormalizedPluginsConfig;
|
||||
} {
|
||||
const autoEnabledConfig =
|
||||
params.config !== undefined
|
||||
? applyPluginAutoEnable({
|
||||
config: params.config,
|
||||
env: params.env ?? process.env,
|
||||
}).config
|
||||
: undefined;
|
||||
const bundledCompatPluginIds = resolveBundledWebFetchCompatPluginIds({
|
||||
config: autoEnabledConfig,
|
||||
workspaceDir: params.workspaceDir,
|
||||
env: params.env,
|
||||
});
|
||||
const allowlistCompat = params.bundledAllowlistCompat
|
||||
? withBundledPluginAllowlistCompat({
|
||||
config: autoEnabledConfig,
|
||||
pluginIds: bundledCompatPluginIds,
|
||||
})
|
||||
: autoEnabledConfig;
|
||||
const enablementCompat = withBundledPluginEnablementCompat({
|
||||
config: allowlistCompat,
|
||||
pluginIds: bundledCompatPluginIds,
|
||||
});
|
||||
const config = withBundledPluginVitestCompat({
|
||||
config: enablementCompat,
|
||||
pluginIds: bundledCompatPluginIds,
|
||||
env: params.env,
|
||||
});
|
||||
|
||||
return {
|
||||
config,
|
||||
normalized: normalizePluginsConfig(config?.plugins),
|
||||
};
|
||||
}
|
||||
36
src/plugins/web-fetch-providers.ts
Normal file
36
src/plugins/web-fetch-providers.ts
Normal file
@@ -0,0 +1,36 @@
|
||||
import { listBundledWebFetchProviders as listBundledWebFetchProviderEntries } from "./bundled-web-fetch.js";
|
||||
import { resolveEffectiveEnableState } from "./config-state.js";
|
||||
import type { PluginLoadOptions } from "./loader.js";
|
||||
import type { PluginWebFetchProviderEntry } from "./types.js";
|
||||
import {
|
||||
resolveBundledWebFetchResolutionConfig,
|
||||
sortWebFetchProviders,
|
||||
} from "./web-fetch-providers.shared.js";
|
||||
|
||||
function listBundledWebFetchProviders(): PluginWebFetchProviderEntry[] {
|
||||
return sortWebFetchProviders(listBundledWebFetchProviderEntries());
|
||||
}
|
||||
|
||||
export function resolveBundledPluginWebFetchProviders(params: {
|
||||
config?: PluginLoadOptions["config"];
|
||||
workspaceDir?: string;
|
||||
env?: PluginLoadOptions["env"];
|
||||
bundledAllowlistCompat?: boolean;
|
||||
onlyPluginIds?: readonly string[];
|
||||
}): PluginWebFetchProviderEntry[] {
|
||||
const { config, normalized } = resolveBundledWebFetchResolutionConfig(params);
|
||||
const onlyPluginIdSet =
|
||||
params.onlyPluginIds && params.onlyPluginIds.length > 0 ? new Set(params.onlyPluginIds) : null;
|
||||
|
||||
return listBundledWebFetchProviders().filter((provider) => {
|
||||
if (onlyPluginIdSet && !onlyPluginIdSet.has(provider.pluginId)) {
|
||||
return false;
|
||||
}
|
||||
return resolveEffectiveEnableState({
|
||||
id: provider.pluginId,
|
||||
origin: "bundled",
|
||||
config: normalized,
|
||||
rootConfig: config,
|
||||
}).enabled;
|
||||
});
|
||||
}
|
||||
Reference in New Issue
Block a user