From 82a958dc79fac118e053378dd2cb2f862d7fd808 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Wed, 8 Apr 2026 14:37:16 +0100 Subject: [PATCH] refactor: dedupe firecrawl and directive helpers --- .../src/firecrawl-fetch-provider-shared.ts | 60 +++++++++ .../firecrawl/src/firecrawl-fetch-provider.ts | 61 +-------- .../firecrawl/web-fetch-contract-api.ts | 56 +------- .../reply/directive-handling.model.test.ts | 121 ++++++------------ 4 files changed, 100 insertions(+), 198 deletions(-) create mode 100644 extensions/firecrawl/src/firecrawl-fetch-provider-shared.ts diff --git a/extensions/firecrawl/src/firecrawl-fetch-provider-shared.ts b/extensions/firecrawl/src/firecrawl-fetch-provider-shared.ts new file mode 100644 index 00000000000..fdd16664d4d --- /dev/null +++ b/extensions/firecrawl/src/firecrawl-fetch-provider-shared.ts @@ -0,0 +1,60 @@ +import type { WebFetchProviderPlugin } from "openclaw/plugin-sdk/provider-web-fetch-contract"; + +type FirecrawlWebFetchProviderSharedFields = Omit< + WebFetchProviderPlugin, + "applySelectionConfig" | "createTool" +>; + +function ensureRecord(target: Record, key: string): Record { + const current = target[key]; + if (current && typeof current === "object" && !Array.isArray(current)) { + return current as Record; + } + const next: Record = {}; + target[key] = next; + return next; +} + +export const FIRECRAWL_WEB_FETCH_PROVIDER_SHARED = { + id: "firecrawl", + label: "Firecrawl", + hint: "Fetch pages with Firecrawl for JS-heavy or bot-protected sites.", + envVars: ["FIRECRAWL_API_KEY"], + placeholder: "fc-...", + signupUrl: "https://www.firecrawl.dev/", + docsUrl: "https://docs.firecrawl.dev", + autoDetectOrder: 50, + credentialPath: "plugins.entries.firecrawl.config.webFetch.apiKey", + inactiveSecretPaths: [ + "plugins.entries.firecrawl.config.webFetch.apiKey", + "tools.web.fetch.firecrawl.apiKey", + ], + getCredentialValue: (fetchConfig) => { + if (!fetchConfig || typeof fetchConfig !== "object") { + return undefined; + } + const legacy = fetchConfig.firecrawl; + if (!legacy || typeof legacy !== "object" || Array.isArray(legacy)) { + return undefined; + } + if ((legacy as { enabled?: boolean }).enabled === false) { + return undefined; + } + return (legacy as { apiKey?: unknown }).apiKey; + }, + setCredentialValue: (fetchConfigTarget, value) => { + const firecrawl = ensureRecord(fetchConfigTarget, "firecrawl"); + firecrawl.apiKey = value; + }, + getConfiguredCredentialValue: (config) => + (config?.plugins?.entries?.firecrawl?.config as { webFetch?: { apiKey?: unknown } } | undefined) + ?.webFetch?.apiKey, + setConfiguredCredentialValue: (configTarget, value) => { + const plugins = ensureRecord(configTarget as unknown as Record, "plugins"); + const entries = ensureRecord(plugins, "entries"); + const firecrawlEntry = ensureRecord(entries, "firecrawl"); + const pluginConfig = ensureRecord(firecrawlEntry, "config"); + const webFetch = ensureRecord(pluginConfig, "webFetch"); + webFetch.apiKey = value; + }, +} satisfies FirecrawlWebFetchProviderSharedFields; diff --git a/extensions/firecrawl/src/firecrawl-fetch-provider.ts b/extensions/firecrawl/src/firecrawl-fetch-provider.ts index d9aecf5c41f..f057b64eb09 100644 --- a/extensions/firecrawl/src/firecrawl-fetch-provider.ts +++ b/extensions/firecrawl/src/firecrawl-fetch-provider.ts @@ -1,68 +1,11 @@ import type { WebFetchProviderPlugin } from "openclaw/plugin-sdk/provider-web-fetch"; import { enablePluginInConfig } from "openclaw/plugin-sdk/provider-web-fetch"; import { runFirecrawlScrape } from "./firecrawl-client.js"; +import { FIRECRAWL_WEB_FETCH_PROVIDER_SHARED } from "./firecrawl-fetch-provider-shared.js"; export function createFirecrawlWebFetchProvider(): WebFetchProviderPlugin { return { - id: "firecrawl", - label: "Firecrawl", - hint: "Fetch pages with Firecrawl for JS-heavy or bot-protected sites.", - envVars: ["FIRECRAWL_API_KEY"], - placeholder: "fc-...", - signupUrl: "https://www.firecrawl.dev/", - docsUrl: "https://docs.firecrawl.dev", - autoDetectOrder: 50, - credentialPath: "plugins.entries.firecrawl.config.webFetch.apiKey", - inactiveSecretPaths: [ - "plugins.entries.firecrawl.config.webFetch.apiKey", - "tools.web.fetch.firecrawl.apiKey", - ], - getCredentialValue: (fetchConfig) => { - if (!fetchConfig || typeof fetchConfig !== "object") { - return undefined; - } - const legacy = fetchConfig.firecrawl; - if (!legacy || typeof legacy !== "object" || Array.isArray(legacy)) { - return undefined; - } - if ((legacy as { enabled?: boolean }).enabled === false) { - return undefined; - } - return (legacy as { apiKey?: unknown }).apiKey; - }, - setCredentialValue: (fetchConfigTarget, value) => { - const existing = fetchConfigTarget.firecrawl; - const firecrawl = - existing && typeof existing === "object" && !Array.isArray(existing) - ? (existing as Record) - : {}; - firecrawl.apiKey = value; - fetchConfigTarget.firecrawl = firecrawl; - }, - getConfiguredCredentialValue: (config) => - ( - config?.plugins?.entries?.firecrawl?.config as - | { webFetch?: { apiKey?: unknown } } - | undefined - )?.webFetch?.apiKey, - setConfiguredCredentialValue: (configTarget, value) => { - const plugins = (configTarget.plugins ??= {}); - const entries = (plugins.entries ??= {}); - const firecrawlEntry = (entries.firecrawl ??= {}); - const pluginConfig = - firecrawlEntry.config && - typeof firecrawlEntry.config === "object" && - !Array.isArray(firecrawlEntry.config) - ? firecrawlEntry.config - : ((firecrawlEntry.config = {}), firecrawlEntry.config); - const webFetch = - pluginConfig.webFetch && - typeof pluginConfig.webFetch === "object" && - !Array.isArray(pluginConfig.webFetch) - ? (pluginConfig.webFetch as Record) - : ((pluginConfig.webFetch = {}), pluginConfig.webFetch as Record); - webFetch.apiKey = value; - }, + ...FIRECRAWL_WEB_FETCH_PROVIDER_SHARED, applySelectionConfig: (config) => enablePluginInConfig(config, "firecrawl").config, createTool: ({ config }) => ({ description: "Fetch a page using Firecrawl.", diff --git a/extensions/firecrawl/web-fetch-contract-api.ts b/extensions/firecrawl/web-fetch-contract-api.ts index b2de504a337..050116943e0 100644 --- a/extensions/firecrawl/web-fetch-contract-api.ts +++ b/extensions/firecrawl/web-fetch-contract-api.ts @@ -2,63 +2,11 @@ import { enablePluginInConfig, type WebFetchProviderPlugin, } from "openclaw/plugin-sdk/provider-web-fetch-contract"; - -function ensureRecord(target: Record, key: string): Record { - const current = target[key]; - if (current && typeof current === "object" && !Array.isArray(current)) { - return current as Record; - } - const next: Record = {}; - target[key] = next; - return next; -} +import { FIRECRAWL_WEB_FETCH_PROVIDER_SHARED } from "./src/firecrawl-fetch-provider-shared.js"; export function createFirecrawlWebFetchProvider(): WebFetchProviderPlugin { return { - id: "firecrawl", - label: "Firecrawl", - hint: "Fetch pages with Firecrawl for JS-heavy or bot-protected sites.", - envVars: ["FIRECRAWL_API_KEY"], - placeholder: "fc-...", - signupUrl: "https://www.firecrawl.dev/", - docsUrl: "https://docs.firecrawl.dev", - autoDetectOrder: 50, - credentialPath: "plugins.entries.firecrawl.config.webFetch.apiKey", - inactiveSecretPaths: [ - "plugins.entries.firecrawl.config.webFetch.apiKey", - "tools.web.fetch.firecrawl.apiKey", - ], - getCredentialValue: (fetchConfig) => { - if (!fetchConfig || typeof fetchConfig !== "object") { - return undefined; - } - const legacy = fetchConfig.firecrawl; - if (!legacy || typeof legacy !== "object" || Array.isArray(legacy)) { - return undefined; - } - if ((legacy as { enabled?: boolean }).enabled === false) { - return undefined; - } - return (legacy as { apiKey?: unknown }).apiKey; - }, - setCredentialValue: (fetchConfigTarget, value) => { - const firecrawl = ensureRecord(fetchConfigTarget, "firecrawl"); - firecrawl.apiKey = value; - }, - getConfiguredCredentialValue: (config) => - ( - config?.plugins?.entries?.firecrawl?.config as - | { webFetch?: { apiKey?: unknown } } - | undefined - )?.webFetch?.apiKey, - setConfiguredCredentialValue: (configTarget, value) => { - const plugins = ensureRecord(configTarget as Record, "plugins"); - const entries = ensureRecord(plugins, "entries"); - const firecrawlEntry = ensureRecord(entries, "firecrawl"); - const pluginConfig = ensureRecord(firecrawlEntry, "config"); - const webFetch = ensureRecord(pluginConfig, "webFetch"); - webFetch.apiKey = value; - }, + ...FIRECRAWL_WEB_FETCH_PROVIDER_SHARED, applySelectionConfig: (config) => enablePluginInConfig(config, "firecrawl").config, createTool: () => null, }; diff --git a/src/auto-reply/reply/directive-handling.model.test.ts b/src/auto-reply/reply/directive-handling.model.test.ts index 86f4d849ec3..66bd82baa1b 100644 --- a/src/auto-reply/reply/directive-handling.model.test.ts +++ b/src/auto-reply/reply/directive-handling.model.test.ts @@ -198,6 +198,39 @@ async function persistModelDirectiveForTest(params: { return { persisted, sessionEntry }; } +type PersistInlineDirectivesParams = Parameters[0]; + +async function persistInternalOperatorWriteDirective( + command: string, + overrides: Partial = {}, +) { + const sessionEntry = overrides.sessionEntry ?? createSessionEntry(); + const sessionStore = overrides.sessionStore ?? { "agent:main:main": sessionEntry }; + await persistInlineDirectives({ + directives: parseInlineDirectives(command), + cfg: baseConfig(), + sessionEntry, + sessionStore, + sessionKey: "agent:main:main", + storePath: "/tmp/sessions.json", + elevatedEnabled: true, + elevatedAllowed: true, + defaultProvider: "anthropic", + defaultModel: "claude-opus-4-6", + aliasIndex: baseAliasIndex(), + allowedModelKeys: new Set(["anthropic/claude-opus-4-6", "openai/gpt-4o"]), + provider: "anthropic", + model: "claude-opus-4-6", + initialModelLabel: "anthropic/claude-opus-4-6", + formatModelSwitchEvent: (label) => `Switched to ${label}`, + agentCfg: undefined, + surface: "webchat", + gatewayClientScopes: ["operator.write"], + ...overrides, + }); + return sessionEntry; +} + async function resolveModelInfoReply( overrides: Partial[0]> = {}, ) { @@ -692,37 +725,9 @@ describe("handleDirectiveOnly model persist behavior (fixes #1435)", () => { describe("persistInlineDirectives internal exec scope gate", () => { it("skips exec persistence for internal operator.write callers", async () => { - const allowedModelKeys = new Set(["anthropic/claude-opus-4-6", "openai/gpt-4o"]); - const directives = parseInlineDirectives( + const sessionEntry = await persistInternalOperatorWriteDirective( "/exec host=node security=allowlist ask=always node=worker-1", ); - const sessionEntry = { - sessionId: "s1", - updatedAt: Date.now(), - } as SessionEntry; - const sessionStore = { "agent:main:main": sessionEntry }; - - await persistInlineDirectives({ - directives, - cfg: baseConfig(), - sessionEntry, - sessionStore, - sessionKey: "agent:main:main", - storePath: "/tmp/sessions.json", - elevatedEnabled: true, - elevatedAllowed: true, - defaultProvider: "anthropic", - defaultModel: "claude-opus-4-6", - aliasIndex: baseAliasIndex(), - allowedModelKeys, - provider: "anthropic", - model: "claude-opus-4-6", - initialModelLabel: "anthropic/claude-opus-4-6", - formatModelSwitchEvent: (label) => `Switched to ${label}`, - agentCfg: undefined, - surface: "webchat", - gatewayClientScopes: ["operator.write"], - }); expect(sessionEntry.execHost).toBeUndefined(); expect(sessionEntry.execSecurity).toBeUndefined(); @@ -731,69 +736,15 @@ describe("persistInlineDirectives internal exec scope gate", () => { }); it("skips verbose persistence for internal operator.write callers", async () => { - const allowedModelKeys = new Set(["anthropic/claude-opus-4-6", "openai/gpt-4o"]); - const directives = parseInlineDirectives("/verbose full"); - const sessionEntry = { - sessionId: "s1", - updatedAt: Date.now(), - } as SessionEntry; - const sessionStore = { "agent:main:main": sessionEntry }; - - await persistInlineDirectives({ - directives, - cfg: baseConfig(), - sessionEntry, - sessionStore, - sessionKey: "agent:main:main", - storePath: "/tmp/sessions.json", - elevatedEnabled: true, - elevatedAllowed: true, - defaultProvider: "anthropic", - defaultModel: "claude-opus-4-6", - aliasIndex: baseAliasIndex(), - allowedModelKeys, - provider: "anthropic", - model: "claude-opus-4-6", - initialModelLabel: "anthropic/claude-opus-4-6", - formatModelSwitchEvent: (label) => `Switched to ${label}`, - agentCfg: undefined, - surface: "webchat", - gatewayClientScopes: ["operator.write"], - }); + const sessionEntry = await persistInternalOperatorWriteDirective("/verbose full"); expect(sessionEntry.verboseLevel).toBeUndefined(); }); it("treats internal provider context as authoritative over external surface metadata", async () => { - const allowedModelKeys = new Set(["anthropic/claude-opus-4-6", "openai/gpt-4o"]); - const directives = parseInlineDirectives("/verbose full"); - const sessionEntry = { - sessionId: "s1", - updatedAt: Date.now(), - } as SessionEntry; - const sessionStore = { "agent:main:main": sessionEntry }; - - await persistInlineDirectives({ - directives, - cfg: baseConfig(), - sessionEntry, - sessionStore, - sessionKey: "agent:main:main", - storePath: "/tmp/sessions.json", - elevatedEnabled: true, - elevatedAllowed: true, - defaultProvider: "anthropic", - defaultModel: "claude-opus-4-6", - aliasIndex: baseAliasIndex(), - allowedModelKeys, - provider: "anthropic", - model: "claude-opus-4-6", - initialModelLabel: "anthropic/claude-opus-4-6", - formatModelSwitchEvent: (label) => `Switched to ${label}`, - agentCfg: undefined, + const sessionEntry = await persistInternalOperatorWriteDirective("/verbose full", { messageProvider: "webchat", surface: "telegram", - gatewayClientScopes: ["operator.write"], }); expect(sessionEntry.verboseLevel).toBeUndefined();