refactor: dedupe firecrawl and directive helpers

2026-04-12 01:31:08 +00:00 · 2026-04-08 14:37:16 +01:00
parent 34f73abfd3
commit 82a958dc79
4 changed files with 100 additions and 198 deletions
--- a/extensions/firecrawl/src/firecrawl-fetch-provider-shared.ts
+++ b/extensions/firecrawl/src/firecrawl-fetch-provider-shared.ts
@@ -0,0 +1,60 @@
+import type { WebFetchProviderPlugin } from "openclaw/plugin-sdk/provider-web-fetch-contract";
+
+type FirecrawlWebFetchProviderSharedFields = Omit<
+  WebFetchProviderPlugin,
+  "applySelectionConfig" | "createTool"
+>;
+
+function ensureRecord(target: Record<string, unknown>, key: string): Record<string, unknown> {
+  const current = target[key];
+  if (current && typeof current === "object" && !Array.isArray(current)) {
+    return current as Record<string, unknown>;
+  }
+  const next: Record<string, unknown> = {};
+  target[key] = next;
+  return next;
+}
+
+export const FIRECRAWL_WEB_FETCH_PROVIDER_SHARED = {
+  id: "firecrawl",
+  label: "Firecrawl",
+  hint: "Fetch pages with Firecrawl for JS-heavy or bot-protected sites.",
+  envVars: ["FIRECRAWL_API_KEY"],
+  placeholder: "fc-...",
+  signupUrl: "https://www.firecrawl.dev/",
+  docsUrl: "https://docs.firecrawl.dev",
+  autoDetectOrder: 50,
+  credentialPath: "plugins.entries.firecrawl.config.webFetch.apiKey",
+  inactiveSecretPaths: [
+    "plugins.entries.firecrawl.config.webFetch.apiKey",
+    "tools.web.fetch.firecrawl.apiKey",
+  ],
+  getCredentialValue: (fetchConfig) => {
+    if (!fetchConfig || typeof fetchConfig !== "object") {
+      return undefined;
+    }
+    const legacy = fetchConfig.firecrawl;
+    if (!legacy || typeof legacy !== "object" || Array.isArray(legacy)) {
+      return undefined;
+    }
+    if ((legacy as { enabled?: boolean }).enabled === false) {
+      return undefined;
+    }
+    return (legacy as { apiKey?: unknown }).apiKey;
+  },
+  setCredentialValue: (fetchConfigTarget, value) => {
+    const firecrawl = ensureRecord(fetchConfigTarget, "firecrawl");
+    firecrawl.apiKey = value;
+  },
+  getConfiguredCredentialValue: (config) =>
+    (config?.plugins?.entries?.firecrawl?.config as { webFetch?: { apiKey?: unknown } } | undefined)
+      ?.webFetch?.apiKey,
+  setConfiguredCredentialValue: (configTarget, value) => {
+    const plugins = ensureRecord(configTarget as unknown as Record<string, unknown>, "plugins");
+    const entries = ensureRecord(plugins, "entries");
+    const firecrawlEntry = ensureRecord(entries, "firecrawl");
+    const pluginConfig = ensureRecord(firecrawlEntry, "config");
+    const webFetch = ensureRecord(pluginConfig, "webFetch");
+    webFetch.apiKey = value;
+  },
+} satisfies FirecrawlWebFetchProviderSharedFields;
--- a/extensions/firecrawl/src/firecrawl-fetch-provider.ts
+++ b/extensions/firecrawl/src/firecrawl-fetch-provider.ts
@@ -1,68 +1,11 @@
 import type { WebFetchProviderPlugin } from "openclaw/plugin-sdk/provider-web-fetch";
 import { enablePluginInConfig } from "openclaw/plugin-sdk/provider-web-fetch";
 import { runFirecrawlScrape } from "./firecrawl-client.js";
+import { FIRECRAWL_WEB_FETCH_PROVIDER_SHARED } from "./firecrawl-fetch-provider-shared.js";

 export function createFirecrawlWebFetchProvider(): WebFetchProviderPlugin {
  return {
-    id: "firecrawl",
-    label: "Firecrawl",
-    hint: "Fetch pages with Firecrawl for JS-heavy or bot-protected sites.",
-    envVars: ["FIRECRAWL_API_KEY"],
-    placeholder: "fc-...",
-    signupUrl: "https://www.firecrawl.dev/",
-    docsUrl: "https://docs.firecrawl.dev",
-    autoDetectOrder: 50,
-    credentialPath: "plugins.entries.firecrawl.config.webFetch.apiKey",
-    inactiveSecretPaths: [
-      "plugins.entries.firecrawl.config.webFetch.apiKey",
-      "tools.web.fetch.firecrawl.apiKey",
-    ],
-    getCredentialValue: (fetchConfig) => {
-      if (!fetchConfig || typeof fetchConfig !== "object") {
-        return undefined;
-      }
-      const legacy = fetchConfig.firecrawl;
-      if (!legacy || typeof legacy !== "object" || Array.isArray(legacy)) {
-        return undefined;
-      }
-      if ((legacy as { enabled?: boolean }).enabled === false) {
-        return undefined;
-      }
-      return (legacy as { apiKey?: unknown }).apiKey;
-    },
-    setCredentialValue: (fetchConfigTarget, value) => {
-      const existing = fetchConfigTarget.firecrawl;
-      const firecrawl =
-        existing && typeof existing === "object" && !Array.isArray(existing)
-          ? (existing as Record<string, unknown>)
-          : {};
-      firecrawl.apiKey = value;
-      fetchConfigTarget.firecrawl = firecrawl;
-    },
-    getConfiguredCredentialValue: (config) =>
-      (
-        config?.plugins?.entries?.firecrawl?.config as
-          | { webFetch?: { apiKey?: unknown } }
-          | undefined
-      )?.webFetch?.apiKey,
-    setConfiguredCredentialValue: (configTarget, value) => {
-      const plugins = (configTarget.plugins ??= {});
-      const entries = (plugins.entries ??= {});
-      const firecrawlEntry = (entries.firecrawl ??= {});
-      const pluginConfig =
-        firecrawlEntry.config &&
-        typeof firecrawlEntry.config === "object" &&
-        !Array.isArray(firecrawlEntry.config)
-          ? firecrawlEntry.config
-          : ((firecrawlEntry.config = {}), firecrawlEntry.config);
-      const webFetch =
-        pluginConfig.webFetch &&
-        typeof pluginConfig.webFetch === "object" &&
-        !Array.isArray(pluginConfig.webFetch)
-          ? (pluginConfig.webFetch as Record<string, unknown>)
-          : ((pluginConfig.webFetch = {}), pluginConfig.webFetch as Record<string, unknown>);
-      webFetch.apiKey = value;
-    },
+    ...FIRECRAWL_WEB_FETCH_PROVIDER_SHARED,
    applySelectionConfig: (config) => enablePluginInConfig(config, "firecrawl").config,
    createTool: ({ config }) => ({
      description: "Fetch a page using Firecrawl.",
--- a/extensions/firecrawl/web-fetch-contract-api.ts
+++ b/extensions/firecrawl/web-fetch-contract-api.ts
@@ -2,63 +2,11 @@ import {
  enablePluginInConfig,
  type WebFetchProviderPlugin,
 } from "openclaw/plugin-sdk/provider-web-fetch-contract";
-
-function ensureRecord(target: Record<string, unknown>, key: string): Record<string, unknown> {
-  const current = target[key];
-  if (current && typeof current === "object" && !Array.isArray(current)) {
-    return current as Record<string, unknown>;
-  }
-  const next: Record<string, unknown> = {};
-  target[key] = next;
-  return next;
-}
+import { FIRECRAWL_WEB_FETCH_PROVIDER_SHARED } from "./src/firecrawl-fetch-provider-shared.js";

 export function createFirecrawlWebFetchProvider(): WebFetchProviderPlugin {
  return {
-    id: "firecrawl",
-    label: "Firecrawl",
-    hint: "Fetch pages with Firecrawl for JS-heavy or bot-protected sites.",
-    envVars: ["FIRECRAWL_API_KEY"],
-    placeholder: "fc-...",
-    signupUrl: "https://www.firecrawl.dev/",
-    docsUrl: "https://docs.firecrawl.dev",
-    autoDetectOrder: 50,
-    credentialPath: "plugins.entries.firecrawl.config.webFetch.apiKey",
-    inactiveSecretPaths: [
-      "plugins.entries.firecrawl.config.webFetch.apiKey",
-      "tools.web.fetch.firecrawl.apiKey",
-    ],
-    getCredentialValue: (fetchConfig) => {
-      if (!fetchConfig || typeof fetchConfig !== "object") {
-        return undefined;
-      }
-      const legacy = fetchConfig.firecrawl;
-      if (!legacy || typeof legacy !== "object" || Array.isArray(legacy)) {
-        return undefined;
-      }
-      if ((legacy as { enabled?: boolean }).enabled === false) {
-        return undefined;
-      }
-      return (legacy as { apiKey?: unknown }).apiKey;
-    },
-    setCredentialValue: (fetchConfigTarget, value) => {
-      const firecrawl = ensureRecord(fetchConfigTarget, "firecrawl");
-      firecrawl.apiKey = value;
-    },
-    getConfiguredCredentialValue: (config) =>
-      (
-        config?.plugins?.entries?.firecrawl?.config as
-          | { webFetch?: { apiKey?: unknown } }
-          | undefined
-      )?.webFetch?.apiKey,
-    setConfiguredCredentialValue: (configTarget, value) => {
-      const plugins = ensureRecord(configTarget as Record<string, unknown>, "plugins");
-      const entries = ensureRecord(plugins, "entries");
-      const firecrawlEntry = ensureRecord(entries, "firecrawl");
-      const pluginConfig = ensureRecord(firecrawlEntry, "config");
-      const webFetch = ensureRecord(pluginConfig, "webFetch");
-      webFetch.apiKey = value;
-    },
+    ...FIRECRAWL_WEB_FETCH_PROVIDER_SHARED,
    applySelectionConfig: (config) => enablePluginInConfig(config, "firecrawl").config,
    createTool: () => null,
  };
--- a/src/auto-reply/reply/directive-handling.model.test.ts
+++ b/src/auto-reply/reply/directive-handling.model.test.ts
@@ -198,6 +198,39 @@ async function persistModelDirectiveForTest(params: {
  return { persisted, sessionEntry };
 }

+type PersistInlineDirectivesParams = Parameters<typeof persistInlineDirectives>[0];
+
+async function persistInternalOperatorWriteDirective(
+  command: string,
+  overrides: Partial<PersistInlineDirectivesParams> = {},
+) {
+  const sessionEntry = overrides.sessionEntry ?? createSessionEntry();
+  const sessionStore = overrides.sessionStore ?? { "agent:main:main": sessionEntry };
+  await persistInlineDirectives({
+    directives: parseInlineDirectives(command),
+    cfg: baseConfig(),
+    sessionEntry,
+    sessionStore,
+    sessionKey: "agent:main:main",
+    storePath: "/tmp/sessions.json",
+    elevatedEnabled: true,
+    elevatedAllowed: true,
+    defaultProvider: "anthropic",
+    defaultModel: "claude-opus-4-6",
+    aliasIndex: baseAliasIndex(),
+    allowedModelKeys: new Set(["anthropic/claude-opus-4-6", "openai/gpt-4o"]),
+    provider: "anthropic",
+    model: "claude-opus-4-6",
+    initialModelLabel: "anthropic/claude-opus-4-6",
+    formatModelSwitchEvent: (label) => `Switched to ${label}`,
+    agentCfg: undefined,
+    surface: "webchat",
+    gatewayClientScopes: ["operator.write"],
+    ...overrides,
+  });
+  return sessionEntry;
+}
+
 async function resolveModelInfoReply(
  overrides: Partial<Parameters<typeof maybeHandleModelDirectiveInfo>[0]> = {},
 ) {
@@ -692,37 +725,9 @@ describe("handleDirectiveOnly model persist behavior (fixes #1435)", () => {

 describe("persistInlineDirectives internal exec scope gate", () => {
  it("skips exec persistence for internal operator.write callers", async () => {
-    const allowedModelKeys = new Set(["anthropic/claude-opus-4-6", "openai/gpt-4o"]);
-    const directives = parseInlineDirectives(
+    const sessionEntry = await persistInternalOperatorWriteDirective(
      "/exec host=node security=allowlist ask=always node=worker-1",
    );
-    const sessionEntry = {
-      sessionId: "s1",
-      updatedAt: Date.now(),
-    } as SessionEntry;
-    const sessionStore = { "agent:main:main": sessionEntry };
-
-    await persistInlineDirectives({
-      directives,
-      cfg: baseConfig(),
-      sessionEntry,
-      sessionStore,
-      sessionKey: "agent:main:main",
-      storePath: "/tmp/sessions.json",
-      elevatedEnabled: true,
-      elevatedAllowed: true,
-      defaultProvider: "anthropic",
-      defaultModel: "claude-opus-4-6",
-      aliasIndex: baseAliasIndex(),
-      allowedModelKeys,
-      provider: "anthropic",
-      model: "claude-opus-4-6",
-      initialModelLabel: "anthropic/claude-opus-4-6",
-      formatModelSwitchEvent: (label) => `Switched to ${label}`,
-      agentCfg: undefined,
-      surface: "webchat",
-      gatewayClientScopes: ["operator.write"],
-    });

    expect(sessionEntry.execHost).toBeUndefined();
    expect(sessionEntry.execSecurity).toBeUndefined();
@@ -731,69 +736,15 @@ describe("persistInlineDirectives internal exec scope gate", () => {
  });

  it("skips verbose persistence for internal operator.write callers", async () => {
-    const allowedModelKeys = new Set(["anthropic/claude-opus-4-6", "openai/gpt-4o"]);
-    const directives = parseInlineDirectives("/verbose full");
-    const sessionEntry = {
-      sessionId: "s1",
-      updatedAt: Date.now(),
-    } as SessionEntry;
-    const sessionStore = { "agent:main:main": sessionEntry };
-
-    await persistInlineDirectives({
-      directives,
-      cfg: baseConfig(),
-      sessionEntry,
-      sessionStore,
-      sessionKey: "agent:main:main",
-      storePath: "/tmp/sessions.json",
-      elevatedEnabled: true,
-      elevatedAllowed: true,
-      defaultProvider: "anthropic",
-      defaultModel: "claude-opus-4-6",
-      aliasIndex: baseAliasIndex(),
-      allowedModelKeys,
-      provider: "anthropic",
-      model: "claude-opus-4-6",
-      initialModelLabel: "anthropic/claude-opus-4-6",
-      formatModelSwitchEvent: (label) => `Switched to ${label}`,
-      agentCfg: undefined,
-      surface: "webchat",
-      gatewayClientScopes: ["operator.write"],
-    });
+    const sessionEntry = await persistInternalOperatorWriteDirective("/verbose full");

    expect(sessionEntry.verboseLevel).toBeUndefined();
  });

  it("treats internal provider context as authoritative over external surface metadata", async () => {
-    const allowedModelKeys = new Set(["anthropic/claude-opus-4-6", "openai/gpt-4o"]);
-    const directives = parseInlineDirectives("/verbose full");
-    const sessionEntry = {
-      sessionId: "s1",
-      updatedAt: Date.now(),
-    } as SessionEntry;
-    const sessionStore = { "agent:main:main": sessionEntry };
-
-    await persistInlineDirectives({
-      directives,
-      cfg: baseConfig(),
-      sessionEntry,
-      sessionStore,
-      sessionKey: "agent:main:main",
-      storePath: "/tmp/sessions.json",
-      elevatedEnabled: true,
-      elevatedAllowed: true,
-      defaultProvider: "anthropic",
-      defaultModel: "claude-opus-4-6",
-      aliasIndex: baseAliasIndex(),
-      allowedModelKeys,
-      provider: "anthropic",
-      model: "claude-opus-4-6",
-      initialModelLabel: "anthropic/claude-opus-4-6",
-      formatModelSwitchEvent: (label) => `Switched to ${label}`,
-      agentCfg: undefined,
+    const sessionEntry = await persistInternalOperatorWriteDirective("/verbose full", {
      messageProvider: "webchat",
      surface: "telegram",
-      gatewayClientScopes: ["operator.write"],
    });

    expect(sessionEntry.verboseLevel).toBeUndefined();