diff --git a/.github/labeler.yml b/.github/labeler.yml index d289d0276fa..733e8579afd 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -306,10 +306,6 @@ - changed-files: - any-glob-to-any-file: - "extensions/tavily/**" -"extensions: tinyfish": - - changed-files: - - any-glob-to-any-file: - - "extensions/tinyfish/**" "extensions: talk-voice": - changed-files: - any-glob-to-any-file: diff --git a/CHANGELOG.md b/CHANGELOG.md index c983ea75b29..1a26ea90640 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -153,7 +153,6 @@ Docs: https://docs.openclaw.ai - CLI/onboarding: reset the remote gateway URL prompt to the safe loopback default after declining a discovered endpoint, so onboarding does not keep a previously rejected remote URL. (#57828) - Agents/exec defaults: honor per-agent `tools.exec` defaults when no inline directive or session override is present, so configured exec host, security, ask, and node settings actually apply. (#57689) - Sandbox/networking: sanitize SSH subprocess env vars through the shared sandbox policy and route marketplace archive downloads plus Ollama discovery, auth, and pull requests through the guarded fetch path so sandboxed execution and remote fetches follow the repo's trust boundaries. (#57848, #57850) -- Plugins/TinyFish: add bundled TinyFish plugin (default-off) with a `tinyfish_automation` tool for hosted browser automation of complex public web workflows, including SSE streaming, SSRF guards, and proxy/stealth support. (#58645) Thanks @simantak-dabhade. ### Fixes diff --git a/docs/.generated/config-baseline.json b/docs/.generated/config-baseline.json index 28c4e8e0e43..332d6a67341 100644 --- a/docs/.generated/config-baseline.json +++ b/docs/.generated/config-baseline.json @@ -57374,160 +57374,6 @@ "help": "Explicitly allows this plugin to request provider/model overrides in background subagent runs. Keep false unless the plugin is trusted to steer model selection.", "hasChildren": false }, - { - "path": "plugins.entries.tinyfish", - "kind": "plugin", - "type": "object", - "required": false, - "deprecated": false, - "sensitive": false, - "tags": [ - "advanced" - ], - "label": "TinyFish", - "help": "Hosted browser automation for complex public web workflows. (plugin: tinyfish)", - "hasChildren": true - }, - { - "path": "plugins.entries.tinyfish.config", - "kind": "plugin", - "type": "object", - "required": false, - "deprecated": false, - "sensitive": false, - "tags": [ - "advanced" - ], - "label": "TinyFish Config", - "help": "Plugin-defined config payload for tinyfish.", - "hasChildren": true - }, - { - "path": "plugins.entries.tinyfish.config.apiKey", - "kind": "plugin", - "type": [ - "object", - "string" - ], - "required": false, - "deprecated": false, - "sensitive": true, - "tags": [ - "auth", - "security" - ], - "label": "TinyFish API Key", - "help": "TinyFish API key for hosted browser automation (fallback: TINYFISH_API_KEY env var).", - "hasChildren": false - }, - { - "path": "plugins.entries.tinyfish.config.baseUrl", - "kind": "plugin", - "type": "string", - "required": false, - "deprecated": false, - "sensitive": false, - "tags": [ - "advanced", - "url-secret" - ], - "label": "TinyFish Base URL", - "help": "TinyFish API base URL override.", - "hasChildren": false - }, - { - "path": "plugins.entries.tinyfish.enabled", - "kind": "plugin", - "type": "boolean", - "required": false, - "deprecated": false, - "sensitive": false, - "tags": [ - "advanced" - ], - "label": "Enable TinyFish", - "hasChildren": false - }, - { - "path": "plugins.entries.tinyfish.hooks", - "kind": "plugin", - "type": "object", - "required": false, - "deprecated": false, - "sensitive": false, - "tags": [ - "advanced" - ], - "label": "Plugin Hook Policy", - "help": "Per-plugin typed hook policy controls for core-enforced safety gates. Use this to constrain high-impact hook categories without disabling the entire plugin.", - "hasChildren": true - }, - { - "path": "plugins.entries.tinyfish.hooks.allowPromptInjection", - "kind": "plugin", - "type": "boolean", - "required": false, - "deprecated": false, - "sensitive": false, - "tags": [ - "access" - ], - "label": "Allow Prompt Injection Hooks", - "help": "Controls whether this plugin may mutate prompts through typed hooks. Set false to block `before_prompt_build` and ignore prompt-mutating fields from legacy `before_agent_start`, while preserving legacy `modelOverride` and `providerOverride` behavior.", - "hasChildren": false - }, - { - "path": "plugins.entries.tinyfish.subagent", - "kind": "plugin", - "type": "object", - "required": false, - "deprecated": false, - "sensitive": false, - "tags": [ - "advanced" - ], - "label": "Plugin Subagent Policy", - "help": "Per-plugin subagent runtime controls for model override trust and allowlists. Keep this unset unless a plugin must explicitly steer subagent model selection.", - "hasChildren": true - }, - { - "path": "plugins.entries.tinyfish.subagent.allowedModels", - "kind": "plugin", - "type": "array", - "required": false, - "deprecated": false, - "sensitive": false, - "tags": [ - "access" - ], - "label": "Plugin Subagent Allowed Models", - "help": "Allowed override targets for trusted plugin subagent runs as canonical \"provider/model\" refs. Use \"*\" only when you intentionally allow any model.", - "hasChildren": true - }, - { - "path": "plugins.entries.tinyfish.subagent.allowedModels.*", - "kind": "plugin", - "type": "string", - "required": false, - "deprecated": false, - "sensitive": false, - "tags": [], - "hasChildren": false - }, - { - "path": "plugins.entries.tinyfish.subagent.allowModelOverride", - "kind": "plugin", - "type": "boolean", - "required": false, - "deprecated": false, - "sensitive": false, - "tags": [ - "access" - ], - "label": "Allow Plugin Subagent Model Override", - "help": "Explicitly allows this plugin to request provider/model overrides in background subagent runs. Keep false unless the plugin is trusted to steer model selection.", - "hasChildren": false - }, { "path": "plugins.entries.tlon", "kind": "plugin", diff --git a/docs/.generated/config-baseline.jsonl b/docs/.generated/config-baseline.jsonl index 438808aa16c..c82a49455b6 100644 --- a/docs/.generated/config-baseline.jsonl +++ b/docs/.generated/config-baseline.jsonl @@ -1,4 +1,4 @@ -{"generatedBy":"scripts/generate-config-doc-baseline.ts","recordType":"meta","totalPaths":5778} +{"generatedBy":"scripts/generate-config-doc-baseline.ts","recordType":"meta","totalPaths":5767} {"recordType":"path","path":"acp","kind":"core","type":"object","required":false,"deprecated":false,"sensitive":false,"tags":["advanced"],"label":"ACP","help":"ACP runtime controls for enabling dispatch, selecting backends, constraining allowed agent targets, and tuning streamed turn projection behavior.","hasChildren":true} {"recordType":"path","path":"acp.allowedAgents","kind":"core","type":"array","required":false,"deprecated":false,"sensitive":false,"tags":["access"],"label":"ACP Allowed Agents","help":"Allowlist of ACP target agent ids permitted for ACP runtime sessions. Empty means no additional allowlist restriction.","hasChildren":true} {"recordType":"path","path":"acp.allowedAgents.*","kind":"core","type":"string","required":false,"deprecated":false,"sensitive":false,"tags":[],"hasChildren":false} @@ -4914,17 +4914,6 @@ {"recordType":"path","path":"plugins.entries.thread-ownership.subagent.allowedModels","kind":"plugin","type":"array","required":false,"deprecated":false,"sensitive":false,"tags":["access"],"label":"Plugin Subagent Allowed Models","help":"Allowed override targets for trusted plugin subagent runs as canonical \"provider/model\" refs. Use \"*\" only when you intentionally allow any model.","hasChildren":true} {"recordType":"path","path":"plugins.entries.thread-ownership.subagent.allowedModels.*","kind":"plugin","type":"string","required":false,"deprecated":false,"sensitive":false,"tags":[],"hasChildren":false} {"recordType":"path","path":"plugins.entries.thread-ownership.subagent.allowModelOverride","kind":"plugin","type":"boolean","required":false,"deprecated":false,"sensitive":false,"tags":["access"],"label":"Allow Plugin Subagent Model Override","help":"Explicitly allows this plugin to request provider/model overrides in background subagent runs. Keep false unless the plugin is trusted to steer model selection.","hasChildren":false} -{"recordType":"path","path":"plugins.entries.tinyfish","kind":"plugin","type":"object","required":false,"deprecated":false,"sensitive":false,"tags":["advanced"],"label":"TinyFish","help":"Hosted browser automation for complex public web workflows. (plugin: tinyfish)","hasChildren":true} -{"recordType":"path","path":"plugins.entries.tinyfish.config","kind":"plugin","type":"object","required":false,"deprecated":false,"sensitive":false,"tags":["advanced"],"label":"TinyFish Config","help":"Plugin-defined config payload for tinyfish.","hasChildren":true} -{"recordType":"path","path":"plugins.entries.tinyfish.config.apiKey","kind":"plugin","type":["object","string"],"required":false,"deprecated":false,"sensitive":true,"tags":["auth","security"],"label":"TinyFish API Key","help":"TinyFish API key for hosted browser automation (fallback: TINYFISH_API_KEY env var).","hasChildren":false} -{"recordType":"path","path":"plugins.entries.tinyfish.config.baseUrl","kind":"plugin","type":"string","required":false,"deprecated":false,"sensitive":false,"tags":["advanced","url-secret"],"label":"TinyFish Base URL","help":"TinyFish API base URL override.","hasChildren":false} -{"recordType":"path","path":"plugins.entries.tinyfish.enabled","kind":"plugin","type":"boolean","required":false,"deprecated":false,"sensitive":false,"tags":["advanced"],"label":"Enable TinyFish","hasChildren":false} -{"recordType":"path","path":"plugins.entries.tinyfish.hooks","kind":"plugin","type":"object","required":false,"deprecated":false,"sensitive":false,"tags":["advanced"],"label":"Plugin Hook Policy","help":"Per-plugin typed hook policy controls for core-enforced safety gates. Use this to constrain high-impact hook categories without disabling the entire plugin.","hasChildren":true} -{"recordType":"path","path":"plugins.entries.tinyfish.hooks.allowPromptInjection","kind":"plugin","type":"boolean","required":false,"deprecated":false,"sensitive":false,"tags":["access"],"label":"Allow Prompt Injection Hooks","help":"Controls whether this plugin may mutate prompts through typed hooks. Set false to block `before_prompt_build` and ignore prompt-mutating fields from legacy `before_agent_start`, while preserving legacy `modelOverride` and `providerOverride` behavior.","hasChildren":false} -{"recordType":"path","path":"plugins.entries.tinyfish.subagent","kind":"plugin","type":"object","required":false,"deprecated":false,"sensitive":false,"tags":["advanced"],"label":"Plugin Subagent Policy","help":"Per-plugin subagent runtime controls for model override trust and allowlists. Keep this unset unless a plugin must explicitly steer subagent model selection.","hasChildren":true} -{"recordType":"path","path":"plugins.entries.tinyfish.subagent.allowedModels","kind":"plugin","type":"array","required":false,"deprecated":false,"sensitive":false,"tags":["access"],"label":"Plugin Subagent Allowed Models","help":"Allowed override targets for trusted plugin subagent runs as canonical \"provider/model\" refs. Use \"*\" only when you intentionally allow any model.","hasChildren":true} -{"recordType":"path","path":"plugins.entries.tinyfish.subagent.allowedModels.*","kind":"plugin","type":"string","required":false,"deprecated":false,"sensitive":false,"tags":[],"hasChildren":false} -{"recordType":"path","path":"plugins.entries.tinyfish.subagent.allowModelOverride","kind":"plugin","type":"boolean","required":false,"deprecated":false,"sensitive":false,"tags":["access"],"label":"Allow Plugin Subagent Model Override","help":"Explicitly allows this plugin to request provider/model overrides in background subagent runs. Keep false unless the plugin is trusted to steer model selection.","hasChildren":false} {"recordType":"path","path":"plugins.entries.tlon","kind":"plugin","type":"object","required":false,"deprecated":false,"sensitive":false,"tags":["advanced"],"label":"@openclaw/tlon","help":"OpenClaw Tlon/Urbit channel plugin (plugin: tlon)","hasChildren":true} {"recordType":"path","path":"plugins.entries.tlon.config","kind":"plugin","type":"object","required":false,"deprecated":false,"sensitive":false,"tags":["advanced"],"label":"@openclaw/tlon Config","help":"Plugin-defined config payload for tlon.","hasChildren":false} {"recordType":"path","path":"plugins.entries.tlon.enabled","kind":"plugin","type":"boolean","required":false,"deprecated":false,"sensitive":false,"tags":["advanced"],"label":"Enable @openclaw/tlon","hasChildren":false} diff --git a/docs/.i18n/glossary.zh-CN.json b/docs/.i18n/glossary.zh-CN.json index 5b1f3f152a5..08ef38067d3 100644 --- a/docs/.i18n/glossary.zh-CN.json +++ b/docs/.i18n/glossary.zh-CN.json @@ -302,9 +302,5 @@ { "source": "Testing", "target": "测试" - }, - { - "source": "TinyFish", - "target": "TinyFish" } ] diff --git a/docs/docs.json b/docs/docs.json index d01c9b5b573..cef0331dad2 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -1157,8 +1157,7 @@ "tools/kimi-search", "tools/perplexity-search", "tools/searxng-search", - "tools/tavily", - "tools/tinyfish" + "tools/tavily" ] }, "tools/btw", diff --git a/docs/tools/tinyfish.md b/docs/tools/tinyfish.md deleted file mode 100644 index 66e6577527a..00000000000 --- a/docs/tools/tinyfish.md +++ /dev/null @@ -1,125 +0,0 @@ ---- -summary: "TinyFish plugin: hosted browser automation for public multi-step workflows" -read_when: - - You want hosted browser automation from OpenClaw - - You are configuring or developing the TinyFish plugin -title: "TinyFish" ---- - -# TinyFish - -TinyFish adds a hosted browser automation tool to OpenClaw for complex public -web workflows: multi-step navigation, forms, JS-heavy pages, geo-aware proxy -routing, and structured extraction. - -Quick mental model: - -- Enable the bundled plugin -- Configure `plugins.entries.tinyfish.config` -- Use the `tinyfish_automation` tool for public browser workflows -- Get back `run_id`, `status`, `result`, and a live `streaming_url` when TinyFish provides one - -## Where it runs - -The TinyFish plugin runs inside the Gateway process, but the browser automation -it triggers runs on TinyFish's hosted infrastructure. - -If you use a remote Gateway, enable and configure the plugin on the machine -running the Gateway. - -## Enable - -TinyFish ships as a bundled plugin and is disabled by default. - -```json5 -{ - plugins: { - entries: { - tinyfish: { - enabled: true, - }, - }, - }, -} -``` - -Restart the Gateway after enabling it. - -## Config - -Set config under `plugins.entries.tinyfish.config`: - -```json5 -{ - plugins: { - entries: { - tinyfish: { - enabled: true, - config: { - apiKey: "tf_live_...", - // Optional; defaults to https://agent.tinyfish.ai - baseUrl: "https://agent.tinyfish.ai", - }, - }, - }, - }, -} -``` - -You can also supply the API key through `TINYFISH_API_KEY`. - -## Tool - -The plugin registers one tool: - -### tinyfish_automation - -Run hosted browser automation against a public website. - -| Parameter | Required | Description | -| ----------------- | -------- | ----------------------------------------------------------------- | -| `url` | Yes | Target public website URL | -| `goal` | Yes | Natural-language description of what to accomplish | -| `browser_profile` | No | `lite` (default) or `stealth` (anti-bot mode) | -| `proxy_config` | No | Object with `enabled` (boolean) and `country_code` (2-letter ISO) | - -Return shape: - -| Field | Description | -| --------------- | ----------------------------------------------------- | -| `run_id` | TinyFish run identifier | -| `status` | `COMPLETED`, `FAILED`, or other terminal status | -| `result` | Structured extraction result (when successful) | -| `error` | Error details (when failed) | -| `streaming_url` | Live browser session URL (when TinyFish provides one) | -| `help_url` | Link to relevant TinyFish docs (on error) | -| `help_message` | Human-readable help hint (on error) | - -## Good fits - -Use TinyFish when the built-in browser is not the best surface: - -- Complex public forms with multiple steps -- JS-heavy pages that need real browser rendering -- Multi-step workflows with many clicks and navigation -- Region-sensitive browsing that benefits from proxy routing -- Structured extraction from live browser sessions - -Prefer other tools when: - -- A simple HTTP fetch or search is enough (`web_fetch`, `web_search`) -- You want direct local or remote CDP control with the built-in [Browser](/tools/browser) -- You need persistent authenticated browser sessions - -## Limitations - -- TinyFish targets public web workflows; persistent authenticated sessions are out of scope -- CAPTCHA solving is not supported -- Browser session state does not persist across runs -- Batch and parallel runs are out of scope for the initial bundled plugin - -## Example prompts - -- "Open example.com/pricing and extract every plan name and price as JSON." -- "Go to example.com/contact, fill the public inquiry form, and summarize what happened." -- "Visit example.com/search, switch the region to Canada, and extract the top five public listings." diff --git a/extensions/tinyfish/index.test.ts b/extensions/tinyfish/index.test.ts deleted file mode 100644 index 4b61b886479..00000000000 --- a/extensions/tinyfish/index.test.ts +++ /dev/null @@ -1,49 +0,0 @@ -import type { OpenClawPluginApi } from "openclaw/plugin-sdk/core"; -import { describe, expect, it, vi } from "vitest"; -import plugin from "./index.js"; - -describe("tinyfish plugin registration", () => { - it("registers only the tinyfish_automation tool", () => { - const registerTool = vi.fn(); - - const api = { - id: "tinyfish", - name: "TinyFish", - description: "TinyFish", - source: "test", - registrationMode: "full", - config: {}, - pluginConfig: {}, - runtime: {} as never, - logger: { info() {}, warn() {}, error() {} }, - registerTool, - registerHook() {}, - registerHttpRoute() {}, - registerChannel() {}, - registerGatewayMethod() {}, - registerCli() {}, - registerService() {}, - registerProvider() {}, - registerSpeechProvider() {}, - registerMediaUnderstandingProvider() {}, - registerImageGenerationProvider() {}, - registerWebSearchProvider() {}, - registerInteractiveHandler() {}, - registerCommand() {}, - registerContextEngine() {}, - registerMemoryPromptSection() {}, - resolvePath(input: string) { - return input; - }, - onConversationBindingResolved() {}, - on() {}, - } as unknown as OpenClawPluginApi; - - plugin.register?.(api); - - expect(registerTool).toHaveBeenCalledTimes(1); - expect(registerTool.mock.calls[0]?.[0]).toMatchObject({ - name: "tinyfish_automation", - }); - }); -}); diff --git a/extensions/tinyfish/index.ts b/extensions/tinyfish/index.ts deleted file mode 100644 index b49351d4904..00000000000 --- a/extensions/tinyfish/index.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry"; -import { createTinyFishTool } from "./src/tinyfish-tool.js"; - -export default definePluginEntry({ - id: "tinyfish", - name: "TinyFish", - description: "Hosted browser automation for complex public web workflows.", - register(api) { - api.registerTool(createTinyFishTool(api)); - }, -}); diff --git a/extensions/tinyfish/openclaw.plugin.json b/extensions/tinyfish/openclaw.plugin.json deleted file mode 100644 index 581b120099e..00000000000 --- a/extensions/tinyfish/openclaw.plugin.json +++ /dev/null @@ -1,41 +0,0 @@ -{ - "id": "tinyfish", - "name": "TinyFish", - "description": "Hosted browser automation for complex public web workflows.", - "skills": ["./skills"], - "providerAuthEnvVars": { - "tinyfish": ["TINYFISH_API_KEY"] - }, - "contracts": { - "tools": ["tinyfish_automation"] - }, - "uiHints": { - "apiKey": { - "label": "TinyFish API Key", - "help": "TinyFish API key for hosted browser automation (fallback: TINYFISH_API_KEY env var).", - "sensitive": true, - "placeholder": "tf_live_..." - }, - "baseUrl": { - "label": "TinyFish Base URL", - "help": "TinyFish API base URL override.", - "advanced": true, - "placeholder": "https://agent.tinyfish.ai" - } - }, - "configSchema": { - "type": "object", - "additionalProperties": false, - "properties": { - "apiKey": { - "type": ["string", "object"], - "description": "TinyFish API key. You can also set TINYFISH_API_KEY." - }, - "baseUrl": { - "type": "string", - "minLength": 1, - "description": "Optional TinyFish API base URL. Defaults to https://agent.tinyfish.ai." - } - } - } -} diff --git a/extensions/tinyfish/package.json b/extensions/tinyfish/package.json deleted file mode 100644 index 24f2bac9ba7..00000000000 --- a/extensions/tinyfish/package.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "name": "@openclaw/tinyfish-plugin", - "version": "2026.3.30", - "private": true, - "description": "OpenClaw TinyFish plugin", - "type": "module", - "openclaw": { - "extensions": [ - "./index.ts" - ] - } -} diff --git a/extensions/tinyfish/skills/tinyfish/SKILL.md b/extensions/tinyfish/skills/tinyfish/SKILL.md deleted file mode 100644 index af015a4f3d8..00000000000 --- a/extensions/tinyfish/skills/tinyfish/SKILL.md +++ /dev/null @@ -1,65 +0,0 @@ ---- -name: tinyfish -description: TinyFish hosted browser automation for complex public web workflows. -metadata: - { "openclaw": { "emoji": "🐟", "requires": { "config": ["plugins.entries.tinyfish.enabled"] } } } ---- - -# TinyFish Automation - -## When to use which tool - -| Need | Tool | When | -| --------------------------- | --------------------- | ------------------------------------------------------------------- | -| Simple page fetch | `web_fetch` | Static content, no JS rendering needed | -| Web search | `web_search` | Finding pages by query | -| Direct browser control | `browser` | Need step-by-step CDP control or local browser access | -| Complex public web workflow | `tinyfish_automation` | Multi-step forms, JS-heavy pages, bot-protected sites, geo-proxying | - -## tinyfish_automation - -Use when you need hosted browser automation for public websites that require -real browser interaction beyond what `web_fetch` or `web_search` can handle. - -| Parameter | Description | -| ----------------- | -------------------------------------------------------- | -| `url` | Target public website URL (required) | -| `goal` | Natural-language description of the task (required) | -| `browser_profile` | `lite` (default, fast) or `stealth` (anti-bot mode) | -| `proxy_config` | Optional proxy routing with `enabled` and `country_code` | - -### Browser profiles - -| Profile | Speed | Anti-bot | Best for | -| --------- | ------ | -------- | ------------------------------------------ | -| `lite` | Faster | Basic | Standard pages, forms, JS-rendered content | -| `stealth` | Slower | Strong | Cloudflare, DataDome, bot-protected sites | - -### Tips - -- **Start with `lite`** and escalate to `stealth` only if the site blocks access. -- **Keep goals specific** — "extract all product prices as JSON" works better than "look at the page." -- **Use `proxy_config`** with a `country_code` when content varies by region. -- **Prefer `web_fetch`** for simple static pages — TinyFish is for when you need a real browser. -- **Prefer `browser`** when you need direct local browser control or persistent sessions. - -### Return shape - -| Field | Description | -| --------------- | ----------------------------------------------------- | -| `run_id` | TinyFish run identifier | -| `status` | `COMPLETED`, `FAILED`, or other terminal status | -| `result` | Structured extraction result (when successful) | -| `error` | Error details (when failed) | -| `streaming_url` | Live browser session URL (when TinyFish provides one) | -| `help_url` | Link to relevant TinyFish docs (on error) | -| `help_message` | Human-readable help hint (on error) | - -## Choosing the right workflow - -Follow this escalation pattern — start simple, escalate only when needed: - -1. **`web_fetch`** — Simple page content, no JS rendering needed. -2. **`web_search`** — Find pages by query. -3. **`tinyfish_automation`** — Complex public workflows, forms, JS-heavy pages, bot-protected sites. -4. **`browser`** — Direct local browser control, persistent sessions, private/authenticated pages. diff --git a/extensions/tinyfish/src/tinyfish-tool.test.ts b/extensions/tinyfish/src/tinyfish-tool.test.ts deleted file mode 100644 index 50366604cc0..00000000000 --- a/extensions/tinyfish/src/tinyfish-tool.test.ts +++ /dev/null @@ -1,439 +0,0 @@ -import { SsrFBlockedError } from "openclaw/plugin-sdk/infra-runtime"; -import { describe, expect, it, vi } from "vitest"; -import { buildBaseUrl, createTinyFishTool } from "./tinyfish-tool.js"; - -const noopLogger = { - info: vi.fn(), - warn: vi.fn(), - error: vi.fn(), - debug: vi.fn(), -}; - -type MockFetchRequest = { - init?: RequestInit; - policy?: Record; -}; - -function createApi(pluginConfig: Record = {}) { - return { - id: "tinyfish", - name: "TinyFish", - description: "test", - source: "test", - config: {}, - pluginConfig, - runtime: {} as never, - logger: noopLogger, - } as never; -} - -function allowPublicHostname() { - return vi.fn(async () => ({ hostname: "example.com" }) as never); -} - -function sseResponse(events: string[]) { - const payload = events.join(""); - return new Response( - new ReadableStream({ - start(controller) { - controller.enqueue(new TextEncoder().encode(payload)); - controller.close(); - }, - }), - { - status: 200, - headers: { "content-type": "text/event-stream" }, - }, - ); -} - -describe("tinyfish automation tool", () => { - it("serializes request params and returns the streaming URL when provided", async () => { - const fetchWithGuard = vi.fn(async () => ({ - response: sseResponse([ - 'data: {"type":"STARTED","run_id":"run-1"}\n\n', - 'data: {"type":"STREAMING_URL","streaming_url":"https://stream.example/run-1"}\n\n', - 'data: {"type":"COMPLETE","run_id":"run-1","status":"COMPLETED","result":{"ok":true}}\n\n', - ]), - finalUrl: "https://agent.tinyfish.ai/v1/automation/run-sse", - release: async () => {}, - })); - - const tool = createTinyFishTool(createApi({ apiKey: "config-key" }), { - fetchWithGuard, - env: {}, - resolveHostname: allowPublicHostname(), - }); - - const result = (await tool.execute("tool-1", { - url: "https://example.com", - goal: "Fill the public form", - browser_profile: "stealth", - proxy_config: { enabled: true, country_code: "us" }, - })) as { details: Record }; - - expect(fetchWithGuard).toHaveBeenCalledTimes(1); - const firstCalls = fetchWithGuard.mock.calls as MockFetchRequest[][]; - const firstRequest = firstCalls[0]?.[0]; - expect(firstRequest).toMatchObject({ - init: { - method: "POST", - headers: expect.objectContaining({ "X-API-Key": "config-key" }), - }, - }); - expect(JSON.parse(String(firstRequest?.init?.body))).toEqual({ - url: "https://example.com/", - goal: "Fill the public form", - browser_profile: "stealth", - proxy_config: { enabled: true, country_code: "US" }, - api_integration: "openclaw", - }); - expect(result.details).toEqual({ - run_id: "run-1", - status: "COMPLETED", - result: { ok: true }, - error: null, - help_url: null, - help_message: null, - streaming_url: "https://stream.example/run-1", - }); - }); - - it("keeps the TinyFish API hostname restricted without skipping private-IP checks", async () => { - const fetchWithGuard = vi.fn(async () => ({ - response: sseResponse([ - 'data: {"type":"COMPLETE","run_id":"run-policy","status":"COMPLETED","result":{"ok":true}}\n\n', - ]), - finalUrl: "https://agent.tinyfish.ai/v1/automation/run-sse", - release: async () => {}, - })); - - const tool = createTinyFishTool(createApi({ apiKey: "config-key" }), { - fetchWithGuard, - env: {}, - resolveHostname: allowPublicHostname(), - }); - - await tool.execute("tool-1", { - url: "https://example.com", - goal: "Collect the pricing table", - }); - - const firstCalls = fetchWithGuard.mock.calls as MockFetchRequest[][]; - const firstRequest = firstCalls[0]?.[0]; - expect(firstRequest).toMatchObject({ - policy: { hostnameAllowlist: ["agent.tinyfish.ai"] }, - }); - expect(firstRequest?.policy).not.toHaveProperty("allowedHostnames"); - }); - - it("uses TINYFISH_API_KEY from the environment when plugin config is unset", async () => { - const fetchWithGuard = vi.fn(async () => ({ - response: sseResponse([ - 'data: {"type":"COMPLETE","run_id":"run-env","status":"COMPLETED","result":{"ok":true}}\n\n', - ]), - finalUrl: "https://agent.tinyfish.ai/v1/automation/run-sse", - release: async () => {}, - })); - - const tool = createTinyFishTool(createApi(), { - fetchWithGuard, - env: { TINYFISH_API_KEY: "env-key" }, - resolveHostname: allowPublicHostname(), - }); - - await tool.execute("tool-1", { - url: "https://example.com", - goal: "Collect the pricing table", - }); - - const firstCalls = fetchWithGuard.mock.calls as MockFetchRequest[][]; - const firstRequest = firstCalls[0]?.[0]; - expect(firstRequest?.init?.headers).toMatchObject({ "X-API-Key": "env-key" }); - }); - - it("sends X-Client-Source header for attribution", async () => { - const fetchWithGuard = vi.fn(async () => ({ - response: sseResponse([ - 'data: {"type":"COMPLETE","run_id":"run-attr","status":"COMPLETED","result":{}}\n\n', - ]), - finalUrl: "https://agent.tinyfish.ai/v1/automation/run-sse", - release: async () => {}, - })); - - const tool = createTinyFishTool(createApi({ apiKey: "config-key" }), { - fetchWithGuard, - env: {}, - resolveHostname: allowPublicHostname(), - }); - - await tool.execute("tool-1", { - url: "https://example.com", - goal: "Extract data", - }); - - const firstCalls = fetchWithGuard.mock.calls as MockFetchRequest[][]; - const firstRequest = firstCalls[0]?.[0]; - expect(firstRequest?.init?.headers).toMatchObject({ "X-Client-Source": "openclaw" }); - }); - - it("rejects TinyFish base URLs with query strings or fragments", async () => { - const fetchWithGuard = vi.fn(); - const tool = createTinyFishTool( - createApi({ apiKey: "config-key", baseUrl: "https://proxy.example/api?tenant=a#frag" }), - { fetchWithGuard, env: {}, resolveHostname: allowPublicHostname() }, - ); - - await expect( - tool.execute("tool-1", { url: "https://example.com", goal: "Collect the pricing table" }), - ).rejects.toThrow(/query parameters or fragments/); - expect(fetchWithGuard).not.toHaveBeenCalled(); - }); - - it("rejects TinyFish base URLs with embedded credentials", async () => { - const fetchWithGuard = vi.fn(); - const tool = createTinyFishTool( - createApi({ apiKey: "config-key", baseUrl: "https://user:pass@proxy.example/api" }), - { fetchWithGuard, env: {}, resolveHostname: allowPublicHostname() }, - ); - - await expect( - tool.execute("tool-1", { url: "https://example.com", goal: "Collect the pricing table" }), - ).rejects.toThrow(/embedded credentials/); - expect(fetchWithGuard).not.toHaveBeenCalled(); - }); - - it("points missing-key errors at plugins.entries.tinyfish.config.apiKey", async () => { - const tool = createTinyFishTool(createApi(), { fetchWithGuard: vi.fn(), env: {} }); - - await expect( - tool.execute("tool-1", { url: "https://example.com", goal: "Collect the pricing table" }), - ).rejects.toThrow(/plugins\.entries\.tinyfish\.config\.apiKey/); - }); - - it("surfaces unresolved SecretRef api keys with the TinyFish config path", async () => { - const tool = createTinyFishTool( - createApi({ apiKey: { source: "env", provider: "default", id: "TINYFISH_API_KEY" } }), - { fetchWithGuard: vi.fn(), env: {} }, - ); - - await expect( - tool.execute("tool-1", { url: "https://example.com", goal: "Collect the pricing table" }), - ).rejects.toThrow(/plugins\.entries\.tinyfish\.config\.apiKey: unresolved SecretRef/); - }); - - it("rejects target URLs with embedded credentials", async () => { - const tool = createTinyFishTool(createApi({ apiKey: "config-key" }), { - fetchWithGuard: vi.fn(), - env: {}, - resolveHostname: vi.fn(), - }); - - await expect( - tool.execute("tool-1", { url: "https://user:pass@example.com/private", goal: "Open" }), - ).rejects.toThrow(/embedded credentials/); - }); - - it("rejects non-public target URLs before forwarding to TinyFish", async () => { - const fetchWithGuard = vi.fn(); - const tool = createTinyFishTool(createApi({ apiKey: "config-key" }), { - fetchWithGuard, - env: {}, - resolveHostname: vi.fn(async () => { - throw new SsrFBlockedError("blocked"); - }), - }); - - await expect( - tool.execute("tool-1", { url: "http://127.0.0.1/private", goal: "Open" }), - ).rejects.toThrow(/public website/); - expect(fetchWithGuard).not.toHaveBeenCalled(); - }); - - it("succeeds when TinyFish omits the streaming URL event", async () => { - const fetchWithGuard = vi.fn(async () => ({ - response: sseResponse([ - 'data: {"type":"STARTED","run_id":"run-2"}\n\n', - 'data: {"type":"COMPLETE","run_id":"run-2","status":"COMPLETED","result":{"count":3}}\n\n', - ]), - finalUrl: "https://agent.tinyfish.ai/v1/automation/run-sse", - release: async () => {}, - })); - - const tool = createTinyFishTool(createApi({ apiKey: "config-key" }), { - fetchWithGuard, - env: {}, - resolveHostname: allowPublicHostname(), - }); - - const result = (await tool.execute("tool-1", { - url: "https://example.com", - goal: "Extract the table", - })) as { details: Record }; - - expect(result.details).toEqual({ - run_id: "run-2", - status: "COMPLETED", - result: { count: 3 }, - error: null, - help_url: null, - help_message: null, - streaming_url: null, - }); - }); - - it("preserves failed COMPLETE payload help fields", async () => { - const fetchWithGuard = vi.fn(async () => ({ - response: sseResponse([ - 'data: {"type":"STARTED","run_id":"run-3"}\n\n', - 'data: {"type":"COMPLETE","run_id":"run-3","status":"FAILED","error":{"message":"proxy exhausted","help_url":"https://docs.example/help","help_message":"Try another region"}}\n\n', - ]), - finalUrl: "https://agent.tinyfish.ai/v1/automation/run-sse", - release: async () => {}, - })); - - const tool = createTinyFishTool(createApi({ apiKey: "config-key" }), { - fetchWithGuard, - env: {}, - resolveHostname: allowPublicHostname(), - }); - - const result = (await tool.execute("tool-1", { - url: "https://example.com", - goal: "Submit the workflow", - })) as { details: Record }; - - expect(result.details).toMatchObject({ - run_id: "run-3", - status: "FAILED", - error: { message: "proxy exhausted" }, - help_url: "https://docs.example/help", - help_message: "Try another region", - }); - }); - - it("fails cleanly when the SSE payload is malformed", async () => { - const fetchWithGuard = vi.fn(async () => ({ - response: sseResponse(["data: not-json\n\n"]), - finalUrl: "https://agent.tinyfish.ai/v1/automation/run-sse", - release: async () => {}, - })); - - const tool = createTinyFishTool(createApi({ apiKey: "config-key" }), { - fetchWithGuard, - env: {}, - resolveHostname: allowPublicHostname(), - }); - - await expect( - tool.execute("tool-1", { url: "https://example.com", goal: "Extract" }), - ).rejects.toThrow(/not valid JSON/); - }); - - it("caps oversized TinyFish error bodies", async () => { - const fetchWithGuard = vi.fn(async () => ({ - response: new Response("x".repeat(5000), { - status: 502, - headers: { "content-type": "text/plain" }, - }), - finalUrl: "https://agent.tinyfish.ai/v1/automation/run-sse", - release: async () => {}, - })); - - const tool = createTinyFishTool(createApi({ apiKey: "config-key" }), { - fetchWithGuard, - env: {}, - resolveHostname: allowPublicHostname(), - }); - - let thrown: Error | undefined; - try { - await tool.execute("tool-1", { url: "https://example.com", goal: "Extract" }); - } catch (error) { - thrown = error as Error; - } - - expect(thrown).toBeInstanceOf(Error); - expect(String(thrown)).toMatch(new RegExp(`x{${2048}}`)); - expect(String(thrown)).not.toMatch(new RegExp(`x{2500}`)); - }); - - it("returns immediately after COMPLETE without waiting for EOF", async () => { - let cancelCalled = false; - const stream = new ReadableStream({ - start(controller) { - controller.enqueue( - new TextEncoder().encode( - 'data: {"type":"STARTED","run_id":"run-5"}\n\n' + - 'data: {"type":"COMPLETE","run_id":"run-5","status":"COMPLETED","result":{"ok":true}}\n\n' + - ": heartbeat\n\n", - ), - ); - }, - cancel() { - cancelCalled = true; - }, - }); - - const fetchWithGuard = vi.fn(async () => ({ - response: new Response(stream, { - status: 200, - headers: { "content-type": "text/event-stream" }, - }), - finalUrl: "https://agent.tinyfish.ai/v1/automation/run-sse", - release: async () => {}, - })); - - const tool = createTinyFishTool(createApi({ apiKey: "config-key" }), { - fetchWithGuard, - env: {}, - resolveHostname: allowPublicHostname(), - }); - - const result = (await Promise.race([ - tool.execute("tool-1", { url: "https://example.com", goal: "Extract" }), - new Promise((_, reject) => { - setTimeout(() => reject(new Error("Timed out")), 250); - }), - ])) as { details: Record }; - - expect(result.details).toMatchObject({ - run_id: "run-5", - status: "COMPLETED", - result: { ok: true }, - }); - expect(cancelCalled).toBe(true); - }); - - it("fails cleanly when the stream ends without COMPLETE", async () => { - const fetchWithGuard = vi.fn(async () => ({ - response: sseResponse(['data: {"type":"STARTED","run_id":"run-4"}\n\n']), - finalUrl: "https://agent.tinyfish.ai/v1/automation/run-sse", - release: async () => {}, - })); - - const tool = createTinyFishTool(createApi({ apiKey: "config-key" }), { - fetchWithGuard, - env: {}, - resolveHostname: allowPublicHostname(), - }); - - await expect( - tool.execute("tool-1", { url: "https://example.com", goal: "Extract" }), - ).rejects.toThrow(/COMPLETE/); - }); -}); - -describe("buildBaseUrl", () => { - it("defaults to the TinyFish production URL", () => { - expect(buildBaseUrl(undefined)).toBe("https://agent.tinyfish.ai/"); - }); - - it("appends trailing slash to custom base URLs", () => { - expect(buildBaseUrl("https://proxy.example/api")).toBe("https://proxy.example/api/"); - }); - - it("preserves existing trailing slash", () => { - expect(buildBaseUrl("https://proxy.example/api/")).toBe("https://proxy.example/api/"); - }); -}); diff --git a/extensions/tinyfish/src/tinyfish-tool.ts b/extensions/tinyfish/src/tinyfish-tool.ts deleted file mode 100644 index 8f4efe81ca2..00000000000 --- a/extensions/tinyfish/src/tinyfish-tool.ts +++ /dev/null @@ -1,519 +0,0 @@ -import { Type } from "@sinclair/typebox"; -import { jsonResult, readStringParam, ToolInputError } from "openclaw/plugin-sdk/agent-runtime"; -import type { OpenClawPluginApi } from "openclaw/plugin-sdk/core"; -import { - fetchWithSsrFGuard, - resolvePinnedHostname, - SsrFBlockedError, -} from "openclaw/plugin-sdk/infra-runtime"; -import { - normalizeResolvedSecretInputString, - normalizeSecretInput, -} from "openclaw/plugin-sdk/secret-input"; - -const DEFAULT_BASE_URL = "https://agent.tinyfish.ai"; -const RUN_STREAM_PATH = "v1/automation/run-sse"; -/** TinyFish API integration identifier (body field contract with TinyFish). */ -const TINYFISH_API_INTEGRATION = "openclaw"; -/** Generic attribution header value for request origin tracking. */ -const CLIENT_SOURCE = "openclaw"; -const STREAM_TIMEOUT_MS = 15 * 60 * 1000; -const MAX_ERROR_TEXT_BYTES = 2048; - -type TinyFishConfig = { - apiKey: string; - baseUrl: string; -}; - -type TinyFishBrowserProfile = "lite" | "stealth"; - -type TinyFishProxyConfig = { - enabled: boolean; - country_code?: string; -}; - -type TinyFishToolParams = { - url: string; - goal: string; - browser_profile?: TinyFishBrowserProfile; - proxy_config?: TinyFishProxyConfig; -}; - -type TinyFishRunResult = { - run_id: string | null; - status: string; - result: unknown; - error: unknown; - help_url: string | null; - help_message: string | null; - streaming_url: string | null; -}; - -type TinyFishSseEvent = Record & { - type?: unknown; - run_id?: unknown; - status?: unknown; - result?: unknown; - resultJson?: unknown; - error?: unknown; - streaming_url?: unknown; - url?: unknown; - help_url?: unknown; - help_message?: unknown; -}; - -type GuardedFetch = typeof fetchWithSsrFGuard; -type ResolveHostname = typeof resolvePinnedHostname; - -export type TinyFishToolDeps = { - env?: NodeJS.ProcessEnv; - fetchWithGuard?: GuardedFetch; - resolveHostname?: ResolveHostname; -}; - -function asRecord(value: unknown): Record | null { - return value && typeof value === "object" && !Array.isArray(value) - ? (value as Record) - : null; -} - -function readOptionalString(value: unknown): string | undefined { - return typeof value === "string" && value.trim() ? value.trim() : undefined; -} - -export function buildBaseUrl(rawBaseUrl: unknown): string { - const value = readOptionalString(rawBaseUrl) ?? DEFAULT_BASE_URL; - let parsed: URL; - try { - parsed = new URL(value); - } catch { - throw new Error("TinyFish base URL is invalid. Check plugins.entries.tinyfish.config.baseUrl."); - } - if (!["http:", "https:"].includes(parsed.protocol)) { - throw new Error( - "TinyFish base URL must use http or https. Check plugins.entries.tinyfish.config.baseUrl.", - ); - } - if (parsed.username || parsed.password) { - throw new Error( - "TinyFish base URL must not include embedded credentials. Check plugins.entries.tinyfish.config.baseUrl.", - ); - } - if (parsed.search || parsed.hash) { - throw new Error( - "TinyFish base URL must not include query parameters or fragments. Check plugins.entries.tinyfish.config.baseUrl.", - ); - } - parsed.pathname = parsed.pathname.endsWith("/") ? parsed.pathname : `${parsed.pathname}/`; - return parsed.toString(); -} - -function resolveTinyFishConfig( - pluginConfig: Record | undefined, - env: NodeJS.ProcessEnv, -): TinyFishConfig { - const configRecord = asRecord(pluginConfig) ?? {}; - const apiKey = - normalizeSecretInput( - normalizeResolvedSecretInputString({ - value: configRecord.apiKey, - path: "plugins.entries.tinyfish.config.apiKey", - }), - ) || - normalizeSecretInput(env.TINYFISH_API_KEY) || - undefined; - if (!apiKey) { - throw new Error( - "TinyFish API key missing. Set plugins.entries.tinyfish.config.apiKey or TINYFISH_API_KEY.", - ); - } - return { - apiKey, - baseUrl: buildBaseUrl(configRecord.baseUrl), - }; -} - -function validateTargetUrl(rawUrl: string): string { - let parsed: URL; - try { - parsed = new URL(rawUrl); - } catch { - throw new ToolInputError("url must be a valid http or https URL"); - } - if (!["http:", "https:"].includes(parsed.protocol)) { - throw new ToolInputError("url must be a valid http or https URL"); - } - if (parsed.username || parsed.password) { - throw new ToolInputError("url must not include embedded credentials"); - } - return parsed.toString(); -} - -async function assertPublicTargetUrl( - rawUrl: string, - resolveHostname: ResolveHostname, -): Promise { - const parsed = new URL(rawUrl); - if (parsed.hostname === "") { - throw new ToolInputError("url must target a public website"); - } - try { - await resolveHostname(parsed.hostname); - } catch (error) { - if (error instanceof SsrFBlockedError) { - throw new ToolInputError("url must target a public website"); - } - throw error; - } -} - -function readBrowserProfile(params: Record): TinyFishBrowserProfile | undefined { - const value = readStringParam(params, "browser_profile"); - if (!value) { - return undefined; - } - if (value === "lite" || value === "stealth") { - return value; - } - throw new ToolInputError("browser_profile must be one of: lite, stealth"); -} - -function readProxyConfig(params: Record): TinyFishProxyConfig | undefined { - const raw = params.proxy_config ?? params.proxyConfig; - if (raw === undefined) { - return undefined; - } - const record = asRecord(raw); - if (!record) { - throw new ToolInputError("proxy_config must be an object"); - } - if (typeof record.enabled !== "boolean") { - throw new ToolInputError("proxy_config.enabled must be true or false"); - } - - const countryCode = readOptionalString(record.country_code ?? record.countryCode); - if (countryCode && !/^[A-Za-z]{2}$/.test(countryCode)) { - throw new ToolInputError("proxy_config.country_code must be a 2-letter country code"); - } - - return { - enabled: record.enabled, - ...(countryCode ? { country_code: countryCode.toUpperCase() } : {}), - }; -} - -function normalizeTinyFishParams(params: Record): TinyFishToolParams { - const browserProfile = readBrowserProfile(params); - const proxyConfig = readProxyConfig(params); - - return { - url: validateTargetUrl(readStringParam(params, "url", { required: true })), - goal: readStringParam(params, "goal", { required: true }), - ...(browserProfile ? { browser_profile: browserProfile } : {}), - ...(proxyConfig ? { proxy_config: proxyConfig } : {}), - }; -} - -function buildRunEndpoint(baseUrl: string): URL { - return new URL(RUN_STREAM_PATH, baseUrl); -} - -function extractHelpField( - completeEvent: TinyFishSseEvent, - field: "help_url" | "help_message", -): string | null { - const directValue = readOptionalString(completeEvent[field]); - if (directValue) { - return directValue; - } - const errorRecord = asRecord(completeEvent.error); - return readOptionalString(errorRecord?.[field]) ?? null; -} - -function finalizeRunResult(params: { - completeEvent: TinyFishSseEvent; - runId?: string; - streamingUrl?: string; -}): TinyFishRunResult { - const status = readOptionalString(params.completeEvent.status) ?? "COMPLETED"; - return { - run_id: readOptionalString(params.completeEvent.run_id) ?? params.runId ?? null, - status, - result: params.completeEvent.result ?? params.completeEvent.resultJson ?? null, - error: params.completeEvent.error ?? null, - help_url: extractHelpField(params.completeEvent, "help_url"), - help_message: extractHelpField(params.completeEvent, "help_message"), - streaming_url: - readOptionalString(params.completeEvent.streaming_url) ?? params.streamingUrl ?? null, - }; -} - -function parseEventBlock(block: string): TinyFishSseEvent | null { - const dataLines: string[] = []; - for (const line of block.split(/\r?\n/)) { - if (!line || line.startsWith(":")) { - continue; - } - if (line.startsWith("data:")) { - dataLines.push(line.slice(5).trimStart()); - } - } - - if (dataLines.length === 0) { - return null; - } - - const payload = dataLines.join("\n").trim(); - if (!payload) { - return null; - } - - let parsed: unknown; - try { - parsed = JSON.parse(payload); - } catch { - throw new Error(`TinyFish SSE payload was not valid JSON: ${payload.slice(0, 120)}`); - } - - const record = asRecord(parsed); - if (!record) { - throw new Error("TinyFish SSE payload must be a JSON object"); - } - return record; -} - -async function parseRunStream( - body: ReadableStream, - logger: OpenClawPluginApi["logger"], -): Promise { - const reader = body.getReader(); - const decoder = new TextDecoder(); - let buffer = ""; - let runId: string | undefined; - let streamingUrl: string | undefined; - let completeEvent: TinyFishSseEvent | null = null; - let completeReceived = false; - - const handleEvent = (event: TinyFishSseEvent) => { - const type = readOptionalString(event.type); - if (type === "STARTED") { - runId = readOptionalString(event.run_id) ?? runId; - return; - } - if (type === "STREAMING_URL") { - runId = readOptionalString(event.run_id) ?? runId; - streamingUrl = readOptionalString(event.streaming_url) ?? readOptionalString(event.url); - return; - } - if (type === "PROGRESS" || type === "HEARTBEAT") { - logger.debug?.(`[tinyfish] stream event: ${type}`); - return; - } - if (type === "COMPLETE") { - completeEvent = event; - completeReceived = true; - runId = readOptionalString(event.run_id) ?? runId; - streamingUrl = - readOptionalString(event.streaming_url) ?? streamingUrl ?? readOptionalString(event.url); - return; - } - logger.debug?.(`[tinyfish] ignoring unknown stream event: ${String(type ?? "unknown")}`); - }; - - try { - while (true) { - const { done, value } = await reader.read(); - buffer += decoder.decode(value ?? new Uint8Array(), { stream: !done }); - - let match = /\r?\n\r?\n/.exec(buffer); - while (match) { - const block = buffer.slice(0, match.index); - buffer = buffer.slice(match.index + match[0].length); - const event = parseEventBlock(block); - if (event) { - handleEvent(event); - if (completeReceived) { - break; - } - } - match = /\r?\n\r?\n/.exec(buffer); - } - - if (done || completeReceived) { - break; - } - } - } finally { - if (completeReceived) { - await reader.cancel().catch(() => {}); - } - reader.releaseLock(); - } - - const finalBlock = buffer.trim(); - if (!completeReceived && finalBlock) { - try { - const event = parseEventBlock(finalBlock); - if (event) { - handleEvent(event); - } - } catch { - // Swallow parse errors from trailing partial data so the caller gets the - // clearer "stream ended before COMPLETE" error below. - } - } - - if (!completeEvent) { - const runHint = runId ? ` after run_id ${runId}` : ""; - throw new Error(`TinyFish SSE stream ended before COMPLETE${runHint}. Retry the tool call.`); - } - - return finalizeRunResult({ completeEvent, runId, streamingUrl }); -} - -async function readErrorText(response: Response): Promise { - const reader = response.body?.getReader(); - if (!reader) { - return ""; - } - const decoder = new TextDecoder(); - let remainingBytes = MAX_ERROR_TEXT_BYTES; - let text = ""; - try { - while (remainingBytes > 0) { - const { done, value } = await reader.read(); - if (done) { - break; - } - if (!value || value.byteLength === 0) { - continue; - } - const chunk = value.byteLength > remainingBytes ? value.subarray(0, remainingBytes) : value; - text += decoder.decode(chunk, { stream: true }); - remainingBytes -= chunk.byteLength; - if (chunk.byteLength < value.byteLength) { - await reader.cancel().catch(() => {}); - break; - } - } - text += decoder.decode(); - } catch { - return text; - } finally { - reader.releaseLock(); - } - if (!text) { - return ""; - } - try { - const parsed = JSON.parse(text) as unknown; - const record = asRecord(parsed); - const message = - readOptionalString(record?.message) ?? - readOptionalString(record?.detail) ?? - readOptionalString(record?.error); - return message ?? text; - } catch { - return text; - } -} - -async function runTinyFishAutomation( - params: TinyFishToolParams, - api: OpenClawPluginApi, - deps: TinyFishToolDeps, -): Promise { - const env = deps.env ?? process.env; - const config = resolveTinyFishConfig(api.pluginConfig, env); - const endpoint = buildRunEndpoint(config.baseUrl); - const fetchWithGuard = deps.fetchWithGuard ?? fetchWithSsrFGuard; - const resolveHostname = deps.resolveHostname ?? resolvePinnedHostname; - - await assertPublicTargetUrl(params.url, resolveHostname); - - const requestBody: Record = { - url: params.url, - goal: params.goal, - api_integration: TINYFISH_API_INTEGRATION, - }; - - if (params.browser_profile) { - requestBody.browser_profile = params.browser_profile; - } - if (params.proxy_config) { - requestBody.proxy_config = params.proxy_config; - } - - const { response, release } = await fetchWithGuard({ - url: endpoint.toString(), - init: { - method: "POST", - headers: { - Accept: "text/event-stream", - "Content-Type": "application/json", - "X-API-Key": config.apiKey, - "X-Client-Source": CLIENT_SOURCE, - }, - body: JSON.stringify(requestBody), - }, - policy: { - hostnameAllowlist: [endpoint.hostname], - }, - timeoutMs: STREAM_TIMEOUT_MS, - auditContext: "tinyfish-automation-run-sse", - }); - - try { - if (!response.ok) { - const errorText = await readErrorText(response); - const suffix = errorText ? `: ${errorText}` : ""; - throw new Error(`TinyFish API request failed (${response.status})${suffix}`); - } - if (!response.body) { - throw new Error("TinyFish API returned an empty SSE body"); - } - return await parseRunStream(response.body, api.logger); - } finally { - await release(); - } -} - -export function createTinyFishTool(api: OpenClawPluginApi, deps: TinyFishToolDeps = {}) { - return { - name: "tinyfish_automation", - label: "TinyFish Automation", - description: - "Run TinyFish hosted browser automation for public multi-step workflows, forms, JS-heavy pages, and structured extraction.", - parameters: Type.Object({ - url: Type.String({ - description: "Target public website URL to automate.", - }), - goal: Type.String({ - description: "Natural-language description of what TinyFish should accomplish.", - }), - browser_profile: Type.Optional( - Type.Unsafe({ - type: "string", - enum: ["lite", "stealth"], - description: "Optional TinyFish browser profile.", - }), - ), - proxy_config: Type.Optional( - Type.Object({ - enabled: Type.Boolean({ - description: "Enable or disable TinyFish proxy routing for this run.", - }), - country_code: Type.Optional( - Type.String({ - description: "Optional 2-letter country code, for example US.", - }), - ), - }), - ), - }), - async execute(_id: string, rawParams: Record) { - const params = normalizeTinyFishParams(rawParams); - return jsonResult(await runTinyFishAutomation(params, api, deps)); - }, - }; -} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 931181b3ed3..3fa8d8e1c4a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -618,8 +618,6 @@ importers: specifier: ^1.41.1 version: 1.41.1 - extensions/tinyfish: {} - extensions/tlon: dependencies: '@aws-sdk/client-s3': diff --git a/src/plugins/bundled-provider-auth-env-vars.generated.ts b/src/plugins/bundled-provider-auth-env-vars.generated.ts index 1fa107d2b27..041de85b61e 100644 --- a/src/plugins/bundled-provider-auth-env-vars.generated.ts +++ b/src/plugins/bundled-provider-auth-env-vars.generated.ts @@ -34,7 +34,6 @@ export const BUNDLED_PROVIDER_AUTH_ENV_VAR_CANDIDATES = { sglang: ["SGLANG_API_KEY"], synthetic: ["SYNTHETIC_API_KEY"], tavily: ["TAVILY_API_KEY"], - tinyfish: ["TINYFISH_API_KEY"], together: ["TOGETHER_API_KEY"], venice: ["VENICE_API_KEY"], "vercel-ai-gateway": ["AI_GATEWAY_API_KEY"],